Skip to content

Commit

Permalink
Move inline file paths to static helper methods so we can describe th…
Browse files Browse the repository at this point in the history
…em and document them (#3361)

This makes the documentation somewhat discoverable: both grepping the
codebase would pull it up, as would jump-to-definition from any of the
use sites in the code (which is why this is better than having it in
some readme file or docsite page). We could also link to it from the
docs to make it more discoverable.

Pull request: #3361
  • Loading branch information
lihaoyi committed Aug 11, 2024
1 parent 7fdc264 commit fc35b16
Show file tree
Hide file tree
Showing 8 changed files with 137 additions and 17 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package mill.integration

import mill.main.client.OutFiles
import mill.runner.RunnerState
import utest._

Expand Down Expand Up @@ -47,7 +48,8 @@ object MultiLevelBuildTests extends IntegrationTestSuite {
def loadFrames(n: Int) = {
for (depth <- Range(0, n))
yield {
val path = wsRoot / "out" / Seq.fill(depth)("mill-build") / "mill-runner-state.json"
val path =
wsRoot / "out" / Seq.fill(depth)(OutFiles.millBuild()) / OutFiles.millRunnerState()
if (os.exists(path)) upickle.default.read[RunnerState.Frame.Logged](os.read(path)) -> path
else RunnerState.Frame.Logged(Map(), Seq(), Seq(), Map(), None, Seq(), 0) -> path
}
Expand Down
12 changes: 6 additions & 6 deletions main/client/src/mill/main/client/MillClientMain.java
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public static int main0(String[] args) throws Exception {
int index = 0;
while (index < serverProcessesLimit) {
index++;
final String lockBase = "out/mill-worker-" + versionAndJvmHomeEncoding + "-" + index;
final String lockBase = "out/" + OutFiles.millWorker() + versionAndJvmHomeEncoding + "-" + index;
java.io.File lockBaseFile = new java.io.File(lockBase);
final File stdout = new java.io.File(lockBaseFile, "stdout");
final File stderr = new java.io.File(lockBaseFile, "stderr");
Expand Down Expand Up @@ -185,7 +185,7 @@ public static int run(
String[] args,
Map<String, String> env) throws Exception {

try (FileOutputStream f = new FileOutputStream(lockBase + "/run")) {
try (FileOutputStream f = new FileOutputStream(ServerFiles.runArgs(lockBase))) {
f.write(System.console() != null ? 1 : 0);
Util.writeString(f, BuildInfo.millVersion);
Util.writeArgs(args, f);
Expand All @@ -199,7 +199,7 @@ public static int run(
}
while (locks.processLock.probe()) Thread.sleep(3);

String socketName = lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io";
String socketName = ServerFiles.pipe(lockBase);
AFUNIXSocketAddress addr = AFUNIXSocketAddress.of(new File(socketName));

long retryStart = System.currentTimeMillis();
Expand Down Expand Up @@ -241,7 +241,7 @@ public static int run(
outPump.getLastData().waitForSilence(50);

try {
return Integer.parseInt(Files.readAllLines(Paths.get(lockBase + "/exitCode")).get(0));
return Integer.parseInt(Files.readAllLines(Paths.get(ServerFiles.exitCode(lockBase))).get(0));
} catch (Throwable e) {
return ExitClientCodeCannotReadFromExitCodeFile();
} finally {
Expand All @@ -252,8 +252,8 @@ public static int run(
// 5 processes max
private static int getServerProcessesLimit(String jvmHomeEncoding) {
File outFolder = new File("out");
String[] totalProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-"));
String[] thisJdkProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-" + jvmHomeEncoding));
String[] totalProcesses = outFolder.list((dir, name) -> name.startsWith(OutFiles.millWorker()));
String[] thisJdkProcesses = outFolder.list((dir, name) -> name.startsWith(OutFiles.millWorker() + jvmHomeEncoding));

int processLimit = 5;
if (totalProcesses != null) {
Expand Down
50 changes: 50 additions & 0 deletions main/client/src/mill/main/client/OutFiles.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package mill.main.client;

/**
* Central place containing all the files that live inside the `out/` folder
* and documentation about what they do
*/
public class OutFiles {
/**
* Path of the Mill "meta-build", used to compile the `build.sc` file so we can
* run the primary Mill build. Can be nested for multiple stages of bootstrapping
*/
public static String millBuild(){
return "mill-build";
}

/**
* A parallel performance and timing profile generated for every Mill execution.
* Can be loaded into the Chrome browser chrome://tracing page to visualize where
* time in a build is being spent
*/
public static String millChromeProfile(){
return "mill-chrome-profile.json";
}

/**
* A sequential profile containing rich information about the tasks that were run
* as part of a build: name, duration, cached, dependencies, etc.. Useful to help
* understand what tasks are taking time in a build run and why those tasks are
* being executed
*/
public static String millProfile(){
return "mill-profile.json";
}

/**
* Long lived metadata about the Mill bootstrap process that persists between runs:
* workers, watched files, classpaths, etc.
*/
public static String millRunnerState(){
return "mill-runner-state.json";
}

/**
* Subfolder of `out/` that contains the machinery necessary for a single Mill background
* server: metadata files, pipes, logs, etc.
*/
public static String millWorker(){
return "mill-worker-";
}
}
63 changes: 63 additions & 0 deletions main/client/src/mill/main/client/ServerFiles.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package mill.main.client;

/**
* Central place containing all the files that live inside the `out/mill-worker-*` folder
* and documentation about what they do
*/
public class ServerFiles {

/**
* Lock file used to ensure a single server is running in a particular
* mill-worker folder.
*/
public static String processLock(String base){
return base + "/processLock";
}

public static String clientLock(String base){
return base + "/clientLock";
}

public static String serverLock(String base){
return base + "/serverLock";
}


/**
* The pipe by which the client snd server exchange IO
*
* Use uniquely-named pipes based on the fully qualified path of the project folder
* because on Windows the un-qualified name of the pipe must be globally unique
* across the whole filesystem
*/
public static String pipe(String base) {
try {
return base + "/mill-" + Util.md5hex(new java.io.File(base).getCanonicalPath()) + "-io";
}catch (Exception e){
throw new RuntimeException(e);
}
}

/**
* Log file containing server housekeeping information
*/
public static String serverLog(String base){
return base + "/server.log";
}

/**
* File that the client writes to pass the arguments, environment variables,
* and other necessary metadata to the Mill server to kick off a run
*/
public static String runArgs(String base){
return base + "/runArgs";
}

/**
* File the server writes to pass the exit code of a completed run back to the
* client
*/
public static String exitCode(String base){
return base + "/exitCode";
}
}
8 changes: 5 additions & 3 deletions main/client/src/mill/main/client/lock/Locks.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package mill.main.client.lock;

import mill.main.client.ServerFiles;

public class Locks implements AutoCloseable {

public Lock processLock;
Expand All @@ -8,9 +10,9 @@ public class Locks implements AutoCloseable {

public static Locks files(String lockBase) throws Exception {
return new Locks(){{
processLock = new FileLock(lockBase + "/pid");
serverLock = new FileLock(lockBase + "/serverLock");
clientLock = new FileLock(lockBase + "/clientLock");
processLock = new FileLock(ServerFiles.processLock(lockBase));
serverLock = new FileLock(ServerFiles.serverLock(lockBase));
clientLock = new FileLock(ServerFiles.clientLock(lockBase));
}};
}

Expand Down
5 changes: 3 additions & 2 deletions main/eval/src/mill/eval/EvaluatorCore.scala
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import mill.api.Strict.Agg
import mill.api._
import mill.define._
import mill.eval.Evaluator.TaskResult
import mill.main.client.OutFiles
import mill.util._

import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
Expand Down Expand Up @@ -69,8 +70,8 @@ private[mill] trait EvaluatorCore extends GroupEvaluator {
contextLoggerMsg0: Int => String
): Evaluator.Results = {
os.makeDir.all(outPath)
val chromeProfileLogger = new ChromeProfileLogger(outPath / "mill-chrome-profile.json")
val profileLogger = new ProfileLogger(outPath / "mill-profile.json")
val chromeProfileLogger = new ChromeProfileLogger(outPath / OutFiles.millChromeProfile())
val profileLogger = new ProfileLogger(outPath / OutFiles.millProfile())
val threadNumberer = new ThreadNumberer()
val (sortedGroups, transitive) = Plan.plan(goals)
val interGroupDeps = findInterGroupDeps(sortedGroups)
Expand Down
3 changes: 2 additions & 1 deletion runner/src/mill/runner/MillBuildBootstrap.scala
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import mill.eval.Evaluator
import mill.main.RunScript
import mill.resolve.SelectMode
import mill.define.{BaseModule, Discover, Segments}
import mill.main.client.OutFiles

import java.net.URLClassLoader

Expand Down Expand Up @@ -50,7 +51,7 @@ class MillBuildBootstrap(

for ((frame, depth) <- runnerState.frames.zipWithIndex) {
os.write.over(
recOut(projectRoot, depth) / "mill-runner-state.json",
recOut(projectRoot, depth) / OutFiles.millRunnerState(),
upickle.default.write(frame.loggedData, indent = 4),
createFolders = true
)
Expand Down
9 changes: 5 additions & 4 deletions runner/src/mill/runner/MillServerMain.scala
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,9 @@ class Server[T](
var running = true
while (running) {
Server.lockBlock(locks.serverLock) {
val socketName =
lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io"

val socketName = ServerFiles.pipe(lockBase)

new File(socketName).delete()
val addr = AFUNIXSocketAddress.of(new File(socketName))
val serverSocket = AFUNIXServerSocket.bindOn(addr)
Expand Down Expand Up @@ -152,7 +153,7 @@ class Server[T](
// that relies on that method
val proxiedSocketInput = proxyInputStreamThroughPumper(clientSocket.getInputStream)

val argStream = new FileInputStream(lockBase + "/run")
val argStream = new FileInputStream(ServerFiles.runArgs(lockBase))
val interactive = argStream.read() != 0
val clientMillVersion = Util.readString(argStream)
val serverMillVersion = BuildInfo.millVersion
Expand All @@ -161,7 +162,7 @@ class Server[T](
s"Mill version changed ($serverMillVersion -> $clientMillVersion), re-starting server"
)
java.nio.file.Files.write(
java.nio.file.Paths.get(lockBase + "/exitCode"),
java.nio.file.Paths.get(ServerFiles.exitCode(lockBase)),
s"${MillClientMain.ExitServerCodeWhenVersionMismatch()}".getBytes()
)
System.exit(MillClientMain.ExitServerCodeWhenVersionMismatch())
Expand Down

0 comments on commit fc35b16

Please sign in to comment.