diff --git a/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala b/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala index 6a0de6a8754..a5dd3a1f40f 100644 --- a/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala +++ b/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala @@ -1,5 +1,6 @@ package mill.integration +import mill.main.client.OutFiles import mill.runner.RunnerState import utest._ @@ -47,7 +48,8 @@ object MultiLevelBuildTests extends IntegrationTestSuite { def loadFrames(n: Int) = { for (depth <- Range(0, n)) yield { - val path = wsRoot / "out" / Seq.fill(depth)("mill-build") / "mill-runner-state.json" + val path = + wsRoot / "out" / Seq.fill(depth)(OutFiles.millBuild()) / OutFiles.millRunnerState() if (os.exists(path)) upickle.default.read[RunnerState.Frame.Logged](os.read(path)) -> path else RunnerState.Frame.Logged(Map(), Seq(), Seq(), Map(), None, Seq(), 0) -> path } diff --git a/main/client/src/mill/main/client/MillClientMain.java b/main/client/src/mill/main/client/MillClientMain.java index 5687c8b251c..90035e8fff6 100644 --- a/main/client/src/mill/main/client/MillClientMain.java +++ b/main/client/src/mill/main/client/MillClientMain.java @@ -112,7 +112,7 @@ public static int main0(String[] args) throws Exception { int index = 0; while (index < serverProcessesLimit) { index++; - final String lockBase = "out/mill-worker-" + versionAndJvmHomeEncoding + "-" + index; + final String lockBase = "out/" + OutFiles.millWorker() + versionAndJvmHomeEncoding + "-" + index; java.io.File lockBaseFile = new java.io.File(lockBase); final File stdout = new java.io.File(lockBaseFile, "stdout"); final File stderr = new java.io.File(lockBaseFile, "stderr"); @@ -185,7 +185,7 @@ public static int run( String[] args, Map env) throws Exception { - try (FileOutputStream f = new FileOutputStream(lockBase + "/run")) { + try (FileOutputStream f = new FileOutputStream(ServerFiles.runArgs(lockBase))) { f.write(System.console() != null ? 1 : 0); Util.writeString(f, BuildInfo.millVersion); Util.writeArgs(args, f); @@ -199,7 +199,7 @@ public static int run( } while (locks.processLock.probe()) Thread.sleep(3); - String socketName = lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io"; + String socketName = ServerFiles.pipe(lockBase); AFUNIXSocketAddress addr = AFUNIXSocketAddress.of(new File(socketName)); long retryStart = System.currentTimeMillis(); @@ -241,7 +241,7 @@ public static int run( outPump.getLastData().waitForSilence(50); try { - return Integer.parseInt(Files.readAllLines(Paths.get(lockBase + "/exitCode")).get(0)); + return Integer.parseInt(Files.readAllLines(Paths.get(ServerFiles.exitCode(lockBase))).get(0)); } catch (Throwable e) { return ExitClientCodeCannotReadFromExitCodeFile(); } finally { @@ -252,8 +252,8 @@ public static int run( // 5 processes max private static int getServerProcessesLimit(String jvmHomeEncoding) { File outFolder = new File("out"); - String[] totalProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-")); - String[] thisJdkProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-" + jvmHomeEncoding)); + String[] totalProcesses = outFolder.list((dir, name) -> name.startsWith(OutFiles.millWorker())); + String[] thisJdkProcesses = outFolder.list((dir, name) -> name.startsWith(OutFiles.millWorker() + jvmHomeEncoding)); int processLimit = 5; if (totalProcesses != null) { diff --git a/main/client/src/mill/main/client/OutFiles.java b/main/client/src/mill/main/client/OutFiles.java new file mode 100644 index 00000000000..1233aba84dc --- /dev/null +++ b/main/client/src/mill/main/client/OutFiles.java @@ -0,0 +1,50 @@ +package mill.main.client; + +/** + * Central place containing all the files that live inside the `out/` folder + * and documentation about what they do + */ +public class OutFiles { + /** + * Path of the Mill "meta-build", used to compile the `build.sc` file so we can + * run the primary Mill build. Can be nested for multiple stages of bootstrapping + */ + public static String millBuild(){ + return "mill-build"; + } + + /** + * A parallel performance and timing profile generated for every Mill execution. + * Can be loaded into the Chrome browser chrome://tracing page to visualize where + * time in a build is being spent + */ + public static String millChromeProfile(){ + return "mill-chrome-profile.json"; + } + + /** + * A sequential profile containing rich information about the tasks that were run + * as part of a build: name, duration, cached, dependencies, etc.. Useful to help + * understand what tasks are taking time in a build run and why those tasks are + * being executed + */ + public static String millProfile(){ + return "mill-profile.json"; + } + + /** + * Long lived metadata about the Mill bootstrap process that persists between runs: + * workers, watched files, classpaths, etc. + */ + public static String millRunnerState(){ + return "mill-runner-state.json"; + } + + /** + * Subfolder of `out/` that contains the machinery necessary for a single Mill background + * server: metadata files, pipes, logs, etc. + */ + public static String millWorker(){ + return "mill-worker-"; + } +} diff --git a/main/client/src/mill/main/client/ServerFiles.java b/main/client/src/mill/main/client/ServerFiles.java new file mode 100644 index 00000000000..45d8b3c369c --- /dev/null +++ b/main/client/src/mill/main/client/ServerFiles.java @@ -0,0 +1,63 @@ +package mill.main.client; + +/** + * Central place containing all the files that live inside the `out/mill-worker-*` folder + * and documentation about what they do + */ +public class ServerFiles { + + /** + * Lock file used to ensure a single server is running in a particular + * mill-worker folder. + */ + public static String processLock(String base){ + return base + "/processLock"; + } + + public static String clientLock(String base){ + return base + "/clientLock"; + } + + public static String serverLock(String base){ + return base + "/serverLock"; + } + + + /** + * The pipe by which the client snd server exchange IO + * + * Use uniquely-named pipes based on the fully qualified path of the project folder + * because on Windows the un-qualified name of the pipe must be globally unique + * across the whole filesystem + */ + public static String pipe(String base) { + try { + return base + "/mill-" + Util.md5hex(new java.io.File(base).getCanonicalPath()) + "-io"; + }catch (Exception e){ + throw new RuntimeException(e); + } + } + + /** + * Log file containing server housekeeping information + */ + public static String serverLog(String base){ + return base + "/server.log"; + } + + /** + * File that the client writes to pass the arguments, environment variables, + * and other necessary metadata to the Mill server to kick off a run + */ + public static String runArgs(String base){ + return base + "/runArgs"; + } + + /** + * File the server writes to pass the exit code of a completed run back to the + * client + */ + public static String exitCode(String base){ + return base + "/exitCode"; + } +} diff --git a/main/client/src/mill/main/client/lock/Locks.java b/main/client/src/mill/main/client/lock/Locks.java index 6d74812351f..a6483d719bc 100644 --- a/main/client/src/mill/main/client/lock/Locks.java +++ b/main/client/src/mill/main/client/lock/Locks.java @@ -1,5 +1,7 @@ package mill.main.client.lock; +import mill.main.client.ServerFiles; + public class Locks implements AutoCloseable { public Lock processLock; @@ -8,9 +10,9 @@ public class Locks implements AutoCloseable { public static Locks files(String lockBase) throws Exception { return new Locks(){{ - processLock = new FileLock(lockBase + "/pid"); - serverLock = new FileLock(lockBase + "/serverLock"); - clientLock = new FileLock(lockBase + "/clientLock"); + processLock = new FileLock(ServerFiles.processLock(lockBase)); + serverLock = new FileLock(ServerFiles.serverLock(lockBase)); + clientLock = new FileLock(ServerFiles.clientLock(lockBase)); }}; } diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index e33143749ff..26c5ace2d94 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -5,6 +5,7 @@ import mill.api.Strict.Agg import mill.api._ import mill.define._ import mill.eval.Evaluator.TaskResult +import mill.main.client.OutFiles import mill.util._ import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} @@ -69,8 +70,8 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { contextLoggerMsg0: Int => String ): Evaluator.Results = { os.makeDir.all(outPath) - val chromeProfileLogger = new ChromeProfileLogger(outPath / "mill-chrome-profile.json") - val profileLogger = new ProfileLogger(outPath / "mill-profile.json") + val chromeProfileLogger = new ChromeProfileLogger(outPath / OutFiles.millChromeProfile()) + val profileLogger = new ProfileLogger(outPath / OutFiles.millProfile()) val threadNumberer = new ThreadNumberer() val (sortedGroups, transitive) = Plan.plan(goals) val interGroupDeps = findInterGroupDeps(sortedGroups) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index bbf636daf30..c9d2a993aaf 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -7,6 +7,7 @@ import mill.eval.Evaluator import mill.main.RunScript import mill.resolve.SelectMode import mill.define.{BaseModule, Discover, Segments} +import mill.main.client.OutFiles import java.net.URLClassLoader @@ -50,7 +51,7 @@ class MillBuildBootstrap( for ((frame, depth) <- runnerState.frames.zipWithIndex) { os.write.over( - recOut(projectRoot, depth) / "mill-runner-state.json", + recOut(projectRoot, depth) / OutFiles.millRunnerState(), upickle.default.write(frame.loggedData, indent = 4), createFolders = true ) diff --git a/runner/src/mill/runner/MillServerMain.scala b/runner/src/mill/runner/MillServerMain.scala index 0a8fd606293..bdc9d818102 100644 --- a/runner/src/mill/runner/MillServerMain.scala +++ b/runner/src/mill/runner/MillServerMain.scala @@ -101,8 +101,9 @@ class Server[T]( var running = true while (running) { Server.lockBlock(locks.serverLock) { - val socketName = - lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io" + + val socketName = ServerFiles.pipe(lockBase) + new File(socketName).delete() val addr = AFUNIXSocketAddress.of(new File(socketName)) val serverSocket = AFUNIXServerSocket.bindOn(addr) @@ -152,7 +153,7 @@ class Server[T]( // that relies on that method val proxiedSocketInput = proxyInputStreamThroughPumper(clientSocket.getInputStream) - val argStream = new FileInputStream(lockBase + "/run") + val argStream = new FileInputStream(ServerFiles.runArgs(lockBase)) val interactive = argStream.read() != 0 val clientMillVersion = Util.readString(argStream) val serverMillVersion = BuildInfo.millVersion @@ -161,7 +162,7 @@ class Server[T]( s"Mill version changed ($serverMillVersion -> $clientMillVersion), re-starting server" ) java.nio.file.Files.write( - java.nio.file.Paths.get(lockBase + "/exitCode"), + java.nio.file.Paths.get(ServerFiles.exitCode(lockBase)), s"${MillClientMain.ExitServerCodeWhenVersionMismatch()}".getBytes() ) System.exit(MillClientMain.ExitServerCodeWhenVersionMismatch())