From fc35b1640eb41bba0d0488fea2fcc6da6dc676f7 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sun, 11 Aug 2024 08:16:29 -0700 Subject: [PATCH] Move inline file paths to static helper methods so we can describe them and document them (#3361) This makes the documentation somewhat discoverable: both grepping the codebase would pull it up, as would jump-to-definition from any of the use sites in the code (which is why this is better than having it in some readme file or docsite page). We could also link to it from the docs to make it more discoverable. Pull request: https://github.com/com-lihaoyi/mill/pull/3361 --- .../test/src/MultiLevelBuildTests.scala | 4 +- .../src/mill/main/client/MillClientMain.java | 12 ++-- .../client/src/mill/main/client/OutFiles.java | 50 +++++++++++++++ .../src/mill/main/client/ServerFiles.java | 63 +++++++++++++++++++ .../src/mill/main/client/lock/Locks.java | 8 ++- main/eval/src/mill/eval/EvaluatorCore.scala | 5 +- .../src/mill/runner/MillBuildBootstrap.scala | 3 +- runner/src/mill/runner/MillServerMain.scala | 9 +-- 8 files changed, 137 insertions(+), 17 deletions(-) create mode 100644 main/client/src/mill/main/client/OutFiles.java create mode 100644 main/client/src/mill/main/client/ServerFiles.java diff --git a/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala b/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala index 6a0de6a8754..a5dd3a1f40f 100644 --- a/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala +++ b/integration/invalidation/multi-level-editing/test/src/MultiLevelBuildTests.scala @@ -1,5 +1,6 @@ package mill.integration +import mill.main.client.OutFiles import mill.runner.RunnerState import utest._ @@ -47,7 +48,8 @@ object MultiLevelBuildTests extends IntegrationTestSuite { def loadFrames(n: Int) = { for (depth <- Range(0, n)) yield { - val path = wsRoot / "out" / Seq.fill(depth)("mill-build") / "mill-runner-state.json" + val path = + wsRoot / "out" / Seq.fill(depth)(OutFiles.millBuild()) / OutFiles.millRunnerState() if (os.exists(path)) upickle.default.read[RunnerState.Frame.Logged](os.read(path)) -> path else RunnerState.Frame.Logged(Map(), Seq(), Seq(), Map(), None, Seq(), 0) -> path } diff --git a/main/client/src/mill/main/client/MillClientMain.java b/main/client/src/mill/main/client/MillClientMain.java index 5687c8b251c..90035e8fff6 100644 --- a/main/client/src/mill/main/client/MillClientMain.java +++ b/main/client/src/mill/main/client/MillClientMain.java @@ -112,7 +112,7 @@ public static int main0(String[] args) throws Exception { int index = 0; while (index < serverProcessesLimit) { index++; - final String lockBase = "out/mill-worker-" + versionAndJvmHomeEncoding + "-" + index; + final String lockBase = "out/" + OutFiles.millWorker() + versionAndJvmHomeEncoding + "-" + index; java.io.File lockBaseFile = new java.io.File(lockBase); final File stdout = new java.io.File(lockBaseFile, "stdout"); final File stderr = new java.io.File(lockBaseFile, "stderr"); @@ -185,7 +185,7 @@ public static int run( String[] args, Map env) throws Exception { - try (FileOutputStream f = new FileOutputStream(lockBase + "/run")) { + try (FileOutputStream f = new FileOutputStream(ServerFiles.runArgs(lockBase))) { f.write(System.console() != null ? 1 : 0); Util.writeString(f, BuildInfo.millVersion); Util.writeArgs(args, f); @@ -199,7 +199,7 @@ public static int run( } while (locks.processLock.probe()) Thread.sleep(3); - String socketName = lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io"; + String socketName = ServerFiles.pipe(lockBase); AFUNIXSocketAddress addr = AFUNIXSocketAddress.of(new File(socketName)); long retryStart = System.currentTimeMillis(); @@ -241,7 +241,7 @@ public static int run( outPump.getLastData().waitForSilence(50); try { - return Integer.parseInt(Files.readAllLines(Paths.get(lockBase + "/exitCode")).get(0)); + return Integer.parseInt(Files.readAllLines(Paths.get(ServerFiles.exitCode(lockBase))).get(0)); } catch (Throwable e) { return ExitClientCodeCannotReadFromExitCodeFile(); } finally { @@ -252,8 +252,8 @@ public static int run( // 5 processes max private static int getServerProcessesLimit(String jvmHomeEncoding) { File outFolder = new File("out"); - String[] totalProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-")); - String[] thisJdkProcesses = outFolder.list((dir, name) -> name.startsWith("mill-worker-" + jvmHomeEncoding)); + String[] totalProcesses = outFolder.list((dir, name) -> name.startsWith(OutFiles.millWorker())); + String[] thisJdkProcesses = outFolder.list((dir, name) -> name.startsWith(OutFiles.millWorker() + jvmHomeEncoding)); int processLimit = 5; if (totalProcesses != null) { diff --git a/main/client/src/mill/main/client/OutFiles.java b/main/client/src/mill/main/client/OutFiles.java new file mode 100644 index 00000000000..1233aba84dc --- /dev/null +++ b/main/client/src/mill/main/client/OutFiles.java @@ -0,0 +1,50 @@ +package mill.main.client; + +/** + * Central place containing all the files that live inside the `out/` folder + * and documentation about what they do + */ +public class OutFiles { + /** + * Path of the Mill "meta-build", used to compile the `build.sc` file so we can + * run the primary Mill build. Can be nested for multiple stages of bootstrapping + */ + public static String millBuild(){ + return "mill-build"; + } + + /** + * A parallel performance and timing profile generated for every Mill execution. + * Can be loaded into the Chrome browser chrome://tracing page to visualize where + * time in a build is being spent + */ + public static String millChromeProfile(){ + return "mill-chrome-profile.json"; + } + + /** + * A sequential profile containing rich information about the tasks that were run + * as part of a build: name, duration, cached, dependencies, etc.. Useful to help + * understand what tasks are taking time in a build run and why those tasks are + * being executed + */ + public static String millProfile(){ + return "mill-profile.json"; + } + + /** + * Long lived metadata about the Mill bootstrap process that persists between runs: + * workers, watched files, classpaths, etc. + */ + public static String millRunnerState(){ + return "mill-runner-state.json"; + } + + /** + * Subfolder of `out/` that contains the machinery necessary for a single Mill background + * server: metadata files, pipes, logs, etc. + */ + public static String millWorker(){ + return "mill-worker-"; + } +} diff --git a/main/client/src/mill/main/client/ServerFiles.java b/main/client/src/mill/main/client/ServerFiles.java new file mode 100644 index 00000000000..45d8b3c369c --- /dev/null +++ b/main/client/src/mill/main/client/ServerFiles.java @@ -0,0 +1,63 @@ +package mill.main.client; + +/** + * Central place containing all the files that live inside the `out/mill-worker-*` folder + * and documentation about what they do + */ +public class ServerFiles { + + /** + * Lock file used to ensure a single server is running in a particular + * mill-worker folder. + */ + public static String processLock(String base){ + return base + "/processLock"; + } + + public static String clientLock(String base){ + return base + "/clientLock"; + } + + public static String serverLock(String base){ + return base + "/serverLock"; + } + + + /** + * The pipe by which the client snd server exchange IO + * + * Use uniquely-named pipes based on the fully qualified path of the project folder + * because on Windows the un-qualified name of the pipe must be globally unique + * across the whole filesystem + */ + public static String pipe(String base) { + try { + return base + "/mill-" + Util.md5hex(new java.io.File(base).getCanonicalPath()) + "-io"; + }catch (Exception e){ + throw new RuntimeException(e); + } + } + + /** + * Log file containing server housekeeping information + */ + public static String serverLog(String base){ + return base + "/server.log"; + } + + /** + * File that the client writes to pass the arguments, environment variables, + * and other necessary metadata to the Mill server to kick off a run + */ + public static String runArgs(String base){ + return base + "/runArgs"; + } + + /** + * File the server writes to pass the exit code of a completed run back to the + * client + */ + public static String exitCode(String base){ + return base + "/exitCode"; + } +} diff --git a/main/client/src/mill/main/client/lock/Locks.java b/main/client/src/mill/main/client/lock/Locks.java index 6d74812351f..a6483d719bc 100644 --- a/main/client/src/mill/main/client/lock/Locks.java +++ b/main/client/src/mill/main/client/lock/Locks.java @@ -1,5 +1,7 @@ package mill.main.client.lock; +import mill.main.client.ServerFiles; + public class Locks implements AutoCloseable { public Lock processLock; @@ -8,9 +10,9 @@ public class Locks implements AutoCloseable { public static Locks files(String lockBase) throws Exception { return new Locks(){{ - processLock = new FileLock(lockBase + "/pid"); - serverLock = new FileLock(lockBase + "/serverLock"); - clientLock = new FileLock(lockBase + "/clientLock"); + processLock = new FileLock(ServerFiles.processLock(lockBase)); + serverLock = new FileLock(ServerFiles.serverLock(lockBase)); + clientLock = new FileLock(ServerFiles.clientLock(lockBase)); }}; } diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index e33143749ff..26c5ace2d94 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -5,6 +5,7 @@ import mill.api.Strict.Agg import mill.api._ import mill.define._ import mill.eval.Evaluator.TaskResult +import mill.main.client.OutFiles import mill.util._ import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} @@ -69,8 +70,8 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { contextLoggerMsg0: Int => String ): Evaluator.Results = { os.makeDir.all(outPath) - val chromeProfileLogger = new ChromeProfileLogger(outPath / "mill-chrome-profile.json") - val profileLogger = new ProfileLogger(outPath / "mill-profile.json") + val chromeProfileLogger = new ChromeProfileLogger(outPath / OutFiles.millChromeProfile()) + val profileLogger = new ProfileLogger(outPath / OutFiles.millProfile()) val threadNumberer = new ThreadNumberer() val (sortedGroups, transitive) = Plan.plan(goals) val interGroupDeps = findInterGroupDeps(sortedGroups) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index bbf636daf30..c9d2a993aaf 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -7,6 +7,7 @@ import mill.eval.Evaluator import mill.main.RunScript import mill.resolve.SelectMode import mill.define.{BaseModule, Discover, Segments} +import mill.main.client.OutFiles import java.net.URLClassLoader @@ -50,7 +51,7 @@ class MillBuildBootstrap( for ((frame, depth) <- runnerState.frames.zipWithIndex) { os.write.over( - recOut(projectRoot, depth) / "mill-runner-state.json", + recOut(projectRoot, depth) / OutFiles.millRunnerState(), upickle.default.write(frame.loggedData, indent = 4), createFolders = true ) diff --git a/runner/src/mill/runner/MillServerMain.scala b/runner/src/mill/runner/MillServerMain.scala index 0a8fd606293..bdc9d818102 100644 --- a/runner/src/mill/runner/MillServerMain.scala +++ b/runner/src/mill/runner/MillServerMain.scala @@ -101,8 +101,9 @@ class Server[T]( var running = true while (running) { Server.lockBlock(locks.serverLock) { - val socketName = - lockBase + "/mill-" + Util.md5hex(new File(lockBase).getCanonicalPath()) + "-io" + + val socketName = ServerFiles.pipe(lockBase) + new File(socketName).delete() val addr = AFUNIXSocketAddress.of(new File(socketName)) val serverSocket = AFUNIXServerSocket.bindOn(addr) @@ -152,7 +153,7 @@ class Server[T]( // that relies on that method val proxiedSocketInput = proxyInputStreamThroughPumper(clientSocket.getInputStream) - val argStream = new FileInputStream(lockBase + "/run") + val argStream = new FileInputStream(ServerFiles.runArgs(lockBase)) val interactive = argStream.read() != 0 val clientMillVersion = Util.readString(argStream) val serverMillVersion = BuildInfo.millVersion @@ -161,7 +162,7 @@ class Server[T]( s"Mill version changed ($serverMillVersion -> $clientMillVersion), re-starting server" ) java.nio.file.Files.write( - java.nio.file.Paths.get(lockBase + "/exitCode"), + java.nio.file.Paths.get(ServerFiles.exitCode(lockBase)), s"${MillClientMain.ExitServerCodeWhenVersionMismatch()}".getBytes() ) System.exit(MillClientMain.ExitServerCodeWhenVersionMismatch())