[WIP] Build binaries for Mill Client using Graal VM native-image (#4044)

This PR builds binaries for Mill using [Graal VM
native-image](https://www.graalvm.org/latest/reference-manual/native-image/overview/Options/)
and adds Java/Scala/Kotlin examples for building native images.

Execution time for `mill --version`, with server running, shows over 5x
improvement.
```bash
$ time ./mill --version
Mill Build Tool version 0.12.3
Java version: 11.0.25, vendor: Eclipse Adoptium, runtime: ~/.sdkman/candidates/java/11.0.25-tem
Default locale: en_US, platform encoding: UTF-8
OS name: "Linux", version: 6.8.0-49-generic, arch: amd64

real	0m0.113s
user	0m0.171s
sys	0m0.037s

$ time ./out/dist/native.dest/mill --version
Mill Build Tool version 0.12.3-14-b0feea
Java version: 11.0.25, vendor: Eclipse Adoptium, runtime: ~/.sdkman/candidates/java/11.0.25-tem
Default locale: en_US, platform encoding: UTF-8
OS name: "Linux", version: 6.8.0-49-generic, arch: amd64

real	0m0.021s
user	0m0.005s
sys	0m0.016s

```

Resolves #4007.

---------

Co-authored-by: Li Haoyi <haoyi.sg@gmail.com>

.github/workflows/run-tests.yml

@@ -1,3 +1,26 @@
+# Uncommment this to replace the rest of the file when you want to debug stuff in CI
+#name: Run Debug
+#
+#on:
+#  push:
+#  pull_request:
+#  workflow_dispatch:
+#
+#jobs:
+#  debug:
+##    runs-on: ubuntu-latest
+#    runs-on: windows-latest
+#    steps:
+#      - uses: actions/checkout@v4
+#        with: { fetch-depth: 1 }
+#      - uses: actions/setup-java@v4
+#        with:
+#          distribution: 'temurin'
+#          java-version: '17'
+#
+#      - run: ./mill 'example.javalib.publishing[7-native-image].packaged.server.test'
+##
+#
 name: Run Tests
 
 # We run full CI on push builds to main and on all pull requests
@@ -65,53 +88,62 @@ jobs:
             install-android-sdk: false
 
           - java-version: 17
-            millargs: "'example.javalib.__.local.test'"
+            millargs: "'example.javalib.__.local.server.test'"
             install-android-sdk: false
 
           - java-version: 17
-            millargs: "'example.scalalib.__.local.test'"
+            millargs: "'example.scalalib.__.local.server.test'"
             install-android-sdk: false
 
           - java-version: 17
-            millargs: "'example.kotlinlib.__.local.test'"
+            millargs: "'example.kotlinlib.__.local.server.test'"
             install-android-sdk: false
 
           - java-version: 17
-            millargs: "'example.android.__.local.test'"
+            millargs: "'example.android.__.local.server.test'"
             install-android-sdk: true
 
           - java-version: 17
-            millargs: "'example.{pythonlib,javascriptlib}.__.local.test'"
+            millargs: "'example.{pythonlib,javascriptlib}.__.local.server.test'"
             install-android-sdk: false
 
           - java-version: 11
-            millargs: "'example.thirdparty[{mockito,acyclic,commons-io}].local.test'"
+            millargs: "'example.thirdparty[{mockito,acyclic,commons-io}].local.server.test'"
             install-android-sdk: false
 
           - java-version: 17
-            millargs: "'example.thirdparty[{fansi,jimfs,netty,gatling}].local.test'"
+            millargs: "'example.thirdparty[{fansi,jimfs,netty,gatling}].local.server.test'"
             install-android-sdk: false
 
           - java-version: '17'
-            millargs: "'example.thirdparty[arrow].local.test'"
+            millargs: "'example.thirdparty[arrow].local.server.test'"
             install-android-sdk: false
 
           - java-version: 11
-            millargs: "'example.{cli,fundamentals,depth,extending}.__.local.test'"
+            millargs: "'example.{cli,fundamentals,depth,extending}.__.local.server.test'"
             install-android-sdk: false
             # Most of these integration tests should not depend on which mode they
             # are run in, so just run them in `local`
           - java-version: '17'
-            millargs: "'integration.{failure,feature,ide}.__.local.test'"
+            millargs: "'integration.{failure,feature,ide}.__.local.server.test'"
             install-android-sdk: false
             # These invalidation tests need to be exercised in both execution modes
             # to make sure they work with and without -i/--no-server being passed
           - java-version: 17
-            millargs: "'integration.invalidation.__.fork.test'"
+            millargs: "'integration.invalidation.__.packaged.fork.test'"
             install-android-sdk: false
 
           - java-version: 17
-            millargs: "'integration.invalidation.__.server.test'"
+            millargs: "'integration.invalidation.__.packaged.server.test'"
+            install-android-sdk: false
+
+          # Run some smoketests with Graal native image launcher
+          - java-version: 17
+            millargs: "'example.javalib.__.native.server.test'"
+            install-android-sdk: false
+
+          - java-version: '17'
+            millargs: "'integration.{failure,feature,ide}.__.native.server.test'"
             install-android-sdk: false
 
     uses: ./.github/workflows/post-build-selective.yml
@@ -131,11 +163,13 @@ jobs:
           - java-version: 11
             millargs: '"{main,scalalib,bsp}.__.test"'
           - java-version: 11
-            millargs: '"example.scalalib.basic.__.fork.test"'
+            millargs: '"example.scalalib.basic.__.packaged.fork.test"'
           - java-version: 17
-            millargs: "'integration.{feature,failure}[_].fork.test'"
+            millargs: "'integration.{feature,failure}.__.packaged.fork.test'"
+          - java-version: 11
+            millargs: "'integration.invalidation.__.packaged.server.test'"
           - java-version: 11
-            millargs: "'integration.invalidation[_].server.test'"
+            millargs: "'integration.invalidation.__.packaged.server.test'"
           - java-version: 11
             millargs: "contrib.__.test"
 

build.mill

@@ -42,7 +42,9 @@ object Settings {
   )
   val mimaBaseVersions: Seq[String] =
     0.to(13).map("0.11." + _) ++
-      Seq("0.12.0", "0.12.1", "0.12.2", "0.12.3", "0.12.4", "0.12.5")
+    Seq("0.12.0", "0.12.1", "0.12.2", "0.12.3", "0.12.4", "0.12.5")
+
+  val graalvmJvmId = "graalvm-community:23.0.1"
 }
 
 object Deps {

ci/test-mill-bootstrap.sh

@@ -20,4 +20,4 @@ ci/prepare-mill-bootstrap.sh
 
 # Run tests
 target/mill-release -i "__.compile"
-target/mill-release -i "example.scalalib.basic[1-simple].server.test"
+target/mill-release -i "example.scalalib.basic[1-simple].packaged.server.test"

dist/package.mill

@@ -1,10 +1,13 @@
 package build.dist
 import mill._, scalalib._, publish._
+import mill.define.ModuleRef
 import mill.util.Jvm
 import mill.api.JarManifest
 import de.tobiasroeser.mill.vcs.version.VcsVersion
 import $file.ci.upload
 
+import scala.util.{Properties, Using}
+
 object `package` extends RootModule with build.MillPublishJavaModule {
 
   /**
@@ -110,7 +113,7 @@ object `package` extends RootModule with build.MillPublishJavaModule {
     PathRef(Task.dest / filename)
   }
   def assembly = Task {
-    Task.traverse(allPublishModules)(m => m.publishLocalCached)()
+    Task.traverse(allPublishModules.filter(_ != native))(m => m.publishLocalCached)()
     val raw = rawAssembly().path
     os.copy(raw, Task.dest / raw.last)
     PathRef(Task.dest / raw.last)
@@ -157,7 +160,7 @@ object `package` extends RootModule with build.MillPublishJavaModule {
 
   def run(args: Task[Args] = Task.Anon(Args())) = Task.Command(exclusive = true) {
     args().value match {
-      case Nil => mill.api.Result.Failure("Need to pass in cwd as first argument to dev.run")
+      case Nil => mill.api.Result.Failure("Need to pass in cwd as first argument to dist.run")
       case wd0 +: rest =>
         val wd = os.Path(wd0, Task.workspace)
         os.makeDir.all(wd)
@@ -170,7 +173,7 @@ object `package` extends RootModule with build.MillPublishJavaModule {
           mill.api.Result.Success(())
         } catch {
           case e: Throwable =>
-            mill.api.Result.Failure(s"dev.run failed with an exception. ${e.getMessage()}")
+            mill.api.Result.Failure(s"dist.run failed with an exception. ${e.getMessage()}")
         }
     }
   }
@@ -344,4 +347,42 @@ object `package` extends RootModule with build.MillPublishJavaModule {
       )
     }
   }
+
+  object native extends build.MillPublishJavaModule with mill.scalalib.NativeImageModule {
+    def moduleDeps = build.dist.moduleDeps
+    def nativeImageExecutableName = if (Properties.isWin) "mill.exe" else "mill"
+
+    def mainClass = Some("mill.runner.client.MillClientMain")
+
+    def nativeImageClasspath = build.runner.client.runClasspath()
+
+    def rawNativeImage = Task {
+      val previous = super.nativeImage().path
+      val executable = Task.dest / previous.last
+
+      Using(os.write.outputStream(executable)) { out =>
+        out.write(os.read.bytes(previous))
+        out.write(System.lineSeparator.getBytes)
+        out.write(os.read.bytes(assembly().path))
+      }
+
+      os.perms.set(executable, "rwxrwxrwx")
+      PathRef(executable)
+    }
+
+    def nativeImage = Task {
+      Task.traverse(allPublishModules.filter(_ != native))(m => m.publishLocalCached)()
+      rawNativeImage()
+    }
+
+    def jar = rawNativeImage()
+
+    def nativeImageOptions = Seq("--no-fallback")
+
+    def zincWorker = ModuleRef(ZincWorkerGraalvm)
+
+    object ZincWorkerGraalvm extends ZincWorkerModule {
+      def jvmId = build.Settings.graalvmJvmId
+    }
+  }
 }

dist/resources/META-INF/native-image/reachability-metadata.json

@@ -0,0 +1,15 @@
+{
+  "reflection": [
+    {
+      "type": "mill.runner.MillMain"
+    },
+    {
+      "type": "mill.runner.MillServerMain"
+    }
+  ],
+  "resources": [
+    {
+      "glob": "logback.xml"
+    }
+  ]
+}

docs/modules/ROOT/pages/javalib/intro.adoc

@@ -30,5 +30,3 @@ include::partial$example/javalib/basic/4-compat-modules.adoc[]
 == Realistic Java Example Project
 
 include::partial$example/javalib/basic/6-realistic.adoc[]
-
-

docs/modules/ROOT/pages/javalib/publishing.adoc

@@ -49,3 +49,8 @@ include::partial$example/javalib/publishing/5-jlink.adoc[]
 == Java Installers using `jpackage`
 
 include::partial$example/javalib/publishing/6-jpackage.adoc[]
+
+
+== Building Native Image with Graal VM
+
+include::partial$example/javalib/publishing/7-native-image.adoc[]

docs/modules/ROOT/pages/kotlinlib/publishing.adoc

@@ -25,4 +25,8 @@ https://docs.oracle.com/en/java/javase/17/docs/specs/man/jpackage.html[JPackage]
 For more details, see:
 
 * xref:javalib/publishing.adoc#_java_app_and_bundles_using_jlink[Java App and Bundles using JLink]
-* xref:javalib/publishing.adoc#_java_installers_using_jpackage[Java Installers using JPackage]
+* xref:javalib/publishing.adoc#_java_installers_using_jpackage[Java Installers using JPackage]
+
+== Building Native Image with Graal VM
+
+include::partial$example/kotlinlib/publishing/7-native-image.adoc[]

docs/modules/ROOT/pages/scalalib/publishing.adoc

@@ -24,4 +24,9 @@ https://docs.oracle.com/en/java/javase/17/docs/specs/man/jpackage.html[JPackage]
 For more details, see:
 
 * xref:javalib/publishing.adoc#_java_app_and_bundles_using_jlink[Java App and Bundles using JLink]
-* xref:javalib/publishing.adoc#_java_installers_using_jpackage[Java Installers using JPackage]
+* xref:javalib/publishing.adoc#_java_installers_using_jpackage[Java Installers using JPackage]
+
+
+== Building Native Image with Graal VM
+
+include::partial$example/scalalib/publishing/7-native-image.adoc[]

example/javalib/publishing/7-native-image/build.mill

@@ -0,0 +1,15 @@
+//// SNIPPET:BUILD
+package build
+import mill._, javalib._
+import mill.define.ModuleRef
+
+object foo extends JavaModule with NativeImageModule {
+
+  def zincWorker = ModuleRef(ZincWorkerGraalvm)
+
+  object ZincWorkerGraalvm extends ZincWorkerModule {
+    def jvmId = "graalvm-community:17.0.7"
+  }
+}
+
+//// SNIPPET:END

example/javalib/publishing/7-native-image/foo/src/foo/App.java

@@ -0,0 +1,8 @@
+package foo;
+
+public class App {
+
+  public static void main(String[] args) {
+    System.out.println("Hello, World!");
+  }
+}

example/kotlinlib/publishing/7-native-image/build.mill

@@ -0,0 +1,17 @@
+//// SNIPPET:BUILD
+package build
+import mill._, kotlinlib._
+import mill.define.ModuleRef
+
+object foo extends KotlinModule with NativeImageModule {
+
+  def zincWorker = ModuleRef(ZincWorkerGraalvm)
+
+  object ZincWorkerGraalvm extends ZincWorkerModule {
+    def jvmId = "graalvm-community:17.0.7"
+  }
+
+  def kotlinVersion = "1.9.24"
+}
+
+//// SNIPPET:END

example/kotlinlib/publishing/7-native-image/foo/src/foo/App.kt

@@ -0,0 +1,5 @@
+package foo
+
+fun main(args: Array<String>) {
+    println("Hello, World!")
+}

example/scalalib/publishing/7-native-image/build.mill

@@ -0,0 +1,33 @@
+//// SNIPPET:BUILD
+package build
+import mill._, scalalib._
+import mill.define.ModuleRef
+
+object foo extends ScalaModule with NativeImageModule {
+
+  def zincWorker = ModuleRef(ZincWorkerGraalvm)
+
+  object ZincWorkerGraalvm extends ZincWorkerModule {
+    def jvmId = "graalvm-community:17.0.7"
+  }
+
+  def scalaVersion = "2.13.11"
+}
+
+//// SNIPPET:END
+//
+// This example uses `NativeImageModule` to generate a native executable using https://www.graalvm.org/[Graal VM].
+// NOTE: For build portability, it is recommended to use a custom JDK via a custom `ZincWorkerModule` overriding
+// `def jvmId`.
+
+/** Usage
+
+> ./mill show foo.ZincWorkerGraalvm.javaHome
+
+> ./mill foo.nativeImage
+GraalVM Native Image: Generating...native-image...
+Finished generating...native-image...
+
+> ./out/foo/nativeImage.dest/native-image
+Hello, World!
+*/

example/scalalib/publishing/7-native-image/foo/src/foo/App.scala

@@ -0,0 +1,7 @@
+package foo
+
+object App {
+
+  def main(args: Array[String]): Unit =
+    println("Hello, World!")
+}