configure buck2 for localhost nativelink for caching

.buckconfig

@@ -23,5 +23,9 @@
 
 [buck2]
     file_watcher = watchman
+    materializations = deferred
+    sqlite_materializer_state = true
+    defer_write_actions = true
+    hash_all_commands = true
 
 <file:.buckconfig.generated>

.github/workflows/ci.yml

@@ -14,26 +14,47 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: true
+
       - uses: facebook/install-dotslash@latest
       - uses: dtolnay/rust-toolchain@stable
+        id: rust-toolchain
         with:
           targets: x86_64-unknown-linux-gnu,aarch64-unknown-linux-gnu
+
       - name: BTRFS-ify antlir2-out
         run: |
           mkdir antlir2-out
           truncate -s 100G ${{ runner.temp }}/image.btrfs
           mkfs.btrfs ${{ runner.temp }}/image.btrfs
           sudo mount ${{ runner.temp }}/image.btrfs antlir2-out
           sudo chown -R $(id -u):$(id -g) antlir2-out
-      - name: Install deps
+      - name: Install system deps
         run: |
           sudo apt install \
             cpio jq libcap-dev systemd-container
 
-      - name: Disable watchman
+      - name: Install nativelink
+        uses: baptiste0928/cargo-install@v3
+        with:
+          git: https://github.com/TraceMachina/nativelink
+          tag: v0.5.3
+
+      - name: Restore buck2 cache
+        id: cache-restore
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /tmp/nativelink
+          key: nativelink-${{ steps.rust-toolchain.outputs.cachekey }}
+          restore-keys: |
+            nativelink-
+
+      - name: Start nativelink
+        run: nativelink &
+
+      - name: Setup CI buckconfig
         run: |
-          echo '[buck2]' >> .buckconfig
-          echo 'file_watcher=notify' >> .buckconfig
+          cp ci/buckconfig .buckconfig.local
 
       - name: Test target graph
         run: |
@@ -66,6 +87,13 @@ jobs:
         run: |
           ./buck2 build --keep-going @${{ runner.temp }}/tests.txt
 
+      - name: Upload buck2 cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /tmp/nativelink
+          key: ${{ steps.cache-restore.outputs.cache-primary-key }}
+
       - name: Run tests
         run: |
           ./buck2 test --keep-going @${{ runner.temp }}/tests.txt

ci/buckconfig

@@ -0,0 +1,12 @@
+[buck2]
+  file_watcher=notify
+
+[buck2_re_client]
+  action_cache_address = grpc://127.0.0.1:50051
+  engine_address = grpc://127.0.0.1:50051
+  cas_address = grpc://127.0.0.1:50051
+  tls = false
+  instance_name = main
+
+[build]
+  execution_platforms = antlir//platforms:exec-platforms

ci/nativelink.json

@@ -0,0 +1,121 @@
+{
+  "stores": {
+    "AC_MAIN_STORE": {
+      "filesystem": {
+        "content_path": "/tmp/nativelink/content_path-ac",
+        "temp_path": "/tmp/nativelink/tmp_path-ac",
+        "eviction_policy": {
+          // 5gb.
+          "max_bytes": 5000000000,
+        }
+      }
+    },
+    "WORKER_FAST_SLOW_STORE": {
+      "fast_slow": {
+        // "fast" must be a "filesystem" store because the worker uses it to make
+        // hardlinks on disk to a directory where the jobs are running.
+        "fast": {
+          "filesystem": {
+            "content_path": "/tmp/nativelink/content_path-cas",
+            "temp_path": "/tmp/nativelink/tmp_path-cas",
+            "eviction_policy": {
+              // 10gb.
+              "max_bytes": 10000000000,
+            }
+          }
+        },
+        "slow": {
+          /// Discard data.
+          /// This example usage has the CAS and the Worker live in the same place,
+          /// so they share the same underlying CAS. Since workers require a fast_slow
+          /// store, we use the fast store as our primary data store, and the slow store
+          /// is just a noop, since there's no shared storage in this config.
+          "noop": {}
+        }
+      }
+    }
+  },
+  "schedulers": {
+    "MAIN_SCHEDULER": {
+      "simple": {
+      }
+    }
+  },
+  "workers": [{
+    "local": {
+      "worker_api_endpoint": {
+        "uri": "grpc://127.0.0.1:50061",
+      },
+      "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE",
+      "upload_action_result": {
+        "ac_store": "AC_MAIN_STORE",
+      },
+      "work_directory": "/tmp/nativelink/work",
+      "platform_properties": {},
+      "additional_environment": {
+        "PATH": {
+          "value": "$PATH",
+        }
+      }
+    }
+  }],
+  "servers": [{
+    "name": "public",
+    "listener": {
+      "http": {
+        "socket_address": "0.0.0.0:50051"
+      }
+    },
+    "services": {
+      "cas": {
+        "main": {
+          "cas_store": "WORKER_FAST_SLOW_STORE"
+        }
+      },
+      "ac": {
+        "main": {
+          "ac_store": "AC_MAIN_STORE"
+        }
+      },
+      "execution": {
+        "main": {
+          "cas_store": "WORKER_FAST_SLOW_STORE",
+          "scheduler": "MAIN_SCHEDULER",
+        }
+      },
+      "capabilities": {
+        "main": {
+          "remote_execution": {
+            "scheduler": "MAIN_SCHEDULER",
+          }
+        }
+      },
+      "bytestream": {
+        "cas_stores": {
+          "main": "WORKER_FAST_SLOW_STORE",
+        }
+      }
+    }
+  }, {
+    "name": "private_workers_servers",
+    "listener": {
+      "http": {
+        "socket_address": "0.0.0.0:50061"
+      }
+    },
+    "services": {
+      // Note: This should be served on a different port, because it has
+      // a different permission set than the other services.
+      // In other words, this service is a backend api. The ones above
+      // are a frontend api.
+      "worker_api": {
+        "scheduler": "MAIN_SCHEDULER",
+      },
+      "admin": {},
+      "health": {},
+    }
+  }],
+  "global": {
+    "max_open_files": 4096
+  }
+}

flavor/centos9-compose/BUCK

@@ -0,0 +1,13 @@
+load("//antlir/antlir2/bzl/flavor:defs.bzl", "flavor")
+load("//antlir/bzl:build_defs.bzl", "buck_genrule", "http_file")
+load("//antlir/antlir2/bzl/image:defs.bzl", "image")
+load("//antlir/antlir2/bzl/package:defs.bzl", "package")
+load("//antlir/antlir2/bzl/image:build_appliance.bzl", "build_appliance")
+
+flavor(
+    name = "centos9",
+    default_build_appliance = "//flavor/centos9:build-appliance.prebuilt",
+    default_dnf_repo_set = "generated//snapshot/rpm/centos9-compose:repos",
+    default_dnf_versionlock = "//antlir/antlir2/package_managers/dnf/build_appliance:empty-dnf-versionlock.json",
+    visibility = ["PUBLIC"],
+)

platforms/BUCK

@@ -0,0 +1,26 @@
+load("@prelude//:rules.bzl", "platform")
+load("@prelude//platforms:defs.bzl", "execution_platform", "host_configuration")
+load(":defs.bzl", "nativelink_platform", "exec_platforms")
+
+nativelink_platform(
+    name = "nativelink",
+    cpu_configuration = "config//cpu:x86_64",
+    os_configuration = "config//os:linux",
+    visibility = ["PUBLIC"],
+)
+
+execution_platform(
+    name = "host",
+    cpu_configuration = host_configuration.cpu,
+    os_configuration = host_configuration.os,
+    use_windows_path_separators = host_info().os.is_windows,
+    visibility = ["PUBLIC"],
+)
+
+exec_platforms(
+    name = "exec-platforms",
+    platforms = [
+        ":nativelink",
+        ":host",
+    ]
+)

platforms/defs.bzl

@@ -0,0 +1,44 @@
+def _nativelink_platform(ctx):
+    constraints = {}
+    constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints)
+    constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints)
+
+    configuration = ConfigurationInfo(
+        constraints = constraints,
+        values = {},
+    )
+
+    platform = ExecutionPlatformInfo(
+        label = ctx.label.raw_target(),
+        configuration = configuration,
+        executor_config = CommandExecutorConfig(
+            local_enabled = True,
+            remote_enabled = True,
+            use_limited_hybrid = True,
+            remote_execution_properties = {
+                # "arch": ctx.attrs.cpu_configuration.label.name,
+                # "os": ctx.attrs.os_configuration.label.name,
+            },
+            remote_execution_use_case = "antlir",
+            remote_output_paths = "output_paths",
+        ),
+    )
+
+    return [DefaultInfo(), platform]
+
+nativelink_platform = rule(attrs = {
+    "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]),
+    "os_configuration": attrs.dep(providers = [ConfigurationInfo]),
+}, impl = _nativelink_platform)
+
+def _exec_platforms(ctx):
+    return [
+        DefaultInfo(),
+        ExecutionPlatformRegistrationInfo(
+            platforms = [p[ExecutionPlatformInfo] for p in ctx.attrs.platforms],
+        ),
+    ]
+
+exec_platforms = rule(attrs = {
+    "platforms": attrs.list(attrs.dep(providers = [ExecutionPlatformInfo])),
+}, impl = _exec_platforms)

third-party/rust/BUCK

@@ -8891,6 +8891,7 @@ cargo.rust_library(
         "equivalent",
         "inline-more",
         "raw-entry",
+        "nightly",
     ],
     visibility = [],
     deps = [