monolith: enable nix cache over ssh

secrets/monolith/default.yaml

@@ -5,6 +5,7 @@ gitlab-runners:
     thoreb-itinerario-nix: ENC[AES256_GCM,data:UdAAD0V895sFoEYR56sCG2LlpZugJ0K/nwkTygzWOnbTSmBRAcIQ8qVFPZGw+K+XMSLiCyio6Jp7k8AYP0K1VYm+6aEP3OkqR9FCLQTJgXo=,iv:UGUby50BYkn13OzItk6zZmxc5+SnbZZa4bebQHIow2A=,tag:LjDg3deWwdH2T71EtPo6jA==,type:str]
     docker-images-token: ENC[AES256_GCM,data:GGB/KSkjdAyhFKEspAh91ItbqEDf7K/LZSGSn+Jp7SxRfXpDzHIiMD8XJ9PTkGLeQGN4ug1i2nTYPS7d/P5OALWDU+1NPiV9nPdG0w7GERfu4g==,iv:6roabdOKX9xFMf0hWlECd73+943R+hFLos0e2dOpzns=,tag:LrASFc4DtN7aQ+3oOW/p/w==,type:str]
     wopus-gitlab-nix: ENC[AES256_GCM,data:asE7J0d58x9VfQFWc07f5T4s5NZ+/VqMQo66EX93J0LbJ4iI5YjvrrIE4pSI1e4Nz/SRQhltaJ0DfSH0+qgjD4wnAONPRi3UlFbSdGWS2bwwRtWe+Nci2krrUFxV2i/ZVE3CwCkNe4mqtII=,iv:gKrD/LhzI+jnDnX6CdxoHfjpiRdrsuRYJF9rTc8SffM=,tag:TczDGSU3gdKmERjBJ7tP/A==,type:str]
+    wopus-ssh-nix-cache-pk: ENC[AES256_GCM,data:MtYDK6P7nwBzr6p+lRX/dkosBfeDUAj/slf/a5SgVXNIbQlkEk7gvfW5iL+C2HgMwowqWx4F+3q2W+kGweqEYzEYAoZ9pR08a7Jci3Szyy49hkamxJXF+Qwhb5VQKxDppESne7DARCF0iYeUjgeXxCYyuWlGpisnkN3HCWrIYCqbk0LS+yqgkNhDxtxMaThGYztfPnLMEV/P5vuge9sRKu3Xi3iX2uDKtx4FTBsX30Lmd8kngOVnP/GaEHDa5ECO+/yW6ZRg3fIaqJ4RV+Vz79ovFUuZV/VE8eY3JOdK5tKIBWb31YUOjP7ccBes7mMhFLO3ceNeh+a6KAJbQ4pCojJwf/cLz663FKr5f/uWDicOBbL64l3+zV5zvSDzFls0ImXMNL6Fe3SaKP7ZcC5rVrRD8P+UN/OSFmbN5LM7uYY8nNsLxTH7MYsRHgTBUmTsFEhLGJIUjtf6J3/NWIlxjBq1MmpgxN0bD6gwVAxDPP489v918tsZtKdG8SJhLUPE4LWKsU7LHpgUBroKlbGE,iv:1jnF2TTlyTR59xM8Bgaz6bubDOwFexHBJipNVa0VPXY=,tag:VsDb6C6wYa9p4Yey3iG4eA==,type:str]
 sops:
     age:
         - recipient: age1zrgu7w8059xydagm60phnffghvfe9h2ca58cx8qwagqpyfuvs9fqw79c8h
@@ -25,8 +26,8 @@ sops:
             aFVxcDFhaGdYekRWRVFIWnRsZndtZFkKgsvxOFHOcO306Z9FkucA1fDOpZA8N1/h
             jYmIgcKTFgWoSCvux67lK30jFsYp7sm5z6WxxDYsGcoQ/+pxoUX2jQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-06-24T11:51:22Z"
+    lastmodified: "2025-07-16T15:08:21Z"
-    mac: ENC[AES256_GCM,data:onyjWlFsH/9YGSi2nGsPmZjhE4nFVQ5Jiwfi4s9KC7NetKD7Reyz2JY6i3YuZspBn3Jvbq8nOKVPGzttMAG+IrqQEv6+MxrCOEnJZXZcqocDNg7dACOXmJB5iwpFVdKscesTH2SScf7Pl/q6l9KOFjFuaZeBB7dlxHVA5zzCVOU=,iv:lEbxg2HfxU6ikgWSpUNAGIfgaz7DnZjXnLWcmsvt0A4=,tag:/Ag37QuJj9Xy/u20Nhy05Q==,type:str]
+    mac: ENC[AES256_GCM,data:jGS7FqZqEeACNIoeSLokZFa8NdD4ItagH0MXDK/71VODxAHXyYx1YC1rjQoHtJ94wBJV+PAJBvsVXFLRpp9OrmSmHdEAxDAfk0/yQsbqpTyruYRC+bkf5V+Ul8DhtXLFlCQ0LVL+Ku9jTUGCUbV0MHLAN5OBfPglk2vICOoV8Qk=,iv:1HAG5eAoAscctpkYQ1BNUFfQAodb0KbMqgQMw9W6G3o=,tag:rpxbvG5l7eMvCTKYQeRtEw==,type:str]
     pgp:
         - created_at: "2025-03-07T22:49:16Z"
           enc: |-

system/gitlab-runner.nix

@@ -1,56 +1,96 @@
 { pkgs, lib, ... }:
 let
-  installNixScript = pkgs.writeScriptBin "install-nix" ''
+  installNixScript =
-    mkdir -p -m 0755 /nix/var/log/nix/drvs
+    {
-    mkdir -p -m 0755 /nix/var/nix/gcroots
+      authenticationTokenConfigFile,
-    mkdir -p -m 0755 /nix/var/nix/profiles
+      nixCacheSshPrivateKeyPath ? null,
-    mkdir -p -m 0755 /nix/var/nix/temproots
+      ...
-    mkdir -p -m 0755 /nix/var/nix/userpool
+    }:
-    mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
+    pkgs.writeScriptBin "install-nix" ''
-    mkdir -p -m 1777 /nix/var/nix/profiles/per-user
+      mkdir -p -m 0755 /nix/var/log/nix/drvs
-    mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
+      mkdir -p -m 0755 /nix/var/nix/gcroots
-    mkdir -p -m 0700 "$HOME/.nix-defexpr"
+      mkdir -p -m 0755 /nix/var/nix/profiles
+      mkdir -p -m 0755 /nix/var/nix/temproots
+      mkdir -p -m 0755 /nix/var/nix/userpool
+      mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
+      mkdir -p -m 1777 /nix/var/nix/profiles/per-user
+      mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
+      mkdir -p -m 0700 "$HOME/.nix-defexpr"
 
-    . ${pkgs.nix}/etc/profile.d/nix.sh
+      . ${pkgs.nix}/etc/profile.d/nix.sh
 
-    ${pkgs.nix}/bin/nix-env -i ${
+      ${pkgs.nix}/bin/nix-env -i ${
-      lib.concatStringsSep " " (
+        lib.concatStringsSep " " (
-        with pkgs;
+          with pkgs;
-        [
+          [
-          nix
+            nix
-          cacert
+            cacert
-          git
+            git
-          openssh
+            openssh
-          docker
+            docker
-        ]
+          ]
-      )
+        )
-    }
+      }
-  '';
+
+      ${lib.optionalString (nixCacheSshPrivateKeyPath != null) ''
+        NIX_CACHE_SSH_PRIVATE_KEY_PATH="${nixCacheSshPrivateKeyPath}"
+        . ${./gitlab-runner/nix-cache-start}
+      ''}
+    '';
+
+  pushStoreContents =
+    {
+      authenticationTokenConfigFile,
+      nixCacheSshPrivateKeyPath ? null,
+      ...
+    }:
+    pkgs.writeScriptBin "push-to-cache" ''
+      ${lib.optionalString (nixCacheSshPrivateKeyPath != null) ''
+        . ${./gitlab-runner/nix-cache-end}
+      ''}
+    '';
 in
-{
+rec {
-  mkNixRunner = authenticationTokenConfigFile: {
+  mkNixRunnerFull =
-    # File should contain at least these two variables:
+    {
-    # `CI_SERVER_URL`
+      authenticationTokenConfigFile,
-    # `REGISTRATION_TOKEN`
+      nixCacheSshPrivateKeyPath ? null,
-    inherit authenticationTokenConfigFile; # 2
+      ...
-    dockerImage = "alpine:3.18.2";
+    }@args:
-    dockerPullPolicy = "if-not-present";
+    {
-    dockerVolumes = [
+      # File should contain at least these two variables:
-      "/etc/nix/nix.conf:/etc/nix/nix.conf:ro"
+      # `CI_SERVER_URL`
-      "/nix/store:/nix/store:ro"
+      # `REGISTRATION_TOKEN`
-      "/nix/var/nix/db:/nix/var/nix/db:ro"
+      inherit authenticationTokenConfigFile; # 2
-      "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+      dockerImage = "alpine:3.18.2";
-      "/tmp:/tmp"
+      dockerPullPolicy = "if-not-present";
-      "/var/run/docker.sock:/var/run/docker.sock"
+      dockerVolumes =
-      "/var/lib/docker/containers:/var/lib/docker/containers"
+        [
-    ];
+          "/etc/nix/nix.conf:/etc/nix/nix.conf:ro"
-    dockerDisableCache = true;
+          "/nix/store:/nix/store:ro"
-    preBuildScript = "\". ${lib.getExe installNixScript}\"";
+          "/nix/var/nix/db:/nix/var/nix/db:ro"
-    environmentVariables = {
+          "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
-      ENV = "/etc/profile";
+          "/tmp:/tmp"
-      USER = "root";
+          "/var/run/docker.sock:/var/run/docker.sock"
-      NIX_REMOTE = "daemon";
+          "/var/lib/docker/containers:/var/lib/docker/containers"
-      NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt";
+        ]
+        ++ lib.optionals (nixCacheSshPrivateKeyPath != null) [
+          "${nixCacheSshPrivateKeyPath}:${nixCacheSshPrivateKeyPath}"
+        ];
+      dockerDisableCache = true;
+      preBuildScript = "\". ${lib.getExe (installNixScript args)}\"";
+      postBuildScript = "\". ${lib.getExe (pushStoreContents args)}\"";
+      environmentVariables = {
+        ENV = "/etc/profile";
+        USER = "root";
+        NIX_REMOTE = "daemon";
+        NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt";
+      };
+    };
+
+  mkNixRunner =
+    authenticationTokenConfigFile:
+    mkNixRunnerFull {
+      inherit authenticationTokenConfigFile;
     };
-  };
 }

system/gitlab-runner/nix-cache-end

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+echo "nix-cache: Storing new store items"
+NEW_NIX_STORE_CONTENTS_FILE=$(mktemp)
+find /nix/store/ -maxdepth 1 > $NEW_NIX_STORE_CONTENTS_FILE
+
+sort $OLD_NIX_STORE_CONTENTS_FILE -o $OLD_NIX_STORE_CONTENTS_FILE
+sort $NEW_NIX_STORE_CONTENTS_FILE -o $NEW_NIX_STORE_CONTENTS_FILE
+
+echo "nix-cache: Comparing store paths"
+FILTERED_NIX_STORE_CONTENTS_FILE=$(mktemp)
+comm -13 $OLD_NIX_STORE_CONTENTS_FILE $NEW_NIX_STORE_CONTENTS_FILE > $FILTERED_NIX_STORE_CONTENTS_FILE
+echo "nix-cache: New store paths:"
+cat $FILTERED_NIX_STORE_CONTENTS_FILE | sed 's/^/    /g'
+
+if test -n "$(head -n1 $FILTERED_NIX_STORE_CONTENTS_FILE)"; then
+    echo "nix-cache: Sending new paths to cache"
+    nix copy --to "$STORE_URL" $(cat $FILTERED_NIX_STORE_CONTENTS_FILE) || true
+else
+    echo "nix-cache: Nothing to send"
+fi

system/gitlab-runner/nix-cache-start

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+echo "nix-cache: Setting up ssh key and host"
+STORE_HOST_PUB_KEY="IyBuaXgtY2FjaGUud29wdXMuZGV2OjIyIFNTSC0yLjAtT3BlblNTSF8xMC4wCm5peC1jYWNoZS53b3B1cy5kZXYgc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU5VNzFONVF4ZENtTTdOMjVTbk9nNnUrWUxtdjkyem5wZURjeUlEYW1sZEkK"
+STORE_URL="ssh://nix-ssh@nix-cache.wopus.dev?trusted=true&compress=true&ssh-key=$NIX_CACHE_SSH_PRIVATE_KEY_PATH&base64-ssh-public-host-key=$STORE_HOST_PUB_KEY"
+echo STORE_URL="$STORE_URL"
+
+NIX_EXTRA_CONFIG_FILE=$(mktemp)
+cat > "$NIX_EXTRA_CONFIG_FILE" <<EOF
+  extra-substituters = $STORE_URL
+EOF
+
+echo "nix-cache: Adding remote cache as substituter"
+export NIX_USER_CONF_FILES="$NIX_EXTRA_CONFIG_FILE:$NIX_USER_CONF_FILES"
+
+echo "nix-cache: Storing existing store items"
+OLD_NIX_STORE_CONTENTS_FILE=$(mktemp)
+find /nix/store/ -maxdepth 1 > $OLD_NIX_STORE_CONTENTS_FILE

system/monolith-gitlab-runner.nix

@@ -4,7 +4,7 @@
   ...
 }:
 let
-  inherit (pkgs.callPackage ./gitlab-runner.nix { }) mkNixRunner;
+  inherit (pkgs.callPackage ./gitlab-runner.nix { }) mkNixRunner mkNixRunnerFull;
 in
 {
   boot.kernel.sysctl."net.ipv4.ip_forward" = true;
@@ -18,7 +18,10 @@ in
       thoreb-telemetria-nix = mkNixRunner config.sops.secrets."gitlab-runners/thoreb-telemetria-nix".path;
       thoreb-itinerario-nix = mkNixRunner config.sops.secrets."gitlab-runners/thoreb-itinerario-nix".path;
 
-      wopus-gitlab-nix = mkNixRunner config.sops.secrets."gitlab-runners/wopus-gitlab-nix".path;
+      wopus-gitlab-nix = mkNixRunnerFull {
+        authenticationTokenConfigFile = config.sops.secrets."gitlab-runners/wopus-gitlab-nix".path;
+        nixCacheSshPrivateKeyPath = config.sops.secrets."gitlab-runners/wopus-ssh-nix-cache-pk".path;
+      };
 
       default = {
         # File should contain at least these two variables:
@@ -45,5 +48,8 @@ in
     "gitlab-runners/wopus-gitlab-nix" = {
       sopsFile = ../secrets/monolith/default.yaml;
     };
+    "gitlab-runners/wopus-ssh-nix-cache-pk" = {
+      sopsFile = ../secrets/monolith/default.yaml;
+    };
   };
 }