monolith: enable nix cache over ssh
This commit is contained in:
parent
22dc422b63
commit
868496d2b9
5 changed files with 138 additions and 52 deletions
|
|
@ -6,6 +6,7 @@ gitlab-runners:
|
|||
docker-images-token: ENC[AES256_GCM,data:GGB/KSkjdAyhFKEspAh91ItbqEDf7K/LZSGSn+Jp7SxRfXpDzHIiMD8XJ9PTkGLeQGN4ug1i2nTYPS7d/P5OALWDU+1NPiV9nPdG0w7GERfu4g==,iv:6roabdOKX9xFMf0hWlECd73+943R+hFLos0e2dOpzns=,tag:LrASFc4DtN7aQ+3oOW/p/w==,type:str]
|
||||
wopus-gitlab-nix: ENC[AES256_GCM,data:asE7J0d58x9VfQFWc07f5T4s5NZ+/VqMQo66EX93J0LbJ4iI5YjvrrIE4pSI1e4Nz/SRQhltaJ0DfSH0+qgjD4wnAONPRi3UlFbSdGWS2bwwRtWe+Nci2krrUFxV2i/ZVE3CwCkNe4mqtII=,iv:gKrD/LhzI+jnDnX6CdxoHfjpiRdrsuRYJF9rTc8SffM=,tag:TczDGSU3gdKmERjBJ7tP/A==,type:str]
|
||||
wopus-gitlab-docker-images: ENC[AES256_GCM,data:aGbCjQr1VKgg5n4f8vZKgdXcDw/M5JHez9E2TqipBXQ8D0jXdfPg6laNOJUOD+uPBOIGKUBMEg4OtLblCZFVw/V6wJN16wVbwkDU3uELQ8tPmlYSt4fcy4+5sC6+tV4YeMSKA6yIjD+xpkk=,iv:ojBhf2WdkWHruvTbABAAvuGDVOnsUl+qnhvH09L+lgA=,tag:gWhEkvL1qlcge3bSKVDSIg==,type:str]
|
||||
wopus-ssh-nix-cache-pk: ENC[AES256_GCM,data:MtYDK6P7nwBzr6p+lRX/dkosBfeDUAj/slf/a5SgVXNIbQlkEk7gvfW5iL+C2HgMwowqWx4F+3q2W+kGweqEYzEYAoZ9pR08a7Jci3Szyy49hkamxJXF+Qwhb5VQKxDppESne7DARCF0iYeUjgeXxCYyuWlGpisnkN3HCWrIYCqbk0LS+yqgkNhDxtxMaThGYztfPnLMEV/P5vuge9sRKu3Xi3iX2uDKtx4FTBsX30Lmd8kngOVnP/GaEHDa5ECO+/yW6ZRg3fIaqJ4RV+Vz79ovFUuZV/VE8eY3JOdK5tKIBWb31YUOjP7ccBes7mMhFLO3ceNeh+a6KAJbQ4pCojJwf/cLz663FKr5f/uWDicOBbL64l3+zV5zvSDzFls0ImXMNL6Fe3SaKP7ZcC5rVrRD8P+UN/OSFmbN5LM7uYY8nNsLxTH7MYsRHgTBUmTsFEhLGJIUjtf6J3/NWIlxjBq1MmpgxN0bD6gwVAxDPP489v918tsZtKdG8SJhLUPE4LWKsU7LHpgUBroKlbGE,iv:1jnF2TTlyTR59xM8Bgaz6bubDOwFexHBJipNVa0VPXY=,tag:VsDb6C6wYa9p4Yey3iG4eA==,type:str]
|
||||
bitbucket-runners:
|
||||
wopus-runner-1: ENC[AES256_GCM,data:gtH0T5n8qMYpvSv5ciN8+ScGlFDf9xE0FTxNP97vT/qsOCcaItTE+5P+DFcWw46onLED+1c+u0sArFbEsT3f8lyco9b+0l99uOQAxLZQzAXYH8zGye1UnwUtytkci2PHu5c8kTpIWHXyZ1IOYNGWkermeab57ANzOkM1LbkHyAjS6VTh0I60LfAOdHOw5FDFL8d1d9oWxLloOe9USLPqHjC023EpCUT2YuyHoPCTpBu8Kb/2HfV0wkAKaB3dvVrKwXCj+bfP6+bjQ3uMzVO/7jxPmnSGBfvyZ+Hlg5goJ6bSAqQWmnPPnQ96FgQfe8su5ML9qNIp9/7eNiL6Rv6Vhxe0hHbE5wsZ/58grcg/LrugeWJvUJ9THhwcTwO8Pkvwlq0XM9seUY2NV+LCK3bLQ4IWDjWkU1IHg6+nihTcvl1iD6UIGMgqGoB/v05WVzHb+GcE2fFuSuhVHfa5RMyboELOJoFrqZiXGhY=,iv:ZakLafxYQCDd1Zw8T83Xfj+YwAQKna9LC6ognJqtifA=,tag:bwBObfdMIvJfRrOG04NtxA==,type:str]
|
||||
wopus-runner-2: ENC[AES256_GCM,data:gg8merZMFbf396hdJY7zmKQndT3GzB7NeGZAs3C0au8Zd7OFAg9vcQcFcxNA3kZGJZqmFTR/ycWJwhYr9fhlfFuPhDynVvgJAqoYtvC2MUDiOMD/d3DlfwFjQ6cOGTrvFuY1kkgSFb4OFdrVC1eiTDrGygFmYnYcqTKn/t5Ttqi+cHZNzFzVzdVLvaLCYxltM5g45zn+fXYxYwCfqyb32/M1XTnnwIGiataGxEX5oWhVV4zqeLO4ZIYPSby5AVvIMJ/zqvqaeVVY52GLDcTKrj3thbZxMQLWN3/lOA0uYhi3L/WM8Gx+JMEIbSICcuT7QXu4w4PA+opcx9GnsMCK2/egzS+cNPJ4vGZCdVD/jh6A9zVEJAgXdsHXNXFHmMPt7DcgrCQiub62og4kBY4G/Rcg4UN7sb3v3qyBpGbCGHGRjCFc+wdHpom0yDOG2cwcqfN49pC2R7Ag2BisFQ/5A+DPmKnvGG3kt9s=,iv:5g5XiDecYqi4JNRkZubgPJECBQdZ6rBeojgFe6Etebk=,tag:HRy5bFSbfxKTb5e13lGtgg==,type:str]
|
||||
|
|
@ -31,8 +32,8 @@ sops:
|
|||
aFVxcDFhaGdYekRWRVFIWnRsZndtZFkKgsvxOFHOcO306Z9FkucA1fDOpZA8N1/h
|
||||
jYmIgcKTFgWoSCvux67lK30jFsYp7sm5z6WxxDYsGcoQ/+pxoUX2jQ==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
lastmodified: "2025-05-28T03:04:52Z"
|
||||
mac: ENC[AES256_GCM,data:THwZcK7nJnCYEUR8CiaQKZ8dQpYbDqnshBBWFzEzPXEWLgFB9+7d6aRh9ZDjZs0rhBTChta3H7YxDJdFh5nAJQy532FJp4S4tBOLHWFZARlKhXngujd0SvxPER55uvxImNFIYX0RDSHUck5jDXCA0tBCmE/Q7DuY7v0+cmRgOV8=,iv:1p3kFMSg0k1n00P6UY5Tttuqvpsb4Se8km5zA9GhAu4=,tag:cDxbHZ+eScDQacwV1sYGIA==,type:str]
|
||||
lastmodified: "2025-06-03T01:18:30Z"
|
||||
mac: ENC[AES256_GCM,data:KugjzfnFVdkR7sTDpeFXyf75PRvWx0Dyj+ZCxFo7rQcjkGBr/MqoXqvImIkmL3xTBrr0I+eRYYkn2wdAe+yw5ZY5r7/QJENNt55bR4h8KgXEebCXERdnCejv/in0LWTD887kC5LlYzK/oZbBgLgDu4btdxn1/6balc08uL4Wn10=,iv:SjTCJCOh5j+a0rrbwjiq77nQ+3M4EEOcOYoWQP0nFS8=,tag:cwfNn/VPzh4s7eFnDKx5yA==,type:str]
|
||||
pgp:
|
||||
- created_at: "2025-03-07T22:49:16Z"
|
||||
enc: |-
|
||||
|
|
|
|||
|
|
@ -1,55 +1,95 @@
|
|||
{ pkgs, lib, ... }:
|
||||
let
|
||||
installNixScript = pkgs.writeScriptBin "install-nix" ''
|
||||
mkdir -p -m 0755 /nix/var/log/nix/drvs
|
||||
mkdir -p -m 0755 /nix/var/nix/gcroots
|
||||
mkdir -p -m 0755 /nix/var/nix/profiles
|
||||
mkdir -p -m 0755 /nix/var/nix/temproots
|
||||
mkdir -p -m 0755 /nix/var/nix/userpool
|
||||
mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
|
||||
mkdir -p -m 1777 /nix/var/nix/profiles/per-user
|
||||
mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
|
||||
mkdir -p -m 0700 "$HOME/.nix-defexpr"
|
||||
installNixScript =
|
||||
{
|
||||
authenticationTokenConfigFile,
|
||||
nixCacheSshPrivateKeyPath ? null,
|
||||
...
|
||||
}:
|
||||
pkgs.writeScriptBin "install-nix" ''
|
||||
mkdir -p -m 0755 /nix/var/log/nix/drvs
|
||||
mkdir -p -m 0755 /nix/var/nix/gcroots
|
||||
mkdir -p -m 0755 /nix/var/nix/profiles
|
||||
mkdir -p -m 0755 /nix/var/nix/temproots
|
||||
mkdir -p -m 0755 /nix/var/nix/userpool
|
||||
mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
|
||||
mkdir -p -m 1777 /nix/var/nix/profiles/per-user
|
||||
mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
|
||||
mkdir -p -m 0700 "$HOME/.nix-defexpr"
|
||||
|
||||
. ${pkgs.nix}/etc/profile.d/nix.sh
|
||||
. ${pkgs.nix}/etc/profile.d/nix.sh
|
||||
|
||||
${pkgs.nix}/bin/nix-env -i ${
|
||||
lib.concatStringsSep " " (
|
||||
with pkgs;
|
||||
[
|
||||
nix
|
||||
cacert
|
||||
git
|
||||
openssh
|
||||
docker
|
||||
]
|
||||
)
|
||||
}
|
||||
'';
|
||||
${pkgs.nix}/bin/nix-env -i ${
|
||||
lib.concatStringsSep " " (
|
||||
with pkgs;
|
||||
[
|
||||
nix
|
||||
cacert
|
||||
git
|
||||
openssh
|
||||
docker
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
${lib.optionalString (nixCacheSshPrivateKeyPath != null) ''
|
||||
NIX_CACHE_SSH_PRIVATE_KEY_PATH="${nixCacheSshPrivateKeyPath}"
|
||||
. ${./gitlab-runner/nix-cache-start}
|
||||
''}
|
||||
'';
|
||||
|
||||
pushStoreContents =
|
||||
{
|
||||
authenticationTokenConfigFile,
|
||||
nixCacheSshPrivateKeyPath ? null,
|
||||
...
|
||||
}:
|
||||
pkgs.writeScriptBin "push-to-cache" ''
|
||||
${lib.optionalString (nixCacheSshPrivateKeyPath != null) ''
|
||||
. ${./gitlab-runner/nix-cache-end}
|
||||
''}
|
||||
'';
|
||||
in
|
||||
{
|
||||
mkNixRunner = authenticationTokenConfigFile: {
|
||||
# File should contain at least these two variables:
|
||||
# `CI_SERVER_URL`
|
||||
# `REGISTRATION_TOKEN`
|
||||
inherit authenticationTokenConfigFile; # 2
|
||||
dockerImage = "alpine:3.18.2";
|
||||
dockerVolumes = [
|
||||
"/etc/nix/nix.conf:/etc/nix/nix.conf:ro"
|
||||
"/nix/store:/nix/store:ro"
|
||||
"/nix/var/nix/db:/nix/var/nix/db:ro"
|
||||
"/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
|
||||
"/tmp:/tmp"
|
||||
"/var/run/docker.sock:/var/run/docker.sock"
|
||||
"/var/lib/docker/containers:/var/lib/docker/containers"
|
||||
];
|
||||
dockerDisableCache = true;
|
||||
preBuildScript = "\". ${lib.getExe installNixScript}\"";
|
||||
environmentVariables = {
|
||||
ENV = "/etc/profile";
|
||||
USER = "root";
|
||||
NIX_REMOTE = "daemon";
|
||||
NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt";
|
||||
rec {
|
||||
mkNixRunnerFull =
|
||||
{
|
||||
authenticationTokenConfigFile,
|
||||
nixCacheSshPrivateKeyPath ? null,
|
||||
...
|
||||
}@args:
|
||||
{
|
||||
# File should contain at least these two variables:
|
||||
# `CI_SERVER_URL`
|
||||
# `REGISTRATION_TOKEN`
|
||||
inherit authenticationTokenConfigFile; # 2
|
||||
dockerImage = "alpine:3.18.2";
|
||||
dockerVolumes =
|
||||
[
|
||||
"/etc/nix/nix.conf:/etc/nix/nix.conf:ro"
|
||||
"/nix/store:/nix/store:ro"
|
||||
"/nix/var/nix/db:/nix/var/nix/db:ro"
|
||||
"/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
|
||||
"/tmp:/tmp"
|
||||
"/var/run/docker.sock:/var/run/docker.sock"
|
||||
"/var/lib/docker/containers:/var/lib/docker/containers"
|
||||
]
|
||||
++ lib.optionals (nixCacheSshPrivateKeyPath != null) [
|
||||
"${nixCacheSshPrivateKeyPath}:${nixCacheSshPrivateKeyPath}"
|
||||
];
|
||||
dockerDisableCache = true;
|
||||
preBuildScript = "\". ${lib.getExe (installNixScript args)}\"";
|
||||
postBuildScript = "\". ${lib.getExe (pushStoreContents args)}\"";
|
||||
environmentVariables = {
|
||||
ENV = "/etc/profile";
|
||||
USER = "root";
|
||||
NIX_REMOTE = "daemon";
|
||||
NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt";
|
||||
};
|
||||
};
|
||||
|
||||
mkNixRunner =
|
||||
authenticationTokenConfigFile:
|
||||
mkNixRunnerFull {
|
||||
inherit authenticationTokenConfigFile;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
21
system/gitlab-runner/nix-cache-end
Executable file
21
system/gitlab-runner/nix-cache-end
Executable file
|
|
@ -0,0 +1,21 @@
|
|||
#!/bin/sh
|
||||
|
||||
echo "nix-cache: Storing new store items"
|
||||
NEW_NIX_STORE_CONTENTS_FILE=$(mktemp)
|
||||
find /nix/store/ -maxdepth 1 > $NEW_NIX_STORE_CONTENTS_FILE
|
||||
|
||||
sort $OLD_NIX_STORE_CONTENTS_FILE -o $OLD_NIX_STORE_CONTENTS_FILE
|
||||
sort $NEW_NIX_STORE_CONTENTS_FILE -o $NEW_NIX_STORE_CONTENTS_FILE
|
||||
|
||||
echo "nix-cache: Comparing store paths"
|
||||
FILTERED_NIX_STORE_CONTENTS_FILE=$(mktemp)
|
||||
comm -13 $OLD_NIX_STORE_CONTENTS_FILE $NEW_NIX_STORE_CONTENTS_FILE > $FILTERED_NIX_STORE_CONTENTS_FILE
|
||||
echo "nix-cache: New store paths:"
|
||||
cat $FILTERED_NIX_STORE_CONTENTS_FILE | sed 's/^/ /g'
|
||||
|
||||
if test -n "$(head -n1 $FILTERED_NIX_STORE_CONTENTS_FILE)"; then
|
||||
echo "nix-cache: Sending new paths to cache"
|
||||
nix copy --to "$STORE_URL" $(cat $FILTERED_NIX_STORE_CONTENTS_FILE) || true
|
||||
else
|
||||
echo "nix-cache: Nothing to send"
|
||||
fi
|
||||
18
system/gitlab-runner/nix-cache-start
Executable file
18
system/gitlab-runner/nix-cache-start
Executable file
|
|
@ -0,0 +1,18 @@
|
|||
#!/bin/sh
|
||||
|
||||
echo "nix-cache: Setting up ssh key and host"
|
||||
STORE_HOST_PUB_KEY="IyBuaXgtY2FjaGUud29wdXMuZGV2OjIyIFNTSC0yLjAtT3BlblNTSF8xMC4wCm5peC1jYWNoZS53b3B1cy5kZXYgc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU5VNzFONVF4ZENtTTdOMjVTbk9nNnUrWUxtdjkyem5wZURjeUlEYW1sZEkK"
|
||||
STORE_URL="ssh://nix-ssh@nix-cache.wopus.dev?trusted=true&compress=true&ssh-key=$NIX_CACHE_SSH_PRIVATE_KEY_PATH&base64-ssh-public-host-key=$STORE_HOST_PUB_KEY"
|
||||
echo STORE_URL="$STORE_URL"
|
||||
|
||||
NIX_EXTRA_CONFIG_FILE=$(mktemp)
|
||||
cat > "$NIX_EXTRA_CONFIG_FILE" <<EOF
|
||||
extra-substituters = $STORE_URL
|
||||
EOF
|
||||
|
||||
echo "nix-cache: Adding remote cache as substituter"
|
||||
export NIX_USER_CONF_FILES="$NIX_EXTRA_CONFIG_FILE:$NIX_USER_CONF_FILES"
|
||||
|
||||
echo "nix-cache: Storing existing store items"
|
||||
OLD_NIX_STORE_CONTENTS_FILE=$(mktemp)
|
||||
find /nix/store/ -maxdepth 1 > $OLD_NIX_STORE_CONTENTS_FILE
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
...
|
||||
}:
|
||||
let
|
||||
inherit (pkgs.callPackage ./gitlab-runner.nix { }) mkNixRunner;
|
||||
inherit (pkgs.callPackage ./gitlab-runner.nix { }) mkNixRunner mkNixRunnerFull;
|
||||
in
|
||||
{
|
||||
boot.kernel.sysctl."net.ipv4.ip_forward" = true;
|
||||
|
|
@ -18,7 +18,10 @@ in
|
|||
thoreb-telemetria-nix = mkNixRunner config.sops.secrets."gitlab-runners/thoreb-telemetria-nix".path;
|
||||
thoreb-itinerario-nix = mkNixRunner config.sops.secrets."gitlab-runners/thoreb-itinerario-nix".path;
|
||||
|
||||
wopus-gitlab-nix = mkNixRunner config.sops.secrets."gitlab-runners/wopus-gitlab-nix".path;
|
||||
wopus-gitlab-nix = mkNixRunnerFull {
|
||||
authenticationTokenConfigFile = config.sops.secrets."gitlab-runners/wopus-gitlab-nix".path;
|
||||
nixCacheSshPrivateKeyPath = config.sops.secrets."gitlab-runners/wopus-ssh-nix-cache-pk".path;
|
||||
};
|
||||
|
||||
default = {
|
||||
# File should contain at least these two variables:
|
||||
|
|
@ -56,5 +59,8 @@ in
|
|||
"gitlab-runners/wopus-gitlab-docker-images" = {
|
||||
sopsFile = ../secrets/monolith/default.yaml;
|
||||
};
|
||||
"gitlab-runners/wopus-ssh-nix-cache-pk" = {
|
||||
sopsFile = ../secrets/monolith/default.yaml;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue