nix: use /var/tmp/ during builds

hosts/delta/knot-resolver.nix

@@ -39,38 +39,40 @@
       "159.69.4.2:443"
       "[2a01:4f8:1c0c:70d1::1]:443"
     ];
-    extraConfig = ''
-      -- TLS certificate for DoT and DoH
-      -- https://knot-resolver.readthedocs.io/en/stable/daemon-bindings-net_tlssrv.html
-      net.tls(
-        "${config.security.acme.certs."caspervk.net".directory}/fullchain.pem",
-        "${config.security.acme.certs."caspervk.net".directory}/key.pem"
-      )
-      -- Cache is stored in /var/cache/knot-resolver, which is mounted as
-      -- tmpfs. Allow using 75% of the partition for caching.
-      -- https://knot-resolver.readthedocs.io/en/stable/daemon-bindings-cache.html
-      cache.size = math.floor(cache.fssize() * 0.75)
-      -- The predict module helps to keep the cache hot by prefetching
-      -- records. Any time the resolver answers with records that are about to
-      -- expire, they get refreshed.
-      -- https://knot-resolver.readthedocs.io/en/stable/modules-predict.html
-      modules.load("predict")
-      -- Block spam and advertising domains
-      -- https://knot-resolver.readthedocs.io/en/stable/modules-policy.html#response-policy-zones
-      policy.add(
-        policy.rpz(
-          policy.ANSWER({ [kres.type.A] = {rdata=kres.str2ip("0.0.0.0"), ttl = 600} }),
-          "${pkgs.runCommand "stevenblack-blocklist-rpz" {} ''grep '^0\.0\.0\.0' ${pkgs.stevenblack-blocklist}/hosts | awk '{print $2 " 600 IN CNAME .\n*." $2 " 600 IN CNAME ."}' > $out''}"
+    extraConfig =
+      # lua
+      ''
+        -- TLS certificate for DoT and DoH
+        -- https://knot-resolver.readthedocs.io/en/stable/daemon-bindings-net_tlssrv.html
+        net.tls(
+          "${config.security.acme.certs."caspervk.net".directory}/fullchain.pem",
+          "${config.security.acme.certs."caspervk.net".directory}/key.pem"
         )
-      )
-      -- Test domain to verify DNS server is being used
-      policy.add(
-        policy.domains(
-          policy.ANSWER({ [kres.type.A] = {rdata = kres.str2ip("192.0.2.0"), ttl = 5} }),
-          policy.todnames({"test.dns.caspervk.net"})
+        -- Cache is stored in /var/cache/knot-resolver, which is mounted as
+        -- tmpfs. Allow using 75% of the partition for caching.
+        -- https://knot-resolver.readthedocs.io/en/stable/daemon-bindings-cache.html
+        cache.size = math.floor(cache.fssize() * 0.75)
+        -- The predict module helps to keep the cache hot by prefetching
+        -- records. Any time the resolver answers with records that are about to
+        -- expire, they get refreshed.
+        -- https://knot-resolver.readthedocs.io/en/stable/modules-predict.html
+        modules.load("predict")
+        -- Block spam and advertising domains
+        -- https://knot-resolver.readthedocs.io/en/stable/modules-policy.html#response-policy-zones
+        policy.add(
+          policy.rpz(
+            policy.ANSWER({ [kres.type.A] = {rdata=kres.str2ip("0.0.0.0"), ttl = 600} }),
+            "${pkgs.runCommand "stevenblack-blocklist-rpz" {} ''grep '^0\.0\.0\.0' ${pkgs.stevenblack-blocklist}/hosts | awk '{print $2 " 600 IN CNAME .\n*." $2 " 600 IN CNAME ."}' > $out''}"
+          )
         )
-      )
-    '';
+        -- Test domain to verify DNS server is being used
+        policy.add(
+          policy.domains(
+            policy.ANSWER({ [kres.type.A] = {rdata = kres.str2ip("192.0.2.0"), ttl = 5} }),
+            policy.todnames({"test.dns.caspervk.net"})
+          )
+        )
+      '';
   };
 
   networking.firewall = {

modules/base/nix.nix

@@ -50,6 +50,19 @@
     frequency = config.nix.gc.dates;
   };
 
+  # Nix uses /tmp/ (tmpfs) during builds by default. This may cause 'No space
+  # left on device' errors with limited system memory or during big builds. Set
+  # the Nix daemon to use /var/tmp/ instead. Note that /var/tmp/ should ideally
+  # be on the same filesystem as /nix/store/ for faster copying of files.
+  # https://github.com/NixOS/nixpkgs/issues/54707
+  #
+  # NOTE: This does not change the directory for builds during `nixos-rebuild`.
+  # See overlays/nixos-rebuild.nix for workaround.
+  # https://github.com/NixOS/nixpkgs/issues/293114
+  systemd.services.nix-daemon = {
+    environment.TMPDIR = "/var/tmp";
+  };
+
   # Run unpatched dynamic binaries on NixOS.
   # https://github.com/Mic92/nix-ld
   programs.nix-ld.enable = true;

overlays/default.nix

@@ -1,5 +1,6 @@
 {...}: {
   imports = [
     ./neovim.nix
+    ./nixos-rebuild.nix
   ];
 }

overlays/neovim.nix

@@ -9,10 +9,10 @@
     imports = ["${home-manager-unstable}/modules/programs/neovim.nix"];
   };
   nixpkgs.overlays = [
-    (self: super: {
+    (final: prev: {
       # Home-manager uses the neovim-unwrapped package for the neovim module
-      neovim-unwrapped = nixpkgs-unstable.legacyPackages.${super.system}.neovim-unwrapped;
-      vimPlugins = nixpkgs-unstable.legacyPackages.${super.system}.vimPlugins;
+      neovim-unwrapped = nixpkgs-unstable.legacyPackages.${prev.system}.neovim-unwrapped;
+      vimPlugins = nixpkgs-unstable.legacyPackages.${prev.system}.vimPlugins;
     })
   ];
 }

overlays/nixos-rebuild.nix

@@ -0,0 +1,25 @@
+{...}: {
+  # The Nix daemon's temporary build directory is changed from /tmp/ to
+  # /var/tmp in modules/base/nix.nix, but it is only respected by `nix build`,
+  # not `nixos-rebuild`.
+  # This overlay wraps `nixos-rebuild` to explicitly set TMPDIR=/var/tmp.
+  # https://github.com/NixOS/nixpkgs/issues/293114
+  nixpkgs.overlays = [
+    (final: prev: {
+      # `overrideAttrs`, instead of simply overriding the `nixos-rebuild`
+      # package, to ensure `nixos-rebuild.override`, which is used in NixOS,
+      # works and is overridden.
+      # https://wiki.nixos.org/wiki/Nix_Cookbook#Wrapping_packages
+      # TODO: There must be a better way to do this?
+      nixos-rebuild = prev.nixos-rebuild.overrideAttrs (oldAttrs: {
+        nativeBuildInputs = oldAttrs.nativeBuildInputs ++ [prev.makeWrapper];
+        postInstall =
+          oldAttrs.postInstall
+          + ''
+            wrapProgram $out/bin/nixos-rebuild \
+              --set TMPDIR /var/tmp
+          '';
+      });
+    })
+  ];
+}