Merge pull request #1 from DeterminateSystems/port

Init the action

.envrc

@@ -0,0 +1,5 @@
+if ! has nix_direnv_version || ! nix_direnv_version 2.1.1; then
+  source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.1.1/direnvrc" "sha256-b6qJ4r34rbE23yWjMqbmu3ia2z4b2wIlZUksBke/ol0="
+fi
+
+use_flake

.gitattributes

@@ -0,0 +1 @@
+dist/** linguist-generated=true

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,12 @@
+##### Description
+
+<!---
+Please include a short description of what your PR does and / or the motivation
+behind it
+--->
+
+##### Checklist
+
+- [ ] Tested changes against a test repository
+- [ ] Added or updated relevant documentation (leave unchecked if not applicable)
+- [ ] (If this PR is for a release) Updated README to point to the new tag (leave unchecked if not applicable)

.github/workflows/cache-test.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+set -ux
+
+seed=$(date)
+
+outpath=$(nix-build .github/workflows/cache-tester.nix --argstr seed "$seed")
+nix copy --to 'http://127.0.0.1:37515' "$outpath"
+rm ./result
+nix store delete "$outpath"
+if [ -f "$outpath" ]; then
+    echo "$outpath still exists? can't test"
+    exit 1
+fi
+
+rm -rf ~/.cache/nix
+
+echo "-------"
+echo "Trying to substitute the build again..."
+echo "if it fails, the cache is broken."
+
+nix-store --realize -vvvvvvvv "$outpath"

.github/workflows/cache-tester.nix

@@ -0,0 +1,10 @@
+{ seed }:
+derivation {
+  name = "cache-test";
+  system = builtins.currentSystem;
+
+  builder = "/bin/sh";
+  args = [ "-euxc" "echo \"$seed\" > $out" ];
+
+  inherit seed;
+}

.github/workflows/ci.yml

@@ -0,0 +1,65 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+      - name: Record existing bundle hash
+        run: |
+          echo "BUNDLE_HASH=$(sha256sum <dist/index.js | sed 's/  -//')" >>$GITHUB_ENV
+      - name: Build action
+        run: |
+          nix develop --command -- just build
+      - name: Check bundle consistency
+        run: |
+          NEW_BUNDLE_HASH=$(sha256sum <dist/index.js | sed 's/  -//')
+          if [[ "$BUNDLE_HASH" != "$NEW_BUNDLE_HASH" ]]; then
+            >&2 echo "The committed dist/index.js is out-of-date!"
+            >&2 echo
+            >&2 echo "    Committed: $BUNDLE_HASH"
+            >&2 echo "        Built: $NEW_BUNDLE_HASH"
+            >&2 echo
+            >&2 echo 'Run `just build` then commit the resulting dist/index.js'
+            exit 1
+          fi
+
+  run-x86_64-linux:
+    name: Run x86_64 Linux
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+        with:
+          extra-conf: |
+            narinfo-cache-negative-ttl = 0
+      - name: Cache the store
+        uses: ./
+      - name: Check the cache for liveness
+        run: |
+          .github/workflows/cache-test.sh
+
+  # run-x86_64-darwin:
+  #   name: Run x86_64 Darwin
+  #   runs-on: macos-12
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - name: Install Nix
+  #       uses: DeterminateSystems/nix-installer-action@v4
+  #       with:
+  #         extra-conf: |
+  #           narinfo-cache-negative-ttl = 0
+  #     - name: Cache the store
+  #       uses: ./
+  #     - name: Check the cache for liveness
+  #       run: |
+  #         .github/workflows/cache-test.sh

.gitignore

@@ -0,0 +1,8 @@
+.direnv
+result*
+
+node_modules
+
+# magic-nix-cache
+creds.env
+creds.json

LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2023 Determinate Systems, Inc., Zhaofeng Li
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,2 +1,43 @@
 # The Magic Nix Cache Action
 
+Cut CI time by 50% or more by caching to GitHub Actions' cache, for free. No configuration required.
+
+<!--
+cat action.yml| nix run nixpkgs#yq-go -- '[[ "Parameter", "Description", "Required", "Default" ], ["-", "-", "-", "-"]] + [.inputs | to_entries | sort_by(.key) | .[] | ["`" + .key + "`", .value.description, .value.required // "", .value.default // ""]] | map(join(" | ")) | .[] | "| " + . + " |"' -r
+-->
+
+```yaml
+name: Flake Check
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  flake-check:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: DeterminateSystems/flake-checker-action@v4
+        with:
+          fail-mode: true
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@v4
+      - uses: DeterminateSystems/nix-installer-action-cache@focus-on-cache
+
+      - name: "Nix Flake Check"
+        run: nix flake check . -L
+```
+
+| Parameter | Description | Required | Default |
+| - | - | - | - |
+| `diagnostic-endpoint` | Diagnostic endpoint url where diagnostics and performance data is sent. To disable set this to an empty string. |  | https://install.determinate.systems/magic-nix-cache/perf |
+| `listen` | The host and port to listen on. |  | 127.0.0.1:37515 |
+| `source-binary` | Run a version of the cache binary from somewhere already on disk. Conflicts with all other `source-*` options. |  |  |
+| `source-branch` | The branch of `magic-nix-cache` to use. Conflicts with all other `source-*` options. |  |  |
+| `source-pr` | The PR of `magic-nix-cache` to use. Conflicts with all other `source-*` options. |  |  |
+| `source-revision` | The revision of `nix-magic-nix-cache` to use. Conflicts with all other `source-*` options. |  |  |
+| `source-tag` | The tag of `magic-nix-cache` to use. Conflicts with all other `source-*` options. |  |  |
+| `source-url` | A URL pointing to a `magic-nix-cache` binary. Overrides all other `source-*` options. |  |  |
+| `upstream-cache` | Your preferred upstream cache. Store paths in this store will not be cached in GitHub Actions' cache. |  | https://cache.nixos.org |

action.yml

@@ -0,0 +1,38 @@
+name: Magic Nix Cache
+branding:
+  icon: "box"
+  color: "purple"
+description: "Free, no-configuration Nix cache. Cut CI time by 50% or more by caching to GitHub Actions' cache."
+inputs:
+  listen:
+    description: The host and port to listen on.
+    default: 127.0.0.1:37515
+  upstream-cache:
+    description: Your preferred upstream cache. Store paths in this store will not be cached in GitHub Actions' cache.
+    default: https://cache.nixos.org
+  source-binary:
+    description: Run a version of the cache binary from somewhere already on disk. Conflicts with all other `source-*` options.
+  source-branch:
+    description: The branch of `magic-nix-cache` to use. Conflicts with all other `source-*` options.
+    required: false
+    default: main
+  source-pr:
+    description: The PR of `magic-nix-cache` to use. Conflicts with all other `source-*` options.
+    required: false
+  source-revision:
+    description: The revision of `nix-magic-nix-cache` to use. Conflicts with all other `source-*` options.
+    required: false
+  source-tag:
+    description: The tag of `magic-nix-cache` to use. Conflicts with all other `source-*` options.
+    required: false
+  source-url:
+    description: A URL pointing to a `magic-nix-cache` binary. Overrides all other `source-*` options.
+    required: false
+  diagnostic-endpoint:
+    description: "Diagnostic endpoint url where diagnostics and performance data is sent. To disable set this to an empty string."
+    default: "https://install.determinate.systems/magic-nix-cache/perf"
+
+runs:
+  using: "node16"
+  main: "./dist/index.js"
+  post: "./dist/index.js"

flake.lock

@@ -0,0 +1,44 @@
+{
+  "nodes": {
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1684385584,
+        "narHash": "sha256-O7y0gK8OLIDqz+LaHJJyeu09IGiXlZIS3+JgEzGmmJA=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "48a0fb7aab511df92a17cf239c37f2bd2ec9ae3a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,33 @@
+{
+  description = "Magic Nix Cache";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+
+    flake-compat = {
+      url = "github:edolstra/flake-compat";
+      flake = false;
+    };
+  };
+
+  outputs = { self, nixpkgs, ... }:
+    let
+      supportedSystems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
+      forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f {
+        pkgs = import nixpkgs { inherit system; };
+      });
+    in
+    {
+      devShells = forAllSystems ({ pkgs }: {
+        default = pkgs.mkShell {
+          packages = with pkgs; [
+            bun
+            nodejs
+            jq
+            act
+            just
+          ];
+        };
+      });
+    };
+}

justfile

@@ -0,0 +1,21 @@
+# A creds.json can be specified to test auto-caching locally.
+# See <https://github.com/DeterminateSystems/magic-nix-cache> for
+# instructions on how to obtain this file.
+github_creds_json := env_var_or_default('GITHUB_CREDS_JSON', '')
+
+# List available recipes
+default:
+  @just --list --unsorted --justfile {{justfile()}}
+
+# Install dependencies
+install:
+  bun i --no-summary --ignore-scripts
+
+# Build the action
+build: install
+  bun run build
+
+# Run CI locally
+act job='run-x86_64-linux': build
+  act -j {{job}} -P ubuntu-22.04=catthehacker/ubuntu:act-22.04 \
+    {{ if github_creds_json != '' { "--env-file <(jq -r '. | to_entries[] | .key + \"=\" + .value' " + github_creds_json + ")" } else { '' } }}

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "magic-nix-cache",
+  "version": "1.0.0",
+  "description": "",
+  "type": "module",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "rollup -c"
+  },
+  "license": "LGPL",
+  "dependencies": {
+    "@actions/core": "^1.10.0",
+    "tail": "^2.2.6",
+    "tslib": "^2.5.2",
+    "got": "^12.6.0"
+  },
+  "devDependencies": {
+    "@rollup/plugin-commonjs": "^25.0.0",
+    "@rollup/plugin-node-resolve": "^15.0.2",
+    "@rollup/plugin-typescript": "^11.1.1",
+    "@types/node": "^20.2.1",
+    "rollup": "^3.22.0",
+    "typescript": "^5.0.4"
+  }
+}

rollup.config.js

@@ -0,0 +1,47 @@
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import typescript from '@rollup/plugin-typescript';
+import { nodeResolve } from '@rollup/plugin-node-resolve';
+import commonjs from '@rollup/plugin-commonjs';
+
+const nodeModules = path.dirname(fileURLToPath(import.meta.url)) + '/node_modules';
+
+export default {
+  input: './src/index.ts',
+  output: {
+    file: './dist/index.js',
+    format: 'es',
+  },
+  plugins: [
+    typescript({
+      noEmitOnError: true,
+    }),
+    nodeResolve({
+      exportConditions: ['node', 'default', 'module', 'require'],
+    }),
+    commonjs(),
+  ],
+  onwarn(warning, warn) {
+    const allowlist = {
+      'CIRCULAR_DEPENDENCY': [
+        // core.ts -> oidc-utils.ts -> core.ts
+        nodeModules + '/@actions/core/lib/',
+      ],
+      'THIS_IS_UNDEFINED': [
+        // __classPrivateField{Get,Set} generated by TypeScript
+        nodeModules + '/form-data-encoder/lib/',
+      ],
+    };
+
+    if (allowlist.hasOwnProperty(warning.code)) {
+      for (const exception of allowlist[warning.code]) {
+        const ids = warning.ids || [ warning.id ];
+        if (ids.filter(p => !p.startsWith(exception)).length === 0) {
+          return;
+        }
+      }
+    }
+    warn(warning);
+  },
+};

shell.nix

@@ -0,0 +1,15 @@
+let
+  lock = builtins.fromJSON (builtins.readFile ./flake.lock);
+
+  flake-compat = builtins.fetchTarball {
+    url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+    sha256 = lock.nodes.flake-compat.locked.narHash;
+  };
+
+  flake = import flake-compat {
+    src = ./.;
+  };
+
+  shell = flake.shellNix.default;
+in
+shell

src/index.ts

@@ -0,0 +1,215 @@
+// Main
+
+import * as fs from 'node:fs/promises';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { spawn } from 'node:child_process';
+import { createWriteStream, openSync, writeSync, close } from 'node:fs';
+import { pipeline } from 'node:stream/promises';
+import { setTimeout } from 'timers/promises';
+
+import * as core from '@actions/core';
+import { Tail } from 'tail';
+import got from "got";
+
+const ENV_CACHE_DAEMONDIR = 'MAGIC_NIX_CACHE_DAEMONDIR';
+
+function getCacherUrl() : string {
+  const runnerArch = process.env.RUNNER_ARCH;
+  const runnerOs = process.env.RUNNER_OS;
+  const binarySuffix = `${runnerArch}-${runnerOs}`;
+  const urlPrefix = `https://install.determinate.systems/magic-nix-cache`;
+
+  if (core.getInput('source-url')) {
+    return core.getInput('source-url');
+  }
+
+  if (core.getInput('source-tag')) {
+    return `${urlPrefix}/tag/${core.getInput('source-tag')}/${binarySuffix}`;
+  }
+
+  if (core.getInput('source-pr')) {
+    return `${urlPrefix}/pr/${core.getInput('source-pr')}/${binarySuffix}`;
+  }
+
+  if (core.getInput('source-branch')) {
+    return `${urlPrefix}/branch/${core.getInput('source-branch')}/${binarySuffix}`;
+  }
+
+  if (core.getInput('source-revision')) {
+    return `${urlPrefix}/rev/${core.getInput('source-revision')}/${binarySuffix}`;
+  }
+
+  return `${urlPrefix}/latest/${binarySuffix}`;
+}
+
+async function fetchAutoCacher(destination: string) {
+  const stream = createWriteStream(destination, {
+    encoding: "binary",
+    mode: 0o755,
+  });
+
+  const binary_url = getCacherUrl();
+  core.debug(`Fetching the Magic Nix Cache from ${binary_url}`);
+
+  return pipeline(
+    got.stream(binary_url),
+    stream
+  );
+}
+
+async function setUpAutoCache() {
+  const tmpdir = process.env['RUNNER_TEMP'] || os.tmpdir();
+  const required_env = ['ACTIONS_CACHE_URL', 'ACTIONS_RUNTIME_URL', 'ACTIONS_RUNTIME_TOKEN'];
+
+  var anyMissing = false;
+  for (const n of required_env) {
+    if (!process.env.hasOwnProperty(n)) {
+      anyMissing = true;
+      core.warning(`Disabling automatic caching since required environment ${n} isn't available`);
+    }
+  }
+
+  if (anyMissing) {
+    return;
+  }
+
+  core.debug(`GitHub Action Cache URL: ${process.env['ACTIONS_CACHE_URL']}`);
+
+  const daemonDir = await fs.mkdtemp(path.join(tmpdir, 'magic-nix-cache-'));
+
+  var daemonBin: string;
+  if (core.getInput('source-binary')) {
+    daemonBin = core.getInput('source-binary');
+  } else {
+    daemonBin = `${daemonDir}/magic-nix-cache`;
+    await fetchAutoCacher(daemonBin);
+  }
+
+  var runEnv;
+  if (core.isDebug()) {
+    runEnv = {
+      RUST_LOG: "trace,nix_actions_cache=debug,gha_cache=debug",
+      RUST_BACKTRACE: "full",
+      ...process.env
+    };
+  } else {
+    runEnv = process.env;
+  }
+
+  const output = openSync(`${daemonDir}/parent.log`, 'a');
+  const launch = spawn(
+    daemonBin,
+    [
+      '--daemon-dir', daemonDir,
+      '--listen', core.getInput('listen'),
+      '--upstream', core.getInput('upstream-cache'),
+      '--diagnostic-endpoint', core.getInput('diagnostic-endpoint')
+    ],
+    {
+      stdio: ['ignore', output, output],
+      env: runEnv
+    }
+  );
+
+  await new Promise<void>((resolve, reject) => {
+    launch.on('exit', (code, signal) => {
+      if (signal) {
+        reject(new Error(`Daemon was killed by signal ${signal}`));
+      } else if (code) {
+        reject(new Error(`Daemon exited with code ${code}`));
+      } else {
+        resolve();
+      }
+    });
+  });
+
+  await fs.mkdir(`${process.env["HOME"]}/.config/nix`, { recursive: true });
+  const nixConf = openSync(`${process.env["HOME"]}/.config/nix/nix.conf`, 'a');
+  writeSync(nixConf, `${"\n"}extra-substituters = http://${core.getInput('listen')}/?trusted=1&compression=zstd&parallel-compression=true${"\n"}`);
+  close(nixConf);
+
+  core.debug('Launched Magic Nix Cache');
+  core.exportVariable(ENV_CACHE_DAEMONDIR, daemonDir);
+}
+
+async function notifyAutoCache() {
+  const daemonDir = process.env[ENV_CACHE_DAEMONDIR];
+
+  if (!daemonDir) {
+    return;
+  }
+
+  const res: any = await got.post(`http://${core.getInput('listen')}/api/workflow-start`).json();
+  core.debug(res);
+}
+
+async function tearDownAutoCache() {
+  const daemonDir = process.env[ENV_CACHE_DAEMONDIR];
+
+  if (!daemonDir) {
+    core.debug('magic-nix-cache not started - Skipping');
+    return;
+  }
+
+  const pidFile = path.join(daemonDir, 'daemon.pid');
+  const pid = parseInt(await fs.readFile(pidFile, { encoding: 'ascii' }));
+  core.debug(`found daemon pid: ${pid}`);
+  if (!pid) {
+    throw new Error("magic-nix-cache did not start successfully");
+  }
+
+  const log = new Tail(path.join(daemonDir, 'daemon.log'));
+  core.debug(`tailing daemon.log...`);
+  log.on('line', (line) => {
+    core.debug(`got a log line`);
+    core.info(line);
+  });
+
+
+  try {
+    core.debug(`about to post to localhost`);
+    const res: any = await got.post(`http://${core.getInput('listen')}/api/workflow-finish`).json();
+    core.debug(`back from post`);
+    core.debug(res);
+  } finally {
+    await setTimeout(5000);
+
+    core.debug(`unwatching the daemon log`);
+    log.unwatch();
+  }
+
+  core.debug(`killing`);
+  try {
+    process.kill(pid, 'SIGTERM');
+  } catch (e) {
+    if (e.code !== 'ESRCH') {
+      throw e;
+    }
+  }
+}
+
+const isPost = !!process.env['STATE_isPost'];
+
+try {
+  if (!isPost) {
+    core.saveState('isPost', 'true');
+    await setUpAutoCache();
+    await notifyAutoCache();
+  } else {
+    await tearDownAutoCache();
+  }
+} catch (e) {
+  core.info(`got an exception:`);
+  core.info(e);
+
+  if (!isPost) {
+    core.setFailed(e.message);
+    throw e;
+  } else {
+    core.info("not considering this a failure: finishing the upload is optional, anyway.");
+    process.exit();
+  }}
+
+core.debug(`rip`);
+

tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "compilerOptions": {
+    "target": "es2020",
+    "module": "es2020",
+    "noUnusedLocals": true
+  },
+  "include": ["src/**/*"]
+}