diff --git a/.github/workflows/cache-test.sh b/.github/workflows/cache-test.sh index c1f7eb1..d6159c9 100755 --- a/.github/workflows/cache-test.sh +++ b/.github/workflows/cache-test.sh @@ -1,14 +1,47 @@ -#!/bin/sh +#! /usr/bin/env bash set -e set -ux seed=$(date) +log="${MAGIC_NIX_CACHE_DAEMONDIR}/daemon.log" + +binary_cache=https://cache.flakehub.com + +# Check that the action initialized correctly. +grep 'FlakeHub cache is enabled' "${log}" +grep 'Using cache' "${log}" +grep 'GitHub Action cache is enabled' "${log}" + +# Build something. outpath=$(nix-build .github/workflows/cache-tester.nix --argstr seed "$seed") -nix copy --to 'http://127.0.0.1:37515' "$outpath" + +# Check that the path was enqueued to be pushed to the cache. +grep "Enqueueing.*${outpath}" "${log}" + +# Wait until it has been pushed succesfully. +found= +for ((i = 0; i < 60; i++)); do + sleep 1 + if grep "✅ $(basename "${outpath}")" "${log}"; then + found=1 + break + fi +done +if [[ -z $found ]]; then + echo "FlakeHub push did not happen." >&2 + exit 1 +fi + +# Check the FlakeHub binary cache to see if the path is really there. +nix path-info --store "${binary_cache}" "${outpath}" + +# FIXME: remove this once the daemon also uploads to GHA automatically. +nix copy --to 'http://127.0.0.1:37515' "${outpath}" + rm ./result -nix store delete "$outpath" +nix store delete "${outpath}" if [ -f "$outpath" ]; then echo "$outpath still exists? can't test" exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5bf051..9fdff01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,12 @@ jobs: - name: Record existing bundle hash run: | echo "BUNDLE_HASH=$(sha256sum >$GITHUB_ENV + - name: Check shell scripts + run: | + nix develop --command shellcheck ./.github/workflows/cache-test.sh - name: Build action run: | - nix develop --command -- just build + nix develop --command just build - name: Check bundle consistency run: | NEW_BUNDLE_HASH=$(sha256sum { // eslint-disable-next-line node/prefer-global/url const {URL: URL$4} = require$$0$5; const EventEmitter = require$$0$1; -const tls$3 = require$$1$1; +const tls$3 = require$$1$2; const http2$2 = require$$3; const QuickLRU$1 = quickLru; const delayAsyncDestroy$1 = delayAsyncDestroy$2; @@ -8299,7 +8299,7 @@ var clientRequest = ClientRequest$1; var auto$1 = {exports: {}}; -const tls$2 = require$$1$1; +const tls$2 = require$$1$2; var resolveAlpn = (options = {}, connect = tls$2.connect) => new Promise((resolve, reject) => { let timeout = false; @@ -8374,7 +8374,7 @@ var calculateServerName$1 = host => { // See https://github.com/facebook/jest/issues/2549 // eslint-disable-next-line node/prefer-global/url const {URL: URL$2, urlToHttpOptions} = require$$0$5; -const http$2 = require$$1$2; +const http$2 = require$$1__default; const https$2 = require$$2$1; const resolveALPN = resolveAlpn; const QuickLRU = quickLru; @@ -8580,7 +8580,7 @@ auto$1.exports.createResolveProtocol = createResolveProtocol; var autoExports = auto$1.exports; const stream = require$$0$3; -const tls$1 = require$$1$1; +const tls$1 = require$$1$2; // Really awesome hack. const JSStreamSocket$2 = (new tls$1.TLSSocket(new stream.PassThrough()))._handle._parentWrap.constructor; @@ -8653,8 +8653,8 @@ var getAuthHeaders = self => { return {}; }; -const tls = require$$1$1; -const http$1 = require$$1$2; +const tls = require$$1$2; +const http$1 = require$$1__default; const https$1 = require$$2$1; const JSStreamSocket$1 = jsStreamSocket; const {globalAgent: globalAgent$2} = agent; @@ -8815,7 +8815,7 @@ let Http2OverHttp2$1 = class Http2OverHttp2 extends Http2OverHttpX$1 { var h2OverH2 = Http2OverHttp2$1; -const http = require$$1$2; +const http = require$$1__default; const https = require$$2$1; const Http2OverHttpX = h2OverHx; const getAuthorizationHeaders = getAuthHeaders; @@ -12119,7 +12119,7 @@ function getCacherUrl() { const runnerArch = process.env.RUNNER_ARCH; const runnerOs = process.env.RUNNER_OS; const binarySuffix = `${runnerArch}-${runnerOs}`; - const urlPrefix = `https://install.determinate.systems/magic-nix-cache`; + const urlPrefix = `https://install.determinate.systems/magic-nix-cache-closure`; if (coreExports.getInput('source-url')) { return coreExports.getInput('source-url'); } @@ -12137,14 +12137,22 @@ function getCacherUrl() { } return `${urlPrefix}/stable/${binarySuffix}`; } -async function fetchAutoCacher(destination) { - const stream = createWriteStream(destination, { - encoding: "binary", - mode: 0o755, - }); +async function fetchAutoCacher() { const binary_url = getCacherUrl(); - coreExports.debug(`Fetching the Magic Nix Cache from ${binary_url}`); - return pipeline(gotClient.stream(binary_url), stream); + coreExports.info(`Fetching the Magic Nix Cache from ${binary_url}`); + const { stdout } = await promisify$1(exec)(`curl -L "${binary_url}" | xz -d | nix-store --import`); + const paths = stdout.split(os$2.EOL); + // Since the export is in reverse topologically sorted order, magic-nix-cache is always the penultimate entry in the list (the empty string left by split being the last). + const last_path = paths.at(-2); + return `${last_path}/bin/magic-nix-cache`; +} +function tailLog(daemonDir) { + const log = new Tail_1(path$1.join(daemonDir, 'daemon.log')); + coreExports.debug(`tailing daemon.log...`); + log.on('line', (line) => { + coreExports.info(line); + }); + return log; } async function setUpAutoCache() { const tmpdir = process.env['RUNNER_TEMP'] || os$2.tmpdir(); @@ -12166,8 +12174,7 @@ async function setUpAutoCache() { daemonBin = coreExports.getInput('source-binary'); } else { - daemonBin = `${daemonDir}/magic-nix-cache`; - await fetchAutoCacher(daemonBin); + daemonBin = await fetchAutoCacher(); } var runEnv; if (coreExports.isDebug()) { @@ -12180,18 +12187,58 @@ async function setUpAutoCache() { else { runEnv = process.env; } - const output = openSync(`${daemonDir}/daemon.log`, 'a'); - const launch = spawn(daemonBin, [ - '--daemon-dir', daemonDir, + const notifyPort = coreExports.getInput('startup-notification-port'); + const notifyPromise = new Promise((resolveListening) => { + const promise = new Promise(async (resolveQuit) => { + const notifyServer = require$$1.createServer((req, res) => { + if (req.method === 'POST' && req.url === '/') { + coreExports.debug(`Notify server shutting down.`); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end('{}'); + notifyServer.close(() => { + resolveQuit(); + }); + } + }); + notifyServer.listen(notifyPort, () => { + coreExports.debug(`Notify server running.`); + resolveListening(promise); + }); + }); + }); + // Start tailing the daemon log. + const outputPath = `${daemonDir}/daemon.log`; + const output = openSync(outputPath, 'a'); + const log = tailLog(daemonDir); + const netrc = await netrcPath(); + // Start the server. Once it is ready, it will notify us via the notification server. + const daemon = spawn(daemonBin, [ + '--startup-notification-url', `http://127.0.0.1:${notifyPort}`, '--listen', coreExports.getInput('listen'), '--upstream', coreExports.getInput('upstream-cache'), - '--diagnostic-endpoint', coreExports.getInput('diagnostic-endpoint') - ], { + '--diagnostic-endpoint', coreExports.getInput('diagnostic-endpoint'), + '--nix-conf', `${process.env["HOME"]}/.config/nix/nix.conf` + ].concat(coreExports.getInput('use-flakehub') === 'true' ? [ + '--use-flakehub', + '--flakehub-cache-server', coreExports.getInput('flakehub-cache-server'), + '--flakehub-api-server', coreExports.getInput('flakehub-api-server'), + '--flakehub-api-server-netrc', netrc, + '--flakehub-flake-name', coreExports.getInput('flakehub-flake-name'), + ] : []).concat(coreExports.getInput('use-gha-cache') === 'true' ? [ + '--use-gha-cache' + ] : []), { stdio: ['ignore', output, output], - env: runEnv + env: runEnv, + detached: true }); + const pidFile = path$1.join(daemonDir, 'daemon.pid'); + await fs$2.writeFile(pidFile, `${daemon.pid}`); + coreExports.info("Waiting for magic-nix-cache to start..."); await new Promise((resolve, reject) => { - launch.on('exit', (code, signal) => { + notifyPromise.then((value) => { + resolve(); + }); + daemon.on('exit', async (code, signal) => { if (signal) { reject(new Error(`Daemon was killed by signal ${signal}`)); } @@ -12199,17 +12246,14 @@ async function setUpAutoCache() { reject(new Error(`Daemon exited with code ${code}`)); } else { - resolve(); + reject(new Error(`Daemon unexpectedly exited`)); } }); }); - await fs$2.mkdir(`${process.env["HOME"]}/.config/nix`, { recursive: true }); - const nixConf = openSync(`${process.env["HOME"]}/.config/nix/nix.conf`, 'a'); - writeSync(nixConf, `${"\n"}extra-substituters = http://${coreExports.getInput('listen')}/?trusted=1&compression=zstd¶llel-compression=true${"\n"}`); - writeSync(nixConf, `fallback = true${"\n"}`); - close(nixConf); - coreExports.debug('Launched Magic Nix Cache'); + daemon.unref(); + coreExports.info('Launched Magic Nix Cache'); coreExports.exportVariable(ENV_CACHE_DAEMONDIR, daemonDir); + log.unwatch(); } async function notifyAutoCache() { const daemonDir = process.env[ENV_CACHE_DAEMONDIR]; @@ -12228,6 +12272,34 @@ async function notifyAutoCache() { coreExports.info(`Magic Nix Cache may not be running for this workflow.`); } } +async function netrcPath() { + const expectedNetrcPath = path$1.join(process.env['RUNNER_TEMP'], 'determinate-nix-installer-netrc'); + try { + await fs$2.access(expectedNetrcPath); + return expectedNetrcPath; + } + catch { + // `nix-installer` was not used, the user may be registered with FlakeHub though. + const destinedNetrcPath = path$1.join(process.env['RUNNER_TEMP'], 'magic-nix-cache-netrc'); + try { + await flakehub_login(destinedNetrcPath); + } + catch (e) { + coreExports.info("FlakeHub cache disabled."); + coreExports.debug(`Error while logging into FlakeHub: ${e}`); + } + return destinedNetrcPath; + } +} +async function flakehub_login(netrc) { + const jwt = await coreExports.getIDToken("api.flakehub.com"); + await fs$2.writeFile(netrc, [ + `machine api.flakehub.com login flakehub password ${jwt}`, + `machine flakehub.com login flakehub password ${jwt}`, + `machine cache.flakehub.com login flakehub password ${jwt}`, + ].join("\n")); + coreExports.info("Logged in to FlakeHub."); +} async function tearDownAutoCache() { const daemonDir = process.env[ENV_CACHE_DAEMONDIR]; if (!daemonDir) { @@ -12240,11 +12312,7 @@ async function tearDownAutoCache() { if (!pid) { throw new Error("magic-nix-cache did not start successfully"); } - const log = new Tail_1(path$1.join(daemonDir, 'daemon.log')); - coreExports.debug(`tailing daemon.log...`); - log.on('line', (line) => { - coreExports.info(line); - }); + const log = tailLog(daemonDir); try { coreExports.debug(`about to post to localhost`); const res = await gotClient.post(`http://${coreExports.getInput('listen')}/api/workflow-finish`).json(); diff --git a/flake.nix b/flake.nix index 5d58dd9..736da47 100644 --- a/flake.nix +++ b/flake.nix @@ -19,10 +19,10 @@ default = pkgs.mkShell { packages = with pkgs; [ bun - nodejs jq act just + shellcheck ]; }; }); diff --git a/package.json b/package.json index d0a0394..52f23d9 100644 --- a/package.json +++ b/package.json @@ -10,15 +10,15 @@ "license": "LGPL", "dependencies": { "@actions/core": "^1.10.0", + "got": "^12.6.0", "tail": "^2.2.6", - "tslib": "^2.5.2", - "got": "^12.6.0" + "tslib": "^2.5.2" }, "devDependencies": { "@rollup/plugin-commonjs": "^25.0.0", "@rollup/plugin-node-resolve": "^15.0.2", "@rollup/plugin-typescript": "^11.1.1", - "@types/node": "^20.2.1", + "@types/node": "^20.11.17", "rollup": "^3.22.0", "typescript": "^5.0.4" } diff --git a/src/index.ts b/src/index.ts index bdbfd44..ee9e1b3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,10 +3,10 @@ import * as fs from 'node:fs/promises'; import * as os from 'node:os'; import * as path from 'node:path'; -import { spawn } from 'node:child_process'; -import { createWriteStream, openSync, writeSync, close, readFileSync } from 'node:fs'; -import { pipeline } from 'node:stream/promises'; -import { inspect } from 'node:util'; +import { spawn, exec } from 'node:child_process'; +import { openSync, readFileSync } from 'node:fs'; +import { inspect, promisify } from 'node:util'; +import * as http from 'http'; import * as core from '@actions/core'; import { Tail } from 'tail'; @@ -33,8 +33,7 @@ function getCacherUrl() : string { const runnerArch = process.env.RUNNER_ARCH; const runnerOs = process.env.RUNNER_OS; const binarySuffix = `${runnerArch}-${runnerOs}`; - const urlPrefix = `https://install.determinate.systems/magic-nix-cache`; - + const urlPrefix = `https://install.determinate.systems/magic-nix-cache-closure`; if (core.getInput('source-url')) { return core.getInput('source-url'); } @@ -58,19 +57,27 @@ function getCacherUrl() : string { return `${urlPrefix}/stable/${binarySuffix}`; } -async function fetchAutoCacher(destination: string) { - const stream = createWriteStream(destination, { - encoding: "binary", - mode: 0o755, - }); - +async function fetchAutoCacher() { const binary_url = getCacherUrl(); - core.debug(`Fetching the Magic Nix Cache from ${binary_url}`); + core.info(`Fetching the Magic Nix Cache from ${binary_url}`); - return pipeline( - gotClient.stream(binary_url), - stream - ); + const { stdout } = await promisify(exec)(`curl -L "${binary_url}" | xz -d | nix-store --import`); + + const paths = stdout.split(os.EOL); + + // Since the export is in reverse topologically sorted order, magic-nix-cache is always the penultimate entry in the list (the empty string left by split being the last). + const last_path = paths.at(-2); + + return `${last_path}/bin/magic-nix-cache`; +} + +function tailLog(daemonDir) { + const log = new Tail(path.join(daemonDir, 'daemon.log')); + core.debug(`tailing daemon.log...`); + log.on('line', (line) => { + core.info(line); + }); + return log; } async function setUpAutoCache() { @@ -97,8 +104,7 @@ async function setUpAutoCache() { if (core.getInput('source-binary')) { daemonBin = core.getInput('source-binary'); } else { - daemonBin = `${daemonDir}/magic-nix-cache`; - await fetchAutoCacher(daemonBin); + daemonBin = await fetchAutoCacher(); } var runEnv; @@ -112,41 +118,87 @@ async function setUpAutoCache() { runEnv = process.env; } - const output = openSync(`${daemonDir}/daemon.log`, 'a'); - const launch = spawn( + const notifyPort = core.getInput('startup-notification-port'); + + const notifyPromise = new Promise>((resolveListening) => { + const promise = new Promise(async (resolveQuit) => { + const notifyServer = http.createServer((req, res) => { + if (req.method === 'POST' && req.url === '/') { + core.debug(`Notify server shutting down.`); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end('{}'); + notifyServer.close(() => { + resolveQuit(); + }); + } + }); + + notifyServer.listen(notifyPort, () => { + core.debug(`Notify server running.`); + resolveListening(promise); + }); + }); + }); + + // Start tailing the daemon log. + const outputPath = `${daemonDir}/daemon.log`; + const output = openSync(outputPath, 'a'); + const log = tailLog(daemonDir); + const netrc = await netrcPath(); + + // Start the server. Once it is ready, it will notify us via the notification server. + const daemon = spawn( daemonBin, [ - '--daemon-dir', daemonDir, + '--startup-notification-url', `http://127.0.0.1:${notifyPort}`, '--listen', core.getInput('listen'), '--upstream', core.getInput('upstream-cache'), - '--diagnostic-endpoint', core.getInput('diagnostic-endpoint') - ], + '--diagnostic-endpoint', core.getInput('diagnostic-endpoint'), + '--nix-conf', `${process.env["HOME"]}/.config/nix/nix.conf` + ].concat( + core.getInput('use-flakehub') === 'true' ? [ + '--use-flakehub', + '--flakehub-cache-server', core.getInput('flakehub-cache-server'), + '--flakehub-api-server', core.getInput('flakehub-api-server'), + '--flakehub-api-server-netrc', netrc, + '--flakehub-flake-name', core.getInput('flakehub-flake-name'), + ] : []).concat( + core.getInput('use-gha-cache') === 'true' ? [ + '--use-gha-cache' + ] : []), { stdio: ['ignore', output, output], - env: runEnv + env: runEnv, + detached: true } ); + const pidFile = path.join(daemonDir, 'daemon.pid'); + await fs.writeFile(pidFile, `${daemon.pid}`); + + core.info("Waiting for magic-nix-cache to start..."); + await new Promise((resolve, reject) => { - launch.on('exit', (code, signal) => { + notifyPromise.then((value) => { + resolve(); + }); + daemon.on('exit', async (code, signal) => { if (signal) { reject(new Error(`Daemon was killed by signal ${signal}`)); } else if (code) { reject(new Error(`Daemon exited with code ${code}`)); } else { - resolve(); + reject(new Error(`Daemon unexpectedly exited`)); } }); }); - await fs.mkdir(`${process.env["HOME"]}/.config/nix`, { recursive: true }); - const nixConf = openSync(`${process.env["HOME"]}/.config/nix/nix.conf`, 'a'); - writeSync(nixConf, `${"\n"}extra-substituters = http://${core.getInput('listen')}/?trusted=1&compression=zstd¶llel-compression=true${"\n"}`); - writeSync(nixConf, `fallback = true${"\n"}`); - close(nixConf); + daemon.unref(); - core.debug('Launched Magic Nix Cache'); + core.info('Launched Magic Nix Cache'); core.exportVariable(ENV_CACHE_DAEMONDIR, daemonDir); + + log.unwatch(); } async function notifyAutoCache() { @@ -168,6 +220,40 @@ async function notifyAutoCache() { } } + +async function netrcPath() { + const expectedNetrcPath = path.join(process.env['RUNNER_TEMP'], 'determinate-nix-installer-netrc') + try { + await fs.access(expectedNetrcPath) + return expectedNetrcPath; + } catch { + // `nix-installer` was not used, the user may be registered with FlakeHub though. + const destinedNetrcPath = path.join(process.env['RUNNER_TEMP'], 'magic-nix-cache-netrc') + try { + await flakehub_login(destinedNetrcPath); + } catch (e) { + core.info("FlakeHub cache disabled."); + core.debug(`Error while logging into FlakeHub: ${e}`) + } + return destinedNetrcPath; + } +} + +async function flakehub_login(netrc: string) { + const jwt = await core.getIDToken("api.flakehub.com"); + + await fs.writeFile( + netrc, + [ + `machine api.flakehub.com login flakehub password ${jwt}`, + `machine flakehub.com login flakehub password ${jwt}`, + `machine cache.flakehub.com login flakehub password ${jwt}`, + ].join("\n"), + ); + + core.info("Logged in to FlakeHub."); +} + async function tearDownAutoCache() { const daemonDir = process.env[ENV_CACHE_DAEMONDIR]; @@ -183,12 +269,7 @@ async function tearDownAutoCache() { throw new Error("magic-nix-cache did not start successfully"); } - const log = new Tail(path.join(daemonDir, 'daemon.log')); - core.debug(`tailing daemon.log...`); - log.on('line', (line) => { - core.info(line); - }); - + const log = tailLog(daemonDir); try { core.debug(`about to post to localhost`); @@ -239,4 +320,3 @@ try { }} core.debug(`rip`); -