Merge pull request #129 from crazy-max/buildx-bin-cache

buildx: cache binary to hosted tool cache and GHA cache backend

Author: crazy-max

__tests__/buildx/install.test.ts

@@ -36,9 +36,8 @@ afterEach(function () {
 describe('download', () => {
   // prettier-ignore
   test.each([
-    ['v0.9.1', false],
-    ['latest', false],
-    ['v0.9.1', true],
+    ['v0.9.0', false],
+    ['v0.10.5', true],
     ['latest', true]
   ])(
   'acquires %p of buildx (standalone: %p)', async (version, standalone) => {
@@ -56,6 +55,18 @@ describe('download', () => {
     100000
   );
 
+  // prettier-ignore
+  test.each([
+    // following versions are already cached to htc from previous test cases
+    ['v0.9.0'],
+    ['v0.10.5'],
+  ])(
+  'acquires %p of buildx with cache', async (version) => {
+    const install = new Install({standalone: false});
+    const toolPath = await install.download(version);
+    expect(fs.existsSync(toolPath)).toBe(true);
+  });
+
   // TODO: add tests for arm
   // prettier-ignore
   test.each([

__tests__/util.test.ts

@@ -232,6 +232,41 @@ describe('isValidRef', () => {
   });
 });
 
+describe('trimPrefix', () => {
+  test.each([
+    ['', 'abc', ''],
+    ['abc', 'a', 'bc'],
+    ['abc', 'ab', 'c'],
+    ['abc', '', 'abc'],
+    ['abc', '', 'abc'],
+    ['abc', 'd', 'abc'],
+    ['abc', 'abc', ''],
+    ['abc', 'abcd', 'abc'],
+    ['abcdabc', 'abc', 'dabc'],
+    ['abcabc', 'abc', 'abc'],
+    ['abcdabc', 'd', 'abcdabc']
+  ])('given %p', async (str, prefix, expected) => {
+    expect(Util.trimPrefix(str, prefix)).toEqual(expected);
+  });
+});
+
+describe('trimSuffix', () => {
+  test.each([
+    ['', 'abc', ''],
+    ['abc', 'c', 'ab'],
+    ['abc', '', 'abc'],
+    ['abc', 'bc', 'a'],
+    ['abc', 'abc', ''],
+    ['abc', 'abcd', 'abc'],
+    ['abc', 'aabc', 'abc'],
+    ['abcdabc', 'abc', 'abcd'],
+    ['abcabc', 'abc', 'abc'],
+    ['abcdabc', 'd', 'abcdabc']
+  ])('given %p', async (str, suffix, expected) => {
+    expect(Util.trimSuffix(str, suffix)).toEqual(expected);
+  });
+});
+
 // See: https://github.com/actions/toolkit/blob/a1b068ec31a042ff1e10a522d8fdf0b8869d53ca/packages/core/src/core.ts#L89
 function getInputName(name: string): string {
   return `INPUT_${name.replace(/ /g, '_').toUpperCase()}`;

package.json

@@ -45,6 +45,7 @@
     "registry": "https://registry.npmjs.org/"
   },
   "dependencies": {
+    "@actions/cache": "^3.2.1",
     "@actions/core": "^1.10.0",
     "@actions/exec": "^1.1.1",
     "@actions/github": "^5.1.1",

src/buildx/install.ts

@@ -14,12 +14,14 @@
  * limitations under the License.
  */
 
+import crypto from 'crypto';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import * as core from '@actions/core';
 import * as httpm from '@actions/http-client';
 import * as tc from '@actions/tool-cache';
+import * as cache from '@actions/cache';
 import * as semver from 'semver';
 import * as util from 'util';
 
@@ -28,6 +30,7 @@ import {Context} from '../context';
 import {Exec} from '../exec';
 import {Docker} from '../docker/docker';
 import {Git} from '../git';
+import {Util} from '../util';
 
 import {GitHubRelease} from '../types/github';
 
@@ -42,70 +45,87 @@ export class Install {
     this._standalone = opts?.standalone;
   }
 
+  /*
+   * Download buildx binary from GitHub release
+   * @param version semver version or latest
+   * @returns path to the buildx binary
+   */
   public async download(version: string): Promise<string> {
     const release: GitHubRelease = await Install.getRelease(version);
-    const fversion = release.tag_name.replace(/^v+|v+$/g, '');
-    core.debug(`Install.download version: ${fversion}`);
-
-    let toolPath: string;
-    toolPath = tc.find('buildx', fversion, this.platform());
-    if (!toolPath) {
-      const c = semver.clean(fversion) || '';
-      if (!semver.valid(c)) {
-        throw new Error(`Invalid Buildx version "${fversion}".`);
-      }
-      toolPath = await this.fetchBinary(fversion);
+    core.debug(`Install.download release tag name: ${release.tag_name}`);
+
+    const vspec = await this.vspec(release.tag_name);
+    core.debug(`Install.download vspec: ${vspec}`);
+
+    const c = semver.clean(vspec) || '';
+    if (!semver.valid(c)) {
+      throw new Error(`Invalid Buildx version "${vspec}".`);
     }
-    core.debug(`Install.download toolPath: ${toolPath}`);
 
-    return toolPath;
+    const installCache = new InstallCache('buildx-dl-bin', vspec);
+
+    const cacheFoundPath = await installCache.find();
+    if (cacheFoundPath) {
+      core.info(`Buildx binary found in ${cacheFoundPath}`);
+      return cacheFoundPath;
+    }
+
+    const downloadURL = util.format('https://github.com/docker/buildx/releases/download/v%s/%s', vspec, this.filename(vspec));
+    core.info(`Downloading ${downloadURL}`);
+
+    const htcDownloadPath = await tc.downloadTool(downloadURL);
+    core.debug(`Install.download htcDownloadPath: ${htcDownloadPath}`);
+
+    const cacheSavePath = await installCache.save(htcDownloadPath);
+    core.info(`Cached to ${cacheSavePath}`);
+    return cacheSavePath;
   }
 
+  /*
+   * Build buildx binary from source
+   * @param gitContext git repo context
+   * @returns path to the buildx binary
+   */
   public async build(gitContext: string): Promise<string> {
-    // eslint-disable-next-line prefer-const
-    let [repo, ref] = gitContext.split('#');
-    if (ref.length == 0) {
-      ref = 'master';
+    const vspec = await this.vspec(gitContext);
+    core.debug(`Install.build vspec: ${vspec}`);
+
+    const installCache = new InstallCache('buildx-build-bin', vspec);
+
+    const cacheFoundPath = await installCache.find();
+    if (cacheFoundPath) {
+      core.info(`Buildx binary found in ${cacheFoundPath}`);
+      return cacheFoundPath;
     }
 
-    let vspec: string;
-    // TODO: include full ref as fingerprint. Use commit sha as best-effort in the meantime.
-    if (ref.match(/^[0-9a-fA-F]{40}$/)) {
-      vspec = ref;
-    } else {
-      vspec = await Git.remoteSha(repo, ref, process.env.GIT_AUTH_TOKEN);
-    }
-    core.debug(`Install.build: tool version spec ${vspec}`);
-
-    let toolPath: string;
-    toolPath = tc.find('buildx', vspec);
-    if (!toolPath) {
-      const outputDir = path.join(Context.tmpDir(), 'build-cache');
-      const buildCmd = await this.buildCommand(gitContext, outputDir);
-      toolPath = await Exec.getExecOutput(buildCmd.command, buildCmd.args, {
-        ignoreReturnCode: true
-      }).then(res => {
-        if (res.stderr.length > 0 && res.exitCode != 0) {
-          throw new Error(`build failed with: ${res.stderr.match(/(.*)\s*$/)?.[0]?.trim() ?? 'unknown error'}`);
-        }
-        return tc.cacheFile(`${outputDir}/buildx`, os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx', 'buildx', vspec, this.platform());
-      });
-    }
-
-    return toolPath;
+    const outputDir = path.join(Context.tmpDir(), 'buildx-build-cache');
+    const buildCmd = await this.buildCommand(gitContext, outputDir);
+
+    const buildBinPath = await Exec.getExecOutput(buildCmd.command, buildCmd.args, {
+      ignoreReturnCode: true
+    }).then(res => {
+      if (res.stderr.length > 0 && res.exitCode != 0) {
+        throw new Error(`build failed with: ${res.stderr.match(/(.*)\s*$/)?.[0]?.trim() ?? 'unknown error'}`);
+      }
+      return `${outputDir}/buildx`;
+    });
+
+    const cacheSavePath = await installCache.save(buildBinPath);
+    core.info(`Cached to ${cacheSavePath}`);
+    return cacheSavePath;
   }
 
-  public async installStandalone(toolPath: string, dest?: string): Promise<string> {
+  public async installStandalone(binPath: string, dest?: string): Promise<string> {
     core.info('Standalone mode');
     dest = dest || Context.tmpDir();
-    const toolBinPath = path.join(toolPath, os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx');
-    const binDir = path.join(dest, 'bin');
+
+    const binDir = path.join(dest, 'buildx-bin-standalone');
     if (!fs.existsSync(binDir)) {
       fs.mkdirSync(binDir, {recursive: true});
     }
-    const filename: string = os.platform() == 'win32' ? 'buildx.exe' : 'buildx';
-    const buildxPath: string = path.join(binDir, filename);
-    fs.copyFileSync(toolBinPath, buildxPath);
+    const binName: string = os.platform() == 'win32' ? 'buildx.exe' : 'buildx';
+    const buildxPath: string = path.join(binDir, binName);
+    fs.copyFileSync(binPath, buildxPath);
 
     core.info('Fixing perms');
     fs.chmodSync(buildxPath, '0755');
@@ -117,17 +137,17 @@ export class Install {
     return buildxPath;
   }
 
-  public async installPlugin(toolPath: string, dest?: string): Promise<string> {
+  public async installPlugin(binPath: string, dest?: string): Promise<string> {
     core.info('Docker plugin mode');
     dest = dest || Docker.configDir;
-    const toolBinPath = path.join(toolPath, os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx');
+
     const pluginsDir: string = path.join(dest, 'cli-plugins');
     if (!fs.existsSync(pluginsDir)) {
       fs.mkdirSync(pluginsDir, {recursive: true});
     }
-    const filename: string = os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx';
-    const pluginPath: string = path.join(pluginsDir, filename);
-    fs.copyFileSync(toolBinPath, pluginPath);
+    const binName: string = os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx';
+    const pluginPath: string = path.join(pluginsDir, binName);
+    fs.copyFileSync(binPath, pluginPath);
 
     core.info('Fixing perms');
     fs.chmodSync(pluginPath, '0755');
@@ -173,21 +193,6 @@ export class Install {
     return standalone;
   }
 
-  private async fetchBinary(version: string): Promise<string> {
-    const targetFile: string = os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx';
-    const downloadURL = util.format('https://github.com/docker/buildx/releases/download/v%s/%s', version, this.filename(version));
-    core.info(`Downloading ${downloadURL}`);
-    const downloadPath = await tc.downloadTool(downloadURL);
-    core.debug(`Install.fetchBinary downloadPath: ${downloadPath}`);
-    return await tc.cacheFile(downloadPath, targetFile, 'buildx', version, this.platform());
-  }
-
-  private platform(): string {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const arm_version = (process.config.variables as any).arm_version;
-    return `${os.platform()}-${os.arch()}${arm_version ? 'v' + arm_version : ''}`;
-  }
-
   private filename(version: string): string {
     let arch: string;
     switch (os.arch()) {
@@ -215,6 +220,39 @@ export class Install {
     return util.format('buildx-v%s.%s-%s%s', version, platform, arch, ext);
   }
 
+  /*
+   * Get version spec (fingerprint) for cache key. If versionOrRef is a valid
+   * Git context, then return the SHA of the ref along the repo and owner and
+   * create a hash of it. Otherwise, return the versionOrRef (semver) as is
+   * without the 'v' prefix.
+   */
+  private async vspec(versionOrRef: string): Promise<string> {
+    if (!Util.isValidRef(versionOrRef)) {
+      const v = versionOrRef.replace(/^v+|v+$/g, '');
+      core.info(`Use ${v} version spec cache key for ${versionOrRef}`);
+      return v;
+    }
+
+    // eslint-disable-next-line prefer-const
+    let [baseURL, ref] = versionOrRef.split('#');
+    if (ref.length == 0) {
+      ref = 'master';
+    }
+
+    let sha: string;
+    if (ref.match(/^[0-9a-fA-F]{40}$/)) {
+      sha = ref;
+    } else {
+      sha = await Git.remoteSha(baseURL, ref, process.env.GIT_AUTH_TOKEN);
+    }
+
+    const [owner, repo] = baseURL.substring('https://github.com/'.length).split('/');
+    const key = `${owner}/${Util.trimSuffix(repo, '.git')}/${sha}`;
+    const hash = crypto.createHash('sha256').update(key).digest('hex');
+    core.info(`Use ${hash} version spec cache key for ${key}`);
+    return hash;
+  }
+
   public static async getRelease(version: string): Promise<GitHubRelease> {
     const url = `https://raw.githubusercontent.com/docker/actions-toolkit/main/.github/buildx-releases.json`;
     const http: httpm.HttpClient = new httpm.HttpClient('docker-actions-toolkit');
@@ -231,3 +269,74 @@ export class Install {
     return releases[version];
   }
 }
+
+class InstallCache {
+  private readonly htcName: string;
+  private readonly htcVersion: string;
+  private readonly ghaCacheKey: string;
+  private readonly cacheDir: string;
+  private readonly cacheFile: string;
+  private readonly cachePath: string;
+
+  constructor(htcName: string, htcVersion: string) {
+    this.htcName = htcName;
+    this.htcVersion = htcVersion;
+    this.ghaCacheKey = util.format('%s-%s-%s', this.htcName, this.htcVersion, this.platform());
+    this.cacheDir = path.join(Buildx.configDir, '.bin', htcVersion, this.platform());
+    this.cacheFile = os.platform() == 'win32' ? 'docker-buildx.exe' : 'docker-buildx';
+    this.cachePath = path.join(this.cacheDir, this.cacheFile);
+    if (!fs.existsSync(this.cacheDir)) {
+      fs.mkdirSync(this.cacheDir, {recursive: true});
+    }
+  }
+
+  public async save(file: string): Promise<string> {
+    core.debug(`InstallCache.save ${file}`);
+    const cachePath = this.copyToCache(file);
+
+    const htcPath = await tc.cacheDir(this.cacheDir, this.htcName, this.htcVersion, this.platform());
+    core.debug(`InstallCache.save cached to hosted tool cache ${htcPath}`);
+
+    if (cache.isFeatureAvailable()) {
+      core.debug(`InstallCache.save caching ${this.ghaCacheKey} to GitHub Actions cache`);
+      await cache.saveCache([this.cacheDir], this.ghaCacheKey);
+    }
+
+    return cachePath;
+  }
+
+  public async find(): Promise<string> {
+    let htcPath = tc.find(this.htcName, this.htcVersion, this.platform());
+    if (htcPath) {
+      core.info(`Restored from hosted tool cache ${htcPath}`);
+      return this.copyToCache(`${htcPath}/${this.cacheFile}`);
+    }
+
+    if (cache.isFeatureAvailable()) {
+      core.debug(`GitHub Actions cache feature available`);
+      if (await cache.restoreCache([this.cacheDir], this.ghaCacheKey)) {
+        core.info(`Restored ${this.ghaCacheKey} from GitHub Actions cache`);
+        htcPath = await tc.cacheDir(this.cacheDir, this.htcName, this.htcVersion, this.platform());
+        core.info(`Restored to hosted tool cache ${htcPath}`);
+        return this.copyToCache(`${htcPath}/${this.cacheFile}`);
+      }
+    } else {
+      core.info(`GitHub Actions cache feature not available`);
+    }
+
+    return '';
+  }
+
+  private copyToCache(file: string): string {
+    core.debug(`Copying ${file} to ${this.cachePath}`);
+    fs.copyFileSync(file, this.cachePath);
+    fs.chmodSync(this.cachePath, '0755');
+    return this.cachePath;
+  }
+
+  private platform(): string {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const arm_version = (process.config.variables as any).arm_version;
+    return `${os.platform()}-${os.arch()}${arm_version ? 'v' + arm_version : ''}`;
+  }
+}