Cache multiple entries using --files-from tar input

remove known failing test

Quote tar paths

Add salt to test cache

Try reading input files from manifest

bump salt

Run test on macos

more testing

Run caching tests on 3 platforms

Run tests on self-hosted

Apparently cant reference hosted runners by name

Bump salt

wait for some time after save

more timing out

smarter waiting

Cache in tmp dir that won't be deleted

Use child_process instead of actions/exec

Revert tempDir hack

bump salt

more logging

More console logging

Use filepath to with cacheHttpClient

Test cache restoration

Revert temp dir hack

debug logging

clean up cache.yml testing

Bump salt

change debug output

build actions

.github/workflows/cache.yml

@@ -6,7 +6,6 @@ on:
       - master
 
 jobs:
-  # Build and unit test
   build:
     runs-on: self-hosted
     steps:
@@ -16,19 +15,15 @@ jobs:
       uses: actions/setup-node@v1
       with:
         node-version: '12.x'
-    - run: npm install
-    - run: npm run build
     - name: Restore npm cache
       uses: ./
+      id: cache
       with:
         path: |
           node_modules
           dist
-          ~/Desktop/cache-me
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-    - name: Prettier Format Check
-      run: npm run format-check
-    - name: ESLint Check
-      run: npm run lint
-    - name: Build & Test
-      run: npm run test
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('salt.txt') }}
+    - run: npm install
+      if: steps.cache.outputs.cache-hit != 'true'
+    - run: npm run build
+      if: steps.cache.outputs.cache-hit != 'true'

.github/workflows/workflow.yml

@@ -1,11 +1,11 @@
 name: Tests
 
 on:
-  pull_request:
-    branches:
-      - master
-    paths-ignore:
-      - '**.md'
+  # pull_request:
+  #   branches:
+  #     - master
+  #   paths-ignore:
+  #     - '**.md'
   push:
     branches:
       - master

__tests__/actionUtils.test.ts

@@ -1,4 +1,5 @@
 import * as core from "@actions/core";
+import * as glob from "@actions/glob";
 import * as os from "os";
 import * as path from "path";
 
@@ -181,16 +182,17 @@ test("isValidEvent returns false for unknown event", () => {
     expect(isValidEvent).toBe(false);
 });
 
-test("expandPaths with no ~ in path", () => {
+test("resolvePaths with no ~ in path", async () => {
+    // TODO: these test paths will need to exist
     const filePath = ".cache/yarn";
 
-    const resolvedPath = actionUtils.expandPaths([filePath]);
+    const resolvedPath = await actionUtils.resolvePaths([filePath]);
 
     const expectedPath = [path.resolve(filePath)];
     expect(resolvedPath).toStrictEqual(expectedPath);
 });
 
-test("expandPaths with ~ in path", () => {
+test("resolvePaths with ~ in path", async () => {
     const filePath = "~/.cache/yarn";
 
     const homedir = jest.requireActual("os").homedir();
@@ -199,20 +201,22 @@ test("expandPaths with ~ in path", () => {
         return homedir;
     });
 
-    const resolvedPath = actionUtils.expandPaths([filePath]);
+    const resolvedPath = await actionUtils.resolvePaths([filePath]);
 
     const expectedPath = [path.join(homedir, ".cache/yarn")];
     expect(resolvedPath).toStrictEqual(expectedPath);
 });
 
-test("expandPaths with home not found", () => {
+test("resolvePaths with home not found", () => {
     const filePath = "~/.cache/yarn";
     const homedirMock = jest.spyOn(os, "homedir");
     homedirMock.mockImplementation(() => {
         return "";
     });
+    // const globMock = jest.spyOn(glob, "homedir");
+    // globMock.mockImplementation(() => "");
 
-    expect(() => actionUtils.expandPaths([filePath])).toThrow(
+    expect(async () => await actionUtils.resolvePaths([filePath])).toThrow(
         "Unable to resolve `~` to HOME"
     );
 });

__tests__/save.test.ts

@@ -40,7 +40,7 @@ beforeAll(() => {
         return actualUtils.getSupportedEvents();
     });
 
-    jest.spyOn(actionUtils, "expandPaths").mockImplementation(
+    jest.spyOn(actionUtils, "resolvePaths").mockImplementation(
         async filePaths => {
             return filePaths.map(x => path.resolve(x));
         }

dist/restore/index.js

@@ -3151,6 +3151,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 var __importStar = (this && this.__importStar) || function (mod) {
     if (mod && mod.__esModule) return mod;
     var result = {};
@@ -3233,19 +3240,35 @@ function logWarning(message) {
     core.info(`${warningPrefix}${message}`);
 }
 exports.logWarning = logWarning;
-function expandPaths(patterns) {
-    var _a;
+function resolvePaths(patterns) {
+    var e_1, _a;
+    var _b;
     return __awaiter(this, void 0, void 0, function* () {
         const paths = [];
-        const workspace = (_a = process.env["GITHUB_WORKSPACE"], (_a !== null && _a !== void 0 ? _a : process.cwd()));
-        const globber = yield glob.create(patterns.join("\n"));
-        const files = yield globber.glob();
-        paths.push(...files);
-        // Convert paths to relative paths here?
-        return paths.map(x => path.relative(workspace, x));
+        const workspace = (_b = process.env["GITHUB_WORKSPACE"], (_b !== null && _b !== void 0 ? _b : process.cwd()));
+        const globber = yield glob.create(patterns.join("\n"), {
+            implicitDescendants: false
+        });
+        try {
+            for (var _c = __asyncValues(globber.globGenerator()), _d; _d = yield _c.next(), !_d.done;) {
+                const file = _d.value;
+                const relativeFile = path.relative(workspace, file);
+                core.debug(`Matched: ${relativeFile}`);
+                // Paths are made relative so the tar entries are all relative to the root of the workspace.
+                paths.push(`${relativeFile}`);
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (_d && !_d.done && (_a = _c.return)) yield _a.call(_c);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        return paths;
     });
 }
-exports.expandPaths = expandPaths;
+exports.resolvePaths = resolvePaths;
 function getSupportedEvents() {
     return [constants_1.Events.Push, constants_1.Events.PullRequest];
 }
@@ -4421,6 +4444,7 @@ var Events;
     Events["Push"] = "push";
     Events["PullRequest"] = "pull_request";
 })(Events = exports.Events || (exports.Events = {}));
+exports.CacheFilename = "cache.tgz";
 
 
 /***/ }),
@@ -4901,8 +4925,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const exec_1 = __webpack_require__(986);
 const io = __importStar(__webpack_require__(1));
+const path = __importStar(__webpack_require__(622));
+const constants_1 = __webpack_require__(694);
+const exec_1 = __webpack_require__(986);
 const fs_1 = __webpack_require__(747);
 function getTarPath() {
     return __awaiter(this, void 0, void 0, function* () {
@@ -4917,13 +4943,14 @@ function getTarPath() {
         return yield io.which("tar", true);
     });
 }
-function execTar(args) {
+function execTar(args, cwd) {
     var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
         try {
-            yield exec_1.exec(`"${yield getTarPath()}"`, args);
+            yield exec_1.exec(`"${yield getTarPath()}"`, args, { cwd: cwd });
         }
         catch (error) {
+            console.log("error", error);
             const IS_WINDOWS = process.platform === "win32";
             if (IS_WINDOWS) {
                 throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`);
@@ -4946,19 +4973,22 @@ function extractTar(archivePath) {
     });
 }
 exports.extractTar = extractTar;
-function createTar(archivePath, sourceDirectories) {
+function createTar(archiveFolder, sourceDirectories) {
     return __awaiter(this, void 0, void 0, function* () {
         // TODO: will want to stream sourceDirectories into tar
+        const manifestFilename = "manifest.txt";
+        fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join("\n"));
         const workingDirectory = getWorkingDirectory();
         const args = [
             "-cz",
             "-f",
-            archivePath,
+            constants_1.CacheFilename,
             "-C",
             workingDirectory,
-            sourceDirectories.join(" ")
+            "--files-from",
+            manifestFilename
         ];
-        yield execTar(args);
+        yield execTar(args, archiveFolder);
     });
 }
 exports.createTar = createTar;

dist/save/index.js

@@ -3151,6 +3151,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 var __importStar = (this && this.__importStar) || function (mod) {
     if (mod && mod.__esModule) return mod;
     var result = {};
@@ -3233,19 +3240,35 @@ function logWarning(message) {
     core.info(`${warningPrefix}${message}`);
 }
 exports.logWarning = logWarning;
-function expandPaths(patterns) {
-    var _a;
+function resolvePaths(patterns) {
+    var e_1, _a;
+    var _b;
     return __awaiter(this, void 0, void 0, function* () {
         const paths = [];
-        const workspace = (_a = process.env["GITHUB_WORKSPACE"], (_a !== null && _a !== void 0 ? _a : process.cwd()));
-        const globber = yield glob.create(patterns.join("\n"));
-        const files = yield globber.glob();
-        paths.push(...files);
-        // Convert paths to relative paths here?
-        return paths.map(x => path.relative(workspace, x));
+        const workspace = (_b = process.env["GITHUB_WORKSPACE"], (_b !== null && _b !== void 0 ? _b : process.cwd()));
+        const globber = yield glob.create(patterns.join("\n"), {
+            implicitDescendants: false
+        });
+        try {
+            for (var _c = __asyncValues(globber.globGenerator()), _d; _d = yield _c.next(), !_d.done;) {
+                const file = _d.value;
+                const relativeFile = path.relative(workspace, file);
+                core.debug(`Matched: ${relativeFile}`);
+                // Paths are made relative so the tar entries are all relative to the root of the workspace.
+                paths.push(`${relativeFile}`);
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (_d && !_d.done && (_a = _c.return)) yield _a.call(_c);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        return paths;
     });
 }
-exports.expandPaths = expandPaths;
+exports.resolvePaths = resolvePaths;
 function getSupportedEvents() {
     return [constants_1.Events.Push, constants_1.Events.PullRequest];
 }
@@ -4449,15 +4472,16 @@ function run() {
                 return;
             }
             core.debug(`Cache ID: ${cacheId}`);
-            const cachePaths = yield utils.expandPaths(core
+            const cachePaths = yield utils.resolvePaths(core
                 .getInput(constants_1.Inputs.Path)
                 .split("\n")
                 .filter(x => x !== ""));
             core.debug("Cache Paths:");
             core.debug(`${JSON.stringify(cachePaths)}`);
-            const archivePath = path.join(yield utils.createTempDirectory(), "cache.tgz");
+            const archiveFolder = yield utils.createTempDirectory();
+            const archivePath = path.join(archiveFolder, constants_1.CacheFilename);
             core.debug(`Archive Path: ${archivePath}`);
-            yield tar_1.createTar(archivePath, cachePaths);
+            yield tar_1.createTar(archiveFolder, cachePaths);
             const fileSizeLimit = 5 * 1024 * 1024 * 1024; // 5GB per repo limit
             const archiveFileSize = utils.getArchiveFileSize(archivePath);
             core.debug(`File Size: ${archiveFileSize}`);
@@ -4506,6 +4530,7 @@ var Events;
     Events["Push"] = "push";
     Events["PullRequest"] = "pull_request";
 })(Events = exports.Events || (exports.Events = {}));
+exports.CacheFilename = "cache.tgz";
 
 
 /***/ }),
@@ -4888,8 +4913,10 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const exec_1 = __webpack_require__(986);
 const io = __importStar(__webpack_require__(1));
+const path = __importStar(__webpack_require__(622));
+const constants_1 = __webpack_require__(694);
+const exec_1 = __webpack_require__(986);
 const fs_1 = __webpack_require__(747);
 function getTarPath() {
     return __awaiter(this, void 0, void 0, function* () {
@@ -4904,13 +4931,14 @@ function getTarPath() {
         return yield io.which("tar", true);
     });
 }
-function execTar(args) {
+function execTar(args, cwd) {
     var _a, _b;
     return __awaiter(this, void 0, void 0, function* () {
         try {
-            yield exec_1.exec(`"${yield getTarPath()}"`, args);
+            yield exec_1.exec(`"${yield getTarPath()}"`, args, { cwd: cwd });
         }
         catch (error) {
+            console.log("error", error);
             const IS_WINDOWS = process.platform === "win32";
             if (IS_WINDOWS) {
                 throw new Error(`Tar failed with error: ${(_a = error) === null || _a === void 0 ? void 0 : _a.message}. Ensure BSD tar is installed and on the PATH.`);
@@ -4933,19 +4961,22 @@ function extractTar(archivePath) {
     });
 }
 exports.extractTar = extractTar;
-function createTar(archivePath, sourceDirectories) {
+function createTar(archiveFolder, sourceDirectories) {
     return __awaiter(this, void 0, void 0, function* () {
         // TODO: will want to stream sourceDirectories into tar
+        const manifestFilename = "manifest.txt";
+        fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join("\n"));
         const workingDirectory = getWorkingDirectory();
         const args = [
             "-cz",
             "-f",
-            archivePath,
+            constants_1.CacheFilename,
             "-C",
             workingDirectory,
-            sourceDirectories.join(" ")
+            "--files-from",
+            manifestFilename
         ];
-        yield execTar(args);
+        yield execTar(args, archiveFolder);
     });
 }
 exports.createTar = createTar;

salt.txt

@@ -0,0 +1 @@
+Fri Mar  6 11:28:08 PST 2020

src/constants.ts

@@ -18,3 +18,5 @@ export enum Events {
     Push = "push",
     PullRequest = "pull_request"
 }
+
+export const CacheFilename = "cache.tgz";

src/save.ts

@@ -1,7 +1,7 @@
 import * as core from "@actions/core";
 import * as path from "path";
 import * as cacheHttpClient from "./cacheHttpClient";
-import { Events, Inputs, State } from "./constants";
+import { Events, Inputs, State, CacheFilename } from "./constants";
 import { createTar } from "./tar";
 import * as utils from "./utils/actionUtils";
 
@@ -43,7 +43,7 @@ async function run(): Promise<void> {
             return;
         }
         core.debug(`Cache ID: ${cacheId}`);
-        const cachePaths = await utils.expandPaths(
+        const cachePaths = await utils.resolvePaths(
             core
                 .getInput(Inputs.Path)
                 .split("\n")
@@ -53,13 +53,11 @@ async function run(): Promise<void> {
         core.debug("Cache Paths:");
         core.debug(`${JSON.stringify(cachePaths)}`);
 
-        const archivePath = path.join(
-            await utils.createTempDirectory(),
-            "cache.tgz"
-        );
+        const archiveFolder = await utils.createTempDirectory();
+        const archivePath = path.join(archiveFolder, CacheFilename);
         core.debug(`Archive Path: ${archivePath}`);
 
-        await createTar(archivePath, cachePaths);
+        await createTar(archiveFolder, cachePaths);
 
         const fileSizeLimit = 5 * 1024 * 1024 * 1024; // 5GB per repo limit
         const archiveFileSize = utils.getArchiveFileSize(archivePath);

src/tar.ts

@@ -1,6 +1,8 @@
-import { exec } from "@actions/exec";
 import * as io from "@actions/io";
-import { existsSync } from "fs";
+import * as path from "path";
+import { CacheFilename } from "./constants";
+import { exec } from "@actions/exec";
+import { existsSync, writeFileSync } from "fs";
 
 async function getTarPath(): Promise<string> {
     // Explicitly use BSD Tar on Windows
@@ -14,10 +16,12 @@ async function getTarPath(): Promise<string> {
     return await io.which("tar", true);
 }
 
-async function execTar(args: string[]): Promise<void> {
+async function execTar(args: string[], cwd?: string): Promise<void> {
     try {
-        await exec(`"${await getTarPath()}"`, args);
+        await exec(`"${await getTarPath()}"`, args, { cwd: cwd });
     } catch (error) {
+        console.log("error", error);
+
         const IS_WINDOWS = process.platform === "win32";
         if (IS_WINDOWS) {
             throw new Error(
@@ -41,18 +45,25 @@ export async function extractTar(archivePath: string): Promise<void> {
 }
 
 export async function createTar(
-    archivePath: string,
+    archiveFolder: string,
     sourceDirectories: string[]
 ): Promise<void> {
     // TODO: will want to stream sourceDirectories into tar
+    const manifestFilename = "manifest.txt";
+    writeFileSync(
+        path.join(archiveFolder, manifestFilename),
+        sourceDirectories.join("\n")
+    );
+
     const workingDirectory = getWorkingDirectory();
     const args = [
         "-cz",
         "-f",
-        archivePath,
+        CacheFilename,
         "-C",
         workingDirectory,
-        sourceDirectories.join(" ")
+        "--files-from",
+        manifestFilename
     ];
-    await execTar(args);
+    await execTar(args, archiveFolder);
 }

src/utils/actionUtils.ts

@@ -28,6 +28,7 @@ export async function createTempDirectory(): Promise<string> {
         }
         tempDirectory = path.join(baseLocation, "actions", "temp");
     }
+
     const dest = path.join(tempDirectory, uuidV4.default());
     await io.mkdirP(dest);
     return dest;
@@ -82,17 +83,21 @@ export function logWarning(message: string): void {
     core.info(`${warningPrefix}${message}`);
 }
 
-export async function expandPaths(patterns: string[]): Promise<string[]> {
+export async function resolvePaths(patterns: string[]): Promise<string[]> {
     const paths: string[] = [];
     const workspace = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
+    const globber = await glob.create(patterns.join("\n"), {
+        implicitDescendants: false
+    });
 
-    const globber = await glob.create(patterns.join("\n"));
-    const files = await globber.glob();
+    for await (const file of globber.globGenerator()) {
+        const relativeFile = path.relative(workspace, file);
+        core.debug(`Matched: ${relativeFile}`);
+        // Paths are made relative so the tar entries are all relative to the root of the workspace.
+        paths.push(`${relativeFile}`);
+    }
 
-    paths.push(...files);
-
-    // Convert paths to relative paths here?
-    return paths.map(x => path.relative(workspace, x));
+    return paths;
 }
 
 export function getSupportedEvents(): string[] {