From fc1a0f503808c042223667e5a9f45d4e4a96a302 Mon Sep 17 00:00:00 2001
From: Sam Stentz <samstentz@google.com>
Date: Tue, 28 Nov 2023 19:11:56 +0000
Subject: [PATCH] gcs used for backing
---
README.md | 2 ++
action.yml | 14 +++++---
src/constants.ts | 1 +
src/restoreImpl.ts | 85 ++++++++++------------------------------------
src/saveImpl.ts | 65 +++++++++++++----------------------
5 files changed, 54 insertions(+), 113 deletions(-)
diff --git a/README.md b/README.md
index 7f27d6c..2b7f614 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+# **Disclaimer**: Modified to use GCS bucket as backing. Only supports default cache action and assumes gsutil is configured as well as gcloud auth.
+
# Cache action
This action allows caching dependencies and build outputs to improve workflow execution time.
diff --git a/action.yml b/action.yml
index 5c6fa87..0687b38 100644
--- a/action.yml
+++ b/action.yml
@@ -2,6 +2,10 @@ name: 'Cache'
description: 'Cache artifacts like dependencies and build outputs to improve workflow execution time'
author: 'GitHub'
inputs:
+ bucket:
+ description: 'The GCS bucket holding cache.'
+ default: 'ingestion-engine-dev-scratch-artifacts'
+ required: false
path:
description: 'A list of files, directories, and wildcard patterns to cache and restore'
required: true
@@ -9,21 +13,21 @@ inputs:
description: 'An explicit key for restoring and saving the cache'
required: true
restore-keys:
- description: 'An ordered list of keys to use for restoring stale cache if no cache hit occurred for key. Note `cache-hit` returns false in this case.'
+ description: 'UNIMPLEMENTED: An ordered list of keys to use for restoring stale cache if no cache hit occurred for key. Note `cache-hit` returns false in this case.'
required: false
upload-chunk-size:
- description: 'The chunk size used to split up large files during upload, in bytes'
+ description: 'UNIMPLEMENTED: The chunk size used to split up large files during upload, in bytes'
required: false
enableCrossOsArchive:
- description: 'An optional boolean when enabled, allows windows runners to save or restore caches that can be restored or saved respectively on other platforms'
+ description: 'UNIMPLEMENTED: An optional boolean when enabled, allows windows runners to save or restore caches that can be restored or saved respectively on other platforms'
default: 'false'
required: false
fail-on-cache-miss:
- description: 'Fail the workflow if cache entry is not found'
+ description: 'UNIMPLEMENTED: Fail the workflow if cache entry is not found'
default: 'false'
required: false
lookup-only:
- description: 'Check if a cache entry exists for the given input(s) (key, restore-keys) without downloading the cache'
+ description: 'UNIMPLEMENTED: Check if a cache entry exists for the given input(s) (key, restore-keys) without downloading the cache'
default: 'false'
required: false
outputs:
diff --git a/src/constants.ts b/src/constants.ts
index 0158ae0..7c264a0 100644
--- a/src/constants.ts
+++ b/src/constants.ts
@@ -1,4 +1,5 @@
export enum Inputs {
+ Bucket = "bucket", // Input for cache action.
Key = "key", // Input for cache, restore, save action
Path = "path", // Input for cache, restore, save action
RestoreKeys = "restore-keys", // Input for cache, restore action
diff --git a/src/restoreImpl.ts b/src/restoreImpl.ts
index 0aff57a..3d88154 100644
--- a/src/restoreImpl.ts
+++ b/src/restoreImpl.ts
@@ -1,17 +1,10 @@
-import * as cache from "@actions/cache";
import * as core from "@actions/core";
+import { exec } from "@actions/exec";
-import { Events, Inputs, Outputs, State } from "./constants";
-import {
- IStateProvider,
- NullStateProvider,
- StateProvider
-} from "./stateProvider";
+import { Events, Inputs, Outputs } from "./constants";
import * as utils from "./utils/actionUtils";
-export async function restoreImpl(
- stateProvider: IStateProvider
-): Promise<string | undefined> {
+export async function restoreImpl(): Promise<string | undefined> {
try {
if (!utils.isCacheFeatureAvailable()) {
core.setOutput(Outputs.CacheHit, "false");
@@ -28,70 +21,28 @@ export async function restoreImpl(
return;
}
- const primaryKey = core.getInput(Inputs.Key, { required: true });
- stateProvider.setState(State.CachePrimaryKey, primaryKey);
-
- const restoreKeys = utils.getInputAsArray(Inputs.RestoreKeys);
- const cachePaths = utils.getInputAsArray(Inputs.Path, {
- required: true
- });
- const enableCrossOsArchive = utils.getInputAsBool(
- Inputs.EnableCrossOsArchive
- );
- const failOnCacheMiss = utils.getInputAsBool(Inputs.FailOnCacheMiss);
- const lookupOnly = utils.getInputAsBool(Inputs.LookupOnly);
-
- const cacheKey = await cache.restoreCache(
- cachePaths,
- primaryKey,
- restoreKeys,
- { lookupOnly: lookupOnly },
- enableCrossOsArchive
- );
-
- if (!cacheKey) {
- if (failOnCacheMiss) {
- throw new Error(
- `Failed to restore cache entry. Exiting as fail-on-cache-miss is set. Input key: ${primaryKey}`
- );
- }
- core.info(
- `Cache not found for input keys: ${[
- primaryKey,
- ...restoreKeys
- ].join(", ")}`
- );
-
+ const key = core.getInput(Inputs.Key, { required: true });
+ const bucket = core.getInput(Inputs.Bucket);
+ const workspace = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
+ const exitCode = await exec("/bin/bash", [
+ "-c",
+ `gsutil -o 'GSUtil:parallel_thread_count=1' -o 'GSUtil:sliced_object_download_max_components=8' cp "gs://${bucket}/${key}" - | tar --skip-old-files -x -P -C "${workspace}"`
+ ]);
+ if (exitCode === 1) {
+ console.log("[warning]Failed to extract cache...");
return;
}
- // Store the matched cache key in states
- stateProvider.setState(State.CacheMatchedKey, cacheKey);
-
- const isExactKeyMatch = utils.isExactKeyMatch(
- core.getInput(Inputs.Key, { required: true }),
- cacheKey
- );
-
- core.setOutput(Outputs.CacheHit, isExactKeyMatch.toString());
- if (lookupOnly) {
- core.info(`Cache found and can be restored from key: ${cacheKey}`);
- } else {
- core.info(`Cache restored from key: ${cacheKey}`);
- }
-
- return cacheKey;
+ // cache-id return set to 1
+ return "1";
} catch (error: unknown) {
core.setFailed((error as Error).message);
}
}
-async function run(
- stateProvider: IStateProvider,
- earlyExit: boolean | undefined
-): Promise<void> {
+async function run(earlyExit: boolean | undefined): Promise<void> {
try {
- await restoreImpl(stateProvider);
+ await restoreImpl();
} catch (err) {
console.error(err);
if (earlyExit) {
@@ -112,11 +63,11 @@ async function run(
export async function restoreOnlyRun(
earlyExit?: boolean | undefined
): Promise<void> {
- await run(new NullStateProvider(), earlyExit);
+ await run(earlyExit);
}
export async function restoreRun(
earlyExit?: boolean | undefined
): Promise<void> {
- await run(new StateProvider(), earlyExit);
+ await run(earlyExit);
}
diff --git a/src/saveImpl.ts b/src/saveImpl.ts
index 5fd4513..a4fe3c5 100644
--- a/src/saveImpl.ts
+++ b/src/saveImpl.ts
@@ -1,8 +1,10 @@
-import * as cache from "@actions/cache";
+import * as cacheUtils from "@actions/cache/lib/internal/cacheUtils";
import * as core from "@actions/core";
+import { exec } from "@actions/exec";
+import { writeFileSync } from "fs";
+import * as path from "path";
-import { Events, Inputs, State } from "./constants";
-import { IStateProvider } from "./stateProvider";
+import { Events, Inputs } from "./constants";
import * as utils from "./utils/actionUtils";
// Catch and log any unhandled exceptions. These exceptions can leak out of the uploadChunk method in
@@ -10,8 +12,7 @@ import * as utils from "./utils/actionUtils";
// throw an uncaught exception. Instead of failing this action, just warn.
process.on("uncaughtException", e => utils.logWarning(e.message));
-async function saveImpl(stateProvider: IStateProvider): Promise<number | void> {
- let cacheId = -1;
+async function saveImpl(): Promise<number | void> {
try {
if (!utils.isCacheFeatureAvailable()) {
return;
@@ -26,50 +27,32 @@ async function saveImpl(stateProvider: IStateProvider): Promise<number | void> {
return;
}
- // If restore has stored a primary key in state, reuse that
- // Else re-evaluate from inputs
- const primaryKey =
- stateProvider.getState(State.CachePrimaryKey) ||
- core.getInput(Inputs.Key);
-
- if (!primaryKey) {
- utils.logWarning(`Key is not specified.`);
- return;
- }
-
- // If matched restore key is same as primary key, then do not save cache
- // NO-OP in case of SaveOnly action
- const restoredKey = stateProvider.getCacheState();
-
- if (utils.isExactKeyMatch(primaryKey, restoredKey)) {
- core.info(
- `Cache hit occurred on the primary key ${primaryKey}, not saving cache.`
- );
- return;
- }
-
- const cachePaths = utils.getInputAsArray(Inputs.Path, {
+ const key = core.getInput(Inputs.Key);
+ const bucket = core.getInput(Inputs.Bucket);
+ const paths = utils.getInputAsArray(Inputs.Path, {
required: true
});
- const enableCrossOsArchive = utils.getInputAsBool(
- Inputs.EnableCrossOsArchive
- );
+ // https://github.com/actions/toolkit/blob/c861dd8859fe5294289fcada363ce9bc71e9d260/packages/cache/src/internal/tar.ts#L75
+ const cachePaths = await cacheUtils.resolvePaths(paths);
+ const tmpFolder = await cacheUtils.createTempDirectory();
+ // Write source directories to manifest.txt to avoid command length limits
+ const manifestPath = path.join(tmpFolder, "manifest.txt");
+ writeFileSync(manifestPath, cachePaths.join("\n"));
- cacheId = await cache.saveCache(
- cachePaths,
- primaryKey,
- { uploadChunkSize: utils.getInputAsInt(Inputs.UploadChunkSize) },
- enableCrossOsArchive
- );
-
- if (cacheId != -1) {
- core.info(`Cache saved with key: ${primaryKey}`);
+ const workspace = process.env["GITHUB_WORKSPACE"] ?? process.cwd();
+ const exitCode = await exec("/bin/bash", [
+ "-c",
+ `tar -cf - -P -C ${workspace} --files-from ${manifestPath} | gsutil -o 'GSUtil:parallel_composite_upload_threshold=250M' cp - "gs://${bucket}/${key}"`
+ ]);
+ if (exitCode !== 0) {
+ utils.logWarning("Failed to upload cache...");
}
} catch (error: unknown) {
utils.logWarning((error as Error).message);
}
- return cacheId;
+ // cache-id return set to 1
+ return 1;
}
export default saveImpl;