mirror of
https://code.forgejo.org/actions/cache.git
synced 2024-11-23 20:19:16 +01:00
Format and update tests
This commit is contained in:
parent
2ce22df8c4
commit
cdec5dec0d
4 changed files with 122 additions and 81 deletions
|
@ -212,14 +212,14 @@ test("save with large cache outputs warning", async () => {
|
||||||
const IS_WINDOWS = process.platform === "win32";
|
const IS_WINDOWS = process.platform === "win32";
|
||||||
const args = IS_WINDOWS
|
const args = IS_WINDOWS
|
||||||
? [
|
? [
|
||||||
"-cz",
|
"-cz",
|
||||||
"--force-local",
|
"--force-local",
|
||||||
"-f",
|
"-f",
|
||||||
archivePath.replace(/\\/g, "/"),
|
archivePath.replace(/\\/g, "/"),
|
||||||
"-C",
|
"-C",
|
||||||
cachePath.replace(/\\/g, "/"),
|
cachePath.replace(/\\/g, "/"),
|
||||||
"."
|
"."
|
||||||
]
|
]
|
||||||
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
|
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
|
||||||
|
|
||||||
expect(execMock).toHaveBeenCalledTimes(1);
|
expect(execMock).toHaveBeenCalledTimes(1);
|
||||||
|
@ -259,6 +259,11 @@ test("save with server error outputs warning", async () => {
|
||||||
const cachePath = path.resolve(inputPath);
|
const cachePath = path.resolve(inputPath);
|
||||||
testUtils.setInput(Inputs.Path, inputPath);
|
testUtils.setInput(Inputs.Path, inputPath);
|
||||||
|
|
||||||
|
const cacheId = 4;
|
||||||
|
const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => {
|
||||||
|
return Promise.resolve(cacheId);
|
||||||
|
});
|
||||||
|
|
||||||
const execMock = jest.spyOn(exec, "exec");
|
const execMock = jest.spyOn(exec, "exec");
|
||||||
|
|
||||||
const saveCacheMock = jest
|
const saveCacheMock = jest
|
||||||
|
@ -269,26 +274,29 @@ test("save with server error outputs warning", async () => {
|
||||||
|
|
||||||
await run();
|
await run();
|
||||||
|
|
||||||
|
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
|
||||||
|
|
||||||
const archivePath = path.join("/foo/bar", "cache.tgz");
|
const archivePath = path.join("/foo/bar", "cache.tgz");
|
||||||
|
|
||||||
const IS_WINDOWS = process.platform === "win32";
|
const IS_WINDOWS = process.platform === "win32";
|
||||||
const args = IS_WINDOWS
|
const args = IS_WINDOWS
|
||||||
? [
|
? [
|
||||||
"-cz",
|
"-cz",
|
||||||
"--force-local",
|
"--force-local",
|
||||||
"-f",
|
"-f",
|
||||||
archivePath.replace(/\\/g, "/"),
|
archivePath.replace(/\\/g, "/"),
|
||||||
"-C",
|
"-C",
|
||||||
cachePath.replace(/\\/g, "/"),
|
cachePath.replace(/\\/g, "/"),
|
||||||
"."
|
"."
|
||||||
]
|
]
|
||||||
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
|
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
|
||||||
|
|
||||||
expect(execMock).toHaveBeenCalledTimes(1);
|
expect(execMock).toHaveBeenCalledTimes(1);
|
||||||
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
|
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
|
||||||
|
|
||||||
expect(saveCacheMock).toHaveBeenCalledTimes(1);
|
expect(saveCacheMock).toHaveBeenCalledTimes(1);
|
||||||
expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath);
|
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
|
||||||
|
|
||||||
expect(logWarningMock).toHaveBeenCalledTimes(1);
|
expect(logWarningMock).toHaveBeenCalledTimes(1);
|
||||||
expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred");
|
expect(logWarningMock).toHaveBeenCalledWith("HTTP Error Occurred");
|
||||||
|
@ -321,32 +329,40 @@ test("save with valid inputs uploads a cache", async () => {
|
||||||
const cachePath = path.resolve(inputPath);
|
const cachePath = path.resolve(inputPath);
|
||||||
testUtils.setInput(Inputs.Path, inputPath);
|
testUtils.setInput(Inputs.Path, inputPath);
|
||||||
|
|
||||||
|
const cacheId = 4;
|
||||||
|
const reserveCacheMock = jest.spyOn(cacheHttpClient, "reserveCache").mockImplementationOnce(() => {
|
||||||
|
return Promise.resolve(cacheId);
|
||||||
|
});
|
||||||
|
|
||||||
const execMock = jest.spyOn(exec, "exec");
|
const execMock = jest.spyOn(exec, "exec");
|
||||||
|
|
||||||
const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache");
|
const saveCacheMock = jest.spyOn(cacheHttpClient, "saveCache");
|
||||||
|
|
||||||
await run();
|
await run();
|
||||||
|
|
||||||
|
expect(reserveCacheMock).toHaveBeenCalledTimes(1);
|
||||||
|
expect(reserveCacheMock).toHaveBeenCalledWith(primaryKey);
|
||||||
|
|
||||||
const archivePath = path.join("/foo/bar", "cache.tgz");
|
const archivePath = path.join("/foo/bar", "cache.tgz");
|
||||||
|
|
||||||
const IS_WINDOWS = process.platform === "win32";
|
const IS_WINDOWS = process.platform === "win32";
|
||||||
const args = IS_WINDOWS
|
const args = IS_WINDOWS
|
||||||
? [
|
? [
|
||||||
"-cz",
|
"-cz",
|
||||||
"--force-local",
|
"--force-local",
|
||||||
"-f",
|
"-f",
|
||||||
archivePath.replace(/\\/g, "/"),
|
archivePath.replace(/\\/g, "/"),
|
||||||
"-C",
|
"-C",
|
||||||
cachePath.replace(/\\/g, "/"),
|
cachePath.replace(/\\/g, "/"),
|
||||||
"."
|
"."
|
||||||
]
|
]
|
||||||
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
|
: ["-cz", "-f", archivePath, "-C", cachePath, "."];
|
||||||
|
|
||||||
expect(execMock).toHaveBeenCalledTimes(1);
|
expect(execMock).toHaveBeenCalledTimes(1);
|
||||||
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
|
expect(execMock).toHaveBeenCalledWith(`"tar"`, args);
|
||||||
|
|
||||||
expect(saveCacheMock).toHaveBeenCalledTimes(1);
|
expect(saveCacheMock).toHaveBeenCalledTimes(1);
|
||||||
expect(saveCacheMock).toHaveBeenCalledWith(primaryKey, archivePath);
|
expect(saveCacheMock).toHaveBeenCalledWith(cacheId, archivePath);
|
||||||
|
|
||||||
expect(failedMock).toHaveBeenCalledTimes(0);
|
expect(failedMock).toHaveBeenCalledTimes(0);
|
||||||
});
|
});
|
||||||
|
|
|
@ -107,9 +107,7 @@ export async function downloadCache(
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reserve Cache
|
// Reserve Cache
|
||||||
export async function reserveCache(
|
export async function reserveCache(key: string): Promise<number> {
|
||||||
key: string
|
|
||||||
): Promise<number> {
|
|
||||||
const restClient = createRestClient();
|
const restClient = createRestClient();
|
||||||
|
|
||||||
const reserveCacheRequest: ReserveCacheRequest = {
|
const reserveCacheRequest: ReserveCacheRequest = {
|
||||||
|
@ -133,14 +131,6 @@ function getContentRange(start: number, end: number): string {
|
||||||
return `bytes ${start}-${end}/*`;
|
return `bytes ${start}-${end}/*`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// function bufferToStream(buffer: Buffer): NodeJS.ReadableStream {
|
|
||||||
// const stream = new Duplex();
|
|
||||||
// stream.push(buffer);
|
|
||||||
// stream.push(null);
|
|
||||||
|
|
||||||
// return stream;
|
|
||||||
// }
|
|
||||||
|
|
||||||
async function uploadChunk(
|
async function uploadChunk(
|
||||||
restClient: RestClient,
|
restClient: RestClient,
|
||||||
resourceUrl: string,
|
resourceUrl: string,
|
||||||
|
@ -148,14 +138,87 @@ async function uploadChunk(
|
||||||
start: number,
|
start: number,
|
||||||
end: number
|
end: number
|
||||||
): Promise<IRestResponse<void>> {
|
): Promise<IRestResponse<void>> {
|
||||||
core.debug(`Uploading chunk of size ${end - start + 1} bytes at offset ${start} with content range: ${getContentRange(start, end)}`);
|
core.debug(
|
||||||
|
`Uploading chunk of size ${end -
|
||||||
|
start +
|
||||||
|
1} bytes at offset ${start} with content range: ${getContentRange(
|
||||||
|
start,
|
||||||
|
end
|
||||||
|
)}`
|
||||||
|
);
|
||||||
const requestOptions = getRequestOptions();
|
const requestOptions = getRequestOptions();
|
||||||
requestOptions.additionalHeaders = {
|
requestOptions.additionalHeaders = {
|
||||||
"Content-Type": "application/octet-stream",
|
"Content-Type": "application/octet-stream",
|
||||||
"Content-Range": getContentRange(start, end)
|
"Content-Range": getContentRange(start, end)
|
||||||
};
|
};
|
||||||
|
|
||||||
return await restClient.uploadStream<void>("PATCH", resourceUrl, data, requestOptions);
|
return await restClient.uploadStream<void>(
|
||||||
|
"PATCH",
|
||||||
|
resourceUrl,
|
||||||
|
data,
|
||||||
|
requestOptions
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function uploadFile(
|
||||||
|
restClient: RestClient,
|
||||||
|
cacheId: number,
|
||||||
|
archivePath: string
|
||||||
|
): Promise<void> {
|
||||||
|
// Upload Chunks
|
||||||
|
const fileSize = fs.statSync(archivePath).size;
|
||||||
|
const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString();
|
||||||
|
const responses: IRestResponse<void>[] = [];
|
||||||
|
const fd = fs.openSync(archivePath, "r");
|
||||||
|
|
||||||
|
const concurrency = 4; // # of HTTP requests in parallel
|
||||||
|
const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks
|
||||||
|
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
|
||||||
|
|
||||||
|
const parallelUploads = [...new Array(concurrency).keys()];
|
||||||
|
core.debug("Awaiting all uploads");
|
||||||
|
let offset = 0;
|
||||||
|
await Promise.all(
|
||||||
|
parallelUploads.map(async () => {
|
||||||
|
while (offset < fileSize) {
|
||||||
|
const chunkSize =
|
||||||
|
offset + MAX_CHUNK_SIZE > fileSize
|
||||||
|
? fileSize - offset
|
||||||
|
: MAX_CHUNK_SIZE;
|
||||||
|
const start = offset;
|
||||||
|
const end = offset + chunkSize - 1;
|
||||||
|
offset += MAX_CHUNK_SIZE;
|
||||||
|
const chunk = fs.createReadStream(archivePath, {
|
||||||
|
fd,
|
||||||
|
start,
|
||||||
|
end,
|
||||||
|
autoClose: false
|
||||||
|
});
|
||||||
|
responses.push(
|
||||||
|
await uploadChunk(
|
||||||
|
restClient,
|
||||||
|
resourceUrl,
|
||||||
|
chunk,
|
||||||
|
start,
|
||||||
|
end
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
fs.closeSync(fd);
|
||||||
|
|
||||||
|
const failedResponse = responses.find(
|
||||||
|
x => !isSuccessStatusCode(x.statusCode)
|
||||||
|
);
|
||||||
|
if (failedResponse) {
|
||||||
|
throw new Error(
|
||||||
|
`Cache service responded with ${failedResponse.statusCode} during chunk upload.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function commitCache(
|
async function commitCache(
|
||||||
|
@ -172,44 +235,6 @@ async function commitCache(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function uploadFile(restClient: RestClient, cacheId: number, archivePath: string): Promise<void> {
|
|
||||||
// Upload Chunks
|
|
||||||
const fileSize = fs.statSync(archivePath).size;
|
|
||||||
const resourceUrl = getCacheApiUrl() + "caches/" + cacheId.toString();
|
|
||||||
const responses: IRestResponse<void>[] = [];
|
|
||||||
const fd = fs.openSync(archivePath, "r");
|
|
||||||
|
|
||||||
const concurrency = 16; // # of HTTP requests in parallel
|
|
||||||
const MAX_CHUNK_SIZE = 32000000; // 32 MB Chunks
|
|
||||||
core.debug(`Concurrency: ${concurrency} and Chunk Size: ${MAX_CHUNK_SIZE}`);
|
|
||||||
const parallelUploads = [...new Array(concurrency).keys()];
|
|
||||||
core.debug("Awaiting all uploads");
|
|
||||||
let offset = 0;
|
|
||||||
await Promise.all(parallelUploads.map(async () => {
|
|
||||||
while (offset < fileSize) {
|
|
||||||
const chunkSize = offset + MAX_CHUNK_SIZE > fileSize ? fileSize - offset : MAX_CHUNK_SIZE;
|
|
||||||
const start = offset;
|
|
||||||
const end = offset + chunkSize - 1;
|
|
||||||
offset += MAX_CHUNK_SIZE;
|
|
||||||
const chunk = fs.createReadStream(archivePath, { fd, start, end, autoClose: false });
|
|
||||||
responses.push(await uploadChunk(restClient, resourceUrl, chunk, start, end));
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
fs.closeSync(fd);
|
|
||||||
|
|
||||||
const failedResponse = responses.find(
|
|
||||||
x => !isSuccessStatusCode(x.statusCode)
|
|
||||||
);
|
|
||||||
if (failedResponse) {
|
|
||||||
throw new Error(
|
|
||||||
`Cache service responded with ${failedResponse.statusCode} during chunk upload.`
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function saveCache(
|
export async function saveCache(
|
||||||
cacheId: number,
|
cacheId: number,
|
||||||
archivePath: string
|
archivePath: string
|
||||||
|
@ -219,8 +244,8 @@ export async function saveCache(
|
||||||
core.debug("Upload cache");
|
core.debug("Upload cache");
|
||||||
await uploadFile(restClient, cacheId, archivePath);
|
await uploadFile(restClient, cacheId, archivePath);
|
||||||
|
|
||||||
core.debug("Commiting cache");
|
|
||||||
// Commit Cache
|
// Commit Cache
|
||||||
|
core.debug("Commiting cache");
|
||||||
const cacheSize = utils.getArchiveFileSize(archivePath);
|
const cacheSize = utils.getArchiveFileSize(archivePath);
|
||||||
const commitCacheResponse = await commitCache(
|
const commitCacheResponse = await commitCache(
|
||||||
restClient,
|
restClient,
|
||||||
|
|
|
@ -82,7 +82,6 @@ async function run(): Promise<void> {
|
||||||
cacheEntry?.archiveLocation,
|
cacheEntry?.archiveLocation,
|
||||||
archivePath
|
archivePath
|
||||||
);
|
);
|
||||||
await exec(`md5sum`, [archivePath]);
|
|
||||||
|
|
||||||
const archiveFileSize = utils.getArchiveFileSize(archivePath);
|
const archiveFileSize = utils.getArchiveFileSize(archivePath);
|
||||||
core.info(
|
core.info(
|
||||||
|
|
|
@ -38,7 +38,9 @@ async function run(): Promise<void> {
|
||||||
core.debug("Reserving Cache");
|
core.debug("Reserving Cache");
|
||||||
const cacheId = await cacheHttpClient.reserveCache(primaryKey);
|
const cacheId = await cacheHttpClient.reserveCache(primaryKey);
|
||||||
if (cacheId < 0) {
|
if (cacheId < 0) {
|
||||||
core.info(`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`);
|
core.info(
|
||||||
|
`Unable to reserve cache with key ${primaryKey}, another job may be creating this cache.`
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
core.debug(`Cache ID: ${cacheId}`);
|
core.debug(`Cache ID: ${cacheId}`);
|
||||||
|
@ -84,7 +86,6 @@ async function run(): Promise<void> {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
await exec(`md5sum`, [archivePath]);
|
|
||||||
core.debug("Saving Cache");
|
core.debug("Saving Cache");
|
||||||
await cacheHttpClient.saveCache(cacheId, archivePath);
|
await cacheHttpClient.saveCache(cacheId, archivePath);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
Loading…
Reference in a new issue