actions/cache
1
0
Fork 0
mirror of https://code.forgejo.org/actions/cache.git synced 2024-11-23 20:19:16 +01:00
Code Activity

Fix upload chunk retries

Browse source
This commit is contained in:
Dave Hadka 2020-05-11 10:49:48 -04:00 committed by Aiqiao Yan
parent ec7f7ebd08
commit da9f90cb83
3 changed files with 153 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
dist/restore/index.js vendored
View file

@ -1382,7 +1382,7 @@ function getContentRange(start, end) {
// Content-Range: bytes 0-199/* // Content-Range: bytes 0-199/*
return `bytes ${start}-${end}/*`; return `bytes ${start}-${end}/*`;
} }
function uploadChunk(httpClient, resourceUrl, data, start, end) { function uploadChunk(httpClient, resourceUrl, openStream, start, end) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
core.debug(`Uploading chunk of size ${end - core.debug(`Uploading chunk of size ${end -
start + start +
@ -1392,7 +1392,7 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) {
"Content-Range": getContentRange(start, end) "Content-Range": getContentRange(start, end)
}; };
const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () {
return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders); return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders);
}); });
const response = yield uploadChunkRequest(); const response = yield uploadChunkRequest();
if (isSuccessStatusCode(response.message.statusCode)) { if (isSuccessStatusCode(response.message.statusCode)) {
@ -1435,13 +1435,12 @@ function uploadFile(httpClient, cacheId, archivePath) {
const start = offset; const start = offset;
const end = offset + chunkSize - 1; const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE; offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, { yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, {
fd, fd,
start, start,
end, end,
autoClose: false autoClose: false
}); }), start, end);
yield uploadChunk(httpClient, resourceUrl, chunk, start, end);
} }
}))); })));
} }
@ -1496,7 +1495,9 @@ class BasicCredentialHandler {
this.password = password; this.password = password;
} }
prepareRequest(options) { prepareRequest(options) {
options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64'); options.headers['Authorization'] =
'Basic ' +
Buffer.from(this.username + ':' + this.password).toString('base64');
} }
// This handler cannot handle 401 // This handler cannot handle 401
canHandleAuthentication(response) { canHandleAuthentication(response) {
@ -1532,7 +1533,8 @@ class PersonalAccessTokenCredentialHandler {
// currently implements pre-authorization // currently implements pre-authorization
// TODO: support preAuth = false where it hooks on 401 // TODO: support preAuth = false where it hooks on 401
prepareRequest(options) { prepareRequest(options) {
options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); options.headers['Authorization'] =
'Basic ' + Buffer.from('PAT:' + this.token).toString('base64');
} }
// This handler cannot handle 401 // This handler cannot handle 401
canHandleAuthentication(response) { canHandleAuthentication(response) {
@ -2001,6 +2003,7 @@ var HttpCodes;
HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout";
HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict";
HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; HttpCodes[HttpCodes["Gone"] = 410] = "Gone";
HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests";
HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError";
HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented";
HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway";
@ -2025,8 +2028,18 @@ function getProxyUrl(serverUrl) {
return proxyUrl ? proxyUrl.href : ''; return proxyUrl ? proxyUrl.href : '';
} }
exports.getProxyUrl = getProxyUrl; exports.getProxyUrl = getProxyUrl;
const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect]; const HttpRedirectCodes = [
const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout]; HttpCodes.MovedPermanently,
HttpCodes.ResourceMoved,
HttpCodes.SeeOther,
HttpCodes.TemporaryRedirect,
HttpCodes.PermanentRedirect
];
const HttpResponseRetryCodes = [
HttpCodes.BadGateway,
HttpCodes.ServiceUnavailable,
HttpCodes.GatewayTimeout
];
const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD'];
const ExponentialBackoffCeiling = 10; const ExponentialBackoffCeiling = 10;
const ExponentialBackoffTimeSlice = 5; const ExponentialBackoffTimeSlice = 5;
@ -2157,19 +2170,23 @@ class HttpClient {
*/ */
async request(verb, requestUrl, data, headers) { async request(verb, requestUrl, data, headers) {
if (this._disposed) { if (this._disposed) {
throw new Error("Client has already been disposed."); throw new Error('Client has already been disposed.');
} }
let parsedUrl = url.parse(requestUrl); let parsedUrl = url.parse(requestUrl);
let info = this._prepareRequest(verb, parsedUrl, headers); let info = this._prepareRequest(verb, parsedUrl, headers);
// Only perform retries on reads since writes may not be idempotent. // Only perform retries on reads since writes may not be idempotent.
let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1; let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1
? this._maxRetries + 1
: 1;
let numTries = 0; let numTries = 0;
let response; let response;
while (numTries < maxTries) { while (numTries < maxTries) {
response = await this.requestRaw(info, data); response = await this.requestRaw(info, data);
// Check if it's an authentication challenge // Check if it's an authentication challenge
if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { if (response &&
response.message &&
response.message.statusCode === HttpCodes.Unauthorized) {
let authenticationHandler; let authenticationHandler;
for (let i = 0; i < this.handlers.length; i++) { for (let i = 0; i < this.handlers.length; i++) {
if (this.handlers[i].canHandleAuthentication(response)) { if (this.handlers[i].canHandleAuthentication(response)) {
@ -2187,21 +2204,32 @@ class HttpClient {
} }
} }
let redirectsRemaining = this._maxRedirects; let redirectsRemaining = this._maxRedirects;
while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 &&
&& this._allowRedirects this._allowRedirects &&
&& redirectsRemaining > 0) { redirectsRemaining > 0) {
const redirectUrl = response.message.headers["location"]; const redirectUrl = response.message.headers['location'];
if (!redirectUrl) { if (!redirectUrl) {
// if there's no location to redirect to, we won't // if there's no location to redirect to, we won't
break; break;
} }
let parsedRedirectUrl = url.parse(redirectUrl); let parsedRedirectUrl = url.parse(redirectUrl);
if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) { if (parsedUrl.protocol == 'https:' &&
throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true."); parsedUrl.protocol != parsedRedirectUrl.protocol &&
!this._allowRedirectDowngrade) {
throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.');
} }
// we need to finish reading the response before reassigning response // we need to finish reading the response before reassigning response
// which will leak the open socket. // which will leak the open socket.
await response.readBody(); await response.readBody();
// strip authorization header if redirected to a different hostname
if (parsedRedirectUrl.hostname !== parsedUrl.hostname) {
for (let header in headers) {
// header names are case insensitive
if (header.toLowerCase() === 'authorization') {
delete headers[header];
}
}
}
// let's make the request with the new redirectUrl // let's make the request with the new redirectUrl
info = this._prepareRequest(verb, parsedRedirectUrl, headers); info = this._prepareRequest(verb, parsedRedirectUrl, headers);
response = await this.requestRaw(info, data); response = await this.requestRaw(info, data);
@ -2252,8 +2280,8 @@ class HttpClient {
*/ */
requestRawWithCallback(info, data, onResult) { requestRawWithCallback(info, data, onResult) {
let socket; let socket;
if (typeof (data) === 'string') { if (typeof data === 'string') {
info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8'); info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8');
} }
let callbackCalled = false; let callbackCalled = false;
let handleResult = (err, res) => { let handleResult = (err, res) => {
@ -2266,7 +2294,7 @@ class HttpClient {
let res = new HttpClientResponse(msg); let res = new HttpClientResponse(msg);
handleResult(null, res); handleResult(null, res);
}); });
req.on('socket', (sock) => { req.on('socket', sock => {
socket = sock; socket = sock;
}); });
// If we ever get disconnected, we want the socket to timeout eventually // If we ever get disconnected, we want the socket to timeout eventually
@ -2281,10 +2309,10 @@ class HttpClient {
// res should have headers // res should have headers
handleResult(err, null); handleResult(err, null);
}); });
if (data && typeof (data) === 'string') { if (data && typeof data === 'string') {
req.write(data, 'utf8'); req.write(data, 'utf8');
} }
if (data && typeof (data) !== 'string') { if (data && typeof data !== 'string') {
data.on('close', function () { data.on('close', function () {
req.end(); req.end();
}); });
@ -2311,31 +2339,34 @@ class HttpClient {
const defaultPort = usingSsl ? 443 : 80; const defaultPort = usingSsl ? 443 : 80;
info.options = {}; info.options = {};
info.options.host = info.parsedUrl.hostname; info.options.host = info.parsedUrl.hostname;
info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort; info.options.port = info.parsedUrl.port
info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); ? parseInt(info.parsedUrl.port)
: defaultPort;
info.options.path =
(info.parsedUrl.pathname || '') + (info.parsedUrl.search || '');
info.options.method = method; info.options.method = method;
info.options.headers = this._mergeHeaders(headers); info.options.headers = this._mergeHeaders(headers);
if (this.userAgent != null) { if (this.userAgent != null) {
info.options.headers["user-agent"] = this.userAgent; info.options.headers['user-agent'] = this.userAgent;
} }
info.options.agent = this._getAgent(info.parsedUrl); info.options.agent = this._getAgent(info.parsedUrl);
// gives handlers an opportunity to participate // gives handlers an opportunity to participate
if (this.handlers) { if (this.handlers) {
this.handlers.forEach((handler) => { this.handlers.forEach(handler => {
handler.prepareRequest(info.options); handler.prepareRequest(info.options);
}); });
} }
return info; return info;
} }
_mergeHeaders(headers) { _mergeHeaders(headers) {
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {});
if (this.requestOptions && this.requestOptions.headers) { if (this.requestOptions && this.requestOptions.headers) {
return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers)); return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers));
} }
return lowercaseKeys(headers || {}); return lowercaseKeys(headers || {});
} }
_getExistingOrDefaultHeader(additionalHeaders, header, _default) { _getExistingOrDefaultHeader(additionalHeaders, header, _default) {
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {});
let clientHeader; let clientHeader;
if (this.requestOptions && this.requestOptions.headers) { if (this.requestOptions && this.requestOptions.headers) {
clientHeader = lowercaseKeys(this.requestOptions.headers)[header]; clientHeader = lowercaseKeys(this.requestOptions.headers)[header];
@ -2373,7 +2404,7 @@ class HttpClient {
proxyAuth: proxyUrl.auth, proxyAuth: proxyUrl.auth,
host: proxyUrl.hostname, host: proxyUrl.hostname,
port: proxyUrl.port port: proxyUrl.port
}, }
}; };
let tunnelAgent; let tunnelAgent;
const overHttps = proxyUrl.protocol === 'https:'; const overHttps = proxyUrl.protocol === 'https:';
@ -2400,7 +2431,9 @@ class HttpClient {
// we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process
// http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options
// we have to cast it to any and change it directly // we have to cast it to any and change it directly
agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false }); agent.options = Object.assign(agent.options || {}, {
rejectUnauthorized: false
});
} }
return agent; return agent;
} }
@ -2461,7 +2494,7 @@ class HttpClient {
msg = contents; msg = contents;
} }
else { else {
msg = "Failed request: (" + statusCode + ")"; msg = 'Failed request: (' + statusCode + ')';
} }
let err = new Error(msg); let err = new Error(msg);
// attach statusCode and body obj (if available) to the error object // attach statusCode and body obj (if available) to the error object
@ -3049,12 +3082,10 @@ function getProxyUrl(reqUrl) {
} }
let proxyVar; let proxyVar;
if (usingSsl) { if (usingSsl) {
proxyVar = process.env["https_proxy"] || proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY'];
process.env["HTTPS_PROXY"];
} }
else { else {
proxyVar = process.env["http_proxy"] || proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY'];
process.env["HTTP_PROXY"];
} }
if (proxyVar) { if (proxyVar) {
proxyUrl = url.parse(proxyVar); proxyUrl = url.parse(proxyVar);
@ -3066,7 +3097,7 @@ function checkBypass(reqUrl) {
if (!reqUrl.hostname) { if (!reqUrl.hostname) {
return false; return false;
} }
let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || ''; let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || '';
if (!noProxy) { if (!noProxy) {
return false; return false;
} }
@ -3087,7 +3118,10 @@ function checkBypass(reqUrl) {
upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`);
} }
// Compare request host against noproxy // Compare request host against noproxy
for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) { for (let upperNoProxyItem of noProxy
.split(',')
.map(x => x.trim().toUpperCase())
.filter(x => x)) {
if (upperReqHosts.some(x => x === upperNoProxyItem)) { if (upperReqHosts.some(x => x === upperNoProxyItem)) {
return true; return true;
} }

110
dist/save/index.js vendored
View file

@ -1381,7 +1381,7 @@ function getContentRange(start, end) {
// Content-Range: bytes 0-199/* // Content-Range: bytes 0-199/*
return `bytes ${start}-${end}/*`; return `bytes ${start}-${end}/*`;
} }
function uploadChunk(httpClient, resourceUrl, data, start, end) { function uploadChunk(httpClient, resourceUrl, openStream, start, end) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
core.debug(`Uploading chunk of size ${end - core.debug(`Uploading chunk of size ${end -
start + start +
@ -1391,7 +1391,7 @@ function uploadChunk(httpClient, resourceUrl, data, start, end) {
"Content-Range": getContentRange(start, end) "Content-Range": getContentRange(start, end)
}; };
const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () {
return yield httpClient.sendStream("PATCH", resourceUrl, data, additionalHeaders); return yield httpClient.sendStream("PATCH", resourceUrl, openStream(), additionalHeaders);
}); });
const response = yield uploadChunkRequest(); const response = yield uploadChunkRequest();
if (isSuccessStatusCode(response.message.statusCode)) { if (isSuccessStatusCode(response.message.statusCode)) {
@ -1434,13 +1434,12 @@ function uploadFile(httpClient, cacheId, archivePath) {
const start = offset; const start = offset;
const end = offset + chunkSize - 1; const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE; offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, { yield uploadChunk(httpClient, resourceUrl, () => fs.createReadStream(archivePath, {
fd, fd,
start, start,
end, end,
autoClose: false autoClose: false
}); }), start, end);
yield uploadChunk(httpClient, resourceUrl, chunk, start, end);
} }
}))); })));
} }
@ -1495,7 +1494,9 @@ class BasicCredentialHandler {
this.password = password; this.password = password;
} }
prepareRequest(options) { prepareRequest(options) {
options.headers['Authorization'] = 'Basic ' + Buffer.from(this.username + ':' + this.password).toString('base64'); options.headers['Authorization'] =
'Basic ' +
Buffer.from(this.username + ':' + this.password).toString('base64');
} }
// This handler cannot handle 401 // This handler cannot handle 401
canHandleAuthentication(response) { canHandleAuthentication(response) {
@ -1531,7 +1532,8 @@ class PersonalAccessTokenCredentialHandler {
// currently implements pre-authorization // currently implements pre-authorization
// TODO: support preAuth = false where it hooks on 401 // TODO: support preAuth = false where it hooks on 401
prepareRequest(options) { prepareRequest(options) {
options.headers['Authorization'] = 'Basic ' + Buffer.from('PAT:' + this.token).toString('base64'); options.headers['Authorization'] =
'Basic ' + Buffer.from('PAT:' + this.token).toString('base64');
} }
// This handler cannot handle 401 // This handler cannot handle 401
canHandleAuthentication(response) { canHandleAuthentication(response) {
@ -2000,6 +2002,7 @@ var HttpCodes;
HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout";
HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict";
HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; HttpCodes[HttpCodes["Gone"] = 410] = "Gone";
HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests";
HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError";
HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented";
HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway";
@ -2024,8 +2027,18 @@ function getProxyUrl(serverUrl) {
return proxyUrl ? proxyUrl.href : ''; return proxyUrl ? proxyUrl.href : '';
} }
exports.getProxyUrl = getProxyUrl; exports.getProxyUrl = getProxyUrl;
const HttpRedirectCodes = [HttpCodes.MovedPermanently, HttpCodes.ResourceMoved, HttpCodes.SeeOther, HttpCodes.TemporaryRedirect, HttpCodes.PermanentRedirect]; const HttpRedirectCodes = [
const HttpResponseRetryCodes = [HttpCodes.BadGateway, HttpCodes.ServiceUnavailable, HttpCodes.GatewayTimeout]; HttpCodes.MovedPermanently,
HttpCodes.ResourceMoved,
HttpCodes.SeeOther,
HttpCodes.TemporaryRedirect,
HttpCodes.PermanentRedirect
];
const HttpResponseRetryCodes = [
HttpCodes.BadGateway,
HttpCodes.ServiceUnavailable,
HttpCodes.GatewayTimeout
];
const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD'];
const ExponentialBackoffCeiling = 10; const ExponentialBackoffCeiling = 10;
const ExponentialBackoffTimeSlice = 5; const ExponentialBackoffTimeSlice = 5;
@ -2156,19 +2169,23 @@ class HttpClient {
*/ */
async request(verb, requestUrl, data, headers) { async request(verb, requestUrl, data, headers) {
if (this._disposed) { if (this._disposed) {
throw new Error("Client has already been disposed."); throw new Error('Client has already been disposed.');
} }
let parsedUrl = url.parse(requestUrl); let parsedUrl = url.parse(requestUrl);
let info = this._prepareRequest(verb, parsedUrl, headers); let info = this._prepareRequest(verb, parsedUrl, headers);
// Only perform retries on reads since writes may not be idempotent. // Only perform retries on reads since writes may not be idempotent.
let maxTries = (this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1) ? this._maxRetries + 1 : 1; let maxTries = this._allowRetries && RetryableHttpVerbs.indexOf(verb) != -1
? this._maxRetries + 1
: 1;
let numTries = 0; let numTries = 0;
let response; let response;
while (numTries < maxTries) { while (numTries < maxTries) {
response = await this.requestRaw(info, data); response = await this.requestRaw(info, data);
// Check if it's an authentication challenge // Check if it's an authentication challenge
if (response && response.message && response.message.statusCode === HttpCodes.Unauthorized) { if (response &&
response.message &&
response.message.statusCode === HttpCodes.Unauthorized) {
let authenticationHandler; let authenticationHandler;
for (let i = 0; i < this.handlers.length; i++) { for (let i = 0; i < this.handlers.length; i++) {
if (this.handlers[i].canHandleAuthentication(response)) { if (this.handlers[i].canHandleAuthentication(response)) {
@ -2186,21 +2203,32 @@ class HttpClient {
} }
} }
let redirectsRemaining = this._maxRedirects; let redirectsRemaining = this._maxRedirects;
while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 while (HttpRedirectCodes.indexOf(response.message.statusCode) != -1 &&
&& this._allowRedirects this._allowRedirects &&
&& redirectsRemaining > 0) { redirectsRemaining > 0) {
const redirectUrl = response.message.headers["location"]; const redirectUrl = response.message.headers['location'];
if (!redirectUrl) { if (!redirectUrl) {
// if there's no location to redirect to, we won't // if there's no location to redirect to, we won't
break; break;
} }
let parsedRedirectUrl = url.parse(redirectUrl); let parsedRedirectUrl = url.parse(redirectUrl);
if (parsedUrl.protocol == 'https:' && parsedUrl.protocol != parsedRedirectUrl.protocol && !this._allowRedirectDowngrade) { if (parsedUrl.protocol == 'https:' &&
throw new Error("Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true."); parsedUrl.protocol != parsedRedirectUrl.protocol &&
!this._allowRedirectDowngrade) {
throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.');
} }
// we need to finish reading the response before reassigning response // we need to finish reading the response before reassigning response
// which will leak the open socket. // which will leak the open socket.
await response.readBody(); await response.readBody();
// strip authorization header if redirected to a different hostname
if (parsedRedirectUrl.hostname !== parsedUrl.hostname) {
for (let header in headers) {
// header names are case insensitive
if (header.toLowerCase() === 'authorization') {
delete headers[header];
}
}
}
// let's make the request with the new redirectUrl // let's make the request with the new redirectUrl
info = this._prepareRequest(verb, parsedRedirectUrl, headers); info = this._prepareRequest(verb, parsedRedirectUrl, headers);
response = await this.requestRaw(info, data); response = await this.requestRaw(info, data);
@ -2251,8 +2279,8 @@ class HttpClient {
*/ */
requestRawWithCallback(info, data, onResult) { requestRawWithCallback(info, data, onResult) {
let socket; let socket;
if (typeof (data) === 'string') { if (typeof data === 'string') {
info.options.headers["Content-Length"] = Buffer.byteLength(data, 'utf8'); info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8');
} }
let callbackCalled = false; let callbackCalled = false;
let handleResult = (err, res) => { let handleResult = (err, res) => {
@ -2265,7 +2293,7 @@ class HttpClient {
let res = new HttpClientResponse(msg); let res = new HttpClientResponse(msg);
handleResult(null, res); handleResult(null, res);
}); });
req.on('socket', (sock) => { req.on('socket', sock => {
socket = sock; socket = sock;
}); });
// If we ever get disconnected, we want the socket to timeout eventually // If we ever get disconnected, we want the socket to timeout eventually
@ -2280,10 +2308,10 @@ class HttpClient {
// res should have headers // res should have headers
handleResult(err, null); handleResult(err, null);
}); });
if (data && typeof (data) === 'string') { if (data && typeof data === 'string') {
req.write(data, 'utf8'); req.write(data, 'utf8');
} }
if (data && typeof (data) !== 'string') { if (data && typeof data !== 'string') {
data.on('close', function () { data.on('close', function () {
req.end(); req.end();
}); });
@ -2310,31 +2338,34 @@ class HttpClient {
const defaultPort = usingSsl ? 443 : 80; const defaultPort = usingSsl ? 443 : 80;
info.options = {}; info.options = {};
info.options.host = info.parsedUrl.hostname; info.options.host = info.parsedUrl.hostname;
info.options.port = info.parsedUrl.port ? parseInt(info.parsedUrl.port) : defaultPort; info.options.port = info.parsedUrl.port
info.options.path = (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); ? parseInt(info.parsedUrl.port)
: defaultPort;
info.options.path =
(info.parsedUrl.pathname || '') + (info.parsedUrl.search || '');
info.options.method = method; info.options.method = method;
info.options.headers = this._mergeHeaders(headers); info.options.headers = this._mergeHeaders(headers);
if (this.userAgent != null) { if (this.userAgent != null) {
info.options.headers["user-agent"] = this.userAgent; info.options.headers['user-agent'] = this.userAgent;
} }
info.options.agent = this._getAgent(info.parsedUrl); info.options.agent = this._getAgent(info.parsedUrl);
// gives handlers an opportunity to participate // gives handlers an opportunity to participate
if (this.handlers) { if (this.handlers) {
this.handlers.forEach((handler) => { this.handlers.forEach(handler => {
handler.prepareRequest(info.options); handler.prepareRequest(info.options);
}); });
} }
return info; return info;
} }
_mergeHeaders(headers) { _mergeHeaders(headers) {
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {});
if (this.requestOptions && this.requestOptions.headers) { if (this.requestOptions && this.requestOptions.headers) {
return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers)); return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers));
} }
return lowercaseKeys(headers || {}); return lowercaseKeys(headers || {});
} }
_getExistingOrDefaultHeader(additionalHeaders, header, _default) { _getExistingOrDefaultHeader(additionalHeaders, header, _default) {
const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => (c[k.toLowerCase()] = obj[k], c), {}); const lowercaseKeys = obj => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {});
let clientHeader; let clientHeader;
if (this.requestOptions && this.requestOptions.headers) { if (this.requestOptions && this.requestOptions.headers) {
clientHeader = lowercaseKeys(this.requestOptions.headers)[header]; clientHeader = lowercaseKeys(this.requestOptions.headers)[header];
@ -2372,7 +2403,7 @@ class HttpClient {
proxyAuth: proxyUrl.auth, proxyAuth: proxyUrl.auth,
host: proxyUrl.hostname, host: proxyUrl.hostname,
port: proxyUrl.port port: proxyUrl.port
}, }
}; };
let tunnelAgent; let tunnelAgent;
const overHttps = proxyUrl.protocol === 'https:'; const overHttps = proxyUrl.protocol === 'https:';
@ -2399,7 +2430,9 @@ class HttpClient {
// we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process
// http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options
// we have to cast it to any and change it directly // we have to cast it to any and change it directly
agent.options = Object.assign(agent.options || {}, { rejectUnauthorized: false }); agent.options = Object.assign(agent.options || {}, {
rejectUnauthorized: false
});
} }
return agent; return agent;
} }
@ -2460,7 +2493,7 @@ class HttpClient {
msg = contents; msg = contents;
} }
else { else {
msg = "Failed request: (" + statusCode + ")"; msg = 'Failed request: (' + statusCode + ')';
} }
let err = new Error(msg); let err = new Error(msg);
// attach statusCode and body obj (if available) to the error object // attach statusCode and body obj (if available) to the error object
@ -3010,12 +3043,10 @@ function getProxyUrl(reqUrl) {
} }
let proxyVar; let proxyVar;
if (usingSsl) { if (usingSsl) {
proxyVar = process.env["https_proxy"] || proxyVar = process.env['https_proxy'] || process.env['HTTPS_PROXY'];
process.env["HTTPS_PROXY"];
} }
else { else {
proxyVar = process.env["http_proxy"] || proxyVar = process.env['http_proxy'] || process.env['HTTP_PROXY'];
process.env["HTTP_PROXY"];
} }
if (proxyVar) { if (proxyVar) {
proxyUrl = url.parse(proxyVar); proxyUrl = url.parse(proxyVar);
@ -3027,7 +3058,7 @@ function checkBypass(reqUrl) {
if (!reqUrl.hostname) { if (!reqUrl.hostname) {
return false; return false;
} }
let noProxy = process.env["no_proxy"] || process.env["NO_PROXY"] || ''; let noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || '';
if (!noProxy) { if (!noProxy) {
return false; return false;
} }
@ -3048,7 +3079,10 @@ function checkBypass(reqUrl) {
upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`);
} }
// Compare request host against noproxy // Compare request host against noproxy
for (let upperNoProxyItem of noProxy.split(',').map(x => x.trim().toUpperCase()).filter(x => x)) { for (let upperNoProxyItem of noProxy
.split(',')
.map(x => x.trim().toUpperCase())
.filter(x => x)) {
if (upperReqHosts.some(x => x === upperNoProxyItem)) { if (upperReqHosts.some(x => x === upperNoProxyItem)) {
return true; return true;
} }

18
src/cacheHttpClient.ts
View file

@ -179,7 +179,7 @@ function getContentRange(start: number, end: number): string {
async function uploadChunk( async function uploadChunk(
httpClient: HttpClient, httpClient: HttpClient,
resourceUrl: string, resourceUrl: string,
data: NodeJS.ReadableStream, openStream: () => NodeJS.ReadableStream,
start: number, start: number,
end: number end: number
): Promise<void> { ): Promise<void> {
@ -200,7 +200,7 @@ async function uploadChunk(
return await httpClient.sendStream( return await httpClient.sendStream(
"PATCH", "PATCH",
resourceUrl, resourceUrl,
data, openStream(),
additionalHeaders additionalHeaders
); );
}; };
@ -263,17 +263,17 @@ async function uploadFile(
const start = offset; const start = offset;
const end = offset + chunkSize - 1; const end = offset + chunkSize - 1;
offset += MAX_CHUNK_SIZE; offset += MAX_CHUNK_SIZE;
const chunk = fs.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
});
await uploadChunk( await uploadChunk(
httpClient, httpClient,
resourceUrl, resourceUrl,
chunk, () =>
fs.createReadStream(archivePath, {
fd,
start,
end,
autoClose: false
}),
start, start,
end end
); );