From 14039a483ebf4f28ec584b6dbc88c49b25759fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Caciquinho?= Date: Fri, 19 Nov 2021 10:12:17 -0300 Subject: [PATCH] feat: Adiciona logica de chunk by ao processar upload de assets no s3 (#286) * feat: Adiciona logica de chunk by ao processar upload de assets no s3 * fix: linter * feat: add build * fix: remove catch * feat: add build * fix: eslint --- .tool-versions | 2 +- dist/index.js | 223 ++++++++++++++++++++++++++++++++++++------------- src/files.js | 37 +++++++- 3 files changed, 198 insertions(+), 64 deletions(-) diff --git a/.tool-versions b/.tool-versions index d11b569..39e3293 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -nodejs 14.16.0 +nodejs 16.13.0 \ No newline at end of file diff --git a/dist/index.js b/dist/index.js index b78e37d..202b659 100644 --- a/dist/index.js +++ b/dist/index.js @@ -25816,6 +25816,31 @@ var enhanceError = __webpack_require__(21516); var isHttps = /https:?/; +/** + * + * @param {http.ClientRequestArgs} options + * @param {AxiosProxyConfig} proxy + * @param {string} location + */ +function setProxy(options, proxy, location) { + options.hostname = proxy.host; + options.host = proxy.host; + options.port = proxy.port; + options.path = location; + + // Basic proxy authorization + if (proxy.auth) { + var base64 = Buffer.from(proxy.auth.username + ':' + proxy.auth.password, 'utf8').toString('base64'); + options.headers['Proxy-Authorization'] = 'Basic ' + base64; + } + + // If a proxy is used, any redirects must also pass through the proxy + options.beforeRedirect = function beforeRedirect(redirection) { + redirection.headers.host = redirection.host; + setProxy(redirection, proxy, redirection.href); + }; +} + /*eslint consistent-return:0*/ module.exports = function httpAdapter(config) { return new Promise(function dispatchHttpRequest(resolvePromise, rejectPromise) { @@ -25926,11 +25951,11 @@ module.exports = function httpAdapter(config) { }); } - if (shouldProxy) { proxy = { host: parsedProxyUrl.hostname, - port: parsedProxyUrl.port + port: parsedProxyUrl.port, + protocol: parsedProxyUrl.protocol }; if (parsedProxyUrl.auth) { @@ -25945,17 +25970,8 @@ module.exports = function httpAdapter(config) { } if (proxy) { - options.hostname = proxy.host; - options.host = proxy.host; options.headers.host = parsed.hostname + (parsed.port ? ':' + parsed.port : ''); - options.port = proxy.port; - options.path = protocol + '//' + parsed.hostname + (parsed.port ? ':' + parsed.port : '') + options.path; - - // Basic proxy authorization - if (proxy.auth) { - var base64 = Buffer.from(proxy.auth.username + ':' + proxy.auth.password, 'utf8').toString('base64'); - options.headers['Proxy-Authorization'] = 'Basic ' + base64; - } + setProxy(options, proxy, protocol + '//' + parsed.hostname + (parsed.port ? ':' + parsed.port : '') + options.path); } var transport; @@ -26329,6 +26345,9 @@ axios.all = function all(promises) { }; axios.spread = __webpack_require__(74850); +// Expose isAxiosError +axios.isAxiosError = __webpack_require__(60650); + module.exports = axios; // Allow use of default import syntax in TypeScript @@ -27258,6 +27277,25 @@ module.exports = function isAbsoluteURL(url) { }; +/***/ }), + +/***/ 60650: +/***/ ((module) => { + +"use strict"; + + +/** + * Determines whether the payload is an error thrown by Axios + * + * @param {*} payload The value to test + * @returns {boolean} True if the payload is an error thrown by Axios, otherwise false + */ +module.exports = function isAxiosError(payload) { + return (typeof payload === 'object') && (payload.isAxiosError === true); +}; + + /***/ }), /***/ 33608: @@ -28760,14 +28798,19 @@ module.exports.parse = parse /***/ ((module, __unused_webpack_exports, __webpack_require__) => { var debug; -try { - /* eslint global-require: off */ - debug = __webpack_require__(38237)("follow-redirects"); -} -catch (error) { - debug = function () { /* */ }; -} -module.exports = debug; + +module.exports = function () { + if (!debug) { + try { + /* eslint global-require: off */ + debug = __webpack_require__(38237)("follow-redirects"); + } + catch (error) { + debug = function () { /* */ }; + } + } + debug.apply(null, arguments); +}; /***/ }), @@ -28784,8 +28827,9 @@ var assert = __webpack_require__(42357); var debug = __webpack_require__(31133); // Create handlers that pass events from native requests +var events = ["abort", "aborted", "connect", "error", "socket", "timeout"]; var eventHandlers = Object.create(null); -["abort", "aborted", "connect", "error", "socket", "timeout"].forEach(function (event) { +events.forEach(function (event) { eventHandlers[event] = function (arg1, arg2, arg3) { this._redirectable.emit(event, arg1, arg2, arg3); }; @@ -28838,6 +28882,11 @@ function RedirectableRequest(options, responseCallback) { } RedirectableRequest.prototype = Object.create(Writable.prototype); +RedirectableRequest.prototype.abort = function () { + abortRequest(this._currentRequest); + this.emit("abort"); +}; + // Writes buffered data to the current native request RedirectableRequest.prototype.write = function (data, encoding, callback) { // Writing is not allowed if end has been called @@ -28917,40 +28966,58 @@ RedirectableRequest.prototype.removeHeader = function (name) { // Global timeout for all underlying requests RedirectableRequest.prototype.setTimeout = function (msecs, callback) { + var self = this; if (callback) { - this.once("timeout", callback); + this.on("timeout", callback); + } + + function destroyOnTimeout(socket) { + socket.setTimeout(msecs); + socket.removeListener("timeout", socket.destroy); + socket.addListener("timeout", socket.destroy); + } + + // Sets up a timer to trigger a timeout event + function startTimer(socket) { + if (self._timeout) { + clearTimeout(self._timeout); + } + self._timeout = setTimeout(function () { + self.emit("timeout"); + clearTimer(); + }, msecs); + destroyOnTimeout(socket); + } + + // Prevent a timeout from triggering + function clearTimer() { + clearTimeout(this._timeout); + if (callback) { + self.removeListener("timeout", callback); + } + if (!this.socket) { + self._currentRequest.removeListener("socket", startTimer); + } } + // Start the timer when the socket is opened if (this.socket) { - startTimer(this, msecs); + startTimer(this.socket); } else { - var self = this; - this._currentRequest.once("socket", function () { - startTimer(self, msecs); - }); + this._currentRequest.once("socket", startTimer); } + this.on("socket", destroyOnTimeout); this.once("response", clearTimer); this.once("error", clearTimer); return this; }; -function startTimer(request, msecs) { - clearTimeout(request._timeout); - request._timeout = setTimeout(function () { - request.emit("timeout"); - }, msecs); -} - -function clearTimer() { - clearTimeout(this._timeout); -} - // Proxy all other public ClientRequest methods [ - "abort", "flushHeaders", "getHeader", + "flushHeaders", "getHeader", "setNoDelay", "setSocketKeepAlive", ].forEach(function (method) { RedirectableRequest.prototype[method] = function (a, b) { @@ -29020,11 +29087,8 @@ RedirectableRequest.prototype._performRequest = function () { // Set up event handlers request._redirectable = this; - for (var event in eventHandlers) { - /* istanbul ignore else */ - if (event) { - request.on(event, eventHandlers[event]); - } + for (var e = 0; e < events.length; e++) { + request.on(events[e], eventHandlers[events[e]]); } // End a redirected request @@ -29082,9 +29146,7 @@ RedirectableRequest.prototype._processResponse = function (response) { if (location && this._options.followRedirects !== false && statusCode >= 300 && statusCode < 400) { // Abort the current request - this._currentRequest.removeAllListeners(); - this._currentRequest.on("error", noop); - this._currentRequest.abort(); + abortRequest(this._currentRequest); // Discard the remainder of the response to avoid waiting for data response.destroy(); @@ -29177,7 +29239,7 @@ function wrap(protocols) { var wrappedProtocol = exports[scheme] = Object.create(nativeProtocol); // Executes a request, following redirects - wrappedProtocol.request = function (input, options, callback) { + function request(input, options, callback) { // Parse parameters if (typeof input === "string") { var urlStr = input; @@ -29212,14 +29274,20 @@ function wrap(protocols) { assert.equal(options.protocol, protocol, "protocol mismatch"); debug("options", options); return new RedirectableRequest(options, callback); - }; + } // Executes a GET request, following redirects - wrappedProtocol.get = function (input, options, callback) { - var request = wrappedProtocol.request(input, options, callback); - request.end(); - return request; - }; + function get(input, options, callback) { + var wrappedRequest = wrappedProtocol.request(input, options, callback); + wrappedRequest.end(); + return wrappedRequest; + } + + // Expose the properties on the wrapped protocol + Object.defineProperties(wrappedProtocol, { + request: { value: request, configurable: true, enumerable: true, writable: true }, + get: { value: get, configurable: true, enumerable: true, writable: true }, + }); }); return exports; } @@ -29270,6 +29338,14 @@ function createErrorType(code, defaultMessage) { return CustomError; } +function abortRequest(request) { + for (var e = 0; e < events.length; e++) { + request.removeListener(events[e], eventHandlers[events[e]]); + } + request.on("error", noop); + request.abort(); +} + // Exports module.exports = wrap({ http: http, https: https }); module.exports.wrap = wrap; @@ -36435,6 +36511,7 @@ const buildChapters = async (chapterPath) => { const chapterArrayOfObj = groupFiles(sanitizedArrayOfFiles); core.info(`Processing ${chapterArrayOfObj.length} Chapters`); + return Promise.all( chapterArrayOfObj.map((chapterObj) => buildChapterObj(chapterObj)), ); @@ -36474,13 +36551,41 @@ const processAssetContent = async (assetPath) => { return { [relativeAssetPath]: s3UrlLocation }; }; +const chunkArray = (myArray, chunkSize) => { + const results = []; + + while (myArray.length) { + results.push(myArray.splice(0, chunkSize)); + } + + return results; +}; + +const createFilesChunk = (arrayOfChapters) => chunkArray(arrayOfChapters, 100); +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + const buildAssets = async (assetsPath) => { const arrayOfAssets = await getAssetsFiles(assetsPath); - core.info(`Processing ${arrayOfAssets.length} Assets`); - return Promise.all( - arrayOfAssets.map((assetPath) => processAssetContent(assetPath)), - ); + + const groupOfAssets = createFilesChunk(arrayOfAssets); + + core.info(`Número de Grupos: #${groupOfAssets.length}`); + + const result = await groupOfAssets.reduce((chain, assets, index) => ( + chain.then(async (previousResults) => { + core.info(`Iniciando upload de grupo ${(index + 1)}...`); + + const responses = await Promise.all( + assets.map((assetPath) => processAssetContent(assetPath)), + ); + + await sleep(1000); + return [...responses, ...previousResults || []]; + }) + ), Promise.resolve()); + + return result; }; module.exports = { @@ -41432,7 +41537,7 @@ module.exports = JSON.parse("{\"rules\":{\"*/*\":{\"endpoint\":\"{service}.{regi /***/ ((module) => { "use strict"; -module.exports = JSON.parse("{\"_args\":[[\"axios@0.21.0\",\"/home/vitorluizc/projects/trybe/process-content\"]],\"_from\":\"axios@0.21.0\",\"_id\":\"axios@0.21.0\",\"_inBundle\":false,\"_integrity\":\"sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==\",\"_location\":\"/axios\",\"_phantomChildren\":{},\"_requested\":{\"type\":\"version\",\"registry\":true,\"raw\":\"axios@0.21.0\",\"name\":\"axios\",\"escapedName\":\"axios\",\"rawSpec\":\"0.21.0\",\"saveSpec\":null,\"fetchSpec\":\"0.21.0\"},\"_requiredBy\":[\"/\"],\"_resolved\":\"https://registry.npmjs.org/axios/-/axios-0.21.0.tgz\",\"_spec\":\"0.21.0\",\"_where\":\"/home/vitorluizc/projects/trybe/process-content\",\"author\":{\"name\":\"Matt Zabriskie\"},\"browser\":{\"./lib/adapters/http.js\":\"./lib/adapters/xhr.js\"},\"bugs\":{\"url\":\"https://github.com/axios/axios/issues\"},\"bundlesize\":[{\"path\":\"./dist/axios.min.js\",\"threshold\":\"5kB\"}],\"dependencies\":{\"follow-redirects\":\"^1.10.0\"},\"description\":\"Promise based HTTP client for the browser and node.js\",\"devDependencies\":{\"bundlesize\":\"^0.17.0\",\"coveralls\":\"^3.0.0\",\"es6-promise\":\"^4.2.4\",\"grunt\":\"^1.0.2\",\"grunt-banner\":\"^0.6.0\",\"grunt-cli\":\"^1.2.0\",\"grunt-contrib-clean\":\"^1.1.0\",\"grunt-contrib-watch\":\"^1.0.0\",\"grunt-eslint\":\"^20.1.0\",\"grunt-karma\":\"^2.0.0\",\"grunt-mocha-test\":\"^0.13.3\",\"grunt-ts\":\"^6.0.0-beta.19\",\"grunt-webpack\":\"^1.0.18\",\"istanbul-instrumenter-loader\":\"^1.0.0\",\"jasmine-core\":\"^2.4.1\",\"karma\":\"^1.3.0\",\"karma-chrome-launcher\":\"^2.2.0\",\"karma-coverage\":\"^1.1.1\",\"karma-firefox-launcher\":\"^1.1.0\",\"karma-jasmine\":\"^1.1.1\",\"karma-jasmine-ajax\":\"^0.1.13\",\"karma-opera-launcher\":\"^1.0.0\",\"karma-safari-launcher\":\"^1.0.0\",\"karma-sauce-launcher\":\"^1.2.0\",\"karma-sinon\":\"^1.0.5\",\"karma-sourcemap-loader\":\"^0.3.7\",\"karma-webpack\":\"^1.7.0\",\"load-grunt-tasks\":\"^3.5.2\",\"minimist\":\"^1.2.0\",\"mocha\":\"^5.2.0\",\"sinon\":\"^4.5.0\",\"typescript\":\"^2.8.1\",\"url-search-params\":\"^0.10.0\",\"webpack\":\"^1.13.1\",\"webpack-dev-server\":\"^1.14.1\"},\"homepage\":\"https://github.com/axios/axios\",\"jsdelivr\":\"dist/axios.min.js\",\"keywords\":[\"xhr\",\"http\",\"ajax\",\"promise\",\"node\"],\"license\":\"MIT\",\"main\":\"index.js\",\"name\":\"axios\",\"repository\":{\"type\":\"git\",\"url\":\"git+https://github.com/axios/axios.git\"},\"scripts\":{\"build\":\"NODE_ENV=production grunt build\",\"coveralls\":\"cat coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js\",\"examples\":\"node ./examples/server.js\",\"fix\":\"eslint --fix lib/**/*.js\",\"postversion\":\"git push && git push --tags\",\"preversion\":\"npm test\",\"start\":\"node ./sandbox/server.js\",\"test\":\"grunt test && bundlesize\",\"version\":\"npm run build && grunt version && git add -A dist && git add CHANGELOG.md bower.json package.json\"},\"typings\":\"./index.d.ts\",\"unpkg\":\"dist/axios.min.js\",\"version\":\"0.21.0\"}"); +module.exports = JSON.parse("{\"name\":\"axios\",\"version\":\"0.21.1\",\"description\":\"Promise based HTTP client for the browser and node.js\",\"main\":\"index.js\",\"scripts\":{\"test\":\"grunt test && bundlesize\",\"start\":\"node ./sandbox/server.js\",\"build\":\"NODE_ENV=production grunt build\",\"preversion\":\"npm test\",\"version\":\"npm run build && grunt version && git add -A dist && git add CHANGELOG.md bower.json package.json\",\"postversion\":\"git push && git push --tags\",\"examples\":\"node ./examples/server.js\",\"coveralls\":\"cat coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js\",\"fix\":\"eslint --fix lib/**/*.js\"},\"repository\":{\"type\":\"git\",\"url\":\"https://github.com/axios/axios.git\"},\"keywords\":[\"xhr\",\"http\",\"ajax\",\"promise\",\"node\"],\"author\":\"Matt Zabriskie\",\"license\":\"MIT\",\"bugs\":{\"url\":\"https://github.com/axios/axios/issues\"},\"homepage\":\"https://github.com/axios/axios\",\"devDependencies\":{\"bundlesize\":\"^0.17.0\",\"coveralls\":\"^3.0.0\",\"es6-promise\":\"^4.2.4\",\"grunt\":\"^1.0.2\",\"grunt-banner\":\"^0.6.0\",\"grunt-cli\":\"^1.2.0\",\"grunt-contrib-clean\":\"^1.1.0\",\"grunt-contrib-watch\":\"^1.0.0\",\"grunt-eslint\":\"^20.1.0\",\"grunt-karma\":\"^2.0.0\",\"grunt-mocha-test\":\"^0.13.3\",\"grunt-ts\":\"^6.0.0-beta.19\",\"grunt-webpack\":\"^1.0.18\",\"istanbul-instrumenter-loader\":\"^1.0.0\",\"jasmine-core\":\"^2.4.1\",\"karma\":\"^1.3.0\",\"karma-chrome-launcher\":\"^2.2.0\",\"karma-coverage\":\"^1.1.1\",\"karma-firefox-launcher\":\"^1.1.0\",\"karma-jasmine\":\"^1.1.1\",\"karma-jasmine-ajax\":\"^0.1.13\",\"karma-opera-launcher\":\"^1.0.0\",\"karma-safari-launcher\":\"^1.0.0\",\"karma-sauce-launcher\":\"^1.2.0\",\"karma-sinon\":\"^1.0.5\",\"karma-sourcemap-loader\":\"^0.3.7\",\"karma-webpack\":\"^1.7.0\",\"load-grunt-tasks\":\"^3.5.2\",\"minimist\":\"^1.2.0\",\"mocha\":\"^5.2.0\",\"sinon\":\"^4.5.0\",\"typescript\":\"^2.8.1\",\"url-search-params\":\"^0.10.0\",\"webpack\":\"^1.13.1\",\"webpack-dev-server\":\"^1.14.1\"},\"browser\":{\"./lib/adapters/http.js\":\"./lib/adapters/xhr.js\"},\"jsdelivr\":\"dist/axios.min.js\",\"unpkg\":\"dist/axios.min.js\",\"typings\":\"./index.d.ts\",\"dependencies\":{\"follow-redirects\":\"^1.10.0\"},\"bundlesize\":[{\"path\":\"./dist/axios.min.js\",\"threshold\":\"5kB\"}]}"); /***/ }), diff --git a/src/files.js b/src/files.js index f774666..a499e4f 100644 --- a/src/files.js +++ b/src/files.js @@ -62,6 +62,7 @@ const buildChapters = async (chapterPath) => { const chapterArrayOfObj = groupFiles(sanitizedArrayOfFiles); core.info(`Processing ${chapterArrayOfObj.length} Chapters`); + return Promise.all( chapterArrayOfObj.map((chapterObj) => buildChapterObj(chapterObj)), ); @@ -101,13 +102,41 @@ const processAssetContent = async (assetPath) => { return { [relativeAssetPath]: s3UrlLocation }; }; +const chunkArray = (myArray, chunkSize) => { + const results = []; + + while (myArray.length) { + results.push(myArray.splice(0, chunkSize)); + } + + return results; +}; + +const createFilesChunk = (arrayOfChapters) => chunkArray(arrayOfChapters, 100); +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + const buildAssets = async (assetsPath) => { const arrayOfAssets = await getAssetsFiles(assetsPath); - core.info(`Processing ${arrayOfAssets.length} Assets`); - return Promise.all( - arrayOfAssets.map((assetPath) => processAssetContent(assetPath)), - ); + + const groupOfAssets = createFilesChunk(arrayOfAssets); + + core.info(`Número de Grupos: #${groupOfAssets.length}`); + + const result = await groupOfAssets.reduce((chain, assets, index) => ( + chain.then(async (previousResults) => { + core.info(`Iniciando upload de grupo ${(index + 1)}...`); + + const responses = await Promise.all( + assets.map((assetPath) => processAssetContent(assetPath)), + ); + + await sleep(1000); + return [...responses, ...previousResults || []]; + }) + ), Promise.resolve()); + + return result; }; module.exports = {