diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..02f061a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.localstack/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..48328d7 --- /dev/null +++ b/README.md @@ -0,0 +1,83 @@ +# Elasticsearch with AWS Lambda and SNS using LocalStack + +![Architecture](img/architecture.png) + +This is an exercise and also a repository for testing using LocalStack, Elasticsearch, Kibana, AWS Lambda and SNS. + +Initially it is a copy of the article [*Ingest data into Elasticsearch with AWS Lambda and SNS using LocalStack*](https://dev.to/jainec/ingest-data-into-elasticsearch-with-aws-lambda-and-sns-using-localstack-3af4) by Jaine Conceição, but I want to make new implementations in the future for studies and experiments. + +Visit the article: https://dev.to/jainec/ingest-data-into-elasticsearch-with-aws-lambda-and-sns-using-localstack-3af4 + +--- + +## Start containers + +Run the `docker-compose.yml` file: + +```sh +$ docker-compose up -d +``` + +## Configure local SNS and Lambda + +Creating the SNS topic + +```sh +$ aws --endpoint-url=http://0.0.0.0:4566 sns create-topic --name events-topic +``` + +Creating the Lambda function + +```sh +$ aws --endpoint-url=http://0.0.0.0:4566 lambda create-function --function-name events-lambda --zip-file fileb://function_lambda.zip --handler index.handler --runtime nodejs12.x --role _ +``` + +Subscribing lambda to SNS topic + +```sh +$ aws --endpoint-url=http://0.0.0.0:4566 sns subscribe --topic-arn arn:aws:sns:us-east-1:000000000000:events-topic --protocol lambda --notification-endpoint arn:aws:lambda:us-east-1:000000000000:function:events-lambda +``` + +## Testing + +Sending a message to the SNS topic via command line: + +```sh +$ aws sns publish --endpoint-url=http://0.0.0.0:4566 --topic-arn arn:aws:sns:us-east-1:000000000000:events-topic --message '{"name":"test", "user_id":1, "properties": {"nickname": "test-user1", "job": "Software engineer"}}' +``` + +Now access Kibana and make sure that you have configured the index pattern: + +![Create index](img/kibana_create_index.png) + +Then go to Discover section and your message should appear like that: + +![View message](img/kibana_view_message.png) + +--- + +## Troubleshoot + +### Install AWS CLI on Alpine Linux 3.15 + +```sh +$ apk add --no-cache python3 py3-pip +$ pip3 install --upgrade pip +$ pip3 install --no-cache-dir awscli + +# Check installation +$ aws --version + +# Output similar +aws-cli/1.25.7 Python/3.97 Linux/5.10.102.1-microsoft-standard-WSL2 botocore/1.27.7 +``` + +### Error when starting elasticsearch container + +> (...) max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144] + +Execute: +```sh +#https://github.com/docker-library/elasticsearch/issues/111 +$ sysctl -w vm.max_map_count=262144 +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4f7fa25 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +version: "3.5" + +services: + localstack: + container_name: localstack + image: localstack/localstack:latest + networks: + - default + ports: + - "4566:4566" + environment: + - EDGE_PORT=4566 + - SERVICES=sns,lambda + volumes: + - ./.localstack:/tmp/localstack + + elasticsearch: + container_name: elasticsearch + image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0 + environment: + discovery.type: "single-node" + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + ports: + - 9200:9200 + networks: + - default + + kibana: + container_name: kibana + image: docker.elastic.co/kibana/kibana:7.7.0 + ports: + - "5601:5601" + networks: + - default + depends_on: + - elasticsearch + +networks: + default: + driver: bridge diff --git a/function_lambda.zip b/function_lambda.zip new file mode 100644 index 0000000..ba13a5d Binary files /dev/null and b/function_lambda.zip differ diff --git a/function_lambda/elastic/elastic.js b/function_lambda/elastic/elastic.js new file mode 100644 index 0000000..1a76f31 --- /dev/null +++ b/function_lambda/elastic/elastic.js @@ -0,0 +1,36 @@ +console.log('Loading elasticsearch file'); + +const elasticsearch = require('elasticsearch'); + +const elasticClient = elasticsearch.Client({ + host: 'http://elasticsearch:9200' +}); + +const indexName = 'event_tracking'; + +function initIndex() { + console.log('Init elasticsearch'); + try { + return elasticClient.indices.create({ + index: indexName + }); + } catch (error) { + console.log('Error creating elasticsearch index') + } +} +exports.initIndex = initIndex; + +function addDocument(message) { + console.log('Adding document to elastic'); + return elasticClient.index({ + index: indexName, + type: "document", + body: { + event_name: message.name ?? null, + user_id: message.user_id ?? null, + properties: message.properties ?? null, + created_at: message.created_at ?? null, + } + }); +} +exports.addDocument = addDocument; diff --git a/function_lambda/index.js b/function_lambda/index.js new file mode 100644 index 0000000..fa5a988 --- /dev/null +++ b/function_lambda/index.js @@ -0,0 +1,15 @@ +console.log('Loading function'); + +const elasticsearch = require('./elastic/elastic'); + +elasticsearch.initIndex(); + +exports.handler = function(event) { + const message = event.Records[0].Sns.Message; + console.log('Message received from SNS:', message); + + elasticsearch.addDocument(JSON.parse(message)); + console.log('Document added to elastic'); + + return 'Success'; +}; diff --git a/function_lambda/node_modules/.package-lock.json b/function_lambda/node_modules/.package-lock.json new file mode 100644 index 0000000..12d0c4d --- /dev/null +++ b/function_lambda/node_modules/.package-lock.json @@ -0,0 +1,120 @@ +{ + "name": "lambda-sns-elasticsearch", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "node_modules/agentkeepalive": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", + "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elasticsearch": { + "version": "16.7.3", + "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.3.tgz", + "integrity": "sha512-e9kUNhwnIlu47fGAr4W6yZJbkpsgQJB0TqNK8rCANe1J4P65B1sGnbCFTgcKY3/dRgCWnuP1AJ4obvzW604xEQ==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dependencies": { + "agentkeepalive": "^3.4.1", + "chalk": "^1.0.0", + "lodash": "^4.17.10" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "engines": { + "node": ">=0.8.0" + } + } + } +} diff --git a/function_lambda/node_modules/agentkeepalive/History.md b/function_lambda/node_modules/agentkeepalive/History.md new file mode 100644 index 0000000..d5d14d8 --- /dev/null +++ b/function_lambda/node_modules/agentkeepalive/History.md @@ -0,0 +1,170 @@ + +3.5.2 / 2018-10-19 +================== + +**fixes** + * [[`5751fc1`](http://github.com/node-modules/agentkeepalive/commit/5751fc1180ed6544602c681ffbd08ca66a0cb12c)] - fix: sockLen being miscalculated when removing sockets (#60) (Ehden Sinai <>) + +3.5.1 / 2018-07-31 +================== + +**fixes** + * [[`495f1ab`](http://github.com/node-modules/agentkeepalive/commit/495f1ab625d43945d72f68096b97db723d4f0657)] - fix: add the lost npm files (#66) (Henry Zhuang <>) + +3.5.0 / 2018-07-31 +================== + +**features** + * [[`16f5aea`](http://github.com/node-modules/agentkeepalive/commit/16f5aeadfda57f1c602652f1472a63cc83cd05bf)] - feat: add typing define. (#65) (Henry Zhuang <>) + +**others** + * [[`28fa062`](http://github.com/node-modules/agentkeepalive/commit/28fa06246fb5103f88ebeeb8563757a9078b8157)] - docs: add "per host" to description of maxFreeSockets (tony-gutierrez <>) + * [[`7df2577`](http://github.com/node-modules/agentkeepalive/commit/7df25774f00a1031ca4daad2878a17e0539072a2)] - test: run test on node 10 (#63) (fengmk2 <>) + +3.4.1 / 2018-03-08 +================== + +**fixes** + * [[`4d3a3b1`](http://github.com/node-modules/agentkeepalive/commit/4d3a3b1f7b16595febbbd39eeed72b2663549014)] - fix: Handle ipv6 addresses in host-header correctly with TLS (#53) (Mattias Holmlund <>) + +**others** + * [[`55a7a5c`](http://github.com/node-modules/agentkeepalive/commit/55a7a5cd33e97f9a8370083dcb041c5552f10ac9)] - test: stop timer after test end (fengmk2 <>) + +3.4.0 / 2018-02-27 +================== + +**features** + * [[`bc7cadb`](http://github.com/node-modules/agentkeepalive/commit/bc7cadb30ecd2071e2b341ac53ae1a2b8155c43d)] - feat: use socket custom freeSocketKeepAliveTimeout first (#59) (fengmk2 <>) + +**others** + * [[`138eda8`](http://github.com/node-modules/agentkeepalive/commit/138eda81e10b632aaa87bea0cb66d8667124c4e8)] - doc: fix `keepAliveMsecs` params description (#55) (Hongcai Deng <>) + +3.3.0 / 2017-06-20 +================== + + * feat: add statusChanged getter (#51) + * chore: format License + +3.2.0 / 2017-06-10 +================== + + * feat: add expiring active sockets + * test: add node 8 (#49) + +3.1.0 / 2017-02-20 +================== + + * feat: timeout support humanize ms (#48) + +3.0.0 / 2016-12-20 +================== + + * fix: emit agent socket close event + * test: add remove excess calls to removeSocket + * test: use egg-ci + * test: refactor test with eslint rules + * feat: merge _http_agent.js from 7.2.1 + +2.2.0 / 2016-06-26 +================== + + * feat: Add browser shim (noop) for isomorphic use. (#39) + * chore: add security check badge + +2.1.1 / 2016-04-06 +================== + + * https: fix ssl socket leak when keepalive is used + * chore: remove circle ci image + +2.1.0 / 2016-04-02 +================== + + * fix: opened sockets number overflow maxSockets + +2.0.5 / 2016-03-16 +================== + + * fix: pick _evictSession to httpsAgent + +2.0.4 / 2016-03-13 +================== + + * test: add Circle ci + * test: add appveyor ci build + * refactor: make sure only one error listener + * chore: use codecov + * fix: handle idle socket error + * test: run on more node versions + +2.0.3 / 2015-08-03 +================== + + * fix: add default error handler to avoid Unhandled error event throw + +2.0.2 / 2015-04-25 +================== + + * fix: remove socket from freeSockets on 'timeout' (@pmalouin) + +2.0.1 / 2015-04-19 +================== + + * fix: add timeoutSocketCount to getCurrentStatus() + * feat(getCurrentStatus): add getCurrentStatus + +2.0.0 / 2015-04-01 +================== + + * fix: socket.destroyed always be undefined on 0.10.x + * Make it compatible with node v0.10.x (@lattmann) + +1.2.1 / 2015-03-23 +================== + + * patch from iojs: don't overwrite servername option + * patch commits from joyent/node + * add max sockets test case + * add nagle algorithm delayed link + +1.2.0 / 2014-09-02 +================== + + * allow set keepAliveTimeout = 0 + * support timeout on working socket. fixed #6 + +1.1.0 / 2014-08-28 +================== + + * add some socket counter for deep monitor + +1.0.0 / 2014-08-13 +================== + + * update _http_agent, only support 0.11+, only support node 0.11.0+ + +0.2.2 / 2013-11-19 +================== + + * support node 0.8 and node 0.10 + +0.2.1 / 2013-11-08 +================== + + * fix socket does not timeout bug, it will hang on life, must use 0.2.x on node 0.11 + +0.2.0 / 2013-11-06 +================== + + * use keepalive agent on node 0.11+ impl + +0.1.5 / 2013-06-24 +================== + + * support coveralls + * add node 0.10 test + * add 0.8.22 original https.js + * add original http.js module to diff + * update jscover + * mv pem to fixtures + * add https agent usage diff --git a/function_lambda/node_modules/agentkeepalive/README.md b/function_lambda/node_modules/agentkeepalive/README.md new file mode 100644 index 0000000..8231458 --- /dev/null +++ b/function_lambda/node_modules/agentkeepalive/README.md @@ -0,0 +1,248 @@ +# agentkeepalive + +[![NPM version][npm-image]][npm-url] +[![build status][travis-image]][travis-url] +[![Appveyor status][appveyor-image]][appveyor-url] +[![Test coverage][codecov-image]][codecov-url] +[![David deps][david-image]][david-url] +[![Known Vulnerabilities][snyk-image]][snyk-url] +[![npm download][download-image]][download-url] + +[npm-image]: https://img.shields.io/npm/v/agentkeepalive.svg?style=flat +[npm-url]: https://npmjs.org/package/agentkeepalive +[travis-image]: https://img.shields.io/travis/node-modules/agentkeepalive.svg?style=flat +[travis-url]: https://travis-ci.org/node-modules/agentkeepalive +[appveyor-image]: https://ci.appveyor.com/api/projects/status/k7ct4s47di6m5uy2?svg=true +[appveyor-url]: https://ci.appveyor.com/project/fengmk2/agentkeepalive +[codecov-image]: https://codecov.io/gh/node-modules/agentkeepalive/branch/master/graph/badge.svg +[codecov-url]: https://codecov.io/gh/node-modules/agentkeepalive +[david-image]: https://img.shields.io/david/node-modules/agentkeepalive.svg?style=flat +[david-url]: https://david-dm.org/node-modules/agentkeepalive +[snyk-image]: https://snyk.io/test/npm/agentkeepalive/badge.svg?style=flat-square +[snyk-url]: https://snyk.io/test/npm/agentkeepalive +[download-image]: https://img.shields.io/npm/dm/agentkeepalive.svg?style=flat-square +[download-url]: https://npmjs.org/package/agentkeepalive + +The Node.js's missing `keep alive` `http.Agent`. Support `http` and `https`. + +## What's different from original `http.Agent`? + +- `keepAlive=true` by default +- Disable Nagle's algorithm: `socket.setNoDelay(true)` +- Add free socket timeout: avoid long time inactivity socket leak in the free-sockets queue. +- Add active socket timeout: avoid long time inactivity socket leak in the active-sockets queue. + +## Install + +```bash +$ npm install agentkeepalive --save +``` + +## new Agent([options]) + +* `options` {Object} Set of configurable options to set on the agent. + Can have the following fields: + * `keepAlive` {Boolean} Keep sockets around in a pool to be used by + other requests in the future. Default = `true`. + * `keepAliveMsecs` {Number} When using the keepAlive option, specifies the initial delay + for TCP Keep-Alive packets. Ignored when the keepAlive option is false or undefined. Defaults to 1000. + Default = `1000`. Only relevant if `keepAlive` is set to `true`. + * `freeSocketKeepAliveTimeout`: {Number} Sets the free socket to timeout + after `freeSocketKeepAliveTimeout` milliseconds of inactivity on the free socket. + Default is `15000`. + Only relevant if `keepAlive` is set to `true`. + * `timeout`: {Number} Sets the working socket to timeout + after `timeout` milliseconds of inactivity on the working socket. + Default is `freeSocketKeepAliveTimeout * 2`. + * `maxSockets` {Number} Maximum number of sockets to allow per + host. Default = `Infinity`. + * `maxFreeSockets` {Number} Maximum number of sockets (per host) to leave open + in a free state. Only relevant if `keepAlive` is set to `true`. + Default = `256`. + * `socketActiveTTL` {Number} Sets the socket active time to live, even if it's in use. + If not setted the behaviour continues the same (the socket will be released only when free) + Default = `null`. + +## Usage + +```js +const http = require('http'); +const Agent = require('agentkeepalive'); + +const keepaliveAgent = new Agent({ + maxSockets: 100, + maxFreeSockets: 10, + timeout: 60000, + freeSocketKeepAliveTimeout: 30000, // free socket keepalive for 30 seconds +}); + +const options = { + host: 'cnodejs.org', + port: 80, + path: '/', + method: 'GET', + agent: keepaliveAgent, +}; + +const req = http.request(options, res => { + console.log('STATUS: ' + res.statusCode); + console.log('HEADERS: ' + JSON.stringify(res.headers)); + res.setEncoding('utf8'); + res.on('data', function (chunk) { + console.log('BODY: ' + chunk); + }); +}); +req.on('error', e => { + console.log('problem with request: ' + e.message); +}); +req.end(); + +setTimeout(() => { + if (keepaliveAgent.statusChanged) { + console.log('[%s] agent status changed: %j', Date(), keepaliveAgent.getCurrentStatus()); + } +}, 2000); + +``` + +### `getter agent.statusChanged` + +counters have change or not after last checkpoint. + +### `agent.getCurrentStatus()` + +`agent.getCurrentStatus()` will return a object to show the status of this agent: + +```js +{ + createSocketCount: 10, + closeSocketCount: 5, + timeoutSocketCount: 0, + requestCount: 5, + freeSockets: { 'localhost:57479:': 3 }, + sockets: { 'localhost:57479:': 5 }, + requests: {} +} +``` + +### Support `https` + +```js +const https = require('https'); +const HttpsAgent = require('agentkeepalive').HttpsAgent; + +const keepaliveAgent = new HttpsAgent(); +// https://www.google.com/search?q=nodejs&sugexp=chrome,mod=12&sourceid=chrome&ie=UTF-8 +const options = { + host: 'www.google.com', + port: 443, + path: '/search?q=nodejs&sugexp=chrome,mod=12&sourceid=chrome&ie=UTF-8', + method: 'GET', + agent: keepaliveAgent, +}; + +const req = https.request(options, res => { + console.log('STATUS: ' + res.statusCode); + console.log('HEADERS: ' + JSON.stringify(res.headers)); + res.setEncoding('utf8'); + res.on('data', chunk => { + console.log('BODY: ' + chunk); + }); +}); + +req.on('error', e => { + console.log('problem with request: ' + e.message); +}); +req.end(); + +setTimeout(() => { + console.log('agent status: %j', keepaliveAgent.getCurrentStatus()); +}, 2000); +``` + +## [Benchmark](https://github.com/node-modules/agentkeepalive/tree/master/benchmark) + +run the benchmark: + +```bash +cd benchmark +sh start.sh +``` + +Intel(R) Core(TM)2 Duo CPU P8600 @ 2.40GHz + +node@v0.8.9 + +50 maxSockets, 60 concurrent, 1000 requests per concurrent, 5ms delay + +Keep alive agent (30 seconds): + +```js +Transactions: 60000 hits +Availability: 100.00 % +Elapsed time: 29.70 secs +Data transferred: 14.88 MB +Response time: 0.03 secs +Transaction rate: 2020.20 trans/sec +Throughput: 0.50 MB/sec +Concurrency: 59.84 +Successful transactions: 60000 +Failed transactions: 0 +Longest transaction: 0.15 +Shortest transaction: 0.01 +``` + +Normal agent: + +```js +Transactions: 60000 hits +Availability: 100.00 % +Elapsed time: 46.53 secs +Data transferred: 14.88 MB +Response time: 0.05 secs +Transaction rate: 1289.49 trans/sec +Throughput: 0.32 MB/sec +Concurrency: 59.81 +Successful transactions: 60000 +Failed transactions: 0 +Longest transaction: 0.45 +Shortest transaction: 0.00 +``` + +Socket created: + +``` +[proxy.js:120000] keepalive, 50 created, 60000 requestFinished, 1200 req/socket, 0 requests, 0 sockets, 0 unusedSockets, 50 timeout +{" <10ms":662," <15ms":17825," <20ms":20552," <30ms":17646," <40ms":2315," <50ms":567," <100ms":377," <150ms":56," <200ms":0," >=200ms+":0} +---------------------------------------------------------------- +[proxy.js:120000] normal , 53866 created, 84260 requestFinished, 1.56 req/socket, 0 requests, 0 sockets +{" <10ms":75," <15ms":1112," <20ms":10947," <30ms":32130," <40ms":8228," <50ms":3002," <100ms":4274," <150ms":181," <200ms":18," >=200ms+":33} +``` + +## License + +``` +(The MIT License) + +Copyright(c) node-modules and other contributors. +Copyright(c) 2012 - 2015 fengmk2 + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +``` diff --git a/function_lambda/node_modules/agentkeepalive/browser.js b/function_lambda/node_modules/agentkeepalive/browser.js new file mode 100644 index 0000000..29c9398 --- /dev/null +++ b/function_lambda/node_modules/agentkeepalive/browser.js @@ -0,0 +1,5 @@ +module.exports = noop; +module.exports.HttpsAgent = noop; + +// Noop function for browser since native api's don't use agents. +function noop () {} diff --git a/function_lambda/node_modules/agentkeepalive/index.d.ts b/function_lambda/node_modules/agentkeepalive/index.d.ts new file mode 100644 index 0000000..c11636f --- /dev/null +++ b/function_lambda/node_modules/agentkeepalive/index.d.ts @@ -0,0 +1,43 @@ +declare module "agentkeepalive" { + import * as http from 'http'; + import * as https from 'https'; + + interface AgentStatus { + createSocketCount: number, + createSocketErrorCount: number, + closeSocketCount: number, + errorSocketCount: number, + timeoutSocketCount: number, + requestCount: number, + freeSockets: object, + sockets: object, + requests: object, + } + + interface HttpOptions extends http.AgentOptions { + freeSocketKeepAliveTimeout?: number; + timeout?: number; + socketActiveTTL?: number; + } + + interface HttpsOptions extends https.AgentOptions { + freeSocketKeepAliveTimeout?: number; + timeout?: number; + socketActiveTTL?: number; + } + + class internal extends http.Agent { + constructor(opts?: HttpOptions); + readonly statusChanged: boolean; + createSocket(req: http.IncomingMessage, options: http.RequestOptions, cb: Function): void; + getCurrentStatus(): AgentStatus; + } + + namespace internal { + export class HttpsAgent extends internal { + constructor(opts?: HttpsOptions); + } + } + + export = internal; +} diff --git a/function_lambda/node_modules/agentkeepalive/index.js b/function_lambda/node_modules/agentkeepalive/index.js new file mode 100644 index 0000000..6138131 --- /dev/null +++ b/function_lambda/node_modules/agentkeepalive/index.js @@ -0,0 +1,4 @@ +'use strict'; + +module.exports = require('./lib/agent'); +module.exports.HttpsAgent = require('./lib/https_agent'); diff --git a/function_lambda/node_modules/agentkeepalive/lib/_http_agent.js b/function_lambda/node_modules/agentkeepalive/lib/_http_agent.js new file mode 100644 index 0000000..c324b7f --- /dev/null +++ b/function_lambda/node_modules/agentkeepalive/lib/_http_agent.js @@ -0,0 +1,416 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +// patch from https://github.com/nodejs/node/blob/v7.2.1/lib/_http_agent.js + +'use strict'; + +const net = require('net'); +const util = require('util'); +const EventEmitter = require('events'); +const debug = util.debuglog('http'); + +// New Agent code. + +// The largest departure from the previous implementation is that +// an Agent instance holds connections for a variable number of host:ports. +// Surprisingly, this is still API compatible as far as third parties are +// concerned. The only code that really notices the difference is the +// request object. + +// Another departure is that all code related to HTTP parsing is in +// ClientRequest.onSocket(). The Agent is now *strictly* +// concerned with managing a connection pool. + +function Agent(options) { + if (!(this instanceof Agent)) + return new Agent(options); + + EventEmitter.call(this); + + var self = this; + + self.defaultPort = 80; + self.protocol = 'http:'; + + self.options = util._extend({}, options); + + // don't confuse net and make it think that we're connecting to a pipe + self.options.path = null; + self.requests = {}; + self.sockets = {}; + self.freeSockets = {}; + self.keepAliveMsecs = self.options.keepAliveMsecs || 1000; + self.keepAlive = self.options.keepAlive || false; + self.maxSockets = self.options.maxSockets || Agent.defaultMaxSockets; + self.maxFreeSockets = self.options.maxFreeSockets || 256; + + // [patch start] + // free keep-alive socket timeout. By default free socket do not have a timeout. + self.freeSocketKeepAliveTimeout = self.options.freeSocketKeepAliveTimeout || 0; + // working socket timeout. By default working socket do not have a timeout. + self.timeout = self.options.timeout || 0; + // the socket active time to live, even if it's in use + this.socketActiveTTL = this.options.socketActiveTTL || null; + // [patch end] + + self.on('free', function(socket, options) { + var name = self.getName(options); + debug('agent.on(free)', name); + + if (socket.writable && + self.requests[name] && self.requests[name].length) { + // [patch start] + debug('continue handle next request'); + // [patch end] + self.requests[name].shift().onSocket(socket); + if (self.requests[name].length === 0) { + // don't leak + delete self.requests[name]; + } + } else { + // If there are no pending requests, then put it in + // the freeSockets pool, but only if we're allowed to do so. + var req = socket._httpMessage; + if (req && + req.shouldKeepAlive && + socket.writable && + self.keepAlive) { + var freeSockets = self.freeSockets[name]; + var freeLen = freeSockets ? freeSockets.length : 0; + var count = freeLen; + if (self.sockets[name]) + count += self.sockets[name].length; + + if (count > self.maxSockets || freeLen >= self.maxFreeSockets) { + socket.destroy(); + } else { + freeSockets = freeSockets || []; + self.freeSockets[name] = freeSockets; + socket.setKeepAlive(true, self.keepAliveMsecs); + socket.unref(); + socket._httpMessage = null; + self.removeSocket(socket, options); + freeSockets.push(socket); + + // [patch start] + // Add a default error handler to avoid Unhandled 'error' event throw on idle socket + // https://github.com/node-modules/agentkeepalive/issues/25 + // https://github.com/nodejs/node/pull/4482 (fixed in >= 4.4.0 and >= 5.4.0) + if (socket.listeners('error').length === 0) { + socket.once('error', freeSocketErrorListener); + } + // set free keepalive timer + // try to use socket custom freeSocketKeepAliveTimeout first + const freeSocketKeepAliveTimeout = socket.freeSocketKeepAliveTimeout || self.freeSocketKeepAliveTimeout; + socket.setTimeout(freeSocketKeepAliveTimeout); + debug(`push to free socket queue and wait for ${freeSocketKeepAliveTimeout}ms`); + // [patch end] + } + } else { + socket.destroy(); + } + } + }); +} + +util.inherits(Agent, EventEmitter); +exports.Agent = Agent; + +// [patch start] +function freeSocketErrorListener(err) { + var socket = this; + debug('SOCKET ERROR on FREE socket:', err.message, err.stack); + socket.destroy(); + socket.emit('agentRemove'); +} +// [patch end] + +Agent.defaultMaxSockets = Infinity; + +Agent.prototype.createConnection = net.createConnection; + +// Get the key for a given set of request options +Agent.prototype.getName = function getName(options) { + var name = options.host || 'localhost'; + + name += ':'; + if (options.port) + name += options.port; + + name += ':'; + if (options.localAddress) + name += options.localAddress; + + // Pacify parallel/test-http-agent-getname by only appending + // the ':' when options.family is set. + if (options.family === 4 || options.family === 6) + name += ':' + options.family; + + return name; +}; + +// [patch start] +function handleSocketCreation(req) { + return function(err, newSocket) { + if (err) { + process.nextTick(function() { + req.emit('error', err); + }); + return; + } + req.onSocket(newSocket); + } +} +// [patch end] + +Agent.prototype.addRequest = function addRequest(req, options, port/*legacy*/, + localAddress/*legacy*/) { + // Legacy API: addRequest(req, host, port, localAddress) + if (typeof options === 'string') { + options = { + host: options, + port, + localAddress + }; + } + + options = util._extend({}, options); + options = util._extend(options, this.options); + + if (!options.servername) + options.servername = calculateServerName(options, req); + + var name = this.getName(options); + if (!this.sockets[name]) { + this.sockets[name] = []; + } + + var freeLen = this.freeSockets[name] ? this.freeSockets[name].length : 0; + var sockLen = freeLen + this.sockets[name].length; + + if (freeLen) { + // we have a free socket, so use that. + var socket = this.freeSockets[name].shift(); + debug('have free socket'); + + // [patch start] + // remove free socket error event handler + socket.removeListener('error', freeSocketErrorListener); + // restart the default timer + socket.setTimeout(this.timeout); + + if (this.socketActiveTTL && Date.now() - socket.createdTime > this.socketActiveTTL) { + debug(`socket ${socket.createdTime} expired`); + socket.destroy(); + return this.createSocket(req, options, handleSocketCreation(req)); + } + // [patch end] + + // don't leak + if (!this.freeSockets[name].length) + delete this.freeSockets[name]; + + socket.ref(); + req.onSocket(socket); + this.sockets[name].push(socket); + } else if (sockLen < this.maxSockets) { + debug('call onSocket', sockLen, freeLen); + // If we are under maxSockets create a new one. + // [patch start] + this.createSocket(req, options, handleSocketCreation(req)); + // [patch end] + } else { + debug('wait for socket'); + // We are over limit so we'll add it to the queue. + if (!this.requests[name]) { + this.requests[name] = []; + } + this.requests[name].push(req); + } +}; + +Agent.prototype.createSocket = function createSocket(req, options, cb) { + var self = this; + options = util._extend({}, options); + options = util._extend(options, self.options); + + if (!options.servername) + options.servername = calculateServerName(options, req); + + var name = self.getName(options); + options._agentKey = name; + + debug('createConnection', name, options); + options.encoding = null; + var called = false; + const newSocket = self.createConnection(options, oncreate); + // [patch start] + if (newSocket) { + oncreate(null, Object.assign(newSocket, { createdTime: Date.now() })); + } + // [patch end] + function oncreate(err, s) { + if (called) + return; + called = true; + if (err) + return cb(err); + if (!self.sockets[name]) { + self.sockets[name] = []; + } + self.sockets[name].push(s); + debug('sockets', name, self.sockets[name].length); + + function onFree() { + self.emit('free', s, options); + } + s.on('free', onFree); + + function onClose(err) { + debug('CLIENT socket onClose'); + // This is the only place where sockets get removed from the Agent. + // If you want to remove a socket from the pool, just close it. + // All socket errors end in a close event anyway. + self.removeSocket(s, options); + + // [patch start] + self.emit('close'); + // [patch end] + } + s.on('close', onClose); + + // [patch start] + // start socket timeout handler + function onTimeout() { + debug('CLIENT socket onTimeout'); + s.destroy(); + // Remove it from freeSockets immediately to prevent new requests from being sent through this socket. + self.removeSocket(s, options); + self.emit('timeout'); + } + s.on('timeout', onTimeout); + // set the default timer + s.setTimeout(self.timeout); + // [patch end] + + function onRemove() { + // We need this function for cases like HTTP 'upgrade' + // (defined by WebSockets) where we need to remove a socket from the + // pool because it'll be locked up indefinitely + debug('CLIENT socket onRemove'); + self.removeSocket(s, options); + s.removeListener('close', onClose); + s.removeListener('free', onFree); + s.removeListener('agentRemove', onRemove); + + // [patch start] + // remove socket timeout handler + s.setTimeout(0, onTimeout); + // [patch end] + } + s.on('agentRemove', onRemove); + cb(null, s); + } +}; + +function calculateServerName(options, req) { + let servername = options.host; + const hostHeader = req.getHeader('host'); + if (hostHeader) { + // abc => abc + // abc:123 => abc + // [::1] => ::1 + // [::1]:123 => ::1 + if (hostHeader.startsWith('[')) { + const index = hostHeader.indexOf(']'); + if (index === -1) { + // Leading '[', but no ']'. Need to do something... + servername = hostHeader; + } else { + servername = hostHeader.substr(1, index - 1); + } + } else { + servername = hostHeader.split(':', 1)[0]; + } + } + return servername; +} + +Agent.prototype.removeSocket = function removeSocket(s, options) { + var name = this.getName(options); + debug('removeSocket', name, 'writable:', s.writable); + var sets = [this.sockets]; + + // If the socket was destroyed, remove it from the free buffers too. + if (!s.writable) + sets.push(this.freeSockets); + + for (var sk = 0; sk < sets.length; sk++) { + var sockets = sets[sk]; + + if (sockets[name]) { + var index = sockets[name].indexOf(s); + if (index !== -1) { + sockets[name].splice(index, 1); + // Don't leak + if (sockets[name].length === 0) + delete sockets[name]; + } + } + } + + // [patch start] + var freeLen = this.freeSockets[name] ? this.freeSockets[name].length : 0; + var sockLen = freeLen + (this.sockets[name] ? this.sockets[name].length : 0); + // [patch end] + + if (this.requests[name] && this.requests[name].length && sockLen < this.maxSockets) { + debug('removeSocket, have a request, make a socket'); + var req = this.requests[name][0]; + // If we have pending requests and a socket gets closed make a new one + this.createSocket(req, options, function(err, newSocket) { + if (err) { + process.nextTick(function() { + req.emit('error', err); + }); + return; + } + newSocket.emit('free'); + }); + } +}; + +Agent.prototype.destroy = function destroy() { + var sets = [this.freeSockets, this.sockets]; + for (var s = 0; s < sets.length; s++) { + var set = sets[s]; + var keys = Object.keys(set); + for (var v = 0; v < keys.length; v++) { + var setName = set[keys[v]]; + for (var n = 0; n < setName.length; n++) { + setName[n].destroy(); + } + } + } +}; + +exports.globalAgent = new Agent(); diff --git a/function_lambda/node_modules/agentkeepalive/lib/agent.js b/function_lambda/node_modules/agentkeepalive/lib/agent.js new file mode 100644 index 0000000..a51ad59 --- /dev/null +++ b/function_lambda/node_modules/agentkeepalive/lib/agent.js @@ -0,0 +1,133 @@ +/** + * refer: + * * @atimb "Real keep-alive HTTP agent": https://gist.github.com/2963672 + * * https://github.com/joyent/node/blob/master/lib/http.js + * * https://github.com/joyent/node/blob/master/lib/https.js + * * https://github.com/joyent/node/blob/master/lib/_http_agent.js + */ + +'use strict'; + +const OriginalAgent = require('./_http_agent').Agent; +const ms = require('humanize-ms'); + +class Agent extends OriginalAgent { + constructor(options) { + options = options || {}; + options.keepAlive = options.keepAlive !== false; + // default is keep-alive and 15s free socket timeout + if (options.freeSocketKeepAliveTimeout === undefined) { + options.freeSocketKeepAliveTimeout = 15000; + } + // Legacy API: keepAliveTimeout should be rename to `freeSocketKeepAliveTimeout` + if (options.keepAliveTimeout) { + options.freeSocketKeepAliveTimeout = options.keepAliveTimeout; + } + options.freeSocketKeepAliveTimeout = ms(options.freeSocketKeepAliveTimeout); + + // Sets the socket to timeout after timeout milliseconds of inactivity on the socket. + // By default is double free socket keepalive timeout. + if (options.timeout === undefined) { + options.timeout = options.freeSocketKeepAliveTimeout * 2; + // make sure socket default inactivity timeout >= 30s + if (options.timeout < 30000) { + options.timeout = 30000; + } + } + options.timeout = ms(options.timeout); + + super(options); + + this.createSocketCount = 0; + this.createSocketCountLastCheck = 0; + + this.createSocketErrorCount = 0; + this.createSocketErrorCountLastCheck = 0; + + this.closeSocketCount = 0; + this.closeSocketCountLastCheck = 0; + + // socket error event count + this.errorSocketCount = 0; + this.errorSocketCountLastCheck = 0; + + this.requestCount = 0; + this.requestCountLastCheck = 0; + + this.timeoutSocketCount = 0; + this.timeoutSocketCountLastCheck = 0; + + this.on('free', s => { + this.requestCount++; + // last enter free queue timestamp + s.lastFreeTime = Date.now(); + }); + this.on('timeout', () => { + this.timeoutSocketCount++; + }); + this.on('close', () => { + this.closeSocketCount++; + }); + this.on('error', () => { + this.errorSocketCount++; + }); + } + + createSocket(req, options, cb) { + super.createSocket(req, options, (err, socket) => { + if (err) { + this.createSocketErrorCount++; + return cb(err); + } + if (this.keepAlive) { + // Disable Nagle's algorithm: http://blog.caustik.com/2012/04/08/scaling-node-js-to-100k-concurrent-connections/ + // https://fengmk2.com/benchmark/nagle-algorithm-delayed-ack-mock.html + socket.setNoDelay(true); + } + this.createSocketCount++; + cb(null, socket); + }); + } + + get statusChanged() { + const changed = this.createSocketCount !== this.createSocketCountLastCheck || + this.createSocketErrorCount !== this.createSocketErrorCountLastCheck || + this.closeSocketCount !== this.closeSocketCountLastCheck || + this.errorSocketCount !== this.errorSocketCountLastCheck || + this.timeoutSocketCount !== this.timeoutSocketCountLastCheck || + this.requestCount !== this.requestCountLastCheck; + if (changed) { + this.createSocketCountLastCheck = this.createSocketCount; + this.createSocketErrorCountLastCheck = this.createSocketErrorCount; + this.closeSocketCountLastCheck = this.closeSocketCount; + this.errorSocketCountLastCheck = this.errorSocketCount; + this.timeoutSocketCountLastCheck = this.timeoutSocketCount; + this.requestCountLastCheck = this.requestCount; + } + return changed; + } + + getCurrentStatus() { + return { + createSocketCount: this.createSocketCount, + createSocketErrorCount: this.createSocketErrorCount, + closeSocketCount: this.closeSocketCount, + errorSocketCount: this.errorSocketCount, + timeoutSocketCount: this.timeoutSocketCount, + requestCount: this.requestCount, + freeSockets: inspect(this.freeSockets), + sockets: inspect(this.sockets), + requests: inspect(this.requests), + }; + } +} + +module.exports = Agent; + +function inspect(obj) { + const res = {}; + for (const key in obj) { + res[key] = obj[key].length; + } + return res; +} diff --git a/function_lambda/node_modules/agentkeepalive/lib/https_agent.js b/function_lambda/node_modules/agentkeepalive/lib/https_agent.js new file mode 100644 index 0000000..e6d58a3 --- /dev/null +++ b/function_lambda/node_modules/agentkeepalive/lib/https_agent.js @@ -0,0 +1,42 @@ +/** + * Https Agent base on custom http agent + */ + +'use strict'; + +const https = require('https'); +const HttpAgent = require('./agent'); +const OriginalHttpsAgent = https.Agent; + +class HttpsAgent extends HttpAgent { + constructor(options) { + super(options); + + this.defaultPort = 443; + this.protocol = 'https:'; + this.maxCachedSessions = this.options.maxCachedSessions; + if (this.maxCachedSessions === undefined) { + this.maxCachedSessions = 100; + } + + this._sessionCache = { + map: {}, + list: [], + }; + } +} + +[ + 'createConnection', + 'getName', + '_getSession', + '_cacheSession', + // https://github.com/nodejs/node/pull/4982 + '_evictSession', +].forEach(function(method) { + if (typeof OriginalHttpsAgent.prototype[method] === 'function') { + HttpsAgent.prototype[method] = OriginalHttpsAgent.prototype[method]; + } +}); + +module.exports = HttpsAgent; diff --git a/function_lambda/node_modules/agentkeepalive/package.json b/function_lambda/node_modules/agentkeepalive/package.json new file mode 100644 index 0000000..a28305b --- /dev/null +++ b/function_lambda/node_modules/agentkeepalive/package.json @@ -0,0 +1,53 @@ +{ + "name": "agentkeepalive", + "version": "3.5.2", + "description": "Missing keepalive http.Agent", + "main": "index.js", + "browser": "browser.js", + "files": [ + "index.js", + "index.d.ts", + "browser.js", + "lib" + ], + "scripts": { + "test": "egg-bin test", + "cov": "egg-bin cov", + "ci": "npm run lint && npm run cov", + "lint": "eslint lib test index.js", + "autod": "autod" + }, + "repository": { + "type": "git", + "url": "git://github.com/node-modules/agentkeepalive.git" + }, + "bugs": { + "url": "https://github.com/node-modules/agentkeepalive/issues" + }, + "keywords": [ + "http", + "https", + "agent", + "keepalive", + "agentkeepalive" + ], + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "devDependencies": { + "autod": "^3.0.1", + "egg-bin": "^1.11.1", + "egg-ci": "^1.8.0", + "eslint": "^4.19.1", + "eslint-config-egg": "^6.0.0", + "pedding": "^1.1.0" + }, + "engines": { + "node": ">= 4.0.0" + }, + "ci": { + "version": "4, 6, 8, 10" + }, + "author": "fengmk2 (https://fengmk2.com)", + "license": "MIT" +} diff --git a/function_lambda/node_modules/ansi-regex/index.js b/function_lambda/node_modules/ansi-regex/index.js new file mode 100644 index 0000000..b9574ed --- /dev/null +++ b/function_lambda/node_modules/ansi-regex/index.js @@ -0,0 +1,4 @@ +'use strict'; +module.exports = function () { + return /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-PRZcf-nqry=><]/g; +}; diff --git a/function_lambda/node_modules/ansi-regex/license b/function_lambda/node_modules/ansi-regex/license new file mode 100644 index 0000000..654d0bf --- /dev/null +++ b/function_lambda/node_modules/ansi-regex/license @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/function_lambda/node_modules/ansi-regex/package.json b/function_lambda/node_modules/ansi-regex/package.json new file mode 100644 index 0000000..eb44fb5 --- /dev/null +++ b/function_lambda/node_modules/ansi-regex/package.json @@ -0,0 +1,64 @@ +{ + "name": "ansi-regex", + "version": "2.1.1", + "description": "Regular expression for matching ANSI escape codes", + "license": "MIT", + "repository": "chalk/ansi-regex", + "author": { + "name": "Sindre Sorhus", + "email": "sindresorhus@gmail.com", + "url": "sindresorhus.com" + }, + "maintainers": [ + "Sindre Sorhus (sindresorhus.com)", + "Joshua Appelman (jbnicolai.com)", + "JD Ballard (github.com/qix-)" + ], + "engines": { + "node": ">=0.10.0" + }, + "scripts": { + "test": "xo && ava --verbose", + "view-supported": "node fixtures/view-codes.js" + }, + "files": [ + "index.js" + ], + "keywords": [ + "ansi", + "styles", + "color", + "colour", + "colors", + "terminal", + "console", + "cli", + "string", + "tty", + "escape", + "formatting", + "rgb", + "256", + "shell", + "xterm", + "command-line", + "text", + "regex", + "regexp", + "re", + "match", + "test", + "find", + "pattern" + ], + "devDependencies": { + "ava": "0.17.0", + "xo": "0.16.0" + }, + "xo": { + "rules": { + "guard-for-in": 0, + "no-loop-func": 0 + } + } +} diff --git a/function_lambda/node_modules/ansi-regex/readme.md b/function_lambda/node_modules/ansi-regex/readme.md new file mode 100644 index 0000000..6a928ed --- /dev/null +++ b/function_lambda/node_modules/ansi-regex/readme.md @@ -0,0 +1,39 @@ +# ansi-regex [![Build Status](https://travis-ci.org/chalk/ansi-regex.svg?branch=master)](https://travis-ci.org/chalk/ansi-regex) + +> Regular expression for matching [ANSI escape codes](http://en.wikipedia.org/wiki/ANSI_escape_code) + + +## Install + +``` +$ npm install --save ansi-regex +``` + + +## Usage + +```js +const ansiRegex = require('ansi-regex'); + +ansiRegex().test('\u001b[4mcake\u001b[0m'); +//=> true + +ansiRegex().test('cake'); +//=> false + +'\u001b[4mcake\u001b[0m'.match(ansiRegex()); +//=> ['\u001b[4m', '\u001b[0m'] +``` + +## FAQ + +### Why do you test for codes not in the ECMA 48 standard? + +Some of the codes we run as a test are codes that we acquired finding various lists of non-standard or manufacturer specific codes. If I recall correctly, we test for both standard and non-standard codes, as most of them follow the same or similar format and can be safely matched in strings without the risk of removing actual string content. There are a few non-standard control codes that do not follow the traditional format (i.e. they end in numbers) thus forcing us to exclude them from the test because we cannot reliably match them. + +On the historical side, those ECMA standards were established in the early 90's whereas the VT100, for example, was designed in the mid/late 70's. At that point in time, control codes were still pretty ungoverned and engineers used them for a multitude of things, namely to activate hardware ports that may have been proprietary. Somewhere else you see a similar 'anarchy' of codes is in the x86 architecture for processors; there are a ton of "interrupts" that can mean different things on certain brands of processors, most of which have been phased out. + + +## License + +MIT © [Sindre Sorhus](http://sindresorhus.com) diff --git a/function_lambda/node_modules/ansi-styles/index.js b/function_lambda/node_modules/ansi-styles/index.js new file mode 100644 index 0000000..7894527 --- /dev/null +++ b/function_lambda/node_modules/ansi-styles/index.js @@ -0,0 +1,65 @@ +'use strict'; + +function assembleStyles () { + var styles = { + modifiers: { + reset: [0, 0], + bold: [1, 22], // 21 isn't widely supported and 22 does the same thing + dim: [2, 22], + italic: [3, 23], + underline: [4, 24], + inverse: [7, 27], + hidden: [8, 28], + strikethrough: [9, 29] + }, + colors: { + black: [30, 39], + red: [31, 39], + green: [32, 39], + yellow: [33, 39], + blue: [34, 39], + magenta: [35, 39], + cyan: [36, 39], + white: [37, 39], + gray: [90, 39] + }, + bgColors: { + bgBlack: [40, 49], + bgRed: [41, 49], + bgGreen: [42, 49], + bgYellow: [43, 49], + bgBlue: [44, 49], + bgMagenta: [45, 49], + bgCyan: [46, 49], + bgWhite: [47, 49] + } + }; + + // fix humans + styles.colors.grey = styles.colors.gray; + + Object.keys(styles).forEach(function (groupName) { + var group = styles[groupName]; + + Object.keys(group).forEach(function (styleName) { + var style = group[styleName]; + + styles[styleName] = group[styleName] = { + open: '\u001b[' + style[0] + 'm', + close: '\u001b[' + style[1] + 'm' + }; + }); + + Object.defineProperty(styles, groupName, { + value: group, + enumerable: false + }); + }); + + return styles; +} + +Object.defineProperty(module, 'exports', { + enumerable: true, + get: assembleStyles +}); diff --git a/function_lambda/node_modules/ansi-styles/license b/function_lambda/node_modules/ansi-styles/license new file mode 100644 index 0000000..654d0bf --- /dev/null +++ b/function_lambda/node_modules/ansi-styles/license @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/function_lambda/node_modules/ansi-styles/package.json b/function_lambda/node_modules/ansi-styles/package.json new file mode 100644 index 0000000..78c535f --- /dev/null +++ b/function_lambda/node_modules/ansi-styles/package.json @@ -0,0 +1,50 @@ +{ + "name": "ansi-styles", + "version": "2.2.1", + "description": "ANSI escape codes for styling strings in the terminal", + "license": "MIT", + "repository": "chalk/ansi-styles", + "author": { + "name": "Sindre Sorhus", + "email": "sindresorhus@gmail.com", + "url": "sindresorhus.com" + }, + "maintainers": [ + "Sindre Sorhus (sindresorhus.com)", + "Joshua Appelman (jbnicolai.com)" + ], + "engines": { + "node": ">=0.10.0" + }, + "scripts": { + "test": "mocha" + }, + "files": [ + "index.js" + ], + "keywords": [ + "ansi", + "styles", + "color", + "colour", + "colors", + "terminal", + "console", + "cli", + "string", + "tty", + "escape", + "formatting", + "rgb", + "256", + "shell", + "xterm", + "log", + "logging", + "command-line", + "text" + ], + "devDependencies": { + "mocha": "*" + } +} diff --git a/function_lambda/node_modules/ansi-styles/readme.md b/function_lambda/node_modules/ansi-styles/readme.md new file mode 100644 index 0000000..3f933f6 --- /dev/null +++ b/function_lambda/node_modules/ansi-styles/readme.md @@ -0,0 +1,86 @@ +# ansi-styles [![Build Status](https://travis-ci.org/chalk/ansi-styles.svg?branch=master)](https://travis-ci.org/chalk/ansi-styles) + +> [ANSI escape codes](http://en.wikipedia.org/wiki/ANSI_escape_code#Colors_and_Styles) for styling strings in the terminal + +You probably want the higher-level [chalk](https://github.com/chalk/chalk) module for styling your strings. + +![](screenshot.png) + + +## Install + +``` +$ npm install --save ansi-styles +``` + + +## Usage + +```js +var ansi = require('ansi-styles'); + +console.log(ansi.green.open + 'Hello world!' + ansi.green.close); +``` + + +## API + +Each style has an `open` and `close` property. + + +## Styles + +### Modifiers + +- `reset` +- `bold` +- `dim` +- `italic` *(not widely supported)* +- `underline` +- `inverse` +- `hidden` +- `strikethrough` *(not widely supported)* + +### Colors + +- `black` +- `red` +- `green` +- `yellow` +- `blue` +- `magenta` +- `cyan` +- `white` +- `gray` + +### Background colors + +- `bgBlack` +- `bgRed` +- `bgGreen` +- `bgYellow` +- `bgBlue` +- `bgMagenta` +- `bgCyan` +- `bgWhite` + + +## Advanced usage + +By default you get a map of styles, but the styles are also available as groups. They are non-enumerable so they don't show up unless you access them explicitly. This makes it easier to expose only a subset in a higher-level module. + +- `ansi.modifiers` +- `ansi.colors` +- `ansi.bgColors` + + +###### Example + +```js +console.log(ansi.colors.green.open); +``` + + +## License + +MIT © [Sindre Sorhus](http://sindresorhus.com) diff --git a/function_lambda/node_modules/chalk/index.js b/function_lambda/node_modules/chalk/index.js new file mode 100644 index 0000000..2d85a91 --- /dev/null +++ b/function_lambda/node_modules/chalk/index.js @@ -0,0 +1,116 @@ +'use strict'; +var escapeStringRegexp = require('escape-string-regexp'); +var ansiStyles = require('ansi-styles'); +var stripAnsi = require('strip-ansi'); +var hasAnsi = require('has-ansi'); +var supportsColor = require('supports-color'); +var defineProps = Object.defineProperties; +var isSimpleWindowsTerm = process.platform === 'win32' && !/^xterm/i.test(process.env.TERM); + +function Chalk(options) { + // detect mode if not set manually + this.enabled = !options || options.enabled === undefined ? supportsColor : options.enabled; +} + +// use bright blue on Windows as the normal blue color is illegible +if (isSimpleWindowsTerm) { + ansiStyles.blue.open = '\u001b[94m'; +} + +var styles = (function () { + var ret = {}; + + Object.keys(ansiStyles).forEach(function (key) { + ansiStyles[key].closeRe = new RegExp(escapeStringRegexp(ansiStyles[key].close), 'g'); + + ret[key] = { + get: function () { + return build.call(this, this._styles.concat(key)); + } + }; + }); + + return ret; +})(); + +var proto = defineProps(function chalk() {}, styles); + +function build(_styles) { + var builder = function () { + return applyStyle.apply(builder, arguments); + }; + + builder._styles = _styles; + builder.enabled = this.enabled; + // __proto__ is used because we must return a function, but there is + // no way to create a function with a different prototype. + /* eslint-disable no-proto */ + builder.__proto__ = proto; + + return builder; +} + +function applyStyle() { + // support varags, but simply cast to string in case there's only one arg + var args = arguments; + var argsLen = args.length; + var str = argsLen !== 0 && String(arguments[0]); + + if (argsLen > 1) { + // don't slice `arguments`, it prevents v8 optimizations + for (var a = 1; a < argsLen; a++) { + str += ' ' + args[a]; + } + } + + if (!this.enabled || !str) { + return str; + } + + var nestedStyles = this._styles; + var i = nestedStyles.length; + + // Turns out that on Windows dimmed gray text becomes invisible in cmd.exe, + // see https://github.com/chalk/chalk/issues/58 + // If we're on Windows and we're dealing with a gray color, temporarily make 'dim' a noop. + var originalDim = ansiStyles.dim.open; + if (isSimpleWindowsTerm && (nestedStyles.indexOf('gray') !== -1 || nestedStyles.indexOf('grey') !== -1)) { + ansiStyles.dim.open = ''; + } + + while (i--) { + var code = ansiStyles[nestedStyles[i]]; + + // Replace any instances already present with a re-opening code + // otherwise only the part of the string until said closing code + // will be colored, and the rest will simply be 'plain'. + str = code.open + str.replace(code.closeRe, code.open) + code.close; + } + + // Reset the original 'dim' if we changed it to work around the Windows dimmed gray issue. + ansiStyles.dim.open = originalDim; + + return str; +} + +function init() { + var ret = {}; + + Object.keys(styles).forEach(function (name) { + ret[name] = { + get: function () { + return build.call(this, [name]); + } + }; + }); + + return ret; +} + +defineProps(Chalk.prototype, init()); + +module.exports = new Chalk(); +module.exports.styles = ansiStyles; +module.exports.hasColor = hasAnsi; +module.exports.stripColor = stripAnsi; +module.exports.supportsColor = supportsColor; diff --git a/function_lambda/node_modules/chalk/license b/function_lambda/node_modules/chalk/license new file mode 100644 index 0000000..654d0bf --- /dev/null +++ b/function_lambda/node_modules/chalk/license @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) Sindre Sorhus (sindresorhus.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/function_lambda/node_modules/chalk/package.json b/function_lambda/node_modules/chalk/package.json new file mode 100644 index 0000000..2b5881e --- /dev/null +++ b/function_lambda/node_modules/chalk/package.json @@ -0,0 +1,70 @@ +{ + "name": "chalk", + "version": "1.1.3", + "description": "Terminal string styling done right. Much color.", + "license": "MIT", + "repository": "chalk/chalk", + "maintainers": [ + "Sindre Sorhus (sindresorhus.com)", + "Joshua Appelman (jbnicolai.com)", + "JD Ballard (github.com/qix-)" + ], + "engines": { + "node": ">=0.10.0" + }, + "scripts": { + "test": "xo && mocha", + "bench": "matcha benchmark.js", + "coverage": "nyc npm test && nyc report", + "coveralls": "nyc npm test && nyc report --reporter=text-lcov | coveralls" + }, + "files": [ + "index.js" + ], + "keywords": [ + "color", + "colour", + "colors", + "terminal", + "console", + "cli", + "string", + "str", + "ansi", + "style", + "styles", + "tty", + "formatting", + "rgb", + "256", + "shell", + "xterm", + "log", + "logging", + "command-line", + "text" + ], + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "devDependencies": { + "coveralls": "^2.11.2", + "matcha": "^0.6.0", + "mocha": "*", + "nyc": "^3.0.0", + "require-uncached": "^1.0.2", + "resolve-from": "^1.0.0", + "semver": "^4.3.3", + "xo": "*" + }, + "xo": { + "envs": [ + "node", + "mocha" + ] + } +} diff --git a/function_lambda/node_modules/chalk/readme.md b/function_lambda/node_modules/chalk/readme.md new file mode 100644 index 0000000..5cf111e --- /dev/null +++ b/function_lambda/node_modules/chalk/readme.md @@ -0,0 +1,213 @@ +

+
+
+ chalk +
+
+
+

+ +> Terminal string styling done right + +[![Build Status](https://travis-ci.org/chalk/chalk.svg?branch=master)](https://travis-ci.org/chalk/chalk) +[![Coverage Status](https://coveralls.io/repos/chalk/chalk/badge.svg?branch=master)](https://coveralls.io/r/chalk/chalk?branch=master) +[![](http://img.shields.io/badge/unicorn-approved-ff69b4.svg)](https://www.youtube.com/watch?v=9auOCbH5Ns4) + + +[colors.js](https://github.com/Marak/colors.js) used to be the most popular string styling module, but it has serious deficiencies like extending `String.prototype` which causes all kinds of [problems](https://github.com/yeoman/yo/issues/68). Although there are other ones, they either do too much or not enough. + +**Chalk is a clean and focused alternative.** + +![](https://github.com/chalk/ansi-styles/raw/master/screenshot.png) + + +## Why + +- Highly performant +- Doesn't extend `String.prototype` +- Expressive API +- Ability to nest styles +- Clean and focused +- Auto-detects color support +- Actively maintained +- [Used by ~4500 modules](https://www.npmjs.com/browse/depended/chalk) as of July 15, 2015 + + +## Install + +``` +$ npm install --save chalk +``` + + +## Usage + +Chalk comes with an easy to use composable API where you just chain and nest the styles you want. + +```js +var chalk = require('chalk'); + +// style a string +chalk.blue('Hello world!'); + +// combine styled and normal strings +chalk.blue('Hello') + 'World' + chalk.red('!'); + +// compose multiple styles using the chainable API +chalk.blue.bgRed.bold('Hello world!'); + +// pass in multiple arguments +chalk.blue('Hello', 'World!', 'Foo', 'bar', 'biz', 'baz'); + +// nest styles +chalk.red('Hello', chalk.underline.bgBlue('world') + '!'); + +// nest styles of the same type even (color, underline, background) +chalk.green( + 'I am a green line ' + + chalk.blue.underline.bold('with a blue substring') + + ' that becomes green again!' +); +``` + +Easily define your own themes. + +```js +var chalk = require('chalk'); +var error = chalk.bold.red; +console.log(error('Error!')); +``` + +Take advantage of console.log [string substitution](http://nodejs.org/docs/latest/api/console.html#console_console_log_data). + +```js +var name = 'Sindre'; +console.log(chalk.green('Hello %s'), name); +//=> Hello Sindre +``` + + +## API + +### chalk.`