diff --git a/firehose/.gitignore b/firehose/.gitignore new file mode 100644 index 0000000..2b48c8b --- /dev/null +++ b/firehose/.gitignore @@ -0,0 +1,6 @@ +# package directories +node_modules +jspm_packages + +# Serverless directories +.serverless \ No newline at end of file diff --git a/firehose/README.md b/firehose/README.md new file mode 100644 index 0000000..0c99bf1 --- /dev/null +++ b/firehose/README.md @@ -0,0 +1,35 @@ +# Serverless Kinesis streams + +Serverless service to showcase Kinesis stream support. + +based on:[serverless-kinesis-streams](https://github.com/pmuens/serverless-kinesis-streams), but auto create Kinesis streams + +## Installation + +1.install + +``` +npm install -u https://github.com/phodal/serverless-guide/tree/master/kinesis-streams -n kinesis-streams +``` + +2.install + +``` +npm install +``` + +3.deploy + +``` +serverless deploy +``` + +## How to use + +1. Run `serverless invoke --function dataReceiver --path event.json` to send data to the Kinesis stream +2. Run `serverless logs --function logger` to see the which data was send to the Kinesis `date-receiver` stream + +## AWS services used + +- Lambda +- Kinesis diff --git a/firehose/event.json b/firehose/event.json new file mode 100644 index 0000000..1982764 --- /dev/null +++ b/firehose/event.json @@ -0,0 +1,3 @@ +{ + "data": "Some example data" +} diff --git a/firehose/handler.js b/firehose/handler.js new file mode 100644 index 0000000..231f3df --- /dev/null +++ b/firehose/handler.js @@ -0,0 +1,32 @@ +'use strict'; + +const AWS = require('aws-sdk'); +const uuid = require('uuid'); + +module.exports.dataReceiver = (event, context, callback) => { + const data = event.data; + + const firehose = new AWS.Firehose(); + + const partitionKey = uuid.v1(); + + const params = { + Data: data, + PartitionKey: partitionKey, + StreamName: 'kinesis-streams-stream' + }; + + return firehose.putRecord(params, (error, data) => { + if (error) { + callback(error); + } + callback(null, { message: 'Data successfully written to Kinesis stream "data-receiver"' }); + }); +}; + +module.exports.logger = (event, context, callback) => { + // print out the event information on the console (so that we can see it in the CloudWatch logs) + console.log(`The following data was written to the Kinesis stream "data-receiver":\n${JSON.stringify(event.Records[0].kinesis, null, 2)}`); + + callback(null, { event }); +}; diff --git a/firehose/logs.md b/firehose/logs.md new file mode 100644 index 0000000..ba8cdb8 --- /dev/null +++ b/firehose/logs.md @@ -0,0 +1,97 @@ +Serverless 应用开发指南:基于 Kinesis Streams 的数据流分析(上) +=== + +Serverless 适合用于事件驱动型应用,以及定时任务。 + +在之前的那篇《[Serverless 应用开发指南:CRON 定时执行 Lambda 任务](https://www.phodal.com/blog/serverless-development-guide-cron-scheduled-job/)》中,我们介绍了如何调度的示例。 + +在今天,让我们来看看一个事件驱动的例子。 + +最初我想的是通过 Lambda + DynamoDB 来自定义数据格式,后来发现使用 Kinesis Streams 是一种更简单的方案。 + +Amazon Kinesis Streams +--- + + +今天,我们要学习的组件是 Amazon Kinesis Streams。引自官网的介绍: + +> 借助 Amazon Kinesis Streams,您可以构建用于处理或分析流数据的自定义应用程序,以满足特定需求。Kinesis Streams 每小时可从数十万种来源 (如网站点击流、财务交易、社交媒体源、IT 日志和定位追踪事件) 中持续捕获和存储数 TB 数据。借助 Kinesis Client Library (KCL),您可以构建 Amazon Kinesis 应用程序,并能使用流数据为实时控制面板提供强力支持、生成警报、实施动态定价和广告等等。您还可以将数据从 Kinesis Streams 发送到其他 AWS 服务中,如 Amazon Simple Storage Service (Amazon S3)、Amazon Redshift、Amazon EMR 和 AWS Lambda。 + +简单的来说,用于收集日志事件数据的功能,还可以用于实时数据分析。 + +Serverless + Kinesis Streams +--- + +最初我试用了 GitHub 上的[serverless-kinesis-streams](https://github.com/pmuens/serverless-kinesis-streams),然后发现它并不会自动创建 Kinesis Streams 服务,于是便自己创建了一个: + +``` +serverless install -u https://github.com/phodal/serverless-guide/tree/master/kinesis-streams -n kinesis-streams  10:14:50 +Serverless: Downloading and installing "serverless-kinesis-streams"... +central entry: serverless-kinesis-streams-master/ +central entry: serverless-kinesis-streams-master/README.md +central entry: serverless-kinesis-streams-master/event.json +central entry: serverless-kinesis-streams-master/handler.js +central entry: serverless-kinesis-streams-master/package.json +central entry: serverless-kinesis-streams-master/serverless.yml +Serverless: Successfully installed "serverless-kinesis-streams" as "kinesis-streams" +``` + +然后执行: + +``` +yarn install +``` + +就可以直接部署了: + +``` +Serverless: Packaging service... +Serverless: Excluding development dependencies... +Serverless: Uploading CloudFormation file to S3... +Serverless: Uploading artifacts... +Serverless: Uploading service .zip file to S3 (19.75 KB)... +Serverless: Validating template... +Serverless: Updating Stack... +Serverless: Checking Stack update progress... +.............................. +Serverless: Stack update finished... +Service Information +service: kinesis-streams +stage: dev +region: us-east-1 +stack: kinesis-streams-dev +api keys: + None +endpoints: + None +functions: + dataReceiver: kinesis-streams-dev-dataReceiver + logger: kinesis-streams-dev-logger +Serverless: Removing old service versions... +``` + +``` +$ serverless invoke --function dataReceiver --path event.json  + +11:30:48 +{ + "message": "Data successfully written to Kinesis stream \"data-receiver\"" +} +``` + +``` +$ serverless logs --function logger  + +11:31:41 +START RequestId: 3776bac6-612f-45dd-a8ac-156007f8e49b Version: $LATEST +2017-11-04 11:30:53.382 (+08:00) 3776bac6-612f-45dd-a8ac-156007f8e49b The following data was written to the Kinesis stream "data-receiver": +{ + "kinesisSchemaVersion": "1.0", + "partitionKey": "8e35d6a0-c110-11e7-90ae-59fa1aa30da7", + "sequenceNumber": "49578559262872379484471662829472308063624661238972153858", + "data": "U29tZSBleGFtcGxlIGRhdGE=", + "approximateArrivalTimestamp": 1509766251.753 +} +END RequestId: 3776bac6-612f-45dd-a8ac-156007f8e49b +REPORT RequestId: 3776bac6-612f-45dd-a8ac-156007f8e49b Duration: 72.07 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 33 MB +``` diff --git a/firehose/package.json b/firehose/package.json new file mode 100644 index 0000000..1997f0d --- /dev/null +++ b/firehose/package.json @@ -0,0 +1,13 @@ +{ + "name": "kinesis-streams", + "version": "0.1.0", + "description": "Serverless service to showcase Kinesis stream support", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "MIT", + "dependencies": { + "uuid": "^2.0.3" + } +} \ No newline at end of file diff --git a/firehose/serverless.yml b/firehose/serverless.yml new file mode 100644 index 0000000..cb32a07 --- /dev/null +++ b/firehose/serverless.yml @@ -0,0 +1,39 @@ +service: serverless-firehose + +custom: + names: + kinesis: ${self:service}-stream + +provider: + name: aws + runtime: nodejs4.3 + region: us-east-1 + stage: dev + memorySize: 128 + iamRoleStatements: + - Effect: "Allow" + Resource: "*" + Action: + - "kinesis:*" + - "logs:*" + - "firehose:*" + +functions: + dataReceiver: + handler: handler.dataReceiver + logger: + handler: handler.logger + events: + - stream: + type: kinesis + arn: + Fn::GetAtt: + - KinesisStream + - Arn +resources: + Resources: + KinesisStream: + Type: AWS::Kinesis::Stream + Properties: + Name: ${self:custom.names.kinesis} + ShardCount: 1 \ No newline at end of file diff --git a/firehose/yarn.lock b/firehose/yarn.lock new file mode 100644 index 0000000..8f86d6b --- /dev/null +++ b/firehose/yarn.lock @@ -0,0 +1,7 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +uuid@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"