first commit

.gitignore

@@ -0,0 +1 @@
+.localstack/

README.md

@@ -0,0 +1,83 @@
+# Elasticsearch with AWS Lambda and SNS using LocalStack
+
+![Architecture](img/architecture.png)
+
+This is an exercise and also a repository for testing using LocalStack, Elasticsearch, Kibana, AWS Lambda and SNS.
+
+Initially it is a copy of the article [*Ingest data into Elasticsearch with AWS Lambda and SNS using LocalStack*](https://dev.to/jainec/ingest-data-into-elasticsearch-with-aws-lambda-and-sns-using-localstack-3af4) by Jaine Conceição, but I want to make new implementations in the future for studies and experiments.
+
+Visit the article: https://dev.to/jainec/ingest-data-into-elasticsearch-with-aws-lambda-and-sns-using-localstack-3af4
+
+---
+
+## Start containers
+
+Run the `docker-compose.yml` file:
+
+```sh
+$ docker-compose up -d
+```
+
+## Configure local SNS and Lambda
+
+Creating the SNS topic
+
+```sh
+$ aws --endpoint-url=http://0.0.0.0:4566 sns create-topic --name events-topic
+```
+
+Creating the Lambda function
+
+```sh
+$ aws --endpoint-url=http://0.0.0.0:4566 lambda create-function --function-name events-lambda --zip-file fileb://function_lambda.zip --handler index.handler --runtime nodejs12.x --role _
+```
+
+Subscribing lambda to SNS topic
+
+```sh
+$ aws --endpoint-url=http://0.0.0.0:4566 sns subscribe --topic-arn arn:aws:sns:us-east-1:000000000000:events-topic --protocol lambda --notification-endpoint arn:aws:lambda:us-east-1:000000000000:function:events-lambda
+```
+
+## Testing
+
+Sending a message to the SNS topic via command line:
+
+```sh
+$ aws sns publish --endpoint-url=http://0.0.0.0:4566 --topic-arn arn:aws:sns:us-east-1:000000000000:events-topic --message '{"name":"test", "user_id":1, "properties": {"nickname": "test-user1", "job": "Software engineer"}}'
+```
+
+Now access Kibana and make sure that you have configured the index pattern:
+
+![Create index](img/kibana_create_index.png)
+
+Then go to Discover section and your message should appear like that:
+
+![View message](img/kibana_view_message.png)
+
+---
+
+## Troubleshoot
+
+### Install AWS CLI on Alpine Linux 3.15
+
+```sh
+$ apk add --no-cache python3 py3-pip
+$ pip3 install --upgrade pip
+$ pip3 install --no-cache-dir awscli
+
+# Check installation
+$ aws --version
+
+# Output similar
+aws-cli/1.25.7 Python/3.97 Linux/5.10.102.1-microsoft-standard-WSL2 botocore/1.27.7
+```
+
+### Error when starting elasticsearch container
+
+> (...) max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
+
+Execute:
+```sh
+#https://github.com/docker-library/elasticsearch/issues/111
+$ sysctl -w vm.max_map_count=262144
+```

docker-compose.yml

@@ -0,0 +1,40 @@
+version: "3.5"
+
+services:
+  localstack:
+    container_name: localstack
+    image: localstack/localstack:latest
+    networks:
+      - default
+    ports:
+      - "4566:4566"
+    environment:
+      - EDGE_PORT=4566
+      - SERVICES=sns,lambda
+    volumes:
+      - ./.localstack:/tmp/localstack
+
+  elasticsearch:
+    container_name: elasticsearch
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.7.0
+    environment:
+      discovery.type: "single-node"
+      ES_JAVA_OPTS: "-Xms512m -Xmx512m"
+    ports:
+      - 9200:9200
+    networks:
+      - default
+
+  kibana:
+    container_name: kibana
+    image: docker.elastic.co/kibana/kibana:7.7.0
+    ports:
+      - "5601:5601"
+    networks:
+      - default
+    depends_on:
+      - elasticsearch
+
+networks:
+  default:
+    driver: bridge

function_lambda/elastic/elastic.js

@@ -0,0 +1,36 @@
+console.log('Loading elasticsearch file');
+
+const elasticsearch = require('elasticsearch');
+
+const elasticClient = elasticsearch.Client({
+    host: 'http://elasticsearch:9200'
+});
+
+const indexName = 'event_tracking';
+
+function initIndex() {
+    console.log('Init elasticsearch');
+    try {
+        return elasticClient.indices.create({
+            index: indexName
+        });
+    } catch (error) {
+        console.log('Error creating elasticsearch index')
+    }
+}
+exports.initIndex = initIndex;
+
+function addDocument(message) {
+    console.log('Adding document to elastic');
+    return elasticClient.index({
+        index: indexName,
+        type: "document",
+        body: {
+            event_name: message.name ?? null,
+            user_id: message.user_id ?? null,
+            properties: message.properties ?? null,
+            created_at: message.created_at ?? null,
+        }
+    });
+}
+exports.addDocument = addDocument;

function_lambda/index.js

@@ -0,0 +1,15 @@
+console.log('Loading function');
+
+const elasticsearch = require('./elastic/elastic');
+
+elasticsearch.initIndex();
+
+exports.handler = function(event) {
+    const message = event.Records[0].Sns.Message;
+    console.log('Message received from SNS:', message);
+
+    elasticsearch.addDocument(JSON.parse(message));
+    console.log('Document added to elastic');
+
+    return 'Success';
+};