From 5784284eebc06504406b2f123115a1edb55ee852 Mon Sep 17 00:00:00 2001 From: frankcash Date: Tue, 26 Mar 2019 20:08:34 -0400 Subject: [PATCH] sample has been fully init --- .gitignore | 3 ++- airflow_home/plugins/postgres_to_s3_operator.py | 1 - config/.env.sample | 3 +++ docker-compose.yml | 4 ++-- 4 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 config/.env.sample diff --git a/.gitignore b/.gitignore index 6e7a079..1ac6ca3 100644 --- a/.gitignore +++ b/.gitignore @@ -121,4 +121,5 @@ temp/output.csv temp/*.csv *.pid -.data/ \ No newline at end of file +.data/ +config/.env.production \ No newline at end of file diff --git a/airflow_home/plugins/postgres_to_s3_operator.py b/airflow_home/plugins/postgres_to_s3_operator.py index 79481e5..178c506 100644 --- a/airflow_home/plugins/postgres_to_s3_operator.py +++ b/airflow_home/plugins/postgres_to_s3_operator.py @@ -35,7 +35,6 @@ def query_db(self, query, hook): return self.gen_file(content) def gen_file(self, content): - print(f"gen_file {content}") output = io.BytesIO() writer = csv.writer(output, delimiter='|', encoding='utf-8') writer.writerows(content) diff --git a/config/.env.sample b/config/.env.sample new file mode 100644 index 0000000..261dce0 --- /dev/null +++ b/config/.env.sample @@ -0,0 +1,3 @@ +AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow123@postgres:5432/airflow +AIRFLOW_CONN_HACKERQUEUE_PROD=postgres://user:password@db_host:5432/db +AIRFLOW_CONN_DEMO_BUCKET=s3://aws_access_key_id:aws_secret_access_key@S3 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1e9b380..86a80f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,8 @@ services: build: . image: airflow_compose:latest restart: on-failure - environment: - AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow123@postgres:5432/airflow + env_file: + - config/.env.production ports: - "8080:8080" - "5555:5555"