diff --git a/.gitignore b/.gitignore index 6e7a079..1ac6ca3 100644 --- a/.gitignore +++ b/.gitignore @@ -121,4 +121,5 @@ temp/output.csv temp/*.csv *.pid -.data/ \ No newline at end of file +.data/ +config/.env.production \ No newline at end of file diff --git a/airflow_home/plugins/postgres_to_s3_operator.py b/airflow_home/plugins/postgres_to_s3_operator.py index 79481e5..178c506 100644 --- a/airflow_home/plugins/postgres_to_s3_operator.py +++ b/airflow_home/plugins/postgres_to_s3_operator.py @@ -35,7 +35,6 @@ def query_db(self, query, hook): return self.gen_file(content) def gen_file(self, content): - print(f"gen_file {content}") output = io.BytesIO() writer = csv.writer(output, delimiter='|', encoding='utf-8') writer.writerows(content) diff --git a/config/.env.sample b/config/.env.sample new file mode 100644 index 0000000..261dce0 --- /dev/null +++ b/config/.env.sample @@ -0,0 +1,3 @@ +AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow123@postgres:5432/airflow +AIRFLOW_CONN_HACKERQUEUE_PROD=postgres://user:password@db_host:5432/db +AIRFLOW_CONN_DEMO_BUCKET=s3://aws_access_key_id:aws_secret_access_key@S3 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1e9b380..86a80f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,8 +20,8 @@ services: build: . image: airflow_compose:latest restart: on-failure - environment: - AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow123@postgres:5432/airflow + env_file: + - config/.env.production ports: - "8080:8080" - "5555:5555"