initial commit

.dockerignore

@@ -0,0 +1,127 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# fuck osx
+.DS_Store
+
+airflow_home/logs
+airflow_home/airflow.cfg
+airflow_home/airflow.db
+airflow_home/unittests.cfg
+temp/output.csv
+
+.vscode
+
+temp/*.csv
+
+*.pid
+
+
+.git
+temp/*.csv
+test/*
+venv/*
+
+__pycache__

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+ignore = D203,W293,W191,E305,E501,E251,E302,E115,E402,E266,E128,E261,E265,E101,E202,E231,E222,E303,E124,E293,E226,W503,E722,E201,E225,W292,W291,E127,E201,E126,E123,E121,E203,E221,E122,E262,
+exclude = .git, __pycache__, venv
+max-complexity = 35

.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,16 @@
+<!-- Fill out this template to explain your pull request. -->
+
+# Related Tickets
+<!-- Please link to any related tickets -->
+
+
+# Changes proposed
+<!-- Describe the changes you've made so it's easier for the team to review. -->
+
+
+#### Expected Output
+
+
+#### Caveats
+<!-- If there is anything hacky or unique being added in your code please define it.-->
+

.gitignore

@@ -0,0 +1,124 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+temp/*
+
+*.db-journal
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# fuck osx
+.DS_Store
+
+airflow_home/logs
+airflow_home/airflow.cfg
+airflow_home/airflow.db
+airflow_home/unittests.cfg
+temp/output.csv
+
+.vscode
+
+temp/*.csv
+
+*.pid
+.data/

Dockerfile

@@ -0,0 +1,81 @@
+FROM python:3.6-slim
+
+# Never prompts the user for choices on installation/configuration of packages
+ENV DEBIAN_FRONTEND noninteractive
+ENV TERM linux
+
+# Airflow
+ARG AIRFLOW_VERSION=1.10.0
+ARG AIRFLOW_HOME=/usr/local/airflow
+ENV AIRFLOW_GPL_UNIDECODE yes
+ENV AIRFLOW_HOME=/usr/local/airflow
+
+# Define en_US.
+ENV LANGUAGE en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV LC_CTYPE en_US.UTF-8
+ENV LC_MESSAGES en_US.UTF-8
+ENV locale-gen en_US.UTF-8
+ENV locale-gen es_CO.UTF-8
+
+
+COPY requirements.txt ${AIRFLOW_HOME}/requirements.txt
+
+RUN set -ex \
+    && buildDeps=' \
+        python3-dev \
+        libkrb5-dev \
+        libsasl2-dev \
+        libssl-dev \
+        libffi-dev \
+        libblas-dev \
+        liblapack-dev \
+        libpq-dev \
+        git \
+    ' \
+    && apt-get update -yqq \
+    && apt-get upgrade -yqq \
+    && apt-get install -yqq --no-install-recommends \
+        $buildDeps \
+        build-essential \
+        python3-pip \
+        python3-requests \
+        mysql-client \
+        mysql-server \
+        default-libmysqlclient-dev \
+        apt-utils \
+        curl \
+        rsync \
+        netcat \
+        locales \
+    && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \
+    && locale-gen \
+    && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \
+    && useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow \
+    && pip install -U pip setuptools wheel \
+    && pip install -r ${AIRFLOW_HOME}/requirements.txt \
+    && apt-get purge --auto-remove -yqq $buildDeps \
+    && apt-get autoremove -yqq --purge \
+    && apt-get clean \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/* \
+        /usr/share/man \
+        /usr/share/doc \
+        /usr/share/doc-base
+
+COPY script/entrypoint.sh /entrypoint.sh
+COPY config/airflow.cfg ${AIRFLOW_HOME}/airflow.cfg
+COPY airflow_home/ ${AIRFLOW_HOME}
+RUN mkdir ${AIRFLOW_HOME}/temp
+
+RUN chown -R airflow: ${AIRFLOW_HOME}
+
+EXPOSE 8080 5555 8793
+
+USER airflow
+WORKDIR ${AIRFLOW_HOME}
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["webserver"] # set default arg for entrypoint

Makefile

@@ -0,0 +1,2 @@
+build:
+	docker build --no-cache --rm -t airflow_compose:latest .