[BD] Tests in python 3 (#965)

Use tox and run tests in python3 https://openedx.atlassian.net/browse/BOM-1359.

* Do not include tests in coverage

* Separate quality tests

* Upload javascript coverage to codecov

* Add coverage configuration that is present in other openedX projects

.coveragerc

@@ -4,3 +4,17 @@ omit = analytics_dashboard/settings*
        *wsgi.py
        *migrations*
        *admin.py
+       *test*
+source = common, analytics_dashboard
+branch = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    raise NotImplementedError
+
+[html]
+directory = ${COVERAGE_DIR}/html/
+
+[xml]
+output = ${COVERAGE_DIR}/coverage.xml

.gitignore

@@ -65,5 +65,11 @@ conf/locale/fake*/LC_MESSAGES/*.po
 conf/locale/fake*/LC_MESSAGES/*.mo
 conf/locale/messages.mo
 
+# Unit test / coverage reports
+coverage/
+.coverage
+htmlcov
+.tox
+
 # Webpack
 webpack-stats.json

.travis.yml

@@ -17,16 +17,58 @@ before_install:
     # Start up the relevant services
     - docker-compose -f .travis/docker-compose-travis.yml up -d
     - docker exec analytics_api pip install --upgrade pip==9.0.3
-    - docker exec analytics_api make -C /edx/app/analytics_api/analytics_api test.requirements
     - docker exec analytics_api make -C /edx/app/analytics_api/analytics_api travis
+    # HACK: https://github.com/nodejs/docker-node/issues/1199
+    - rm package-lock.json
+
+install:
+    - pip install -r requirements/travis.txt
 
 script:
-    - docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_tests.sh
+    - docker exec -t -e TRAVIS=1 insights_testing bash -c "
+      cd /edx/app/insights/edx_analytics_dashboard/ &&
+      PATH=\$PATH:/edx/app/insights/nodeenvs/insights/bin:/snap/bin &&
+      make $TARGETS "
 
-after_success:
-    - pip install -U codecov
-    - docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_coverage.sh
-    - codecov
+matrix:
+  include:
+    - python: 2.7
+      env:
+        TESTNAME=quality-and-js
+        TARGETS="requirements.js quality validate_js"
+      after_success:
+        - codecov --disable pycov
+    - python: 2.7
+      env:
+        TESTNAME=a11y
+        TARGETS="requirements.a11y migrate requirements.js static accept a11y"
+    - python: 2.7
+      env:
+        TESTNAME=test-i18n
+        TARGETS="validate_translations generate_fake_translations"
+    - python: 2.7
+      env:
+        TESTNAME=test-python
+        TARGETS="requirements.js test_python"
+      after_success:
+        - docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_coverage.sh
+        - codecov --disable pycov
+    - python: 3.5
+      env:
+        TESTNAME=quality-and-js-python-3.5
+        TARGETS="PYTHON_ENV=py35 requirements.js quality validate_js"
+    - python: 3.5
+      env:
+        TESTNAME=a11y
+        TARGETS="PYTHON_ENV=py35 requirements.a11y migrate requirements.js static accept a11y"
+    - python: 3.5
+      env:
+        TESTNAME=test-i18n-python-3.5
+        TARGETS="PYTHON_ENV=py35 validate_translations generate_fake_translations"
+    - python: 3.5
+      env:
+        TESTNAME=test-python-python-3.5
+        TARGETS="PYTHON_ENV=py35 requirements.js test_python"
 
 after_failure:
     # Print the list of running containers to rule out a killed container as a cause of failure

.travis/a11y_reqs.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -xe
+
+apt update
+apt install -y xvfb language-pack-en firefox
+
+# Need firefox 46 specifically, later versions don't work with Karma(frontend testing library).
+curl -O https://ftp.mozilla.org/pub/firefox/releases/46.0/linux-x86_64/en-US/firefox-46.0.tar.bz2
+tar xvf firefox-46.0.tar.bz2
+mv -f firefox /opt
+mv -f /usr/bin/firefox /usr/bin/firefox_default
+ln -s /opt/firefox/firefox /usr/bin/firefox

.travis/run_coverage.sh

@@ -1,9 +1,7 @@
 #!/bin/bash -xe
-. /edx/app/insights/venvs/insights/bin/activate
-. /edx/app/insights/nodeenvs/insights/bin/activate
 
 cd /edx/app/insights/edx_analytics_dashboard
 
-coverage xml
+make coverage
 
 bash ./scripts/build-stats-to-datadog.sh

Makefile

@@ -1,8 +1,9 @@
 .PHONY: requirements
 
 ROOT = $(shell echo "$$PWD")
-COVERAGE = $(ROOT)/build/coverage
+COVERAGE_DIR = $(ROOT)/build/coverage
 NODE_BIN=./node_modules/.bin
+PYTHON_ENV=py27
 
 DJANGO_SETTINGS_MODULE ?= "analytics_dashboard.settings.local"
 
@@ -14,6 +15,9 @@ pin_pip:
 
 requirements: requirements.py requirements.js
 
+requirements.tox:
+	pip install -q -r requirements/tox.txt
+
 requirements.py:
 	pip install -q -r requirements/base.txt --exists-action w
 
@@ -26,35 +30,45 @@ test.requirements:
 develop: requirements.js
 	pip install -q -r requirements/local.txt --exists-action w
 
-migrate:
-	python manage.py migrate --run-syncdb
+migrate: requirements.tox
+	tox -e $(PYTHON_ENV)-migrate
 
-clean:
+clean: requirements.tox 
 	find . -name '*.pyc' -delete
-	coverage erase
+	tox -e $(PYTHON_ENV)-clean
 
 test_python_no_compress: clean
-	pytest analytics_dashboard common --cov=analytics_dashboard --cov=common --cov-branch --cov-report=html:$(COVERAGE)/html/ \
-	--cov-report=xml:$(COVERAGE)/coverage.xml
+	tox -e $(PYTHON_ENV)-tests
+
+coverage: requirements.tox
+	export COVERAGE_DIR=$(COVERAGE_DIR) && \
+	tox -e $(PYTHON_ENV)-coverage
 
 test_compress: static
 	# No longer does anything. Kept for legacy support.
 
-test_python: test_compress test_python_no_compress
+test_python: requirements.tox test_compress test_python_no_compress
+
+requirements.a11y:
+	./.travis/a11y_reqs.sh
 
-accept:
+runserver_a11y: requirements.tox
+	tox -e $(PYTHON_ENV)-runserver_a11y > dashboard.log 2>&1 &
+
+accept: runserver_a11y
 ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True")
-	./manage.py waffle_flag enable_learner_analytics --create --everyone
+	tox -e $(PYTHON_ENV)-waffle_learner_analytics
 endif
 ifeq ("${ENABLE_COURSE_LIST_FILTERS}", "True")
-	./manage.py waffle_switch enable_course_filters on --create
+	tox -e $(PYTHON_ENV)-waffle_course_filters
 endif
 ifeq ("${ENABLE_COURSE_LIST_PASSING}", "True")
-	./manage.py waffle_switch enable_course_passing on --create
+	tox -e $(PYTHON_ENV)-waffle_course_passing
 endif
-	./manage.py create_acceptance_test_soapbox_messages
-	pytest -v acceptance_tests -k 'not NUM_PROCESSES=1' --ignore=acceptance_tests/course_validation
-	./manage.py delete_acceptance_test_soapbox_messages
+	tox -e $(PYTHON_ENV)-create_acceptance_test_soapbox_messages
+	tox -e $(PYTHON_ENV)-accept
+	tox -e $(PYTHON_ENV)-delete_acceptance_test_soapbox_messages
+
 
 # local acceptance tests are typically run with by passing in environment variables on the commandline
 # e.g. API_SERVER_URL="http://localhost:9001/api/v0" API_AUTH_TOKEN="edx" make accept_local
@@ -63,20 +77,27 @@ accept_local:
 	pytest -v acceptance_tests --ignore=acceptance_tests/course_validation
 	./manage.py delete_acceptance_test_soapbox_messages
 
-a11y:
+a11y: requirements.tox
 ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True")
-	./manage.py waffle_flag enable_learner_analytics --create --everyone
+	tox -e $(PYTHON_ENV)-waffle_learner_analytics
 endif
-	BOKCHOY_A11Y_CUSTOM_RULES_FILE=./node_modules/edx-custom-a11y-rules/lib/custom_a11y_rules.js SELENIUM_BROWSER=firefox pytest -v a11y_tests -k 'not NUM_PROCESSES=1' --ignore=acceptance_tests/course_validation
+	tox -e $(PYTHON_ENV)-a11y
 
 course_validation:
 	python -m acceptance_tests.course_validation.generate_report
 
-quality:
-	pycodestyle acceptance_tests analytics_dashboard common
-	PYTHONPATH=".:./analytics_dashboard:$PYTHONPATH" pylint --rcfile=pylintrc acceptance_tests analytics_dashboard common
+run_check_isort: requirements.tox
+	tox -e $(PYTHON_ENV)-check_isort
+
+run_pycodestyle: requirements.tox
+	tox -e $(PYTHON_ENV)-pycodestyle
+
+run_pylint: requirements.tox
+	tox -e $(PYTHON_ENV)-pylint
 
-validate_python: test.requirements test_python quality
+quality: run_pylint run_pycodestyle
+
+validate_python: test_python quality
 
 #FIXME validate_js: requirements.js
 validate_js:
@@ -91,16 +112,15 @@ demo:
 	python manage.py waffle_switch display_course_name_in_nav off --create
 
 # compiles djangojs and django .po and .mo files
-compile_translations:
-	python manage.py compilemessages
+compile_translations: requirements.tox
+	tox -e $(PYTHON_ENV)-compile_translations
 
 # creates the source django & djangojs files
-extract_translations:
-	cd analytics_dashboard && python ../manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d django
-	cd analytics_dashboard && python ../manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d djangojs
+extract_translations: requirements.tox
+	tox -e $(PYTHON_ENV)-extract_translations
 
-dummy_translations:
-	cd analytics_dashboard && i18n_tool dummy -v
+dummy_translations: requirements.tox
+	tox -e $(PYTHON_ENV)-dummy_translations
 
 generate_fake_translations: extract_translations dummy_translations compile_translations
 
@@ -110,21 +130,19 @@ pull_translations:
 update_translations: pull_translations generate_fake_translations
 
 # check if translation files are up-to-date
-detect_changed_source_translations:
-	cd analytics_dashboard && i18n_tool changed
+detect_changed_source_translations: requirements.tox
+	tox -e $(PYTHON_ENV)-detect_changed_translations
 
 # extract, compile, and check if translation files are up-to-date
 validate_translations: extract_translations compile_translations detect_changed_source_translations
-	cd analytics_dashboard && i18n_tool validate
+	tox -e $(PYTHON_ENV)-validate_translations
 
 static_no_compress: static
 	# No longer does anything. Kept for legacy support.
 
-static:
+static: requirements.tox
 	$(NODE_BIN)/webpack --config webpack.prod.config.js
-	# collectstatic creates way too much output with the cldr-data directory output so silence that directory
-	echo "Running collectstatic while silencing cldr-data/main/* ..."
-	python manage.py collectstatic --noinput | sed -n '/.*node_modules\/cldr-data\/main\/.*/!p'
+	tox -e $(PYTHON_ENV)-static
 
 export CUSTOM_COMPILE_COMMAND = make upgrade
 upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
@@ -136,4 +154,5 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy
 	pip-compile --upgrade -o requirements/local.txt requirements/local.in
 	pip-compile --upgrade -o requirements/optional.txt requirements/optional.in
 	pip-compile --upgrade -o requirements/production.txt requirements/production.in
-
+	pip-compile --upgrade -o requirements/tox.txt requirements/tox.in
+	pip-compile --upgrade -o requirements/travis.txt requirements/travis.in

requirements/tox.in

@@ -0,0 +1,5 @@
+# Base requirements for tox
+-c constraints.txt
+
+tox
+tox-battery

requirements/tox.txt

@@ -0,0 +1,27 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    make upgrade
+#
+appdirs==1.4.3            # via virtualenv
+configparser==4.0.2       # via importlib-metadata
+contextlib2==0.6.0.post1  # via importlib-metadata, importlib-resources, virtualenv, zipp
+distlib==0.3.0            # via virtualenv
+filelock==3.0.12          # via tox, virtualenv
+importlib-metadata==1.5.0  # via importlib-resources, pluggy, tox, virtualenv
+importlib-resources==1.3.1  # via virtualenv
+packaging==20.3           # via tox
+pathlib2==2.3.5           # via importlib-metadata, importlib-resources, virtualenv
+pluggy==0.13.1            # via tox
+py==1.8.1                 # via tox
+pyparsing==2.4.6          # via packaging
+scandir==1.10.0           # via pathlib2
+singledispatch==3.4.0.3   # via importlib-resources
+six==1.14.0               # via packaging, pathlib2, tox, virtualenv
+toml==0.10.0              # via tox
+tox-battery==0.5.2        # via -r requirements/tox.in
+tox==3.14.5               # via -r requirements/tox.in, tox-battery
+typing==3.7.4.1           # via importlib-resources
+virtualenv==20.0.10       # via tox
+zipp==1.2.0               # via importlib-metadata, importlib-resources

requirements/travis.in

@@ -0,0 +1,6 @@
+# Basic requirements for Travis CI
+
+-c constraints.txt
+-r tox.txt
+
+codecov