diff --git a/.coveragerc b/.coveragerc index 8b0e54d30..d3ab7a345 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,3 +4,17 @@ omit = analytics_dashboard/settings* *wsgi.py *migrations* *admin.py + *test* +source = common, analytics_dashboard +branch = True + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + raise NotImplementedError + +[html] +directory = ${COVERAGE_DIR}/html/ + +[xml] +output = ${COVERAGE_DIR}/coverage.xml diff --git a/.gitignore b/.gitignore index ad1faf2d2..94a8cbf72 100644 --- a/.gitignore +++ b/.gitignore @@ -65,5 +65,11 @@ conf/locale/fake*/LC_MESSAGES/*.po conf/locale/fake*/LC_MESSAGES/*.mo conf/locale/messages.mo +# Unit test / coverage reports +coverage/ +.coverage +htmlcov +.tox + # Webpack webpack-stats.json diff --git a/.travis.yml b/.travis.yml index a8aebac98..6b9545d91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,16 +17,58 @@ before_install: # Start up the relevant services - docker-compose -f .travis/docker-compose-travis.yml up -d - docker exec analytics_api pip install --upgrade pip==9.0.3 - - docker exec analytics_api make -C /edx/app/analytics_api/analytics_api test.requirements - docker exec analytics_api make -C /edx/app/analytics_api/analytics_api travis + # HACK: https://github.com/nodejs/docker-node/issues/1199 + - rm package-lock.json + +install: + - pip install -r requirements/travis.txt script: - - docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_tests.sh + - docker exec -t -e TRAVIS=1 insights_testing bash -c " + cd /edx/app/insights/edx_analytics_dashboard/ && + PATH=\$PATH:/edx/app/insights/nodeenvs/insights/bin:/snap/bin && + make $TARGETS " -after_success: - - pip install -U codecov - - docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_coverage.sh - - codecov +matrix: + include: + - python: 2.7 + env: + TESTNAME=quality-and-js + TARGETS="requirements.js quality validate_js" + after_success: + - codecov --disable pycov + - python: 2.7 + env: + TESTNAME=a11y + TARGETS="requirements.a11y migrate requirements.js static accept a11y" + - python: 2.7 + env: + TESTNAME=test-i18n + TARGETS="validate_translations generate_fake_translations" + - python: 2.7 + env: + TESTNAME=test-python + TARGETS="requirements.js test_python" + after_success: + - docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_coverage.sh + - codecov --disable pycov + - python: 3.5 + env: + TESTNAME=quality-and-js-python-3.5 + TARGETS="PYTHON_ENV=py35 requirements.js quality validate_js" + - python: 3.5 + env: + TESTNAME=a11y + TARGETS="PYTHON_ENV=py35 requirements.a11y migrate requirements.js static accept a11y" + - python: 3.5 + env: + TESTNAME=test-i18n-python-3.5 + TARGETS="PYTHON_ENV=py35 validate_translations generate_fake_translations" + - python: 3.5 + env: + TESTNAME=test-python-python-3.5 + TARGETS="PYTHON_ENV=py35 requirements.js test_python" after_failure: # Print the list of running containers to rule out a killed container as a cause of failure diff --git a/.travis/a11y_reqs.sh b/.travis/a11y_reqs.sh new file mode 100755 index 000000000..650f5c2e3 --- /dev/null +++ b/.travis/a11y_reqs.sh @@ -0,0 +1,11 @@ +#!/bin/bash -xe + +apt update +apt install -y xvfb language-pack-en firefox + +# Need firefox 46 specifically, later versions don't work with Karma(frontend testing library). +curl -O https://ftp.mozilla.org/pub/firefox/releases/46.0/linux-x86_64/en-US/firefox-46.0.tar.bz2 +tar xvf firefox-46.0.tar.bz2 +mv -f firefox /opt +mv -f /usr/bin/firefox /usr/bin/firefox_default +ln -s /opt/firefox/firefox /usr/bin/firefox diff --git a/.travis/run_coverage.sh b/.travis/run_coverage.sh index aca42231d..5048f312a 100755 --- a/.travis/run_coverage.sh +++ b/.travis/run_coverage.sh @@ -1,9 +1,7 @@ #!/bin/bash -xe -. /edx/app/insights/venvs/insights/bin/activate -. /edx/app/insights/nodeenvs/insights/bin/activate cd /edx/app/insights/edx_analytics_dashboard -coverage xml +make coverage bash ./scripts/build-stats-to-datadog.sh diff --git a/.travis/run_tests.sh b/.travis/run_tests.sh deleted file mode 100755 index 8fd702f5c..000000000 --- a/.travis/run_tests.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -xe -. /edx/app/insights/venvs/insights/bin/activate -. /edx/app/insights/nodeenvs/insights/bin/activate - -apt update -apt install -y xvfb language-pack-en firefox # gettext - -# Need firefox 46 specifically, later versions don't work with Karma(frontend testing library). -curl -O https://ftp.mozilla.org/pub/firefox/releases/46.0/linux-x86_64/en-US/firefox-46.0.tar.bz2 -tar xvf firefox-46.0.tar.bz2 -mv -f firefox /opt -mv -f /usr/bin/firefox /usr/bin/firefox_default -ln -s /opt/firefox/firefox /usr/bin/firefox - -cd /edx/app/insights/edx_analytics_dashboard -export PATH=$PATH:$PWD/node_modules/.bin - -# Output node.js version -node --version -npm --version - -# HACK: remove package-lock.json because otherwise npm doesn't want to git clone (I WISH I KNEW WHY) -rm package-lock.json - -# Pin pip version -make pin_pip - -make develop -make migrate - -# Compile assets and run validation -make static_no_compress -make validate_translations -make validate -make generate_fake_translations - -# The following tests need insights running. We have to do it here -# because we need to wait till all the requirements have been installed -# otherwise the server will startup with potentially the wrong libraries. -/edx/bin/python.insights /edx/bin/manage.insights runserver 0.0.0.0:9000 --noreload --traceback > dashboard.log 2>&1 & -xvfb-run make accept -xvfb-run make a11y diff --git a/Makefile b/Makefile index b29d925b7..76e0c4612 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,9 @@ .PHONY: requirements ROOT = $(shell echo "$$PWD") -COVERAGE = $(ROOT)/build/coverage +COVERAGE_DIR = $(ROOT)/build/coverage NODE_BIN=./node_modules/.bin +PYTHON_ENV=py27 DJANGO_SETTINGS_MODULE ?= "analytics_dashboard.settings.local" @@ -14,6 +15,9 @@ pin_pip: requirements: requirements.py requirements.js +requirements.tox: + pip install -q -r requirements/tox.txt + requirements.py: pip install -q -r requirements/base.txt --exists-action w @@ -26,35 +30,45 @@ test.requirements: develop: requirements.js pip install -q -r requirements/local.txt --exists-action w -migrate: - python manage.py migrate --run-syncdb +migrate: requirements.tox + tox -e $(PYTHON_ENV)-migrate -clean: +clean: requirements.tox find . -name '*.pyc' -delete - coverage erase + tox -e $(PYTHON_ENV)-clean test_python_no_compress: clean - pytest analytics_dashboard common --cov=analytics_dashboard --cov=common --cov-branch --cov-report=html:$(COVERAGE)/html/ \ - --cov-report=xml:$(COVERAGE)/coverage.xml + tox -e $(PYTHON_ENV)-tests + +coverage: requirements.tox + export COVERAGE_DIR=$(COVERAGE_DIR) && \ + tox -e $(PYTHON_ENV)-coverage test_compress: static # No longer does anything. Kept for legacy support. -test_python: test_compress test_python_no_compress +test_python: requirements.tox test_compress test_python_no_compress + +requirements.a11y: + ./.travis/a11y_reqs.sh -accept: +runserver_a11y: requirements.tox + tox -e $(PYTHON_ENV)-runserver_a11y > dashboard.log 2>&1 & + +accept: runserver_a11y ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True") - ./manage.py waffle_flag enable_learner_analytics --create --everyone + tox -e $(PYTHON_ENV)-waffle_learner_analytics endif ifeq ("${ENABLE_COURSE_LIST_FILTERS}", "True") - ./manage.py waffle_switch enable_course_filters on --create + tox -e $(PYTHON_ENV)-waffle_course_filters endif ifeq ("${ENABLE_COURSE_LIST_PASSING}", "True") - ./manage.py waffle_switch enable_course_passing on --create + tox -e $(PYTHON_ENV)-waffle_course_passing endif - ./manage.py create_acceptance_test_soapbox_messages - pytest -v acceptance_tests -k 'not NUM_PROCESSES=1' --ignore=acceptance_tests/course_validation - ./manage.py delete_acceptance_test_soapbox_messages + tox -e $(PYTHON_ENV)-create_acceptance_test_soapbox_messages + tox -e $(PYTHON_ENV)-accept + tox -e $(PYTHON_ENV)-delete_acceptance_test_soapbox_messages + # local acceptance tests are typically run with by passing in environment variables on the commandline # e.g. API_SERVER_URL="http://localhost:9001/api/v0" API_AUTH_TOKEN="edx" make accept_local @@ -63,20 +77,27 @@ accept_local: pytest -v acceptance_tests --ignore=acceptance_tests/course_validation ./manage.py delete_acceptance_test_soapbox_messages -a11y: +a11y: requirements.tox ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True") - ./manage.py waffle_flag enable_learner_analytics --create --everyone + tox -e $(PYTHON_ENV)-waffle_learner_analytics endif - BOKCHOY_A11Y_CUSTOM_RULES_FILE=./node_modules/edx-custom-a11y-rules/lib/custom_a11y_rules.js SELENIUM_BROWSER=firefox pytest -v a11y_tests -k 'not NUM_PROCESSES=1' --ignore=acceptance_tests/course_validation + tox -e $(PYTHON_ENV)-a11y course_validation: python -m acceptance_tests.course_validation.generate_report -quality: - pycodestyle acceptance_tests analytics_dashboard common - PYTHONPATH=".:./analytics_dashboard:$PYTHONPATH" pylint --rcfile=pylintrc acceptance_tests analytics_dashboard common +run_check_isort: requirements.tox + tox -e $(PYTHON_ENV)-check_isort + +run_pycodestyle: requirements.tox + tox -e $(PYTHON_ENV)-pycodestyle + +run_pylint: requirements.tox + tox -e $(PYTHON_ENV)-pylint -validate_python: test.requirements test_python quality +quality: run_pylint run_pycodestyle + +validate_python: test_python quality #FIXME validate_js: requirements.js validate_js: @@ -91,16 +112,15 @@ demo: python manage.py waffle_switch display_course_name_in_nav off --create # compiles djangojs and django .po and .mo files -compile_translations: - python manage.py compilemessages +compile_translations: requirements.tox + tox -e $(PYTHON_ENV)-compile_translations # creates the source django & djangojs files -extract_translations: - cd analytics_dashboard && python ../manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d django - cd analytics_dashboard && python ../manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d djangojs +extract_translations: requirements.tox + tox -e $(PYTHON_ENV)-extract_translations -dummy_translations: - cd analytics_dashboard && i18n_tool dummy -v +dummy_translations: requirements.tox + tox -e $(PYTHON_ENV)-dummy_translations generate_fake_translations: extract_translations dummy_translations compile_translations @@ -110,21 +130,19 @@ pull_translations: update_translations: pull_translations generate_fake_translations # check if translation files are up-to-date -detect_changed_source_translations: - cd analytics_dashboard && i18n_tool changed +detect_changed_source_translations: requirements.tox + tox -e $(PYTHON_ENV)-detect_changed_translations # extract, compile, and check if translation files are up-to-date validate_translations: extract_translations compile_translations detect_changed_source_translations - cd analytics_dashboard && i18n_tool validate + tox -e $(PYTHON_ENV)-validate_translations static_no_compress: static # No longer does anything. Kept for legacy support. -static: +static: requirements.tox $(NODE_BIN)/webpack --config webpack.prod.config.js - # collectstatic creates way too much output with the cldr-data directory output so silence that directory - echo "Running collectstatic while silencing cldr-data/main/* ..." - python manage.py collectstatic --noinput | sed -n '/.*node_modules\/cldr-data\/main\/.*/!p' + tox -e $(PYTHON_ENV)-static export CUSTOM_COMPILE_COMMAND = make upgrade upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in @@ -136,4 +154,5 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy pip-compile --upgrade -o requirements/local.txt requirements/local.in pip-compile --upgrade -o requirements/optional.txt requirements/optional.in pip-compile --upgrade -o requirements/production.txt requirements/production.in - + pip-compile --upgrade -o requirements/tox.txt requirements/tox.in + pip-compile --upgrade -o requirements/travis.txt requirements/travis.in diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 4af20638d..000000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -DJANGO_SETTINGS_MODULE = analytics_dashboard.settings.test diff --git a/requirements/tox.in b/requirements/tox.in new file mode 100644 index 000000000..9bd2445ec --- /dev/null +++ b/requirements/tox.in @@ -0,0 +1,5 @@ +# Base requirements for tox +-c constraints.txt + +tox +tox-battery diff --git a/requirements/tox.txt b/requirements/tox.txt new file mode 100644 index 000000000..0d982e93e --- /dev/null +++ b/requirements/tox.txt @@ -0,0 +1,27 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# make upgrade +# +appdirs==1.4.3 # via virtualenv +configparser==4.0.2 # via importlib-metadata +contextlib2==0.6.0.post1 # via importlib-metadata, importlib-resources, virtualenv, zipp +distlib==0.3.0 # via virtualenv +filelock==3.0.12 # via tox, virtualenv +importlib-metadata==1.5.0 # via importlib-resources, pluggy, tox, virtualenv +importlib-resources==1.3.1 # via virtualenv +packaging==20.3 # via tox +pathlib2==2.3.5 # via importlib-metadata, importlib-resources, virtualenv +pluggy==0.13.1 # via tox +py==1.8.1 # via tox +pyparsing==2.4.6 # via packaging +scandir==1.10.0 # via pathlib2 +singledispatch==3.4.0.3 # via importlib-resources +six==1.14.0 # via packaging, pathlib2, tox, virtualenv +toml==0.10.0 # via tox +tox-battery==0.5.2 # via -r requirements/tox.in +tox==3.14.5 # via -r requirements/tox.in, tox-battery +typing==3.7.4.1 # via importlib-resources +virtualenv==20.0.10 # via tox +zipp==1.2.0 # via importlib-metadata, importlib-resources diff --git a/requirements/travis.in b/requirements/travis.in new file mode 100644 index 000000000..715f5fcc3 --- /dev/null +++ b/requirements/travis.in @@ -0,0 +1,6 @@ +# Basic requirements for Travis CI + +-c constraints.txt +-r tox.txt + +codecov diff --git a/requirements/travis.txt b/requirements/travis.txt new file mode 100644 index 000000000..b637af8a2 --- /dev/null +++ b/requirements/travis.txt @@ -0,0 +1,34 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# make upgrade +# +appdirs==1.4.3 # via -r requirements/tox.txt, virtualenv +certifi==2019.11.28 # via requests +chardet==3.0.4 # via requests +codecov==2.0.16 # via -r requirements/travis.in +configparser==4.0.2 # via -r requirements/tox.txt, importlib-metadata +contextlib2==0.6.0.post1 # via -r requirements/tox.txt, importlib-metadata, importlib-resources, virtualenv, zipp +coverage==5.0.3 # via codecov +distlib==0.3.0 # via -r requirements/tox.txt, virtualenv +filelock==3.0.12 # via -r requirements/tox.txt, tox, virtualenv +idna==2.9 # via requests +importlib-metadata==1.5.0 # via -r requirements/tox.txt, importlib-resources, pluggy, tox, virtualenv +importlib-resources==1.3.1 # via -r requirements/tox.txt, virtualenv +packaging==20.3 # via -r requirements/tox.txt, tox +pathlib2==2.3.5 # via -r requirements/tox.txt, importlib-metadata, importlib-resources, virtualenv +pluggy==0.13.1 # via -r requirements/tox.txt, tox +py==1.8.1 # via -r requirements/tox.txt, tox +pyparsing==2.4.6 # via -r requirements/tox.txt, packaging +requests==2.23.0 # via codecov +scandir==1.10.0 # via -r requirements/tox.txt, pathlib2 +singledispatch==3.4.0.3 # via -r requirements/tox.txt, importlib-resources +six==1.14.0 # via -r requirements/tox.txt, packaging, pathlib2, singledispatch, tox, virtualenv +toml==0.10.0 # via -r requirements/tox.txt, tox +tox-battery==0.5.2 # via -r requirements/tox.txt +tox==3.14.5 # via -r requirements/tox.txt, tox-battery +typing==3.7.4.1 # via -r requirements/tox.txt, importlib-resources +urllib3==1.25.8 # via requests +virtualenv==20.0.10 # via -r requirements/tox.txt, tox +zipp==1.2.0 # via -r requirements/tox.txt, importlib-metadata, importlib-resources diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..5525d37c5 --- /dev/null +++ b/tox.ini @@ -0,0 +1,122 @@ +[tox] +skipsdist=True +envlist = {py27,py35}-{check_isort,create_acceptance_test_soapbox_messages,delete_acceptance_test_soapbox_messages,detect_changed_translations,dummy_translations,extract_translations,pycodestyle,pylint,tests,test_acceptvalidate_translations,waffle_learner_analytics,waffle_course_filters,waffle_course_passing} + +[pytest] +DJANGO_SETTINGS_MODULE = analytics_dashboard.settings.test + +[testenv] +envdir= + # Use the same environment for all commands running under a specific python version + py27: {toxworkdir}/py27 + py35: {toxworkdir}/py35 + +passenv = + API_SERVER_URL + API_AUTH_TOKEN + LMS_HOSTNAME + LMS_PASSWORD + LMS_USERNAME + ENABLE_AUTO_AUTH + ENABLE_OAUTH_TESTS + ENABLE_ERROR_PAGE_TESTS + DISPLAY_LEARNER_ANALYTICS + ENABLE_COURSE_LIST_FILTERS + ENABLE_COURSE_LIST_PASSING + COVERAGE_DIR + +setenv = + tests: DJANGO_SETTINGS_MODULE = analytics_dashboard.settings.test + PYTHONPATH = ".:./analytics_dashboard:$PYTHONPATH" + NODE_BIN = ./node_modules/.bin + NODENV = /edx/app/insights/nodeenvs/insights/bin + PATH = $PATH:$NODE_BIN:$NODENV + PYTEST_ADDOPTS = "--cov-config=.coveragerc" +deps = + -r requirements/base.txt + -r requirements/test.txt + +changedir = + detect_changed_translations,dummy_translations,validate_translations: analytics_dashboard + +whitelist_externals = + xvfb-run + +commands = + check_isort: isort --check-only --recursive --diff acceptance_tests/ analytics_dashboard/ common/ + clean: coverage erase + compile_translations: python manage.py compilemessages + coverage: coverage report + coverage: coverage xml + coverage: coverage html + create_acceptance_test_soapbox_messages: python manage.py create_acceptance_test_soapbox_messages + delete_acceptance_test_soapbox_messages: python manage.py delete_acceptance_test_soapbox_messages + detect_changed_translations: i18n_tool changed + extract_translations: python manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d django + extract_translations: python manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d djangojs + migrate: python manage.py migrate --run-syncdb + pylint: pylint -j 0 --rcfile=pylintrc acceptance_tests analytics_dashboard common + pycodestyle: pycodestyle acceptance_tests analytics_dashboard common + runserver_a11y: python manage.py runserver 0.0.0.0:9000 --noreload --traceback + static: python manage.py collectstatic --noinput --verbosity 0 + tests: python -Wd -m pytest {posargs} common analytics_dashboard --cov common --cov analytics_dashboard + validate_translations: i18n_tool validate - + waffle_learner_analytics: python manage.py waffle_flag enable_learner_analytics --create --everyone + waffle_course_filters: python manage.py waffle_switch enable_course_filters on --create + waffle_course_passing: python ./manage.py waffle_switch enable_course_passing on --create + +[testenv:py27-accept] +envdir = {toxworkdir}/py27 +passenv = + API_SERVER_URL + API_AUTH_TOKEN + LMS_HOSTNAME + LMS_PASSWORD + LMS_USERNAME + ENABLE_AUTO_AUTH + ENABLE_OAUTH_TESTS + ENABLE_ERROR_PAGE_TESTS + DISPLAY_LEARNER_ANALYTICS + ENABLE_COURSE_LIST_FILTERS + ENABLE_COURSE_LIST_PASSING + +setenv = + DJANGO_SETTINGS_MODULE = analytics_dashboard.settings.test + PYTHONPATH=".:./analytics_dashboard:$PYTHONPATH" + NODE_BIN = ./node_modules/.bin + NODENV = /edx/app/insights/nodeenvs/insights/bin + PATH=$PATH:$NODE_BIN:$NODENV + BOKCHOY_A11Y_CUSTOM_RULES_FILE=./node_modules/edx-custom-a11y-rules/lib/custom_a11y_rules.js + SELENIUM_BROWSER=firefox +deps = + -r requirements/base.txt + -r requirements/test.txt + +whitelist_externals = + xvfb-run + +commands = + xvfb-run pytest -v acceptance_tests --ignore=acceptance_tests/course_validation + + +[testenv:py27-a11y] +envdir = {toxworkdir}/py27 +passenv = {[testenv:py27-accept]passenv} +deps = {[testenv:py27-accept]deps} +whitelist_externals = {[testenv:py27-accept]whitelist_externals} +commands = + xvfb-run pytest -v a11y_tests -k 'not NUM_PROCESSES=1' --ignore=acceptance_tests/course_validation + +[testenv:py35-accept] +envdir = {toxworkdir}/py35 +passenv = {[testenv:py27-accept]passenv} +deps = {[testenv:py27-accept]deps} +whitelist_externals = {[testenv:py27-accept]whitelist_externals} +commands = {[testenv:py27-accept]commands} + +[testenv:py35-a11y] +envdir = {toxworkdir}/py35 +passenv = {[testenv:py27-a11y]passenv} +deps = {[testenv:py27-a11y]deps} +whitelist_externals = {[testenv:py27-a11y]whitelist_externals} +commands = {[testenv:py27-a11y]commands}