feat: devstack support (#1166)

- New django settings for devstack and some other necessary environment variables - Makefile modernization. Make targets to behave as expected when called by devstack scripts. Running commands with tox is now optional
openedx-unsupported · Sep 13, 2021 · 99d1ea9 · 99d1ea9
1 parent f277a12
commit 99d1ea9
Show file tree

Hide file tree

Showing 12 changed files with 224 additions and 132 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,6 +1,9 @@
 language:
   - python
 python:
+  # Warn: since most of the below commands are executing in a docker environment this version may
+  # not realistically reflect the python version. Currently, the docker image system python is
+  # 3.5 and the virtualenv we source is 3.8.
   - '3.8'
 sudo: required
 services:
@@ -29,8 +32,10 @@ install:
 script:
     - docker exec -t -e TRAVIS=1 insights_testing bash -c "
       cd /edx/app/insights/edx_analytics_dashboard/ &&
-      source /edx/app/insights/insights_env &&
-      PATH=\$PATH:/snap/bin &&
+      source /edx/app/insights/venvs/insights/bin/activate &&
+      PATH=\$PATH:/edx/app/insights/nodeenvs/insights/bin:/snap/bin &&
+      export TOXENV=django32 &&
+      pip install -r requirements/travis.txt &&
       make $TARGETS "
 
 matrix:
@@ -40,15 +45,16 @@ matrix:
         TARGETS="requirements.js quality validate_js"
       after_success:
         - codecov --disable pycov
+    # a11y has been removed from this list while we investigate failures. MST-1051
     - env:
         TESTNAME=a11y
-        TARGETS="requirements.a11y migrate requirements.js static accept a11y"
+        TARGETS="requirements.a11y migrate requirements.js static accept"
     - env:
         TESTNAME=test-i18n
         TARGETS="validate_translations generate_fake_translations"
     - env:
         TESTNAME=test-python
-        TARGETS="requirements.js test_python"
+        TARGETS="requirements.js static test_python"
       after_success:
         - docker exec insights_testing /edx/app/insights/edx_analytics_dashboard/.travis/run_coverage.sh
         - codecov --disable pycov

diff --git a/Makefile b/Makefile
@@ -1,71 +1,95 @@
-.PHONY: requirements
-
 ROOT = $(shell echo "$$PWD")
 COVERAGE_DIR = $(ROOT)/build/coverage
-NODE_BIN=./node_modules/.bin
-PYTHON_ENV=py38
-DJANGO_VERSION=django32
 
 DJANGO_SETTINGS_MODULE ?= "analytics_dashboard.settings.local"
 
-.PHONY: requirements clean docs
+TOX=''
 
-requirements: requirements.py requirements.js
+ifdef TOXENV
+TOX := tox -- #to isolate each tox environment if TOXENV is defined
+endif
+
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
+try:
+	from urllib import pathname2url
+except:
+	from urllib.request import pathname2url
+
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
 
-requirements.tox:
-	pip3 install -q -r requirements/tox.txt
+.PHONY: requirements coverage clean docs
 
-requirements.py:
-	pip3 install -q -r requirements/base.txt --exists-action w
+# Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage.
+help: ## display this help message
+	@echo "Please use \`make <target>\` where <target> is one of"
+	@perl -nle'print $& if m{^[\.a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
+
+requirements: requirements.py requirements.js
+
+requirements.py: piptools
+	pip-sync -q requirements/base.txt
 
 requirements.js:
 	npm install --unsafe-perm
 
-test.requirements:
-	pip3 install -q -r requirements/test.txt --exists-action w
+test.requirements: piptools
+	pip-sync -q requirements/test.txt
 
-develop: requirements.js
-	pip3 install -q -r requirements/local.txt --exists-action w
+develop: piptools requirements.js
+	pip-sync -q requirements/local.txt
 
-migrate: requirements.tox
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-migrate
+migrate: ## apply database migrations
+	$(TOX)python manage.py migrate  --run-syncdb
 
-clean: requirements.tox
-	find . -name '*.pyc' -delete
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-clean
+run-local: ## Run local (non-devstack) development server on port 8000
+	python manage.py runserver 0.0.0.0:8110 --settings=analytics_dashboard.settings.local
+
+dbshell-local: ## Run local (non-devstack) database shell
+	python manage.py dbshell --settings=analytics_dashboard.settings.local
 
-test_python_no_compress: clean
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-tests
+shell: ## Run Python shell
+	python manage.py shell
 
-coverage: requirements.tox
+clean: ## delete generated byte code and coverage reports
+	find . -name '*.pyc' -delete
+	find . -name '__pycache__' -type d -exec rm -rf {} ';' || true
+	coverage erase
+	rm -rf assets
+	rm -rf pii_report
+
+coverage: clean
 	export COVERAGE_DIR=$(COVERAGE_DIR) && \
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-coverage
+	$(TOX)pytest common analytics_dashboard --cov common --cov analytics_dashboard --cov-report html --cov-report xml
 
-test_compress: static
-	# No longer does anything. Kept for legacy support.
+coverage_html: coverage ## run and view HTML coverage report
+	$(BROWSER) build/coverage/html/index.html
 
-test_python: requirements.tox test_compress test_python_no_compress
+test_python: clean ## run pyton tests and generate coverage report
+	$(TOX)pytest common analytics_dashboard --cov common --cov analytics_dashboard
 
 requirements.a11y:
 	./.travis/a11y_reqs.sh
 
-runserver_a11y: requirements.tox
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-runserver_a11y > dashboard.log 2>&1 &
+runserver_a11y:
+	$(TOX)python manage.py runserver 0.0.0.0:9000 --noreload --traceback > dashboard.log 2>&1 &
 
 accept: runserver_a11y
 ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True")
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-waffle_learner_analytics
+	$(TOX)python manage.py waffle_flag enable_learner_analytics --create --everyone
 endif
 ifeq ("${ENABLE_COURSE_LIST_FILTERS}", "True")
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-waffle_course_filters
+	$(TOX)python manage.py waffle_switch enable_course_filters on --create
 endif
 ifeq ("${ENABLE_COURSE_LIST_PASSING}", "True")
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-waffle_course_passing
+	$(TOX)python ./manage.py waffle_switch enable_course_passing on --create
 endif
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-create_acceptance_test_soapbox_messages
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-accept
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-delete_acceptance_test_soapbox_messages
-
+	$(TOX)python manage.py create_acceptance_test_soapbox_messages
+	$(TOX)pytest -v acceptance_tests --ignore=acceptance_tests/course_validation
+	$(TOX)python manage.py delete_acceptance_test_soapbox_messages
 
 # local acceptance tests are typically run with by passing in environment variables on the commandline
 # e.g. API_SERVER_URL="http://localhost:9001/api/v0" API_AUTH_TOKEN="edx" make accept_local
@@ -74,31 +98,39 @@ accept_local:
 	pytest -v acceptance_tests --ignore=acceptance_tests/course_validation
 	./manage.py delete_acceptance_test_soapbox_messages
 
-a11y: requirements.tox
+accept_devstack:
+	# TODO: implement this
+
+a11y:
 ifeq ("${DISPLAY_LEARNER_ANALYTICS}", "True")
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-waffle_learner_analytics
+	$(TOX)python manage.py waffle_flag enable_learner_analytics --create --everyone
 endif
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-a11y
+	cat dashboard.log
+	$(TOX)pytest -v a11y_tests -k 'not NUM_PROCESSES==1' --ignore=acceptance_tests/course_validation
 
 course_validation:
 	python -m acceptance_tests.course_validation.generate_report
 
-run_check_isort: requirements.tox
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-check_isort
+isort_check: ## check that isort has been run
+	$(TOX)isort --check-only --recursive --diff acceptance_tests/ analytics_dashboard/ common/
+
+isort: ## run isort to sort imports in all Python files
+	$(TOX)isort --recursive --diff acceptance_tests/ analytics_dashboard/ common/
 
-run_pycodestyle: requirements.tox
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-pycodestyle
+pycodestyle:  # run pycodestyle
+	$(TOX)pycodestyle acceptance_tests analytics_dashboard common
 
-run_pylint: requirements.tox
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-pylint
+pylint:  # run pylint
+	$(TOX)pylint -j 0 --rcfile=pylintrc acceptance_tests analytics_dashboard common
 
-quality: run_pylint run_pycodestyle
+# TODO: fix imports so this can run isort_check
+quality: pycodestyle pylint ## run all code quality checks
 
 validate_python: test_python quality
 
 #FIXME validate_js: requirements.js
 validate_js:
-	$(NODE_BIN)/gulp test
+	npm run test
 	npm run lint -s
 
 validate: validate_python validate_js
@@ -108,50 +140,50 @@ demo:
 	python manage.py waffle_switch enable_course_api off --create
 	python manage.py waffle_switch display_course_name_in_nav off --create
 
-# compiles djangojs and django .po and .mo files
-compile_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-compile_translations
+compile_translations: # compiles djangojs and django .po and .mo files
+	$(TOX)python manage.py compilemessages
 
-# creates the source django & djangojs files
-extract_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-extract_translations
+extract_translations: ## extract strings to be translated, outputting .mo files
+	$(TOX)python manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d django
+	$(TOX)python manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d djangojs
 
-dummy_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-dummy_translations
+dummy_translations: ## generate dummy translation (.po) files
+	cd analytics_dashboard && i18n_tool dummy
 
-generate_fake_translations: extract_translations dummy_translations compile_translations
+generate_fake_translations: extract_translations dummy_translations compile_translations ## generate and compile dummy translation files
 
-pull_translations:
+pull_translations: ## pull translations from Transifex
 	cd analytics_dashboard && tx pull -af
 
 update_translations: pull_translations generate_fake_translations
 
-# check if translation files are up-to-date
-detect_changed_source_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-detect_changed_translations
+detect_changed_source_translations: ## check if translation files are up-to-date
+	cd analytics_dashboard && i18n_tool changed
 
 # extract, compile, and check if translation files are up-to-date
 validate_translations: extract_translations compile_translations detect_changed_source_translations
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-validate_translations
+	cd analytics_dashboard && i18n_tool validate -
 
-static_no_compress: static
-	# No longer does anything. Kept for legacy support.
+static: ## generate static files
+	npm run build
+	$(TOX)python manage.py collectstatic --noinput
 
-static: requirements.tox
-	$(NODE_BIN)/webpack --config webpack.prod.config.js
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-static
+pii_check: ## check for PII annotations on all Django models
+	## Not yet implemented
 
-export CUSTOM_COMPILE_COMMAND = make upgrade
-upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+piptools:
 	pip3 install -q -r requirements/pip_tools.txt
+
+export CUSTOM_COMPILE_COMMAND = make upgrade
+upgrade: piptools ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
 	pip-compile --upgrade -o requirements/pip_tools.txt requirements/pip_tools.in
 	pip-compile --upgrade -o requirements/base.txt requirements/base.in
 	pip-compile --upgrade -o requirements/doc.txt requirements/doc.in
 	pip-compile --upgrade -o requirements/test.txt requirements/test.in
+	pip-compile --upgrade -o requirements/tox.txt requirements/tox.in
 	pip-compile --upgrade -o requirements/local.txt requirements/local.in
 	pip-compile --upgrade -o requirements/optional.txt requirements/optional.in
 	pip-compile --upgrade -o requirements/production.txt requirements/production.in
-	pip-compile --upgrade -o requirements/tox.txt requirements/tox.in
 	pip-compile --upgrade -o requirements/travis.txt requirements/travis.in
 	# Let tox control the Django version for tests
 	grep -e "^django==" requirements/base.txt > requirements/django.txt

diff --git a/README.md b/README.md
@@ -6,11 +6,10 @@ Dashboard to display course analytics to course teams
 
 Prerequisites
 -------------
-* Python 3.5.x
+* Python 3.8.x
 * [gettext](http://www.gnu.org/software/gettext/)
 * [node](https://nodejs.org) 12.11.1
 * [npm](https://www.npmjs.org/) 6.11.3
-* [JDK 7+](http://openjdk.java.net/)
 
 Warning: You must have NPM version 5.5.1. Using another version might result in
 a different `package-lock.json` file. Committing those changes might break our
@@ -19,12 +18,19 @@ install`. [nodeenv](https://github.com/ekalinin/nodeenv) or
 [n](https://github.com/tj/n) are tools that you can use to work on different
 Node.js and NPM versions than your system installed versions.
 
-We have a TODO to set-up the development environment with a Docker container so
+It's recommended you set up this service with devstack so
 that you will not have to manage Node and NPM versions yourself.
 
-Getting Started
----------------
+Getting Started With Devstack
+-----------------------------
+The best way to run this service is with edX Devstack: https://github.com/edx/devstack.
+
+See the [Devstack README](https://github.com/edx/devstack/blob/master/README.rst) for information on how to install and run Insights. For the purposes of devstack this service will be referred to as `insights` and not `analytics-dashboard`.
+
+Getting Started Standalone
+--------------------------
 1. Get the code (e.g. clone the repository).
+2. Create a Python 3 virtual environment and activate it
 2. Install the Python/Node requirements:
 
         $ make develop
@@ -102,6 +108,8 @@ The following flags are available:
 
 Authentication & Authorization
 ------------------------------
+This section is only necessary if running I stand alone service OAuth2 is automatically configured by provisioning in devstack.
+
 By default, this application relies on an external OAuth2 provider
 (contained within the [LMS](https://github.com/edx/edx-platform)) for authentication and authorization. If you are a
 developer, and do not want to setup edx-platform, you can get around this requirement by doing the following:

diff --git a/analytics_dashboard/settings/dev.py b/analytics_dashboard/settings/dev.py
@@ -54,9 +54,6 @@
 ########## END BRANDING
 
 
-########## AUTHENTICATION/AUTHORIZATION
-ENABLE_AUTO_AUTH = True
-
 # Uncomment the line below to avoid having to worry about course permissions
 ENABLE_COURSE_PERMISSIONS = False
 ########## END AUTHENTICATION/AUTHORIZATION
@@ -70,13 +67,8 @@
 SEGMENT_IO_KEY = os.environ.get('SEGMENT_WRITE_KEY')
 ########## END SEGMENT.IO
 
-GRADING_POLICY_API_URL = 'http://127.0.0.1:18000/api/grades/v1/'
-COURSE_API_URL = 'http://127.0.0.1:18000/api/courses/v1/'
-
 LOGGING = get_logger_config(debug=DEBUG, dev_env=True, local_loglevel='DEBUG')
 
-########## MODULE_PREVIEW
-MODULE_PREVIEW_URL = 'http://127.0.0.1:18000/xblock'
 ########## END MODULE_PREVIEW
 
 ########## REST FRAMEWORK CONFIGURATION