Replace require.js with webpack, remove bower (#701)

* Replace bower with npm. Really slow r.js build

* WIP replace r.js with webpack

Successfully replaced r.js with webpack and have gotten a local instance of
Insights working with webpack-built js and css bundles.

There are many issues remaining. Just to name a few:

* The size of the bundles are very large compared to r.js bundles
* Libraries are repeatedly included in many bundles
* The speed of building bundles, while faster than r.js, is still slow
* A separate prod and dev configs are needed
* A proper webpack dev environment still needs to be set up (e.g. hot module
  reloading)

* reference correct location for pattern library npm packages

* add bourbon npm package and link to it

* globalization functionality is split into its own chunk

* Optimized bundle size

* Set-up webpack-dev-server

* Remove vendor folder. fast-sass-loader.

* Use fast source-map method for development.

* Split dev/prod webpack config. Lots of comments!

* No need for "vendor" bundle, just use "manifest".

* Git ignore webpack-stats.json

* Common chunks vendor bundle. Fix sassc.

* Prettier the webpack configs

* Run prettier-eslint instead

* Passing karma tests with webpack

* Fix linting error

* Fix bootstrap dropdowns

* Remove legacy bower, requirejs, django-compressor

* Fix the travis build

* Precompile the pattern-library to .css file

* Fix post-install font copy paths

* Update translations

* Missed a few legacy rjs related lines

* Linting: stray import in urls.py

* Dropdowns in application-main. Compile to .scss

* Use sass-loader. Update README.

* Require full bootstrap to try fix acceptance tests

* Load webpack in base.html so landing works on dev

* Output webpack bundles to /static not /assets

* Update translations

* Add compression-webpack-plugin

TBD whether django can actually figure out how to find the .gz files, but until
then here is the best webpack prod config I could come up with.

* Add babel-loader. Convert two js files to es2015

* Fix eslint errors: separate es6 eslint config

* Fix packages for NPM 5. Add lock file.

* Ignore /static/bundles in extract_translations

* Try adding q to package.json?

* Remove extraneous merge header

* Remove cldr-data-downloader & q from package.json

* Move webpack packages from dev to dependencies

* Use backbone 1.2.3. Reintroduce router execute.

* Remove package-lock.json until I figure it out

* Remove webpack-compression-plugin (using CDN now)

* Use webpack-merge for config inheritance

* Use chunkhash in filename in production

* Create source-maps in uglify on prod

* Fix eslint & errors. Add prettier. More ES2015.

* Address PR comments in _font.scss

* Address PR comments in npm-post-install.sh

* Make DJANGO_DEV_SERVER webpack proxy configurable

.es6-eslintrc.json

@@ -0,0 +1,27 @@
+{
+    "extends": "eslint-config-edx",
+    "parser": "babel-eslint",
+    "parserOptions": {
+        "sourceType": "module",
+        "allowImportExportEverywhere": true,
+        "plugins": ["syntax-dynamic-import"]
+    },
+    "plugins": [
+        "import"
+    ],
+    "globals": {
+        "gettext": true, // added by django for i18n
+        "setFixtures": true,  // added globally by jasmine-jquery
+        "sinon": true, // used for mocking
+        "requirejs": true
+    },
+    "settings": {
+        "import/resolver": "webpack"
+    },
+    "rules": {
+        "import/no-unresolved": ["error", {"amd": true}],
+        "global-require" : "off"
+        // global-require turned off due to https://github.com/eslint/eslint/issues/4812. Fixed in v3.1.1
+        // consider turning rule on after upgrading to ESLint 3.1.1
+    }
+}

.eslintignore

@@ -3,3 +3,4 @@ node_modules/
 analytics_dashboard/static/bower_components/
 analytics_dashboard/static/dist/
 analytics_dashboard/static/vendor/
+analytics_dashboard/static/bundles/

.eslintrc.json

@@ -1,30 +1,16 @@
 {
-    "extends": "eslint-config-edx",
+    "extends": "eslint-config-edx-es5",
+    "plugins": [
+        "import"
+    ],
     "globals": {
         "gettext": true, // added by django for i18n
         "setFixtures": true,  // added globally by jasmine-jquery
         "sinon": true, // used for mocking
         "requirejs": true
     },
     "settings": {
-        "import/resolver": {
-            "node": {
-                "extensions": [".js", ".json"],
-                "moduleDirectory": ["node_modules", "analytics_dashboard/static",
-                                    "analytics_dashboard/static/js"]
-            }
-        },
-        "import/core-modules": ["jquery", "underscore", "backbone", "backbone.paginator", "backbone.wreqr",
-                                "backbone.babysitter", "backgrid", "backgrid-filter", "backgrid-paginator", "bootstrap",
-                                "bootstrap_accessibility", "models", "collections", "views", "utils", "load",
-                                "datatables", "dataTablesBootstrap", "naturalSort", "d3", "nvd3", "topojson",
-                                "datamaps", "moment", "text", "json", "cldr", "cldr-data", "globalize", "globalization",
-                                "marionette", "uitk", "URI", "IPv6", "punycode", "SecondLevelDomains", "learners",
-                                "axe-core", "sinon", "nprogress",
-                                "vendor/domReady!", "globalize/number", "boot",
-                                "uitk/disclosure/disclosure-view",
-                                "json!cldr-data/supplemental/likelySubtags.json",
-                                "json!cldr-data/supplemental/numberingSystems.json"]
+        "import/resolver": "webpack"
     },
     "rules": {
         "import/no-unresolved": ["error", {"amd": true}],

.gitignore

@@ -43,6 +43,7 @@ assets/
 node_modules/
 analytics_dashboard/static/bower_components/
 analytics_dashboard/static/dist/
+analytics_dashboard/static/bundles/
 .coverage
 nosetests.xml
 reports/
@@ -63,3 +64,6 @@ conf/locale/en/LC_MESSAGES/*.mo
 conf/locale/fake*/LC_MESSAGES/*.po
 conf/locale/fake*/LC_MESSAGES/*.mo
 conf/locale/messages.mo
+
+# Webpack
+webpack-stats.json

.travis/run_tests.sh

@@ -15,10 +15,6 @@ ln -s /opt/firefox/firefox /usr/bin/firefox
 cd /edx/app/insights/edx_analytics_dashboard
 export PATH=$PATH:$PWD/node_modules/.bin
 
-# Make it so bower can run without sudo.
-# https://github.com/GeoNode/geonode/pull/1070
-echo '{ "allow_root": true }' > /root/.bowerrc
-
 # Output node.js version
 node --version
 npm --version

Makefile

@@ -14,8 +14,7 @@ requirements.py:
 	pip install -q -r requirements/base.txt --exists-action w
 
 requirements.js:
-	npm install
-	$(NODE_BIN)/bower install
+	npm install --unsafe-perm
 
 test.requirements: requirements
 	pip install -q -r requirements/test.txt --exists-action w
@@ -35,8 +34,8 @@ test_python_no_compress: clean
 	--cover-package=analytics_dashboard --cover-package=common --cover-branches --cover-html --cover-html-dir=$(COVERAGE)/html/ \
 	--with-ignore-docstrings --cover-xml --cover-xml-file=$(COVERAGE)/coverage.xml
 
-test_compress: static_no_compress
-	python manage.py compress --settings=analytics_dashboard.settings.test
+test_compress: static
+	# No longer does anything. Kept for legacy support.
 
 test_python: test_compress test_python_no_compress
 
@@ -79,7 +78,7 @@ validate_python: test.requirements test_python quality
 #FIXME validate_js: requirements.js
 validate_js:
 	$(NODE_BIN)/gulp test
-	$(NODE_BIN)/gulp lint
+	npm run lint -s
 
 validate: validate_python validate_js
 
@@ -94,8 +93,8 @@ compile_translations:
 
 # creates the source django & djangojs files
 extract_translations:
-	cd analytics_dashboard && python ../manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" -d django
-	cd analytics_dashboard && python ../manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" -d djangojs
+	cd analytics_dashboard && python ../manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d django
+	cd analytics_dashboard && python ../manage.py makemessages -l en -v1 --ignore="docs/*" --ignore="src/*" --ignore="i18n/*" --ignore="assets/*" --ignore="static/bundles/*" -d djangojs
 
 dummy_translations:
 	cd analytics_dashboard && i18n_tool dummy -v
@@ -115,10 +114,11 @@ detect_changed_source_translations:
 validate_translations: extract_translations compile_translations detect_changed_source_translations
 	cd analytics_dashboard && i18n_tool validate
 
-static_no_compress:
-	$(NODE_BIN)/r.js -o build.js
-	# collectstatic creates way too much output so silence it with verbosity=0
-	python manage.py collectstatic --noinput -v 0
+static_no_compress: static
+	# No longer does anything. Kept for legacy support.
 
-static: static_no_compress
-	python manage.py compress
+static:
+	$(NODE_BIN)/webpack --config webpack.prod.config.js
+	# collectstatic creates way too much output with the cldr-data directory output so silence that directory
+	echo "Running collectstatic while silencing cldr-data/main/* ..."
+	python manage.py collectstatic --noinput | sed -n '/.*node_modules\/cldr-data\/main\/.*/!p'

README.md

@@ -15,24 +15,36 @@ Prerequisites
 Getting Started
 ---------------
 1. Get the code (e.g. clone the repository).
-2. Install the Python/Node/Bower requirements:
+2. Install the Python/Node requirements:
 
         $ make develop
 
 3. Setup your database:
 
         $ make migrate
 
-4. Run the server:
+4. Run the webpack-dev-server:
 
-        $ ./manage.py runserver 0.0.0.0:9000
+        $ npm start
+
+If you plan on running the Django development server on a different port or
+host, make sure to set the `DJANGO_DEV_SERVER` environmental variable. For
+example:
+
+        $ DJANGO_DEV_SERVER='http://localhost:9000' npm start
+
+5. In a separate terminal run the Django development server:
+
+        $ ./manage.py runserver 0.0.0.0:8110
 
 By default the Django Default Toolbar is disabled. To enable it set the environmental variable ENABLE_DJANGO_TOOLBAR.
 
 Alternatively, you can launch the server using:
 
         $ ENABLE_DJANGO_TOOLBAR=1 ./manage.py runserver
 
+Visit http://localhost:9000 in your browser and then login through the LMS to
+access Insights (see **Authentication & Authorization** below for more details).
 
 Site-Wide Announcements
 -----------------------
@@ -139,16 +151,34 @@ Note that only the following files (for each language) should be committed to th
 
 Asset Pipeline
 --------------
-Static files are managed via [django-compressor](http://django-compressor.readthedocs.org/) and [RequireJS](http://requirejs.org/).
-RequireJS (and r.js) are used to manage JavaScript dependencies. django-compressor compiles SASS, minifies JavaScript (
-using [Closure Compiler](https://developers.google.com/closure/compiler/)), and handles naming files to facilitate
-cache busting during deployment.
+Static files are managed via [webpack](https://webpack.js.org/).
+
+To run the webpack-dev-server, which will watch for changes to static files
+(`.js`, `.css`, `.sass`, `.underscore`, etc. files) and incrementally recompile
+webpack bundles and try to hot-reload them in your browser, run this in a
+terminal:
+
+    $ npm start
+
+Alternatively, you can compile production webpack bundles by running (runs
+webpack using the prod config and then exits):
+
+    $ make static
+
+Before committing new JavaScript, make sure it conforms to our style guide by
+running [eslint](http://eslint.org/), and fixing any errors.
+
+    $ npm run lint -s
+
+You can also try automatically fixing the errors and applying an additional
+level of standardized formatting with
+[prettier](https://github.com/prettier/prettier) by running
+[prettier-eslint](https://github.com/prettier/prettier-eslint).
 
-Both tools should operate seamlessly in a local development environment. When deploying to production, call
-`make static` to compile all static assets and move them to the proper location to be served.
+    $ npm run format
 
-When creating new pages that utilize RequireJS dependencies, remember to use the `static_rjs` templatetag to load
-the script, and to add a new module to `build.js`.
+Note: this will only format a subset of the JavaScript, we haven't converted the
+formatting of all of our files yet. Edit the directory list in `package.json`.
 
 Theming and Branding
 --------------------