diff --git a/.eslintignore b/.eslintignore index 15767f3..db449e6 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,7 +1,12 @@ -addon/StudyUtils.jsm -# for circleCI; don't eslint code in the Firefox directory -firefox +# do not lint/format generated artifacts dist -OLD package-lock.json +# makes sure that eslintrc.js gets linted/formatted !.eslintrc.js +# do not lint/format bundled util libraries (PioneerUtils.jsm is included for Pioneer Shield studies only; eventually it will be merged with StudyUtils.jsm) +addon/StudyUtils.jsm +addon/PioneerUtils.jsm +# for circleCI; don't lint/format code in the Firefox directory +firefox +# don't lint/format package.json since npm install formats it differently by default +package.json diff --git a/.eslintrc.js b/.eslintrc.js index e9f672f..9006c54 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -7,18 +7,17 @@ */ module.exports = { - "parserOptions": { - "ecmaVersion": 8, - "sourceType": "module", - "ecmaFeatures": { - "jsx": false, - "experimentalObjectRestSpread": true, + parserOptions: { + ecmaVersion: 8, + sourceType: "module", + ecmaFeatures: { + jsx: false, + experimentalObjectRestSpread: true, }, }, env: { - "es6": true, + es6: true, // 'browser-window': false - }, extends: [ "eslint:recommended", @@ -28,23 +27,20 @@ module.exports = { "plugin:mozilla/recommended", ], - plugins: [ - "json", - "mozilla", - ], + plugins: ["json", "mozilla"], rules: { "babel/new-cap": "off", "comma-dangle": ["error", "always-multiline"], - "eqeqeq": "error", - "indent": ["warn", 2, {SwitchCase: 1}], + eqeqeq: "error", + indent: ["warn", 2, { SwitchCase: 1 }], "mozilla/no-aArgs": "warn", - "mozilla/balanced-listeners": 0, + "mozilla/balanced-listeners": "off", "no-console": "warn", - "no-shadow": ["error"], + "no-shadow": "error", "no-unused-vars": "error", "prefer-const": "warn", "prefer-spread": "error", - "semi": ["error", "always"], + semi: ["error", "always"], }, }; diff --git a/.gitignore b/.gitignore index 0c48090..de66215 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ install.rdf *.xpi node_modules addon/StudyUtils.jsm +addon/PioneerUtils.jsm diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a612ad9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/OLD/.addonlinterrc b/OLD/.addonlinterrc deleted file mode 100644 index 45981d8..0000000 --- a/OLD/.addonlinterrc +++ /dev/null @@ -1,3 +0,0 @@ -ignorerules: - LOW_LEVEL_MODULE: true - KNOWN_LIBRARY: true diff --git a/OLD/.gitignore b/OLD/.gitignore deleted file mode 100644 index 2ff8917..0000000 --- a/OLD/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -*.log -*.update.rdf -*.xpi -.DS_Store -coverage/ -deprecated/* -npm-shrinkwrap.json -node_modules/* -test/test-z-ensure-coverage.js -testing-env diff --git a/OLD/README.md b/OLD/README.md deleted file mode 100644 index b623ea4..0000000 --- a/OLD/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Base Template for Shield Studies Addons - -## Features - -- `eslint` - - - es6 for lib, data, test - - browser, not node for `data/` - -- `addons-linter` with `.addonslinterrc` - -- ci with Travis OR CircleCi (TODO) - -- ability to do code coverage, using `grunt-istanbul` and [`istanbul-jpm`](https://github.com/freaktechnik/istanbul-jpm) - -- uses Grunt to do some of the heavy lifting. Sorry if you hate Grunt. [I do as well](#1). - -- TODO: Allow better build of React type things for front ends - -## General Setup and Install - -1. Clone / copy the directory -2. `npm install` - -## Adding a new npm library - -``` -npm install --save-dev somelibrary -#edit .jpmignore to allow it in -``` - -## Contribute - -Issues on this Github :) - -## Assumptions and Opinions - -1. All code lives in `lib` and is ES6. -2. All website stuff (web-workers, ui) lives in `data` -3. Index at `lib/index.js` -4. Grunt, b/c it makes instrument / coverage easier - - - `grunt-istanbul` + `istanbul-jpm` - - if you want or need `make`, `gulp`, `webpack` you absolutely can - -5. All the testing happens in a create `testing-env` folder, so that - - - it can use a custom `.jpmignore` file - - it can do coverage with less silliness - -6. As built, the tests will fail, until you fix the facade tests. -7. We use `chai` for testing. If you don't, remove it where it happens. -8. You are somewhere with some resemblence to Unix cli (Linux or OSX). - - diff --git a/OLD/package.json b/OLD/package.json deleted file mode 100644 index 20a2b1a..0000000 --- a/OLD/package.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "shield-studies-addon-template", - "description": "A basic add-on", - "version": "0.1.1", - "author": "Gregg Lind ", - "bugs": { - "url": "https://github.com/mozilla/shield-studies-addon-template/issues" - }, - "dependencies": {}, - "devDependencies": { - "addons-linter": "^0.15.5", - "chai": "^3.5.0", - "depcheck-ci": "^1.0.1", - "eslint": "^3.6.1", - "fixpack": "^2.3.1", - "grunt": "^1.0.1", - "grunt-cli": "^1.2.0", - "grunt-istanbul": "^0.7.0", - "grunt-shell": "^1.3.0", - "istanbul-jpm": "^0.1.0", - "jpm": "^1.0.7", - "npm-run-all": "^3.1.0", - "nsp": "^2.6.2", - "shield-studies-addon-utils": "^2.0.0", - "yamljs": "^0.2.8" - }, - "engines": { - "firefox": ">=38.0a1", - "fennec": ">=38.0a1" - }, - "homepage": "http://github.com/mozilla/shield-studies-addon-template", - "keywords": [ - "jetpack", - "shield-study" - ], - "license": "MIT", - "main": "lib/index.js", - "repository": { - "type": "git", - "url": "git://github.com/mozilla/shield-studies-addon-template.git" - }, - "scripts": { - "eslint": "grunt eslint", - "lint": "npm-run-all lint:*", - "lint:addons-linter": "# `addons-linter` will be caught during `test` # grunt shell:addonLintTest", - "lint:depcheck": "depcheck-ci # use coverage to catch missing", - "lint:eslint": "eslint .", - "lint:fixpack": "fixpack", - "lint:nsp": "nsp check", - "prepublish": "npm shrinkwrap", - "pretest": "npm-run-all lint:*", - "test": "grunt test && istanbul check-coverage --statements 100 --functions 100 --branches 100 --lines 100 coverage/reports/coverage.json" - }, - "title": "Template for creating shield study add-ons" -} diff --git a/OLD/scripts/addon-lint-consumer.js b/OLD/scripts/addon-lint-consumer.js deleted file mode 100755 index 1ed3c88..0000000 --- a/OLD/scripts/addon-lint-consumer.js +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env node - -/* - usage: - - ``` - jpm xpi # makes myaddon.xpi - npm install addon-linter - ./node_modules/.bin/addon-linter myaddon.xpi | node addon-lint-consumer.js - ``` - - license: PUBLIC DOMAIN. - - -//example .addonlinterrc -ignorerules: - LOW_LEVEL_MODULE: true - KNOWN_LIBRARY: true - -*/ - -var yamljs = require('yamljs'); - -function loadRules (fn) { - var ignored = {}; - try { - ignored = (yamljs.load(fn)).ignorerules; - } catch (err) { - // ignore - } - return ignored; -} - -function filterLint(lint, ignored) { - ['errors', 'notices', 'warnings'].map(function (k) { - var filtered = lint[k].filter(function (seen) { - return !(seen.code in ignored); - }); - lint[k] = filtered; - }); - return lint; -} - -function output(filteredLint) { - var show = 0; - ['errors', 'notices', 'warnings'].map(function (k) { - if (filteredLint[k].length) { - show = 1; - } - }); - if (show) { - console.error(filteredLint); - } - process.exit(show); -} - -function doTheWork(content) { - // your code here - var ignored = loadRules('.addonlinterrc'); - output(filterLint(JSON.parse(content),ignored)); -} - -// read in all the stdin -var content = ''; -process.stdin.resume(); -process.stdin.setEncoding('utf8'); -process.stdin.on('data', function (buf) { - content += buf.toString(); -}); -process.stdin.on('end', function () { - // your code here - doTheWork(content); -}); diff --git a/OLD/scripts/addonLintTest b/OLD/scripts/addonLintTest deleted file mode 100755 index cae1740..0000000 --- a/OLD/scripts/addonLintTest +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -o nounset -set -o errexit - -# $1 will be the expected name for the xpi -node_modules/.bin/jpm xpi -node_modules/.bin/addons-linter --output json --pretty "$1".xpi |\ - node scripts/addon-lint-consumer.js - -echo "OK" $0 diff --git a/OLD/scripts/ensure-files-are-covered b/OLD/scripts/ensure-files-are-covered deleted file mode 100755 index 09c5c46..0000000 --- a/OLD/scripts/ensure-files-are-covered +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -o nounset -set -o errexit - -cat << DONE > test/test-z-ensure-coverage.js -// automatically created by makeCoverageTest -DONE - -git ls-tree -r HEAD --name-only lib | \ -grep "js$" | \ -xargs -I '{}' echo "require('../{}');" | \ -egrep -v "(jetpack|index.js|main.js)" >> test/test-z-ensure-coverage.js - -echo "OK" $0 diff --git a/OLD/scripts/makeTestEnv b/OLD/scripts/makeTestEnv deleted file mode 100755 index 3b9b7cc..0000000 --- a/OLD/scripts/makeTestEnv +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -o nounset -set -o errexit - -rm -rf testing-env -mkdir testing-env -cd testing-env -cat ../.jpmignore ../.jpmignore-testing-env > .jpmignore -ln -s ../Gruntfile.js . -ln -s ../node_modules . -ln -s ../data . -ln -s ../coverage/instrument/lib . -ln -s ../package.json . -ln -s ../test . -echo "OK" $0 diff --git a/OLD/scripts/shield-studies-linting-questions.js b/OLD/scripts/shield-studies-linting-questions.js deleted file mode 100755 index 4b74267..0000000 --- a/OLD/scripts/shield-studies-linting-questions.js +++ /dev/null @@ -1,6 +0,0 @@ -// all the usual gregg questions - -// grep for surveyUrl -// where is the variations? - -// livecheck all the SG urls diff --git a/OLD/test-share-study.js b/OLD/test-share-study.js deleted file mode 100644 index bb7258e..0000000 --- a/OLD/test-share-study.js +++ /dev/null @@ -1,627 +0,0 @@ - -async function animationTest(driver, url) { - await utils.addShareButton(driver); - await utils.gotoURL(driver, url); - await utils.copyUrlBar(driver); - await utils.waitForClassAdded(driver); - const { hasClass, hasColor } = await utils.testAnimation(driver); - return hasClass && hasColor; -} - -async function popupTest(driver, url) { - await utils.gotoURL(driver, url); - await utils.copyUrlBar(driver); - const panelOpened = await utils.testPanel(driver, "share-button-panel"); - return panelOpened; -} - -async function overflowMenuTest(driver, test, url) { - const window = driver.manage().window(); - const currentSize = await window.getSize(); - await window.setSize(640, 480); - - const overflowButton = driver.wait(until.elementLocated( - By.id("nav-bar-overflow-button")), 1000); - await overflowButton.click(); - - await utils.copyUrlBar(driver); - - assert(!(await test(driver, url))); - await window.setSize(currentSize.width, currentSize.height); -} - -async function setTreatment(driver, treatment) { - return driver.executeAsyncScript((treatmentArg, callback) => { - Components.utils.import("resource://gre/modules/Preferences.jsm"); - Preferences.set("extensions.sharebuttonstudy.treatment", treatmentArg); - callback(); - }, treatment); -} - -async function summaryFieldTest(driver, addonId, treatment) { - await utils.uninstallAddon(driver, addonId); - // hack workaround to wait for uninstall to really happen? - await new Promise(resolve => setTimeout(resolve, 1000)); - await setTreatment(driver, treatment); - // install the addon - await utils.installAddon(driver); - - if (["highlight", "doorhangerDoNothing"].includes(treatment)) { - await utils.addShareButton(driver); - } - - await utils.gotoURL(driver, MOZILLA_ORG); - await utils.copyUrlBar(driver); - await utils.waitForAnimationEnd(driver); - - await utils.uninstallAddon(driver, addonId); - // hacky workaround to wait until the summary ping is sent - await new Promise(resolve => setTimeout(resolve, 1000)); - const pings = await utils.getMostRecentPingsByType(driver, "shield-study-addon"); - const foundPings = utils.searchTelemetry( - [ping => Object.hasOwnProperty.call(ping.payload.data.attributes, "summary")], - pings); - assert(foundPings.length > 0); - - const summaryPings = JSON.parse(foundPings[0].payload.data.attributes.summary); - // Event pings do not use the treatment name to avoid confusion between - // showing the doorhanger vs. showing ask-to-add panel etc. (ie. if the share button - // is already in the toolbar) - const treatmentToEventName = { - highlight: "highlight", - doorhangerDoNothing: "doorhanger", - doorhangerAskToAdd: "ask-to-add", - doorhangerAddToToolbar: "add-to-toolbar", - }; - const events = [{ event: "copy" }, { treatment: treatmentToEventName[treatment] }]; - // add to toolbar will additionally trigger the doorhanger treatment, since it will - // add the button to the toolbar every time - if (treatment === "doorhangerAddToToolbar") { - events.push({ treatment: "doorhanger" }); - } - - assert(summaryPings.length === events.length); - for (let i = 0; i < events.length; i++) { - delete summaryPings[i].timestamp; - delete summaryPings[i].id; - assert(events[i][Object.keys(events[i])[0]] - === summaryPings[i][Object.keys(summaryPings[i])[0]]); - } -} - - - -describe("Basic Functional Tests", function() { - // This gives Firefox time to start, and us a bit longer during some of the tests. - this.timeout(15000); - - let driver; - let addonId; - - before(async() => { - driver = await utils.promiseSetupDriver(); - await setTreatment(driver, "doorHangerAddToToolbar"); - // install the addon - addonId = await utils.installAddon(driver); - // add the share-button to the toolbar - await utils.addShareButton(driver); - }); - - after(async() => driver.quit()); - - afterEach(async() => postTestReset(driver)); - - it("should have a URL bar", async() => { - const urlBar = await utils.promiseUrlBar(driver); - const text = await urlBar.getAttribute("placeholder"); - assert.equal(text, "Search or enter address"); - }); - - it("should have a share button", async() => { - const button = await utils.promiseAddonButton(driver); - const text = await button.getAttribute("tooltiptext"); - assert.equal(text, "Share this page"); - }); - - it("should have copy paste working", async() => { - // FIXME testText will automatically be treated as a URL - // which means that it will be formatted and the clipboard - // value will be different unless we pass in a URL text at - // the start - const testText = "about:test"; - - // write dummy value just in case testText is already in clipboard - await clipboardy.write("foobar"); - const urlBar = await utils.promiseUrlBar(driver); - await urlBar.sendKeys(testText); - - await utils.copyUrlBar(driver); - const clipboard = await clipboardy.read(); - assert(clipboard === testText); - }); - - it(`should only trigger MAX_TIMES_TO_SHOW = ${MAX_TIMES_TO_SHOW} times`, async() => { - // NOTE: if this test fails, make sure MAX_TIMES_TO_SHOW has the correct value. - - await utils.gotoURL(driver, MOZILLA_ORG); - for (let i = 0; i < MAX_TIMES_TO_SHOW; i++) { - /* eslint-disable no-await-in-loop */ - await utils.copyUrlBar(driver); - // wait for the animation to end - await utils.waitForAnimationEnd(driver); - // close the popup - await utils.closePanel(driver); - /* eslint-enable no-await-in-loop */ - } - // try to open the panel again, this should fail - await utils.copyUrlBar(driver); - const panelOpened = await utils.testPanel(driver); - const { hasClass, hasColor } = await utils.testAnimation(driver); - - assert(!panelOpened && !hasClass && !hasColor); - }); - - // These tests uninstall the addon before and install the addon after. - // This lets us assume the addon is installed at the start of each test. - describe("Addon uninstall tests", () => { - before(async() => utils.uninstallAddon(driver, addonId)); - - after(async() => utils.installAddon(driver)); - - it("should no longer trigger animation once uninstalled", async() => { - await utils.copyUrlBar(driver); - assert(!(await animationTest(driver, MOZILLA_ORG))); - }); - - it("should no longer trigger popup once uninstalled", async() => { - await utils.copyUrlBar(driver); - assert(!(await utils.testPanel(driver, "share-button-panel"))); - }); - - it("should no longer trigger ask panel once uninstalled", async() => { - await utils.copyUrlBar(driver); - assert(!(await utils.testPanel(driver, "share-button-ask-panel"))); - }); - - it("should not add the button to the toolbar once uninstalled", async() => { - await utils.removeShareButton(driver); - await utils.copyUrlBar(driver); - const shareButton = await utils.promiseAddonButton(driver); - assert(!shareButton); - }); - }); -}); - -describe("Highlight Treatment Tests", function() { - // This gives Firefox time to start, and us a bit longer during some of the tests. - this.timeout(25000); - - let driver; - - before(async() => { - driver = await utils.promiseSetupDriver(); - await setTreatment(driver, "highlight"); - // install the addon - await utils.installAddon(driver); - }); - - after(async() => { - await driver.quit(); - }); - - afterEach(async() => { - await postTestReset(driver); - await utils.removeShareButton(driver); - }); - - it("animation should trigger on regular page", async() => { - await utils.addShareButton(driver); - assert(await animationTest(driver, MOZILLA_ORG)); - }); - - it("animation should not trigger on disabled page", async() => { - await utils.addShareButton(driver); - assert(!(await animationTest(driver, "about:blank"))); - }); - - it("animation should not trigger if the share button is not added to toolbar", async() => { - await utils.gotoURL(driver, MOZILLA_ORG); - - await utils.copyUrlBar(driver); - const { hasClass, hasColor } = await utils.testAnimation(driver); - assert(!hasClass && !hasColor); - }); - - it("should not trigger animation if the share button is in the overflow menu", async() => { - await utils.addShareButton(driver); - await overflowMenuTest(driver, animationTest, MOZILLA_ORG); - }); - - it("should send highlight and copy telemetry pings", async() => { - await utils.addShareButton(driver); - await utils.gotoURL(driver, MOZILLA_ORG); - await utils.copyUrlBar(driver); - await utils.waitForClassAdded(driver); - - const pings = await utils.getMostRecentPingsByType(driver, "shield-study-addon"); - const foundPings = utils.searchTelemetry([ - ping => ping.payload.data.attributes.treatment === "highlight", - ping => ping.payload.data.attributes.event === "copy", - ], pings); - assert(foundPings.length > 0); - }); -}); - -describe("Summary Ping Tests", function() { - // This gives Firefox time to start, and us a bit longer during some of the tests. - this.timeout(25000); - - let driver; - let addonId; - - before(async() => { - driver = await utils.promiseSetupDriver(); - }); - - beforeEach(async() => { - await setTreatment(driver, "highlight"); - // install the addon - addonId = await utils.installAddon(driver); - }); - - after(async() => { - await driver.quit(); - }); - - afterEach(async() => { - await postTestReset(driver); - await utils.removeShareButton(driver); - }); - - it("should set hasShareButton to false if the share button is not added", async() => { - await utils.uninstallAddon(driver, addonId); - // hacky workaround to wait until the summary ping is sent - await new Promise(resolve => setTimeout(resolve, 500)); - const pings = await utils.getMostRecentPingsByType(driver, "shield-study-addon"); - const foundPings = utils.searchTelemetry( - [ping => Object.hasOwnProperty.call(ping.payload.data.attributes, "summary")], - pings); - assert(foundPings.length > 0); - assert(!JSON.parse(foundPings[0].payload.data.attributes.hasShareButton)); - }); - - it("should set hasShareButton to true if the share button is added", async() => { - await utils.addShareButton(driver); - await utils.uninstallAddon(driver, addonId); - // hacky workaround to wait until the summary ping is sent - await new Promise(resolve => setTimeout(resolve, 500)); - const pings = await utils.getMostRecentPingsByType(driver, "shield-study-addon"); - const foundPings = utils.searchTelemetry( - [ping => Object.hasOwnProperty.call(ping.payload.data.attributes, "summary")], - pings); - assert(foundPings.length > 0); - assert(JSON.parse(foundPings[0].payload.data.attributes.hasShareButton)); - }); - - it("should report the correct number of URL copy events", async() => { - await utils.copyUrlBar(driver); - await new Promise(resolve => setTimeout(resolve, 100)); // wait in between copy events - await utils.copyUrlBar(driver); - await new Promise(resolve => setTimeout(resolve, 100)); // wait in between copy events - await utils.copyUrlBar(driver); - await new Promise(resolve => setTimeout(resolve, 100)); // wait in between copy events - await utils.uninstallAddon(driver, addonId); - // hacky workaround to wait until the summary ping is sent - await new Promise(resolve => setTimeout(resolve, 1000)); - const pings = await utils.getMostRecentPingsByType(driver, "shield-study-addon"); - const foundPings = utils.searchTelemetry( - [ping => Object.hasOwnProperty.call(ping.payload.data.attributes, "summary")], - pings); - assert(foundPings.length > 0); - const urlBarCopies = JSON.parse(foundPings[0].payload.data.attributes - .numberOfTimesURLBarCopied); - assert(urlBarCopies === 3, `Expected 3 urlBarCopies, instead urlBarCopies = ${urlBarCopies}`); - }); - - it("should log a summary ping for highlight treatment", async() => { - await summaryFieldTest(driver, addonId, "highlight"); - }); - - it("should log a summary ping for doorhangerDoNothing treatment", async() => { - await summaryFieldTest(driver, addonId, "doorhangerDoNothing"); - }); - - it("should log a summary ping for doorhangerAskToAdd treatment", async() => { - await summaryFieldTest(driver, addonId, "doorhangerAskToAdd"); - }); - - it("should log a summary ping for doorhangerAddToToolbar treatment", async() => { - await summaryFieldTest(driver, addonId, "doorhangerAddToToolbar"); - }); -}); - -describe("DoorhangerDoNothing Treatment Tests", function() { - // This gives Firefox time to start, and us a bit longer during some of the tests. - this.timeout(25000); - - let driver; - let addonId; - - before(async() => { - driver = await utils.promiseSetupDriver(); - await setTreatment(driver, "doorhangerDoNothing"); - // install the addon - addonId = await utils.installAddon(driver); - }); - - after(async() => { - await utils.uninstallAddon(driver, addonId); - await driver.quit(); - }); - - afterEach(async() => { - await postTestReset(driver); - await utils.removeShareButton(driver); - }); - - it("popup should trigger on regular page", async() => { - await utils.addShareButton(driver); - assert(await popupTest(driver, MOZILLA_ORG)); - }); - - it("popup should not trigger on disabled page", async() => { - await utils.addShareButton(driver); - await utils.gotoURL(driver, "about:blank"); - - await utils.copyUrlBar(driver); - const panelOpened = await utils.testPanel(driver, "share-button-panel"); - assert(!panelOpened); - await utils.removeShareButton(driver); - }); - - it("popup should not trigger if the share button is not added to toolbar", async() => { - await utils.gotoURL(driver, MOZILLA_ORG); - - await utils.copyUrlBar(driver); - const panelOpened = await utils.testPanel(driver, "share-button-panel"); - assert(!panelOpened); - }); - - it("should not trigger doorhanger if the share button is in the overflow menu", async() => { - await utils.addShareButton(driver); - await overflowMenuTest(driver, popupTest, MOZILLA_ORG); - }); - - it("should send doorhanger and copy telemetry pings", async() => { - await utils.addShareButton(driver); - await utils.gotoURL(driver, MOZILLA_ORG); - await utils.copyUrlBar(driver); - await utils.testPanel(driver, "share-button-panel"); - - const pings = await utils.getMostRecentPingsByType(driver, "shield-study-addon"); - const foundPings = utils.searchTelemetry([ - ping => ping.payload.data.attributes.treatment === "doorhanger", - ping => ping.payload.data.attributes.event === "copy", - ], pings); - assert(foundPings.length > 0); - }); -}); - -describe("DoorhangerAskToAdd Treatment Tests", function() { - // This gives Firefox time to start, and us a bit longer during some of the tests. - this.timeout(25000); - - let driver; - let addonId; - - before(async() => { - driver = await utils.promiseSetupDriver(); - await setTreatment(driver, "doorhangerAskToAdd"); - // install the addon - addonId = await utils.installAddon(driver); - }); - - after(async() => { - await utils.uninstallAddon(driver, addonId); - await driver.quit(); - }); - - afterEach(async() => { - await postTestReset(driver); - await utils.removeShareButton(driver); - }); - - it("should open an ask panel on a regular page without the share button", async() => { - await utils.gotoURL(driver, MOZILLA_ORG); - await utils.copyUrlBar(driver); - const panelOpened = await utils.testPanel(driver, "share-button-ask-panel"); - assert(panelOpened); - }); - - it("should open a standard panel on a regular page with the share button", async() => { - await utils.addShareButton(driver); - assert(await popupTest(driver, MOZILLA_ORG)); - }); - - it("should not open an ask panel on a regular page with the share button", async() => { - await utils.addShareButton(driver); - - await utils.gotoURL(driver, MOZILLA_ORG); - - await utils.copyUrlBar(driver); - const askPanelOpened = await utils.testPanel(driver, "share-button-ask-panel"); - assert(!askPanelOpened); - }); - - it("should not open an ask panel on a regular page if the share button is in the overflow menu", async() => { - await utils.addShareButton(driver); - - const window = driver.manage().window(); - const currentSize = await window.getSize(); - await window.setSize(640, 480); - await utils.copyUrlBar(driver); - assert(!await utils.testPanel(driver, "share-button-ask-panel")); - await window.setSize(currentSize.width, currentSize.height); - }); - - it("should not open an ask panel on a disabled page", async() => { - await utils.gotoURL(driver, "about:blank"); - - await utils.copyUrlBar(driver); - const panelOpened = await utils.testPanel(driver, "share-button-ask-panel"); - assert(!panelOpened); - }); - - it("should add the button to the toolbar upon clicking on ask panel", async() => { - await utils.gotoURL(driver, MOZILLA_ORG); - - await utils.copyUrlBar(driver); - const panelOpened = await utils.testPanel(driver, "share-button-ask-panel"); - assert(panelOpened); - - const askPanel = driver.wait(until.elementLocated( - By.id("share-button-ask-panel")), 1000); - await askPanel.click(); - assert(await utils.promiseAddonButton(driver)); - }); - - it("should not show the ask panel after the button was added once", async() => { - await utils.gotoURL(driver, MOZILLA_ORG); - await utils.copyUrlBar(driver); - - const askPanel = driver.wait(until.elementLocated( - By.id("share-button-ask-panel")), 1000); - await askPanel.click(); - assert(await utils.promiseAddonButton(driver)); - - assert(await utils.removeShareButton(driver)); - - await utils.copyUrlBar(driver); - const panelOpened = await utils.testPanel(driver, "share-button-ask-panel"); - assert(!panelOpened); - }); - - it("should send ask-to-add and copy telemetry pings", async() => { - await utils.addShareButton(driver); - await utils.gotoURL(driver, MOZILLA_ORG); - await utils.copyUrlBar(driver); - await utils.testPanel(driver, "share-button-ask-panel"); - - const pings = await utils.getMostRecentPingsByType(driver, "shield-study-addon"); - const foundPings = utils.searchTelemetry([ - ping => ping.payload.data.attributes.treatment === "ask-to-add", - ping => ping.payload.data.attributes.event === "copy", - ], pings); - assert(foundPings.length > 0); - }); -}); - -describe("DoorhangerAddToToolbar Treatment Tests", function() { - // This gives Firefox time to start, and us a bit longer during some of the tests. - this.timeout(25000); - - let driver; - let addonId; - - before(async() => { - driver = await utils.promiseSetupDriver(); - await setTreatment(driver, "doorhangerAddToToolbar"); - // install the addon - addonId = await utils.installAddon(driver); - }); - - after(async() => { - await utils.uninstallAddon(driver, addonId); - await driver.quit(); - }); - - afterEach(async() => { - await postTestReset(driver); - await utils.removeShareButton(driver); - }); - - it("should add the button to the toolbar upon copy paste on regular page", async() => { - await utils.gotoURL(driver, MOZILLA_ORG); - - await utils.copyUrlBar(driver); - assert(await utils.promiseAddonButton(driver)); - }); - - it("should only add the button to the toolbar once", async() => { - await utils.gotoURL(driver, MOZILLA_ORG); - await utils.copyUrlBar(driver); - - assert(await utils.removeShareButton(driver)); - - await utils.copyUrlBar(driver); - - const shareButton = await utils.promiseAddonButton(driver); - assert(shareButton === null); - }); - - it("popup should trigger on regular page", async() => { - assert(await popupTest(driver, MOZILLA_ORG)); - }); - - it("should send add-to-toolbar and copy telemetry pings", async() => { - await utils.gotoURL(driver, MOZILLA_ORG); - await utils.copyUrlBar(driver); - - const pings = await utils.getMostRecentPingsByType(driver, "shield-study-addon"); - const foundPings = utils.searchTelemetry([ - ping => ping.payload.data.attributes.treatment === "add-to-toolbar", - ping => ping.payload.data.attributes.event === "copy", - ], pings); - assert(foundPings.length > 0); - }); -}); - -describe("Expiration date tests", function() { - // This gives Firefox time to start, and us a bit longer during some of the tests. - this.timeout(15000); - - let driver; - - before(async() => { - driver = await utils.promiseSetupDriver(); - // set expiration date to a date sufficiently in the past to trigger expiration - const now = new Date(Date.now()); - const expiredDateString = new Date(now.setDate(now.getDate() - 30)).toISOString(); - await driver.executeAsyncScript((dateStringArg, callback) => { - Components.utils.import("resource://gre/modules/Preferences.jsm"); - Preferences.set("extensions.sharebuttonstudy.expirationDateString", dateStringArg); - callback(); - }, expiredDateString); - // install the addon - await utils.installAddon(driver); - }); - - after(async() => driver.quit()); - - it("should open a new tab", async() => { - const newTabOpened = await driver.wait(async() => { - const handles = await driver.getAllWindowHandles(); - return handles.length === 2; // opened a new tab - }, 3000); - assert(newTabOpened); - }); - - it("should open a new tab to the correct URL", async() => { - const currentHandle = await driver.getWindowHandle(); - driver.setContext(Context.CONTENT); - // Find the new window handle. - let newWindowHandle = null; - const handles = await driver.getAllWindowHandles(); - for (const handle of handles) { - if (handle !== currentHandle) { - newWindowHandle = handle; - } - } - const correctURLOpened = await driver.wait(async() => { - await driver.switchTo().window(newWindowHandle); - const currentURL = await driver.getCurrentUrl(); - return currentURL.startsWith("https://qsurvey.mozilla.com/s3/sharing-study"); - }); - assert(correctURLOpened); - }); -}); diff --git a/README.md b/README.md index daec5d7..62dca3c 100644 --- a/README.md +++ b/README.md @@ -1,266 +1,41 @@ # Shield Study Embedded Web Extension Template -## Under Construction - -### Check out changes under review before forking - -This repo is undergoing big changes. A [huge PR](https://github.com/mozilla/shield-studies-addon-template/pull/49) is currently under review for improvements to build an embedded WebExtension Shield study. - -### We are moving to WebExtension Experiments - -In an effort to move to WebExtensions, we are also working on making a Shield study [WebExtension Experiment](https://firefox-source-docs.mozilla.org/toolkit/components/extensions/webextensions/index.html) template. That template will ultimately replace this one. - ![CircleCI badge](https://img.shields.io/circleci/project/github/mozilla/shield-studies-addon-template/master.svg?label=CircleCI) -**Note**: This is toy / demonstration [Shield Study](https://wiki.mozilla.org/Firefox/Shield/Shield_Studies) Legacy Addon. Use this as a template for yours - - -## Getting started - -``` -# install depndencies -npm install - -## build -npm run eslint -npm run build - -## build and run -npm run firefox -``` - -### Details - -First, make sure you are on NPM 5+ installed so that the proper dependencies are installed using the package-lock.json file. - -`$ npm install -g npm` - -After cloning the repo, you can run the following commands from the top level directory, one after another: - -``` -$ npm install -$ npm run build -``` - -This packages the add-on into `dist/linked-addon.xpi`. This file is the addon you load into Firefox. - -Note: `linked-addon.xpi` is a symbolic link to the extension's true XPI, which is named based on the study's unique addon ID specified in `package.json`. +## Important notice +### We are moving to Web Extensions -## About This Study +In an effort to remove the necessity of creating legacy add-ons for Shield studies, we are working on [supporting a pure Web Extension workflow in this template](https://github.com/mozilla/shield-studies-addon-template/issues/53). +Until support for those workflows become stable, it is still recommended to use this template as it is and create legacy add-ons for your study. +Chat with us: #shield on Slack about the latest progress and how to help us progress faster away from legacy add-ons. -(Note: get these from your PHD). +## About This Repository -Goal: Determine which if any TOOLBAR BUTTONS DESIGNS is the most enticing to the user. +**Note**: This contains an example [Shield Study](https://wiki.mozilla.org/Firefox/Shield/Shield_Studies) Legacy Add-on. Use this as a template for yours. +(Note: Make this README reflect your study). -## User Experience / Functionality +Goal: Determine which if any TOOLBAR BUTTONS DESIGNS is the most enticing to the user. -During INSTALL ONLY users see: - -- a notification bar - - - introducing the feature. - - allowing them to opt out - -During FIRST INSTALL and EVERY OTHER STARTUP, users see: - -- a 'toolbar button' (webExtension BrowserAction) - - - with one of 3 images (Cat, Dog, Lizard) - -Clicking on the button - -- changes the badge -- sends telemetry - -Icon will be the same every run. - -If the user clicks on the badge more than 3 times, it ends the study. +## Seeing the add-on in action +See [TESTPLAN.md](./docs/TESTPLAN.md) for more details on how to get the add-on installed and tested. ## Data Collected / Telemetry Pings Measure: -- Button (BrowserAction) usage. - -see [TELEMETRY.md](./TELEMETRY.md) - -## Test plan - -see [TESTPLAN](./TESTPLAN.md) - - -## Directory Structure and Files - - -``` -├── .circleci/ # setup for .circle ci integration -├── .eslintignore -├── .eslintrc.js # mozilla, json -├── .git/ -├── .gitignore -├── README.md # (this file) -├── TELEMETRY.md # Telemetry examples for this addon -├── TESTPLAN.md # Manual QA test plan -├── addon # Files that will go into the addon -│   ├── Config.jsm -│   ├── StudyUtils.jsm # (copied in during `prebuild`) -│   ├── bootstrap.js # LEGACY Bootstrap.js -│   ├── chrome.manifest # (derived from templates) -│   ├── install.rdf # (derived from templates) -│   │ -│   ├── lib # JSM (Firefox modules) -│   │   └── AddonPrefs.jsm -│   │   └── Feature.jsm # does `introduction` -| | -│   └── webextension # modern, embedded webextesion -│   ├── .eslintrc.json -│   ├── background.js -│   ├── icons -│   │   ├── Anonymous-Lizard.svg -│   │   ├── DogHazard1.svg -│   │   ├── Grooming-Cat-Line-Art.svg -│   │   ├── isolatedcorndog.svg -│   │   ├── kittens.svg -│   │   ├── lizard.svg -│   │   └── puppers.svg -│   └── manifest.json -│ -├── bin # Scripts / commands -│   └── xpi.sh # build the XPI -│ -├── dist # built xpis (addons) -│   ├── @template-shield-study.mozilla.com-1.1.0.xpi -│   └── linked-addon.xpi -> @template-shield-study.mozilla.com-1.1.0.xpi -│ -├── package-lock.json -├── package.json -├── run-firefox.js # command -├── sign/ # "LEGACY-SIGNED" addons. used by `npm sign` -│ -│ -├── templates # mustache templates, filled from `package.json` -│   ├── chrome.manifest.mustache -│   └── install.rdf.mustache -│ -│ -└── test # Automated tests `npm test` and circle - ├── Dockerfile - ├── docker_setup.sh - ├── functional_tests.js - ├── test-share-study.js - ├── test_harness.js - ├── test_printer.py - └── utils.js - - ->> tree -a -I node_modules - -``` - -### Loading the Web Extension in Firefox - -You can have Firefox automatically launched and the add-on installed by running: - -`$ npm run firefox` - -To load the extension manually instead, open (preferably) the [Developer Edition of Firefox](https://www.mozilla.org/firefox/developer/) and load the `.xpi` using the following steps: - -* Navigate to *about:config* and set `extensions.legacy.enabled` to `true`. This permits the loading of the embedded Web Extension since new versions of Firefox are becoming restricted to pure Web Extensions only. -* Navigate to *about:debugging* in your URL bar -* Select "Load Temporary Add-on" -* Find and select the `linked-addon.xpi` file you just built. - -### Seeing the add-on in action - -To debug installation and loading of extensions loaded in this manner, use the Browser Console which can be open from Firefox's top toolbar in `Tools > Web Developer > Browser Console`. This will display Shield (loading/telemetry) and `console.log()` output from the extensions that we build. - -You should see a green puzzle piece icon in the browser address bar. You should also see the following in the Browser Console (`Tools > Web Developer > Browser Console`), which comes from this add-on: - -``` -install 5 bootstrap.js:125 -startup ADDON_INSTALL bootstrap.js:33 -info {"studyName":"mostImportantExperiment","addon":{"id":"template-shield-study@mozilla.com","version":"1.0.0"},"variation":{"name":"kittens"},"shieldId":"8bb19b5c-99d0-cc48-ba95-c73f662bd9b3"} bootstrap.js:67 -1508111525396 shield-study-utils DEBUG log made: shield-study-utils -1508111525398 shield-study-utils DEBUG setting up! -1508111525421 shield-study-utils DEBUG firstSeen -1508111525421 shield-study-utils DEBUG telemetry in: shield-study {"study_state":"enter"} -1508111525421 shield-study-utils DEBUG getting info -1508111525423 shield-study-utils DEBUG telemetry: {"version":3,"study_name":"mostImportantExperiment","branch":"kittens","addon_version":"1.0.0","shield_version":"4.1.0","type":"shield-study","data":{"study_state":"enter"},"testing":true} -1508111525430 shield-study-utils DEBUG startup 5 -1508111525431 shield-study-utils DEBUG getting info -1508111525431 shield-study-utils DEBUG marking TelemetryEnvironment: mostImportantExperiment -1508111525476 shield-study-utils DEBUG telemetry in: shield-study {"study_state":"installed"} -1508111525477 shield-study-utils DEBUG getting info -1508111525477 shield-study-utils DEBUG telemetry: {"version":3,"study_name":"mostImportantExperiment","branch":"kittens","addon_version":"1.0.0","shield_version":"4.1.0","type":"shield-study","data":{"study_state":"installed"},"testing":true} -1508111525479 shield-study-utils DEBUG getting info -1508111525686 shield-study-utils DEBUG getting info -1508111525686 shield-study-utils DEBUG respondingTo: info -init kittens background.js:29:5 -``` - -Note: This add-on force assigns users to the `kitten` group/variation (in `addon/Config.jsm`), which is why the console will always report `init kittens`. - -Click on the web extension's green 'puzzle piece' icon to trigger additional console output and sending of telemetry data. - -To end early: Click on button multiple times until the 'too-popular' endpoint is reached. This will result in the uninstallation of the extension, and the user will be sent to the URL specified in `addon/Config.jsm` under `endings -> too-popular`. - -That's it! The rest is up to you. Fork the repo and hack away. - -### Developing - -You can automatically build recent changes and package them into a `.xpi` by running the following from the top level directory: - -`$ npm run watch` - -Now, anytime a file is changed and saved, node will repackage the add-on. You must reload the add-on as before, or by clicking the "Reload" under the add-on in *about:debugging*. Note that a hard re-load is recommended to clear local storage. To do this, simply remove the add-on and reload as before. - -### Description of what goes on when this addon is started - -During `bootstrap.js:startup(data, reason)`: - - a. `shieldUtils` imports and sets configuration from `Config.jsm` - b. `bootstrap.js:chooseVariation` explicitly and deterministically chooses a variation from `studyConfig.weightedVariations` - c. the WebExtension starts up - d. `boostrap.js` listens for requests from the `webExtension` that are study related: `["info", "telemetry", "endStudy"]` - e. `webExtension` (`background.js`) asks for `info` from `studyUtils` using `askShield` function. - f. Feature starts using the `variation` from that info. - g. Feature instruments user button to send `telemetry` and to `endStudy` if the button is clicked enough. - -Tip: It is particularly useful to compare the source code of previously deployed shield studies with this template (and each other) to get an idea of what is actually relevant to change between studies vs what is mostly untouched boilerplate. - -### Getting Data - -Telemetry pings are loaded into S3 and re:dash. You can use this [Example Query](https://sql.telemetry.mozilla.org/queries/46999/source#table) as a starting point. - -### Testing - -Run the following to run the example set of functional tests: - -`$ npm test` - -Note: The functional tests are using async/await, so make sure you are running Node 7.6+ - -The functional testing set-up is imported from [https://github.com/mozilla/share-button-study]() which contains plenty of examples of functional tests relevant to Shield study addons. - -## General information about Shield study development - -### Design - -Any UI in a Shield study should be consistent with standard Firefox design specifications. These standards can be found at [design.firefox.com](https://design.firefox.com/photon/welcome.html). Firefox logo specifications can be found [here](https://design.firefox.com/photon/visuals/product-identity-assets.html). -### Engineering +* Button (BrowserAction) usage. -Shield study add-ons are legacy (`addon/bootstrap.js`) add-ons with an optional embedded web extension (`addon/webextension/background.js`). +See [TELEMETRY.md](./docs/TELEMETRY.md) for more details on what pings are sent by this add-on. -The web extension needs to be packaged together with a legacy add-on in order to be able to access Telemetry data, user preferences etc that are required for collecting relevant data for [Shield Studies](https://wiki.mozilla.org/Firefox/Shield/Shield_Studies). +## Analyzing data -It is recommended to build necessary logic and user interface using in the context of the webextension and communicate with the legacy add-on code through messaging whenever privileged access is required. +Telemetry pings are loaded into S3 and re:dash. Sample query: -For more information, see [./about.md] +* [All pings](https://sql.telemetry.mozilla.org/queries/{#your-id}/source#table) -### Similar repositories +## Improving this add-on -[https://github.com/benmiroglio/shield-study-embedded-webextension-hello-world-example]() - A repository that was created this week specifically to help new Shield/Pioneer engineers to quickly get up and running with a Shield add-on. It was however built upon an older and much more verbose addon template, which makes it's file structure hard to follow. -[https://github.com/gregglind/template-shield-study]() - The incubation repo for the updated structure and contents of this repo. Use this repo instead. +See [DEV.md](./docs/DEV.md) for more details on how to work with this add-on as a developer. diff --git a/TESTPLAN.md b/TESTPLAN.md deleted file mode 100644 index 4f6fb50..0000000 --- a/TESTPLAN.md +++ /dev/null @@ -1,161 +0,0 @@ -# Test Plan for the 57-Perception-Study Addon - -## Automated Testing - -`npm test` verifies the telemetry payload as expected at Firefox startup and add-on installation in a clean profile, then does **optimistic testing** of the *commonest path* though the study for a user - -- prove the notification bar ui opens -- *clicking on the left-most button presented*. -- verifying that sent Telemetry is correct. - -Code at [/test/functional_tests.js](/test/functional_tests.js). - -## Manual / QA TEST Instructions - -Assumptions / Thoughts - -1. Please ask if you want more command-line tools to do this testing. - -### BEFORE EACH TEST: INSTALL THE ADDON to a CLEAN (NEW) PROFILE - -1. (create profile: , or via some other method) -2. In your Firefox profile -3. `about:debugging` > `install temporary addon` - -As an alternative (command line) CLI method: - -1. `git clone` the directory. -2. `npm install` then `npm run firefox` from the GitHub (source) directory. - - -### Note: checking "Correct Pings" - -All interactions with the UI create sequences of Telemetry Pings. - -All UI `shield-study` `study_state` sequences look like this: - -- `enter => install => (one of: "voted" | "notification-x" | "window-or-fx-closed") => exit`. - -(Note: this is complicated to explain, so please ask questions and I will try to write it up better!, see [TELEMETRY.md](/TELEMETRY.md) and EXAMPLE SEQUENCE below.) - -### Do these tests. - -1. UI APPEARANCE. OBSERVE a notification bar with these traits: - - * Icon is 'heartbeat' - * Text is one of 8 selected "questions", such as: "Do you like Firefox?". These are listed in [/addon/Config.jsm](/addon/Config.jsm) as the variable `weightedVariations`. - * clickable buttons with labels 'yes | not sure | no' OR 'no | not sure | yes' (50/50 chance of each) - * an `x` button at the right that closes the notice. - - Test fails IF: - - - there is no bar. - - elements are not correct or are not displayed - - -2. UI functionality: VOTE - - Expect: Click on a 'vote' button (any of: `yes | not sure | no`) has all these effects - - - notice closes - - addon uninstalls - - no additional tabs open - - telemetry pings are 'correct' with this SPECIFIC `study_state` as the ending - - - ending is `voted` - - 'vote' is correct. - -3. UI functionality: 'X' button - - Click on the 'x' button. - - - notice closes - - addon uninstalls - - no additional tabs open - - telemetry pings are 'correct' with this SPECIFIC ending - - - ending is `notification-x` - -4. UI functionality 'close window' - - 1. Open a 2nd Firefox window. - 2. Close the initial window. - - Then observe: - - - notice closes - - addon uninstalls - - no additional tabs open - - telemetry pings are 'correct' with this SPECIFIC ending - - - ending is `window-or-fx-closed` - - ---- - -## Helper code and tips - -### ***To open a Chrome privileged console*** - -1. `about:addons` -2. `Tools > web developer console` - -Or use other methods, like Scratchpad. - - -### **Telemetry Ping Printing Helper Code** - -```javascript -async function printPings() { - async function getTelemetryPings (options) { - // type is String or Array - const {type, n, timestamp, headersOnly} = options; - Components.utils.import("resource://gre/modules/TelemetryArchive.jsm"); - // {type, id, timestampCreated} - let pings = await TelemetryArchive.promiseArchivedPingList(); - if (type) { - if (!(type instanceof Array)) { - type = [type]; // Array-ify if it's a string - } - } - if (type) pings = pings.filter(p => type.includes(p.type)); - if (timestamp) pings = pings.filter(p => p.timestampCreated > timestamp); - - pings.sort((a, b) => b.timestampCreated - a.timestampCreated); - if (n) pings = pings.slice(0, n); - const pingData = headersOnly ? pings : pings.map(ping => TelemetryArchive.promiseArchivedPingById(ping.id)); - return Promise.all(pingData); - } - async function getPings() { - const ar = ["shield-study", "shield-study-addon"]; - return getTelemetryPings({type: ar}); - } - - const pings = (await getPings()).reverse(); - const p0 = pings[0].payload; - // print common fields - console.log( - ` -// common fields - -branch ${p0.branch} // should describe Question text -study_name ${p0.study_name} -addon_version ${p0.addon_version} -version ${p0.version} - - ` - ); - - pings.forEach(p => { - console.log(p.creationDate, p.payload.type); - console.log(JSON.stringify(p.payload.data, null, 2)); - }); -} - -printPings(); -``` - - -### Example sequence for a 'voted => not sure' interaction - -See [TELEMETRY.md](/TELEMETRY.md), EXAMPLE SEQUENCE section at the bottom. diff --git a/about.md b/about.md deleted file mode 100644 index 3f0ef4a..0000000 --- a/about.md +++ /dev/null @@ -1,701 +0,0 @@ -# Shield Study Template - -`Shield-Study-Template` contains files for for making a **Shield Study Addon**. Shield Study Addons are **LEGACY ADDONS** for Firefox that include the **SHIELD-STUDIES-ADDON-UTILS** (`studyUtils.jsm`) file (4.1.x series). - - - -**Contents** - -- [`npm` commands for `Shield-Study-Template`](#npm-commands-for-shield-study-template) -- [What is a Shield Study?](#what-is-a-shield-study) -- [tl;dr - Running the Template Study](#tldr---running-the-template-study) -- [Folder Contents](#folder-contents) -- [Parts of A Shield Study (General)](#parts-of-a-shield-study-general) - - [Shield-Studies-Addon-Utils (`studyUtils.jsm`)](#shield-studies-addon-utils-studyutilsjsm) - - [Legacy Addons](#legacy-addons) - - [Building Your Feature, with Variations](#building-your-feature-with-variations) -- [All About Shield Telemetry](#span-idshield-telemetryall-about-shield-telemetryspan) - - [Shield Study Telemetry Probe Life cycle](#shield-study-telemetry-probe-life-cycle) - - [Expected ping counts](#expected-ping-counts) - - [How Probes are Sent from `studyUtils.jsm`](#how-probes-are-sent-from-studyutilsjsm) -- [Send your own probes](#send-your-own-probes) -- [Viewing Sent Telemetry Probes](#viewing-sent-telemetry-probes) - - [client](#client) - - [Collector (example s.t.m.o query)](#collector-example-stmo-query) -- [Engineering Side-by-Side (a/b) Feature Variations](#engineering-side-by-side-ab-feature-variations) -- [Kittens or Puppers, the Critical Study We have all been waiting for](#kittens-or-puppers-the-critical-study-we-have-all-been-waiting-for) -- [Get More Help](#get-more-help) -- [Gotchas / FAQ / Ranting](#gotchas--faq--ranting) - - [General](#general) - - [studyUtils](#studyutils) - - [Legacy Addons](#legacy-addons-1) - - [s.t.m.o - sql.telemetry.mozilla.org](#stmo---sqltelemetrymozillaorg) -- [Glossary](#glossary) -- [OTHER DOCS](#other-docs) - - [Configuration](#configuration) - - [Lifecycle](#lifecycle) - - [Running](#running) - - [TODO](#todo) -- [Links and References](#links-and-references) - - - -## `npm` commands for `Shield-Study-Template` - - -``` - "eslint": "eslint addon --ext jsm --ext js --ext json", - "prebuild": "cp node_modules/shield-studies-addon-utils/dist/StudyUtils.jsm addon/", - "build": "bash ./bin/xpi.sh", - "test": "export XPI=dist/linked-addon.xpi && npm run build && mocha test/functional_tests.js --retry 2", - "harness_test": "export XPI=dist/linked-addon.xpi && mocha test/functional_tests.js --retry 2 --reporter json", - "firefox": "export XPI=dist/linked-addon.xpi && npm run build && node run-firefox.js", - "watch": "onchange 'addon/**' 'package.json' 'template/**' -e addon/install.rdf -e addon/chrome.manifest -e addon/StudyUtils.jsm -- npm run build -- '{{event}} {{changed}} $(date)'", - "sign": "echo 'TBD, see: https://bugzilla.mozilla.org/show_bug.cgi?id=1407757'" -``` - -## What is a Shield Study? - -**Shield Study Addons** do these actions: - -- implement variations (1+) of a feature -- report common study and addon lifecycle events to Telemetry -- report study-specific data about how users react to and interact with a specific variations -- respond coherently to addon life-cycle events (`install`, `startup`, `disable`, `uninstall`). - - -## tl;dr - Running the Template Study - -1. **One time**: - - * Clone this directory - - ``` - git clone template - rm -rf {.git,docs}/ - git init - ``` - - * install dependencies, including [`mozilla/shield-studies-addon-utils`][mozilla-ssau]. - - ``` - npm install - ``` - - * install **Firefox Nightly** for easier development - -2. Edit and examine files: - - - `addon/bootstrap.js` - - `addon/Config.jsm` - - `package.json` - - `addon/lib/*` - -3. Build the legacy addon xpi. Run **Nightly** with addon - - `npm run firefox` - -4. Debug using - - - [`browser console`][link-browser-console] - - `about:debugging`. - -5. Restart / re-run after addon changes. - -Repeat Steps 2-5 as necessary. - - -## Direcotry Contents - -``` -├── .circleci/ # setup for Circle-CI integration -| -├── .eslintignore # -├── .eslintrc.js # linting rules: mozilla, json -| -├── .git/ -├── .gitignore -| -├── README.md # (this file) -├── TELEMETRY.md # Telemetry examples for this addon -├── TESTPLAN.md # Manual QA test plan -| -├── addon # Files that will go into the addon -│   ├── Config.jsm -│   ├── StudyUtils.jsm # (copied in during `prebuild`) -│   ├── bootstrap.js # LEGACY Bootstrap.js -│   ├── chrome.manifest # (derived from templates) -│   ├── install.rdf # (derived from templates) -│   │ -│   ├── lib # JSM (Firefox modules) -│   │   ├── AddonPrefs.jsm -│   │   └── Feature.jsm -| | -│   └── webextension # webExtension for Feature and UI -│   ├── .eslintrc.json -│   ├── background.js -│   └── manifest.json -│ -├── bin # Scripts / commands -│   └── xpi.sh # build the XPI from contents of `addon/` -│ -├── dist # built xpi's (addons) -│   ├── @template-shield-study.mozilla.com-1.1.0.xpi -│   └── linked-addon.xpi -> @template-shield-study.mozilla.com-1.1.0.xpi -│ -├── package-lock.json -├── package.json -├── run-firefox.js # command -├── sign/ # "LEGACY-SIGNED" addons. used by `npm sign` (TBD) -│ -│ -├── templates # mustache templates, filled from `package.json` -│   ├── chrome.manifest.mustache -│   └── install.rdf.mustache -│ -│ -└── test # Automated tests `npm test` and circle - ├── Dockerfile - ├── docker_setup.sh - ├── functional_tests.js # Edit these - ├── test-share-study.js # Examples from another study - ├── test_harness.js - ├── test_printer.py - └── utils.js - -``` - - -## Parts of A Shield Study (General) - -Note: see [about the #kittens study](#kittens) for architecture of the particulars of the example study. - -- Shield-Studies-Addon-Utils -- Legacy Addon framing code -- UI / Feature - - - (optional) Web Extension, embedded - - (optional) Various Firefox modules (`.jsm` files) - -More details on each follow. - -### Shield-Studies-Addon-Utils (`studyUtils.jsm`) - -`studyUtils.jsm` is a Firefox JavaScript module that provides these capabilities: - -1. **suggest variation for a client** - - deterministic and predicatable: every startup will suggest the same variation for a particular client - - per client: uses sha256 hash of (Telemetry Id, study name) - - ```javascript - const variation = await studyUtils.deterministicVariation(myWeightedVariations); - studyUtils.setVariation(variation); - ``` -2. **Report lifecycle data** using Telemetry - - `shield-study` Telemetry bucket - - [about Shield Telemetry](#shield-telemetry) - - ```javascript - // some study state events - studyUtils.firstSeen(); - studyUtils.endStudy(reason); - studyUtils.startup(ADDON_INSTALL); - ``` -3. **Report feature interaction and success data** using Telemetry - - `shield-study-addon` Telemetry bucket - - ```javascript - // values must be strings - studyUtils.telemetry({evt:"click", button:"share", times:"3"}) - ``` -4. **Annotate Telemetry Enviroment** to mark the user as special, and copy every `main` and other ping to a special bucket for faster analysis. - -**Links** for `studyUtils` code: - -- `npm install shield-studies-addon-utils` -- `node_modules/shield-studies-addon-utils/dist/studyUtils.jsm` -- Github: [mozilla/shield-studies-addon-utils](https://github.com/mozilla/shield-studies-addon-utils) - - -### Legacy Addons - -**Note**: to send Telemetry and see the ClientId, study addons require `Components.utils` (Chrome) privileges. Firefox webExtensions do not have those privileges. All Study Addons must be [Legacy Extensions][link-legacy]. - -A **Legacy Addon** consists of: - -* files - - - `bootstrap.js` - - `install.rdf` - - optional `chrome.manifest`, `update.rdf` etc. - -* build process to turn these files an `xpi`. -* signing process using the [Legacy Signing Key][legacy-signing], to enable running in Beta and Release. - -### Your Feature, with Variations - -If you have UI: - -- embedded web extension - suggested (where possible). See [link-extensions] -- jsm files - -If you do not have UI - -- jsm files - - - -## Shield Telemetry Details - -### Shield Study Life-Cycle Telemetry - -``` - time -------------> - - ENTRY +-> INSTALL +----> ENDINGS +------> EXIT - - - enter +-> install +----> user-disable exit (all states) - + + + + - | | | +------> ended-positive - | | | - | | +---------> ended-neutral - | | - | +------------> ended-negative - | | - | +------------> expired - | - | - | (only if not installed) - +-------------------> inelegible exit - -``` - - -### Expected ping counts - -All **N** enters will eventually have an ending and an exit. - -There will be **i** installs ( \\( i \le N \\) ). - -There will be **x** ineligibles ( \\( x \le N \\) ). - -\\( N = i + x \\) - - -``` - enter == exit - == (install + ineligible) - - install == user-disable + expired + ended-* -``` - -### How Probes are Sent from `studyUtils.jsm` - -Note: - -`const su = Cu.import("resource://path/to/StudyUtils.jsm")` - -`study_state` | `studyUtils` call | when to call it ---- | --- | --- -`enter` | `su.firstSeen()` | call ONCE per study during `ADDON_INSTALL` -`install` | `su.startup(ADDON_INSTALL)` | During `boostrap.js:startup` -none sent | `su.startup()` | Never -**ENDINGS** | | Affected by the `endings` config value. -`user-disable` | `su.endStudy("user-disable")` | Implies user uninstalled or disabled addon, or (BUG) Normandy uninstalled it. -`expired` | `su.endStudy("expired")` | Time-limited study reached expiration. -`ended-positive` | `su.endStudy("ended-positive")` | General study-defined 'good ending', such as attempting to use feature. -`ended-negative` | `su.endStudy("ended-negative")` | General study-defined 'bad ending', such as clicking 'I do not like this feature'. -`ended-neutral` | `su.endStudy("ended-neutral")` | General study-defined 'neutral ending'. -`ineligible` | `su.endStudy("ineligible")` | During install, client actually not appropriate for study, for some study-specific reason. -**EXIT** | | -`exit` | | automatically sent as part of `endStudy` - - -**Note**: Every user should have - -- exactly 1 each of ENTER, EXIT -- exactly 1 of either INSTALL or INELGIBLE -- exactly one 'ending' ping (which might be INELIGIBLE, EXPIRED, USER-DISABLE, ENDED-*) - -**Note**: [Full Schemas - gregglind/shield-study-schemas](https://github.com/gregglind/shield-study-schemas/tree/master/schemas-client) - - -### Send your own probes - -Use: `shieldStudy.telemetry(anObjectWithStringValues)` - -This will send data to the `shield-study-addon` bucket. The `key=>string` map will be the `payload.data.attributes` key. - -Example: - -```javascript -// values must be strings -studyUtils.telemetry({evt:"click", button:"share", times:"3"}) -``` - -### Defining Custom Study Endings - -Suppose you want some 'early endings', such as: - -- positive: user reached "end of the built UI". -- negative: user clicked on "no thanks". - -Define in `endings`: - -``` -endings: { - /** User defined endings */ - "user-attempted-signup": { - "baseUrl": "http://www.example.com/?reason=too-popular", - "study_state": "ended-positive", // neutral is default - } -} -``` - -Then: - -``` -studyUtils.endStudy("user-attempted-signup"); -``` - - -## Viewing Sent Telemetry Probes - -### client - -1. **Use the QA Helper Addon** - - The QA-Shield-Study-Helper lists the `payload.data` field for every `shield-study` and `shield-study-addon` ping. - - [Bugzilla for QA Helper Addon](https://bugzilla.mozilla.org/show_bug.cgi?id=1407757 - ) - [direct install link for Signed XPI for @qa-shield-study-helper-1.0.0.xpi][qa-helper-addon-direct] - - Example output: - - ```text - // common fields - - branch up-to-expectations-1 // should describe Question text - study_name 57-perception-shield-study - addon_version 1.0.0 - version 3 - - - 2017-10-09T14:16:18.042Z shield-study - { - "study_state": "enter" - } - 2017-10-09T14:16:18.055Z shield-study - { - "study_state": "installed" - } - 2017-10-09T14:16:18.066Z shield-study-addon - { - "attributes": { - "event": "prompted", - "promptType": "notificationBox-strings-1" - } - } - 2017-10-09T16:29:44.109Z shield-study-addon - { - "attributes": { - "promptType": "notificationBox-strings-1", - "event": "answered", - "yesFirst": "1", - "score": "0", - "label": "not sure", - "branch": "up-to-expectations-1", - "message": "Is Firefox performing up to your expectations?" - } - } - 2017-10-09T16:29:44.188Z shield-study - { - "study_state": "ended-neutral", - "study_state_fullname": "voted" - } - 2017-10-09T16:29:44.191Z shield-study - { - "study_state": "exit" - } - ``` - -2. Use `about:telemetry`, and look for `shield-study` or `shield-study-addon` probes. - - - -### Collector (example s.t.m.o query) - -[Example s.t.m.o study states query for "Pioneer Enrollement"][stmo-study-states] shows the Study lifecycle for every client in the Pioneer Enrollment study. - - - -## Engineering Side-by-Side (a/b) Feature Variations - -Note: this is a gloss / summary. - - -1. Your feature has a `startup` or `configuration` method that does different things depending on which variation is chosen. - - ```javascript - // bootstrap.js startup()... - const variation = await studyUtils.deterministicVariation(myWeightedVariations); - studyUtils.setVariation(variation); - - //... - - // start the feature - TheFeature.startup(variation) - ``` - -2. Ensure that your Feature measures every variation, including the Control (no-effect). - - -## Kittens or Puppers, the Critical Study We have all been waiting for - -Style: - -- Embedded Web Extension -- Telmetry on 'button click' -- has one "end early" condition: 3 or more button presses during a sesson. -- Goal: test if 'interest rate is higher for kittens or puppies, using a PROXY MEASURE -- "button clicks" - - - - - - - -## Get More Help - -- slack: `#shield` - - - -## Gotchas / FAQ / Ranting - -### General - -I am on Windows. How can I build? - -- (see TODO link to the issue and instructions by JCrawford) - -### studyUtils - - - -### The lifecycle and deployment of the add-on once it gets released - -The add-on for the experiment is remotely installed to the users which are selected for the experiment. (Note that this leads to an environment-change and a subsequent main ping) - -Main telemetry is tagged with the user's currently running experiments so that the main telemetry data and shield ping data can be cross-referenced later. - -After the experiment, the add-on is remotely uninstalled. In rare occasions, it remains installed until a new Firefox update is released. - - - -### Legacy Addons - -Debugging `Cu.import`. - -- use `run-firefox` to 'try again' after any change to modules. "Reload addon" will probably not work. -- Based on `chrome.manifest` files. -- `chrome.manifest` paths can't have `@ # ; : ? /` -- `chrome.manifest` isn't read yet in `bootstrap.js` main scope, OR during `install`. It is read during `startup` and `shutdown` -- Remember to uninstall your modules. -- [browser console][link-browser-console] will show errors sometimes. - - -### s.t.m.o - [sql.telemetry.mozilla.org](http://sql.telemetry.mozilla.org/) - - -#### Where are my pings? - -1. Are you seeing them in `about:telemetry` and / or the QA-Study-Helper. If YES, then they are being reported at client, good! If NO: check the config settings for your study for `telemetry.send => true` -2. Is pref set weirdly: `toolkit.telemetry.server => https://incoming.telemetry.mozilla.org`. If you are running from `run_firefox` and maybe lots of other contexts, this pref will not be properly set (because we don’t usually want to send telemetry!) BAD RESULT: “toolkit.telemetry.server”, Pref::new(“https://%(server)s/dummy/telemetry/“)) -3. Have you waited… 3-5 minutes? - - -- All error messages are misleading. They almost always indicate issues with syntax. Sometimes they indicate mis-spelled fields. -- No SEMI-COLONS at the end of your sql! -- Athena >> Presto (10-20x faster!) -- Be careful with single and double-quotes. - -## Glossary - -- **Probe**. A Telemetry measure, or ping. More broadly: any measure sent anywhere. -- **Variation**. synonyms (branch, arm): - - which *specific* version / configuration a specific client is randomized into. - - A JSON object describing the configuration for that specific choice, with keys like `name`. - -## OTHER DOCS - -- template/README.md - - - should be edited for YOUR STUDY - - move the general npm commands there - - links to 'about shield stuides' (in general) - - shield-study-addon-utils api - - -## `StudyUtils.jsm` api used in `bootstrap.js` - -### Configuration - -- `studyUtils.setup` - - Needed to send any telemetry - - Minimal setup: - - ``` - { - "studyName": "a-study-name", - "endings": {}, - "telemetry": { - "send": true, // assumed false. Actually send pings? - "removeTestingFlag": false, // Marks pings as testing, set true for actual release - } - } - ``` - - -- `studyUtils.deterministicVariation(weightedVariations)` - - Suggest a variation. - -- `studyUtils.setVariation(anObjectWithNameKey)` - - Actually set the variation. - - -### Lifecycle - -- `studyUtils.firstSeen()` - - - Send the `enter` ping. - - Future: Record first entry. - -- `await studyUtils.startup({reason})` - - If 'install', send an install ping. - -- `studyUtils.endStudy(endingName)`; - - - Send ending ping - - Open a url for that ending if defined - - Uninstalls addon - - -### Running - -- `await studyUtils.info()` - - Return configuration info - -- `studyUtils.respondToWebExtensionMessage` - - "Do shield things" (`telemetry`, `info`, `endStudy`) - -- `studyUtils._isEnding` - - Useful flag for knowing if something is already calling an ending, to help prevent race conditions and "double endings" - -- `studyUtils.telemetry(stringStringObject)` - - Send a 'study specific' ping to `shield-study-addon` bucket. - - - -### TODO - -Change SSAU api to this: - -- suggestVariation -- setup(includes branch) -- install() => firstSeen() => ping('enter'); -- -- alreadyEnding() -- endStudy()? tryEndStudy()? # first in. - -- info -- respondToWebExtension / respond? -- telemetry - - -## FIXES - -``` -startup(reason) { - const isEligible = some Fn(); - studyUtils.startup(reason, isEligible) - - if INSTALL { - if !isEliglbe endStudy('ineligible') - - } - startup(reason) - - ==> utils.startup(reason) - if INSTALL, then send install - else send nothing -} - -``` - -``` -install -- elgible -- not eligible - - -specialPing("enter") -sendEnterPing -sendInstallPing -endStudy - -endStudy() - -telemetry(); - -``` - -## TODO - -- debuggin and setting localstore? Prefs are 1000x easier -- debug both halves. - - -## Template - -- see the cloneable template HERE -- see some other examples HERE - -if at template... -say -Acutally, read the docs at SSAU there. - - -## Getting QA of your addons - -https://mana.mozilla.org/wiki/display/PI/PI+Request - -## Links and References - -[link-browser-console]: https://developer.mozilla.org/en-US/docs/Tools/Browser_Console - -[link-legacy]: https://developer.mozilla.org/en-US/Add-ons/Legacy_add_ons - - -[link-webextensions]: https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Examples - -[link-embedded]: https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Embedded_WebExtensions - -[stmo-study-states]: https://sql.telemetry.mozilla.org/queries/47604/source#table - -[qa-helper-addon-direct]: https://bugzilla.mozilla.org/attachment.cgi?id=8917534 - -[legacy-signing]: see TODO link - -[mozilla-ssau]: https://github.com/mozilla/shield-studies-addon-utils diff --git a/addon/Config.jsm b/addon/Config.jsm index b99665e..9610f51 100644 --- a/addon/Config.jsm +++ b/addon/Config.jsm @@ -12,68 +12,72 @@ var EXPORTED_SYMBOLS = ["config"]; var config = { // required STUDY key - "study": { + study: { /** Required for studyUtils.setup(): - * - * - studyName - * - endings: - * - map of endingName: configuration - * - telemetry - * - boolean send - * - boolean removeTestingFlag - * - * All other keys are optional. - */ + * + * - studyName + * - endings: + * - map of endingName: configuration + * - telemetry + * - boolean send + * - boolean removeTestingFlag + * + * All other keys are optional. + */ // required keys: studyName, endings, telemetry // will be used activeExperiments tagging - "studyName": "buttonFeatureExperiment", + studyName: "buttonFeatureExperiment", /** **endings** - * - keys indicate the 'endStudy' even that opens these. - * - urls should be static (data) or external, because they have to - * survive uninstall - * - If there is no key for an endStudy reason, no url will open. - * - usually surveys, orientations, explanations - */ - "endings": { + * - keys indicate the 'endStudy' even that opens these. + * - urls should be static (data) or external, because they have to + * survive uninstall + * - If there is no key for an endStudy reason, no url will open. + * - usually surveys, orientations, explanations + */ + endings: { /** standard endings */ "user-disable": { - "baseUrl": "http://www.example.com/?reason=user-disable", + baseUrl: "http://www.example.com/?reason=user-disable", }, - "ineligible": { - "baseUrl": "http://www.example.com/?reason=ineligible", + ineligible: { + baseUrl: "http://www.example.com/?reason=ineligible", }, - "expired": { - "baseUrl": "http://www.example.com/?reason=expired", + expired: { + baseUrl: "http://www.example.com/?reason=expired", }, /** User defined endings */ "used-often": { - "baseUrl": "http://www.example.com/?reason=used-often", - "study_state": "ended-positive", // neutral is default + baseUrl: "http://www.example.com/?reason=used-often", + study_state: "ended-positive", // neutral is default }, "a-non-url-opening-ending": { - "study_state": "ended-neutral", - "baseUrl": null, + study_state: "ended-neutral", + baseUrl: null, }, "introduction-leave-study": { - "study_state": "ended-negative", - "baseUrl": "http://www.example.com/?reason=introduction-leave-study", + study_state: "ended-negative", + baseUrl: "http://www.example.com/?reason=introduction-leave-study", }, }, - "telemetry": { - "send": true, // assumed false. Actually send pings? - "removeTestingFlag": false, // Marks pings as testing, set true for actual release + telemetry: { + send: true, // assumed false. Actually send pings? + removeTestingFlag: false, // Marks pings to be discarded, set true for to have the pings processed in the pipeline // TODO "onInvalid": "throw" // invalid packet for schema? throw||log }, }, // required LOG key - "log": { + log: { // Fatal: 70, Error: 60, Warn: 50, Info: 40, Config: 30, Debug: 20, Trace: 10, All: -1, - "studyUtils": { - "level": "Trace", + bootstrap: { + // Console.jsm uses "debug", whereas Log.jsm uses "Debug", *sigh* + level: "debug", + }, + studyUtils: { + level: "Trace", }, }, @@ -81,7 +85,7 @@ var config = { // a place to put an 'isEligible' function // Will run only during first install attempt - "isEligible": async function() { + async isEligible() { // get whatever prefs, addons, telemetry, anything! // Cu.import can see 'firefox things', but not package things. return true; @@ -92,16 +96,21 @@ var config = { - downweight lizards. Lizards is a 'poison' branch, meant to help control for novelty effect */ - "weightedVariations": [ - {"name": "kittens", - "weight": 1.5}, - {"name": "puppers", - "weight": 1.5}, - {"name": "lizard", - "weight": 1}, // we want more puppers in our sample + weightedVariations: [ + { + name: "kittens", + weight: 1.5, + }, + { + name: "puppers", + weight: 1.5, + }, + { + name: "lizard", + weight: 1, + }, // we want more puppers in our sample ], - // Optional: relative to bootstrap.js in the xpi - "studyUtilsPath": `./StudyUtils.jsm`, + studyUtilsPath: `./StudyUtils.jsm`, }; diff --git a/addon/bootstrap.js b/addon/bootstrap.js index 4c4107c..07c33c6 100644 --- a/addon/bootstrap.js +++ b/addon/bootstrap.js @@ -1,166 +1,216 @@ "use strict"; -/* global __SCRIPT_URI_SPEC__ */ -/* global Feature, Services */ // Cu.import +/* global config, studyUtils, Feature */ /* eslint no-unused-vars: ["error", { "varsIgnorePattern": "(startup|shutdown|install|uninstall)" }]*/ const { utils: Cu } = Components; -Cu.import("resource://gre/modules/Console.jsm"); -Cu.import("resource://gre/modules/Log.jsm"); -Cu.import("resource://gre/modules/Services.jsm"); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); - -const CONFIGPATH = `${__SCRIPT_URI_SPEC__}/../Config.jsm`; -const { config } = Cu.import(CONFIGPATH, {}); - -const STUDYUTILSPATH = `${__SCRIPT_URI_SPEC__}/../${config.studyUtilsPath}`; -const { studyUtils } = Cu.import(STUDYUTILSPATH, {}); - -const REASONS = studyUtils.REASONS; - -// logging for bootstrap.js, pref sets how verbose -const PREF_LOGGING_LEVEL = "shield.testing.logging.level"; -const BOOTSTRAP_LOGGER_NAME = `shield-study-${config.study.studyName}`; -const log = Log.repository.getLogger(BOOTSTRAP_LOGGER_NAME); -log.addAppender(new Log.ConsoleAppender(new Log.BasicFormatter())); -log.level = Services.prefs.getIntPref(PREF_LOGGING_LEVEL, Log.Level.Warn); - - -// QA NOTE: Study Specific Modules - package.json:addon.chromeResource -const BASE = `button-icon-preference`; -XPCOMUtils.defineLazyModuleGetter(this, "Feature", `resource://${BASE}/lib/Feature.jsm`); - - -/* Example addon-specific module imports. Remember to Unload during shutdown! +XPCOMUtils.defineLazyModuleGetter( + this, + "Services", + "resource://gre/modules/Services.jsm", +); + +const STUDY = "button-icon-preference"; + +XPCOMUtils.defineLazyModuleGetter( + this, + "config", + `resource://${STUDY}/Config.jsm`, +); +XPCOMUtils.defineLazyModuleGetter( + this, + "studyUtils", + `resource://${STUDY}/StudyUtils.jsm`, +); +XPCOMUtils.defineLazyModuleGetter( + this, + "Feature", + `resource://${STUDY}/lib/Feature.jsm`, +); + +/* Example addon-specific module imports. Remember to Unload during shutdown() below. // https://developer.mozilla.org/en-US/docs/Mozilla/JavaScript_code_modules/Using + Ideally, put ALL your feature code in a Feature.jsm file, + NOT in this bootstrap.js. - Ideally, put ALL your feature code in a Feature.jsm file, - NOT in this bootstrap.js. - - const BASE=`template-shield-study`; - XPCOMUtils.defineLazyModuleGetter(this, "SomeExportedSymbol", - `resource://${BASE}/SomeModule.jsm"); + XPCOMUtils.defineLazyModuleGetter(this, "SomeModule", + `resource://${STUDY}/lib/SomeModule.jsm`); XPCOMUtils.defineLazyModuleGetter(this, "Preferences", "resource://gre/modules/Preferences.jsm"); */ -async function startup(addonData, reason) { - // `addonData`: Array [ "id", "version", "installPath", "resourceURI", "instanceID", "webExtension" ] bootstrap.js:48 - log.debug("startup", REASONS[reason] || reason); +this.Bootstrap = { + /** + * Change this preference to test the add-on behavior in different study + * variations/branches (or leave it unset to use the automatic assigning + * of a study variation/branch from weightedVariations in Config.jsm) + */ + VARIATION_OVERRIDE_PREF: "extensions.button_icon_preference.variation", + + /** + * Use console as our logger until there is a log() method in studyUtils that we can rely on + */ + log: console, + + /** + * @param addonData Array [ "id", "version", "installPath", "resourceURI", "instanceID", "webExtension" ] + * @param reason + * @returns {Promise} + */ + async startup(addonData, reason) { + this.log.debug("startup", studyUtils.REASONS[reason] || reason); + + this.initStudyUtils(addonData.id, addonData.version); + + // choose and set variation + const variation = await this.selectVariation(); + this.variation = variation; + + // Check if the user is eligible to run this study using the |isEligible| + // function when the study is initialized + if (reason === studyUtils.REASONS.ADDON_INSTALL) { + // telemetry "enter" ONCE + studyUtils.firstSeen(); + const eligible = await config.isEligible(); + if (!eligible) { + this.log.debug("User is ineligible, ending study."); + // 1. uses config.endings.ineligible.url if any, + // 2. sends UT for "ineligible" + // 3. then uninstalls addon + await studyUtils.endStudy({ reason: "ineligible" }); + return; + } + } - /* Configuration of Study Utils*/ - studyUtils.setup({ - ...config, - addon: { id: addonData.id, version: addonData.version }, - }); - // choose the variation for this particular user, then set it. - const variation = getVariationFromPref(config.weightedVariations) || - await studyUtils.deterministicVariation( - config.weightedVariations - ); - studyUtils.setVariation(variation); - log.debug(`studyUtils has config and variation.name: ${variation.name}. Ready to send telemetry`); + /* + * Adds the study to the active list of telemetry experiments, + * and sends the "installed" telemetry ping if applicable + */ + await studyUtils.startup({ reason }); + // log what the study variation and other info is. + this.log.debug(`info ${JSON.stringify(studyUtils.info())}`); - /** addon_install ONLY: - * - note first seen, - * - check eligible - */ - if ((REASONS[reason]) === "ADDON_INSTALL") { - // telemetry "enter" ONCE - studyUtils.firstSeen(); - const eligible = await config.isEligible(); // addon-specific - if (!eligible) { - // 1. uses config.endings.ineligible.url if any, - // 2. sends UT for "ineligible" - // 3. then uninstalls addon - await studyUtils.endStudy({reason: "ineligible"}); - return; - } - } - - // startup for eligible users. - // 1. sends `install` ping IFF ADDON_INSTALL. - // 2. sets activeExperiments in telemetry environment. - await studyUtils.startup({reason}); - - // if you have code to handle expiration / long-timers, it could go here - (function fakeTrackExpiration() {})(); - - // IFF your study has an embedded webExtension, start it. - const { webExtension } = addonData; - if (webExtension) { - webExtension.startup().then(api => { - const {browser} = api; - /** spec for messages intended for Shield => - * {shield:true,msg=[info|endStudy|telemetry],data=data} - */ - browser.runtime.onMessage.addListener(studyUtils.respondToWebExtensionMessage); - // other browser.runtime.onMessage handlers for your addon, if any - }); - } - - // log what the study variation and other info is. - log.debug(`info ${JSON.stringify(studyUtils.info())}`); - - // Start up your feature, with specific variation info. - this.feature = new Feature({variation, studyUtils, reasonName: REASONS[reason]}); -} + // initiate the chrome-privileged part of the study add-on + this.feature = new Feature( + variation, + studyUtils, + studyUtils.REASONS[reason], + this.log, + ); -/** Shutdown needs to distinguish between USER-DISABLE and other - * times that `endStudy` is called. - * - * studyUtils._isEnding means this is a '2nd shutdown'. - */ -function shutdown(addonData, reason) { - log.debug("shutdown", REASONS[reason] || reason); - // FRAGILE: handle uninstalls initiated by USER or by addon - if (reason === REASONS.ADDON_UNINSTALL || reason === REASONS.ADDON_DISABLE) { - log.debug("uninstall or disable"); - if (!studyUtils._isEnding) { - // we are the first 'uninstall' requestor => must be user action. - log.debug("probably: user requested shutdown"); - studyUtils.endStudy({reason: "user-disable"}); + // Expiration checks should be implemented in a very reliable way by the add-on since Normandy does not handle study expiration in a reliable manner + /* + if (this.feature.hasExpired()) { + await studyUtils.endStudy({ reason: "expired" }); return; } - // normal shutdown, or 2nd uninstall request - - // QA NOTE: unload addon specific modules here. - Cu.unload(`resource://${BASE}/lib/Feature.jsm`); - this.feature.shutdown(); + */ - // clean up our modules. - Cu.unload(CONFIGPATH); - Cu.unload(STUDYUTILSPATH); - } -} + // IF your study has an embedded webExtension, start it. + const { webExtension } = addonData; + if (webExtension) { + webExtension.startup().then(api => { + const { browser } = api; + /** spec for messages intended for Shield => + * {shield:true,msg=[info|endStudy|telemetry],data=data} + */ + browser.runtime.onMessage.addListener( + studyUtils.respondToWebExtensionMessage, + ); + // other browser.runtime.onMessage handlers for your addon, if any + }); + } -function uninstall(addonData, reason) { - log.debug("uninstall", REASONS[reason] || reason); -} + // start up the chrome-privileged part of the study + this.feature.start(); + }, -function install(addonData, reason) { - log.debug("install", REASONS[reason] || reason); - // handle ADDON_UPGRADE (if needful) here -} + initStudyUtils(id, version) { + // validate study config + studyUtils.setup({ ...config, addon: { id, version } }); + // TODO bdanforth: patch studyUtils to setLoggingLevel as part of setup method + studyUtils.setLoggingLevel(config.log.studyUtils.level); + }, + // choose the variation for this particular user, then set it. + async selectVariation() { + const variation = + this.getVariationFromPref(config.weightedVariations) || + (await studyUtils.deterministicVariation(config.weightedVariations)); + studyUtils.setVariation(variation); + this.log.debug(`studyUtils has config and variation.name: ${variation.name}. + Ready to send telemetry`); + return variation; + }, + + // helper to let Dev or QA set the variation name + getVariationFromPref(weightedVariations) { + const name = Services.prefs.getCharPref(this.VARIATION_OVERRIDE_PREF, ""); + if (name !== "") { + const variation = weightedVariations.filter(x => x.name === name)[0]; + if (!variation) { + throw new Error(`about:config => ${ + this.VARIATION_OVERRIDE_PREF + } set to ${name}, + but no variation with that name exists.`); + } + return variation; + } + return name; + }, + + /** + * Shutdown needs to distinguish between USER-DISABLE and other + * times that `endStudy` is called. + * + * studyUtils._isEnding means this is a '2nd shutdown'. + */ + async shutdown(addonData, reason) { + this.log.debug("shutdown", studyUtils.REASONS[reason] || reason); + + const isUninstall = + reason === studyUtils.REASONS.ADDON_UNINSTALL || + reason === studyUtils.REASONS.ADDON_DISABLE; + if (isUninstall) { + this.log.debug("uninstall or disable"); + } -// helper to let Dev or QA set the variation name -function getVariationFromPref(weightedVariations) { - const key = "shield.test.variation"; - const name = Services.prefs.getCharPref(key, ""); - if (name !== "") { - const variation = weightedVariations.filter(x => x.name === name)[0]; - if (!variation) { - throw new Error(`about:config => shield.test.variation set to ${name}, but not variation with that name exists`); + if (isUninstall && !studyUtils._isEnding) { + // we are the first 'uninstall' requestor => must be user action. + this.log.debug("probably: user requested shutdown"); + studyUtils.endStudy({ reason: "user-disable" }); } - return variation; - } - return name; // undefined -} + // normal shutdown, or 2nd uninstall request + // Run shutdown-related code in Feature.jsm + // We check if feature exists because it's possible the study is shutting down before it has instantiated the feature. Ex: if the user is ineligible or if the study has expired. + if (this.feature) { + await this.feature.shutdown(); + } + // Unload addon-specific modules + Cu.unload(`resource://${STUDY}/lib/Feature.jsm`); + Cu.unload(`resource://${STUDY}/Config.jsm`); + Cu.unload(`resource://${STUDY}/StudyUtils.jsm`); + }, + + uninstall(addonData, reason) { + this.log.debug("uninstall", reason); + }, + + install(addonData, reason) { + this.log.debug("install", reason); + // handle ADDON_UPGRADE (if needful) here + }, +}; + +// Expose bootstrap methods on the global +for (const methodName of ["install", "startup", "shutdown", "uninstall"]) { + this[methodName] = Bootstrap[methodName].bind(Bootstrap); +} diff --git a/addon/lib/AddonPrefs.jsm b/addon/lib/AddonPrefs.jsm deleted file mode 100644 index 55d6cf9..0000000 --- a/addon/lib/AddonPrefs.jsm +++ /dev/null @@ -1,52 +0,0 @@ -"use strict"; - -/** An Example JSM, implementing "addon-specific prefs" - * - * Note: This is an example JSM, not acutally used by this particular study. - */ - -/* eslint no-unused-vars: ["error", { "varsIgnorePattern": "(EXPORTED_SYMBOLS|AddonPrefs)" }]*/ -var EXPORTED_SYMBOLS = ["AddonPrefs"]; - -Components.utils.import("resource://gre/modules/Services.jsm"); - -const BASE_PREF = "extensions.original-bootstrap-addon-id."; - -function get(key, type = "char") { - key = BASE_PREF + key; - - switch (type) { - case "char": - return Services.prefs.getCharPref(key); - case "bool": - return Services.prefs.getBoolPref(key); - case "int": - return Services.prefs.getIntPref(key); - } - - throw new Error(`Unknown type: ${type}`); -} - -function set(key, type, value) { - key = BASE_PREF + key; - - switch (type) { - case "char": - return Services.prefs.setCharPref(key, value); - case "bool": - return Services.prefs.setBoolPref(key, value); - case "int": - return Services.prefs.setIntPref(key, value); - } - - throw new Error(`Unknown type: ${type}`); -} - -var AddonPrefs = { - get, set, -}; - - -// webpack:`libraryTarget: 'this'` -this.EXPORTED_SYMBOLS = EXPORTED_SYMBOLS; -this.AddonPrefs = AddonPrefs; diff --git a/addon/lib/Feature.jsm b/addon/lib/Feature.jsm index ed411f8..31e97f6 100644 --- a/addon/lib/Feature.jsm +++ b/addon/lib/Feature.jsm @@ -1,22 +1,21 @@ "use strict"; - /** Example Feature module for a Shield Study. - * - * UI: - * - during INSTALL only, show a notification bar with 2 buttons: - * - "Thanks". Accepts the study (optional) - * - "I don't want this". Uninstalls the study. - * - * Firefox code: - * - Implements the 'introduction' to the 'button choice' study, via notification bar. - * - * Demonstrates `studyUtils` API: - * - * - `telemetry` to instrument "shown", "accept", and "leave-study" events. - * - `endStudy` to send a custom study ending. - * - **/ + * + * UI: + * - during INSTALL only, show a notification bar with 2 buttons: + * - "Thanks". Accepts the study (optional) + * - "I don't want this". Uninstalls the study. + * + * Firefox code: + * - Implements the 'introduction' to the 'button choice' study, via notification bar. + * + * Demonstrates `studyUtils` API: + * + * - `telemetry` to instrument "shown", "accept", and "leave-study" events. + * - `endStudy` to send a custom study ending. + * + **/ /* eslint no-unused-vars: ["error", { "varsIgnorePattern": "(EXPORTED_SYMBOLS|Feature)" }]*/ @@ -27,12 +26,15 @@ Cu.import("resource://gre/modules/XPCOMUtils.jsm"); const EXPORTED_SYMBOLS = ["Feature"]; -XPCOMUtils.defineLazyModuleGetter(this, "RecentWindow", - "resource:///modules/RecentWindow.jsm"); +XPCOMUtils.defineLazyModuleGetter( + this, + "RecentWindow", + "resource:///modules/RecentWindow.jsm", +); /** Return most recent NON-PRIVATE browser window, so that we can - * maniuplate chrome elements on it. - */ + * manipulate chrome elements on it. + */ function getMostRecentBrowserWindow() { return RecentWindow.getMostRecentBrowserWindow({ private: false, @@ -40,49 +42,56 @@ function getMostRecentBrowserWindow() { }); } - class Feature { /** A Demonstration feature. - * - * - variation: study info about particular client study variation - * - studyUtils: the configured studyUtils singleton. - * - reasonName: string of bootstrap.js startup/shutdown reason - * - */ - constructor({variation, studyUtils, reasonName}) { - // unused. Some other UI might use the specific variation info for things. - this.variation = variation; + * + * - variation: study info about particular client study variation + * - studyUtils: the configured studyUtils singleton. + * - reasonName: string of bootstrap.js startup/shutdown reason + * + */ + constructor(variation, studyUtils, reasonName, log) { + this.variation = variation; // unused. Some other UI might use the specific variation info for things. this.studyUtils = studyUtils; + this.reasonName = reasonName; + this.log = log; + + // Example log statement + this.log.debug("Feature constructor"); + } + + start() { + this.log.debug("Feature start"); - // only during INSTALL - if (reasonName === "ADDON_INSTALL") { + // perform something only during INSTALL = a new study period begins + if (this.reasonName === "ADDON_INSTALL") { this.introductionNotificationBar(); } } /** Display instrumented 'notification bar' explaining the feature to the user - * - * Telemetry Probes: - * - * - {event: introduction-shown} - * - * - {event: introduction-accept} - * - * - {event: introduction-leave-study} - * - * Note: Bar WILL NOT SHOW if the only window open is a private window. - * - * Note: Handling of 'x' is not implemented. For more complete implementation: - * - * https://github.com/gregglind/57-perception-shield-study/blob/680124a/addon/lib/Feature.jsm#L148-L152 - * - */ + * + * Telemetry Probes: + * + * - {event: introduction-shown} + * + * - {event: introduction-accept} + * + * - {event: introduction-leave-study} + * + * Note: Bar WILL NOT SHOW if the only window open is a private window. + * + * Note: Handling of 'x' is not implemented. For more complete implementation: + * + * https://github.com/gregglind/57-perception-shield-study/blob/680124a/addon/lib/Feature.jsm#L148-L152 + * + */ introductionNotificationBar() { const feature = this; const recentWindow = getMostRecentBrowserWindow(); const doc = recentWindow.document; const notificationBox = doc.querySelector( - "#high-priority-global-notificationbox" + "#high-priority-global-notificationbox", ); if (!notificationBox) return; @@ -94,30 +103,32 @@ class Feature { null, // icon notificationBox.PRIORITY_INFO_HIGH, // priority // buttons - [{ - label: "Thanks!", - isDefault: true, - callback: function acceptButton() { - // eslint-disable-next-line no-console - console.log("clicked THANKS!"); - feature.telemetry({ - event: "introduction-accept", - }); + [ + { + label: "Thanks!", + isDefault: true, + callback: function acceptButton() { + // eslint-disable-next-line no-console + console.log("clicked THANKS!"); + feature.telemetry({ + event: "introduction-accept", + }); + }, }, - }, - { - label: "I do not want this.", - callback: function leaveStudyButton() { - // eslint-disable-next-line no-console - console.log("clicked NO!"); - feature.telemetry({ - event: "introduction-leave-study", - }); - feature.studyUtils.endStudy("introduction-leave-study"); + { + label: "I do not want this.", + callback: function leaveStudyButton() { + // eslint-disable-next-line no-console + console.log("clicked NO!"); + feature.telemetry({ + event: "introduction-leave-study", + }); + feature.studyUtils.endStudy("introduction-leave-study"); + }, }, - }], + ], // callback for nb events - null + null, ); // used by testing to confirm the bar is set with the correct config @@ -125,19 +136,19 @@ class Feature { feature.telemetry({ event: "introduction-shown", }); - } + /* good practice to have the literal 'sending' be wrapped up */ telemetry(stringStringMap) { this.studyUtils.telemetry(stringStringMap); } - /* no-op shutdown */ + /** + * Called at end of study, and if the user disables the study or it gets uninstalled by other means. + */ shutdown() {} } - - // webpack:`libraryTarget: 'this'` this.EXPORTED_SYMBOLS = EXPORTED_SYMBOLS; this.Feature = Feature; diff --git a/addon/webextension/.eslintrc.json b/addon/webextension/.eslintrc.json index 1d71c50..5c7cc9e 100644 --- a/addon/webextension/.eslintrc.json +++ b/addon/webextension/.eslintrc.json @@ -1,13 +1,11 @@ { - "env": { - "browser": true, - "es6": true, - "webextensions": true - }, - "extends": [ - "eslint:recommended" - ], - "rules": { - "no-console": "warn" - } + "env": { + "browser": true, + "es6": true, + "webextensions": true + }, + "extends": ["eslint:recommended"], + "rules": { + "no-console": "warn" + } } diff --git a/addon/webextension/background.js b/addon/webextension/background.js index 71dd8eb..979fa1b 100644 --- a/addon/webextension/background.js +++ b/addon/webextension/background.js @@ -3,111 +3,119 @@ "use strict"; /** `background.js` example for embedded webExtensions. - * - As usual for webExtensions, controls BrowserAction (toolbar button) - * look, feel, interactions. - * - * - Also handles 2-way communication with the HOST (Legacy Addon) - * - * - all communication to the Legacy Addon is via `browser.runtime.sendMessage` - * - * - Only the webExtension can initiate messages. see `msgStudy('info')` below. - */ - + * - As usual for webExtensions, controls BrowserAction (toolbar button) + * look, feel, interactions. + * + * - Also handles 2-way communication with the HOST (Legacy Addon) + * + * - all communication to the Legacy Addon is via `browser.runtime.sendMessage` + * + * - Only the webExtension can initiate messages. see `msgStudyUtils("info")` below. + */ /** Re-usable code for talking to `studyUtils` using `browser.runtime.sendMessage` - * - Host listens and responds at `bootstrap.js`: - * - * `browser.runtime.onMessage.addListener(studyUtils.respondToWebExtensionMessage)`; - * - * - `msg` calls the corresponding studyUtils API call. - * - * - info: current studyUtils configuration, including 'variation' - * - endStudy: for ending a study - * - telemetry: send a 'shield-study-addon' packet - */ + * - Host listens and responds at `bootstrap.js`: + * + * `browser.runtime.onMessage.addListener(studyUtils.respondToWebExtensionMessage)`; + * + * - `msg` calls the corresponding studyUtils API call. + * + * - info: current studyUtils configuration, including 'variation' + * - endStudy: for ending a study + * - telemetry: send a 'shield-study-addon' packet + */ async function msgStudyUtils(msg, data) { const allowed = ["endStudy", "telemetry", "info"]; - if (!allowed.includes(msg)) throw new Error(`shieldUtils doesn't know ${msg}, only knows ${allowed}`); + if (!allowed.includes(msg)) + throw new Error(`shieldUtils doesn't know ${msg}, only knows ${allowed}`); try { - // the 'shield' key is how the Host listener knows it's for shield. - const ans = await browser.runtime.sendMessage({shield: true, msg, data}); - return ans; + // the "shield" key is how the Host listener knows it's for shield. + return await browser.runtime.sendMessage({ shield: true, msg, data }); } catch (e) { console.error("ERROR msgStudyUtils", msg, data, e); - throw e + throw e; } } /** `telemetry` - * - * - check all pings for validity as 'shield-study-addon' pings - * - tell Legacy Addon to send - * - * Good practice: send all Telemetry from one function for easier - * logging, debugging, validation - * - * Note: kyes, values must be strings to fulfill the `shield-study-addon` - * ping-type validation. This allows `payload.data.attributes` to store - * correctly at Parquet at s.t.m.o. - * - * Bold claim: catching errors here - * - */ -function telemetry (data) { - function throwIfInvalid (obj) { + * + * - check all pings for validity as "shield-study-addon" pings + * - tell Legacy Addon to send + * + * Good practice: send all Telemetry from one function for easier + * logging, debugging, validation + * + * Note: kyes, values must be strings to fulfill the `shield-study-addon` + * ping-type validation. This allows `payload.data.attributes` to store + * correctly at Parquet at s.t.m.o. + * + * Bold claim: catching errors here + * + */ +function telemetry(data) { + function throwIfInvalid(obj) { // Check: all keys and values must be strings, for (const k in obj) { - if (typeof k !== 'string') throw new Error(`key ${k} not a string`); - if (typeof obj[k] !== 'string') throw new Error(`value ${k} ${obj[k]} not a string`); + if (typeof k !== "string") throw new Error(`key ${k} not a string`); + if (typeof obj[k] !== "string") + throw new Error(`value ${k} ${obj[k]} not a string`); } - return true + return true; } + throwIfInvalid(data); return msgStudyUtils("telemetry", data); } - class BrowserActionButtonChoiceFeature { /** - * - set image, text, click handler (telemetry) - * - tell Legacy Addon to send - */ + * - set image, text, click handler (telemetry) + * - tell Legacy Addon to send + */ constructor(variation) { - console.log("initilizing BrowserActionButtonChoiceFeature:", variation.name); + console.log( + "initilizing BrowserActionButtonChoiceFeature:", + variation.name, + ); this.timesClickedInSession = 0; // modify BrowserAction (button) ui for this particular {variation} - console.log("path:", `icons/${variation.name}.svg`) - browser.browserAction.setIcon({path: `icons/${variation.name}.svg`}); - browser.browserAction.setTitle({title: variation.name}); + console.log("path:", `icons/${variation.name}.svg`); + browser.browserAction.setIcon({ path: `icons/${variation.name}.svg` }); + browser.browserAction.setTitle({ title: variation.name }); browser.browserAction.onClicked.addListener(() => this.handleButtonClick()); } /** handleButtonClick - * - * - instrument browserAction button clicks - * - change label - */ + * + * - instrument browserAction button clicks + * - change label + */ handleButtonClick() { // note: doesn't persist across a session, unless you use localStorage or similar. this.timesClickedInSession += 1; console.log("got a click", this.timesClickedInSession); - browser.browserAction.setBadgeText({text: this.timesClickedInSession.toString()}); + browser.browserAction.setBadgeText({ + text: this.timesClickedInSession.toString(), + }); // telemetry: FIRST CLICK if (this.timesClickedInSession == 1) { - this.telemetry({"event": "button-first-click-in-session"}); + telemetry({ event: "button-first-click-in-session" }); } // telemetry EVERY CLICK - telemetry({"event": "button-click", timesClickedInSession: ""+this.timesClickedInSession}); + telemetry({ + event: "button-click", + timesClickedInSession: "" + this.timesClickedInSession, + }); // webExtension-initiated ending for "used-often" // // - 3 timesClickedInSession in a session ends the study. // - see `../Config.jsm` for what happens during this ending. if (this.timesClickedInSession >= 3) { - msgStudyUtils("endStudy", {reason: "used-often"}); + msgStudyUtils("endStudy", { reason: "used-often" }); } } } @@ -119,14 +127,16 @@ class BrowserActionButtonChoiceFeature { * 3. initialize the feature, using our specific variation */ function runOnce() { - msgStudyUtils("info").then( - ({variation}) => new BrowserActionButtonChoiceFeature(variation) - ).catch(function defaultSetup() { - // Errors here imply that this is NOT embedded. - console.log("you must be running as part of `web-ext`. You get 'corn dog'!"); - new BrowserActionButtonChoiceFeature({"name": "isolatedcorndog"}) - }); + msgStudyUtils("info") + .then(({ variation }) => new BrowserActionButtonChoiceFeature(variation)) + .catch(function defaultSetup() { + // Errors here imply that this is NOT embedded. + console.log( + "you must be running as part of `web-ext`. You get 'corn dog'!", + ); + new BrowserActionButtonChoiceFeature({ name: "isolatedcorndog" }); + }); } // actually start -runOnce() +runOnce(); diff --git a/addon/webextension/icons/Anonymous-Lizard.svg b/addon/webextension/icons/Anonymous-Lizard.svg deleted file mode 100644 index 395aaf4..0000000 --- a/addon/webextension/icons/Anonymous-Lizard.svg +++ /dev/null @@ -1,171 +0,0 @@ - - - - - - - - - - - - - - - - image/svg+xml - - - - - Openclipart - - - Lizard - 2009-04-04T06:59:01 - Lizard sign/symbol by Guillaume Boitel. From old OCAL site. - https://openclipart.org/detail/24022/lizard-by-anonymous-24022 - - - Anonymous - - - - - animal - lizard - outline - reptile - sign - symbol - - - - - - - - - - - diff --git a/addon/webextension/icons/DogHazard1.svg b/addon/webextension/icons/DogHazard1.svg deleted file mode 100644 index 15afc0e..0000000 --- a/addon/webextension/icons/DogHazard1.svg +++ /dev/null @@ -1,183 +0,0 @@ - - - - - - - - - - - - - - - image/svg+xml - - - - - Openclipart - - - Dog hazard 1 - 2010-12-06T18:00:19 - - https://openclipart.org/detail/99625/dog-hazard-1-by-rones - - - rones - - - - - caution - dog - dogs - hazard - humor - roadsign - sign - - - - - - - - - - - diff --git a/addon/webextension/icons/Grooming-Cat-Line-Art.svg b/addon/webextension/icons/Grooming-Cat-Line-Art.svg deleted file mode 100644 index 892b1b9..0000000 --- a/addon/webextension/icons/Grooming-Cat-Line-Art.svg +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/addon/LICENSE b/addon/webextension/icons/LICENSE similarity index 100% rename from addon/LICENSE rename to addon/webextension/icons/LICENSE diff --git a/bin/xpi.sh b/bin/xpi.sh index e5a6ac2..aff0743 100644 --- a/bin/xpi.sh +++ b/bin/xpi.sh @@ -47,6 +47,7 @@ popd > /dev/null echo echo "SUCCESS: xpi at ${BASE_DIR}/dist/${XPI_NAME}" echo "SUCCESS: symlinked xpi at ${BASE_DIR}/dist/linked-addon.xpi" +echo ls -alF "${BASE_DIR}"/dist diff --git a/docs/DEV.md b/docs/DEV.md new file mode 100644 index 0000000..a6be585 --- /dev/null +++ b/docs/DEV.md @@ -0,0 +1,158 @@ +# Developing this add-on + +### Preparations + +* Download a Developer and Nightly versions of Firefox (only Developer/Nightly will allow running unsigned legacy extensions, and Nightly is the default target for the automated tests) + +## Getting started + +```bash +# install dependencies +npm install + +## build +npm run eslint +npm run build + +## build and run +npm run firefox +``` + +## Details + +First, make sure you are on NPM 5+ installed so that the proper dependencies are installed using the package-lock.json file. + +`$ npm install -g npm` + +Clone the repo: + +`$ git clone https://github.com/mozilla/shield-studies-addon-template.git` + +After cloning the repo, you can run the following commands from the top level directory, one after another: + +``` +$ npm install +$ npm run build +``` + +This packages the add-on into an xpi file which is stored in `dist/`. This file is what you load into Firefox. + +## Loading the Web Extension in Firefox + +You can have Firefox automatically launched and the add-on installed by running: + +`$ npm run firefox` + +To load the extension manually instead, open (preferably) the [Developer Edition of Firefox](https://www.mozilla.org/firefox/developer/) and load the `.xpi` using the following steps: + +* Navigate to _about:config_ and set `extensions.legacy.enabled` to `true`. This permits the loading of the embedded Web Extension since new versions of Firefox are becoming restricted to pure Web Extensions only. +* Navigate to _about:debugging_ in your URL bar +* Select "Load Temporary Add-on" +* Find and select the latest xpi file you just built. + +## Seeing the add-on in action + +To debug installation and loading of the add-on: + +* Open the Browser Console using Firefox's top menu at `Tools > Web Developer > Browser Console`. This will display Shield (loading/telemetry) and log output from the add-on. + +See [TESTPLAN.md](./TESTPLAN.md) for more details on how to see this add-on in action and hot it is expected to behave. + +## Automated launch of Firefox with add-on installed + +`$ npm run firefox` starts Firefox and automatically installs the add-on in a new profile and opens the browser console automatically. + +Note: This runs in a recently created profile, and the study variation/branch is overridden by a preference in the FIREFOX_PREFERENCES section of `test/utils.js`. + +## Automated testing + +`npm run test` verifies the telemetry payload as expected at Firefox startup and add-on installation in a clean profile, then does **optimistic testing** of the _commonest path_ though the study for a user + +* prove the notification bar ui opens +* _clicking on the left-most button presented_. +* verifying that sent Telemetry is correct. + +Code at [/test/functional_test.js](/test/functional_test.js). + +Note: The functional tests are using async/await, so make sure you are running Node 7.6+ + +The functional testing set-up is imported from [https://github.com/mozilla/share-button-study]() which contains plenty of examples of functional tests relevant to Shield study addons. + +## Watch + +You can automatically build recent changes and package them into a `.xpi` by running the following from the top level directory: + +`$ npm run watch` + +Now, anytime a file is changed and saved, node will repackage the add-on. You must reload the add-on as before, or by clicking the "Reload" under the add-on in _about:debugging_. Note that a hard re-load is recommended to clear local storage. To do this, simply remove the add-on and reload as before. + +Note: This is currently only useful if you load the extension manually - it has no effect when running `npm run firefox`. + +## Directory Structure and Files + +``` +├── .circleci # setup for .circle ci integration +│   └── config.yml +├── .eslintignore +├── .eslintrc.js # mozilla, json +├── .gitignore +├── LICENSE +├── README.md # (this file) +├── addon # Files that will go into the addon +│   ├── Config.jsm # Study-specific configuration regarding branches, eligibility etc +│   ├── StudyUtils.jsm # (copied in during `prebuild`) +│   ├── bootstrap.js # LEGACY Bootstrap.js +│   ├── chrome.manifest # (derived from templates) +│   ├── icon.png +│   ├── install.rdf # (derived from templates) +│   ├── lib # JSM (Firefox modules) +│   │   └── Feature.jsm # contains study-specific privileged code +│   └── webextension # study-specific embedded webextension +│   ├── .eslintrc.json +│   ├── background.js +│   ├── icons +│   │   ├── LICENSE +│   │   ├── isolatedcorndog.svg +│   │   ├── kittens.svg +│   │   ├── lizard.svg +│   │   └── puppers.svg +│   └── manifest.json +├── bin # Scripts / commands +│   └── xpi.sh # build the XPI +├── dist # built xpis (addons) +│   ├── .gitignore +│   ├── @template-button-study.shield.mozilla.com-1.2.0.xpi +│   └── linked-addon.xpi -> @template-button-study.shield.mozilla.com-1.2.0.xpi +├── docs +│ ├── DEV.md +│ ├── TELEMETRY.md # Telemetry examples for this addon +│ ├── TESTPLAN.md # Manual QA test plan +│ └── WINDOWS_SETUP.md +├── package-lock.json +├── package.json +├── run-firefox.js # used by `npm run firefox` +├── templates # mustache templates, filled from `package.json` +│   ├── chrome.manifest.mustache +│   └── install.rdf.mustache +└── test # Automated tests `npm test` and circle + ├── Dockerfile + ├── docker_setup.sh + ├── functional_tests.js + ├── test_harness.js + ├── test_printer.py + └── utils.js + +>> tree -a -I 'node_modules|.git|.DS_Store' +``` + +This structure is set forth in [shield-studies-addon-template](https://github.com/mozilla/shield-studies-addon-template), with study-specific changes found mostly in `addon/lib`, `addon/webextension` and `addon/Config.jsm`. + +## General Shield Study Engineering + +Shield study add-ons are legacy (`addon/bootstrap.js`) add-ons with an optional embedded web extension (`addon/webextension/background.js`). + +The web extension needs to be packaged together with a legacy add-on in order to be able to access Telemetry data, user preferences etc that are required for collecting relevant data for [Shield Studies](https://wiki.mozilla.org/Firefox/Shield/Shield_Studies). + +It is recommended to build necessary logic and user interface using in the context of the web extension and communicate with the legacy add-on code through messaging whenever privileged access is required. + +For more information, see (especially ). diff --git a/TELEMETRY.md b/docs/TELEMETRY.md similarity index 64% rename from TELEMETRY.md rename to docs/TELEMETRY.md index 4e5d2d5..6abfe70 100644 --- a/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -1,42 +1,46 @@ -# Telemetry sent by Addon +# Telemetry sent by this add-on +## Usual Firefox Telemetry is unaffected. +* No change: `main` and other pings are UNAFFECTED by this add-on. +* Respects telemetry preferences. If user has disabled telemetry, no telemetry will be sent. -## Usual Firefox Telemetry is unaffected. +## Study-specific endings -- No change: `main` and other pings are UNAFFECTED by this addon. -- Respects telemetry preferences. If user has disabled telemetry, no telemetry will be sent. +This study has no surveys and as such has NO SPECIFIC ENDINGS. +The STUDY SPECIFIC ENDINGS this study supports are: +* "voted", +* "notification-x" +* "window-or-fx-closed" ## `shield-study` pings (common to all shield-studies) -`shield-studies-addon-utils` sends the usual packets. +[shield-studies-addon-utils](https://github.com/mozilla/shield-studies-addon-utils) sends the usual packets. -The STUDY SPECIFIC ENDINGS this study supports are: +## `shield-study-addon` pings, specific to THIS study. -- "voted", -- "notification-x" -- "window-or-fx-closed" +Events instrumented in this study: +* UI -## `shield-study-addon` pings, specific to THIS study. + * prompted (notification bar is shown) -Events instrumented in this study: +* Interactions + * voted -- UI - - prompted (notification bar is shown) +All interactions with the UI create sequences of Telemetry Pings. -- Interactions - - voted +All UI `shield-study` `study_state` sequences look like this: +* `enter => install => (one of: "voted" | "notification-x" | "window-or-fx-closed") => exit`. ## Example sequence for a 'voted => not sure' interaction These are the `payload` fields from all pings in the `shield-study` and `shield-study-addon` buckets. ``` - // common fields branch up-to-expectations-1 // should describe Question text @@ -81,5 +85,3 @@ version 3 "study_state": "exit" } ``` - - diff --git a/docs/TESTPLAN.md b/docs/TESTPLAN.md new file mode 100644 index 0000000..55b7e82 --- /dev/null +++ b/docs/TESTPLAN.md @@ -0,0 +1,133 @@ +# Test plan for this add-on + +## Manual / QA TEST Instructions + +### Preparations + +* Download a Release version of Firefox + +### Install the add-on and enroll in the study + +* (Create profile: , or via some other method) +* Navigate to _about:config_ and set the following preferences. (If a preference does not exist, create it be right-clicking in the white area and selecting New -> String or Integer depending on the type of preference) +* Set `extensions.legacy.enabled` to `true`. This permits the loading of the embedded Web Extension since new versions of Firefox are becoming restricted to pure Web Extensions only. +* Set `extensions.button_icon_preference.variation` to `kitten` (or any other study variation/branch to test specifically) +* Go to [this study's tracking bug](tbd: replace with your study's launch bug link in bugzilla) and install the latest signed XPI + +## Expected User Experience / Functionality + +Users see: + +* an icon in the browser address bar (webExtension BrowserAction) with one of 3 images (Cat, Dog, Lizard) + +Clicking on the button: + +* changes the badge +* sends telemetry + +ONCE ONLY users see: + +* a notification bar, introducing the featur +* allowing them to opt out + +Icon will be the same every run. + +If the user clicks on the badge more than 3 times, it ends the study. + +### Do these tests + +1. UI APPEARANCE. OBSERVE a notification bar with these traits: + + * Icon is 'heartbeat' + * Text is one of 8 selected "questions", such as: "Do you like Firefox?". These are listed in [/addon/Config.jsm](/addon/Config.jsm) as the variable `weightedVariations`. + * clickable buttons with labels 'yes | not sure | no' OR 'no | not sure | yes' (50/50 chance of each) + * an `x` button at the right that closes the notice. + + Test fails IF: + + * there is no bar. + * elements are not correct or are not displaye + +2. UI functionality: VOTE + + Expect: Click on a 'vote' button (any of: `yes | not sure | no`) has all these effects + + * notice closes + * add-on uninstalls + * no additional tabs open + * telemetry pings are 'correct' with this SPECIFIC `study_state` as the ending + + * ending is `voted` + * 'vote' is correct. + +3. UI functionality: 'X' button + + Click on the 'x' button. + + * notice closes + * add-on uninstalls + * no additional tabs open + * telemetry pings are 'correct' with this SPECIFIC ending + + * ending is `notification-x` + +4. UI functionality 'close window' + + 1. Open a 2nd Firefox window. + 2. Close the initial window. + + Then observe: + + * notice closes + * add-on uninstalls + * no additional tabs open + * telemetry pings are 'correct' with this SPECIFIC ending + + * ending is `window-or-fx-closed` + +5. UI functionality 'too-popular' + + * Click on the web extension's icon three times + * Verify that the study ends + * Verify that sent Telemetry is correct + * Verify that the user is sent to the URL specified in `addon/Config.jsm` under `endings -> too-popular`. + +### Design + +Any UI in a Shield study should be consistent with standard Firefox design specifications. These standards can be found at [design.firefox.com](https://design.firefox.com/photon/welcome.html). Firefox logo specifications can be found [here](https://design.firefox.com/photon/visuals/product-identity-assets.html). + +### Note: checking "sent Telemetry is correct" + +* Open the Browser Console using Firefox's top menu at `Tools > Web Developer > Browser Console`. This will display Shield (loading/telemetry) log output from the add-on. + +See [TELEMETRY.md](./TELEMETRY.md) for more details on what pings are sent by this add-on. + +## Debug + +To debug installation and loading of the add-on: + +* Open the Browser Console using Firefox's top menu at `Tools > Web Developer > Browser Console`. This will display Shield (loading/telemetry) and log output from the add-on. + +Example log output after installing the add-on: + +``` +install 5 bootstrap.js:125 +startup ADDON_INSTALL bootstrap.js:33 +info {"studyName":"mostImportantExperiment","addon":{"id":"template-shield-study@mozilla.com","version":"1.0.0"},"variation":{"name":"kittens"},"shieldId":"8bb19b5c-99d0-cc48-ba95-c73f662bd9b3"} bootstrap.js:67 +1508111525396 shield-study-utils DEBUG log made: shield-study-utils +1508111525398 shield-study-utils DEBUG setting up! +1508111525421 shield-study-utils DEBUG firstSeen +1508111525421 shield-study-utils DEBUG telemetry in: shield-study {"study_state":"enter"} +1508111525421 shield-study-utils DEBUG getting info +1508111525423 shield-study-utils DEBUG telemetry: {"version":3,"study_name":"mostImportantExperiment","branch":"kittens","addon_version":"1.0.0","shield_version":"4.1.0","type":"shield-study","data":{"study_state":"enter"},"testing":true} +1508111525430 shield-study-utils DEBUG startup 5 +1508111525431 shield-study-utils DEBUG getting info +1508111525431 shield-study-utils DEBUG marking TelemetryEnvironment: mostImportantExperiment +1508111525476 shield-study-utils DEBUG telemetry in: shield-study {"study_state":"installed"} +1508111525477 shield-study-utils DEBUG getting info +1508111525477 shield-study-utils DEBUG telemetry: {"version":3,"study_name":"mostImportantExperiment","branch":"kittens","addon_version":"1.0.0","shield_version":"4.1.0","type":"shield-study","data":{"study_state":"installed"},"testing":true} +1508111525479 shield-study-utils DEBUG getting info +1508111525686 shield-study-utils DEBUG getting info +1508111525686 shield-study-utils DEBUG respondingTo: info +init kittens background.js:29:5 +``` diff --git a/WINDOWS_SETUP.md b/docs/WINDOWS_SETUP.md similarity index 76% rename from WINDOWS_SETUP.md rename to docs/WINDOWS_SETUP.md index ac9ad03..d54ba08 100644 --- a/WINDOWS_SETUP.md +++ b/docs/WINDOWS_SETUP.md @@ -1,6 +1,6 @@ -# Windows Development and Testing +# Windows Development and Testing -The Shield Studies Addon Template makes some assumptions about your environment that can be challenging to meet on Windows machines. So far the most promising approach uses the **Windows Subsystem for Linux (WSL)**. WSL is a young project with bugs and unexpected pitfalls; caveat emptor. +The Shield Studies Add-on Template makes some assumptions about your environment that can be challenging to meet on Windows machines. So far the most promising approach uses the **Windows Subsystem for Linux (WSL)**. WSL is a young project with bugs and unexpected pitfalls; caveat emptor. ## Requirements @@ -9,19 +9,23 @@ The Shield Studies Addon Template makes some assumptions about your environment ## Installing WSL 1. [Follow Microsoft's official steps for installing WSL](https://answers.microsoft.com/en-us/insider/wiki/insider_wintp-insider_install/how-to-enable-the-windows-subsystem-for-linux/16e8f2e8-4a6a-4325-a89a-fd28c7841775?auth=1). These instructions are clear and detailed. _If you prefer, here is a TL;DR:_ - 1. Enable developer mode in Windows 10 in `Start > Settings > Update & security > For developers`. - 2. Enable the optional Windows feature, "Windows Subsystem for Linux" using `optionalfeatures.exe`. - 3. Restart. - 4. Type `bash` at the Windows command line and wait for Ubuntu to install. + + 1. Enable developer mode in Windows 10 in `Start > Settings > Update & security > For developers`. + 2. Enable the optional Windows feature, "Windows Subsystem for Linux" using `optionalfeatures.exe`. + 3. Restart. + 4. Type `bash` at the Windows command line and wait for Ubuntu to install. 2. Install a recent version of node.js: + ``` curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash - sudo apt-get install -y nodejs ``` + 3. Get the latest npm: `npm install npm@latest -g` 4. Install git and zip (and any other linux command-line tools you like): + ``` sudo apt install git sudo apt install zip @@ -43,4 +47,4 @@ sudo apt install zip * Windows-first developers, improve any/all of the above information. * Windows-first developers, help find workarounds to bugs encountered. -* Windows-first developers, script any of the above steps to improve this setup process. \ No newline at end of file +* Windows-first developers, script any of the above steps to improve this setup process. diff --git a/package-lock.json b/package-lock.json index af06f51..f30995e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "shield-studies-addon-template", - "version": "1.2.0", + "version": "1.3.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -216,6 +216,30 @@ "integrity": "sha1-MU3QpLM2j609/NxU7eYXG4htrzw=", "dev": true }, + "alce": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/alce/-/alce-1.0.0.tgz", + "integrity": "sha1-QmGEyY7iiNDurHf9Y/7WgLZnyrY=", + "dev": true, + "requires": { + "esprima": "1.0.4", + "estraverse": "1.3.2" + }, + "dependencies": { + "esprima": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz", + "integrity": "sha1-n1V+CPw7TSbs6d00+Pv0drYlha0=", + "dev": true + }, + "estraverse": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.3.2.tgz", + "integrity": "sha1-N8K4k+8T1yPydth41g2FNRUqbEI=", + "dev": true + } + } + }, "amqplib": { "version": "0.5.2", "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.5.2.tgz", @@ -1418,6 +1442,12 @@ } } }, + "deep-extend": { + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.2.11.tgz", + "integrity": "sha1-eha6aXKRMjQFBhcElLyD9wdv4I8=", + "dev": true + }, "deep-is": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", @@ -2630,6 +2660,12 @@ "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=", "dev": true }, + "extend-object": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/extend-object/-/extend-object-1.0.0.tgz", + "integrity": "sha1-QlFPhAFdE1bK9Rh5ad+yvBvaCCM=", + "dev": true + }, "external-editor": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.0.5.tgz", @@ -2758,6 +2794,18 @@ "readable-stream": "2.3.3" } }, + "fixpack": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/fixpack/-/fixpack-2.3.1.tgz", + "integrity": "sha1-U/A9iKq31RIyWSgvAIipo7GYNsI=", + "dev": true, + "requires": { + "alce": "1.0.0", + "colors": "1.1.2", + "extend-object": "1.0.0", + "rc": "0.6.0" + } + }, "flat-cache": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.2.2.tgz", @@ -6142,6 +6190,12 @@ "integrity": "sha1-gV7R9uvGWSb4ZbMQwHE7yzMVzks=", "dev": true }, + "prettier": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.10.2.tgz", + "integrity": "sha512-TcdNoQIWFoHblurqqU6d1ysopjq7UX0oRcT/hJ8qvBAELiYWn+Ugf0AXdnzISEJ7vuhNnQ98N8jR8Sh53x4IZg==", + "dev": true + }, "private": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", @@ -6268,6 +6322,32 @@ } } }, + "rc": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/rc/-/rc-0.6.0.tgz", + "integrity": "sha1-4ckwBZr4MchUE/4nWuL0D048U3E=", + "dev": true, + "requires": { + "deep-extend": "0.2.11", + "ini": "1.3.4", + "minimist": "0.0.10", + "strip-json-comments": "0.1.3" + }, + "dependencies": { + "minimist": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", + "integrity": "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=", + "dev": true + }, + "strip-json-comments": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-0.1.3.tgz", + "integrity": "sha1-Fkxk43Coo8wAyeAbU55WmCPw7lQ=", + "dev": true + } + } + }, "read-all-stream": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/read-all-stream/-/read-all-stream-3.1.0.tgz", diff --git a/package.json b/package.json index 41f7ca8..905ad1e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "shield-studies-addon-template", "description": "Template Shield Study", - "version": "1.2.0", + "version": "1.3.0", "author": "Mozilla Gregg Lind ", "addon": { "$ABOUT": "use these variables fill the moustache templates", @@ -31,6 +31,7 @@ "eslint-plugin-json": "^1.2.0", "eslint-plugin-mozilla": "^0.4.4", "eslint-plugin-no-unsanitized": "^2.0.1", + "fixpack": "^2.3.1", "fs-extra": "^3.0.1", "fx-runner": "^1.0.6", "geckodriver": "^1.7.1", @@ -42,14 +43,18 @@ "npm-run-all": "^4.1.1", "nsp": "^2.8.1", "onchange": "^3.2.1", + "prettier": "^1.10.2", "selenium-webdriver": "^3.5.0", "shield-studies-addon-utils": "^4.1.0" }, + "engines": { + "node": ">=8.9.0" + }, "homepage": "http://github.com/mozilla/shield-studies-addon-template", "keywords": [ - "mozilla", - "legacy-addon", "firefox", + "legacy-addon", + "mozilla", "shield-study" ], "license": "MIT", @@ -61,13 +66,17 @@ "scripts": { "build": "bash ./bin/xpi.sh", "eslint": "eslint . --ext jsm --ext js --ext json", + "eslint-fix": "npm run eslint -- --fix", + "firefox": "export XPI=dist/linked-addon.xpi && npm run build && node run-firefox.js", + "format": "prettier '**/*.{css,js,json,jsm,md}' --trailing-comma=all --ignore-path=.eslintignore --write", + "postformat": "npm run eslint-fix", + "harness_test": "export XPI=dist/linked-addon.xpi && mocha test/functional_tests.js --retry 2 --reporter json", "lint": "npm-run-all lint:*", - "lint:addons-linter": "# actually a post build test: bin/addonLintTest ' + require('./package.json').name", + "lint-build:addons-linter": "# actually a post build test: bin/addonLintTest ' + require('./package.json').name", + "lint:addons-linter": "addons-linter addon/webextension/", "lint:eslint": "npm run eslint", "lint:fixpack": "fixpack", "lint:nsp": "nsp check", - "firefox": "export XPI=dist/linked-addon.xpi && npm run build && node run-firefox.js", - "harness_test": "export XPI=dist/linked-addon.xpi && mocha test/functional_tests.js --retry 2 --reporter json", "prebuild": "cp node_modules/shield-studies-addon-utils/dist/StudyUtils.jsm addon/", "sign": "echo 'TBD, see: https://bugzilla.mozilla.org/show_bug.cgi?id=1407757'", "test": "export XPI=dist/linked-addon.xpi && npm run build && mocha test/functional_tests.js --retry 2", diff --git a/run-firefox.js b/run-firefox.js index 302a5ab..474ab14 100644 --- a/run-firefox.js +++ b/run-firefox.js @@ -9,6 +9,8 @@ * reloading, as the .xpi file has not been recreated. */ +console.log("Starting up firefox"); + const firefox = require("selenium-webdriver/firefox"); const path = require("path"); const Context = firefox.Context; @@ -22,7 +24,6 @@ const { MODIFIER_KEY, } = require("./test/utils"); - const HELP = ` env vars: @@ -42,12 +43,11 @@ Future will clean up this interface a bit! `; const minimistHandler = { - boolean: [ "help" ], + boolean: ["help"], alias: { h: "help", v: "version" }, "--": true, }; - (async() => { const minimist = require("minimist"); const parsedArgs = minimist(process.argv.slice(2), minimistHandler); @@ -58,7 +58,7 @@ const minimistHandler = { try { const driver = await promiseSetupDriver(); - console.log("Starting up firefox"); + console.log("Firefox started"); // install the addon if (process.env.XPI) { @@ -78,6 +78,9 @@ const minimistHandler = { const openBrowserConsole = Key.chord(MODIFIER_KEY, Key.SHIFT, "j"); await urlBar.sendKeys(openBrowserConsole); + console.log( + "The addon should now be loaded and you should be able to interact with the addon in the newly opened Firefox instance.", + ); } catch (e) { console.error(e); } diff --git a/survival.md b/survival.md deleted file mode 100644 index d5d4de6..0000000 --- a/survival.md +++ /dev/null @@ -1,173 +0,0 @@ -Surviving on shield island - -You are a wizened and skillful dev. Perhaps you have built a 99.99% uptime website serving millions of users, or you built the firefox awesomebar.... but can you SURVIVE SHIELD ISLAND? - -> Inventory - -- Firefox nightly, firefox beta, firefox (release) -- npm -- git / github -- shield-studies-addon-utils (studyUtils.jsm) - - -> Look - -DARK CLOUDS block the sun. - -On the beach is a HANDBOOK here. - -There are TELEMETRY PROBES here. - -There is a DEVELOPMENT ENVIROMENT here. - -> put on shield-study-addon-utils - -StudyUtils.jsm worn. - -You feel optimistic, despite the long odds of survival. - -You can: -- TELEMETRY: send well-formatted probes -- SETVARIATION: use the studyName and Telemetry client to consistently and determistically to assign the client to a particular branch - -(see TODO) - -> use studyUtils to set variation - -> - - - - - -> examine dark clouds - -As you look up, a light rain starts to fall. You wonder how you will make a fire to keep warm, as your clothes start to soak. The first chills of hypothermia ripple your flesh. - -Legacy addon development has challenges. - - -> examine handbook. - -(taking handbook) - -SURVIVAL IN HARD ENVIROMENTS - -by Lief Savor - -Remember that your SURVIVAL GOAL is to... - -Get DATA to Telemetry -About user actions -so that you can make decisions about WHICH APPROACH. - - -## Telemetry First Development - -Everything in a SHIELD STUDY leads to allowing ANALYSTS to get data quickly, reliably, and consistently so that they can do analysis of this form: - -- for Which VARIATION of a feature -- did users 'do best'. - -Your SHIELD-STUDY Legacy Addon is a DELIVERY MECHANISM to collect that data. - - -An example analysis table - -An exmple telemetry `shield-study-addon` probe that contributes to that. - -Here is the SQL. - - -## An EMPTY STUDY. - -> use flint and steel to make fire - -## Do I - - - -## But I like making User Interface? - -Don't we all?! Mocks are fun! Styling is fun! - -starting with Telemetry makes soe of the... unexpected decisions make sense. - -## Wait, did you say legacy addon? - -Yes. Web Extensions CAN'T SEND TELEMETRY. We need Firefox (chrome) privileges to access the `TelemetryController.jsm`. That meanss - -> make webExtension - -Good idea, for some UI's. - -## But I have been buildling UI in pure Legacy Extensions since Firefox 2. - -Awesome work! Firebug was awesome. You have no further use of this guide, and should go to [TODO:Shield-Studies-Addon-Utils-api.md]. - -## Tools and inventory - -### > x template - -We have a template folder at TODO:template. The files are... - -The template shows an EMBEDDED WEB EXTENSION with -- build scripts - -``` -the file tree -``` - -## Part 1, instrumenting buttons in an embedded web extension. - - -### Action and Probes - -Pretend story: which of several buttons is the most compelling to firefox users. - -Good news: probes are mostly plain-old-javascrpt-objects. - -Back news, getting - - -### side-by-side deployment - -### building 2 buttons. - -This part is EASY using the webExtension - -`manifest.json` - -Getting the probes to firefox - - - -"At least both branches are equally bad": A plea for experimental controls - - -FInding shelter - -Send message / signal mirror / telemetry? - - -> go woods - -A monkey appears. It is curious about you - -(Helper addon for QA telemetry) - - - - -## Full List of All Shield Telemetry Spoilers - - - - -## Shield Study Utils Api - - - - - - diff --git a/test/Dockerfile b/test/Dockerfile index b2bb47c..208303f 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:16.04 -WORKDIR /share-button-study +WORKDIR /shield-study RUN apt-get update -y && \ apt-get install -y curl && \ @@ -11,4 +11,4 @@ RUN apt-get update -y && \ npm install -g get-firefox && \ get-firefox -b beta -t /firefox.tar.bz2 && tar -xvjf /firefox.tar.bz2 -C / -ENV PATH="/share-button-study/node_modules/.bin:$PATH" +ENV PATH="/shield-study/node_modules/.bin:$PATH" diff --git a/test/functional_tests.js b/test/functional_tests.js index 07d5076..bbc483e 100644 --- a/test/functional_tests.js +++ b/test/functional_tests.js @@ -17,11 +17,15 @@ const utils = require("./utils"); /* Part 1: Utilities */ async function getShieldPingsAfterTimestamp(driver, ts) { - return utils.getTelemetryPings(driver, {type: ["shield-study", "shield-study-addon"], timestamp: ts}); + return utils.getTelemetryPings(driver, { + type: ["shield-study", "shield-study-addon"], + timestamp: ts, + }); } -function summarizePings(pings) { return pings.map(p => [p.payload.type, p.payload.data]); } - +function summarizePings(pings) { + return pings.map(p => [p.payload.type, p.payload.data]); +} async function getNotification(driver) { return utils.getChromeElementBy.tagName(driver, "notification"); @@ -31,7 +35,6 @@ async function getFirstButton(driver) { return utils.getChromeElementBy.className(driver, "notification-button"); } - /* Part 2: The Tests */ describe("basic functional tests", function() { @@ -56,7 +59,6 @@ describe("basic functional tests", function() { // collect sent pings pings = await getShieldPingsAfterTimestamp(driver, beginTime); // console.log(pingsReport(pings).report); - }); after(async() => { @@ -81,41 +83,57 @@ describe("basic functional tests", function() { }); it("at least one shield-study telemetry ping with study_state=installed", async() => { - const foundPings = utils.searchTelemetry([ - ping => ping.type === "shield-study" && ping.payload.data.study_state === "installed", - ], pings); - assert(foundPings.length > 0, "at least one shield-study telemetry ping with study_state=installed"); + const foundPings = utils.searchTelemetry( + [ + ping => + ping.type === "shield-study" && + ping.payload.data.study_state === "installed", + ], + pings, + ); + assert( + foundPings.length > 0, + "at least one shield-study telemetry ping with study_state=installed", + ); }); it("at least one shield-study telemetry ping with study_state=enter", async() => { - const foundPings = utils.searchTelemetry([ - ping => ping.type === "shield-study" && ping.payload.data.study_state === "enter", - ], pings); - assert(foundPings.length > 0, "at least one shield-study telemetry ping with study_state=enter"); + const foundPings = utils.searchTelemetry( + [ + ping => + ping.type === "shield-study" && + ping.payload.data.study_state === "enter", + ], + pings, + ); + assert( + foundPings.length > 0, + "at least one shield-study telemetry ping with study_state=enter", + ); }); - it("telemetry: has entered, installed, shown", function() { + it("telemetry: has entered, installed, etc", function() { // Telemetry: order, and summary of pings is good. const observed = summarizePings(pings); const expected = [ [ "shield-study-addon", { - "attributes": { - "event": "introduction-shown", + attributes: { + event: "introduction-shown", }, }, ], [ "shield-study", { - "study_state": "installed", + study_state: "installed", }, ], [ "shield-study", { - "study_state": "enter", + study_state: "enter", }, ], ]; @@ -125,7 +143,9 @@ describe("basic functional tests", function() { describe("introduction / orientation bar", function() { it("exists, carries study config", async() => { const notice = await getNotification(driver); - const noticeConfig = JSON.parse(await notice.getAttribute("data-study-config")); + const noticeConfig = JSON.parse( + await notice.getAttribute("data-study-config"), + ); assert(noticeConfig.name); assert(noticeConfig.weight); }); @@ -149,15 +169,14 @@ describe("basic functional tests", function() { [ "shield-study-addon", { - "attributes": { - "event": "introduction-accept", + attributes: { + event: "introduction-accept", }, }, ], ]; // this would add new telemetry assert.deepEqual(expected, observed, "telemetry pings do not match"); - }); it("TBD click on NO uninstalls addon", async() => { diff --git a/test/test_harness.js b/test/test_harness.js index 0b332d5..10afea1 100644 --- a/test/test_harness.js +++ b/test/test_harness.js @@ -13,21 +13,21 @@ const { spawn } = require("child_process"); // Promise wrapper around childProcess.spawn() function spawnProcess(command, args) { - return new Promise((resolve) => { + return new Promise(resolve => { const childProcess = spawn(command, args); const stderrArray = []; const stdoutArray = []; - childProcess.stdout.on("data", (data) => { + childProcess.stdout.on("data", data => { stdoutArray.push(data.toString()); // data is of type Buffer }); - childProcess.stderr.on("data", (data) => { + childProcess.stderr.on("data", data => { // TODO reject upon error? stderrArray.push(data.toString()); // data is of type Buffer }); - childProcess.on("close", (code) => { + childProcess.on("close", code => { // TODO reject upon error? console.log("Test suite completed."); resolve({ code, stdoutArray, stderrArray }); @@ -44,7 +44,9 @@ async function main() { console.log(`Currently running test suite #${i}.`); const childProcesses = []; // NOTE Parallel tests seem to introduce more errors. - childProcesses.push(spawnProcess("npm", ["run", "--silent", "harness_test"])); + childProcesses.push( + spawnProcess("npm", ["run", "--silent", "harness_test"]), + ); // TODO Promise.all() will reject upon a single error, is this an issue? try { @@ -57,11 +59,13 @@ async function main() { const mochaOutput = JSON.parse(rawOutput.join("")); for (const failedTest of mochaOutput.failures) { console.log(failedTest.err); - if (!(failedTestCounts.has(failedTest.fullTitle))) { + if (!failedTestCounts.has(failedTest.fullTitle)) { failedTestCounts.set(failedTest.fullTitle, 0); } - failedTestCounts.set(failedTest.fullTitle, - failedTestCounts.get(failedTest.fullTitle) + 1); + failedTestCounts.set( + failedTest.fullTitle, + failedTestCounts.get(failedTest.fullTitle) + 1, + ); } } catch (e) { console.log(`JSON parsing error: ${e}`); @@ -74,12 +78,15 @@ async function main() { } console.log(failedTestCounts); for (const pair of failedTestCounts) { - fs.appendFile(`test_harness_output_${new Date().toISOString()}.txt`, `${pair[0]}: ${pair[1]}\n`, - (err) => { + fs.appendFile( + `test_harness_output_${new Date().toISOString()}.txt`, + `${pair[0]}: ${pair[1]}\n`, + err => { if (err) { console.log(`fs.writeFile errror: ${err}`); } - }); + }, + ); } } diff --git a/test/utils.js b/test/utils.js index c0dc0df..5a538c8 100644 --- a/test/utils.js +++ b/test/utils.js @@ -35,13 +35,16 @@ const FIREFOX_PREFERENCES = { // NECESSARY for all 57+ builds "extensions.legacy.enabled": true, + // Force variation for testing + "extensions.button_icon_preference.variation": "kittens", + /** WARNING: gecko webdriver sets many additional prefs at: - * https://dxr.mozilla.org/mozilla-central/source/testing/geckodriver/src/prefs.rs - * - * In, particular, this DISABLES actual telemetry uploading - * ("toolkit.telemetry.server", Pref::new("https://%(server)s/dummy/telemetry/")), - * - */ + * https://dxr.mozilla.org/mozilla-central/source/testing/geckodriver/src/prefs.rs + * + * In, particular, this DISABLES actual telemetry uploading + * ("toolkit.telemetry.server", Pref::new("https://%(server)s/dummy/telemetry/")), + * + */ }; // useful if we need to test on a specific version of Firefox @@ -60,14 +63,14 @@ async function promiseActualBinary(binary) { } /** - * Uses process.env.FIREFOX_BINARY - * - */ + * Uses process.env.FIREFOX_BINARY + * + */ module.exports.promiseSetupDriver = async() => { const profile = new firefox.Profile(); // TODO, allow 'actually send telemetry' here. - Object.keys(FIREFOX_PREFERENCES).forEach((key) => { + Object.keys(FIREFOX_PREFERENCES).forEach(key => { profile.setPreference(key, FIREFOX_PREFERENCES[key]); }); @@ -79,7 +82,9 @@ module.exports.promiseSetupDriver = async() => { .forBrowser("firefox") .setFirefoxOptions(options); - const binaryLocation = await promiseActualBinary(process.env.FIREFOX_BINARY || "nightly"); + const binaryLocation = await promiseActualBinary( + process.env.FIREFOX_BINARY || "nightly", + ); // console.log(binaryLocation); await options.setBinary(new firefox.Binary(binaryLocation)); @@ -90,25 +95,24 @@ module.exports.promiseSetupDriver = async() => { return driver; }; - /* let's actually just make this a constant */ const MODIFIER_KEY = (function getModifierKey() { - const modifierKey = process.platform === "darwin" ? - webdriver.Key.COMMAND : webdriver.Key.CONTROL; + const modifierKey = + process.platform === "darwin" + ? webdriver.Key.COMMAND + : webdriver.Key.CONTROL; return modifierKey; })(); module.exports.MODIFIER_KEY = MODIFIER_KEY; - // TODO glind general wrapper for 'async with callback'? - /* Firefox UI helper functions */ // such as: "social-share-button" module.exports.addButtonFromCustomizePanel = async(driver, buttonId) => - driver.executeAsyncScript((callback) => { + driver.executeAsyncScript(callback => { // see https://dxr.mozilla.org/mozilla-central/rev/211d4dd61025c0a40caea7a54c9066e051bdde8c/browser/base/content/browser-social.js#193 Components.utils.import("resource:///modules/CustomizableUI.jsm"); CustomizableUI.addWidgetToArea(buttonId, CustomizableUI.AREA_NAVBAR); @@ -117,7 +121,7 @@ module.exports.addButtonFromCustomizePanel = async(driver, buttonId) => module.exports.removeButtonFromNavbar = async(driver, buttonId) => { try { - await driver.executeAsyncScript((callback) => { + await driver.executeAsyncScript(callback => { Components.utils.import("resource:///modules/CustomizableUI.jsm"); CustomizableUI.removeWidgetFromArea(buttonId); callback(); @@ -130,7 +134,7 @@ module.exports.removeButtonFromNavbar = async(driver, buttonId) => { if (e.name === "TimeoutError") { return false; } - throw (e); + throw e; } }; @@ -141,17 +145,29 @@ module.exports.installAddon = async(driver, fileLocation) => { fileLocation = fileLocation || path.join(process.cwd(), process.env.XPI); const executor = driver.getExecutor(); - executor.defineCommand("installAddon", "POST", "/session/:sessionId/moz/addon/install"); + executor.defineCommand( + "installAddon", + "POST", + "/session/:sessionId/moz/addon/install", + ); const installCmd = new cmd.Command("installAddon"); const session = await driver.getSession(); - installCmd.setParameters({ sessionId: session.getId(), path: fileLocation, temporary: true }); + installCmd.setParameters({ + sessionId: session.getId(), + path: fileLocation, + temporary: true, + }); return executor.execute(installCmd); }; module.exports.uninstallAddon = async(driver, id) => { const executor = driver.getExecutor(); - executor.defineCommand("uninstallAddon", "POST", "/session/:sessionId/moz/addon/uninstall"); + executor.defineCommand( + "uninstallAddon", + "POST", + "/session/:sessionId/moz/addon/uninstall", + ); const uninstallCmd = new cmd.Command("uninstallAddon"); const session = await driver.getSession(); @@ -159,7 +175,6 @@ module.exports.uninstallAddon = async(driver, id) => { await executor.execute(uninstallCmd); }; - /* this is NOT WORKING FOR UNKNOWN HARD TO EXLAIN REASONS => Uncaught WebDriverError: InternalError: too much recursion module.exports.allAddons = async(driver) => { @@ -173,20 +188,20 @@ module.exports.allAddons = async(driver) => { */ /** Returns array of pings of type `type` in reverse sorted order by timestamp - * first element is most recent ping - * - * as seen in shield-study-addon-util's `utils.jsm` - * options - * - type: string or array of ping types - * - n: positive integer. at most n pings. - * - timestamp: only pings after this timestamp. - * - headersOnly: boolean, just the 'headers' for the pings, not the full bodies. - */ + * first element is most recent ping + * + * as seen in shield-study-addon-util's `utils.jsm` + * options + * - type: string or array of ping types + * - n: positive integer. at most n pings. + * - timestamp: only pings after this timestamp. + * - headersOnly: boolean, just the 'headers' for the pings, not the full bodies. + */ module.exports.getTelemetryPings = async(driver, passedOptions) => { // callback is how you get the return back from the script return driver.executeAsyncScript(async(options, callback) => { - let {type} = options; - const { n, timestamp, headersOnly} = options; + let { type } = options; + const { n, timestamp, headersOnly } = options; Components.utils.import("resource://gre/modules/TelemetryArchive.jsm"); // {type, id, timestampCreated} let pings = await TelemetryArchive.promiseArchivedPingList(); @@ -201,49 +216,57 @@ module.exports.getTelemetryPings = async(driver, passedOptions) => { pings.sort((a, b) => b.timestampCreated - a.timestampCreated); if (n) pings = pings.slice(0, n); - const pingData = headersOnly ? pings : pings.map(ping => TelemetryArchive.promiseArchivedPingById(ping.id)); + const pingData = headersOnly + ? pings + : pings.map(ping => TelemetryArchive.promiseArchivedPingById(ping.id)); callback(await Promise.all(pingData)); }, passedOptions); }; - - - // TODO glind, this interface feels janky // this feels like it wants to be $ like. // not obvious right now, moving on! class getChromeElementBy { - static async _get1(driver, method, selector ) { + static async _get1(driver, method, selector) { driver.setContext(Context.CHROME); try { - return await driver.wait(until.elementLocated( - By[method](selector)), 1000); + return await driver.wait( + until.elementLocated(By[method](selector)), + 1000, + ); } catch (e) { // if there an error, the button was not found console.error(e); return null; } } - static async id(driver, id) { return this._get1(driver, "id", id); } + static async id(driver, id) { + return this._get1(driver, "id", id); + } - static async className(driver, className) { return this._get1(driver, "className", className); } + static async className(driver, className) { + return this._get1(driver, "className", className); + } - static async tagName(driver, tagName) { return this._get1(driver, "tagName", tagName); } + static async tagName(driver, tagName) { + return this._get1(driver, "tagName", tagName); + } } module.exports.getChromeElementBy = getChromeElementBy; -module.exports.promiseUrlBar = (driver) => { +module.exports.promiseUrlBar = driver => { driver.setContext(Context.CHROME); - return driver.wait(until.elementLocated( - By.id("urlbar")), 1000); + return driver.wait(until.elementLocated(By.id("urlbar")), 1000); }; -module.exports.takeScreenshot = async(driver, filepath = "./screenshot.png") => { +module.exports.takeScreenshot = async( + driver, + filepath = "./screenshot.png", +) => { try { const data = await driver.takeScreenshot(); - return await Fs.outputFile(filepath, - data, "base64"); + return await Fs.outputFile(filepath, data, "base64"); } catch (screenshotError) { throw screenshotError; } @@ -269,7 +292,9 @@ module.exports.searchTelemetry = (conditionArray, telemetryArray) => { const resultingPings = []; for (const condition of conditionArray) { const index = telemetryArray.findIndex(ping => condition(ping)); - if (index === -1) { throw new SearchError(condition); } + if (index === -1) { + throw new SearchError(condition); + } resultingPings.push(telemetryArray[index]); } return resultingPings; @@ -322,7 +347,6 @@ module.exports.searchTelemetry = (conditionArray, telemetryArray) => { // } // }; - // module.exports.testPanel = async(driver, panelId) => { // driver.setContext(Context.CHROME); // try { // if we can't find the panel, return false @@ -345,7 +369,6 @@ module.exports.searchTelemetry = (conditionArray, telemetryArray) => { // } // }; - // module.exports.closePanel = async(driver, target = null) => { // if (target !== null) { // target.sendKeys(webdriver.Key.ESCAPE); diff --git a/tutorial.md b/tutorial.md deleted file mode 100644 index 2bd3620..0000000 --- a/tutorial.md +++ /dev/null @@ -1,71 +0,0 @@ -# Tutorial Study: Fake button study. - - -Learner goals: -1. build a shield-instrumented Embedded Web Extension -2. understand how probes, analysis, code relate - - -Concepts: -- technical terms - - web extension - - embedded web extension - - Legacy (bootstrap) addon -- debugging techniques - - `web-ext` - - `about:debugging` - - `run-firefox.js` - - -## Questions? - -- start from empty dir, or from the finished project? -- how much to build in the first step. -- how to display the order of the steps? - -(Steps, order unclear) - -## project goal - -## Analysis - -## Engineering steps - -### WebExtension with a button - -### Multiple Button Choices - -Make a class that has a startup at random. - -### Send Message for Telemetry - -Have a sendMessage language for this. - -Work on TELEMETRY.md - -### Embedded WebExtension - -Plumb the listener - -### incorporating shield to listen - -- PERMANENT Randomization, revisited. -- Telemetry sending -- debugging -- helper addon - -## Advanced - -### Non embedded webExtension - -## good probes - -- good probes mirror the analysis. -- Analysis wants to be sql. Try to make the probes reflect that. - -## Why Shield-Utils? - -## surveys? - - -