Remove obsolete 'experimentalAlignedExceptEdge' admin flag

shared/test_run_filter.go

@@ -119,23 +119,6 @@ func (filter TestRunFilter) OrExperimentalRuns() TestRunFilter {
 	return filter
 }
 
-// OrAlignedExperimentalRunsExceptEdge returns the current filter, or, if it is a default
-// query, returns a query for the latest experimental runs.
-func (filter TestRunFilter) OrAlignedExperimentalRunsExceptEdge() TestRunFilter {
-	if !filter.IsDefaultQuery() {
-		return filter
-	}
-	aligned := true
-	filter.Aligned = &aligned
-	filter.Products = GetDefaultProducts()
-	for i := range filter.Products {
-		if filter.Products[i].BrowserName != "edge" {
-			filter.Products[i].Labels = mapset.NewSetWith("experimental")
-		}
-	}
-	return filter
-}
-
 // MasterOnly returns the current filter, ensuring it has with the master-only
 // restriction (a label of "master").
 func (filter TestRunFilter) MasterOnly() TestRunFilter {

util/populate_dev_data.go

@@ -89,6 +89,11 @@ func main() {
 	edge.ResultsURL = fmt.Sprintf(summaryURLFmtString, *localHost, staticRunSHA[:10], "edge[stable].json")
 	edge.Labels = []string{"edge", shared.StableLabel}
 
+	edgeExp := edge
+	edgeExp.BrowserVersion = "20"
+	edgeExp.ResultsURL = strings.Replace(edge.ResultsURL, "[stable]", "[experimental]", -1)
+	edgeExp.Labels = []string{"edge", shared.ExperimentalLabel}
+
 	firefox := chrome
 	firefox.BrowserName = "firefox"
 	firefox.BrowserVersion = "66"
@@ -115,6 +120,7 @@ func main() {
 		chrome,
 		chromeExp,
 		edge,
+		edgeExp,
 		firefox,
 		firefoxExp,
 		safari,
@@ -160,7 +166,7 @@ func main() {
 	addFlag(store, "diffFilter", enabledFlag)
 	addFlag(store, "diffFromAPI", enabledFlag)
 	addFlag(store, "experimentalByDefault", enabledFlag)
-	addFlag(store, "experimentalAlignedExceptEdge", enabledFlag)
+	addFlag(store, "experimentalAligned", enabledFlag)
 	addFlag(store, "structuredQueries", enabledFlag)
 	addFlag(store, "diffRenames", enabledFlag)
 	addFlag(store, "paginationTokens", enabledFlag)

webapp/components/wpt-flags.js

@@ -42,7 +42,6 @@ Object.defineProperty(wpt, 'ClientSideFeatures', {
       'displayMetadata',
       'experimentalByDefault',
       'experimentalAligned',
-      'experimentalAlignedExceptEdge',
       'fetchManifestForTestList',
       'githubCommitLinks',
       'githubLogin',
@@ -349,11 +348,6 @@ class WPTEnvironmentFlagsEditor extends FlagsEditorClass(/*environmentFlags*/ tr
         Align the default experimental runs
       </paper-checkbox>
     </paper-item>
-    <paper-item sub-item>
-      <paper-checkbox checked="{{experimentalAlignedExceptEdge}}">
-        All experimental, except edge[stable], and aligned
-      </paper-checkbox>
-    </paper-item>
     <paper-item>
       <paper-checkbox checked="{{paginationTokens}}">
         Return "wpt-next-page" pagination token HTTP header in /api/runs

webapp/test_results_handler.go

@@ -105,14 +105,10 @@ func populateHomepageData(r *http.Request) (data homepageData, err error) {
 	} else {
 		if pr == nil && testRunFilter.IsDefaultQuery() {
 			if aeAPI.IsFeatureEnabled("experimentalByDefault") {
-				if aeAPI.IsFeatureEnabled("experimentalAlignedExceptEdge") {
-					testRunFilter = testRunFilter.OrAlignedExperimentalRunsExceptEdge()
-				} else {
-					testRunFilter = testRunFilter.OrExperimentalRuns()
-					if aeAPI.IsFeatureEnabled("experimentalAligned") {
-						aligned := true
-						testRunFilter.Aligned = &aligned
-					}
+				testRunFilter = testRunFilter.OrExperimentalRuns()
+				if aeAPI.IsFeatureEnabled("experimentalAligned") || true {
+					aligned := true
+					testRunFilter.Aligned = &aligned
 				}
 			} else {
 				testRunFilter = testRunFilter.OrAlignedStableRuns()

webdriver/README.md

@@ -1,17 +1,130 @@
 # wpt.fyi Webdriver tests
 
-This directory covers Webdriver tests for the `webapp`, written in a 3rd-party
-Golang Webdriver client, [tebeka/selenium](https://github.com/tebeka/selenium).
+This directory containers integration tests for webapp/. These tests bring up
+the full webserver then use a Golang Webdriver client,
+[tebeka/selenium](https://github.com/tebeka/selenium), to load pages in a
+browser (Chrome or Firefox) and assert that the webapp behaves as expected.
 
 To run the tests, from the root `wpt.fyi` directory, run:
 
-    make go_webdriver_test
+    make go_large_test
 
-If you want to actually see the tests in action, disable the frame buffer.
+You can run just one of Chrome or Firefox via:
 
-    make webdriver_deps
-    go test --frame_buffer=false -tags=large ./webdriver
+    make go_chrome_test
+    make go_firefox_test
 
-If you want to use a custom installed location of selenium / browser / driver
-binaries, the required flags are shown in [the Makefile](../Makefile)
-`go_webdriver_test' rule.
+Note that when running these tests outside of docker (see [Running in
+docker](#running-in-docker)), they will use your locally installed browser and
+webdriver clients, so it is worth making sure they are the versions you expect.
+
+## Debugging
+
+Debugging the webdriver/ tests can be difficult as they are integration tests
+and the problem can occur anywhere from controlling the browser, to the webapp
+frontend, to the backend - and other weird bugs inbetween! This section
+contains some tips on how to effectively debug them.
+
+After running one of the above `make` commands at least once, one can directly
+run the golang tests via:
+
+```
+# You can copy GECKODRIVER_PATH out of the make output; it should be installed
+# locally under webapp/node_modules/selenium-standalone/...
+go test -v -timeout=15m -tags=large ./webdriver -args \
+    -firefox_path=/usr/bin/firefox \
+    -geckodriver_path=$GECKODRIVER_PATH \
+    -chrome_path=/usr/bin/google-chrome \
+    -chromedriver_path=/usr/bin/chromedriver \
+    -frame_buffer=true \
+    -staging=false \
+    -browser=chrome  # Or firefox
+```
+
+It is worth comparing this command-line against the Makefile, in case this
+documentation becomes out of date.
+
+### Running only one test
+
+If you only need to run one test, you can use the golang test [`-run`
+parameter](https://golang.org/pkg/testing/#hdr-Subtests_and_Sub_benchmarks).
+For example:
+
+```
+go test -v -timeout=15m -tags=large ./webdriver \
+    -run TestProductParam_Order/Order \
+    -args \
+    -firefox_path=/usr/bin/firefox \
+    -geckodriver_path=$GECKODRIVER_PATH \
+    -chrome_path=/usr/bin/google-chrome \
+    -chromedriver_path=/usr/bin/chromedriver \
+    -frame_buffer=true \
+    -staging=false \
+    -browser=chrome  # Or firefox
+```
+
+### Visual Output
+
+Many of the tests run some javascript (or click on an element, etc) and expect
+to find some resulting change on the page. When that doesn't occur, they
+usually just timeout and it can be difficult to know why. One very useful trick
+is to enable visual output, so that you can actually see the webpage as the
+test runs.
+
+To do this, set the `frame_buffer` argument to `false`, e.g.:
+
+```
+go test -v -timeout=15m -tags=large ./webdriver -args \
+    -firefox_path=/usr/bin/firefox \
+    -geckodriver_path=$GECKODRIVER_PATH \
+    -chrome_path=/usr/bin/google-chrome \
+    -chromedriver_path=/usr/bin/chromedriver \
+    -frame_buffer=false \
+    -staging=false \
+    -browser=chrome  # Or firefox
+```
+
+### Verbose webdriver output
+
+By default, webdriver output is hidden as it is very noisy. You can re-enable
+it by passing `-debug=true` to the tests, e.g.:
+
+```
+go test -v -timeout=15m -tags=large ./webdriver -args \
+    -firefox_path=/usr/bin/firefox \
+    -geckodriver_path=$GECKODRIVER_PATH \
+    -chrome_path=/usr/bin/google-chrome \
+    -chromedriver_path=/usr/bin/chromedriver \
+    -frame_buffer=true \
+    -staging=false \
+    -browser=chrome \
+    -debug=true
+```
+
+Redirecting stderr to stdout and saving it to a log-file is recommended due to
+the verbosity of webdriver logs (append `2>&1 | tee my-log.txt` to the above
+command).
+
+### Running in docker
+
+Sometimes bugs only occur in a docker-like environment. This can be difficult
+to reproduce, but a first step is to run the tests inside of docker. To do
+this, first start the docker container in one terminal tab:
+
+```
+./util/docker-dev/run.sh
+```
+
+Then, in another tab, we need to get the instance id of the container, exec
+'bash' inside of it, and run our test:
+
+```
+docker container ls
+[ note the CONTAINER ID ]
+docker exec -it $CONTAINER_ID bash
+user@abce84dd426d:~/wpt.fyi$
+[now you can run 'make go_chrome_test', or 'go test ...' directly, etc]
+```
+
+Note that this maps the host machine's wpt.fyi checkout into docker, so any
+code edits you make on the host are reflected in the container and vice-versa.

webdriver/label_test.go

@@ -15,13 +15,8 @@ import (
 
 func TestLabelParam_Results(t *testing.T) {
 	runWebdriverTest(t, func(t *testing.T, app AppServer, wd selenium.WebDriver) {
-		// Local static data only have 2 experimental browsers, and neither has aligned
-		// experimental runs.
-		if *staging {
-			testLabel(t, wd, app, "/", "experimental", "wpt-results", 4, false)
-		} else {
-			testLabel(t, wd, app, "/", "experimental", "wpt-results", 3, false)
-		}
+		aligned := false
+		testLabel(t, wd, app, "/", "experimental", "wpt-results", 4, aligned)
 	})
 
 }