feat: provide proper network timings (#2919)

+ make e2e test offline. Tested manually + WPT. --------- Co-authored-by: browser-automation-bot <[email protected]>
GoogleChromeLabs · Dec 20, 2024 · 22e096b · 22e096b
1 parent d0c9c13
commit 22e096b
Show file tree

Hide file tree

Showing 17 changed files with 86 additions and 89 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -63,6 +63,10 @@ jobs:
         with:
           node-version-file: '.nvmrc'
           cache: npm
+      - name: Disable AppArmor
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        # https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md
+        run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
       - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1
       - name: Install and build npm dependencies
         run: npm ci

diff --git a/.github/workflows/selenium.yml b/.github/workflows/selenium.yml
@@ -30,6 +30,9 @@ jobs:
         with:
           node-version-file: '.nvmrc'
           cache: npm
+      - name: Disable AppArmor
+        # https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md
+        run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
       - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1
       - name: Install and build npm dependencies
         run: npm ci

diff --git a/.github/workflows/webdriverio.yml b/.github/workflows/webdriverio.yml
@@ -30,6 +30,9 @@ jobs:
         with:
           node-version-file: '.nvmrc'
           cache: npm
+      - name: Disable AppArmor
+        # https://chromium.googlesource.com/chromium/src/+/main/docs/security/apparmor-userns-restrictions.md
+        run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
       - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1
       - name: Install and build npm dependencies
         run: npm ci

diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml
@@ -82,6 +82,8 @@ jobs:
         with:
           node-version-file: '.nvmrc'
           cache: npm
+      - name: Disable AppArmor
+        run: echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns
       - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1
       - name: Install and build npm dependencies
         run: npm ci

diff --git a/src/bidiMapper/modules/context/NavigationTracker.spec.ts b/src/bidiMapper/modules/context/NavigationTracker.spec.ts
@@ -318,7 +318,7 @@ describe('NavigationTracker', () => {
         assertNoNavigationEvents();
       });
 
-      it('canceled by script-initiated navigation', async () => {
+      it('canceled by script-initiated navigation', () => {
         navigationTracker.frameRequestedNavigation(SOME_URL);
 
         assertNoNavigationEvents();
@@ -350,7 +350,7 @@ describe('NavigationTracker', () => {
         assert.equal(navigationTracker.url, INITIAL_URL);
       });
 
-      it('canceled by command navigation', async () => {
+      it('canceled by command navigation', () => {
         navigationTracker.frameRequestedNavigation(SOME_URL);
         navigationTracker.frameStartedNavigating(ANOTHER_URL, LOADER_ID);
 

diff --git a/src/bidiMapper/modules/network/NetworkRequest.ts b/src/bidiMapper/modules/network/NetworkRequest.ts
@@ -23,27 +23,27 @@
 import type {Protocol} from 'devtools-protocol';
 
 import {
-  Network,
   type BrowsingContext,
   ChromiumBidi,
+  Network,
   type NetworkEvent,
 } from '../../../protocol/protocol.js';
 import {assert} from '../../../utils/assert.js';
 import {Deferred} from '../../../utils/Deferred.js';
-import {LogType, type LoggerFn} from '../../../utils/log.js';
+import {type LoggerFn, LogType} from '../../../utils/log.js';
 import type {CdpTarget} from '../cdp/CdpTarget.js';
 import type {EventManager} from '../session/EventManager.js';
 
 import type {NetworkStorage} from './NetworkStorage.js';
 import {
-  computeHeadersSize,
+  bidiBodySizeFromCdpPostDataEntries,
   bidiNetworkHeadersFromCdpNetworkHeaders,
-  cdpToBiDiCookie,
-  cdpFetchHeadersFromBidiNetworkHeaders,
   cdpAuthChallengeResponseFromBidiAuthContinueWithAuthAction,
-  bidiBodySizeFromCdpPostDataEntries,
-  networkHeaderFromCookieHeaders,
+  cdpFetchHeadersFromBidiNetworkHeaders,
+  cdpToBiDiCookie,
+  computeHeadersSize,
   getTiming,
+  networkHeaderFromCookieHeaders,
 } from './NetworkUtils.js';
 
 const REALM_REGEX = /(?<=realm=").*(?=")/;
@@ -304,26 +304,66 @@ export class NetworkRequest {
   }
 
   get #timings(): Network.FetchTimingInfo {
+    // The timing in the CDP events are provided relative to the event's baseline.
+    // However, the baseline can be different for different events, and the events have to
+    // be normalized throughout resource events. Normalize events timestamps  by the
+    // request.
+    // TODO: Verify this is correct.
+    const responseTimeOffset = getTiming(
+      getTiming(this.#response.info?.timing?.requestTime) -
+        getTiming(this.#request.info?.timestamp),
+    );
+
     return {
       // TODO: Verify this is correct
-      timeOrigin: getTiming(this.#response.info?.timing?.requestTime),
-      requestTime: getTiming(this.#response.info?.timing?.requestTime),
+      timeOrigin: Math.round(getTiming(this.#request.info?.wallTime) * 1000),
+      // Timing baseline.
+      // TODO: Verify this is correct.
+      requestTime: 0,
+      // TODO: set if redirect detected.
       redirectStart: 0,
+      // TODO: set if redirect detected.
       redirectEnd: 0,
       // TODO: Verify this is correct
       // https://source.chromium.org/chromium/chromium/src/+/main:net/base/load_timing_info.h;l=145
-      fetchStart: getTiming(this.#response.info?.timing?.requestTime),
-      dnsStart: getTiming(this.#response.info?.timing?.dnsStart),
-      dnsEnd: getTiming(this.#response.info?.timing?.dnsEnd),
-      connectStart: getTiming(this.#response.info?.timing?.connectStart),
-      connectEnd: getTiming(this.#response.info?.timing?.connectEnd),
-      tlsStart: getTiming(this.#response.info?.timing?.sslStart),
-      requestStart: getTiming(this.#response.info?.timing?.sendStart),
+      fetchStart: getTiming(
+        this.#response.info?.timing?.workerFetchStart,
+        responseTimeOffset,
+      ),
+      // fetchStart: 0,
+      dnsStart: getTiming(
+        this.#response.info?.timing?.dnsStart,
+        responseTimeOffset,
+      ),
+      dnsEnd: getTiming(
+        this.#response.info?.timing?.dnsEnd,
+        responseTimeOffset,
+      ),
+      connectStart: getTiming(
+        this.#response.info?.timing?.connectStart,
+        responseTimeOffset,
+      ),
+      connectEnd: getTiming(
+        this.#response.info?.timing?.connectEnd,
+        responseTimeOffset,
+      ),
+      tlsStart: getTiming(
+        this.#response.info?.timing?.sslStart,
+        responseTimeOffset,
+      ),
+      requestStart: getTiming(
+        this.#response.info?.timing?.sendStart,
+        responseTimeOffset,
+      ),
       // https://source.chromium.org/chromium/chromium/src/+/main:net/base/load_timing_info.h;l=196
       responseStart: getTiming(
         this.#response.info?.timing?.receiveHeadersStart,
+        responseTimeOffset,
+      ),
+      responseEnd: getTiming(
+        this.#response.info?.timing?.receiveHeadersEnd,
+        responseTimeOffset,
       ),
-      responseEnd: getTiming(this.#response.info?.timing?.receiveHeadersEnd),
     };
   }
 

diff --git a/src/bidiMapper/modules/network/NetworkUtils.ts b/src/bidiMapper/modules/network/NetworkUtils.ts
@@ -366,13 +366,16 @@ export function bidiBodySizeFromCdpPostDataEntries(
   return size;
 }
 
-export function getTiming(timing: number | undefined): number {
+export function getTiming(
+  timing: number | undefined,
+  offset: number = 0,
+): number {
   if (!timing) {
     return 0;
   }
-  if (timing < 0) {
+  if (timing <= 0 || timing + offset <= 0) {
     return 0;
   }
 
-  return timing;
+  return timing + offset;
 }
diff --git a/tests/network/test_network.py b/tests/network/test_network.py
@@ -422,10 +422,15 @@ async def test_network_before_request_sent_event_with_data_url_emitted(
 
 
 @pytest.mark.asyncio
+@pytest.mark.parametrize('capabilities', [{
+    'acceptInsecureCerts': True
+}],
+                         indirect=True)
 async def test_network_sends_only_included_cookies(websocket, context_id,
-                                                   url_base):
+                                                   url_example_another_origin,
+                                                   url_bad_ssl):
 
-    await goto_url(websocket, context_id, "https://example.com")
+    await goto_url(websocket, context_id, url_bad_ssl)
 
     await execute_command(
         websocket, {
@@ -446,7 +451,7 @@ async def test_network_sends_only_included_cookies(websocket, context_id,
         websocket, {
             "method": "browsingContext.navigate",
             "params": {
-                "url": url_base,
+                "url": url_example_another_origin,
                 "wait": "complete",
                 "context": context_id
             }
@@ -463,7 +468,7 @@ async def test_network_sends_only_included_cookies(websocket, context_id,
             "redirectCount": 0,
             "request": {
                 "request": ANY_STR,
-                "url": url_base,
+                "url": url_example_another_origin,
                 "method": "GET",
                 "headers": ANY_LIST,
                 "cookies": [],

diff --git a/...river/headful/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini b/...river/headful/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini
@@ -1,6 +1,3 @@
 [before_request_sent.py]
-  [test_serviceworker_request]
-    expected: FAIL
-
   [test_destination_initiator]
     expected: FAIL
diff --git a/...edriver/headful/webdriver/tests/bidi/network/response_completed/response_completed.py.ini b/...edriver/headful/webdriver/tests/bidi/network/response_completed/response_completed.py.ini
@@ -14,14 +14,5 @@
   [test_response_status[407-Proxy Authentication Required\]]
     expected: FAIL
 
-  [test_request_timing_info]
-    expected: FAIL
-
-  [test_serviceworker_request]
-    expected: FAIL
-
-  [test_url_with_fragment]
-    expected: FAIL
-
   [test_destination_initiator]
     expected: FAIL
diff --git a/...hromedriver/headful/webdriver/tests/bidi/network/response_started/response_started.py.ini b/...hromedriver/headful/webdriver/tests/bidi/network/response_started/response_started.py.ini
@@ -8,14 +8,5 @@
   [test_www_authenticate]
     expected: FAIL
 
-  [test_request_timing_info]
-    expected: FAIL
-
-  [test_serviceworker_request]
-    expected: FAIL
-
-  [test_url_with_fragment]
-    expected: FAIL
-
   [test_destination_initiator]
     expected: FAIL
diff --git a/...iver/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini b/...iver/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini
@@ -1,6 +1,3 @@
 [before_request_sent.py]
-  [test_serviceworker_request]
-    expected: FAIL
-
   [test_destination_initiator]
     expected: FAIL
diff --git a/...driver/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini b/...driver/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini
@@ -14,14 +14,5 @@
   [test_response_status[407-Proxy Authentication Required\]]
     expected: FAIL
 
-  [test_request_timing_info]
-    expected: FAIL
-
-  [test_serviceworker_request]
-    expected: FAIL
-
-  [test_url_with_fragment]
-    expected: FAIL
-
   [test_destination_initiator]
     expected: FAIL
diff --git a/...romedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini b/...romedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini
@@ -8,14 +8,5 @@
   [test_www_authenticate]
     expected: FAIL
 
-  [test_request_timing_info]
-    expected: FAIL
-
-  [test_serviceworker_request]
-    expected: FAIL
-
-  [test_url_with_fragment]
-    expected: FAIL
-
   [test_destination_initiator]
     expected: FAIL
diff --git a/...pper/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini b/...pper/headless/webdriver/tests/bidi/network/before_request_sent/before_request_sent.py.ini
@@ -1,6 +1,3 @@
 [before_request_sent.py]
-  [test_serviceworker_request]
-    expected: FAIL
-
   [test_destination_initiator]
     expected: FAIL
diff --git a/...mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini b/...mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini
@@ -14,14 +14,5 @@
   [test_response_status[407-Proxy Authentication Required\]]
     expected: FAIL
 
-  [test_request_timing_info]
-    expected: FAIL
-
-  [test_serviceworker_request]
-    expected: FAIL
-
-  [test_url_with_fragment]
-    expected: FAIL
-
   [test_destination_initiator]
     expected: FAIL
diff --git a/...ata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini b/...ata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini
@@ -5,14 +5,5 @@
   [test_response_status[407-Proxy Authentication Required\]]
     expected: FAIL
 
-  [test_request_timing_info]
-    expected: FAIL
-
-  [test_serviceworker_request]
-    expected: FAIL
-
-  [test_url_with_fragment]
-    expected: FAIL
-
   [test_destination_initiator]
     expected: FAIL