Implement non-hosted mode test harness

NO_IFTTT: getting in sync with older changes
Bug: 390535614
Change-Id: I807dc1686a4a052e49ff484db75e9b6638772719
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6278526
Commit-Queue: Liviu Rau <liviurau@chromium.org>
Reviewed-by: Alex Rudenko <alexrudenko@chromium.org>
Reviewed-by: Philip Pfaffe <pfaffe@chromium.org>
Reviewed-by: Simon Zünd <szuend@chromium.org>

Author: Liviu Rau

test/BUILD.gn

@@ -10,6 +10,7 @@ group("test") {
     ":run",
     ":unittests",
     "e2e",
+    "e2e_non_hosted",
     "interactions",
     "perf",
     "shared",

test/conductor/BUILD.gn

@@ -25,10 +25,12 @@ node_ts_library("implementation") {
     "mocha-interface.ts",
     "mocha_hooks.ts",
     "paths.ts",
+    "platform.ts",
     "pool.ts",
     "puppeteer-state.ts",
     "resultsdb.ts",
     "screenshot-error.ts",
+    "server_port.ts",
     "target_tab.ts",
     "test_config.ts",
     "test_server.ts",

test/conductor/hooks.ts

@@ -21,8 +21,8 @@ import {
   getBrowserAndPages,
   registerHandlers,
   setBrowserAndPages,
-  setTestServerPort,
 } from './puppeteer-state.js';
+import {setTestServerPort} from './server_port.js';
 import {TargetTab} from './target_tab.js';
 import {TestConfig} from './test_config.js';
 
@@ -55,6 +55,7 @@ const envChromeFeatures = process.env['CHROME_FEATURES'];
 
 function launchChrome() {
   // Use port 0 to request any free port.
+  // LINT.IfChange(features)
   const enabledFeatures = [
     'Portals',
     'PortalsCrossOrigin',
@@ -64,13 +65,13 @@ function launchChrome() {
     'PrivacySandboxAdsAPIsOverride',
     'AutofillEnableDevtoolsIssues',
   ];
-
   const disabledFeatures = [
     'PMProcessPriorityPolicy',                     // crbug.com/361252079
     'MojoChannelAssociatedSendUsesRunOrPostTask',  // crbug.com/376228320
     'RasterInducingScroll',                        // crbug.com/381055647
     'CompositeBackgroundColorAnimation',           // crbug.com/381055647
   ];
+  // LINT.ThenChange(/test/e2e_non_hosted/shared/browser-helper.ts:features)
   const launchArgs = [
     '--remote-allow-origins=*',
     '--remote-debugging-port=0',

test/conductor/platform.ts

@@ -0,0 +1,21 @@
+// Copyright 2025 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import * as os from 'os';
+
+export type Platform = 'mac'|'win32'|'linux';
+export let platform: Platform;
+switch (os.platform()) {
+  case 'darwin':
+    platform = 'mac';
+    break;
+
+  case 'win32':
+    platform = 'win32';
+    break;
+
+  default:
+    platform = 'linux';
+    break;
+}

test/conductor/puppeteer-state.ts

@@ -5,15 +5,12 @@
 import * as puppeteer from 'puppeteer-core';
 
 import {querySelectorShadowTextAll, querySelectorShadowTextOne} from './custom-query-handlers.js';
+import {clearServerPort} from './server_port.js';
 
 let target: puppeteer.Page|null;
 let frontend: puppeteer.Page|null;
 let browser: puppeteer.Browser|null;
 
-// Set when we launch the server. It will be different for each
-// sub-process runner when running in parallel.
-let testServerPort: number|null;
-
 export interface BrowserAndPages {
   target: puppeteer.Page;
   frontend: puppeteer.Page;
@@ -24,7 +21,7 @@ export const clearPuppeteerState = () => {
   target = null;
   frontend = null;
   browser = null;
-  testServerPort = null;
+  clearServerPort();
 };
 
 export const setBrowserAndPages = (newValues: BrowserAndPages) => {
@@ -55,23 +52,6 @@ export const getBrowserAndPages = (): BrowserAndPages => {
   };
 };
 
-export const setTestServerPort = (port: number) => {
-  if (testServerPort) {
-    throw new Error('Can\'t set the test server port twice.');
-  }
-  testServerPort = port;
-};
-
-export const getTestServerPort = () => {
-  if (!testServerPort) {
-    throw new Error(
-        'Unable to locate test server port. Was it stored first?' +
-        '\nYou might be calling this function at module instantiation time, instead of ' +
-        'at runtime when the port is available.');
-  }
-  return testServerPort;
-};
-
 let handlerRegistered = false;
 export const registerHandlers = () => {
   if (handlerRegistered) {

test/conductor/server_port.ts

@@ -0,0 +1,29 @@
+
+// Copyright 2025 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Set when we launch the server. It will be different for each
+// sub-process runner when running in parallel.
+let testServerPort: number|null;
+
+export const setTestServerPort = (port: number) => {
+  if (testServerPort) {
+    throw new Error('Can\'t set the test server port twice.');
+  }
+  testServerPort = port;
+};
+
+export const getTestServerPort = () => {
+  if (!testServerPort) {
+    throw new Error(
+        'Unable to locate test server port. Was it stored first?' +
+        '\nYou might be calling this function at module instantiation time, instead of ' +
+        'at runtime when the port is available.');
+  }
+  return testServerPort;
+};
+
+export function clearServerPort() {
+  testServerPort = null;
+}

test/conductor/test_server.ts

@@ -49,6 +49,7 @@ export function startServer(server: 'hosted-mode'|'component-docs', commandLineA
   });
 }
 
+process.on('exit', stopServer);
 export function stopServer() {
   runningServer.kill();
 }

test/e2e_non_hosted/BUILD.gn

@@ -0,0 +1,29 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("../../scripts/build/ninja/copy.gni")
+import("../../scripts/build/typescript/typescript.gni")
+
+group("e2e_non_hosted") {
+  deps = [
+    ":test_inputs",
+    ":tests",
+  ]
+}
+
+node_ts_library("tests") {
+  deps = [
+    "../conductor:implementation",
+    "dummy",
+  ]
+  sources = [ "mocharc.ts" ]
+}
+
+generated_file("test_inputs") {
+  outputs = [ "$target_gen_dir/tests.txt" ]
+  data_keys = [ "tests" ]
+  rebase = target_gen_dir
+
+  deps = [ ":tests" ]
+}

test/e2e_non_hosted/conductor/BUILD.gn

@@ -0,0 +1,25 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("../../../scripts/build/typescript/typescript.gni")
+
+group("conductor") {
+  deps = [
+    ":implementation",
+    "../../../scripts/component_server",
+    "../../../scripts/hosted_mode",
+  ]
+}
+
+node_ts_library("implementation") {
+  deps = [
+    "../../conductor:implementation",
+    "../shared",
+  ]
+  sources = [
+    "mocha-interface-helpers.ts",
+    "mocha-interface.ts",
+    "state-provider.ts",
+  ]
+}

test/e2e_non_hosted/conductor/mocha-interface-helpers.ts

@@ -0,0 +1,135 @@
+// Copyright 2024 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import type * as Mocha from 'mocha';
+
+import {AsyncScope} from '../../conductor/async-scope.js';
+import type {Platform} from '../../conductor/platform.js';
+import {ScreenshotError} from '../../conductor/screenshot-error.js';
+import {TestConfig} from '../../conductor/test_config.js';
+import type {BrowserSettings, BrowserWrapper} from '../shared/browser-helper.js';
+import type {DevToolsFronendPage, DevtoolsSettings} from '../shared/frontend-helper.js';
+import type {InspectedPage} from '../shared/target-helper.js';
+declare global {
+  namespace Mocha {
+    export interface TestFunction {
+      (title: string, fn: TestCallbackWithState): void;
+
+      skipOnPlatforms: (platforms: Platform[], title: string, fn: Mocha.AsyncFunc) => void;
+    }
+    export interface Suite {
+      settings: SuiteSettings;
+      state: State;
+      browser: BrowserWrapper;
+    }
+  }
+}
+
+export type HarnessSettings = BrowserSettings&DevtoolsSettings;
+export type SuiteSettings = Partial<HarnessSettings>;
+
+export interface State {
+  devToolsPage: DevToolsFronendPage;
+  inspectedPage: InspectedPage;
+  browser: BrowserWrapper;
+}
+
+/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+export type TestCallbackWithState = (state: State) => PromiseLike<any>;
+
+async function takeScreenshots(state: State): Promise<{target?: string, frontend?: string}> {
+  try {
+    const {devToolsPage, inspectedPage} = state;
+    const targetScreenshot = await inspectedPage.screenshot();
+    const frontendScreenshot = await devToolsPage.screenshot();
+    return {target: targetScreenshot, frontend: frontendScreenshot};
+  } catch (err) {
+    console.error('Error taking a screenshot', err);
+    return {};
+  }
+}
+
+async function createScreenshotError(test: Mocha.Runnable|undefined, error: Error): Promise<Error> {
+  if (!test?.parent?.state) {
+    console.error('Missing browsing state. Unable to take screenshots for the error:', error);
+    return error;
+  }
+  const sate = test.parent.state;
+  console.error('Taking screenshots for the error:', error);
+  if (!TestConfig.debug) {
+    try {
+      const screenshotTimeout = 5_000;
+      let timer: NodeJS.Timeout;
+      const {target, frontend} = await Promise.race([
+        takeScreenshots(sate).then(result => {
+          clearTimeout(timer);
+          return result;
+        }),
+        new Promise(resolve => {
+          timer = setTimeout(resolve, screenshotTimeout);
+        }).then(() => {
+          console.error(`Could not take screenshots within ${screenshotTimeout}ms.`);
+          return {target: undefined, frontend: undefined};
+        }),
+      ]);
+      return ScreenshotError.fromBase64Images(error, target, frontend);
+    } catch (e) {
+      console.error('Unexpected error saving screenshots', e);
+      return e;
+    }
+  }
+  return error;
+}
+
+export function makeInstrumentedTestFunction(fn: Mocha.AsyncFunc, label: string) {
+  return async function testFunction(this: Mocha.Context) {
+    const abortController = new AbortController();
+    let resolver;
+    let rejecter: (reason?: unknown) => void;
+    const testPromise = new Promise((resolve, reject) => {
+      resolver = resolve;
+      rejecter = reject;
+    });
+    // AbortSignal for the current test function.
+    AsyncScope.abortSignal = abortController.signal;
+    // Promisify the function in case it is sync.
+    const promise = (async () => await fn.call(this))();
+    const actualTimeout = this.timeout();
+    // Disable test timeout.
+    this.timeout(0);
+    const t = actualTimeout !== 0 ? setTimeout(async () => {
+      abortController.abort();
+      const stacks = [];
+      const scopes = AsyncScope.scopes;
+      for (const scope of scopes.values()) {
+        const {descriptions, stack} = scope;
+        if (stack) {
+          const stepDescription = descriptions ? `${descriptions.join(' > ')}:\n` : '';
+          stacks.push(`${stepDescription}${stack.join('\n')}\n`);
+        }
+      }
+      const err = new Error(`A test function (${label}) for "${this.test?.title}" timed out`);
+      if (stacks.length > 0) {
+        const msg = `Pending async operations during timeout:\n${stacks.join('\n\n')}`;
+        err.cause = new Error(msg);
+      }
+      rejecter(await createScreenshotError(this.test, err));
+    }, actualTimeout) : 0;
+    promise
+        .then(
+            resolver,
+            async err => {
+              // Suppress errors after the test was aborted.
+              if (abortController.signal.aborted) {
+                return;
+              }
+              rejecter(await createScreenshotError(this.test, err));
+            })
+        .finally(() => {
+          clearTimeout(t);
+          this.timeout(actualTimeout);
+        });
+    return await testPromise;
+  };
+}