fix(puppeteer driver): clean the view-hierarchy from unused data. (#64)

* fix(puppeteer driver): clean the view-hierarchy from unused data.

* feat: allow partial review.

* fix(drivers): improve puppeteer driver.

* fix: enforce summary tag on finish.

* fix(puppeteer driver): wait for `body` tag to load.

* test: add missing unit tests.

examples/puppeteer/tests/example.test.ts

@@ -32,9 +32,9 @@ describe("Example Test Suite", () => {
 
   it("perform test with pilot", async () => {
     await copilot.pilot(
-      "Enter https://example.com/, press on more information, " +
-        "expect to be redirected to IANA site, summarize the findings. " +
-        "Open in non-headless mode.",
+      "On `https://github.com/wix-incubator/detox-copilot`, " +
+        "open the Commits page and summarize the latest commits. " +
+        "Open the browser with GUI.",
     );
   });
 });

src/drivers/puppeteer/getCleanDOM.ts

@@ -0,0 +1,73 @@
+import * as puppeteer from "puppeteer";
+
+/**
+ * Get clean DOM from the page content
+ * - Removes hidden elements
+ * - Removes ads, analytics, tracking elements
+ * - Removes unnecessary attributes
+ * - Removes empty elements
+ * @param page
+ */
+export default async function getCleanDOM(page: puppeteer.Page) {
+  await page.waitForSelector("body");
+
+  return await page.evaluate(() => {
+    const copiedDocument = document.cloneNode(true) as Document;
+
+    copiedDocument
+      .querySelectorAll('[hidden], [aria-hidden="true"]')
+      .forEach((el) => el.remove());
+
+    const removeSelectors = [
+      "script",
+      "style",
+      "link",
+      "meta",
+      "noscript",
+      "iframe",
+      '[class*="ads"]',
+      '[id*="ads"]',
+      '[class*="analytics"]',
+      '[class*="tracking"]',
+      "footer",
+      "header",
+      "nav",
+      "path",
+      "aside",
+    ];
+
+    const allowedAttributes = [
+      "src",
+      "href",
+      "alt",
+      "title",
+      "aria-label",
+      "aria-labelledby",
+      "aria-describedby",
+      "aria-hidden",
+      "role",
+      "class",
+      "id",
+      "data-*",
+    ];
+
+    copiedDocument.querySelectorAll("*").forEach((el) => {
+      Array.from(el.attributes).forEach((attr) => {
+        if (!allowedAttributes.includes(attr.name)) {
+          el.removeAttribute(attr.name);
+        }
+      });
+
+      if (!el.innerHTML.trim()) {
+        el.remove();
+      }
+    });
+
+    removeSelectors.forEach((selector) => {
+      copiedDocument.querySelectorAll(selector).forEach((el) => el.remove());
+    });
+
+    const mainContent = copiedDocument.body.innerHTML;
+    return mainContent.replace(/\s+/g, " ").trim();
+  });
+}

src/drivers/puppeteer/index.ts

@@ -2,6 +2,7 @@ import { TestingFrameworkAPICatalog, TestingFrameworkDriver } from "@/types";
 import * as puppeteer from "puppeteer";
 import path from "path";
 import fs from "fs";
+import getCleanDOM from "./getCleanDOM";
 
 export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
   private currentPage?: puppeteer.Page;
@@ -59,7 +60,7 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
       );
     }
 
-    return await this.currentPage.content();
+    return await getCleanDOM(this.currentPage);
   }
 
   /**
@@ -87,7 +88,8 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
               guidelines: [
                 "Options can specify `headless`, `slowMo`, `args`, etc.",
                 "Useful for running tests in a headless browser environment.",
-                'Prefer passing `headless: "new"` to `puppeteer.launch() unless mentioned that it is required not to.',
+                'Prefer passing `headless: "new"` to `puppeteer.launch() unless mentioned that ' +
+                  "it is required not to (e.g. launching with GUI was mentioned).",
               ],
             },
             {
@@ -99,6 +101,15 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
                 "Useful for cleaning up resources and freeing memory.",
               ],
             },
+            {
+              signature: "await getCurrentPage().setUserAgent(userAgent)",
+              description: "Overrides the default user agent string.",
+              example: 'await getCurrentPage().setUserAgent("UA-TEST");',
+              guidelines: [
+                "Affects the value of `navigator.userAgent`.",
+                "Useful for simulating different browsers or bots.",
+              ],
+            },
           ],
         },
         {
@@ -184,6 +195,7 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
               example:
                 'await getCurrentPage().waitForFunction(() => document.querySelector("#status").innerText === "Loaded");',
               guidelines: [
+                "Useful as an alternative for using click, type, etc. to wait for a condition. Preferred practice.",
                 "Useful for waiting on dynamic content or conditions.",
                 "Avoid tight polling intervals to prevent excessive CPU usage.",
               ],
@@ -209,6 +221,7 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
                 "Simulates a mouse click on the element matched by the selector.",
               example: 'await getCurrentPage().click("#submit-button");',
               guidelines: [
+                "Prefer evaluated click instead of click when possible.",
                 "Waits for the element to be visible and enabled.",
                 "Throws an error if the element is not found or not interactable.",
                 "Avoid clicking on elements that change page state without proper synchronization.",
@@ -221,6 +234,7 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
                 "Types text into an element matched by the selector.",
               example: 'await getCurrentPage().type("#username", "myUser123");',
               guidelines: [
+                "Prefer evaluated type instead of type when possible.",
                 "Suitable for input and textarea elements.",
                 "Simulates individual key presses with optional delay.",
                 "Ensure the element is focusable before typing.",
@@ -231,6 +245,7 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
               description: "Focuses on the element matched by the selector.",
               example: 'await getCurrentPage().focus("#search-input");',
               guidelines: [
+                "Prefer evaluated focus instead of focus when possible.",
                 "Useful before performing keyboard actions.",
                 "Waits for the element to be interactable.",
               ],
@@ -241,6 +256,7 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
               example:
                 'await getCurrentPage().select("#country-select", "US");',
               guidelines: [
+                "Prefer evaluated select instead of select when possible.",
                 "Supports selecting multiple options if the `<select>` element allows it.",
                 "Values correspond to the `value` attribute of `<option>` elements.",
               ],
@@ -250,6 +266,7 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
               description: "Simulates hovering the mouse over the element.",
               example: 'await getCurrentPage().hover(".dropdown-trigger");',
               guidelines: [
+                "Prefer evaluated hover instead of select when possible.",
                 "Triggers hover effects such as tooltips or menus.",
                 "Ensure the element is visible and within the viewport.",
               ],
@@ -409,11 +426,23 @@ export class PuppeteerFrameworkDriver implements TestingFrameworkDriver {
               signature:
                 "await getCurrentPage().evaluate(pageFunction[, ...args])",
               description: "Executes a function in the page context.",
-              example:
-                "const dimensions = await getCurrentPage().evaluate(() => { return { width: document.documentElement.clientWidth, height: document.documentElement.clientHeight }; });",
+              example: `
+                # Example 1: Get the title of the page
+                const title = await getCurrentPage().evaluate(() => document.title);
+                
+                # Example 2: Get the text content of an element
+                const text = await getCurrentPage().evaluate(el => el.textContent, document.querySelector(".message"));
+                
+                # Example 3: Click on an element
+                await getCurrentPage().evaluate(() => document.querySelector("#submit").click());
+                
+                # Example 4: Type text into an input field
+                await getCurrentPage().evaluate((el, text) => el.value = text, document.querySelector("#username"), "john_doe");
+                `,
               guidelines: [
                 "Allows access to the DOM and JavaScript environment of the page.",
                 "Avoid exposing sensitive data or functions.",
+                "It's recommended to use RegExp to match the text content of an element (with partial substring) and not the exact text.",
               ],
             },
             {

src/utils/CodeEvaluator.ts

@@ -8,12 +8,6 @@ export class CodeEvaluator {
     context: any,
     sharedContext: Record<string, any> = {},
   ): Promise<CodeEvaluationResult> {
-    const loggerSpinner = logger.startSpinner({
-      message: `Copilot evaluating code: \n\`\`\`\n${code}\n\`\`\`\n`,
-      isBold: false,
-      color: "gray",
-    });
-
     const asyncFunction = this.createAsyncFunction(
       code,
       context,
@@ -22,12 +16,11 @@ export class CodeEvaluator {
 
     try {
       const result = await asyncFunction();
-      loggerSpinner.stop("success", `Copilot evaluated the code successfully`);
 
       return { code, result, sharedContext };
     } catch (error) {
-      loggerSpinner.stop("failure", {
-        message: `Copilot failed to evaluate the code: \n\`\`\`\n${code}\n\`\`\``,
+      logger.error({
+        message: `\nCopilot failed to evaluate the code: \n\`\`\`\n${code}\n\`\`\``,
         isBold: false,
         color: "gray",
       });

src/utils/extractOutputs.test.ts

@@ -13,8 +13,8 @@ describe("extractOutputs", () => {
          Tap on GREAT button
         </ACTION>`;
     const outputsMapper: OutputsMapping = {
-      thoughts: "THOUGHTS",
-      action: "ACTION",
+      thoughts: { tag: "THOUGHTS", isRequired: true },
+      action: { tag: "ACTION", isRequired: true },
     };
     const outputs = extractOutputs({ text: textToBeParsed, outputsMapper });
     expect(outputs).toEqual({
@@ -37,13 +37,56 @@ describe("extractOutputs", () => {
          Tap on WOW button
         </ACTION>`;
     const outputsMapper: OutputsMapping = {
-      thoughts: "THOUGHTS",
-      action: "ACTION",
+      thoughts: { tag: "THOUGHTS", isRequired: true },
+      action: { tag: "ACTION", isRequired: true },
     };
     const outputs = extractOutputs({ text: textToBeParsed, outputsMapper });
     expect(outputs).toEqual({
       thoughts: "I think this is great",
       action: "Tap on GREAT button",
     });
   });
+
+  it("should throw error if required output is missing", () => {
+    const textToBeParsed = `
+        These are my thoughts:
+        <THOUGHTS>
+        I think this is great
+        </THOUGHTS>
+        This is the action the copilot should perform:
+         <ACTION>
+         Tap on GREAT button
+        </ACTION>`;
+    const outputsMapper: OutputsMapping = {
+      thoughts: { tag: "THOUGHTS", isRequired: true },
+      action: { tag: "ACTION", isRequired: true },
+      action2: { tag: "ACTION2", isRequired: true },
+    };
+    expect(() =>
+      extractOutputs({ text: textToBeParsed, outputsMapper }),
+    ).toThrowError("Missing field for required tag <ACTION2>");
+  });
+
+  it("should not throw error if required output is missing but not required", () => {
+    const textToBeParsed = `
+        These are my thoughts:
+        <THOUGHTS>
+        I think this is great
+        </THOUGHTS>
+        This is the action the copilot should perform:
+         <ACTION>
+         Tap on GREAT button
+        </ACTION>`;
+    const outputsMapper: OutputsMapping = {
+      thoughts: { tag: "THOUGHTS", isRequired: true },
+      action: { tag: "ACTION", isRequired: true },
+      action2: { tag: "ACTION2", isRequired: false },
+    };
+    const outputs = extractOutputs({ text: textToBeParsed, outputsMapper });
+    expect(outputs).toEqual({
+      thoughts: "I think this is great",
+      action: "Tap on GREAT button",
+      action2: "N/A",
+    });
+  });
 });

src/utils/extractOutputs.ts

@@ -1,19 +1,24 @@
-export type OutputsMapping = Record<string, string>;
+export type Output = {
+  tag: string;
+  isRequired: boolean;
+};
+
+export type OutputsMapping = Record<string, Output>;
 
 export const OUTPUTS_MAPPINGS: Record<string, OutputsMapping> = {
   PILOT_REVIEW_SECTION: {
-    summary: "SUMMARY",
-    findings: "FINDINGS",
-    score: "SCORE",
+    summary: { tag: "SUMMARY", isRequired: false },
+    findings: { tag: "FINDINGS", isRequired: false },
+    score: { tag: "SCORE", isRequired: false },
   },
   PILOT_STEP: {
-    thoughts: "THOUGHTS",
-    action: "ACTION",
-    ux: "UX",
-    a11y: "ACCESSIBILITY",
+    thoughts: { tag: "THOUGHTS", isRequired: true },
+    action: { tag: "ACTION", isRequired: true },
+    ux: { tag: "UX", isRequired: false },
+    a11y: { tag: "ACCESSIBILITY", isRequired: false },
   },
   PILOT_SUMMARY: {
-    summary: "SUMMARY",
+    summary: { tag: "SUMMARY", isRequired: true },
   },
 };
 
@@ -27,13 +32,15 @@ export function extractOutputs<M extends OutputsMapping>({
   const outputs: Partial<{ [K in keyof M]: string }> = {};
 
   for (const fieldName in outputsMapper) {
-    const tag = outputsMapper[fieldName];
+    const tag = outputsMapper[fieldName].tag;
     const regex = new RegExp(`<${tag}>(.*?)</${tag}>`, "s");
     const match = text.match(regex);
     if (match) {
       outputs[fieldName] = match[1].trim();
+    } else if (!outputsMapper[fieldName].isRequired) {
+      outputs[fieldName] = "N/A";
     } else {
-      throw new Error(`Missing field for tag <${tag}>`);
+      throw new Error(`Missing field for required tag <${tag}>`);
     }
   }