Merge pull request #48 from wix-incubator/semantic-search-api

feat: add `advanced` mode for Copilot's visual analysis.

src/Copilot.test.ts

@@ -1,7 +1,7 @@
 import { Copilot } from '@/Copilot';
 import { CopilotStepPerformer } from '@/actions/CopilotStepPerformer';
 import { CopilotError } from '@/errors/CopilotError';
-import { Config, ScreenCapturerResult } from '@/types';
+import { Config, ScreenCapturerResult, PromptHandler } from '@/types';
 import { mockCache, mockedCacheFile } from './test-utils/cache';
 import { ScreenCapturer } from '@/utils/ScreenCapturer';
 import {
@@ -10,6 +10,7 @@ import {
   barCategory1,
   dummyContext,
 } from './test-utils/APICatalogTestUtils';
+import { PilotPerformer } from './actions/PilotPerformer';
 
 jest.mock('@/actions/CopilotStepPerformer');
 jest.mock('@/utils/ScreenCapturer');
@@ -21,28 +22,41 @@ const VIEW_HIERARCHY = 'hash';
 
 describe('Copilot', () => {
   let mockConfig: Config;
+  let mockPromptHandler: jest.Mocked<PromptHandler>;
+  let mockFrameworkDriver: any;
+  let mockPilotPerformer: jest.Mocked<PilotPerformer>;
   let screenCapture: ScreenCapturerResult;
 
   beforeEach(() => {
-    mockConfig = {
-      frameworkDriver: {
-        captureSnapshotImage: jest.fn(),
-        captureViewHierarchyString: jest.fn(),
-        apiCatalog: {
-          context: {},
-          categories: [],
-        },
-      },
-      promptHandler: {
-        runPrompt: jest.fn(),
-        isSnapshotImageSupported: jest.fn().mockReturnValue(true),
+    mockPromptHandler = {
+      runPrompt: jest.fn(),
+      isSnapshotImageSupported: jest.fn()
+    } as any;
+
+    mockFrameworkDriver = {
+      apiCatalog: {
+        context: {},
+        categories: []
       },
+      captureSnapshotImage: jest.fn(),
+      captureViewHierarchyString: jest.fn()
     };
 
+    mockPilotPerformer = {
+      perform: jest.fn()
+    } as any;
+
+    mockConfig = {
+      promptHandler: mockPromptHandler,
+      frameworkDriver: mockFrameworkDriver
+    };
+
+    jest.spyOn(PilotPerformer.prototype, 'perform').mockImplementation(mockPilotPerformer.perform);
+
     screenCapture = {
-      snapshot: SNAPSHOT_DATA,
-      viewHierarchy: VIEW_HIERARCHY,
-      isSnapshotImageAttached: true,
+      snapshot: 'base64-encoded-image',
+      viewHierarchy: '<View><Button testID="login" title="Login" /></View>',
+      isSnapshotImageAttached: true
     };
 
     jest.spyOn(console, 'error').mockImplementation(() => {});
@@ -286,17 +300,45 @@ describe('Copilot', () => {
       expect(spyCopilotStepPerformer).toHaveBeenCalledWith(dummyContext);
     });
 
-    it('should extend the API catalog with a new category', () => {
+    it('should extend the API catalog with multiple categories sequentially', () => {
       Copilot.init(mockConfig);
       const instance = Copilot.getInstance();
 
       instance.extendAPICatalog([barCategory1]);
       instance.extendAPICatalog([bazCategory]);
 
-      expect(mockConfig.frameworkDriver.apiCatalog.categories).toEqual([
-        barCategory1,
-        bazCategory,
-      ]);
+      expect(mockConfig.frameworkDriver.apiCatalog.categories).toEqual([barCategory1, bazCategory]);
+    });
+
+    it('should pilot through steps successfully', async () => {
+      Copilot.init(mockConfig);
+      const instance = Copilot.getInstance();
+      const goal = 'test goal';
+
+      const mockPilotResult = {
+        steps: [
+          {
+            plan: {
+              thoughts: "Step 1 thoughts",
+              action: "Tap on GREAT button"
+            },
+            code: "code executed"
+          },
+          {
+            plan: {
+              thoughts: "Completed successfully",
+              action: "success"
+            }
+          }
+        ]
+      };
+
+      mockPilotPerformer.perform.mockResolvedValue(mockPilotResult);
+
+      const pilotResult = await instance.pilot(goal);
+
+      expect(instance['pilotPerformer'].perform).toHaveBeenCalledWith(goal);
+      expect(pilotResult).toEqual(mockPilotResult);
     });
   });
 
@@ -334,4 +376,4 @@ describe('Copilot', () => {
       });
     });
   });
-});
+});

src/Copilot.ts

@@ -1,13 +1,15 @@
 import {CopilotError} from "@/errors/CopilotError";
-import {PromptCreator} from "@/utils/PromptCreator";
-import {CodeEvaluator} from "@/utils/CodeEvaluator";
-import {SnapshotManager} from "@/utils/SnapshotManager";
-import {CopilotStepPerformer} from "@/actions/CopilotStepPerformer";
-import {Config, PreviousStep, TestingFrameworkAPICatalogCategory, PilotReport, ScreenCapturerResult} from "@/types";
+import {PromptCreator} from './utils/PromptCreator';
+import {CodeEvaluator} from './utils/CodeEvaluator';
+import {SnapshotManager} from './utils/SnapshotManager';
+import {CopilotStepPerformer} from './actions/CopilotStepPerformer';
+import {Config, PreviousStep, TestingFrameworkAPICatalogCategory, PilotReport, ScreenCapturerResult} from './types';
 import {CacheHandler} from "@/utils/CacheHandler";
-import {PilotPerformer} from "@/actions/PilotPerformer";
-import {PilotPromptCreator} from "@/utils/PilotPromptCreator";
-import {ScreenCapturer} from "@/utils/ScreenCapturer";
+import {PilotPerformer} from './actions/PilotPerformer';
+import {PilotPromptCreator} from './utils/PilotPromptCreator';
+import {ScreenCapturer} from './utils/ScreenCapturer';
+import {CopilotAPISearchPromptCreator} from './utils/CopilotAPISearchPromptCreator';
+import {ViewAnalysisPromptCreator} from './utils/ViewAnalysisPromptCreator';
 
 /**
  * The main Copilot class that provides AI-assisted testing capabilities for a given underlying testing framework.
@@ -18,6 +20,7 @@ export class Copilot {
     static instance?: Copilot;
 
     private readonly promptCreator: PromptCreator;
+    private readonly apiSearchPromptCreator: CopilotAPISearchPromptCreator;
     private readonly codeEvaluator: CodeEvaluator;
     private readonly snapshotManager: SnapshotManager;
     private previousSteps: PreviousStep[] = [];
@@ -30,6 +33,7 @@ export class Copilot {
 
     private constructor(config: Config) {
         this.promptCreator = new PromptCreator(config.frameworkDriver.apiCatalog);
+        this.apiSearchPromptCreator = new CopilotAPISearchPromptCreator(config.frameworkDriver.apiCatalog);
         this.codeEvaluator = new CodeEvaluator();
         this.snapshotManager = new SnapshotManager(config.frameworkDriver);
         this.pilotPromptCreator = new PilotPromptCreator();
@@ -38,11 +42,13 @@ export class Copilot {
         this.copilotStepPerformer = new CopilotStepPerformer(
             config.frameworkDriver.apiCatalog.context,
             this.promptCreator,
+            this.apiSearchPromptCreator,
+            new ViewAnalysisPromptCreator(config.frameworkDriver.apiCatalog),
             this.codeEvaluator,
-            this.snapshotManager,
             config.promptHandler,
             this.cacheHandler,
-            config.options?.cacheMode
+            config.options?.cacheMode,
+            config.options?.analysisMode
         );
         this.pilotPerformer = new PilotPerformer(this.pilotPromptCreator, this.copilotStepPerformer, config.promptHandler, this.screenCapturer);
     }
@@ -139,7 +145,7 @@ export class Copilot {
      /**
      * Performs an entire test flow using the provided goal.
      * @param goal A string which describes the flow should be executed.
-     * @returns pilot report with info about the actions thoughts ect ... 
+     * @returns pilot report with info about the actions thoughts ect ...
      */
      async pilot(goal :string) : Promise<PilotReport> {
         return await this.pilotPerformer.perform(goal);