From f2019ed9aadd0141e8da80d598a1b8e6ed5c58e9 Mon Sep 17 00:00:00 2001 From: lironsh Date: Mon, 30 Dec 2024 17:14:15 +0200 Subject: [PATCH 1/4] feat: Add API method Add the function to `types.ts` and `index.ts` --- src/Copilot.ts | 15 ++++++++++++++- src/index.ts | 5 ++++- src/types.ts | 8 ++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/Copilot.ts b/src/Copilot.ts index 8162369..5d730fe 100644 --- a/src/Copilot.ts +++ b/src/Copilot.ts @@ -3,7 +3,7 @@ import {PromptCreator} from "@/utils/PromptCreator"; import {CodeEvaluator} from "@/utils/CodeEvaluator"; import {SnapshotManager} from "@/utils/SnapshotManager"; import {StepPerformer} from "@/actions/StepPerformer"; -import {Config, PreviousStep} from "@/types"; +import {Config, PreviousStep, TestingFrameworkAPICatalogCategory} from "@/types"; import {CacheHandler} from "@/utils/CacheHandler"; /** @@ -21,8 +21,10 @@ export class Copilot { private stepPerformer: StepPerformer; private cacheHandler: CacheHandler; private isRunning: boolean = false; + private config: Config; private constructor(config: Config) { + this.config = config; this.promptCreator = new PromptCreator(config.frameworkDriver.apiCatalog); this.codeEvaluator = new CodeEvaluator(); this.snapshotManager = new SnapshotManager(config.frameworkDriver); @@ -109,6 +111,17 @@ export class Copilot { this.cacheHandler.flushTemporaryCache(); } + /** + * Allow the user to add context and categories (titles and items) to the existing API catalog. + * @param context - The contexts to register. + * @param categories - The categories to register. + */ + extendAPICatalog(categories: TestingFrameworkAPICatalogCategory[] | TestingFrameworkAPICatalogCategory, context?: any): void { + this.promptCreator.extendAPICategories(categories); + if (context) + this.stepPerformer.extendJSContext(context); + } + private didPerformStep(step: string, code: string, result: any): void { this.previousSteps = [...this.previousSteps, { step, diff --git a/src/index.ts b/src/index.ts index ab23097..eb5aafa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ import {Copilot} from "@/Copilot"; -import {CopilotFacade, Config} from "@/types"; +import {CopilotFacade, Config, TestingFrameworkAPICatalogCategory} from "@/types"; const copilot: CopilotFacade = { init: (config: Config) => { @@ -23,6 +23,9 @@ const copilot: CopilotFacade = { } return result; + }, + extendAPICatalog: (context: any, categories: TestingFrameworkAPICatalogCategory[]) => { + Copilot.getInstance().extendAPICatalog(context, categories); } }; diff --git a/src/types.ts b/src/types.ts index 259d74a..6d5245d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -41,6 +41,14 @@ export interface CopilotFacade { * @example 'Tap on the login button', 'A login form should be visible' */ perform: (...steps: string[]) => Promise; + + /** + * Extends the API catalog of the testing framework with additional categories and items. + * @param context The variables of the testing framework (i.e. exposes the matching function, expect, etc.). + * @param categories The categories to add to the API catalog. + * @note This can be used to add custom categories and items to the API catalog. + */ + extendAPICatalog: (categories: TestingFrameworkAPICatalogCategory[] | TestingFrameworkAPICatalogCategory, context?: any,) => void; } /** From e28574986ecd2881dcf8ec729f3e8af0583c4f7b Mon Sep 17 00:00:00 2001 From: lironsh Date: Mon, 6 Jan 2025 16:10:16 +0200 Subject: [PATCH 2/4] feat: implement the function `extendAPICatalog` - Add `extendAPICategories` function to `PromptCreator` - Add `extendJSContext` function to `StepPerformer - Add `extendAPICatalog` function to `Copilot` --- src/Copilot.ts | 2 +- src/actions/StepPerformer.ts | 16 +++++++++++++--- src/utils/PromptCreator.ts | 16 +++++++++++++++- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/Copilot.ts b/src/Copilot.ts index 5d730fe..026ee89 100644 --- a/src/Copilot.ts +++ b/src/Copilot.ts @@ -116,7 +116,7 @@ export class Copilot { * @param context - The contexts to register. * @param categories - The categories to register. */ - extendAPICatalog(categories: TestingFrameworkAPICatalogCategory[] | TestingFrameworkAPICatalogCategory, context?: any): void { + extendAPICatalog(categories: TestingFrameworkAPICatalogCategory[], context?: any): void { this.promptCreator.extendAPICategories(categories); if (context) this.stepPerformer.extendJSContext(context); diff --git a/src/actions/StepPerformer.ts b/src/actions/StepPerformer.ts index f80c848..c39d5e5 100644 --- a/src/actions/StepPerformer.ts +++ b/src/actions/StepPerformer.ts @@ -19,9 +19,19 @@ export class StepPerformer { ) { } + extendJSContext(newContext: any): void { + for (const key in newContext) { + if (key in this.context) { + console.log(`Notice: Context ${key} is overridden by the new context value`); + break; + } + } + this.context = {...this.context, ...newContext}; + } + private generateCacheKey(step: string, previous: PreviousStep[], viewHierarchy: string): string { const viewHierarchyHash = crypto.createHash('md5').update(viewHierarchy).digest('hex'); - return JSON.stringify({ step, previous, viewHierarchyHash }); + return JSON.stringify({step, previous, viewHierarchyHash}); } private async captureSnapshotAndViewHierarchy() { @@ -32,7 +42,7 @@ export class StepPerformer { const isSnapshotImageAttached = snapshot != null && this.promptHandler.isSnapshotImageSupported(); - return { snapshot, viewHierarchy, isSnapshotImageAttached }; + return {snapshot, viewHierarchy, isSnapshotImageAttached}; } private shouldOverrideCache() { @@ -73,7 +83,7 @@ export class StepPerformer { for (let attempt = 1; attempt <= attempts; attempt++) { try { - const { snapshot, viewHierarchy, isSnapshotImageAttached } = await this.captureSnapshotAndViewHierarchy(); + const {snapshot, viewHierarchy, isSnapshotImageAttached} = await this.captureSnapshotAndViewHierarchy(); const code = await this.generateCode(step, previous, snapshot, viewHierarchy, isSnapshotImageAttached); lastCode = code; diff --git a/src/utils/PromptCreator.ts b/src/utils/PromptCreator.ts index 62b05f2..c8e9d68 100644 --- a/src/utils/PromptCreator.ts +++ b/src/utils/PromptCreator.ts @@ -6,7 +6,21 @@ import { } from "@/types"; export class PromptCreator { - constructor(private apiCatalog: TestingFrameworkAPICatalog) {} + constructor(private apiCatalog: TestingFrameworkAPICatalog) { + } + + extendAPICategories(newCategories: TestingFrameworkAPICatalogCategory[]): void { + for (const category of newCategories) { + const existingCategory = this.apiCatalog.categories.find((existingCategory) => existingCategory.title === category.title); + if (existingCategory) { + for (const item of category.items) { + existingCategory.items.push(item); + } + } else { + this.apiCatalog.categories.push(category); + } + } + } createPrompt( intent: string, From 1d600dc87ab82a0b3148c3ceda995c3c937448e6 Mon Sep 17 00:00:00 2001 From: lironsh Date: Mon, 6 Jan 2025 17:20:54 +0200 Subject: [PATCH 3/4] test: add suitable test suites. - Add `Copilot`, `StepPerformer`, `index` and `PromptCreator` suites - Add utils file - Add snapshot file for `PromptCreator` test --- src/Copilot.test.ts | 65 ++++- src/Copilot.ts | 6 +- src/actions/StepPerformer.test.ts | 35 +++ src/index.ts | 4 +- src/integration tests/index.test.ts | 32 ++- src/test-utils/APICatalogTestUtils.ts | 39 +++ src/types.ts | 4 +- src/utils/CacheHandler.test.ts | 8 +- src/utils/CodeEvaluator.test.ts | 4 +- src/utils/PromptCreator.test.ts | 35 ++- src/utils/PromptCreator.ts | 19 +- .../__snapshots__/PromptCreator.test.ts.snap | 270 ++++++++++++++++++ src/utils/extractCodeBlock.test.ts | 2 +- 13 files changed, 490 insertions(+), 33 deletions(-) create mode 100644 src/test-utils/APICatalogTestUtils.ts diff --git a/src/Copilot.test.ts b/src/Copilot.test.ts index 66ec9e4..526a3e8 100644 --- a/src/Copilot.test.ts +++ b/src/Copilot.test.ts @@ -1,9 +1,14 @@ -import { Copilot } from '@/Copilot'; -import { StepPerformer } from '@/actions/StepPerformer'; -import { CopilotError } from '@/errors/CopilotError'; -import { Config } from "@/types"; -import fs from "fs"; -import { mockCache, mockedCacheFile } from "./test-utils/cache"; +import {Copilot} from '@/Copilot'; +import {StepPerformer} from '@/actions/StepPerformer'; +import {CopilotError} from '@/errors/CopilotError'; +import {Config} from "@/types"; +import {mockCache, mockedCacheFile} from "./test-utils/cache"; +import { + bazCategory, + barCategory2, + barCategory1, + dummyContext +} from "./test-utils/APICatalogTestUtils"; jest.mock('@/actions/StepPerformer'); jest.mock('fs'); @@ -28,7 +33,8 @@ describe('Copilot', () => { isSnapshotImageSupported: jest.fn().mockReturnValue(true) } }; - jest.spyOn(console, 'error').mockImplementation(() => {}); + jest.spyOn(console, 'error').mockImplementation(() => { + }); (StepPerformer.prototype.perform as jest.Mock).mockResolvedValue({code: 'code', result: true}); }); @@ -188,4 +194,49 @@ describe('Copilot', () => { expect(mockedCacheFile).toBeUndefined(); }); }); + + describe('extend API catalog', () => { + const spyStepPerformer = jest.spyOn(StepPerformer.prototype, 'extendJSContext'); + it('should extend the API catalog with a new category', () => { + Copilot.init(mockConfig); + const instance = Copilot.getInstance(); + + instance.extendAPICatalog([barCategory1]); + + expect(mockConfig.frameworkDriver.apiCatalog.categories).toEqual([barCategory1]); + expect(spyStepPerformer).not.toHaveBeenCalled(); + + }); + + it('should extend the API catalog with a new category and context', () => { + Copilot.init(mockConfig); + const instance = Copilot.getInstance(); + instance.extendAPICatalog([barCategory1], dummyContext); + + expect(mockConfig.frameworkDriver.apiCatalog.categories).toEqual([barCategory1]); + expect(spyStepPerformer).toHaveBeenCalledWith(dummyContext); + }); + + it('should extend the API catalog with an existing category', () => { + Copilot.init(mockConfig); + const instance = Copilot.getInstance(); + + instance.extendAPICatalog([barCategory1]) + instance.extendAPICatalog([barCategory2], dummyContext); + + expect(mockConfig.frameworkDriver.apiCatalog.categories.length).toEqual(1); + expect(mockConfig.frameworkDriver.apiCatalog.categories[0].items).toEqual([...barCategory1.items, ...barCategory2.items]); + expect(spyStepPerformer).toHaveBeenCalledWith(dummyContext); + }); + + it('should extend the API catalog with a new category', () => { + Copilot.init(mockConfig); + const instance = Copilot.getInstance(); + + instance.extendAPICatalog([barCategory1]); + instance.extendAPICatalog([bazCategory]); + + expect(mockConfig.frameworkDriver.apiCatalog.categories).toEqual([barCategory1, bazCategory]); + }); + }); }); diff --git a/src/Copilot.ts b/src/Copilot.ts index 026ee89..ad7615d 100644 --- a/src/Copilot.ts +++ b/src/Copilot.ts @@ -21,10 +21,8 @@ export class Copilot { private stepPerformer: StepPerformer; private cacheHandler: CacheHandler; private isRunning: boolean = false; - private config: Config; private constructor(config: Config) { - this.config = config; this.promptCreator = new PromptCreator(config.frameworkDriver.apiCatalog); this.codeEvaluator = new CodeEvaluator(); this.snapshotManager = new SnapshotManager(config.frameworkDriver); @@ -112,9 +110,9 @@ export class Copilot { } /** - * Allow the user to add context and categories (titles and items) to the existing API catalog. - * @param context - The contexts to register. + * Enriches the API catalog by adding the provided categories and JS context. * @param categories - The categories to register. + * @param context - (Optional) Additional JS context to register. */ extendAPICatalog(categories: TestingFrameworkAPICatalogCategory[], context?: any): void { this.promptCreator.extendAPICategories(categories); diff --git a/src/actions/StepPerformer.test.ts b/src/actions/StepPerformer.test.ts index b9288a9..48a7f81 100644 --- a/src/actions/StepPerformer.test.ts +++ b/src/actions/StepPerformer.test.ts @@ -5,6 +5,7 @@ import {SnapshotManager} from '@/utils/SnapshotManager'; import {CacheHandler} from '@/utils/CacheHandler'; import {PromptHandler, TestingFrameworkAPICatalog} from '@/types'; import * as crypto from 'crypto'; +import {dummyContext, dummyBarContext1, dummyBarContext2} from "../test-utils/APICatalogTestUtils"; jest.mock('fs'); jest.mock('crypto'); @@ -283,4 +284,38 @@ describe('StepPerformer', () => { expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); expect(mockCacheHandler.addToTemporaryCache).toHaveBeenCalled(); }); + + describe('extendJSContext', () =>{ + it('should extend the context with the given object', async () => { + // Initial context + stepPerformer.extendJSContext(dummyBarContext1); + + setupMocks(); + await stepPerformer.perform(INTENT); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith(PROMPT_RESULT, dummyBarContext1); + + // Extended context + const extendedContext = { ...dummyBarContext1, ...dummyContext }; + stepPerformer.extendJSContext(dummyContext); + + await stepPerformer.perform(INTENT); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith(PROMPT_RESULT, extendedContext); + }); + + it('should log when a context is overridden', async () => { + jest.spyOn(console, 'log'); + stepPerformer.extendJSContext(dummyBarContext1); + + setupMocks(); + await stepPerformer.perform(INTENT); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith(PROMPT_RESULT, dummyBarContext1); + + stepPerformer.extendJSContext(dummyBarContext2); + expect(console.log).toHaveBeenCalledWith('Notice: Context bar is overridden by the new context value'); + + await stepPerformer.perform(INTENT); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith(PROMPT_RESULT, dummyBarContext2); + + }); + }); }); diff --git a/src/index.ts b/src/index.ts index eb5aafa..70e3cdc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,8 +24,8 @@ const copilot: CopilotFacade = { return result; }, - extendAPICatalog: (context: any, categories: TestingFrameworkAPICatalogCategory[]) => { - Copilot.getInstance().extendAPICatalog(context, categories); + extendAPICatalog: (categories: TestingFrameworkAPICatalogCategory[], context?: any) => { + Copilot.getInstance().extendAPICatalog(categories, context); } }; diff --git a/src/integration tests/index.test.ts b/src/integration tests/index.test.ts index da5a093..0901316 100644 --- a/src/integration tests/index.test.ts +++ b/src/integration tests/index.test.ts @@ -1,9 +1,12 @@ import copilot from "@/index"; import fs from 'fs'; -import { Copilot } from "@/Copilot"; -import { PromptHandler, TestingFrameworkDriver } from "@/types"; +import {Copilot} from "@/Copilot"; +import {PromptHandler, TestingFrameworkDriver} from "@/types"; import * as crypto from 'crypto'; import {mockedCacheFile, mockCache} from "../test-utils/cache"; +import {PromptCreator} from "../utils/PromptCreator"; +import {StepPerformer} from "../actions/StepPerformer"; +import {bazCategory, barCategory1, dummyContext} from "../test-utils/APICatalogTestUtils"; jest.mock('crypto'); jest.mock('fs'); @@ -79,7 +82,6 @@ describe('Copilot Integration Tests', () => { it('should successfully perform an action', async () => { mockPromptHandler.runPrompt.mockResolvedValue('// No operation'); - await expect(copilot.perform('Tap on the login button')).resolves.not.toThrow(); expect(mockFrameworkDriver.captureSnapshotImage).toHaveBeenCalled(); @@ -330,4 +332,28 @@ describe('Copilot Integration Tests', () => { ); }); }); + + describe('API Catalog Extension', () => { + const spyPromptCreator = jest.spyOn(PromptCreator.prototype, 'extendAPICategories'); + const spyStepPerformer = jest.spyOn(StepPerformer.prototype, 'extendJSContext'); + + beforeEach(() => { + jest.clearAllMocks(); + copilot.init({ + frameworkDriver: mockFrameworkDriver, + promptHandler: mockPromptHandler, + }); + copilot.start(); + }); + + it('should call relevant functions to extend the catalog', () => { + + copilot.extendAPICatalog([bazCategory]); + expect(spyPromptCreator).toHaveBeenCalledTimes(1); + + copilot.extendAPICatalog([barCategory1], dummyContext); + expect(spyPromptCreator).toHaveBeenCalledTimes(2); + expect(spyStepPerformer).toHaveBeenCalledTimes(1); + }); + }); }); diff --git a/src/test-utils/APICatalogTestUtils.ts b/src/test-utils/APICatalogTestUtils.ts new file mode 100644 index 0000000..2eefe69 --- /dev/null +++ b/src/test-utils/APICatalogTestUtils.ts @@ -0,0 +1,39 @@ +export const bazCategory = { + title: 'Custom Actions', + items: [ + { + signature: 'swipe(direction: string)', + description: 'Swipes in the specified direction.', + example: 'await swipe("up");', + guidelines: ['Use this method to scroll the screen.'] + } + ] +}; + +export const barCategory1 = { + title: 'Actions', + items: [ + { + signature: 'tapButton(id: string)', + description: 'Taps the button with the specified ID.', + example: 'await tapButton("submit");', + guidelines: ['Use this method to tap buttons.'] + } + ] +}; + +export const barCategory2 = { + title: 'Actions', + items: [ + { + signature: 'swipe(direction: string)', + description: 'Swipes in the specified direction.', + example: 'await swipe("up");', + guidelines: ['Use this method to scroll the screen.'] + } + ] +}; + +export const dummyContext = {foo: jest.fn()}; +export const dummyBarContext1 = {bar: jest.fn()}; +export const dummyBarContext2 = {bar: jest.fn()}; diff --git a/src/types.ts b/src/types.ts index 6d5245d..ebbd3c9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -43,12 +43,12 @@ export interface CopilotFacade { perform: (...steps: string[]) => Promise; /** - * Extends the API catalog of the testing framework with additional categories and items. + * Extends the API catalog of the testing framework with additional APIs (categories and JS context). * @param context The variables of the testing framework (i.e. exposes the matching function, expect, etc.). * @param categories The categories to add to the API catalog. * @note This can be used to add custom categories and items to the API catalog. */ - extendAPICatalog: (categories: TestingFrameworkAPICatalogCategory[] | TestingFrameworkAPICatalogCategory, context?: any,) => void; + extendAPICatalog: (categories: TestingFrameworkAPICatalogCategory[], context?: any,) => void; } /** diff --git a/src/utils/CacheHandler.test.ts b/src/utils/CacheHandler.test.ts index 1b1f4d2..e17b16d 100644 --- a/src/utils/CacheHandler.test.ts +++ b/src/utils/CacheHandler.test.ts @@ -1,5 +1,5 @@ -import { CacheHandler } from './CacheHandler'; -import { mockCache, mockedCacheFile} from "../test-utils/cache"; +import {CacheHandler} from './CacheHandler'; +import {mockCache, mockedCacheFile} from "../test-utils/cache"; jest.mock('fs'); @@ -13,7 +13,7 @@ describe('CacheHandler', () => { describe('cache and file operations', () => { it('should load cache from file successfully if the file exists and is valid', () => { - mockCache({ 'cacheKey': 'value' }); + mockCache({'cacheKey': 'value'}); expect(cacheHandler.getStepFromCache('cacheKey')).toBeUndefined(); @@ -28,7 +28,7 @@ describe('CacheHandler', () => { cacheHandler.addToTemporaryCache('cacheKey', 'value'); cacheHandler.flushTemporaryCache(); - expect(mockedCacheFile).toEqual({ 'cacheKey': 'value' }); + expect(mockedCacheFile).toEqual({'cacheKey': 'value'}); }); }); diff --git a/src/utils/CodeEvaluator.test.ts b/src/utils/CodeEvaluator.test.ts index 7652bbf..7fc9b2e 100644 --- a/src/utils/CodeEvaluator.test.ts +++ b/src/utils/CodeEvaluator.test.ts @@ -1,4 +1,4 @@ -import { CodeEvaluator } from '@/utils/CodeEvaluator'; +import {CodeEvaluator} from '@/utils/CodeEvaluator'; describe('CodeEvaluator', () => { let codeEvaluator: CodeEvaluator; @@ -26,7 +26,7 @@ describe('CodeEvaluator', () => { const contextVariable = 43; const validCode = 'return contextVariable - 1;'; - await expect(codeEvaluator.evaluate(validCode, { contextVariable })).resolves.toStrictEqual({ + await expect(codeEvaluator.evaluate(validCode, {contextVariable})).resolves.toStrictEqual({ code: 'return contextVariable - 1;', result: 42 }); diff --git a/src/utils/PromptCreator.test.ts b/src/utils/PromptCreator.test.ts index f1b414f..c460052 100644 --- a/src/utils/PromptCreator.test.ts +++ b/src/utils/PromptCreator.test.ts @@ -1,8 +1,9 @@ -import { PromptCreator } from './PromptCreator'; +import {PromptCreator} from './PromptCreator'; import { PreviousStep, TestingFrameworkAPICatalog } from "@/types"; +import {bazCategory, barCategory2, promptCreatorConstructorMockAPI} from "../test-utils/APICatalogTestUtils"; const mockAPI: TestingFrameworkAPICatalog = { context: {}, @@ -49,6 +50,19 @@ const mockAPI: TestingFrameworkAPICatalog = { ] }; +describe('PromptCreator constructor', () => { + it('should merge redundant categories', () => { + let promptCreator: PromptCreator; + promptCreator = new PromptCreator(promptCreatorConstructorMockAPI); + const intent = 'expect button to be visible'; + const viewHierarchy = '