From 7107c769ad44e3767f16c51e46c6e2a00c2798e7 Mon Sep 17 00:00:00 2001 From: lironsh Date: Tue, 19 Nov 2024 11:39:49 +0200 Subject: [PATCH 1/7] test: add `fs` mocking util. --- src/test-utils/cache.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/test-utils/cache.ts diff --git a/src/test-utils/cache.ts b/src/test-utils/cache.ts new file mode 100644 index 0000000..194d8fc --- /dev/null +++ b/src/test-utils/cache.ts @@ -0,0 +1,15 @@ +import fs from "fs"; + +export let mockedCacheFile: { [key: string]: any } | undefined; + +export const mockCache = (data: { [key: string]: any } | undefined = undefined) => { + mockedCacheFile = data; + + (fs.writeFileSync as jest.Mock).mockImplementation((filePath, data) => { + mockedCacheFile = JSON.parse(data); + }); + + (fs.readFileSync as jest.Mock).mockReturnValue(JSON.stringify(mockedCacheFile)); + + (fs.existsSync as jest.Mock).mockReturnValue(mockedCacheFile !== undefined); +}; From f65949470f85e5a6246fc7fe0d3c19351d55f45c Mon Sep 17 00:00:00 2001 From: lironsh Date: Tue, 19 Nov 2024 11:34:43 +0200 Subject: [PATCH 2/7] feat: Add CacheHandler test: Add tests to CacheHandler test: Change tests due to the addition of CacheHandler chore: Change files due to the addition of CacheHandler chore: changes due to CacheHandler --- detox_copilot_cache.json | 5 +- src/Copilot.test.ts | 67 ++++++++++-- src/Copilot.ts | 40 ++++++- src/actions/StepPerformer.test.ts | 157 ++++++++++++++++------------ src/actions/StepPerformer.ts | 41 ++------ src/integration tests/index.test.ts | 89 ++++++++-------- src/utils/CacheHandler.test.ts | 102 ++++++++++++++++++ src/utils/CacheHandler.ts | 57 ++++++++++ src/utils/PromptCreator.ts | 12 +++ tsconfig.json | 1 + 10 files changed, 412 insertions(+), 159 deletions(-) create mode 100644 src/utils/CacheHandler.test.ts create mode 100644 src/utils/CacheHandler.ts diff --git a/detox_copilot_cache.json b/detox_copilot_cache.json index 9beb977..9e26dfe 100644 --- a/detox_copilot_cache.json +++ b/detox_copilot_cache.json @@ -1,4 +1 @@ -{ - "{\"step\":\"Tap on the login button\",\"previous\":[]}": "// No operation", - "{\"step\":\"The welcome message should be visible\",\"previous\":[]}": "// No operation" -} \ No newline at end of file +{} \ No newline at end of file diff --git a/src/Copilot.test.ts b/src/Copilot.test.ts index 9cb0fcc..9fa51ff 100644 --- a/src/Copilot.test.ts +++ b/src/Copilot.test.ts @@ -2,8 +2,13 @@ import { Copilot } from '@/Copilot'; import { StepPerformer } from '@/actions/StepPerformer'; import { CopilotError } from '@/errors/CopilotError'; import { Config } from "@/types"; +import fs from "fs"; +import { mockCache, mockedCacheFile } from "./test-utils/cache"; jest.mock('@/actions/StepPerformer'); +jest.mock('fs'); + +const INTENT = 'tap button'; describe('Copilot', () => { let mockConfig: Config; @@ -78,19 +83,19 @@ describe('Copilot', () => { Copilot.init(mockConfig); const instance = Copilot.getInstance(); - const intent = 'tap button'; + instance.start(); - await instance.performStep(intent); + await instance.performStep(INTENT); - expect(StepPerformer.prototype.perform).toHaveBeenCalledWith(intent, []); + expect(StepPerformer.prototype.perform).toHaveBeenCalledWith(INTENT, []); }); it('should return the result from StepPerformer.perform', async () => { Copilot.init(mockConfig); const instance = Copilot.getInstance(); - const intent = 'tap button'; + instance.start(); - const result = await instance.performStep(intent); + const result = await instance.performStep(INTENT); expect(result).toBe(true); }); @@ -98,6 +103,7 @@ describe('Copilot', () => { it('should accumulate previous intents', async () => { Copilot.init(mockConfig); const instance = Copilot.getInstance(); + instance.start(); const intent1 = 'tap button 1'; const intent2 = 'tap button 2'; @@ -112,18 +118,65 @@ describe('Copilot', () => { }); }); - describe('reset', () => { + describe('start', () => { it('should clear previous intents', async () => { Copilot.init(mockConfig); const instance = Copilot.getInstance(); + instance.start(); const intent1 = 'tap button 1'; const intent2 = 'tap button 2'; await instance.performStep(intent1); - instance.reset(); + instance.end(true); + instance.start(); await instance.performStep(intent2); expect(StepPerformer.prototype.perform).toHaveBeenLastCalledWith(intent2, []); }); }); + + describe('start and end behavior', () => { + it('should not perform before start', async () => { + Copilot.init(mockConfig); + const instance = Copilot.getInstance(); + + await expect(instance.performStep(INTENT)).rejects.toThrowError('Copilot is not running. Please call the `start()` method before performing any steps.'); + }); + + it('should not start without end the previous flow(start->start)', async () => { + Copilot.init(mockConfig); + const instance = Copilot.getInstance(); + instance.start(); + + await instance.performStep(INTENT); + + expect(() => instance.start()).toThrowError('Copilot was already started. Please call the `end()` method before starting a new test flow.'); + }); + + it('should not end without start a new flow(end->end)', async () => { + Copilot.init(mockConfig); + const instance = Copilot.getInstance(); + instance.start(); + + await instance.performStep(INTENT); + instance.end(true); + + expect(() => instance.end(true)).toThrowError('Copilot is not running. Please call the `start()` method before ending the test flow.'); + }); + }); + + describe('end', () => { + it('end with false should not save to cache', async () => { + mockCache(); + + Copilot.init(mockConfig); + const instance = Copilot.getInstance(); + instance.start(); + + await instance.performStep(INTENT); + instance.end(false); + + expect(mockedCacheFile).toBeUndefined(); + }); + }); }); diff --git a/src/Copilot.ts b/src/Copilot.ts index 6834a34..e582395 100644 --- a/src/Copilot.ts +++ b/src/Copilot.ts @@ -4,6 +4,7 @@ import {CodeEvaluator} from "@/utils/CodeEvaluator"; import {SnapshotManager} from "@/utils/SnapshotManager"; import {StepPerformer} from "@/actions/StepPerformer"; import {Config, PreviousStep} from "@/types"; +import {CacheHandler} from "@/utils/CacheHandler"; /** * The main Copilot class that provides AI-assisted testing capabilities for a given underlying testing framework. @@ -18,18 +19,25 @@ export class Copilot { private readonly snapshotManager: SnapshotManager; private previousSteps: PreviousStep[] = []; private stepPerformer: StepPerformer; + private cacheHandler: CacheHandler; + //private isTestSuiteSuccessful: boolean; + private isRunning: boolean = false; private constructor(config: Config) { this.promptCreator = new PromptCreator(config.frameworkDriver.apiCatalog); this.codeEvaluator = new CodeEvaluator(); this.snapshotManager = new SnapshotManager(config.frameworkDriver); + this.cacheHandler = new CacheHandler(); + //this.isTestSuiteSuccessful = true; this.stepPerformer = new StepPerformer( config.frameworkDriver.apiCatalog.context, this.promptCreator, this.codeEvaluator, this.snapshotManager, - config.promptHandler + config.promptHandler, + this.cacheHandler ); + } /** @@ -57,18 +65,42 @@ export class Copilot { * @param step The step describing the operation to perform. */ async performStep(step: string): Promise { + if (!this.isRunning) { + throw new CopilotError('Copilot is not running. Please call the `start()` method before performing any steps.'); + } + const {code, result} = await this.stepPerformer.perform(step, this.previousSteps); this.didPerformStep(step, code, result); - return result; } /** - * Resets the Copilot by clearing the previous steps. + * Starts the Copilot by clearing the previous steps and temporary cache. * @note This must be called before starting a new test flow, in order to clean context from previous tests. */ - reset(): void { + start(): void { + if (this.isRunning) { + throw new CopilotError('Copilot was already started. Please call the `end()` method before starting a new test flow.'); + } + + this.isRunning = true; this.previousSteps = []; + this.cacheHandler.clearTemporaryCache(); + } + + /** + * Ends the Copilot test flow and optionally saves the temporary cache to the main cache. + * @param saveToCache - boolean flag indicating whether the temporary cache data should be saved to the main cache. + */ + end(saveToCache: boolean = true): void { + if (!this.isRunning) { + throw new CopilotError('Copilot is not running. Please call the `start()` method before ending the test flow.'); + } + + this.isRunning = false; + + if (saveToCache) + this.cacheHandler.flushTemporaryCache(); } private didPerformStep(step: string, code: string, result: any): void { diff --git a/src/actions/StepPerformer.test.ts b/src/actions/StepPerformer.test.ts index 179f2a1..3788f5e 100644 --- a/src/actions/StepPerformer.test.ts +++ b/src/actions/StepPerformer.test.ts @@ -1,14 +1,24 @@ -import { StepPerformer } from '@/actions/StepPerformer'; -import { PromptCreator } from '@/utils/PromptCreator'; -import { CodeEvaluator } from '@/utils/CodeEvaluator'; -import { SnapshotManager } from '@/utils/SnapshotManager'; -import { PromptHandler, TestingFrameworkAPICatalog } from '@/types'; +import {StepPerformer} from '@/actions/StepPerformer'; +import {PromptCreator} from '@/utils/PromptCreator'; +import {CodeEvaluator} from '@/utils/CodeEvaluator'; +import {SnapshotManager} from '@/utils/SnapshotManager'; +import {CacheHandler} from '@/utils/CacheHandler'; +import {PromptHandler, TestingFrameworkAPICatalog} from '@/types'; import * as fs from 'fs'; import * as crypto from 'crypto'; +import mock = jest.mock; jest.mock('fs'); jest.mock('crypto'); +const INTENT = 'tap button'; +const VIEW_HIERARCHY = ''; +const PROMPT_RESULT = 'generated code'; +const CODE_EVALUATION_RESULT = 'success'; +const SNAPSHOT_DATA = 'snapshot_data'; +const VIEW_HIERARCHY_HASH = 'hash'; +const CACHE_KEY = JSON.stringify({step: INTENT, previous: [], viewHierarchyHash: VIEW_HIERARCHY_HASH}); + describe('StepPerformer', () => { let stepPerformer: StepPerformer; let mockContext: jest.Mocked; @@ -16,16 +26,11 @@ describe('StepPerformer', () => { let mockCodeEvaluator: jest.Mocked; let mockSnapshotManager: jest.Mocked; let mockPromptHandler: jest.Mocked; - const cacheFileName = 'test_step_performer_cache.json'; + let mockCacheHandler: jest.Mocked; beforeEach(() => { jest.resetAllMocks(); - // Mock fs methods to prevent actual file system interactions - (fs.existsSync as jest.Mock).mockReturnValue(false); - (fs.readFileSync as jest.Mock).mockReturnValue(''); - (fs.writeFileSync as jest.Mock).mockImplementation(() => {}); - const apiCatalog: TestingFrameworkAPICatalog = { context: {}, categories: [], @@ -56,13 +61,23 @@ describe('StepPerformer', () => { isSnapshotImageSupported: jest.fn(), } as jest.Mocked; + mockCacheHandler = { + loadCacheFromFile: jest.fn(), + saveCacheToFile: jest.fn(), + existInCache: jest.fn(), + addToTemporaryCache: jest.fn(), + flushTemporaryCache: jest.fn(), + clearTemporaryCache: jest.fn(), + getStepFromCache: jest.fn(), + } as unknown as jest.Mocked; + stepPerformer = new StepPerformer( mockContext, mockPromptCreator, mockCodeEvaluator, mockSnapshotManager, mockPromptHandler, - cacheFileName, // Use a test-specific cache file name + mockCacheHandler ); }); @@ -77,14 +92,14 @@ describe('StepPerformer', () => { } const setupMocks = ({ - isSnapshotSupported = true, - snapshotData = 'snapshot_data', - viewHierarchy = '', - promptResult = 'generated code', - codeEvaluationResult = 'success', - cacheExists = false, - overrideCache = false, - }: SetupMockOptions = {}) => { + isSnapshotSupported = true, + snapshotData = SNAPSHOT_DATA, + viewHierarchy = VIEW_HIERARCHY, + promptResult = PROMPT_RESULT, + codeEvaluationResult = CODE_EVALUATION_RESULT, + cacheExists = false, + overrideCache = false, + }: SetupMockOptions = {}) => { mockPromptHandler.isSnapshotImageSupported.mockReturnValue(isSnapshotSupported); mockSnapshotManager.captureSnapshotImage.mockResolvedValue( snapshotData != null ? snapshotData : undefined, @@ -105,71 +120,71 @@ describe('StepPerformer', () => { }), }); - // Adjust fs mocks based on cacheExists if (cacheExists) { - (fs.existsSync as jest.Mock).mockReturnValue(true); - const cacheData = {}; - const cacheKey = JSON.stringify({ step: 'tap button', previous: [], viewHierarchyHash }); - // @ts-ignore - cacheData[cacheKey] = promptResult; - (fs.readFileSync as jest.Mock).mockReturnValue(JSON.stringify(cacheData)); - } else { - (fs.existsSync as jest.Mock).mockReturnValue(false); + const cacheData: Map = new Map(); + cacheData.set(CACHE_KEY, PROMPT_RESULT); + + mockCacheHandler.getStepFromCache.mockImplementation((key: string) => { + return cacheData.get(key); + }); } }; it('should perform an intent successfully with snapshot image support', async () => { - const intent = 'tap button'; setupMocks(); - const result = await stepPerformer.perform(intent); + const result = await stepPerformer.perform(INTENT); + expect(mockCacheHandler.loadCacheFromFile).toHaveBeenCalled(); expect(result).toBe('success'); expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith( - intent, - '', + INTENT, + VIEW_HIERARCHY, true, [], ); - expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', 'snapshot_data'); - expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); - expect(fs.writeFileSync).toHaveBeenCalled(); // Ensure cache is saved + expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', SNAPSHOT_DATA); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith(PROMPT_RESULT, mockContext); + expect(mockCacheHandler.getStepFromCache).toHaveBeenCalledWith(CACHE_KEY); + expect(mockCacheHandler.addToTemporaryCache).toHaveBeenCalledWith(CACHE_KEY, PROMPT_RESULT); }); it('should perform an intent successfully without snapshot image support', async () => { - const intent = 'tap button'; - setupMocks({ isSnapshotSupported: false }); + setupMocks({isSnapshotSupported: false}); - const result = await stepPerformer.perform(intent); + const result = await stepPerformer.perform(INTENT); + expect(mockCacheHandler.loadCacheFromFile).toHaveBeenCalled(); expect(result).toBe('success'); expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith( - intent, - '', + INTENT, + VIEW_HIERARCHY, false, [], ); expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', undefined); - expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); - expect(fs.writeFileSync).toHaveBeenCalled(); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith(PROMPT_RESULT, mockContext); + expect(mockCacheHandler.getStepFromCache).toHaveBeenCalledWith(CACHE_KEY); + expect(mockCacheHandler.addToTemporaryCache).toHaveBeenCalledWith(CACHE_KEY, PROMPT_RESULT); }); it('should perform an intent with undefined snapshot', async () => { - const intent = 'tap button'; - setupMocks({ snapshotData: null }); + setupMocks({snapshotData: null}); - const result = await stepPerformer.perform(intent); + const result = await stepPerformer.perform(INTENT); + expect(mockCacheHandler.loadCacheFromFile).toHaveBeenCalled(); expect(result).toBe('success'); expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith( - intent, - '', + INTENT, + VIEW_HIERARCHY, false, [], ); expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', undefined); - expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); - expect(fs.writeFileSync).toHaveBeenCalled(); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith(PROMPT_RESULT, mockContext); + expect(mockCacheHandler.getStepFromCache).toHaveBeenCalledWith(CACHE_KEY); + expect(mockCacheHandler.addToTemporaryCache).toHaveBeenCalledWith(CACHE_KEY, PROMPT_RESULT); }); it('should perform an intent successfully with previous intents', async () => { @@ -182,78 +197,84 @@ describe('StepPerformer', () => { setupMocks(); + const thisCacheKey = JSON.stringify({ + step: intent, + previous: previousIntents, + viewHierarchyHash: VIEW_HIERARCHY_HASH + }); const result = await stepPerformer.perform(intent, previousIntents); + expect(mockCacheHandler.loadCacheFromFile).toHaveBeenCalled(); expect(result).toBe('success'); expect(mockPromptCreator.createPrompt).toHaveBeenCalledWith( intent, - '', + VIEW_HIERARCHY, true, previousIntents, ); - expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', 'snapshot_data'); - expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); - expect(fs.writeFileSync).toHaveBeenCalled(); + expect(mockPromptHandler.runPrompt).toHaveBeenCalledWith('generated prompt', SNAPSHOT_DATA); + expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith(PROMPT_RESULT, mockContext); + expect(mockCacheHandler.getStepFromCache).toHaveBeenCalledWith(thisCacheKey); + expect(mockCacheHandler.addToTemporaryCache).toHaveBeenCalledWith(thisCacheKey, PROMPT_RESULT); }); it('should throw an error if code evaluation fails', async () => { - const intent = 'tap button'; setupMocks(); mockCodeEvaluator.evaluate.mockRejectedValue(new Error('Evaluation failed')); - await expect(stepPerformer.perform(intent)).rejects.toThrow('Evaluation failed'); - expect(fs.writeFileSync).toHaveBeenCalled(); // Cache should be saved + await expect(stepPerformer.perform(INTENT)).rejects.toThrow('Evaluation failed'); + expect(mockCacheHandler.addToTemporaryCache).toHaveBeenCalledWith(CACHE_KEY, PROMPT_RESULT); }); it('should use cached prompt result if available', async () => { - const intent = 'tap button'; - setupMocks({ cacheExists: true }); + setupMocks({cacheExists: true}); - const result = await stepPerformer.perform(intent); + const result = await stepPerformer.perform(INTENT); expect(result).toBe('success'); + expect(mockCacheHandler.getStepFromCache).toHaveBeenCalledWith(CACHE_KEY); // Should not call runPrompt or createPrompt since result is cached expect(mockPromptCreator.createPrompt).not.toHaveBeenCalled(); expect(mockPromptHandler.runPrompt).not.toHaveBeenCalled(); expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); - expect(fs.writeFileSync).not.toHaveBeenCalled(); // No need to save cache again + expect(mockCacheHandler.addToTemporaryCache).not.toHaveBeenCalled(); // No need to save cache again }); it('should retry if initial runPrompt throws an error and succeed on retry', async () => { - const intent = 'tap button'; setupMocks(); const error = new Error('Initial prompt failed'); mockPromptHandler.runPrompt.mockRejectedValueOnce(error); // On retry, it succeeds mockPromptHandler.runPrompt.mockResolvedValueOnce('retry generated code'); - const result = await stepPerformer.perform(intent); + const result = await stepPerformer.perform(INTENT); expect(result).toBe('success'); + expect(mockCacheHandler.loadCacheFromFile).toHaveBeenCalled(); expect(mockPromptCreator.createPrompt).toHaveBeenCalledTimes(2); expect(mockPromptHandler.runPrompt).toHaveBeenCalledTimes(2); expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('retry generated code', mockContext); - expect(fs.writeFileSync).toHaveBeenCalledTimes(1); // Cache should be saved after success + expect(mockCacheHandler.addToTemporaryCache).toHaveBeenCalledTimes(1); // Cache should be saved after success }); it('should throw original error if retry also fails', async () => { - const intent = 'tap button'; setupMocks(); const error = new Error('Initial prompt failed'); const retryError = new Error('Retry prompt failed'); mockPromptHandler.runPrompt.mockRejectedValueOnce(error); mockPromptHandler.runPrompt.mockRejectedValueOnce(retryError); - await expect(stepPerformer.perform(intent)).rejects.toThrow(retryError); + await expect(stepPerformer.perform(INTENT)).rejects.toThrow(retryError); + expect(mockCacheHandler.loadCacheFromFile).toHaveBeenCalled(); expect(mockPromptCreator.createPrompt).toHaveBeenCalledTimes(2); expect(mockPromptHandler.runPrompt).toHaveBeenCalledTimes(2); expect(mockCodeEvaluator.evaluate).not.toHaveBeenCalled(); - expect(fs.writeFileSync).not.toHaveBeenCalled(); + expect(mockCacheHandler.addToTemporaryCache).not.toHaveBeenCalled(); }); it('should not use cached prompt result if COPILOT_OVERRIDE_CACHE is enabled', async () => { const intent = 'tap button'; - setupMocks({ cacheExists: true, overrideCache: true }); + setupMocks({cacheExists: true, overrideCache: true}); const result = await stepPerformer.perform(intent); diff --git a/src/actions/StepPerformer.ts b/src/actions/StepPerformer.ts index 3a430bf..f80c848 100644 --- a/src/actions/StepPerformer.ts +++ b/src/actions/StepPerformer.ts @@ -1,6 +1,7 @@ import {PromptCreator} from '@/utils/PromptCreator'; import {CodeEvaluator} from '@/utils/CodeEvaluator'; import {SnapshotManager} from '@/utils/SnapshotManager'; +import {CacheHandler} from '@/utils/CacheHandler'; import {CodeEvaluationResult, PreviousStep, PromptHandler} from '@/types'; import * as fs from 'fs'; import * as path from 'path'; @@ -8,18 +9,14 @@ import * as crypto from 'crypto'; import {extractCodeBlock} from '@/utils/extractCodeBlock'; export class StepPerformer { - private cache: Map = new Map(); - private readonly cacheFilePath: string; - constructor( private context: any, private promptCreator: PromptCreator, private codeEvaluator: CodeEvaluator, private snapshotManager: SnapshotManager, private promptHandler: PromptHandler, - cacheFileName: string = 'detox_copilot_cache.json', + private cacheHandler: CacheHandler, ) { - this.cacheFilePath = path.resolve(process.cwd(), cacheFileName); } private generateCacheKey(step: string, previous: PreviousStep[], viewHierarchy: string): string { @@ -27,30 +24,6 @@ export class StepPerformer { return JSON.stringify({ step, previous, viewHierarchyHash }); } - private loadCacheFromFile(): void { - try { - if (fs.existsSync(this.cacheFilePath)) { - const data = fs.readFileSync(this.cacheFilePath, 'utf-8'); - const json = JSON.parse(data); - this.cache = new Map(Object.entries(json)); - } else { - this.cache.clear(); // Ensure cache is empty if file doesn't exist - } - } catch (error) { - console.warn('Error loading cache from file:', error); - this.cache.clear(); // Clear cache on error to avoid stale data - } - } - - private saveCacheToFile(): void { - try { - const json = Object.fromEntries(this.cache); - fs.writeFileSync(this.cacheFilePath, JSON.stringify(json, null, 2), { flag: 'w+' }); - } catch (error) { - console.error('Error saving cache to file:', error); - } - } - private async captureSnapshotAndViewHierarchy() { const snapshot = this.promptHandler.isSnapshotImageSupported() ? await this.snapshotManager.captureSnapshotImage() @@ -75,15 +48,15 @@ export class StepPerformer { ): Promise { const cacheKey = this.generateCacheKey(step, previous, viewHierarchy); - if (!this.shouldOverrideCache() && this.cache.has(cacheKey)) { - return this.cache.get(cacheKey); + const cachedCode = this.cacheHandler.getStepFromCache(cacheKey); + if (!this.shouldOverrideCache() && cachedCode) { + return cachedCode; } else { const prompt = this.promptCreator.createPrompt(step, viewHierarchy, isSnapshotImageAttached, previous); const promptResult = await this.promptHandler.runPrompt(prompt, snapshot); const code = extractCodeBlock(promptResult); - this.cache.set(cacheKey, code); - this.saveCacheToFile(); + this.cacheHandler.addToTemporaryCache(cacheKey, code); return code; } @@ -93,7 +66,7 @@ export class StepPerformer { // TODO: replace with the user's logger console.log('\x1b[90m%s\x1b[0m%s', 'Copilot performing:', `"${step}"`); - this.loadCacheFromFile(); + this.cacheHandler.loadCacheFromFile(); let lastError: any = null; let lastCode: string | undefined; diff --git a/src/integration tests/index.test.ts b/src/integration tests/index.test.ts index 6187140..6227b33 100644 --- a/src/integration tests/index.test.ts +++ b/src/integration tests/index.test.ts @@ -1,19 +1,15 @@ import copilot from "@/index"; import { Copilot } from "@/Copilot"; import { PromptHandler, TestingFrameworkDriver } from "@/types"; -import fs from 'fs'; -import path from 'path'; import * as crypto from 'crypto'; +import {mockedCacheFile, mockCache} from "../test-utils/cache"; -jest.mock('fs'); -jest.mock('path'); jest.mock('crypto'); +jest.mock('fs'); describe('Copilot Integration Tests', () => { let mockFrameworkDriver: jest.Mocked; let mockPromptHandler: jest.Mocked; - let mockFs: jest.Mocked; - let mockPath: jest.Mocked; beforeEach(() => { jest.clearAllMocks(); @@ -32,13 +28,15 @@ describe('Copilot Integration Tests', () => { isSnapshotImageSupported: jest.fn().mockReturnValue(true) }; - mockFs = fs as jest.Mocked; - mockPath = path as jest.Mocked; + // mockFs = fs as jest.Mocked; + // mockPath = path as jest.Mocked; + // + // mockFs.existsSync.mockReturnValue(false); + // mockFs.readFileSync.mockReturnValue('{}'); + // mockFs.writeFileSync.mockImplementation(() => {}); + // mockPath.resolve.mockImplementation((...paths) => paths.join('/')); - mockFs.existsSync.mockReturnValue(false); - mockFs.readFileSync.mockReturnValue('{}'); - mockFs.writeFileSync.mockImplementation(() => {}); - mockPath.resolve.mockImplementation((...paths) => paths.join('/')); + mockCache(); (crypto.createHash as jest.Mock).mockReturnValue({ update: jest.fn().mockReturnValue({ @@ -70,6 +68,7 @@ describe('Copilot Integration Tests', () => { frameworkDriver: mockFrameworkDriver, promptHandler: mockPromptHandler }); + copilot.start(); }); it('should successfully perform an action', async () => { @@ -123,6 +122,7 @@ describe('Copilot Integration Tests', () => { frameworkDriver: mockFrameworkDriver, promptHandler: mockPromptHandler }); + copilot.start(); }); it('should perform multiple steps using spread operator', async () => { @@ -167,6 +167,7 @@ describe('Copilot Integration Tests', () => { frameworkDriver: mockFrameworkDriver, promptHandler: mockPromptHandler }); + copilot.start(); }); it('should throw error when PromptHandler fails', async () => { @@ -194,13 +195,15 @@ describe('Copilot Integration Tests', () => { frameworkDriver: mockFrameworkDriver, promptHandler: mockPromptHandler }); + copilot.start(); }); it('should reset context when reset is called', async () => { mockPromptHandler.runPrompt.mockResolvedValueOnce('// Login action'); await copilot.perform('Log in to the application'); - copilot.reset(); + copilot.end(); + copilot.start(); mockPromptHandler.runPrompt.mockResolvedValueOnce('// New action after reset'); await copilot.perform('Perform action after reset'); @@ -221,7 +224,8 @@ describe('Copilot Integration Tests', () => { expect(lastCallArgsBeforeReset).toContain('Action 1'); expect(lastCallArgsBeforeReset).toContain('Action 2'); - copilot.reset(); + copilot.end(); + copilot.start(); mockPromptHandler.runPrompt.mockResolvedValueOnce('// New action'); await copilot.perform('New action after reset'); @@ -239,57 +243,57 @@ describe('Copilot Integration Tests', () => { frameworkDriver: mockFrameworkDriver, promptHandler: mockPromptHandler }); + copilot.start(); }); it('should create cache file if it does not exist', async () => { - mockFs.existsSync.mockReturnValue(false); mockPromptHandler.runPrompt.mockResolvedValue('// Perform action'); await copilot.perform('Perform action'); + copilot.end(true) - expect(mockFs.writeFileSync).toHaveBeenCalled(); + expect(mockedCacheFile).toEqual({ + '{"step":"Perform action","previous":[],"viewHierarchyHash":"hash"}': '// Perform action' + }); }); it('should read from existing cache file', async () => { - mockFs.existsSync.mockReturnValue(true); - mockFs.readFileSync.mockReturnValue(JSON.stringify({ + mockCache({ '{"step":"Cached action","previous":[],"viewHierarchyHash":"hash"}': '// Cached action code' - })); + }); await copilot.perform('Cached action'); - expect(mockFs.readFileSync).toHaveBeenCalled(); - expect(mockPromptHandler.runPrompt).not.toHaveBeenCalled(); + expect(mockPromptHandler.runPrompt).toHaveBeenCalledTimes(0) }); it('should update cache file after performing new action', async () => { mockPromptHandler.runPrompt.mockResolvedValue('// New action code'); await copilot.perform('New action'); + copilot.end(); - expect(mockFs.writeFileSync).toHaveBeenCalledWith( - expect.any(String), - expect.stringContaining('New action'), - expect.any(Object) - ); - }); - - it('should handle fs.readFileSync errors', async () => { - mockFs.existsSync.mockReturnValue(true); - mockFs.readFileSync.mockImplementation(() => { throw new Error('Read error'); }); - mockPromptHandler.runPrompt.mockResolvedValue('// New action code'); - - await copilot.perform('Action with read error'); - - expect(mockPromptHandler.runPrompt).toHaveBeenCalled(); + expect(mockedCacheFile).toEqual({ + '{"step":"New action","previous":[],"viewHierarchyHash":"hash"}': '// New action code' + }); }); - it('should handle fs.writeFileSync errors', async () => { - mockFs.writeFileSync.mockImplementation(() => { throw new Error('Write error'); }); - mockPromptHandler.runPrompt.mockResolvedValue('// Action code'); - - await expect(copilot.perform('Action with write error')).resolves.not.toThrow(); - }); + // it('should handle fs.readFileSync errors', async () => { + // mockFs.existsSync.mockReturnValue(true); + // mockFs.readFileSync.mockImplementation(() => { throw new Error('Read error'); }); + // mockPromptHandler.runPrompt.mockResolvedValue('// New action code'); + // + // await copilot.perform('Action with read error'); + // + // expect(mockPromptHandler.runPrompt).toHaveBeenCalled(); + // }); + + // it('should handle fs.writeFileSync errors', async () => { + // mockFs.writeFileSync.mockImplementation(() => { throw new Error('Write error'); }); + // mockPromptHandler.runPrompt.mockResolvedValue('// Action code'); + // + // await expect(copilot.perform('Action with write error')).resolves.not.toThrow(); + // }); }); describe('Feature Support', () => { @@ -298,6 +302,7 @@ describe('Copilot Integration Tests', () => { frameworkDriver: mockFrameworkDriver, promptHandler: mockPromptHandler }); + copilot.start(); }); it('should work without snapshot images when not supported', async () => { diff --git a/src/utils/CacheHandler.test.ts b/src/utils/CacheHandler.test.ts new file mode 100644 index 0000000..f0c7e15 --- /dev/null +++ b/src/utils/CacheHandler.test.ts @@ -0,0 +1,102 @@ +import { CacheHandler } from './CacheHandler'; +jest.mock('fs'); + +import { mockCache, mockedCacheFile} from "../test-utils/cache"; + +describe('CacheHandler', () => { + let cacheHandler: CacheHandler; + + beforeEach(() => { + jest.resetAllMocks(); + cacheHandler = new CacheHandler(); + }); + + describe('cache and file operations', () => { + it('should load cache from file successfully if the file exists and is valid', () => { + mockCache({ 'cacheKey': 'value' }); + + expect(cacheHandler.getStepFromCache('cacheKey')).toBeUndefined(); + + cacheHandler.loadCacheFromFile(); + + expect(cacheHandler.getStepFromCache('cacheKey')).toBe('value'); + }); + + it('should save cache to file successfully', () => { + mockCache(); + + cacheHandler.addToTemporaryCache('cacheKey', 'value'); + cacheHandler.flushTemporaryCache(); + + expect(mockedCacheFile).toEqual({ 'cacheKey': 'value' }); + }); + }); + + describe('addToTemporaryCache', () => { + it('should not save to cache', () => { + mockCache(); + + cacheHandler.addToTemporaryCache('cacheKey', 'value'); + + expect(cacheHandler.getStepFromCache('cacheKey')).toBeUndefined(); + expect(mockedCacheFile).toBeUndefined(); + }); + }); + + describe('getStepFromCache', () => { + it('should retrieve a value from cache using getStepFromCache', () => { + cacheHandler.addToTemporaryCache('some_key', 'value'); + cacheHandler.flushTemporaryCache(); + + const result = cacheHandler.getStepFromCache('some_key'); + + expect(result).toBe('value'); + }); + + it('should return undefined if the key does not exist in cache', () => { + const result = cacheHandler.getStepFromCache('non_existent_key'); + + expect(result).toBeUndefined(); + }); + }); + + describe('flushTemporaryCache', () => { + it('should move all temporary cache entries to the main cache', () => { + expect(cacheHandler.getStepFromCache('cacheKey1')).toBeUndefined(); + + cacheHandler.addToTemporaryCache('cacheKey1', 'value1'); + cacheHandler.addToTemporaryCache('cacheKey2', 'value2'); + cacheHandler.addToTemporaryCache('cacheKey3', 'value3'); + + cacheHandler.flushTemporaryCache() + + expect(cacheHandler.getStepFromCache('cacheKey1')).toBe('value1'); + expect(cacheHandler.getStepFromCache('cacheKey3')).toBe('value3'); + expect(cacheHandler.getStepFromCache('cacheKey2')).not.toBe('value3'); + }); + + it('should get the updated value from cache', () => { + expect(cacheHandler.getStepFromCache('cacheKey1')).toBeUndefined(); + + cacheHandler.addToTemporaryCache('cacheKey1', 'value1'); + cacheHandler.addToTemporaryCache('cacheKey1', 'value2'); + + cacheHandler.flushTemporaryCache() + + expect(cacheHandler.getStepFromCache('cacheKey1')).toBe('value2'); + }); + }); + + it('should clear the temporary cache', () => { + mockCache(); + cacheHandler.addToTemporaryCache('cacheKey', 'value'); + + expect(cacheHandler.getStepFromCache('cacheKey')).toBeUndefined(); + + cacheHandler.clearTemporaryCache(); + cacheHandler.flushTemporaryCache() + + expect(cacheHandler.getStepFromCache('cacheKey')).toBeUndefined(); + expect(mockedCacheFile).toStrictEqual({}); + }); +}); diff --git a/src/utils/CacheHandler.ts b/src/utils/CacheHandler.ts new file mode 100644 index 0000000..868850d --- /dev/null +++ b/src/utils/CacheHandler.ts @@ -0,0 +1,57 @@ +import fs from 'fs'; +import path from 'path'; + +export class CacheHandler { + private cache: Map = new Map(); + private temporaryCache: Map = new Map(); + private readonly cacheFilePath: string; + + constructor(cacheFileName: string = 'detox_copilot_cache.json') { + this.cacheFilePath = path.resolve(process.cwd(), cacheFileName); + } + + public loadCacheFromFile(): void { + try { + if (fs.existsSync(this.cacheFilePath)) { + const readFileSync = fs.readFileSync; + const data = fs.readFileSync(this.cacheFilePath, 'utf-8'); + const json = JSON.parse(data); + this.cache = new Map(Object.entries(json)); + } else { + this.cache.clear(); // Ensure cache is empty if file doesn't exist + } + } catch (error) { + console.warn('Error loading cache from file:', error); + this.cache.clear(); // Clear cache on error to avoid stale data + } + } + + private saveCacheToFile(): void { + try { + const json = Object.fromEntries(this.cache); + fs.writeFileSync(this.cacheFilePath, JSON.stringify(json, null, 2), { flag: 'w+' }); + } catch (error) { + console.error('Error saving cache to file:', error); + } + } + + public getStepFromCache(key: string): any | undefined { + return this.cache.get(key); + } + + public addToTemporaryCache(key: string, value: any): void { + this.temporaryCache.set(key, value); + } + + public flushTemporaryCache(): void { + this.temporaryCache.forEach((value, key) => { + this.cache.set(key, value); + }); + this.saveCacheToFile(); + this.clearTemporaryCache(); + } + + public clearTemporaryCache(): void { + this.temporaryCache.clear(); + } +} diff --git a/src/utils/PromptCreator.ts b/src/utils/PromptCreator.ts index fbf795e..62b05f2 100644 --- a/src/utils/PromptCreator.ts +++ b/src/utils/PromptCreator.ts @@ -62,6 +62,13 @@ export class PromptCreator { "A snapshot image is attached for visual reference.", "" ); + } else { + context.push( + "### Snapshot image", + "", + "No snapshot image is attached for this intent.", + "" + ); } if (previousSteps.length > 0) { @@ -139,6 +146,10 @@ export class PromptCreator { "", ...this.createStepByStepInstructions(isSnapshotImageAttached).map((instruction, index) => `${index + 1}. ${instruction}`), "", + "### Verify the prompt", + "", + "Before generating the code, please review the provided context and instructions to ensure they are clear and unambiguous. If you encounter any issues or have questions, please throw an informative error explaining the problem.", + "", "### Examples", "", "#### Example of throwing an informative error:", @@ -166,6 +177,7 @@ export class PromptCreator { if (isSnapshotImageAttached) { steps.push( "Analyze the provided intent, the view hierarchy, and the snapshot image to understand the required action.", + "Assess the positions of elements within the screen layout. Ensure that tests accurately reflect their intended locations, such as whether an element is centered or positioned relative to others. Tests should fail if the actual locations do not align with the expected configuration.", "Determine if the intent can be fully validated visually using the snapshot image.", "If the intent can be visually analyzed and passes the visual check, return only comments explaining the successful visual assertion.", "If the visual assertion fails, return code that throws an informative error explaining the failure.", diff --git a/tsconfig.json b/tsconfig.json index 989ad80..8084026 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,7 @@ "include": ["src/**/*"], "exclude": [ "node_modules", + "test-utils", "**/*.test.ts", "**/*.test.ts.snap" ], From d24767f75bcb57f0a2fe3530c5541ed2c31f3fdd Mon Sep 17 00:00:00 2001 From: lironsh Date: Tue, 19 Nov 2024 11:38:01 +0200 Subject: [PATCH 3/7] chore: Change from "reset" to "start" & "end" --- src/index.ts | 9 ++++++--- src/types.ts | 12 ++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index aeb7816..2da9536 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,12 +1,15 @@ -import { Copilot } from "@/Copilot"; +import {Copilot} from "@/Copilot"; import {CopilotFacade, Config} from "@/types"; const copilot: CopilotFacade = { init: (config: Config) => { Copilot.init(config); }, - reset: () => { - Copilot.getInstance().reset(); + start: () => { + Copilot.getInstance().start(); + }, + end: (saveToCache?: boolean) => { + Copilot.getInstance().end(saveToCache); }, perform: async (...steps: string[]) => { const copilotInstance = Copilot.getInstance(); diff --git a/src/types.ts b/src/types.ts index 1e12710..db8e000 100644 --- a/src/types.ts +++ b/src/types.ts @@ -10,11 +10,19 @@ export interface CopilotFacade { init: (config: Config) => void; /** - * Resets the Copilot instance. + * Start the Copilot instance. * Must be called before each test to ensure a clean state (the Copilot uses the operations history as part of * its context). */ - reset: () => void; + start: () => void; + + /** + * Finalizes the test flow and optionally saves temporary cache data to the main cache. + * If `saveToCache` is true, the temporary cache will be saved. True is the default value. + * @param saveToCache + * @note This must be called after the test flow is complete. + */ + end: (saveToCache?: boolean) => void; /** * Performs a testing operation or series of testing operations in the app based on the given `steps`. From 2fda9a5c372eba2aaa6d1996a616d873ae9b59e8 Mon Sep 17 00:00:00 2001 From: lironsh Date: Tue, 10 Dec 2024 12:08:21 +0200 Subject: [PATCH 4/7] chore: Change the default behavior to cache saving - The parameter of the 'end' function changed from "saveToCache" to "isCacheDisabled" - Test were modified accordingly --- src/Copilot.test.ts | 4 +- src/Copilot.ts | 8 ++-- src/actions/StepPerformer.test.ts | 3 +- src/index.ts | 4 +- src/integration tests/index.test.ts | 48 +++++++++---------- src/types.ts | 6 +-- .../__snapshots__/PromptCreator.test.ts.snap | 35 +++++++++++--- 7 files changed, 63 insertions(+), 45 deletions(-) diff --git a/src/Copilot.test.ts b/src/Copilot.test.ts index 9fa51ff..fcf8729 100644 --- a/src/Copilot.test.ts +++ b/src/Copilot.test.ts @@ -166,7 +166,7 @@ describe('Copilot', () => { }); describe('end', () => { - it('end with false should not save to cache', async () => { + it('end with disable cache=true should not save to cache', async () => { mockCache(); Copilot.init(mockConfig); @@ -174,7 +174,7 @@ describe('Copilot', () => { instance.start(); await instance.performStep(INTENT); - instance.end(false); + instance.end(true); expect(mockedCacheFile).toBeUndefined(); }); diff --git a/src/Copilot.ts b/src/Copilot.ts index e582395..fdfc0d7 100644 --- a/src/Copilot.ts +++ b/src/Copilot.ts @@ -20,7 +20,6 @@ export class Copilot { private previousSteps: PreviousStep[] = []; private stepPerformer: StepPerformer; private cacheHandler: CacheHandler; - //private isTestSuiteSuccessful: boolean; private isRunning: boolean = false; private constructor(config: Config) { @@ -28,7 +27,6 @@ export class Copilot { this.codeEvaluator = new CodeEvaluator(); this.snapshotManager = new SnapshotManager(config.frameworkDriver); this.cacheHandler = new CacheHandler(); - //this.isTestSuiteSuccessful = true; this.stepPerformer = new StepPerformer( config.frameworkDriver.apiCatalog.context, this.promptCreator, @@ -90,16 +88,16 @@ export class Copilot { /** * Ends the Copilot test flow and optionally saves the temporary cache to the main cache. - * @param saveToCache - boolean flag indicating whether the temporary cache data should be saved to the main cache. + * @param isCacheDisabled - boolean flag indicating whether the temporary cache data should be saved to the main cache. */ - end(saveToCache: boolean = true): void { + end(isCacheDisabled: boolean = false): void { if (!this.isRunning) { throw new CopilotError('Copilot is not running. Please call the `start()` method before ending the test flow.'); } this.isRunning = false; - if (saveToCache) + if (!isCacheDisabled) this.cacheHandler.flushTemporaryCache(); } diff --git a/src/actions/StepPerformer.test.ts b/src/actions/StepPerformer.test.ts index 3788f5e..d20ec89 100644 --- a/src/actions/StepPerformer.test.ts +++ b/src/actions/StepPerformer.test.ts @@ -7,6 +7,7 @@ import {PromptHandler, TestingFrameworkAPICatalog} from '@/types'; import * as fs from 'fs'; import * as crypto from 'crypto'; import mock = jest.mock; +import copilot from "../index"; jest.mock('fs'); jest.mock('crypto'); @@ -283,6 +284,6 @@ describe('StepPerformer', () => { expect(mockPromptCreator.createPrompt).toHaveBeenCalled(); expect(mockPromptHandler.runPrompt).toHaveBeenCalled(); expect(mockCodeEvaluator.evaluate).toHaveBeenCalledWith('generated code', mockContext); - expect(fs.writeFileSync).toHaveBeenCalled(); // Need to save cache + expect(mockCacheHandler.addToTemporaryCache).toHaveBeenCalled(); }); }); diff --git a/src/index.ts b/src/index.ts index 2da9536..7e97bdf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,8 +8,8 @@ const copilot: CopilotFacade = { start: () => { Copilot.getInstance().start(); }, - end: (saveToCache?: boolean) => { - Copilot.getInstance().end(saveToCache); + end: (isCacheDisabled?: boolean) => { + Copilot.getInstance().end(isCacheDisabled); }, perform: async (...steps: string[]) => { const copilotInstance = Copilot.getInstance(); diff --git a/src/integration tests/index.test.ts b/src/integration tests/index.test.ts index 6227b33..3a947a5 100644 --- a/src/integration tests/index.test.ts +++ b/src/integration tests/index.test.ts @@ -1,4 +1,5 @@ import copilot from "@/index"; +import fs from 'fs'; import { Copilot } from "@/Copilot"; import { PromptHandler, TestingFrameworkDriver } from "@/types"; import * as crypto from 'crypto'; @@ -28,14 +29,6 @@ describe('Copilot Integration Tests', () => { isSnapshotImageSupported: jest.fn().mockReturnValue(true) }; - // mockFs = fs as jest.Mocked; - // mockPath = path as jest.Mocked; - // - // mockFs.existsSync.mockReturnValue(false); - // mockFs.readFileSync.mockReturnValue('{}'); - // mockFs.writeFileSync.mockImplementation(() => {}); - // mockPath.resolve.mockImplementation((...paths) => paths.join('/')); - mockCache(); (crypto.createHash as jest.Mock).mockReturnValue({ @@ -250,7 +243,7 @@ describe('Copilot Integration Tests', () => { mockPromptHandler.runPrompt.mockResolvedValue('// Perform action'); await copilot.perform('Perform action'); - copilot.end(true) + copilot.end(false) expect(mockedCacheFile).toEqual({ '{"step":"Perform action","previous":[],"viewHierarchyHash":"hash"}': '// Perform action' @@ -278,22 +271,27 @@ describe('Copilot Integration Tests', () => { }); }); - // it('should handle fs.readFileSync errors', async () => { - // mockFs.existsSync.mockReturnValue(true); - // mockFs.readFileSync.mockImplementation(() => { throw new Error('Read error'); }); - // mockPromptHandler.runPrompt.mockResolvedValue('// New action code'); - // - // await copilot.perform('Action with read error'); - // - // expect(mockPromptHandler.runPrompt).toHaveBeenCalled(); - // }); - - // it('should handle fs.writeFileSync errors', async () => { - // mockFs.writeFileSync.mockImplementation(() => { throw new Error('Write error'); }); - // mockPromptHandler.runPrompt.mockResolvedValue('// Action code'); - // - // await expect(copilot.perform('Action with write error')).resolves.not.toThrow(); - // }); + it('should handle fs.readFileSync errors', async () => { + mockCache({}); // Set up an initial mocked file + (fs.readFileSync as jest.Mock).mockImplementation(() => { + throw new Error('Read error'); + }); + mockPromptHandler.runPrompt.mockResolvedValue('// New action code'); + + await copilot.perform('Action with read error'); + + expect(mockPromptHandler.runPrompt).toHaveBeenCalled(); + }); + + it('should handle fs.writeFileSync errors', async () => { + mockCache(undefined); // No mocked file exists + (fs.writeFileSync as jest.Mock).mockImplementation(() => { + throw new Error('Write error'); + }); + mockPromptHandler.runPrompt.mockResolvedValue('// Action code'); + + await expect(copilot.perform('Action with write error')).resolves.not.toThrow(); + }); }); describe('Feature Support', () => { diff --git a/src/types.ts b/src/types.ts index db8e000..708a50b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -18,11 +18,11 @@ export interface CopilotFacade { /** * Finalizes the test flow and optionally saves temporary cache data to the main cache. - * If `saveToCache` is true, the temporary cache will be saved. True is the default value. - * @param saveToCache + * If `isCacheDisabled` is true, the temporary cache will not be saved. False is the default value. + * @param isCacheDisabled * @note This must be called after the test flow is complete. */ - end: (saveToCache?: boolean) => void; + end: (isCacheDisabled?: boolean) => void; /** * Performs a testing operation or series of testing operations in the app based on the given `steps`. diff --git a/src/utils/__snapshots__/PromptCreator.test.ts.snap b/src/utils/__snapshots__/PromptCreator.test.ts.snap index f392bff..143f676 100644 --- a/src/utils/__snapshots__/PromptCreator.test.ts.snap +++ b/src/utils/__snapshots__/PromptCreator.test.ts.snap @@ -93,13 +93,18 @@ Your task is to generate the minimal executable code to perform the following in Please follow these steps carefully: 1. Analyze the provided intent, the view hierarchy, and the snapshot image to understand the required action. -2. Determine if the intent can be fully validated visually using the snapshot image. -3. If the intent can be visually analyzed and passes the visual check, return only comments explaining the successful visual assertion. -4. If the visual assertion fails, return code that throws an informative error explaining the failure. -5. If visual validation is not possible, proceed to generate the minimal executable code required to perform the intent. -6. If you cannot generate the relevant code due to ambiguity or invalid intent, return code that throws an informative error explaining the problem in one sentence. -7. Wrap the generated code with backticks, without any additional formatting. -8. Do not provide any additional code beyond the minimal executable code required to perform the intent. +2. Assess the positions of elements within the screen layout. Ensure that tests accurately reflect their intended locations, such as whether an element is centered or positioned relative to others. Tests should fail if the actual locations do not align with the expected configuration. +3. Determine if the intent can be fully validated visually using the snapshot image. +4. If the intent can be visually analyzed and passes the visual check, return only comments explaining the successful visual assertion. +5. If the visual assertion fails, return code that throws an informative error explaining the failure. +6. If visual validation is not possible, proceed to generate the minimal executable code required to perform the intent. +7. If you cannot generate the relevant code due to ambiguity or invalid intent, return code that throws an informative error explaining the problem in one sentence. +8. Wrap the generated code with backticks, without any additional formatting. +9. Do not provide any additional code beyond the minimal executable code required to perform the intent. + +### Verify the prompt + +Before generating the code, please review the provided context and instructions to ensure they are clear and unambiguous. If you encounter any issues or have questions, please throw an informative error explaining the problem. ### Examples @@ -134,6 +139,10 @@ Generate the minimal executable code to perform the following intent: "expect bu