From 66cdf21c48ccd1954fa0adb8965a860dd2984074 Mon Sep 17 00:00:00 2001 From: Asaf Korem <55082339+asafkorem@users.noreply.github.com> Date: Mon, 14 Oct 2024 19:22:11 +0300 Subject: [PATCH] fix: ambigous error and cache (#20) * fix: ask for error throwing in case of invalid intent. * fix(StepPerformer): cache error messages. --- src/actions/StepPerformer.test.ts | 2 +- src/actions/StepPerformer.ts | 29 +++++++------------ src/utils/PromptCreator.ts | 1 + .../__snapshots__/PromptCreator.test.ts.snap | 3 ++ 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/actions/StepPerformer.test.ts b/src/actions/StepPerformer.test.ts index 6ef5eaf..58a6f89 100644 --- a/src/actions/StepPerformer.test.ts +++ b/src/actions/StepPerformer.test.ts @@ -196,7 +196,7 @@ describe('StepPerformer', () => { mockCodeEvaluator.evaluate.mockRejectedValue(new Error('Evaluation failed')); await expect(stepPerformer.perform(intent)).rejects.toThrow('Evaluation failed'); - expect(fs.writeFileSync).not.toHaveBeenCalled(); // Cache should not be saved + expect(fs.writeFileSync).toHaveBeenCalled(); // Cache should be saved }); it('should use cached prompt result if available', async () => { diff --git a/src/actions/StepPerformer.ts b/src/actions/StepPerformer.ts index f31f871..ca3ff69 100644 --- a/src/actions/StepPerformer.ts +++ b/src/actions/StepPerformer.ts @@ -1,7 +1,7 @@ -import { PromptCreator } from '@/utils/PromptCreator'; -import { CodeEvaluator } from '@/utils/CodeEvaluator'; -import { SnapshotManager } from '@/utils/SnapshotManager'; -import {CodeEvaluationResult, PreviousStep, PromptHandler, TestingFrameworkDriver} from '@/types'; +import {PromptCreator} from '@/utils/PromptCreator'; +import {CodeEvaluator} from '@/utils/CodeEvaluator'; +import {SnapshotManager} from '@/utils/SnapshotManager'; +import {CodeEvaluationResult, PreviousStep, PromptHandler} from '@/types'; import * as fs from 'fs'; import * as path from 'path'; import * as crypto from 'crypto'; @@ -81,11 +81,11 @@ export class StepPerformer { try { promptResult = await this.promptHandler.runPrompt(prompt, snapshot); - const result = await this.codeEvaluator.evaluate(promptResult, this.context); // Cache the result this.cache.set(cacheKey, promptResult); this.saveCacheToFile(); - return result; + + return await this.codeEvaluator.evaluate(promptResult, this.context); } catch (error) { // Extend 'previous' array with the failure message const failedAttemptMessage = promptResult @@ -98,13 +98,6 @@ export class StepPerformer { result: undefined, }]; - const retryCacheKey = this.generateCacheKey(step, newPrevious, viewHierarchy); - - if (this.cache.has(retryCacheKey)) { - const cachedRetryPromptResult = this.cache.get(retryCacheKey); - return this.codeEvaluator.evaluate(cachedRetryPromptResult, this.context); - } - const retryPrompt = this.promptCreator.createPrompt( step, viewHierarchy, @@ -114,15 +107,15 @@ export class StepPerformer { try { const retryPromptResult = await this.promptHandler.runPrompt(retryPrompt, snapshot); - const retryResult = await this.codeEvaluator.evaluate( - retryPromptResult, - this.context, - ); // Cache the result under the original cache key this.cache.set(cacheKey, retryPromptResult); this.saveCacheToFile(); - return retryResult; + + return await this.codeEvaluator.evaluate( + retryPromptResult, + this.context, + ); } catch (retryError) { // Throw the original error if retry fails throw error; diff --git a/src/utils/PromptCreator.ts b/src/utils/PromptCreator.ts index d0a7296..02d7d41 100644 --- a/src/utils/PromptCreator.ts +++ b/src/utils/PromptCreator.ts @@ -30,6 +30,7 @@ export class PromptCreator { "", "You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API.", "Please generate the minimal executable code to perform the desired intent based on the given information and context.", + "If generating the relevant code is not possible due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence.", "" ]; } diff --git a/src/utils/__snapshots__/PromptCreator.test.ts.snap b/src/utils/__snapshots__/PromptCreator.test.ts.snap index 4082d49..cdc54d5 100644 --- a/src/utils/__snapshots__/PromptCreator.test.ts.snap +++ b/src/utils/__snapshots__/PromptCreator.test.ts.snap @@ -5,6 +5,7 @@ exports[`PromptCreator should create a prompt for an intent correctly 1`] = ` You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API. Please generate the minimal executable code to perform the desired intent based on the given information and context. +If generating the relevant code is not possible due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence. ## Context @@ -111,6 +112,7 @@ exports[`PromptCreator should handle when no snapshot image is attached 1`] = ` You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API. Please generate the minimal executable code to perform the desired intent based on the given information and context. +If generating the relevant code is not possible due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence. ## Context @@ -212,6 +214,7 @@ exports[`PromptCreator should include previous intents in the context 1`] = ` You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API. Please generate the minimal executable code to perform the desired intent based on the given information and context. +If generating the relevant code is not possible due to ambiguity, invalid intent, or inability to find the desired element, return code that throws an informative error explaining the problem in one sentence. ## Context