From e0f53c17148ac6984c38145066eb7fc239dca912 Mon Sep 17 00:00:00 2001 From: Asaf Korem Date: Fri, 18 Oct 2024 13:14:51 +0300 Subject: [PATCH] feat: improve logs. --- src/actions/StepPerformer.ts | 44 +++++++++++-------- src/utils/CodeEvaluator.ts | 10 +---- .../__snapshots__/PromptCreator.test.ts.snap | 6 +-- src/utils/extractCodeBlock.ts | 6 +++ 4 files changed, 36 insertions(+), 30 deletions(-) create mode 100644 src/utils/extractCodeBlock.ts diff --git a/src/actions/StepPerformer.ts b/src/actions/StepPerformer.ts index ca3ff69..0339eed 100644 --- a/src/actions/StepPerformer.ts +++ b/src/actions/StepPerformer.ts @@ -5,6 +5,7 @@ import {CodeEvaluationResult, PreviousStep, PromptHandler} from '@/types'; import * as fs from 'fs'; import * as path from 'path'; import * as crypto from 'crypto'; +import {extractCodeBlock} from "@/utils/extractCodeBlock"; export class StepPerformer { private cache: Map = new Map(); @@ -66,8 +67,8 @@ export class StepPerformer { const cacheKey = this.generateCacheKey(step, previous, viewHierarchy); if (this.cache.has(cacheKey)) { - const cachedPromptResult = this.cache.get(cacheKey); - return this.codeEvaluator.evaluate(cachedPromptResult, this.context); + const cachedCode = this.cache.get(cacheKey); + return this.codeEvaluator.evaluate(cachedCode, this.context); } const prompt = this.promptCreator.createPrompt( @@ -77,25 +78,29 @@ export class StepPerformer { previous, ); - let promptResult: string | undefined; + let code: string | undefined = undefined; try { - promptResult = await this.promptHandler.runPrompt(prompt, snapshot); + const promptResult = await this.promptHandler.runPrompt(prompt, snapshot); + code = extractCodeBlock(promptResult); + // Cache the result - this.cache.set(cacheKey, promptResult); + this.cache.set(cacheKey, code); this.saveCacheToFile(); - return await this.codeEvaluator.evaluate(promptResult, this.context); + return await this.codeEvaluator.evaluate(code, this.context); } catch (error) { - // Extend 'previous' array with the failure message - const failedAttemptMessage = promptResult - ? `Failed to evaluate "${step}", tried with generated code: "${promptResult}". Should we try a different approach? If can't, return a code that throws a descriptive error.` - : `Failed to perform "${step}", could not generate prompt result. Should we try a different approach? If can't, return a code that throws a descriptive error.`; + console.log("\x1b[33m%s\x1b[0m", "Failed to evaluate the code, Copilot is retrying..."); + + // Extend 'previous' array with the failure message as the result + const result = code + ? `Failed to evaluate "${step}", tried with generated code: "${code}". Validate the code against the APIs and hierarchy and let's try a different approach. If can't, return a code that throws a descriptive error.` + : `Failed to perform "${step}", could not generate prompt result. Let's try a different approach. If can't, return a code that throws a descriptive error.`; const newPrevious = [...previous, { step, - code: failedAttemptMessage, - result: undefined, + code: code ?? 'undefined', + result }]; const retryPrompt = this.promptCreator.createPrompt( @@ -107,16 +112,19 @@ export class StepPerformer { try { const retryPromptResult = await this.promptHandler.runPrompt(retryPrompt, snapshot); + code = extractCodeBlock(retryPromptResult); - // Cache the result under the original cache key - this.cache.set(cacheKey, retryPromptResult); + const result = await this.codeEvaluator.evaluate(code, this.context); + + // Cache the result under the _original_ cache key + this.cache.set(cacheKey, code); this.saveCacheToFile(); - return await this.codeEvaluator.evaluate( - retryPromptResult, - this.context, - ); + return result; } catch (retryError) { + // Log the retry error + console.error('Retry failed:', retryError); + // Throw the original error if retry fails throw error; } diff --git a/src/utils/CodeEvaluator.ts b/src/utils/CodeEvaluator.ts index 1fb972f..10195d1 100644 --- a/src/utils/CodeEvaluator.ts +++ b/src/utils/CodeEvaluator.ts @@ -2,8 +2,7 @@ import { CodeEvaluationError } from '@/errors/CodeEvaluationError'; import {CodeEvaluationResult} from "@/types"; export class CodeEvaluator { - async evaluate(rawCode: string, context: any): Promise { - const code = this.extractCodeBlock(rawCode); + async evaluate(code: string, context: any): Promise { const asyncFunction = this.createAsyncFunction(code, context); const result = await asyncFunction(); @@ -28,11 +27,4 @@ export class CodeEvaluator { ); } } - - private extractCodeBlock(text: string): string { - const regex = /```(?:\w*\s)?([\s\S]*?)```/; - const match = text.match(regex); - - return (match ? match[1] : text).trim(); - } } diff --git a/src/utils/__snapshots__/PromptCreator.test.ts.snap b/src/utils/__snapshots__/PromptCreator.test.ts.snap index 14a3b39..49721db 100644 --- a/src/utils/__snapshots__/PromptCreator.test.ts.snap +++ b/src/utils/__snapshots__/PromptCreator.test.ts.snap @@ -3,7 +3,7 @@ exports[`PromptCreator should create a prompt for an intent correctly 1`] = ` "# Test Code Generation -You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API. +You are an AI assistant tasked with generating test code for an application using the provided UI testing framework API. Please generate the minimal executable code to perform the desired intent based on the given information and context. ## Context @@ -119,7 +119,7 @@ Please provide your response below:" exports[`PromptCreator should handle when no snapshot image is attached 1`] = ` "# Test Code Generation -You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API. +You are an AI assistant tasked with generating test code for an application using the provided UI testing framework API. Please generate the minimal executable code to perform the desired intent based on the given information and context. ## Context @@ -223,7 +223,7 @@ Please provide your response below:" exports[`PromptCreator should include previous intents in the context 1`] = ` "# Test Code Generation -You are an AI assistant tasked with generating test code for a mobile application using the provided UI testing framework API. +You are an AI assistant tasked with generating test code for an application using the provided UI testing framework API. Please generate the minimal executable code to perform the desired intent based on the given information and context. ## Context diff --git a/src/utils/extractCodeBlock.ts b/src/utils/extractCodeBlock.ts new file mode 100644 index 0000000..4cd88aa --- /dev/null +++ b/src/utils/extractCodeBlock.ts @@ -0,0 +1,6 @@ +export function extractCodeBlock(text: string): string { + const regex = /```(?:\w*\s)?([\s\S]*?)```/; + const match = text.match(regex); + + return (match ? match[1] : text).trim(); +}