From 565b40c90f1e3158eacacca307eb18650b5db664 Mon Sep 17 00:00:00 2001 From: Curtis Man Date: Fri, 31 Jan 2025 11:23:32 -0800 Subject: [PATCH 01/16] Separate debug trace between main action vs explainable check translations (#646) Create explicit translate function to translate without context for explainable check, explicitly disable any context sensitive information in the prompt. --- ts/packages/commonUtils/src/jsonTranslator.ts | 22 +++---- .../src/context/chatHistoryPrompt.ts | 64 ++++++++++--------- .../handlers/requestCommandHandler.ts | 2 +- .../src/translation/agentTranslators.ts | 41 ++++++++++-- 4 files changed, 79 insertions(+), 50 deletions(-) diff --git a/ts/packages/commonUtils/src/jsonTranslator.ts b/ts/packages/commonUtils/src/jsonTranslator.ts index 080262b68..a9156a9b9 100644 --- a/ts/packages/commonUtils/src/jsonTranslator.ts +++ b/ts/packages/commonUtils/src/jsonTranslator.ts @@ -10,7 +10,6 @@ import { success, TypeChatJsonTranslator, TypeChatJsonValidator, - TypeChatLanguageModel, } from "typechat"; import { createTypeScriptJsonValidator } from "typechat/ts"; import { TypeChatConstraintsValidator } from "./constraints.js"; @@ -231,7 +230,7 @@ async function attachAttachments( export type JsonTranslatorOptions = { constraintsValidator?: TypeChatConstraintsValidator | undefined; // Optional instructions?: PromptSection[] | undefined; // Instructions before the per request preamble - model?: string | TypeChatLanguageModel | undefined; // optional + model?: string | undefined; // optional }; /** @@ -266,17 +265,14 @@ export function createJsonTranslatorWithValidator( validator: TypeChatJsonValidator, options?: JsonTranslatorOptions, ) { - let model = options?.model; - if (typeof model !== "object") { - model = ai.createChatModel( - model, - { - response_format: { type: "json_object" }, - }, - undefined, - ["translator", name], - ); - } + const model = ai.createChatModel( + options?.model, + { + response_format: { type: "json_object" }, + }, + undefined, + ["translate", name], + ); const debugPrompt = registerDebug(`typeagent:translate:${name}:prompt`); const debugResult = registerDebug(`typeagent:translate:${name}:result`); diff --git a/ts/packages/dispatcher/src/context/chatHistoryPrompt.ts b/ts/packages/dispatcher/src/context/chatHistoryPrompt.ts index 6678df606..338935896 100644 --- a/ts/packages/dispatcher/src/context/chatHistoryPrompt.ts +++ b/ts/packages/dispatcher/src/context/chatHistoryPrompt.ts @@ -15,12 +15,13 @@ export function createTypeAgentRequestPrompt( request: string, history: HistoryContext | undefined, attachments: CachedImageWithDetails[] | undefined, + context: boolean = true, ) { let promptSections: PromptSection[] = []; let entities: Entity[] = []; let entityStr: string[] = []; - if (history) { + if (context && history !== undefined) { promptSections = history.promptSections; entities = history.entities; @@ -45,51 +46,54 @@ export function createTypeAgentRequestPrompt( translator.validator.getSchemaText(), `\`\`\``, ]; - if (promptSections.length > 1) { - prompts.push("The following is a summary of the chat history:"); - if (entityStr.length > 0) { + if (context) { + if (promptSections.length > 1) { + prompts.push("The following is a summary of the chat history:"); + if (entityStr.length > 0) { + prompts.push("###"); + prompts.push( + "Recent entities found in chat history, in order, oldest first:", + ); + prompts.push(...entityStr.reverse()); + } + + const additionalInstructions = history?.additionalInstructions; + if ( + additionalInstructions !== undefined && + additionalInstructions.length > 0 + ) { + prompts.push("###"); + prompts.push("Information about the latest assistant action:"); + prompts.push(...additionalInstructions); + } + prompts.push("###"); + prompts.push("The latest assistant response:"); prompts.push( - "Recent entities found in chat history, in order, oldest first:", + promptSections[promptSections.length - 1].content as string, ); - prompts.push(...entityStr.reverse()); - } - - const additionalInstructions = history?.additionalInstructions; - if ( - additionalInstructions !== undefined && - additionalInstructions.length > 0 - ) { - prompts.push("###"); - prompts.push("Information about the latest assistant action:"); - prompts.push(...additionalInstructions); } prompts.push("###"); - prompts.push("The latest assistant response:"); prompts.push( - promptSections[promptSections.length - 1].content as string, + `Current Date is ${new Date().toLocaleDateString("en-US")}. The time is ${new Date().toLocaleTimeString()}.`, ); } - - prompts.push("###"); - prompts.push( - `Current Date is ${new Date().toLocaleDateString("en-US")}. The time is ${new Date().toLocaleTimeString()}.`, - ); prompts.push("###"); prompts.push(`The following is the current user request:`); prompts.push(`"""\n${request}\n"""`); prompts.push("###"); - prompts.push( - "Resolve all references and pronouns in the current user request with the recent entities in the chat history. If there are multiple possible resolution, choose the most likely resolution based on conversation context, bias toward the newest.", - ); - prompts.push( - "Avoid clarifying unless absolutely necessary. Infer the user's intent based on conversation context.", - ); + if (context && history !== undefined) { + prompts.push( + "Resolve all references and pronouns in the current user request with the recent entities in the chat history. If there are multiple possible resolution, choose the most likely resolution based on conversation context, bias toward the newest.", + ); + prompts.push( + "Avoid clarifying unless absolutely necessary. Infer the user's intent based on conversation context.", + ); + } prompts.push( `Based primarily on the current user request with references and pronouns resolved with recent entities in the chat history, but considering the context of the whole chat history, the following is the current user request translated into a JSON object with 2 spaces of indentation and no properties with the value undefined:`, ); - //console.log(prompt); return prompts.join("\n"); } diff --git a/ts/packages/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts b/ts/packages/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts index b767534a3..ce417e295 100644 --- a/ts/packages/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts +++ b/ts/packages/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts @@ -59,7 +59,7 @@ async function canTranslateWithoutContext( try { const translations = new Map(); for (const [translatorName, translator] of usedTranslators) { - const result = await translator.translate(request); + const result = await translator.checkTranslate(request); if (!result.success) { throw new Error("Failed to translate without history context"); } diff --git a/ts/packages/dispatcher/src/translation/agentTranslators.ts b/ts/packages/dispatcher/src/translation/agentTranslators.ts index 840978ef9..f740869fd 100644 --- a/ts/packages/dispatcher/src/translation/agentTranslators.ts +++ b/ts/packages/dispatcher/src/translation/agentTranslators.ts @@ -4,7 +4,9 @@ import { CachedImageWithDetails, createJsonTranslatorFromSchemaDef, + createJsonTranslatorWithValidator, enableJsonTranslatorStreaming, + JsonTranslatorOptions, } from "common-utils"; import { AppAction } from "@typeagent/agent-sdk"; import { Result, TypeChatJsonTranslator } from "typechat"; @@ -234,6 +236,7 @@ export type TypeAgentTranslator = { attachments?: CachedImageWithDetails[], cb?: IncrementalJsonValueCallBack, ) => Promise>; + checkTranslate: (request: string) => Promise>; }; // TranslatedAction are actions returned from the LLM without the translator name @@ -261,6 +264,7 @@ export function loadAgentJsonTranslator< model?: string, exact: boolean = true, ): TypeAgentTranslator { + const options = { model }; const translator = regenerateSchema ? createActionJsonTranslatorFromSchemaDef( "AllActions", @@ -271,7 +275,7 @@ export function loadAgentJsonTranslator< changeAgentAction, multipleActionOptions, ), - { model }, + options, { exact }, ) : createJsonTranslatorFromSchemaDef( @@ -283,15 +287,18 @@ export function loadAgentJsonTranslator< changeAgentAction, multipleActionOptions, ), - { model }, + options, ); - return createTypeAgentTranslator(translator); + return createTypeAgentTranslator(translator, options); } function createTypeAgentTranslator< T extends TranslatedAction = TranslatedAction, ->(translator: TypeChatJsonTranslator): TypeAgentTranslator { +>( + translator: TypeChatJsonTranslator, + options: JsonTranslatorOptions, +): TypeAgentTranslator { const streamingTranslator = enableJsonTranslatorStreaming(translator); // the request prompt is already expanded by the override replacement below @@ -300,6 +307,16 @@ function createTypeAgentTranslator< return request; }; + // Create another translator so that we can have a different + // debug/token count tag + const altTranslator = createJsonTranslatorWithValidator( + "check", + translator.validator, + options, + ); + altTranslator.createRequestPrompt = (request: string) => { + return request; + }; const typeAgentTranslator = { translate: async ( request: string, @@ -322,6 +339,17 @@ function createTypeAgentTranslator< attachments, ); }, + // No streaming, no history, no attachments. + checkTranslate: async (request: string) => { + const requestPrompt = createTypeAgentRequestPrompt( + altTranslator, + request, + undefined, + undefined, + false, + ); + return altTranslator.translate(requestPrompt); + }, }; return typeAgentTranslator; @@ -338,6 +366,7 @@ export function createTypeAgentTranslatorForSelectedActions< multipleActionOptions: MultipleActionOptions, model?: string, ) { + const options = { model }; const translator = createActionJsonTranslatorFromSchemaDef( "AllActions", composeSelectedActionSchema( @@ -348,9 +377,9 @@ export function createTypeAgentTranslatorForSelectedActions< changeAgentAction, multipleActionOptions, ), - { model }, + options, ); - return createTypeAgentTranslator(translator); + return createTypeAgentTranslator(translator, options); } // For CLI, replicate the behavior of loadAgentJsonTranslator to get the schema From 4de379805fe200e506c9389fe0fff2d8a6cf8d7a Mon Sep 17 00:00:00 2001 From: gvanrossum-ms Date: Fri, 31 Jan 2025 12:40:59 -0800 Subject: [PATCH 02/16] Remove child chunk text from parent chunks. (#647) This way, the text of a function is not repeated in its parent chunk. --- .../agents/spelunker/src/typescriptChunker.ts | 59 ++++++++++++++++++- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/ts/packages/agents/spelunker/src/typescriptChunker.ts b/ts/packages/agents/spelunker/src/typescriptChunker.ts index 07de0e8d1..d268e027a 100644 --- a/ts/packages/agents/spelunker/src/typescriptChunker.ts +++ b/ts/packages/agents/spelunker/src/typescriptChunker.ts @@ -31,20 +31,24 @@ export async function chunkifyTypeScriptFiles( const results: (ChunkedFile | ChunkerErrorItem)[] = []; for (const fileName of fileNames) { // console.log(fileName); + const sourceFile: ts.SourceFile = await tsCode.loadSourceFile(fileName); + const baseName = path.basename(fileName); const extName = path.extname(fileName); const codeName = baseName.slice(0, -extName.length || undefined); + const blobs: Blob[] = [ + { start: 0, lines: sourceFile.text.match(/.*(?:\r?\n|$)/g) || [] }, + ]; const rootChunk: Chunk = { chunkId: generate_id(), treeName: "file", codeName, - blobs: [], + blobs, parentId: "", children: [], fileName, }; const chunks: Chunk[] = [rootChunk]; - const sourceFile: ts.SourceFile = await tsCode.loadSourceFile(fileName); chunks.push(...recursivelyChunkify(sourceFile, rootChunk)); const chunkedFile: ChunkedFile = { fileName, @@ -81,7 +85,7 @@ export async function chunkifyTypeScriptFiles( children: [], fileName, }; - // TODO: Remove chunk.blobs from parentChunk.blobs. + spliceBlobs(parentChunk, chunk); chunks.push(chunk); recursivelyChunkify(childNode, chunk); } else { @@ -95,6 +99,55 @@ export async function chunkifyTypeScriptFiles( return results; } +function assert(condition: boolean, message: string): asserts condition { + if (!condition) { + throw new Error(`Assertion failed: ${message}`); + } +} + +function spliceBlobs(parentChunk: Chunk, childChunk: Chunk): void { + const parentBlobs = parentChunk.blobs; + const childBlobs = childChunk.blobs; + assert(parentBlobs.length > 0, "Parent chunk must have at least one blob"); + assert(childBlobs.length === 1, "Child chunk must have exactly one blob"); + const parentBlob = parentBlobs[parentBlobs.length - 1]; + const childBlob = childBlobs[0]; + assert( + childBlob.start >= parentBlob.start, + "Child blob must start after parent blob", + ); + assert( + childBlob.start + childBlob.lines.length <= + parentBlob.start + parentBlob.lines.length, + "Child blob must end before parent blob", + ); + + const linesBefore = parentBlob.lines.slice( + 0, + childBlob.start - parentBlob.start, + ); + const startBefore = parentBlob.start; + while (linesBefore.length && !linesBefore[linesBefore.length - 1].trim()) { + linesBefore.pop(); + } + + let startAfter = childBlob.start + childBlob.lines.length; + const linesAfter = parentBlob.lines.slice(startAfter - parentBlob.start); + while (linesAfter.length && !linesAfter[0].trim()) { + linesAfter.shift(); + startAfter++; + } + + const blobs: Blob[] = []; + if (linesBefore.length) { + blobs.push({ start: startBefore, lines: linesBefore }); + } + if (linesAfter.length) { + blobs.push({ start: startAfter, lines: linesAfter }); + } + parentChunk.blobs.splice(-1, 1, ...blobs); +} + function makeBlobs( sourceFile: ts.SourceFile, startPos: number, From 605b4db8e61c0df3a8c1e809d68bdfc94f3d3af0 Mon Sep 17 00:00:00 2001 From: gvanrossum-ms Date: Fri, 31 Jan 2025 17:53:07 -0800 Subject: [PATCH 03/16] [Spelunker] Add breadcrumbs and fix recursive chunking (#648) I really need to start working on a test suite... --- .../agents/spelunker/src/typescriptChunker.ts | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/ts/packages/agents/spelunker/src/typescriptChunker.ts b/ts/packages/agents/spelunker/src/typescriptChunker.ts index d268e027a..1b4f96e78 100644 --- a/ts/packages/agents/spelunker/src/typescriptChunker.ts +++ b/ts/packages/agents/spelunker/src/typescriptChunker.ts @@ -72,10 +72,12 @@ export async function chunkifyTypeScriptFiles( // ts.SyntaxKind[childNode.kind], // tsCode.getStatementName(childNode), // ); - const chunk: Chunk = { + const treeName = ts.SyntaxKind[childNode.kind]; + const codeName = tsCode.getStatementName(childNode) ?? ""; + const childChunk: Chunk = { chunkId: generate_id(), - treeName: ts.SyntaxKind[childNode.kind], - codeName: tsCode.getStatementName(childNode) ?? "", + treeName, + codeName, blobs: makeBlobs( sourceFile, childNode.getFullStart(), @@ -85,11 +87,11 @@ export async function chunkifyTypeScriptFiles( children: [], fileName, }; - spliceBlobs(parentChunk, chunk); - chunks.push(chunk); - recursivelyChunkify(childNode, chunk); + spliceBlobs(parentChunk, childChunk); + chunks.push(childChunk); + chunks.push(...recursivelyChunkify(childNode, childChunk)); } else { - recursivelyChunkify(childNode, parentChunk); + chunks.push(...recursivelyChunkify(childNode, parentChunk)); } } return chunks; @@ -142,12 +144,34 @@ function spliceBlobs(parentChunk: Chunk, childChunk: Chunk): void { if (linesBefore.length) { blobs.push({ start: startBefore, lines: linesBefore }); } + const sig: string = signature(childChunk); + // console.log("signature", sig); + if (sig) { + blobs.push({ start: childBlob.start, lines: [sig], breadcrumb: true }); + } if (linesAfter.length) { blobs.push({ start: startAfter, lines: linesAfter }); } parentChunk.blobs.splice(-1, 1, ...blobs); } +function signature(chunk: Chunk): string { + const firstLine = chunk.blobs[0]?.lines[0] ?? ""; + const indent = firstLine.match(/^(\s*)/)?.[0] || ""; + + switch (chunk.treeName) { + case "InterfaceDeclaration": + return `${indent}interface ${chunk.codeName} ...`; + case "TypeAliasDeclaration": + return `${indent}type ${chunk.codeName} ...`; + case "FunctionDeclaration": + return `${indent}function ${chunk.codeName} ...`; + case "ClassDeclaration": + return `${indent}class ${chunk.codeName} ...`; + } + return ""; +} + function makeBlobs( sourceFile: ts.SourceFile, startPos: number, From 6a81110ce5b116f48c5a55b2aadf3edd9a41c17b Mon Sep 17 00:00:00 2001 From: Curtis Man Date: Fri, 31 Jan 2025 19:33:03 -0800 Subject: [PATCH 04/16] Add JSON Schema support as an options. (#649) Initial support to generate JSON schema for OAI's structure output. Off by default since the performance hit isn't great, and TS schema in the prompt with json mode already have high accuracy rate. (40s+ with cache miss, 1-2s with cache hit, and cache miss vs cache hit rate is about 50%). --- ts/packages/actionSchema/src/generator.ts | 44 ++++- ts/packages/actionSchema/src/index.ts | 1 + .../actionSchema/src/jsonSchemaGenerator.ts | 160 ++++++++++++++++++ ts/packages/aiclient/src/models.ts | 9 +- ts/packages/aiclient/src/openai.ts | 51 ++++-- ts/packages/aiclient/src/restClient.ts | 4 + ts/packages/cli/src/commands/run/translate.ts | 47 +++-- ts/packages/commonUtils/src/indexNode.ts | 1 + ts/packages/commonUtils/src/jsonTranslator.ts | 27 ++- .../src/context/commandHandlerContext.ts | 3 +- ts/packages/dispatcher/src/context/session.ts | 10 +- .../system/handlers/configCommandHandlers.ts | 48 ++++-- .../translation/actionSchemaJsonTranslator.ts | 15 +- .../src/translation/agentTranslators.ts | 3 +- 14 files changed, 366 insertions(+), 57 deletions(-) create mode 100644 ts/packages/actionSchema/src/jsonSchemaGenerator.ts diff --git a/ts/packages/actionSchema/src/generator.ts b/ts/packages/actionSchema/src/generator.ts index ee1e1d761..da0bccccd 100644 --- a/ts/packages/actionSchema/src/generator.ts +++ b/ts/packages/actionSchema/src/generator.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { wrapTypeWithJsonSchema } from "./jsonSchemaGenerator.js"; import { SchemaType, SchemaObjectFields, @@ -14,6 +15,7 @@ function generateSchemaType( type: SchemaType, pending: SchemaTypeDefinition[], indent: number, + jsonSchema: boolean, strict: boolean, paren: boolean = false, ): string { @@ -25,17 +27,20 @@ function generateSchemaType( type.fields, pending, indent + 1, + jsonSchema, strict, ); return `{\n${lines.join("\n")}\n${" ".repeat(indent)}}`; case "array": - return `${generateSchemaType(type.elementType, pending, indent, strict, true)}[]`; + return `${generateSchemaType(type.elementType, pending, indent, jsonSchema, strict, true)}[]`; case "string-union": const stringUnion = type.typeEnum.map((v) => `"${v}"`).join(" | "); return paren ? `(${stringUnion})` : stringUnion; case "type-union": const typeUnion = type.types - .map((t) => generateSchemaType(t, pending, indent, strict)) + .map((t) => + generateSchemaType(t, pending, indent, jsonSchema, strict), + ) .join(" | "); return paren ? `(${typeUnion})` : typeUnion; case "type-reference": @@ -45,7 +50,12 @@ function generateSchemaType( throw new Error(`Unresolved type reference: ${type.name}`); } return type.name; - + case "undefined": + if (jsonSchema) { + // When jsonSchema is enabled, emit "null" for optional fields to match the json schema. + // translator will convert it to "undefined" because of stripNulls is enabled + return "null"; + } default: return type.type; } @@ -68,14 +78,21 @@ function generateSchemaObjectFields( fields: SchemaObjectFields, pending: SchemaTypeDefinition[], indent: number, + jsonSchema: boolean, strict: boolean, ) { const indentStr = " ".repeat(indent); for (const [key, field] of Object.entries(fields)) { generateComments(lines, field.comments, indentStr); - const optional = field.optional ? "?" : ""; + const optional = field.optional && !jsonSchema ? "?" : ""; + + // When jsonSchema is enabled, emit "null" for optional fields to match the json schema. + // translator will convert it to "undefined" because of stripNulls is enabled + const jsonSchemaOptional = + field.optional && jsonSchema ? " | null" : ""; + lines.push( - `${indentStr}${key}${optional}: ${generateSchemaType(field.type, pending, indent, strict)};${field.trailingComments ? ` //${field.trailingComments.join(" ")}` : ""}`, + `${indentStr}${key}${optional}: ${generateSchemaType(field.type, pending, indent, jsonSchema, strict)}${jsonSchemaOptional};${field.trailingComments ? ` //${field.trailingComments.join(" ")}` : ""}`, ); } } @@ -84,6 +101,7 @@ function generateTypeDefinition( lines: string[], definition: SchemaTypeDefinition, pending: SchemaTypeDefinition[], + jsonSchema: boolean, strict: boolean, exact: boolean, ) { @@ -93,6 +111,7 @@ function generateTypeDefinition( definition.type, pending, 0, + jsonSchema, strict, ); const line = definition.alias @@ -104,6 +123,7 @@ function generateTypeDefinition( export type GenerateSchemaOptions = { strict?: boolean; // default true exact?: boolean; // default false + jsonSchema?: boolean; // default false }; export function generateSchemaTypeDefinition( @@ -111,6 +131,7 @@ export function generateSchemaTypeDefinition( options?: GenerateSchemaOptions, order?: Map, ) { + const jsonSchema = options?.jsonSchema ?? false; const strict = options?.strict ?? true; const exact = options?.exact ?? false; const emitted = new Map< @@ -129,7 +150,14 @@ export function generateSchemaTypeDefinition( emitOrder: emitted.size, }); const dep: SchemaTypeDefinition[] = []; - generateTypeDefinition(lines, definition, dep, strict, exact); + generateTypeDefinition( + lines, + definition, + dep, + jsonSchema, + strict, + exact, + ); // Generate the dependencies first to be close to the usage pending.unshift(...dep); @@ -157,7 +185,9 @@ export function generateActionSchema( options?: GenerateSchemaOptions, ): string { return generateSchemaTypeDefinition( - actionSchemaGroup.entry, + options?.jsonSchema + ? wrapTypeWithJsonSchema(actionSchemaGroup.entry) + : actionSchemaGroup.entry, options, actionSchemaGroup.order, ); diff --git a/ts/packages/actionSchema/src/index.ts b/ts/packages/actionSchema/src/index.ts index 714a1532f..fe08949fd 100644 --- a/ts/packages/actionSchema/src/index.ts +++ b/ts/packages/actionSchema/src/index.ts @@ -19,6 +19,7 @@ export { generateActionSchema, generateSchemaTypeDefinition, } from "./generator.js"; +export { generateActionJsonSchema } from "./jsonSchemaGenerator.js"; export { validateAction } from "./validate.js"; export { getParameterType, getParameterNames } from "./utils.js"; diff --git a/ts/packages/actionSchema/src/jsonSchemaGenerator.ts b/ts/packages/actionSchema/src/jsonSchemaGenerator.ts new file mode 100644 index 000000000..57d1bdd10 --- /dev/null +++ b/ts/packages/actionSchema/src/jsonSchemaGenerator.ts @@ -0,0 +1,160 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import * as sc from "./creator.js"; +import { + ActionSchemaEntryTypeDefinition, + ActionSchemaGroup, + SchemaType, + SchemaTypeDefinition, +} from "./type.js"; +export function wrapTypeWithJsonSchema( + type: ActionSchemaEntryTypeDefinition, +): SchemaTypeDefinition { + // The root of a Json schema is always an object + // place the root type definition with an object with a response field of the type. + return sc.type(type.name, sc.obj({ response: type.type }), undefined, true); +} + +type JsonSchemaObject = { + type: "object"; + properties: Record; + required: string[]; + additionalProperties: false; +}; +type JsonSchemaArray = { + type: "array"; + items: JsonSchema; +}; + +type JsonSchemaString = { + type: "string"; + enum?: string[]; +}; + +type JsonSchemaNumber = { + type: "number"; +}; + +type JsonSchemaBoolean = { + type: "boolean"; +}; + +type JsonSchemaUnion = { + anyOf: JsonSchema[]; +}; + +type JsonSchemaNull = { + type: "null"; +}; + +type JsonSchemaReference = { + $ref: string; +}; + +type JsonSchemaRoot = { + name: string; + description?: string; + strict: true; + schema: JsonSchema & { $defs?: Record }; // REVIEW should be JsonSchemaObject; +}; + +type JsonSchema = + | JsonSchemaObject + | JsonSchemaArray + | JsonSchemaString + | JsonSchemaNumber + | JsonSchemaBoolean + | JsonSchemaNull + | JsonSchemaUnion + | JsonSchemaReference; + +function generateJsonSchemaType( + type: SchemaType, + pending: SchemaTypeDefinition[], + strict: boolean, +): JsonSchema { + switch (type.type) { + case "object": + return { + type: "object", + properties: Object.fromEntries( + Object.entries(type.fields).map(([key, field]) => [ + key, + generateJsonSchemaType(field.type, pending, strict), + ]), + ), + required: Object.keys(type.fields), + additionalProperties: false, + }; + case "array": + return { + type: "array", + items: generateJsonSchemaType( + type.elementType, + pending, + strict, + ), + }; + case "string-union": + return { + type: "string", + enum: type.typeEnum, + }; + case "type-union": + return { + anyOf: type.types.map((t) => + generateJsonSchemaType(t, pending, strict), + ), + }; + case "type-reference": + if (type.definition) { + pending.push(type.definition); + } else if (strict) { + throw new Error(`Unresolved type reference: ${type.name}`); + } + return { + $ref: `#/$defs/${type.name}`, + }; + case "undefined": { + // Note: undefined is presented by null in JSON schema + return { type: "null" }; + } + default: + return { type: type.type }; + } +} +function generateJsonSchemaTypeDefinition( + def: SchemaTypeDefinition, + strict: boolean = true, +): JsonSchemaRoot { + const pending: SchemaTypeDefinition[] = []; + const schema: JsonSchemaRoot = { + name: def.name, + strict: true, + schema: generateJsonSchemaType(def.type, pending, strict), + }; + + if (pending.length !== 0) { + const $defs: Record = {}; + do { + const definition = pending.shift()!; + if ($defs[definition.name]) { + continue; + } + $defs[definition.name] = generateJsonSchemaType( + definition.type, + pending, + strict, + ); + } while (pending.length > 0); + schema.schema.$defs = $defs; + } + return schema; +} + +export function generateActionJsonSchema(actionSchemaGroup: ActionSchemaGroup) { + const type = wrapTypeWithJsonSchema(actionSchemaGroup.entry); + + return generateJsonSchemaTypeDefinition(type); +} diff --git a/ts/packages/aiclient/src/models.ts b/ts/packages/aiclient/src/models.ts index 2167e6081..d336a37f0 100644 --- a/ts/packages/aiclient/src/models.ts +++ b/ts/packages/aiclient/src/models.ts @@ -3,6 +3,8 @@ import { PromptSection, Result, TypeChatLanguageModel } from "typechat"; +export type JsonSchema = any; + /** * Translation settings for Chat models */ @@ -11,6 +13,7 @@ export type CompletionSettings = { temperature?: number; max_tokens?: number; response_format?: { type: "json_object" }; + // Use fixed seed parameter to improve determinism //https://cookbook.openai.com/examples/reproducible_outputs_with_the_seed_parameter seed?: number; @@ -22,12 +25,16 @@ export type CompletionSettings = { export interface ChatModel extends TypeChatLanguageModel { completionSettings: CompletionSettings; completionCallback?: ((request: any, response: any) => void) | undefined; - complete(prompt: string | PromptSection[]): Promise>; + complete( + prompt: string | PromptSection[], + jsonSchema?: JsonSchema, + ): Promise>; } export interface ChatModelWithStreaming extends ChatModel { completeStream( prompt: string | PromptSection[], + jsonSchema?: JsonSchema, ): Promise>>; } diff --git a/ts/packages/aiclient/src/openai.ts b/ts/packages/aiclient/src/openai.ts index ee26f512a..e9654453f 100644 --- a/ts/packages/aiclient/src/openai.ts +++ b/ts/packages/aiclient/src/openai.ts @@ -8,6 +8,7 @@ import { ChatModelWithStreaming, ImageModel, ImageGeneration, + JsonSchema, } from "./models"; import { callApi, callJsonApi, FetchThrottler } from "./restClient"; import { getEnvSetting } from "./common"; @@ -397,7 +398,11 @@ function createAzureOpenAIChatModel( completionSettings ??= {}; completionSettings.n ??= 1; completionSettings.temperature ??= 0; - if (!settings.supportsResponseFormat) { + + const disableResponseFormat = + !settings.supportsResponseFormat && + completionSettings.response_format !== undefined; + if (disableResponseFormat) { // Remove it even if user specify it. delete completionSettings.response_format; } @@ -416,8 +421,35 @@ function createAzureOpenAIChatModel( }; return model; + function getParams( + messages: PromptSection[], + jsonSchema?: JsonSchema, + additionalParams?: any, + ) { + const params: any = { + ...defaultParams, + messages, + ...completionSettings, + ...additionalParams, + }; + if (jsonSchema !== undefined) { + if (disableResponseFormat) { + throw new Error( + `Json schema not supported by model '${settings.modelName}'`, + ); + } + if (params.response_format?.type === "json_object") { + params.response_format = { + type: "json_schema", + json_schema: jsonSchema, + }; + } + } + return params; + } async function complete( prompt: string | PromptSection[], + jsonSchema?: JsonSchema, ): Promise> { verifyPromptLength(settings, prompt); @@ -426,17 +458,12 @@ function createAzureOpenAIChatModel( return headerResult; } - const messages = + const messages: PromptSection[] = typeof prompt === "string" ? [{ role: "user", content: prompt }] : prompt; - const params = { - ...defaultParams, - messages: messages, - ...completionSettings, - }; - + const params: any = getParams(messages, jsonSchema); const result = await callJsonApi( headerResult.data, settings.endpoint, @@ -478,6 +505,7 @@ function createAzureOpenAIChatModel( async function completeStream( prompt: string | PromptSection[], + jsonSchema?: JsonSchema, ): Promise>> { verifyPromptLength(settings, prompt); @@ -505,13 +533,10 @@ function createAzureOpenAIChatModel( } }); - const params = { - ...defaultParams, - messages: messages, + const params = getParams(messages, jsonSchema, { stream: true, stream_options: { include_usage: true && !historyIncludesImages }, - ...completionSettings, - }; + }); const result = await callApi( headerResult.data, settings.endpoint, diff --git a/ts/packages/aiclient/src/restClient.ts b/ts/packages/aiclient/src/restClient.ts index 8cf5b4228..0db489db6 100644 --- a/ts/packages/aiclient/src/restClient.ts +++ b/ts/packages/aiclient/src/restClient.ts @@ -247,6 +247,7 @@ export async function fetchWithRetry( if (result === undefined) { throw new Error("fetch: No response"); } + debugHeader(result.status, result.statusText); debugHeader(result.headers); if (result.status === 200) { return success(result); @@ -256,7 +257,10 @@ export async function fetchWithRetry( retryCount >= retryMaxAttempts ) { return error(`fetch error: ${await getErrorMessage(result)}`); + } else if (debugHeader.enabled) { + debugHeader(await getErrorMessage(result)); } + // See if the service tells how long to wait to retry const pauseMs = getRetryAfterMs(result, retryPauseMs); await sleep(pauseMs); diff --git a/ts/packages/cli/src/commands/run/translate.ts b/ts/packages/cli/src/commands/run/translate.ts index ac1afce0a..ed244323b 100644 --- a/ts/packages/cli/src/commands/run/translate.ts +++ b/ts/packages/cli/src/commands/run/translate.ts @@ -2,7 +2,7 @@ // Licensed under the MIT License. import { Args, Command, Flags } from "@oclif/core"; -import { createDispatcher } from "agent-dispatcher"; +import { ClientIO, createDispatcher } from "agent-dispatcher"; import { getDefaultAppAgentProviders } from "default-agent-provider"; import { getChatModelNames } from "aiclient"; import { @@ -10,6 +10,7 @@ import { getInstanceDir, getSchemaNamesForActionConfigProvider, } from "agent-dispatcher/internal"; +import { withConsoleClientIO } from "agent-dispatcher/helpers/console"; const modelNames = await getChatModelNames(); const defaultAppAgentProviders = getDefaultAppAgentProviders(getInstanceDir()); @@ -32,10 +33,18 @@ export default class TranslateCommand extends Command { options: schemaNames, multiple: true, }), + multiple: Flags.boolean({ + description: "Include multiple action schema", + allowNo: true, + }), model: Flags.string({ description: "Translation model to use", options: modelNames, }), + jsonSchema: Flags.boolean({ + description: "Output JSON schema", + allowNo: true, + }), }; static description = "Translate a request into action"; @@ -49,19 +58,29 @@ export default class TranslateCommand extends Command { ? Object.fromEntries(flags.translator.map((name) => [name, true])) : undefined; - const dispatcher = await createDispatcher("cli run translate", { - appAgentProviders: defaultAppAgentProviders, - schemas, - actions: null, - commands: { dispatcher: true }, - translation: { model: flags.model }, - cache: { enabled: false }, - persist: true, - dblogging: true, + await withConsoleClientIO(async (clientIO: ClientIO) => { + const dispatcher = await createDispatcher("cli run translate", { + appAgentProviders: defaultAppAgentProviders, + schemas, + actions: null, + commands: { dispatcher: true }, + translation: { + model: flags.model, + multiple: { enabled: flags.multiple }, + schema: { generation: { jsonSchema: flags.jsonSchema } }, + }, + cache: { enabled: false }, + clientIO, + persist: true, + dblogging: true, + }); + try { + await dispatcher.processCommand( + `@dispatcher translate ${args.request}`, + ); + } finally { + await dispatcher.close(); + } }); - await dispatcher.processCommand( - `@dispatcher translate ${args.request}`, - ); - await dispatcher.close(); } } diff --git a/ts/packages/commonUtils/src/indexNode.ts b/ts/packages/commonUtils/src/indexNode.ts index 68537075e..5c7bf892d 100644 --- a/ts/packages/commonUtils/src/indexNode.ts +++ b/ts/packages/commonUtils/src/indexNode.ts @@ -12,6 +12,7 @@ export { enableJsonTranslatorStreaming, TypeChatJsonTranslatorWithStreaming, createJsonTranslatorWithValidator, + TypeAgentJsonValidator, JsonTranslatorOptions, } from "./jsonTranslator.js"; export { IncrementalJsonValueCallBack } from "./incrementalJsonParser.js"; diff --git a/ts/packages/commonUtils/src/jsonTranslator.ts b/ts/packages/commonUtils/src/jsonTranslator.ts index a9156a9b9..b88d6f91b 100644 --- a/ts/packages/commonUtils/src/jsonTranslator.ts +++ b/ts/packages/commonUtils/src/jsonTranslator.ts @@ -14,7 +14,7 @@ import { import { createTypeScriptJsonValidator } from "typechat/ts"; import { TypeChatConstraintsValidator } from "./constraints.js"; import registerDebug from "debug"; -import { openai as ai } from "aiclient"; +import { openai as ai, JsonSchema } from "aiclient"; import { createIncrementalJsonParser, IncrementalJsonParser, @@ -260,9 +260,17 @@ export function createJsonTranslatorFromSchemaDef( ); } +export interface TypeAgentJsonValidator + extends TypeChatJsonValidator { + getSchemaText: () => string; + getTypeName: () => string; + validate(jsonObject: object): Result; + getJsonSchema?: () => JsonSchema | undefined; +} + export function createJsonTranslatorWithValidator( name: string, - validator: TypeChatJsonValidator, + validator: TypeAgentJsonValidator, options?: JsonTranslatorOptions, ) { const model = ai.createChatModel( @@ -275,18 +283,29 @@ export function createJsonTranslatorWithValidator( ); const debugPrompt = registerDebug(`typeagent:translate:${name}:prompt`); + const debugJsonSchema = registerDebug( + `typeagent:translate:${name}:jsonschema`, + ); const debugResult = registerDebug(`typeagent:translate:${name}:result`); const complete = model.complete.bind(model); model.complete = async (prompt: string | PromptSection[]) => { debugPrompt(prompt); - return complete(prompt); + const jsonSchema = validator.getJsonSchema?.(); + if (jsonSchema !== undefined) { + debugJsonSchema(jsonSchema); + } + return complete(prompt, jsonSchema); }; if (ai.supportsStreaming(model)) { const completeStream = model.completeStream.bind(model); model.completeStream = async (prompt: string | PromptSection[]) => { debugPrompt(prompt); - return completeStream(prompt); + const jsonSchema = validator.getJsonSchema?.(); + if (jsonSchema !== undefined) { + debugJsonSchema(jsonSchema); + } + return completeStream(prompt, jsonSchema); }; } diff --git a/ts/packages/dispatcher/src/context/commandHandlerContext.ts b/ts/packages/dispatcher/src/context/commandHandlerContext.ts index 702715ae6..ab09d65f6 100644 --- a/ts/packages/dispatcher/src/context/commandHandlerContext.ts +++ b/ts/packages/dispatcher/src/context/commandHandlerContext.ts @@ -132,9 +132,10 @@ export function getTranslatorForSchema( getActiveTranslators(context), config.switch.inline, config.multiple, - config.schema.generation, + config.schema.generation.enabled, config.model, !config.schema.optimize.enabled, + config.schema.generation.jsonSchema, ); context.translatorCache.set(translatorName, newTranslator); return newTranslator; diff --git a/ts/packages/dispatcher/src/context/session.ts b/ts/packages/dispatcher/src/context/session.ts index e77f3a8c5..8570a1966 100644 --- a/ts/packages/dispatcher/src/context/session.ts +++ b/ts/packages/dispatcher/src/context/session.ts @@ -84,7 +84,10 @@ type DispatcherConfig = { multiple: MultipleActionConfig; history: boolean; schema: { - generation: boolean; + generation: { + enabled: boolean; + jsonSchema: boolean; + }; optimize: { enabled: boolean; numInitialActions: number; // 0 means no limit @@ -145,7 +148,10 @@ const defaultSessionConfig: SessionConfig = { }, history: true, schema: { - generation: true, + generation: { + enabled: true, + jsonSchema: false, + }, optimize: { enabled: false, numInitialActions: 5, diff --git a/ts/packages/dispatcher/src/context/system/handlers/configCommandHandlers.ts b/ts/packages/dispatcher/src/context/system/handlers/configCommandHandlers.ts index 6760dcb5c..889f40e02 100644 --- a/ts/packages/dispatcher/src/context/system/handlers/configCommandHandlers.ts +++ b/ts/packages/dispatcher/src/context/system/handlers/configCommandHandlers.ts @@ -748,21 +748,45 @@ const configTranslationCommandHandlers: CommandHandlerTable = { schema: { description: "Action schema configuration", commands: { - generation: getToggleHandlerTable( - "generated action schema", - async (context, enable: boolean) => { - await changeContextConfig( - { - translation: { - schema: { - generation: enable, + generation: { + description: "Generated action schema", + commands: { + ...getToggleCommandHandlers( + "generated action schema", + async (context, enable: boolean) => { + await changeContextConfig( + { + translation: { + schema: { + generation: { + enabled: enable, + }, + }, + }, }, - }, + context, + ); }, - context, - ); + ), + json: getToggleHandlerTable( + "use generate json schema if model supports it", + async (context, enable: boolean) => { + await changeContextConfig( + { + translation: { + schema: { + generation: { + jsonSchema: enable, + }, + }, + }, + }, + context, + ); + }, + ), }, - ), + }, optimize: { description: "Optimize schema", commands: { diff --git a/ts/packages/dispatcher/src/translation/actionSchemaJsonTranslator.ts b/ts/packages/dispatcher/src/translation/actionSchemaJsonTranslator.ts index cb45fee2e..e06c05cb9 100644 --- a/ts/packages/dispatcher/src/translation/actionSchemaJsonTranslator.ts +++ b/ts/packages/dispatcher/src/translation/actionSchemaJsonTranslator.ts @@ -12,10 +12,12 @@ import { ActionSchemaUnion, ActionSchemaGroup, GenerateSchemaOptions, + generateActionJsonSchema, } from "action-schema"; import { createJsonTranslatorWithValidator, JsonTranslatorOptions, + TypeAgentJsonValidator, } from "common-utils"; import { getInjectedActionConfigs, @@ -32,13 +34,21 @@ import { ActionConfigProvider } from "./actionConfigProvider.js"; function createActionSchemaJsonValidator( actionSchemaGroup: ActionSchemaGroup, generateOptions?: GenerateSchemaOptions, -): TypeChatJsonValidator { +): TypeAgentJsonValidator { const schema = generateActionSchema(actionSchemaGroup, generateOptions); + const generateJsonSchema = generateOptions?.jsonSchema ?? false; + const jsonSchema = generateJsonSchema + ? generateActionJsonSchema(actionSchemaGroup) + : undefined; return { getSchemaText: () => schema, getTypeName: () => actionSchemaGroup.entry.name, + getJsonSchema: () => jsonSchema, validate(jsonObject: object): Result { - const value: any = jsonObject; + const value: any = generateJsonSchema + ? (jsonObject as any).response + : jsonObject; + if (value.actionName === undefined) { return error("Missing actionName property"); } @@ -51,6 +61,7 @@ function createActionSchemaJsonValidator( try { validateAction(actionSchema, value); + // Return the unwrapped value with generateJsonSchema as the translated result return success(value); } catch (e: any) { return error(e.message); diff --git a/ts/packages/dispatcher/src/translation/agentTranslators.ts b/ts/packages/dispatcher/src/translation/agentTranslators.ts index f740869fd..bf8b9a168 100644 --- a/ts/packages/dispatcher/src/translation/agentTranslators.ts +++ b/ts/packages/dispatcher/src/translation/agentTranslators.ts @@ -263,6 +263,7 @@ export function loadAgentJsonTranslator< regenerateSchema: boolean = true, model?: string, exact: boolean = true, + jsonSchema: boolean = false, ): TypeAgentTranslator { const options = { model }; const translator = regenerateSchema @@ -276,7 +277,7 @@ export function loadAgentJsonTranslator< multipleActionOptions, ), options, - { exact }, + { exact, jsonSchema }, ) : createJsonTranslatorFromSchemaDef( "AllActions", From 612b84253d4e9f38d20b80052ded1192354d8697 Mon Sep 17 00:00:00 2001 From: gvanrossum-ms Date: Sat, 1 Feb 2025 13:17:34 -0800 Subject: [PATCH 05/16] [Spelunker] Fix mini model endpoint (#650) The endpoint name for the mini model was misspelled, so we quietly got the default model (gpt-4-o-2). After fixing it, I noticed that the summarizing and selecting phases (which use the mini model) became significantly *slower*. (There's a lot of variability in the measurements but the difference was pretty consistent). So now the fix is to set the mini model endpoint name to `undefined`, which means it's the same old default model, but there's no misleading misspelled endpoint name, and a comment explaining the reason. --- ts/packages/agents/spelunker/src/searchCode.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts/packages/agents/spelunker/src/searchCode.ts b/ts/packages/agents/spelunker/src/searchCode.ts index 7437b9aac..20e89aabf 100644 --- a/ts/packages/agents/spelunker/src/searchCode.ts +++ b/ts/packages/agents/spelunker/src/searchCode.ts @@ -55,7 +55,7 @@ function createQueryContext(): QueryContext { "AnswerSpecs", ); const miniModel = openai.createChatModel( - "GPT_4_0_MINI", + undefined, // "GPT_4_O_MINI" is slower than default model?! undefined, undefined, ["spelunkerMini"], From 1a5a07c0318d36311d91fbcad4a2bf7c86699f51 Mon Sep 17 00:00:00 2001 From: gvanrossum-ms Date: Sat, 1 Feb 2025 14:52:12 -0800 Subject: [PATCH 06/16] [Spelunker] Fix dramatic typo in selection prompt (#651) Also a few minor prompt tweaks. --- ts/packages/agents/spelunker/src/makeSelectorSchema.ts | 4 ++-- ts/packages/agents/spelunker/src/searchCode.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ts/packages/agents/spelunker/src/makeSelectorSchema.ts b/ts/packages/agents/spelunker/src/makeSelectorSchema.ts index 717bd7d65..2addbed7c 100644 --- a/ts/packages/agents/spelunker/src/makeSelectorSchema.ts +++ b/ts/packages/agents/spelunker/src/makeSelectorSchema.ts @@ -4,8 +4,8 @@ // Identifier for a chunk of code. export type ChunkId = string; -// A chunk is a function/method, class or module. -// Nested chunks are elided from the chunk text (they are their own chunk). +// A chunk is a function/method, class, module/file, interface, type, etc. +// Nested chunks have been elided from the chunk text (they are their own chunk). export type ChunkDescription = { chunkId: ChunkId; relevance: number; // Float between 0.0 and 1.0 diff --git a/ts/packages/agents/spelunker/src/searchCode.ts b/ts/packages/agents/spelunker/src/searchCode.ts index 20e89aabf..58f71ecfa 100644 --- a/ts/packages/agents/spelunker/src/searchCode.ts +++ b/ts/packages/agents/spelunker/src/searchCode.ts @@ -326,11 +326,11 @@ async function selectRelevantChunks( const prompt = `\ Please select up to 30 chunks that are relevant to the user question. Consider carefully how relevant each chunk is to the user question. - Provide a relevance scsore between 0 and 1 (float). + Provide a relevance score between 0 and 1 (float). Report only the chunk ID and relevance for each selected chunk. - Omit irrelevant chunks. It's fine to select fewer than 30. + Omit irrelevant or empty chunks. It's fine to select fewer than 30. - User question: "{input}" + User question: "${input}" Chunks: ${prepareChunks(chunks)} From d3473d8b038b2a73460d063d27f7f60de28cbc07 Mon Sep 17 00:00:00 2001 From: gvanrossum-ms Date: Sun, 2 Feb 2025 08:38:10 -0800 Subject: [PATCH 07/16] [Spelunker] Breakthrough in prepareChunks (#652) We now show the chunks and blobs as a (partial) file with line numbers and some annotations. Also: Renamed some schema files ("make" prefix was silly). Renamed some schema interfaces (answerer is now oracle). Added a lineNo field to Chunk interface. Removed some dead(-ish) code. --- .../agents/spelunker/src/chunkSchema.ts | 1 + .../{makeAnswerSchema.ts => oracleSchema.ts} | 2 +- .../agents/spelunker/src/pythonChunker.ts | 3 + .../agents/spelunker/src/searchCode.ts | 107 +++++++++++------- ...akeSelectorSchema.ts => selectorSchema.ts} | 0 ...SummarizeSchema.ts => summarizerSchema.ts} | 2 +- .../agents/spelunker/src/typescriptChunker.ts | 29 +++-- 7 files changed, 93 insertions(+), 51 deletions(-) rename ts/packages/agents/spelunker/src/{makeAnswerSchema.ts => oracleSchema.ts} (95%) rename ts/packages/agents/spelunker/src/{makeSelectorSchema.ts => selectorSchema.ts} (100%) rename ts/packages/agents/spelunker/src/{makeSummarizeSchema.ts => summarizerSchema.ts} (94%) diff --git a/ts/packages/agents/spelunker/src/chunkSchema.ts b/ts/packages/agents/spelunker/src/chunkSchema.ts index f99801f4b..84ac852c5 100644 --- a/ts/packages/agents/spelunker/src/chunkSchema.ts +++ b/ts/packages/agents/spelunker/src/chunkSchema.ts @@ -20,6 +20,7 @@ export interface Chunk { parentId: ChunkId; children: ChunkId[]; fileName: string; // Set upon receiving end from ChunkedFile.fileName. + lineNo: number; // 1-based, calculated from first blob. docs?: FileDocumentation; // Computed later by fileDocumenter. } diff --git a/ts/packages/agents/spelunker/src/makeAnswerSchema.ts b/ts/packages/agents/spelunker/src/oracleSchema.ts similarity index 95% rename from ts/packages/agents/spelunker/src/makeAnswerSchema.ts rename to ts/packages/agents/spelunker/src/oracleSchema.ts index d630c622f..887dcd3ba 100644 --- a/ts/packages/agents/spelunker/src/makeAnswerSchema.ts +++ b/ts/packages/agents/spelunker/src/oracleSchema.ts @@ -5,7 +5,7 @@ export type ChunkId = string; // Answer to the original question. -export type AnswerSpecs = { +export type OracleSpecs = { question: string; // Original question (e.g. "How can items be related") answer: string; // Answer to the question. It is readable and complete, with suitable formatting (line breaks, bullet points etc) diff --git a/ts/packages/agents/spelunker/src/pythonChunker.ts b/ts/packages/agents/spelunker/src/pythonChunker.ts index 6756878ca..cf3137643 100644 --- a/ts/packages/agents/spelunker/src/pythonChunker.ts +++ b/ts/packages/agents/spelunker/src/pythonChunker.ts @@ -59,6 +59,9 @@ export async function chunkifyPythonFiles( if (!("error" in result)) { for (const chunk of result.chunks) { chunk.fileName = result.fileName; + chunk.lineNo = chunk.blobs.length + ? chunk.blobs[0].start + 1 + : 1; } } } diff --git a/ts/packages/agents/spelunker/src/searchCode.ts b/ts/packages/agents/spelunker/src/searchCode.ts index 58f71ecfa..6d7ba18aa 100644 --- a/ts/packages/agents/spelunker/src/searchCode.ts +++ b/ts/packages/agents/spelunker/src/searchCode.ts @@ -17,12 +17,12 @@ import { import { createActionResultFromError } from "@typeagent/agent-sdk/helpers/action"; import { loadSchema } from "typeagent"; -import { AnswerSpecs } from "./makeAnswerSchema.js"; -import { ChunkDescription, SelectorSpecs } from "./makeSelectorSchema.js"; +import { OracleSpecs } from "./oracleSchema.js"; +import { ChunkDescription, SelectorSpecs } from "./selectorSchema.js"; import { SpelunkerContext } from "./spelunkerActionHandler.js"; import { Blob, Chunk, ChunkedFile, ChunkerErrorItem } from "./chunkSchema.js"; import { chunkifyPythonFiles } from "./pythonChunker.js"; -import { SummarizeSpecs } from "./makeSummarizeSchema.js"; +import { SummarizerSpecs } from "./summarizerSchema.js"; import { createRequire } from "module"; import { chunkifyTypeScriptFiles } from "./typescriptChunker.js"; @@ -39,20 +39,20 @@ function console_log(...rest: any[]): void { export interface QueryContext { chatModel: ChatModel; - answerMaker: TypeChatJsonTranslator; + oracle: TypeChatJsonTranslator; miniModel: ChatModel; chunkSelector: TypeChatJsonTranslator; - chunkSummarizer: TypeChatJsonTranslator; + chunkSummarizer: TypeChatJsonTranslator; databaseLocation: string; database: sqlite.Database | undefined; } function createQueryContext(): QueryContext { const chatModel = openai.createChatModelDefault("spelunkerChat"); - const answerMaker = createTranslator( + const oracle = createTranslator( chatModel, - "makeAnswerSchema.ts", - "AnswerSpecs", + "oracleSchema.ts", + "OracleSpecs", ); const miniModel = openai.createChatModel( undefined, // "GPT_4_O_MINI" is slower than default model?! @@ -62,13 +62,13 @@ function createQueryContext(): QueryContext { ); const chunkSelector = createTranslator( miniModel, - "makeSelectorSchema.ts", + "selectorSchema.ts", "SelectorSpecs", ); - const chunkSummarizer = createTranslator( + const chunkSummarizer = createTranslator( miniModel, - "makeSummarizeSchema.ts", - "SummarizeSpecs", + "summarizerSchema.ts", + "SummarizerSpecs", ); const databaseFolder = path.join( process.env.HOME ?? "/", @@ -81,11 +81,11 @@ function createQueryContext(): QueryContext { mode: 0o700, }; fs.mkdirSync(databaseFolder, mkdirOptions); - const databaseLocation = path.join(databaseFolder, "codeSearchdatabase.db"); + const databaseLocation = path.join(databaseFolder, "codeSearchDatabase.db"); const database = undefined; return { chatModel, - answerMaker, + oracle, miniModel, chunkSelector, chunkSummarizer, @@ -125,6 +125,15 @@ export async function searchCode( const blobRows: any[] = db .prepare(`SELECT * FROM blobs WHERE chunkId = ?`) .all(chunkRow.chunkId); + for (const blob of blobRows) { + blob.lines = blob.lines.match(/.*(?:\r?\n|$)/g) ?? []; + while ( + blob.lines.length && + !blob.lines[blob.lines.length - 1].trim() + ) { + blob.lines.pop(); + } + } const childRows: any[] = db .prepare(`SELECT * FROM chunks WHERE parentId = ?`) .all(chunkRow.chunkId); @@ -136,6 +145,7 @@ export async function searchCode( parentId: chunkRow.parentId, children: childRows.map((row) => row.chunkId), fileName: chunkRow.fileName, + lineNo: chunkRow.lineNo, }; allChunks.push(chunk); } @@ -159,7 +169,7 @@ export async function searchCode( const preppedChunks: Chunk[] = chunkDescs .map((chunkDesc) => prepChunk(chunkDesc, allChunks)) .filter(Boolean) as Chunk[]; - // TODO: Prompt engineering; more efficient preparation of summaries and chunks + // TODO: Prompt engineering const prompt = `\ Please answer the user question using the given context and summaries. @@ -176,9 +186,8 @@ export async function searchCode( // console_log(`[${prompt.slice(0, 1000)}]`); // 5. Send prompt to smart, code-savvy LLM. - console_log(`[Step 5: Ask the smart LLM]`); - const wrappedResult = - await context.queryContext!.answerMaker.translate(prompt); + console_log(`[Step 5: Ask the oracle]`); + const wrappedResult = await context.queryContext!.oracle.translate(prompt); if (!wrappedResult.success) { console_log(` [It's a failure: ${wrappedResult.message}]`); return createActionResultFromError( @@ -347,8 +356,42 @@ async function selectRelevantChunks( } function prepareChunks(chunks: Chunk[]): string { - // TODO: Format the chunks more efficiently - return JSON.stringify(chunks, undefined, 2); + chunks.sort( + // Sort by file name and chunk ID (should order by line number) + (a, b) => { + let cmp = a.fileName.localeCompare(b.fileName); + if (!cmp) { + cmp = a.chunkId.localeCompare(b.chunkId); + } + return cmp; + }, + ); + const output: string[] = []; + function put(line: string): void { + // console_log(line.trimEnd()); + output.push(line); + } + let lastFn = ""; + let lineNo = 0; + for (const chunk of chunks) { + if (chunk.fileName !== lastFn) { + lastFn = chunk.fileName; + lineNo = 0; + put("\n"); + put(`** file=${chunk.fileName}\n`); + } + put( + `* chunkId=${chunk.chunkId} kind=${chunk.treeName} name=${chunk.codeName}\n`, + ); + for (const blob of chunk.blobs) { + lineNo = blob.start; + for (const line of blob.lines) { + lineNo += 1; + put(`${lineNo} ${line}`); + } + } + } + return output.join(""); } function prepareSummaries(db: sqlite.Database): string { @@ -459,11 +502,8 @@ async function loadDatabase( const prepSelectAllFiles = db.prepare( `SELECT fileName, mtime, size FROM Files`, ); - const prepCountChunks = db.prepare( - `SELECT COUNT(*) FROM Chunks WHERE fileName = ?`, - ); const prepInsertChunks = db.prepare( - `INSERT OR REPLACE INTO Chunks (chunkId, treeName, codeName, parentId, fileName) VALUES (?, ?, ?, ?, ?)`, + `INSERT OR REPLACE INTO Chunks (chunkId, treeName, codeName, parentId, fileName, lineNo) VALUES (?, ?, ?, ?, ?, ?)`, ); const prepInsertBlobs = db.prepare( `INSERT INTO Blobs (chunkId, start, lines, breadcrumb) VALUES (?, ?, ?, ?)`, @@ -505,17 +545,6 @@ async function loadDatabase( size: stat.size, }); } - if (!filesToDo.includes(file)) { - // If there are no chunks, also add to filesToDo - // TODO: Zero chunks is not reliable, empty files haalso ve zero chunks - const count: number = (prepCountChunks.get(file) as any)[ - "COUNT(*)" - ]; - if (!count) { - // console_log(` [Need to update ${file} (no chunks)]`); - filesToDo.push(file); - } - } } const filesToDelete: string[] = [...filesInDb.keys()].filter( (file) => !files.includes(file), @@ -574,10 +603,6 @@ async function loadDatabase( prepDeleteBlobs.run(chunkedFile.fileName); prepDeleteChunks.run(chunkedFile.fileName); for (const chunk of chunkedFile.chunks) { - // TODO: Assuming this never throws, just remove this - if (!chunk.fileName) { - throw new Error(`Chunk ${chunk.chunkId} has no fileName`); - } allChunks.push(chunk); prepInsertChunks.run( chunk.chunkId, @@ -585,6 +610,7 @@ async function loadDatabase( chunk.codeName, chunk.parentId || null, chunk.fileName, + chunk.lineNo, ); for (const blob of chunk.blobs) { prepInsertBlobs.run( @@ -620,7 +646,8 @@ CREATE TABLE IF NOT EXISTS Chunks ( treeName TEXT NOT NULL, codeName TEXT NOT NULL, parentId TEXT KEY REFERENCES chunks(chunkId), -- May be null - fileName TEXT KEY REFERENCES files(fileName) NOT NULL + fileName TEXT KEY REFERENCES files(fileName) NOT NULL, + lineNo INTEGER NOT NULL -- 1-based ); CREATE TABLE IF NOT EXISTS Blobs ( chunkId TEXT KEY REFERENCES chunks(chunkId) NOT NULL, diff --git a/ts/packages/agents/spelunker/src/makeSelectorSchema.ts b/ts/packages/agents/spelunker/src/selectorSchema.ts similarity index 100% rename from ts/packages/agents/spelunker/src/makeSelectorSchema.ts rename to ts/packages/agents/spelunker/src/selectorSchema.ts diff --git a/ts/packages/agents/spelunker/src/makeSummarizeSchema.ts b/ts/packages/agents/spelunker/src/summarizerSchema.ts similarity index 94% rename from ts/packages/agents/spelunker/src/makeSummarizeSchema.ts rename to ts/packages/agents/spelunker/src/summarizerSchema.ts index 65fd6d3e1..77523ffbe 100644 --- a/ts/packages/agents/spelunker/src/makeSummarizeSchema.ts +++ b/ts/packages/agents/spelunker/src/summarizerSchema.ts @@ -11,6 +11,6 @@ export type Summary = { }; // Produce a brief summary for each chunk. -export type SummarizeSpecs = { +export type SummarizerSpecs = { summaries: Summary[]; // A summary for every chunk }; diff --git a/ts/packages/agents/spelunker/src/typescriptChunker.ts b/ts/packages/agents/spelunker/src/typescriptChunker.ts index 1b4f96e78..d17842757 100644 --- a/ts/packages/agents/spelunker/src/typescriptChunker.ts +++ b/ts/packages/agents/spelunker/src/typescriptChunker.ts @@ -36,9 +36,16 @@ export async function chunkifyTypeScriptFiles( const baseName = path.basename(fileName); const extName = path.extname(fileName); const codeName = baseName.slice(0, -extName.length || undefined); - const blobs: Blob[] = [ - { start: 0, lines: sourceFile.text.match(/.*(?:\r?\n|$)/g) || [] }, - ]; + const blob: Blob = { + start: 0, + lines: sourceFile.text.match(/.*(?:\r?\n|$)/g) || [], + }; + while (blob.lines.length && !blob.lines[0].trim()) { + blob.lines.shift(); + blob.start++; + } + const blobs: Blob[] = [blob]; + const lineNo = blobs.length ? blobs[0].start + 1 : 1; const rootChunk: Chunk = { chunkId: generate_id(), treeName: "file", @@ -47,6 +54,7 @@ export async function chunkifyTypeScriptFiles( parentId: "", children: [], fileName, + lineNo, }; const chunks: Chunk[] = [rootChunk]; chunks.push(...recursivelyChunkify(sourceFile, rootChunk)); @@ -74,18 +82,21 @@ export async function chunkifyTypeScriptFiles( // ); const treeName = ts.SyntaxKind[childNode.kind]; const codeName = tsCode.getStatementName(childNode) ?? ""; + const blobs = makeBlobs( + sourceFile, + childNode.getFullStart(), + childNode.getEnd(), + ); + const lineNo = blobs.length ? blobs[0].start + 1 : 1; const childChunk: Chunk = { chunkId: generate_id(), treeName, codeName, - blobs: makeBlobs( - sourceFile, - childNode.getFullStart(), - childNode.getEnd(), - ), + blobs, parentId: parentChunk.chunkId, children: [], fileName, + lineNo, }; spliceBlobs(parentChunk, childChunk); chunks.push(childChunk); @@ -218,7 +229,7 @@ export class Testing { const fileNames = [ "./packages/agents/spelunker/src/typescriptChunker.ts", "./packages/agents/spelunker/src/spelunkerSchema.ts", - "./packages/agents/spelunker/src/makeSummarizeSchema.ts", + "./packages/agents/spelunker/src/summarizerSchema.ts", "./packages/codeProcessor/src/tsCode.ts", "./packages/agents/spelunker/src/pythonChunker.ts", ]; From 28680269b4c19d41a279fe5af32722b078794b54 Mon Sep 17 00:00:00 2001 From: gvanrossum-ms Date: Sun, 2 Feb 2025 20:59:39 -0800 Subject: [PATCH 08/16] [Spelunker] Improve formatting ("preparation") of summaries sent to oracle (#654) This requires adding a "language" field to the Summaries interface/table (filled in by the AI during summary creation). Also tweak the translation of the single text string in the Blob table back into a list of lines, all ending in a newline. --- .../agents/spelunker/src/searchCode.ts | 24 +++++++++++++++---- .../agents/spelunker/src/summarizerSchema.ts | 3 ++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/ts/packages/agents/spelunker/src/searchCode.ts b/ts/packages/agents/spelunker/src/searchCode.ts index 6d7ba18aa..84a83b26a 100644 --- a/ts/packages/agents/spelunker/src/searchCode.ts +++ b/ts/packages/agents/spelunker/src/searchCode.ts @@ -126,13 +126,16 @@ export async function searchCode( .prepare(`SELECT * FROM blobs WHERE chunkId = ?`) .all(chunkRow.chunkId); for (const blob of blobRows) { - blob.lines = blob.lines.match(/.*(?:\r?\n|$)/g) ?? []; + blob.lines = blob.lines.split("\n"); while ( blob.lines.length && !blob.lines[blob.lines.length - 1].trim() ) { blob.lines.pop(); } + for (let i = 0; i < blob.lines.length; i++) { + blob.lines[i] = blob.lines[i] + "\n"; + } } const childRows: any[] = db .prepare(`SELECT * FROM chunks WHERE parentId = ?`) @@ -395,10 +398,20 @@ function prepareChunks(chunks: Chunk[]): string { } function prepareSummaries(db: sqlite.Database): string { + const languageCommentMap: { [key: string]: string } = { + python: "#", + typescript: "//", + }; const selectAllSummaries = db.prepare(`SELECT * FROM Summaries`); const summaryRows: any[] = selectAllSummaries.all(); - // TODO: format as code: # / - return JSON.stringify(summaryRows, undefined, 2); + const lines: string[] = []; + for (const summaryRow of summaryRows) { + const comment = languageCommentMap[summaryRow.language] ?? "#"; + lines.push(""); + lines.push(`${comment} ${summaryRow.summary}`); + lines.push(summaryRow.signature); + } + return lines.join("\n"); } function createTranslator( @@ -657,8 +670,8 @@ CREATE TABLE IF NOT EXISTS Blobs ( ); CREATE TABLE IF NOT EXISTS Summaries ( chunkId TEXT PRIMARY KEY REFERENCES chunks(chunkId), + language TEXT, -- "python", "typescript", etc. summary TEXT, - shortName TEXT, signature TEXT ) `; @@ -750,7 +763,7 @@ async function summarizeChunkSlice( // Enter them into the database const db = context.queryContext!.database!; const prepInsertSummary = db.prepare(` - INSERT OR REPLACE INTO Summaries (chunkId, summary, signature) VALUES (?, ?, ?) + INSERT OR REPLACE INTO Summaries (chunkId, language, summary, signature) VALUES (?, ?, ?, ?) `); let errors = 0; for (const summary of summarizeSpecs.summaries) { @@ -758,6 +771,7 @@ async function summarizeChunkSlice( try { prepInsertSummary.run( summary.chunkId, + summary.language, summary.summary, summary.signature, ); diff --git a/ts/packages/agents/spelunker/src/summarizerSchema.ts b/ts/packages/agents/spelunker/src/summarizerSchema.ts index 77523ffbe..c9d5392ac 100644 --- a/ts/packages/agents/spelunker/src/summarizerSchema.ts +++ b/ts/packages/agents/spelunker/src/summarizerSchema.ts @@ -5,7 +5,8 @@ export type ChunkId = string; export type Summary = { - chunkId: ChunkId; + chunkId: ChunkId; // The unique identifier of the chunk + language: string; // The language of the chunk, e.g. 'python' or 'typescript' (all lowercase) summary: string; // A one-line summary of the chunk, explaining what it does at a high level, concisely but with attention for detail. Do not duplicate the signature signature: string; // For functions, 'def foo(bar: int) -> str:'; for classes, 'class Foo:'; for modules, 'module foo.bar' }; From 6b36438cb6d6a126e2101caaf6f61f4bfb3bb194 Mon Sep 17 00:00:00 2001 From: Curtis Man Date: Mon, 3 Feb 2025 10:17:04 -0800 Subject: [PATCH 09/16] Fix corepack signature. (#656) See: https://github.com/nodejs/corepack/issues/612 --- pipelines/azure-build-ts.yml | 8 ++++---- ts/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pipelines/azure-build-ts.yml b/pipelines/azure-build-ts.yml index 267f55d36..643f88ac1 100644 --- a/pipelines/azure-build-ts.yml +++ b/pipelines/azure-build-ts.yml @@ -38,16 +38,16 @@ jobs: displayName: "Checkout TypeAgent Repository" path: "typeagent" - - template: include-install-pnpm.yml - parameters: - buildDirectory: $(Build.SourcesDirectory)/ts - - task: UseNode@1 displayName: "Setup Node.js" inputs: version: $(nodeVersion) checkLatest: true + - template: include-install-pnpm.yml + parameters: + buildDirectory: $(Build.SourcesDirectory)/ts + - script: | pnpm install --frozen-lockfile --strict-peer-dependencies displayName: "Install dependencies" diff --git a/ts/package.json b/ts/package.json index 3fa729299..8fbc31e17 100644 --- a/ts/package.json +++ b/ts/package.json @@ -48,7 +48,7 @@ "prettier": "^3.2.5", "shx": "^0.3.4" }, - "packageManager": "pnpm@9.15.4+sha512.b2dc20e2fc72b3e18848459b37359a32064663e5627a51e4c74b2c29dd8e8e0491483c3abb40789cfd578bf362fb6ba8261b05f0387d76792ed6e23ea3b1b6a0", + "packageManager": "pnpm@9.15.5+sha512.845196026aab1cc3f098a0474b64dfbab2afe7a1b4e91dd86895d8e4aa32a7a6d03049e2d0ad770bbe4de023a7122fb68c1a1d6e0d033c7076085f9d5d4800d4", "engines": { "node": ">=18", "pnpm": ">=9" From 54273c400dce18a1ee60c960aad629505b28b0c0 Mon Sep 17 00:00:00 2001 From: Curtis Man Date: Mon, 3 Feb 2025 10:32:01 -0800 Subject: [PATCH 10/16] Force to use latest corepack version (#657) --- pipelines/include-install-pnpm.yml | 1 + ts/Dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/pipelines/include-install-pnpm.yml b/pipelines/include-install-pnpm.yml index cc90d4123..77cdb1670 100644 --- a/pipelines/include-install-pnpm.yml +++ b/pipelines/include-install-pnpm.yml @@ -45,6 +45,7 @@ steps: # workspace-concurrency 0 means use use the CPU core count. This is better than the default (4) for larger agents. script: | echo "Using node $(node --version)" + npm install -g corepack@0.31.0 sudo corepack enable echo "Using pnpm $(pnpm -v)" pnpm config set store-dir ${{ parameters.pnpmStorePath }} diff --git a/ts/Dockerfile b/ts/Dockerfile index 1fa16456b..4a4ff3cf8 100644 --- a/ts/Dockerfile +++ b/ts/Dockerfile @@ -2,6 +2,7 @@ FROM node:20 AS base ENV PNPM_HOME="/pnpm" ENV PATH="$PNPM_HOME:$PATH" +RUN npm install -g corepack@0.31.0 RUN corepack enable # Install dependencies required for Chrome/Puppeteer From ad5e688cf4fb56bf12d2aa0e5c60a7e5bb5ef252 Mon Sep 17 00:00:00 2001 From: gvanrossum-ms Date: Mon, 3 Feb 2025 10:19:20 -0800 Subject: [PATCH 11/16] [Spelunker] Add .focus command to set focus directory (#655) In a Spelunker session (after `@config request spelunker`) you can now ask to see the focus by typing `.focus` or change the focus using `focus `. Folders can start with `~` indicating `$HOME`, or be absolute paths or relative paths (relative to whatever the CLI or Shell uses as its current directory). --- .../agents/spelunker/src/searchCode.ts | 2 +- .../spelunker/src/spelunkerActionHandler.ts | 61 +++++++++++++++---- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/ts/packages/agents/spelunker/src/searchCode.ts b/ts/packages/agents/spelunker/src/searchCode.ts index 84a83b26a..27d9d2231 100644 --- a/ts/packages/agents/spelunker/src/searchCode.ts +++ b/ts/packages/agents/spelunker/src/searchCode.ts @@ -168,7 +168,7 @@ export async function searchCode( } // 4. Construct a prompt from those chunks. - console_log(`[Step 4: Construct a prompt for the smart LLM]`); + console_log(`[Step 4: Construct a prompt for the oracle]`); const preppedChunks: Chunk[] = chunkDescs .map((chunkDesc) => prepChunk(chunkDesc, allChunks)) .filter(Boolean) as Chunk[]; diff --git a/ts/packages/agents/spelunker/src/spelunkerActionHandler.ts b/ts/packages/agents/spelunker/src/spelunkerActionHandler.ts index 2aa9dc4db..ca469e7c4 100644 --- a/ts/packages/agents/spelunker/src/spelunkerActionHandler.ts +++ b/ts/packages/agents/spelunker/src/spelunkerActionHandler.ts @@ -48,12 +48,18 @@ class RequestCommandHandler implements CommandHandler { params: ParsedCommandParams, ): Promise { if (typeof params.args?.question === "string") { - const result: ActionResult = await searchCode( - actionContext.sessionContext.agentContext, - params.args.question, - [], - [], - ); + let result: ActionResult; + const question = params.args.question.trim(); + if (question.startsWith(".")) { + result = handleFocus(actionContext.sessionContext, question); + } else { + result = await searchCode( + actionContext.sessionContext.agentContext, + question, + [], + [], + ); + } if (typeof result.error == "string") { actionContext.actionIO.appendDisplay({ type: "text", @@ -154,13 +160,11 @@ async function handleSpelunkerAction( ): Promise { switch (action.actionName) { case "searchCode": { - if ( - typeof action.parameters.question == "string" && - action.parameters.question.trim() - ) { + const question = action.parameters.question.trim(); + if (typeof question == "string" && question) { return await searchCode( context.agentContext, - action.parameters.question, + question, action.parameters.entityUniqueIds, entities, ); @@ -223,3 +227,38 @@ function focusReport( ]; return createActionResult(literalText, undefined, entities); } + +function handleFocus( + sessionContext: SessionContext, + question: string, +): ActionResult { + question = question.trim(); + if (!question.startsWith(".")) { + throw new Error("handleFocus requires a question starting with '.'"); + } + const spelunkerContext: SpelunkerContext = sessionContext.agentContext; + const words = question.split(/\s+/); + if (words[0] != ".focus") { + const text = `Unknown '.' command (${words[0]}) -- try .focus`; + return createActionResult(text); + } + if (words.length < 2) { + return focusReport( + sessionContext.agentContext, + "Focus is empty", + "Focus is", + ); + } + spelunkerContext.focusFolders = [ + ...words + .slice(1) + .map((folder) => path.resolve(expandHome(folder))) + .filter((f) => fs.existsSync(f) && fs.statSync(f).isDirectory()), + ]; + saveContext(sessionContext); + return focusReport( + sessionContext.agentContext, + "Focus cleared", + "Focus set to", + ); +} From 3588ffdd3af1585087c2756c3fc6e39774cda49a Mon Sep 17 00:00:00 2001 From: Curtis Man Date: Mon, 3 Feb 2025 11:40:04 -0800 Subject: [PATCH 12/16] Enable no unused locals in dispatcher package. (#658) --- ts/packages/dispatcher/src/command/command.ts | 3 +-- ts/packages/dispatcher/src/command/completion.ts | 3 --- .../dispatcher/src/context/dispatcher/dispatcherAgent.ts | 6 +----- .../context/dispatcher/handlers/explainCommandHandler.ts | 5 +---- .../src/context/system/action/configActionHandler.ts | 3 +-- .../src/context/system/handlers/envCommandHandler.ts | 2 -- .../src/context/system/handlers/serviceHost/service.ts | 1 - .../src/context/system/handlers/sessionCommandHandlers.ts | 2 -- ts/packages/dispatcher/src/context/system/systemAgent.ts | 6 +----- ts/packages/dispatcher/src/execute/actionHandlers.ts | 2 +- .../dispatcher/src/translation/actionConfigProvider.ts | 8 +------- .../dispatcher/src/translation/actionSchemaFileCache.ts | 2 -- .../src/translation/actionSchemaJsonTranslator.ts | 2 +- ts/packages/dispatcher/src/tsconfig.json | 3 +-- 14 files changed, 9 insertions(+), 39 deletions(-) diff --git a/ts/packages/dispatcher/src/command/command.ts b/ts/packages/dispatcher/src/command/command.ts index 432447aaa..b7dca84d8 100644 --- a/ts/packages/dispatcher/src/command/command.ts +++ b/ts/packages/dispatcher/src/command/command.ts @@ -8,7 +8,7 @@ import { DispatcherName, makeClientIOMessage, } from "../context/interactiveIO.js"; -import { FullAction, getDefaultExplainerName } from "agent-cache"; +import { getDefaultExplainerName } from "agent-cache"; import { CommandHandlerContext } from "../context/commandHandlerContext.js"; import { @@ -18,7 +18,6 @@ import { } from "@typeagent/agent-sdk"; import { executeCommand } from "../execute/actionHandlers.js"; import { isCommandDescriptorTable } from "@typeagent/agent-sdk/helpers/command"; -import { RequestMetrics } from "../utils/metrics.js"; import { parseParams } from "./parameters.js"; import { getHandlerTableUsage, getUsage } from "./commandHelp.js"; import { CommandResult } from "../dispatcher.js"; diff --git a/ts/packages/dispatcher/src/command/completion.ts b/ts/packages/dispatcher/src/command/completion.ts index c6542ba07..866b4fe71 100644 --- a/ts/packages/dispatcher/src/command/completion.ts +++ b/ts/packages/dispatcher/src/command/completion.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import registerDebug from "debug"; import { CommandHandlerContext } from "../context/commandHandlerContext.js"; import { @@ -29,8 +28,6 @@ export type CommandCompletionResult = { completions: string[]; // All the partial completions available after partial (and space if true) }; -const debugPartialError = registerDebug("typeagent:dispatcher:partial:error"); - // Determine the command to resolve for partial completion // If there is a trailing space, then it will just be the input (minus the @) // If there is no space, then it will the input without the last word diff --git a/ts/packages/dispatcher/src/context/dispatcher/dispatcherAgent.ts b/ts/packages/dispatcher/src/context/dispatcher/dispatcherAgent.ts index cf3e8d853..e9a7414a6 100644 --- a/ts/packages/dispatcher/src/context/dispatcher/dispatcherAgent.ts +++ b/ts/packages/dispatcher/src/context/dispatcher/dispatcherAgent.ts @@ -10,16 +10,12 @@ import { TranslateCommandHandler } from "./handlers/translateCommandHandler.js"; import { ExplainCommandHandler } from "./handlers/explainCommandHandler.js"; import { ActionContext, - AppAction, AppAgent, AppAgentManifest, } from "@typeagent/agent-sdk"; import { CommandHandlerContext } from "../commandHandlerContext.js"; import { createActionResultNoDisplay } from "@typeagent/agent-sdk/helpers/action"; -import { - DispatcherActions, - UnknownAction, -} from "./schema/dispatcherActionSchema.js"; +import { DispatcherActions } from "./schema/dispatcherActionSchema.js"; import { ClarifyRequestAction } from "./schema/clarifyActionSchema.js"; const dispatcherHandlers: CommandHandlerTable = { diff --git a/ts/packages/dispatcher/src/context/dispatcher/handlers/explainCommandHandler.ts b/ts/packages/dispatcher/src/context/dispatcher/handlers/explainCommandHandler.ts index 390b2e1bc..4027121c7 100644 --- a/ts/packages/dispatcher/src/context/dispatcher/handlers/explainCommandHandler.ts +++ b/ts/packages/dispatcher/src/context/dispatcher/handlers/explainCommandHandler.ts @@ -2,10 +2,7 @@ // Licensed under the MIT License. import { ActionContext, ParsedCommandParams } from "@typeagent/agent-sdk"; import { CommandHandler } from "@typeagent/agent-sdk/helpers/command"; -import { - displayResult, - displayStatus, -} from "@typeagent/agent-sdk/helpers/display"; +import { displayResult } from "@typeagent/agent-sdk/helpers/display"; import { CommandHandlerContext } from "../../commandHandlerContext.js"; import { RequestAction, printProcessRequestActionResult } from "agent-cache"; import { createLimiter, getElapsedString } from "common-utils"; diff --git a/ts/packages/dispatcher/src/context/system/action/configActionHandler.ts b/ts/packages/dispatcher/src/context/system/action/configActionHandler.ts index 44ba8dc29..6b310950d 100644 --- a/ts/packages/dispatcher/src/context/system/action/configActionHandler.ts +++ b/ts/packages/dispatcher/src/context/system/action/configActionHandler.ts @@ -3,7 +3,7 @@ import { processCommandNoLock } from "../../../command/command.js"; import { CommandHandlerContext } from "../../commandHandlerContext.js"; -import { ConfigAction, ToggleAgent } from "../schema/configActionSchema.js"; +import { ConfigAction } from "../schema/configActionSchema.js"; import { AppAction, ActionContext } from "@typeagent/agent-sdk"; export async function executeConfigAction( @@ -19,7 +19,6 @@ export async function executeConfigAction( ); break; case "toggleAgent": - const agentAction = configAction as ToggleAgent; const cmdParam: string = configAction.parameters.enable ? `` : `--off`; diff --git a/ts/packages/dispatcher/src/context/system/handlers/envCommandHandler.ts b/ts/packages/dispatcher/src/context/system/handlers/envCommandHandler.ts index 79e9036ab..0380d609f 100644 --- a/ts/packages/dispatcher/src/context/system/handlers/envCommandHandler.ts +++ b/ts/packages/dispatcher/src/context/system/handlers/envCommandHandler.ts @@ -12,8 +12,6 @@ import { displayError, displayResult, } from "@typeagent/agent-sdk/helpers/display"; -import dotenv from "dotenv"; -import { Action } from "agent-cache"; export class EnvCommandHandler implements CommandHandlerNoParams { public readonly description = diff --git a/ts/packages/dispatcher/src/context/system/handlers/serviceHost/service.ts b/ts/packages/dispatcher/src/context/system/handlers/serviceHost/service.ts index bedb54be4..92aa85df3 100644 --- a/ts/packages/dispatcher/src/context/system/handlers/serviceHost/service.ts +++ b/ts/packages/dispatcher/src/context/system/handlers/serviceHost/service.ts @@ -2,7 +2,6 @@ // Licensed under the MIT License. import { WebSocket, WebSocketServer } from "ws"; -import { WebSocketMessageV2 } from "common-utils"; import registerDebug from "debug"; import { IncomingMessage } from "node:http"; diff --git a/ts/packages/dispatcher/src/context/system/handlers/sessionCommandHandlers.ts b/ts/packages/dispatcher/src/context/system/handlers/sessionCommandHandlers.ts index 49bc7f105..63c994a2e 100644 --- a/ts/packages/dispatcher/src/context/system/handlers/sessionCommandHandlers.ts +++ b/ts/packages/dispatcher/src/context/system/handlers/sessionCommandHandlers.ts @@ -28,9 +28,7 @@ import { displaySuccess, displayWarn, } from "@typeagent/agent-sdk/helpers/display"; -import { getToggleHandlerTable } from "../../../command/handlerUtils.js"; import { askYesNoWithContext } from "../../interactiveIO.js"; -import path from "node:path"; class SessionNewCommandHandler implements CommandHandler { public readonly description = "Create a new empty session"; diff --git a/ts/packages/dispatcher/src/context/system/systemAgent.ts b/ts/packages/dispatcher/src/context/system/systemAgent.ts index 65d171c2e..e975a2e94 100644 --- a/ts/packages/dispatcher/src/context/system/systemAgent.ts +++ b/ts/packages/dispatcher/src/context/system/systemAgent.ts @@ -41,7 +41,6 @@ import { resolveCommand, } from "../../command/command.js"; import { - getHandlerTableUsage, getUsage, printStructuredHandlerTableUsage, } from "../../command/commandHelp.js"; @@ -61,10 +60,7 @@ import { getParameterNames, validateAction, } from "action-schema"; -import { - EnvCommandHandler, - getEnvCommandHandlers, -} from "./handlers/envCommandHandler.js"; +import { getEnvCommandHandlers } from "./handlers/envCommandHandler.js"; import { executeNotificationAction } from "./action/notificationActionHandler.js"; import { executeHistoryAction } from "./action/historyActionHandler.js"; diff --git a/ts/packages/dispatcher/src/execute/actionHandlers.ts b/ts/packages/dispatcher/src/execute/actionHandlers.ts index 1f464abce..7a18a9278 100644 --- a/ts/packages/dispatcher/src/execute/actionHandlers.ts +++ b/ts/packages/dispatcher/src/execute/actionHandlers.ts @@ -126,7 +126,7 @@ function getActionContext( function closeContextObject(o: any) { const descriptors = Object.getOwnPropertyDescriptors(o); - for (const [name, desc] of Object.entries(descriptors)) { + for (const [name] of Object.entries(descriptors)) { // TODO: Note this doesn't prevent the function continue to be call if is saved. Object.defineProperty(o, name, { get: () => { diff --git a/ts/packages/dispatcher/src/translation/actionConfigProvider.ts b/ts/packages/dispatcher/src/translation/actionConfigProvider.ts index 25bfab901..3c7c424ae 100644 --- a/ts/packages/dispatcher/src/translation/actionConfigProvider.ts +++ b/ts/packages/dispatcher/src/translation/actionConfigProvider.ts @@ -2,13 +2,7 @@ // Licensed under the MIT License. import { ActionSchemaFile } from "action-schema"; -import { ActionConfig, convertToActionConfig } from "./actionConfig.js"; -import { AppAgentProvider } from "../agentProvider/agentProvider.js"; -import { AppAgentManifest } from "@typeagent/agent-sdk"; -import { - ActionSchemaFileCache, - createSchemaInfoProvider, -} from "./actionSchemaFileCache.js"; +import { ActionConfig } from "./actionConfig.js"; export interface ActionConfigProvider { tryGetActionConfig(schemaName: string): ActionConfig | undefined; diff --git a/ts/packages/dispatcher/src/translation/actionSchemaFileCache.ts b/ts/packages/dispatcher/src/translation/actionSchemaFileCache.ts index 63cd6dd6a..44e668a4e 100644 --- a/ts/packages/dispatcher/src/translation/actionSchemaFileCache.ts +++ b/ts/packages/dispatcher/src/translation/actionSchemaFileCache.ts @@ -194,8 +194,6 @@ export function getActionSchema( export function createSchemaInfoProvider( provider: ActionConfigProvider, ): SchemaInfoProvider { - const hashCache = new Map(); - const getActionSchemaFile = (schemaName: string) => { return provider.getActionSchemaFileForConfig( provider.getActionConfig(schemaName), diff --git a/ts/packages/dispatcher/src/translation/actionSchemaJsonTranslator.ts b/ts/packages/dispatcher/src/translation/actionSchemaJsonTranslator.ts index e06c05cb9..0daadb3fe 100644 --- a/ts/packages/dispatcher/src/translation/actionSchemaJsonTranslator.ts +++ b/ts/packages/dispatcher/src/translation/actionSchemaJsonTranslator.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { error, Result, success, TypeChatJsonValidator } from "typechat"; +import { error, Result, success } from "typechat"; import { ActionSchemaFile, generateActionSchema, diff --git a/ts/packages/dispatcher/src/tsconfig.json b/ts/packages/dispatcher/src/tsconfig.json index f5f61dab7..040ded705 100644 --- a/ts/packages/dispatcher/src/tsconfig.json +++ b/ts/packages/dispatcher/src/tsconfig.json @@ -3,8 +3,7 @@ "compilerOptions": { "composite": true, "rootDir": ".", - "outDir": "../dist", - "noUnusedLocals": false, + "outDir": "../dist", }, "include": ["./**/*"], "ts-node": { From f4c49f01e9d651d071dff4fa59b57cffedc39527 Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Mon, 3 Feb 2025 12:50:11 -0800 Subject: [PATCH 13/16] Schema discovery: Add page summaries to help ground the candidate user action responses (#659) --- .../src/agent/discovery/actionHandler.mts | 39 +++++++- .../discovery/schema/discoveryActions.mts | 3 + .../agent/discovery/schema/pageSummary.mts | 10 +++ .../src/agent/discovery/schema/pageTypes.mts | 90 +++++++++++-------- .../discovery/schema/userActionsPool.mts | 27 +++++- .../src/agent/discovery/translator.mts | 84 +++++++++++++++++ 6 files changed, 214 insertions(+), 39 deletions(-) create mode 100644 ts/packages/agents/browser/src/agent/discovery/schema/pageSummary.mts diff --git a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts index 814ad5d16..98f532fba 100644 --- a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts +++ b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts @@ -24,16 +24,36 @@ export async function handleSchemaDiscoveryAction( case "findUserActions": await handleFindUserActions(action); break; + case "summarizePage": + await handleGetPageSummary(action); + break; } async function handleFindUserActions(action: any) { const htmlFragments = await browser.getHtmlFragments(); + // const screenshot = await browser.getCurrentPageScreenshot(); + const screenshot = ""; + let pageSummary = ""; + + const summaryResponse = await agent.getPageSummary( + undefined, + htmlFragments, + screenshot, + ); + + if (summaryResponse.success) { + pageSummary = + "Page summary: \n" + JSON.stringify(summaryResponse.data, null, 2); + } + const timerName = `Analyzing page actions`; console.time(timerName); + const response = await agent.getCandidateUserActions( undefined, htmlFragments, - undefined, + screenshot, + pageSummary, ); if (!response.success) { @@ -48,5 +68,22 @@ export async function handleSchemaDiscoveryAction( return response.data; } + async function handleGetPageSummary(action: any) { + const htmlFragments = await browser.getHtmlFragments(); + const timerName = `Summarizing page`; + console.time(timerName); + const response = await agent.getPageSummary(undefined, htmlFragments); + + if (!response.success) { + console.error("Attempt to get page summary failed"); + console.error(response.message); + return; + } + + console.timeEnd(timerName); + message = "Page summary: \n" + JSON.stringify(response.data, null, 2); + return response.data; + } + return message; } diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts b/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts index e16160e06..1ebc8189d 100644 --- a/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts +++ b/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts @@ -14,6 +14,9 @@ export type FindUserActions = { export type SummarizePage = { actionName: "summarizePage"; + parameters: { + allowDuplicates?: boolean; + }; }; export type SaveUserActions = { diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/pageSummary.mts b/ts/packages/agents/browser/src/agent/discovery/schema/pageSummary.mts new file mode 100644 index 000000000..379c7cf6f --- /dev/null +++ b/ts/packages/agents/browser/src/agent/discovery/schema/pageSummary.mts @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// A description of the page, including layout information and summary of content. +export type PageDescription = { + description: string; + features: string[]; + entities: string[]; + possibleUserAction: string[]; +}; diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/pageTypes.mts b/ts/packages/agents/browser/src/agent/discovery/schema/pageTypes.mts index 49a7803e4..edc5b363b 100644 --- a/ts/packages/agents/browser/src/agent/discovery/schema/pageTypes.mts +++ b/ts/packages/agents/browser/src/agent/discovery/schema/pageTypes.mts @@ -1,55 +1,71 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -export type SearchBox = { - featureName: "searchInputBox"; - description: "Input box for searching on the page"; - parameters: { - cssSelector: string; - }; -}; - -export type SearchResultsList = { - featureName: "searchResultsList"; - description: "List of products available from the search results"; - parameters: { - cssSelector: string; - }; -}; - -export type ProductDetailsCard = { - featureName: "productDetailsCard"; - description: "A section that shows the product name, price, images and rating. This also gives an option to add the product to the shopping cart."; - parameters: { - cssSelector: string; - }; -}; - -export type SearchForContent = { - actionName: "searchForProduct"; - description: "Find content on the page"; - parameters: { - value: string; - cssSelector: string; - }; -}; - export type LandingPage = { description: "The default landing page for the site"; - features: SearchBox; }; export type SearchResultsPage = { description: "The search results page"; - features: SearchResultsList; }; export type ProductDetailsPage = { description: "A product details page, with focus on one product."; - features: ProductDetailsCard; }; export type ShoppingCartPage = { description: "The shopping cart page for the site"; - features: SearchBox; }; + +export type PastOrderPage = { + description: "The page showing a user's past orders"; +}; + +export type UnknownPage = { + description: "A page that does not meet the previous more-specific categories"; +}; + +export type CommercePageTypes = + | LandingPage + | SearchResultsPage + | ProductDetailsPage + | ShoppingCartPage + | PastOrderPage + | UnknownPage; + +export type CrosswordPage = { + description: "The page showing a crossword puzzle"; +}; + +export type NewsLandingPage = { + description: "The page showing news headlines for the day"; +}; + +export type SportsLandingPage = { + description: "The page showing sports headlines for the day"; +}; + +export type OpinionPage = { + description: "The page showing editorial opinions for the day"; +}; + +export type ArticlePage = { + description: "The page showing an individual news article"; +}; + +export type WeatherPage = { + description: "The page showing weather headlines"; +}; + +export type PuzzlesPage = { + description: "The page showing a list of puzzles, such as sudoku, crossword, word matching games and more."; +}; + +export type NewsPageTypes = + | CrosswordPage + | NewsLandingPage + | SportsLandingPage + | OpinionPage + | ArticlePage + | PuzzlesPage + | UnknownPage; diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts b/ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts index b63504bdc..55b0675ad 100644 --- a/ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts +++ b/ts/packages/agents/browser/src/agent/discovery/schema/userActionsPool.mts @@ -28,6 +28,7 @@ export type SearchForProductAction = { }; }; +// This allows users to select individual results on the search results page. export type SelectSearchResult = { actionName: "selectSearchResult"; parameters: { @@ -38,39 +39,63 @@ export type SelectSearchResult = { export type NavigateToHomePage = { actionName: "navigateToHomePage"; + parameters: { + linkCssSelector: string; + }; }; // Follow a link to view a store landing page export type NavigateToStorePage = { actionName: "navigateToStorePage"; + parameters: { + linkCssSelector: string; + }; }; // Follow a link to view a product details page export type NavigateToProductPage = { actionName: "navigateToProductPage"; + parameters: { + linkCssSelector: string; + }; }; -// Follow a link to view a recipe details page +// Follow a link to view a recipe details page. This link is typically named "Recipe" or "Recipes" export type NavigateToRecipePage = { actionName: "navigateToRecipePage"; + parameters: { + linkCssSelector: string; + }; }; export type NavigateToListPage = { actionName: "navigateToListPage"; + parameters: { + linkCssSelector: string; + }; }; +// Navigate to the "Buy it again" page. This page may also be called Past Orders. export type NavigateToBuyItAgainPage = { actionName: "navigateToBuyItAgainPage"; + parameters: { + linkCssSelector: string; + }; }; +// This link opens the shopping cart. Its usually indicated by a cart or bag icon. export type NavigateToShoppingCartPage = { actionName: "navigateToShoppingCartPage"; + parameters: { + linkCssSelector: string; + }; }; export type NavigateToOtherPage = { actionName: "navigateToOtherPage"; parameters: { pageType: string; + linkCssSelector: string; }; }; diff --git a/ts/packages/agents/browser/src/agent/discovery/translator.mts b/ts/packages/agents/browser/src/agent/discovery/translator.mts index 22daadd5c..8fe660591 100644 --- a/ts/packages/agents/browser/src/agent/discovery/translator.mts +++ b/ts/packages/agents/browser/src/agent/discovery/translator.mts @@ -250,6 +250,7 @@ export class SchemaDiscoveryAgent { userRequest?: string, fragments?: HtmlFragments[], screenshot?: string, + pageSummary?: string, ) { // prompt - present html, optional screenshot and list of candidate actions const bootstrapTranslator = this.getBootstrapTranslator( @@ -273,6 +274,19 @@ export class SchemaDiscoveryAgent { `, }); } + if (pageSummary) { + requestSection.push({ + type: "text", + text: ` + + Here is a previously-generated summary of the page + ''' + ${pageSummary} + ''' + `, + }); + } + const promptSections = [ ...prefixSection, ...screenshotSection, @@ -303,4 +317,74 @@ export class SchemaDiscoveryAgent { ]); return response; } + + async getPageSummary( + userRequest?: string, + fragments?: HtmlFragments[], + screenshot?: string, + ) { + const packageRoot = path.join("..", "..", ".."); + const resultsSchema = await fs.promises.readFile( + fileURLToPath( + new URL( + path.join( + packageRoot, + "./src/agent/discovery/schema/pageSummary.mts", + ), + import.meta.url, + ), + ), + "utf8", + ); + + const bootstrapTranslator = this.getBootstrapTranslator( + "PageDescription", + resultsSchema, + ); + + const screenshotSection = getScreenshotPromptSection(screenshot, fragments); + const htmlSection = getHtmlPromptSection(fragments); + const prefixSection = getBootstrapPrefixPromptSection(); + let requestSection = []; + if (userRequest) { + requestSection.push({ + type: "text", + text: ` + + Here is user request + ''' + ${userRequest} + ''' + `, + }); + } + const promptSections = [ + ...prefixSection, + ...screenshotSection, + ...htmlSection, + { + type: "text", + text: ` + Examine the layout information provided and determine the content of the page and the actions users can take on it. + Once you have this list, a SINGLE "PageDescription" response using the typescript schema below. + + ''' + ${bootstrapTranslator.validator.getSchemaText()} + ''' + `, + }, + ...requestSection, + { + type: "text", + text: ` + The following is the COMPLETE JSON response object with 2 spaces of indentation and no properties with the value undefined: + `, + }, + ]; + + const response = await bootstrapTranslator.translate("", [ + { role: "user", content: JSON.stringify(promptSections) }, + ]); + return response; + } } From 313cf3ecde7c8fe7080400a716ed8c8943c4728b Mon Sep 17 00:00:00 2001 From: gvanrossum-ms Date: Mon, 3 Feb 2025 12:52:28 -0800 Subject: [PATCH 14/16] [Spelunker] Be less eager to get comments before a declaration (#660) We were failing an assertion when splicing child nodes out of parent nodes in the following case: ```ts type Foo = ...; // comment function bar() { ... } ``` The TSC `node.getFullStart() ` API would return the location of Foo's comment, and our strategy for moving to the next line didn't work. (This makes `getFullStart()` pretty stupid IMO, but we can't change that.) This PR is a quick hack to fix this by always moving to the start of the next line if the start position points in the middle of a line (i.e., `character` is > 0). --- .../agents/spelunker/src/typescriptChunker.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ts/packages/agents/spelunker/src/typescriptChunker.ts b/ts/packages/agents/spelunker/src/typescriptChunker.ts index d17842757..54ba26f2c 100644 --- a/ts/packages/agents/spelunker/src/typescriptChunker.ts +++ b/ts/packages/agents/spelunker/src/typescriptChunker.ts @@ -192,6 +192,19 @@ function makeBlobs( const lineStarts = sourceFile.getLineStarts(); // TODO: Move to caller? let startLoc = sourceFile.getLineAndCharacterOfPosition(startPos); const endLoc = sourceFile.getLineAndCharacterOfPosition(endPos); + if (startLoc.character) { + // Adjust start: if in the middle of a line, move to start of next line. + // This is still a heuristic that will fail e.g. with `; function ...`. + // But we don't want to support that; a more likely scenario is: + // ``` + // type A = ...; // comment + // function ... + // ``` + // Here getFullStart() points to the start of the comment on A, + // but we must start at the function. + startPos = lineStarts[startLoc.line + 1]; + startLoc = sourceFile.getLineAndCharacterOfPosition(startPos); + } // console.log( // `Start and end: ${startPos}=${startLoc.line + 1}:${startLoc.character}, ` + // `${endPos}=${endLoc.line + 1}:${endLoc.character}`, From 28c43a0ac8f041321057f217186c8af8baff3c4b Mon Sep 17 00:00:00 2001 From: robgruen Date: Mon, 3 Feb 2025 15:27:23 -0800 Subject: [PATCH 15/16] More shell tests (#653) --- .github/workflows/build-ts.yml | 44 ----- .github/workflows/shell-tests.yml | 12 +- .../smoke-tests-pull_request_targets.yml.bak | 121 ------------ .github/workflows/smoke-tests.yml | 7 +- ts/packages/shell/test/listAgent.spec.ts | 64 ++++++ .../shell/test/sessionCommands.spec.ts | 34 ++-- ts/packages/shell/test/simple.spec.ts | 8 +- ts/packages/shell/test/testHelper.ts | 183 ++++++++++++++++-- 8 files changed, 259 insertions(+), 214 deletions(-) delete mode 100644 .github/workflows/smoke-tests-pull_request_targets.yml.bak create mode 100644 ts/packages/shell/test/listAgent.spec.ts diff --git a/.github/workflows/build-ts.yml b/.github/workflows/build-ts.yml index a10bcb36d..7a891ef71 100644 --- a/.github/workflows/build-ts.yml +++ b/.github/workflows/build-ts.yml @@ -66,10 +66,6 @@ jobs: working-directory: ts run: | pnpm install --frozen-lockfile --strict-peer-dependencies - # - name: Install Playwright Browsers - # if: ${{ runner.os == 'windows' && matrix.version == '22' }} - # run: pnpm exec playwright install --with-deps - # working-directory: ts/packages/shell - name: Build if: ${{ github.event_name != 'pull_request' || steps.filter.outputs.ts == 'true' }} working-directory: ts @@ -85,43 +81,3 @@ jobs: working-directory: ts run: | npm run lint - # - name: Login to Azure - # if: ${{ github.event_name != 'merge_group' }} - # uses: azure/login@v2.2.0 - # with: - # client-id: ${{ secrets.AZUREAPPSERVICE_CLIENTID_5B0D2D6BA40F4710B45721D2112356DD }} - # tenant-id: ${{ secrets.AZUREAPPSERVICE_TENANTID_39BB903136F14B6EAD8F53A8AB78E3AA }} - # subscription-id: ${{ secrets.AZUREAPPSERVICE_SUBSCRIPTIONID_F36C1F2C4B2C49CA8DD5C52FAB98FA30 }} - # - name: Get Keys - # if: ${{ github.event_name != 'merge_group' }} - # run: | - # node tools/scripts/getKeys.mjs --vault build-pipeline-kv - # working-directory: ts - # - name: Test CLI - smoke - # if: ${{ github.event_name != 'merge_group' }} - # run: | - # npm run start:dev 'prompt' 'why is the sky blue' - # working-directory: ts/packages/cli - # continue-on-error: true - # - name: Shell Tests - smoke (windows) - # if: ${{ github.event_name != 'merge_group' && runner.os == 'windows' && matrix.version == '22' }} - # timeout-minutes: 60 - # run: | - # npx playwright test simple.spec.ts - # rm ../../.env - # - name: Shell Tests - smoke (linux) - # if: ${{ github.event_name != 'merge_group' && runner.os == 'Linux' && matrix.version == '22' }} - # timeout-minutes: 60 - # run: | - # Xvfb :99 -screen 0 1600x1200x24 & export DISPLAY=:99 - # npx playwright test simple.spec.ts - # rm ../../.env - # working-directory: ts/packages/shell - # continue-on-error: true - # - name: Live Tests - # if: ${{ github.event_name != 'merge_group' && runner.os == 'linux' && matrix.version == '22' }} - # timeout-minutes: 60 - # run: | - # npm run test:live - # working-directory: ts - # continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index e0ce15004..6a1c1cf4e 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -28,7 +28,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["windows-latest"] + os: ["windows-latest", "ubuntu-latest"] version: [20] runs-on: ${{ matrix.os }} @@ -98,19 +98,15 @@ jobs: timeout-minutes: 60 run: | npm run shell:test - rm ../../.env working-directory: ts/packages/shell continue-on-error: true - name: Shell Tests (linux) if: ${{ runner.os == 'Linux' }} timeout-minutes: 60 - # https://github.com/microsoft/playwright/issues/34251 - sysctl command run: | - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 Xvfb :99 -screen 0 1600x1200x24 & export DISPLAY=:99 npm run shell:test - rm ../../.env working-directory: ts/packages/shell continue-on-error: true @@ -129,3 +125,9 @@ jobs: path: ts/packages/shell/playwright-report/ overwrite: true retention-days: 30 + + - name: Clean up Keys + run: | + rm ./.env + working-directory: ts + if: always() diff --git a/.github/workflows/smoke-tests-pull_request_targets.yml.bak b/.github/workflows/smoke-tests-pull_request_targets.yml.bak deleted file mode 100644 index d336a2ee3..000000000 --- a/.github/workflows/smoke-tests-pull_request_targets.yml.bak +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -# This workflow runs live/smoke sanity tests - -name: smoke-tests - -on: - workflow_dispatch: - pull_request_target: - branches: ["main"] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -permissions: - pull-requests: read - contents: read - id-token: write - -env: - NODE_OPTIONS: --max_old_space_size=8192 - -jobs: - shell_and_cli: - environment: development-forks # required for federated credentials - strategy: - fail-fast: false - matrix: - #os: ["ubuntu-latest", "windows-latest", "macos-latest"] - os: ["ubuntu-latest"] - version: [20] - - runs-on: ${{ matrix.os }} - steps: - - if: runner.os == 'Linux' - run: | - sudo apt install libsecret-1-0 - - - name: Setup Git LF - run: | - git config --global core.autocrlf false - - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - uses: dorny/paths-filter@v3 - id: filter - with: - filters: | - ts: - - "ts/**" - - ".github/workflows/build-ts.yml" - - - uses: pnpm/action-setup@v4 - name: Install pnpm - with: - package_json_file: ts/package.json - - - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.version }} - cache: "pnpm" - cache-dependency-path: ts/pnpm-lock.yaml - - name: Install dependencies - working-directory: ts - run: | - pnpm install --frozen-lockfile --strict-peer-dependencies - - - name: Install Playwright Browsers - run: pnpm exec playwright install --with-deps - working-directory: ts/packages/shell - - - name: Build - working-directory: ts - run: | - npm run build - - - name: Login to Azure - uses: azure/login@v2.2.0 - with: - client-id: ${{ secrets.AZUREAPPSERVICE_CLIENTID_5B0D2D6BA40F4710B45721D2112356DD }} - tenant-id: ${{ secrets.AZUREAPPSERVICE_TENANTID_39BB903136F14B6EAD8F53A8AB78E3AA }} - subscription-id: ${{ secrets.AZUREAPPSERVICE_SUBSCRIPTIONID_F36C1F2C4B2C49CA8DD5C52FAB98FA30 }} - - - name: Get Keys - run: | - node tools/scripts/getKeys.mjs --vault build-pipeline-kv - working-directory: ts - - - name: Test CLI - smoke - run: | - npm run start:dev 'prompt' 'why is the sky blue' - working-directory: ts/packages/cli - continue-on-error: true - - - name: Shell Tests - smoke (windows) - if: ${{ runner.os == 'windows' }} - timeout-minutes: 60 - run: | - npx playwright test simple.spec.ts - rm ../../.env - - - name: Shell Tests - smoke (linux) - if: ${{ runner.os == 'Linux' }} - timeout-minutes: 60 - run: | - Xvfb :99 -screen 0 1600x1200x24 & export DISPLAY=:99 - npx playwright test simple.spec.ts - rm ../../.env - working-directory: ts/packages/shell - continue-on-error: true - - - name: Live Tests - if: ${{ runner.os == 'linux' }} - timeout-minutes: 60 - run: | - npm run test:live - working-directory: ts - continue-on-error: true \ No newline at end of file diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml index d1317690c..92587115a 100644 --- a/.github/workflows/smoke-tests.yml +++ b/.github/workflows/smoke-tests.yml @@ -9,8 +9,6 @@ on: workflow_dispatch: push: branches: ["main"] - # pull_request: - # branches: ["main"] pull_request_target: branches: ["main"] merge_group: @@ -27,7 +25,9 @@ permissions: env: NODE_OPTIONS: --max_old_space_size=8192 - DEBUG: pw:browser* + # DEBUG: pw:browser* # PlayWright debug messages + # ELECTRON_ENABLE_LOGGING: true # Electron debug messages + # DEBUG: typeagent:* # TypeAgent debug messages jobs: shell_and_cli: @@ -37,7 +37,6 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "windows-latest"] - #os: ["ubuntu-latest"] version: [20] runs-on: ${{ matrix.os }} diff --git a/ts/packages/shell/test/listAgent.spec.ts b/ts/packages/shell/test/listAgent.spec.ts new file mode 100644 index 000000000..6cb4bcd48 --- /dev/null +++ b/ts/packages/shell/test/listAgent.spec.ts @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import test, { + ElectronApplication, + Page, + _electron, + _electron as electron, + expect, +} from "@playwright/test"; +import { + exitApplication, + getAppPath, + sendUserRequestAndWaitForCompletion, + getLaunchArgs, + startShell, + testUserRequest, +} from "./testHelper"; + +// Annotate entire file as serial. +test.describe.configure({ mode: "serial" }); + +test.describe("List Agent Tests", () => { + test("create_update_clear_list", async ({}, testInfo) => { + console.log(`Running test '${testInfo.title}`); + + await testUserRequest( + [ + "create a shopping list", + "what's on the shopping list?", + "add eggs, milk, flour to the shopping list", + "what's on the shopping list?", + "remove milk from the shopping list", + "what's on the shopping list?", + "clear the shopping list", + "what's on the shopping list?", + ], + [ + "Created list: shopping", + "List 'shopping' is empty", + "Added items: eggs,milk,flour to list shopping", + "eggs\nmilk\nflour", + "Removed items: milk from list shopping", + "eggs\nflour", + "Cleared list: shopping", + "List 'shopping' is empty", + ], + ); + }); + + // test("delete_list", async ({}, testInfo) => { + // console.log(`Running test '${testInfo.title}`); + + // await testUserRequest( + // [ + // "delete the shopping list", + // "is there a shopping list?" + // ], + // [ + // "Cleared list: shopping", + // "List 'shopping' is empty" + // ]); + // }); +}); diff --git a/ts/packages/shell/test/sessionCommands.spec.ts b/ts/packages/shell/test/sessionCommands.spec.ts index 748ab73d2..fa8967e0b 100644 --- a/ts/packages/shell/test/sessionCommands.spec.ts +++ b/ts/packages/shell/test/sessionCommands.spec.ts @@ -12,7 +12,7 @@ import { getAppPath, getLastAgentMessage, sendUserRequest, - sendUserRequestAndWaitForResponse, + sendUserRequestAndWaitForCompletion, startShell, waitForAgentMessage, } from "./testHelper"; @@ -29,20 +29,20 @@ test.describe("@session Commands", () => { const mainWindow: Page = await startShell(); // get the session count - let msg = await sendUserRequestAndWaitForResponse( + let msg = await sendUserRequestAndWaitForCompletion( `@session list`, mainWindow, ); const sessions: string[] = msg.split("\n"); - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session new`, mainWindow, ); expect(msg.toLowerCase()).toContain("new session created: "); - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session list`, mainWindow, ); @@ -52,7 +52,7 @@ test.describe("@session Commands", () => { sessions.length + 1, ); - msg = await sendUserRequestAndWaitForResponse(`@history`, mainWindow); + msg = await sendUserRequestAndWaitForCompletion(`@history`, mainWindow); expect(msg.length, "History NOT cleared!").toBe(0); // close the application @@ -66,14 +66,14 @@ test.describe("@session Commands", () => { const mainWindow: Page = await startShell(); // create a new session so we have at least two - let msg = await sendUserRequestAndWaitForResponse( + let msg = await sendUserRequestAndWaitForCompletion( `@session new`, mainWindow, ); expect(msg.toLowerCase()).toContain("new session created: "); // get the session count - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session list`, mainWindow, ); @@ -82,7 +82,7 @@ test.describe("@session Commands", () => { const sessionName: string = sessions[sessions.length - 1]; // issue delete session command - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session delete ${sessions[0]}`, mainWindow, ); @@ -92,7 +92,7 @@ test.describe("@session Commands", () => { await mainWindow.locator(".choice-button", { hasText: "No" }).click(); // verify session not deleted - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session list`, mainWindow, ); @@ -102,7 +102,7 @@ test.describe("@session Commands", () => { ); // reissue delete session command - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session delete ${sessions[0]}`, mainWindow, ); @@ -112,7 +112,7 @@ test.describe("@session Commands", () => { await mainWindow.locator(".choice-button", { hasText: "Yes" }).click(); // get new session count - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session list`, mainWindow, ); @@ -122,7 +122,7 @@ test.describe("@session Commands", () => { ); // get session info - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session info`, mainWindow, ); @@ -140,14 +140,14 @@ test.describe("@session Commands", () => { const mainWindow: Page = await startShell(); // reset - let msg = await sendUserRequestAndWaitForResponse( + let msg = await sendUserRequestAndWaitForCompletion( `@session reset`, mainWindow, ); expect(msg).toContain("Session settings revert to default."); // issue clear session command - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session clear`, mainWindow, ); @@ -167,21 +167,21 @@ test.describe("@session Commands", () => { const mainWindow: Page = await startShell(); // create a new session - let msg = await sendUserRequestAndWaitForResponse( + let msg = await sendUserRequestAndWaitForCompletion( `@session new`, mainWindow, ); expect(msg.toLowerCase()).toContain("new session created: "); // get the session list - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session list`, mainWindow, ); const sessions: string[] = msg.split("\n"); // open the earlier session - msg = await sendUserRequestAndWaitForResponse( + msg = await sendUserRequestAndWaitForCompletion( `@session open ${sessions[0]}`, mainWindow, ); diff --git a/ts/packages/shell/test/simple.spec.ts b/ts/packages/shell/test/simple.spec.ts index 575384496..613a1bea6 100644 --- a/ts/packages/shell/test/simple.spec.ts +++ b/ts/packages/shell/test/simple.spec.ts @@ -11,8 +11,8 @@ import test, { import { exitApplication, getAppPath, + sendUserRequestAndWaitForCompletion, getLaunchArgs, - sendUserRequestAndWaitForResponse, startShell, } from "./testHelper"; import { fileURLToPath } from "node:url"; @@ -31,13 +31,13 @@ test("startShell", { tag: "@smoke" }, async ({}) => { await startShell(); }); -test.skip("why is the sky blue?", { tag: "@smoke" }, async ({}, testInfo) => { +test("why is the sky blue?", { tag: "@smoke" }, async ({}, testInfo) => { console.log(`Running test '${testInfo.title}`); // launch the app const mainWindow: Page = await startShell(); - const msg = await sendUserRequestAndWaitForResponse( + const msg = await sendUserRequestAndWaitForCompletion( `why is the sky blue?`, mainWindow, ); @@ -45,7 +45,7 @@ test.skip("why is the sky blue?", { tag: "@smoke" }, async ({}, testInfo) => { expect( msg.toLowerCase(), "Chat agent didn't respond with the expected message.", - ).toContain("raleigh scattering."); + ).toContain("scattering"); // close the application await exitApplication(mainWindow); diff --git a/ts/packages/shell/test/testHelper.ts b/ts/packages/shell/test/testHelper.ts index a61cf33f0..c470888ad 100644 --- a/ts/packages/shell/test/testHelper.ts +++ b/ts/packages/shell/test/testHelper.ts @@ -4,6 +4,7 @@ import { _electron as electron, ElectronApplication, + expect, Locator, Page, } from "@playwright/test"; @@ -20,7 +21,9 @@ const runningApplications: Map = new Map< /** * Starts the electron app and returns the main page after the greeting agent message has been posted. */ -export async function startShell(): Promise { +export async function startShell( + waitForAgentGreeting: boolean = true, +): Promise { // this is needed to isolate these tests session from other concurrently running tests process.env["INSTANCE_NAME"] = `test_${process.env["TEST_WORKER_INDEX"]}_${process.env["TEST_PARALLEL_INDEX"]}`; @@ -56,12 +59,14 @@ export async function startShell(): Promise { const mainWindow: Page = await getMainWindow(app); // wait for agent greeting - await waitForAgentMessage(mainWindow, 30000, 1); + if (waitForAgentGreeting) { + await waitForAgentMessage(mainWindow, 30000, 1, true, ["..."]); + } return mainWindow; } catch (e) { console.warn( - `Unable to start electron application (${process.env["INSTANCE_NAME"]}). Attempt ${retryAttempt} of ${maxRetries}. Error: ${e}`, + `Unable to start electrom application (${process.env["INSTANCE_NAME"]}). Attempt ${retryAttempt} of ${maxRetries}. Error: ${e}`, ); retryAttempt++; @@ -172,13 +177,18 @@ export async function sendUserRequest(prompt: string, page: Page) { */ export async function sendUserRequestFast(prompt: string, page: Page) { const locator: Locator = page.locator("#phraseDiv"); - await locator.waitFor({ timeout: 120000, state: "visible" }); - await locator.fill(prompt, { timeout: 30000 }); + await locator.waitFor({ timeout: 5000, state: "visible" }); + await locator.fill(prompt, { timeout: 5000 }); page.keyboard.down("Enter"); } /** - * Submits a user request to the system via the chat input box and then waits for the agent's response + * Submits a user request to the system via the chat input box and then waits for the first available response + * NOTE: If your expected response changes or you invoke multi-action flow you should be calling + * sendUserRequestAndAwaitSpecificResponse() instead of this call + * + * Remarks: Use this method when calling @commands...agent calls should use aforementioned function. + * * @param prompt The user request/prompt. * @param page The main page from the electron host application. */ @@ -187,7 +197,7 @@ export async function sendUserRequestAndWaitForResponse( page: Page, ): Promise { const locators: Locator[] = await page - .locator(".chat-message-agent-text") + .locator(".chat-message-agent .chat-message-content") .all(); // send the user request @@ -197,19 +207,80 @@ export async function sendUserRequestAndWaitForResponse( await waitForAgentMessage(page, 30000, locators.length + 1); // return the response - return await getLastAgentMessage(page); + return await getLastAgentMessageText(page); +} + +/** + * Submits a user request and awaits for completion of the response. + * + * Remarks: Call this function when expecting an agent action response. + * + * @param prompt The user request/prompt. + * @param page The page hosting the user shell + */ +export async function sendUserRequestAndWaitForCompletion( + prompt: string, + page: Page, +): Promise { + // TODO: implement + const locators: Locator[] = await page + .locator(".chat-message-agent .chat-message-content") + .all(); + + // send the user request + await sendUserRequest(prompt, page); + + // wait for agent response + await waitForAgentMessage(page, 30000, locators.length + 1, true); + + // return the response + return await getLastAgentMessageText(page); } /** * Gets the last agent message from the chat view * @param page The main page from the electron host application. */ -export async function getLastAgentMessage(page: Page): Promise { +export async function getLastAgentMessageText(page: Page): Promise { const locators: Locator[] = await page - .locator(".chat-message-agent-text") + .locator(".chat-message-agent .chat-message-content") .all(); - return locators[0].innerText(); + return await locators[0].innerText(); +} + +/** + * Gets the last agent message from the chat view + * @param page The maing page from the electron host application. + */ +export async function getLastAgentMessage(page: Page): Promise { + const locators: Locator[] = await page + .locator(".chat-message-container-agent") + .all(); + + return locators[0]; +} + +/** + * Determines if the supplied agent message/action has been completed + * + * @param msg The agent message to check for completion + */ +export async function isMessageCompleted(msg: Locator): Promise { + // Agent message is complete once the metrics have been reported + try { + const details: Locator = await msg.locator(".metrics-details", { + hasText: "Total", + }); + + if ((await details.count()) > 0) { + return true; + } + } catch (e) { + // not found + } + + return false; } /** @@ -217,36 +288,59 @@ export async function getLastAgentMessage(page: Page): Promise { * @param page The page where the chatview is hosted * @param timeout The maximum amount of time to wait for the agent message * @param expectedMessageCount The expected # of agent messages at this time. - * @returns When the expected # of messages is reached or the timeout is reached. Whichever occurs first. + * @param waitForMessageCompletion A flag indicating if we should block util the message is completed. + * @param ignore A list of messges that this method will consider noise and will reject as false positivies + * i.e. [".."] and this method will ignore agent messages that are "..." and will continue waiting. + * This is useful when an agent sends status messages. + * + * @returns When the expected # of messages is reached or the timeout is reached. Whichever occurrs first. */ export async function waitForAgentMessage( page: Page, timeout: number, - expectedMessageCount?: number | undefined, + expectedMessageCount: number, + waitForMessageCompletion: boolean = false, + ignore: string[] = [], ): Promise { let timeWaited = 0; let locators: Locator[] = await page - .locator(".chat-message-agent-text") + .locator(".chat-message-container-agent") .all(); let originalAgentMessageCount = locators.length; let messageCount = originalAgentMessageCount; - if (expectedMessageCount == messageCount) { - return; - } - do { + if ( + expectedMessageCount == messageCount && + (!waitForMessageCompletion || + (await isMessageCompleted(await getLastAgentMessage(page)))) + ) { + return; + } + await page.waitForTimeout(1000); timeWaited += 1000; - locators = await page.locator(".chat-message-agent-text").all(); + locators = await page.locator(".chat-message-container-agent").all(); messageCount = locators.length; + + // is this message ignorable? + if (locators.length > 0) { + const lastMessage = await getLastAgentMessageText(page); + if (ignore.indexOf(lastMessage) > -1) { + console.log(`Ignore agent message '${lastMessage}'`); + messageCount = originalAgentMessageCount; + } + } } while ( timeWaited <= timeout && messageCount == originalAgentMessageCount ); } +/** + * Deletes test profiles from agent storage + */ export function deleteTestProfiles() { const profileDir = path.join(os.homedir(), ".typeagent", "profiles"); @@ -263,3 +357,54 @@ export function deleteTestProfiles() { }); } } + +export type TestCallback = () => void; + +/** + * Encapsulates the supplied method within a startup and shutdown of teh + * shell. Test code executes between them. + */ +export async function runTestCalback(callback: TestCallback): Promise { + // launch the app + const mainWindow: Page = await startShell(); + + // run the supplied function + callback(); + + // close the application + await exitApplication(mainWindow); +} + +/** + * Encapsulates the supplied method within a startup and shutdown of teh + * shell. Test code executes between them. + */ +export async function testUserRequest( + userRequests: string[], + expectedResponses: string[], +): Promise { + if (userRequests.length != expectedResponses.length) { + throw new Error("Request/Response count mismatch!"); + } + + // launch the app + const mainWindow: Page = await startShell(); + + // issue the supplied requests and check their responses + for (let i = 0; i < userRequests.length; i++) { + const msg = await sendUserRequestAndWaitForCompletion( + userRequests[i], + mainWindow, + 1, + ); + + // verify expected result + expect( + msg, + `Chat agent didn't respond with the expected message. Request: '${userRequests[i]}', Response: '${expectedResponses[i]}'`, + ).toBe(expectedResponses[i]); + } + + // close the application + await exitApplication(mainWindow); +} From 7901207426517bf614bb9e22f64867a5bd4861eb Mon Sep 17 00:00:00 2001 From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com> Date: Mon, 3 Feb 2025 17:57:13 -0800 Subject: [PATCH 16/16] Determining page layout (#661) --- .../browser/src/agent/browserConnector.mts | 6 +- .../src/agent/discovery/actionHandler.mts | 21 +++++ .../discovery/schema/discoveryActions.mts | 6 -- .../src/agent/discovery/schema/pageLayout.mts | 9 ++ .../src/agent/discovery/translator.mts | 86 ++++++++++++++++--- .../browser/src/extension/htmlReducer.ts | 5 ++ .../browser/src/extension/serviceWorker.ts | 7 +- 7 files changed, 119 insertions(+), 21 deletions(-) create mode 100644 ts/packages/agents/browser/src/agent/discovery/schema/pageLayout.mts diff --git a/ts/packages/agents/browser/src/agent/browserConnector.mts b/ts/packages/agents/browser/src/agent/browserConnector.mts index a9090d875..605fe44e5 100644 --- a/ts/packages/agents/browser/src/agent/browserConnector.mts +++ b/ts/packages/agents/browser/src/agent/browserConnector.mts @@ -80,13 +80,17 @@ export class BrowserConnector { return []; } - async getFilteredHtmlFragments(inputHtmlFragments: any[]) { + async getFilteredHtmlFragments( + inputHtmlFragments: any[], + cssSelectorsToKeep: string[], + ) { let htmlFragments: any[] = []; const timeoutPromise = new Promise((f) => setTimeout(f, 5000)); const filterAction = { actionName: "getFilteredHTMLFragments", parameters: { fragments: inputHtmlFragments, + cssSelectorsToKeep: cssSelectorsToKeep, }, }; diff --git a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts index 98f532fba..60c57253c 100644 --- a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts +++ b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts @@ -27,6 +27,9 @@ export async function handleSchemaDiscoveryAction( case "summarizePage": await handleGetPageSummary(action); break; + case "findPageComponents": + await handleGetPageComponents(action); + break; } async function handleFindUserActions(action: any) { @@ -85,5 +88,23 @@ export async function handleSchemaDiscoveryAction( return response.data; } + async function handleGetPageComponents(action: any) { + const htmlFragments = await browser.getHtmlFragments(); + const timerName = `Getting page layout`; + console.time(timerName); + const response = await agent.getPageLayout(undefined, htmlFragments); + + if (!response.success) { + console.error("Attempt to get page layout failed"); + console.error(response.message); + return; + } + + console.timeEnd(timerName); + message = "Page layout: \n" + JSON.stringify(response.data, null, 2); + + return response.data; + } + return message; } diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts b/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts index 1ebc8189d..e45ab0de1 100644 --- a/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts +++ b/ts/packages/agents/browser/src/agent/discovery/schema/discoveryActions.mts @@ -7,16 +7,10 @@ export type FindPageComponents = { export type FindUserActions = { actionName: "findUserActions"; - parameters: { - allowDuplicates?: boolean; - }; }; export type SummarizePage = { actionName: "summarizePage"; - parameters: { - allowDuplicates?: boolean; - }; }; export type SaveUserActions = { diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/pageLayout.mts b/ts/packages/agents/browser/src/agent/discovery/schema/pageLayout.mts new file mode 100644 index 000000000..f26e66281 --- /dev/null +++ b/ts/packages/agents/browser/src/agent/discovery/schema/pageLayout.mts @@ -0,0 +1,9 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +export type PageLayout = { + headerCSSSelector: string; + footerCSSSelector: string; + navigationLinksCSSSelector: string; + mainContentCSSSelector: string; +}; diff --git a/ts/packages/agents/browser/src/agent/discovery/translator.mts b/ts/packages/agents/browser/src/agent/discovery/translator.mts index 8fe660591..ba5071389 100644 --- a/ts/packages/agents/browser/src/agent/discovery/translator.mts +++ b/ts/packages/agents/browser/src/agent/discovery/translator.mts @@ -79,17 +79,18 @@ function getScreenshotPromptSection( url: screenshot, }, }); - } - if (fragments) { - const textFragments = fragments.map((a) => a.text); - screenshotSection.push({ - type: "text", - text: `Here is the text content of the page + + if (fragments) { + const textFragments = fragments.map((a) => a.text); + screenshotSection.push({ + type: "text", + text: `Here is the text content of the page ''' ${textFragments} ''' `, - }); + }); + } } return screenshotSection; } @@ -176,7 +177,6 @@ export class SchemaDiscoveryAgent { requestSection.push({ type: "text", text: ` - Here is user request ''' ${userRequest} @@ -266,7 +266,6 @@ export class SchemaDiscoveryAgent { requestSection.push({ type: "text", text: ` - Here is user request ''' ${userRequest} @@ -366,7 +365,74 @@ export class SchemaDiscoveryAgent { type: "text", text: ` Examine the layout information provided and determine the content of the page and the actions users can take on it. - Once you have this list, a SINGLE "PageDescription" response using the typescript schema below. + Once you have this list, a SINGLE "${bootstrapTranslator.validator.getTypeName()}" response using the typescript schema below. + + ''' + ${bootstrapTranslator.validator.getSchemaText()} + ''' + `, + }, + ...requestSection, + { + type: "text", + text: ` + The following is the COMPLETE JSON response object with 2 spaces of indentation and no properties with the value undefined: + `, + }, + ]; + + const response = await bootstrapTranslator.translate("", [ + { role: "user", content: JSON.stringify(promptSections) }, + ]); + return response; + } + + async getPageLayout( + userRequest?: string, + fragments?: HtmlFragments[], + screenshot?: string, + ) { + const packageRoot = path.join("..", "..", ".."); + const resultsSchema = await fs.promises.readFile( + fileURLToPath( + new URL( + path.join(packageRoot, "./src/agent/discovery/schema/PageLayout.mts"), + import.meta.url, + ), + ), + "utf8", + ); + + const bootstrapTranslator = this.getBootstrapTranslator( + "PageLayout", + resultsSchema, + ); + + const screenshotSection = getScreenshotPromptSection(screenshot, fragments); + const htmlSection = getHtmlPromptSection(fragments); + const prefixSection = getBootstrapPrefixPromptSection(); + let requestSection = []; + if (userRequest) { + requestSection.push({ + type: "text", + text: ` + + Here is user request + ''' + ${userRequest} + ''' + `, + }); + } + const promptSections = [ + ...prefixSection, + ...screenshotSection, + ...htmlSection, + { + type: "text", + text: ` + Examine the layout information provided and determine the content of the page and the actions users can take on it. + Once you have this list, a SINGLE "${bootstrapTranslator.validator.getTypeName()}" response using the typescript schema below. ''' ${bootstrapTranslator.validator.getSchemaText()} diff --git a/ts/packages/agents/browser/src/extension/htmlReducer.ts b/ts/packages/agents/browser/src/extension/htmlReducer.ts index aad29703a..ad4578e3b 100644 --- a/ts/packages/agents/browser/src/extension/htmlReducer.ts +++ b/ts/packages/agents/browser/src/extension/htmlReducer.ts @@ -46,6 +46,7 @@ export class HTMLReducer { "nocontent", "noscript", "template", + "img", ]; mediaElementSelectors: string[] = [ @@ -68,6 +69,10 @@ export class HTMLReducer { "clickid", "fetchpriority", "srcset", + "aria-busy", + "aria-haspopup", + "aria-autocomplete", + "href", ]; attribsToReplace: Set = new Set(["href", "src"]); diff --git a/ts/packages/agents/browser/src/extension/serviceWorker.ts b/ts/packages/agents/browser/src/extension/serviceWorker.ts index ad27d37a3..7eb21a2cb 100644 --- a/ts/packages/agents/browser/src/extension/serviceWorker.ts +++ b/ts/packages/agents/browser/src/extension/serviceWorker.ts @@ -730,6 +730,7 @@ async function getTabHTMLFragmentsBySize( async function getFilteredHTMLFragments( targetTab: chrome.tabs.Tab, inputHtmlFragments: any[], + cssSelectorsToKeep: string[], ) { let htmlFragments: any[] = []; @@ -740,10 +741,7 @@ async function getFilteredHTMLFragments( { type: "get_filtered_html_fragments", inputHtml: inputHtmlFragments[i].content, - cssSelectors: [ - inputHtmlFragments[i].cssSelectorAcross, - inputHtmlFragments[i].cssSelectorDown, - ].join(", "), + cssSelectors: cssSelectorsToKeep.join(", "), frameId: inputHtmlFragments[i].frameId, }, { frameId: inputHtmlFragments[i].frameId }, @@ -1165,6 +1163,7 @@ async function runBrowserAction(action: any) { responseObject = await getFilteredHTMLFragments( targetTab, action.parameters.fragments, + action.parameters.cssSelectorsToKeep, ); break; }