Dispatcher Image tests (#771)

FYI - @curtisman

ts/.vscode/settings.json

@@ -24,6 +24,7 @@
         {"name": "cache", "rootPath": "packages/cache"},
         {"name": "commonUtils", "rootPath": "packages/commonUtils"},
         {"name": "dispatcher", "rootPath": "packages/dispatcher"},
+        {"name": "defaultAgentProvider", "rootPath": "packages/defaultAgentProvider"},
         {"name": "knowledgeProcessor", "rootPath": "packages/knowledgeProcessor"},
         {"name": "memoryProviders", "rootPath": "examples/memoryProviders"},
         {"name": "typeagent", "rootPath": "packages/typeagent"}

ts/packages/agents/chat/src/chatResponseActionSchema.ts

@@ -54,13 +54,13 @@ export type TermFilter = {
     timeRange?: DateTimeRange | undefined; // in this time range
 };
 
-// this action is used to lookup information and stored attachments from past conversations or the internet and generate a response based on the lookup results, for example "what did we say about the project last week?" or "what is the current price of Microsoft stock?"
+// this action is used to lookup information images the user has previously shared from past conversations or the internet and generate a response based on the lookup results, for example "what did we say about the project last week?" or "what is the current price of Microsoft stock?"
 export interface LookupAndGenerateResponseAction {
     actionName: "lookupAndGenerateResponse";
     parameters: {
         // the original request from the user
         originalRequest: string;
-        // if the request is for private information from past conversations including private events, plans, projects in progress, files or file names, and other items from discussions with team members or the assistant, use the conversation lookup filters
+        // if the request is for private information from past conversations including private events, plans, projects in progress, attachments, files, file names, and other items from discussions with team members or the assistant, use the conversation lookup filters
         conversationLookupFilters?: TermFilter[];
         // if the request is for contemporary internet information including sports scores, news events, or current commerce offerings, use the lookups parameter to request a lookup of the information on the user's behalf; the assistant will generate a response based on the lookup results
         // Lookup *facts* you don't know or if your facts are out of date.
@@ -70,7 +70,7 @@ export interface LookupAndGenerateResponseAction {
         internetLookups?: string[];
         // Any file references to images referred to by the message
         relatedFiles?: string[];
-        // Are the contents of the files needed at this time?
+        // Are the contents of the files needed at this time? (i.e. does the user want to see an image or picture)
         retrieveRelatedFilesFromStorage?: boolean;
     };
 }

ts/packages/agents/chat/src/chatResponseHandler.ts

@@ -111,16 +111,19 @@ async function handleChatResponse(
                         "...",
                 );
 
-                const needDisplay = context.streamingContext !== generatedText;
-                const result = needDisplay
-                    ? createActionResult(generatedText, true)
-                    : createActionResultNoDisplay(generatedText);
+                const needDisplay = context.streamingContext !== generatedText; //|| generateResponseAction.parameters.showImageToUser;
+                let result;
+                if (needDisplay) {
+                    result = createActionResult(generatedText, true);
+                } else {
+                    result = createActionResultNoDisplay(generatedText);
+                }
 
                 let entities = parameters.generatedTextEntities || [];
                 if (parameters.userRequestEntities !== undefined) {
-                    entities = parameters.userRequestEntities.concat(entities);
+                    result.entities =
+                        parameters.userRequestEntities.concat(entities);
                 }
-                result.entities = entities;
 
                 if (
                     generateResponseAction.parameters.relatedFiles !== undefined
@@ -139,7 +142,7 @@ async function handleChatResponse(
                     }
 
                     logEntities("File Entities:", fileEntities);
-                    result.entities.concat(fileEntities);
+                    result.entities = result.entities.concat(fileEntities);
                 }
 
                 return result;
@@ -224,6 +227,13 @@ async function handleChatResponse(
                 } else {
                     console.log("Conversation manager is undefined!");
                 }
+            } else if (
+                lookupAction.parameters.retrieveRelatedFilesFromStorage &&
+                lookupAction.parameters.relatedFiles !== undefined
+            ) {
+                return createActionResultFromHtmlDisplay(
+                    `<div>${await rehydrateImages(context, lookupAction.parameters.relatedFiles)}</div>`,
+                );
             }
         }
     }

ts/packages/commonUtils/src/jsonTranslator.ts

@@ -170,8 +170,17 @@ async function attachAttachments(
     if (attachments && attachments.length > 0 && pp) {
         for (let i = 0; i < attachments.length; i++) {
             pp.unshift(
-                (await addImagePromptContent("user", attachments[i]))
-                    .promptSection!,
+                (
+                    await addImagePromptContent(
+                        "user",
+                        attachments[i],
+                        true,
+                        true,
+                        false,
+                        true,
+                        true,
+                    )
+                ).promptSection!,
             );
         }
     }

ts/packages/defaultAgentProvider/test/data/translate-image-history-e2e.json

@@ -0,0 +1,89 @@
+[
+    [
+        {
+            "request": "Tell me about the image in the chat history.",            
+            "action": {
+                "translatorName": "chat",
+                "actionName": "lookupAndGenerateResponse",
+                "parameters": {
+                    "originalRequest": "Tell me about the image in the chat history.",
+                    "conversationLookupFilters": [
+                      {
+                        "terms": [
+                          "image"
+                        ]
+                      }
+                    ]
+                }
+            },
+            "history": {
+              "text": "It's an image!",
+              "source": "chat",
+              "entities": [
+                {
+                  "name": "image",
+                  "type": [
+                    "object"
+                  ]
+                },
+                {
+                  "name": "non-existent image",
+                  "type": [
+                    "object"
+                  ]
+                },
+                {
+                  "name": "attachment__0.png",
+                  "type": [
+                    "file",
+                    "image",
+                    "data"
+                  ]
+                }
+              ]
+            },
+            "attachments": [ "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAABhSURBVDhPY6AdqJ9vwFA/5wAEA9kkAbDmuR8YGub+B2MQm2hD0DWTZAiyZnQXEGUIyL/ICuEGIBsMVIMT1M+dgBJoMANAAGwISA6ohmiAbABZYNSAwWBA/dwFYEw7wMAAAAOtdVt9DPxQAAAAAElFTkSuQmCC" ]
+        },
+        {
+          "request": "show me attachment__0.png",
+          "action" : {
+            "translatorName": "chat",
+            "actionName": "lookupAndGenerateResponse",
+            "parameters": {
+              "originalRequest": "show me attachment__0.png",
+              "relatedFiles": [
+                "attachment__0.png"
+              ],
+              "retrieveRelatedFilesFromStorage": true            
+            },
+            "entities": {
+              "relatedFiles": [
+                {
+                  "name": "attachment__0.png",
+                  "type": [
+                    "file",
+                    "image",
+                    "data"
+                  ],
+                  "sourceAppAgentName": "chat"
+                }
+              ]
+            }
+          },
+          "history": {
+            "text":"Action chat.lookupAndGenerateResponse completed.",
+            "source": "chat",
+            "entities": [
+                {
+                  "name": "attachment__0.png",
+                  "type": [
+                    "file",
+                    "image",
+                    "data"
+                  ]
+                }
+            ]
+          }
+        }
+    ]
+]

ts/packages/defaultAgentProvider/test/translate.test.ts

@@ -1,6 +1,14 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+/*
+ * NOTE: This test has the extension .test.ts and not .spec.ts.
+ * *.test.ts files are run under test:live && test:live:debug
+ * project settings (see ../package.json).  The assumption is
+ * test:live has API endpoints where as test:local tests run
+ * wholly locally.
+ */
+
 import { defineTranslateTest } from "./translateTestCommon.js";
 const dataFiles = ["test/data/translate-e2e.json"];
 

ts/packages/defaultAgentProvider/test/translateImageConversation.test.ts

@@ -0,0 +1,8 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { defineTranslateTest } from "./translateTestCommon.js";
+const dataFiles = ["test/data/translate-image-history-e2e.json"];
+import registerDebug from "debug";
+registerDebug.enable("*");
+await defineTranslateTest("translate image request (w/history)", dataFiles);

ts/packages/defaultAgentProvider/test/translateTestCommon.ts

@@ -17,6 +17,7 @@ type TranslateTestStep = {
     match?: "exact" | "partial"; // default to "exact"
     // History insertion after translation (if any)
     history?: ChatHistoryInput | ChatHistoryInput[];
+    attachments?: string[] | undefined;
 };
 type TranslateTestEntry = TranslateTestStep | TranslateTestStep[];
 type TranslateTestFile = TranslateTestEntry[];
@@ -79,9 +80,14 @@ export async function defineTranslateTest(name: string, dataFiles: string[]) {
             await Promise.all(
                 dispatchers.map(async (dispatcher) => {
                     for (const step of steps) {
-                        const { request, action, match, history } = step;
+                        const { request, action, match, history, attachments } =
+                            step;
 
-                        const result = await dispatcher.processCommand(request);
+                        const result = await dispatcher.processCommand(
+                            request,
+                            undefined,
+                            attachments,
+                        );
                         expect(result?.hasError).toBeFalsy();
 
                         const actions = result?.actions;

ts/packages/defaultAgentProvider/test/translate_history.test.ts

@@ -1,6 +1,14 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+/*
+ * NOTE: This test has the extension .test.ts and not .spec.ts.
+ * *.test.ts files are run under test:live && test:live:debug
+ * project settings (see ../package.json).  The assumption is
+ * test:live has API endpoints where as test:local tests run
+ * wholly locally.
+ */
+
 import { defineTranslateTest } from "./translateTestCommon.js";
 const dataFiles = ["test/data/translate-history-e2e.json"];
 

ts/packages/dispatcher/src/context/dispatcher/handlers/requestCommandHandler.ts

@@ -313,7 +313,10 @@ export class RequestCommandHandler implements CommandHandler {
 
             // store attachments for later reuse
             const cachedAttachments: CachedImageWithDetails[] = [];
-            if (attachments) {
+            if (
+                attachments &&
+                systemContext.session.sessionDirPath !== undefined
+            ) {
                 for (let i = 0; i < attachments?.length; i++) {
                     const [attachmentName, tags]: [string, ExifReader.Tags] =
                         await systemContext.session.storeUserSuppliedFile(

ts/packages/shell/src/renderer/assets/styles.less

@@ -482,22 +482,6 @@ table.table-message td {
   font-weight: bolder;
 }
 
-.agent-name[action-data]:hover:after {
-  content: attr(action-data);
-  position: absolute;
-  top: 30px;
-  left: 10px;
-  white-space: pre-wrap;
-  background-color: #86d9fd;
-  color: gray;
-  border: 1px solid #d6f3ff;
-  border-radius: 5px;
-  padding: 5px;
-  font-size: 100%;
-  font-family: "Courier New", Courier, monospace;
-  z-index: 100;
-}
-
 .timeString {
   font-style: normal;
 }