Merge remote-tracking branch 'origin' into dev/robgruen/android

dotnet/autoShell/AutoShell.cs

@@ -361,6 +361,18 @@ static void CloseApplication(string friendlyName)
             }
         }
 
+        [DllImport("user32.dll", CharSet = CharSet.Auto)]
+        private static extern int SystemParametersInfo(int uAction, int uParam, string lpvParam, int fuWinIni);
+
+        private const int SPI_SETDESKWALLPAPER = 20;
+        private const int SPIF_UPDATEINIFILE = 0x01;
+        private const int SPIF_SENDCHANGE = 0x02;
+
+        private static void SetDesktopWallpaper(string imagePath)
+        {
+            SystemParametersInfo(SPI_SETDESKWALLPAPER, 0, imagePath, SPIF_UPDATEINIFILE | SPIF_SENDCHANGE);
+        }
+
         static bool execLine(string line)
         {
             var quit = false;
@@ -418,6 +430,9 @@ static bool execLine(string line)
                         var installedApps = GetAllInstalledAppsIds();
                         Console.WriteLine(JsonConvert.SerializeObject(installedApps.Keys));
                         break;
+                    case "setWallpaper":
+                        SetDesktopWallpaper(value);
+                        break;
                     default:
                         Debug.WriteLine("Unknown command: " + key);
                         break;

dotnet/autoShell/autoShell.csproj

@@ -8,10 +8,11 @@
     <OutputType>Exe</OutputType>
     <RootNamespace>autoShell</RootNamespace>
     <AssemblyName>autoShell</AssemblyName>
-    <TargetFrameworkVersion>v4.8.1</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.8</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
     <Deterministic>true</Deterministic>
+    <TargetFrameworkProfile />
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <PlatformTarget>AnyCPU</PlatformTarget>

ts/packages/agentSdk/src/action.ts

@@ -20,6 +20,7 @@ export type ActionResultSuccessNoDisplay = {
     entities: Entity[];
     dynamicDisplayId?: string | undefined;
     dynamicDisplayNextRefreshMs?: number | undefined;
+    additionalInstructions?: string[] | undefined;
     error?: undefined;
 };
 
@@ -29,6 +30,7 @@ export type ActionResultSuccess = {
     entities: Entity[];
     dynamicDisplayId?: string | undefined;
     dynamicDisplayNextRefreshMs?: number | undefined;
+    additionalInstructions?: string[] | undefined;
     error?: undefined;
 };
 

ts/packages/agents/desktop/src/actionHandler.ts

@@ -46,6 +46,7 @@ async function executeDesktopAction(
     const message = await runDesktopActions(
         action as DesktopActions,
         context.sessionContext.agentContext,
+        context.sessionContext.sessionStorage!,
     );
     return createActionResult(message);
 }

ts/packages/agents/desktop/src/actionsSchema.ts

@@ -10,7 +10,8 @@ export type DesktopActions =
     | SwitchToWindowAction
     | SetVolumeAction
     | RestoreVolumeAction
-    | MuteVolumeAction;
+    | MuteVolumeAction
+    | SetWallpaperAction;
 
 // Launches a new program window on a Windows Desktop
 // Example:
@@ -88,6 +89,13 @@ export type MuteVolumeAction = {
     };
 };
 
+export type SetWallpaperAction = {
+    actionName: "setWallpaper";
+    parameters: {
+        filePath?: string; // The path to the file
+        url?: string; // The url to the image
+    };
+};
 export type KnownPrograms =
     | "chrome"
     | "word"

ts/packages/agents/desktop/src/connector.ts

@@ -7,6 +7,10 @@ import { ProgramNameIndex, loadProgramNameIndex } from "./programNameIndex.js";
 import { Storage } from "@typeagent/agent-sdk";
 import registerDebug from "debug";
 import { DesktopActions } from "./actionsSchema.js";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { downloadImage } from "common-utils";
 
 const debug = registerDebug("typeagent:desktop");
 const debugData = registerDebug("typeagent:desktop:data");
@@ -63,11 +67,72 @@ async function ensureAutomationProcess(agentContext: DesktopActionContext) {
 export async function runDesktopActions(
     action: DesktopActions,
     agentContext: DesktopActionContext,
+    sessionStorage: Storage,
 ) {
     let confirmationMessage = "OK";
     let actionData = "";
     const actionName = action.actionName;
     switch (actionName) {
+        case "setWallpaper": {
+            let file = action.parameters.filePath;
+            const rootTypeAgentDir = path.join(os.homedir(), ".typeagent");
+
+            // if the requested image a URL, then download it
+            if (action.parameters.url !== undefined) {
+                file = `../downloaded_images/${path.basename(action.parameters.url)}`;
+                if (path.extname(file).length == 0) {
+                    file += ".png";
+                }
+                if (
+                    await downloadImage(
+                        action.parameters.url,
+                        file,
+                        sessionStorage!,
+                    )
+                ) {
+                    file = file.substring(3);
+                } else {
+                    confirmationMessage =
+                        "Failed to dowload the requested image.";
+                    break;
+                }
+            }
+
+            if (file !== undefined) {
+                if (
+                    file.startsWith("/") ||
+                    file.indexOf(":") == 2 ||
+                    fs.existsSync(file)
+                ) {
+                    actionData = file;
+                } else {
+                    // if the file path is relative we'll have to search for the image since we don't have root storage dir
+                    // TODO: add shared agent storage or known storage location (requires permissions, trusted agents, etc.)
+                    const files = fs
+                        .readdirSync(rootTypeAgentDir, { recursive: true })
+                        .filter((allFilesPaths) =>
+                            (allFilesPaths as string).endsWith(
+                                path.basename(file),
+                            ),
+                        );
+
+                    if (files.length > 0) {
+                        actionData = path.join(
+                            rootTypeAgentDir,
+                            files[0] as string,
+                        );
+                    } else {
+                        actionData = file;
+                    }
+                }
+            } else {
+                confirmationMessage = "Unknown wallpaper location.";
+                break;
+            }
+
+            confirmationMessage = "Set wallpaper to " + actionData;
+            break;
+        }
         case "launchProgram": {
             actionData = await mapInputToAppName(
                 action.parameters.name,

ts/packages/agents/image/src/imageActionHandler.ts

@@ -8,19 +8,20 @@ import {
     ActionResult,
     ActionResultSuccess,
 } from "@typeagent/agent-sdk";
-import { StopWatch } from "common-utils";
+import { downloadImage, StopWatch } from "common-utils";
 import {
     createActionResult,
     createActionResultFromHtmlDisplayWithScript,
 } from "@typeagent/agent-sdk/helpers/action";
 import { bing, GeneratedImage, openai } from "aiclient";
 import { Image } from "../../../aiclient/dist/bing.js";
-import { randomBytes } from "crypto";
+import { randomBytes, randomUUID } from "crypto";
 import {
     CreateImageAction,
     FindImageAction,
     ImageAction,
 } from "./imageActionSchema.js";
+import path from "path";
 
 export function instantiate(): AppAgent {
     return {
@@ -75,11 +76,6 @@ async function handlePhotoAction(
                 result = createActionResult(
                     `Unable to find any images for ${findImageAction.parameters.searchTerm}`,
                 );
-                // } else if (searchResults.length == 1) {
-                //     result = createActionResultFromHtmlDisplay(
-                //         `<img class="chat-input-image" src="${searchResults[0].contentUrl}" />`,
-                //         "Found 1 image.",
-                //     );
             } else {
                 const urls: string[] = [];
                 const captions: string[] = [];
@@ -88,6 +84,15 @@ async function handlePhotoAction(
                     captions.push(findImageAction.parameters.searchTerm);
                 });
                 result = createCarouselForImages(urls, captions);
+
+                // add the found images to the entities
+                for (let i = 0; i < searchResults.length; i++) {
+                    result.entities.push({
+                        name: path.basename(searchResults[i].contentUrl),
+                        type: ["image", "url", "search"],
+                        additionalEntityText: searchResults[i].contentUrl,
+                    });
+                }
             }
             break;
         }
@@ -141,6 +146,23 @@ async function handlePhotoAction(
                     captions.push(i.revised_prompt);
                 });
                 result = createCarouselForImages(urls, captions);
+
+                // save the generated image in the session store and add the image to the knoweledge store
+                const id = randomUUID();
+                const fileName = `../generated_images/${id.toString()}.png`;
+                if (
+                    await downloadImage(
+                        urls[0],
+                        fileName,
+                        photoContext.sessionContext.sessionStorage!,
+                    )
+                ) {
+                    // add the generated image to the entities
+                    result.entities.push({
+                        name: fileName.substring(3),
+                        type: ["file", "image", "ai_generated"],
+                    });
+                }
             }
             break;
         default:
@@ -153,6 +175,7 @@ function createCarouselForImages(
     images: string[],
     captions: string[],
 ): ActionResultSuccess {
+    let literal: string = `There are ${images.length} shown. `;
     const hash: string = randomBytes(4).readUInt32LE(0).toString();
     const jScript: string = `
     <script>
@@ -216,6 +239,8 @@ function createCarouselForImages(
     </div>`;
 
         carouselDots += `<span class="dot ${hash}" onclick="slideShow_${hash}.currentSlide(${index + 1})"></span>`;
+
+        literal += `Image ${index + 1}: ${url}, Caption: ${captions[index]} `;
     });
 
     const carousel_end: string = `
@@ -233,6 +258,6 @@ function createCarouselForImages(
 
     return createActionResultFromHtmlDisplayWithScript(
         carousel_start + carousel + carousel_end + jScript,
-        `There are ${images.length} shown.`,
+        literal,
     );
 }

ts/packages/cache/src/explanation/requestAction.ts

@@ -6,6 +6,7 @@ import { AppAction, Entity } from "@typeagent/agent-sdk";
 export type HistoryContext = {
     promptSections: PromptSection[];
     entities: Entity[];
+    additionalInstructions?: string[] | undefined;
 };
 
 export function normalizeParamValue(value: ParamValueType) {

ts/packages/commonUtils/src/image.ts

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import { Storage } from "@typeagent/agent-sdk";
+import { getBlob } from "aiclient";
 import ExifReader from "exifreader";
 
 export class CachedImageWithDetails {
@@ -30,3 +32,28 @@ export function extractRelevantExifTags(exifTags: ExifReader.Tags) {
     console.log(tags.replace("\n\n", "\n"));
     return tags;
 }
+
+/**
+ * Dowloads the supplied uri and saves it to local session storage
+ * @param uri The uri of the image to download
+ * @param fileName The name of the file to save the image locally as (including relative path)
+ */
+export async function downloadImage(
+    uri: string,
+    fileName: string,
+    storage: Storage,
+): Promise<boolean> {
+    return new Promise<boolean>(async (resolve) => {
+        // save the generated image in the session store
+        const blobResponse = await getBlob(uri);
+        if (blobResponse.success) {
+            const ab = Buffer.from(await blobResponse.data.arrayBuffer());
+
+            storage.write(fileName, ab.toString("base64"));
+
+            resolve(true);
+        }
+
+        resolve(false);
+    });
+}

ts/packages/commonUtils/src/indexNode.ts

@@ -44,12 +44,13 @@ export * from "./profiler/profileReader.js";
 
 export { createRpc } from "./rpc.js";
 
-export { CachedImageWithDetails, getImageElement } from "./image.js";
+export * from "./image.js";
 
 export {
     getFileExtensionForMimeType,
     getMimeTypeFromFileExtension as getMimeType,
-    isMimeTypeSupported,
+    isImageMimeTypeSupported,
+    isImageFileType,
 } from "./mimeTypes.js";
 
 export { getObjectProperty, setObjectProperty } from "./objectProperty.js";

ts/packages/commonUtils/src/mimeTypes.ts

@@ -38,7 +38,7 @@ export function getMimeTypeFromFileExtension(fileExtension: string): string {
     throw "Unsupported file extension.";
 }
 
-export function isMimeTypeSupported(mime: string): boolean {
+export function isImageMimeTypeSupported(mime: string): boolean {
     switch (mime) {
         case "image/png":
         case "image/jpg":
@@ -48,3 +48,13 @@ export function isMimeTypeSupported(mime: string): boolean {
             return false;
     }
 }
+
+export function isImageFileType(fileExtension: string): boolean {
+    if (fileExtension.startsWith(".")) {
+        fileExtension = fileExtension.substring(1);
+    }
+
+    const imageFileTypes: Set<string> = new Set<string>(["png", "jpg", "jpeg"]);
+
+    return imageFileTypes.has(fileExtension);
+}

ts/packages/dispatcher/src/action/actionHandlers.ts

@@ -270,6 +270,8 @@ async function executeAction(
             result.entities,
             "assistant",
             systemContext.requestId,
+            undefined,
+            result.additionalInstructions,
         );
     }
 

ts/packages/dispatcher/src/action/storageImpl.ts

@@ -13,6 +13,7 @@ import {
     DataProtectionScope,
     PersistenceCreator,
 } from "@azure/msal-node-extensions";
+import { isImageFileType } from "common-utils";
 
 export function getStorage(name: string, baseDir: string): Storage {
     const getFullPath = (storagePath: string) => {
@@ -45,7 +46,16 @@ export function getStorage(name: string, baseDir: string): Storage {
             if (!fs.existsSync(dirName)) {
                 await fs.promises.mkdir(dirName, { recursive: true });
             }
-            return fs.promises.writeFile(fullPath, data);
+
+            // images are passed in as base64 strings so we need to encode them properly on disk
+            if (isImageFileType(path.extname(storagePath))) {
+                return fs.promises.writeFile(
+                    fullPath,
+                    Buffer.from(data, "base64"),
+                );
+            } else {
+                return fs.promises.writeFile(fullPath, data);
+            }
         },
         delete: async (storagePath: string) => {
             const fullPath = getFullPath(storagePath);