Sidepanel: Converting intent schema response to typescript (#782)

microsoft · Mar 4, 2025 · 0c8e2b9 · 0c8e2b9
1 parent 9ccba3b
commit 0c8e2b9
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 7 deletions.
diff --git a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts
@@ -7,8 +7,11 @@ import { BrowserConnector } from "../browserConnector.mjs";
 import { createDiscoveryPageTranslator } from "./translator.mjs";
 import {
     ActionSchemaTypeDefinition,
+    ActionSchemaObject,
+    ActionParamType,
     generateActionSchema,
     parseActionSchemaSource,
+    generateSchemaTypeDefinition,
 } from "action-schema";
 import { ActionSchemaCreator as sc } from "action-schema";
 import path from "path";
@@ -18,6 +21,7 @@ import { UserActionsList } from "./schema/userActionsPool.mjs";
 import { PageDescription } from "./schema/pageSummary.mjs";
 import { createTempAgentForSchema } from "./tempAgentActionHandler.mjs";
 import { SchemaDiscoveryActions } from "./schema/discoveryActions.mjs";
+import { UserIntent } from "./schema/recordedActions.mjs";
 
 export async function handleSchemaDiscoveryAction(
     action: SchemaDiscoveryActions,
@@ -271,6 +275,48 @@ export async function handleSchemaDiscoveryAction(
         return response.data;
     }
 
+    async function getIntentSchemaFromJSON(
+        userIntentJson: UserIntent,
+        actionDescription: string,
+    ) {
+        let fields: Map<string, any> = new Map<string, any>();
+
+        userIntentJson.parameters.forEach((p) => {
+            let t: ActionParamType = sc.string();
+            switch (p.type) {
+                case "string":
+                    t = sc.string();
+                    break;
+                case "number":
+                    t = sc.number();
+                    break;
+                case "boolean":
+                    t = sc.number();
+                    break;
+            }
+
+            if (p.required && !p.defaultValue) {
+                fields.set(p.shortName, sc.field(t, p.description));
+            } else {
+                fields.set(p.shortName, sc.optional(t, p.description));
+            }
+        });
+
+        const obj: ActionSchemaObject = sc.obj({
+            actionName: sc.string(userIntentJson.actiontName),
+            parameters: sc.obj(Object.fromEntries(fields)),
+        } as const);
+
+        const schema = sc.type(
+            userIntentJson.actiontName,
+            obj,
+            actionDescription,
+            true,
+        );
+
+        return await generateSchemaTypeDefinition(schema, { exact: true });
+    }
+
     async function handleGetIntentFromReccording(action: any) {
         const timerName = `Getting intent schema`;
         console.time(timerName);
@@ -291,8 +337,13 @@ export async function handleSchemaDiscoveryAction(
         }
 
         console.timeEnd(timerName);
-        message =
-            "Intent schema: \n" + JSON.stringify(intentResponse.data, null, 2);
+
+        const intentSchema = await getIntentSchemaFromJSON(
+            intentResponse.data as UserIntent,
+            action.parameters.recordedActionDescription,
+        );
+
+        message = "Intent schema: \n" + intentSchema;
 
         const timerName2 = `Getting action schema`;
         console.time(timerName2);
@@ -318,7 +369,8 @@ export async function handleSchemaDiscoveryAction(
         console.timeEnd(timerName2);
 
         return {
-            intent: intentResponse.data,
+            intent: intentSchema,
+            intentJson: intentResponse.data,
             actions: stepsResponse.data,
         };
     }

diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts b/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts
@@ -7,6 +7,7 @@ export type UserIntentParameter = {
     shortName: string;
     // a longer, descriptive name for the parameter. This value can contain non-alphanumeric characters
     name: string;
+    // The valid values are "string", "number" and "boolean"
     type: string;
     // The default value for the parameter. If this value is set based on a HTML
     // page, check whether the target element has a default value. For dropdown elements, use the

diff --git a/ts/packages/agents/browser/src/extension/serviceWorker.ts b/ts/packages/agents/browser/src/extension/serviceWorker.ts
@@ -1642,7 +1642,7 @@ chrome.runtime.onMessage.addListener(
                     const data =
                         await chrome.storage.local.get("recordedActions");
                     if (data) {
-                        chrome.storage.local.remove("recordedActions");
+                        await chrome.storage.local.remove("recordedActions");
                     }
                     sendResponse({});
                     break;

diff --git a/ts/packages/agents/browser/src/extension/sidepanel.ts b/ts/packages/agents/browser/src/extension/sidepanel.ts
@@ -357,10 +357,8 @@ function renderTimeline(action: any, index: number) {
         }
 
         const card = document.createElement("div");
-        const intentSchema = JSON.stringify(response.intent, null, 2);
-
         card.innerHTML = `        
-            <pre class="card-text"><code class="language-json">${intentSchema}</code></pre>
+            <pre class="card-text"><code class="language-json">${response.intent}</code></pre>
         `;
 
         intentViewContainer.replaceChildren(card);