web automation: ensure UI automation operations are awaited in conten…

…t script (#804)
microsoft · Mar 9, 2025 · 0360a3f · 0360a3f
1 parent 15dbaca
commit 0360a3f
Show file tree

Hide file tree

Showing 5 changed files with 111 additions and 52 deletions.
diff --git a/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts
@@ -15,7 +15,6 @@ import {
     TextInput,
 } from "./schema/pageComponents.mjs";
 import {
-    PageManipulationActions,
     PageManipulationActionsList,
     UserIntent,
 } from "./schema/recordedActions.mjs";
@@ -149,7 +148,7 @@ export function createTempAgentForSchema(
 
         console.log(`Running ${targetPlan.planName}`);
 
-        targetPlan.steps.forEach(async (step: PageManipulationActions) => {
+        for (const step of targetPlan.steps) {
             switch (step.actionName) {
                 case "ClickOnLink":
                     const linkParameter = targetIntent.parameters.find(
@@ -245,6 +244,6 @@ export function createTempAgentForSchema(
 
                     break;
             }
-        });
+        }
     }
 }
diff --git a/ts/packages/agents/browser/src/extension/contentScript.ts b/ts/packages/agents/browser/src/extension/contentScript.ts
@@ -820,10 +820,47 @@ document.addEventListener("fromPaleoDbAutomation", function (e: any) {
     console.log("received", message);
 });
 
-function sendUIEventsRequest(data: any) {
-    document.dispatchEvent(
-        new CustomEvent("toUIEventsDispatcher", { detail: data }),
-    );
+async function sendUIEventsRequest(message: any) {
+    return new Promise((resolve, reject) => {
+        const requestId = `request_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+
+        const listener = (event: MessageEvent) => {
+            if (event.source !== window) return;
+
+            const data = event.data;
+            if (
+                data &&
+                data.type === "main-world-response" &&
+                data.requestId === requestId
+            ) {
+                window.removeEventListener("message", listener);
+
+                if (data.error) {
+                    reject(new Error(data.error));
+                } else {
+                    resolve(data.result);
+                }
+            }
+        };
+
+        window.addEventListener("message", listener);
+
+        // Send the message with the request ID
+        window.postMessage(
+            {
+                type: "content-script-request",
+                requestId: requestId,
+                payload: message,
+            },
+            "*",
+        );
+
+        // Add a timeout to prevent hanging promises
+        setTimeout(() => {
+            window.removeEventListener("message", listener);
+            reject(new Error("Request to main world timed out"));
+        }, 10000);
+    });
 }
 
 document.addEventListener("fromUIEventsDispatcher", async function (e: any) {
@@ -939,7 +976,7 @@ async function handleScriptAction(
         }
 
         case "run_ui_event": {
-            sendUIEventsRequest(message.action);
+            await sendUIEventsRequest(message.action);
             sendResponse({});
             break;
         }

diff --git a/ts/packages/agents/browser/src/extension/sidepanel.html b/ts/packages/agents/browser/src/extension/sidepanel.html
@@ -244,7 +244,7 @@ <h3>User-Defined Actions</h3>
             <label class="form-label">Steps:</label>
             <textarea
               id="actionStepsDescription"
-              class="form-control"
+              class="form-control hidden"
               rows="5"
               placeholder="Give the steps to perform this action"
             ></textarea>

diff --git a/ts/packages/agents/browser/src/extension/sidepanel.ts b/ts/packages/agents/browser/src/extension/sidepanel.ts
@@ -87,10 +87,9 @@ function renderSchemaResults(schemaActions: any) {
     const itemsList = document.getElementById(
         "detectedSchemaItemsList",
     ) as HTMLElement;
+    itemsList.innerHTML = "";
 
-    if (schemaActions) {
-        itemsList.innerHTML = "";
-
+    if (schemaActions !== undefined && schemaActions.length > 0) {
         schemaActions.forEach((action: any, index: number) => {
             const { actionName, parameters } = action;
             const paramsText = parameters

diff --git a/ts/packages/agents/browser/src/extension/uiEventsDispatcher.ts b/ts/packages/agents/browser/src/extension/uiEventsDispatcher.ts
@@ -296,48 +296,72 @@ async function selectDropdownOption(selector: string, optionLabel: string) {
     }
 }
 
-function sendDataToContentScript(data: any) {
-    document.dispatchEvent(
-        new CustomEvent("fromUIEventsDispatcher", { detail: data }),
-    );
-}
+window.addEventListener("message", async (event: any) => {
+    const data = event.data;
 
-document.addEventListener("toUIEventsDispatcher", async function (e: any) {
-    var message = e.detail;
-    console.log("received", message);
-    const actionName =
-        message.actionName ?? message.fullActionName.split(".").at(-1);
+    // Check if this is a request from our content script
+    if (data && data.type === "content-script-request") {
+        const { requestId, payload } = data;
 
-    if (actionName === "clickOnElement") {
-        clickOnElement(escapeCssSelector(message.parameters.cssSelector));
-    }
-    if (actionName === "enterTextInElement") {
-        await enterTextInElement(
-            message.parameters.value,
-            escapeCssSelector(message.parameters.cssSelector),
-            {
-                delay: 20,
-                clearExisting: true,
-                triggerBlur: true,
-                triggerSubmit: message.parameters.submitForm ?? false,
-            },
-        );
-    }
-    if (actionName === "enterTextOnPage") {
-        // await enterTextOnPage(message.parameters.value.toUpperCase());
-        await enterTextInElement(message.parameters.value, "body", {
-            delay: 20,
-            clearExisting: true,
-            triggerBlur: true,
-            triggerSubmit: message.parameters.submitForm ?? false,
-            enterAtPageScope: true,
-        });
-    }
-    if (actionName === "setDropdownValue") {
-        await selectDropdownOption(
-            escapeCssSelector(message.parameters.cssSelector),
-            message.parameters.optionLabel,
-        );
+        try {
+            var message = payload;
+            console.log("received", message);
+            const actionName =
+                message.actionName ?? message.fullActionName.split(".").at(-1);
+
+            if (actionName === "clickOnElement") {
+                clickOnElement(
+                    escapeCssSelector(message.parameters.cssSelector),
+                );
+            }
+            if (actionName === "enterTextInElement") {
+                await enterTextInElement(
+                    message.parameters.value,
+                    escapeCssSelector(message.parameters.cssSelector),
+                    {
+                        delay: 20,
+                        clearExisting: true,
+                        triggerBlur: true,
+                        triggerSubmit: message.parameters.submitForm ?? false,
+                    },
+                );
+            }
+            if (actionName === "enterTextOnPage") {
+                // await enterTextOnPage(message.parameters.value.toUpperCase());
+                await enterTextInElement(message.parameters.value, "body", {
+                    delay: 20,
+                    clearExisting: true,
+                    triggerBlur: true,
+                    triggerSubmit: message.parameters.submitForm ?? false,
+                    enterAtPageScope: true,
+                });
+            }
+            if (actionName === "setDropdownValue") {
+                await selectDropdownOption(
+                    escapeCssSelector(message.parameters.cssSelector),
+                    message.parameters.optionLabel,
+                );
+            }
+
+            window.postMessage(
+                {
+                    type: "main-world-response",
+                    requestId: requestId,
+                    result: {},
+                },
+                "*",
+            );
+        } catch (error) {
+            // Send error back
+            window.postMessage(
+                {
+                    type: "main-world-response",
+                    requestId: requestId,
+                    error: error,
+                },
+                "*",
+            );
+        }
     }
 });