sidepanel: Add support for recorded actions that need the exact text …

…on element

- Fix typos
- Add support for recorded actions that need the exact text on element

ts/packages/agents/browser/src/agent/discovery/actionHandler.mts

@@ -175,6 +175,11 @@ export async function handleSchemaDiscoveryAction(
             ...authoredActions.values(),
         ];
 
+        if (typeDefinitions.length === 0) {
+            console.log("No actions for this schema.");
+            return;
+        }
+
         const union = sc.union(
             typeDefinitions.map((definition) => sc.ref(definition)),
         );
@@ -383,12 +388,12 @@ export async function handleSchemaDiscoveryAction(
         });
 
         const obj: ActionSchemaObject = sc.obj({
-            actionName: sc.string(userIntentJson.actiontName),
+            actionName: sc.string(userIntentJson.actionName),
             parameters: sc.obj(Object.fromEntries(fields)),
         } as const);
 
         const schema = sc.type(
-            userIntentJson.actiontName,
+            userIntentJson.actionName,
             obj,
             actionDescription,
             true,

ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts

@@ -32,7 +32,7 @@ export type UserIntentParameter = {
 
 export type UserIntent = {
     // a concise name for the action, in camelCase
-    actiontName: string;
+    actionName: string;
     // a consise list of the parameters that should be captured from the user in order to implenent this action
     parameters: UserIntentParameter[];
 };
@@ -49,7 +49,7 @@ export type WebPlan = {
 export type SelectElementByText = {
     actionName: "selectElementByText";
     parameters: {
-        // the shortName of the UserIntentParameter to use for this value
+        // IMPORTANT: the shortName of the UserIntentParameter to use for this value
         text: string;
         elementType?: string;
     };
@@ -58,7 +58,7 @@ export type SelectElementByText = {
 export type EnterText = {
     actionName: "enterText";
     parameters: {
-        // the shortName of the UserIntentParameter to use for this value
+        // IMPORTANT: the shortName of the UserIntentParameter to use for this value
         textParameter: string;
     };
 };
@@ -68,24 +68,32 @@ export type EnterText = {
 export type EnterTextAtPageScope = {
     actionName: "EnterTextAtPageScope";
     parameters: {
-        // the shortName of the UserIntentParameter to use for this value
+        // IMPORTANT: the shortName of the UserIntentParameter to use for this value
         textParameter: string;
     };
 };
 
 export type SelectValueFromDropdown = {
     actionName: "selectValueFromDropdown";
     parameters: {
-        // the shortName of the UserIntentParameter to use for this value
+        // IMPORTANT: the shortName of the UserIntentParameter to use for this value
         valueTextParameter: string;
     };
 };
 
+export type ClickOnButton = {
+    actionName: "clickOnButton";
+    parameters: {
+        // the displayed text of the button to click on
+        buttonText: string;
+    };
+};
+
 export type ClickOnElement = {
     actionName: "clickOnElement";
     parameters: {
-        // the shortName of the UserIntentParameter to use for this value
-        elementTextParameter: string;
+        // the displayed text of the element to click on
+        elementText: string;
     };
 };
 
@@ -101,6 +109,7 @@ export type PageManipulationActions =
     | SelectElementByText
     | EnterText
     | SelectValueFromDropdown
+    | ClickOnButton
     | ClickOnElement
     | ClickOnLink;
 

ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts

@@ -63,9 +63,6 @@ export function createTempAgentForSchema(
         selectionCondition?: string,
     ) {
         const htmlFragments = await browser.getHtmlFragments();
-        const timerName = `getting ${componentType} section`;
-
-        console.time(timerName);
         const response = await agent.getPageComponentSchema(
             componentType,
             selectionCondition,
@@ -79,7 +76,6 @@ export function createTempAgentForSchema(
             return;
         }
 
-        console.timeEnd(timerName);
         return response.data;
     }
 
@@ -141,7 +137,7 @@ export function createTempAgentForSchema(
             !actionsJson.has(action.actionName)
         ) {
             console.log(
-                `Action ${actionsJson} was not found on the list of user-defined actions`,
+                `Action ${action.actionName} was not found on the list of user-defined actions`,
             );
             return;
         }
@@ -169,21 +165,27 @@ export function createTempAgentForSchema(
                     await followLink(link?.linkCssSelector);
                     break;
                 case "clickOnElement":
-                    const elementParameter = targetIntent.parameters.find(
-                        (param) =>
-                            param.shortName ==
-                            step.parameters.elementTextParameter,
-                    );
                     const element = (await getComponentFromPage(
                         "Element",
-                        `element text ${elementParameter?.name}`,
+                        `element text ${step.parameters?.elementText}`,
                     )) as Element;
                     if (element !== undefined) {
                         await browser.clickOn(element.cssSelector);
                         await browser.awaitPageInteraction();
                         await browser.awaitPageLoad();
                     }
                     break;
+                case "clickOnButton":
+                    const button = (await getComponentFromPage(
+                        "Element",
+                        `element text ${step.parameters?.buttonText}`,
+                    )) as Element;
+                    if (button !== undefined) {
+                        await browser.clickOn(button.cssSelector);
+                        await browser.awaitPageInteraction();
+                        await browser.awaitPageLoad();
+                    }
+                    break;
                 case "enterText":
                     const textParameter = targetIntent.parameters.find(
                         (param) =>

ts/packages/agents/browser/src/extension/sidepanel.ts

@@ -197,7 +197,7 @@ async function saveUserAction() {
         }, 5000);
     }
 
-    button.innerHTML = `<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Processing...`;
+    button.innerHTML = `<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Saving...`;
     button.disabled = true;
 
     // Get schema based on the recorded action info
@@ -213,8 +213,9 @@ async function saveUserAction() {
         console.error("Error fetching schema:", chrome.runtime.lastError);
         showTemporaryStatus("✖ Failed", "btn-outline-danger");
     } else {
+        const processedActionName = response.intentJson.actionName;
         await addEntryToStoredPageProperties(actionName!, "userActions", {
-            name: actionName,
+            name: processedActionName,
             description: actionDescription,
             steps,
             screenshot,
@@ -224,17 +225,17 @@ async function saveUserAction() {
         });
 
         await addEntryToStoredPageProperties(
-            actionName!,
+            processedActionName,
             "authoredActionDefinitions",
             response.intentTypeDefinition,
         );
         await addEntryToStoredPageProperties(
-            actionName!,
+            processedActionName,
             "authoredActionsJson",
             response.actions,
         );
         await addEntryToStoredPageProperties(
-            actionName!,
+            processedActionName,
             "authoredIntentJson",
             response.intentJson,
         );