From 94fc72e19cb133c0aa0cfa50feba90edd5eeb5f8 Mon Sep 17 00:00:00 2001
From: Hillary Mutisya <150286414+hillary-mutisya@users.noreply.github.com>
Date: Fri, 7 Mar 2025 21:55:25 -0800
Subject: [PATCH] sidepanel: Add support for recorded actions that need the
exact text on element
- Fix typos
- Add support for recorded actions that need the exact text on element
---
.../src/agent/discovery/actionHandler.mts | 9 +++++--
.../discovery/schema/recordedActions.mts | 23 ++++++++++++------
.../discovery/tempAgentActionHandler.mts | 24 ++++++++++---------
.../agents/browser/src/extension/sidepanel.ts | 11 +++++----
4 files changed, 42 insertions(+), 25 deletions(-)
diff --git a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts
index 221bb4627..22f41e133 100644
--- a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts
+++ b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts
@@ -175,6 +175,11 @@ export async function handleSchemaDiscoveryAction(
...authoredActions.values(),
];
+ if (typeDefinitions.length === 0) {
+ console.log("No actions for this schema.");
+ return;
+ }
+
const union = sc.union(
typeDefinitions.map((definition) => sc.ref(definition)),
);
@@ -383,12 +388,12 @@ export async function handleSchemaDiscoveryAction(
});
const obj: ActionSchemaObject = sc.obj({
- actionName: sc.string(userIntentJson.actiontName),
+ actionName: sc.string(userIntentJson.actionName),
parameters: sc.obj(Object.fromEntries(fields)),
} as const);
const schema = sc.type(
- userIntentJson.actiontName,
+ userIntentJson.actionName,
obj,
actionDescription,
true,
diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts b/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts
index 6effb1727..343346005 100644
--- a/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts
+++ b/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts
@@ -32,7 +32,7 @@ export type UserIntentParameter = {
export type UserIntent = {
// a concise name for the action, in camelCase
- actiontName: string;
+ actionName: string;
// a consise list of the parameters that should be captured from the user in order to implenent this action
parameters: UserIntentParameter[];
};
@@ -49,7 +49,7 @@ export type WebPlan = {
export type SelectElementByText = {
actionName: "selectElementByText";
parameters: {
- // the shortName of the UserIntentParameter to use for this value
+ // IMPORTANT: the shortName of the UserIntentParameter to use for this value
text: string;
elementType?: string;
};
@@ -58,7 +58,7 @@ export type SelectElementByText = {
export type EnterText = {
actionName: "enterText";
parameters: {
- // the shortName of the UserIntentParameter to use for this value
+ // IMPORTANT: the shortName of the UserIntentParameter to use for this value
textParameter: string;
};
};
@@ -68,7 +68,7 @@ export type EnterText = {
export type EnterTextAtPageScope = {
actionName: "EnterTextAtPageScope";
parameters: {
- // the shortName of the UserIntentParameter to use for this value
+ // IMPORTANT: the shortName of the UserIntentParameter to use for this value
textParameter: string;
};
};
@@ -76,16 +76,24 @@ export type EnterTextAtPageScope = {
export type SelectValueFromDropdown = {
actionName: "selectValueFromDropdown";
parameters: {
- // the shortName of the UserIntentParameter to use for this value
+ // IMPORTANT: the shortName of the UserIntentParameter to use for this value
valueTextParameter: string;
};
};
+export type ClickOnButton = {
+ actionName: "clickOnButton";
+ parameters: {
+ // the displayed text of the button to click on
+ buttonText: string;
+ };
+};
+
export type ClickOnElement = {
actionName: "clickOnElement";
parameters: {
- // the shortName of the UserIntentParameter to use for this value
- elementTextParameter: string;
+ // the displayed text of the element to click on
+ elementText: string;
};
};
@@ -101,6 +109,7 @@ export type PageManipulationActions =
| SelectElementByText
| EnterText
| SelectValueFromDropdown
+ | ClickOnButton
| ClickOnElement
| ClickOnLink;
diff --git a/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts
index d59697230..0a1fe82e3 100644
--- a/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts
+++ b/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts
@@ -63,9 +63,6 @@ export function createTempAgentForSchema(
selectionCondition?: string,
) {
const htmlFragments = await browser.getHtmlFragments();
- const timerName = `getting ${componentType} section`;
-
- console.time(timerName);
const response = await agent.getPageComponentSchema(
componentType,
selectionCondition,
@@ -79,7 +76,6 @@ export function createTempAgentForSchema(
return;
}
- console.timeEnd(timerName);
return response.data;
}
@@ -141,7 +137,7 @@ export function createTempAgentForSchema(
!actionsJson.has(action.actionName)
) {
console.log(
- `Action ${actionsJson} was not found on the list of user-defined actions`,
+ `Action ${action.actionName} was not found on the list of user-defined actions`,
);
return;
}
@@ -169,14 +165,9 @@ export function createTempAgentForSchema(
await followLink(link?.linkCssSelector);
break;
case "clickOnElement":
- const elementParameter = targetIntent.parameters.find(
- (param) =>
- param.shortName ==
- step.parameters.elementTextParameter,
- );
const element = (await getComponentFromPage(
"Element",
- `element text ${elementParameter?.name}`,
+ `element text ${step.parameters?.elementText}`,
)) as Element;
if (element !== undefined) {
await browser.clickOn(element.cssSelector);
@@ -184,6 +175,17 @@ export function createTempAgentForSchema(
await browser.awaitPageLoad();
}
break;
+ case "clickOnButton":
+ const button = (await getComponentFromPage(
+ "Element",
+ `element text ${step.parameters?.buttonText}`,
+ )) as Element;
+ if (button !== undefined) {
+ await browser.clickOn(button.cssSelector);
+ await browser.awaitPageInteraction();
+ await browser.awaitPageLoad();
+ }
+ break;
case "enterText":
const textParameter = targetIntent.parameters.find(
(param) =>
diff --git a/ts/packages/agents/browser/src/extension/sidepanel.ts b/ts/packages/agents/browser/src/extension/sidepanel.ts
index 7aba4fc1f..fd62ec863 100644
--- a/ts/packages/agents/browser/src/extension/sidepanel.ts
+++ b/ts/packages/agents/browser/src/extension/sidepanel.ts
@@ -197,7 +197,7 @@ async function saveUserAction() {
}, 5000);
}
- button.innerHTML = ` Processing...`;
+ button.innerHTML = ` Saving...`;
button.disabled = true;
// Get schema based on the recorded action info
@@ -213,8 +213,9 @@ async function saveUserAction() {
console.error("Error fetching schema:", chrome.runtime.lastError);
showTemporaryStatus("✖ Failed", "btn-outline-danger");
} else {
+ const processedActionName = response.intentJson.actionName;
await addEntryToStoredPageProperties(actionName!, "userActions", {
- name: actionName,
+ name: processedActionName,
description: actionDescription,
steps,
screenshot,
@@ -224,17 +225,17 @@ async function saveUserAction() {
});
await addEntryToStoredPageProperties(
- actionName!,
+ processedActionName,
"authoredActionDefinitions",
response.intentTypeDefinition,
);
await addEntryToStoredPageProperties(
- actionName!,
+ processedActionName,
"authoredActionsJson",
response.actions,
);
await addEntryToStoredPageProperties(
- actionName!,
+ processedActionName,
"authoredIntentJson",
response.intentJson,
);