Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sidepanel: Add support for recorded actions that need the exact text on element #802

Merged
merged 1 commit into from
Mar 8, 2025
Merged

sidepanel: Add support for recorded actions that need the exact text on element #802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions ts/packages/agents/browser/src/agent/discovery/actionHandler.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,11 @@ export async function handleSchemaDiscoveryAction(
...authoredActions.values(),
];

if (typeDefinitions.length === 0) {
console.log("No actions for this schema.");
return;
}

const union = sc.union(
typeDefinitions.map((definition) => sc.ref(definition)),
);
Expand Down Expand Up @@ -383,12 +388,12 @@ export async function handleSchemaDiscoveryAction(
});

const obj: ActionSchemaObject = sc.obj({
actionName: sc.string(userIntentJson.actiontName),
actionName: sc.string(userIntentJson.actionName),
parameters: sc.obj(Object.fromEntries(fields)),
} as const);

const schema = sc.type(
userIntentJson.actiontName,
userIntentJson.actionName,
obj,
actionDescription,
true,
Expand Down
23 changes: 16 additions & 7 deletions ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export type UserIntentParameter = {

export type UserIntent = {
// a concise name for the action, in camelCase
actiontName: string;
actionName: string;
// a consise list of the parameters that should be captured from the user in order to implenent this action
parameters: UserIntentParameter[];
};
Expand All @@ -49,7 +49,7 @@ export type WebPlan = {
export type SelectElementByText = {
actionName: "selectElementByText";
parameters: {
// the shortName of the UserIntentParameter to use for this value
// IMPORTANT: the shortName of the UserIntentParameter to use for this value
text: string;
elementType?: string;
};
Expand All @@ -58,7 +58,7 @@ export type SelectElementByText = {
export type EnterText = {
actionName: "enterText";
parameters: {
// the shortName of the UserIntentParameter to use for this value
// IMPORTANT: the shortName of the UserIntentParameter to use for this value
textParameter: string;
};
};
Expand All @@ -68,24 +68,32 @@ export type EnterText = {
export type EnterTextAtPageScope = {
actionName: "EnterTextAtPageScope";
parameters: {
// the shortName of the UserIntentParameter to use for this value
// IMPORTANT: the shortName of the UserIntentParameter to use for this value
textParameter: string;
};
};

export type SelectValueFromDropdown = {
actionName: "selectValueFromDropdown";
parameters: {
// the shortName of the UserIntentParameter to use for this value
// IMPORTANT: the shortName of the UserIntentParameter to use for this value
valueTextParameter: string;
};
};

export type ClickOnButton = {
actionName: "clickOnButton";
parameters: {
// the displayed text of the button to click on
buttonText: string;
};
};

export type ClickOnElement = {
actionName: "clickOnElement";
parameters: {
// the shortName of the UserIntentParameter to use for this value
elementTextParameter: string;
// the displayed text of the element to click on
elementText: string;
};
};

Expand All @@ -101,6 +109,7 @@ export type PageManipulationActions =
| SelectElementByText
| EnterText
| SelectValueFromDropdown
| ClickOnButton
| ClickOnElement
| ClickOnLink;

Expand Down
24 changes: 13 additions & 11 deletions ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,6 @@ export function createTempAgentForSchema(
selectionCondition?: string,
) {
const htmlFragments = await browser.getHtmlFragments();
const timerName = `getting ${componentType} section`;

console.time(timerName);
const response = await agent.getPageComponentSchema(
componentType,
selectionCondition,
Expand All @@ -79,7 +76,6 @@ export function createTempAgentForSchema(
return;
}

console.timeEnd(timerName);
return response.data;
}

Expand Down Expand Up @@ -141,7 +137,7 @@ export function createTempAgentForSchema(
!actionsJson.has(action.actionName)
) {
console.log(
`Action ${actionsJson} was not found on the list of user-defined actions`,
`Action ${action.actionName} was not found on the list of user-defined actions`,
);
return;
}
Expand Down Expand Up @@ -169,21 +165,27 @@ export function createTempAgentForSchema(
await followLink(link?.linkCssSelector);
break;
case "clickOnElement":
const elementParameter = targetIntent.parameters.find(
(param) =>
param.shortName ==
step.parameters.elementTextParameter,
);
const element = (await getComponentFromPage(
"Element",
`element text ${elementParameter?.name}`,
`element text ${step.parameters?.elementText}`,
)) as Element;
if (element !== undefined) {
await browser.clickOn(element.cssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
}
break;
case "clickOnButton":
const button = (await getComponentFromPage(
"Element",
`element text ${step.parameters?.buttonText}`,
)) as Element;
if (button !== undefined) {
await browser.clickOn(button.cssSelector);
await browser.awaitPageInteraction();
await browser.awaitPageLoad();
}
break;
case "enterText":
const textParameter = targetIntent.parameters.find(
(param) =>
Expand Down
11 changes: 6 additions & 5 deletions ts/packages/agents/browser/src/extension/sidepanel.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ async function saveUserAction() {
}, 5000);
}

button.innerHTML = `<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Processing...`;
button.innerHTML = `<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span> Saving...`;
button.disabled = true;

// Get schema based on the recorded action info
Expand All @@ -213,8 +213,9 @@ async function saveUserAction() {
console.error("Error fetching schema:", chrome.runtime.lastError);
showTemporaryStatus("✖ Failed", "btn-outline-danger");
} else {
const processedActionName = response.intentJson.actionName;
await addEntryToStoredPageProperties(actionName!, "userActions", {
name: actionName,
name: processedActionName,
description: actionDescription,
steps,
screenshot,
Expand All @@ -224,17 +225,17 @@ async function saveUserAction() {
});

await addEntryToStoredPageProperties(
actionName!,
processedActionName,
"authoredActionDefinitions",
response.intentTypeDefinition,
);
await addEntryToStoredPageProperties(
actionName!,
processedActionName,
"authoredActionsJson",
response.actions,
);
await addEntryToStoredPageProperties(
actionName!,
processedActionName,
"authoredIntentJson",
response.intentJson,
);
Expand Down
Loading