diff --git a/ts/packages/agents/browser/src/agent/browserConnector.mts b/ts/packages/agents/browser/src/agent/browserConnector.mts
index 701b1c85f..8a57e9cb8 100644
--- a/ts/packages/agents/browser/src/agent/browserConnector.mts
+++ b/ts/packages/agents/browser/src/agent/browserConnector.mts
@@ -225,6 +225,17 @@ export class BrowserConnector {
return this.sendActionToBrowser(clickAction);
}
+ async setDropdown(cssSelector: string, optionLabel: string) {
+ const clickAction = {
+ actionName: "setDropdownValue",
+ parameters: {
+ cssSelector: cssSelector,
+ optionLabel: optionLabel,
+ },
+ };
+ return this.sendActionToBrowser(clickAction);
+ }
+
async enterTextIn(textValue: string, cssSelector?: string) {
let actionName = cssSelector ? "enterTextInElement" : "enterTextOnPage";
diff --git a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts
index a5c1d8c45..6d5275014 100644
--- a/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts
+++ b/ts/packages/agents/browser/src/agent/discovery/actionHandler.mts
@@ -149,7 +149,7 @@ export async function handleSchemaDiscoveryAction(
async function handleRegisterSiteSchema(action: any) {
const url = await browser.getPageUrl();
- let detectedActions = new Map(
+ const detectedActions = new Map(
Object.entries(
(await browser.getCurrentPageStoredProperty(
url!,
@@ -157,7 +157,7 @@ export async function handleSchemaDiscoveryAction(
)) ?? {},
),
);
- let authoredActions = new Map(
+ const authoredActions = new Map(
Object.entries(
(await browser.getCurrentPageStoredProperty(
url!,
@@ -165,7 +165,7 @@ export async function handleSchemaDiscoveryAction(
)) ?? {},
),
);
- let typeDefinitions: ActionSchemaTypeDefinition[] = [
+ const typeDefinitions: ActionSchemaTypeDefinition[] = [
...detectedActions.values(),
...authoredActions.values(),
];
@@ -184,7 +184,7 @@ export async function handleSchemaDiscoveryAction(
const hostName = new URL(url!).hostname.replace(/\./g, "_");
const agentName = `temp_${hostName}`;
- let schemaDescription = `A schema that enables interactions with the ${hostName} page`;
+ const schemaDescription = `A schema that enables interactions with the ${hostName} page`;
const manifest: AppAgentManifest = {
emojiChar: "🚧",
diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/pageComponents.mts b/ts/packages/agents/browser/src/agent/discovery/schema/pageComponents.mts
index b8cbca2c3..3210bfb66 100644
--- a/ts/packages/agents/browser/src/agent/discovery/schema/pageComponents.mts
+++ b/ts/packages/agents/browser/src/agent/discovery/schema/pageComponents.mts
@@ -55,7 +55,36 @@ export type LocationInStore = {
};
export type NavigationLink = {
+ title: string;
// CSS Selector for the link
+ linkCssSelector: string;
+};
+
+export type DropdownControl = {
+ title: string;
+ // CSS Selector for the selection element
+ cssSelector: string;
+
+ values: {
+ text: string;
+ value: string;
+ }[];
+};
+
+export type TextInput = {
title: string;
+ // CSS Selector for the text input area
+ cssSelector: string;
+};
+
+export type Button = {
+ title: string;
+ // CSS Selector for the button
+ cssSelector: string;
+};
+
+export type Link = {
+ title: string;
+ // CSS Selector for the link
linkCssSelector: string;
};
diff --git a/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts b/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts
index 203b9532d..833b53807 100644
--- a/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts
+++ b/ts/packages/agents/browser/src/agent/discovery/schema/recordedActions.mts
@@ -5,15 +5,26 @@
export type UserIntentParameter = {
// a concise name for the parameter, in camelCase. This should only contain alphanumeric characters
shortName: string;
+
// a longer, descriptive name for the parameter. This value can contain non-alphanumeric characters
name: string;
+
// The valid values are "string", "number" and "boolean"
type: string;
+
// The default value for the parameter. If this value is set based on a HTML
// page, check whether the target element has a default value. For dropdown elements, use the
// selected value for this entry
defaultValue?: any;
+
+ // The text for the various options for this control. This is useful for HTML elements
+ // that only accept a fixed set of values e.g. dropdown elements, radio-button lists etc.
+ valueOptions?: string[];
+
+ // The description for this parameter. Always include a list of Options as part of the description if the
+ // HTML control only accepts a fixed set of values e.g. dropdown elements, radio-button lists etc.
description: string;
+
// Indicates whether a parameter is required. If a parameter has a default value
// then it is not required.
required: boolean;
@@ -48,7 +59,7 @@ export type EnterText = {
actionName: "enterText";
parameters: {
// the shortName of the UserIntentParameter to use for this value
- text: string;
+ textParameter: string;
};
};
@@ -64,7 +75,7 @@ export type ClickOnButton = {
actionName: "clickOnButton";
parameters: {
// the shortName of the UserIntentParameter to use for this value
- buttonText: string;
+ buttonTextParameter: string;
};
};
diff --git a/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts b/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts
index c11d32b0b..1aa3b7b3a 100644
--- a/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts
+++ b/ts/packages/agents/browser/src/agent/discovery/tempAgentActionHandler.mts
@@ -8,7 +8,16 @@ import {
NavigateToPage,
} from "./schema/userActionsPool.mjs";
import { handleCommerceAction } from "../commerce/actionHandler.mjs";
-import { NavigationLink } from "./schema/pageComponents.mjs";
+import {
+ Button,
+ DropdownControl,
+ NavigationLink,
+} from "./schema/pageComponents.mjs";
+import {
+ PageManipulationActions,
+ PageManipulationActionsList,
+ UserIntent,
+} from "./schema/recordedActions.mjs";
export function createTempAgentForSchema(
browser: BrowserConnector,
@@ -41,6 +50,9 @@ export function createTempAgentForSchema(
break;
case "signUpForNewsletterAction":
break;
+ default:
+ handleUserDefinedAction(action);
+ break;
}
},
};
@@ -102,4 +114,133 @@ export function createTempAgentForSchema(
await followLink(link?.linkCssSelector);
}
+
+ async function handleUserDefinedAction(action: any) {
+ const url = await browser.getPageUrl();
+ const intentJson = new Map(
+ Object.entries(
+ (await browser.getCurrentPageStoredProperty(
+ url!,
+ "authoredIntentJson",
+ )) ?? {},
+ ),
+ );
+
+ const actionsJson = new Map(
+ Object.entries(
+ (await browser.getCurrentPageStoredProperty(
+ url!,
+ "authoredActionsJson",
+ )) ?? {},
+ ),
+ );
+
+ if (
+ !intentJson.has(action.actionName) ||
+ !actionsJson.has(action.actionName)
+ ) {
+ console.log(
+ `Action ${actionsJson} was not found on the list of user-defined actions`,
+ );
+ return;
+ }
+
+ const targetIntent = intentJson.get(action.actionName) as UserIntent;
+ const targetPlan = actionsJson.get(
+ action.actionName,
+ ) as PageManipulationActionsList;
+
+ console.log(`Running ${targetPlan.planName}`);
+
+ targetPlan.steps.forEach(async (step: PageManipulationActions) => {
+ switch (step.actionName) {
+ case "ClickOnLink":
+ const linkParameter = targetIntent.parameters.find(
+ (param) =>
+ param.shortName ==
+ step.parameters.linkTextParameter,
+ );
+ const link = (await getComponentFromPage(
+ "NavigationLink",
+ `link text ${linkParameter?.name}`,
+ )) as NavigationLink;
+
+ await followLink(link?.linkCssSelector);
+ break;
+ case "clickOnButton":
+ const buttonParameter = targetIntent.parameters.find(
+ (param) =>
+ param.shortName ==
+ step.parameters.buttonTextParameter,
+ );
+ const button = (await getComponentFromPage(
+ "Button",
+ `button text ${buttonParameter?.name}`,
+ )) as Button;
+ await browser.clickOn(button.cssSelector);
+ await browser.awaitPageInteraction();
+ await browser.awaitPageLoad();
+
+ break;
+ case "enterText":
+ const textParameter = targetIntent.parameters.find(
+ (param) =>
+ param.shortName == step.parameters.textParameter,
+ );
+ const textElement = (await getComponentFromPage(
+ "TextInput",
+ `input label ${textParameter?.name}`,
+ )) as Button;
+
+ const userProvidedTextValue =
+ action.parameters[step.parameters.textParameter];
+
+ if (userProvidedTextValue !== undefined) {
+ await browser.enterTextIn(
+ userProvidedTextValue,
+ textElement.cssSelector,
+ );
+ }
+ break;
+ case "selectElementByText":
+ break;
+ case "selectValueFromDropdown":
+ const selectParameter = targetIntent.parameters.find(
+ (param) =>
+ param.shortName ==
+ step.parameters.valueTextParameter,
+ );
+
+ const userProvidedValue =
+ action.parameters[step.parameters.valueTextParameter];
+
+ if (userProvidedValue !== undefined) {
+ const selectElement = (await getComponentFromPage(
+ "DropdownControl",
+ `text ${selectParameter?.name}`,
+ )) as DropdownControl;
+
+ await browser.clickOn(selectElement.cssSelector);
+ const selectValue = selectElement.values.find(
+ (value) =>
+ value.text ===
+ action.parameters[
+ step.parameters.valueTextParameter
+ ],
+ );
+ if (selectValue) {
+ await browser.setDropdown(
+ selectElement.cssSelector,
+ selectValue.text,
+ );
+ } else {
+ console.error(`Could not find a dropdown option with text ${action.parameters[step.parameters.valueTextParameter]}
+ on the ${selectElement.title} dropdown.`);
+ }
+ }
+
+ break;
+ }
+ });
+ }
}
diff --git a/ts/packages/agents/browser/src/extension/serviceWorker.ts b/ts/packages/agents/browser/src/extension/serviceWorker.ts
index 6692f67c5..e54a0d760 100644
--- a/ts/packages/agents/browser/src/extension/serviceWorker.ts
+++ b/ts/packages/agents/browser/src/extension/serviceWorker.ts
@@ -1205,6 +1205,14 @@ async function runBrowserAction(action: any) {
});
break;
}
+ case "setDropdownValue": {
+ const targetTab = await getActiveTab();
+ const response = await chrome.tabs.sendMessage(targetTab.id!, {
+ type: "run_ui_event",
+ action: action,
+ });
+ break;
+ }
case "getPageSchema": {
const targetTab = await getActiveTab();
const key = action.parameters.url ?? targetTab.url;
@@ -1569,6 +1577,7 @@ chrome.runtime.onMessage.addListener(
sendResponse({
intent: schemaResult.intent,
+ intentJson: schemaResult.intentJson,
actions: schemaResult.actions,
intentTypeDefinition: schemaResult.intentTypeDefinition,
});
diff --git a/ts/packages/agents/browser/src/extension/sidepanel.ts b/ts/packages/agents/browser/src/extension/sidepanel.ts
index eb3e76214..4affc725c 100644
--- a/ts/packages/agents/browser/src/extension/sidepanel.ts
+++ b/ts/packages/agents/browser/src/extension/sidepanel.ts
@@ -176,25 +176,83 @@ async function saveUserAction() {
const screenshot = JSON.parse(stepsContainer.dataset.screenshot || "");
const html = JSON.parse(stepsContainer.dataset.html || "");
- // Retrieve existing actions from localStorage
- const storedActions = localStorage.getItem("userActions");
- const actions = storedActions ? JSON.parse(storedActions) : [];
-
- // Add new action
- actions.push({
- name: actionName,
- description: actionDescription,
- steps,
+ const button = document.getElementById("saveAction") as HTMLButtonElement;
+ const originalContent = button.innerHTML;
+ const originalClass = button.className;
+
+ function showTemporaryStatus(text: string, newClass: string) {
+ button.innerHTML = text;
+ button.className = `btn btn-sm ${newClass}`;
+
+ setTimeout(() => {
+ button.innerHTML = originalContent;
+ button.className = originalClass;
+ button.disabled = false;
+ }, 5000);
+ }
+
+ button.innerHTML = ` Processing...`;
+ button.disabled = true;
+
+ // Get schema based on the recorded action info
+ const response = await chrome.runtime.sendMessage({
+ type: "getIntentFromRecording",
+ html: [{ content: html, frameId: 0 }],
screenshot,
- html,
+ actionName,
+ actionDescription,
+ steps: JSON.stringify(steps),
});
+ if (chrome.runtime.lastError) {
+ console.error("Error fetching schema:", chrome.runtime.lastError);
+ showTemporaryStatus("✖ Failed", "btn-outline-danger");
+ } else {
+ await addEntryToStoredPageProperties(actionName!, "userActions", {
+ name: actionName,
+ description: actionDescription,
+ steps,
+ screenshot,
+ html,
+ intentSchema: response.intent,
+ actionsJson: response.actions,
+ });
- // Save back to localStorage
- localStorage.setItem("userActions", JSON.stringify(actions));
+ await addEntryToStoredPageProperties(
+ actionName!,
+ "authoredActionDefinitions",
+ response.intentTypeDefinition,
+ );
+ await addEntryToStoredPageProperties(
+ actionName!,
+ "authoredActionsJson",
+ response.actions,
+ );
+ await addEntryToStoredPageProperties(
+ actionName!,
+ "authoredIntentJson",
+ response.intentJson,
+ );
+ showTemporaryStatus("✔ Succeeded", "btn-outline-success");
+ }
- // Update UI
+ toggleActionForm();
await updateUserActionsUI();
- toggleActionForm(); // Hide form after saving
+}
+
+async function addEntryToStoredPageProperties(
+ actionName: string,
+ key: string,
+ value: any,
+) {
+ let currentActionJson = new Map(
+ Object.entries((await getStoredPageProperty(launchUrl!, key)) ?? {}),
+ );
+ currentActionJson.set(actionName!, value);
+ await setStoredPageProperty(
+ launchUrl!,
+ key,
+ Object.fromEntries(currentActionJson),
+ );
}
// Function to update user actions display
@@ -239,13 +297,10 @@ async function stopRecording() {
).value.trim();
renderTimelineSteps(
- actionName,
- actionDescription,
response.recordedActions,
stepsContainer,
response.recordedActionScreenshot,
response.recordedActionHtml,
- true,
);
}
@@ -266,13 +321,11 @@ async function cancelRecording() {
}
async function clearRecordedUserAction() {
- if (localStorage.getItem("userActions")) {
- localStorage.removeItem("userActions");
- }
-
await chrome.runtime.sendMessage({ type: "clearRecordedActions" });
-
+ await setStoredPageProperty(launchUrl!, "userActions", null);
await setStoredPageProperty(launchUrl!, "authoredActionDefinitions", null);
+ await setStoredPageProperty(launchUrl!, "authoredActionsJson", null);
+ await setStoredPageProperty(launchUrl!, "authoredIntentJson", null);
// Update UI
await updateUserActionsUI();
}
@@ -283,8 +336,13 @@ async function showUserDefinedActionsList() {
) as HTMLDivElement;
// Fetch recorded actions
- const storedActions = localStorage.getItem("userActions");
- const actions = storedActions ? JSON.parse(storedActions) : [];
+ const storedActions = new Map(
+ Object.entries(
+ (await getStoredPageProperty(launchUrl!, "userActions")) ?? {},
+ ),
+ );
+
+ const actions = Array.from(storedActions.values());
userActionsListContainer.innerHTML = "";
@@ -332,9 +390,6 @@ function renderTimeline(action: any, index: number) {
Actions
-
@@ -364,98 +419,46 @@ function renderTimeline(action: any, index: number) {
"#Stepscontent",
)! as HTMLElement;
renderTimelineSteps(
- action.name,
- action.description,
action.steps,
stepsContainer,
action.screenshot,
action.html,
);
- const processActionButton = timelineHeader.querySelector(
- "#processAction",
- )! as HTMLElement;
-
- const intentViewContainer = timelineHeader.querySelector(
- "#intentContent",
- )! as HTMLElement;
-
- const actionsViewContainer = timelineHeader.querySelector(
- "#planContent",
- )! as HTMLElement;
-
- processActionButton.style.display = "block";
- processActionButton.addEventListener("click", () =>
- getIntentFromRecording(
- action.html,
- action.screenshot,
- action.name,
- action.description,
- action.steps,
- ),
- );
-
- async function getIntentFromRecording(
- html: string,
- screenshot: string,
- actionName: string,
- description: string,
- steps: any[],
- ) {
- const response = await chrome.runtime.sendMessage({
- type: "getIntentFromRecording",
- html: [{ content: html, frameId: 0 }],
- screenshot,
- actionName,
- description,
- steps: JSON.stringify(steps),
- });
- if (chrome.runtime.lastError) {
- console.error("Error fetching schema:", chrome.runtime.lastError);
- return;
- }
-
+ if (action.intentSchema !== undefined) {
const card = document.createElement("div");
card.innerHTML = `
- ${response.intent}
+ ${action.intentSchema}
`;
+ const intentViewContainer = timelineHeader.querySelector(
+ "#intentContent",
+ )! as HTMLElement;
+
intentViewContainer.replaceChildren(card);
+ }
+
+ if (action.actionsJson !== undefined) {
+ const actionsViewContainer = timelineHeader.querySelector(
+ "#planContent",
+ )! as HTMLElement;
const actionsCard = document.createElement("div");
actionsCard.innerHTML = `
- ${JSON.stringify(response.actions, null, 2)}
+ ${JSON.stringify(action.actionsJson, null, 2)}
`;
actionsViewContainer.replaceChildren(actionsCard);
-
- let currentTypeDefinitions = new Map(
- Object.entries(
- (await getStoredPageProperty(
- launchUrl!,
- "authoredActionDefinitions",
- )) ?? {},
- ),
- );
- currentTypeDefinitions.set(actionName, response.intentTypeDefinition);
- await setStoredPageProperty(
- launchUrl!,
- "authoredActionDefinitions",
- Object.fromEntries(currentTypeDefinitions),
- );
}
userActionsListContainer.appendChild(timelineHeader);
}
function renderTimelineSteps(
- actionName: string,
- actionDescription: string,
steps: any[],
userActionsListContainer: HTMLElement,
screenshotData: string,
htmlData: string,
- isEditingMode?: boolean,
) {
userActionsListContainer.innerHTML = `
@@ -506,10 +509,6 @@ function renderTimelineSteps(
"#downloadHtml",
)! as HTMLElement;
- const processActionButton = userActionsListContainer.querySelector(
- "#processAction",
- )! as HTMLElement;
-
if (screenshotData) {
const img = document.createElement("img");
img.src = screenshotData;
@@ -534,19 +533,6 @@ function renderTimelineSteps(
);
}
- if (isEditingMode) {
- processActionButton.classList.remove("hidden");
- processActionButton.addEventListener("click", () =>
- getIntentFromRecording(
- htmlData,
- screenshotData,
- actionName,
- actionDescription,
- steps,
- ),
- );
- }
-
// Function to download the screenshot
function downloadScreenshot(dataUrl: string) {
const link = document.createElement("a");
@@ -566,28 +552,6 @@ function renderTimelineSteps(
link.click();
document.body.removeChild(link);
}
-
- async function getIntentFromRecording(
- html: string,
- screenshot: string,
- actionName: string,
- description: string,
- steps: any[],
- ) {
- const response = await chrome.runtime.sendMessage({
- type: "getIntentFromRecording",
- html: [{ content: html, frameId: 0 }],
- screenshot,
- actionName,
- description,
- steps: JSON.stringify(steps),
- });
- if (chrome.runtime.lastError) {
- console.error("Error fetching schema:", chrome.runtime.lastError);
- return;
- }
- console.log(response.data);
- }
}
document.addEventListener("DOMContentLoaded", async () => {
diff --git a/ts/packages/agents/browser/src/extension/uiEventsDispatcher.ts b/ts/packages/agents/browser/src/extension/uiEventsDispatcher.ts
index e1bd2a8c7..132cbc29d 100644
--- a/ts/packages/agents/browser/src/extension/uiEventsDispatcher.ts
+++ b/ts/packages/agents/browser/src/extension/uiEventsDispatcher.ts
@@ -44,20 +44,196 @@ function clickOnElement(selector: string) {
}
}
-function simulateKeyEvent(targetNode: HTMLElement, character: string) {
- function triggerKeyboardEvent(targetNode: HTMLElement, eventType: string) {
- var keyEvent = new KeyboardEvent(eventType, {
- key: character,
- code: `Key${character.toUpperCase()}`,
- bubbles: true,
- keyCode: character.charCodeAt(0),
- });
+function enterTextInElement(
+ text: string,
+ selector: string,
+ options: {
+ delay?: number; // Delay between keystrokes in ms
+ clearExisting?: boolean; // Whether to clear existing content first
+ triggerBlur?: boolean; // Whether to trigger blur event after typing
+ triggerSubmit?: boolean; // Whether to trigger form submit after typing
+ } = {},
+): Promise {
+ return new Promise((resolve, reject) => {
+ try {
+ // Default options
+ const config = {
+ delay: options.delay ?? 50,
+ clearExisting: options.clearExisting ?? false,
+ triggerBlur: options.triggerBlur ?? false,
+ triggerSubmit: options.triggerSubmit ?? false,
+ };
+
+ const inputElement = document.querySelector(
+ selector,
+ ) as HTMLElement;
+
+ // Check if inputElement is an input or textarea
+ if (
+ !(inputElement instanceof HTMLInputElement) &&
+ !(inputElement instanceof HTMLTextAreaElement) &&
+ !inputElement.isContentEditable
+ ) {
+ throw new Error(
+ "Element must be an input, textarea, or contenteditable element",
+ );
+ }
+
+ inputElement.focus();
+
+ if (config.clearExisting) {
+ if (
+ inputElement instanceof HTMLInputElement ||
+ inputElement instanceof HTMLTextAreaElement
+ ) {
+ inputElement.value = "";
+ const inputEvent = new Event("input", { bubbles: true });
+ inputElement.dispatchEvent(inputEvent);
+ } else if (inputElement.isContentEditable) {
+ inputElement.textContent = "";
+ }
+ }
+
+ // Function to simulate typing a single character
+ const typeCharacter = (char: string, index: number) => {
+ setTimeout(() => {
+ simulateKeyEvent(inputElement, char);
+
+ // Check if we're done typing
+ if (index === text.length - 1) {
+ // Handle additional actions after typing
+ if (config.triggerBlur) {
+ inputElement.blur();
+ }
+
+ if (config.triggerSubmit) {
+ // Find the parent form and submit it
+ const form = inputElement.closest("form");
+ if (form) {
+ form.dispatchEvent(
+ new Event("submit", {
+ bubbles: true,
+ cancelable: true,
+ }),
+ );
+ }
+ }
+
+ resolve();
+ }
+ }, index * config.delay);
+ };
+
+ // Type each character with delay
+ Array.from(text).forEach((char, index) => {
+ typeCharacter(char, index);
+ });
+ } catch (error) {
+ reject(error);
+ }
+ });
+}
+
+function simulateKeyEvent(inputElement: HTMLElement, char: string) {
+ const keydownEvent = new KeyboardEvent("keydown", {
+ key: char,
+ code: `Key${char.toUpperCase()}`,
+ charCode: char.charCodeAt(0),
+ keyCode: char.charCodeAt(0),
+ which: char.charCodeAt(0),
+ bubbles: true,
+ cancelable: true,
+ });
+ inputElement.dispatchEvent(keydownEvent);
+
+ const keypressEvent = new KeyboardEvent("keypress", {
+ key: char,
+ code: `Key${char.toUpperCase()}`,
+ charCode: char.charCodeAt(0),
+ keyCode: char.charCodeAt(0),
+ which: char.charCodeAt(0),
+ bubbles: true,
+ cancelable: true,
+ });
+ inputElement.dispatchEvent(keypressEvent);
+
+ const textInputEvent = new InputEvent("textInput", {
+ data: char,
+ inputType: "insertText",
+ bubbles: true,
+ cancelable: true,
+ });
+ inputElement.dispatchEvent(textInputEvent);
+
+ if (
+ inputElement instanceof HTMLInputElement ||
+ inputElement instanceof HTMLTextAreaElement
+ ) {
+ // Get current position of cursor
+ const startPos = inputElement.selectionStart || 0;
+ const endPos = inputElement.selectionEnd || 0;
- targetNode.dispatchEvent(keyEvent);
+ // If text is selected, replace it
+ if (startPos !== endPos) {
+ inputElement.value =
+ inputElement.value.substring(0, startPos) +
+ char +
+ inputElement.value.substring(endPos);
+
+ // Set cursor position after the inserted character
+ inputElement.selectionStart = inputElement.selectionEnd =
+ startPos + 1;
+ } else {
+ // Just insert at current position
+ inputElement.value =
+ inputElement.value.substring(0, startPos) +
+ char +
+ inputElement.value.substring(startPos);
+
+ // Move cursor forward
+ inputElement.selectionStart = inputElement.selectionEnd =
+ startPos + 1;
+ }
+ } else if (inputElement.isContentEditable) {
+ // For contenteditable elements
+ // Get current selection
+ const selection = window.getSelection();
+ const range = selection?.getRangeAt(0);
+
+ if (selection && range) {
+ // Delete any selected content
+ range.deleteContents();
+
+ // Insert the character
+ const textNode = document.createTextNode(char);
+ range.insertNode(textNode);
+
+ // Move cursor after inserted character
+ range.setStartAfter(textNode);
+ range.setEndAfter(textNode);
+ selection.removeAllRanges();
+ selection.addRange(range);
+ }
}
- ["keydown", "keypress", "keyup"].forEach(function (eventType) {
- triggerKeyboardEvent(targetNode, eventType);
+
+ const inputEvent = new InputEvent("input", {
+ inputType: "insertText",
+ data: char,
+ bubbles: true,
+ cancelable: true,
+ });
+ inputElement.dispatchEvent(inputEvent);
+
+ const keyupEvent = new KeyboardEvent("keyup", {
+ key: char,
+ code: `Key${char.toUpperCase()}`,
+ charCode: char.charCodeAt(0),
+ keyCode: char.charCodeAt(0),
+ which: char.charCodeAt(0),
+ bubbles: true,
+ cancelable: true,
});
+ inputElement.dispatchEvent(keyupEvent);
}
function enterLetterInElement(letter: string, selector: string) {
@@ -87,20 +263,27 @@ async function enterTextOnPage(text: string) {
}
}
-async function enterTextInElement(text: string, selector: string) {
- const targetElement = document.querySelector(selector) as HTMLElement;
- if (targetElement) {
- targetElement.focus();
- if (!document.execCommand("insertText", false, text)) {
- if (targetElement instanceof HTMLInputElement) {
- targetElement.value = text;
- } else {
- for (var i = 0; i < text.length; i++) {
- simulateKeyEvent(targetElement, text[i]);
- await new Promise((r) => setTimeout(r, 20));
- }
- }
- }
+async function selectDropdownOption(selector: string, optionLabel: string) {
+ const select = document.querySelector(selector) as HTMLSelectElement;
+ if (!select) {
+ console.error("Select element not found");
+ return null;
+ }
+
+ const options = Array.from(select.querySelectorAll("option"));
+ const matchingOption = options.find(
+ (opt) => opt.textContent?.trim() === optionLabel,
+ );
+
+ if (matchingOption) {
+ // Set the value programmatically
+ (select as HTMLSelectElement).value = matchingOption.value;
+
+ // Dispatch change and input events to trigger handlers
+ select.dispatchEvent(new Event("change", { bubbles: true }));
+ select.dispatchEvent(new Event("input", { bubbles: true }));
+
+ await new Promise((r) => setTimeout(r, 20));
}
}
@@ -123,11 +306,22 @@ document.addEventListener("toUIEventsDispatcher", async function (e: any) {
await enterTextInElement(
message.parameters.value,
escapeCssSelector(message.parameters.cssSelector),
+ {
+ clearExisting: true,
+ triggerBlur: true,
+ triggerSubmit: message.parameters.submitForm ?? false,
+ },
);
}
if (actionName === "enterTextOnPage") {
await enterTextOnPage(message.parameters.value.toUpperCase());
}
+ if (actionName === "setDropdownValue") {
+ await selectDropdownOption(
+ escapeCssSelector(message.parameters.cssSelector),
+ message.parameters.optionLabel,
+ );
+ }
});
document.addEventListener("DOMContentLoaded", async () => {