From 4b866c89984268150b7a587f4dccd7795e52d552 Mon Sep 17 00:00:00 2001
From: Robert Gruen <robgruen@microsoft.com>
Date: Fri, 1 Nov 2024 17:38:00 -0700
Subject: [PATCH] Added SMS

---
 .../mobile/.idea/deploymentTargetSelector.xml |  4 +--
 .../typeagent/sample/MainActivity.kt          |  1 +
 ts/packages/agentSdk/src/display.ts           |  5 ++--
 .../src/androidMobileActionHandler.ts         | 10 +++++--
 .../androidMobile/src/androidMobileSchema.ts  | 26 +++++++++++--------
 ts/packages/api/src/webServer.ts              |  5 ++++
 ts/packages/shell/src/lib/lib.android.d.ts    |  1 +
 ts/packages/shell/src/renderer/src/main.ts    |  4 +++
 8 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/android/samples/mobile/.idea/deploymentTargetSelector.xml b/android/samples/mobile/.idea/deploymentTargetSelector.xml
index 8e187da5f..384b1b1c5 100644
--- a/android/samples/mobile/.idea/deploymentTargetSelector.xml
+++ b/android/samples/mobile/.idea/deploymentTargetSelector.xml
@@ -4,10 +4,10 @@
     <selectionStates>
       <SelectionState runConfigName="app">
         <option name="selectionMode" value="DROPDOWN" />
-        <DropdownSelection timestamp="2024-10-24T17:18:28.751902400Z">
+        <DropdownSelection timestamp="2024-11-02T00:10:09.974283400Z">
           <Target type="DEFAULT_BOOT">
             <handle>
-              <DeviceId pluginId="LocalEmulator" identifier="path=C:\Users\robgruen\.android\avd\Pixel_9_Pro_XL_API_35.avd" />
+              <DeviceId pluginId="PhysicalDevice" identifier="serial=RFCX71SE0VE" />
             </handle>
           </Target>
         </DropdownSelection>
diff --git a/android/samples/mobile/app/src/main/java/com/microsoft/typeagent/sample/MainActivity.kt b/android/samples/mobile/app/src/main/java/com/microsoft/typeagent/sample/MainActivity.kt
index a62590b21..3d3c5aac3 100644
--- a/android/samples/mobile/app/src/main/java/com/microsoft/typeagent/sample/MainActivity.kt
+++ b/android/samples/mobile/app/src/main/java/com/microsoft/typeagent/sample/MainActivity.kt
@@ -55,6 +55,7 @@ class MainActivity : ComponentActivity() {
 @Preview
 fun Browser() {
     val url = "http://10.0.2.2:3000"
+    //val url = "http://192.168.1.142:3000/"
     var wvv: WebView? = null
     var jsi: JavaScriptInterface? = null
 
diff --git a/ts/packages/agentSdk/src/display.ts b/ts/packages/agentSdk/src/display.ts
index a9260f325..3dba7d4aa 100644
--- a/ts/packages/agentSdk/src/display.ts
+++ b/ts/packages/agentSdk/src/display.ts
@@ -28,11 +28,10 @@ export type DisplayAppendMode = "inline" | "block" | "temporary";
 
 export type ClientAction =
     | "show-camera"
-    | "open-app"
     | "show-notification"
-    | "start-intent"
     | "set-alarm"
-    | "call-phonenumber";
+    | "call-phonenumber"
+    | "send-sms";
 
 export interface ActionIO {
     readonly type: DisplayType;
diff --git a/ts/packages/agents/androidMobile/src/androidMobileActionHandler.ts b/ts/packages/agents/androidMobile/src/androidMobileActionHandler.ts
index 7f1685ff6..47d5bd0b2 100644
--- a/ts/packages/agents/androidMobile/src/androidMobileActionHandler.ts
+++ b/ts/packages/agents/androidMobile/src/androidMobileActionHandler.ts
@@ -9,7 +9,7 @@ import {
     ActionResult,
 } from "@typeagent/agent-sdk";
 import { createActionResult } from "@typeagent/agent-sdk/helpers/action";
-import { AndroidMobileAction, CallPhoneNumberAction, SetAlarmAction } from "./androidMobileSchema.js";
+import { AndroidMobileAction, CallPhoneNumberAction, SendSMSAction, SetAlarmAction } from "./androidMobileSchema.js";
 
 export function instantiate(): AppAgent {
     return {
@@ -54,6 +54,12 @@ async function handlePhotoAction(
 ) {
     let result: ActionResult | undefined = undefined;
     switch (action.actionName) {
+        case "sendSMS": {
+            let smsAction = action as SendSMSAction;
+            result = createActionResult(`Sending SMS to ${smsAction.parameters.phoneNumber} message '${smsAction.parameters.message}'`);
+            context.actionIO.takeAction("send-sms", smsAction.parameters);
+            break;
+        }
         case "callPhoneNumber": {
             let callAction = action as CallPhoneNumberAction;
             result = createActionResult(`Calling ${callAction.parameters.phoneNumber}`);
@@ -67,7 +73,7 @@ async function handlePhotoAction(
             break;
         }
         default:
-            throw new Error(`Unknown action: ${action.actionName}`);
+            throw new Error(`Unknown action: ${action}`);
     }
     return result;
 }
diff --git a/ts/packages/agents/androidMobile/src/androidMobileSchema.ts b/ts/packages/agents/androidMobile/src/androidMobileSchema.ts
index 7cbe54abe..6fcbb8861 100644
--- a/ts/packages/agents/androidMobile/src/androidMobileSchema.ts
+++ b/ts/packages/agents/androidMobile/src/androidMobileSchema.ts
@@ -1,7 +1,20 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-export type AndroidMobileAction = CallPhoneNumberAction | SetAlarmAction | UnknownAction;
+export type AndroidMobileAction = SendSMSAction | CallPhoneNumberAction | SetAlarmAction;
+
+// sends a SMS to the supplied phone number
+export type SendSMSAction = {
+    actionName: "sendSMS",
+    parameters: {
+        // the original request of the user
+        originalRequest: string;
+        // the phone number to message
+        phoneNumber: string;
+        // the sms message
+        message: string;
+    }
+}
 
 // calls a user's phone number but only if we know the phone number
 export type CallPhoneNumberAction = {
@@ -23,13 +36,4 @@ export type SetAlarmAction = {
         // the time for the alarm in the format YYYY-mm-ddThh:mm:ss (i.e. 2024-02-15T08:30:15 )
         time: string;
     };
-};
-
-// if the user types text that can not easily be understood as a list action, this action is used
-export interface UnknownAction {
-    actionName: "unknown";
-    parameters: {
-        // text typed by the user that the system did not understand
-        text: string;
-    };
-}
+};
\ No newline at end of file
diff --git a/ts/packages/api/src/webServer.ts b/ts/packages/api/src/webServer.ts
index 85c1ec34b..33e3e8073 100644
--- a/ts/packages/api/src/webServer.ts
+++ b/ts/packages/api/src/webServer.ts
@@ -62,6 +62,11 @@ export class TypeAgentAPIWebServer {
         this.server.listen(3000, "127.0.0.1", () => {
             console.log("Listening on 127.0.0.1:3000");
         });
+
+        // // starts a simple http server locally on port 3000
+        // this.server.listen(3000, "192.168.1.142", () => {
+        //     console.log("Listening on 192.168.1.142:3000");
+        // });
     }
 
     stop() {
diff --git a/ts/packages/shell/src/lib/lib.android.d.ts b/ts/packages/shell/src/lib/lib.android.d.ts
index 5c7a4fa3b..c487a4f63 100644
--- a/ts/packages/shell/src/lib/lib.android.d.ts
+++ b/ts/packages/shell/src/lib/lib.android.d.ts
@@ -5,4 +5,5 @@ declare var Android: {
     showToast: (message: string) => void;
     setAlarm: (time: string) => void;
     callPhoneNumber: (phoneNumber: string) => void;
+    sendSMS: (phoneNumber: string, message: string) => void;
 };
\ No newline at end of file
diff --git a/ts/packages/shell/src/renderer/src/main.ts b/ts/packages/shell/src/renderer/src/main.ts
index fb80f04b0..7a936ab5c 100644
--- a/ts/packages/shell/src/renderer/src/main.ts
+++ b/ts/packages/shell/src/renderer/src/main.ts
@@ -222,6 +222,10 @@ function addEvents(
                     let d: any = data;
                     Android?.callPhoneNumber(d.phoneNumber);
                 }
+                case "send-sms": {
+                    let d: any = data;
+                    Android.sendSMS(d.phoneNumber, d.message);
+                }
             }
         } catch (e) {
             console.log(e);