diff --git a/Processing/GestureRecorder/.gitignore b/Processing/GestureRecorder/.gitignore new file mode 100644 index 0000000..bb0754e --- /dev/null +++ b/Processing/GestureRecorder/.gitignore @@ -0,0 +1 @@ +/Gestures \ No newline at end of file diff --git a/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/DialogBox.PNG b/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/DialogBox.PNG new file mode 100644 index 0000000..98cde35 Binary files /dev/null and b/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/DialogBox.PNG differ diff --git a/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/LIS3DHGestureRecorder.ino b/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/LIS3DHGestureRecorder.ino index 6c4b181..53c7049 100644 --- a/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/LIS3DHGestureRecorder.ino +++ b/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/LIS3DHGestureRecorder.ino @@ -1,67 +1,151 @@ /* - * Reads in the x, y, and z accelerometer values from the LIS3DH and a "record gesture" button - * state and prints the following CSV to serial: timestamp, x, y, z, buttonState - * - * By Jon E. Froehlich - * @jonfroehlich - * http://makeabilitylab.io - * - */ + Reads in the x, y, and z accelerometer values from the LIS3DH and a "record gesture" button + state and prints the following CSV to serial: timestamp, x, y, z, buttonState + By Jon E. Froehlich + @jonfroehlich + http://makeabilitylab.io + +*/ + +#include #include #include #include #include +#include "Stream.h" +#include "BluetoothSerial.h" //Header File for Serial Bluetooth, will be added by default into Arduino + +// Settings +enum CommunicationMode { + RECORDER_SERIAL, + RECORDER_BLUETOOTH, + RECORDER_WIFI +}; +// By default we use serial +const CommunicationMode CURRENT_MODE = RECORDER_SERIAL; // CHANGE THIS IF YOU WANT A DIFFERENT MODE + +// ------------------------------------------- +// Serial info +const int SERIAL_BAUD_RATE = 115200; + +// ------------------------------------------- +// Bluetooth settings +const char* ARDUINO_BT_NAME = "ESP32 Gesture Recorder"; +// Bluetooth uses the serial baud rate and port index above + +// ------------------------------------------- +// WIFI +// If using wifi, make sure to put your credentials there if you're using WIFI mode +const char* WIFI_SSID = ""; +const char* WIFI_PASS = ""; +const char* WIFI_HOST_IP_ADDRESS = ""; // The IP address of the host you want to connect to +const int WIFI_HOST_PORT = 10002; // The IP address of the host you want to connect to // Used for LIS3DH hardware & software SPI #define LIS3DH_CS 10 Adafruit_LIS3DH lis = Adafruit_LIS3DH(); const boolean INCLUDE_TIMESTAMP = true; // print out timestamp to serial -const int BUTTON_INPUT_PIN = 21; // hooked up with pull-up configuration -const int SERIAL_BAUD_RATE = 115200; // make sure this matches the value in GestureRecorder.pde -const int DELAY_MS = 10; // the loop delay +const int BUTTON_INPUT_PIN = 21; // hooked up with pull-up configuration +const int DELAY_MS = 2; // the loop delay + +// Common data objects +BluetoothSerial ESP_BT; //Object for Bluetooth +// Use WiFiClient class to create TCP connections +WiFiClient ESP_WIFI; -void setup() { +void setup() +{ Serial.begin(SERIAL_BAUD_RATE); Serial.println("Initializing accelerometer..."); - if (! lis.begin(0x18)) { // change this to 0x19 for alternative i2c address + if (!lis.begin(0x18)) + { // change this to 0x19 for alternative i2c address Serial.println("Couldnt start"); - while (1) yield(); + while (1) + yield(); } Serial.println("LIS3DH found!"); - lis.setRange(LIS3DH_RANGE_4_G); // 2, 4, 8 or 16 G! + lis.setRange(LIS3DH_RANGE_4_G); // 2, 4, 8 or 16 G! Serial.print("Range = "); Serial.print(2 << lis.getRange()); Serial.println("G"); + // Setup Bluetooth if we want + if (CURRENT_MODE == RECORDER_BLUETOOTH) { + if (!ESP_BT.begin(ARDUINO_BT_NAME)) { + Serial.println("We failed to start Bluetooth"); + } + ESP_BT.setPin("1234"); + Serial.println("Bluetooth started"); + while (!ESP_BT.isReady()) { + Serial.println("Waiting for Bluetooth..."); + vTaskDelay(1000); // this is similar to yield and allows the device to go to low power mode + } + } + else if (CURRENT_MODE == RECORDER_WIFI) { + WiFi.begin(WIFI_SSID, WIFI_PASS); + Serial.print("Connecting."); + while (WiFi.status() != WL_CONNECTED) { + Serial.print("."); + vTaskDelay(500); // this is similar to yield and allows the device to go to low power mode + } + Serial.print("WiFi connected - IP address: "); + Serial.println(WiFi.localIP()); + while (!ESP_WIFI.connect(WIFI_HOST_IP_ADDRESS, WIFI_HOST_PORT)) { + Serial.print("Failed to connect to "); + Serial.print(WIFI_HOST_IP_ADDRESS); + Serial.print(":"); + Serial.print(WIFI_HOST_PORT); + Serial.println(); + vTaskDelay(5000); // wait 5 seconds + } + Serial.println("Connected to host"); + } pinMode(BUTTON_INPUT_PIN, INPUT_PULLUP); } -void loop() { +// We return a pointer to the object since we can't return an abstract class +Stream* getCommProtocol() { + switch (CURRENT_MODE) + { + case RECORDER_SERIAL: + return &Serial; + case RECORDER_BLUETOOTH: + return &ESP_BT; + case RECORDER_WIFI: + return &ESP_WIFI; + } +} + +void loop() +{ // Read accel data - lis.read(); + lis.read(); int buttonVal = digitalRead(BUTTON_INPUT_PIN); - if(INCLUDE_TIMESTAMP){ - Serial.print(millis()); - Serial.print(", "); + Stream* stream = getCommProtocol(); + + if (INCLUDE_TIMESTAMP) + { + stream->print(millis()); + stream->print(", "); } - - Serial.print(lis.x); - Serial.print(", "); - Serial.print(lis.y); - Serial.print(", "); - Serial.print(lis.z); - Serial.print(", "); - Serial.print(!buttonVal); // because pull-up - Serial.println(); - - if(DELAY_MS > 0){ + stream->print(lis.x); + stream->print(", "); + stream->print(lis.y); + stream->print(", "); + stream->print(lis.z); + stream->print(", "); + stream->print(!buttonVal); // because pull-up + stream->println(); + + while (!lis.haveNewData()) + { delay(DELAY_MS); } } diff --git a/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/readme.md b/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/readme.md new file mode 100644 index 0000000..16acf03 --- /dev/null +++ b/Processing/GestureRecorder/Arduino/LIS3DHGestureRecorder/readme.md @@ -0,0 +1,27 @@ +# SERIAL MODE + +# BLUETOOTH MODE +Starting the device in this mode will create a new bluetooth device named by default ESP32 Gesture Recorder. +You can change it if you really like. +On windows, the easiest way to add this is as follows: +1. go to bluetooth settings. +2. On the right side it says "More Bluetooth Options". +3. A new dialog box will open and it will say "Com Ports" in one of the tabs at the top. +4. Go to that tab (see image below): + +![dialog box](DialogBox.PNG "The COM Ports Dialog box") + +5. Click Add +6. A new dialog will pop up, giving you different options (it may take a moment to come up) +7. Select outgoing since your PC will be initializing the connection +8. Select the bluetooth device in the dropdown for "Device that will use the com port". +It will be ESP32 Gesture Recorder unless you changed it +9. Hit okay +10. Take note of the name of the COM port that was created. + +# WIFI MODE +Start the processing server with WIFI turned on. +It will spit out a "Started the server at 192.168.x.x" +Put that address in the arduino sketch and upload it to your ESP32. +Make sure to open up the port you've selected in your firewall for the computer running processing. +I had trouble with this even when I added explicit rules. \ No newline at end of file diff --git a/Processing/GestureRecorder/GestureRecorder.pde b/Processing/GestureRecorder/GestureRecorder.pde index 674ae76..7184134 100644 --- a/Processing/GestureRecorder/GestureRecorder.pde +++ b/Processing/GestureRecorder/GestureRecorder.pde @@ -22,6 +22,7 @@ * Future ideas: * - Shows capture snapshot? * - GUI to select which gesture to record + * - Print out which mode we are using for comm (BLUETOOTH, Serial, WiFi) * */ @@ -29,6 +30,7 @@ import processing.serial.*; import java.awt.Rectangle; import java.io.BufferedWriter; import java.io.FileWriter; +import processing.net.*; final String GESTURE_DIR_NAME = "Gestures"; final String FULL_DATASTREAM_RECORDING_FILENAME = "fulldatastream.csv"; @@ -45,7 +47,13 @@ final int DISPLAY_TIMEWINDOW_MS = 1000 * 20; // 20 secs. You can change this to // Make sure to change this! If you're not sure what port your Arduino is using // Run this Processing sketch and look in the console, then change the number accordingly -final int ARDUINO_SERIAL_PORT_INDEX = 0; // CHANGE THIS TO APPROPRIATE PORT! + +final String ARDUINO_SERIAL_PORT_NAME = "COM3"; // CHANGE THIS TO APPROPRIATE PORT! +// WIFI SETTINGS +// This should be false even in bluetooth mode since that just emulates a com port +final boolean ARDUINO_WIFI_MODE = false; // CHANGE THIS TO TRUE IF AND ONLY IF YOU ARE USING WIFI MODE +final int ARDUINO_WIFI_PORT = 10002; // THIS IS THE PORT THAT THEY WILL TRY TO COMMUNICATE OVER + // Set the baud rate to same as Arduino (e.g., 9600 or 115200) final int SERIAL_BAUD_RATE = 115200; // CHANGE THIS TO MATCH BAUD RATE ON ARDUINO @@ -64,6 +72,9 @@ PrintWriter _printWriterAllData; // The serial port is necessary to read data in from the Arduino Serial _serialPort; +// Wifi server +Server _wifiServer; + long _currentXMin; // the far left x-axis value on the graph Rectangle _legendRect; // location and drawing area of the legend boolean _dynamicYAxis = true; @@ -123,48 +134,62 @@ void setup() { e.printStackTrace(); } - // Print to console all the available serial ports - String [] serialPorts = getAndPrintSerialPortInfo(); + if (ARDUINO_WIFI_MODE) { + // https://www.processing.org/reference/libraries/net/Server.html + _wifiServer = new Server(this, ARDUINO_WIFI_PORT); + print("Starting server"); + while (!_wifiServer.active()) { + print("."); + delay(50); + } + println("\nWe have started a server at "+Server.ip()); + thread("processServer"); // start a thread that reads for the server + } else { - if (serialPorts.length <= 0) { - println("You appear to have *ZERO* active serial ports. Make sure your Arduino is plugged in. Exiting..."); - exit(); - } else if (ARDUINO_SERIAL_PORT_INDEX > serialPorts.length) { - println("You set ARDUINO_SERIAL_PORT_INDEX = " + ARDUINO_SERIAL_PORT_INDEX + "; however, you only have " + - serialPorts.length + " total serial ports."); - println("Please make sure your Arduino is plugged in. Then update ARDUINO_SERIAL_PORT_INDEX to the appropriate index."); - println("Exiting..."); - exit(); - return; - } + // Print to console all the available serial ports + String [] serialPorts = getAndPrintSerialPortInfo(); - // Open the serial port - try { - println("Attempting to initialize the serial port at index " + ARDUINO_SERIAL_PORT_INDEX + " with baud rate = " + SERIAL_BAUD_RATE); - println("This index corresponds to serial port " + serialPorts[ARDUINO_SERIAL_PORT_INDEX]); - _serialPort = new Serial(this, serialPorts[ARDUINO_SERIAL_PORT_INDEX], SERIAL_BAUD_RATE); - - // We need to clear the port (or sometimes there is leftover data) - // (Yes, this is strange, but once we implemented this clear, - // we were no longer seeing garbage data in the beginning of our printwriter - // strea,) - _serialPort.clear(); - } - catch(Exception e) { - println("Serial port exception: " + e); - e.printStackTrace(); - exit(); - return; - } + if (serialPorts.length <= 0) { + println("You appear to have *ZERO* active serial ports. Make sure your Arduino is plugged in. Exiting..."); + exit(); + return; + } - if (_serialPort == null) { - println("Could not initialize the serial port at " + ARDUINO_SERIAL_PORT_INDEX + ". Exiting..."); - exit(); - return; - } + int ARDUINO_SERIAL_PORT_INDEX = getIndexOfArduinoSerialPort(ARDUINO_SERIAL_PORT_NAME); + + if (ARDUINO_SERIAL_PORT_INDEX == -1) { + println("Could not find the serial port: " + ARDUINO_SERIAL_PORT_NAME + ". Exiting..."); + exit(); + return; + } + + // Open the serial port + try { + println("Attempting to initialize the serial port " + ARDUINO_SERIAL_PORT_NAME + " with baud rate = " + SERIAL_BAUD_RATE); + println("This index corresponds to serial port at index: " + ARDUINO_SERIAL_PORT_INDEX); + _serialPort = new Serial(this, serialPorts[ARDUINO_SERIAL_PORT_INDEX], SERIAL_BAUD_RATE); + + // We need to clear the port (or sometimes there is leftover data) + // (Yes, this is strange, but once we implemented this clear, + // we were no longer seeing garbage data in the beginning of our printwriter stream) + _serialPort.clear(); + } + catch(Exception e) { + println("Serial port exception: " + e); + e.printStackTrace(); + exit(); + return; + } - // Don't generate a serialEvent() unless you get a newline character: - _serialPort.bufferUntil('\n'); + if (_serialPort == null) { + println("Could not initialize the serial port at " + ARDUINO_SERIAL_PORT_INDEX + ". Exiting..."); + exit(); + return; + } + + // Don't generate a serialEvent() unless you get a newline character: + _serialPort.bufferUntil('\n'); + } _currentXMin = System.currentTimeMillis() - DISPLAY_TIMEWINDOW_MS; @@ -378,9 +403,9 @@ void toggleGestureRecording() { } /** - * Convenience method that returns the number of gestures recoreded with the given name + * Convenience method that returns the number of gestures recorded with the given name */ -int getNumGesturesRecordedWithName(String name) { +int getNumGesturesRecordedWithName(String name){ return _mapGestureNameToRecordedCount.containsKey(name) ? (int)_mapGestureNameToRecordedCount.get(name) : 0; } @@ -420,6 +445,9 @@ void drawDebugInfo() { } } +/** + * Draws the y-axis ticks and labels + */ void drawYAxis() { final int numYTickMarks = 5; final int tickMarkWidth = 6; @@ -468,16 +496,28 @@ String[] getAndPrintSerialPortInfo() { String[] listOfSerialPorts = Serial.list(); printArray(listOfSerialPorts); println("** END SERIAL PORT LIST**"); - println("Make sure to change ARDUINO_SERIAL_PORT_INDEX to the correct port number!"); - if (listOfSerialPorts.length > 0) { + if(listOfSerialPorts.length > 0){ String firstPortName = listOfSerialPorts[0]; println("For example, if your Arduino is on port " + firstPortName + - " then you would set ARDUINO_SERIAL_PORT_INDEX = " + 0); + " then you would set ARDUINO_SERIAL_PORT_NAME = " + firstPortName); } return listOfSerialPorts; } +/** + * Convenience method that returns index of serialPortName or -1 if not found + */ +int getIndexOfArduinoSerialPort(String serialPortName) { + String[] listOfSerialPorts = Serial.list(); + for (int i=0; i< listOfSerialPorts.length; i++) { + if (listOfSerialPorts[i].equals(serialPortName)){ + return i; + } + } + return -1; +} + /** * Converts a sensor value to a y-pixel value and returns the y-pixel value */ @@ -664,9 +704,6 @@ void drawGestureRecordingAnnotations() { * Note: this method is called by a different thread (EventThread) than the draw() method (Animation Thread) */ void serialEvent (Serial myPort) { - long currentTimestampMs = System.currentTimeMillis(); - _currentXMin = currentTimestampMs - DISPLAY_TIMEWINDOW_MS; - //println(Thread.currentThread()); String inString = ""; @@ -682,6 +719,40 @@ void serialEvent (Serial myPort) { return; } + processInputFromClient(inString); +} + +void processServer() { + if (_wifiServer == null){ + return; + } + + Client thisClient = _wifiServer.available(); + + // If the client is not null, and says something, display what it said + if (thisClient !=null) { + String whatClientSaid = thisClient.readString(); + if (whatClientSaid != null) { + processInputFromClient(whatClientSaid); + println(thisClient.ip() + "t" + whatClientSaid); + } + } + delay(10); +} + +// ServerEvent message is generated when a new client connects +// to an existing server. +void serverEvent(Server someServer, Client someClient) { + println("We have a new client: " + someClient.ip()); +} + +/** + * This processes data from a client (serial or wifi) + */ +void processInputFromClient(String inString) { + long currentTimestampMs = System.currentTimeMillis(); + _currentXMin = currentTimestampMs - DISPLAY_TIMEWINDOW_MS; + try { if (inString != null) { int [] data; @@ -764,6 +835,11 @@ void exit() { _printWriterAllData.close(); } } + + if (_wifiServer != null) { + _wifiServer.stop(); + } + super.exit(); }