From 2ea1cd1d0f69b215741367f15cc438b6e2ea82af Mon Sep 17 00:00:00 2001 From: m10653 Date: Tue, 5 Mar 2024 11:23:03 +0000 Subject: [PATCH 01/14] Proof of concept optimization --- package.json | 3 +- src/shared/log/LogField.ts | 154 ++++++++++++++++++++++--------------- tsconfig.json | 10 ++- 3 files changed, 102 insertions(+), 65 deletions(-) diff --git a/package.json b/package.json index 529532c2..05fcea4b 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "jsonfile": "^6.1.0", "ssh2": "^1.14.0", "tesseract.js": "^5.0.3", - "ytdl-core": "https://github.com/khlevon/node-ytdl-core.git#v4.11.5-patch.2" + "ytdl-core": "https://github.com/khlevon/node-ytdl-core.git#v4.11.5-patch.2", + "js-sdsl": "^4.4.2" }, "prettier": { "printWidth": 120, diff --git a/src/shared/log/LogField.ts b/src/shared/log/LogField.ts index 9f34a103..adc3d958 100644 --- a/src/shared/log/LogField.ts +++ b/src/shared/log/LogField.ts @@ -10,11 +10,14 @@ import { LogValueSetString, LogValueSetStringArray } from "./LogValueSets"; - +import { OrderedSet } from "js-sdsl"; +function cmp(x: { timestamp: number; values: number }, y: { timestamp: number; values: number }): number { + return x.timestamp - y.timestamp; +} /** A full log field that contains data. */ export default class LogField { private type: LoggableType; - private data: LogValueSetAny = { timestamps: [], values: [] }; + private data: OrderedSet<{ timestamp: number; values: number }> = new OrderedSet([], cmp); public structuredType: string | null = null; public wpilibType: string | null = null; // Original type from WPILOG & NT4 public metadataString = ""; @@ -39,42 +42,39 @@ export default class LogField { /** Returns the full set of ordered timestamps. */ getTimestamps(): number[] { - return this.data.timestamps; + return Array.from(this.data, (elem) => elem.timestamp); + } + getValues(): number[] { + return Array.from(this.data, (elem) => elem.values); } /** Clears all data before the provided timestamp. */ clearBeforeTime(timestamp: number) { - while (this.data.timestamps.length >= 2 && this.data.timestamps[1] < timestamp) { - this.data.timestamps.shift(); - this.data.values.shift(); - this.stripingReference = !this.stripingReference; - } - if (this.data.timestamps.length > 0 && this.data.timestamps[0] < timestamp) { - this.data.timestamps[0] = timestamp; + var itr = this.data.begin(); + while (itr.isAccessible) { + if (itr.pointer.timestamp >= timestamp) { + break; + } + this.data.eraseElementByIterator(itr); + itr.next(); } } /** Returns the values in the specified timestamp range. */ getRange(start: number, end: number): LogValueSetAny { - let timestamps: number[]; - let values: any[]; - - let startValueIndex = this.data.timestamps.findIndex((x) => x > start); - if (startValueIndex === -1) { - startValueIndex = this.data.timestamps.length - 1; - } else if (startValueIndex !== 0) { - startValueIndex -= 1; - } + var timestamps = []; + var values = []; - let endValueIndex = this.data.timestamps.findIndex((x) => x >= end); - if (endValueIndex === -1 || endValueIndex === this.data.timestamps.length - 1) { - // Extend to end of timestamps - timestamps = this.data.timestamps.slice(startValueIndex); - values = this.data.values.slice(startValueIndex); - } else { - timestamps = this.data.timestamps.slice(startValueIndex, endValueIndex + 1); - values = this.data.values.slice(startValueIndex, endValueIndex + 1); + var itr = this.data.reverseUpperBound({ timestamp: start, values: 0 }); + while (itr.isAccessible()) { + if (itr.pointer.timestamp > end) { + break; + } + timestamps.push(itr.pointer.timestamp); + values.push(itr.pointer.values); + itr.next(); } + return { timestamps: timestamps, values: values }; } @@ -112,45 +112,73 @@ export default class LogField { getStringArray(start: number, end: number): LogValueSetStringArray | undefined { if (this.type === LoggableType.StringArray) return this.getRange(start, end); } + // private findIndex(func: any) /** Inserts a new value at the correct index. */ private putData(timestamp: number, value: any) { + // return if (value === null) return; // Find position to insert based on timestamp - let insertIndex: number; - if (this.data.timestamps.length > 0 && timestamp > this.data.timestamps[this.data.timestamps.length - 1]) { - // There's a good chance this data is at the end of the log, so check that first - insertIndex = this.data.timestamps.length; - } else { - // Adding in the middle, find where to insert it - let alreadyExists = false; - insertIndex = - this.data.timestamps.findLastIndex((x) => { - if (alreadyExists) return; - if (x === timestamp) alreadyExists = true; - return x < timestamp; - }) + 1; - if (alreadyExists) { - this.data.values[this.data.timestamps.indexOf(timestamp)] = value; - return; - } - } + // let insertIndex: number; + // // this.data.rEnd(). + // if (this.data.length > 0 && timestamp > this.data.timestamps[this.data.timestamps.length - 1]) { + // // There's a good chance this data is at the end of the log, so check that first + // insertIndex = this.data.timestamps.length; + // } else { + // // Adding in the middle, find where to insert it + // let alreadyExists = false; + // insertIndex = + // this.data.timestamps.findLastIndex((x) => { + // if (alreadyExists) return; + // if (x === timestamp) alreadyExists = true; + // return x < timestamp; + // }) + 1; + // if (alreadyExists) { + // this.data.values[this.data.timestamps.indexOf(timestamp)] = value; + // return; + // } + // } + // if (insertIndex >= 0) { + // console.log("Looking for past:" + value); + // } + //find insert point if set overwrite it with new data + // if not + // check element behind + // if that element is not equal set the value + // else do nothing + // this.data.end(); + var record = { timestamp: timestamp, values: value }; + this.data.insert(record); + return; + var needle = this.data.find(record); - // Compare to adjacent values - if (insertIndex > 0 && logValuesEqual(this.type, value, this.data.values[insertIndex - 1])) { - // Same as the previous value - } else if ( - insertIndex < this.data.values.length && - logValuesEqual(this.type, value, this.data.values[insertIndex]) - ) { - // Same as the next value - this.data.timestamps[insertIndex] = timestamp; + if (needle.isAccessible()) { + this.data.insert(record); } else { - // New value - this.data.timestamps.splice(insertIndex, 0, timestamp); - this.data.values.splice(insertIndex, 0, value); + var pastElem = this.data.reverseUpperBound(record); + if (pastElem.isAccessible()) { + if (logValuesEqual(this.type, value, pastElem.pointer)) { + this.data.updateKeyByIterator(pastElem, record); + } else { + this.data.insert(record); + } + } } + // // Compare to adjacent values + // if (insertIndex > 0 && logValuesEqual(this.type, value, this.data.values[insertIndex - 1])) { + // // Same as the previous value + // } else if ( + // insertIndex < this.data.values.length && + // logValuesEqual(this.type, value, this.data.values[insertIndex]) + // ) { + // // Same as the next value + // this.data.timestamps[insertIndex] = timestamp; + // } else { + // // New value + // this.data.timestamps.splice(insertIndex, 0, timestamp); + // this.data.values.splice(insertIndex, 0, value); + // } } /** Writes a new Raw value to the field. */ @@ -204,10 +232,12 @@ export default class LogField { /** Returns a serialized version of the data from this field. */ toSerialized(): any { + // console.log(this.getValues()); + // console.log(this.getTimestamps()); return { type: this.type, - timestamps: this.data.timestamps, - values: this.data.values, + timestamps: this.getTimestamps(), + values: this.getValues(), structuredType: this.structuredType, wpilibType: this.wpilibType, metadataString: this.metadataString, @@ -218,11 +248,9 @@ export default class LogField { /** Creates a new field based on the data from `toSerialized()` */ static fromSerialized(serializedData: any) { + console.log("ASDFASDFASDFSDAFASD_A_SDF_ASD_FA_SD_F"); let field = new LogField(serializedData.type); - field.data = { - timestamps: serializedData.timestamps, - values: serializedData.values - }; + field.data = new OrderedSet<{ timestamp: number; values: number }>([], cmp); field.structuredType = serializedData.structuredType; field.wpilibType = serializedData.wpilibType; field.metadataString = serializedData.metadataString; diff --git a/tsconfig.json b/tsconfig.json index 5f058c14..2584b40d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,4 +1,11 @@ { + "ts-node": { + // these options are overrides used only by ts-node + // same as the --compilerOptions flag and the TS_NODE_COMPILER_OPTIONS environment variable + "compilerOptions": { + "module": "commonjs" + } + }, "compilerOptions": { "target": "ESNext", "module": "ESNext", @@ -7,6 +14,7 @@ "forceConsistentCasingInFileNames": true, "strict": true, "removeComments": true, - "resolveJsonModule": true + "resolveJsonModule": true, + "sourceMap": true } } From 3b26e95e7bf04b52e013dc17e6c5ebea0ff0a1ce Mon Sep 17 00:00:00 2001 From: m10653 Date: Tue, 5 Mar 2024 13:11:24 +0000 Subject: [PATCH 02/14] Tmp build changes for debug. --- package.json | 8 ++++---- rollup.config.mjs | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 05fcea4b..893eb1fc 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,6 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.2.1", "@rollup/plugin-replace": "^5.0.2", - "@rollup/plugin-typescript": "11.1.3", "@types/chart.js": "^2.9.38", "@types/download": "^8.0.2", "@types/heatmap.js": "2.0.38", @@ -51,20 +50,21 @@ "rollup": "^3.29.2", "rollup-plugin-cleanup": "^3.2.1", "rollup-plugin-re": "^1.0.7", + "rollup-plugin-typescript2": "^0.36.0", "simple-statistics": "^7.8.3", "three": "^0.156.1", "tslib": "^2.6.2", - "typescript": "5.2.2" + "typescript": "^5.2.2" }, "dependencies": { "check-disk-space": "^3.4.0", "download": "^8.0.0", "electron-fetch": "^1.9.1", + "js-sdsl": "^4.4.2", "jsonfile": "^6.1.0", "ssh2": "^1.14.0", "tesseract.js": "^5.0.3", - "ytdl-core": "https://github.com/khlevon/node-ytdl-core.git#v4.11.5-patch.2", - "js-sdsl": "^4.4.2" + "ytdl-core": "https://github.com/khlevon/node-ytdl-core.git#v4.11.5-patch.2" }, "prettier": { "printWidth": 120, diff --git a/rollup.config.mjs b/rollup.config.mjs index 95f7e4f1..4818dd55 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -2,7 +2,7 @@ import commonjs from "@rollup/plugin-commonjs"; import json from "@rollup/plugin-json"; import nodeResolve from "@rollup/plugin-node-resolve"; import replace from "@rollup/plugin-replace"; -import typescript from "@rollup/plugin-typescript"; +import typescript from "rollup-plugin-typescript2"; import fs from "fs"; import cleanup from "rollup-plugin-cleanup"; import replaceRegEx from "rollup-plugin-re"; @@ -16,9 +16,10 @@ function bundle(input, output, isMain, external = []) { format: isMain ? "cjs" : "es" }, context: "this", + treeshake: true, external: external, plugins: [ - typescript(), + typescript({ check: false, sourcemap: false }), nodeResolve(), commonjs(), cleanup(), From ee66954a41ca221f42d5b086a43229b326f548d3 Mon Sep 17 00:00:00 2001 From: m10653 Date: Tue, 5 Mar 2024 13:11:47 +0000 Subject: [PATCH 03/14] Bug fixes --- src/shared/log/LogField.ts | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/shared/log/LogField.ts b/src/shared/log/LogField.ts index adc3d958..e2470179 100644 --- a/src/shared/log/LogField.ts +++ b/src/shared/log/LogField.ts @@ -56,7 +56,7 @@ export default class LogField { break; } this.data.eraseElementByIterator(itr); - itr.next(); + // itr.next(); } } @@ -65,8 +65,9 @@ export default class LogField { var timestamps = []; var values = []; - var itr = this.data.reverseUpperBound({ timestamp: start, values: 0 }); + var itr = this.data.lowerBound({ timestamp: start, values: 0 }); while (itr.isAccessible()) { + console.log(itr.pointer); if (itr.pointer.timestamp > end) { break; } @@ -149,20 +150,22 @@ export default class LogField { // else do nothing // this.data.end(); var record = { timestamp: timestamp, values: value }; - this.data.insert(record); - return; + // this.data.insert(record); + // return; var needle = this.data.find(record); if (needle.isAccessible()) { - this.data.insert(record); + this.data.updateKeyByIterator(needle, record); } else { var pastElem = this.data.reverseUpperBound(record); if (pastElem.isAccessible()) { - if (logValuesEqual(this.type, value, pastElem.pointer)) { - this.data.updateKeyByIterator(pastElem, record); + if (logValuesEqual(this.type, value, pastElem.pointer.values)) { + this.data.updateKeyByIterator(pastElem, record); // } else { this.data.insert(record); } + } else { + this.data.insert(record); } } // // Compare to adjacent values @@ -248,9 +251,15 @@ export default class LogField { /** Creates a new field based on the data from `toSerialized()` */ static fromSerialized(serializedData: any) { - console.log("ASDFASDFASDFSDAFASD_A_SDF_ASD_FA_SD_F"); + // console.log("ASDFASDFASDFSDAFASD_A_SDF_ASD_FA_SD_F"); let field = new LogField(serializedData.type); - field.data = new OrderedSet<{ timestamp: number; values: number }>([], cmp); + field.data = new OrderedSet( + serializedData.timestamps.map((time: number, index: number) => ({ + timestamp: time, + values: serializedData.values[index] + })), + cmp + ); field.structuredType = serializedData.structuredType; field.wpilibType = serializedData.wpilibType; field.metadataString = serializedData.metadataString; From 9e71486d69113fc4b2ff280b029ffda574b03981 Mon Sep 17 00:00:00 2001 From: m10653 Date: Tue, 5 Mar 2024 13:46:10 +0000 Subject: [PATCH 04/14] Remove Unused print --- src/shared/log/LogField.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/shared/log/LogField.ts b/src/shared/log/LogField.ts index e2470179..7d115663 100644 --- a/src/shared/log/LogField.ts +++ b/src/shared/log/LogField.ts @@ -67,7 +67,6 @@ export default class LogField { var itr = this.data.lowerBound({ timestamp: start, values: 0 }); while (itr.isAccessible()) { - console.log(itr.pointer); if (itr.pointer.timestamp > end) { break; } From e9ff87eefddeba30edb841b50eb5b4a1ece23e62 Mon Sep 17 00:00:00 2001 From: m10653 Date: Tue, 5 Mar 2024 14:08:05 +0000 Subject: [PATCH 05/14] Basic Cleanup --- src/shared/log/LogField.ts | 62 ++++---------------------------------- 1 file changed, 6 insertions(+), 56 deletions(-) diff --git a/src/shared/log/LogField.ts b/src/shared/log/LogField.ts index 7d115663..f1e9147c 100644 --- a/src/shared/log/LogField.ts +++ b/src/shared/log/LogField.ts @@ -112,75 +112,28 @@ export default class LogField { getStringArray(start: number, end: number): LogValueSetStringArray | undefined { if (this.type === LoggableType.StringArray) return this.getRange(start, end); } - // private findIndex(func: any) /** Inserts a new value at the correct index. */ private putData(timestamp: number, value: any) { - // return if (value === null) return; - - // Find position to insert based on timestamp - // let insertIndex: number; - // // this.data.rEnd(). - // if (this.data.length > 0 && timestamp > this.data.timestamps[this.data.timestamps.length - 1]) { - // // There's a good chance this data is at the end of the log, so check that first - // insertIndex = this.data.timestamps.length; - // } else { - // // Adding in the middle, find where to insert it - // let alreadyExists = false; - // insertIndex = - // this.data.timestamps.findLastIndex((x) => { - // if (alreadyExists) return; - // if (x === timestamp) alreadyExists = true; - // return x < timestamp; - // }) + 1; - // if (alreadyExists) { - // this.data.values[this.data.timestamps.indexOf(timestamp)] = value; - // return; - // } - // } - // if (insertIndex >= 0) { - // console.log("Looking for past:" + value); - // } - //find insert point if set overwrite it with new data - // if not - // check element behind - // if that element is not equal set the value - // else do nothing - // this.data.end(); var record = { timestamp: timestamp, values: value }; - // this.data.insert(record); - // return; var needle = this.data.find(record); if (needle.isAccessible()) { - this.data.updateKeyByIterator(needle, record); + this.data.updateKeyByIterator(needle, record); // Overwrite Exising record if they share a timestamp } else { - var pastElem = this.data.reverseUpperBound(record); + var pastElem = this.data.reverseUpperBound(record); // Get prev record if (pastElem.isAccessible()) { if (logValuesEqual(this.type, value, pastElem.pointer.values)) { - this.data.updateKeyByIterator(pastElem, record); // + // Check is prev data is the same + this.data.updateKeyByIterator(pastElem, record); // replace prev timestamp with new timestamp and same value } else { - this.data.insert(record); + this.data.insert(record); // If different insert } } else { - this.data.insert(record); + this.data.insert(record); // if no data exists insert } } - // // Compare to adjacent values - // if (insertIndex > 0 && logValuesEqual(this.type, value, this.data.values[insertIndex - 1])) { - // // Same as the previous value - // } else if ( - // insertIndex < this.data.values.length && - // logValuesEqual(this.type, value, this.data.values[insertIndex]) - // ) { - // // Same as the next value - // this.data.timestamps[insertIndex] = timestamp; - // } else { - // // New value - // this.data.timestamps.splice(insertIndex, 0, timestamp); - // this.data.values.splice(insertIndex, 0, value); - // } } /** Writes a new Raw value to the field. */ @@ -234,8 +187,6 @@ export default class LogField { /** Returns a serialized version of the data from this field. */ toSerialized(): any { - // console.log(this.getValues()); - // console.log(this.getTimestamps()); return { type: this.type, timestamps: this.getTimestamps(), @@ -250,7 +201,6 @@ export default class LogField { /** Creates a new field based on the data from `toSerialized()` */ static fromSerialized(serializedData: any) { - // console.log("ASDFASDFASDFSDAFASD_A_SDF_ASD_FA_SD_F"); let field = new LogField(serializedData.type); field.data = new OrderedSet( serializedData.timestamps.map((time: number, index: number) => ({ From de134304c8aa68c4d4cad99ced0dcad9b6576789 Mon Sep 17 00:00:00 2001 From: m10653 Date: Tue, 5 Mar 2024 14:09:18 +0000 Subject: [PATCH 06/14] resolve type errors --- src/shared/log/LogField.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shared/log/LogField.ts b/src/shared/log/LogField.ts index f1e9147c..4ab36ead 100644 --- a/src/shared/log/LogField.ts +++ b/src/shared/log/LogField.ts @@ -42,10 +42,10 @@ export default class LogField { /** Returns the full set of ordered timestamps. */ getTimestamps(): number[] { - return Array.from(this.data, (elem) => elem.timestamp); + return Array.from(this.data, (elem: { timestamp: number; values: number }) => elem.timestamp); } getValues(): number[] { - return Array.from(this.data, (elem) => elem.values); + return Array.from(this.data, (elem: { timestamp: number; values: number }) => elem.values); } /** Clears all data before the provided timestamp. */ From 925e6f3a345de4f823d9d444aea15f12d53364c8 Mon Sep 17 00:00:00 2001 From: m10653 Date: Wed, 6 Mar 2024 08:44:30 +0000 Subject: [PATCH 07/14] Replace SortedSet with sorting at end --- src/shared/log/LogField.ts | 126 +++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 54 deletions(-) diff --git a/src/shared/log/LogField.ts b/src/shared/log/LogField.ts index 4ab36ead..6262da05 100644 --- a/src/shared/log/LogField.ts +++ b/src/shared/log/LogField.ts @@ -10,14 +10,17 @@ import { LogValueSetString, LogValueSetStringArray } from "./LogValueSets"; -import { OrderedSet } from "js-sdsl"; -function cmp(x: { timestamp: number; values: number }, y: { timestamp: number; values: number }): number { - return x.timestamp - y.timestamp; -} + +type LogRecord = { + timestamp: number; + value: any; + index: number; +}; /** A full log field that contains data. */ export default class LogField { private type: LoggableType; - private data: OrderedSet<{ timestamp: number; values: number }> = new OrderedSet([], cmp); + private data: LogValueSetAny = { timestamps: [], values: [] }; + private rawData: LogRecord[] = []; public structuredType: string | null = null; public wpilibType: string | null = null; // Original type from WPILOG & NT4 public metadataString = ""; @@ -42,39 +45,42 @@ export default class LogField { /** Returns the full set of ordered timestamps. */ getTimestamps(): number[] { - return Array.from(this.data, (elem: { timestamp: number; values: number }) => elem.timestamp); - } - getValues(): number[] { - return Array.from(this.data, (elem: { timestamp: number; values: number }) => elem.values); + return this.data.timestamps; } /** Clears all data before the provided timestamp. */ clearBeforeTime(timestamp: number) { - var itr = this.data.begin(); - while (itr.isAccessible) { - if (itr.pointer.timestamp >= timestamp) { - break; - } - this.data.eraseElementByIterator(itr); - // itr.next(); + while (this.data.timestamps.length >= 2 && this.data.timestamps[1] < timestamp) { + this.data.timestamps.shift(); + this.data.values.shift(); + this.stripingReference = !this.stripingReference; + } + if (this.data.timestamps.length > 0 && this.data.timestamps[0] < timestamp) { + this.data.timestamps[0] = timestamp; } } /** Returns the values in the specified timestamp range. */ getRange(start: number, end: number): LogValueSetAny { - var timestamps = []; - var values = []; - - var itr = this.data.lowerBound({ timestamp: start, values: 0 }); - while (itr.isAccessible()) { - if (itr.pointer.timestamp > end) { - break; - } - timestamps.push(itr.pointer.timestamp); - values.push(itr.pointer.values); - itr.next(); + let timestamps: number[]; + let values: any[]; + + let startValueIndex = this.data.timestamps.findIndex((x) => x > start); + if (startValueIndex === -1) { + startValueIndex = this.data.timestamps.length - 1; + } else if (startValueIndex !== 0) { + startValueIndex -= 1; } + let endValueIndex = this.data.timestamps.findIndex((x) => x >= end); + if (endValueIndex === -1 || endValueIndex === this.data.timestamps.length - 1) { + // Extend to end of timestamps + timestamps = this.data.timestamps.slice(startValueIndex); + values = this.data.values.slice(startValueIndex); + } else { + timestamps = this.data.timestamps.slice(startValueIndex, endValueIndex + 1); + values = this.data.values.slice(startValueIndex, endValueIndex + 1); + } return { timestamps: timestamps, values: values }; } @@ -116,24 +122,7 @@ export default class LogField { /** Inserts a new value at the correct index. */ private putData(timestamp: number, value: any) { if (value === null) return; - var record = { timestamp: timestamp, values: value }; - var needle = this.data.find(record); - - if (needle.isAccessible()) { - this.data.updateKeyByIterator(needle, record); // Overwrite Exising record if they share a timestamp - } else { - var pastElem = this.data.reverseUpperBound(record); // Get prev record - if (pastElem.isAccessible()) { - if (logValuesEqual(this.type, value, pastElem.pointer.values)) { - // Check is prev data is the same - this.data.updateKeyByIterator(pastElem, record); // replace prev timestamp with new timestamp and same value - } else { - this.data.insert(record); // If different insert - } - } else { - this.data.insert(record); // if no data exists insert - } - } + this.rawData.push({ timestamp: timestamp, value: value, index: this.rawData.length }); } /** Writes a new Raw value to the field. */ @@ -187,10 +176,14 @@ export default class LogField { /** Returns a serialized version of the data from this field. */ toSerialized(): any { + if (this.data.timestamps.length == 0) { + this.sortAndProcess(); + } + return { type: this.type, - timestamps: this.getTimestamps(), - values: this.getValues(), + timestamps: this.data.timestamps, + values: this.data.values, structuredType: this.structuredType, wpilibType: this.wpilibType, metadataString: this.metadataString, @@ -198,17 +191,42 @@ export default class LogField { typeWarning: this.typeWarning }; } + private sortAndProcess() { + this.rawData.sort((a: LogRecord, b: LogRecord) => { + let cmp = a.timestamp - b.timestamp; + if (cmp == 0) { + return a.index - b.index; + } else { + return cmp; + } + }); + if (this.rawData.length > 0) { + // Bootstrap first value + this.data.timestamps.push(this.rawData[0].timestamp); + this.data.values.push(this.rawData[0].value); + } + for (let i = 1; i < this.rawData.length; i++) { + if (this.rawData[i].timestamp == this.data.timestamps[this.data.values.length - 1]) { + this.data.values[this.data.values.length - 1] = this.rawData[i].value; + } else if ( + logValuesEqual(this.type, this.data.values[this.data.values.length - 1], this.rawData[i].value) && + i < this.rawData.length + ) { + } else { + this.data.timestamps.push(this.rawData[i].timestamp); + this.data.values.push(this.rawData[i].value); + } + } + this.rawData = []; + } /** Creates a new field based on the data from `toSerialized()` */ static fromSerialized(serializedData: any) { let field = new LogField(serializedData.type); - field.data = new OrderedSet( - serializedData.timestamps.map((time: number, index: number) => ({ - timestamp: time, - values: serializedData.values[index] - })), - cmp - ); + field.data = { + timestamps: serializedData.timestamps, + values: serializedData.values + }; field.structuredType = serializedData.structuredType; field.wpilibType = serializedData.wpilibType; field.metadataString = serializedData.metadataString; From d538ec159b53976904eaf63624fec8fea9f675cc Mon Sep 17 00:00:00 2001 From: m10653 Date: Wed, 6 Mar 2024 08:59:58 +0000 Subject: [PATCH 08/14] Removed Dev build settings --- package.json | 5 ++--- rollup.config.mjs | 5 ++--- tsconfig.json | 10 +--------- 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 893eb1fc..529532c2 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.2.1", "@rollup/plugin-replace": "^5.0.2", + "@rollup/plugin-typescript": "11.1.3", "@types/chart.js": "^2.9.38", "@types/download": "^8.0.2", "@types/heatmap.js": "2.0.38", @@ -50,17 +51,15 @@ "rollup": "^3.29.2", "rollup-plugin-cleanup": "^3.2.1", "rollup-plugin-re": "^1.0.7", - "rollup-plugin-typescript2": "^0.36.0", "simple-statistics": "^7.8.3", "three": "^0.156.1", "tslib": "^2.6.2", - "typescript": "^5.2.2" + "typescript": "5.2.2" }, "dependencies": { "check-disk-space": "^3.4.0", "download": "^8.0.0", "electron-fetch": "^1.9.1", - "js-sdsl": "^4.4.2", "jsonfile": "^6.1.0", "ssh2": "^1.14.0", "tesseract.js": "^5.0.3", diff --git a/rollup.config.mjs b/rollup.config.mjs index 4818dd55..95f7e4f1 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -2,7 +2,7 @@ import commonjs from "@rollup/plugin-commonjs"; import json from "@rollup/plugin-json"; import nodeResolve from "@rollup/plugin-node-resolve"; import replace from "@rollup/plugin-replace"; -import typescript from "rollup-plugin-typescript2"; +import typescript from "@rollup/plugin-typescript"; import fs from "fs"; import cleanup from "rollup-plugin-cleanup"; import replaceRegEx from "rollup-plugin-re"; @@ -16,10 +16,9 @@ function bundle(input, output, isMain, external = []) { format: isMain ? "cjs" : "es" }, context: "this", - treeshake: true, external: external, plugins: [ - typescript({ check: false, sourcemap: false }), + typescript(), nodeResolve(), commonjs(), cleanup(), diff --git a/tsconfig.json b/tsconfig.json index 2584b40d..5f058c14 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,11 +1,4 @@ { - "ts-node": { - // these options are overrides used only by ts-node - // same as the --compilerOptions flag and the TS_NODE_COMPILER_OPTIONS environment variable - "compilerOptions": { - "module": "commonjs" - } - }, "compilerOptions": { "target": "ESNext", "module": "ESNext", @@ -14,7 +7,6 @@ "forceConsistentCasingInFileNames": true, "strict": true, "removeComments": true, - "resolveJsonModule": true, - "sourceMap": true + "resolveJsonModule": true } } From 2a7a5510c8628562f21a959257dd96278f37cec5 Mon Sep 17 00:00:00 2001 From: m10653 Date: Wed, 6 Mar 2024 10:59:54 +0000 Subject: [PATCH 09/14] Revert "Removed Dev build settings" This reverts commit d538ec159b53976904eaf63624fec8fea9f675cc. --- package.json | 5 +++-- rollup.config.mjs | 5 +++-- tsconfig.json | 10 +++++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 529532c2..893eb1fc 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,6 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.2.1", "@rollup/plugin-replace": "^5.0.2", - "@rollup/plugin-typescript": "11.1.3", "@types/chart.js": "^2.9.38", "@types/download": "^8.0.2", "@types/heatmap.js": "2.0.38", @@ -51,15 +50,17 @@ "rollup": "^3.29.2", "rollup-plugin-cleanup": "^3.2.1", "rollup-plugin-re": "^1.0.7", + "rollup-plugin-typescript2": "^0.36.0", "simple-statistics": "^7.8.3", "three": "^0.156.1", "tslib": "^2.6.2", - "typescript": "5.2.2" + "typescript": "^5.2.2" }, "dependencies": { "check-disk-space": "^3.4.0", "download": "^8.0.0", "electron-fetch": "^1.9.1", + "js-sdsl": "^4.4.2", "jsonfile": "^6.1.0", "ssh2": "^1.14.0", "tesseract.js": "^5.0.3", diff --git a/rollup.config.mjs b/rollup.config.mjs index 95f7e4f1..4818dd55 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -2,7 +2,7 @@ import commonjs from "@rollup/plugin-commonjs"; import json from "@rollup/plugin-json"; import nodeResolve from "@rollup/plugin-node-resolve"; import replace from "@rollup/plugin-replace"; -import typescript from "@rollup/plugin-typescript"; +import typescript from "rollup-plugin-typescript2"; import fs from "fs"; import cleanup from "rollup-plugin-cleanup"; import replaceRegEx from "rollup-plugin-re"; @@ -16,9 +16,10 @@ function bundle(input, output, isMain, external = []) { format: isMain ? "cjs" : "es" }, context: "this", + treeshake: true, external: external, plugins: [ - typescript(), + typescript({ check: false, sourcemap: false }), nodeResolve(), commonjs(), cleanup(), diff --git a/tsconfig.json b/tsconfig.json index 5f058c14..2584b40d 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,4 +1,11 @@ { + "ts-node": { + // these options are overrides used only by ts-node + // same as the --compilerOptions flag and the TS_NODE_COMPILER_OPTIONS environment variable + "compilerOptions": { + "module": "commonjs" + } + }, "compilerOptions": { "target": "ESNext", "module": "ESNext", @@ -7,6 +14,7 @@ "forceConsistentCasingInFileNames": true, "strict": true, "removeComments": true, - "resolveJsonModule": true + "resolveJsonModule": true, + "sourceMap": true } } From fefa9fe607065f5c0545c125cc5374b98ea820d7 Mon Sep 17 00:00:00 2001 From: m10653 Date: Wed, 6 Mar 2024 21:53:53 +0000 Subject: [PATCH 10/14] Improve createBlankField Preformance --- src/shared/log/Log.ts | 167 ++++++++++++++++++++++++------------------ 1 file changed, 96 insertions(+), 71 deletions(-) diff --git a/src/shared/log/Log.ts b/src/shared/log/Log.ts index 7f398839..82eda2ba 100644 --- a/src/shared/log/Log.ts +++ b/src/shared/log/Log.ts @@ -25,7 +25,7 @@ export default class Log { private structDecoder = new StructDecoder(); private protoDecoder = new ProtoDecoder(); - private fields: { [id: string]: LogField } = {}; + private fields: Map = new Map(); private generatedParents: Set = new Set(); // Children of these fields are generated private timestampRange: [number, number] | null = null; private enableTimestampSetCache: boolean; @@ -41,8 +41,8 @@ export default class Log { /** Checks if the field exists and registers it if necessary. */ public createBlankField(key: string, type: LoggableType) { - if (key in this.fields) return; - this.fields[key] = new LogField(type); + if (this.fields.has(key)) return; + this.fields.set(key, new LogField(type)); } /** Clears all data before the provided timestamp. */ @@ -105,76 +105,85 @@ export default class Log { /** Returns the constant field type. */ getType(key: string): LoggableType | null { - if (key in this.fields) { - return this.fields[key].getType(); - } else { + let field = this.fields.get(key); + if (field === undefined) { return null; + } else { + return field.getType(); } } /** Returns a boolean that toggles when a value is removed from the field. */ getStripingReference(key: string): boolean { - if (key in this.fields) { - return this.fields[key].getStripingReference(); - } else { + let field = this.fields.get(key); + if (field === undefined) { return false; + } else { + return field.getStripingReference(); } } /** Returns the structured type string for a field. */ getStructuredType(key: string): string | null { - if (key in this.fields) { - return this.fields[key].structuredType; - } else { + let field = this.fields.get(key); + if (field === undefined) { return null; + } else { + return field.structuredType; } } /** Sets the structured type string for a field. */ setStructuredType(key: string, type: string | null) { - if (key in this.fields) { - this.fields[key].structuredType = type; + let field = this.fields.get(key); + if (field !== undefined) { + field.structuredType = type; } } /** Returns the WPILib type string for a field. */ getWpilibType(key: string): string | null { - if (key in this.fields) { - return this.fields[key].wpilibType; - } else { + let field = this.fields.get(key); + if (field === undefined) { return null; + } else { + return field.wpilibType; } } /** Sets the WPILib type string for a field. */ setWpilibType(key: string, type: string) { - if (key in this.fields) { - this.fields[key].wpilibType = type; + let field = this.fields.get(key); + if (field !== undefined) { + field.wpilibType = type; } } /** Returns the metadata string for a field. */ getMetadataString(key: string): string { - if (key in this.fields) { - return this.fields[key].metadataString; - } else { + let field = this.fields.get(key); + if (field === undefined) { return ""; + } else { + return field.metadataString; } } /** Sets the WPILib metadata string for a field. */ setMetadataString(key: string, type: string) { - if (key in this.fields) { - this.fields[key].metadataString = type; + let field = this.fields.get(key); + if (field !== undefined) { + field.metadataString = type; } } /** Returns whether there was an attempt to write a conflicting type to a field. */ getTypeWarning(key: string): boolean { - if (key in this.fields) { - return this.fields[key].typeWarning; - } else { + let field = this.fields.get(key); + if (field === undefined) { return false; + } else { + return field.typeWarning; } } @@ -204,7 +213,7 @@ export default class Log { * that data can be retrieved more quickly for subsequent calls. */ getTimestamps(keys: string[], uuid: string | null = null): number[] { let output: number[] = []; - keys = keys.filter((key) => key in this.fields); + keys = keys.filter((key) => this.fields.has(key)); if (keys.length > 1) { // Multiple fields, read from cache if possible let saveCache = false; @@ -220,12 +229,28 @@ export default class Log { } // Get new data - output = [...new Set(keys.map((key) => this.fields[key].getTimestamps()).flat())]; + output = [ + ...new Set( + keys + .map((key) => { + let elem = this.fields.get(key)?.getTimestamps(); + if (elem === undefined) { + throw new Error("Key:" + key + ", Does not exist in Fields"); + } + return elem; + }) + .flat() + ) + ]; output.sort((a, b) => a - b); if (saveCache && uuid) this.timestampSetCache[uuid].timestamps = output; } else if (keys.length === 1) { // Single field - output = [...this.fields[keys[0]].getTimestamps()]; + let field = this.fields.get(keys[0]); + if (field === undefined) { + throw new Error("Key:" + keys[0] + ", Does not exist in Fields"); + } + output = [...field.getTimestamps()]; } return output; } @@ -286,49 +311,49 @@ export default class Log { /** Reads a set of generic values from the field. */ getRange(key: string, start: number, end: number): LogValueSetAny | undefined { - if (key in this.fields) return this.fields[key].getRange(start, end); + if (this.fields.has(key)) return this.fields.get(key)?.getRange(start, end); } /** Reads a set of Raw values from the field. */ getRaw(key: string, start: number, end: number): LogValueSetRaw | undefined { - if (key in this.fields) return this.fields[key].getRaw(start, end); + if (this.fields.has(key)) return this.fields.get(key)?.getRaw(start, end); } /** Reads a set of Boolean values from the field. */ getBoolean(key: string, start: number, end: number): LogValueSetBoolean | undefined { - if (key in this.fields) return this.fields[key].getBoolean(start, end); + if (this.fields.has(key)) return this.fields.get(key)?.getBoolean(start, end); } /** Reads a set of Number values from the field. */ getNumber(key: string, start: number, end: number): LogValueSetNumber | undefined { - if (key in this.fields) return this.fields[key].getNumber(start, end); + if (this.fields.has(key)) return this.fields.get(key)?.getNumber(start, end); } /** Reads a set of String values from the field. */ getString(key: string, start: number, end: number): LogValueSetString | undefined { - if (key in this.fields) return this.fields[key].getString(start, end); + if (this.fields.has(key)) return this.fields.get(key)?.getString(start, end); } /** Reads a set of BooleanArray values from the field. */ getBooleanArray(key: string, start: number, end: number): LogValueSetBooleanArray | undefined { - if (key in this.fields) return this.fields[key].getBooleanArray(start, end); + if (this.fields.has(key)) return this.fields.get(key)?.getBooleanArray(start, end); } /** Reads a set of NumberArray values from the field. */ getNumberArray(key: string, start: number, end: number): LogValueSetNumberArray | undefined { - if (key in this.fields) return this.fields[key].getNumberArray(start, end); + if (this.fields.has(key)) return this.fields.get(key)?.getNumberArray(start, end); } /** Reads a set of StringArray values from the field. */ getStringArray(key: string, start: number, end: number): LogValueSetStringArray | undefined { - if (key in this.fields) return this.fields[key].getStringArray(start, end); + if (this.fields.has(key)) return this.fields.get(key)?.getStringArray(start, end); } /** Writes a new Raw value to the field. */ putRaw(key: string, timestamp: number, value: Uint8Array) { this.createBlankField(key, LoggableType.Raw); - this.fields[key].putRaw(timestamp, value); - if (this.fields[key].getType() === LoggableType.Raw) { + this.fields.get(key)?.putRaw(timestamp, value); + if (this.fields.get(key)?.getType() === LoggableType.Raw) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } @@ -342,8 +367,8 @@ export default class Log { /** Writes a new Boolean value to the field. */ putBoolean(key: string, timestamp: number, value: boolean) { this.createBlankField(key, LoggableType.Boolean); - this.fields[key].putBoolean(timestamp, value); - if (this.fields[key].getType() === LoggableType.Boolean) { + this.fields.get(key)?.putBoolean(timestamp, value); + if (this.fields.get(key)?.getType() === LoggableType.Boolean) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } } @@ -351,8 +376,8 @@ export default class Log { /** Writes a new Number value to the field. */ putNumber(key: string, timestamp: number, value: number) { this.createBlankField(key, LoggableType.Number); - this.fields[key].putNumber(timestamp, value); - if (this.fields[key].getType() === LoggableType.Number) { + this.fields.get(key)?.putNumber(timestamp, value); + if (this.fields.get(key)?.getType() === LoggableType.Number) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } } @@ -360,8 +385,8 @@ export default class Log { /** Writes a new String value to the field. */ putString(key: string, timestamp: number, value: string) { this.createBlankField(key, LoggableType.String); - this.fields[key].putString(timestamp, value); - if (this.fields[key].getType() === LoggableType.String) { + this.fields.get(key)?.putString(timestamp, value); + if (this.fields.get(key)?.getType() === LoggableType.String) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } @@ -377,15 +402,15 @@ export default class Log { /** Writes a new BooleanArray value to the field. */ putBooleanArray(key: string, timestamp: number, value: boolean[]) { this.createBlankField(key, LoggableType.BooleanArray); - this.fields[key].putBooleanArray(timestamp, value); - if (this.fields[key].getType() === LoggableType.BooleanArray) { + this.fields.get(key)?.putBooleanArray(timestamp, value); + if (this.fields.get(key)?.getType() === LoggableType.BooleanArray) { this.processTimestamp(key, timestamp); this.setGeneratedParent(key); { let lengthKey = key + "/length"; this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); - this.fields[lengthKey].putNumber(timestamp, value.length); + this.fields.get(lengthKey)?.putNumber(timestamp, value.length); } for (let i = 0; i < value.length; i++) { if (this.enableTimestampSetCache) { @@ -394,7 +419,7 @@ export default class Log { } let itemKey = key + "/" + i.toString(); this.createBlankField(itemKey, LoggableType.Boolean); - this.fields[itemKey].putBoolean(timestamp, value[i]); + this.fields.get(itemKey)?.putBoolean(timestamp, value[i]); } } } @@ -402,15 +427,15 @@ export default class Log { /** Writes a new NumberArray value to the field. */ putNumberArray(key: string, timestamp: number, value: number[]) { this.createBlankField(key, LoggableType.NumberArray); - this.fields[key].putNumberArray(timestamp, value); - if (this.fields[key].getType() === LoggableType.NumberArray) { + this.fields.get(key)?.putNumberArray(timestamp, value); + if (this.fields.get(key)?.getType() === LoggableType.NumberArray) { this.processTimestamp(key, timestamp); this.setGeneratedParent(key); { let lengthKey = key + "/length"; this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); - this.fields[lengthKey].putNumber(timestamp, value.length); + this.fields.get(lengthKey)?.putNumber(timestamp, value.length); } for (let i = 0; i < value.length; i++) { if (this.enableTimestampSetCache) { @@ -419,7 +444,7 @@ export default class Log { } let itemKey = key + "/" + i.toString(); this.createBlankField(itemKey, LoggableType.Number); - this.fields[itemKey].putNumber(timestamp, value[i]); + this.fields.get(itemKey)?.putNumber(timestamp, value[i]); } } } @@ -427,15 +452,15 @@ export default class Log { /** Writes a new StringArray value to the field. */ putStringArray(key: string, timestamp: number, value: string[]) { this.createBlankField(key, LoggableType.StringArray); - this.fields[key].putStringArray(timestamp, value); - if (this.fields[key].getType() === LoggableType.StringArray) { + this.fields.get(key)?.putStringArray(timestamp, value); + if (this.fields.get(key)?.getType() === LoggableType.StringArray) { this.processTimestamp(key, timestamp); this.setGeneratedParent(key); { let lengthKey = key + "/length"; this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); - this.fields[lengthKey].putNumber(timestamp, value.length); + this.fields.get(lengthKey)?.putNumber(timestamp, value.length); } for (let i = 0; i < value.length; i++) { if (this.enableTimestampSetCache) { @@ -444,7 +469,7 @@ export default class Log { } let itemKey = key + "/" + i.toString(); this.createBlankField(itemKey, LoggableType.String); - this.fields[itemKey].putString(timestamp, value[i]); + this.fields.get(itemKey)?.putString(timestamp, value[i]); } } } @@ -489,7 +514,7 @@ export default class Log { let lengthKey = key + "/length"; this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); - this.fields[lengthKey].putNumber(timestamp, value.length); + this.fields.get(lengthKey)?.putNumber(timestamp, value.length); } for (let i = 0; i < value.length; i++) { this.putUnknownStruct(key + "/" + i.toString(), timestamp, value[i], true); @@ -506,7 +531,7 @@ export default class Log { /** Writes a JSON-encoded string value to the field. */ putJSON(key: string, timestamp: number, value: string) { this.putString(key, timestamp, value); - if (this.fields[key].getType() === LoggableType.String) { + if (this.fields.get(key)?.getType() === LoggableType.String) { this.setGeneratedParent(key); this.setStructuredType(key, "JSON"); let decodedValue: unknown = null; @@ -522,7 +547,7 @@ export default class Log { /** Writes a msgpack-encoded raw value to the field. */ putMsgpack(key: string, timestamp: number, value: Uint8Array) { this.putRaw(key, timestamp, value); - if (this.fields[key].getType() === LoggableType.Raw) { + if (this.fields.get(key)?.getType() === LoggableType.Raw) { this.setGeneratedParent(key); this.setStructuredType(key, "MessagePack"); let decodedValue: unknown = null; @@ -541,7 +566,7 @@ export default class Log { */ putStruct(key: string, timestamp: number, value: Uint8Array, schemaType: string, isArray: boolean) { this.putRaw(key, timestamp, value); - if (this.fields[key].getType() === LoggableType.Raw) { + if (this.fields.get(key)?.getType() === LoggableType.Raw) { this.setGeneratedParent(key); this.setStructuredType(key, schemaType + (isArray ? "[]" : "")); let decodedData: { data: unknown; schemaTypes: { [key: string]: string } } | null = null; @@ -586,7 +611,7 @@ export default class Log { // Not a schema, continue normally this.putRaw(key, timestamp, value); - if (this.fields[key].getType() === LoggableType.Raw) { + if (this.fields.get(key)?.getType() === LoggableType.Raw) { this.setGeneratedParent(key); this.setStructuredType(key, ProtoDecoder.getFriendlySchemaType(schemaType)); let decodedData: { data: unknown; schemaTypes: { [key: string]: string } } | null = null; @@ -620,18 +645,18 @@ export default class Log { this.putNumber(translationKey + "/x", timestamp, pose.translation[0]); this.putNumber(translationKey + "/y", timestamp, pose.translation[1]); this.putNumber(rotationKey + "/value", timestamp, pose.rotation); - if (!(key in this.fields)) { + if (!this.fields.has(key)) { this.createBlankField(key, LoggableType.Empty); this.setStructuredType(key, "Pose2d"); this.setGeneratedParent(key); this.processTimestamp(key, timestamp); } - if (!(translationKey in this.fields)) { + if (!this.fields.has(translationKey)) { this.createBlankField(translationKey, LoggableType.Empty); this.setStructuredType(translationKey, "Translation2d"); this.processTimestamp(translationKey, timestamp); } - if (!(rotationKey in this.fields)) { + if (!this.fields.has(rotationKey)) { this.createBlankField(rotationKey, LoggableType.Empty); this.setStructuredType(rotationKey, "Rotation2d"); this.processTimestamp(rotationKey, timestamp); @@ -640,7 +665,7 @@ export default class Log { /** Writes a translation array with the "Translation2d[]" structured type. */ putTranslationArray(key: string, timestamp: number, translations: Translation2d[]) { - if (!(key in this.fields)) { + if (!this.fields.has(key)) { this.createBlankField(key, LoggableType.Empty); this.setStructuredType(key, "Translation2d[]"); this.setGeneratedParent(key); @@ -649,7 +674,7 @@ export default class Log { this.putNumber(key + "/length", timestamp, translations.length); for (let i = 0; i < translations.length; i++) { const itemKey = key + "/" + i.toString(); - if (!(itemKey in this.fields)) { + if (!this.fields.has(itemKey)) { this.createBlankField(itemKey, LoggableType.Empty); this.setStructuredType(itemKey, "Translation2d"); this.processTimestamp(itemKey, timestamp); @@ -664,7 +689,7 @@ export default class Log { this.putNumber(key + "/x", timestamp, x); this.putNumber(key + "/y", timestamp, y); this.putString(key + "/alliance", timestamp, alliance); - if (!(key in this.fields)) { + if (!this.fields.has(key)) { this.createBlankField(key, LoggableType.Empty); this.setStructuredType(key, "ZebraTranslation"); this.setGeneratedParent(key); @@ -694,7 +719,7 @@ export default class Log { static fromSerialized(serializedData: any): Log { let log = new Log(); Object.entries(serializedData.fields).forEach(([key, value]) => { - log.fields[key] = LogField.fromSerialized(value); + log.fields.set(key, LogField.fromSerialized(value)); }); log.generatedParents = new Set(serializedData.generatedParents); log.timestampRange = serializedData.timestampRange; @@ -744,7 +769,7 @@ export default class Log { // Merge fields Object.entries(source.fields).forEach(([key, value]) => { - log.fields[adjustKey(key)] = LogField.fromSerialized(value); + log.fields.set(adjustKey(key), LogField.fromSerialized(value)); }); // Merge generated parents From c98f04b191677d0c2919c57e34619c89e90db6bf Mon Sep 17 00:00:00 2001 From: m10653 Date: Wed, 6 Mar 2024 21:54:23 +0000 Subject: [PATCH 11/14] Improve Wpilog Decoder --- src/hub/dataSources/wpilog/WPILOGDecoder.ts | 86 ++++++++++++--------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/src/hub/dataSources/wpilog/WPILOGDecoder.ts b/src/hub/dataSources/wpilog/WPILOGDecoder.ts index ababd740..574fdb48 100644 --- a/src/hub/dataSources/wpilog/WPILOGDecoder.ts +++ b/src/hub/dataSources/wpilog/WPILOGDecoder.ts @@ -16,6 +16,9 @@ export class WPILOGDecoderRecord { private timestamp: number; private data: Uint8Array; private dataView: DataView; + private length: number; + private start: number; + private end: number; /** * Creates a new WPILOGDecoderRecord. @@ -23,11 +26,21 @@ export class WPILOGDecoderRecord { * @param timestamp The timestamp in microseconds * @param data The payload data */ - constructor(entry: number, timestamp: number, data: Uint8Array) { + // constructor(entry: number, timestamp: number, data: Uint8Array) { + // this.entry = entry; + // this.timestamp = timestamp; + // this.data = data; + // this.dataView = new DataView(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)); + // } + constructor(entry: number, timestamp: number, dataView: DataView, data: Uint8Array, start: number, end: number) { this.entry = entry; this.timestamp = timestamp; this.data = data; - this.dataView = new DataView(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)); + // this.dataView = new DataView(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)); + this.dataView = dataView; + this.start = start; + this.end = end; + this.length = end - start; } /** Gets the entry ID. */ @@ -47,22 +60,22 @@ export class WPILOGDecoderRecord { /** Returns the type of the control record. */ private getControlType(): number { - return this.data[0]; + return this.data[this.start]; } /** Returns true if the record is a start control record. */ isStart(): boolean { - return this.isControl() && this.data.length >= 17 && this.getControlType() === CONTROL_START; + return this.isControl() && this.length >= 17 && this.getControlType() === CONTROL_START; } /** Returns true if the record is a finish control record. */ isFinish(): boolean { - return this.isControl() && this.data.length === 5 && this.getControlType() === CONTROL_FINISH; + return this.isControl() && this.length === 5 && this.getControlType() === CONTROL_FINISH; } /** Returns true if the record is a set metadata control record. */ isSetMetadata(): boolean { - return this.isControl() && this.data.length >= 9 && this.getControlType() === CONTROL_SET_METADATA; + return this.isControl() && this.length >= 9 && this.getControlType() === CONTROL_SET_METADATA; } /** Decodes a start control record. */ @@ -70,8 +83,8 @@ export class WPILOGDecoderRecord { if (!this.isStart()) throw "Not a start control record"; let stringResult: { string: string; position: number }; - let entry = this.dataView.getUint32(1, true); - stringResult = this.readInnerString(5); + let entry = this.dataView.getUint32(this.start + 1, true); + stringResult = this.readInnerString(this.start + 5); let name = stringResult.string; stringResult = this.readInnerString(stringResult.position); let type = stringResult.string; @@ -88,7 +101,7 @@ export class WPILOGDecoderRecord { /** Decodes a finish control record. */ getFinishEntry(): number { if (!this.isFinish()) throw "Not a finish control record"; - return this.dataView.getUint32(1, true); + return this.dataView.getUint32(this.start + 1, true); } /** Decodes a set metadata control record. */ @@ -96,59 +109,59 @@ export class WPILOGDecoderRecord { if (!this.isSetMetadata()) throw "Not a set metadata control record"; return { - entry: this.dataView.getUint32(1, true), - metadata: this.readInnerString(5).string + entry: this.dataView.getUint32(this.start + 1, true), + metadata: this.readInnerString(this.start + 5).string }; } /** Gets the raw data. */ getRaw(): Uint8Array { - return new Uint8Array(this.data.buffer.slice(this.data.byteOffset, this.data.byteOffset + this.data.byteLength)); + return new Uint8Array(this.data.subarray(this.start, this.end)); } /** Decodes a data record as a boolean. */ getBoolean(): boolean { - if (this.data.length !== 1) throw "Not a boolean"; - return this.data[0] !== 0; + if (this.length !== 1) throw "Not a boolean"; + return this.data[this.start] !== 0; } /** Decodes a data record as an integer. */ getInteger(): number { - if (this.data.length !== 8) throw "Not an integer"; - return Number(this.dataView.getBigInt64(0, true)); + if (this.length !== 8) throw "Not an integer"; + return Number(this.dataView.getBigInt64(this.start, true)); } /** Decodes a data record as a float. */ getFloat(): number { - if (this.data.length !== 4) throw "Not a float"; - return this.dataView.getFloat32(0, true); + if (this.length !== 4) throw "Not a float"; + return this.dataView.getFloat32(this.start, true); } /** Decodes a data record as a double. */ getDouble(): number { - if (this.data.length !== 8) throw "Not a double"; - return this.dataView.getFloat64(0, true); + if (this.length !== 8) throw "Not a double"; + return this.dataView.getFloat64(this.start, true); } /** Decodes a data record as a string. */ getString(): string { - return TEXT_DECODER.decode(this.data); + return TEXT_DECODER.decode(this.data.subarray(this.start, this.end)); } /** Decodes a data record as a boolean array. */ getBooleanArray(): boolean[] { let array: boolean[] = []; - this.data.forEach((x) => { - array.push(x !== 0); - }); + for (let i = this.start; i < this.end; i++) { + array.push(this.data[i] !== 0); + } return array; } /** Decodes a data record as an integer array. */ getIntegerArray(): number[] { - if (this.data.length % 8 !== 0) throw "Not an integer array"; + if (this.length % 8 !== 0) throw "Not an integer array"; let array: number[] = []; - for (let position = 0; position < this.data.length; position += 8) { + for (let position = this.start; position < this.length; position += 8) { array.push(Number(this.dataView.getBigInt64(position, true))); } return array; @@ -156,9 +169,9 @@ export class WPILOGDecoderRecord { /** Decodes a data record as a float array. */ getFloatArray(): number[] { - if (this.data.length % 4 !== 0) throw "Not a float array"; + if (this.length % 4 !== 0) throw "Not a float array"; let array: number[] = []; - for (let position = 0; position < this.data.length; position += 4) { + for (let position = this.start; position < this.length; position += 4) { array.push(this.dataView.getFloat32(position, true)); } return array; @@ -166,9 +179,9 @@ export class WPILOGDecoderRecord { /** Decodes a data record as a double array. */ getDoubleArray(): number[] { - if (this.data.length % 8 !== 0) throw "Not a double array"; + if (this.length % 8 !== 0) throw "Not a double array"; let array: number[] = []; - for (let position = 0; position < this.data.length; position += 8) { + for (let position = this.start; position < this.length; position += 8) { array.push(this.dataView.getFloat64(position, true)); } return array; @@ -176,10 +189,10 @@ export class WPILOGDecoderRecord { /** Decodes a data record as a string array. */ getStringArray(): string[] { - let size = this.dataView.getUint32(0, true); - if (size > (this.data.length - 4) / 4) throw "Not a string array"; + let size = this.dataView.getUint32(this.start, true); + if (size > (this.length - 4) / 4) throw "Not a string array"; let array: string[] = []; - let position = 4; + let position = this.start + 4; for (let i = 0; i < size; i++) { let stringResult = this.readInnerString(position); array.push(stringResult.string); @@ -192,7 +205,7 @@ export class WPILOGDecoderRecord { private readInnerString(position: number): { string: string; position: number } { let size = this.dataView.getUint32(position, true); let end = position + 4 + size; - if (end > this.data.length) throw "Invalid string size"; + if (end > this.start + this.length) throw "Invalid string size"; return { string: TEXT_DECODER.decode(this.data.subarray(position + 4, end)), position: end @@ -276,7 +289,10 @@ export class WPILOGDecoder { new WPILOGDecoderRecord( entry, timestamp, - this.data.subarray(position + headerLength, position + headerLength + size) + this.dataView, + this.data, + position + headerLength, + position + headerLength + size ), newPosition ); From c606ed7dc9affe16e7b6c7508b561f4ba82f2e4f Mon Sep 17 00:00:00 2001 From: m10653 Date: Mon, 3 Jun 2024 10:35:25 +0100 Subject: [PATCH 12/14] Fix live log loading. --- package-lock.json | 17 +- package.json | 4 +- rollup.config.mjs | 5 +- src/hub/dataSources/wpilog/WPILOGDecoder.ts | 86 ++++------ src/hub/dataSources/wpilog/wpilogWorker.ts | 40 ++++- src/shared/log/Log.ts | 180 +++++++++----------- src/shared/log/LogField.ts | 125 ++++++++++---- 7 files changed, 267 insertions(+), 190 deletions(-) diff --git a/package-lock.json b/package-lock.json index e469d88b..dd5c6f62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "check-disk-space": "^3.4.0", "download": "^8.0.0", "electron-fetch": "^1.9.1", + "js-sdsl": "^4.4.2", "jsonfile": "^6.1.0", "ssh2": "^1.14.0", "tesseract.js": "^5.0.3", @@ -51,7 +52,7 @@ "simple-statistics": "^7.8.3", "three": "^0.156.1", "tslib": "^2.6.2", - "typescript": "5.2.2" + "typescript": "^5.2.2" } }, "node_modules/@babel/runtime": { @@ -3293,6 +3294,15 @@ "sourcemap-codec": "^1.4.8" } }, + "node_modules/js-sdsl": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.2.tgz", + "integrity": "sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -7600,6 +7610,11 @@ } } }, + "js-sdsl": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.2.tgz", + "integrity": "sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==" + }, "js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", diff --git a/package.json b/package.json index ec023fda..ef80f1e9 100644 --- a/package.json +++ b/package.json @@ -50,11 +50,11 @@ "rollup": "^3.29.2", "rollup-plugin-cleanup": "^3.2.1", "rollup-plugin-re": "^1.0.7", - "rollup-plugin-typescript2": "^0.36.0", "simple-statistics": "^7.8.3", "three": "^0.156.1", "tslib": "^2.6.2", - "typescript": "^5.2.2" + "typescript": "^5.2.2", + "@rollup/plugin-typescript": "11.1.3" }, "dependencies": { "check-disk-space": "^3.4.0", diff --git a/rollup.config.mjs b/rollup.config.mjs index 4818dd55..95f7e4f1 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -2,7 +2,7 @@ import commonjs from "@rollup/plugin-commonjs"; import json from "@rollup/plugin-json"; import nodeResolve from "@rollup/plugin-node-resolve"; import replace from "@rollup/plugin-replace"; -import typescript from "rollup-plugin-typescript2"; +import typescript from "@rollup/plugin-typescript"; import fs from "fs"; import cleanup from "rollup-plugin-cleanup"; import replaceRegEx from "rollup-plugin-re"; @@ -16,10 +16,9 @@ function bundle(input, output, isMain, external = []) { format: isMain ? "cjs" : "es" }, context: "this", - treeshake: true, external: external, plugins: [ - typescript({ check: false, sourcemap: false }), + typescript(), nodeResolve(), commonjs(), cleanup(), diff --git a/src/hub/dataSources/wpilog/WPILOGDecoder.ts b/src/hub/dataSources/wpilog/WPILOGDecoder.ts index 574fdb48..ababd740 100644 --- a/src/hub/dataSources/wpilog/WPILOGDecoder.ts +++ b/src/hub/dataSources/wpilog/WPILOGDecoder.ts @@ -16,9 +16,6 @@ export class WPILOGDecoderRecord { private timestamp: number; private data: Uint8Array; private dataView: DataView; - private length: number; - private start: number; - private end: number; /** * Creates a new WPILOGDecoderRecord. @@ -26,21 +23,11 @@ export class WPILOGDecoderRecord { * @param timestamp The timestamp in microseconds * @param data The payload data */ - // constructor(entry: number, timestamp: number, data: Uint8Array) { - // this.entry = entry; - // this.timestamp = timestamp; - // this.data = data; - // this.dataView = new DataView(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)); - // } - constructor(entry: number, timestamp: number, dataView: DataView, data: Uint8Array, start: number, end: number) { + constructor(entry: number, timestamp: number, data: Uint8Array) { this.entry = entry; this.timestamp = timestamp; this.data = data; - // this.dataView = new DataView(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)); - this.dataView = dataView; - this.start = start; - this.end = end; - this.length = end - start; + this.dataView = new DataView(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)); } /** Gets the entry ID. */ @@ -60,22 +47,22 @@ export class WPILOGDecoderRecord { /** Returns the type of the control record. */ private getControlType(): number { - return this.data[this.start]; + return this.data[0]; } /** Returns true if the record is a start control record. */ isStart(): boolean { - return this.isControl() && this.length >= 17 && this.getControlType() === CONTROL_START; + return this.isControl() && this.data.length >= 17 && this.getControlType() === CONTROL_START; } /** Returns true if the record is a finish control record. */ isFinish(): boolean { - return this.isControl() && this.length === 5 && this.getControlType() === CONTROL_FINISH; + return this.isControl() && this.data.length === 5 && this.getControlType() === CONTROL_FINISH; } /** Returns true if the record is a set metadata control record. */ isSetMetadata(): boolean { - return this.isControl() && this.length >= 9 && this.getControlType() === CONTROL_SET_METADATA; + return this.isControl() && this.data.length >= 9 && this.getControlType() === CONTROL_SET_METADATA; } /** Decodes a start control record. */ @@ -83,8 +70,8 @@ export class WPILOGDecoderRecord { if (!this.isStart()) throw "Not a start control record"; let stringResult: { string: string; position: number }; - let entry = this.dataView.getUint32(this.start + 1, true); - stringResult = this.readInnerString(this.start + 5); + let entry = this.dataView.getUint32(1, true); + stringResult = this.readInnerString(5); let name = stringResult.string; stringResult = this.readInnerString(stringResult.position); let type = stringResult.string; @@ -101,7 +88,7 @@ export class WPILOGDecoderRecord { /** Decodes a finish control record. */ getFinishEntry(): number { if (!this.isFinish()) throw "Not a finish control record"; - return this.dataView.getUint32(this.start + 1, true); + return this.dataView.getUint32(1, true); } /** Decodes a set metadata control record. */ @@ -109,59 +96,59 @@ export class WPILOGDecoderRecord { if (!this.isSetMetadata()) throw "Not a set metadata control record"; return { - entry: this.dataView.getUint32(this.start + 1, true), - metadata: this.readInnerString(this.start + 5).string + entry: this.dataView.getUint32(1, true), + metadata: this.readInnerString(5).string }; } /** Gets the raw data. */ getRaw(): Uint8Array { - return new Uint8Array(this.data.subarray(this.start, this.end)); + return new Uint8Array(this.data.buffer.slice(this.data.byteOffset, this.data.byteOffset + this.data.byteLength)); } /** Decodes a data record as a boolean. */ getBoolean(): boolean { - if (this.length !== 1) throw "Not a boolean"; - return this.data[this.start] !== 0; + if (this.data.length !== 1) throw "Not a boolean"; + return this.data[0] !== 0; } /** Decodes a data record as an integer. */ getInteger(): number { - if (this.length !== 8) throw "Not an integer"; - return Number(this.dataView.getBigInt64(this.start, true)); + if (this.data.length !== 8) throw "Not an integer"; + return Number(this.dataView.getBigInt64(0, true)); } /** Decodes a data record as a float. */ getFloat(): number { - if (this.length !== 4) throw "Not a float"; - return this.dataView.getFloat32(this.start, true); + if (this.data.length !== 4) throw "Not a float"; + return this.dataView.getFloat32(0, true); } /** Decodes a data record as a double. */ getDouble(): number { - if (this.length !== 8) throw "Not a double"; - return this.dataView.getFloat64(this.start, true); + if (this.data.length !== 8) throw "Not a double"; + return this.dataView.getFloat64(0, true); } /** Decodes a data record as a string. */ getString(): string { - return TEXT_DECODER.decode(this.data.subarray(this.start, this.end)); + return TEXT_DECODER.decode(this.data); } /** Decodes a data record as a boolean array. */ getBooleanArray(): boolean[] { let array: boolean[] = []; - for (let i = this.start; i < this.end; i++) { - array.push(this.data[i] !== 0); - } + this.data.forEach((x) => { + array.push(x !== 0); + }); return array; } /** Decodes a data record as an integer array. */ getIntegerArray(): number[] { - if (this.length % 8 !== 0) throw "Not an integer array"; + if (this.data.length % 8 !== 0) throw "Not an integer array"; let array: number[] = []; - for (let position = this.start; position < this.length; position += 8) { + for (let position = 0; position < this.data.length; position += 8) { array.push(Number(this.dataView.getBigInt64(position, true))); } return array; @@ -169,9 +156,9 @@ export class WPILOGDecoderRecord { /** Decodes a data record as a float array. */ getFloatArray(): number[] { - if (this.length % 4 !== 0) throw "Not a float array"; + if (this.data.length % 4 !== 0) throw "Not a float array"; let array: number[] = []; - for (let position = this.start; position < this.length; position += 4) { + for (let position = 0; position < this.data.length; position += 4) { array.push(this.dataView.getFloat32(position, true)); } return array; @@ -179,9 +166,9 @@ export class WPILOGDecoderRecord { /** Decodes a data record as a double array. */ getDoubleArray(): number[] { - if (this.length % 8 !== 0) throw "Not a double array"; + if (this.data.length % 8 !== 0) throw "Not a double array"; let array: number[] = []; - for (let position = this.start; position < this.length; position += 8) { + for (let position = 0; position < this.data.length; position += 8) { array.push(this.dataView.getFloat64(position, true)); } return array; @@ -189,10 +176,10 @@ export class WPILOGDecoderRecord { /** Decodes a data record as a string array. */ getStringArray(): string[] { - let size = this.dataView.getUint32(this.start, true); - if (size > (this.length - 4) / 4) throw "Not a string array"; + let size = this.dataView.getUint32(0, true); + if (size > (this.data.length - 4) / 4) throw "Not a string array"; let array: string[] = []; - let position = this.start + 4; + let position = 4; for (let i = 0; i < size; i++) { let stringResult = this.readInnerString(position); array.push(stringResult.string); @@ -205,7 +192,7 @@ export class WPILOGDecoderRecord { private readInnerString(position: number): { string: string; position: number } { let size = this.dataView.getUint32(position, true); let end = position + 4 + size; - if (end > this.start + this.length) throw "Invalid string size"; + if (end > this.data.length) throw "Invalid string size"; return { string: TEXT_DECODER.decode(this.data.subarray(position + 4, end)), position: end @@ -289,10 +276,7 @@ export class WPILOGDecoder { new WPILOGDecoderRecord( entry, timestamp, - this.dataView, - this.data, - position + headerLength, - position + headerLength + size + this.data.subarray(position + headerLength, position + headerLength + size) ), newPosition ); diff --git a/src/hub/dataSources/wpilog/wpilogWorker.ts b/src/hub/dataSources/wpilog/wpilogWorker.ts index 01326338..f7039aab 100644 --- a/src/hub/dataSources/wpilog/wpilogWorker.ts +++ b/src/hub/dataSources/wpilog/wpilogWorker.ts @@ -3,7 +3,32 @@ import { PROTO_PREFIX, STRUCT_PREFIX } from "../../../shared/log/LogUtil"; import LoggableType from "../../../shared/log/LoggableType"; import CustomSchemas from "../schema/CustomSchemas"; import { WPILOGDecoder } from "./WPILOGDecoder"; +// import * as fs from "fs"; +// var self = { +// onmessage: function (tmp: any) {}, +// postMessage: function (data: any) { +// return; +// console.log(data.id + " " + data.progress); +// } +// }; +// function torun() { +// //"D:\\FRC_20240302_141552_VAASH_P1.wpilog" +// var file = ""; +// // file = "D:\\Robotics\\robotics logs\\FRC_20240302_141552_VAASH_P1.wpilog"; +// file = "D:\\Robotics\\robotics logs\\FRC_20240303_205029_VAASH_E10.wpilog"; +// // file = "D:\\Robotics\\robotics logs\\MIMIL_Q63_C8A241B2394C4853202020500E3318FF_2024-03-02_10-38-03.hoot.wpilog"; +// const buffer = fs.readFileSync(file); +// console.profile(); +// // for (let i = 0; i < 5; i++) { +// console.time("slow-big"); +// self.onmessage({ data: { id: 0, payload: [buffer] } }); +// console.timeEnd("slow-big"); +// // } + +// console.profileEnd(); +// console.log("Done"); +// } self.onmessage = (event) => { // WORKER SETUP let { id, payload } = event.data; @@ -20,7 +45,7 @@ self.onmessage = (event) => { // MAIN LOGIC // Run worker - let log = new Log(false); // No timestamp set cache for efficiency + let log = new Log(false, false); // No timestamp set cache for efficiency let reader = new WPILOGDecoder(payload[0]); let totalBytes = (payload[0] as Uint8Array).byteLength; let entryIds: { [id: number]: string } = {}; @@ -150,7 +175,7 @@ self.onmessage = (event) => { let now = new Date().getTime(); if (now - lastProgressTimestamp > 1000 / 60) { lastProgressTimestamp = now; - progress(byteCount / totalBytes); + progress((byteCount / totalBytes) * 0.2); } }); } catch (exception) { @@ -158,9 +183,16 @@ self.onmessage = (event) => { reject(); return; } - progress(1); + // progress(1); setTimeout(() => { // Allow progress message to get through first - resolve(log.toSerialized()); + resolve( + log.toSerialized((x) => { + progress(0.2 + x * 0.8); + }) + ); + progress(1); }, 0); }; + +// torun(); diff --git a/src/shared/log/Log.ts b/src/shared/log/Log.ts index 82eda2ba..2168d3b2 100644 --- a/src/shared/log/Log.ts +++ b/src/shared/log/Log.ts @@ -25,24 +25,26 @@ export default class Log { private structDecoder = new StructDecoder(); private protoDecoder = new ProtoDecoder(); - private fields: Map = new Map(); + private fields: { [id: string]: LogField } = {}; private generatedParents: Set = new Set(); // Children of these fields are generated private timestampRange: [number, number] | null = null; private enableTimestampSetCache: boolean; + private enableLiveSorting: boolean; private timestampSetCache: { [id: string]: { keys: string[]; timestamps: number[] } } = {}; private queuedStructs: QueuedStructure[] = []; private queuedStructArrays: QueuedStructure[] = []; private queuedProtos: QueuedStructure[] = []; - constructor(enableTimestampSetCache = true) { + constructor(enableTimestampSetCache = true, enableLiveSorting = true) { this.enableTimestampSetCache = enableTimestampSetCache; + this.enableLiveSorting = enableLiveSorting; } /** Checks if the field exists and registers it if necessary. */ public createBlankField(key: string, type: LoggableType) { - if (this.fields.has(key)) return; - this.fields.set(key, new LogField(type)); + if (key in this.fields) return; + this.fields[key] = new LogField(type, this.enableLiveSorting); } /** Clears all data before the provided timestamp. */ @@ -105,85 +107,76 @@ export default class Log { /** Returns the constant field type. */ getType(key: string): LoggableType | null { - let field = this.fields.get(key); - if (field === undefined) { - return null; + if (key in this.fields) { + return this.fields[key].getType(); } else { - return field.getType(); + return null; } } /** Returns a boolean that toggles when a value is removed from the field. */ getStripingReference(key: string): boolean { - let field = this.fields.get(key); - if (field === undefined) { - return false; + if (key in this.fields) { + return this.fields[key].getStripingReference(); } else { - return field.getStripingReference(); + return false; } } /** Returns the structured type string for a field. */ getStructuredType(key: string): string | null { - let field = this.fields.get(key); - if (field === undefined) { - return null; + if (key in this.fields) { + return this.fields[key].structuredType; } else { - return field.structuredType; + return null; } } /** Sets the structured type string for a field. */ setStructuredType(key: string, type: string | null) { - let field = this.fields.get(key); - if (field !== undefined) { - field.structuredType = type; + if (key in this.fields) { + this.fields[key].structuredType = type; } } /** Returns the WPILib type string for a field. */ getWpilibType(key: string): string | null { - let field = this.fields.get(key); - if (field === undefined) { - return null; + if (key in this.fields) { + return this.fields[key].wpilibType; } else { - return field.wpilibType; + return null; } } /** Sets the WPILib type string for a field. */ setWpilibType(key: string, type: string) { - let field = this.fields.get(key); - if (field !== undefined) { - field.wpilibType = type; + if (key in this.fields) { + this.fields[key].wpilibType = type; } } /** Returns the metadata string for a field. */ getMetadataString(key: string): string { - let field = this.fields.get(key); - if (field === undefined) { - return ""; + if (key in this.fields) { + return this.fields[key].metadataString; } else { - return field.metadataString; + return ""; } } /** Sets the WPILib metadata string for a field. */ setMetadataString(key: string, type: string) { - let field = this.fields.get(key); - if (field !== undefined) { - field.metadataString = type; + if (key in this.fields) { + this.fields[key].metadataString = type; } } /** Returns whether there was an attempt to write a conflicting type to a field. */ getTypeWarning(key: string): boolean { - let field = this.fields.get(key); - if (field === undefined) { - return false; + if (key in this.fields) { + return this.fields[key].typeWarning; } else { - return field.typeWarning; + return false; } } @@ -213,7 +206,7 @@ export default class Log { * that data can be retrieved more quickly for subsequent calls. */ getTimestamps(keys: string[], uuid: string | null = null): number[] { let output: number[] = []; - keys = keys.filter((key) => this.fields.has(key)); + keys = keys.filter((key) => key in this.fields); if (keys.length > 1) { // Multiple fields, read from cache if possible let saveCache = false; @@ -229,28 +222,12 @@ export default class Log { } // Get new data - output = [ - ...new Set( - keys - .map((key) => { - let elem = this.fields.get(key)?.getTimestamps(); - if (elem === undefined) { - throw new Error("Key:" + key + ", Does not exist in Fields"); - } - return elem; - }) - .flat() - ) - ]; + output = [...new Set(keys.map((key) => this.fields[key].getTimestamps()).flat())]; output.sort((a, b) => a - b); if (saveCache && uuid) this.timestampSetCache[uuid].timestamps = output; } else if (keys.length === 1) { // Single field - let field = this.fields.get(keys[0]); - if (field === undefined) { - throw new Error("Key:" + keys[0] + ", Does not exist in Fields"); - } - output = [...field.getTimestamps()]; + output = [...this.fields[keys[0]].getTimestamps()]; } return output; } @@ -311,49 +288,49 @@ export default class Log { /** Reads a set of generic values from the field. */ getRange(key: string, start: number, end: number): LogValueSetAny | undefined { - if (this.fields.has(key)) return this.fields.get(key)?.getRange(start, end); + if (key in this.fields) return this.fields[key].getRange(start, end); } /** Reads a set of Raw values from the field. */ getRaw(key: string, start: number, end: number): LogValueSetRaw | undefined { - if (this.fields.has(key)) return this.fields.get(key)?.getRaw(start, end); + if (key in this.fields) return this.fields[key].getRaw(start, end); } /** Reads a set of Boolean values from the field. */ getBoolean(key: string, start: number, end: number): LogValueSetBoolean | undefined { - if (this.fields.has(key)) return this.fields.get(key)?.getBoolean(start, end); + if (key in this.fields) return this.fields[key].getBoolean(start, end); } /** Reads a set of Number values from the field. */ getNumber(key: string, start: number, end: number): LogValueSetNumber | undefined { - if (this.fields.has(key)) return this.fields.get(key)?.getNumber(start, end); + if (key in this.fields) return this.fields[key].getNumber(start, end); } /** Reads a set of String values from the field. */ getString(key: string, start: number, end: number): LogValueSetString | undefined { - if (this.fields.has(key)) return this.fields.get(key)?.getString(start, end); + if (key in this.fields) return this.fields[key].getString(start, end); } /** Reads a set of BooleanArray values from the field. */ getBooleanArray(key: string, start: number, end: number): LogValueSetBooleanArray | undefined { - if (this.fields.has(key)) return this.fields.get(key)?.getBooleanArray(start, end); + if (key in this.fields) return this.fields[key].getBooleanArray(start, end); } /** Reads a set of NumberArray values from the field. */ getNumberArray(key: string, start: number, end: number): LogValueSetNumberArray | undefined { - if (this.fields.has(key)) return this.fields.get(key)?.getNumberArray(start, end); + if (key in this.fields) return this.fields[key].getNumberArray(start, end); } /** Reads a set of StringArray values from the field. */ getStringArray(key: string, start: number, end: number): LogValueSetStringArray | undefined { - if (this.fields.has(key)) return this.fields.get(key)?.getStringArray(start, end); + if (key in this.fields) return this.fields[key].getStringArray(start, end); } /** Writes a new Raw value to the field. */ putRaw(key: string, timestamp: number, value: Uint8Array) { this.createBlankField(key, LoggableType.Raw); - this.fields.get(key)?.putRaw(timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.Raw) { + this.fields[key].putRaw(timestamp, value); + if (this.fields[key].getType() === LoggableType.Raw) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } @@ -367,8 +344,8 @@ export default class Log { /** Writes a new Boolean value to the field. */ putBoolean(key: string, timestamp: number, value: boolean) { this.createBlankField(key, LoggableType.Boolean); - this.fields.get(key)?.putBoolean(timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.Boolean) { + this.fields[key].putBoolean(timestamp, value); + if (this.fields[key].getType() === LoggableType.Boolean) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } } @@ -376,8 +353,8 @@ export default class Log { /** Writes a new Number value to the field. */ putNumber(key: string, timestamp: number, value: number) { this.createBlankField(key, LoggableType.Number); - this.fields.get(key)?.putNumber(timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.Number) { + this.fields[key].putNumber(timestamp, value); + if (this.fields[key].getType() === LoggableType.Number) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } } @@ -385,8 +362,8 @@ export default class Log { /** Writes a new String value to the field. */ putString(key: string, timestamp: number, value: string) { this.createBlankField(key, LoggableType.String); - this.fields.get(key)?.putString(timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.String) { + this.fields[key].putString(timestamp, value); + if (this.fields[key].getType() === LoggableType.String) { this.processTimestamp(key, timestamp); // Only update timestamp if type is correct } @@ -402,15 +379,15 @@ export default class Log { /** Writes a new BooleanArray value to the field. */ putBooleanArray(key: string, timestamp: number, value: boolean[]) { this.createBlankField(key, LoggableType.BooleanArray); - this.fields.get(key)?.putBooleanArray(timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.BooleanArray) { + this.fields[key].putBooleanArray(timestamp, value); + if (this.fields[key].getType() === LoggableType.BooleanArray) { this.processTimestamp(key, timestamp); this.setGeneratedParent(key); { let lengthKey = key + "/length"; this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); - this.fields.get(lengthKey)?.putNumber(timestamp, value.length); + this.fields[lengthKey].putNumber(timestamp, value.length); } for (let i = 0; i < value.length; i++) { if (this.enableTimestampSetCache) { @@ -419,7 +396,7 @@ export default class Log { } let itemKey = key + "/" + i.toString(); this.createBlankField(itemKey, LoggableType.Boolean); - this.fields.get(itemKey)?.putBoolean(timestamp, value[i]); + this.fields[itemKey].putBoolean(timestamp, value[i]); } } } @@ -427,15 +404,15 @@ export default class Log { /** Writes a new NumberArray value to the field. */ putNumberArray(key: string, timestamp: number, value: number[]) { this.createBlankField(key, LoggableType.NumberArray); - this.fields.get(key)?.putNumberArray(timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.NumberArray) { + this.fields[key].putNumberArray(timestamp, value); + if (this.fields[key].getType() === LoggableType.NumberArray) { this.processTimestamp(key, timestamp); this.setGeneratedParent(key); { let lengthKey = key + "/length"; this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); - this.fields.get(lengthKey)?.putNumber(timestamp, value.length); + this.fields[lengthKey].putNumber(timestamp, value.length); } for (let i = 0; i < value.length; i++) { if (this.enableTimestampSetCache) { @@ -444,7 +421,7 @@ export default class Log { } let itemKey = key + "/" + i.toString(); this.createBlankField(itemKey, LoggableType.Number); - this.fields.get(itemKey)?.putNumber(timestamp, value[i]); + this.fields[itemKey].putNumber(timestamp, value[i]); } } } @@ -452,15 +429,15 @@ export default class Log { /** Writes a new StringArray value to the field. */ putStringArray(key: string, timestamp: number, value: string[]) { this.createBlankField(key, LoggableType.StringArray); - this.fields.get(key)?.putStringArray(timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.StringArray) { + this.fields[key].putStringArray(timestamp, value); + if (this.fields[key].getType() === LoggableType.StringArray) { this.processTimestamp(key, timestamp); this.setGeneratedParent(key); { let lengthKey = key + "/length"; this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); - this.fields.get(lengthKey)?.putNumber(timestamp, value.length); + this.fields[lengthKey].putNumber(timestamp, value.length); } for (let i = 0; i < value.length; i++) { if (this.enableTimestampSetCache) { @@ -469,7 +446,7 @@ export default class Log { } let itemKey = key + "/" + i.toString(); this.createBlankField(itemKey, LoggableType.String); - this.fields.get(itemKey)?.putString(timestamp, value[i]); + this.fields[itemKey].putString(timestamp, value[i]); } } } @@ -514,7 +491,7 @@ export default class Log { let lengthKey = key + "/length"; this.createBlankField(lengthKey, LoggableType.Number); this.processTimestamp(lengthKey, timestamp); - this.fields.get(lengthKey)?.putNumber(timestamp, value.length); + this.fields[lengthKey].putNumber(timestamp, value.length); } for (let i = 0; i < value.length; i++) { this.putUnknownStruct(key + "/" + i.toString(), timestamp, value[i], true); @@ -531,7 +508,7 @@ export default class Log { /** Writes a JSON-encoded string value to the field. */ putJSON(key: string, timestamp: number, value: string) { this.putString(key, timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.String) { + if (this.fields[key].getType() === LoggableType.String) { this.setGeneratedParent(key); this.setStructuredType(key, "JSON"); let decodedValue: unknown = null; @@ -547,7 +524,7 @@ export default class Log { /** Writes a msgpack-encoded raw value to the field. */ putMsgpack(key: string, timestamp: number, value: Uint8Array) { this.putRaw(key, timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.Raw) { + if (this.fields[key].getType() === LoggableType.Raw) { this.setGeneratedParent(key); this.setStructuredType(key, "MessagePack"); let decodedValue: unknown = null; @@ -566,7 +543,7 @@ export default class Log { */ putStruct(key: string, timestamp: number, value: Uint8Array, schemaType: string, isArray: boolean) { this.putRaw(key, timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.Raw) { + if (this.fields[key].getType() === LoggableType.Raw) { this.setGeneratedParent(key); this.setStructuredType(key, schemaType + (isArray ? "[]" : "")); let decodedData: { data: unknown; schemaTypes: { [key: string]: string } } | null = null; @@ -611,7 +588,7 @@ export default class Log { // Not a schema, continue normally this.putRaw(key, timestamp, value); - if (this.fields.get(key)?.getType() === LoggableType.Raw) { + if (this.fields[key].getType() === LoggableType.Raw) { this.setGeneratedParent(key); this.setStructuredType(key, ProtoDecoder.getFriendlySchemaType(schemaType)); let decodedData: { data: unknown; schemaTypes: { [key: string]: string } } | null = null; @@ -645,18 +622,18 @@ export default class Log { this.putNumber(translationKey + "/x", timestamp, pose.translation[0]); this.putNumber(translationKey + "/y", timestamp, pose.translation[1]); this.putNumber(rotationKey + "/value", timestamp, pose.rotation); - if (!this.fields.has(key)) { + if (!(key in this.fields)) { this.createBlankField(key, LoggableType.Empty); this.setStructuredType(key, "Pose2d"); this.setGeneratedParent(key); this.processTimestamp(key, timestamp); } - if (!this.fields.has(translationKey)) { + if (!(translationKey in this.fields)) { this.createBlankField(translationKey, LoggableType.Empty); this.setStructuredType(translationKey, "Translation2d"); this.processTimestamp(translationKey, timestamp); } - if (!this.fields.has(rotationKey)) { + if (!(rotationKey in this.fields)) { this.createBlankField(rotationKey, LoggableType.Empty); this.setStructuredType(rotationKey, "Rotation2d"); this.processTimestamp(rotationKey, timestamp); @@ -665,7 +642,7 @@ export default class Log { /** Writes a translation array with the "Translation2d[]" structured type. */ putTranslationArray(key: string, timestamp: number, translations: Translation2d[]) { - if (!this.fields.has(key)) { + if (!(key in this.fields)) { this.createBlankField(key, LoggableType.Empty); this.setStructuredType(key, "Translation2d[]"); this.setGeneratedParent(key); @@ -674,7 +651,7 @@ export default class Log { this.putNumber(key + "/length", timestamp, translations.length); for (let i = 0; i < translations.length; i++) { const itemKey = key + "/" + i.toString(); - if (!this.fields.has(itemKey)) { + if (!(itemKey in this.fields)) { this.createBlankField(itemKey, LoggableType.Empty); this.setStructuredType(itemKey, "Translation2d"); this.processTimestamp(itemKey, timestamp); @@ -689,7 +666,7 @@ export default class Log { this.putNumber(key + "/x", timestamp, x); this.putNumber(key + "/y", timestamp, y); this.putString(key + "/alliance", timestamp, alliance); - if (!this.fields.has(key)) { + if (!(key in this.fields)) { this.createBlankField(key, LoggableType.Empty); this.setStructuredType(key, "ZebraTranslation"); this.setGeneratedParent(key); @@ -698,7 +675,10 @@ export default class Log { } /** Returns a serialized version of the data from this log. */ - toSerialized(): any { + toSerialized(progressCallback: ((progress: number) => void) | undefined = undefined): any { + if (this.enableLiveSorting) { + this.enableLiveSorting = false; + } let result: any = { fields: {}, generatedParents: Array.from(this.generatedParents), @@ -709,8 +689,12 @@ export default class Log { queuedStructArrays: this.queuedStructArrays, queuedProtos: this.queuedProtos }; + let totalFields = Object.keys(this.fields).length; Object.entries(this.fields).forEach(([key, value]) => { result.fields[key] = value.toSerialized(); + if (progressCallback != undefined) { + progressCallback(Object.keys(result.fields).length / totalFields); + } }); return result; } @@ -719,7 +703,7 @@ export default class Log { static fromSerialized(serializedData: any): Log { let log = new Log(); Object.entries(serializedData.fields).forEach(([key, value]) => { - log.fields.set(key, LogField.fromSerialized(value)); + log.fields[key] = LogField.fromSerialized(value); }); log.generatedParents = new Set(serializedData.generatedParents); log.timestampRange = serializedData.timestampRange; @@ -769,7 +753,7 @@ export default class Log { // Merge fields Object.entries(source.fields).forEach(([key, value]) => { - log.fields.set(adjustKey(key), LogField.fromSerialized(value)); + log.fields[adjustKey(key)] = LogField.fromSerialized(value); }); // Merge generated parents diff --git a/src/shared/log/LogField.ts b/src/shared/log/LogField.ts index 6262da05..eb92d3f8 100644 --- a/src/shared/log/LogField.ts +++ b/src/shared/log/LogField.ts @@ -1,3 +1,4 @@ +import { re } from "mathjs"; import LoggableType from "./LoggableType"; import { logValuesEqual } from "./LogUtil"; import { @@ -10,27 +11,25 @@ import { LogValueSetString, LogValueSetStringArray } from "./LogValueSets"; - type LogRecord = { timestamp: number; value: any; - index: number; }; /** A full log field that contains data. */ export default class LogField { private type: LoggableType; private data: LogValueSetAny = { timestamps: [], values: [] }; - private rawData: LogRecord[] = []; public structuredType: string | null = null; public wpilibType: string | null = null; // Original type from WPILOG & NT4 public metadataString = ""; public typeWarning = false; // Flag that there was an attempt to write a conflicting type - + private enableLiveSorting: boolean; // Toggles when first value is removed, useful for creating striping effects that persist as data is updated private stripingReference = false; - - constructor(type: LoggableType) { + private rawData: LogRecord[] = []; + constructor(type: LoggableType, enableLiveSorting = true) { this.type = type; + this.enableLiveSorting = enableLiveSorting; } /** Returns the constant field type. */ @@ -121,8 +120,48 @@ export default class LogField { /** Inserts a new value at the correct index. */ private putData(timestamp: number, value: any) { - if (value === null) return; - this.rawData.push({ timestamp: timestamp, value: value, index: this.rawData.length }); + if (this.enableLiveSorting) { + if (value === null) return; + + // Find position to insert based on timestamp + let insertIndex: number; + if (this.data.timestamps.length > 0 && timestamp > this.data.timestamps[this.data.timestamps.length - 1]) { + // There's a good chance this data is at the end of the log, so check that first + insertIndex = this.data.timestamps.length; + } else { + // Adding in the middle, find where to insert it + let alreadyExists = false; + insertIndex = + this.data.timestamps.findLastIndex((x) => { + if (alreadyExists) return; + if (x === timestamp) alreadyExists = true; + return x < timestamp; + }) + 1; + if (alreadyExists) { + this.data.values[this.data.timestamps.indexOf(timestamp)] = value; + return; + } + } + + // Compare to adjacent values + if (insertIndex > 0 && logValuesEqual(this.type, value, this.data.values[insertIndex - 1])) { + // Same as the previous value + } else if ( + insertIndex < this.data.values.length && + logValuesEqual(this.type, value, this.data.values[insertIndex]) + ) { + // Same as the next value + this.data.timestamps[insertIndex] = timestamp; + } else { + // New value + this.data.timestamps.splice(insertIndex, 0, timestamp); + this.data.values.splice(insertIndex, 0, value); + } + } else { + // this.data.timestamps.push(timestamp); + // this.data.values.push(value); + this.rawData.push({ timestamp: timestamp, value: value }); + } } /** Writes a new Raw value to the field. */ @@ -176,10 +215,9 @@ export default class LogField { /** Returns a serialized version of the data from this field. */ toSerialized(): any { - if (this.data.timestamps.length == 0) { + if (!this.enableLiveSorting) { this.sortAndProcess(); } - return { type: this.type, timestamps: this.data.timestamps, @@ -191,47 +229,72 @@ export default class LogField { typeWarning: this.typeWarning }; } + + /** Creates a new field based on the data from `toSerialized()` */ + static fromSerialized(serializedData: any) { + let field = new LogField(serializedData.type); + field.data = { + timestamps: serializedData.timestamps, + values: serializedData.values + }; + field.structuredType = serializedData.structuredType; + field.wpilibType = serializedData.wpilibType; + field.metadataString = serializedData.metadataString; + field.stripingReference = serializedData.stripingReference; + field.typeWarning = serializedData.typeWarning; + return field; + } private sortAndProcess() { this.rawData.sort((a: LogRecord, b: LogRecord) => { - let cmp = a.timestamp - b.timestamp; - if (cmp == 0) { - return a.index - b.index; - } else { - return cmp; - } + return a.timestamp - b.timestamp; }); + // this.rawData.reverse(); + // let record = this.rawData.pop(); + // // Bootstrap first value + // if (record) { + // this.data.timestamps.push(record.timestamp); + // this.data.values.push(record.value); + // } + // record = this.rawData.pop(); + // while (record) { + // // Check if the timestamp is the same as the last one + // if (record.timestamp == this.data.timestamps[this.data.values.length - 1]) { + // // Overwrite the last value + // this.data.values[this.data.values.length - 1] = record.value; + // // If the values are equal do not add the value + // } else if ( + // !logValuesEqual(this.type, this.data.values[this.data.values.length - 1], record.value) || + // this.rawData.length <= 0 + // ) { + // // add the value if the prevous value is different or it is the last value in the rawData (make sure final data point is added) + // this.data.timestamps.push(record.timestamp); + // this.data.values.push(record.value); + // } + // record = this.rawData.pop(); + // } + if (this.rawData.length > 0) { // Bootstrap first value this.data.timestamps.push(this.rawData[0].timestamp); this.data.values.push(this.rawData[0].value); } for (let i = 1; i < this.rawData.length; i++) { + // Check if the timestamp is the same as the last one if (this.rawData[i].timestamp == this.data.timestamps[this.data.values.length - 1]) { + // Overwrite the last value this.data.values[this.data.values.length - 1] = this.rawData[i].value; + // If the values are equal do not add the value } else if ( logValuesEqual(this.type, this.data.values[this.data.values.length - 1], this.rawData[i].value) && i < this.rawData.length ) { } else { + // add the value this.data.timestamps.push(this.rawData[i].timestamp); this.data.values.push(this.rawData[i].value); } } this.rawData = []; - } - - /** Creates a new field based on the data from `toSerialized()` */ - static fromSerialized(serializedData: any) { - let field = new LogField(serializedData.type); - field.data = { - timestamps: serializedData.timestamps, - values: serializedData.values - }; - field.structuredType = serializedData.structuredType; - field.wpilibType = serializedData.wpilibType; - field.metadataString = serializedData.metadataString; - field.stripingReference = serializedData.stripingReference; - field.typeWarning = serializedData.typeWarning; - return field; + this.enableLiveSorting = true; } } From d271f735c38b428d4a2991f5eff9fb6408c3d94b Mon Sep 17 00:00:00 2001 From: m10653 Date: Tue, 4 Jun 2024 13:41:08 +0100 Subject: [PATCH 13/14] Fix URCL not parsing --- src/shared/log/LogField.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/shared/log/LogField.ts b/src/shared/log/LogField.ts index eb92d3f8..b5a6b370 100644 --- a/src/shared/log/LogField.ts +++ b/src/shared/log/LogField.ts @@ -284,8 +284,9 @@ export default class LogField { // Overwrite the last value this.data.values[this.data.values.length - 1] = this.rawData[i].value; // If the values are equal do not add the value + // } } else if ( - logValuesEqual(this.type, this.data.values[this.data.values.length - 1], this.rawData[i].value) && + logValuesEqual(this.type, this.rawData[i].value, this.data.values[this.data.values.length - 1]) && i < this.rawData.length ) { } else { From d7fd7565c1a024955c95afd5dc2a2bd9120699ce Mon Sep 17 00:00:00 2001 From: m10653 Date: Thu, 20 Jun 2024 06:25:20 +0100 Subject: [PATCH 14/14] Clean up and Fix URCL Issue --- package-lock.json | 17 +---- package.json | 5 +- src/hub/dataSources/wpilog/wpilogWorker.ts | 55 ++++++--------- src/shared/log/Log.ts | 19 +++-- src/shared/log/LogField.ts | 82 ++++++++-------------- tsconfig.json | 10 +-- 6 files changed, 66 insertions(+), 122 deletions(-) diff --git a/package-lock.json b/package-lock.json index dd5c6f62..e469d88b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,6 @@ "check-disk-space": "^3.4.0", "download": "^8.0.0", "electron-fetch": "^1.9.1", - "js-sdsl": "^4.4.2", "jsonfile": "^6.1.0", "ssh2": "^1.14.0", "tesseract.js": "^5.0.3", @@ -52,7 +51,7 @@ "simple-statistics": "^7.8.3", "three": "^0.156.1", "tslib": "^2.6.2", - "typescript": "^5.2.2" + "typescript": "5.2.2" } }, "node_modules/@babel/runtime": { @@ -3294,15 +3293,6 @@ "sourcemap-codec": "^1.4.8" } }, - "node_modules/js-sdsl": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.2.tgz", - "integrity": "sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/js-sdsl" - } - }, "node_modules/js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", @@ -7610,11 +7600,6 @@ } } }, - "js-sdsl": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/js-sdsl/-/js-sdsl-4.4.2.tgz", - "integrity": "sha512-dwXFwByc/ajSV6m5bcKAPwe4yDDF6D614pxmIi5odytzxRlwqF6nwoiCek80Ixc7Cvma5awClxrzFtxCQvcM8w==" - }, "js-yaml": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", diff --git a/package.json b/package.json index ef80f1e9..5c32be41 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@rollup/plugin-json": "^6.0.0", "@rollup/plugin-node-resolve": "^15.2.1", "@rollup/plugin-replace": "^5.0.2", + "@rollup/plugin-typescript": "11.1.3", "@types/chart.js": "^2.9.38", "@types/download": "^8.0.2", "@types/heatmap.js": "2.0.38", @@ -53,14 +54,12 @@ "simple-statistics": "^7.8.3", "three": "^0.156.1", "tslib": "^2.6.2", - "typescript": "^5.2.2", - "@rollup/plugin-typescript": "11.1.3" + "typescript": "5.2.2" }, "dependencies": { "check-disk-space": "^3.4.0", "download": "^8.0.0", "electron-fetch": "^1.9.1", - "js-sdsl": "^4.4.2", "jsonfile": "^6.1.0", "ssh2": "^1.14.0", "tesseract.js": "^5.0.3", diff --git a/src/hub/dataSources/wpilog/wpilogWorker.ts b/src/hub/dataSources/wpilog/wpilogWorker.ts index f7039aab..e8f6e6e0 100644 --- a/src/hub/dataSources/wpilog/wpilogWorker.ts +++ b/src/hub/dataSources/wpilog/wpilogWorker.ts @@ -3,32 +3,7 @@ import { PROTO_PREFIX, STRUCT_PREFIX } from "../../../shared/log/LogUtil"; import LoggableType from "../../../shared/log/LoggableType"; import CustomSchemas from "../schema/CustomSchemas"; import { WPILOGDecoder } from "./WPILOGDecoder"; -// import * as fs from "fs"; -// var self = { -// onmessage: function (tmp: any) {}, -// postMessage: function (data: any) { -// return; -// console.log(data.id + " " + data.progress); -// } -// }; -// function torun() { -// //"D:\\FRC_20240302_141552_VAASH_P1.wpilog" -// var file = ""; -// // file = "D:\\Robotics\\robotics logs\\FRC_20240302_141552_VAASH_P1.wpilog"; -// file = "D:\\Robotics\\robotics logs\\FRC_20240303_205029_VAASH_E10.wpilog"; -// // file = "D:\\Robotics\\robotics logs\\MIMIL_Q63_C8A241B2394C4853202020500E3318FF_2024-03-02_10-38-03.hoot.wpilog"; -// const buffer = fs.readFileSync(file); -// console.profile(); -// // for (let i = 0; i < 5; i++) { -// console.time("slow-big"); -// self.onmessage({ data: { id: 0, payload: [buffer] } }); -// console.timeEnd("slow-big"); -// // } - -// console.profileEnd(); -// console.log("Done"); -// } self.onmessage = (event) => { // WORKER SETUP let { id, payload } = event.data; @@ -45,12 +20,13 @@ self.onmessage = (event) => { // MAIN LOGIC // Run worker - let log = new Log(false, false); // No timestamp set cache for efficiency + let log = new Log(false, false); // No timestamp set cache for efficiency, disable live sorting let reader = new WPILOGDecoder(payload[0]); let totalBytes = (payload[0] as Uint8Array).byteLength; let entryIds: { [id: number]: string } = {}; let entryTypes: { [id: number]: string } = {}; let lastProgressTimestamp = new Date().getTime(); + let customSchemaRecords: { key: string; timestamp: number; value: Uint8Array; type: string }[] = []; try { reader.forEach((record, byteCount) => { if (record.isControl()) { @@ -155,12 +131,7 @@ self.onmessage = (event) => { } else { log.putRaw(key, timestamp, record.getRaw()); if (CustomSchemas.has(type)) { - try { - CustomSchemas.get(type)!(log, key, timestamp, record.getRaw()); - } catch { - console.error('Failed to decode custom schema "' + type + '"'); - } - log.setGeneratedParent(key); + customSchemaRecords.push({ key: key, timestamp: timestamp, value: record.getRaw(), type: type }); } } break; @@ -175,7 +146,7 @@ self.onmessage = (event) => { let now = new Date().getTime(); if (now - lastProgressTimestamp > 1000 / 60) { lastProgressTimestamp = now; - progress((byteCount / totalBytes) * 0.2); + progress((byteCount / totalBytes) * 0.2); // Show progress of 0-20% for file reading } }); } catch (exception) { @@ -183,12 +154,26 @@ self.onmessage = (event) => { reject(); return; } - // progress(1); setTimeout(() => { // Allow progress message to get through first + log.sortAndProcess((x) => { + progress(0.2 + x * 0.5); // Show progress of 20-70% for log processing/sorting + }); + + // Process custom schemas + for (let i = 0; i < customSchemaRecords.length; i++) { + let record = customSchemaRecords[i]; + try { + CustomSchemas.get(record.type)!(log, record.key, record.timestamp, record.value); + } catch { + console.error('Failed to decode custom schema "' + record.type + '"'); + } + log.setGeneratedParent(record.key); + } + resolve( log.toSerialized((x) => { - progress(0.2 + x * 0.8); + progress(0.2 + x * 0.8); // Show progress of 20-100% for log processing/sorting }) ); progress(1); diff --git a/src/shared/log/Log.ts b/src/shared/log/Log.ts index 2168d3b2..f6ddc143 100644 --- a/src/shared/log/Log.ts +++ b/src/shared/log/Log.ts @@ -676,8 +676,9 @@ export default class Log { /** Returns a serialized version of the data from this log. */ toSerialized(progressCallback: ((progress: number) => void) | undefined = undefined): any { - if (this.enableLiveSorting) { - this.enableLiveSorting = false; + if (!this.enableLiveSorting) { + this.sortAndProcess(progressCallback); // Enable live sorting after first serialization + this.enableLiveSorting = true; } let result: any = { fields: {}, @@ -692,12 +693,20 @@ export default class Log { let totalFields = Object.keys(this.fields).length; Object.entries(this.fields).forEach(([key, value]) => { result.fields[key] = value.toSerialized(); - if (progressCallback != undefined) { - progressCallback(Object.keys(result.fields).length / totalFields); - } }); return result; } + sortAndProcess(progressCallback: ((progress: number) => void) | undefined = undefined) { + if (!this.enableLiveSorting) { + let entires = Object.values(this.fields); + for (let i = 0; i < entires.length; i++) { + entires[i].sortAndProcess(); + if (progressCallback != undefined) { + progressCallback(i / entires.length); + } + } + } + } /** Creates a new log based on the data from `toSerialized()` */ static fromSerialized(serializedData: any): Log { diff --git a/src/shared/log/LogField.ts b/src/shared/log/LogField.ts index b5a6b370..82b640da 100644 --- a/src/shared/log/LogField.ts +++ b/src/shared/log/LogField.ts @@ -1,4 +1,3 @@ -import { re } from "mathjs"; import LoggableType from "./LoggableType"; import { logValuesEqual } from "./LogUtil"; import { @@ -158,8 +157,6 @@ export default class LogField { this.data.values.splice(insertIndex, 0, value); } } else { - // this.data.timestamps.push(timestamp); - // this.data.values.push(value); this.rawData.push({ timestamp: timestamp, value: value }); } } @@ -217,6 +214,7 @@ export default class LogField { toSerialized(): any { if (!this.enableLiveSorting) { this.sortAndProcess(); + this.enableLiveSorting = true; // After first serilization enable live sorting } return { type: this.type, @@ -244,58 +242,34 @@ export default class LogField { field.typeWarning = serializedData.typeWarning; return field; } - private sortAndProcess() { - this.rawData.sort((a: LogRecord, b: LogRecord) => { - return a.timestamp - b.timestamp; - }); - // this.rawData.reverse(); - // let record = this.rawData.pop(); - // // Bootstrap first value - // if (record) { - // this.data.timestamps.push(record.timestamp); - // this.data.values.push(record.value); - // } - // record = this.rawData.pop(); - // while (record) { - // // Check if the timestamp is the same as the last one - // if (record.timestamp == this.data.timestamps[this.data.values.length - 1]) { - // // Overwrite the last value - // this.data.values[this.data.values.length - 1] = record.value; - // // If the values are equal do not add the value - // } else if ( - // !logValuesEqual(this.type, this.data.values[this.data.values.length - 1], record.value) || - // this.rawData.length <= 0 - // ) { - // // add the value if the prevous value is different or it is the last value in the rawData (make sure final data point is added) - // this.data.timestamps.push(record.timestamp); - // this.data.values.push(record.value); - // } - // record = this.rawData.pop(); - // } - - if (this.rawData.length > 0) { - // Bootstrap first value - this.data.timestamps.push(this.rawData[0].timestamp); - this.data.values.push(this.rawData[0].value); - } - for (let i = 1; i < this.rawData.length; i++) { - // Check if the timestamp is the same as the last one - if (this.rawData[i].timestamp == this.data.timestamps[this.data.values.length - 1]) { - // Overwrite the last value - this.data.values[this.data.values.length - 1] = this.rawData[i].value; - // If the values are equal do not add the value - // } - } else if ( - logValuesEqual(this.type, this.rawData[i].value, this.data.values[this.data.values.length - 1]) && - i < this.rawData.length - ) { - } else { - // add the value - this.data.timestamps.push(this.rawData[i].timestamp); - this.data.values.push(this.rawData[i].value); + /** Sorts and processes data all at once */ + sortAndProcess() { + if (!this.enableLiveSorting) { + this.rawData.sort((a: LogRecord, b: LogRecord) => { + return a.timestamp - b.timestamp; + }); + if (this.rawData.length > 0) { + // Bootstrap first value + this.data.timestamps.push(this.rawData[0].timestamp); + this.data.values.push(this.rawData[0].value); + } + for (let i = 1; i < this.rawData.length; i++) { + // Check if the timestamp is the same as the last one + if (this.rawData[i].timestamp == this.data.timestamps[this.data.values.length - 1]) { + // Overwrite the last value + this.data.values[this.data.values.length - 1] = this.rawData[i].value; + // If the values are equal do not add the value + } else if ( + logValuesEqual(this.type, this.rawData[i].value, this.data.values[this.data.values.length - 1]) && + i < this.rawData.length + ) { + } else { + // add the value + this.data.timestamps.push(this.rawData[i].timestamp); + this.data.values.push(this.rawData[i].value); + } } + this.rawData = []; } - this.rawData = []; - this.enableLiveSorting = true; } } diff --git a/tsconfig.json b/tsconfig.json index 2584b40d..5f058c14 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,11 +1,4 @@ { - "ts-node": { - // these options are overrides used only by ts-node - // same as the --compilerOptions flag and the TS_NODE_COMPILER_OPTIONS environment variable - "compilerOptions": { - "module": "commonjs" - } - }, "compilerOptions": { "target": "ESNext", "module": "ESNext", @@ -14,7 +7,6 @@ "forceConsistentCasingInFileNames": true, "strict": true, "removeComments": true, - "resolveJsonModule": true, - "sourceMap": true + "resolveJsonModule": true } }