Added the ability to store cached image knowledge responses in a sepa…

…rate directory (#772)

ts/examples/chat/src/memory/imageMemory.ts

@@ -6,7 +6,6 @@ import {
     CommandHandler,
     CommandMetadata,
     InteractiveIo,
-    NamedArgs,
     parseNamedArguments,
     StopWatch,
 } from "interactive-app";
@@ -132,9 +131,18 @@ export function createImageCommands(
             TokenCounter.getInstance().total;
 
         if (isDir) {
-            await indexImages(namedArgs, sourcePath, context, clock);
+            await indexImages(
+                sourcePath,
+                namedArgs.value("cachePath", "string", false),
+                context,
+                clock,
+            );
         } else {
-            await indexImage(sourcePath, context);
+            await indexImage(
+                sourcePath,
+                namedArgs.value("cachePath", "string", false),
+                context,
+            );
         }
 
         const tokenCountFinish: CompletionUsageStats =
@@ -161,8 +169,8 @@ export function createImageCommands(
     }
 
     async function indexImages(
-        namesArgs: NamedArgs,
         sourcePath: string,
+        cachePath: string,
         context: ChatContext,
         clock: StopWatch,
     ) {
@@ -177,11 +185,15 @@ export function createImageCommands(
             console.log(
                 `${fullFilePath} [${i + 1} of ${fileNames.length}] (estimated time remaining: ${(clock.elapsedSeconds / (i + 1)) * (fileNames.length - i)})`,
             );
-            await indexImage(fullFilePath, context);
+            await indexImage(fullFilePath, cachePath, context);
         }
     }
 
-    async function indexImage(fileName: string, context: ChatContext) {
+    async function indexImage(
+        fileName: string,
+        cachePath: string,
+        context: ChatContext,
+    ) {
         if (!fs.existsSync(fileName)) {
             console.log(`Could not find part of the file path '${fileName}'`);
             return;
@@ -192,7 +204,12 @@ export function createImageCommands(
 
         // load the image
         const image: knowLib.image.Image | undefined =
-            await knowLib.image.loadImage(fileName, context.models.chatModel);
+            await knowLib.image.loadImage(
+                fileName,
+                context.models.chatModel,
+                true,
+                cachePath,
+            );
 
         if (image) {
             await knowLib.image.addImageToConversation(

ts/examples/chat/src/memory/knowproMemory.ts

@@ -247,6 +247,7 @@ export async function createKnowproCommands(
                 related: argBool("Index related terms", true),
                 indexFilePath: arg("Output path for index file"),
                 maxMessages: argNum("Maximum images to index"),
+                cachePath: arg("Path to image knowledge response cache."),
             },
         };
     }
@@ -261,6 +262,7 @@ export async function createKnowproCommands(
         let progress = new ProgressBar(context.printer, 165);
         context.images = await im.importImages(
             namedArgs.filePath,
+            namedArgs.cachePath,
             true,
             (text, _index, max) => {
                 progress.total = max;

ts/packages/knowledgeProcessor/src/images/image.ts

@@ -506,14 +506,20 @@ export interface imageDetailExtractionSchema {
  * @param fileName The image file to load
  * @param model The language model being used to describe the image.
  * @param loadCachedDetails A flag indicating if cached image descriptions should be loaded if available.
+ * @param cacheFolder The folder to find cached image data. If not supplied defaults to the image folder.
  * @returns The described image.
  */
 export async function loadImage(
     fileName: string,
     model: ChatModel,
     loadCachedDetails: boolean = true,
+    cacheFolder: string | undefined = undefined,
 ): Promise<Image | undefined> {
-    const cachedFileName: string = fileName + ".eat";
+    if (cacheFolder === undefined) {
+        cacheFolder = path.dirname(fileName);
+    }
+
+    const cachedFileName: string = path.join(cacheFolder, fileName + ".eat");
     if (loadCachedDetails && fs.existsSync(cachedFileName)) {
         return JSON.parse(fs.readFileSync(cachedFileName, "utf8"));
     }
@@ -599,16 +605,21 @@ export async function loadImage(
  * Loads the image and then uses the LLM and other APIs to get POI, KnowledgeResponse, address, etc.
  *
  * @param fileName The image file to load
+ * @param cachePath The path where the image knowledge response is to be cached
  * @param model The language model being used to describe the image.
  * @param loadCachedDetails A flag indicating if cached image descriptions should be loaded if available.
  * @returns The described image.
  */
 export async function loadImageWithKnowledge(
     fileName: string,
+    cachePath: string,
     model: ChatModel,
     loadCachedDetails: boolean = true,
 ): Promise<Image | undefined> {
-    const cachedFileName: string = fileName + ".kr.json";
+    const cachedFileName: string = path.join(
+        cachePath,
+        path.basename(fileName) + ".kr.json",
+    );
     if (loadCachedDetails && fs.existsSync(cachedFileName)) {
         return JSON.parse(fs.readFileSync(cachedFileName, "utf8"));
     }

ts/packages/memory/image/src/importImages.ts

@@ -441,6 +441,7 @@ export class ImageCollection implements IConversation<ImageMeta> {
  */
 export async function importImages(
     imagePath: string,
+    cachePath: string | undefined,
     recursive: boolean = true,
     callback?: (text: string, count: number, max: number) => void,
 ): Promise<ImageCollection> {
@@ -452,14 +453,28 @@ export async function importImages(
         );
     }
 
+    if (cachePath !== undefined) {
+        if (!fs.existsSync(cachePath)) {
+            fs.mkdirSync(cachePath);
+        }
+    } else {
+        cachePath = path.dirname(imagePath);
+    }
+
     // create a model used to extract data from the images
     const chatModel = createKnowledgeModel();
 
     let images: Image[] = [];
     if (isDir) {
-        images = await indexImages(imagePath, recursive, chatModel, callback);
+        images = await indexImages(
+            imagePath,
+            cachePath,
+            recursive,
+            chatModel,
+            callback,
+        );
     } else {
-        const img = await indexImage(imagePath, chatModel);
+        const img = await indexImage(imagePath, cachePath, chatModel);
         if (img !== undefined) {
             images.push(img);
         }
@@ -472,12 +487,14 @@ export async function importImages(
  * Imports images from the supplied folder.
  *
  * @param sourcePath - The folder to import.
+ * @param cachePath - The folder to cache the knowledge responses in
  * @param recursive - A flag indicating whether or not subfolders are imported.
  * @param chatModel - The model used to extract data from the image.
  * @returns - The imported images from the supplied folder.
  */
 async function indexImages(
     sourcePath: string,
+    cachePath: string,
     recursive: boolean,
     chatModel: ChatModel,
     callback?: (text: string, count: number, max: number) => void,
@@ -492,7 +509,7 @@ async function indexImages(
     for (let i = 0; i < fileNames.length; i++) {
         const fullFilePath: string = path.join(sourcePath, fileNames[i]);
         //console.log(`${fullFilePath} [${i+1} of ${fileNames.length}] (estimated time remaining: ${clock.elapsedSeconds / (i + 1) * (fileNames.length - i)})`);
-        const img = await indexImage(fullFilePath, chatModel);
+        const img = await indexImage(fullFilePath, cachePath, chatModel);
 
         if (callback) {
             callback(fileNames[i], i, fileNames.length);
@@ -510,11 +527,13 @@ async function indexImages(
  * Imports the supplied image file (if it's an image)
  *
  * @param fileName - The file to import
+ * @param cachePath - The folder to cache the knowledge response in.
  * @param chatModel - The model used to extract data from the image.
  * @returns - The imported image.
  */
 async function indexImage(
     fileName: string,
+    cachePath: string,
     chatModel: ChatModel,
 ): Promise<Image | undefined> {
     if (!fs.existsSync(fileName)) {
@@ -527,6 +546,7 @@ async function indexImage(
 
     const img: image.Image | undefined = await image.loadImageWithKnowledge(
         fileName,
+        cachePath,
         chatModel,
     );