knowpro: natural language querying (phase 1) (#783)

Part 1:
- Start by using knowlege-processor to translate natural language
queries into TermFilters
- Transforms term filters (terms, date ranges) into know-pro
**searchConversation** call
- Execute query

Next check-in will also transform actions.

ts/examples/chat/src/memory/knowproCommon.ts

@@ -1,23 +1,9 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+import * as knowLib from "knowledge-processor";
 import * as kp from "knowpro";
 
-export function getTimeRangeForConversation(
-    conversation: kp.IConversation,
-): kp.DateRange | undefined {
-    const messages = conversation.messages;
-    const start = messages[0].timestamp;
-    const end = messages[messages.length - 1].timestamp;
-    if (start !== undefined) {
-        return {
-            start: new Date(start),
-            end: end ? new Date(end) : undefined,
-        };
-    }
-    return undefined;
-}
-
 export function textLocationToString(location: kp.TextLocation): string {
     let text = `MessageIndex: ${location.messageIndex}`;
     if (location.chunkIndex) {
@@ -28,3 +14,61 @@ export function textLocationToString(location: kp.TextLocation): string {
     }
     return text;
 }
+
+export async function matchFilterToConversation(
+    conversation: kp.IConversation,
+    filter: knowLib.conversation.TermFilterV2,
+    knowledgeType?: kp.KnowledgeType | undefined,
+    useAnd: boolean = false,
+) {
+    let searchTermGroup: kp.SearchTermGroup = termFilterToSearchGroup(
+        filter,
+        useAnd,
+    );
+    let when: kp.WhenFilter = termFilterToWhenFilter(filter);
+    when.knowledgeType = knowledgeType;
+    let searchResults = await kp.searchConversation(
+        conversation,
+        searchTermGroup,
+        when,
+    );
+    if (useAnd && (!searchResults || searchResults.size === 0)) {
+        // Try again with OR
+        searchTermGroup = termFilterToSearchGroup(filter, false);
+        searchResults = await kp.searchConversation(
+            conversation,
+            searchTermGroup,
+            when,
+        );
+    }
+    return searchResults;
+}
+
+export function termFilterToSearchGroup(
+    filter: knowLib.conversation.TermFilterV2,
+    and: boolean,
+): kp.SearchTermGroup {
+    const searchTermGroup: kp.SearchTermGroup = {
+        booleanOp: and ? "and" : "or",
+        terms: [],
+    };
+    if (filter.searchTerms && filter.searchTerms.length > 0) {
+        for (const st of filter.searchTerms) {
+            searchTermGroup.terms.push({ term: { text: st } });
+        }
+    }
+    return searchTermGroup;
+}
+
+export function termFilterToWhenFilter(
+    filter: knowLib.conversation.TermFilterV2,
+) {
+    let when: kp.WhenFilter = {};
+    if (filter.timeRange) {
+        when.dateRange = {
+            start: knowLib.conversation.toStartDate(filter.timeRange.startDate),
+            end: knowLib.conversation.toStopDate(filter.timeRange.stopDate),
+        };
+    }
+    return when;
+}

ts/examples/chat/src/memory/knowproMemory.ts

@@ -29,12 +29,14 @@ import { dateTime, ensureDir, getFileName } from "typeagent";
 import path from "path";
 import chalk from "chalk";
 import { KnowProPrinter } from "./knowproPrinter.js";
-import { getTimeRangeForConversation } from "./knowproCommon.js";
 import * as cm from "conversation-memory";
 import * as im from "image-memory";
+import { matchFilterToConversation } from "./knowproCommon.js";
 
 type KnowProContext = {
     knowledgeModel: ChatModel;
+    knowledgeExtractor: knowLib.conversation.KnowledgeExtractor;
+    knowledgeActions: knowLib.conversation.KnowledgeActionTranslator;
     basePath: string;
     printer: KnowProPrinter;
     podcast?: cm.Podcast | undefined;
@@ -46,8 +48,14 @@ export async function createKnowproCommands(
     chatContext: ChatContext,
     commands: Record<string, CommandHandler>,
 ): Promise<void> {
+    const knowledgeModel = chatContext.models.chatModel;
     const context: KnowProContext = {
-        knowledgeModel: chatContext.models.chatModel,
+        knowledgeModel,
+        knowledgeExtractor: kp.createKnowledgeProcessor(knowledgeModel),
+        knowledgeActions:
+            knowLib.conversation.createKnowledgeActionTranslator(
+                knowledgeModel,
+            ),
         basePath: "/data/testChat/knowpro",
         printer: new KnowProPrinter(),
     };
@@ -58,6 +66,8 @@ export async function createKnowproCommands(
     commands.kpPodcastSave = podcastSave;
     commands.kpPodcastLoad = podcastLoad;
     commands.kpSearchTerms = searchTerms;
+    commands.kpSearch = search;
+    commands.kpSearchK = knowledgeSearch;
     commands.kpEntities = entities;
     commands.kpPodcastBuildIndex = podcastBuildIndex;
 
@@ -436,6 +446,80 @@ export async function createKnowproCommands(
         }
     }
 
+    function searchDef(): CommandMetadata {
+        return {
+            description: "Search using natural language",
+            args: {
+                query: arg("Search query"),
+            },
+            options: {
+                maxToDisplay: argNum("Maximum matches to display", 25),
+                ktype: arg("Knowledge type"),
+            },
+        };
+    }
+    commands.kpSearch.metadata = searchDef();
+    async function search(args: string[]): Promise<void> {
+        if (!ensureConversationLoaded()) {
+            context.printer.writeError("No conversation loaded");
+            return;
+        }
+        const namedArgs = parseNamedArguments(args, searchDef());
+        const query = namedArgs.query;
+        const result = await context.knowledgeActions.translateSearchTermsV2(
+            query,
+            kp.getTimeRangeSectionForConversation(context.conversation!),
+        );
+        if (!result.success) {
+            context.printer.writeError(result.message);
+            return;
+        }
+        let searchAction = result.data;
+        if (searchAction.actionName !== "getAnswer") {
+            return;
+        }
+        context.printer.writeSearchFilter(searchAction);
+        if (searchAction.parameters.filters.length > 0) {
+            const filter = searchAction.parameters.filters[0];
+            const searchResults = await matchFilterToConversation(
+                context.conversation!,
+                filter,
+                namedArgs.ktype,
+            );
+            if (searchResults) {
+                context.printer.writeSearchResults(
+                    context.conversation!,
+                    searchResults,
+                    namedArgs.maxToDisplay,
+                );
+            }
+        }
+    }
+
+    commands.kpSearchK.metadata = searchDef();
+    async function knowledgeSearch(args: string[]): Promise<void> {
+        const namedArgs = parseNamedArguments(args, searchDef());
+        const query = namedArgs.query;
+        const knowledge = await context.knowledgeExtractor.extract(query);
+        if (knowledge) {
+            context.printer.writeTitle("Topics");
+            for (const topic of knowledge.topics) {
+                context.printer.writeLine(topic);
+                context.printer.writeLine();
+            }
+            context.printer.writeTitle("Actions");
+            for (const action of knowledge.actions) {
+                context.printer.writeAction(action);
+                context.printer.writeLine();
+            }
+            context.printer.writeTitle("Entities");
+            for (const entity of knowledge.entities) {
+                context.printer.writeEntity(entity);
+                context.printer.writeLine();
+            }
+        }
+    }
+
     function createSearchGroup(
         termArgs: string[],
         namedArgs: NamedArgs,
@@ -487,7 +571,7 @@ export async function createKnowproCommands(
         };
         const conv: kp.IConversation | undefined =
             context.podcast ?? context.images;
-        const dateRange = getTimeRangeForConversation(conv!);
+        const dateRange = kp.getTimeRangeForConversation(conv!);
         if (dateRange) {
             let startDate: Date | undefined;
             let endDate: Date | undefined;

ts/examples/chat/src/memory/knowproPrinter.ts

@@ -5,10 +5,7 @@ import * as kp from "knowpro";
 import * as knowLib from "knowledge-processor";
 import { ChatPrinter } from "../chatPrinter.js";
 import chalk from "chalk";
-import {
-    getTimeRangeForConversation,
-    textLocationToString,
-} from "./knowproCommon.js";
+import { textLocationToString } from "./knowproCommon.js";
 import * as cm from "conversation-memory";
 import * as im from "image-memory";
 
@@ -298,7 +295,7 @@ export class KnowProPrinter extends ChatPrinter {
 
     public writeConversationInfo(conversation: kp.IConversation) {
         this.writeTitle(conversation.nameTag);
-        const timeRange = getTimeRangeForConversation(conversation);
+        const timeRange = kp.getTimeRangeForConversation(conversation);
         if (timeRange) {
             this.write("Time range: ");
             this.writeDateRange(timeRange);
@@ -334,6 +331,17 @@ export class KnowProPrinter extends ChatPrinter {
         }
         return this;
     }
+
+    public writeSearchFilter(
+        action: knowLib.conversation.GetAnswerWithTermsActionV2,
+    ) {
+        this.writeInColor(
+            chalk.cyanBright,
+            `Question: ${action.parameters.question}`,
+        );
+        this.writeLine();
+        this.writeJson(action.parameters.filters);
+    }
 }
 
 function getPodcastParticipants(podcast: cm.Podcast) {

ts/packages/knowPro/src/conversationIndex.ts

@@ -20,7 +20,7 @@ import {
 } from "./interfaces.js";
 import { IndexingEventHandlers } from "./interfaces.js";
 import { conversation as kpLib } from "knowledge-processor";
-import { openai } from "aiclient";
+import { ChatModel, openai } from "aiclient";
 import { async } from "typeagent";
 import { facetValueToString } from "./knowledge.js";
 import { buildSecondaryIndexes } from "./secondaryIndexes.js";
@@ -201,6 +201,7 @@ export function addKnowledgeToIndex(
 
 export async function buildSemanticRefIndex<TMeta extends IKnowledgeSource>(
     conversation: IConversation<TMeta>,
+    extractor?: kpLib.KnowledgeExtractor,
     eventHandler?: IndexingEventHandlers,
 ): Promise<IndexingResults> {
     conversation.semanticRefIndex ??= new ConversationIndex();
@@ -210,11 +211,7 @@ export async function buildSemanticRefIndex<TMeta extends IKnowledgeSource>(
         conversation.semanticRefs = [];
     }
     const semanticRefs = conversation.semanticRefs;
-    const chatModel = createKnowledgeModel();
-    const extractor = kpLib.createKnowledgeExtractor(chatModel, {
-        maxContextLength: 4096,
-        mergeActionKnowledge: false,
-    });
+    extractor ??= createKnowledgeProcessor();
     const maxRetries = 4;
     let indexingResult: IndexingResults = {};
     for (let i = 0; i < conversation.messages.length; i++) {
@@ -376,11 +373,26 @@ export function createKnowledgeModel() {
     return chatModel;
 }
 
+export function createKnowledgeProcessor(
+    chatModel?: ChatModel,
+): kpLib.KnowledgeExtractor {
+    chatModel ??= createKnowledgeModel();
+    const extractor = kpLib.createKnowledgeExtractor(chatModel, {
+        maxContextLength: 4096,
+        mergeActionKnowledge: false,
+    });
+    return extractor;
+}
+
 export async function buildConversationIndex(
     conversation: IConversation,
     eventHandler?: IndexingEventHandlers,
 ): Promise<IndexingResults> {
-    const result = await buildSemanticRefIndex(conversation, eventHandler);
+    const result = await buildSemanticRefIndex(
+        conversation,
+        undefined,
+        eventHandler,
+    );
     if (!result.error && conversation.semanticRefIndex) {
         await buildSecondaryIndexes(conversation, true, eventHandler);
     }

ts/packages/knowPro/src/search.ts

@@ -17,6 +17,7 @@ import * as q from "./query.js";
 import { IQueryOpExpr } from "./query.js";
 import { resolveRelatedTerms } from "./relatedTermsIndex.js";
 import { conversation as kpLib } from "knowledge-processor";
+import { PromptSection } from "typechat";
 
 export type SearchTerm = {
     /**
@@ -466,3 +467,33 @@ function isActionPropertyTerm(term: PropertySearchTerm): boolean {
 
     return false;
 }
+
+export function getTimeRangeForConversation(
+    conversation: IConversation,
+): DateRange | undefined {
+    const messages = conversation.messages;
+    const start = messages[0].timestamp;
+    const end = messages[messages.length - 1].timestamp;
+    if (start !== undefined) {
+        return {
+            start: new Date(start),
+            end: end ? new Date(end) : undefined,
+        };
+    }
+    return undefined;
+}
+
+export function getTimeRangeSectionForConversation(
+    conversation: IConversation,
+): PromptSection[] {
+    const timeRange = getTimeRangeForConversation(conversation);
+    if (timeRange) {
+        return [
+            {
+                role: "system",
+                content: `ONLY IF user request explicitly asks for time ranges, THEN use the CONVERSATION TIME RANGE: "${timeRange.start} to ${timeRange.end}"`,
+            },
+        ];
+    }
+    return [];
+}