Tag scope

ts/packages/knowPro/src/accumulators.ts

@@ -117,15 +117,11 @@ export class MatchAccumulator<T = any> {
         }
     }
 
-    public getMatches(): IterableIterator<Match<T>> {
-        return this.matches.values();
-    }
-
-    public *getMatchesWhere(
-        predicate: (match: Match<T>) => boolean,
+    public *getMatches(
+        predicate?: (match: Match<T>) => boolean,
     ): IterableIterator<Match<T>> {
         for (const match of this.matches.values()) {
-            if (predicate(match)) {
+            if (predicate === undefined || predicate(match)) {
                 yield match;
             }
         }
@@ -165,7 +161,7 @@ export class MatchAccumulator<T = any> {
         minHitCount: number | undefined,
     ): IterableIterator<Match<T>> {
         return minHitCount !== undefined && minHitCount > 0
-            ? this.getMatchesWhere((m) => m.hitCount >= minHitCount)
+            ? this.getMatches((m) => m.hitCount >= minHitCount)
             : this.matches.values();
     }
 }
@@ -235,6 +231,17 @@ export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {
         );
     }
 
+    public *getSemanticRefs(
+        semanticRefs: SemanticRef[],
+        predicate?: (semanticRef: SemanticRef) => boolean,
+    ) {
+        for (const match of this.getMatches()) {
+            const semanticRef = semanticRefs[match.value];
+            if (predicate === undefined || predicate(semanticRef))
+                yield semanticRef;
+        }
+    }
+
     public groupMatchesByKnowledgeType(
         semanticRefs: SemanticRef[],
     ): Map<KnowledgeType, SemanticRefAccumulator> {
@@ -252,6 +259,19 @@ export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {
         return groups;
     }
 
+    public selectInScope(
+        semanticRefs: SemanticRef[],
+        scope: TextRangeAccumulator,
+    ) {
+        const accumulator = new SemanticRefAccumulator(this.queryTermMatches);
+        for (const match of this.getMatches()) {
+            if (scope.isInRange(semanticRefs[match.value].range)) {
+                accumulator.setMatch(match);
+            }
+        }
+        return accumulator;
+    }
+
     public toScoredSemanticRefs(): ScoredSemanticRef[] {
         return this.getSortedByScore(0).map((m) => {
             return {
@@ -348,13 +368,14 @@ export class QueryTermAccumulator {
 
 export class TextRangeAccumulator {
     constructor(
-        public rangesForMessage: Map<MessageIndex, TextRange[]> = new Map<
-            MessageIndex,
-            TextRange[]
-        >(),
+        private rangesForMessage = new Map<MessageIndex, TextRange[]>(),
     ) {}
 
-    public addTextRange(textRange: TextRange) {
+    public get size() {
+        return this.rangesForMessage.size;
+    }
+
+    public addRange(textRange: TextRange) {
         const messageIndex = textRange.start.messageIndex;
         let textRanges = this.rangesForMessage.get(messageIndex);
         if (textRanges === undefined) {

ts/packages/knowPro/src/query.ts

@@ -19,6 +19,7 @@ import {
     MatchAccumulator,
     QueryTermAccumulator,
     SemanticRefAccumulator,
+    TextRangeAccumulator,
 } from "./accumulators.js";
 import { collections, dateTime } from "typeagent";
 
@@ -312,7 +313,7 @@ export class WhereSemanticRefExpr
             accumulator.queryTermMatches,
         );
         filtered.setMatches(
-            accumulator.getMatchesWhere((match) =>
+            accumulator.getMatches((match) =>
                 this.matchPredicatesOr(
                     context,
                     accumulator.queryTermMatches,
@@ -430,16 +431,30 @@ export class ActionPredicate implements IQuerySemanticRefPredicate {
     }
 }
 
-export class ScopeExpr implements IQueryOpExpr<void> {
-    constructor(public predicates: IQueryScopePredicate[]) {}
+export class ApplyTagScopeExpr implements IQueryOpExpr<SemanticRefAccumulator> {
+    constructor(public sourceExpr: IQueryOpExpr<SemanticRefAccumulator>) {}
 
-    public eval(context: QueryEvalContext): Promise<void> {
-        return Promise.resolve();
+    public async eval(
+        context: QueryEvalContext,
+    ): Promise<SemanticRefAccumulator> {
+        let accumulator = await this.sourceExpr.eval(context);
+        const tagScope = new TextRangeAccumulator();
+        for (const semanticRef of accumulator.getSemanticRefs(
+            context.semanticRefs,
+            (sr) => sr.knowledgeType === "tag",
+        )) {
+            tagScope.addRange(semanticRef.range);
+        }
+        if (tagScope.size > 0) {
+            accumulator = accumulator.selectInScope(
+                context.semanticRefs,
+                tagScope,
+            );
+        }
+        return Promise.resolve(accumulator);
     }
 }
 
-export interface IQueryScopePredicate {}
-
 function isPropertyMatch(
     termMatches: QueryTermAccumulator,
     testText: string | string[] | undefined,

ts/packages/knowPro/src/search.ts

@@ -72,6 +72,21 @@ function createTermSearchQuery(
     wherePredicates?: q.IQuerySemanticRefPredicate[] | undefined,
     maxMatches?: number,
     minHitCount?: number,
+) {
+    const query = new q.SelectTopNKnowledgeGroupExpr(
+        new q.GroupByKnowledgeTypeExpr(
+            createTermsMatch(conversation, terms, wherePredicates),
+        ),
+        maxMatches,
+        minHitCount,
+    );
+    return query;
+}
+
+function createTermsMatch(
+    conversation: IConversation,
+    terms: QueryTerm[],
+    wherePredicates?: q.IQuerySemanticRefPredicate[] | undefined,
 ) {
     const queryTerms = new q.QueryTermsExpr(terms);
     let termsMatchExpr: q.IQueryOpExpr<SemanticRefAccumulator> =
@@ -80,18 +95,14 @@ function createTermSearchQuery(
                 ? new q.ResolveRelatedTermsExpr(queryTerms)
                 : queryTerms,
         );
+    termsMatchExpr = new q.ApplyTagScopeExpr(termsMatchExpr);
     if (wherePredicates !== undefined && wherePredicates.length > 0) {
         termsMatchExpr = new q.WhereSemanticRefExpr(
             termsMatchExpr,
             wherePredicates,
         );
     }
-    const query = new q.SelectTopNKnowledgeGroupExpr(
-        new q.GroupByKnowledgeTypeExpr(termsMatchExpr),
-        maxMatches,
-        minHitCount,
-    );
-    return query;
+    return termsMatchExpr;
 }
 
 function toGroupedSearchResults(

ts/packages/knowPro/src/termIndex.ts

@@ -18,6 +18,7 @@ import {
     ITextEmbeddingDataItem,
     ITextEmbeddingData,
 } from "./dataFormat.js";
+import { createEmbeddingCache } from "knowledge-processor";
 
 export async function buildTermSemanticIndex(
     settings: SemanticIndexSettings,
@@ -166,7 +167,7 @@ export type SemanticIndexSettings = {
 
 export function createSemanticIndexSettings(): SemanticIndexSettings {
     return {
-        embeddingModel: openai.createEmbeddingModel(),
+        embeddingModel: createEmbeddingCache(openai.createEmbeddingModel(), 64),
         minScore: 0.8,
         retryMaxAttempts: 2,
         retryPauseMs: 2000,

ts/packages/knowledgeProcessor/src/modelCache.ts

@@ -19,13 +19,12 @@ export function createEmbeddingCache(
     model: TextEmbeddingModel,
     cacheSize: number,
 ): TextEmbeddingModelWithCache {
-    const maxBatchSize = 1;
     const cache: collections.Cache<string, number[]> =
         collections.createLRUCache(cacheSize);
     const modelWithCache: TextEmbeddingModelWithCache = {
         cache,
         generateEmbedding,
-        maxBatchSize,
+        maxBatchSize: model.maxBatchSize,
     };
     if (model.generateEmbeddingBatch) {
         modelWithCache.generateEmbeddingBatch = generateEmbeddingBatch;