diff --git a/ts/packages/knowPro/src/collections.ts b/ts/packages/knowPro/src/collections.ts index 5a417076c..a61c42fdc 100644 --- a/ts/packages/knowPro/src/collections.ts +++ b/ts/packages/knowPro/src/collections.ts @@ -182,7 +182,10 @@ export class SemanticRefAccumulator extends MatchAccumulator { public addSearchTermMatch( searchTerm: Term, - semanticRefs: ScoredSemanticRef[] | undefined, + semanticRefs: + | ScoredSemanticRef[] + | IterableIterator + | undefined, scoreBoost?: number, ) { if (semanticRefs) { @@ -197,7 +200,10 @@ export class SemanticRefAccumulator extends MatchAccumulator { public addRelatedTermMatch( searchTerm: Term, relatedTerm: Term, - semanticRefs: ScoredSemanticRef[] | undefined, + semanticRefs: + | ScoredSemanticRef[] + | IterableIterator + | undefined, scoreBoost?: number, ) { if (semanticRefs) { diff --git a/ts/packages/knowPro/src/query.ts b/ts/packages/knowPro/src/query.ts index dc94bcc1c..ab349e2a6 100644 --- a/ts/packages/knowPro/src/query.ts +++ b/ts/packages/knowPro/src/query.ts @@ -8,6 +8,7 @@ import { IPropertyToSemanticRefIndex, ITermToSemanticRefIndex, KnowledgeType, + ScoredSemanticRef, SemanticRef, SemanticRefIndex, Term, @@ -148,16 +149,19 @@ export function textRangeForMessage( }; } -export function lookupSearchTermInIndex( +export function lookupSearchTermInSemanticRefIndex( semanticRefIndex: ITermToSemanticRefIndex, searchTerm: SearchTerm, + predicate?: (scoredRef: ScoredSemanticRef) => boolean, matchAccumulator?: SemanticRefAccumulator, ): SemanticRefAccumulator { matchAccumulator ??= new SemanticRefAccumulator(); // Lookup search term matchAccumulator.addSearchTermMatch( searchTerm.term, - semanticRefIndex.lookupTerm(searchTerm.term.text), + predicate + ? lookupAndFilter(semanticRefIndex, searchTerm.term.text, predicate) + : semanticRefIndex.lookupTerm(searchTerm.term.text), ); // And any related terms if (searchTerm.relatedTerms && searchTerm.relatedTerms.length > 0) { @@ -167,12 +171,33 @@ export function lookupSearchTermInIndex( matchAccumulator.addRelatedTermMatch( searchTerm.term, relatedTerm, - semanticRefIndex.lookupTerm(relatedTerm.text), + predicate + ? lookupAndFilter( + semanticRefIndex, + relatedTerm.text, + predicate, + ) + : semanticRefIndex.lookupTerm(relatedTerm.text), relatedTerm.score, ); } } return matchAccumulator; + + function* lookupAndFilter( + semanticRefIndex: ITermToSemanticRefIndex, + text: string, + predicate: (scoredRef: ScoredSemanticRef) => boolean, + ) { + const scoredRefs = semanticRefIndex.lookupTerm(text); + if (scoredRefs) { + for (const scoredRef of scoredRefs) { + if (predicate(scoredRef)) { + yield scoredRef; + } + } + } + } } export function lookupSearchTermInPropertyIndex( @@ -277,6 +302,14 @@ export class QueryEvalContext { } } + public get semanticRefIndex() { + return this.conversation.semanticRefIndex!; + } + + public get propertyIndex() { + return this.conversation.propertyToSemanticRefIndex; + } + public getSemanticRef(semanticRefIndex: SemanticRefIndex): SemanticRef { return this.conversation.semanticRefs![semanticRefIndex]; } @@ -338,23 +371,17 @@ export class MatchSearchTermExpr extends QueryOpExpr { } public override eval(context: QueryEvalContext): SemanticRefAccumulator { - const matchAccumulator = new SemanticRefAccumulator(); - const semanticRefIndex = context.conversation.semanticRefIndex; - if (semanticRefIndex) { - lookupSearchTermInIndex( - semanticRefIndex, - this.searchTerm, - matchAccumulator, - ); - } - return matchAccumulator; + return lookupSearchTermInSemanticRefIndex( + context.semanticRefIndex, + this.searchTerm, + ); } } export class MatchQualifiedSearchTermExpr extends QueryOpExpr< SemanticRefAccumulator | undefined > { - constructor(public searchTerm: QualifiedSearchTerm) { + constructor(public qualifiedSearchTerm: QualifiedSearchTerm) { super(); } @@ -365,23 +392,45 @@ export class MatchQualifiedSearchTermExpr extends QueryOpExpr< return undefined; } let matches: SemanticRefAccumulator | undefined; - if (this.searchTerm.type === "property") { - matches = this.matchProperty(context, this.searchTerm); + if (this.qualifiedSearchTerm.type === "property") { + matches = this.matchProperty(context, this.qualifiedSearchTerm); } else { - matches = this.matchFacet(context, this.searchTerm); + matches = this.matchFacet(context, this.qualifiedSearchTerm); } return matches; } private matchProperty( context: QueryEvalContext, - searchTerm: PropertySearchTerm, + propertySearchTerm: PropertySearchTerm, ): SemanticRefAccumulator | undefined { - const propertyIndex = context.conversation.propertyToSemanticRefIndex!; - return lookupSearchTermInPropertyIndex( - propertyIndex, - searchTerm.propertyName, + if (propertySearchTerm.propertyName === "tag") { + return this.matchTag(context, propertySearchTerm); + } + const propertyIndex = context.propertyIndex; + if (propertyIndex) { + return lookupSearchTermInPropertyIndex( + propertyIndex, + propertySearchTerm.propertyName, + propertySearchTerm.propertyValue, + ); + } + return undefined; + } + + private matchTag( + context: QueryEvalContext, + searchTerm: PropertySearchTerm, + ) { + return lookupSearchTermInSemanticRefIndex( + context.semanticRefIndex, searchTerm.propertyValue, + (scoredRef) => { + return ( + context.getSemanticRef(scoredRef.semanticRefIndex) + .knowledgeType === "tag" + ); + }, ); } @@ -389,28 +438,31 @@ export class MatchQualifiedSearchTermExpr extends QueryOpExpr< context: QueryEvalContext, facetSearchTerm: FacetSearchTerm, ): SemanticRefAccumulator | undefined { - const propertyIndex = context.conversation.propertyToSemanticRefIndex!; - let facetMatches = lookupSearchTermInPropertyIndex( - propertyIndex, - PropertyNames.FacetName, - facetSearchTerm.facetName, - ); - if ( - facetMatches.size > 0 && - facetSearchTerm.facetValue && - !isSearchTermWildcard(facetSearchTerm.facetValue) - ) { - let valueMatches = lookupSearchTermInPropertyIndex( + const propertyIndex = context.propertyIndex; + if (propertyIndex) { + let facetMatches = lookupSearchTermInPropertyIndex( propertyIndex, - PropertyNames.FacetValue, - facetSearchTerm.facetValue, + PropertyNames.FacetName, + facetSearchTerm.facetName, ); - if (valueMatches.size > 0) { - facetMatches = facetMatches.intersect(valueMatches); + if ( + facetMatches.size > 0 && + facetSearchTerm.facetValue && + !isSearchTermWildcard(facetSearchTerm.facetValue) + ) { + let valueMatches = lookupSearchTermInPropertyIndex( + propertyIndex, + PropertyNames.FacetValue, + facetSearchTerm.facetValue, + ); + if (valueMatches.size > 0) { + facetMatches = facetMatches.intersect(valueMatches); + } } - } - return facetMatches; + return facetMatches; + } + return undefined; } } diff --git a/ts/packages/knowPro/src/search.ts b/ts/packages/knowPro/src/search.ts index 0722e487f..a0c01118c 100644 --- a/ts/packages/knowPro/src/search.ts +++ b/ts/packages/knowPro/src/search.ts @@ -9,7 +9,6 @@ import { ScoredSemanticRef, Term, } from "./dataFormat.js"; -import { PropertyNames } from "./propertyIndex.js"; import * as q from "./query.js"; import { resolveRelatedTerms } from "./relatedTermsIndex.js"; @@ -32,30 +31,26 @@ export function createSearchTerm(text: string, score?: number): SearchTerm { export type QualifiedSearchTerm = PropertySearchTerm | FacetSearchTerm; -export type KnowledgePropertyNames = +export type KnowledgePropertyName = | "name" | "type" | "verb" | "subject" | "object" - | "indirectObject"; + | "indirectObject" + | "tag"; -export interface PropertySearchTerm { +export type PropertySearchTerm = { type: "property"; - propertyName: KnowledgePropertyNames; + propertyName: KnowledgePropertyName; propertyValue: SearchTerm; -} +}; -export interface FacetSearchTerm { +export type FacetSearchTerm = { type: "facet"; facetName: SearchTerm; facetValue: SearchTerm; -} - -export interface TagSearchTerm { - type: "tag"; - tagValue: SearchTerm; -} +}; export type SearchResult = { termMatches: Set; @@ -157,31 +152,9 @@ class SearchQueryBuilder { for (const propertyName of Object.keys(properties)) { const propertyValue = properties[propertyName]; let matchExpr: q.MatchQualifiedSearchTermExpr | undefined; - let qualifiedTerm: QualifiedSearchTerm | undefined; - switch (propertyName) { - default: - qualifiedTerm = { - type: "facet", - facetName: createSearchTerm(propertyName), - facetValue: createSearchTerm(propertyValue), - }; - this.allSearchTerms.push(qualifiedTerm.facetName); - this.allSearchTerms.push(qualifiedTerm.facetValue); - break; - case PropertyNames.EntityName: - case PropertyNames.EntityType: - case PropertyNames.Verb: - case PropertyNames.Subject: - case PropertyNames.Object: - case PropertyNames.IndirectObject: - qualifiedTerm = { - type: "property", - propertyName: propertyName as KnowledgePropertyNames, - propertyValue: createSearchTerm(propertyValue), - }; - this.allSearchTerms.push(qualifiedTerm.propertyValue); - break; - } + const [qualifiedTerm, searchTermsCreated] = + qualifiedSearchTermFromKeyValue(propertyName, propertyValue); + this.allSearchTerms.push(...searchTermsCreated); matchExpr = new q.MatchQualifiedSearchTermExpr(qualifiedTerm); matchExpressions.push(matchExpr); } @@ -231,6 +204,40 @@ class SearchQueryBuilder { } } +export function qualifiedSearchTermFromKeyValue( + key: string, + value: string, +): [QualifiedSearchTerm, SearchTerm[]] { + let qualifiedSearchTerm: QualifiedSearchTerm | undefined; + const searchTermsCreated: SearchTerm[] = []; + switch (key) { + default: + qualifiedSearchTerm = { + type: "facet", + facetName: createSearchTerm(key), + facetValue: createSearchTerm(value), + }; + searchTermsCreated.push(qualifiedSearchTerm.facetName); + searchTermsCreated.push(qualifiedSearchTerm.facetValue); + break; + case "name": + case "type": + case "verb": + case "subject": + case "object": + case "indirectObject": + case "tag": + qualifiedSearchTerm = { + type: "property", + propertyName: key as KnowledgePropertyName, + propertyValue: createSearchTerm(value), + }; + searchTermsCreated.push(qualifiedSearchTerm.propertyValue); + break; + } + return [qualifiedSearchTerm, searchTermsCreated]; +} + function toGroupedSearchResults( evalResults: Map, ) {