Experiments with ai validator

docs/teachertool/catalog-shared-test.json

@@ -2,8 +2,8 @@
     "criteria": [
         {
             "id": "E3FBB45F-B63B-46FB-A101-72CAAC709301",
-            "use": "another_rule",
-            "template": "TEST some other rule",
+            "use": "ai_question",
+            "template": "Ask experimental AI: Analyze the use of variables.",
             "docPath": "/teachertool"
         },
         {

pxtblocks/code-validation/askAiQuestion.ts

@@ -0,0 +1,43 @@
+namespace pxt.blocks {
+    const aiEndpoint = "http://127.0.0.1:5000/api/v1/query";
+
+    interface AiQueryRequest {
+        query: string;
+        intent: "makecode_evaluation",
+        context: {
+            share_id: string,
+            target: string
+        }
+    }
+
+    export async function askAiQuestion({ shareId, target, question }: { shareId: string, target: string, question: string}): Promise<{ responseMessage: string }> {
+        let responseMessage = "";
+
+        const request: AiQueryRequest = {
+            query: question,
+            intent: "makecode_evaluation",
+            context: {
+                share_id: shareId,
+                target: target
+            }
+        }
+
+        const requestJson = JSON.stringify(request);
+
+        const response = await fetch(aiEndpoint, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: requestJson
+        });
+
+        if (!response.ok) {
+            responseMessage = await response.text();
+        } else {
+            console.error(`Failed to query AI: ${response.status} ${response.statusText}`);
+        }
+
+        return { responseMessage };
+    }
+}

pxtblocks/code-validation/evaluationResult.ts

@@ -1,5 +1,11 @@
 namespace pxt.blocks {
+
+  export interface CriteriaResult {
+    passed: boolean;
+    message: string;
+  }
+
   export interface EvaluationResult {
-   blockIdResults: pxt.Map<boolean>;
+    criteriaResults: pxt.Map<CriteriaResult>;
   }
 }

pxtblocks/code-validation/rubricCriteria.ts

@@ -31,18 +31,27 @@ namespace pxt.blocks {
         return requiredBlockCounts;
     }
 
-    export function validateProject(usedBlocks: Blockly.Block[], rubric: string): EvaluationResult {
-        const rubricData = parseRubric(rubric);
-        const finalResult: pxt.Map<boolean> = {};
-        rubricData.criteria.forEach((criteria: RubricCriteria) => {
-            (criteria as BlockCheckCriteria).blockRequirements.forEach((blockSet) => {
-                const result = validateBlockSet(usedBlocks, blockSet);
-                Object.keys(result).forEach((blockId) => {
-                    finalResult[blockId] = result[blockId];
-                });
-            });
-        });
-        return { blockIdResults: finalResult } as EvaluationResult;
+    export async function validateProject(usedBlocks: Blockly.Block[], rubric: CriteriaInstance[], catalog: CatalogCriteria[]): Promise<EvaluationResult> {
+        const finalResult: pxt.Map<CriteriaResult> = {};
+
+        await Promise.all(rubric.map(async (criteria: CriteriaInstance) => {
+            const catalogCriteria = catalog.find((c) => c.id === criteria.catalogCriteriaId);
+            if (!catalogCriteria) {
+                console.error(`Could not find catalog criteria with id ${criteria.catalogCriteriaId}`);
+                return;
+            }
+            const validatorPlanId = catalogCriteria.use;
+
+            if (validatorPlanId == "block_used_n_times") {
+                // TODO - create validation plans somewhere which defines checks to run.
+                finalResult[criteria.instanceId] = { passed: true, message: undefined };
+            } else if (validatorPlanId == "ai_question") {
+                const aiResponse = await askAiQuestion({ shareId: "???", target: "???", question: "Does this program use variables in a meaningful way, and are they well named?" });
+                finalResult[criteria.instanceId] = { passed: true, message: aiResponse.responseMessage };
+            }
+        }));
+
+        return { criteriaResults: finalResult } as EvaluationResult;
     }
 
 

pxtblocks/code-validation/types.ts

@@ -1,4 +1,26 @@
 namespace pxt.blocks {
+    export interface ValidatorCheck {
+        validator: string;
+    };
+
+    export interface BlockExistsValidatorCheck extends ValidatorCheck {
+        validator: "block_exists";
+        inputs: {
+            blockId: string;
+            count: number;
+        }[];
+    }
+
+    export interface AiQuestionValidatorCheck extends ValidatorCheck {
+        validator: "ai_question";
+    }
+
+    export interface ValidatorPlan {
+        name: string;
+        threshold: number;
+        checks: ValidatorCheck[];
+    }
+
     // A criteria defined in the catalog of all possible criteria for the user to choose from when creating a rubric.
     export interface CatalogCriteria {
         id: string; // A unique id (GUID) for the catalog criteria

pxteditor/editorcontroller.ts

@@ -292,7 +292,8 @@ namespace pxt.editor {
 
     export interface EditorMessageRunEvalRequest extends EditorMessageRequest {
         action: "runeval";
-        rubric: string;
+        rubric: pxt.blocks.CriteriaInstance[];
+        catalog: pxt.blocks.CatalogCriteria[];
     }
 
     export interface EditorMessageRenderBlocksResponse {
@@ -547,10 +548,11 @@ namespace pxt.editor {
                                 case "runeval": {
                                     const evalmsg = data as EditorMessageRunEvalRequest;
                                     const rubric = evalmsg.rubric;
+                                    const catalog = evalmsg.catalog;
                                     return Promise.resolve()
                                         .then(() => {
                                             const blocks = projectView.getBlocks();
-                                            return pxt.blocks.validateProject(blocks, rubric)})
+                                            return pxt.blocks.validateProject(blocks, rubric, catalog)})
                                         .then (results => {
                                             resp = results;
                                         });

teachertool/src/components/DebugInput.tsx

@@ -3,7 +3,6 @@
 import { useState } from "react";
 import { Button } from "react-common/components/controls/Button";
 import { Input } from "react-common/components/controls/Input";
-import { Textarea } from "react-common/components/controls/Textarea";
 import { loadProjectMetadataAsync } from "../transforms/loadProjectMetadataAsync";
 import { runEvaluateAsync } from "../transforms/runEvaluateAsync";
 
@@ -15,7 +14,7 @@ const DebugInput: React.FC<IProps> = ({}) => {
 
     const evaluate = async () => {
         await loadProjectMetadataAsync(shareLink);
-        await runEvaluateAsync(rubric);
+        await runEvaluateAsync();
     }
 
     return (
@@ -29,14 +28,6 @@ const DebugInput: React.FC<IProps> = ({}) => {
                     initialValue={shareLink}
                     onChange={setShareLink} />
             </div>
-            <div className="rubric-json-input-container">
-                {lf("Rubric:")}
-                <Textarea
-                    id="rubricJsonInput"
-                    className="json-input"
-                    rows={20}
-                    onChange={setRubric} />
-            </div>
             <Button id="evaluateSingleProjectButton" className="primary" onClick={evaluate} title={"Evaluate"} label={lf("Evaluate")} />
         </div>
     )

teachertool/src/components/EvalResultDisplay.tsx

@@ -14,14 +14,16 @@ const EvalResultDisplay: React.FC<IProps> = ({}) => {
                 <div className="eval-results-container">
                     <h3>{lf("Project: {0}", teacherTool.projectMetadata.name)}</h3>
                     {teacherTool.currentEvalResult === undefined && <div className="common-spinner" />}
-                    {Object.keys(teacherTool.currentEvalResult?.blockIdResults ?? {}).map((id) => {
-                        const result = teacherTool.currentEvalResult?.blockIdResults[id];
-                        return (
+                    {Object.keys(teacherTool.currentEvalResult?.criteriaResults ?? {}).map((id) => {
+                        const result = teacherTool.currentEvalResult?.criteriaResults[id];
+                        const criteriaInstance = teacherTool.selectedCriteria?.find((criteria) => criteria.instanceId === id);
+                        const catalogCriteria = teacherTool.catalog?.find((criteria) => criteria.id === criteriaInstance?.catalogCriteriaId);
+                        return catalogCriteria ? (
                             <div className="result-block-id" key={id}>
-                                <p className="block-id-label">{id}:</p>
+                                <p className="block-id-label">{catalogCriteria?.template}:</p>
                                 <p className={result ? "positive-text" : "negative-text"}>{result ? "passed" : "failed"}</p>
                             </div>
-                        );
+                        ) : null;
                     })}
                 </div>
             )}

teachertool/src/services/makecodeEditorService.ts

@@ -83,14 +83,15 @@ export async function setHighContrastAsync(on: boolean) {
     console.log(result);
 }
 
-export async function runEvalInEditorAsync(serializedRubric: string): Promise<pxt.blocks.EvaluationResult | undefined> {
+export async function runEvalInEditorAsync(rubric: pxt.blocks.CriteriaInstance[], catalog: pxt.blocks.CatalogCriteria[]): Promise<pxt.blocks.EvaluationResult | undefined> {
     let evalResults = undefined;
 
     try {
         const response = await sendMessageAsync({
             type: "pxteditor",
             action: "runeval",
-            rubric: serializedRubric } as pxt.editor.EditorMessageRunEvalRequest
+            rubric,
+            catalog  } as pxt.editor.EditorMessageRunEvalRequest
         );
         const result = response as pxt.editor.EditorMessageResponse;
         validateResponse(result, true); // Throws on failure

teachertool/src/teacherTool.css

@@ -431,7 +431,6 @@ code {
     flex-direction: column;
     align-items: flex-end;
     padding: 1rem;
-    height: 45vh;
 }
 
 .single-share-link-input-container {

teachertool/src/transforms/runEvaluateAsync.ts

@@ -4,10 +4,10 @@ import * as Actions from "../state/actions";
 import { makeNotification } from "../utils";
 import { postNotification } from "./postNotification";
 
-export async function runEvaluateAsync(rubric: string) {
-    const { dispatch } = stateAndDispatch();
+export async function runEvaluateAsync() {
+    const { state: teacherTool, dispatch } = stateAndDispatch();
 
-    const evalResult = await runEvalInEditorAsync(rubric);
+    const evalResult = await runEvalInEditorAsync(teacherTool.selectedCriteria, teacherTool.catalog!);
     if (evalResult) {
         dispatch(Actions.setEvalResult(evalResult));
     } else {