Unable to match question and answer back to the feedback

[faidhi066]

Type of issue

Missing information

Feedback

Store the feedback by saving message IDs and content of messages your bot sends and receives. When your bot gets an invoke request with feedback, match the message ID with the corresponding feedback.

I was referring to this documentation and was using replyToId. However, it was using a different id with the message and answer produced.

_activity: {
    name: 'message/submitAction',
    type: 'invoke',
    timestamp: 2024-11-28T11:09:18.496Z,
    localTimestamp: 2024-11-28T11:09:18.496Z,
    id: 'f:4fc57xxxxxxxxxxxxx4',
    channelId: 'msteams',
    serviceUrl: 'https://smba.trafficmanager.net/apac/6f0725c8-01e3-4b12-8169-f2e142c5ce70/',
    from: {
      id: '2fsdfcxxxxx',
      name: 'xxx M365Unit14',
      aadObjectId: 'f9983e5a-bada-4d13-bcbb-3b123f8f994c'
    },
    conversation: {
      conversationType: 'personal',
      tenantId: 'xxxx-xxxxx-xxx',
      id: 'a:1SMgrRBEMcQRClDOV8HEzytV8Ejz36cXgyEnQXfekvSro1_UOjrvdo8FgU2MqFOcBMV_2E5ygLw1XJ8MRa-yH6uxGCXwpeiiM6zvJBKdNwtdLmu-aRZoOF5TjnDkhw1dn'
    },
    recipient: {
      id: '28:8c84536f-9298-4653-a156-xxxx',
      name: 'PolicyBot'
    },
    entities: [ [Object] ],
    channelData: { tenant: [Object], source: [Object], legacy: [Object] },
    **replyToId: '1732787051513',**
    value: { actionName: 'feedback', actionValue: [Object] },
    locale: 'en-US',
    localTimezone: 'xxx/xxx',
    rawTimestamp: '2024-11-28T11:09:18.496Z',
    rawLocalTimestamp: '2024-11-28T19:09:18.496+08:00',
    callerId: 'urn:botframework:azure'
  }

Can you help me, how can I get the based on the invoke id activity to be matched with answer and answer's replyToId (which is same, but different with invoke activity???)?

Page URL

https://learn.microsoft.com/en-us/microsoftteams/platform/bots/how-to/bot-messages-ai-generated-content?tabs=before%2Cbotmessage

Content source URL

https://github.com/MicrosoftDocs/msteams-docs/blob/main/msteams-platform/bots/how-to/bot-messages-ai-generated-content.md

Author

@surbhigupta

Document Id

605a8486-6e6a-31d5-e8a1-021691206099

[sayali-MSFT]

Thank you for your inquiry about your Teams app development issue! To assist you better, could you please provide the following details?

Reproduction Steps: Please share the steps you took to encounter the issue.

This information will help us better understand the situation and provide a more accurate response.

[faidhi066]

loggingMiddleware.js

const { ActivityHandler, ActivityTypes, TurnContext } = require("botbuilder");
const winston = require("winston");
require("winston-daily-rotate-file");
const path = require("path");

// Configure Winston logger with daily rotation and 30-day retention
const logger = winston.createLogger({
  level: "info",
  format: winston.format.combine(
    winston.format.timestamp(),
    winston.format.printf(({ timestamp, level, message }) => {
      return `[${timestamp}] ${level.toUpperCase()}: ${message}`;
    })
  ),
  transports: [
    // new winston.transports.Console(),
    new winston.transports.DailyRotateFile({
      filename: path.join(process.cwd(), "logs/%DATE%-combined.log"),
      datePattern: "YYYY-MM-DD",
      maxFiles: "30d", // Keep logs for 30 days
      format: winston.format.combine(winston.format.colorize()),
    }),
    new winston.transports.DailyRotateFile({
      filename: path.join(process.cwd(), "logs/%DATE%-error.log"),
      datePattern: "YYYY-MM-DD",
      maxFiles: "30d", // Keep logs for 30 days
      level: "error",
      format: winston.format.combine(
        winston.format.colorize(),
        winston.format.simple()
      ),
    }),
  ],
});

// changing feature to another feature can let to the logging become incorrect
// so need to manually set when first time changing
function checkBotIntentionTrigger(context) {
  const text = context.activity.text ? context.activity.text.trim() : null;
  const feature = context.turnState.get("feature");

  if (text != null && feature != "normal") {
    if (
      text === "hello" ||
      text == "good morning" ||
      text == "how are you doing" ||
      text == "hi"
    ) {
      return "greet";
    } else {
      return "normal";
    }
  } else if (context.activity.value != null && feature != "/options") {
    return "/options";
  } else if (
    context.activity.attachments &&
    context.activity.attachments.length > 0 &&
    feature != "fileupload"
  ) {
    return "fileupload";
  } else {
    return feature;
  }
}

// Custom middleware to log messages
class LoggingMiddleware {
  async onTurn(context, next) {
    console.log("Logging");
    // try {
    // Log outgoing messages
    context.onSendActivities(async (context, activities, next) => {
      console.log(context.activity);
      const feature = context.turnState.get("feature") || "unknown";
      activities.forEach((activity) => {
        const logEntry = {
          // timestamp: new Date().toISOString(),
          type: "bot",
          userId: context.activity.from.aadObjectId,
          feature: feature,
          message: activity.text || "N/A",
          id: activity.replyToId || "N/A",
        };
        logger.info(JSON.stringify(logEntry));
      });
      await next();
    });

    // Log incoming user messages
    if (context.activity.type === ActivityTypes.Message) {
      const feature = context.activity.value
        ? context.activity.id
        : context.turnState.get("feature");
      const logEntry = {
        // timestamp: new Date().toISOString(),
        type: "user",
        userId: context.activity.from.aadObjectId,
        feature: checkBotIntentionTrigger(context),
        message:
          context.activity.text ||
          context.activity.value ||
          context.activity.attachments?.[0],
        id: context.activity.id || "N/A",
      };
      logger.info(JSON.stringify(logEntry));
    }
    await next();
    // } catch (error) {
    // Log the error along with the feature and user message that caused it
    // const feature = context.turnState.get("feature") || "unknown";
    // logger.error(
    //   JSON.stringify({
    //     timestamp: new Date().toISOString(),
    //     type: "user-error",
    //     message: error.message,
    //     stack: error.stack,
    //     feature: feature,
    //     userActivity: {
    //       id: context.activity.id || "N/A",
    //       message: context.activity.text || "N/A",
    //       from:
    //         context.activity.from.name ||
    //         context.activity.from.id ||
    //         "Unknown",
    //     },
    //   })
    // );
    // throw error; // Re-throw the error to allow further processing
  }
  // }
}

module.exports.logger = logger;
module.exports.LoggingMiddleware = LoggingMiddleware;

index.js

// const { notificationApp } = require("./internal/initialize");
const { AdaptiveCards } = require("@microsoft/adaptivecards-tools");
const { TeamsKijangBot } = require("./teamsKijangBot");
const { AuthMiddleware } = require("./middleware/authMiddleware"); // Import your middleware
const restify = require("restify");

const {
  CloudAdapter,
  ConfigurationServiceClientCredentialFactory,
  ConfigurationBotFrameworkAuthentication,
  MemoryStorage,
  ConversationState,
  UserState,
} = require("botbuilder");
const config = require("./internal/config");
const { LoggingMiddleware } = require("./middleware/loggingMiddleware");

const memoryStorage = new MemoryStorage();

// Create conversation and user state with in-memory storage provider.
const conversationState = new ConversationState(memoryStorage);
const userState = new UserState(memoryStorage);

// Create adapter.
// See https://aka.ms/about-bot-adapter to learn more about adapters.
const credentialsFactory = new ConfigurationServiceClientCredentialFactory({
  MicrosoftAppId: config.botId,
  MicrosoftAppPassword: config.botPassword,
  MicrosoftAppType: "MultiTenant",
});

const botFrameworkAuthentication = new ConfigurationBotFrameworkAuthentication(
  {},
  credentialsFactory
);
// See https://aka.ms/about-bot-adapter to learn more about how bots work.
const adapter = new CloudAdapter(botFrameworkAuthentication);

process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";

// Register middleware to check user ID
adapter.use(new AuthMiddleware(process.env.MicrosoftAppTenantId));

// Middleware to logging
adapter.use(new LoggingMiddleware());
adapter.onTurnError = async (context, error) => {
  // This check writes out errors to console log .vs. app insights.
  // NOTE: In production environment, you should consider logging this to Azure
  //       application insights. See https://aka.ms/bottelemetry for telemetry
  //       configuration instructions.
  console.error(`\n [onTurnError] unhandled error: ${error}`);

  if (error.message == "AccessDenied") {
    console.log("Access Denied");
    return;
  } else {
    await context.sendTraceActivity(
      "Sorry, there is an error on the processed request."
    );
  }

  // Send a trace activity, which will be displayed in Bot Framework Emulator
  await context.sendTraceActivity(
    "OnTurnError Trace",
    `${error}`,
    "https://www.botframework.com/schemas/error",
    "TurnError"
  );
};

// Create HTTP server.
const server = restify.createServer();
server.use(restify.plugins.bodyParser());
server.listen(process.env.port || process.env.PORT || 3978, () => {
  console.log(`\nApp Started, ${server.name} listening to ${server.url}`);
});

// Bot Framework message handler.
const teamsBot = new TeamsKijangBot(conversationState, userState);
server.post("/api/messages", async (req, res) => {
  await adapter.process(req, res, (context) => teamsBot.run(context));
});

kijangBot.js

 const {
  TurnContext,
  TeamsActivityHandler,
  CardFactory,
  ActivityTypes,
} = require("botbuilder");

const path = require("path");
const { logger } = require("./middleware/loggingMiddleware");

const CONVERSATION_DATA_PROPERTY = "conversationData";
const USER_PROFILE_PROPERTY = "userProfile";

class TeamsKijangBot extends TeamsActivityHandler {
  constructor(conversationState, userState) {
    super();

    // Create the state propery accessors for the conversation data and user property
    this.conversationDataAccessor = conversationState.createProperty(
      CONVERSATION_DATA_PROPERTY
    );
    this.userProfileAccessor = userState.createProperty(USER_PROFILE_PROPERTY);

    // The state management objects for the conversation and user state
    this.conversationState = conversationState;
    this.userState = userState;

    this.onMessage(async (context, next) => {
      let feature = "botloading";

      // Set the feature in turnState
      context.turnState.set("feature", feature);

      await context.sendActivity({ type: ActivityTypes.Typing });

      console.log(context.activity);
      const conversationData = await this.conversationDataAccessor.get(
        context,
        { fileUploadFlag: false, file: "", questionFlag: "", answerFlag: "" }
      );

      console.log("Running with Message Activity.");

      // Extract the user's object ID
      const userId = context.activity.from.aadObjectId;

      // Log or use the user's object ID
      console.log(`User's Object ID: ${userId}`);

      // Setup userId in logger
      // loggerAppInsights = loggerAppInsights.child({ userId: userId });
      // loggerDevLog = loggerDevLog.child({ userId: userId });

      // USER sends message without an attachment !!important
      if (
        context.activity.text != null &&
        context.activity.attachments.length === 1
      ) {
        const removedMentionText = TurnContext.removeRecipientMention(
          context.activity
        );
        const txt = removedMentionText
          .toLowerCase()
          .replace(/\n|\r/g, "")
          .trim();
        feature = "normal";
        // Set the feature in turnState
        context.turnState.set("feature", feature);

        await context.sendActivity({
          text: txt,
          channelData: {
            feedbackLoopEnabled: true, // Enable feedback buttons
          },
          entities: [
            {
              type: "https://schema.org/Message",
              "@type": "Message",
              "@context": "https://schema.org",
              additionalType: ["AIGeneratedContent"], // Enables AI label
            },
          ],
        });
      }

      // By calling next() you ensure that the next BotHandler is run.
      await next();
    });

    // Listen to MembersAdded event, view https://docs.microsoft.com/en-us/microsoftteams/platform/resources/bot-v3/bots-notifications for more events
    this.onMembersAdded(async (context, next) => {
      let feature = "firsttimeadd";
      // Set the feature in turnState
      context.turnState.set("feature", feature);

      const membersAdded = context.activity.membersAdded;
      for (let cnt = 0; cnt < membersAdded.length; cnt++) {
        if (membersAdded[cnt].id) {
          // loggerDevLog.info("USER added the bot first time");
          const greetCard = CardFactory.adaptiveCard(greetJSON);
          await context.sendActivity({ attachments: [greetCard] });
          // await context.sendActivity(
          //   "Hello there. I am KijangChatBot. Ask me any questions."
          // );
          break;
        }
      }
      await next();
    });
  }

  async onInvokeActivity(context) {
    let feature = "invokefeedback";
    // Set the feature in turnState
    context.turnState.set("feature", feature);

    console.log(this.conversationDataAccessor.get(context));

    // console.log(this.latestQuestion, this.latestAnswer);
    const userId = context.activity.from.aadObjectId;
    const timestamp = new Date().toISOString(); // Get the current date and time in ISO format
    console.log(context);
    try {
      switch (context.activity.name) {
        case "message/submitAction":
          feature = "invokefeedback";
          // Set the feature in turnState
          context.turnState.set("feature", feature);

          if (
            JSON.parse(context.activity.value.actionValue.feedback).feedbackText
              .length > 5000
          ) {
            return context.sendActivity(
              "Sorry, your feedback exceeds 5k characters. Please try again."
            );
          }
          const header = "Time,User,Provided Reaction,Feedback\n";
          const data = `${timestamp},${userId},${
            context.activity.value.actionValue.reaction
          },${
            JSON.parse(context.activity.value.actionValue.feedback).feedbackText
          }\n`;
          const logFilePath = path.join(
            // process.cwd(),
            "logs/bot-log-2024-11-21.log"
          );
          const filteredLogs = filterLogsById(
            logFilePath,
            context.activity.replyToId
          );
          const question = filteredLogs.userMessages;
          const answer = filteredLogs.botMessages;
          const feedback = {
            name: `${userId}`,
            question: question,
            answer: answer,
            reaction: context.activity.value.actionValue.reaction,
            feedback: JSON.parse(context.activity.value.actionValue.feedback)
              .feedbackText,
            time: `${timestamp}`,
            bot: "KijangChat-test", // NOTE: To be changed to prod
          };
          await insertDataToMongoDB(feedback);

          context.sendActivity("Thank you for the feedback!");
        default:
          context.sendActivity("Unknown invoke activity handled as default");
          return {
            status: 200,
            body: `Unknown invoke activity handled as default- ${context.activity.name}`,
          };
      }
    } catch (error) {
      console.error("INVOKE FEEDBACK ERROR: ", error);
      logger.error(
        JSON.stringify({
          timestamp: new Date().toISOString(),
          type: "bot-error",
          userId: context.activity.from.aadObjectId,
          feature: context.turnState.get("feature"),
          errorMessage: error.message,
          stack: error.stack,
          message: context.activity.text || "N/A",
          id: context.activity.id,
        })
      );
      await context.sendActivity(
        "⚠️ Sorry, there was an error sending feedback to the bot."
      );
      return {
        status: 500,
        body: `Invoke activity received- ${context.activity.name}`,
      };
    }
  }
}

module.exports.TeamsKijangBot = TeamsKijangBot;

combined.log

[2024-11-27T04:42:10.055Z] INFO: {"type":"user","userId":"f9983e5a-bada-4d13-bcbb-3b123f8f994c","feature":"normal","message":"what is the file about","id":"1732682528311"}
[2024-11-27T04:42:10.065Z] INFO: {"type":"bot","userId":"f9983e5a-bada-4d13-bcbb-3b123f8f994c","feature":"botloading","message":"N/A","id":"1732682528311"}
[2024-11-27T04:42:15.612Z] INFO: {"type":"bot","userId":"f9983e5a-bada-4d13-bcbb-3b123f8f994c","feature":"fileuploadchat","message":"The document is a departmental memo (Note) addressed to the Staff, Victor , from xxx. It seeks approval for initiating the procurement of User Interface and User Experience (UI UX) design services for an in-house, web-based application called aNota. The memo outlines the scope of services required, which include UX research, development of information architecture, wireframes, prototypes, usability testing, and support during design handover. The cost estimation for the project is $200,000, and it will utilize the DTS JTK (FMD) UIUX project budget. The procurement approach suggested is a Purchase Requisition.","id":"1732682528311"}

As you can see the id is the same, but when we refer back to replyToId in asyncInvokeActivity it is not showing the with the same id.

[sayali-MSFT]

It sounds like you're encountering an issue with matching the message ID from an invoke activity to the corresponding feedback. This can be tricky, especially when the replyToId seems to differ between the invoke activity and the actual message.

Ensure that you are storing the message IDs of both the sent and received messages. This includes the id of the invoke activity and the replyToId of the message.
When you receive an invoke request with feedback, use the replyToId to match it with the corresponding message ID. This will help you identify the message that the feedback is related to.
Verify that the replyToId in the invoke activity matches the id of the message. If they are different, you might need to implement a mapping mechanism to track the relationship between the invoke activity and the message.

Here's an example of how you can implement this:

// Store the message IDs
const sentMessages = new Map();
const receivedMessages = new Map();

// Function to store sent message
function storeSentMessage(activity) {
    sentMessages.set(activity.id, activity);
}

// Function to store received message
function storeReceivedMessage(activity) {
    receivedMessages.set(activity.replyToId, activity);
}

// Function to match feedback with message
function matchFeedbackWithMessage(feedbackActivity) {
    const messageId = feedbackActivity.replyToId;
    const message = receivedMessages.get(messageId);
    if (message) {
        // Process the feedback with the matched message
        console.log('Matched message:', message);
    } else {
        console.log('No matching message found for feedback');
    }
}

By storing the message IDs and using the replyToId to match the feedback with the corresponding message, you can ensure that the feedback is correctly associated with the right message.

If you need more detailed guidance, you can refer to the documentation

[faidhi066]

Hello, sorry, was busy with sth else.

Yes, I did the exact same thing, but it returned a different id

[2025-01-06T05:03:56.570Z] INFO: {"type":"user","userId":"f9983e5a-bada-4d13-bcbb-3b123f8f994c","feature":"normal","message":"gpt3.5","id":"1736139830660"}
[2025-01-06T05:03:56.595Z] INFO: {"type":"bot","userId":"f9983e5a-bada-4d13-bcbb-3b123f8f994c","feature":"botloading","message":"N/A","id":"1736139830660"}
[2025-01-06T05:03:59.735Z] INFO: {"type":"bot","userId":"f9983e5a-bada-4d13-bcbb-3b123f8f994c","feature":"normal","message":"Hello! It seems like you're interested in GPT-3.5. This model is part of the Generative Pre-trained Transformer series developed by OpenAI. While GPT-3.5 is not a specific named release, it could refer to iterative improvements made to the GPT-3 model before the introduction of GPT-4. These models are designed to understand and generate human-like text based on the input they receive.\n\nIf you have any specific questions or need more detailed information about the capabilities or applications of these models, feel free to ask!","id":"1736139830660"}

the id here is 1736139830660. But, when I invoked the button this is what came out for the invoke id - 1736137182490.

{
  name: 'message/submitAction',
  type: 'invoke',
  timestamp: 2025-01-06T04:21:12.885Z,
  localTimestamp: 2025-01-06T04:21:12.885Z,
  id: 'f:b684be01-63d5-3237-75bf-d95372fb6bbc',
  channelId: 'msteams',
  serviceUrl: '[https://smba.trafficmanager.net/apac/6f0725c8-01e3-4b12-8169-f2e142c5ce70/'](https://smba.trafficmanager.net/apac/6f0725c8-01e3-4b12-8169-f2e142c5ce70/%27),
  from: {
    id: '29:1ixqVifeKBO80eeIeSQhIJSLUPGZBH2mzS0ObWfqM-JfW30I79_tXXZU8mEbodWv9-dMaIWK8-S9koL3fyOf5Sw',
    name: 'Haha M365Unit14',
    aadObjectId: 'xxxxxx-bada-4d13-bcbb-3b123f8f994c'
  },
  conversation: {
    conversationType: 'personal',
    tenantId: '6f0725c8-01e3-4b12-8169-f2e142c5ce70',
    id: 'a:1k3Fb-lsxxykr2c_NaxFxv6cxILDyIL5Ty8jOt93MN2JK3oqadFKCFwo_l29HNrccm97m-wFbcsyjzvs-kz1JduZboXM9IAbjylHH-8bZLDqd3NZnlRoav_Jo3w16CF_U'
  },
  recipient: { id: '28:0c5bdef3-7c6e--8cb2-', name: '' },
  entities: [
    {
      locale: 'en-US',
      country: 'US',
      platform: 'Web',
      timezone: 'Haram',
      type: 'clientInfo'
    }
  ],
  channelData: {
    tenant: { id: 'xxx-01e3-4b12-xxfd-f2e142c5ce70' },
    source: { name: 'message' },
    legacy: { replyToId: '1:1V3Bzc21TIksseH9-0vGRxBiGaXF6-bSFw1Ofrv8uB5s' }
  },
  replyToId: '1736137182490',
  value: {
    actionName: 'feedback',
    actionValue: { reaction: 'like', feedback: '{"feedbackText":"haha"}' }
  },
  locale: 'en-US',
  localTimezone: 'Haram',
  rawTimestamp: '2025-01-06T04:21:12.885Z',
  rawLocalTimestamp: '2025-01-06T12:21:12.885+08:00',
  callerId: 'urn:botframework:azure'
}

so, how can i trace back to the right id then?

[sayali-MSFT]

@faidhi066 , Based on the error context and the log details you provided, it seems like the issue is related to a mismatch between the expected ID and the invoke ID. The error message specifically states: "The invoke id '1736137182490' does not match the expected id '1736139830660'." This discrepancy could be causing issues with tracing the actions correctly.

Ensure that the IDs used in your bot logic are consistently matched. The invoke ID (1736137182490) should align with the expected ID (1736139830660). It is essential to track where and how these IDs are generated and used within your bot code.

Also,confirm that the bot's configuration in Azure App Service correctly aligns with the project’s environment files. Misconfigurations here can lead to unexpected behaviors.