[code_review] Improve the prompting

bugbug/tools/code_review.py

@@ -63,30 +63,25 @@ class LargeDiffError(Exception):
     """Occurs when the diff is too large to be processed."""
 
 
-TARGET_SOFTWARE = None
+TARGET_SOFTWARE: str | None = None
 
-PROMPT_TEMPLATE_SUMMARIZATION = (
-    """You are an expert reviewer for"""
-    + (f" the {TARGET_SOFTWARE}" if TARGET_SOFTWARE is not None else "")
-    + """ source code, with experience on source code reviews.
+PROMPT_TEMPLATE_SUMMARIZATION = """You are an expert reviewer for {experience_scope}, with experience on source code reviews.
 
 Please, analyze the code provided and report a summarization about the new changes; for that, focus on the coded added represented by lines that start with "+".
 
 {patch}"""
-)
 
-PROMPT_TEMPLATE_REVIEW = (
-    """You will be given a task to generate a code review for the patch below. Use the following steps to solve it:
+PROMPT_TEMPLATE_REVIEW = """You will be given a task to generate a code review for the patch below. Use the following steps to solve it:
 1. Understand the changes done in the patch by reasoning about the summarization as previously reported.
 2. Identify possible code snippets that might result in possible bugs, major readability regressions, and similar concerns.
-3. Reason about each identified problem to make sure they are valid. Have in mind, your review must be consistent with the source code"""
-    + (f" in {TARGET_SOFTWARE}" if TARGET_SOFTWARE is not None else "")
-    + """.
-4. Filter out comments that focuses on documentation, comments, error handling, tests, and confirmation whether objects, methods and files exist or not.
-5. Filter out comments that are descriptive.
-6. Filter out comments that are praising (example: "This is a good addition to the code.").
-7. Filter out comments that are not about added lines (have '+' symbol at the start of the line).
-8. Final answer: Write down the comments and report them using the JSON format previously adopted for the valid comment examples.
+3. Reason about each identified problem to make sure they are valid. Have in mind, your review must be consistent with the {target_code_consistency} source code.
+4. Generate review comments that are short, to the point and focused on the changed code. Don't form the suggestions as questions.
+5. Avoid comments that focuses on documentation, tests, or confirmation whether something is exist.
+6. Avoid comments that ask the author to ensure that the change correct (using words like Ensure, Verify, or Check).
+7. Avoid comments that are descriptive.
+8. Avoid comments that are praising (example: "This is a good addition to the code.").
+9. Avoid comments that are not about added lines (have '+' symbol at the start of the line).
+10. Final answer: Write down the comments and report them using the JSON format previously adopted for the valid comment examples.
 
 As valid comments, consider the examples below:
 {comment_examples}
@@ -97,7 +92,6 @@ class LargeDiffError(Exception):
 
 Do not report any explanation about your choice. Only return a valid JSON list.
 """
-)
 
 
 TEMPLATE_COMMENT_EXAMPLE = """Patch example {example_number}:
@@ -109,13 +103,10 @@ class LargeDiffError(Exception):
 {comments}"""
 
 
-PROMPT_TEMPLATE_FILTERING_ANALYSIS = (
-    """Please, double check the code review provided for the patch below.
+PROMPT_TEMPLATE_FILTERING_ANALYSIS = """Please, double check the code review provided for the patch below.
 Just report the comments that are:
 - applicable for the patch;
-- consistent with the source code"""
-    + (f" in {TARGET_SOFTWARE}" if TARGET_SOFTWARE is not None else "")
-    + """;
+- consistent with the {target_code_consistency} source code;
 - focusing on reporting possible bugs, major readability regressions, or similar concerns;
 - filter out any descriptive comments;
 - filter out any praising comments.
@@ -140,7 +131,6 @@ class LargeDiffError(Exception):
 As examples of not expected comments, not related to the current patch, please, check some below:
     - {rejected_examples}
 """
-)
 
 
 DEFAULT_REJECTED_EXAMPLES = """Please note that these are minor improvements and the overall quality of the patch is good. The documentation is being expanded in a clear and structured way, which will likely be beneficial for future development.
@@ -1040,9 +1030,11 @@ def __init__(
         show_patch_example: bool = False,
         verbose: bool = True,
         suggestions_feedback_db: Optional["SuggestionsFeedbackDB"] = None,
+        target_software: Optional[str] = None,
     ) -> None:
         super().__init__()
 
+        self.target_software = target_software or TARGET_SOFTWARE
         self.comment_gen_llms = comment_gen_llms
         self.llm = llm if llm is not None else comment_gen_llms[0]
         self._tokenizer = get_tokenizer(
@@ -1052,12 +1044,26 @@ def __init__(
         )
 
         self.summarization_chain = LLMChain(
-            prompt=PromptTemplate.from_template(PROMPT_TEMPLATE_SUMMARIZATION),
+            prompt=PromptTemplate.from_template(
+                PROMPT_TEMPLATE_SUMMARIZATION,
+                partial_variables={
+                    "experience_scope": (
+                        f"the {self.target_software} source code"
+                        if self.target_software
+                        else "a software project"
+                    )
+                },
+            ),
             llm=self.llm,
             verbose=verbose,
         )
         self.filtering_chain = LLMChain(
-            prompt=PromptTemplate.from_template(PROMPT_TEMPLATE_FILTERING_ANALYSIS),
+            prompt=PromptTemplate.from_template(
+                PROMPT_TEMPLATE_FILTERING_ANALYSIS,
+                partial_variables={
+                    "target_code_consistency": self.target_software or "rest of the"
+                },
+            ),
             llm=self.llm,
             verbose=verbose,
         )
@@ -1146,20 +1152,17 @@ def run(self, patch: Patch) -> list[InlineComment] | None:
                 verbose=self.verbose,
             )
 
+            experience_scope = (
+                f"the {self.target_software} source code"
+                if self.target_software
+                else "a software project"
+            )
             memory.save_context(
                 {
-                    "input": "You are an expert reviewer for"
-                    + (f" in {TARGET_SOFTWARE}" if TARGET_SOFTWARE is not None else "")
-                    + " source code, with experience on source code reviews."
+                    "input": f"You are an expert reviewer for {experience_scope}, with experience on source code reviews."
                 },
                 {
-                    "output": "Sure, I'm aware of source code practices"
-                    + (
-                        f" in {TARGET_SOFTWARE}"
-                        if TARGET_SOFTWARE is not None
-                        else " in the development community"
-                    )
-                    + "."
+                    "output": f"Sure, I'm aware of source code practices in {self.target_software or 'the development community'}."
                 },
             )
             memory.save_context(
@@ -1204,6 +1207,7 @@ def run(self, patch: Patch) -> list[InlineComment] | None:
                 input=PROMPT_TEMPLATE_REVIEW.format(
                     patch=formatted_patch,
                     comment_examples=self._get_comment_examples(patch),
+                    target_code_consistency=self.target_software or "rest of the",
                 )
             )
             output += cur_output

scripts/code_review_tool_evaluator.py

@@ -28,6 +28,8 @@
 from bugbug.tools import code_review
 from bugbug.vectordb import QdrantVectorDB
 
+code_review.TARGET_SOFTWARE = "Mozilla Firefox"
+
 
 def get_tool_variants(
     llm,