Fix the format of mgsm zh and ja. (#2587)

* Fix the format of mgsm zh and ja.

* Add change log to mgsm.

* Add newline after changelog.

lm_eval/tasks/mgsm/README.md

@@ -92,3 +92,7 @@ If other tasks on this dataset are already supported:
 * [ ] Is the "Main" variant of this task clearly denoted?
 * [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
 * [ ] Have you noted which, if any, published evaluation setups are matched by this variant?
+
+# changelog
+- (en_cot, direct) ver 3; (native_cot) ver 4: issue #2578; PR #2587
+  - fix fewshot format: Changed inconsistent usage of ':' (ASCII) and '：' (Chinese) to use '：' consistently.

lm_eval/tasks/mgsm/direct/direct_yaml

@@ -32,4 +32,4 @@ metric_list:
     ignore_case: true
     ignore_punctuation: true
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/mgsm/direct/mgsm_direct_ja.yaml

@@ -1,11 +1,11 @@
 # Generated by utils.py
 dataset_name: ja
 doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"問題: "+question+"\nAnswer:"}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"問題： "+question+"\nAnswer:"}}{% endif %}'
 generation_kwargs:
   do_sample: false
   until:
-  - '問題:'
+  - 問題：
   - </s>
   - <|im_end|>
 include: direct_yaml

lm_eval/tasks/mgsm/direct/mgsm_direct_zh.yaml

@@ -1,11 +1,11 @@
 # Generated by utils.py
 dataset_name: zh
 doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"问题: "+question+"\nAnswer:"}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"问题： "+question+"\nAnswer:"}}{% endif %}'
 generation_kwargs:
   do_sample: false
   until:
-  - '问题:'
+  - 问题：
   - </s>
   - <|im_end|>
 include: direct_yaml

lm_eval/tasks/mgsm/en_cot/cot_yaml

@@ -33,4 +33,4 @@ filter_list:
     - function: take_first
     name: flexible-extract
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_ja.yaml

@@ -1,11 +1,11 @@
 # Generated by utils.py
 dataset_name: ja
 doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"問題: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"問題： "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 generation_kwargs:
   do_sample: false
   until:
-  - '問題:'
+  - 問題：
   - </s>
   - <|im_end|>
 include: cot_yaml

lm_eval/tasks/mgsm/en_cot/mgsm_en_cot_zh.yaml

@@ -1,11 +1,11 @@
 # Generated by utils.py
 dataset_name: zh
 doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"问题: "+question+"\nStep-by-Step Answer:"}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"问题： "+question+"\nStep-by-Step Answer:"}}{% endif %}'
 generation_kwargs:
   do_sample: false
   until:
-  - '问题:'
+  - 问题：
   - </s>
   - <|im_end|>
 include: cot_yaml

lm_eval/tasks/mgsm/native_cot/cot_yaml

@@ -28,4 +28,4 @@ filter_list:
         regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)"
       - function: "take_first"
 metadata:
-  version: 3.0
+  version: 4.0

lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_ja.yaml

@@ -1,7 +1,7 @@
 # Generated by utils.py
 dataset_name: ja
 doc_to_target: '{% if answer is not none %}{{answer[11:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\nステップごとの答え:"}}{% else %}{{"問題: "+question+"\nステップごとの答え:"}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\nステップごとの答え:"}}{% else %}{{"問題： "+question+"\nステップごとの答え:"}}{% endif %}'
 filter_list:
 - filter:
   - function: regex
@@ -17,7 +17,7 @@ filter_list:
 generation_kwargs:
   do_sample: false
   until:
-  - '問題:'
+  - 問題：
   - </s>
   - <|im_end|>
 include: cot_yaml

lm_eval/tasks/mgsm/native_cot/mgsm_native_cot_zh.yaml

@@ -1,7 +1,7 @@
 # Generated by utils.py
 dataset_name: zh
 doc_to_target: '{% if answer is not none %}{{answer[6:]}}{% else %}{{answer_number|string}}{% endif %}'
-doc_to_text: '{% if answer is not none %}{{question+"\n逐步解答:"}}{% else %}{{"问题: "+question+"\n逐步解答:"}}{% endif %}'
+doc_to_text: '{% if answer is not none %}{{question+"\n逐步解答:"}}{% else %}{{"问题： "+question+"\n逐步解答:"}}{% endif %}'
 filter_list:
 - filter:
   - function: regex
@@ -17,7 +17,7 @@ filter_list:
 generation_kwargs:
   do_sample: false
   until:
-  - '问题:'
+  - 问题：
   - </s>
   - <|im_end|>
 include: cot_yaml

lm_eval/tasks/mgsm/utils.py

@@ -75,7 +75,7 @@
     },
     "ja": {  # Japanese
         # "QUESTION": "問題:",
-        "QUESTION": "\u554f\u984c:",
+        "QUESTION": "\u554f\u984c：",
         # "ANSWER": "ステップごとの答え:",
         "ANSWER": "\u30b9\u30c6\u30c3\u30d7\u3054\u3068\u306e\u7b54\u3048:",
         "DIRECT": "Answer:",
@@ -84,7 +84,7 @@
     },
     "zh": {  # Chinese
         # "QUESTION": "问题:",
-        "QUESTION": "\u95ee\u9898:",
+        "QUESTION": "\u95ee\u9898：",
         # "ANSWER": "逐步解答:",
         "ANSWER": "\u9010\u6b65\u89e3\u7b54:",
         "DIRECT": "Answer:",