Merge pull request #4737 from samsrabin/allow-baseline-skip

Allow skipping tests that have a baseline already

.git-blame-ignore-revs

@@ -1,3 +1,4 @@
 # Migrated code style to Black
 eabf877cbb86b281fdd37a3fa3cc0edf9b8eb874
 321463922724b225988e517da54a18bad90bc316
+6751504adf095d5d034e6406fbb0e914924aecff

CIME/scripts/create_test.py

@@ -351,14 +351,32 @@ def parse_command_line(args, description):
         config, "ALLOW_BASELINE_OVERWRITE", False, check_main=False
     )
 
-    parser.add_argument(
-        "-o",
+    default = get_default_setting(
+        config, "SKIP_TESTS_WITH_EXISTING_BASELINES", False, check_main=False
+    )
+
+    # Don't allow -o/--allow-baseline-overwrite AND --skip-tests-with-existing-baselines
+    existing_baseline_group = parser.add_mutually_exclusive_group()
+
+    existing_baseline_group.add_argument(
         "--allow-baseline-overwrite",
+        "-o",
+        action="store_true",
+        default=default,
+        help="If the --generate option is given, then an attempt to overwrite "
+        "\nan existing baseline directory will raise an error. WARNING: Specifying this "
+        "\noption will allow existing baseline directories to be silently overwritten. "
+        "\nIncompatible with --skip-tests-with-existing-baselines.",
+    )
+
+    existing_baseline_group.add_argument(
+        "--skip-tests-with-existing-baselines",
         action="store_true",
         default=default,
         help="If the --generate option is given, then an attempt to overwrite "
         "\nan existing baseline directory will raise an error. WARNING: Specifying this "
-        "\noption will allow existing baseline directories to be silently overwritten.",
+        "\noption will allow tests with existing baseline directories to be silently skipped. "
+        "\nIncompatible with -o/--allow-baseline-overwrite.",
     )
 
     default = get_default_setting(config, "WAIT", False, check_main=False)
@@ -759,6 +777,7 @@ def parse_command_line(args, description):
         args.save_timing,
         args.queue,
         args.allow_baseline_overwrite,
+        args.skip_tests_with_existing_baselines,
         args.output_root,
         args.wait,
         args.force_procs,
@@ -921,6 +940,7 @@ def create_test(
     save_timing,
     queue,
     allow_baseline_overwrite,
+    skip_tests_with_existing_baselines,
     output_root,
     wait,
     force_procs,
@@ -969,6 +989,7 @@ def create_test(
         save_timing=save_timing,
         queue=queue,
         allow_baseline_overwrite=allow_baseline_overwrite,
+        skip_tests_with_existing_baselines=skip_tests_with_existing_baselines,
         output_root=output_root,
         force_procs=force_procs,
         force_threads=force_threads,
@@ -1068,6 +1089,7 @@ def _main_func(description=None):
         save_timing,
         queue,
         allow_baseline_overwrite,
+        skip_tests_with_existing_baselines,
         output_root,
         wait,
         force_procs,
@@ -1122,6 +1144,7 @@ def _main_func(description=None):
             save_timing,
             queue,
             allow_baseline_overwrite,
+            skip_tests_with_existing_baselines,
             output_root,
             wait,
             force_procs,

CIME/test_scheduler.py

@@ -196,6 +196,7 @@ def __init__(
         save_timing=False,
         queue=None,
         allow_baseline_overwrite=False,
+        skip_tests_with_existing_baselines=False,
         output_root=None,
         force_procs=None,
         force_threads=None,
@@ -227,6 +228,7 @@ def __init__(
         self._input_dir = input_dir
         self._pesfile = pesfile
         self._allow_baseline_overwrite = allow_baseline_overwrite
+        self._skip_tests_with_existing_baselines = skip_tests_with_existing_baselines
         self._single_exe = single_exe
         if self._single_exe:
             self._allow_pnl = True
@@ -348,6 +350,8 @@ def __init__(
                     self._baseline_root, self._baseline_gen_name
                 )
                 existing_baselines = []
+                if skip_tests_with_existing_baselines:
+                    tests_to_skip = []
                 for test_name in test_names:
                     test_baseline = os.path.join(full_baseline_dir, test_name)
                     if os.path.isdir(test_baseline):
@@ -357,11 +361,21 @@ def __init__(
                                 clear_folder(os.path.join(test_baseline, "CaseDocs"))
                             else:
                                 clear_folder(test_baseline)
+                        elif skip_tests_with_existing_baselines:
+                            tests_to_skip.append(test_name)
                 expect(
-                    allow_baseline_overwrite or len(existing_baselines) == 0,
+                    allow_baseline_overwrite
+                    or len(existing_baselines) == 0
+                    or skip_tests_with_existing_baselines,
                     "Baseline directories already exists {}\n"
-                    "Use -o to avoid this error".format(existing_baselines),
+                    "Use -o or --skip-tests-with-existing-baselines to avoid this error".format(
+                        existing_baselines
+                    ),
                 )
+                if skip_tests_with_existing_baselines:
+                    test_names = [
+                        test for test in test_names if test not in tests_to_skip
+                    ]
 
         if self._config.sort_tests:
             _order_tests_by_runtime(test_names, self._baseline_root)

CIME/tests/base.py

@@ -203,6 +203,7 @@ def _create_test(
         run_errors=False,
         env_changes="",
         default_baseline_area=False,
+        expect_cases_made=True,
     ):
         """
         Convenience wrapper around create_test. Returns list of full paths to created cases. If multiple cases,
@@ -260,7 +261,13 @@ def _create_test(
                 )
                 cases.append(casedir)
 
-        self.assertTrue(len(cases) > 0, "create_test made no cases")
+        if expect_cases_made:
+            self.assertTrue(len(cases) > 0, "create_test made no cases")
+        else:
+            self.assertTrue(
+                len(cases) == 0,
+                "create_test unexpectedly made {} case(s)".format(len(cases)),
+            )
 
         return cases[0] if len(cases) == 1 else cases
 

CIME/tests/test_sys_cime_case.py

@@ -748,3 +748,26 @@ def test_case_clean(self):
         self.run_cmd_assert_result("./case.setup --clean", from_dir=casedir)
         self.run_cmd_assert_result("./case.setup --clean", from_dir=casedir)
         self.run_cmd_assert_result("./case.setup", from_dir=casedir)
+
+    def test_skip_run_with_existing_baseline(self):
+        test_name = "TESTRUNPASS_P1.f19_g16_rx1.A"
+
+        if self._config.test_mode == "cesm":
+            create_test_extra_args = ["--generate", "baseline", "--no-build", test_name]
+        else:
+            create_test_extra_args = ["-g", "--no-build", test_name]
+
+        orig_testroot = self._testroot
+        self._testroot = os.path.join(orig_testroot, "case0")
+        casedir_0 = self._create_test(
+            create_test_extra_args,
+            test_id=self._baseline_name,
+            expect_cases_made=True,
+        )
+        self._testroot = os.path.join(orig_testroot, "case1")
+        casedir_1 = self._create_test(
+            ["--skip-tests-with-existing-baselines"] + create_test_extra_args,
+            test_id=self._baseline_name,
+            expect_cases_made=False,
+        )
+        self._testroot = orig_testroot

CIME/utils.py

@@ -260,6 +260,7 @@ def _read_cime_config_file():
         "walltime",
         "job_queue",
         "allow_baseline_overwrite",
+        "skip_tests_with_existing_baselines",
         "wait",
         "force_procs",
         "force_threads",