From a18b62129be61dc90653c47b72a199e8fad079e1 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 3 May 2018 14:49:53 -0600 Subject: [PATCH 1/3] Upgrade preview_run. 1) Redo the formatting of output to make it prettier 2) Add some output for key derived attributes 3) Add logger.info suggestion to user to run preview_run --- scripts/Tools/preview_run | 19 ++++++++++++++++--- scripts/lib/CIME/XML/env_mach_specific.py | 20 +++++++------------- scripts/lib/CIME/case/case.py | 2 +- scripts/lib/CIME/case/case_setup.py | 2 ++ scripts/lib/CIME/utils.py | 20 +++++++++++++++++++- 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/scripts/Tools/preview_run b/scripts/Tools/preview_run index acd6c2a5870..8aa459a6e25 100755 --- a/scripts/Tools/preview_run +++ b/scripts/Tools/preview_run @@ -19,6 +19,7 @@ Example: from standard_script_setup import * from CIME.case import Case +from CIME.utils import set_logger_indent logger = logging.getLogger(__name__) @@ -56,18 +57,30 @@ def _main_func(description): logging.disable(logging.INFO) with Case(caseroot, read_only=False) as case: - print("BATCH SUBMIT: (nodes {})".format(case.num_nodes)) + print("CASE INFO:") + print(" nodes: {}".format(case.num_nodes)) + print(" total tasks: {}".format(case.total_tasks)) + print(" tasks per node: {}".format(case.tasks_per_node)) + print(" thread count: {}".format(case.thread_count)) + print("") + + print("BATCH INFO:") if job is None: job = case.get_primary_job() + set_logger_indent(" ") job_id_to_cmd = case.submit_jobs(dry_run=True, job=job) for job_id, cmd in job_id_to_cmd: + print(" FOR JOB: {}".format(job_id)) + print(" ENV:") case.load_env(job=job_id, reset=True, verbose=True) - print(" {} -> {}".format(job_id, case.get_resolved_value(cmd))) + print(" SUBMIT CMD:") + print(" {}".format(job_id, case.get_resolved_value(cmd))) print("") if job == case.get_primary_job(): - print("MPIRUN: {}".format(case.get_mpirun_cmd())) + print("MPIRUN:") + print (" {}".format(case.get_mpirun_cmd())) if __name__ == "__main__": _main_func(__doc__) diff --git a/scripts/lib/CIME/XML/env_mach_specific.py b/scripts/lib/CIME/XML/env_mach_specific.py index a6eb6bccc14..faff951f82d 100644 --- a/scripts/lib/CIME/XML/env_mach_specific.py +++ b/scripts/lib/CIME/XML/env_mach_specific.py @@ -179,15 +179,13 @@ def make_env_mach_specific_file(self, shell, case): def _load_envs(self, envs_to_set, verbose=False): for env_name, env_value in envs_to_set: + logger_func = logger.warning if verbose else logger.debug if env_value is None and env_name in os.environ: del os.environ[env_name] + logger_func("Unsetting Environment {}".format(env_name)) elif env_value is not None: os.environ[env_name] = env_value - if verbose: - if env_value is not None: - logger.warning("Setting Environment {}={}".format(env_name, env_value)) - else: - logger.warning("Unsetting Environment {}".format(env_name)) + logger_func("Setting Environment {}={}".format(env_name, env_value)) def _compute_module_actions(self, module_nodes, case, job=None): return self._compute_actions(module_nodes, "command", case, job=job) @@ -298,11 +296,9 @@ def _get_module_commands(self, modules_to_load, shell): return cmds def _load_module_modules(self, modules_to_load, verbose=False): + logger_func = logger.warning if verbose else logger.debug for cmd in self._get_module_commands(modules_to_load, "python"): - if verbose: - logger.warning("module command is {}".format(cmd)) - else: - logger.debug("module command is {}".format(cmd)) + logger_func("module command is {}".format(cmd)) stat, py_module_code, errout = run_cmd(cmd) expect(stat==0 and (len(errout) == 0 or self.allow_error()), "module command {} failed with message:\n{}".format(cmd, errout)) @@ -311,6 +307,7 @@ def _load_module_modules(self, modules_to_load, verbose=False): def _load_modules_generic(self, modules_to_load, verbose=False): sh_init_cmd = self.get_module_system_init_path("sh") sh_mod_cmd = self.get_module_system_cmd_path("sh") + logger_func = logger.warning if verbose else logger.debug # Purpose is for environment management system that does not have # a python interface and therefore can only determine what they @@ -330,10 +327,7 @@ def _load_modules_generic(self, modules_to_load, verbose=False): # Use null terminated lines to give us something more definitive to split on. # Env vars can contain newlines, so splitting on newlines can be ambiguous cmd += " && env -0" - if verbose: - logger.warning("cmd: {}".format(cmd)) - else: - logger.debug("cmd: {}".format(cmd)) + logger_func("cmd: {}".format(cmd)) output = run_cmd_no_fail(cmd) ################################################### diff --git a/scripts/lib/CIME/case/case.py b/scripts/lib/CIME/case/case.py index 4936d810345..efc9cedbb40 100644 --- a/scripts/lib/CIME/case/case.py +++ b/scripts/lib/CIME/case/case.py @@ -13,7 +13,7 @@ from CIME.utils import convert_to_type, get_model from CIME.utils import get_project, get_charge_account, check_name from CIME.utils import get_current_commit, safe_copy -from CIME.locked_files import LOCKED_DIR, lock_file +from CIME.locked_files import LOCKED_DIR, lock_file from CIME.XML.machines import Machines from CIME.XML.pes import Pes from CIME.XML.files import Files diff --git a/scripts/lib/CIME/case/case_setup.py b/scripts/lib/CIME/case/case_setup.py index e0a60d5fe92..b58dfa13eb2 100644 --- a/scripts/lib/CIME/case/case_setup.py +++ b/scripts/lib/CIME/case/case_setup.py @@ -205,6 +205,8 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): env_module.make_env_mach_specific_file("csh", case) env_module.save_all_env_info("software_environment.txt") + logger.info("You can now run './preview_run' to get more info on how your case will be run") + ############################################################################### def case_setup(self, clean=False, test_mode=False, reset=False): ############################################################################### diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 779ad6ef46b..9d3a2835cdc 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -56,6 +56,25 @@ def redirect_logger(new_target, logger_name): root_log.handlers = orig_root_loggers log.handlers = orig_handlers +class IndentFormatter(logging.Formatter): + def __init__(self, indent, fmt=None, datefmt=None): + logging.Formatter.__init__(self, fmt, datefmt) + self._indent = indent + + def format(self, rec): + rec.msg = "{}{}".format(self._indent, rec.msg) + out = logging.Formatter.format(self, rec) + return out + +def set_logger_indent(indent): + root_log = logging.getLogger() + root_log.handlers = [] + formatter = IndentFormatter(indent) + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + root_log.addHandler(handler) + class EnvironmentContext(object): """ Context manager for environment variables @@ -1003,7 +1022,6 @@ def parse_args_and_handle_standard_logging_options(args, parser=None): root_logger.setLevel(logging.INFO) return args - def get_logging_options(): """ Use to pass same logging options as was used for current From 1aa0e11ffa5b34832141d3163c34c0b084b6cf2d Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 3 May 2018 14:56:04 -0600 Subject: [PATCH 2/3] Fix mistake --- scripts/Tools/preview_run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Tools/preview_run b/scripts/Tools/preview_run index 8aa459a6e25..56549b40c1c 100755 --- a/scripts/Tools/preview_run +++ b/scripts/Tools/preview_run @@ -75,7 +75,7 @@ def _main_func(description): print(" ENV:") case.load_env(job=job_id, reset=True, verbose=True) print(" SUBMIT CMD:") - print(" {}".format(job_id, case.get_resolved_value(cmd))) + print(" {}".format(case.get_resolved_value(cmd))) print("") if job == case.get_primary_job(): From bd4ea78ceb42d6107851f7701754015353cf5da0 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 4 May 2018 10:45:16 -0600 Subject: [PATCH 3/3] Attempt to pacify pylint --- scripts/lib/CIME/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index 9d3a2835cdc..f1139e8df2b 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -61,9 +61,9 @@ def __init__(self, indent, fmt=None, datefmt=None): logging.Formatter.__init__(self, fmt, datefmt) self._indent = indent - def format(self, rec): - rec.msg = "{}{}".format(self._indent, rec.msg) - out = logging.Formatter.format(self, rec) + def format(self, record): + record.msg = "{}{}".format(self._indent, record.msg) + out = logging.Formatter.format(self, record) return out def set_logger_indent(indent):