From 57446f67fe106999b365182577abc8f7e388468a Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Mon, 18 Nov 2024 14:57:08 -0300 Subject: [PATCH] avoid exception when scrapystats is unset --- shub_workflow/utils/monitor.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/shub_workflow/utils/monitor.py b/shub_workflow/utils/monitor.py index e9dcefa..62aa2bc 100644 --- a/shub_workflow/utils/monitor.py +++ b/shub_workflow/utils/monitor.py @@ -46,21 +46,18 @@ class SpiderStatsAggregatorMixin(BaseScriptProtocol): def aggregate_spider_stats(self, jobdict: JobDict, stats_added_prefix: str = ""): canonical = self.get_canonical_spidername(jobdict["spider"]) - if "scrapystats" in jobdict: - for statkey in jobdict["scrapystats"]: - for statnameprefix in self.target_spider_stats + BASE_TARGET_SPIDER_STATS: - if statkey.startswith(statnameprefix): - value = jobdict["scrapystats"][statkey] - if stats_added_prefix != canonical: - if not self.stats_only_total: - self.stats.inc_value( - f"{stats_added_prefix}/{statkey}/{canonical}".strip("/"), value - ) - self.stats.inc_value(f"{stats_added_prefix}/{statkey}/total".strip("/"), value) - else: - self.stats.inc_value(f"{stats_added_prefix}/{statkey}".strip("/"), value) - else: - LOG.error(f"Job {jobdict['key']} does not have scrapystats.") + for statkey in jobdict.get("scrapystats") or {}: + for statnameprefix in self.target_spider_stats + BASE_TARGET_SPIDER_STATS: + if statkey.startswith(statnameprefix): + value = jobdict["scrapystats"][statkey] + if stats_added_prefix != canonical: + if not self.stats_only_total: + self.stats.inc_value( + f"{stats_added_prefix}/{statkey}/{canonical}".strip("/"), value + ) + self.stats.inc_value(f"{stats_added_prefix}/{statkey}/total".strip("/"), value) + else: + self.stats.inc_value(f"{stats_added_prefix}/{statkey}".strip("/"), value) class BaseMonitor(AlertSenderMixin, SpiderStatsAggregatorMixin, BaseScript, BaseMonitorProtocol):