diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index d724772..013fdcc 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -185,6 +185,7 @@ class Redis(RedisBase): data = DataFrame(input_data['data']) model_tags = input_data['model_tags'] retention_time = input_data['retention_time'] + fill_missing_tags = input_data['fill_missing_tags'] key = f'held_data_{input_data["workflow_name"]}_{input_data["schedule_name"]}' @@ -219,6 +220,7 @@ class Redis(RedisBase): data_hold = {tag: content for tag, content in data_hold.items() if tag in tags} self.debug(f'Data hold after removing removed tags: {data_hold}', metadata=metadata) + to_register_metrics = [] for _, row in data.iterrows(): value = row['value'] @@ -226,6 +228,13 @@ class Redis(RedisBase): data_hold[row['name']] = value to_register_metrics.append((row['name'], value)) + if fill_missing_tags: + self.debug("Filling missing tags in data package", metadata=metadata) + missing_tags = [tag for tag in tags if tag not in list(data_hold.keys())] + + for tag in missing_tags: + data_hold[tag] = None + data_hold['timestamp'] = ( data['timestamp'].max() if not data.empty else data_hold['timestamp'] )