SIENTIAPDE-1645: Refactor temporary directory cleanup activity by extracting processing and deletion logic into dedicated helper methods. Update train_test_split to use a local numpy.random generator for improved reproducibility. Remove an unnecessary else: pass block.

This commit is contained in:
Bruno Domingues
2026-08-05 12:10:54 -03:00
parent 4133e200f7
commit e3e1c712a8
3 changed files with 91 additions and 52 deletions

View File

@@ -61,6 +61,87 @@ class Cleanup(SientiaMonitoring):
r'^(.+)_(\d{8}_\d{6}_\d{6})$'
) # name_YYYYMMDD_HHMMSS_microseconds
def _process_directory_item(
self,
item_path: str,
item_name: str,
cutoff_time: datetime,
metadata: dict[str, Any],
) -> tuple[bool, str | None]:
"""
Handle a single temp-directory entry: skip if the name doesn't match the
timestamp pattern, otherwise delete (or dry-run log) it when stale.
Args:
item_path: Full path to the directory being evaluated
item_name: Directory name (used to extract the embedded timestamp)
cutoff_time: Directories older than this are considered stale
metadata: Workflow execution metadata for logging
Return:
tuple[bool, str | None]: (deleted, error_message). `deleted` is True
if the directory was removed (or would be, in dry-run mode).
"""
match = self.dir_timestamp_pattern.match(item_name)
if not match:
self.debug(f'Skipping directory without timestamp pattern: {item_name}', metadata)
return False, None
timestamp_str = match.group(2)
try:
# Parse YYYYMMDD_HHMMSS_microseconds
dir_time = datetime.strptime(timestamp_str, '%Y%m%d_%H%M%S_%f')
except ValueError as e:
error_msg = f'Invalid timestamp format in directory {item_name}: {str(e)}'
self.error(error_msg, metadata)
return False, error_msg
return self._delete_if_stale(item_path, item_name, dir_time, cutoff_time, metadata)
def _delete_if_stale(
self,
item_path: str,
item_name: str,
dir_time: datetime,
cutoff_time: datetime,
metadata: dict[str, Any],
) -> tuple[bool, str | None]:
"""
Delete (or dry-run log) a directory whose embedded timestamp is older than
cutoff_time; otherwise leave it alone.
Args:
item_path: Full path to the directory
item_name: Directory name (for logging)
dir_time: Timestamp parsed from the directory name
cutoff_time: Directories older than this are considered stale
metadata: Workflow execution metadata for logging
Return:
tuple[bool, str | None]: (deleted, error_message)
"""
if dir_time >= cutoff_time:
age_hours = (datetime.now() - dir_time).total_seconds() / 3600
self.debug(f'Keeping recent directory: {item_name} (age: {age_hours:.1f}h)', metadata)
return False, None
age_hours = (datetime.now() - dir_time).total_seconds() / 3600
if self.dry_run:
self.info(
f'[DRY RUN] Would delete directory: {item_name} (age: {age_hours:.1f}h)', metadata
)
return True, None
try:
shutil.rmtree(item_path)
self.info(f'Deleted stale directory: {item_name} (age: {age_hours:.1f}h)', metadata)
return True, None
except OSError as e:
error_msg = f'Failed to delete directory {item_name}: {str(e)}'
self.error(error_msg, metadata)
return False, error_msg
@activity.defn(name='cleanup_temp_directories')
def cleanup_temp_directories(self, input_data: dict[str, Any]) -> None:
"""
@@ -109,51 +190,13 @@ class Cleanup(SientiaMonitoring):
directories_scanned += 1
# Extract timestamp from directory name
match = self.dir_timestamp_pattern.match(item_name)
if not match:
self.debug(
f'Skipping directory without timestamp pattern: {item_name}', metadata
)
continue
timestamp_str = match.group(2)
try:
# Parse YYYYMMDD_HHMMSS_microseconds
dir_time = datetime.strptime(timestamp_str, '%Y%m%d_%H%M%S_%f')
if dir_time < cutoff_time:
age_hours = (datetime.now() - dir_time).total_seconds() / 3600
if self.dry_run:
self.info(
f'[DRY RUN] Would delete directory: {item_name} (age: {age_hours:.1f}h)',
metadata,
)
directories_deleted += 1
else:
try:
shutil.rmtree(item_path)
self.info(
f'Deleted stale directory: {item_name} (age: {age_hours:.1f}h)',
metadata,
)
directories_deleted += 1
except OSError as e:
error_msg = f'Failed to delete directory {item_name}: {str(e)}'
errors.append(error_msg)
self.error(error_msg, metadata)
else:
age_hours = (datetime.now() - dir_time).total_seconds() / 3600
self.debug(
f'Keeping recent directory: {item_name} (age: {age_hours:.1f}h)',
metadata,
)
except ValueError as e:
error_msg = f'Invalid timestamp format in directory {item_name}: {str(e)}'
errors.append(error_msg)
self.error(error_msg, metadata)
deleted, error = self._process_directory_item(
item_path, item_name, cutoff_time, metadata
)
if error:
errors.append(error)
if deleted:
directories_deleted += 1
self.info(
f'Directory cleanup completed - Scanned: {directories_scanned}, '

View File

@@ -42,15 +42,13 @@ def train_test_split(
random_state: int | None = None,
shuffle: bool = True,
) -> tuple[pd.DataFrame, pd.DataFrame]:
# 1. Definir a semente (seed) para reprodutibilidade
if random_state is not None:
np.random.seed(random_state)
# 2. Gerar índices e embaralhar se necessário
indices = np.arange(len(data))
if shuffle:
np.random.shuffle(indices)
# 1. Generator local (em vez do estado global np.random) para reprodutibilidade
rng = np.random.default_rng(random_state)
rng.shuffle(indices)
# 3. Calcular o ponto de corte (split point)
# Cálculo: N_treino = tamanho_total * proporcao_treino

View File

@@ -136,8 +136,6 @@ class TrainModel:
run_dir=train_result.get('run_dir'),
metadata=metadata,
)
else:
pass
except Exception: # noqa: BLE001
# If cleanup fails after training failed, there is nothing extra to log (DB not committed).
if training_succeeded: # pragma: no branch