diff --git a/model_manager/activities/cleanup.py b/model_manager/activities/cleanup.py index 8b53012..3014c58 100644 --- a/model_manager/activities/cleanup.py +++ b/model_manager/activities/cleanup.py @@ -61,6 +61,87 @@ class Cleanup(SientiaMonitoring): r'^(.+)_(\d{8}_\d{6}_\d{6})$' ) # name_YYYYMMDD_HHMMSS_microseconds + def _process_directory_item( + self, + item_path: str, + item_name: str, + cutoff_time: datetime, + metadata: dict[str, Any], + ) -> tuple[bool, str | None]: + """ + Handle a single temp-directory entry: skip if the name doesn't match the + timestamp pattern, otherwise delete (or dry-run log) it when stale. + + Args: + item_path: Full path to the directory being evaluated + item_name: Directory name (used to extract the embedded timestamp) + cutoff_time: Directories older than this are considered stale + metadata: Workflow execution metadata for logging + + Return: + tuple[bool, str | None]: (deleted, error_message). `deleted` is True + if the directory was removed (or would be, in dry-run mode). + """ + match = self.dir_timestamp_pattern.match(item_name) + if not match: + self.debug(f'Skipping directory without timestamp pattern: {item_name}', metadata) + return False, None + + timestamp_str = match.group(2) + try: + # Parse YYYYMMDD_HHMMSS_microseconds + dir_time = datetime.strptime(timestamp_str, '%Y%m%d_%H%M%S_%f') + except ValueError as e: + error_msg = f'Invalid timestamp format in directory {item_name}: {str(e)}' + self.error(error_msg, metadata) + return False, error_msg + + return self._delete_if_stale(item_path, item_name, dir_time, cutoff_time, metadata) + + def _delete_if_stale( + self, + item_path: str, + item_name: str, + dir_time: datetime, + cutoff_time: datetime, + metadata: dict[str, Any], + ) -> tuple[bool, str | None]: + """ + Delete (or dry-run log) a directory whose embedded timestamp is older than + cutoff_time; otherwise leave it alone. + + Args: + item_path: Full path to the directory + item_name: Directory name (for logging) + dir_time: Timestamp parsed from the directory name + cutoff_time: Directories older than this are considered stale + metadata: Workflow execution metadata for logging + + Return: + tuple[bool, str | None]: (deleted, error_message) + """ + if dir_time >= cutoff_time: + age_hours = (datetime.now() - dir_time).total_seconds() / 3600 + self.debug(f'Keeping recent directory: {item_name} (age: {age_hours:.1f}h)', metadata) + return False, None + + age_hours = (datetime.now() - dir_time).total_seconds() / 3600 + + if self.dry_run: + self.info( + f'[DRY RUN] Would delete directory: {item_name} (age: {age_hours:.1f}h)', metadata + ) + return True, None + + try: + shutil.rmtree(item_path) + self.info(f'Deleted stale directory: {item_name} (age: {age_hours:.1f}h)', metadata) + return True, None + except OSError as e: + error_msg = f'Failed to delete directory {item_name}: {str(e)}' + self.error(error_msg, metadata) + return False, error_msg + @activity.defn(name='cleanup_temp_directories') def cleanup_temp_directories(self, input_data: dict[str, Any]) -> None: """ @@ -109,51 +190,13 @@ class Cleanup(SientiaMonitoring): directories_scanned += 1 - # Extract timestamp from directory name - match = self.dir_timestamp_pattern.match(item_name) - if not match: - self.debug( - f'Skipping directory without timestamp pattern: {item_name}', metadata - ) - continue - - timestamp_str = match.group(2) - try: - # Parse YYYYMMDD_HHMMSS_microseconds - dir_time = datetime.strptime(timestamp_str, '%Y%m%d_%H%M%S_%f') - - if dir_time < cutoff_time: - age_hours = (datetime.now() - dir_time).total_seconds() / 3600 - - if self.dry_run: - self.info( - f'[DRY RUN] Would delete directory: {item_name} (age: {age_hours:.1f}h)', - metadata, - ) - directories_deleted += 1 - else: - try: - shutil.rmtree(item_path) - self.info( - f'Deleted stale directory: {item_name} (age: {age_hours:.1f}h)', - metadata, - ) - directories_deleted += 1 - except OSError as e: - error_msg = f'Failed to delete directory {item_name}: {str(e)}' - errors.append(error_msg) - self.error(error_msg, metadata) - else: - age_hours = (datetime.now() - dir_time).total_seconds() / 3600 - self.debug( - f'Keeping recent directory: {item_name} (age: {age_hours:.1f}h)', - metadata, - ) - - except ValueError as e: - error_msg = f'Invalid timestamp format in directory {item_name}: {str(e)}' - errors.append(error_msg) - self.error(error_msg, metadata) + deleted, error = self._process_directory_item( + item_path, item_name, cutoff_time, metadata + ) + if error: + errors.append(error) + if deleted: + directories_deleted += 1 self.info( f'Directory cleanup completed - Scanned: {directories_scanned}, ' diff --git a/model_manager/utils/repository/data_manager_repository.py b/model_manager/utils/repository/data_manager_repository.py index da80787..399f9bb 100644 --- a/model_manager/utils/repository/data_manager_repository.py +++ b/model_manager/utils/repository/data_manager_repository.py @@ -42,15 +42,13 @@ def train_test_split( random_state: int | None = None, shuffle: bool = True, ) -> tuple[pd.DataFrame, pd.DataFrame]: - # 1. Definir a semente (seed) para reprodutibilidade - if random_state is not None: - np.random.seed(random_state) - # 2. Gerar índices e embaralhar se necessário indices = np.arange(len(data)) if shuffle: - np.random.shuffle(indices) + # 1. Generator local (em vez do estado global np.random) para reprodutibilidade + rng = np.random.default_rng(random_state) + rng.shuffle(indices) # 3. Calcular o ponto de corte (split point) # Cálculo: N_treino = tamanho_total * proporcao_treino diff --git a/model_manager/workflows/train_model.py b/model_manager/workflows/train_model.py index 00557e7..e3e0c73 100644 --- a/model_manager/workflows/train_model.py +++ b/model_manager/workflows/train_model.py @@ -136,8 +136,6 @@ class TrainModel: run_dir=train_result.get('run_dir'), metadata=metadata, ) - else: - pass except Exception: # noqa: BLE001 # If cleanup fails after training failed, there is nothing extra to log (DB not committed). if training_succeeded: # pragma: no branch