SIENTIAPDE-1231

Update .gitignore and refactor metrics.py for improved logging and consistency

- Added coverage.xml to .gitignore to prevent tracking of coverage reports.
- Refactored metric labels in metrics.py for consistency in string formatting and improved readability.
- Enhanced logging messages in various activities to ensure uniformity in message formatting.

laborious/activities/storage.py

@@ -1,20 +1,21 @@
 from temporalio import activity, workflow
 
-from laborious.utils.repository.minio_repository import MinioRepository
-
-
 with workflow.unsafe.imports_passed_through():
     # Extend the Temporal Postgres activities for convenient query -> MinIO export
-    from sientia_do.temporal.activities.postgres import Postgres
-    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
-    from sientia_do.observability.logger import Logger
-    from sientia_do.temporal.constants import now
-    from sientia_do.notifications.models import NotificationLevel
-    from typing import Any
     import traceback
-    import pandas as pd
+    from typing import Any
 
-DATETIME_FILENAME_FORMAT = "%Y-%m-%d_%H-%M-%S"
+    import pandas as pd
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.temporal.activities.postgres import Postgres
+    from sientia_do.temporal.constants import now
+
+    from laborious.utils.repository.minio_repository import MinioRepository
+
+
+DATETIME_FILENAME_FORMAT = '%Y-%m-%d_%H-%M-%S'
 
 
 class Storage(Postgres):
@@ -23,36 +24,33 @@ class Storage(Postgres):
     directly to MinIO as Parquet and return the object name.
     """
 
-    def __init__(self,
-                 host: str,
-                 port: int,
-                 user: str,
-                 password: str,
-                 dbname: str,
-                 min_connections: int,
-                 max_connections: int,
-                 minio_config: dict[str, Any],
-                 logger: Logger,
-                 notification_handler: NotificationHandler):
-        super().__init__(host=host,
-                         port=port,
-                         user=user,
-                         password=password,
-                         dbname=dbname,
-                         min_connections=min_connections,
-                         max_connections=max_connections,
-                         logger=logger,
-                         notification_handler=notification_handler)
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        user: str,
+        password: str,
+        dbname: str,
+        min_connections: int,
+        max_connections: int,
+        minio_config: dict[str, Any],
+        logger: Logger,
+        notification_handler: NotificationHandler,
+    ):
+        super().__init__(
+            host=host,
+            port=port,
+            user=user,
+            password=password,
+            dbname=dbname,
+            min_connections=min_connections,
+            max_connections=max_connections,
+            logger=logger,
+            notification_handler=notification_handler,
+        )
 
         if not hasattr(self, 'minio_repository'):
-            self.minio_repository = MinioRepository(
-                logger=logger,
-                notification_handler=notification_handler,
-                minio_endpoint_url=minio_config['endpoint_url'],
-                minio_access_key=minio_config['access_key'],
-                minio_secret_key=minio_config['secret_key'],
-                minio_region_name=minio_config['region_name'],
-                minio_default_bucket=minio_config['default_bucket'])
+            self.minio_repository: MinioRepository | None = None
 
         if self.minio_repository is None:
             self.minio_repository = MinioRepository(
@@ -62,7 +60,8 @@ class Storage(Postgres):
                 minio_access_key=minio_config['access_key'],
                 minio_secret_key=minio_config['secret_key'],
                 minio_region_name=minio_config['region_name'],
-                minio_default_bucket=minio_config['default_bucket'])
+                minio_default_bucket=minio_config['default_bucket'],
+            )
 
     @activity.defn(name='query_to_minio')
     async def query_to_minio(self, input_data: dict[str, Any]) -> dict[str, Any]:
@@ -79,64 +78,60 @@ class Storage(Postgres):
             dict: { success: bool, object_name: str, uri: str }
         """
 
+        if self.minio_repository is None:
+            raise ValueError('Minio repository not initialized')
+
         metadata = input_data.get('metadata', {})
         object_prefix = input_data.get('object_prefix', 'datasets/retrain')
 
         timestamp = now().strftime(DATETIME_FILENAME_FORMAT)
-        object_name = f"{object_prefix}_{timestamp}.parquet"
-        uri = f"s3://{self.minio_repository.minio_bucket}/{object_name}"
+        object_name = f'{object_prefix}_{timestamp}.parquet'
+        uri = f's3://{self.minio_repository.minio_bucket}/{object_name}'
 
         try:
             data = await self.load_custom_query(input_data)
             if not data:
-                self.error(
-                    f"query_to_minio failed: No data returned from query", metadata)
-                return {"success": False, "message": "No data returned from query"}
+                self.error('query_to_minio failed: No data returned from query', metadata)
+                return {'success': False, 'message': 'No data returned from query'}
 
             # Ensure we have a DataFrame
             data = pd.DataFrame(data)
 
             # Write parquet to memory and upload via persistent client
             self.minio_repository.store_dataframe_as_parquet(
-                dataframe=data,
-                uri=uri,
-                object_name=object_name,
-                metadata=metadata
+                dataframe=data, uri=uri, object_name=object_name, metadata=metadata
             )
 
-            return {"success": True, "object_key": object_name, "uri": uri}
+            return {'success': True, 'object_key': object_name, 'uri': uri}
         except Exception as e:
             trace = traceback.format_exc()
             self.send_notification(
                 metadata=metadata,
-                notification_id="ERROR_LOADING_CUSTOM_QUERY",
-                message=f"Error fetching data from query: {e}",
-                block="load_custom_query",
+                notification_id='ERROR_STORING_QUERY_TO_MINIO',
+                message=f'Error storing query to MinIO: {e}',
+                block='query_to_minio',
                 level=NotificationLevel.ERROR,
-                attachment_content=trace
+                attachment_content=trace,
             )
 
             self.error(trace, metadata)
 
-            return {"success": False, "message": str(e)}
+            return {'success': False, 'message': str(e)}
 
     def close(self) -> None:
         """Close Storage resources (MinIO client and Postgres engine)."""
         try:
-            if hasattr(self, 's3_client') and self.s3_client is not None:
+            if hasattr(self, 'minio_repository') and self.minio_repository is not None:
                 try:
-                    self.s3_client.close()
+                    self.minio_repository.close()
                 finally:
-                    self.s3_client = None
+                    self.minio_repository = None
         finally:
             # Ensure Postgres resources are disposed as well
             try:
                 super().close()
             except Exception:
-                pass
+                self.logger.error('Error closing Postgres resources')
 
     def __del__(self):
-        try:
-            self.close()
-        except Exception:
-            pass
+        self.close()