SIENTIAPDE-1712

Add unique constraint on (model_id, timestamp, variable) in schema and implement idempotent export test to ensure no duplicates are created. Update CoreScouter and related tests to handle conflict resolution by ignoring duplicates.

e2e/conftest.py

@@ -89,6 +89,7 @@ def _create_schema_and_table(engine):
                 value numeric NULL,
                 "timestamp" timestamptz NOT NULL,
                 created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+                CONSTRAINT unique_timestamp_variable UNIQUE (model_id, "timestamp", variable),
                 PRIMARY KEY (id, created_at)
             );
         """

e2e/test_pi_web_api_scouter_success.py

@@ -296,3 +296,101 @@ async def test_scenario_1_1_3_success_with_debug_data_package(
         # The package should be a dict with 'data' and 'held_data' keys
         assert isinstance(package_data, dict), "Expected data package to be a dict"
 
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_1_1_4_idempotent_export_ignores_duplicates(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    mock_pi_web_api_client,
+    postgres_engine,
+):
+    """
+    Scenario 1.1.4: Idempotent export ignores duplicates.
+
+    Running the same batch twice must not increase row count for the same
+    (model_id, timestamp, variable) keys.
+    """
+    client = temporal_test_env.client
+    unique_id = int(datetime.now().timestamp() * 1000) % 1000000
+
+    input_data = {
+        'model_name': 'PI Web API Scouter Test Model',
+        'model_id': str(unique_id),
+        'schedule_name': f'pi-web-api-scouter-idempotent-{unique_id}',
+        'model_tags': {
+            'tag1': {
+                'webid': 'webid1',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+                'frequency': 60000,
+            },
+            'tag2': {
+                'webid': 'webid2',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+                'frequency': 60000,
+            },
+        },
+        'trigger_laborious': False,
+        'filters': {},
+        'schema': 'sientia_data',
+        'table_name': 'laborious_data',
+        'retention_time': 3600,
+        'fill_missing_tags': False,
+        'pi_web_api_query': {
+            'endpoint': '/streamsets/recorded',
+            'period': '*-1d',
+            'max_count': 10,
+            'api_timeout': 30,
+        },
+    }
+
+    first_handle = await client.start_workflow(
+        PIWebAPIScouter.run,
+        input_data,
+        id=f'test-workflow-idempotent-1-{unique_id}',
+        task_queue='test-queue',
+    )
+    await first_handle.result()
+
+    second_handle = await client.start_workflow(
+        PIWebAPIScouter.run,
+        input_data,
+        id=f'test-workflow-idempotent-2-{unique_id}',
+        task_queue='test-queue',
+    )
+    await second_handle.result()
+
+    schema_name = 'sientia_data'
+    table_name = 'laborious_data'
+    full_table_name = f"{schema_name}.{table_name}"
+
+    with postgres_engine.connect() as conn:
+        total_count_query = text(
+            f"""
+            SELECT COUNT(*)
+            FROM {full_table_name}
+            WHERE model_id = :model_id
+            """
+        )
+        total_count = conn.execute(total_count_query, {'model_id': unique_id}).scalar()
+
+        unique_count_query = text(
+            f"""
+            SELECT COUNT(*)
+            FROM (
+                SELECT DISTINCT model_id, "timestamp", variable
+                FROM {full_table_name}
+                WHERE model_id = :model_id
+            ) unique_rows
+            """
+        )
+        unique_count = conn.execute(unique_count_query, {'model_id': unique_id}).scalar()
+
+    assert total_count > 0, "Expected exported rows for idempotency scenario"
+    assert total_count == unique_count, (
+        "Expected no duplicate rows for same (model_id, timestamp, variable)"
+    )
+    assert mock_pi_web_api_client.get_latest_values_df.call_count == 2
+