sientia-dataops-opc-ingestor/scripts/seed_e2e.py

"""Seed test data for the OPC Gateway e2e validation session (SIENTIAPDE-1972).

Populates:
  1. Mongo `connectors` + `connector_tags` (each tag with `metadata: {}` — required
     for documentation, per the feature owner).
  2. Redis slot v2 (`slot:opc_tags:1`, contract C1) so the subscription path collects them.
  3. Postgres test predictions table (with the `write_status` column) + one row the
     NATS write test can UPDATE (correlated by timestamp + model_id).

Idempotent: connectors/tags are upserted, the slot is overwritten, the table uses
CREATE IF NOT EXISTS + ON CONFLICT DO NOTHING.

Usage (from the repo root, sientia-dataops conda env):
  python scripts/seed_e2e.py --print-only          # dry-run: show docs/SQL, touch nothing
  python scripts/seed_e2e.py                       # seed Mongo + Redis + Postgres

Env (same names the gateway/app use):
  MONGODB_URL/MONGODB_USERNAME/MONGODB_PASSWORD/MONGODB_DATABASE
  REDIS_HOST/REDIS_PORT  POSTGRES_HOST/PORT/USER/PASSWORD/DBNAME
  OPC_SERVER_URL (default opc.tcp://localhost:49320 — KepServer default port)

# ponytail: plain sync pymongo/redis/sqlalchemy, no sientia_do async plumbing —
# this is a test fixture, not production code.
"""

import argparse
import json
import os
import sys

try:  # optional: local dev reads .env; in-cluster env comes from Helm
    from dotenv import load_dotenv

    load_dotenv()
except ImportError:
    pass

OPC_URL = os.getenv('OPC_SERVER_URL', 'opc.tcp://localhost:49320')

# No client certificate: the gateway connects without security (SIENTIAPDE-1974 D5).
# To test security, point client_certificate_id at an active client certificate of
# opc_certificates and set mode/policy (e.g. SignAndEncrypt / Basic256Sha256).
SECURITY = {'mode': 'None', 'policy': 'None', 'client_certificate_id': None}

CONNECTOR = {
    'connector_id': 'e2e-kepserver',
    'type': 'opc_ua',
    'name': 'e2e-kepserver',
    'url': OPC_URL,
    'subscription_period_ms': 1000,
    'security': SECURITY,
    'status': 'Active',
    'description': 'OPC Gateway e2e validation connector (SIENTIAPDE-1972)',
}

# metadata: {} on every tag — feature owner asked test tags to carry the field.
TAGS = [
    {'tag_id': 9101, 'connector_id': 'e2e-kepserver', 'address': 'ns=2;s=Simulation Examples.Functions.Ramp1',
     'tag_name': 'e2e_ramp', 'category': 'Read', 'data_type': 'float', 'unit': '-',
     'status': 'Active', 'description': 'e2e read tag', 'metadata': {}},
    {'tag_id': 9102, 'connector_id': 'e2e-kepserver', 'address': 'ns=2;s=Simulation Examples.Functions.User1',
     'tag_name': 'e2e_write_target', 'category': 'Write', 'data_type': 'float', 'unit': '-',
     'status': 'Active', 'description': 'e2e write tag', 'metadata': {}},
]

# Slot v2 (contract C1), as the Orchestrator writes it for the connector above.
SLOT = {
    'e2e-kepserver': {
        'name': 'e2e-kepserver',
        'server_id': 'e2e-kepserver',
        'url': OPC_URL,
        'subscription_period_ms': 1000,
        'security': {'mode': SECURITY['mode'], 'policy': SECURITY['policy']},
        'certificate': None,
        'tags': {
            TAGS[0]['address']: {
                'frequency': 5000,
                'active_read': {'enabled': False},
                'names': [
                    {
                        'collection': 'raw_e2e-gateway-test',
                        'tag_name': TAGS[0]['tag_name'],
                        'model_id': '9001',
                        'model_name': 'e2e-model',
                    }
                ],
            }
        },
    }
}

PREDICTIONS_SQL = """
CREATE SCHEMA IF NOT EXISTS sientia_data;
CREATE TABLE IF NOT EXISTS sientia_data.e2e_predictions (
    model_id text NOT NULL,
    prediction double precision,
    prediction_confidence double precision,
    timestamp text NOT NULL,
    write_status text,
    PRIMARY KEY (model_id, timestamp)
);
INSERT INTO sientia_data.e2e_predictions (model_id, prediction, prediction_confidence, timestamp)
VALUES ('9001', 42.5, 0.97, '2026-07-15T12:00:00Z')
ON CONFLICT DO NOTHING;
"""

# The matching NATS write command for the row above (paste into `nats pub opc.write '...'`).
NATS_CMD = {
    'msg_id': 'e2e-0001', 'server_id': 'e2e-kepserver', 'addr': TAGS[1]['address'],
    'value': 42.5, 'data_type': 'float', 'tag_type': 'prediction',
    'model_id': '9001', 'model_name': 'e2e-model',
    'schema': 'sientia_data', 'table_name': 'e2e_predictions',
    'timestamp': '2026-07-15T12:00:00Z',
}


def seed_mongo():
    from pymongo import MongoClient

    url = os.getenv('MONGODB_URL', 'localhost:27017')
    user, pwd = os.getenv('MONGODB_USERNAME', 'sientia'), os.getenv('MONGODB_PASSWORD', 'sientia')
    db = MongoClient(f'mongodb://{user}:{pwd}@{url}')[os.getenv('MONGODB_DATABASE', 'sientia')]
    db['connectors'].replace_one({'connector_id': CONNECTOR['connector_id']}, CONNECTOR, upsert=True)
    for tag in TAGS:
        db['connector_tags'].replace_one({'tag_id': tag['tag_id']}, tag, upsert=True)
    print(f'mongo: upserted 1 connector + {len(TAGS)} tags (metadata: {{}})')


def seed_redis():
    import redis

    r = redis.Redis(host=os.getenv('REDIS_HOST', 'localhost'), port=int(os.getenv('REDIS_PORT', '6379')))
    r.set('slot:opc_tags:1', json.dumps(SLOT))
    print('redis: wrote slot:opc_tags:1 (v2)')


def seed_postgres():
    from sqlalchemy import create_engine, text

    host, port = os.getenv('POSTGRES_HOST', 'localhost'), os.getenv('POSTGRES_PORT', '5432')
    user, pwd = os.getenv('POSTGRES_USER', 'sientia'), os.getenv('POSTGRES_PASSWORD', 'sientia')
    dbname = os.getenv('POSTGRES_DBNAME', 'sientia')
    engine = create_engine(f'postgresql://{user}:{pwd}@{host}:{port}/{dbname}')
    with engine.begin() as conn:
        for stmt in filter(str.strip, PREDICTIONS_SQL.split(';')):
            conn.execute(text(stmt))
    print('postgres: sientia_data.e2e_predictions ready (write_status column + 1 row)')


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--print-only', action='store_true', help='show docs/SQL, touch nothing')
    args = parser.parse_args()

    if args.print_only:
        print('connector:', json.dumps(CONNECTOR, indent=2))
        print('tags:', json.dumps(TAGS, indent=2))
        print('slot v2:', json.dumps(SLOT, indent=2))
        print('sql:', PREDICTIONS_SQL)
        print('nats write cmd:', json.dumps(NATS_CMD, indent=2))
        return 0

    seed_mongo()
    seed_redis()
    seed_postgres()
    print('\nNATS write command for the session:')
    print(f"nats pub opc.write '{json.dumps(NATS_CMD)}'")
    return 0


if __name__ == '__main__':
    sys.exit(main())