Code import - branch release/SIENTIAPDE-1645

This commit is contained in:
2026-06-28 03:02:55 +00:00
commit d607d5fed0
183 changed files with 178293 additions and 0 deletions

93
.dockerignore Normal file
View File

@@ -0,0 +1,93 @@
# ============================================================================
# WHITELIST APPROACH: Block everything by default, then allow only what's needed
# ============================================================================
# Block everything first
*
# ============================================================================
# ALLOW: Application source code (model_manager package)
# ============================================================================
# Allow the main package directory and all Python files
!model_manager/
!model_manager/**/*.py
!model_manager/**/__init__.py
# Allow subdirectories structure
!model_manager/activities/
!model_manager/activities/**
!model_manager/schedules/
!model_manager/schedules/**
!model_manager/sientia/
!model_manager/sientia/**
!model_manager/utils/
!model_manager/utils/**
!model_manager/utils/models/
!model_manager/utils/models/**
!model_manager/utils/repository/
!model_manager/utils/repository/**
!model_manager/worker/
!model_manager/worker/**
!model_manager/workflows/
!model_manager/workflows/**
# Allow reports directory with header.html
!model_manager/reports/
!model_manager/reports/header.html
# Allow temp directory structure (but not its contents)
!model_manager/reports/temp/
# ============================================================================
# ALLOW: Dependencies file (needed for pip install in Dockerfile)
# ============================================================================
!requirements.txt
# ============================================================================
# BLOCK: Explicitly block unwanted files even if they match above patterns
# ============================================================================
# Python cache and compiled files
**/__pycache__/
**/*.pyc
**/*.pyo
**/*.pyd
**/.Python
**/*.so
**/*.egg
**/*.egg-info/
# Tests (not needed in production)
model_manager/**/test_*.py
model_manager/**/*_test.py
# IDE and editor files
**/.vscode/
**/.idea/
**/*.swp
**/*.swo
**/*~
# OS files
**/.DS_Store
**/Thumbs.db
# Logs and temporary files
**/*.log
**/*.tmp
**/*.temp
# Local configuration
**/.env
**/.env.local
**/*.local
# Documentation inside code
**/*.md
**/README*
# Backup files
**/*.bak
**/*.backup
**/*.old

54
.env Normal file
View File

@@ -0,0 +1,54 @@
POSTGRES_HOST=db.sientia.ai
POSTGRES_PORT=30432
POSTGRES_USER=postgres
POSTGRES_PASSWORD=nFqc81y6kwmr2zuAIx43DhiOosFCVPpeEfTtTWZflkNjB2j1KtEeIANkhFR9mAX3
POSTGRES_DBNAME=sientia-core-mlops-bff
POSTGRES_MIN_CONNECTIONS=10
POSTGRES_MAX_CONNECTIONS=30
MLFLOW_URL=https://tracking.sientia.ai
MLFLOW_USERNAME=aignosi
MLFLOW_PASSWORD=1L0FP50j3ncp123
LOG_LEVEL=DEBUG
HTTP_METRICS_PORT=9090
HTTP_SDK_METRICS_PORT=9091
PROJECT_NAME=sientia-model-manager
TEMPORAL_HOST=orchestrator.sientia.ai
TEMPORAL_NAMESPACE=model-manager
TRAIN_TASK_QUEUE=train_model-local_queue
CLEANUP_TASK_QUEUE=cleanup-local_queue
TEMPORAL_USE_TLS=true
MONGODB_USERNAME=root
MONGODB_PASSWORD=wKZDbMNU1c
MONGODB_URL=db.sientia.ai:32017
MONGODB_DATABASE=sientia
MONGODB_TTL_INDEX_HOURS=1
MINIO_ENDPOINT_URL=https://storage.sientia.ai
MINIO_ACCESS_KEY=model-training-user
MINIO_SECRET_KEY=modelTrainingUser123
MINIO_REGION=us-east-1
MINIO_USE_SSL=false
MINIO_MAX_RETRY_ATTEMPTS=3
MINIO_RETRY_MODE=adaptive
MINIO_CONNECT_TIMEOUT=10
MINIO_READ_TIMEOUT=60
TIMEOUT_VALIDATE_PARAMS=30
TIMEOUT_TRAIN_MODEL=2700
TIMEOUT_DELETE_FILE=120
TIMEOUT_UPDATE_DATABASE=30
CLEANUP_RETENTION_HOURS=24
CLEANUP_DRY_RUN=true
TIMEOUT_CLEANUP_LOCAL=120
CLEANUP_SCHEDULE_ID=cleanup-files-daily
CLEANUP_CRON="0 21 * * *"
CLEANUP_TIMEZONE=UTC
CLEANUP_EXECUTION_TIMEOUT_HOURS=1
EXTRA_PIP_REQUIREMENTS=git+https://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@github.com/Aignosi/sientia-mlops-library.git

57
.env.example Normal file
View File

@@ -0,0 +1,57 @@
POSTGRES_HOST=localhost
POSTGRES_PORT=5432
POSTGRES_USER=postgres
POSTGRES_PASSWORD=changeme
POSTGRES_DBNAME=sientia
POSTGRES_MIN_CONNECTIONS=10
POSTGRES_MAX_CONNECTIONS=30
MLFLOW_URL=http://localhost:5080
MLFLOW_USERNAME=aignosi
MLFLOW_PASSWORD=changeme
LOG_LEVEL=DEBUG
HTTP_METRICS_PORT=9090
HTTP_SDK_METRICS_PORT=9091
PROJECT_NAME=sientia-model-manager
TEMPORAL_HOST=localhost:7233
TEMPORAL_NAMESPACE=model-manager
TEMPORAL_USE_TLS=false
RUNTIME=single
MONGODB_USERNAME=root
MONGODB_PASSWORD=changeme
MONGODB_URL=localhost:27017
MONGODB_DATABASE=sientia
MONGODB_TTL_INDEX_HOURS=1
MINIO_ENDPOINT_URL=http://localhost:9000
MINIO_ACCESS_KEY=minioadmin
MINIO_SECRET_KEY=minioadmin
MINIO_DEFAULT_BUCKET=model-training
MINIO_SECURE=false
STORE_BASE_URL=http://localhost:3000
STORE_OWNER=aignosi
STORE_REPO=suse-model-store
STORE_USERNAME=
STORE_PASSWORD=
STORE_CACHE_TTL_SECONDS=3600
PYPI_SERVER=http://localhost:5000
PYPI_USERNAME=
PYPI_PASSWORD=
TIMEOUT_VALIDATE_PARAMS=30
TIMEOUT_TRAIN_MODEL=2700
TIMEOUT_DELETE_FILE=120
TIMEOUT_UPDATE_DATABASE=30
CLEANUP_RETENTION_HOURS=24
CLEANUP_DRY_RUN=false
TIMEOUT_CLEANUP_LOCAL=120
CLEANUP_SCHEDULE_ID=cleanup-files-daily
CLEANUP_CRON="0 0 * * *"
CLEANUP_TIMEZONE=UTC
CLEANUP_EXECUTION_TIMEOUT_HOURS=1

14
.event.json Normal file
View File

@@ -0,0 +1,14 @@
{
"action": "closed",
"pull_request": {
"merged": true,
"head": {
"ref": "release/SIENTIAPDE-1252"
},
"base": {
"ref": "main"
},
"number": 1,
"title": "Test PR"
}
}

33
.github/workflows/deploy.yml vendored Normal file
View File

@@ -0,0 +1,33 @@
name: Deploy Python Application
on:
pull_request:
types:
- closed
branches:
- main
- 'release/**'
- 'feature/**'
jobs:
deploy:
name: Deploy
if: github.event.pull_request.merged == true
permissions: write-all
uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy.yml@main
with:
branch_name: ${{ github.event.pull_request.head.ref }}
project_type: 'python'
image_name: 'sientia-dataops-model-manager'
helm_chart_path: 'sientia-module'
helm_release_name: 'sientia-dataops-model-manager'
helm_namespace: 'sientia'
update_version_in: '["pyproject", "values"]'
app_owner: 'Aignosi'
private_repos: 'sientia-dataops-library'
helm_repo_name: 'sientia'
helm_repo_url: 'https://raw.githubusercontent.com/Aignosi/sientia-dataops-helm-repo/refs/heads/main/'
helm_chart_version: '0.6.0'
helm_values_file: './values.yaml'
use_vpn: true
secrets: inherit

18
.github/workflows/quality-gate.yml vendored Normal file
View File

@@ -0,0 +1,18 @@
name: Quality gate
on:
pull_request:
branches:
- main
- 'release/**'
- 'feature/**'
types: [ opened, synchronize, reopened ]
jobs:
quality-gate:
uses: Aignosi/github_workflow_templates/.github/workflows/python-quality-gate.yml@main
permissions: write-all
with:
project_name: 'model_manager'
repositories: 'sientia-dataops-library,sientia-model-library'
secrets: inherit

250
.gitignore vendored Normal file
View File

@@ -0,0 +1,250 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[codz]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py.cover
.hypothesis/
.pytest_cache/
cover/
# Code quality tools cache
.bandit/
validate.txt
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# UV
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
#uv.lock
# poetry
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
# This is especially recommended for binary packages to ensure reproducibility, and is more
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
#poetry.lock
#poetry.toml
# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
#pdm.lock
#pdm.toml
.pdm-python
.pdm-build/
# pixi
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
#pixi.lock
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
# in the .venv directory. It is recommended not to include this directory in version control.
.pixi
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.envrc
.venv
env/
venv/
ENV/
venv_311/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Abstra
# Abstra is an AI-powered process automation framework.
# Ignore directories containing user credentials, local state, and settings.
# Learn more at https://abstra.io/docs
.abstra/
# Visual Studio Code
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
# and can be added to the global gitignore or merged into this file. However, if you prefer,
# you could uncomment the following to ignore the entire vscode folder
.vscode/
# Ruff stuff:
.ruff_cache/
# PyPI configuration file
.pypirc
# Cursor
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
# refer to https://docs.cursor.com/context/ignore-files
.cursorignore
.cursorindexingignore
# Marimo
marimo/_static/
marimo/_lsp/
__marimo__/
# Ignore Docker volumes
docker-compose.override.yml
**/db_data/
**/kafka-volume/
**/zookeeper-volume/
**/mage_data/
**/minio_data/
**/venv/
**/certs/*.pem
**/certs/*.der
**/certs/*.csr
**/deploy/*.yaml
scouter/.file_versions/
scouter/pipelines/**/triggers.yaml
**/postgres_data/**
# Ignore Python cache files
*.pyc
*.pyo
*.pyd
# Ignore temporary files
*.swp
# Ignore test run reports in model_manager/reports/temp (but keep temp folder and .gitkeep)
model_manager/reports/temp/*
!model_manager/reports/temp/.gitkeep
# Miscellaneous
git_key*
git_log
tmp/
sientia-module/
.secrets
.event.json
models/

10
.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,10 @@
# Default ignored files
/shelf/
/workspace.xml
# Ignored default folder with query files
/queries/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml
# Editor-based HTTP Client requests
/httpRequests/

View File

@@ -0,0 +1,13 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="GrazieInspection" enabled="false" level="GRAMMAR_ERROR" enabled_by_default="false" />
<inspection_tool class="GrazieStyle" enabled="false" level="STYLE_SUGGESTION" enabled_by_default="false" />
<inspection_tool class="LanguageDetectionInspection" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
<inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
<option name="processCode" value="true" />
<option name="processLiterals" value="true" />
<option name="processComments" value="true" />
</inspection_tool>
</profile>
</component>

View File

@@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

7
.idea/misc.xml generated Normal file
View File

@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Black">
<option name="sdkName" value="$PROJECT_DIR$/venv" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="$PROJECT_DIR$/venv" project-jdk-type="Python SDK" />
</project>

8
.idea/modules.xml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/sientia-dataops-model-manager.iml" filepath="$PROJECT_DIR$/.idea/sientia-dataops-model-manager.iml" />
</modules>
</component>
</project>

24
.idea/sientia-dataops-model-manager.iml generated Normal file
View File

@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/venv" />
</content>
<orderEntry type="jdk" jdkName="$MODULE_DIR$/venv" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
<component name="TemplatesService">
<option name="TEMPLATE_FOLDERS">
<list>
<option value="$MODULE_DIR$/sientia-module/templates" />
</list>
</option>
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="py.test" />
</component>
</module>

8
.idea/sonarlint.xml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SonarLintProjectSettings">
<option name="bindingEnabled" value="true" />
<option name="projectKey" value="Aignosi_sientia-dataops-model-manager_dc6e051c-995a-4b57-9cde-f511018184c7" />
<option name="serverId" value="Sonarqube Aignosi" />
</component>
</project>

6
.idea/vcs.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

71
.idea/workspace.xml generated Normal file
View File

@@ -0,0 +1,71 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="92ccf6be-e6f3-48c1-ab39-aeaebc5aef5b" name="Changes" comment="" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FlaskConsoleOptions" custom-start-script="import sys; print('Python %s on %s' % (sys.version, sys.platform)); sys.path.extend([WORKING_DIR_AND_PYTHON_PATHS])&#10;from flask.cli import ScriptInfo, NoAppException&#10;for module in [&quot;main.py&quot;, &quot;wsgi.py&quot;, &quot;app.py&quot;]:&#10; try: locals().update(ScriptInfo(app_import_path=module, create_app=None).load_app().make_shell_context()); print(&quot;\nFlask App: %s&quot; % app.import_name); break&#10; except NoAppException: pass">
<envs>
<env key="FLASK_APP" value="app" />
</envs>
<option name="myCustomStartScript" value="import sys; print('Python %s on %s' % (sys.version, sys.platform)); sys.path.extend([WORKING_DIR_AND_PYTHON_PATHS])&#10;from flask.cli import ScriptInfo, NoAppException&#10;for module in [&quot;main.py&quot;, &quot;wsgi.py&quot;, &quot;app.py&quot;]:&#10; try: locals().update(ScriptInfo(app_import_path=module, create_app=None).load_app().make_shell_context()); print(&quot;\nFlask App: %s&quot; % app.import_name); break&#10; except NoAppException: pass" />
<option name="myEnvs">
<map>
<entry key="FLASK_APP" value="app" />
</map>
</option>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="ProjectColorInfo">{
&quot;associatedIndex&quot;: 4,
&quot;fromUser&quot;: false
}</component>
<component name="ProjectId" id="3BfRCyO6razhElE9LcPDboXcAv2" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">{
&quot;keyToString&quot;: {
&quot;ModuleVcsDetector.initialDetectionPerformed&quot;: &quot;true&quot;,
&quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
&quot;RunOnceActivity.git.unshallow&quot;: &quot;true&quot;,
&quot;RunOnceActivity.typescript.service.memoryLimit.init&quot;: &quot;true&quot;,
&quot;ai.playground.ignore.import.keys.banner.in.settings&quot;: &quot;true&quot;,
&quot;git-widget-placeholder&quot;: &quot;feature/SIENTIAPDE-1717&quot;,
&quot;last_opened_file_path&quot;: &quot;/home/bruno-domingues/repos/sientia-dataops-model-manager&quot;,
&quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
&quot;settings.editor.selected.configurable&quot;: &quot;preferences.pluginManager&quot;,
&quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
}
}</component>
<component name="SharedIndexes">
<attachedChunks>
<set>
<option value="bundled-js-predefined-d6986cc7102b-9b0f141eb926-JavaScript-PY-253.32098.74" />
<option value="bundled-python-sdk-1cd77e80b48f-6d6dccd035ac-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-253.32098.74" />
</set>
</attachedChunks>
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="92ccf6be-e6f3-48c1-ab39-aeaebc5aef5b" name="Changes" comment="" />
<created>1774878015053</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1774878015053</updated>
<workItem from="1774878016366" duration="289000" />
<workItem from="1774878319177" duration="4459000" />
<workItem from="1774886754051" duration="6220000" />
</task>
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="3" />
</component>
</project>

2
.ruff_cache/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
# Automatically created by ruff.
*

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

1
.ruff_cache/CACHEDIR.TAG Normal file
View File

@@ -0,0 +1 @@
Signature: 8a477f597d28d172789f06886806bc55

14
.secrets Normal file
View File

@@ -0,0 +1,14 @@
GITHUB_TOKEN=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
CI_DEPS_APP_ID=1275848
CI_DEPS_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEA+k29atWqXQo22ozTviHcyldhfITPthkkAUGplQcwW4qV5J2y\nG5z+uteNda1BW+cyLyfdnjkBxlSl/b5w8BJ8nkUr+GojlOe5ItqnfMKJP/l2J5x5\nYbpG0ZOhHxm2TEBh6epPQhjMsNtwszHzeAPBbAmeglvviz4mGkuG4YR4anjj+Ynm\ny/TurN+IUuWNfN2cEyVmBWFLYyQYZvxdHlVRR5xEg3IdNn3Wk5iq3aCOq+N4novc\nsSDe8RwTtNbQK0i7146IdAwg3M3Kg3eBi56Hri+IdJqt2EDbPwJItg7uJWGbi9Rq\nYPdG5iA87zXBmVFV/oTvLWdfTaSeDIcwwozHKQIDAQABAoIBAGTKZRneDAoHEOuU\nhbcsP5Ii+ZiAinuTSBJRdI4UJP7XoWA7t/qyj0iBf+8A4y60vFqMvQr3faV1dJnm\n0+d4KkI5LGmNo+JUJRLJ5WOxmj7unwt4UNBviBDgV6MEYYn26vacaWn+Pn6A/rLJ\nzHZpXLu6a9+mUaFKV2GDRD8PbsXAOH7UvY88gpiFF6lG0NytbU8Iq0HtdesB5+uh\nFCKjFT1X+INDi2liv7QMx7juwel8SOnPmNXrsdtcNYWhnaCNJixspjuEGO3AAQ/a\no7Cg+bS9b9bMihSWJaBjm1jGe8PX2+KjEZviH4FTeTQp8aK9CxXp6GRY2qBwQF49\nGe4mqGECgYEA/jHsO+Ax6Q/lG4J0mN9i8yb1ZkhvPGqArmMYWNmYrelTXajtFucH\nb/wKgVnFVpLsb0PmPre6GQXEj04p6/SPs6IJBwiy4v9ZS4TR3A/HH88YAkaYEyJE\nF8AIga8bjcD908mMhzvV32nCijcTwkvGy0bvs5ED5aauqLuE0tdngm0CgYEA/BS+\nUPUX2JKj7q7QbYxdSVP3F5iKsJ5E0ucvvf/yjjd9Zy5fFq0SeTqZW+IzPA2TE8WL\nB/LgooBwrA8RR+pVFsiYPaYmlKEMv9++YzJXx+QlAJ501KHYu7/rU3goqqg4ZtNE\nX45ni7uV1x6eYYFu26Fvuim7+XSRLh5ghQnuAi0CgYEAnXghxyno4V9mZ7dWMcnZ\nC5Zr2XQv7LZxhxZ+Y7RQ4BO2AESAs3plYhs6Cs2o8SvNalQe02WP1KZ9EOW0FKcJ\nSperjf72iqbzE1RxiSkBCxSI5AgFd3z3v9rHqkbnA9a1p7io5LHNmTx0NplOFURK\nH22PWqcQSfkLJB4ed1rXlbUCgYAoRowt1SsNMEi+7vFgP2f8Ok0lWPr3wyHN7KAl\nJEq1zEnd/Xu13Msx0VoFYzu6YZTZONvA1l5ruEfIRdQGAHu92yjv2KcbYivuUCpU\nIQwDZQFAexDBlGZTgRNxT6Z3tc7lJuYquk7y9XK4Xy0A+TQkJUP7o4VkEurLSpEz\nUGXVtQKBgQDZnEYjtvsEO3NWifdnU8ghO2u99vIFVEm/sCZEj5HdHq7wLL//r2zQ\nlmHDG/8q+5l+FgmnqMf/xkGhtC+DWl9aIuzXNdWMqcIMRTjlorgXjjcJJs7mgd9E\nCobSt2sklE83MXy+aCWlP5mZqHltyAEVdF6QzN1uVN2rWg3N1SwJ/A==\n-----END RSA PRIVATE KEY-----\n"
SONAR_TOKEN=sqp_5d4b1372f6e50a541c1004138106d3521ce531cc
SONAR_HOST_URL=https://sonarqube.sientia.ai
AZURE_CREDENTIALS={"clientId":"d3522fb7-1eb5-41f4-b981-37b07d5da3d8","clientSecret":"7DM8Q~Ch6I91OgLX143YZ~fJntYU__QpGSOAnc~P","subscriptionId":"38ada0de-addc-4c09-9e7e-eddfab756596","tenantId":"a474959a-eea9-4c10-8298-7ea0c8152c36","activeDirectoryEndpointUrl":"https://login.microsoftonline.com","resourceManagerEndpointUrl":"https://management.azure.com/","activeDirectoryGraphResourceId":"https://graph.windows.net/","sqlManagementEndpointUrl":"https://management.core.windows.net:8443/","galleryEndpointUrl":"https://gallery.azure.com/","managementEndpointUrl":"https://management.core.windows.net/"}
REGISTRY_LOGIN_SERVER=aignosi.azurecr.io
REGISTRY_USERNAME=d3522fb7-1eb5-41f4-b981-37b07d5da3d8
REGISTRY_PASSWORD=7DM8Q~Ch6I91OgLX143YZ~fJntYU__QpGSOAnc~P
SUSE_RKE2_KUBE_CONFIG=apiVersion: v1\nclusters:\n- cluster:\n certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJlVENDQVIrZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWtNU0l3SUFZRFZRUUREQmx5YTJVeUxYTmwKY25abGNpMWpZVUF4TnpRMU16VXhOekE1TUI0WERUSTFNRFF5TWpFNU5UVXdPVm9YRFRNMU1EUXlNREU1TlRVdwpPVm93SkRFaU1DQUdBMVVFQXd3WmNtdGxNaTF6WlhKMlpYSXRZMkZBTVRjME5UTTFNVGN3T1RCWk1CTUdCeXFHClNNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJPTTlLUzZTQ0xIQUlOd0IwYzRYMWtPWEZISGcrbmdab2UvVXNrdnkKcTFZalFVazZxUWViblJBT2E2UHdzVTB0bkc1VjB0NSsrMWcrUDBBSFpFSi9DY3FqUWpCQU1BNEdBMVVkRHdFQgovd1FFQXdJQ3BEQVBCZ05WSFJNQkFmOEVCVEFEQVFIL01CMEdBMVVkRGdRV0JCUjRCRXNlRFdrOERuK2hqblJEClpReFF6aTVqV2pBS0JnZ3Foa2pPUFFRREFnTklBREJGQWlCZmdHTXhsSzlRM3NkVXRHZndKSThONDByQWVESHAKUEJvYnI3VXhrUjk2SWdJaEFOblpNUklxbjl2d3hINDNwRjgraC9iUjFsM2dBWXpkcU02WDRYdjZ4aGNvCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K\n server: https://20.121.64.107:6443\n name: suse-rk2\ncontexts:\n- context:\n cluster: suse-rk2\n user: suse-rk2\n name: suse-rk2\ncurrent-context: suse-rk2\nkind: Config\nusers:\n- name: suse-rk2\n user:\n client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrakNDQVRpZ0F3SUJBZ0lJUzZkLzFXazMyckl3Q2dZSUtvWkl6ajBFQXdJd0pERWlNQ0FHQTFVRUF3d1oKY210bE1pMWpiR2xsYm5RdFkyRkFNVGMwTlRNMU1UY3dPVEFlRncweU5UQTBNakl4T1RVMU1EbGFGdzB5TmpBMApNakl4T1RVMU1EbGFNREF4RnpBVkJnTlZCQW9URG5ONWMzUmxiVHB0WVhOMFpYSnpNUlV3RXdZRFZRUURFd3h6CmVYTjBaVzA2WVdSdGFXNHdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CQndOQ0FBVGhLMHlFY2tvajEybHYKcWZlWjRXQlp0enl5bHBtZ1dEelZSaGl0UWllS0tDQk9DNno4am5iN3loTDkzOE8vZTVsQUptQ1JjUGRreFlxKwpPREUvdXpXTW8wZ3dSakFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUhBd0l3Ckh3WURWUjBqQkJnd0ZvQVVaUWxyNk1ZMWtpT2VaL1VJeXZqV1Vzd0FSUU13Q2dZSUtvWkl6ajBFQXdJRFNBQXcKUlFJaEFPek9LQ2JMODRSTFpkaUFMNUZwa251L0d2Ty9ZdStuakI4K1cxNVgwVkJ3QWlBcGFndHVxZWhDYzM4ZApESzdSa2JzZGQ2b25WcEFvcXpJNkNraHRtSXJKb3c9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCi0tLS0tQkVHSU4gQ0VSVElGSUNBVEUtLS0tLQpNSUlCZVRDQ0FSK2dBd0lCQWdJQkFEQUtCZ2dxaGtqT1BRUURBakFrTVNJd0lBWURWUVFEREJseWEyVXlMV05zCmFXVnVkQzFqWVVBeE56UTFNelV4TnpBNU1CNFhEVEkxTURReU1qRTVOVFV3T1ZvWERUTTFNRFF5TURFNU5UVXcKT1Zvd0pERWlNQ0FHQTFVRUF3d1pjbXRsTWkxamJHbGxiblF0WTJGQU1UYzBOVE0xTVRjd09UQlpNQk1HQnlxRwpTTTQ5QWdFR0NDcUdTTTQ5QXdFSEEwSUFCRGdaQkpscmlEQzZldE5Ib1hNalZ1RTltcjlzMFIyTlo3Zmtsb1ZTCks3R3J2aUlyVjBFdHJMeDdwbzZyeVZWTTU1TmRtZzFGZ0MxeEpYaW14Rjg4djRDalFqQkFNQTRHQTFVZER3RUIKL3dRRUF3SUNwREFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQjBHQTFVZERnUVdCQlJsQ1d2b3hqV1NJNTVuOVFqSworTlpTekFCRkF6QUtCZ2dxaGtqT1BRUURBZ05JQURCRkFpQjNYNVdSWGdaT3lwN1FnSGE1UTU5YnU5a00rS1FxCmh6MVVXcmdHS2lNZWFRSWhBSXdjTUk1WjVBZ3kzY2JZUFViWlVOdWNlYzZ6MlFsbFJEcnhmZFhUOVJwSgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==\n client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUczTUpyWUVlbHhLVHQ1MEtXY3hySVhtKzkzZFhDa3JCeWdBUU1IU1pnc2NvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFNFN0TWhISktJOWRwYjZuM21lRmdXYmM4c3BhWm9GZzgxVVlZclVJbmlpZ2dUZ3VzL0k1MgorOG9TL2QvRHYzdVpRQ1pna1hEM1pNV0t2amd4UDdzMWpBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo=
REPO_TOKEN=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
REPO_USERNAME=bruno-domingues-aignosi
HELM_REPO_USERNAME=bruno-domingues-aignosi
HELM_REPO_PASSWORD=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

14
.vscode/settings.json vendored Normal file
View File

@@ -0,0 +1,14 @@
{
"coverage-gutters.coverageBaseDir": "${workspaceFolder}",
"coverage-gutters.coverageFileNames": ["coverage.xml"],
"coverage-gutters.coverageReportFileName": "coverage.xml",
"coverage-gutters.showLineCoverage": true,
"coverage-gutters.showRulerCoverage": true,
"coverage-gutters.highlightdark": "rgba(0, 255, 0, 1)",
"coverage-gutters.highlightlight": "rgba(0, 200, 0, 1)",
"coverage-gutters.partialHighlightDark": "rgba(255, 255, 0, 1)",
"coverage-gutters.partialHighlightLight": "rgba(255, 200, 0, 1)",
"coverage-gutters.noHighlightDark": "rgba(255, 0, 0, 1)",
"coverage-gutters.noHighlightLight": "rgba(255, 0, 0, 1)",
"explorer.autoReveal": false
}

78
Dockerfile Normal file
View File

@@ -0,0 +1,78 @@
# Multi-stage build for optimized Python application
FROM python:3.11-slim AS builder
# Set build-time environment variables
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
# Install build dependencies only
RUN apt-get update && apt-get install -y \
build-essential \
curl \
git \
&& rm -rf /var/lib/apt/lists/* && \
apt-get clean
# Configure SSH to trust GitHub host key
RUN mkdir -p ~/.ssh && \
ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts && \
chmod 600 ~/.ssh/known_hosts
# Create virtual environment
RUN python -m venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH"
# Upgrade pip and wheel for better caching
RUN pip install --upgrade pip setuptools wheel
# Copy requirements files for better Docker layer caching
COPY requirements.txt ./
# Install only production dependencies with no cache
RUN --mount=type=ssh echo "=== Installing dependencies ===" && \
pip install --no-cache-dir -r requirements.txt && \
echo "=== Dependencies installed successfully ===" && \
pip list | wc -l && \
echo "=== Cleaning cache files ===" && \
find /opt/venv -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
find /opt/venv -name "*.pyc" -delete 2>/dev/null || true && \
rm -rf /root/.cache/pip/* && \
echo "=== Cleaning venv site-packages ===" && \
find /opt/venv/lib/python3.11/site-packages/ -type f -name "*.md" -delete 2>/dev/null || true && \
echo "=== Stripping .so files ===" && \
find /opt/venv -name "*.so" -exec strip {} + 2>/dev/null || true
# Production stage using python-slim for better functionality
FROM python:3.11-slim AS production
# Set environment variables
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PATH="/opt/venv/bin:$PATH" \
POD_ID=unknown \
HOME="/app"
# Copy virtual environment from builder stage
COPY --from=builder /opt/venv /opt/venv
# Set working directory
WORKDIR /app
# Copy application code
COPY . .
# Create necessary directories for runtime file creation
RUN mkdir -p /app/model_manager/reports /app/logs /app/temp /app/models /app/data && \
chmod 755 /app/model_manager/reports /app/logs /app/temp /app/models /app/data
# Create non-root user
RUN groupadd -r appuser && useradd -r -g appuser appuser && \
chown -R appuser:appuser /app
# Switch to non-root user
USER appuser
# Set entrypoint for proper signal handling and PID 1
ENTRYPOINT ["/opt/venv/bin/python", "-m", "model_manager.worker.worker"]

21
LICENSE Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 aignosi
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -0,0 +1,227 @@
# Changelog de Parâmetros do Pipeline de Treinamento
Este documento descreve as alterações nos parâmetros de entrada do pipeline Temporal para treinamento de modelos.
## Resumo das Alterações
### Parâmetros ALTERADOS (Breaking Changes)
| Parâmetro | Tipo Anterior | Tipo Novo | Descrição |
|-----------|---------------|-----------|-----------|
| `lag_train` | `int` | `dict[str, int]` | Agora é um dicionário com lag por variável. Ex: `{"var1": 2, "var2": 3}` |
| `lag_val` | `int` | `dict[str, int]` | Agora é um dicionário com lag por variável. Ex: `{"var1": 1, "var2": 1}` |
### Parâmetros NOVOS (Obrigatórios)
| Parâmetro | Tipo | Descrição | Valores Válidos |
|-----------|------|-----------|-----------------|
| `model_name` | `str` | Nome do tipo de modelo | `"Linear Regression"`, `"Polynomial Regression"` |
| `degree` | `int` | Grau do polinômio (1 = linear) | `>= 1` |
| `interaction_only` | `bool` | Apenas termos de interação para polinomial | `true`, `false` |
| `nan_treatment` | `str` | Tratamento de valores NaN | `"drop"`, `"linear interpolation"`, `"fill linear"` |
| `scaler_name` | `str` | Nome do scaler a usar | `"Standard Scaler"`, `"None"` |
### Parâmetros NOVOS (Opcionais)
| Parâmetro | Tipo | Descrição | Default |
|-----------|------|-----------|---------|
| `start_date` | `str \| null` | Data inicial para filtrar dados (formato: `"YYYY-MM-DD HH:MM:SS"`) | `null` |
| `end_date` | `str \| null` | Data final para filtrar dados (formato: `"YYYY-MM-DD HH:MM:SS"`) | `null` |
| `support_filters` | `dict \| null` | Filtros customizados por variável | `{}` |
| `static_threshold` | `int \| null` | Threshold para remoção de janelas estáticas (1-1000). Só usado quando `rem_static_win` é `true`. | `1` |
---
## Exemplo de Input Completo
### Formato ANTERIOR (não funciona mais):
```json
{
"experiment_run_id": 123,
"target_variable": "temperatura",
"variable_columns": ["pressao", "umidade", "velocidade"],
"lag_train": 2,
"lag_val": 1,
"rem_static_win": true,
"low_lim": {"pressao": 0, "umidade": 0, "velocidade": 0},
"upp_lim": {"pressao": 100, "umidade": 100, "velocidade": 50},
"window": 0,
"use_scaler": true,
"include_ar": false,
"bucket_name": "training-data",
"file_name": "dataset.csv",
"line_separator": ";",
"decimal_separator": ",",
"train_size": 80,
"shuffle": false,
"experiment_name": "modelo-temperatura",
"removed_intervals": []
}
```
### Formato NOVO (obrigatório):
```json
{
"experiment_run_id": 123,
"target_variable": "temperatura",
"variable_columns": ["pressao", "umidade", "velocidade"],
"lag_train": {
"pressao": 2,
"umidade": 2,
"velocidade": 2
},
"lag_val": {
"pressao": 1,
"umidade": 1,
"velocidade": 1
},
"rem_static_win": true,
"static_threshold": null,
"low_lim": {"pressao": 0, "umidade": 0, "velocidade": 0},
"upp_lim": {"pressao": 100, "umidade": 100, "velocidade": 50},
"window": 0,
"use_scaler": true,
"include_ar": false,
"bucket_name": "training-data",
"file_name": "dataset.csv",
"line_separator": ";",
"decimal_separator": ",",
"train_size": 80,
"shuffle": false,
"experiment_name": "modelo-temperatura",
"removed_intervals": [],
"model_name": "Linear Regression",
"degree": 1,
"interaction_only": false,
"nan_treatment": "drop",
"scaler_name": "Standard Scaler",
"start_date": null,
"end_date": null,
"support_filters": {}
}
```
---
## Exemplo para Polynomial Regression
```json
{
"experiment_run_id": 124,
"target_variable": "temperatura",
"variable_columns": ["pressao", "umidade"],
"lag_train": {
"pressao": 0,
"umidade": 0
},
"lag_val": {
"pressao": 0,
"umidade": 0
},
"rem_static_win": false,
"low_lim": {"pressao": 0, "umidade": 0},
"upp_lim": {"pressao": 100, "umidade": 100},
"window": 0,
"use_scaler": true,
"include_ar": false,
"bucket_name": "training-data",
"file_name": "dataset.csv",
"line_separator": ";",
"decimal_separator": ",",
"train_size": 80,
"shuffle": false,
"experiment_name": "modelo-polinomial",
"removed_intervals": [],
"model_name": "Polynomial Regression",
"degree": 2,
"interaction_only": false,
"nan_treatment": "linear interpolation",
"scaler_name": "Standard Scaler",
"start_date": "2024-01-01 00:00:00",
"end_date": "2024-12-31 23:59:59",
"support_filters": {}
}
```
---
## Exemplo com Intervalos Removidos
```json
{
"removed_intervals": [
["2024-03-01 00:00:00", "2024-03-15 23:59:59"],
["2024-06-01 00:00:00", "2024-06-30 23:59:59"]
]
}
```
---
## Parâmetros Logados no MLflow
Os seguintes parâmetros são agora logados no MLflow:
| Parâmetro MLflow | Descrição |
|------------------|-----------|
| `model_name` | Nome do modelo (`Linear Regression` ou `Polynomial Regression`) |
| `models_params` | `{"degree": int, "interaction_only": bool}` |
| `target_variable` | Variável alvo |
| `input_variables` | Lista de variáveis de entrada |
| `nan_treatment` | Tratamento de NaN |
| `lag_train` | Dicionário de lags para treino |
| `lag_transform` | Dicionário de lags para transformação |
| `static_threshold` | Threshold para janelas estáticas (1 ou null) |
| `lower_limits` | Limites inferiores por variável |
| `upper_limits` | Limites superiores por variável |
| `scaler_name` | Nome do scaler |
| `scaler_params` | Parâmetros do scaler (mean, variance) |
| `include_ar` | Se inclui variável autoregressiva |
| `train_size` | Proporção de treino (0.0 - 1.0) |
| `test_size` | Proporção de teste (0.0 - 1.0) |
| `start_date` | Data inicial (ou null) |
| `end_date` | Data final (ou null) |
| `removed_intervals` | Lista de intervalos removidos |
| `retrain` | Sempre `false` para novos modelos |
| `support_filters` | Filtros customizados |
---
## Validações de Negócio
O sistema valida automaticamente:
1. **`train_size`**: Deve estar entre 10 e 100
2. **`variable_columns`**: Não pode estar vazio
3. **`lag_train` / `lag_val`**: Todos os valores devem ser >= 0
4. **`window`**: Deve ser >= 0
5. **`degree`**: Deve ser >= 1
6. **`nan_treatment`**: Deve ser `"drop"`, `"linear interpolation"` ou `"fill linear"`
7. **`scaler_name`**: Deve ser `"Standard Scaler"` ou `"None"`
8. **`model_name`**: Deve ser `"Linear Regression"` ou `"Polynomial Regression"`
9. **`low_lim` / `upp_lim`**: Devem ter as mesmas chaves, e `low_lim[var] < upp_lim[var]`
10. **`bucket_name` / `file_name` / `experiment_name`**: Não podem estar vazios
11. **`degree` vs `model_name`**: Se `model_name` = "Polynomial Regression", `degree` deve ser >= 2; se "Linear Regression", `degree` deve ser = 1
12. **`removed_intervals`**: Cada elemento deve ser lista/tupla com pelo menos 2 elementos (start, end)
13. **`target_variable`**: Não pode estar vazio
14. **`static_threshold`**: Se `rem_static_win` = `true` e `static_threshold` tiver valor, deve estar entre 1 e 1000 (inclusive). Se `null`, assume valor `1`.
---
## Arquivos Modificados
- `model_manager/sientia/models.py` - `DataPreprocessor` e `LinearRegressionModel`
- `model_manager/sientia/metrics.py` - Funções RCE adicionadas
- `model_manager/utils/models/train_model_params.py` - Novos parâmetros
- `model_manager/utils/repository/training_repository.py` - Uso dos novos parâmetros
- `model_manager/utils/repository/model_repository.py` - Logging no MLflow

1554
README.md Normal file

File diff suppressed because it is too large Load Diff

1742
coverage.xml Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,33 @@
DATA,DATE2,03CV022/CORRENTE_N_M1_PV(Value),303-WIT-230(Value)
01/05/2022 00:00:00,07-01-2022 01:00:00,170,33
01/05/2022 00:00:10,07-01-2022 01:00:10,169,605
01/05/2022 00:00:20,07-01-2022 01:00:20,169,178
01/05/2022 00:00:30,07-01-2022 01:00:30,166,468
01/05/2022 00:00:40,07-01-2022 01:00:40,162,136
01/05/2022 00:00:50,07-01-2022 01:00:50,157,804
01/05/2022 00:01:00,07-01-2022 01:01:00,155,883
01/05/2022 00:01:10,07-01-2022 01:01:10,155,684
01/05/2022 00:01:20,07-01-2022 01:01:20,155,484
01/05/2022 00:01:30,07-01-2022 01:01:30,155,284
01/05/2022 00:01:40,07-01-2022 01:01:40,155,85
01/05/2022 00:01:50,07-01-2022 01:01:50,154,885
01/05/2022 00:02:00,09-01-2022 02:02:00,154,685
01/05/2022 00:02:10,09-01-2022 02:02:10,154,486
01/05/2022 00:02:20,09-01-2022 02:02:20,154,286
01/05/2022 00:02:30,09-01-2022 02:02:30,154,86
01/05/2022 00:02:40,09-01-2022 02:02:40,152,866
01/05/2022 00:02:50,09-01-2022 02:02:50,150,87
01/05/2022 00:03:00,09-01-2022 02:03:00,148,874
01/05/2022 00:03:10,09-01-2022 02:03:10,148,2134
01/05/2022 00:03:20,09-01-2022 02:03:20,148,2068
01/05/2022 00:03:30,09-01-2022 02:03:30,148,2022
01/05/2022 00:03:40,09-01-2022 02:03:40,148,1976
01/05/2022 00:03:50,09-01-2022 02:03:50,154,139
01/05/2022 00:04:00,11-01-2022 03:04:00,165,112
01/05/2022 00:04:10,11-01-2022 03:04:10,170,42
01/05/2022 00:04:20,11-01-2022 03:04:20,170,116
01/05/2022 00:04:30,11-01-2022 03:04:30,170,191
01/05/2022 00:04:40,11-01-2022 03:04:40,170,266
01/05/2022 00:04:50,11-01-2022 03:04:50,170,341
01/05/2022 00:05:00,11-01-2022 03:05:00,170,416
01/05/2022 00:05:10,11-01-2022 03:05:10,170,491
1 DATA DATE2 03CV022/CORRENTE_N_M1_PV(Value) 303-WIT-230(Value)
2 01/05/2022 00:00:00 07-01-2022 01:00:00 170 33
3 01/05/2022 00:00:10 07-01-2022 01:00:10 169 605
4 01/05/2022 00:00:20 07-01-2022 01:00:20 169 178
5 01/05/2022 00:00:30 07-01-2022 01:00:30 166 468
6 01/05/2022 00:00:40 07-01-2022 01:00:40 162 136
7 01/05/2022 00:00:50 07-01-2022 01:00:50 157 804
8 01/05/2022 00:01:00 07-01-2022 01:01:00 155 883
9 01/05/2022 00:01:10 07-01-2022 01:01:10 155 684
10 01/05/2022 00:01:20 07-01-2022 01:01:20 155 484
11 01/05/2022 00:01:30 07-01-2022 01:01:30 155 284
12 01/05/2022 00:01:40 07-01-2022 01:01:40 155 85
13 01/05/2022 00:01:50 07-01-2022 01:01:50 154 885
14 01/05/2022 00:02:00 09-01-2022 02:02:00 154 685
15 01/05/2022 00:02:10 09-01-2022 02:02:10 154 486
16 01/05/2022 00:02:20 09-01-2022 02:02:20 154 286
17 01/05/2022 00:02:30 09-01-2022 02:02:30 154 86
18 01/05/2022 00:02:40 09-01-2022 02:02:40 152 866
19 01/05/2022 00:02:50 09-01-2022 02:02:50 150 87
20 01/05/2022 00:03:00 09-01-2022 02:03:00 148 874
21 01/05/2022 00:03:10 09-01-2022 02:03:10 148 2134
22 01/05/2022 00:03:20 09-01-2022 02:03:20 148 2068
23 01/05/2022 00:03:30 09-01-2022 02:03:30 148 2022
24 01/05/2022 00:03:40 09-01-2022 02:03:40 148 1976
25 01/05/2022 00:03:50 09-01-2022 02:03:50 154 139
26 01/05/2022 00:04:00 11-01-2022 03:04:00 165 112
27 01/05/2022 00:04:10 11-01-2022 03:04:10 170 42
28 01/05/2022 00:04:20 11-01-2022 03:04:20 170 116
29 01/05/2022 00:04:30 11-01-2022 03:04:30 170 191
30 01/05/2022 00:04:40 11-01-2022 03:04:40 170 266
31 01/05/2022 00:04:50 11-01-2022 03:04:50 170 341
32 01/05/2022 00:05:00 11-01-2022 03:05:00 170 416
33 01/05/2022 00:05:10 11-01-2022 03:05:10 170 491

647
docs/DB_CV022_WIT230.csv Normal file
View File

@@ -0,0 +1,647 @@
DATA,03CV022/CORRENTE_N_M1_PV(Value),303-WIT-230(Value)
01/05/2022 00:00:00,170,33
01/05/2022 00:00:10,169,605
01/05/2022 00:00:20,169,178
01/05/2022 00:00:30,166,468
01/05/2022 00:00:40,162,136
01/05/2022 00:00:50,157,804
01/05/2022 00:01:00,155,883
01/05/2022 00:01:10,155,684
01/05/2022 00:01:20,155,484
01/05/2022 00:01:30,155,284
01/05/2022 00:01:40,155,85
01/05/2022 00:01:50,154,885
01/05/2022 00:02:00,154,685
01/05/2022 00:02:10,154,486
01/05/2022 00:02:20,154,286
01/05/2022 00:02:30,154,86
01/05/2022 00:02:40,152,866
01/05/2022 00:02:50,150,87
01/05/2022 00:03:00,148,874
01/05/2022 00:03:10,148,2134
01/05/2022 00:03:20,148,2068
01/05/2022 00:03:30,148,2022
01/05/2022 00:03:40,148,1976
01/05/2022 00:03:50,154,139
01/05/2022 00:04:00,165,112
01/05/2022 00:04:10,170,42
01/05/2022 00:04:20,170,116
01/05/2022 00:04:30,170,191
01/05/2022 00:04:40,170,266
01/05/2022 00:04:50,170,341
01/05/2022 00:05:00,170,416
01/05/2022 00:05:10,170,491
01/05/2022 00:05:20,170,566
01/05/2022 00:05:30,170,641
01/05/2022 00:05:40,170,716
01/05/2022 00:05:50,170,79
01/05/2022 00:06:00,170,865
01/05/2022 00:06:10,170,94
01/05/2022 00:06:20,171,15
01/05/2022 00:06:30,171,9
01/05/2022 00:06:40,171,165
01/05/2022 00:06:50,171,24
01/05/2022 00:07:00,171,315
01/05/2022 00:07:10,171,39
01/05/2022 00:07:20,171,465
01/05/2022 00:07:30,171,539
01/05/2022 00:07:40,171,614
01/05/2022 00:07:50,171,689
01/05/2022 00:08:00,171,764
01/05/2022 00:08:10,171,839
01/05/2022 00:08:20,171,914
01/05/2022 00:08:30,171,989
01/05/2022 00:08:40,172,64
01/05/2022 00:08:50,172,139
01/05/2022 00:09:00,172,214
01/05/2022 00:09:10,172,288
01/05/2022 00:09:20,172,363
01/05/2022 00:09:30,172,438
01/05/2022 00:09:40,172,513
01/05/2022 00:09:50,172,588
01/05/2022 00:10:00,172,663
01/05/2022 00:10:10,172,738
01/05/2022 00:10:20,172,813
01/05/2022 00:10:30,172,888
01/05/2022 00:10:40,172,962
01/05/2022 00:10:50,172,981
01/05/2022 00:11:00,172,943
01/05/2022 00:11:10,172,905
01/05/2022 00:11:20,172,867
01/05/2022 00:11:30,172,829
01/05/2022 00:11:40,172,79
01/05/2022 00:11:50,172,752
01/05/2022 00:12:00,172,714
01/05/2022 00:12:10,172,676
01/05/2022 00:12:20,172,638
01/05/2022 00:12:30,172,6
01/05/2022 00:12:40,172,562
01/05/2022 00:12:50,172,524
01/05/2022 00:13:00,172,485
01/05/2022 00:13:10,172,447
01/05/2022 00:13:20,172,409
01/05/2022 00:13:30,172,371
01/05/2022 00:13:40,172,333
01/05/2022 00:13:50,172,295
01/05/2022 00:14:00,172,257
01/05/2022 00:14:10,172,219
01/05/2022 00:14:20,172,18
01/05/2022 00:14:30,172,142
01/05/2022 00:14:40,172,104
01/05/2022 00:14:50,172,66
01/05/2022 00:15:00,172,28
01/05/2022 00:15:10,171,99
01/05/2022 00:15:20,171,952
01/05/2022 00:15:30,171,914
01/05/2022 00:15:40,171,876
01/05/2022 00:15:50,171,837
01/05/2022 00:16:00,171,799
01/05/2022 00:16:10,171,761
01/05/2022 00:16:20,171,723
01/05/2022 00:16:30,171,685
01/05/2022 00:16:40,171,647
01/05/2022 00:16:50,171,609
01/05/2022 00:17:00,171,571
01/05/2022 00:17:10,171,533
01/05/2022 00:17:20,171,494
01/05/2022 00:17:30,171,456
01/05/2022 00:17:40,171,418
01/05/2022 00:17:50,171,38
01/05/2022 00:18:00,171,342
01/05/2022 00:18:10,171,304
01/05/2022 00:18:20,171,266
01/05/2022 00:18:30,171,228
01/05/2022 00:18:40,171,189
01/05/2022 00:18:50,171,151
01/05/2022 00:19:00,171,113
01/05/2022 00:19:10,171,75
01/05/2022 00:19:20,171,37
01/05/2022 00:19:30,170,999
01/05/2022 00:19:40,170,961
01/05/2022 00:19:50,170,923
01/05/2022 00:20:00,170,884
01/05/2022 00:20:10,170,846
01/05/2022 00:20:20,170,808
01/05/2022 00:20:30,170,77
01/05/2022 00:20:40,170,732
01/05/2022 00:20:50,170,694
01/05/2022 00:21:00,170,656
01/05/2022 00:21:10,170,618
01/05/2022 00:21:20,170,58
01/05/2022 00:21:30,170,541
01/05/2022 00:21:40,170,503
01/05/2022 00:21:50,170,465
01/05/2022 00:22:00,170,427
01/05/2022 00:22:10,170,389
01/05/2022 00:22:20,170,351
01/05/2022 00:22:30,170,313
01/05/2022 00:22:40,170,275
01/05/2022 00:22:50,170,236
01/05/2022 00:23:00,170,198
01/05/2022 00:23:10,170,16
01/05/2022 00:23:20,170,122
01/05/2022 00:23:30,170,84
01/05/2022 00:23:40,170,46
01/05/2022 00:23:50,170,8
01/05/2022 00:24:00,169,97
01/05/2022 00:24:10,169,932
01/05/2022 00:24:20,169,893
01/05/2022 00:24:30,169,855
01/05/2022 00:24:40,169,817
01/05/2022 00:24:50,169,779
01/05/2022 00:25:00,169,741
01/05/2022 00:25:10,169,703
01/05/2022 00:25:20,169,665
01/05/2022 00:25:30,169,627
01/05/2022 00:25:40,169,588
01/05/2022 00:25:50,169,55
01/05/2022 00:26:00,169,512
01/05/2022 00:26:10,169,474
01/05/2022 00:26:20,169,436
01/05/2022 00:26:30,169,398
01/05/2022 00:26:40,169,36
01/05/2022 00:26:50,169,322
01/05/2022 00:27:00,169,284
01/05/2022 00:27:10,169,245
01/05/2022 00:27:20,169,207
01/05/2022 00:27:30,169,169
01/05/2022 00:27:40,169,131
01/05/2022 00:27:50,169,93
01/05/2022 00:28:00,169,55
01/05/2022 00:28:10,169,17
01/05/2022 00:28:20,168,979
01/05/2022 00:28:30,168,94
01/05/2022 00:28:40,168,902
01/05/2022 00:28:50,168,864
01/05/2022 00:29:00,168,826
01/05/2022 00:29:10,168,788
01/05/2022 00:29:20,168,75
01/05/2022 00:29:30,168,712
01/05/2022 00:29:40,168,674
01/05/2022 00:29:50,168,636
01/05/2022 00:30:00,168,597
01/05/2022 00:30:10,168,559
01/05/2022 00:30:20,168,521
01/05/2022 00:30:30,168,483
01/05/2022 00:30:40,168,445
01/05/2022 00:30:50,168,407
01/05/2022 00:31:00,168,369
01/05/2022 00:31:10,168,331
01/05/2022 00:31:20,168,292
01/05/2022 00:31:30,168,254
01/05/2022 00:31:40,168,216
01/05/2022 00:31:50,168,178
01/05/2022 00:32:00,168,14
01/05/2022 00:32:10,168,102
01/05/2022 00:32:20,168,64
01/05/2022 00:32:30,168,26
01/05/2022 00:32:40,168,26
01/05/2022 00:32:50,168,105
01/05/2022 00:33:00,168,183
01/05/2022 00:33:10,168,262
01/05/2022 00:33:20,168,341
01/05/2022 00:33:30,168,42
01/05/2022 00:33:40,168,499
01/05/2022 00:33:50,168,578
01/05/2022 00:34:00,168,657
01/05/2022 00:34:10,168,735
01/05/2022 00:34:20,168,814
01/05/2022 00:34:30,168,893
01/05/2022 00:34:40,168,972
01/05/2022 00:34:50,169,51
01/05/2022 00:35:00,169,13
01/05/2022 00:35:10,169,209
01/05/2022 00:35:20,169,287
01/05/2022 00:35:30,169,366
01/05/2022 00:35:40,169,445
01/05/2022 00:35:50,169,524
01/05/2022 00:36:00,169,603
01/05/2022 00:36:10,169,682
01/05/2022 00:36:20,169,761
01/05/2022 00:36:30,169,839
01/05/2022 00:36:40,169,918
01/05/2022 00:36:50,169,997
01/05/2022 00:37:00,170,76
01/05/2022 00:37:10,170,155
01/05/2022 00:37:20,170,234
01/05/2022 00:37:30,170,312
01/05/2022 00:37:40,170,391
01/05/2022 00:37:50,170,47
01/05/2022 00:38:00,170,549
01/05/2022 00:38:10,170,628
01/05/2022 00:38:20,170,707
01/05/2022 00:38:30,170,786
01/05/2022 00:38:40,170,864
01/05/2022 00:38:50,170,943
01/05/2022 00:39:00,170,993
01/05/2022 00:39:10,170,967
01/05/2022 00:39:20,170,942
01/05/2022 00:39:30,170,917
01/05/2022 00:39:40,170,891
01/05/2022 00:39:50,170,866
01/05/2022 00:40:00,170,841
01/05/2022 00:40:10,170,815
01/05/2022 00:40:20,170,79
01/05/2022 00:40:30,170,764
01/05/2022 00:40:40,170,739
01/05/2022 00:40:50,170,714
01/05/2022 00:41:00,170,688
01/05/2022 00:41:10,170,663
01/05/2022 00:41:20,170,637
01/05/2022 00:41:30,170,612
01/05/2022 00:41:40,170,587
01/05/2022 00:41:50,170,561
01/05/2022 00:42:00,170,536
01/05/2022 00:42:10,170,51
01/05/2022 00:42:20,170,485
01/05/2022 00:42:30,170,46
01/05/2022 00:42:40,170,434
01/05/2022 00:42:50,170,409
01/05/2022 00:43:00,170,384
01/05/2022 00:43:10,170,358
01/05/2022 00:43:20,170,333
01/05/2022 00:43:30,170,307
01/05/2022 00:43:40,170,282
01/05/2022 00:43:50,170,257
01/05/2022 00:44:00,170,231
01/05/2022 00:44:10,170,206
01/05/2022 00:44:20,170,18
01/05/2022 00:44:30,170,155
01/05/2022 00:44:40,170,13
01/05/2022 00:44:50,170,104
01/05/2022 00:45:00,170,79
01/05/2022 00:45:10,170,54
01/05/2022 00:45:20,170,28
01/05/2022 00:45:30,170,3
01/05/2022 00:45:40,169,977
01/05/2022 00:45:50,169,952
01/05/2022 00:46:00,169,927
01/05/2022 00:46:10,169,901
01/05/2022 00:46:20,169,876
01/05/2022 00:46:30,169,85
01/05/2022 00:46:40,169,825
01/05/2022 00:46:50,169,8
01/05/2022 00:47:00,169,774
01/05/2022 00:47:10,169,749
01/05/2022 00:47:20,169,723
01/05/2022 00:47:30,169,698
01/05/2022 00:47:40,169,673
01/05/2022 00:47:50,169,647
01/05/2022 00:48:00,169,622
01/05/2022 00:48:10,169,597
01/05/2022 00:48:20,169,571
01/05/2022 00:48:30,169,546
01/05/2022 00:48:40,169,52
01/05/2022 00:48:50,169,495
01/05/2022 00:49:00,169,47
01/05/2022 00:49:10,169,444
01/05/2022 00:49:20,169,419
01/05/2022 00:49:30,169,393
01/05/2022 00:49:40,169,368
01/05/2022 00:49:50,169,343
01/05/2022 00:50:00,169,317
01/05/2022 00:50:10,169,292
01/05/2022 00:50:20,169,266
01/05/2022 00:50:30,169,241
01/05/2022 00:50:40,169,216
01/05/2022 00:50:50,169,19
01/05/2022 00:51:00,169,165
01/05/2022 00:51:10,169,14
01/05/2022 00:51:20,169,114
01/05/2022 00:51:30,169,89
01/05/2022 00:51:40,169,63
01/05/2022 00:51:50,169,38
01/05/2022 00:52:00,169,13
01/05/2022 00:52:10,168,987
01/05/2022 00:52:20,168,962
01/05/2022 00:52:30,168,936
01/05/2022 00:52:40,168,911
01/05/2022 00:52:50,168,886
01/05/2022 00:53:00,168,86
01/05/2022 00:53:10,168,835
01/05/2022 00:53:20,168,809
01/05/2022 00:53:30,168,784
01/05/2022 00:53:40,168,759
01/05/2022 00:53:50,168,733
01/05/2022 00:54:00,168,708
01/05/2022 00:54:10,168,683
01/05/2022 00:54:20,168,657
01/05/2022 00:54:30,168,632
01/05/2022 00:54:40,168,606
01/05/2022 00:54:50,168,581
01/05/2022 00:55:00,168,556
01/05/2022 00:55:10,168,53
01/05/2022 00:55:20,168,505
01/05/2022 00:55:30,168,479
01/05/2022 00:55:40,168,454
01/05/2022 00:55:50,168,429
01/05/2022 00:56:00,168,403
01/05/2022 00:56:10,168,378
01/05/2022 00:56:20,168,352
01/05/2022 00:56:30,168,327
01/05/2022 00:56:40,168,302
01/05/2022 00:56:50,168,276
01/05/2022 00:57:00,168,251
01/05/2022 00:57:10,168,226
01/05/2022 00:57:20,168,2
01/05/2022 00:57:30,168,175
01/05/2022 00:57:40,168,149
01/05/2022 00:57:50,168,124
01/05/2022 00:58:00,168,99
01/05/2022 00:58:10,168,73
01/05/2022 00:58:20,168,48
01/05/2022 00:58:30,168,22
01/05/2022 00:58:40,167,76
01/05/2022 00:58:50,160,151
01/05/2022 00:59:00,161,484
01/05/2022 00:59:10,162,817
01/05/2022 00:59:20,164,281
01/05/2022 00:59:30,166,777
01/05/2022 00:59:40,167,148
01/05/2022 00:59:50,148,45
01/05/2022 01:00:00,117,525
01/05/2022 01:00:10,105,3
01/05/2022 01:00:20,105,315
01/05/2022 01:00:30,105,6
01/05/2022 01:00:40,105,886
01/05/2022 01:00:50,106,171
01/05/2022 01:01:00,106,456
01/05/2022 01:01:10,106,742
01/05/2022 01:01:20,107,698
01/05/2022 01:01:30,115,81
01/05/2022 01:01:40,122,464
01/05/2022 01:01:50,129,847
01/05/2022 01:02:00,137,231
01/05/2022 01:02:10,144,138
01/05/2022 01:02:20,145,801
01/05/2022 01:02:30,147,464
01/05/2022 01:02:40,149,835
01/05/2022 01:02:50,160,808
01/05/2022 01:03:00,171,2
01/05/2022 01:03:10,171,36
01/05/2022 01:03:20,171,7
01/05/2022 01:03:30,171,103
01/05/2022 01:03:40,171,137
01/05/2022 01:03:50,171,171
01/05/2022 01:04:00,171,204
01/05/2022 01:04:10,171,238
01/05/2022 01:04:20,171,272
01/05/2022 01:04:30,171,305
01/05/2022 01:04:40,171,339
01/05/2022 01:04:50,171,372
01/05/2022 01:05:00,171,406
01/05/2022 01:05:10,171,44
01/05/2022 01:05:20,171,473
01/05/2022 01:05:30,171,507
01/05/2022 01:05:40,171,541
01/05/2022 01:05:50,171,574
01/05/2022 01:06:00,171,608
01/05/2022 01:06:10,171,642
01/05/2022 01:06:20,171,675
01/05/2022 01:06:30,171,709
01/05/2022 01:06:40,171,743
01/05/2022 01:06:50,171,776
01/05/2022 01:07:00,171,81
01/05/2022 01:07:10,171,844
01/05/2022 01:07:20,171,877
01/05/2022 01:07:30,171,911
01/05/2022 01:07:40,171,944
01/05/2022 01:07:50,171,978
01/05/2022 01:08:00,172,12
01/05/2022 01:08:10,172,45
01/05/2022 01:08:20,172,79
01/05/2022 01:08:30,172,113
01/05/2022 01:08:40,172,146
01/05/2022 01:08:50,172,18
01/05/2022 01:09:00,172,214
01/05/2022 01:09:10,172,247
01/05/2022 01:09:20,172,281
01/05/2022 01:09:30,172,315
01/05/2022 01:09:40,172,348
01/05/2022 01:09:50,172,382
01/05/2022 01:10:00,172,415
01/05/2022 01:10:10,172,449
01/05/2022 01:10:20,172,483
01/05/2022 01:10:30,172,516
01/05/2022 01:10:40,172,55
01/05/2022 01:10:50,172,584
01/05/2022 01:11:00,172,617
01/05/2022 01:11:10,172,651
01/05/2022 01:11:20,172,685
01/05/2022 01:11:30,172,718
01/05/2022 01:11:40,172,752
01/05/2022 01:11:50,172,786
01/05/2022 01:12:00,172,819
01/05/2022 01:12:10,172,853
01/05/2022 01:12:20,172,887
01/05/2022 01:12:30,172,92
01/05/2022 01:12:40,172,954
01/05/2022 01:12:50,172,987
01/05/2022 01:13:00,173,21
01/05/2022 01:13:10,173,55
01/05/2022 01:13:20,173,88
01/05/2022 01:13:30,173,122
01/05/2022 01:13:40,173,156
01/05/2022 01:13:50,173,189
01/05/2022 01:14:00,173,223
01/05/2022 01:14:10,173,257
01/05/2022 01:14:20,173,29
01/05/2022 01:14:30,173,324
01/05/2022 01:14:40,173,358
01/05/2022 01:14:50,173,391
01/05/2022 01:15:00,173,425
01/05/2022 01:15:10,173,458
01/05/2022 01:15:20,173,492
01/05/2022 01:15:30,173,526
01/05/2022 01:15:40,173,559
01/05/2022 01:15:50,173,593
01/05/2022 01:16:00,173,627
01/05/2022 01:16:10,173,66
01/05/2022 01:16:20,173,694
01/05/2022 01:16:30,173,728
01/05/2022 01:16:40,173,761
01/05/2022 01:16:50,173,795
01/05/2022 01:17:00,173,829
01/05/2022 01:17:10,173,862
01/05/2022 01:17:20,173,896
01/05/2022 01:17:30,173,93
01/05/2022 01:17:40,173,963
01/05/2022 01:17:50,173,997
01/05/2022 01:18:00,174,103
01/05/2022 01:18:10,174,218
01/05/2022 01:18:20,174,332
01/05/2022 01:18:30,174,446
01/05/2022 01:18:40,174,56
01/05/2022 01:18:50,174,674
01/05/2022 01:19:00,174,788
01/05/2022 01:19:10,174,902
01/05/2022 01:19:20,175,17
01/05/2022 01:19:30,175,131
01/05/2022 01:19:40,175,245
01/05/2022 01:19:50,175,359
01/05/2022 01:20:00,175,473
01/05/2022 01:20:10,175,587
01/05/2022 01:20:20,175,701
01/05/2022 01:20:30,175,816
01/05/2022 01:20:40,175,93
01/05/2022 01:20:50,176,44
01/05/2022 01:21:00,176,158
01/05/2022 01:21:10,176,272
01/05/2022 01:21:20,176,386
01/05/2022 01:21:30,176,501
01/05/2022 01:21:40,176,615
01/05/2022 01:21:50,176,729
01/05/2022 01:22:00,176,843
01/05/2022 01:22:10,176,957
01/05/2022 01:22:20,177,71
01/05/2022 01:22:30,177,185
01/05/2022 01:22:40,177,3
01/05/2022 01:22:50,177,414
01/05/2022 01:23:00,177,528
01/05/2022 01:23:10,177,642
01/05/2022 01:23:20,177,756
01/05/2022 01:23:30,177,87
01/05/2022 01:23:40,177,984
01/05/2022 01:23:50,178,7
01/05/2022 01:24:00,178,15
01/05/2022 01:24:10,178,24
01/05/2022 01:24:20,178,32
01/05/2022 01:24:30,178,4
01/05/2022 01:24:40,178,48
01/05/2022 01:24:50,178,57
01/05/2022 01:25:00,178,65
01/05/2022 01:25:10,178,73
01/05/2022 01:25:20,178,81
01/05/2022 01:25:30,178,9
01/05/2022 01:25:40,178,98
01/05/2022 01:25:50,178,106
01/05/2022 01:26:00,178,114
01/05/2022 01:26:10,178,123
01/05/2022 01:26:20,178,131
01/05/2022 01:26:30,178,139
01/05/2022 01:26:40,178,147
01/05/2022 01:26:50,178,156
01/05/2022 01:27:00,178,164
01/05/2022 01:27:10,178,172
01/05/2022 01:27:20,178,18
01/05/2022 01:27:30,178,189
01/05/2022 01:27:40,178,197
01/05/2022 01:27:50,178,205
01/05/2022 01:28:00,178,213
01/05/2022 01:28:10,178,222
01/05/2022 01:28:20,178,23
01/05/2022 01:28:30,178,238
01/05/2022 01:28:40,178,246
01/05/2022 01:28:50,178,255
01/05/2022 01:29:00,178,263
01/05/2022 01:29:10,178,271
01/05/2022 01:29:20,178,279
01/05/2022 01:29:30,178,288
01/05/2022 01:29:40,178,296
01/05/2022 01:29:50,178,304
01/05/2022 01:30:00,178,312
01/05/2022 01:30:10,178,321
01/05/2022 01:30:20,178,329
01/05/2022 01:30:30,178,337
01/05/2022 01:30:40,178,345
01/05/2022 01:30:50,178,354
01/05/2022 01:31:00,178,362
01/05/2022 01:31:10,178,37
01/05/2022 01:31:20,178,378
01/05/2022 01:31:30,178,387
01/05/2022 01:31:40,178,395
01/05/2022 01:31:50,178,403
01/05/2022 01:32:00,178,411
01/05/2022 01:32:10,178,42
01/05/2022 01:32:20,178,428
01/05/2022 01:32:30,178,436
01/05/2022 01:32:40,178,444
01/05/2022 01:32:50,178,453
01/05/2022 01:33:00,178,461
01/05/2022 01:33:10,178,469
01/05/2022 01:33:20,178,477
01/05/2022 01:33:30,178,486
01/05/2022 01:33:40,178,494
01/05/2022 01:33:50,178,502
01/05/2022 01:34:00,178,51
01/05/2022 01:34:10,178,519
01/05/2022 01:34:20,178,527
01/05/2022 01:34:30,178,535
01/05/2022 01:34:40,178,543
01/05/2022 01:34:50,178,552
01/05/2022 01:35:00,178,56
01/05/2022 01:35:10,178,568
01/05/2022 01:35:20,178,576
01/05/2022 01:35:30,178,585
01/05/2022 01:35:40,178,593
01/05/2022 01:35:50,178,601
01/05/2022 01:36:00,178,609
01/05/2022 01:36:10,178,618
01/05/2022 01:36:20,178,626
01/05/2022 01:36:30,178,634
01/05/2022 01:36:40,178,642
01/05/2022 01:36:50,178,651
01/05/2022 01:37:00,178,659
01/05/2022 01:37:10,178,667
01/05/2022 01:37:20,178,675
01/05/2022 01:37:30,178,684
01/05/2022 01:37:40,178,692
01/05/2022 01:37:50,178,7
01/05/2022 01:38:00,178,708
01/05/2022 01:38:10,178,717
01/05/2022 01:38:20,178,725
01/05/2022 01:38:30,178,733
01/05/2022 01:38:40,178,741
01/05/2022 01:38:50,178,75
01/05/2022 01:39:00,178,758
01/05/2022 01:39:10,178,766
01/05/2022 01:39:20,178,774
01/05/2022 01:39:30,178,783
01/05/2022 01:39:40,178,791
01/05/2022 01:39:50,178,799
01/05/2022 01:40:00,178,807
01/05/2022 01:40:10,178,816
01/05/2022 01:40:20,178,824
01/05/2022 01:40:30,178,832
01/05/2022 01:40:40,178,84
01/05/2022 01:40:50,178,849
01/05/2022 01:41:00,178,857
01/05/2022 01:41:10,178,865
01/05/2022 01:41:20,178,873
01/05/2022 01:41:30,178,882
01/05/2022 01:41:40,178,89
01/05/2022 01:41:50,178,898
01/05/2022 01:42:00,178,906
01/05/2022 01:42:10,178,915
01/05/2022 01:42:20,178,923
01/05/2022 01:42:30,178,931
01/05/2022 01:42:40,178,939
01/05/2022 01:42:50,178,948
01/05/2022 01:43:00,178,956
01/05/2022 01:43:10,178,964
01/05/2022 01:43:20,178,972
01/05/2022 01:43:30,178,981
01/05/2022 01:43:40,178,989
01/05/2022 01:43:50,178,997
01/05/2022 01:44:00,169,147
01/05/2022 01:44:10,148,934
01/05/2022 01:44:20,119,458
01/05/2022 01:44:30,108,326
01/05/2022 01:44:40,108,825
01/05/2022 01:44:50,109,324
01/05/2022 01:45:00,109,824
01/05/2022 01:45:10,110,323
01/05/2022 01:45:20,110,822
01/05/2022 01:45:30,123,3
01/05/2022 01:45:40,141,628
01/05/2022 01:45:50,160,252
01/05/2022 01:46:00,169,863
01/05/2022 01:46:10,174,353
01/05/2022 01:46:20,176,8
01/05/2022 01:46:30,176,19
01/05/2022 01:46:40,176,31
01/05/2022 01:46:50,176,43
01/05/2022 01:47:00,176,55
01/05/2022 01:47:10,176,67
01/05/2022 01:47:20,176,79
01/05/2022 01:47:30,176,91
1 DATA 03CV022/CORRENTE_N_M1_PV(Value) 303-WIT-230(Value)
2 01/05/2022 00:00:00 170 33
3 01/05/2022 00:00:10 169 605
4 01/05/2022 00:00:20 169 178
5 01/05/2022 00:00:30 166 468
6 01/05/2022 00:00:40 162 136
7 01/05/2022 00:00:50 157 804
8 01/05/2022 00:01:00 155 883
9 01/05/2022 00:01:10 155 684
10 01/05/2022 00:01:20 155 484
11 01/05/2022 00:01:30 155 284
12 01/05/2022 00:01:40 155 85
13 01/05/2022 00:01:50 154 885
14 01/05/2022 00:02:00 154 685
15 01/05/2022 00:02:10 154 486
16 01/05/2022 00:02:20 154 286
17 01/05/2022 00:02:30 154 86
18 01/05/2022 00:02:40 152 866
19 01/05/2022 00:02:50 150 87
20 01/05/2022 00:03:00 148 874
21 01/05/2022 00:03:10 148 2134
22 01/05/2022 00:03:20 148 2068
23 01/05/2022 00:03:30 148 2022
24 01/05/2022 00:03:40 148 1976
25 01/05/2022 00:03:50 154 139
26 01/05/2022 00:04:00 165 112
27 01/05/2022 00:04:10 170 42
28 01/05/2022 00:04:20 170 116
29 01/05/2022 00:04:30 170 191
30 01/05/2022 00:04:40 170 266
31 01/05/2022 00:04:50 170 341
32 01/05/2022 00:05:00 170 416
33 01/05/2022 00:05:10 170 491
34 01/05/2022 00:05:20 170 566
35 01/05/2022 00:05:30 170 641
36 01/05/2022 00:05:40 170 716
37 01/05/2022 00:05:50 170 79
38 01/05/2022 00:06:00 170 865
39 01/05/2022 00:06:10 170 94
40 01/05/2022 00:06:20 171 15
41 01/05/2022 00:06:30 171 9
42 01/05/2022 00:06:40 171 165
43 01/05/2022 00:06:50 171 24
44 01/05/2022 00:07:00 171 315
45 01/05/2022 00:07:10 171 39
46 01/05/2022 00:07:20 171 465
47 01/05/2022 00:07:30 171 539
48 01/05/2022 00:07:40 171 614
49 01/05/2022 00:07:50 171 689
50 01/05/2022 00:08:00 171 764
51 01/05/2022 00:08:10 171 839
52 01/05/2022 00:08:20 171 914
53 01/05/2022 00:08:30 171 989
54 01/05/2022 00:08:40 172 64
55 01/05/2022 00:08:50 172 139
56 01/05/2022 00:09:00 172 214
57 01/05/2022 00:09:10 172 288
58 01/05/2022 00:09:20 172 363
59 01/05/2022 00:09:30 172 438
60 01/05/2022 00:09:40 172 513
61 01/05/2022 00:09:50 172 588
62 01/05/2022 00:10:00 172 663
63 01/05/2022 00:10:10 172 738
64 01/05/2022 00:10:20 172 813
65 01/05/2022 00:10:30 172 888
66 01/05/2022 00:10:40 172 962
67 01/05/2022 00:10:50 172 981
68 01/05/2022 00:11:00 172 943
69 01/05/2022 00:11:10 172 905
70 01/05/2022 00:11:20 172 867
71 01/05/2022 00:11:30 172 829
72 01/05/2022 00:11:40 172 79
73 01/05/2022 00:11:50 172 752
74 01/05/2022 00:12:00 172 714
75 01/05/2022 00:12:10 172 676
76 01/05/2022 00:12:20 172 638
77 01/05/2022 00:12:30 172 6
78 01/05/2022 00:12:40 172 562
79 01/05/2022 00:12:50 172 524
80 01/05/2022 00:13:00 172 485
81 01/05/2022 00:13:10 172 447
82 01/05/2022 00:13:20 172 409
83 01/05/2022 00:13:30 172 371
84 01/05/2022 00:13:40 172 333
85 01/05/2022 00:13:50 172 295
86 01/05/2022 00:14:00 172 257
87 01/05/2022 00:14:10 172 219
88 01/05/2022 00:14:20 172 18
89 01/05/2022 00:14:30 172 142
90 01/05/2022 00:14:40 172 104
91 01/05/2022 00:14:50 172 66
92 01/05/2022 00:15:00 172 28
93 01/05/2022 00:15:10 171 99
94 01/05/2022 00:15:20 171 952
95 01/05/2022 00:15:30 171 914
96 01/05/2022 00:15:40 171 876
97 01/05/2022 00:15:50 171 837
98 01/05/2022 00:16:00 171 799
99 01/05/2022 00:16:10 171 761
100 01/05/2022 00:16:20 171 723
101 01/05/2022 00:16:30 171 685
102 01/05/2022 00:16:40 171 647
103 01/05/2022 00:16:50 171 609
104 01/05/2022 00:17:00 171 571
105 01/05/2022 00:17:10 171 533
106 01/05/2022 00:17:20 171 494
107 01/05/2022 00:17:30 171 456
108 01/05/2022 00:17:40 171 418
109 01/05/2022 00:17:50 171 38
110 01/05/2022 00:18:00 171 342
111 01/05/2022 00:18:10 171 304
112 01/05/2022 00:18:20 171 266
113 01/05/2022 00:18:30 171 228
114 01/05/2022 00:18:40 171 189
115 01/05/2022 00:18:50 171 151
116 01/05/2022 00:19:00 171 113
117 01/05/2022 00:19:10 171 75
118 01/05/2022 00:19:20 171 37
119 01/05/2022 00:19:30 170 999
120 01/05/2022 00:19:40 170 961
121 01/05/2022 00:19:50 170 923
122 01/05/2022 00:20:00 170 884
123 01/05/2022 00:20:10 170 846
124 01/05/2022 00:20:20 170 808
125 01/05/2022 00:20:30 170 77
126 01/05/2022 00:20:40 170 732
127 01/05/2022 00:20:50 170 694
128 01/05/2022 00:21:00 170 656
129 01/05/2022 00:21:10 170 618
130 01/05/2022 00:21:20 170 58
131 01/05/2022 00:21:30 170 541
132 01/05/2022 00:21:40 170 503
133 01/05/2022 00:21:50 170 465
134 01/05/2022 00:22:00 170 427
135 01/05/2022 00:22:10 170 389
136 01/05/2022 00:22:20 170 351
137 01/05/2022 00:22:30 170 313
138 01/05/2022 00:22:40 170 275
139 01/05/2022 00:22:50 170 236
140 01/05/2022 00:23:00 170 198
141 01/05/2022 00:23:10 170 16
142 01/05/2022 00:23:20 170 122
143 01/05/2022 00:23:30 170 84
144 01/05/2022 00:23:40 170 46
145 01/05/2022 00:23:50 170 8
146 01/05/2022 00:24:00 169 97
147 01/05/2022 00:24:10 169 932
148 01/05/2022 00:24:20 169 893
149 01/05/2022 00:24:30 169 855
150 01/05/2022 00:24:40 169 817
151 01/05/2022 00:24:50 169 779
152 01/05/2022 00:25:00 169 741
153 01/05/2022 00:25:10 169 703
154 01/05/2022 00:25:20 169 665
155 01/05/2022 00:25:30 169 627
156 01/05/2022 00:25:40 169 588
157 01/05/2022 00:25:50 169 55
158 01/05/2022 00:26:00 169 512
159 01/05/2022 00:26:10 169 474
160 01/05/2022 00:26:20 169 436
161 01/05/2022 00:26:30 169 398
162 01/05/2022 00:26:40 169 36
163 01/05/2022 00:26:50 169 322
164 01/05/2022 00:27:00 169 284
165 01/05/2022 00:27:10 169 245
166 01/05/2022 00:27:20 169 207
167 01/05/2022 00:27:30 169 169
168 01/05/2022 00:27:40 169 131
169 01/05/2022 00:27:50 169 93
170 01/05/2022 00:28:00 169 55
171 01/05/2022 00:28:10 169 17
172 01/05/2022 00:28:20 168 979
173 01/05/2022 00:28:30 168 94
174 01/05/2022 00:28:40 168 902
175 01/05/2022 00:28:50 168 864
176 01/05/2022 00:29:00 168 826
177 01/05/2022 00:29:10 168 788
178 01/05/2022 00:29:20 168 75
179 01/05/2022 00:29:30 168 712
180 01/05/2022 00:29:40 168 674
181 01/05/2022 00:29:50 168 636
182 01/05/2022 00:30:00 168 597
183 01/05/2022 00:30:10 168 559
184 01/05/2022 00:30:20 168 521
185 01/05/2022 00:30:30 168 483
186 01/05/2022 00:30:40 168 445
187 01/05/2022 00:30:50 168 407
188 01/05/2022 00:31:00 168 369
189 01/05/2022 00:31:10 168 331
190 01/05/2022 00:31:20 168 292
191 01/05/2022 00:31:30 168 254
192 01/05/2022 00:31:40 168 216
193 01/05/2022 00:31:50 168 178
194 01/05/2022 00:32:00 168 14
195 01/05/2022 00:32:10 168 102
196 01/05/2022 00:32:20 168 64
197 01/05/2022 00:32:30 168 26
198 01/05/2022 00:32:40 168 26
199 01/05/2022 00:32:50 168 105
200 01/05/2022 00:33:00 168 183
201 01/05/2022 00:33:10 168 262
202 01/05/2022 00:33:20 168 341
203 01/05/2022 00:33:30 168 42
204 01/05/2022 00:33:40 168 499
205 01/05/2022 00:33:50 168 578
206 01/05/2022 00:34:00 168 657
207 01/05/2022 00:34:10 168 735
208 01/05/2022 00:34:20 168 814
209 01/05/2022 00:34:30 168 893
210 01/05/2022 00:34:40 168 972
211 01/05/2022 00:34:50 169 51
212 01/05/2022 00:35:00 169 13
213 01/05/2022 00:35:10 169 209
214 01/05/2022 00:35:20 169 287
215 01/05/2022 00:35:30 169 366
216 01/05/2022 00:35:40 169 445
217 01/05/2022 00:35:50 169 524
218 01/05/2022 00:36:00 169 603
219 01/05/2022 00:36:10 169 682
220 01/05/2022 00:36:20 169 761
221 01/05/2022 00:36:30 169 839
222 01/05/2022 00:36:40 169 918
223 01/05/2022 00:36:50 169 997
224 01/05/2022 00:37:00 170 76
225 01/05/2022 00:37:10 170 155
226 01/05/2022 00:37:20 170 234
227 01/05/2022 00:37:30 170 312
228 01/05/2022 00:37:40 170 391
229 01/05/2022 00:37:50 170 47
230 01/05/2022 00:38:00 170 549
231 01/05/2022 00:38:10 170 628
232 01/05/2022 00:38:20 170 707
233 01/05/2022 00:38:30 170 786
234 01/05/2022 00:38:40 170 864
235 01/05/2022 00:38:50 170 943
236 01/05/2022 00:39:00 170 993
237 01/05/2022 00:39:10 170 967
238 01/05/2022 00:39:20 170 942
239 01/05/2022 00:39:30 170 917
240 01/05/2022 00:39:40 170 891
241 01/05/2022 00:39:50 170 866
242 01/05/2022 00:40:00 170 841
243 01/05/2022 00:40:10 170 815
244 01/05/2022 00:40:20 170 79
245 01/05/2022 00:40:30 170 764
246 01/05/2022 00:40:40 170 739
247 01/05/2022 00:40:50 170 714
248 01/05/2022 00:41:00 170 688
249 01/05/2022 00:41:10 170 663
250 01/05/2022 00:41:20 170 637
251 01/05/2022 00:41:30 170 612
252 01/05/2022 00:41:40 170 587
253 01/05/2022 00:41:50 170 561
254 01/05/2022 00:42:00 170 536
255 01/05/2022 00:42:10 170 51
256 01/05/2022 00:42:20 170 485
257 01/05/2022 00:42:30 170 46
258 01/05/2022 00:42:40 170 434
259 01/05/2022 00:42:50 170 409
260 01/05/2022 00:43:00 170 384
261 01/05/2022 00:43:10 170 358
262 01/05/2022 00:43:20 170 333
263 01/05/2022 00:43:30 170 307
264 01/05/2022 00:43:40 170 282
265 01/05/2022 00:43:50 170 257
266 01/05/2022 00:44:00 170 231
267 01/05/2022 00:44:10 170 206
268 01/05/2022 00:44:20 170 18
269 01/05/2022 00:44:30 170 155
270 01/05/2022 00:44:40 170 13
271 01/05/2022 00:44:50 170 104
272 01/05/2022 00:45:00 170 79
273 01/05/2022 00:45:10 170 54
274 01/05/2022 00:45:20 170 28
275 01/05/2022 00:45:30 170 3
276 01/05/2022 00:45:40 169 977
277 01/05/2022 00:45:50 169 952
278 01/05/2022 00:46:00 169 927
279 01/05/2022 00:46:10 169 901
280 01/05/2022 00:46:20 169 876
281 01/05/2022 00:46:30 169 85
282 01/05/2022 00:46:40 169 825
283 01/05/2022 00:46:50 169 8
284 01/05/2022 00:47:00 169 774
285 01/05/2022 00:47:10 169 749
286 01/05/2022 00:47:20 169 723
287 01/05/2022 00:47:30 169 698
288 01/05/2022 00:47:40 169 673
289 01/05/2022 00:47:50 169 647
290 01/05/2022 00:48:00 169 622
291 01/05/2022 00:48:10 169 597
292 01/05/2022 00:48:20 169 571
293 01/05/2022 00:48:30 169 546
294 01/05/2022 00:48:40 169 52
295 01/05/2022 00:48:50 169 495
296 01/05/2022 00:49:00 169 47
297 01/05/2022 00:49:10 169 444
298 01/05/2022 00:49:20 169 419
299 01/05/2022 00:49:30 169 393
300 01/05/2022 00:49:40 169 368
301 01/05/2022 00:49:50 169 343
302 01/05/2022 00:50:00 169 317
303 01/05/2022 00:50:10 169 292
304 01/05/2022 00:50:20 169 266
305 01/05/2022 00:50:30 169 241
306 01/05/2022 00:50:40 169 216
307 01/05/2022 00:50:50 169 19
308 01/05/2022 00:51:00 169 165
309 01/05/2022 00:51:10 169 14
310 01/05/2022 00:51:20 169 114
311 01/05/2022 00:51:30 169 89
312 01/05/2022 00:51:40 169 63
313 01/05/2022 00:51:50 169 38
314 01/05/2022 00:52:00 169 13
315 01/05/2022 00:52:10 168 987
316 01/05/2022 00:52:20 168 962
317 01/05/2022 00:52:30 168 936
318 01/05/2022 00:52:40 168 911
319 01/05/2022 00:52:50 168 886
320 01/05/2022 00:53:00 168 86
321 01/05/2022 00:53:10 168 835
322 01/05/2022 00:53:20 168 809
323 01/05/2022 00:53:30 168 784
324 01/05/2022 00:53:40 168 759
325 01/05/2022 00:53:50 168 733
326 01/05/2022 00:54:00 168 708
327 01/05/2022 00:54:10 168 683
328 01/05/2022 00:54:20 168 657
329 01/05/2022 00:54:30 168 632
330 01/05/2022 00:54:40 168 606
331 01/05/2022 00:54:50 168 581
332 01/05/2022 00:55:00 168 556
333 01/05/2022 00:55:10 168 53
334 01/05/2022 00:55:20 168 505
335 01/05/2022 00:55:30 168 479
336 01/05/2022 00:55:40 168 454
337 01/05/2022 00:55:50 168 429
338 01/05/2022 00:56:00 168 403
339 01/05/2022 00:56:10 168 378
340 01/05/2022 00:56:20 168 352
341 01/05/2022 00:56:30 168 327
342 01/05/2022 00:56:40 168 302
343 01/05/2022 00:56:50 168 276
344 01/05/2022 00:57:00 168 251
345 01/05/2022 00:57:10 168 226
346 01/05/2022 00:57:20 168 2
347 01/05/2022 00:57:30 168 175
348 01/05/2022 00:57:40 168 149
349 01/05/2022 00:57:50 168 124
350 01/05/2022 00:58:00 168 99
351 01/05/2022 00:58:10 168 73
352 01/05/2022 00:58:20 168 48
353 01/05/2022 00:58:30 168 22
354 01/05/2022 00:58:40 167 76
355 01/05/2022 00:58:50 160 151
356 01/05/2022 00:59:00 161 484
357 01/05/2022 00:59:10 162 817
358 01/05/2022 00:59:20 164 281
359 01/05/2022 00:59:30 166 777
360 01/05/2022 00:59:40 167 148
361 01/05/2022 00:59:50 148 45
362 01/05/2022 01:00:00 117 525
363 01/05/2022 01:00:10 105 3
364 01/05/2022 01:00:20 105 315
365 01/05/2022 01:00:30 105 6
366 01/05/2022 01:00:40 105 886
367 01/05/2022 01:00:50 106 171
368 01/05/2022 01:01:00 106 456
369 01/05/2022 01:01:10 106 742
370 01/05/2022 01:01:20 107 698
371 01/05/2022 01:01:30 115 81
372 01/05/2022 01:01:40 122 464
373 01/05/2022 01:01:50 129 847
374 01/05/2022 01:02:00 137 231
375 01/05/2022 01:02:10 144 138
376 01/05/2022 01:02:20 145 801
377 01/05/2022 01:02:30 147 464
378 01/05/2022 01:02:40 149 835
379 01/05/2022 01:02:50 160 808
380 01/05/2022 01:03:00 171 2
381 01/05/2022 01:03:10 171 36
382 01/05/2022 01:03:20 171 7
383 01/05/2022 01:03:30 171 103
384 01/05/2022 01:03:40 171 137
385 01/05/2022 01:03:50 171 171
386 01/05/2022 01:04:00 171 204
387 01/05/2022 01:04:10 171 238
388 01/05/2022 01:04:20 171 272
389 01/05/2022 01:04:30 171 305
390 01/05/2022 01:04:40 171 339
391 01/05/2022 01:04:50 171 372
392 01/05/2022 01:05:00 171 406
393 01/05/2022 01:05:10 171 44
394 01/05/2022 01:05:20 171 473
395 01/05/2022 01:05:30 171 507
396 01/05/2022 01:05:40 171 541
397 01/05/2022 01:05:50 171 574
398 01/05/2022 01:06:00 171 608
399 01/05/2022 01:06:10 171 642
400 01/05/2022 01:06:20 171 675
401 01/05/2022 01:06:30 171 709
402 01/05/2022 01:06:40 171 743
403 01/05/2022 01:06:50 171 776
404 01/05/2022 01:07:00 171 81
405 01/05/2022 01:07:10 171 844
406 01/05/2022 01:07:20 171 877
407 01/05/2022 01:07:30 171 911
408 01/05/2022 01:07:40 171 944
409 01/05/2022 01:07:50 171 978
410 01/05/2022 01:08:00 172 12
411 01/05/2022 01:08:10 172 45
412 01/05/2022 01:08:20 172 79
413 01/05/2022 01:08:30 172 113
414 01/05/2022 01:08:40 172 146
415 01/05/2022 01:08:50 172 18
416 01/05/2022 01:09:00 172 214
417 01/05/2022 01:09:10 172 247
418 01/05/2022 01:09:20 172 281
419 01/05/2022 01:09:30 172 315
420 01/05/2022 01:09:40 172 348
421 01/05/2022 01:09:50 172 382
422 01/05/2022 01:10:00 172 415
423 01/05/2022 01:10:10 172 449
424 01/05/2022 01:10:20 172 483
425 01/05/2022 01:10:30 172 516
426 01/05/2022 01:10:40 172 55
427 01/05/2022 01:10:50 172 584
428 01/05/2022 01:11:00 172 617
429 01/05/2022 01:11:10 172 651
430 01/05/2022 01:11:20 172 685
431 01/05/2022 01:11:30 172 718
432 01/05/2022 01:11:40 172 752
433 01/05/2022 01:11:50 172 786
434 01/05/2022 01:12:00 172 819
435 01/05/2022 01:12:10 172 853
436 01/05/2022 01:12:20 172 887
437 01/05/2022 01:12:30 172 92
438 01/05/2022 01:12:40 172 954
439 01/05/2022 01:12:50 172 987
440 01/05/2022 01:13:00 173 21
441 01/05/2022 01:13:10 173 55
442 01/05/2022 01:13:20 173 88
443 01/05/2022 01:13:30 173 122
444 01/05/2022 01:13:40 173 156
445 01/05/2022 01:13:50 173 189
446 01/05/2022 01:14:00 173 223
447 01/05/2022 01:14:10 173 257
448 01/05/2022 01:14:20 173 29
449 01/05/2022 01:14:30 173 324
450 01/05/2022 01:14:40 173 358
451 01/05/2022 01:14:50 173 391
452 01/05/2022 01:15:00 173 425
453 01/05/2022 01:15:10 173 458
454 01/05/2022 01:15:20 173 492
455 01/05/2022 01:15:30 173 526
456 01/05/2022 01:15:40 173 559
457 01/05/2022 01:15:50 173 593
458 01/05/2022 01:16:00 173 627
459 01/05/2022 01:16:10 173 66
460 01/05/2022 01:16:20 173 694
461 01/05/2022 01:16:30 173 728
462 01/05/2022 01:16:40 173 761
463 01/05/2022 01:16:50 173 795
464 01/05/2022 01:17:00 173 829
465 01/05/2022 01:17:10 173 862
466 01/05/2022 01:17:20 173 896
467 01/05/2022 01:17:30 173 93
468 01/05/2022 01:17:40 173 963
469 01/05/2022 01:17:50 173 997
470 01/05/2022 01:18:00 174 103
471 01/05/2022 01:18:10 174 218
472 01/05/2022 01:18:20 174 332
473 01/05/2022 01:18:30 174 446
474 01/05/2022 01:18:40 174 56
475 01/05/2022 01:18:50 174 674
476 01/05/2022 01:19:00 174 788
477 01/05/2022 01:19:10 174 902
478 01/05/2022 01:19:20 175 17
479 01/05/2022 01:19:30 175 131
480 01/05/2022 01:19:40 175 245
481 01/05/2022 01:19:50 175 359
482 01/05/2022 01:20:00 175 473
483 01/05/2022 01:20:10 175 587
484 01/05/2022 01:20:20 175 701
485 01/05/2022 01:20:30 175 816
486 01/05/2022 01:20:40 175 93
487 01/05/2022 01:20:50 176 44
488 01/05/2022 01:21:00 176 158
489 01/05/2022 01:21:10 176 272
490 01/05/2022 01:21:20 176 386
491 01/05/2022 01:21:30 176 501
492 01/05/2022 01:21:40 176 615
493 01/05/2022 01:21:50 176 729
494 01/05/2022 01:22:00 176 843
495 01/05/2022 01:22:10 176 957
496 01/05/2022 01:22:20 177 71
497 01/05/2022 01:22:30 177 185
498 01/05/2022 01:22:40 177 3
499 01/05/2022 01:22:50 177 414
500 01/05/2022 01:23:00 177 528
501 01/05/2022 01:23:10 177 642
502 01/05/2022 01:23:20 177 756
503 01/05/2022 01:23:30 177 87
504 01/05/2022 01:23:40 177 984
505 01/05/2022 01:23:50 178 7
506 01/05/2022 01:24:00 178 15
507 01/05/2022 01:24:10 178 24
508 01/05/2022 01:24:20 178 32
509 01/05/2022 01:24:30 178 4
510 01/05/2022 01:24:40 178 48
511 01/05/2022 01:24:50 178 57
512 01/05/2022 01:25:00 178 65
513 01/05/2022 01:25:10 178 73
514 01/05/2022 01:25:20 178 81
515 01/05/2022 01:25:30 178 9
516 01/05/2022 01:25:40 178 98
517 01/05/2022 01:25:50 178 106
518 01/05/2022 01:26:00 178 114
519 01/05/2022 01:26:10 178 123
520 01/05/2022 01:26:20 178 131
521 01/05/2022 01:26:30 178 139
522 01/05/2022 01:26:40 178 147
523 01/05/2022 01:26:50 178 156
524 01/05/2022 01:27:00 178 164
525 01/05/2022 01:27:10 178 172
526 01/05/2022 01:27:20 178 18
527 01/05/2022 01:27:30 178 189
528 01/05/2022 01:27:40 178 197
529 01/05/2022 01:27:50 178 205
530 01/05/2022 01:28:00 178 213
531 01/05/2022 01:28:10 178 222
532 01/05/2022 01:28:20 178 23
533 01/05/2022 01:28:30 178 238
534 01/05/2022 01:28:40 178 246
535 01/05/2022 01:28:50 178 255
536 01/05/2022 01:29:00 178 263
537 01/05/2022 01:29:10 178 271
538 01/05/2022 01:29:20 178 279
539 01/05/2022 01:29:30 178 288
540 01/05/2022 01:29:40 178 296
541 01/05/2022 01:29:50 178 304
542 01/05/2022 01:30:00 178 312
543 01/05/2022 01:30:10 178 321
544 01/05/2022 01:30:20 178 329
545 01/05/2022 01:30:30 178 337
546 01/05/2022 01:30:40 178 345
547 01/05/2022 01:30:50 178 354
548 01/05/2022 01:31:00 178 362
549 01/05/2022 01:31:10 178 37
550 01/05/2022 01:31:20 178 378
551 01/05/2022 01:31:30 178 387
552 01/05/2022 01:31:40 178 395
553 01/05/2022 01:31:50 178 403
554 01/05/2022 01:32:00 178 411
555 01/05/2022 01:32:10 178 42
556 01/05/2022 01:32:20 178 428
557 01/05/2022 01:32:30 178 436
558 01/05/2022 01:32:40 178 444
559 01/05/2022 01:32:50 178 453
560 01/05/2022 01:33:00 178 461
561 01/05/2022 01:33:10 178 469
562 01/05/2022 01:33:20 178 477
563 01/05/2022 01:33:30 178 486
564 01/05/2022 01:33:40 178 494
565 01/05/2022 01:33:50 178 502
566 01/05/2022 01:34:00 178 51
567 01/05/2022 01:34:10 178 519
568 01/05/2022 01:34:20 178 527
569 01/05/2022 01:34:30 178 535
570 01/05/2022 01:34:40 178 543
571 01/05/2022 01:34:50 178 552
572 01/05/2022 01:35:00 178 56
573 01/05/2022 01:35:10 178 568
574 01/05/2022 01:35:20 178 576
575 01/05/2022 01:35:30 178 585
576 01/05/2022 01:35:40 178 593
577 01/05/2022 01:35:50 178 601
578 01/05/2022 01:36:00 178 609
579 01/05/2022 01:36:10 178 618
580 01/05/2022 01:36:20 178 626
581 01/05/2022 01:36:30 178 634
582 01/05/2022 01:36:40 178 642
583 01/05/2022 01:36:50 178 651
584 01/05/2022 01:37:00 178 659
585 01/05/2022 01:37:10 178 667
586 01/05/2022 01:37:20 178 675
587 01/05/2022 01:37:30 178 684
588 01/05/2022 01:37:40 178 692
589 01/05/2022 01:37:50 178 7
590 01/05/2022 01:38:00 178 708
591 01/05/2022 01:38:10 178 717
592 01/05/2022 01:38:20 178 725
593 01/05/2022 01:38:30 178 733
594 01/05/2022 01:38:40 178 741
595 01/05/2022 01:38:50 178 75
596 01/05/2022 01:39:00 178 758
597 01/05/2022 01:39:10 178 766
598 01/05/2022 01:39:20 178 774
599 01/05/2022 01:39:30 178 783
600 01/05/2022 01:39:40 178 791
601 01/05/2022 01:39:50 178 799
602 01/05/2022 01:40:00 178 807
603 01/05/2022 01:40:10 178 816
604 01/05/2022 01:40:20 178 824
605 01/05/2022 01:40:30 178 832
606 01/05/2022 01:40:40 178 84
607 01/05/2022 01:40:50 178 849
608 01/05/2022 01:41:00 178 857
609 01/05/2022 01:41:10 178 865
610 01/05/2022 01:41:20 178 873
611 01/05/2022 01:41:30 178 882
612 01/05/2022 01:41:40 178 89
613 01/05/2022 01:41:50 178 898
614 01/05/2022 01:42:00 178 906
615 01/05/2022 01:42:10 178 915
616 01/05/2022 01:42:20 178 923
617 01/05/2022 01:42:30 178 931
618 01/05/2022 01:42:40 178 939
619 01/05/2022 01:42:50 178 948
620 01/05/2022 01:43:00 178 956
621 01/05/2022 01:43:10 178 964
622 01/05/2022 01:43:20 178 972
623 01/05/2022 01:43:30 178 981
624 01/05/2022 01:43:40 178 989
625 01/05/2022 01:43:50 178 997
626 01/05/2022 01:44:00 169 147
627 01/05/2022 01:44:10 148 934
628 01/05/2022 01:44:20 119 458
629 01/05/2022 01:44:30 108 326
630 01/05/2022 01:44:40 108 825
631 01/05/2022 01:44:50 109 324
632 01/05/2022 01:45:00 109 824
633 01/05/2022 01:45:10 110 323
634 01/05/2022 01:45:20 110 822
635 01/05/2022 01:45:30 123 3
636 01/05/2022 01:45:40 141 628
637 01/05/2022 01:45:50 160 252
638 01/05/2022 01:46:00 169 863
639 01/05/2022 01:46:10 174 353
640 01/05/2022 01:46:20 176 8
641 01/05/2022 01:46:30 176 19
642 01/05/2022 01:46:40 176 31
643 01/05/2022 01:46:50 176 43
644 01/05/2022 01:47:00 176 55
645 01/05/2022 01:47:10 176 67
646 01/05/2022 01:47:20 176 79
647 01/05/2022 01:47:30 176 91

37097
docs/data-1749222138290.csv Normal file

File diff suppressed because it is too large Load Diff

66
docs/scenarios.md Normal file
View File

@@ -0,0 +1,66 @@
# E2E Test Scenarios
This document maps the workflow scenarios tested in the E2E suite to their corresponding JSON input files and expected behaviors.
## Infrastructure (second-pass review)
- **Containers:** PostgreSQL, MinIO, MongoDB, and Gitea via testcontainers; real clients and `Activities` code paths.
- **MLflow:** `file://` tracking URI (real SDK, no remote server).
- **Temporal:** `WorkflowEnvironment.start_time_skipping()` — official temporalio test runtime; workflows and activities are not stubbed.
- **Logging/metrics:** `get_logger` + `MetricsController` (sientia_do); no `unittest.mock` for observability in `e2e/conftest.py`.
- **Unit tests** under `tests/` may still use mocks where appropriate; that policy is separate from this E2E suite.
## 1. TrainModel Workflow (`test_train_model_workflow.py`)
### 1.1 Happy Paths (Successful execution)
| Test Function | Input JSON | Expected Status | Description |
|---|---|---|---|
| `test_scenario_1_1_1_linear_regression_basic` | `01-linear-regression-basic.json` | `TRAINING_SUCCESS` | Basic linear regression without scaler. Verifies end-to-end pipeline. |
| `test_scenario_1_1_2_linear_regression_with_scaler` | `02-linear-regression-with-scaler.json` | `TRAINING_SUCCESS` | Linear regression with `Standard Scaler`. |
| `test_scenario_1_1_3_polynomial_regression_degree2_with_scaler` | `03-polynomial-regression-degree2.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 2) with Standard Scaler. |
| `test_scenario_1_1_4_polynomial_regression_degree3_with_scaler` | `04-polynomial-regression-degree3.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 3) with Standard Scaler. |
| `test_scenario_1_1_5_linear_regression_with_lags` | `05-linear-regression-with-lags.json` | `TRAINING_SUCCESS` | Linear regression with `lag_train`/`lag_val` per variable. |
| `test_scenario_1_1_6_linear_regression_nan_interpolation` | `06-linear-regression-nan-interpolation.json` | `TRAINING_SUCCESS` | Linear regression with `nan_treatment='linear interpolation'`. |
| `test_scenario_1_1_7_linear_regression_static_window_removal` | `07-linear-regression-static-window-removal.json` | `TRAINING_SUCCESS` | `rem_static_win=true` with default `static_threshold`. |
| `test_scenario_1_1_8_linear_regression_with_limits` | `08-linear-regression-with-limits.json` | `TRAINING_SUCCESS` | `support_filters` with `min`/`max` per variable. |
| `test_scenario_1_1_9_polynomial_degree2_scaler_and_lags` | `09-polynomial-degree2-with-scaler-and-lags.json` | `TRAINING_SUCCESS` | Polynomial (degree 2), Standard Scaler, and lags. |
| `test_scenario_1_1_10_linear_regression_with_ar_opt_params` | `10-linear-regression-with-ar.json` | `TRAINING_SUCCESS` | `opt_params.include_ar=true` (placeholder for future AR behavior). |
| `test_scenario_1_1_11_linear_regression_static_threshold_custom` | `11-linear-regression-static-threshold-custom.json` | `TRAINING_SUCCESS` | `rem_static_win=true` with custom `static_threshold`. |
| `test_scenario_1_1_12_alternate_date_format_dd_mm_yyyy` | `12-angular-test-date-format.json` | `TRAINING_SUCCESS` | `date_column=DATA`, `dd/MM/yyyy` format, object `training_data_dd_mm_yyyy.csv`. |
| `test_scenario_1_1_13_alternate_csv_narrow_date_window` | `13-angular-test-double-date-column.json` | `TRAINING_SUCCESS` | Same alternate CSV with a bounded `start_date`/`end_date` window. |
| `test_scenario_1_1_14_polynomial_with_support_filters` | `14-angular-test-polynomial-support-filters.json` | `TRAINING_SUCCESS` | Polynomial (degree 4), scaler, `upper_line`/`lower_line` support filters. |
| `test_scenario_1_1_15_linear_regression_custom_target_column_name` | `15-linear-regression-custom-target-column.json` | `TRAINING_SUCCESS` | Custom `target_variable` column name (not literal ``target``); Evidently/report columns must match. |
| `test_scenario_1_1_16_naive_timestamp_header_column` | `16-linear-regression-naive-timestamp-header.json` | `TRAINING_SUCCESS` | `date_column`=`Timestamp`, naive CSV `training_data_timestamp_naive.csv`. |
| `test_scenario_1_1_17_linear_regression_blank_timestamp_row_dropped` | `17-linear-regression-blank-timestamp-row.json` | `TRAINING_SUCCESS` | One empty timestamp cell; row dropped before index. |
### 1.2 Error Paths
| Test Function | Input JSON | Expected Status | Description |
|---|---|---|---|
| `test_scenario_1_2_1_minio_file_not_found` | `01-linear-regression-basic.json` | `TRAINING_ERROR` | MinIO file does not exist. Workflow fails during file download. |
| `test_scenario_1_2_2_experiment_run_id_not_in_db` | `01-linear-regression-basic.json` | N/A (raises Exception) | `experiment_run_id` does not exist in DB. Workflow fails immediately on status update attempt. |
## 2. Parameter Validation (`test_train_model_validation.py`)
These scenarios test the business rule validations inside `validate_train_params`. All are expected to terminate with `ORCHESTRATOR_VALIDATION_ERROR`.
| Test Function | Modification | Expected Error Substring |
|---|---|---|
| `test_scenario_2_1_1_train_size_out_of_range` | `train_size = 5` | `'train_size'` |
| `test_scenario_2_1_2_empty_variable_columns` | `variable_columns = []` | `'variable_columns'` |
| `test_scenario_2_1_3_invalid_date_format` | `date_format = 'INVALID'` | `'date_format'` |
| `test_scenario_2_1_4_whitespace_only_model_name` | `model_name = ' '` | `'model_name'` |
| `test_scenario_2_1_5_unknown_model_type` | `model_type = 'totally_unknown_model'` | `'totally_unknown_model'` |
| `test_scenario_2_1_6_missing_target_variable` | `target_variable = ''` | `'target_variable'` |
| `test_scenario_2_1_7_missing_experiment_run_id` | Missing `experiment_run_id` | N/A (raises ValueError immediately) |
| `test_scenario_2_1_8_missing_date_column` | Missing `date_column` | N/A (raises ValueError immediately) |
| `test_scenario_2_1_9_whitespace_date_column` | `date_column = ' '` | `'date_column'` |
## 3. CleanupFiles Workflow (`test_cleanup_files_workflow.py`)
| Test Function | Description |
|---|---|
| `test_scenario_3_1_1_cleanup_with_no_temp_dirs` | Temp directory is empty. Activity completes without error. |
| `test_scenario_3_1_2_cleanup_removes_old_temp_dirs` | Two stale directories matching `name_YYYYMMDD_HHMMSS_microseconds` are removed when older than retention. |
| `test_scenario_3_1_3_cleanup_nonexistent_temp_path` | Target path does not exist. Handled gracefully without error. |

37097
docs/test-model-data.csv Executable file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,40 @@
{
"_description": "Cenário básico de regressão linear sem scaler",
"experiment_run_id": 1001,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Regressão linear com Standard Scaler habilitado",
"experiment_run_id": 1002,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "Standard Scaler"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Regressão polinomial de grau 2 com scaler (obrigatório para evitar overflow)",
"experiment_run_id": 1003,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Polynomial Regression",
"model_type": "polynomial_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 2,
"interaction_only": false,
"scaler_name": "Standard Scaler"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Regressão polinomial de grau 3 com scaler",
"experiment_run_id": 1004,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Polynomial Regression",
"model_type": "polynomial_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 3,
"interaction_only": false,
"scaler_name": "Standard Scaler"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Regressão linear com lags de treino e validação",
"experiment_run_id": 1005,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 5
},
"lag_val": {
"303-WIT-200(Value)": 3
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Regressão linear com tratamento de NaN por interpolação linear",
"experiment_run_id": 1006,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "linear interpolation",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Regressão linear com remoção de janelas estáticas",
"experiment_run_id": 1007,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": true,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,45 @@
{
"_description": "Regressão linear com limites inferior e superior para variáveis",
"experiment_run_id": 1008,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {
"303-WIT-200(Value)": {
"min": 0.0,
"max": 1000.0
}
},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Cenário completo: regressão polinomial grau 2 com scaler e lags",
"experiment_run_id": 1009,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Polynomial Regression",
"model_type": "polynomial_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 3
},
"lag_val": {
"303-WIT-200(Value)": 2
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 2,
"interaction_only": false,
"scaler_name": "Standard Scaler"
},
"opt_params": {}
}

View File

@@ -0,0 +1,42 @@
{
"_description": "Linear regression placeholder for autoregressive features; include_ar is reserved for future wrapper support (see opt_params).",
"experiment_run_id": 1010,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {
"include_ar": true
}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Regressão linear com remoção de janelas estáticas e static_threshold customizado",
"experiment_run_id": 1011,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": true,
"static_threshold": 100,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Alternate date column (DATA) and dd/MM/yyyy HH:mm:ss format; uses MinIO object training_data_dd_mm_yyyy.csv from E2E fixtures.",
"experiment_run_id": 1012,
"variable_columns": [
"303-WIT-230(Value)"
],
"target_variable": "03CV022/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data_dd_mm_yyyy.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "DATA",
"date_format": "dd/MM/yyyy HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-230(Value)": 3
},
"lag_val": {
"303-WIT-230(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": "01/05/2022",
"end_date": "31/07/2022",
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Same alternate CSV as scenario 12 (DATA + dd/MM/yyyy); narrow date window for regression coverage. Not a multi-date-column dataset.",
"experiment_run_id": 1013,
"variable_columns": [
"303-WIT-230(Value)"
],
"target_variable": "03CV022/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data_dd_mm_yyyy.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "DATA",
"date_format": "dd/MM/yyyy HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-230(Value)": 0
},
"lag_val": {
"303-WIT-230(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": "01/05/2022 00:00:00",
"end_date": "31/05/2022 23:59:59",
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,51 @@
{
"_description": "Cenário angular-test-01: regressão polinomial degree 4, scaler, support filters em 303-WIT-200",
"experiment_run_id": 1014,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Polynomial Regression",
"model_type": "polynomial_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": "2025-06-02 00:00:00",
"end_date": "2025-06-08 23:59:59",
"support_filters": {
"303-WIT-200(Value)": {
"upper_line": {
"intercept": 40.400002,
"angle": 0
},
"lower_line": {
"intercept": 30.5,
"angle": 0
}
}
},
"removed_intervals": []
},
"model_kwargs": {
"degree": 4,
"interaction_only": false,
"scaler_name": "Standard Scaler"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Target column name is not ``target``; report/Evidently sections must use params.target_variable.",
"experiment_run_id": 1015,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "MY_CUSTOM_TARGET_COLUMN",
"bucket_name": "model-training",
"file_name": "training_data_custom_target.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "Naive Timestamp column header; snake_case date_column/date_format and training_data_timestamp_naive.csv.",
"experiment_run_id": 1016,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data_timestamp_naive.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "Timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,40 @@
{
"_description": "One CSV row has an empty timestamp; pipeline should drop it and continue training.",
"experiment_run_id": 1017,
"variable_columns": [
"303-WIT-200(Value)"
],
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"bucket_name": "model-training",
"file_name": "training_data_blank_timestamp_row.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"date_format": "yyyy-MM-dd HH:mm:ss",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"303-WIT-200(Value)": 0
},
"lag_val": {
"303-WIT-200(Value)": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "None"
},
"opt_params": {}
}

View File

@@ -0,0 +1,417 @@
# Train Model Workflow IO Diff (`main` vs current branch)
Base comparison: `git diff main...HEAD`
Workflow analyzed: `train_model`
## 1) Executive overview
This branch introduces a structural refactor of the training stack and a contract update for workflow input/output.
Main impacts:
- The old in-house training stack (`TrainingRepository`, `ModelRepository`, `StorageRepository`, `model_manager.sientia.models`) was replaced by:
- `DataManagerRepository` (data prep + metrics + report generation)
- `SientiaModel` wrapper from plugin store (`sientia_model`)
- `SientiaMLflowRepository` (MLflow integration)
- `MinioRepository` (storage integration)
- Input contract moved from many fixed legacy ML params to a plugin/wrapper-oriented schema (`model_type`, `*_kwargs`, `model_metadata`, optional `val_file_name`).
- Workflow return changed from `None` to a serializable result object (`dict[str, Any] | None`) containing training execution metadata.
- Queue naming and worker bootstrap architecture now depend on runtime (`train_model-<runtime>-queue`).
---
## 2) Input contract diff (before vs now)
### 2.1 Previous contract (`main`)
`TrainModelParams` in `main` required a large set of explicit fields for the old preprocessing/model pipeline, focused only in linear regression model:
- Core:
- `experiment_run_id`, `variable_columns`, `target_variable`
- `bucket_name`, `file_name`, `line_separator`, `decimal_separator`
- `train_size`, `shuffle`
- Legacy preprocessing/model fields focused only in linear regression model (required in `from_dict`):
- `lag_train`, `lag_val`
- `rem_static_win`, `low_lim`, `upp_lim`, `window`
- `use_scaler`, `include_ar`, `scaler_name`
- `removed_intervals`, `start_date`, `end_date`, `nan_treatment`
- `degree`, `interaction_only`
- `experiment_name`, `model_name`
- `support_filters` (optional dict), `static_threshold` (optional int)
Validation was strongly tied to this structure (lag ranges, limits consistency, polynomial/scaler constraints, etc.).
### 2.2 Current contract (this branch)
`TrainModelParams` now supports a plugin-driven schema and wrapper kwargs:
- Kept/mandatory core fields:
- `experiment_run_id` (now accepts numeric string too; coerced to int)
- `variable_columns`, `target_variable`
- `bucket_name`, `file_name`, `line_separator`, `decimal_separator`
- `train_size`, `shuffle`
- `model_name`
- `model_type`
- `data_model_kwargs`, `model_kwargs`, `opt_params` (required as dict by current `from_dict`)
- New/updated fields:
- `random_state` (default `42`)
- `val_file_name` (optional explicit validation file)
- `model_id` (currently optional, but needs discussion, since the model metadata in MongoDB should be created before the model training)
- Removed from required input contract:
- `lag_train`, `lag_val`, `rem_static_win`, `low_lim`, `upp_lim`, `window`
- `use_scaler`, `include_ar`
- `degree`, `interaction_only`, `nan_treatment`
- `start_date`, `end_date`, `scaler_name`
- `removed_intervals`, `support_filters`, `static_threshold`
- Parameters internally derived:
- `model_metadata` model type info from plugin store.
- `run_name` is internally derived from experiment name and datetime.
- `experiment_name` is internally derived from `model_name`.
### 2.3 Validation behavior changes
Before:
- Validation was mostly hardcoded business checks tied to legacy linear/polynomial stack.
Now:
- Validation still checks core constraints (`train_size`, non-empty strings, etc.), but model-specific validation moved to JSON Schema driven checks, using OpenAPI/JSON Schema definitions from plugin store:
- `model_metadata.schemas.components.schemas.data_model`
- `model_metadata.schemas.components.schemas.model`
- `model_metadata.schemas.components.schemas.opt_params`
- `model_metadata` is now a required semantic dependency for `validate_business_rules()`.
- Date format validation remains, but allowed formats are defined locally in `train_model_params.py`.
### 2.4 Input loading pipeline changes in workflow
Before:
- `validate_train_params` directly consumed workflow input.
Now:
1. `load_model_metadata` runs first (fetches model index/schema from plugin store and injects `model_metadata`).
2. `validate_train_params` runs with enriched payload.
This means IO preprocessing now depends on plugin-store metadata resolution before final validation.
---
## 3) Output contract diff (before vs now)
### 3.1 Workflow return (`train_model.run`)
Before (`main`):
- Return type: `None`
- Workflow side effects were persisted mainly via DB status updates and MLflow artifacts.
Now:
- Return type: `dict[str, Any] | None`
- Workflow returns the training activity summary when successful.
### 3.2 Activity-level training result payload
Before (from `Training.train_model` in `main` path):
- Returned minimal dict:
- `run_name`
- `run_dir`
Now:
- Returns extended dict:
- `run_name`
- `experiment_name`
- `run_id`
- `run_dir`
### 3.3 Persistence map by destination (DB, MLflow, MinIO, local filesystem)
This section maps where each artifact/metadata goes, in which format, and how that changed from `main`.
#### 3.3.1 PostgreSQL (`experiment_run` table)
## Before (`main`)
- Update path: `update_experiment_run` activity with `UpdateType.MODEL_SAVED`.
- Persisted on success:
- `status` transition to `TRAINING_SUCCESS`
- `run_name` (MLflow run identifier used by current implementation)
- Persisted on failures:
- `status` transition to validation/training error statuses
- `error_message`
## Now (current branch)
- Same update path and status/error behavior.
- Even though train activity now returns more metadata (`run_id`, `experiment_name`), current workflow update for `MODEL_SAVED` still forwards mainly `run_name`.
- Practical effect:
- DB remains status-centric and run-name-centric
- richer identifiers exist in workflow return payload, not fully mirrored to DB columns in current flow
#### 3.3.2 MLflow (tracking server/artifact store)
## Before (`main`)
- Persistence orchestration lived in `ModelRepository.save_model()` + `_save_run()`.
- Typical persisted content:
- model params (many legacy params such as lags, limits, scaler config, removed intervals)
- regression metrics (`MSE`, `R2`, `MAE`)
- model objects:
- `data_model`
- `prediction_model`
- artifacts:
- `report.html`
- `train_data.csv`
- `test_data.csv`
- optional `model_equation.json`
- Run naming:
- computed by querying existing runs and appending sequence (`<experiment>-<n>` style)
## Now (current branch)
- Persistence orchestrated in `Training._persist_training_artifacts()` and MLflow run context is opened by `SientiaMLflowRepository.start_run(...)`.
- Persisted content now:
- model wrapper itself via `wrapper.store_model(name=train_params.model_name)`
- regression metrics also logged as MLflow params via `mlflow.log_param(...)`:
- `mse_val`
- `mae_val`
- `r2_val`
- artifacts explicitly logged with `mlflow.log_artifact(...)`:
- `report.html`
- `train_data.csv`
- `test_data.csv`
- metrics are computed before save (`mse_val`, `mae_val`, `r2_val`) and persisted in the run as params
- Run identifiers now exposed back to workflow:
- `experiment_name`
- `run_name`
- `run_id`
- Notable behavioral change:
- `wrapper._input_example` is cleared (`None`) before storing model.
#### 3.3.3 MinIO object storage
## Before (`main`)
- Read path:
- single source object downloaded via `StorageRepository.fetch_file(bucket_name, file_name)`
- Write path:
- training workflow did not write generated outputs to MinIO in this code path
- generated artifacts were persisted to MLflow, not uploaded back to MinIO
- Location:
- source data in input bucket/key provided by workflow input (`bucket_name` + `file_name`)
## Now (current branch)
- Read path migrated to `MinioRepository.download_file(...)`.
- Supports two input objects:
- mandatory training object: `bucket_name` + `file_name`
- optional validation object: same `bucket_name` + `val_file_name`
- Write path:
- still no artifact upload to MinIO in this workflow path
- report/CSV outputs continue to flow to MLflow artifacts
- Location details:
- bucket resolved from payload (`bucket_name`)
- object key exactly from payload (`file_name`, optional `val_file_name`)
- default bucket in env/config is `MINIO_DEFAULT_BUCKET`, but runtime payload can override via `bucket_name`
#### 3.3.4 Local filesystem (ephemeral runtime workspace)
## Before (`main`)
- Temporary run dir created under reports root using run name + timestamp suffix.
- Artifacts generated locally in that directory:
- `report.html`
- `train_data.csv`
- `test_data.csv`
- optional `model_equation.json`
- After MLflow logging, cleanup activity removed temp directory.
## Now (current branch)
- Temporary run dir managed by `DataManagerRepository` under runtime reports root (`.../reports/temp/<run_name>`).
- Same artifact family generated locally:
- `report.html`
- `train_data.csv`
- `test_data.csv`
- optional `model_equation.json` (for `linear_regression`)
- Cleanup behavior is now tolerant:
- cleanup runs in guarded `finally`
- training success is not reverted if cleanup later fails
#### 3.3.5 Quick matrix (before vs now)
- **Postgres**
- before: status + run_name + errors
- now: same persisted shape; workflow return contains extra IDs
- **MLflow**
- before: legacy model objects + params/metrics + report/data artifacts
- now: wrapper-based model persistence + `mse_val`/`mae_val`/`r2_val` as params + report/data artifacts + run_id exposed
- **MinIO**
- before: reads 1 CSV input object
- now: reads 1 or 2 CSV input objects (train + optional validation), still no output upload
- **Local temp**
- before: generated artifacts, then cleanup
- now: generated artifacts, then best-effort cleanup (non-blocking for success result)
### 3.4 Cleanup behavior impact on output semantics
Before:
- Cleanup was called directly after training result; failures propagated straightforwardly.
Now:
- Cleanup is in a guarded `finally`.
- If training succeeded but cleanup fails, workflow warns and does not rollback success semantics.
- Effective output semantics: successful training result can be returned even if temp cleanup fails.
---
## 4) Detailed field mapping (old -> new)
## Kept (or equivalent role)
- `experiment_run_id` -> kept (broader accepted types: int or numeric string)
- `variable_columns` -> kept
- `target_variable` -> kept
- `bucket_name` -> kept
- `file_name` -> kept
- `line_separator` -> kept
- `decimal_separator` -> kept
- `date_column` -> required (snake_case key; must exist in CSV)
- `date_format` -> optional in payload; omitted/null/blank resolves to default `yyyy-MM-dd HH:mm:ss`
- `train_size` -> kept
- `shuffle` -> kept
- `model_name` -> kept (now less coupled to legacy model enum)
## Added
- `model_type` (primary selector for plugin wrapper/index lookup)
- `data_model_kwargs`
- `model_kwargs`
- `opt_params`
- `val_file_name` (optional second dataset input)
- `model_id` (optional metadata)
- `model_metadata` (loaded/required for schema validation)
- `random_state` (explicit split reproducibility control)
## Removed from new required contract
- `lag_train`, `lag_val`
- `rem_static_win`, `static_threshold`
- `low_lim`, `upp_lim`
- `window`
- `use_scaler`, `include_ar`
- `degree`, `interaction_only`
- `nan_treatment`
- `start_date`, `end_date`
- `scaler_name`
- `removed_intervals`
- `support_filters`
- `experiment_name` (no longer required as top-level client input)
---
## 5) Internal architecture update notes
### 5.1 Repository layer redesign
Removed:
- `model_manager/utils/repository/model_repository.py`
- `model_manager/utils/repository/training_repository.py`
- `model_manager/utils/repository/storage_repository.py`
Added:
- `model_manager/utils/repository/data_manager_repository.py`
Interpretation:
- Data preprocessing/report/metrics responsibilities were consolidated into `DataManagerRepository`.
- Training/model persistence shifted to wrapper + plugin store + MLflow repository integrations.
### 5.2 Model engine abstraction migration
Before:
- Strong coupling to local classes in `model_manager.sientia.models` and custom preprocessing/model objects in `TrainModelResult`.
Now:
- Training uses `SientiaModel` wrapper dynamically obtained by `plugin_store.get_model(model_type=...)`.
- Contract is wrapper-driven (`train`, `transform`, `predict`, `store_model`).
- The codebase removed `model_manager/sientia/models.py`, `model_serving.py`, and `utils.py`, indicating full migration to externalized model runtime abstraction.
### 5.3 Worker/runtime architecture changes
- New `prepare_worker.py` centralizes worker setup and autoscaling parameters.
- Queue names are now runtime-derived:
- `train_model-<runtime>-queue`
- `cleanup_files-<runtime>-queue`
- `worker.py` now installs runtime via plugin store (`plugin_store.install_runtime(runtime_name=...)`) before starting workers.
- This introduces environment/runtime-aware deployment and model packaging behavior.
### 5.4 Synchronous activity and tracking adjustments
- `experiment_tracking` migrated from async postgres helper to sync postgres client path (`postgres_sync`).
- Several activities switched to sync method signatures.
- Error handling in workflow and DB status update paths is more defensive (secondary failures while persisting error status are logged and do not mask primary failure cause).
### 5.5 `TrainModelResult` shape update
Before:
- Stored classic split artifacts (`x_train`, `x_test`, `y_train`, `y_test`) + concrete preprocessing/model objects (`process_data`, `regr`, `scaler_dict`).
Now:
- Stores `train_data`, `val_data` and prediction DataFrames, plus tracking identifiers (`experiment_name`, `run_id`).
- Result object is less tied to internal estimator classes and more aligned with serializable workflow/model-store integration.
---
## 6) Net IO compatibility assessment
## Input compatibility
Not backward compatible with old payloads without adaptation.
Key reasons:
- Legacy required fields removed/ignored by new path.
- New required fields introduced (`model_type`, `*_kwargs` dicts, runtime metadata flow dependency).
- Validation pipeline now expects model metadata semantics.
## Output compatibility
Behavior changed:
- Workflow now returns a result object (previously `None`).
- Training summary includes `experiment_name` and `run_id` in addition to `run_name` and `run_dir`.
- DB update still centered on `run_name`; callers relying only on DB may not see all new output info unless workflow return is consumed.
---
## 7) Practical migration guidance (client side)
To call `train_model` in this branch:
1. Send snake_case payload aligned to new `TrainModelParams`.
2. Always provide:
- `model_name` slugified model name (ex.: `test_model_name or test-model-name`)
- `model_type`
- `data_model_kwargs` (dict)
- `model_kwargs` (dict)
- `opt_params` (dict)
3. Keep `experiment_run_id` numeric (int or numeric string).
4. Use runtime queue naming consistent with worker runtime:
- `train_model-<runtime>-queue`
5. If you need explicit validation split file, send `val_file_name`; otherwise split uses `train_size`/`shuffle`/`random_state`.
---
## 8) Source references used for this document
Primary diffs:
- `model_manager/workflows/train_model.py`
- `model_manager/utils/models/train_model_params.py`
- `model_manager/utils/models/train_model_result.py`
- `model_manager/activities/training.py`
- `model_manager/activities/activities.py`
- `model_manager/activities/experiment_tracking.py`
- `model_manager/utils/repository/data_manager_repository.py`
- `model_manager/utils/repository/model_repository.py` (removed)
- `model_manager/utils/repository/training_repository.py` (removed)
- `model_manager/utils/repository/storage_repository.py` (removed)
- `model_manager/worker/worker.py`
- `model_manager/worker/prepare_worker.py`
- `README.md`
- `input-sample.md`
- `scripts/run_training_test.py`

3
e2e/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
"""
End-to-end tests for the Model Manager Temporal workflows.
"""

697
e2e/conftest.py Normal file
View File

@@ -0,0 +1,697 @@
"""
Pytest configuration and fixtures for E2E tests.
External dependencies use testcontainers or real SDK integrations (no mocks of
model_manager or other first-party code):
- PostgreSQL, MinIO, MongoDB, Gitea: testcontainers.
- MLflow: real client with ``file://`` tracking URI (no MLflow server process).
- Temporal: ``WorkflowEnvironment.start_time_skipping()`` — official in-process
test runtime from temporalio; exercises real workflows and activity code, not
stubs of business logic.
- Observability: ``Logger`` (``get_logger`` from ``model_manager.utils.logger_helper``)
and ``MetricsController`` from sientia_do, same stack as production.
"""
from concurrent.futures import ThreadPoolExecutor
import base64
import csv
import io
import os
import shutil
import tempfile
import time
import mlflow
import pytest
import pytest_asyncio
import requests
from minio import Minio
from sqlalchemy import create_engine, text
from testcontainers.core.container import DockerContainer # type: ignore[import,import-untyped]
from testcontainers.minio import MinioContainer # type: ignore[import-untyped]
from testcontainers.mongodb import MongoDbContainer # type: ignore[import-untyped]
from testcontainers.postgres import PostgresContainer # type: ignore[import,import-untyped]
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import Worker
from model_manager.activities.activities import Activities
from model_manager.utils.logger_helper import get_logger
from model_manager.workflows.cleanup_files import CleanupFiles
from model_manager.workflows.train_model import TrainModel
from sientia_do.notifications.handlers import CoreNotificationHandler
from sientia_do.observability.metrics_controller import MetricsController
from sientia_model.model_repository.plugin_store import PluginStore
# ---------------------------------------------------------------------------
# CSV training data: columns must match the variable_columns and target_variable
# used across all test scenarios.
_TRAIN_CSV_COLUMNS = [
'timestamp',
'303-WIT-200(Value)',
'03CV020/CORRENTE_N_M1_PV(Value)',
'303-WIT-230(Value)',
'03CV022/CORRENTE_N_M1_PV(Value)',
]
_MINIO_BUCKET = 'model-training'
_MINIO_OBJECT = 'training_data.csv'
# ---------------------------------------------------------------------------
# Helpers CSV generation
# ---------------------------------------------------------------------------
def _build_training_csv() -> bytes:
"""
Generate a 150-row CSV with all columns needed by test scenarios.
The numeric values cycle deterministically so lags and static-window
removal always find enough rows in both train and validation splits.
"""
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(_TRAIN_CSV_COLUMNS)
for i in range(150):
ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00'
wit200 = round(30.0 + (i % 20) * 0.5, 2)
cv020 = round(100.0 + (i % 15) * 0.3, 2)
wit230 = round(25.0 + (i % 18) * 0.4, 2)
cv022 = round(90.0 + (i % 12) * 0.25, 2)
writer.writerow([ts, wit200, cv020, wit230, cv022])
return output.getvalue().encode('utf-8')
def _build_training_csv_dd_mm_yyyy() -> bytes:
"""
Generate a 150-row CSV with dd/MM/yyyy HH:mm:ss timestamps and
a DATA column header, for scenarios 12/13 that use a different date format.
"""
output = io.StringIO()
writer = csv.writer(output)
writer.writerow([
'DATA',
'303-WIT-230(Value)',
'03CV022/CORRENTE_N_M1_PV(Value)',
])
for i in range(150):
day = (i % 30) + 1
ts = f'{day:02d}/05/2022 {i % 24:02d}:00:00'
wit230 = round(25.0 + (i % 18) * 0.4, 2)
cv022 = round(90.0 + (i % 12) * 0.25, 2)
writer.writerow([ts, wit230, cv022])
return output.getvalue().encode('utf-8')
def _build_training_csv_custom_target_column() -> bytes:
"""
Same layout as the standard CSV but the target column has a non-default name
(not ``target``) to exercise report and metrics paths.
"""
output = io.StringIO()
writer = csv.writer(output)
writer.writerow([
'timestamp',
'303-WIT-200(Value)',
'MY_CUSTOM_TARGET_COLUMN',
])
for i in range(150):
ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00'
wit200 = round(30.0 + (i % 20) * 0.5, 2)
target_val = round(100.0 + (i % 15) * 0.3, 2)
writer.writerow([ts, wit200, target_val])
return output.getvalue().encode('utf-8')
def _build_training_csv_timestamp_header_naive() -> bytes:
"""
Naive datetimes under column ``Timestamp`` (common UI export) for scenario 16.
"""
output = io.StringIO()
writer = csv.writer(output)
writer.writerow([
'Timestamp',
'303-WIT-200(Value)',
'03CV020/CORRENTE_N_M1_PV(Value)',
])
for i in range(150):
ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00'
wit200 = round(30.0 + (i % 20) * 0.5, 2)
cv020 = round(100.0 + (i % 15) * 0.3, 2)
writer.writerow([ts, wit200, cv020])
return output.getvalue().encode('utf-8')
def _build_training_csv_blank_timestamp_row() -> bytes:
"""Standard columns with one row where ``timestamp`` is empty (NaN after parse)."""
output = io.StringIO()
writer = csv.writer(output)
writer.writerow(_TRAIN_CSV_COLUMNS)
for i in range(150):
wit200 = round(30.0 + (i % 20) * 0.5, 2)
cv020 = round(100.0 + (i % 15) * 0.3, 2)
wit230 = round(25.0 + (i % 18) * 0.4, 2)
cv022 = round(90.0 + (i % 12) * 0.25, 2)
if i == 17:
writer.writerow(['', wit200, cv020, wit230, cv022])
else:
ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00'
writer.writerow([ts, wit200, cv020, wit230, cv022])
return output.getvalue().encode('utf-8')
# ---------------------------------------------------------------------------
# Helpers Gitea seed
# ---------------------------------------------------------------------------
def _wait_for_gitea(base_url: str, timeout: int = 120) -> None:
"""Poll Gitea until it responds to HTTP requests."""
deadline = time.time() + timeout
last_err = None
while time.time() < deadline:
try:
resp = requests.get(f'{base_url}/', timeout=3)
if resp.status_code in (200, 404, 302):
return
except Exception as e:
last_err = e
time.sleep(2)
raise TimeoutError(f'Gitea did not start within {timeout}s at {base_url}. Last error: {last_err}')
def _gitea_api(method: str, url: str, auth: tuple, **kwargs) -> requests.Response:
resp = requests.request(method, url, auth=auth, timeout=30, **kwargs)
try:
resp.raise_for_status()
except requests.exceptions.HTTPError as e:
raise RuntimeError(f"Gitea API error {resp.status_code}: {resp.text}") from e
return resp
def _seed_gitea(base_url: str, admin_user: str, admin_pass: str) -> None:
"""
Create a fictitious model-store repository with dummy models.
"""
auth = (admin_user, admin_pass)
api = f'{base_url}/api/v1'
# Create repository
_gitea_api(
'POST', f'{api}/user/repos', auth,
json={'name': 'model-store', 'private': False, 'auto_init': False},
)
# Root index.yaml
root_index = """
store_name: "E2E Test Store"
version: 1
models:
- name: "linear_regression"
version: 1
runtime: "basic"
- name: "polynomial_regression"
version: 1
runtime: "basic"
runtimes:
basic:
version: "1.0.0"
libraries:
- name: "pandas"
- name: "numpy"
"""
# Model index.yaml (shared for all dummies)
model_index = """
name: "{model_name}"
version: 1
runtime: "basic"
path: "wrapper.py"
class: "DummyWrapper"
model:
class: "DummyModel"
path: "model_logic.py"
external: false
data_model:
class: "DummyTransformer"
path: "model_logic.py"
external: false
"""
# schemas.yaml
schemas_yaml = """
model:
type: object
properties: {}
data_model:
type: object
properties: {}
opt_params:
type: object
properties: {}
"""
# wrapper.py
wrapper_py = """
from sientia_model.wrappers.sientia_model import SientiaModel
import pandas as pd
import numpy as np
from typing import Any
class DummyWrapper(SientiaModel):
def _predict(self, data: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]:
self._log("info", f"Predicting dummy model for {self.model_type}")
# Return a simple prediction (mean or 0.5) to allow metrics computation
preds = pd.DataFrame({self.target: [0.5] * len(data)}, index=data.index)
return preds, {}
def _transform(self, data: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]:
return data, {}
def _train_transformer(self, train_data: pd.DataFrame, val_data: pd.DataFrame) -> None:
pass
def _train_model(self, x: pd.DataFrame, y: pd.DataFrame, x_val: pd.DataFrame | None = None, y_val: pd.DataFrame | None = None) -> None:
self.target = y.columns[0]
def _retrain_transformer(self, data: pd.DataFrame) -> None:
pass
def _retrain_model(self, x: pd.DataFrame, y: pd.DataFrame | None) -> None:
pass
"""
# model_logic.py
model_logic_py = """
class DummyModel:
def __init__(self, **kwargs):
pass
class DummyTransformer:
def __init__(self, **kwargs):
pass
"""
# requirements.txt required by current model packaging path in training activity.
requirements_txt = """
pandas
numpy
"""
def push_file(path: str, content: str):
encoded = base64.b64encode(content.encode()).decode()
_gitea_api(
'POST',
f'{api}/repos/{admin_user}/model-store/contents/{path}',
auth,
json={'message': f'seed: {path}', 'content': encoded},
)
# Push root index
push_file('index.yaml', root_index)
# Push files for both models used in tests
for model_name in ['linear_regression', 'polynomial_regression']:
prefix = f'models/{model_name}'
push_file(f'{prefix}/index.yaml', model_index.format(model_name=model_name))
push_file(f'{prefix}/schemas.yaml', schemas_yaml)
push_file(f'{prefix}/wrapper.py', wrapper_py)
push_file(f'{prefix}/model_logic.py', model_logic_py)
push_file(f'{prefix}/requirements.txt', requirements_txt.strip() + '\n')
push_file(f'{prefix}/__init__.py', "")
# Push runtime
push_file('runtime/basic.yaml', 'name: basic\nversion: "1.0.0"\nlibraries: []')
# ---------------------------------------------------------------------------
# Session-scoped containers
# ---------------------------------------------------------------------------
@pytest.fixture(scope='session')
def postgres_container():
"""PostgreSQL 15 container for experiment_run table."""
container = PostgresContainer('postgres:15')
container.start()
yield container
container.stop()
@pytest.fixture(scope='session')
def minio_container():
"""MinIO container for training CSV storage."""
container = MinioContainer()
container.start()
yield container
container.stop()
@pytest.fixture(scope='session')
def mongodb_container():
"""MongoDB container for CoreNotificationHandler."""
container = MongoDbContainer('mongo:7')
container.start()
yield container
container.stop()
@pytest.fixture(scope='session')
def gitea_container():
"""
Gitea container with a ``model-store`` repo seeded via REST API
(``index.yaml``, dummy model files, ``_seed_gitea``).
The container starts with INSTALL_LOCK so no setup wizard is needed.
An admin user is created via Gitea's CLI before the HTTP API is used.
"""
admin_user = 'gitea_admin'
admin_pass = 'gitea_admin_pass' # noqa: S105
container = (
DockerContainer('gitea/gitea:latest')
.with_env('GITEA__security__INSTALL_LOCK', 'true')
.with_env('GITEA__server__HTTP_PORT', '3000')
.with_env('GITEA__log__LEVEL', 'Warn')
.with_exposed_ports(3000)
)
container.start()
port = container.get_exposed_port(3000)
base_url = f'http://localhost:{port}'
_wait_for_gitea(base_url)
time.sleep(5) # Wait a bit for DB to fully initialize after HTTP is up
# Create admin user via Gitea CLI inside the container
# Must run after Gitea is fully initialized
gitea_cmd = (
f'gitea admin user create '
f'--username {admin_user} '
f'--password {admin_pass} '
f'--email admin@test.local '
f'--admin '
f'--must-change-password=false'
)
exec_result = container.exec(f"su git -c '{gitea_cmd}'")
if exec_result.exit_code != 0:
raise RuntimeError(f"Failed to create Gitea admin user: {exec_result.output.decode('utf-8')}")
_seed_gitea(base_url, admin_user, admin_pass)
yield {
'container': container,
'base_url': base_url,
'admin_user': admin_user,
'admin_pass': admin_pass,
}
container.stop()
@pytest.fixture(scope='session')
def mlflow_tracking_dir():
"""Local MLflow filesystem tracking directory (no network needed)."""
tmpdir = tempfile.mkdtemp(prefix='mlflow-e2e-')
mlflow.set_tracking_uri(f'file://{tmpdir}')
yield tmpdir
shutil.rmtree(tmpdir, ignore_errors=True)
@pytest.fixture(scope='session', autouse=True)
def e2e_runtime_reports_dir():
"""
Route runtime report artifacts to a writable temp directory during E2E.
Production defaults point to /var/lib/model-manager; in local CI/dev runs this
path may be unavailable. This fixture keeps the same code paths while avoiding
host permission issues.
"""
import model_manager.runtime_paths as runtime_paths
import model_manager.utils.repository.data_manager_repository as data_repo_module
base_dir = tempfile.mkdtemp(prefix='model-manager-e2e-runtime-')
reports_root = f'{base_dir}/reports'
reports_temp_dir = f'{reports_root}/temp'
project_base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'model_manager'))
old_runtime_reports_root = runtime_paths.REPORTS_ROOT
old_runtime_reports_temp = runtime_paths.REPORTS_TEMP_DIR
old_runtime_project_base = runtime_paths.PROJECT_BASE_PATH
old_repo_reports_root = data_repo_module.REPORTS_ROOT
old_repo_project_base = data_repo_module.PROJECT_BASE_PATH
runtime_paths.REPORTS_ROOT = reports_root
runtime_paths.REPORTS_TEMP_DIR = reports_temp_dir
runtime_paths.PROJECT_BASE_PATH = project_base_path
data_repo_module.REPORTS_ROOT = reports_root
data_repo_module.PROJECT_BASE_PATH = project_base_path
try:
yield reports_root
finally:
runtime_paths.REPORTS_ROOT = old_runtime_reports_root
runtime_paths.REPORTS_TEMP_DIR = old_runtime_reports_temp
runtime_paths.PROJECT_BASE_PATH = old_runtime_project_base
data_repo_module.REPORTS_ROOT = old_repo_reports_root
data_repo_module.PROJECT_BASE_PATH = old_repo_project_base
shutil.rmtree(base_dir, ignore_errors=True)
# ---------------------------------------------------------------------------
# Session-scoped: seed MinIO with training CSV
# ---------------------------------------------------------------------------
@pytest.fixture(scope='session', autouse=True)
def upload_training_csv(minio_container, mlflow_tracking_dir): # noqa: ARG001
"""
Upload training CSV files to the MinIO container before any test runs.
Depends on mlflow_tracking_dir to ensure the MLflow URI is set at session start.
"""
port = minio_container.get_exposed_port(9000)
client = Minio(
f'localhost:{port}',
access_key='minioadmin',
secret_key='minioadmin',
secure=False,
)
if not client.bucket_exists(_MINIO_BUCKET):
client.make_bucket(_MINIO_BUCKET)
# Standard training CSV
csv_bytes = _build_training_csv()
client.put_object(
_MINIO_BUCKET,
_MINIO_OBJECT,
io.BytesIO(csv_bytes),
length=len(csv_bytes),
content_type='text/csv',
)
# dd/MM/yyyy format CSV for scenarios 12/13
alt_csv_bytes = _build_training_csv_dd_mm_yyyy()
client.put_object(
_MINIO_BUCKET,
'training_data_dd_mm_yyyy.csv',
io.BytesIO(alt_csv_bytes),
length=len(alt_csv_bytes),
content_type='text/csv',
)
custom_target = _build_training_csv_custom_target_column()
client.put_object(
_MINIO_BUCKET,
'training_data_custom_target.csv',
io.BytesIO(custom_target),
length=len(custom_target),
content_type='text/csv',
)
ts_header = _build_training_csv_timestamp_header_naive()
client.put_object(
_MINIO_BUCKET,
'training_data_timestamp_naive.csv',
io.BytesIO(ts_header),
length=len(ts_header),
content_type='text/csv',
)
blank_ts = _build_training_csv_blank_timestamp_row()
client.put_object(
_MINIO_BUCKET,
'training_data_blank_timestamp_row.csv',
io.BytesIO(blank_ts),
length=len(blank_ts),
content_type='text/csv',
)
# ---------------------------------------------------------------------------
# Function-scoped: database engine + schema setup
# ---------------------------------------------------------------------------
@pytest.fixture
def postgres_engine(postgres_container):
"""SQLAlchemy engine connected to the test PostgreSQL container."""
engine = create_engine(postgres_container.get_connection_url())
yield engine
engine.dispose()
@pytest.fixture(autouse=True)
def setup_experiment_run_table(postgres_engine):
"""
Create the experiment_run table before each test and drop it afterwards
to guarantee full isolation between tests.
"""
with postgres_engine.begin() as conn:
conn.execute(text("""
CREATE TABLE IF NOT EXISTS public.experiment_run (
id INT PRIMARY KEY,
experiment_name TEXT NOT NULL,
run_name TEXT,
username TEXT,
status TEXT NOT NULL DEFAULT 'ORCHESTRATOR_WAITING_PROC',
error_message TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
bucket_name TEXT,
file_name TEXT
)
"""))
yield
with postgres_engine.begin() as conn:
conn.execute(text('DROP TABLE IF EXISTS public.experiment_run'))
# ---------------------------------------------------------------------------
# Observability (real sientia_do implementations)
# ---------------------------------------------------------------------------
@pytest.fixture(scope='session')
def e2e_logger():
"""Shared production-style Logger for the whole E2E session."""
return get_logger('model-manager-e2e')
@pytest.fixture
def metrics_controller(e2e_logger):
"""MetricsController bound to the E2E logger (fresh instance per test)."""
return MetricsController(logger=e2e_logger)
# ---------------------------------------------------------------------------
# Application fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def notification_handler(mongodb_container, e2e_logger):
"""
Real CoreNotificationHandler connected to the MongoDB testcontainer.
"""
connection_url = mongodb_container.get_connection_url()
handler = CoreNotificationHandler(
connection_string=connection_url,
database='test_notifications',
logger=e2e_logger,
project_name='model-manager-e2e',
)
yield handler
handler.shutdown()
@pytest.fixture
def plugin_store(gitea_container, e2e_logger, metrics_controller, notification_handler):
"""
Real PluginStore pointed at the Gitea testcontainer.
cache_ttl_seconds=0 forces a fresh download every test.
"""
store = PluginStore(
base_url=gitea_container['base_url'],
owner=gitea_container['admin_user'],
repo='model-store',
username=gitea_container['admin_user'],
password=gitea_container['admin_pass'],
cache_ttl_seconds=0,
logger=e2e_logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
yield store
@pytest.fixture
def test_activities(
postgres_container,
minio_container,
mlflow_tracking_dir, # noqa: ARG001 ensures MLflow URI is set
plugin_store,
e2e_logger,
notification_handler,
metrics_controller,
):
"""
Real Activities instance wired to all testcontainers.
"""
pg_port = postgres_container.get_exposed_port(5432)
minio_port = minio_container.get_exposed_port(9000)
activities = Activities(
postgres_config={
'host': 'localhost',
'port': int(pg_port),
'user': 'test',
'password': 'test',
'dbname': 'test',
'min_connections': 1,
'max_connections': 5,
},
mlflow_config={
'url': mlflow.get_tracking_uri(),
'username': None,
'password': None,
},
minio_config={
'endpoint_url': f'http://localhost:{minio_port}',
'access_key': 'minioadmin',
'secret_key': 'minioadmin',
'use_ssl': False,
'default_bucket': _MINIO_BUCKET,
},
plugin_store=plugin_store,
logger=e2e_logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
yield activities
activities.shutdown()
def _activity_list(activities: Activities) -> list:
return [
activities.update_experiment_run,
activities.load_model_metadata,
activities.validate_train_params,
activities.train_model,
activities.cleanup_resources,
activities.cleanup_temp_directories,
]
@pytest_asyncio.fixture(scope='function')
async def temporal_test_env():
"""Temporal SDK test environment (time-skipping); runs real workflow/activity code."""
env = await WorkflowEnvironment.start_time_skipping()
async with env:
yield env
@pytest_asyncio.fixture(scope='function')
async def temporal_worker(temporal_test_env, test_activities):
"""Temporal worker registered with all workflows and activities."""
with ThreadPoolExecutor() as executor:
async with Worker(
temporal_test_env.client,
task_queue='test-queue',
workflows=[TrainModel, CleanupFiles],
activities=_activity_list(test_activities),
activity_executor=executor,
) as worker:
yield worker

195
e2e/helpers.py Normal file
View File

@@ -0,0 +1,195 @@
"""
Shared helpers for E2E tests (Temporal workflows + PostgreSQL).
"""
import asyncio
import json
from datetime import datetime
from pathlib import Path
from typing import Any
import pytest
from sqlalchemy import text
from sqlalchemy.engine import Engine
async def start_and_await_workflow(
client,
workflow_run,
input_data: dict,
workflow_id: str,
timeout: float = 600.0,
):
"""
Start a Temporal workflow and wait for its result.
Args:
client: Temporal client from WorkflowEnvironment.
workflow_run: Workflow run method (e.g. TrainModel.run).
input_data: Workflow input payload.
workflow_id: Unique workflow id.
timeout: Max seconds to wait for completion (default allows cold testcontainer startup).
Returns:
Workflow result value.
"""
handle = await client.start_workflow(
workflow_run,
input_data,
id=workflow_id,
task_queue='test-queue',
)
return await asyncio.wait_for(handle.result(), timeout=timeout)
def make_workflow_id(prefix: str) -> str:
"""Build a unique workflow id using a prefix and current timestamp."""
return f'{prefix}-{datetime.now().timestamp()}'
def insert_experiment_run(
engine: Engine,
experiment_run_id: int,
experiment_name: str = 'test_experiment',
status: str = 'ORCHESTRATOR_WAITING_PROC',
bucket_name: str = 'model-training',
file_name: str = 'training_data.csv',
) -> None:
"""
Insert a minimal experiment_run row to satisfy foreign-key-style lookups.
Args:
engine: SQLAlchemy engine connected to the test database.
experiment_run_id: Primary key for the row.
experiment_name: Human-readable experiment name.
status: Initial status string.
bucket_name: MinIO bucket name.
file_name: Training file name inside the bucket.
"""
with engine.begin() as conn:
conn.execute(
text("""
INSERT INTO public.experiment_run
(id, experiment_name, status, bucket_name, file_name)
VALUES
(:id, :experiment_name, :status, :bucket_name, :file_name)
ON CONFLICT (id) DO NOTHING
"""),
{
'id': experiment_run_id,
'experiment_name': experiment_name,
'status': status,
'bucket_name': bucket_name,
'file_name': file_name,
},
)
def assert_experiment_status(
engine: Engine,
experiment_run_id: int,
expected_status: str,
) -> None:
"""
Assert the final status of an experiment_run row.
Args:
engine: SQLAlchemy engine.
experiment_run_id: Row primary key.
expected_status: Expected status string.
"""
with engine.connect() as conn:
row = conn.execute(
text('SELECT status FROM public.experiment_run WHERE id = :id'),
{'id': experiment_run_id},
).fetchone()
assert row is not None, (
f'No experiment_run row found for id={experiment_run_id}'
)
assert row[0] == expected_status, (
f'Expected status={expected_status!r}, got {row[0]!r} '
f'for experiment_run id={experiment_run_id}'
)
def assert_experiment_run_name_set(
engine: Engine,
experiment_run_id: int,
) -> None:
"""Assert that run_name is not null/empty after a successful training."""
with engine.connect() as conn:
row = conn.execute(
text('SELECT run_name FROM public.experiment_run WHERE id = :id'),
{'id': experiment_run_id},
).fetchone()
assert row is not None, (
f'No experiment_run row found for id={experiment_run_id}'
)
assert row[0] is not None and row[0].strip() != '', (
f'Expected run_name to be set for experiment_run id={experiment_run_id}, got {row[0]!r}'
)
def assert_experiment_error(
engine: Engine,
experiment_run_id: int,
expected_status: str,
error_substr: str,
) -> None:
"""
Assert status and that error_message contains a given substring.
Args:
engine: SQLAlchemy engine.
experiment_run_id: Row primary key.
expected_status: Expected status string.
error_substr: Substring that must appear in error_message.
"""
with engine.connect() as conn:
row = conn.execute(
text(
'SELECT status, error_message FROM public.experiment_run WHERE id = :id'
),
{'id': experiment_run_id},
).fetchone()
assert row is not None, (
f'No experiment_run row found for id={experiment_run_id}'
)
assert row[0] == expected_status, (
f'Expected status={expected_status!r}, got {row[0]!r}'
)
assert row[1] is not None and error_substr.lower() in row[1].lower(), (
f'Expected error_message to contain {error_substr!r}, got {row[1]!r}'
)
def assert_no_experiment_row(engine: Engine, experiment_run_id: int) -> None:
"""Assert that no experiment_run row exists for the given id."""
with engine.connect() as conn:
count = conn.execute(
text('SELECT COUNT(*) FROM public.experiment_run WHERE id = :id'),
{'id': experiment_run_id},
).scalar()
assert count == 0, (
f'Expected no experiment_run row for id={experiment_run_id}, found {count}'
)
def load_scenario(scenario_filename: str) -> dict[str, Any]:
"""
Load a test scenario JSON file from docs/test-scenarios/.
Args:
scenario_filename: Filename without path (e.g. '01-linear-regression-basic.json').
Returns:
dict: Parsed scenario payload.
"""
scenario_path = (
Path(__file__).parent.parent / 'docs' / 'test-scenarios' / scenario_filename
)
with open(scenario_path) as f:
return json.load(f)

View File

@@ -0,0 +1,108 @@
"""
End-to-end tests for CleanupFiles workflow.
Covers scenarios 3.x: cleanup of temporary local directories.
"""
import pytest
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import Worker
from e2e.helpers import make_workflow_id, start_and_await_workflow
from model_manager.workflows.cleanup_files import CleanupFiles
# Matches Cleanup.dir_timestamp_pattern: name_YYYYMMDD_HHMMSS_microseconds
_STALE_DIR_OLD = 'stale_run_20200102_030405_000001'
_STALE_DIR_OLDER = 'stale_run_20191231_235959_999999'
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_3_1_1_cleanup_with_no_temp_dirs(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
tmp_path,
):
"""Scenario 3.1.1 Cleanup when the temp directory is empty.
The cleanup_temp_directories activity should complete without error
and the workflow should finish successfully.
"""
# Use an empty temp directory as the reports path
empty_dir = tmp_path / 'reports_temp'
empty_dir.mkdir()
result = await start_and_await_workflow(
temporal_test_env.client,
CleanupFiles.run,
{'temp_path': str(empty_dir)},
make_workflow_id('test-s3-1-1'),
)
# Workflow returns None on success
assert result is None
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_3_1_2_cleanup_removes_old_temp_dirs(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
tmp_path,
):
"""Scenario 3.1.2 Cleanup removes stale subdirectories from the temp dir.
Creates two subdirectories with timestamp suffixes inside the reports
temp directory and verifies the activity removes them.
"""
reports_dir = tmp_path / 'reports_temp'
reports_dir.mkdir()
# Create two stale run directories (names must match cleanup activity regex)
stale1 = reports_dir / _STALE_DIR_OLD
stale2 = reports_dir / _STALE_DIR_OLDER
stale1.mkdir()
stale2.mkdir()
(stale1 / 'model.pkl').write_bytes(b'fake-model-data')
(stale2 / 'report.json').write_bytes(b'{"status": "old"}')
result = await start_and_await_workflow(
temporal_test_env.client,
CleanupFiles.run,
{'temp_path': str(reports_dir)},
make_workflow_id('test-s3-1-2'),
)
assert result is None
# The activity should have cleaned up the stale directories
remaining = list(reports_dir.iterdir())
assert len(remaining) == 0, (
f'Expected all stale dirs to be removed, but found: {remaining}'
)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_3_1_3_cleanup_nonexistent_temp_path(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
tmp_path,
):
"""Scenario 3.1.3 Cleanup with a temp_path that does not exist.
The activity must handle a missing directory gracefully without
raising an unhandled exception, since the directory may have already
been cleaned by a previous run.
"""
nonexistent = str(tmp_path / 'does_not_exist' / 'reports')
# Should not raise — the activity is expected to handle a missing path
result = await start_and_await_workflow(
temporal_test_env.client,
CleanupFiles.run,
{'temp_path': nonexistent},
make_workflow_id('test-s3-1-3'),
)
assert result is None

View File

@@ -0,0 +1,298 @@
"""
End-to-end tests for TrainModel parameter validation paths.
Covers scenarios 2.1.x: workflows that must terminate with
ORCHESTRATOR_VALIDATION_ERROR due to invalid parameter values.
"""
import pytest
from temporalio.client import WorkflowFailureError
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import Worker
from e2e.helpers import (
assert_experiment_error,
insert_experiment_run,
load_scenario,
make_workflow_id,
start_and_await_workflow,
)
from model_manager.workflows.train_model import TrainModel
# Base experiment_run ids for validation test scenarios (offset to avoid collision)
_VALIDATION_ID_BASE = 3000
def _exception_chain_text(exc: BaseException) -> str:
"""Concatenate messages from an exception __cause__/__context__ chain."""
parts: list[str] = []
cur: BaseException | None = exc
seen: set[int] = set()
while cur is not None and id(cur) not in seen:
seen.add(id(cur))
text = str(cur).strip()
if text:
parts.append(text)
cur = cur.__cause__ or getattr(cur, '__context__', None)
return ' | '.join(parts).lower()
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_1_train_size_out_of_range(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 2.1.1 train_size=5 violates the 10100 business rule.
Expected: workflow updates status → ORCHESTRATOR_VALIDATION_ERROR
and error_message references 'train_size'.
"""
experiment_run_id = _VALIDATION_ID_BASE + 1
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'train_size': 5}
insert_experiment_run(postgres_engine, experiment_run_id)
with pytest.raises(Exception):
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s2-1-1'),
)
assert_experiment_error(
postgres_engine,
experiment_run_id,
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
error_substr='train_size',
)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_2_empty_variable_columns(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 2.1.2 variable_columns=[] → ORCHESTRATOR_VALIDATION_ERROR."""
experiment_run_id = _VALIDATION_ID_BASE + 2
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'variable_columns': []}
insert_experiment_run(postgres_engine, experiment_run_id)
with pytest.raises(Exception):
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s2-1-2'),
)
assert_experiment_error(
postgres_engine,
experiment_run_id,
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
error_substr='variable_columns',
)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_3_invalid_date_format(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 2.1.3 date_format='INVALID' is not in the allowed list."""
experiment_run_id = _VALIDATION_ID_BASE + 3
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'date_format': 'INVALID'}
insert_experiment_run(postgres_engine, experiment_run_id)
with pytest.raises(Exception):
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s2-1-3'),
)
assert_experiment_error(
postgres_engine,
experiment_run_id,
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
error_substr='date_format',
)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_4_whitespace_only_model_name(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 2.1.4 model_name=' ' (whitespace) → ORCHESTRATOR_VALIDATION_ERROR."""
experiment_run_id = _VALIDATION_ID_BASE + 4
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'model_name': ' '}
insert_experiment_run(postgres_engine, experiment_run_id)
with pytest.raises(Exception):
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s2-1-4'),
)
assert_experiment_error(
postgres_engine,
experiment_run_id,
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
error_substr='model_name',
)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_5_unknown_model_type(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 2.1.5 model_type='totally_unknown' → ORCHESTRATOR_VALIDATION_ERROR.
The PluginStore will not find this model in the Gitea repo, causing
load_model_metadata to fail before validate_train_params is even called.
"""
experiment_run_id = _VALIDATION_ID_BASE + 5
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {
**scenario,
'experiment_run_id': experiment_run_id,
'model_type': 'totally_unknown_model',
}
insert_experiment_run(postgres_engine, experiment_run_id)
with pytest.raises(Exception):
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s2-1-5'),
)
assert_experiment_error(
postgres_engine,
experiment_run_id,
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
error_substr='totally_unknown_model',
)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_6_missing_target_variable(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 2.1.6 target_variable='' (empty string) → ORCHESTRATOR_VALIDATION_ERROR."""
experiment_run_id = _VALIDATION_ID_BASE + 6
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'target_variable': ''}
insert_experiment_run(postgres_engine, experiment_run_id)
with pytest.raises(Exception):
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s2-1-6'),
)
assert_experiment_error(
postgres_engine,
experiment_run_id,
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
error_substr='target_variable',
)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_7_missing_experiment_run_id(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
):
"""Scenario 2.1.7 experiment_run_id missing → workflow raises ValueError immediately.
No DB row is inserted because experiment_run_id is mandatory to even
know which row to update. The workflow should raise before any DB call.
"""
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {k: v for k, v in scenario.items() if k != 'experiment_run_id'}
with pytest.raises(WorkflowFailureError) as excinfo:
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s2-1-7'),
)
combined = _exception_chain_text(excinfo.value)
assert 'experiment_run_id' in combined
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_8_missing_date_column(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
):
"""Scenario 2.1.8 date_column missing in payload raises before workflow business validation."""
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {k: v for k, v in scenario.items() if k != 'date_column'}
with pytest.raises(WorkflowFailureError) as excinfo:
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s2-1-8'),
)
combined = _exception_chain_text(excinfo.value)
assert 'date_column' in combined
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_9_whitespace_date_column(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 2.1.9 date_column=' ' must produce ORCHESTRATOR_VALIDATION_ERROR."""
experiment_run_id = _VALIDATION_ID_BASE + 9
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'date_column': ' '}
insert_experiment_run(postgres_engine, experiment_run_id)
with pytest.raises(Exception):
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s2-1-9'),
)
assert_experiment_error(
postgres_engine,
experiment_run_id,
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
error_substr='date_column',
)

View File

@@ -0,0 +1,493 @@
"""
End-to-end tests for TrainModel workflow main workflow scenarios.
Covers:
1.1.x Happy-path training (various scenarios from docs/test-scenarios/)
1.2.x Error paths (MinIO failure, missing DB row)
"""
import pytest
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import Worker
from e2e.helpers import (
assert_experiment_error,
assert_experiment_run_name_set,
assert_experiment_status,
assert_no_experiment_row,
insert_experiment_run,
load_scenario,
make_workflow_id,
start_and_await_workflow,
)
from model_manager.workflows.train_model import TrainModel
# ---------------------------------------------------------------------------
# 1.1 Happy paths
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_1_linear_regression_basic(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.1 Linear Regression Basic (cenário 01).
Validates the complete training pipeline end-to-end:
load_model_metadata → validate_train_params → train_model →
update_experiment_run (TRAINING_SUCCESS).
"""
scenario = load_scenario('01-linear-regression-basic.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-1'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_2_linear_regression_with_scaler(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.2 Linear regression with Standard Scaler (cenário 02)."""
scenario = load_scenario('02-linear-regression-with-scaler.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-2'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_3_polynomial_regression_degree2_with_scaler(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.3 Polynomial Regression Degree 2 with Standard Scaler (cenário 03)."""
scenario = load_scenario('03-polynomial-regression-degree2.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-3'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_4_polynomial_regression_degree3_with_scaler(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.4 Polynomial regression degree 3 with Standard Scaler (cenário 04)."""
scenario = load_scenario('04-polynomial-regression-degree3.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-4'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_5_linear_regression_with_lags(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.5 Linear Regression with lag_train/lag_val per variable (cenário 05)."""
scenario = load_scenario('05-linear-regression-with-lags.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-5'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_6_linear_regression_nan_interpolation(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.6 nan_treatment='linear interpolation' (cenário 06)."""
scenario = load_scenario('06-linear-regression-nan-interpolation.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-6'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_7_linear_regression_static_window_removal(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.7 rem_static_win=true with default static_threshold (cenário 07)."""
scenario = load_scenario('07-linear-regression-static-window-removal.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-7'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_8_linear_regression_with_limits(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.8 support_filters with min/max limits per variable (cenário 08)."""
scenario = load_scenario('08-linear-regression-with-limits.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-8'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_9_polynomial_degree2_scaler_and_lags(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.9 Polynomial degree 2, Standard Scaler and lags (cenário 09)."""
scenario = load_scenario('09-polynomial-degree2-with-scaler-and-lags.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-9'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_10_linear_regression_with_ar_opt_params(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.10 Linear regression with include_ar in opt_params (cenário 10)."""
scenario = load_scenario('10-linear-regression-with-ar.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-10'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_11_linear_regression_static_threshold_custom(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.11 rem_static_win=true with custom static_threshold (cenário 11)."""
scenario = load_scenario('11-linear-regression-static-threshold-custom.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-11'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_12_alternate_date_format_dd_mm_yyyy(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.12 DATA column and dd/MM/yyyy format CSV (cenário 12)."""
scenario = load_scenario('12-angular-test-date-format.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(
postgres_engine,
experiment_run_id,
file_name='training_data_dd_mm_yyyy.csv',
)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-12'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_13_alternate_csv_narrow_date_window(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.13 Same alternate CSV as 12 with date window (cenário 13)."""
scenario = load_scenario('13-angular-test-double-date-column.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(
postgres_engine,
experiment_run_id,
file_name='training_data_dd_mm_yyyy.csv',
)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-13'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_14_polynomial_with_support_filters(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.14 Polynomial degree 4, Standard Scaler, line support filters (cenário 14)."""
scenario = load_scenario('14-angular-test-polynomial-support-filters.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(postgres_engine, experiment_run_id)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-14'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_15_linear_regression_custom_target_column_name(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.15 Target column is not named ``target``; report path uses target_variable."""
scenario = load_scenario('15-linear-regression-custom-target-column.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(
postgres_engine,
experiment_run_id,
file_name='training_data_custom_target.csv',
)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-15'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_16_naive_timestamp_header_column(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.16 CSV uses ``Timestamp`` header; snake_case date_column/date_format."""
scenario = load_scenario('16-linear-regression-naive-timestamp-header.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(
postgres_engine,
experiment_run_id,
file_name='training_data_timestamp_naive.csv',
)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-16'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_1_17_linear_regression_blank_timestamp_row_dropped(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.1.17 CSV has one empty timestamp cell; row is dropped and training succeeds."""
scenario = load_scenario('17-linear-regression-blank-timestamp-row.json')
experiment_run_id = scenario['experiment_run_id']
insert_experiment_run(
postgres_engine,
experiment_run_id,
file_name='training_data_blank_timestamp_row.csv',
)
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-1-17'),
)
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
# ---------------------------------------------------------------------------
# 1.2 Error paths
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_2_1_minio_file_not_found(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.2.1 Training file does not exist in MinIO → TRAINING_ERROR."""
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {**scenario, 'experiment_run_id': 2001, 'file_name': 'does_not_exist.csv'}
experiment_run_id = 2001
insert_experiment_run(postgres_engine, experiment_run_id)
with pytest.raises(Exception):
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-2-1'),
)
assert_experiment_error(
postgres_engine,
experiment_run_id,
expected_status='TRAINING_ERROR',
error_substr='does_not_exist',
)
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_1_2_2_experiment_run_id_not_in_db(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
postgres_engine,
):
"""Scenario 1.2.2 experiment_run_id row absent → update_experiment_run raises."""
scenario = load_scenario('01-linear-regression-basic.json')
scenario = {**scenario, 'experiment_run_id': 9999}
# Intentionally NOT inserting the row
with pytest.raises(Exception):
await start_and_await_workflow(
temporal_test_env.client,
TrainModel.run,
scenario,
make_workflow_id('test-s1-2-2'),
)
assert_no_experiment_row(postgres_engine, 9999)

View File

@@ -0,0 +1,2 @@
sientia_do: git+ssh://git@github.com/Aignosi/sientia-dataops-library.git
sientia_model: git+ssh://git@github.com/Aignosi/sientia-model-library.git

38
input-sample.json Normal file
View File

@@ -0,0 +1,38 @@
{
"experiment_run_id": 1001,
"variable_columns": ["feature_a", "feature_b"],
"target_variable": "target",
"bucket_name": "model-training",
"file_name": "training_data.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "Linear Regression",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"feature_a": 0,
"feature_b": 0
},
"lag_val": {
"feature_a": 0,
"feature_b": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "Standard Scaler"
},
"opt_params": {}
}

76
input-sample.md Normal file
View File

@@ -0,0 +1,76 @@
##### Insert a new experiment run
```sql
-- Optional: remove a previous run with the same id
DELETE FROM public.experiment_run WHERE experiment_run_id = 1001;
```
```sql
INSERT INTO public.experiment_run
(experiment_name, run_name, username, status, error_message, created_at,
updated_at, bucket_name, file_name, request_data, orchestrator_response_data)
VALUES(
'test-experiment-name',
'test-run-name',
'test-username',
'ORCHESTRATOR_WAITING_PROC',
null,
now(),
now(),
'model-training',
'training_data.csv',
'{"experiment_run_id":1001,"variable_columns":["feature_a","feature_b"],"target_variable":"target","bucket_name":"model-training","file_name":"training_data.csv","line_separator":",","decimal_separator":".","train_size":80,"shuffle":true,"model_name":"Linear Regression","model_type":"linear_regression","data_model_kwargs":{"lag_train":{"feature_a":0,"feature_b":0},"lag_val":{"feature_a":0,"feature_b":0},"nan_treatment":"drop"},"model_kwargs":{"degree":1,"scaler_name":"Standard Scaler"},"opt_params":{}}',
null
);
```
##### Upload the input dataset to MinIO
```bash
mc cp input_dataset.csv suse/model-training/training-sample-dataset-1001.csv
```
##### Temporal input payload sample
Keys match `TrainModelParams.from_dict` in `model_manager/utils/models/train_model_params.py`: every field passed to `_check_none` must be present, including **`date_column`**; `model_metadata` must be non-empty for `validate_business_rules()`. You may omit **`date_format`** (defaults to `yyyy-MM-dd HH:mm:ss`). Omit optional keys (`random_state`, `val_file_name`, `model_id`) when defaults or `None` apply.
```json
{
"experiment_run_id": 1001,
"variable_columns": ["feature_a", "feature_b"],
"target_variable": "target",
"bucket_name": "model-training",
"file_name": "training-sample-dataset-1001.csv",
"line_separator": ",",
"decimal_separator": ".",
"date_column": "timestamp",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "test-runtime-linear-regression-model",
"model_type": "linear_regression",
"data_model_kwargs": {
"lag_train": {
"feature_a": 0,
"feature_b": 0
},
"lag_val": {
"feature_a": 0,
"feature_b": 0
},
"nan_treatment": "drop",
"rem_static_win": false,
"static_threshold": null,
"start_date": null,
"end_date": null,
"support_filters": {},
"removed_intervals": []
},
"model_kwargs": {
"degree": 1,
"interaction_only": false,
"scaler_name": "Standard Scaler"
},
"opt_params": {}
}
```

11
input_dataset.csv Normal file
View File

@@ -0,0 +1,11 @@
timestamp,Counter,Rollout,Square
2026-01-01 00:00:00,1.0,2.0,5.1
2026-01-01 00:01:00,2.0,2.0,5.9
2026-01-01 00:02:00,2.0,3.0,8.3
2026-01-01 00:03:00,3.0,3.0,9.2
2026-01-01 00:04:00,3.0,4.0,10.8
2026-01-01 00:05:00,4.0,4.0,11.7
2026-01-01 00:06:00,4.0,5.0,14.1
2026-01-01 00:07:00,5.0,5.0,15.15
2026-01-01 00:08:00,5.0,6.0,17.2
2026-01-01 00:09:00,6.0,6.0,17.9
1 timestamp Counter Rollout Square
2 2026-01-01 00:00:00 1.0 2.0 5.1
3 2026-01-01 00:01:00 2.0 2.0 5.9
4 2026-01-01 00:02:00 2.0 3.0 8.3
5 2026-01-01 00:03:00 3.0 3.0 9.2
6 2026-01-01 00:04:00 3.0 4.0 10.8
7 2026-01-01 00:05:00 4.0 4.0 11.7
8 2026-01-01 00:06:00 4.0 5.0 14.1
9 2026-01-01 00:07:00 5.0 5.0 15.15
10 2026-01-01 00:08:00 5.0 6.0 17.2
11 2026-01-01 00:09:00 6.0 6.0 17.9

View File

@@ -0,0 +1,385 @@
---
tags:
- engineering
- sientia
- runtime-system
- model-manager
- plugin-store
- migration-plan
created: 2026-03-02
modified: 2026-03-02
created_by: Vitor Pimentel
modified_by: Vitor Pimentel
status: draft
---
# Sientia Model Manager — PluginStore Migration Plan
> Implementation plan for migrating `sientia-dataops-model-manager` to use the Sientia PluginStore for runtime installation and model retrieval, aligned with the runtime architecture described in `analytics.md`.
## Summary
1. [[#Objectives and Scope|Objectives and Scope]] — What this migration must achieve
2. [[#Existing State Overview (model-manager)|Existing State Overview]] — Current responsibilities and coupling points
3. [[#Requirements Mapping|Requirements Mapping]] — Functional and non-functional requirements
4. [[#Target Architecture|Target Architecture]] — Desired runtime and model-loading architecture
5. [[#Implementation Plan|Implementation Plan]] — Phased, detailed changes to apply
6. [[#Testing Strategy|Testing Strategy]] — How to validate the new behavior
7. [[#Rollout and Migration Strategy|Rollout and Migration Strategy]] — How to safely roll out and deprecate old paths
8. [[#Potential Model Library Changes|Potential Model Library Changes]] — Expected impact on `sientia-model-library`
9. [[#Related Documents|Related Documents]] — Cross-links to supporting documents
---
## Objectives and Scope
The goal of this work is to evolve `sientia-dataops-model-manager` so that:
- It **detects the runtime** before the worker starts, using the `RUNTIME` environment variable.
- It **installs the selected runtime** using the PluginStore runtime interface from `sientia_model.model_repository.plugin_store`:
- `PluginStore.install_runtime(runtime_name)`.
- It **uses PluginStore to obtain models from the store**, instead of constructing them from the local ML template / mlops library:
- Pipelines call `PluginStore.get_model(...)` to obtain `SientiaModel` instances.
- The previous “in-repo model implementation plus mlops library” path is removed.
Out of scope:
- Changing Temporal workflow semantics (queues, retry policies, etc.).
- Replacing the existing MLflow-based tracking and reporting; these remain the responsibility of `ModelRepository` and the reporting utilities.
---
## Existing State Overview (model-manager)
The `sientia-dataops-model-manager` application currently:
- **Worker orchestration** (`model_manager/worker/worker.py`)
- Reads Temporal configuration from env (`TEMPORAL_HOST`, `TEMPORAL_NAMESPACE`, task queues).
- Sets up observability (Prometheus, metrics, Sientia logger).
- Builds connector configs for Postgres, MLflow, MinIO, MongoDB.
- Instantiates `Activities` and schedules, then starts Temporal workers.
- Does not validate or install any “runtime” concept before worker startup.
- **Training pipeline**
- `Training` activity (`model_manager/activities/training.py`) coordinates:
- Parameter validation (`validate_train_params`).
- Training execution via `TrainingRepository`.
- Saving trained models and artifacts via `ModelRepository` to MLflow.
- `TrainingRepository` (`model_manager/utils/repository/training_repository.py`):
- Loads and preprocesses CSV data (via `DataPreprocessor`).
- Trains a local `LinearRegressionModel` defined in `model_manager.sientia.models`.
- Computes metrics and builds a `TrainModelResult` for downstream steps.
- `ModelRepository` (`model_manager/utils/repository/model_repository.py`):
- Uses `ModelServing` and `Reports` to:
- Generate reports and CSV artifacts.
- Log parameters, metrics and models into MLflow.
- Today, production models are identified using **stages** (for example `Production`) in the Model Registry; aliases like `@production` are not yet used.
- MLflow-related logic here overlaps conceptually with the MLflow interactions implemented in `sientia-dataops-laborious_temporal`, which motivates extracting a **shared MLflow repository** into `sientia-dataops-library` (see `mlflow-shared-repository-migration-plan`).
- **Coupling to models and runtimes**
- Training is tightly coupled to internal classes (`DataPreprocessor`, `LinearRegressionModel`).
- No runtime installation; env assumed ready.
- The system does not yet use PluginStores `get_model` or `install_runtime` capabilities.
**MLflow:** All MLflow operations (lookup, promotion, etc.) → [[mlflow-shared-repository-migration-plan|shared repository]]. Model Manager uses the interface; it does not implement these concepts.
### Current vs Target — High-level Flow
```mermaid
flowchart LR
subgraph currentState [Current State — Model Manager]
direction TB
WorkerMM["Temporal Worker\n(model_manager/worker.py)"]
ActivitiesMM["Activities\n(training, cleanup, etc.)"]
TrainRepo["TrainingRepository\n(local DataPreprocessor + LinearRegressionModel)"]
ModelRepoMM["ModelRepository\n(MLflow + reports)"]
WorkerMM -->|"Temporal tasks"| ActivitiesMM
ActivitiesMM -->|"train_model activity"| TrainRepo
TrainRepo -->|"TrainModelResult"| ModelRepoMM
ModelRepoMM -->|"experiments, runs, artifacts"| MLflowMM["MLflow Server"]
end
subgraph targetState [Target State — Model Manager]
direction TB
WorkerMM2["Temporal Worker\n+ Runtime bootstrap\n(read RUNTIME, install runtime)"]
ActivitiesMM2["Activities\n(+ ModelProvider)"]
PluginStoreNode["PluginStore\n(Gitea model store)"]
SientiaWrapper["SientiaModel wrapper\n(from store.get_model)"]
ModelRepoMM2["ModelRepository\n(MLflow + reports)"]
WorkerMM2 -->|"install_runtime(RUNTIME)"| PluginStoreNode
WorkerMM2 -->|"Temporal tasks"| ActivitiesMM2
ActivitiesMM2 -->|"get_training_model"| PluginStoreNode
PluginStoreNode -->|"SientiaModel instance"| SientiaWrapper
ActivitiesMM2 -->|"call train(...) on wrapper"| SientiaWrapper
SientiaWrapper -->|"TrainModelResult-compatible data"| ModelRepoMM2
ModelRepoMM2 -->|"experiments, runs, artifacts"| MLflowMM2["MLflow Server"]
end
```
### Current vs Target — Training Hot Path (Code Sketch)
Current hot path in `TrainingRepository.train` (simplified):
```python
data = load_data(uploaded_file, params.line_separator, params.decimal_separator)
data = _ensure_date_column_parsed(data, params)
data = self._configure_datetime_index(data, params)
process_data = self._init_data_preprocessor(params)
process_data.fit(data)
data_view = process_data.transform(data)
x_train, x_test, y_train, y_test = split_train_test(...)
regr = LinearRegressionModel(
target_variable=params.target_variable,
variable_columns=params.variable_columns,
degree=params.degree,
interaction_only=params.interaction_only,
)
regr.fit(pd.concat([x_train, y_train], axis=1))
```
Target hot path with PluginStore + `SientiaModel` (conceptual):
```python
data = load_and_preprocess(uploaded_file, params) # keep existing preprocessing
train_df, val_df = build_train_val_splits(data, params) # explicit train/val
wrapper = model_provider.get_training_model(
model_name=params.model_name,
runtime=os.environ["RUNTIME"],
opt_params={"env": params.environment},
model_kwargs={},
data_model_kwargs={},
)
wrapper.train(
train_data=train_df,
val_data=val_df,
target=params.target_variable,
)
# From here, metrics and artifacts are computed based on wrapper outputs
```
---
## Requirements Mapping
### Functional Requirements
| ID | Requirement | Description | Impacted Areas |
|-------|----------------------------------------------|------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------|
| FR-01 | Runtime detection via env var | Before starting workers, read `RUNTIME` env var and fail fast if missing or empty | `model_manager/worker/worker.py` |
| FR-02 | Runtime installation via PluginStore | Install the runtime defined in `RUNTIME` using `PluginStore.install_runtime(runtime_name)` | `worker.py`, `sientia_model.model_repository.plugin_store.PluginStore` |
| FR-03 | PluginStore-based model retrieval | Use `PluginStore.get_model(...)` to obtain `SientiaModel` instances for training | `TrainingRepository` (or new adapter), `Training` activity |
| FR-04 | Remove mlops-library-based construction | Do not construct models directly from `model_manager.sientia.models`; remove the mlops path | `TrainingRepository`, `model_manager.sientia.models`, dependency graph |
| FR-05 | Use shared MLflow repository | Delegate all MLflow ops (runs, metrics, artifacts, production lookup) to `SientiaMLflowRepository`; keep reports and `TrainModelResult` in Model Manager | `ModelRepository`, [[mlflow-shared-repository-migration-plan]] |
| FR-06 | Centralized PluginStore configuration | Configure PluginStore (Gitea URL, repo, auth, PyPI mirror) via env/config | Config helper, `worker.py` |
### Non-Functional Requirements
| ID | Requirement | Description | Impacted Areas |
|--------|------------------------------------------|-------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------|
| NFR-01 | Robust runtime bootstrap | If runtime installation fails, the worker must not start; errors must be explicit in logs/metrics | `worker.py`, PluginStore error handling |
| NFR-02 | Single path | Use PluginStore + SientiaModel path only; no toggling or parallel old path | `worker.py`, Training activities |
| NFR-03 | Observability and debuggability | Logs and metrics must cover runtime detection, install attempts and PluginStore interactions | Logging around runtime and PluginStore |
| NFR-04 | Testability | Unit and integration tests must cover runtime detection, install, and model retrieval | `tests/worker`, `tests/activities`, new PluginStore tests |
| NFR-05 | Security | No credentials hard-coded in source; rely on env/secret management | Config helpers, deployment manifests |
---
## Target Architecture
### Runtime bootstrap
- **New required env var:** `RUNTIME` (name of the runtime; must match a runtime in the store index).
- **New PluginStore configuration env vars** (names to be finalized):
- `STORE_BASE_URL`, `STORE_OWNER`, `STORE_REPO`
- Optional: `STORE_BRANCH`, `STORE_USERNAME`, `STORE_PASSWORD`, `PYPI_INDEX_URL`, `PYPI_USERNAME`, `PYPI_PASSWORD`
- **Worker startup sequence** in `main()`:
1. Initialize logger and basic metadata as today.
2. Read `RUNTIME` from env; if missing/empty → log critical error and exit.
3. Build `PluginStore` instance using configuration env vars.
4. Call `store.install_runtime(runtime_name=RUNTIME)`; on failure → log error, set `APP_UP` metric to 0 and exit.
5. Only after successful runtime installation → build connector configs, instantiate Activities, connect to Temporal.
### PluginStore usage in training
- **ModelProvider abstraction** (e.g. `model_manager/utils/model_provider.py`):
- Holds a `PluginStore` instance and logger.
- Exposes `get_training_model(model_name, runtime, opt_params, model_kwargs, data_model_kwargs) -> SientiaModel` via `store.get_model(...)`.
- **TrainingRepository integration:**
- Prepare input data using existing preprocessing logic.
- Ask ModelProvider for a `SientiaModel` instance (`model_name` from params, `runtime` from `RUNTIME`).
- Call the public `train(...)` method of the wrapper; collect outputs and metadata only through the public API.
- Build `TrainModelResult` from train/test splits, predictions, and artifacts required by `ModelRepository`.
- Model behavior lives in `SientiaModel`; manager focuses on orchestration and reports.
---
## Implementation Plan
### Phase 0 — Design Alignment
- **P0-01**: Confirm with stakeholders:
- The expected values and semantics for `RUNTIME` (naming convention, mapping to store runtimes).
- Whether the manager will ever need to support more than one runtime per process (current assumption: no).
- **P0-02**: Decide the config strategy:
- Pure environment variables vs a config file + env overrides.
- **P0-03**: Validate how `model_name` will be passed into the training workflow:
- Confirm or extend `TrainModelParams` to carry `model_name` and runtime-related fields as needed.
### Phase 1 — Runtime Detection and Installation
- **P1-01**: Extend `worker.py` docs and configuration:
- Add `RUNTIME` to the environment variables list in the module docstring.
- Document failure behavior when `RUNTIME` is missing or empty.
- **P1-02**: Implement `build_plugin_store_from_env` helper:
- New function that:
- Reads `STORE_*` and `PYPI_*` env vars.
- Creates and returns a `PluginStore` instance with appropriate logger.
- Place it either in `worker.py` or in a dedicated utility module (e.g. `model_manager/utils/plugin_store_config.py`).
- **P1-03**: Integrate runtime installation into `main()`:
- Before any Temporal client initialization:
- Read `runtime_name = os.getenv("RUNTIME")`.
- Build PluginStore via the helper.
- Call `install_runtime(runtime_name)`.
- Log:
- Start and end of runtime installation.
- List of installed requirements (name and version).
- On failure:
- Emit a clear message (including runtime name and store repo).
- Mark the app as DOWN in metrics.
- Exit with non-zero status.
### Phase 2 — ModelProvider and Training Integration
- **P2-01**: Introduce `ModelProvider` abstraction:
- Implement `ModelProvider` with:
- A reference to the shared `PluginStore`.
- Methods for retrieving models for training:
- `get_training_model(model_name, runtime, opt_params, model_kwargs, data_model_kwargs)`.
- Ensure it logs:
- Model and runtime names.
- Cache hits/misses when appropriate.
- **P2-02**: Wire ModelProvider into Activities:
- Update the construction of `Activities` in `worker.py` to accept:
- A `ModelProvider` or a `PluginStore` that can be wrapped inside the `Training` activity.
- Update `Training.__init__` signature to accept this new dependency and store it as an attribute.
- **P2-03**: Adjust `TrainingRepository` to use `SientiaModel`:
- In `train`:
1. Keep or refine current CSV loading and preprocessing pipeline (DataPreprocessor, split into train/test).
2. Use `ModelProvider` to retrieve the model:
- `model_name` from `TrainModelParams`.
- `runtime` from `RUNTIME`.
- `opt_params` mirroring how `test_plugin_store.py` interacts with models.
3. Call the models `train` method and produce the same core artifacts the current code expects:
- Train/test splits.
- Metrics and predictions for `TrainModelResult`.
- In `after_train_calculation`:
- Either re-use metrics from the SientiaModel, or continue calculating MSE/MAE/R² as a verification step.
### Phase 3 — Removing mlops Dependencies
- **P3-01**: Identify all usage points of:
- `LinearRegressionModel`.
- `DataPreprocessor` where behavior overlaps with what SientiaModel already does.
- **P3-02**: Remove mlops-based code:
- Remove unused mlops-library-based training hooks.
- Simplify `TrainingRepository` to delegate as much as possible to SientiaModel logic.
- Use PluginStore + `SientiaModel` path only (no migration flag or parallel old path).
### Phase 4 — ModelRepository, Shared MLflow Repository and Reporting Alignment
- **P4-01**: Ensure `TrainModelResult` is correctly populated:
- Confirm that:
- `x_train`, `x_test`, `y_train`, `y_test`, `y_pred`, `y_train_pred` are provided by the new flow.
- Any fields required by `_generate_report` and `_save_run` remain available.
- **P4-02**: Delegate all MLflow operations to `SientiaMLflowRepository` (see [[mlflow-shared-repository-migration-plan]]); keep in Model Manager only reporting orchestration and `TrainModelResult` construction.
- **P4-03**: Validate that MLflow reports remain consistent:
- Run a side-by-side comparison between:
- A run from the previous (mlops-based) pipeline.
- A PluginStore-based run for the same dataset/experiment using the shared repository.
- Compare:
- Logged parameters.
- Metrics.
- Artifacts (reports, CSVs, equation JSON).
---
## Testing Strategy
- **T1 — Unit tests**
- Worker: Test behavior when `RUNTIME` is missing or empty; test that `install_runtime` is called with the correct runtime name (mock PluginStore).
- ModelProvider: Test that it calls `PluginStore.get_model` with the expected arguments.
- TrainingRepository: Test that it uses the model returned by PluginStore instead of `LinearRegressionModel`.
- **T2 — Integration tests**
- Set up a test store with a minimal model and runtime.
- Run a full training workflow: verify runtime installation first; confirm model is retrieved and training completes; validate metrics and artifacts via MLflow.
- **T3 — Regression tests**
- Run the same experiment once with the old pipeline and once with the PluginStore-based pipeline.
- Compare key results (metrics and artifacts) to ensure differences are understood and acceptable.
---
## Rollout and Migration Strategy
- **R1 — Criar modelos com a nova arquitetura**
- Garantir que o pipeline de modelagem (Factory/Warehouse/Store) consiga produzir modelos:
- Encapsulados em wrappers que estendem `SientiaModel`.
- Com interface estável para `train`, `retrain` (quando existir), `predict`/`transform` e `store_model`.
- Publicar um conjunto inicial de modelos “pilot” no store que será consumido pelo Model Manager.
- **R2 — Subir um ou mais runtimes para esses modelos**
- Configurar e instalar runtimes específicos para os novos modelos, alinhados ao `RUNTIME` esperado pelo Model Manager:
- Verificar que cada runtime contém todas as dependências necessárias (via PluginStore / runtime installer).
- Validar que, em um ambiente de teste, o runtime consegue:
- Instalar bibliotecas.
- Carregar o wrapper via PluginStore e executar pelo menos um ciclo de treino de ponta a ponta.
- **R3 — Colocar os novos modelos para rodar no Model Manager**
- Integrar o uso de `PluginStore.get_model(...)` na pipeline de treino:
- Direcionar um subconjunto de fluxos de treinamento para os modelos “pilot” oriundos do store.
- Monitorar:
- Estabilidade dos workers.
- Tempo de treino e consumo de recursos.
- Artefatos e métricas geradas no MLflow.
- Quando estáveis, coordenar com as equipes de produto/negócio para considerar esses modelos como candidatos a produção e, quando já estiverem usando MLflow 3+ com wrappers, promover esses modelos para produção usando aliases (`@production`).
- **R4 — Migrar progressivamente os demais modelos**
- Definir uma ordem de migração por domínio/família de modelo (por exemplo: modelos de predição de série temporal, modelos de classificação, etc.):
- Para cada modelo legado:
- Criar/ajustar o wrapper `SientiaModel` correspondente no Factory/Warehouse.
- Garantir que o modelo passe a ser entregue pelo store e consumido via PluginStore.
- Executar o ciclo de testes (T1T3) descrito na seção anterior.
- Após migrar todas as famílias de modelo:
- Remover o caminho de código que instancia diretamente `LinearRegressionModel` e demais classes locais.
- Limpar variáveis de configuração relacionadas à pipeline antiga (mlops library local).
- Assumir como padrão único: Model Manager treinando apenas modelos vindos do store, via wrappers `SientiaModel` e, quando aplicável, com resolução de produção por aliases em MLflow 3+.
---
## Potential Model Library Changes
- **Clarifying the SientiaModel training interface:**
- Ensure that `SientiaModel` provides a stable way to train (including validation split handling).
- A clear contract for returning predictions and metrics.
- Access to any internal state needed for `TrainModelResult` construction.
- **Improving PluginStore ergonomics:**
- Optionally add a helper to build `PluginStore` from environment variables (reusable across applications).
- More structured exceptions for missing runtime definitions, missing models, network and authentication issues.
- These changes should be coordinated so that model manager and any other consumers can rely on a consistent, documented behavior.
---
## Related Documents
- [[mlflow-shared-repository-migration-plan|MLflow Shared Repository Migration Plan]] — Concepts implemented in the common interface (production lookup, wrapper loading, promotion, etc.)
- [[analytics-implementation-plan|Runtime Analytics Helm Implementation Plan]]
- [[analytics|Runtime Analytics Architecture and Analysis]]
- [[../model-plugin-system/06-end-to-end-flow|Model Plugin System — End-to-End Flow]]

View File

View File

View File

@@ -0,0 +1,166 @@
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from typing import Any
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
from sientia_model.model_repository.plugin_store import PluginStore
from model_manager.activities.cleanup import Cleanup
from model_manager.activities.experiment_tracking import ExperimentTracking
from model_manager.activities.training import Training
class Activities(ExperimentTracking, Training, Cleanup):
"""
Main activities orchestrator for the Model Manager system.
This class combines functionality from multiple activity classes to provide
a unified interface for all workflow operations. It manages database connections,
MLFlow model interactions, MinIO storage operations, and cleanup operations.
The class implements multiple inheritance to combine specialized functionality:
- ExperimentTracking: ML experiment lifecycle tracking and database operations
- Training: ML model training operations with MLFlow and MinIO integration
- Cleanup: File and directory cleanup operations for MinIO and local filesystem
Attributes:
postgres_config (dict): PostgreSQL connection configuration
mlflow_config (dict): MLFlow server configuration
minio_config (dict): MinIO storage configuration
logger (Logger): Logging and observability instance
notification_handler (NotificationHandler): Notification management instance
"""
def __init__(
self,
postgres_config: dict[str, Any],
mlflow_config: dict[str, Any],
minio_config: dict[str, Any],
plugin_store: PluginStore,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
"""
Initialize the Activities orchestrator with all required configurations.
This constructor initializes all parent classes with their respective
configurations and sets up the foundation for all activity operations.
Args:
postgres_config: PostgreSQL connection configuration dictionary
Required keys: host, port, user, password, dbname, min_connections, max_connections
mlflow_config: MLFlow server configuration dictionary
Required keys: host, port, username, password
minio_config: MinIO storage configuration dictionary
Required keys: endpoint_url, access_key, secret_key, region, use_ssl, default_bucket
logger: Logger instance for observability and debugging
notification_handler: Notification handler for alerts and monitoring
Raises:
Exception: If any parent class initialization fails
"""
ExperimentTracking.__init__(
self,
host=postgres_config['host'],
port=postgres_config['port'],
user=postgres_config['user'],
password=postgres_config['password'],
dbname=postgres_config['dbname'],
min_connections=postgres_config['min_connections'],
max_connections=postgres_config['max_connections'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
self.mlflow_repository = SientiaMLflowRepository(
host=mlflow_config['url'],
username=mlflow_config['username'],
password=mlflow_config['password'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
# MinIO repository used for all object storage operations
endpoint_url = minio_config['endpoint_url']
# MinioRepository expects the endpoint without scheme
if endpoint_url.startswith('http://'):
endpoint = endpoint_url.removeprefix('http://')
elif endpoint_url.startswith('https://'):
endpoint = endpoint_url.removeprefix('https://')
else:
endpoint = endpoint_url
self.minio_repository = MinioRepository(
endpoint=endpoint,
access_key=minio_config['access_key'],
secret_key=minio_config['secret_key'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
secure=minio_config['use_ssl'],
bucket=minio_config['default_bucket'],
)
Training.__init__(
self,
mlflow_repository=self.mlflow_repository,
plugin_store=plugin_store,
minio_repository=self.minio_repository,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
Cleanup.__init__(
self,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
def __del__(self):
"""
Destructor to safely handle cleanup during garbage collection.
This prevents AttributeError when the parent Postgres.__del__ tries to access
self.engine in objects with multiple inheritance. Only attempts cleanup if
the engine attribute exists.
"""
# Only call parent __del__ if engine attribute exists
# This prevents AttributeError in multiple inheritance scenarios
if hasattr(self, 'engine'):
try:
# Call parent class __del__ if it exists
if hasattr(super(), '__del__'): # pragma: no cover
super().__del__() # pragma: no cover
except Exception: # noqa: S110, BLE001 # pragma: no cover
# Silently ignore errors during garbage collection
# Logging here could cause issues if logger is already destroyed
pass
def shutdown(self):
"""
Gracefully shutdown all activities and clean up resources.
This method ensures proper cleanup of all resources including:
- PostgreSQL connection pools (via ExperimentTracking)
- Any other resources that need explicit cleanup
The method should be called before the application terminates to ensure
proper resource cleanup and prevent resource leaks.
Prefer calling this method explicitly rather than relying on __del__.
"""
ExperimentTracking.close(self)
self.info('Postgres client closed')
SientiaMonitoring.shutdown(self)

View File

@@ -0,0 +1,221 @@
"""
Cleanup activities for removing stale files from local filesystem.
This module provides activities for cleaning up temporary files and directories
that are older than the configured retention period. It operates independently
of the database, using timestamps embedded in filenames.
"""
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import os
import re
import shutil
import traceback
from datetime import datetime, timedelta
from typing import Any
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from model_manager.metrics import ACTIVITY_EXECUTION_TOTAL, WORKFLOW_EXECUTION_TOTAL
from model_manager.runtime_paths import REPORTS_TEMP_DIR
RETENTION_HOURS = int(os.getenv('CLEANUP_RETENTION_HOURS', '24'))
DRY_RUN = os.getenv('CLEANUP_DRY_RUN', 'false').lower() == 'true'
class Cleanup(SientiaMonitoring):
"""
Activity for cleaning up stale files and directories.
This activity extends SientiaMonitoring and handles cleanup of:
- Local temporary directories with timestamp suffixes
"""
def __init__(
self,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
"""
Initialize Cleanup activity.
Args:
logger: Logger instance for observability
notification_handler: Handler for sending notifications
metrics_controller: Controller for metrics emission
"""
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
# Configuration from environment variables
self.retention_hours = RETENTION_HOURS
self.dry_run = DRY_RUN
# Regex patterns for timestamp extraction
self.dir_timestamp_pattern = re.compile(
r'^(.+)_(\d{8}_\d{6}_\d{6})$'
) # name_YYYYMMDD_HHMMSS_microseconds
@activity.defn(name='cleanup_temp_directories')
def cleanup_temp_directories(self, input_data: dict[str, Any]) -> None:
"""
Clean up stale temporary directories based on timestamp in directory name.
This activity scans the reports/temp directory for subdirectories following
the pattern '{name}_{timestamp}' where timestamp is in YYYYMMDD_HHMMSS_microseconds format.
Directories older than the retention period are deleted.
Args:
input_data: Cleanup configuration containing:
- metadata (dict): Workflow execution metadata
- temp_path (str): Path to temp directory (optional, defaults to reports/temp)
Returns:
None: Results are logged and tracked via metrics
Raises:
Exception: If cleanup fails (after sending notification)
"""
metadata = input_data.get('metadata', {})
temp_path = input_data.get('temp_path', REPORTS_TEMP_DIR)
metrics_status = 'success'
cutoff_time = datetime.now() - timedelta(hours=self.retention_hours)
try:
self.info(
f'Starting local directory cleanup - Path: {temp_path}, '
f'Retention: {self.retention_hours}h, Dry run: {self.dry_run}',
metadata,
)
if not os.path.exists(temp_path):
self.warning(f'Temp directory does not exist: {temp_path}', metadata)
return
directories_scanned = 0
directories_deleted = 0
errors = []
for item_name in os.listdir(temp_path):
item_path = os.path.join(temp_path, item_name)
if not os.path.isdir(item_path):
continue
directories_scanned += 1
# Extract timestamp from directory name
match = self.dir_timestamp_pattern.match(item_name)
if not match:
self.debug(
f'Skipping directory without timestamp pattern: {item_name}', metadata
)
continue
timestamp_str = match.group(2)
try:
# Parse YYYYMMDD_HHMMSS_microseconds
dir_time = datetime.strptime(timestamp_str, '%Y%m%d_%H%M%S_%f')
if dir_time < cutoff_time:
age_hours = (datetime.now() - dir_time).total_seconds() / 3600
if self.dry_run:
self.info(
f'[DRY RUN] Would delete directory: {item_name} (age: {age_hours:.1f}h)',
metadata,
)
directories_deleted += 1
else:
try:
shutil.rmtree(item_path)
self.info(
f'Deleted stale directory: {item_name} (age: {age_hours:.1f}h)',
metadata,
)
directories_deleted += 1
except OSError as e:
error_msg = f'Failed to delete directory {item_name}: {str(e)}'
errors.append(error_msg)
self.error(error_msg, metadata)
else:
age_hours = (datetime.now() - dir_time).total_seconds() / 3600
self.debug(
f'Keeping recent directory: {item_name} (age: {age_hours:.1f}h)',
metadata,
)
except ValueError as e:
error_msg = f'Invalid timestamp format in directory {item_name}: {str(e)}'
errors.append(error_msg)
self.error(error_msg, metadata)
self.info(
f'Directory cleanup completed - Scanned: {directories_scanned}, '
f'Deleted: {directories_deleted}, Errors: {len(errors)}',
metadata,
)
except Exception as e:
metrics_status = 'error'
error_msg = f'Error in directory cleanup: {str(e)}'
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id='CLEANUP_DIRECTORIES_ERROR',
message=error_msg,
block='cleanup_temp_directories',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
raise
finally:
self._emit_metrics(
metadata=metadata,
metrics_status=metrics_status,
activity_name='cleanup_temp_directories',
emit_workflow_metric=True,
)
def _emit_metrics(
self,
metadata: dict[str, Any],
metrics_status: str,
activity_name: str,
emit_workflow_metric: bool,
) -> None:
"""
Emit workflow and activity execution metrics.
Args:
metadata: Activity metadata containing pod_id and workflow_name
metrics_status: Execution status ('success' or 'error')
activity_name: Name of the activity being executed
"""
if emit_workflow_metric:
self.emit_metric_sync(
metric_object=WORKFLOW_EXECUTION_TOTAL,
tags={
'pod_id': metadata.get('pod_id'),
'workflow_name': metadata.get('workflow_name'),
'status': metrics_status,
},
)
self.emit_metric_sync(
metric_object=ACTIVITY_EXECUTION_TOTAL,
tags={
'pod_id': metadata.get('pod_id'),
'activity_name': activity_name,
'status': metrics_status,
},
)

View File

@@ -0,0 +1,280 @@
"""
Experiment tracking activities for managing ML experiment lifecycle.
This module provides activities for tracking and updating experiment run status
in the PostgreSQL database, extending the synchronous Postgres client with specialized
methods for experiment management.
"""
import enum
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import traceback
from collections.abc import Mapping
from datetime import UTC, datetime
from typing import Any
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.temporal.activities.postgres_sync import Postgres
from sqlalchemy import text
class UpdateType(enum.StrEnum):
"""Types of experiment run updates."""
STATUS = 'status'
STATUS_WITH_ERROR = 'status_with_error'
MODEL_SAVED = 'model_saved'
class ExperimentTracking(Postgres):
"""
Activity for tracking ML experiment lifecycle and status updates.
This activity extends the Postgres activity to provide specialized methods
for managing experiment runs, including status updates, error tracking, and
model registration. It maintains the experiment lifecycle from initialization
through training, model saving, and cleanup.
The activity uses a SQLAlchemy engine / connection pool and adds experiment-specific
operations with proper error handling and notifications.
"""
def __init__(
self,
host: str,
port: int,
user: str,
password: str,
dbname: str,
min_connections: int,
max_connections: int,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
"""
Initialize ExperimentTracking activity with database configuration.
Args:
host: PostgreSQL server hostname
port: PostgreSQL server port
user: Database user
password: Database password
dbname: Database name
min_connections: Minimum connections in pool
max_connections: Maximum connections in pool
logger: Logger instance for observability
notification_handler: Notification handler for alerts
metrics_controller: Metrics controller for observability
Raises:
ConnectionError: If database connection cannot be established
"""
Postgres.__init__(
self,
host=host,
port=port,
user=user,
password=password,
dbname=dbname,
min_connections=min_connections,
max_connections=max_connections,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
self.info(f'Postgres client initialized at {host}:{port}')
def __del__(self):
"""
Destructor to safely handle cleanup during garbage collection.
This prevents AttributeError when used in multiple inheritance scenarios
where the parent Postgres.__del__ might be called on objects without
the engine attribute.
"""
# Only call parent __del__ if engine attribute exists
if hasattr(self, 'engine'):
try:
if hasattr(super(), '__del__'):
super().__del__()
except Exception: # noqa: S110, BLE001
# Silently ignore errors during garbage collection
pass
def _execute_update(self, query: str, params: Mapping[str, Any]) -> dict[str, Any]:
"""
Execute an UPDATE SQL statement.
Args:
query: Parameterized SQL string to execute.
params: Mapping of parameters for the SQL query.
Returns:
dict: A dictionary containing the affected row count: {'rowcount': int}.
"""
with self.engine.begin() as connection:
result = connection.execute(text(query), params)
return {'rowcount': result.rowcount}
def _build_status_update_query(
self, status: str | None, experiment_run_id: int
) -> tuple[str, dict[str, Any]]:
"""Build SQL query for simple status update."""
if not isinstance(status, str) or not status:
raise ValueError('status is required for STATUS update type')
sql_query = """
UPDATE experiment_run
SET status = :status, updated_at = :updated_at
WHERE id = :experiment_run_id
"""
query_params = {
'status': status,
'updated_at': datetime.now(UTC),
'experiment_run_id': experiment_run_id,
}
return sql_query, query_params
def _build_status_with_error_query(
self, status: str | None, error_message: str | None, experiment_run_id: int
) -> tuple[str, dict[str, Any]]:
"""Build SQL query for status update with error message."""
if not isinstance(status, str) or not status:
raise ValueError('status is required for STATUS_WITH_ERROR update type')
if not isinstance(error_message, str) or not error_message:
raise ValueError('error_message is required for STATUS_WITH_ERROR update type')
# Truncate error message if too long
truncated_error = error_message[:1024] if len(error_message) > 1024 else error_message
sql_query = """
UPDATE experiment_run
SET status = :status, error_message = :error_message, updated_at = :updated_at
WHERE id = :experiment_run_id
"""
query_params = {
'status': status,
'error_message': truncated_error,
'updated_at': datetime.now(UTC),
'experiment_run_id': experiment_run_id,
}
return sql_query, query_params
def _build_model_saved_query(
self, run_name: str | None, status: str | None, experiment_run_id: int
) -> tuple[str, dict[str, Any]]:
"""Build SQL query for model saved update."""
if not isinstance(run_name, str) or not run_name:
raise ValueError('run_name is required for MODEL_SAVED update type')
if not isinstance(status, str) or not status:
raise ValueError('status is required for MODEL_SAVED update type')
sql_query = """
UPDATE experiment_run
SET run_name = :run_name, status = :status, updated_at = :updated_at
WHERE id = :experiment_run_id
"""
query_params = {
'run_name': run_name,
'status': status,
'updated_at': datetime.now(UTC),
'experiment_run_id': experiment_run_id,
}
return sql_query, query_params
def _get_update_query_and_params(
self, update_type: str, experiment_run_id: int, input_data: dict[str, Any]
) -> tuple[str, dict[str, Any]]:
"""Get SQL query and parameters based on update type."""
status = input_data.get('status')
error_message = input_data.get('error_message')
run_name = input_data.get('run_name')
if update_type == UpdateType.STATUS:
return self._build_status_update_query(status, experiment_run_id)
if update_type == UpdateType.STATUS_WITH_ERROR:
return self._build_status_with_error_query(status, error_message, experiment_run_id)
if update_type == UpdateType.MODEL_SAVED:
return self._build_model_saved_query(run_name, status, experiment_run_id)
raise ValueError(f'Invalid update_type: {update_type}')
@activity.defn(name='update_experiment_run')
def update_experiment_run(self, input_data: dict[str, Any]) -> None:
"""
Update experiment run with status, errors, or model information.
This activity provides a unified interface for all experiment run updates,
supporting different update types through a single method. It automatically
selects the appropriate SQL query based on the update type and parameters.
Args:
input_data: Configuration for experiment run update operation
Required keys:
- metadata (dict): Workflow execution metadata
- experiment_run_id (int): Unique identifier for the experiment run
- update_type (str): Type of update (status, status_with_error, model_saved)
Optional keys:
- status (str): New status for the experiment run
- error_message (str): Error message if update failed
- run_name (str): MLFlow run name if model was saved
Raises:
ValueError: If required parameters are missing for the update type
RuntimeError: If update operation fails
"""
metadata = input_data.get('metadata')
experiment_run_id = input_data['experiment_run_id']
update_type = input_data['update_type']
status = input_data.get('status')
try:
sql_query, query_params = self._get_update_query_and_params(
update_type, experiment_run_id, input_data
)
result = self._execute_update(sql_query, query_params)
if result.get('rowcount', 0) == 0:
error_msg = (
f'No experiment_run row updated for id={experiment_run_id} '
f'(row missing or id mismatch). update_type={update_type!r}, status={status!r}.'
)
raise ValueError(error_msg)
self.info(
f'Successfully updated experiment run {experiment_run_id} with status {status}',
metadata,
)
except Exception as e: # noqa: BLE001
error_msg = f'Error updating experiment run - ID: {experiment_run_id}, Status: {status}, Error: {str(e)}'
trace = traceback.format_exc()
self.send_notification(
metadata=metadata or {},
notification_id='UPDATE_EXPERIMENT_RUN_ERROR',
message=error_msg,
block='update_experiment_run',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
raise RuntimeError(error_msg) from e

View File

@@ -0,0 +1,399 @@
"""
Training activities for ML model training operations.
This module provides activities for training machine learning models.
The activity extends BaseActivity and receives pre-downloaded files
and raises `ModelTrainingError` when training fails.
"""
from sientia_model.wrappers.sientia_model import SientiaModel
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import traceback
from typing import Any
import mlflow
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
from sientia_model.model_repository.plugin_store import PluginStore
from model_manager.utils.models.train_model_params import TrainModelParams
from model_manager.utils.models.train_model_result import TrainModelResult
from model_manager.utils.repository.data_manager_repository import DataManagerRepository
class Training(SientiaMonitoring):
"""
Activity for ML model training operations.
This activity extends SientiaMonitoring and handles machine learning model
training with comprehensive error handling. It receives pre-downloaded
files from the workflow and raises `ModelTrainingError` on failure so the
workflow can map the correct experiment status.
"""
def __init__(
self,
mlflow_repository: SientiaMLflowRepository,
plugin_store: PluginStore,
minio_repository: MinioRepository,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
"""
Initialize Training activity.
Args:
logger: Logger instance for observability
notification_handler: Handler for sending notifications
"""
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
self.data_manager_repository = DataManagerRepository(logger)
self.mlflow_repository = mlflow_repository
self.plugin_store = plugin_store
self.minio_repository = minio_repository
@activity.defn(name='load_model_metadata')
def load_model_metadata(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Load model metadata/schemas from the model store.
This activity is responsible for fetching model metadata/schemas from the
model store index and extracting a serializable `model_metadata` dict that
`TrainModelParams.validate_business_rules()` depends on.
Args:
input_data: Workflow input at the same level as `validate_train_params`,
including at least `model_name` and the fields required by
`TrainModelParams.from_dict` to build wrapper kwargs.
Return:
dict[str, Any]: Updated `input_data` containing `input_data['model_metadata']`.
"""
metadata = input_data.get('metadata', {})
self.info(f'Loading model metadata for {input_data}', metadata)
try:
train_params = TrainModelParams.from_dict(input_data)
model_metadata = self.plugin_store.get_model_index(
model_type=train_params.model_type,
metadata=metadata,
)
train_params.model_metadata = model_metadata
self.info(f'Model metadata loaded successfully for {input_data}', metadata)
self.debug(f'Model metadata: {model_metadata}', metadata)
return train_params.to_dict()
except Exception as exc:
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id='LOAD_MODEL_METADATA_ERROR',
message=f'Error loading model metadata: {str(exc)}',
block='load_model_metadata',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
raise
@activity.defn(name='validate_train_params')
def validate_train_params(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Validate and convert training parameters from dict to TrainModelParams.
This activity validates the input training parameters and converts them
to a TrainModelParams object.
Args:
input_data: Training parameters and metadata at the same level
Required keys:
- metadata (dict): Workflow execution metadata
- All TrainModelParams fields (experiment_run_id, target_variable, etc.)
Returns:
dict[str, Any]: Validated and converted training parameters as dictionary
Raises:
Exception: If validation fails (after sending notification)
"""
metadata = input_data.get('metadata', {})
self.info(f'Validating training parameters for {input_data}', metadata)
try:
train_params = TrainModelParams.from_dict(input_data)
train_params.validate_business_rules()
self.info(
f'Training parameters validated successfully - '
f'Target: {train_params.target_variable}, '
f'Experiment: {train_params.experiment_name}',
metadata,
)
self.debug(
f'Training parameters validated successfully: {train_params.to_dict()}', metadata
)
return train_params.to_dict()
except Exception as e:
error_msg = f'Error validating training parameters: {str(e)}'
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id='VALIDATE_TRAIN_PARAMS_ERROR',
message=error_msg,
block='validate_train_params',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
raise
@activity.defn(name='train_model')
def train_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Train a machine learning model.
This activity orchestrates the ML training pipeline:
1. Validate input parameters.
2. Prepare data via DataManagerRepository.
3. Train the model and compute metrics.
Args:
input_data: Training configuration containing:
- metadata (dict): Workflow execution metadata.
- uploaded_file (BytesIO): Training data already downloaded from MinIO.
- train_params (dict): Training parameters.
Returns:
dict[str, Any]: Serializable summary (run identifiers, run_dir for cleanup, regression metrics).
Raises:
ValueError: If input validation fails.
Exception: If training fails (after sending notification).
"""
metadata = input_data.get('metadata')
train_params = TrainModelParams.from_dict(input_data['train_params'])
self.info('Starting train_model process', metadata)
try:
# Download training file bytes from MinIO
self.info(
f'Downloading training file from MinIO for {train_params.file_name}', metadata
)
train_bytes = self.minio_repository.download_file(
object_name=train_params.file_name,
bucket=train_params.bucket_name,
metadata=metadata,
)
# Download optional validation file bytes from the same bucket
val_bytes: bytes | None = None
validation_name = train_params.val_file_name
if validation_name is not None:
self.info(f'Downloading validation file from MinIO for {validation_name}', metadata)
val_bytes = self.minio_repository.download_file(
object_name=validation_name,
bucket=train_params.bucket_name,
metadata=metadata,
)
self.info(f'Preparing training data for {train_params.file_name}', metadata)
train_result = self.data_manager_repository.prepare_training_data(
train_file_bytes=train_bytes,
validation_file_bytes=val_bytes,
params=train_params,
metadata=metadata,
)
self.info(f'Getting model wrapper for {train_params.model_type}', metadata)
wrapper = self.plugin_store.get_model(
model_type=train_params.model_type,
force_download=False,
opt_params=train_params.opt_params or {},
model_kwargs=train_params.model_kwargs or {},
data_model_kwargs=train_params.data_model_kwargs or {},
metadata=metadata,
)
if self.logger is not None:
wrapper.logger = self.logger.base_logger
self.info(f'Training model for {train_params.model_type}', metadata)
train_data = train_result.train_data
val_data = train_result.val_data
self.debug(
f'train_model prepared data (head 10):\ntrain:\n{train_data.head(10).to_string()}'
f'\nval:\n{val_data.head(10).to_string()}',
metadata,
)
wrapper.train(
train_data=train_data,
val_data=val_data,
target=train_params.target_variable,
)
self.info(
f'Generating predictions using the trained wrapper for {train_params.model_type}',
metadata,
)
# Generate predictions using the trained wrapper
transformed_train, _ = wrapper.transform(train_data)
transformed_val, _ = wrapper.transform(val_data)
self.debug(
f'train_model transform (head 10):\ntrain:\n{transformed_train.head(10).to_string()}'
f'\nval:\n{transformed_val.head(10).to_string()}',
metadata,
)
y_train_pred_df, _ = wrapper.predict({}, transformed_train)
y_val_pred_df, _ = wrapper.predict({}, transformed_val)
self.debug(
f'train_model predict (head 10):\ntrain:\n{y_train_pred_df.head(10).to_string()}'
f'\nval:\n{y_val_pred_df.head(10).to_string()}',
metadata,
)
y_train_pred_df.sort_index(inplace=True, ascending=False)
y_val_pred_df.sort_index(inplace=True, ascending=False)
train_result.y_train_pred = y_train_pred_df
train_result.y_pred = y_val_pred_df
self.info(f'Computing regression metrics for {train_params.model_type}', metadata)
train_result = self.data_manager_repository.compute_regression_metrics(
train_result,
wrapper,
metadata=metadata,
)
self.info(f'Starting MLflow run for {train_params.model_type}', metadata)
with self.mlflow_repository.start_run(
model_name=train_params.model_name,
run_name=train_result.run_name,
experiment_name=train_result.experiment_name,
tags=None,
metadata=metadata,
) as run_info:
train_result.run_id = run_info.run_id
self._persist_training_artifacts(train_result, train_params, wrapper, metadata)
return {
'run_name': train_result.run_name,
'experiment_name': train_result.experiment_name,
'run_id': train_result.run_id,
'run_dir': train_result.run_dir,
}
except Exception as e: # noqa: BLE001
error_msg = f'Error training model - error: {str(e)}'
trace = traceback.format_exc()
self.send_notification(
metadata=metadata or {},
notification_id='TRAIN_MODEL_ERROR',
message=error_msg,
block='train_model',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
raise e
def _persist_training_artifacts(
self,
train_result: TrainModelResult,
train_params: TrainModelParams,
wrapper: SientiaModel,
metadata: dict[str, Any] | None,
) -> None:
self.info(f'Generating report for {train_params.model_type}', metadata)
train_result = self.data_manager_repository.generate_report(
train_result,
metadata=metadata,
)
if (
train_result.report_path is None
or train_result.train_data_path is None
or train_result.test_data_path is None
):
raise ValueError('Report path, train data path, or test data path is not set')
self.info(f'Storing model for {train_params.model_type}', metadata)
wrapper._input_example = None
wrapper.store_model(name=train_params.model_name)
self._log_regression_metrics_as_params(train_result)
self.info(f'Logging artifacts for {train_params.model_type}', metadata)
mlflow.log_artifact(train_result.report_path)
mlflow.log_artifact(train_result.train_data_path)
mlflow.log_artifact(train_result.test_data_path)
if train_result.equation_path is not None:
mlflow.log_artifact(train_result.equation_path)
def _log_regression_metrics_as_params(self, train_result: TrainModelResult) -> None:
"""
Persist computed regression metrics as MLflow params.
Args:
train_result: Training output containing computed regression metrics.
"""
metric_params = {
'mse_val': train_result.mse_val,
'mae_val': train_result.mae_val,
'r2_val': train_result.r2_val,
}
for key, value in metric_params.items():
if value is not None:
mlflow.log_param(key, value)
@activity.defn(name='cleanup_resources')
def cleanup_resources(self, input_data: dict[str, Any]) -> None:
"""
Cleanup temporary resources created during training.
Args:
input_data: Cleanup configuration containing:
- metadata (dict): Workflow execution metadata.
- run_dir (str): Temporary directory to remove.
Raises:
Exception: If cleanup fails (after sending notification).
"""
metadata = input_data.get('metadata', {})
run_dir = input_data.get('run_dir', '')
try:
self.data_manager_repository.cleanup_run_directory(run_dir, metadata)
except Exception as e: # noqa: BLE001
error_msg = f'Error cleaning up resources - Run directory: {run_dir}, Error: {str(e)}'
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id='CLEANUP_RESOURCES_ERROR',
message=error_msg,
block='cleanup_resources',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
raise

38
model_manager/metrics.py Normal file
View File

@@ -0,0 +1,38 @@
"""
Model Manager Metrics Module
This module defines all Prometheus metrics used by the Sientia DataOps Model Manager system
for monitoring and observability. The metrics provide insights into system performance,
training operations, and operational health.
The metrics are designed to be scraped by Prometheus and can be visualized in
Grafana or other monitoring dashboards to provide real-time visibility into
the system's operation.
Key Metric Categories:
- Application Health: Overall system status and availability
Metric Labels:
- pod_id: Kubernetes pod identifier for multi-instance deployments
"""
from prometheus_client import Counter, Gauge
# Application health metric
APP_UP = Gauge(
'app_up',
'Indicates if the application is running (1) or shutting down (0)',
['pod_id'],
)
WORKFLOW_EXECUTION_TOTAL = Counter(
'model_manager_workflow_executions_total',
'Total number of workflow executions',
['pod_id', 'workflow_name', 'status'], # status: success, error
)
ACTIVITY_EXECUTION_TOTAL = Counter(
'model_manager_activity_executions_total',
'Total number of activity executions',
['pod_id', 'activity_name', 'status'], # status: success, error
)

View File

@@ -0,0 +1,167 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="style.css" />
<title>Report</title>
<style>
* {
font-family: "Franklin Gothic Medium", "Arial Narrow", Arial, sans-serif;
margin: 0;
padding: 0;
box-sizing: border-box;
}
.logo {
padding-top: 70;
padding-bottom: 70;
position: absolute;
margin-left: -48px;
}
h1 {
color: #fff;
position: absolute;
margin-left: 45%;
}
html,
body {
scroll-behavior: smooth;
}
section {
padding-top: 90px;
width: 100%;
display: fixed;
justify-content: center;
align-items: center;
background-color: rgb(217, 217, 214, 0.7);
}
.material-symbols-outlined {
font-variation-settings: "FILL" 0, "wght" 400, "GRAD" 0, "opsz" 24;
color: #ffff;
}
/* Tooltip text */
.tooltiptext {
visibility: hidden;
background-color: rgb(0, 30, 96, 0.9);
padding: 10px;
margin-left: -90px;
font-size: 16px;
position: absolute;
top: 85px;
border-bottom-left-radius: 12px;
border-bottom-right-radius: 12px;
}
/* Show the tooltip text when you mouse over the tooltip container */
.material-symbols-outlined:hover .tooltiptext {
visibility: visible;
}
header {
position: fixed;
top: 0;
width: 100%;
height: 85px;
background: rgb(0, 30, 96, 0.95);
display: flex;
justify-content: space-between;
align-items: center;
padding: 10px 50px 0;
}
header nav {
display: absolute;
margin-left: 80%;
gap: 10px;
}
header nav a {
position: relative;
text-decoration: none;
padding: 12px 18px;
color: #fff;
font-weight: 500;
}
header nav a.active {
background-color: #001540;
position: relative;
border-radius: 12px;
}
</style>
<link
rel="stylesheet"
href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200"
/>
</head>
<body>
<main>
<header>
<a href="#" class="logo">
<img
src="https://aignosi.blob.core.windows.net/sientia/20231016-Aignosi_Logo_WHITE.png"
alt="Aignosi Logo"
width="247"
height="70"
/>
</a>
<h1>Report</h1>
<nav>
<a href="#data_quality" class="active"> Summary </a>
<a href="#data_drift"> Drift </a>
<a href="#regression"> Regression </a>
</nav>
<div class="material-symbols-outlined">
info
<p class="tooltiptext">
Note that "current" <br />
is related to the test <br />
set while "reference" <br />
refers to the training <br />
set
</p>
</div>
</header>
<div class="quality_div">
<section id="data_quality"></section>
</div>
<div class="data_drift_div">
<section id="data_drift"></section>
</div>
<div class="regression_div">
<section id="regression"></section>
</div>
</main>
<script>
let sec = document.querySelectorAll("section");
let links = document.querySelectorAll("nav a");
window.onscroll = () => {
sec.forEach((section) => {
let top = window.scrollY;
let offset = section.offsetTop;
let height = section.offsetHeight;
let id = section.getAttribute("id");
if (top >= offset && top < offset + height) {
links.forEach((link) => {
link.classList.remove("active");
document.querySelector("nav a[href*=" + id + "]").classList.add("active");
});
}
});
};
</script>
</body>
</html>

View File

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,167 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="style.css" />
<title>Report</title>
<style>
* {
font-family: "Franklin Gothic Medium", "Arial Narrow", Arial, sans-serif;
margin: 0;
padding: 0;
box-sizing: border-box;
}
.logo {
padding-top: 70;
padding-bottom: 70;
position: absolute;
margin-left: -48px;
}
h1 {
color: #fff;
position: absolute;
margin-left: 45%;
}
html,
body {
scroll-behavior: smooth;
}
section {
padding-top: 90px;
width: 100%;
display: fixed;
justify-content: center;
align-items: center;
background-color: rgb(217, 217, 214, 0.7);
}
.material-symbols-outlined {
font-variation-settings: "FILL" 0, "wght" 400, "GRAD" 0, "opsz" 24;
color: #ffff;
}
/* Tooltip text */
.tooltiptext {
visibility: hidden;
background-color: rgb(0, 30, 96, 0.9);
padding: 10px;
margin-left: -90px;
font-size: 16px;
position: absolute;
top: 85px;
border-bottom-left-radius: 12px;
border-bottom-right-radius: 12px;
}
/* Show the tooltip text when you mouse over the tooltip container */
.material-symbols-outlined:hover .tooltiptext {
visibility: visible;
}
header {
position: fixed;
top: 0;
width: 100%;
height: 85px;
background: rgb(0, 30, 96, 0.95);
display: flex;
justify-content: space-between;
align-items: center;
padding: 10px 50px 0;
}
header nav {
display: absolute;
margin-left: 80%;
gap: 10px;
}
header nav a {
position: relative;
text-decoration: none;
padding: 12px 18px;
color: #fff;
font-weight: 500;
}
header nav a.active {
background-color: #001540;
position: relative;
border-radius: 12px;
}
</style>
<link
rel="stylesheet"
href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200"
/>
</head>
<body>
<main>
<header>
<a href="#" class="logo">
<img
src="https://aignosi.blob.core.windows.net/sientia/20231016-Aignosi_Logo_WHITE.png"
alt="Aignosi Logo"
width="247"
height="70"
/>
</a>
<h1>Report</h1>
<nav>
<a href="#data_quality" class="active"> Summary </a>
<a href="#data_drift"> Drift </a>
<a href="#regression"> Regression </a>
</nav>
<div class="material-symbols-outlined">
info
<p class="tooltiptext">
Note that "current" <br />
is related to the test <br />
set while "reference" <br />
refers to the training <br />
set
</p>
</div>
</header>
<div class="quality_div">
<section id="data_quality"></section>
</div>
<div class="data_drift_div">
<section id="data_drift"></section>
</div>
<div class="regression_div">
<section id="regression"></section>
</div>
</main>
<script>
let sec = document.querySelectorAll("section");
let links = document.querySelectorAll("nav a");
window.onscroll = () => {
sec.forEach((section) => {
let top = window.scrollY;
let offset = section.offsetTop;
let height = section.offsetHeight;
let id = section.getAttribute("id");
if (top >= offset && top < offset + height) {
links.forEach((link) => {
link.classList.remove("active");
document.querySelector("nav a[href*=" + id + "]").classList.add("active");
});
}
});
};
</script>
</body>
</html>

View File

@@ -0,0 +1,15 @@
{
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"coefficients": {
"303-WIT-200(Value)": 0.0125579833984375
},
"intercept": 37.25,
"equation_string": "03CV020/CORRENTE_N_M1_PV(Value) = 37.250000 + 0.012558 * 303-WIT-200(Value)",
"latex_equation": "03CV020/CORRENTE_N_M1_PV(Value) = 37.250000 + 0.012558 \\cdot 303-WIT-200(Value)",
"model_type": "Linear Regression",
"degree": 1,
"interaction_only": false,
"original_features": [
"303-WIT-200(Value)"
]
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,167 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="style.css" />
<title>Report</title>
<style>
* {
font-family: "Franklin Gothic Medium", "Arial Narrow", Arial, sans-serif;
margin: 0;
padding: 0;
box-sizing: border-box;
}
.logo {
padding-top: 70;
padding-bottom: 70;
position: absolute;
margin-left: -48px;
}
h1 {
color: #fff;
position: absolute;
margin-left: 45%;
}
html,
body {
scroll-behavior: smooth;
}
section {
padding-top: 90px;
width: 100%;
display: fixed;
justify-content: center;
align-items: center;
background-color: rgb(217, 217, 214, 0.7);
}
.material-symbols-outlined {
font-variation-settings: "FILL" 0, "wght" 400, "GRAD" 0, "opsz" 24;
color: #ffff;
}
/* Tooltip text */
.tooltiptext {
visibility: hidden;
background-color: rgb(0, 30, 96, 0.9);
padding: 10px;
margin-left: -90px;
font-size: 16px;
position: absolute;
top: 85px;
border-bottom-left-radius: 12px;
border-bottom-right-radius: 12px;
}
/* Show the tooltip text when you mouse over the tooltip container */
.material-symbols-outlined:hover .tooltiptext {
visibility: visible;
}
header {
position: fixed;
top: 0;
width: 100%;
height: 85px;
background: rgb(0, 30, 96, 0.95);
display: flex;
justify-content: space-between;
align-items: center;
padding: 10px 50px 0;
}
header nav {
display: absolute;
margin-left: 80%;
gap: 10px;
}
header nav a {
position: relative;
text-decoration: none;
padding: 12px 18px;
color: #fff;
font-weight: 500;
}
header nav a.active {
background-color: #001540;
position: relative;
border-radius: 12px;
}
</style>
<link
rel="stylesheet"
href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200"
/>
</head>
<body>
<main>
<header>
<a href="#" class="logo">
<img
src="https://aignosi.blob.core.windows.net/sientia/20231016-Aignosi_Logo_WHITE.png"
alt="Aignosi Logo"
width="247"
height="70"
/>
</a>
<h1>Report</h1>
<nav>
<a href="#data_quality" class="active"> Summary </a>
<a href="#data_drift"> Drift </a>
<a href="#regression"> Regression </a>
</nav>
<div class="material-symbols-outlined">
info
<p class="tooltiptext">
Note that "current" <br />
is related to the test <br />
set while "reference" <br />
refers to the training <br />
set
</p>
</div>
</header>
<div class="quality_div">
<section id="data_quality"></section>
</div>
<div class="data_drift_div">
<section id="data_drift"></section>
</div>
<div class="regression_div">
<section id="regression"></section>
</div>
</main>
<script>
let sec = document.querySelectorAll("section");
let links = document.querySelectorAll("nav a");
window.onscroll = () => {
sec.forEach((section) => {
let top = window.scrollY;
let offset = section.offsetTop;
let height = section.offsetHeight;
let id = section.getAttribute("id");
if (top >= offset && top < offset + height) {
links.forEach((link) => {
link.classList.remove("active");
document.querySelector("nav a[href*=" + id + "]").classList.add("active");
});
}
});
};
</script>
</body>
</html>

View File

@@ -0,0 +1,15 @@
{
"target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
"coefficients": {
"303-WIT-200(Value)": 0.0125579833984375
},
"intercept": 37.25,
"equation_string": "03CV020/CORRENTE_N_M1_PV(Value) = 37.250000 + 0.012558 * 303-WIT-200(Value)",
"latex_equation": "03CV020/CORRENTE_N_M1_PV(Value) = 37.250000 + 0.012558 \\cdot 303-WIT-200(Value)",
"model_type": "Linear Regression",
"degree": 1,
"interaction_only": false,
"original_features": [
"303-WIT-200(Value)"
]
}

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show More