b203b7d22c
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
98 lines
2.2 KiB
YAML
98 lines
2.2 KiB
YAML
version: '3.8'
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:15
|
|
container_name: postgres
|
|
environment:
|
|
POSTGRES_USER: sientia
|
|
POSTGRES_PASSWORD: sientia
|
|
POSTGRES_DB: sientia
|
|
ports:
|
|
- "5432:5432"
|
|
volumes:
|
|
- ./postgres_data:/var/lib/postgresql/data
|
|
networks:
|
|
- sientia-network
|
|
|
|
zookeeper:
|
|
image: confluentinc/cp-zookeeper:7.5.1
|
|
container_name: zookeeper
|
|
environment:
|
|
ZOOKEEPER_CLIENT_PORT: 2181
|
|
ZOOKEEPER_TICK_TIME: 2000
|
|
ports:
|
|
- "2181:2181"
|
|
networks:
|
|
- sientia-network
|
|
|
|
kafka:
|
|
image: confluentinc/cp-kafka:7.5.1
|
|
container_name: kafka
|
|
depends_on:
|
|
- zookeeper
|
|
ports:
|
|
- "9092:9092"
|
|
- "29092:29092"
|
|
environment:
|
|
KAFKA_BROKER_ID: 1
|
|
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
|
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
|
|
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
|
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
|
# Message retention settings (5 minutes)
|
|
KAFKA_LOG_RETENTION_MINUTES: 5 # 5 minutes
|
|
KAFKA_LOG_RETENTION_MS: 300000 # 5 minutes in milliseconds
|
|
KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 30000 # Check every 30 seconds
|
|
networks:
|
|
- sientia-network
|
|
|
|
kafka-ui:
|
|
image: provectuslabs/kafka-ui:latest
|
|
container_name: kafka-ui
|
|
ports:
|
|
- "8080:8080"
|
|
environment:
|
|
KAFKA_CLUSTERS_0_NAME: local
|
|
KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092
|
|
networks:
|
|
- sientia-network
|
|
|
|
redis:
|
|
image: redis:latest
|
|
ports:
|
|
- "${REDIS_PORT}:6379"
|
|
networks:
|
|
- sientia-network
|
|
|
|
redis-ui:
|
|
image: redislabs/redisinsight:latest
|
|
container_name: redis-ui
|
|
ports:
|
|
- "8001:8001"
|
|
networks:
|
|
- sientia-network
|
|
depends_on:
|
|
- redis
|
|
|
|
# scouter:
|
|
# build: .
|
|
# container_name: scouter
|
|
# networks:
|
|
# - sientia-network
|
|
# env_file:
|
|
# - .env
|
|
# depends_on:
|
|
# - postgres
|
|
# - kafka
|
|
# - redis
|
|
|
|
|
|
networks:
|
|
sientia-network:
|
|
driver: bridge
|
|
|
|
volumes:
|
|
postgres_data:
|
|
driver: local |