commit ca9f5488bff11359ca76b0b34d8c33218c0f211e Author: Matheus Date: Tue Apr 15 10:28:47 2025 -0300 Adiciona arquivos de configuração do projeto, incluindo .gitignore, Dockerfile, Makefile e requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..42893e0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,35 @@ +# Ignorar volumes do Docker +docker-compose.override.yml +**/db_data/ +**/kafka-volume/ +**/zookeeper-volume/ +**/mage_data/ +**/minio_data/ +**/venv/ +**/certs/*.pem +**/certs/*.der +**/certs/*.csr +**/deploy/*.yaml +scouter/.file_versions/ +scouter/pipelines/**/triggers.yaml + +# Ignorar arquivos e diretórios de cache do Python +__pycache__/ +*.pyc +*.pyo +*.pyd + +# Ignorar logs +*.log + +# Ignorar arquivos de configuração locais +.vscode/ +.pytest_cache/ +.idea/ +*.swp + +# Ignorar arquivos temporários +*.tmp +*.bak +*.old +.secret \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..95984aa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,83 @@ +FROM python:3.11-bookworm +LABEL description="Deploy Mage on ECS" +ARG FEATURE_BRANCH +USER root +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Definir Python 3.11 como padrão +ENV PATH="/usr/local/bin/python3.11:$PATH" +RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.11 1 && \ + update-alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.11 1 && \ + update-alternatives --config python3 <<< '1' && \ + update-alternatives --config python <<< '1' + +## System Packages +RUN \ + curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ + curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ + apt-get -y update && \ + ACCEPT_EULA=Y apt-get -y install --no-install-recommends \ + # NFS dependencies + nfs-common \ + # odbc dependencies + msodbcsql18 \ + unixodbc-dev \ + graphviz \ + # postgres dependencies + postgresql-client \ + # R + r-base && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +## R Packages +RUN \ + R -e "install.packages('pacman', repos='http://cran.us.r-project.org')" && \ + R -e "install.packages('renv', repos='http://cran.us.r-project.org')" + +## Python Packages +RUN \ + pip3 install --no-cache-dir sparkmagic && \ + mkdir ~/.sparkmagic && \ + curl https://raw.githubusercontent.com/jupyter-incubator/sparkmagic/master/sparkmagic/example_config.json > ~/.sparkmagic/config.json && \ + sed -i 's/localhost:8998/host.docker.internal:9999/g' ~/.sparkmagic/config.json && \ + jupyter-kernelspec install --user "$(pip3 show sparkmagic | grep Location | cut -d' ' -f2)/sparkmagic/kernels/pysparkkernel" + +# Mage integrations and other related packages +RUN \ + pip3 install --no-cache-dir "git+https://github.com/wbond/oscrypto.git@d5f3437ed24257895ae1edd9e503cfb352e635a8" && \ + pip3 install --no-cache-dir "git+https://github.com/dremio-hub/arrow-flight-client-examples.git#egg=dremio-flight&subdirectory=python/dremio-flight" && \ + pip3 install --no-cache-dir "git+https://github.com/mage-ai/singer-python.git#egg=singer-python" && \ + pip3 install --no-cache-dir "git+https://github.com/mage-ai/dbt-mysql.git#egg=dbt-mysql" && \ + pip3 install --no-cache-dir "git+https://github.com/mage-ai/sqlglot#egg=sqlglot" && \ + pip3 install --no-cache-dir faster-fifo && \ + if [ -z "$FEATURE_BRANCH" ] || [ "$FEATURE_BRANCH" = "null" ]; then \ + pip3 install --no-cache-dir "git+https://github.com/mage-ai/mage-ai.git#egg=mage-integrations&subdirectory=mage_integrations"; \ + else \ + pip3 install --no-cache-dir "git+https://github.com/mage-ai/mage-ai.git@$FEATURE_BRANCH#egg=mage-integrations&subdirectory=mage_integrations"; \ + fi + +# Mage +COPY ./mage_ai/server/constants.py /tmp/constants.py +RUN if [ -z "$FEATURE_BRANCH" ] || [ "$FEATURE_BRANCH" = "null" ] ; then \ + tag=$(tail -n 1 /tmp/constants.py) && \ + VERSION=$(echo "$tag" | tr -d "'") && \ + pip3 install --no-cache-dir "mage-ai[all]==$VERSION"; \ + else \ + pip3 install --no-cache-dir "git+https://github.com/mage-ai/mage-ai.git@$FEATURE_BRANCH#egg=mage-ai[all]"; \ + fi + +## Startup Script +COPY --chmod=0755 ./scripts/install_other_dependencies.py ./scripts/run_app.sh /app/ +ENV MAGE_DATA_DIR="/home/src/mage_data" +ENV PYTHONPATH="${PYTHONPATH}:/home/src" +WORKDIR /home/src +EXPOSE 6789 +EXPOSE 7789 + +# Copia o arquivo requirements.txt para o contêiner +COPY requirements.txt /app/requirements.txt +RUN pip3 install --no-cache-dir -r /app/requirements.txt + + +CMD ["/bin/sh", "-c", "/app/run_app.sh"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..66aae81 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +VERSION = 1.0.8 +name = sientia-laborious +# ENVIRONMENT = production + +docker-hub: + @docker build --no-cache -t aignosi.azurecr.io/$(name):$(VERSION) . + @docker push aignosi.azurecr.io/$(name):$(VERSION) \ No newline at end of file diff --git a/laborious/__init__.py b/laborious/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/laborious/activities/__init__.py b/laborious/activities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/laborious/util/__init__.py b/laborious/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/laborious/worker/__init__.py b/laborious/worker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/laborious/workflows/__init__.py b/laborious/workflows/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..72a1b86 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +GitPython +temporalio +pytest +python-dotenv \ No newline at end of file