# =============================================================================
# Stack Docker — Bloque de Ingeniería de datos
# HDFS · YARN · MinIO · MySQL · Hive Metastore · Trino · nodo lab · Airflow
#
# Todos los servicios están detrás de PROFILES: nada arranca "por defecto".
#
# PROFILES DE COMPONENTE (finos):
#   hadoop   namenode datanode resourcemanager nodemanager hdfs-init
#   minio    minio minio-init
#   hive     mysql-metastore hive-metastore hiveserver2
#   lab      mysql-retail lab retail-init
#   airflow  postgres-airflow airflow-init airflow-webserver airflow-scheduler
#
# PROFILES PARAGUAS (alto nivel):
#   all      arranca TODO el stack
#   lake     hadoop + minio + hive + trino (+ fuentes)   -> consultar el lago (sesiones 3-7)
#   elt      minio + lab + airflow                        -> ingesta / dbt / ELT / orquestación (8-11)
#
# RECETAS POR SESIÓN (COMPOSE_PROFILES o --profile):
#   Sesión 3/6/7  Consultar el lago              COMPOSE_PROFILES=lake
#   Sesión 8-11   Ingesta / dbt / ELT / Airflow  COMPOSE_PROFILES=elt
#   Todo el stack                                COMPOSE_PROFILES=all
# =============================================================================
# docker compose --profile all up -d

name: iabd-de

# mysql, minio, trino, lab, airflow y postgres NO se fijan a amd64: publican
# arm64 y así corren de forma nativa en Apple Silicon.
x-amd64: &amd64 linux/amd64

# IMPRESCINDIBLE: la imagen apache/hadoop hereda de apache/hadoop-runner, que fija
# HADOOP_CONF_DIR=/etc/hadoop (un directorio vacío pensado para el mecanismo de
# variables CORE-SITE.XML_*). Lo reapuntamos al conf dir real de la distribución,
# que es donde montamos nuestros XML y donde ya viven hadoop-env.sh y log4j.properties.
x-hadoop-env: &hadoop-env
  HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
  HDFS_NAMENODE_USER: root
  HDFS_DATANODE_USER: root
  HDFS_SECONDARYNAMENODE_USER: root
  YARN_RESOURCEMANAGER_USER: root
  YARN_NODEMANAGER_USER: root

x-hadoop-config-vols: &hadoop-config-vols
  - ./hadoop/config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro
  - ./hadoop/config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml:ro
  - ./hadoop/config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml:ro
  - ./hadoop/config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml:ro

# Entorno común de Airflow (compartido por init, webserver y scheduler).
x-airflow-env: &airflow-env
  AIRFLOW__CORE__EXECUTOR: LocalExecutor
  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres-airflow/airflow
  AIRFLOW__CORE__LOAD_EXAMPLES: "false"
  AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true"
  _PIP_ADDITIONAL_REQUIREMENTS: "dlt[duckdb,filesystem,s3]==1.29.0 dbt-duckdb==1.10.1 duckdb==1.5.4"
  AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
  AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
  AWS_ENDPOINT_URL: http://minio:9000
  AWS_DEFAULT_REGION: us-east-1

x-airflow-common: &airflow-common
  image: ${AIRFLOW_IMAGE}
  environment: *airflow-env
  volumes:
    - ./airflow/dags:/opt/airflow/dags
    - ./airflow/logs:/opt/airflow/logs
    - ./airflow/plugins:/opt/airflow/plugins
    - ./workspace:/workspace
    - ./sample-data:/sample-data:ro
  networks: [bigdata]

services:

  # =========================================================================
  # BLOQUE HADOOP: HDFS (namenode + datanode) y YARN (RM + NM)
  # =========================================================================
  namenode:
    image: ${HADOOP_IMAGE}
    container_name: iabd-de-namenode
    platform: *amd64
    hostname: namenode
    user: root
    profiles: [hadoop, lake, all]
    environment: *hadoop-env
    command: ["/bin/bash", "/opt/hadoop/scripts/start-hdfs.sh"]
    volumes:
      - ./hadoop/config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro
      - ./hadoop/config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml:ro
      - ./hadoop/config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml:ro
      - ./hadoop/config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml:ro
      - ./hadoop/scripts/start-hdfs.sh:/opt/hadoop/scripts/start-hdfs.sh:ro
      - ./sample-data:/sample-data:ro
      - namenode_data:/opt/hadoop/dfs/name
    ports:
      - "9870:9870"   # NameNode Web UI
      - "8020:8020"   # NameNode RPC (acceso HDFS desde el host, opcional)
    healthcheck:
      test: ["CMD-SHELL", "bash -c '</dev/tcp/127.0.0.1/9870'"]
      interval: 10s
      timeout: 5s
      retries: 15
    networks: [bigdata]

  datanode:
    image: ${HADOOP_IMAGE}
    container_name: iabd-de-datanode
    platform: *amd64
    hostname: datanode
    user: root
    profiles: [hadoop, lake, all]
    environment: *hadoop-env
    command: ["hdfs", "datanode"]
    volumes:
      - ./hadoop/config/core-site.xml:/opt/hadoop/etc/hadoop/core-site.xml:ro
      - ./hadoop/config/hdfs-site.xml:/opt/hadoop/etc/hadoop/hdfs-site.xml:ro
      - ./hadoop/config/yarn-site.xml:/opt/hadoop/etc/hadoop/yarn-site.xml:ro
      - ./hadoop/config/mapred-site.xml:/opt/hadoop/etc/hadoop/mapred-site.xml:ro
      - datanode_data:/opt/hadoop/dfs/data
    ports:
      - "9864:9864"   # DataNode Web UI
    depends_on:
      namenode:
        condition: service_healthy
    networks: [bigdata]

  resourcemanager:
    image: ${HADOOP_IMAGE}
    container_name: iabd-de-resourcemanager
    platform: *amd64
    hostname: resourcemanager
    user: root
    profiles: [hadoop, lake, all]
    environment: *hadoop-env
    command: ["yarn", "resourcemanager"]
    volumes: *hadoop-config-vols
    ports:
      - "8088:8088"   # ResourceManager Web UI
    depends_on:
      namenode:
        condition: service_healthy
    networks: [bigdata]

  nodemanager:
    image: ${HADOOP_IMAGE}
    container_name: iabd-de-nodemanager
    platform: *amd64
    hostname: nodemanager
    user: root
    profiles: [hadoop, lake, all]
    environment: *hadoop-env
    command: ["yarn", "nodemanager"]
    volumes: *hadoop-config-vols
    ports:
      - "8042:8042"   # NodeManager Web UI
    depends_on:
      resourcemanager:
        condition: service_started
    networks: [bigdata]

  # One-shot: prepara los directorios base en HDFS. Idempotente.
  hdfs-init:
    image: ${HADOOP_IMAGE}
    container_name: iabd-de-hdfs-init
    platform: *amd64
    user: root
    profiles: [hadoop, lake, all]
    environment: *hadoop-env
    volumes: *hadoop-config-vols
    command:
      - /bin/bash
      - -c
      - |
        set -e
        echo "Esperando a que HDFS acepte operaciones...";
        until hdfs dfs -ls / >/dev/null 2>&1; do sleep 3; done;
        hdfs dfs -mkdir -p /user/hive/warehouse /tmp;
        hdfs dfs -chmod -R 1777 /tmp;
        hdfs dfs -chmod -R 777  /user/hive/warehouse;
        echo "HDFS inicializado: /user/hive/warehouse y /tmp creados.";
    depends_on:
      namenode:
        condition: service_healthy
      datanode:
        condition: service_started
    restart: "no"
    networks: [bigdata]

  # =========================================================================
  # MINIO: object storage compatible con S3 (el lago)
  # =========================================================================
  minio:
    image: ${MINIO_IMAGE}
    container_name: iabd-de-minio
    hostname: minio
    profiles: [minio, lake, elt, airflow, all]
    command: ["server", "/data", "--console-address", ":9001"]
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
    volumes:
      - minio_data:/data
    ports:
      - "9000:9000"   # API S3
      - "9001:9001"   # Consola web
    healthcheck:
      test: ["CMD-SHELL", "mc ready local || curl -sf http://localhost:9000/minio/health/live || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 15
    networks: [bigdata]

  # One-shot: crea los buckets raw-data, processed y warehouse. Idempotente.
  minio-init:
    image: ${MC_IMAGE}
    container_name: iabd-de-minio-init
    profiles: [minio, lake, elt, airflow, all]
    entrypoint:
      - /bin/sh
      - -c
      - |
        set -e
        mc alias set local http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}
        for b in raw-data processed warehouse; do mc mb --ignore-existing local/$$b; done
        echo "Buckets listos."
    depends_on:
      minio:
        condition: service_healthy
    restart: "no"
    networks: [bigdata]

  # =========================================================================
  # BLOQUE HIVE: MySQL del metastore + Metastore + HiveServer2
  # Dos MySQL separados por rol:
  #   mysql-metastore -> SOLO el catálogo de Hive (metadatos)
  #   mysql-retail    -> SOLO retail_db (origen operacional del que extrae dlt)
  # =========================================================================
  mysql-metastore:
    image: ${MYSQL_IMAGE}
    container_name: iabd-de-mysql-metastore
    hostname: mysql-metastore
    profiles: [hive, lake, all]
    command: ["--default-authentication-plugin=mysql_native_password"]
    environment:
      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
      MYSQL_DATABASE: ${MYSQL_DATABASE}
      MYSQL_USER: ${MYSQL_USER}
      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
    volumes:
      - mysql_metastore_data:/var/lib/mysql
    ports:
      - "3306:3306"   # metastore
    healthcheck:
      test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 --silent"]
      interval: 10s
      timeout: 5s
      retries: 15
    networks: [bigdata]

  # Origen operacional. Mismos profiles que retail-init (su init).
  # También lo consulta Trino vía conector MySQL en la sesión de motores (lake).
  mysql-retail:
    image: ${MYSQL_IMAGE}
    container_name: iabd-de-mysql-retail
    hostname: mysql-retail
    profiles: [lab, lake, elt, airflow, all]
    command: ["--default-authentication-plugin=mysql_native_password"]
    environment:
      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
      MYSQL_DATABASE: retail_db
    volumes:
      - mysql_retail_data:/var/lib/mysql
    ports:
      - "3307:3306"   # origen retail_db (host 3307 para no chocar con el metastore)
    healthcheck:
      test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 --silent"]
      interval: 10s
      timeout: 5s
      retries: 15
    networks: [bigdata]

  hive-metastore:
    build:
      context: ./hive
      args:
        HIVE_VERSION: ${HIVE_VERSION}
        MYSQL_CONNECTOR_VERSION: ${MYSQL_CONNECTOR_VERSION}
    image: hive-mysql:${HIVE_VERSION}
    container_name: iabd-de-hive-metastore
    platform: *amd64
    hostname: hive-metastore
    profiles: [hive, lake, all]
    environment:
      SERVICE_NAME: metastore
      DB_DRIVER: mysql
      DEFAULT_FS: hdfs://namenode:8020
      HIVE_WAREHOUSE_PATH: /user/hive/warehouse
      HADOOP_CONF_DIR: /opt/hive/conf
      HIVE_AUX_JARS_PATH: /opt/hadoop/share/hadoop/tools/lib/hadoop-aws-3.4.1.jar,/opt/hadoop/share/hadoop/tools/lib/bundle-2.24.6.jar
      SERVICE_OPTS: >-
        -Djavax.jdo.option.ConnectionDriverName=com.mysql.cj.jdbc.Driver
        -Djavax.jdo.option.ConnectionURL=jdbc:mysql://mysql-metastore:3306/metastore?sslMode=DISABLED
        -Djavax.jdo.option.ConnectionUserName=${MYSQL_USER}
        -Djavax.jdo.option.ConnectionPassword=${MYSQL_PASSWORD}
    volumes:
      - ./hive/conf/core-site.xml:/opt/hive/conf/core-site.xml:ro
      - ./hive/conf/hdfs-site.xml:/opt/hive/conf/hdfs-site.xml:ro
    ports:
      - "9083:9083"
    depends_on:
      mysql-metastore:
        condition: service_healthy
      hdfs-init:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD-SHELL", "bash -c '</dev/tcp/127.0.0.1/9083'"]
      interval: 10s
      timeout: 5s
      retries: 20
    networks: [bigdata]

  hiveserver2:
    image: hive-mysql:${HIVE_VERSION}
    container_name: iabd-de-hiveserver2
    platform: *amd64
    hostname: hiveserver2
    profiles: [hive, lake, all]
    entrypoint: ["sh", "-c", "rm -f /opt/hive/conf/*.pid; exec /entrypoint.sh"]
    environment:
      SERVICE_NAME: hiveserver2
      IS_RESUME: "true"
      DEFAULT_FS: hdfs://namenode:8020
      HIVE_WAREHOUSE_PATH: /user/hive/warehouse
      HADOOP_CONF_DIR: /opt/hive/conf
      HIVE_AUX_JARS_PATH: /opt/hadoop/share/hadoop/tools/lib
      SERVICE_OPTS: "-Dhive.metastore.uris=thrift://hive-metastore:9083"
    volumes:
      - ./hive/conf/core-site.xml:/opt/hive/conf/core-site.xml:ro
      - ./hive/conf/hdfs-site.xml:/opt/hive/conf/hdfs-site.xml:ro
    ports:
      - "10000:10000"
      - "10002:10002"
    depends_on:
      hive-metastore:
        condition: service_healthy
    networks: [bigdata]

  # =========================================================================
  # TRINO: motor de consulta sobre parquet (HDFS o MinIO) vía metastore de Hive
  # =========================================================================
  trino:
    image: ${TRINO_IMAGE}
    container_name: iabd-de-trino
    hostname: trino
    profiles: [lake, all]
    environment:
      HADOOP_USER_NAME: hive
    volumes:
      - ./trino/catalog:/etc/trino/catalog:ro
      - ./trino/hadoop:/etc/hadoop:ro
    ports:
      - "8080:8080"
    depends_on:
      hive-metastore:
        condition: service_healthy
    networks: [bigdata]

  # =========================================================================
  # NODO LAB: DuckDB + dlt + dbt + JupyterLab (sesiones 8-11)
  # =========================================================================
  lab:
    build:
      context: ./duckdb
      args:
        DUCKDB_VERSION: ${DUCKDB_VERSION}
    image: iabd-de-lab:${DUCKDB_VERSION}
    container_name: iabd-de-lab
    hostname: lab
    profiles: [lab, elt, all]
    environment:
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
      AWS_ENDPOINT_URL: http://minio:9000
      AWS_DEFAULT_REGION: us-east-1
      # Alias 'local' de mc preconfigurado: `mc ls local/` funciona sin `mc alias set`
      MC_HOST_local: http://${MINIO_ROOT_USER}:${MINIO_ROOT_PASSWORD}@minio:9000
      MYSQL_HOST: mysql-retail
      MYSQL_DATABASE_SOURCE: retail_db
    volumes:
      - ./workspace:/workspace
      - ./sample-data:/sample-data:ro
    ports:
      - "8888:8888"   # JupyterLab
    # lab NO depende de minio: puede arrancar solo (DuckDB local, cuadernos HDFS).
    # Para usar mc/el lago hay que activar además el profile minio.
    depends_on:
      mysql-retail:
        condition: service_healthy
      retail-init:
        condition: service_completed_successfully
    networks: [bigdata]

  # One-shot: carga retail_db en mysql-retail. Origen del que extrae dlt.
  # También lo usa Trino (conector MySQL) en la sesión de motores de consulta.
  retail-init:
    image: ${MYSQL_IMAGE}
    container_name: iabd-de-retail-init
    profiles: [lab, lake, elt, airflow, all]
    volumes:
      - ./sample-data:/sample-data:ro
    command:
      - /bin/bash
      - -c
      - |
        set -e
        echo "Cargando retail_db en mysql-retail...";
        mysql -h mysql-retail -u root -p${MYSQL_ROOT_PASSWORD} < /sample-data/create_db.sql;
        echo "retail_db lista.";
    depends_on:
      mysql-retail:
        condition: service_healthy
    restart: "no"
    networks: [bigdata]

  # =========================================================================
  # AIRFLOW: orquestación del ELT (dlt -> dbt). LocalExecutor sobre Postgres.
  # =========================================================================
  postgres-airflow:
    image: ${POSTGRES_IMAGE}
    container_name: iabd-de-postgres-airflow
    hostname: postgres-airflow
    profiles: [airflow, elt, all]
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    volumes:
      - postgres_airflow_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U airflow"]
      interval: 10s
      timeout: 5s
      retries: 15
    networks: [bigdata]

  # One-shot: migra la BD de metadatos y crea el usuario admin. Idempotente.
  airflow-init:
    <<: *airflow-common
    container_name: iabd-de-airflow-init
    profiles: [airflow, elt, all]
    entrypoint: /bin/bash
    command:
      - -c
      - |
        airflow db migrate
        airflow users create --username ${AIRFLOW_ADMIN_USER} --password ${AIRFLOW_ADMIN_PASSWORD} \
          --firstname IA --lastname BD --role Admin --email iabd@iabd.es || true
        echo "Airflow inicializado."
    depends_on:
      postgres-airflow:
        condition: service_healthy
    restart: "no"

  airflow-webserver:
    <<: *airflow-common
    container_name: iabd-de-airflow-webserver
    hostname: airflow-webserver
    profiles: [airflow, elt, all]
    command: webserver
    ports:
      - "8082:8080"   # Airflow UI
    depends_on:
      airflow-init:
        condition: service_completed_successfully
      postgres-airflow:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "curl -sf http://localhost:8080/health || exit 1"]
      interval: 15s
      timeout: 10s
      retries: 10

  airflow-scheduler:
    <<: *airflow-common
    container_name: iabd-de-airflow-scheduler
    hostname: airflow-scheduler
    profiles: [airflow, elt, all]
    command: scheduler
    depends_on:
      airflow-init:
        condition: service_completed_successfully
      postgres-airflow:
        condition: service_healthy

networks:
  bigdata:
    name: bigdata

volumes:
  namenode_data:
  datanode_data:
  mysql_metastore_data:
  mysql_retail_data:
  minio_data:
  postgres_airflow_data: