docker-compose.yml

networks:
  docker-net:
    name: docker-net
    driver: bridge

volumes:
  hadoop_datanode:
  hadoop_namenode:
  prefect:
  dremio_data:

services:
  namenode:
    container_name: namenode
    image: apache/hadoop:3
    hostname: namenode
    command: bash -c "if [ ! -f /tmp/hadoop-root/dfs/name/.formatted ]; then hdfs namenode -format && touch /tmp/hadoop-root/dfs/name/.formatted; fi && hdfs namenode"
    ports:
      - 9870:9870
      - 8020:8020
      - 9000:9000
    user: root
    env_file:
      - .env
    volumes:
      - hadoop_namenode:/tmp/hadoop-root/dfs/name
    networks:
      - docker-net

  datanode:
    image: apache/hadoop:3
    container_name: datanode
    hostname: datanode 
    command: ["hdfs", "datanode"]
    ports:
      - 9864:9864
      - 9866:9866
    expose:
      - 50010
    env_file:
      - .env
    user: root
    volumes:
      - hadoop_datanode:/tmp/hadoop-root/dfs/data
    networks:
      - docker-net

  resourcemanager:
    image: apache/hadoop:3
    hostname: resourcemanager
    command: ["yarn", "resourcemanager"]
    ports:
       - 8088:8088
    env_file:
      - .env
    volumes:
      - ./test.sh:/opt/test.sh
    networks:
      - docker-net

  nodemanager:
    image: apache/hadoop:3
    command: ["yarn", "nodemanager"]
    env_file:
      - .env
    ports:
      - 8188:8188
    networks:
      - docker-net

  prefect-server:
    build:
      context: ./prefect
    image: prefect
    hostname: prefect-server
    container_name: prefect-server
    volumes:
      - prefect:/root/.prefect
    command: prefect server start
    environment:
      - PREFECT_UI_URL=http://127.0.0.1:4200/api
      - PREFECT_API_URL=http://127.0.0.1:4200/api
      - PREFECT_SERVER_API_HOST=0.0.0.0
    ports:
      - 4200:4200
    networks:
      - docker-net

  prefect:
    image: prefect:latest
    container_name: prefect
    restart: always
    volumes:
      - "./prefect/flows:/opt/prefect/flows"
      - "/etc/timezone:/etc/timezone:ro"
      - "/etc/localtime:/etc/localtime:ro"
    env_file:
      - .env
    networks:
      - docker-net
    depends_on:
      - prefect-server

  notebook:
    container_name: notebook
    build:
      context: ./notebook
    volumes:
      - ${PWD}/workspace:/home/jovyan/work
    ports:
      - 8888:8888
    environment:
      - JUPYTER_TOKEN=pass
    # command: start-notebook.sh
    user: root
    env_file:
      - .env
    networks:
      - docker-net

  UI:
    build:
      context: ./UI
    image: streamlit
    command: bash -c "bash init.bash && streamlit run main.py"
    container_name: streamlit
    restart: always
    ports:
      - "8501:8501"
    volumes:
      - ./UI:/usr/src/app
    env_file:
      - .env
    networks:
      - docker-net
    depends_on:
      - dremio

  dremio:
    image: dremio/dremio-oss
    hostname: dremio
    container_name: dremio
    restart: always
    user: root
    volumes:
      - dremio_data:/var/lib/dremio
      - dremio_data:/localFiles
      - dremio_data:/opt/dremio
    ports:
      - "9047:9047"   # Web UI (HTTP)
      - "31010:31010" # ODBC/JDBC client
      - "32010:32010" # Apache Arrow Flight clients
    networks:
      - docker-net