docker-compose.yml

services:
  spark-master:
    image: bitnami/spark:latest
    container_name: spark-master
    hostname: spark-master
    ports:
      - "7077:7077"  # Spark Master port
      - "8080:8080"  # Spark Master Web UI port
    environment:
      - SPARK_MODE=master
      - SPARK_MASTER_NAME=spark-master
      - SPARK_MASTER_PORT=7077
    networks:
      - spark-network
    command: /opt/bitnami/scripts/spark/run.sh -c start-master.sh

  spark-worker-1:
    image: bitnami/spark:latest
    container_name: spark-worker-1
    depends_on:
      - spark-master
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER=spark://spark-master:7077
    ports:
      - "8081:8081"  # Spark Worker Web UI port
    networks:
      - spark-network
    command: /opt/bitnami/scripts/spark/run.sh -c start-worker.sh spark://spark-master:7077

  spark-worker-2:
    image: bitnami/spark:latest
    container_name: spark-worker-2
    depends_on:
      - spark-master
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER=spark://spark-master:7077
    ports:
      - "8082:8082"  # Spark Worker Web UI port
    networks:
      - spark-network
    command: /opt/bitnami/scripts/spark/run.sh -c start-worker.sh spark://spark-master:7077

  jupyter:
    image: jupyter/pyspark-notebook:latest
    container_name: jupyter
    ports:
      - "8888:8888"  # Jupyter Notebook port
    volumes:
      - ./notebooks:/home/jovyan/work
    depends_on:
      - spark-master
      - spark-worker-1
      - spark-worker-2
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - JUPYTER_ENABLE_LAB=yes
    networks:
      - spark-network

networks:
  spark-network:
    driver: bridge