Create docker-compose.yml

docker-compose.yml

@@ -0,0 +1,64 @@
+services:
+  spark-master:
+    image: bitnami/spark:latest
+    container_name: spark-master
+    hostname: spark-master
+    ports:
+      - "7077:7077"  # Spark Master port
+      - "8080:8080"  # Spark Master Web UI port
+    environment:
+      - SPARK_MODE=master
+      - SPARK_MASTER_NAME=spark-master
+      - SPARK_MASTER_PORT=7077
+    networks:
+      - spark-network
+    command: /opt/bitnami/scripts/spark/run.sh -c start-master.sh
+
+  spark-worker-1:
+    image: bitnami/spark:latest
+    container_name: spark-worker-1
+    depends_on:
+      - spark-master
+    environment:
+      - SPARK_MODE=worker
+      - SPARK_MASTER=spark://spark-master:7077
+    ports:
+      - "8081:8081"  # Spark Worker Web UI port
+    networks:
+      - spark-network
+    command: /opt/bitnami/scripts/spark/run.sh -c start-worker.sh spark://spark-master:7077
+
+  spark-worker-2:
+    image: bitnami/spark:latest
+    container_name: spark-worker-2
+    depends_on:
+      - spark-master
+    environment:
+      - SPARK_MODE=worker
+      - SPARK_MASTER=spark://spark-master:7077
+    ports:
+      - "8082:8082"  # Spark Worker Web UI port
+    networks:
+      - spark-network
+    command: /opt/bitnami/scripts/spark/run.sh -c start-worker.sh spark://spark-master:7077
+
+  jupyter:
+    image: jupyter/pyspark-notebook:latest
+    container_name: jupyter
+    ports:
+      - "8888:8888"  # Jupyter Notebook port
+    volumes:
+      - ./notebooks:/home/jovyan/work
+    depends_on:
+      - spark-master
+      - spark-worker-1
+      - spark-worker-2
+    environment:
+      - SPARK_MASTER=spark://spark-master:7077
+      - JUPYTER_ENABLE_LAB=yes
+    networks:
+      - spark-network
+
+networks:
+  spark-network:
+    driver: bridge