Allow to deploy WT flavor using local data catalog

.env

@@ -1,3 +1,4 @@
+export DATACAT=1
 export docker_group=$(getent group docker | cut -d: -f3)
 export domain=local.wholetale.org
 export registry_user=fido

Makefile

@@ -1,8 +1,8 @@
-.PHONY: clean dirs dev images gwvolman_src wholetale_src dms_src home_src sources \
+.PHONY: clean dirs dev images gwvolman_src wholetale_src dms_src home_src sources_wt \
 	rebuild_dashboard watch_dashboard \
 	restart_worker restart_girder globus_handler_src status update_src
 
-SUBDIRS = src volumes/ps volumes/workspaces volumes/homes volumes/base volumes/versions volumes/runs volumes/licenses volumes/mountpoints volumes/tmp
+SUBDIRS = src volumes/ps volumes/workspaces volumes/homes volumes/base volumes/versions volumes/runs volumes/licenses volumes/mountpoints volumes/tmp volumes/minio
 TAG = latest
 MEM_LIMIT = 2048
 NODE = node --max_old_space_size=${MEM_LIMIT}
@@ -20,6 +20,15 @@ images:
 	docker pull wholetale/repo2docker_wholetale:$(TAG)
 	docker pull wholetale/ngx-dashboard:$(TAG)
 
+src/sem_viewer:
+	git clone https://github.com/htmdec/sem_viewer src/sem_viewer
+
+src/table_view:
+	git clone https://github.com/htmdec/table_view src/table_view
+
+src/synced_folders:
+	git clone https://github.com/whole-tale/synced_folders src/synced_folders
+
 src/girderfs:
 	git clone https://github.com/whole-tale/girderfs src/girderfs
 
@@ -47,14 +56,14 @@ src/globus_handler:
 src/ngx-dashboard:
 	git clone https://github.com/whole-tale/ngx-dashboard src/ngx-dashboard
 
-sources: src src/gwvolman src/wholetale src/wt_data_manager src/wt_home_dir src/globus_handler src/girderfs src/ngx-dashboard src/virtual_resources src/wt_versioning
+sources_wt: src src/gwvolman src/wholetale src/wt_data_manager src/wt_home_dir src/globus_handler src/girderfs src/ngx-dashboard src/virtual_resources src/wt_versioning src/sem_viewer src/table_view src/synced_folders
 
 dirs: $(SUBDIRS)
 
 $(SUBDIRS):
 	@sudo mkdir -p $@
 
-services: dirs sources
+services: dirs sources_wt
 
 dev: services
 	. ./.env && docker stack config --compose-file docker-stack.yml | docker stack deploy --compose-file - wt
@@ -65,8 +74,20 @@ dev: services
 	    cid=$$(docker ps --filter=name=wt_girder -q) ; \
 	done; \
 	true
-	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install plugin plugins/wt_data_manager plugins/wholetale plugins/wt_home_dir plugins/globus_handler plugins/virtual_resources plugins/wt_versioning
-	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install web --dev --plugins=oauth,gravatar,jobs,worker,wt_data_manager,wholetale,wt_home_dir,globus_handler
+	docker cp ./fontello.zip $$(docker ps --filter=name=wt_girder -q):/girder/clients/web/static/built/
+	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install plugin \
+		plugins/wt_data_manager \
+		plugins/wholetale \
+		plugins/wt_home_dir \
+		plugins/globus_handler \
+		plugins/virtual_resources \
+		plugins/wt_versioning \
+		plugins/minio_assetstore \
+		plugins/sem_viewer \
+		plugins/dataflows \
+		plugins/table_view \
+		plugins/synced_folders
+	docker exec -ti $$(docker ps --filter=name=wt_girder -q) girder-install web --dev --plugins=oauth,gravatar,jobs,worker,wt_data_manager,wholetale,wt_home_dir,globus_handler,sem_viewer,table_view,synced_folders,minio_assetstore,dataflows
 	docker exec --user=root -ti $$(docker ps --filter=name=wt_girder -q) pip install -r /gwvolman/requirements.txt -e /gwvolman
 	docker exec --user=root -ti $$(docker ps --filter=name=wt_girder -q) pip install -e /girderfs
 	./setup_girder.py

README.md

@@ -91,6 +91,14 @@ Run:
 make dev
 ```
 
+or
+
+```
+DATACAT=1 make dev
+```
+
+for WholeTale flavor operating on local data.
+
 To confirm things are working, all `REPLICAS` should show `1/1`
 ```
 $ docker service ls

dev_images.json

@@ -138,8 +138,8 @@
       "command": "/start.sh",
       "environment": [
         "WT_ROCKER_VER=4.2",
-        "WT_RSTUDIO_URL=https://github.com/whole-tale/rstudio/releases/download/v1.4.1106-wt/rstudio-server-1.4.1106-bionic-amd64.deb",
-        "WT_RSTUDIO_MD5=1d2bbd588f9a3ac00580939d4812a7d1",
+        "WT_RSTUDIO_URL=https://github.com/whole-tale/rstudio/releases/download/v2022.12.0%2B353/rstudio-server-2022.12.0-353-jammy-amd64.deb",
+        "WT_RSTUDIO_MD5=d2339bad213562f52354260c08cb1f6e",
         "PASSWORD=R994ULxH-YYQz^mH"
       ],
       "memLimit": "8196m",

docker-stack.yml

@@ -4,9 +4,17 @@ services:
   traefik:
     image: traefik:v2.4
     ports:
-      - "80:80"
-      - "443:443"
-      - "8080:8080"
+      - target: 80
+        published: 80
+        protocol: tcp
+        mode: host
+      - target: 443
+        published: 443
+        protocol: tcp
+        mode: host
+      - target: 8080
+        published: 8080
+        protocol: tcp
     networks:
       - traefik-net
     volumes:
@@ -17,9 +25,43 @@ services:
       replicas: 1
       labels:
         - "traefik.enable=false"
+      placement:
+        constraints:
+          - "node.role == manager"
     environment:
       - GODADDY_API_KEY=$WT_GODADDY_API_KEY
       - GODADDY_API_SECRET=$WT_GODADDY_API_SECRET
+    command:
+      - "--accesslog=true"
+      - "--accesslog.format=json"
+      - "--accesslog.filepath=/dev/stdout"
+
+  minio:
+    image: minio/minio
+    env_file: ./minio.env
+    ports:
+      - 9000:9000
+    volumes:
+      - ./volumes/minio:/data
+    networks:
+      - traefik-net
+    command: minio server /data --console-address ":9001"
+    deploy:
+      labels:
+        - "traefik.enable=true"
+        - "traefik.docker.network=wt_traefik-net"
+        - "traefik.http.routers.minio.rule=Host(`minio.${domain}`)"
+        - "traefik.http.routers.minio.entrypoints=websecure"
+        - "traefik.http.routers.minio.tls=true"
+        - "traefik.http.routers.minio.service=minio"
+        - "traefik.http.services.minio.loadbalancer.server.port=9000"
+        - "traefik.http.services.minio.loadbalancer.passhostheader=true"
+        - "traefik.http.routers.minio-console.rule=Host(`minio-console.${domain}`)"
+        - "traefik.http.routers.minio-console.entrypoints=websecure"
+        - "traefik.http.routers.minio-console.tls=true"
+        - "traefik.http.routers.minio-console.service=minio-console"
+        - "traefik.http.services.minio-console.loadbalancer.server.port=9001"
+        - "traefik.http.services.minio-console.loadbalancer.passhostheader=true"
 
   mongo:
     image: mongo:3.6
@@ -34,33 +76,43 @@ services:
         - "traefik.enable=false"
 
   girder:
-    image: wholetale/girder:latest
+    image: wholetale/girder:approval
     networks:
       - traefik-net
       - celery
       - mongo
     volumes:
-      - ./volumes:/tmp/data
+      - ./volumes:/srv/data
       - ./girder/girder.local.cfg:/girder/girder/conf/girder.local.cfg
+      - ./workspace.yaml:/girder/workspace.yaml
       - ./src/wholetale:/girder/plugins/wholetale
+      - ./src/minio_assetstore:/girder/plugins/minio_assetstore
+      - ./src/synced_folders:/girder/plugins/synced_folders
+      - ./src/sem_viewer:/girder/plugins/sem_viewer
+      - ./src/table_view:/girder/plugins/table_view
+      - ./src/item_previews:/girder/plugins/item_previews
       - ./src/wt_data_manager:/girder/plugins/wt_data_manager
       - ./src/wt_home_dir:/girder/plugins/wt_home_dir
       - ./src/wt_versioning:/girder/plugins/wt_versioning
       - ./src/globus_handler:/girder/plugins/globus_handler
       - ./src/virtual_resources:/girder/plugins/virtual_resources
+      - ./src/dataflows:/girder/plugins/dataflows
       - ./src/gwvolman:/gwvolman
       - ./src/girderfs:/girderfs
       - /var/run/docker.sock:/var/run/docker.sock
       - /tmp:/tmp
+      - /home/xarth/codes/htmdec:/home/xarth/codes/htmdec
     environment:
       - DOMAIN=${domain}
       - DASHBOARD_URL=https://dashboard.${domain}
       - GOSU_USER=girder:girder:${docker_group}
-      - GOSU_CHOWN=/tmp/data
+      - GOSU_CHOWN=/srv/data
       - DATAONE_URL=https://cn-stage-2.test.dataone.org/cn
+      - GIRDER_LOCAL_FONTELLO_ARCHIVE=/girder/clients/web/static/built/fontello.zip
       - HOSTDIR=/
     extra_hosts:
       - "images.${domain}:host-gateway"
+      - "minio.${domain}:host-gateway"
     deploy:
       replicas: 1
       labels:
@@ -216,6 +268,67 @@ services:
         - "traefik.enable=true"
     command: -c 3
 
+  postgresql:
+    image: postgres:11
+    networks:
+      - dagster
+    environment:
+      - POSTGRES_USER=postgres_user
+      - POSTGRES_PASSWORD=postgres_password
+      - POSTGRES_DB=postgres_db
+
+  dagster_web:
+    image: xarthisius/dagster:latest
+    depends_on:
+      - postgresql
+    entrypoint: ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000", "-w", "workspace.yaml"]
+    networks:
+      - traefik-net
+      - dagster
+    deploy:
+      replicas: 1
+      labels:
+        - "traefik.enable=true"
+        - "traefik.http.routers.dagster.rule=Host(`dagster.${domain}`)"
+        - "traefik.http.routers.dagster.entrypoints=websecure"
+        - "traefik.http.routers.dagster.tls=true"
+        - "traefik.http.services.dagster.loadbalancer.server.port=3000"
+        - "traefik.http.services.dagster.loadbalancer.passhostheader=true"
+        - "traefik.docker.network=wt_traefik-net"
+    environment:
+      - DAGSTER_POSTGRES_USER=postgres_user
+      - DAGSTER_POSTGRES_PASSWORD=postgres_password
+      - DAGSTER_POSTGRES_DB=postgres_db
+    extra_hosts:
+      - "girder.${domain}:host-gateway"
+    volumes:
+      - ./workspace.yaml:/opt/dagster/dagster_home/workspace.yaml
+      - ./dagster.yaml:/opt/dagster/dagster_home/dagster.yaml
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./volumes/io_manager_storage:/tmp/io_manager_storage
+
+  dagster_daemon:
+    image: xarthisius/dagster:latest
+    entrypoint: ["dagster-daemon", "run"]
+    environment:
+      - DAGSTER_POSTGRES_USER=postgres_user
+      - DAGSTER_POSTGRES_PASSWORD=postgres_password
+      - DAGSTER_POSTGRES_DB=postgres_db
+    volumes:
+      - ./workspace.yaml:/opt/dagster/dagster_home/workspace.yaml
+      - ./dagster.yaml:/opt/dagster/dagster_home/dagster.yaml
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./volumes/io_manager_storage:/tmp/io_manager_storage
+    extra_hosts:
+      - "girder.${domain}:host-gateway"
+    networks:
+      - dagster
+    deploy:
+      replicas: 1
+      labels:
+        - "traefik.enable=false"
+
+
 networks:
   traefik-net:
     driver: overlay
@@ -224,6 +337,9 @@ networks:
   celery:
     driver: overlay
     attachable: true
+  dagster:
+    driver: overlay
+    attachable: true
 
 volumes:
   mongo-data: {}

girder/girder.local.cfg

@@ -2,6 +2,8 @@
 server.socket_host = "0.0.0.0"
 server.socket_port = 8080
 server.thread_pool = 100
+server.max_request_body_size = 0
+server.max_request_header_size = 0
 tools.proxy.on = True
 
 [database]

scripts/create_versioned_tale.py

@@ -88,7 +88,7 @@
         }
     ],
     "category": "science",
-    "config": {},
+    "config": {"some_key": "some_value"},
     "dataSet": dataset,
     "description": "Something something...",
     "imageId": image["_id"],
@@ -126,7 +126,6 @@
 tale.update({
     "category": "rocket science",
     "config": {"foo": "bar"},
-    "dataSet": [],
     "description": "A better description",
     "imageId": image["_id"],
     "title": "New better title",

scripts/git_pull_master.sh

@@ -2,6 +2,7 @@
 
 for dir in $(find src/ -maxdepth 1 -mindepth 1 -type d | sort) ; do \
   pushd $dir &> /dev/null
+  echo "Updating ${dir}..."
   git checkout -- . &> /dev/null
   git checkout master &> /dev/null
   git pull origin master &> /dev/null

setup_girder.py

@@ -51,7 +51,7 @@ def final_msg():
     params={
         "type": 0,
         "name": "Base",
-        "root": "/tmp/data/base",
+        "root": "/srv/data/base",
     },
 )
 
@@ -68,6 +68,14 @@ def final_msg():
     "wt_home_dir",
     "wt_versioning",
 ]
+if os.environ.get("DATACAT"):
+    plugins += [
+        "table_view",
+        "synced_folders",
+        "dataflows",
+        "sem_viewer",
+        "minio_assetstore",
+    ]
 r = requests.put(
     api_url + "/system/plugins",
     headers=headers,
@@ -126,18 +134,25 @@ def final_msg():
     {"key": "dm.globus_gc_dir", "value": "/opt/globusconnectpersonal"},
     {
         "key": "wholetale.dataverse_extra_hosts",
-        "value": ["dev2.dataverse.org", "demo.dataverse.org"],
+        "value": ["demo.dataverse.org"],
     },
     {
         "key": "wholetale.zenodo_extra_hosts",
         "value": ["https://sandbox.zenodo.org/record/"]
     },
-    {"key": "dm.private_storage_path", "value": "/tmp/data/ps"},
-    {"key": "wthome.homedir_root", "value": "/tmp/data/homes"},
-    {"key": "wthome.taledir_root", "value": "/tmp/data/workspaces"},
-    {"key": "wtversioning.runs_root", "value": "/tmp/data/runs"},
-    {"key": "wtversioning.versions_root", "value": "/tmp/data/versions"},
+    {"key": "dm.private_storage_path", "value": "/srv/data/ps"},
+    {"key": "wthome.homedir_root", "value": "/srv/data/homes"},
+    {"key": "wthome.taledir_root", "value": "/srv/data/workspaces"},
+    {"key": "wtversioning.runs_root", "value": "/srv/data/runs"},
+    {"key": "wtversioning.versions_root", "value": "/srv/data/versions"},
 ]
+if os.environ.get("DATACAT"):
+    settings += [
+        {"key": "wholetale.dashboard_link_title", "value": "Tale Dashboard"},
+        {"key": "wholetale.catalog_link_title", "value": "Data Catalog"},
+        {"key": "wholetale.enable_data_catalog", "value": True},
+        {"key": "core.registration_policy", "value": "approve"},
+    ]
 
 r = requests.put(
     api_url + "/system/setting", headers=headers, params={"list": json.dumps(settings)}