Merge stable into master for release 1.8.0 (#462)

* Adding Polaris consumer and Polaris zip consumer (#448) * Initial commit of Polaris consumer * Add polaris autoscaling * Update spot detection model * Update Docker image tags frontend 0.9.0 and consumer 0.14.0 * Polaris super consumer (#450) * Initial commit of super consumer * Add changes from stable * Initial commit of spot zip consumer * Revert React setting to stable * Decrease TF_MAX_BATCH_SIZE * Pin Jinja version * Update Docker tags and pull policies * Correct integration-test to use google-github-actions/setup-gcloud@main instead of @master * chore: update to Kubernetes 1.22 (#461) * chore: update default k8s version * fix: update frontend ingress for v1. Followed guide: https://kubernetes.io/docs/reference/using-api/deprecation-guide/#ingress-v122 * chore: update cert-manager to v1.8.2. * chore: update rbac auth API to v1. * chore: use apps/v1 instead of extensions/v1beta1. * chore: update prometheus-adapter to newest v2.x * chore: bump frontend chart version * chore: update prometheus-redis-exporter to latest 3.x * chore: bump frontend default container version * fix: update service for frontend. * chore: update ingress to latest version. * feat: upgrade prometheus-operator to kube-prometheus-stack. Lots of major version changes, but that is because of the CRDs. Seemingly no values change is required. * fix: update prometheus URL. * chore: update redis exporter to latest version. * fix: update make cmd for new prometheus name * chore: bump prometheus-adapter to latest version * chore: rename prometheus-operator to kube-prometheus-stack * chore: update redis to latest helm chart * feat: remove service from redis-consumer chart. bump chart version and update default app version to 0.15.0 * chore: bump version to 0.16.2 (prod version) * fix: update display IP command. * chore: update docs with minor OVPN note. * fix: spot-consumer custom metric name. * chore: update autoscaler and janitor to k8s client 22.x * chore: bump version to 1.8.0 * Merge several updates from dev branch into stable (#463) * Bump Mesmer model version to 8 and update consumer to 0.16.1 (#455) Co-authored-by: William Graf <[email protected]> Co-authored-by: Emily Laubscher <[email protected]> Co-authored-by: msschwartz21 <[email protected]>
vanvalenlab · Jul 1, 2022 · 8fe54fc · 8fe54fc
1 parent c89146b
commit 8fe54fc
Show file tree

Hide file tree

Showing 35 changed files with 120 additions and 146 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -26,7 +26,7 @@ If applicable, add screenshots to help explain your problem.
 **Ennvironment:**
 
 - OS: [e.g. macOS, Ubuntu]
-- Kiosk version: [e.g. 1.7.0]
+- Kiosk version: [e.g. 1.8.0]
 - `kubectl version`:
 - `helm version`:
 - `helmfile version`:

diff --git a/Dockerfile b/Dockerfile
@@ -39,7 +39,7 @@ ENV GPU_NODE_MIN_SIZE="0"
 ENV GPU_PER_NODE="1"
 
 # gcloud config
-ENV KUBERNETES_VERSION="1.21"
+ENV KUBERNETES_VERSION="1.22"
 ENV CLOUDSDK_CORE_PROJECT=""
 ENV CLOUDSDK_CONTAINER_CLUSTER=""
 ENV CLOUDSDK_BUCKET=""

diff --git a/README.md b/README.md
@@ -49,7 +49,7 @@ Tracked Image
 Start a terminal shell and install the DeepCell Kiosk wrapper script:
 
 ```bash
-docker run -e DOCKER_TAG=1.7.0 vanvalenlab/kiosk-console:1.7.0 | sudo bash
+docker run -e DOCKER_TAG=1.8.0 vanvalenlab/kiosk-console:1.8.0 | sudo bash
 ```
 
 To start the kiosk, just run `kiosk-console` from the terminal shell.

diff --git a/conf/addons/cluster-autoscaler.yaml b/conf/addons/cluster-autoscaler.yaml
@@ -8,7 +8,7 @@ metadata:
   name: cluster-autoscaler
   namespace: kube-system
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: cluster-autoscaler
@@ -101,7 +101,7 @@ rules:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: cluster-autoscaler
@@ -129,7 +129,7 @@ rules:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: cluster-autoscaler
@@ -147,7 +147,7 @@ subjects:
 
 ---
 
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: cluster-autoscaler
@@ -166,7 +166,7 @@ subjects:
 
 ---
 
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: cluster-autoscaler

diff --git a/conf/addons/nvidia-device-plugin.yaml b/conf/addons/nvidia-device-plugin.yaml
@@ -1,5 +1,5 @@
 # Based on https://github.com/NVIDIA/k8s-device-plugin
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: nvidia-device-plugin

diff --git a/conf/charts/frontend/Chart.yaml b/conf/charts/frontend/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 name: frontend
-version: 0.4.0
+version: 0.5.0
 #kubeVersion: ^1.9.8
 description: This project provides the frontend interface for the Tensorflow-serving backend of Deepcell.
 keywords:

diff --git a/conf/charts/frontend/templates/ingress.yaml b/conf/charts/frontend/templates/ingress.yaml
@@ -1,7 +1,7 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "fullname" . -}}
 {{- $ingressPath := .Values.ingress.path -}}
-apiVersion: networking.k8s.io/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: {{ template "fullname" . }}
@@ -21,17 +21,23 @@ spec:
     - http:
         paths:
           - path: {{ .Values.ingress.path }}
+            pathType: ImplementationSpecific
             backend:
-              serviceName: {{ template "fullname" . }}
-              servicePort: http
+              service:
+                name: {{ template "fullname" . }}
+                port:
+                  name: http
   {{- range .Values.ingress.hosts }}
     - host: {{ . }}
       http:
         paths:
           - path: {{ $ingressPath }}
+            pathType: ImplementationSpecific
             backend:
-              serviceName: {{ $fullName }}
-              servicePort: http
+              service:
+                name: {{ $fullName }}
+                port:
+                  name: http
   {{- end }}
   {{- if .Values.ingress.tls }}
   tls:

diff --git a/conf/charts/frontend/values.yaml b/conf/charts/frontend/values.yaml
@@ -6,7 +6,7 @@ replicas: 1
 
 image:
   repository: vanvalenlab/kiosk-frontend
-  tag: 0.8.1
+  tag: 0.11.2
   pullPolicy: IfNotPresent
 
 ingress:

diff --git a/conf/charts/redis-consumer/Chart.yaml b/conf/charts/redis-consumer/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v1
 name: redis-consumer
-version: 0.6.0
+version: 0.7.0
 #kubeVersion: ^1.9.8
 description: This project consumes and processes redis event, placing the final result back to redis.
 keywords:
@@ -13,6 +13,6 @@ maintainers:
     email: [email protected]
     url: vanvalen.caltech.edu
 engine: gotpl
-appVersion: 0.12.0
+appVersion: 0.16.2
 deprecated: false
 tillerVersion: ^2.9.1
diff --git a/conf/charts/redis-consumer/templates/deployment.yaml b/conf/charts/redis-consumer/templates/deployment.yaml
@@ -35,8 +35,6 @@ spec:
       - name: {{ .Chart.Name }}
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         imagePullPolicy: {{ .Values.image.pullPolicy }}
-        ports:
-        - containerPort: {{ .Values.service.httpTargetPort }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
         env:

diff --git a/conf/charts/redis-consumer/templates/service.yaml b/conf/charts/redis-consumer/templates/service.yaml
diff --git a/conf/charts/redis-consumer/values.yaml b/conf/charts/redis-consumer/values.yaml
@@ -6,7 +6,7 @@ replicas: 1
 
 image:
   repository: vanvalenlab/kiosk-redis-consumer
-  tag: 0.12.0
+  tag: 0.16.2
   pullPolicy: IfNotPresent
 
 resources: {}
@@ -17,16 +17,6 @@ affinity: {}
 
 nodeSelector: {}
 
-service:
-  type: "ClusterIP"
-
-  httpIngressEnabled: true
-  internalHttpPort: 8080
-  httpTargetPort: 8080
-  externalHttpPort: 8080
-
-  httpsIngressEnabled: false
-
 hpa:
   enabled: false
   minReplicas: 1
@@ -47,7 +37,7 @@ env:
   DO_NOTHING_TIMEOUT: 0.5
   STORAGE_MAX_BACKOFF: 60
 
-  REDIS_HOST: "redis-master"
+  REDIS_HOST: "redis"
   REDIS_PORT: "6379"
   REDIS_TIMEOUT: 3
 

diff --git a/conf/helmfile.d/0000.ingress.yaml b/conf/helmfile.d/0000.ingress.yaml
@@ -1,7 +1,7 @@
 repositories:
 # Stable repo of official helm charts
-- name: stable
-  url: https://charts.helm.sh/stable
+- name: ingress-nginx
+  url: https://kubernetes.github.io/ingress-nginx
 
 releases:
 
@@ -17,13 +17,13 @@ releases:
   namespace: kube-system
   labels:
     chart: nginx-ingress
-    repo: stable
+    repo: ingress-nginx
     component: ingress
     namespace: kube-system
-    vendor: kubernetes
+    vendor: ingress-nginx
     default: "false"
-  chart: stable/nginx-ingress
-  version: 1.40.2
+  chart: ingress-nginx/ingress-nginx
+  version: 4.1.4
   wait: true
   timeout: 300
   atomic: true
@@ -33,24 +33,23 @@ releases:
         ### Optional: NGINX_INGRESS_CONTROLLER_REPLICA_COUNT; e.g. 2
         replicaCount: {{ env "NGINX_INGRESS_CONTROLLER_REPLICA_COUNT" | default "4" }}
         resources:
-          limits:
-            cpu: 200m
-            memory: 3Gi
           requests:
             cpu: 50m
             memory: 128Mi
         config:
           proxy-body-size: 1g
         metrics:
-          ### Optional: NGINX_INGRESS_CONTROLLER_METRICS_ENABLED; enable Prometheus metrics (controller.stats.enabled must be true as well)
+          ### Optional: NGINX_INGRESS_CONTROLLER_METRICS_ENABLED; enable Prometheus metrics
           enabled: {{ env "NGINX_INGRESS_CONTROLLER_METRICS_ENABLED" | default "true" }}
         service:
           annotations:
             ### Required: NGINX_INGRESS_HOSTNAME; e.g. ingress.us-west-2.cloudposse.co
             external-dns.alpha.kubernetes.io/hostname: '{{ env "NGINX_INGRESS_HOSTNAME" }}'
             ### Optional: NGINX_INGRESS_TTL; e.g. 60
             external-dns.alpha.kubernetes.io/ttl: '{{ env "NGINX_INGRESS_TTL" | default "60" }}'
+
       defaultBackend:
+        enabled: true
         ### Optional: NGINX_INGRESS_BACKEND_REPLICA_COUNT; e.g. 2
         replicaCount: {{ env "NGINX_INGRESS_DEFAULT_BACKEND_REPLICA_COUNT" | default "2" }}
         resources:
@@ -66,9 +65,11 @@ releases:
             external-dns.alpha.kubernetes.io/hostname: '{{ env "NGINX_INGRESS_HOSTNAME" }}'
             ### Optional: NGINX_INGRESS_TTL; e.g. 60
             external-dns.alpha.kubernetes.io/ttl: '{{ env "NGINX_INGRESS_TTL" | default "60" }}'
+
       rbac:
         ### Optional: NGINX_INGRESS_RBAC_CREATE;
         create: {{ env "NGINX_INGRESS_RBAC_CREATE" | default "true" }}
+
       serviceAccount:
         ### Optional: NGINX_INGRESS_SERVICE_ACCOUNT_CREATE;
         create: {{ env "NGINX_INGRESS_SERVICE_ACCOUNT_CREATE" | default "true" }}

diff --git a/conf/helmfile.d/0010.cert-manager.yaml b/conf/helmfile.d/0010.cert-manager.yaml
@@ -25,7 +25,7 @@ releases:
     namespace: cert-manager
     vendor: kubernetes
   chart: jetstack/cert-manager
-  version: 1.1.0
+  version: v1.8.2
   wait: true
   timeout: 300
   atomic: true

diff --git a/conf/helmfile.d/0100.redis.yaml b/conf/helmfile.d/0100.redis.yaml
@@ -17,28 +17,20 @@ releases:
     chart: redis
     component: database
   chart: bitnami/redis
-  version: 10.5.7
+  version: 16.13.0
   namespace: deepcell
   wait: true
   atomic: true
   timeout: 600
   cleanupOnFail: true
   values:
-    - resources:
-        requests:
-          cpu: 200m
-          memory: 256Mi
+    - architecture: replication
 
-      usePassword: false
+      auth:
+        enabled: false
+        sentinel: false
 
-      cluster:
-        enabled: true
-        slaveCount: 3
-
-      sentinel:
-        enabled: true
-
-      configmap: |-
+      commonConfiguration: |-
         # Enable AOF https://redis.io/topics/persistence#append-only-file
         appendonly yes
         # Disable RDB persistence, AOF persistence already enabled.
@@ -49,8 +41,23 @@ releases:
         # "no" that is the safest pick from the point of view of durability.
         no-appendfsync-on-rewrite yes
 
-      sysctlImage:
+      master:
+        resources:
+          requests:
+            cpu: 200m
+            memory: 256Mi
+
+      replica:
+        replicaCount: 3
+        resources:
+          requests:
+            cpu: 200m
+            memory: 256Mi
+
+      sentinel:
         enabled: true
+
+      sysctl:
         mountHostSys: true
         command:
           - /bin/sh

diff --git a/conf/helmfile.d/0110.prometheus-redis-exporter.yaml b/conf/helmfile.d/0110.prometheus-redis-exporter.yaml
@@ -1,7 +1,7 @@
 repositories:
-  # Stable repo of official helm charts
-  - name: stable
-    url: https://charts.helm.sh/stable
+  # Stable repo of official prometheus helm charts
+  - name: prometheus-community
+    url: https://prometheus-community.github.io/helm-charts
   # Kubernetes incubator repo of helm charts
   - name: incubator
     url: https://charts.helm.sh/incubator
@@ -103,15 +103,15 @@ releases:
 #   - https://github.com/helm/charts/blob/master/stable/prometheus-redis-exporter/values.yaml
 #
 - name: prometheus-redis-exporter
-  chart: stable/prometheus-redis-exporter
+  chart: prometheus-community/prometheus-redis-exporter
   namespace: monitoring
   labels:
     chart: prometheus-redis-exporter
     component: monitoring
     namespace: monitoring
   needs:
     - monitoring/prometheus-redis-exporter-script
-  version: 3.5.0
+  version: 4.8.0
   wait: true
   timeout: 120
   atomic: true

diff --git a/conf/helmfile.d/0210.autoscaler.yaml b/conf/helmfile.d/0210.autoscaler.yaml
@@ -27,7 +27,7 @@ releases:
 
       image:
         repository: vanvalenlab/kiosk-autoscaler
-        tag: 0.9.0
+        tag: 0.10.0
 
       resources:
         requests: