diff --git a/components/kserve/resources/servicemesh/routing/istio-kserve-local-gateway.tmpl.yaml b/components/kserve/resources/servicemesh/routing/istio-kserve-local-gateway.tmpl.yaml
new file mode 100644
index 00000000000..f411e6a0fcb
--- /dev/null
+++ b/components/kserve/resources/servicemesh/routing/istio-kserve-local-gateway.tmpl.yaml
@@ -0,0 +1,18 @@
+# This Gateway is used for KServe internal communication. Each InferenceService (ISVC) will update this Gateway, particularly by adding a Server that includes TLS settings, hosts, and port configurations. This Gateway is created by the OpenDataHub Operator as a feature, but it will not be managed by the OpenDataHub Operator after its creation.
+apiVersion: networking.istio.io/v1beta1
+kind: Gateway
+metadata:
+  name: kserve-local-gateway
+  namespace: {{ .ControlPlane.Namespace }}
+spec:
+  selector:
+    knative: ingressgateway
+  servers:
+    - hosts:
+        - 'default.host'
+      port:
+        name: https
+        number: 8445
+        protocol: HTTPS
+      tls:
+        mode: ISTIO_MUTUAL
diff --git a/components/kserve/resources/servicemesh/routing/kserve-local-gateway-svc.tmpl.yaml b/components/kserve/resources/servicemesh/routing/kserve-local-gateway-svc.tmpl.yaml
new file mode 100644
index 00000000000..060608c1db7
--- /dev/null
+++ b/components/kserve/resources/servicemesh/routing/kserve-local-gateway-svc.tmpl.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    experimental.istio.io/disable-gateway-port-translation: "true"
+  name: kserve-local-gateway
+  namespace: {{ .ControlPlane.Namespace }}
+spec:
+  ports:
+    - name: https
+      protocol: TCP
+      port: 443
+      targetPort: 8445
+  selector:
+    knative: ingressgateway
+  type: ClusterIP