diff --git a/charts/hedera-network/config-files/envoy.yaml b/charts/hedera-network/config-files/envoy.yaml index 7d22e5c8a..59edb6796 100644 --- a/charts/hedera-network/config-files/envoy.yaml +++ b/charts/hedera-network/config-files/envoy.yaml @@ -51,5 +51,5 @@ static_resources: - endpoint: address: socket_address: - address: network-{{ .nodeConfig.name }}-service - port_value: 50211 \ No newline at end of file + address: network-{{ .nodeConfig.name }}-svc + port_value: 50211 diff --git a/charts/hedera-network/config-files/haproxy.cfg b/charts/hedera-network/config-files/haproxy.cfg index a595718b2..7ef03025c 100644 --- a/charts/hedera-network/config-files/haproxy.cfg +++ b/charts/hedera-network/config-files/haproxy.cfg @@ -11,9 +11,8 @@ defaults timeout client 30s timeout server 30s frontend fe_proxy - mode http - option httplog - option http-use-htx + mode tcp + option tcplog option logasap # Read the blacklist of OFAC Sanctioned Country IP List # Log & Reject the Traffic @@ -21,7 +20,7 @@ frontend fe_proxy tcp-request connection reject if ofac_ip acl ofac_ip_xff hdr_ip(x-forwarded-for) -f /var/opt/ofac/ofacBlock.acl http-request deny if ofac_ip_xff - bind *:50211 proto h2 + bind *:50211 default_backend be_servers frontend fe_proxy_tls mode tcp @@ -33,13 +32,12 @@ frontend fe_proxy_tls bind *:50212 default_backend be_servers_tls backend be_servers - mode http - option http-use-htx + mode tcp # Health Checks: Poll the server with regular health checks every 10 seconds # Health Checks: Observe all Layer 4 TCP connections for problems. # Health Checks: 5 connections must fail before on-error parameter is invoked and marks server down. # Health Checks: Revive the server with regular health checks by polling the server every 10 seconds. - server server1 network-{{ .nodeConfig.name }}-svc:50211 proto h2 check inter 10s downinter 10s observe layer4 error-limit 5 on-error mark-down + server server1 network-{{ .nodeConfig.name }}-svc:50211 check inter 10s downinter 10s observe layer4 error-limit 5 on-error mark-down backend be_servers_tls mode tcp option ssl-hello-chk diff --git a/charts/hedera-network/templates/gateway-api/envoy-routes.yaml b/charts/hedera-network/templates/gateway-api/envoy-routes.yaml new file mode 100644 index 000000000..98b408ab1 --- /dev/null +++ b/charts/hedera-network/templates/gateway-api/envoy-routes.yaml @@ -0,0 +1,24 @@ +{{- range $index, $node := ($.Values.hedera.nodes) }} +{{- $envoyProxy := $node.envoyProxy | default dict -}} +{{- $defaults := $.Values.defaults.envoyProxy }} +{{- if default $defaults.enable $envoyProxy.enable | eq "true" }} +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: envoy-routes-{{ $node.name }} + namespace: default + labels: + fullstack.hedera.com/type: http-route +spec: + parentRefs: + - name: fst + sectionName: http-{{ $node.name }} + hostnames: + - {{ tpl $.Values.gatewayApi.route.hostname (dict "node" $node "Template" $.Template) }} + rules: + - backendRefs: + - name: envoy-proxy-{{ $node.name }}-svc + port: 8080 +{{- end }} +{{- end }} diff --git a/charts/hedera-network/templates/gateway-api/gateway.yaml b/charts/hedera-network/templates/gateway-api/gateway.yaml new file mode 100644 index 000000000..769ec881c --- /dev/null +++ b/charts/hedera-network/templates/gateway-api/gateway.yaml @@ -0,0 +1,52 @@ +{{- if $.Values.gatewayApi.gatewayClass.enable | eq "true" }} +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: GatewayClass +metadata: + name: {{ $.Values.gatewayApi.gatewayClass.name }} + namespace: default + labels: + fullstack.hedera.com/type: gateway-class +spec: + controllerName: {{ $.Values.gatewayApi.gatewayClass.controllerName }} +{{- end }} +{{- if $.Values.gatewayApi.gateway.enable | eq "true" }} +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: {{ $.Values.gatewayApi.gateway.name }} + namespace: default + labels: + fullstack.hedera.com/type: gateway +spec: + gatewayClassName: {{ $.Values.gatewayApi.gatewayClass.name }} + listeners: + - name: http-debug + protocol: HTTP + port: 80 + - name: tcp-debug + protocol: TCP + port: 9000 + allowedRoutes: + kinds: + - kind: TCPRoute + - name: grpc-debug + protocol: TCP + port: 9090 + allowedRoutes: + kinds: + - kind: TCPRoute # we use TCPRoute to for GRPC + {{- range $index, $node := $.Values.hedera.nodes }} + {{- $tcp_port := mul $index 1000 | add 50211 }} + {{- $http_port := mul $index 100 | add 8080 }} + - name: tcp-{{ $node.name }} # for haproxy or network-node TCPRoute + protocol: TCP + port: {{ $tcp_port }} + allowedRoutes: + kinds: + - kind: TCPRoute + - name: http-{{ $node.name }} # for envoy-proxy HTTPRoute + protocol: HTTP + port: {{ $http_port }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/hedera-network/templates/gateway-api/haproxy-routes.yaml b/charts/hedera-network/templates/gateway-api/haproxy-routes.yaml new file mode 100644 index 000000000..6f7c4abe6 --- /dev/null +++ b/charts/hedera-network/templates/gateway-api/haproxy-routes.yaml @@ -0,0 +1,23 @@ +{{- range $index, $node := ($.Values.hedera.nodes) }} +{{- $haproxy := $node.haproxy | default dict -}} +{{- $defaults := $.Values.defaults.haproxy }} +{{- if default $defaults.enable $haproxy.enable | eq "true" }} +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: haproxy-grpc-route-{{ $node.name }} + namespace: default + labels: + fullstack.hedera.com/type: tcp-route + fullstack.hedera.com/nodeName: {{ $node.name }} +spec: + parentRefs: + - name: fst + sectionName: tcp-{{ $node.name }} + rules: + - backendRefs: + - name: haproxy-{{ $node.name }}-svc + port: 50211 +{{- end }} +{{- end }} diff --git a/charts/hedera-network/templates/gateway-api/network-node-routes.yaml b/charts/hedera-network/templates/gateway-api/network-node-routes.yaml new file mode 100644 index 000000000..29e3f647e --- /dev/null +++ b/charts/hedera-network/templates/gateway-api/network-node-routes.yaml @@ -0,0 +1,23 @@ +{{- range $index, $node := ($.Values.hedera.nodes) }} +{{- $haproxy := $node.haproxy | default dict -}} +{{- $defaults := $.Values.defaults.haproxy }} +{{- if default $defaults.enable $haproxy.enable | eq "false" }} +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: node-grpc-route-{{ $node.name }} + namespace: default + labels: + fullstack.hedera.com/type: tcp-route + fullstack.hedera.com/nodeName: {{ $node.name }} +spec: + parentRefs: + - name: fst + sectionName: tcp-{{ $node.name }} + rules: + - backendRefs: + - name: network-{{ $node.name }}-svc + port: 50211 +{{- end }} +{{- end }} diff --git a/charts/hedera-network/templates/services/envoy-svc.yaml b/charts/hedera-network/templates/services/envoy-svc.yaml index 35b3419a9..53883b47d 100644 --- a/charts/hedera-network/templates/services/envoy-svc.yaml +++ b/charts/hedera-network/templates/services/envoy-svc.yaml @@ -1,6 +1,6 @@ {{ range $index, $node := ($.Values.hedera.nodes) }} {{- $envoyProxy := $node.envoyProxy | default dict -}} -{{- $defaults := $.Values.defaults.haproxy }} +{{- $defaults := $.Values.defaults.envoyProxy }} {{- if default $defaults.enable $envoyProxy.enable | eq "true" }} --- apiVersion: v1 diff --git a/charts/hedera-network/values.yaml b/charts/hedera-network/values.yaml index 391f951a2..d7d86e5c3 100644 --- a/charts/hedera-network/values.yaml +++ b/charts/hedera-network/values.yaml @@ -30,6 +30,19 @@ tester: pullPolicy: "IfNotPresent" resources: {} +# gateway-api configuration +gatewayApi: + gatewayClass: + name: "fst" + enable: "true" + controllerName: "gateway.envoyproxy.io/gatewayclass-controller" +# controllerName: "haproxy-ingress.github.io/controller" + gateway: + name: "fst" + enable: "true" + route: + hostname: "{{ .node.name }}.fst.local" + # default settings for a single node # This default configurations can be overridden for each node in the hedera.nodes section. defaults: diff --git a/dev/Makefile b/dev/Makefile index 65a7e8024..769ed9b36 100644 --- a/dev/Makefile +++ b/dev/Makefile @@ -23,7 +23,10 @@ SCRIPTS_DIR=$(PWD)/scripts CHART_DIR=$(PWD)/../charts/hedera-network SCRIPT_NAME=direct-install.sh TMP_DIR=${SCRIPTS_DIR}/../temp + +# scripts TELEMETRY_SCRIPT="telemetry.sh" +GATEWAY_API_SCRIPT="gateway.sh" .PHONY: all all: setup setup-cluster reset @@ -42,7 +45,7 @@ update-helm-dependencies: helm dependency update ../charts/hedera-network .PHONY: deploy-chart -deploy-chart: deploy-minio-operator-if-required deploy-prometheus-operator +deploy-chart: deploy-minio-operator-if-required deploy-prometheus-operator deploy-gateway-api echo ">> Deploying helm chart..." && \ echo "" && \ if [ "${SCRIPT_NAME}" = "nmt-install.sh" ]; then \ @@ -143,6 +146,17 @@ restart: stop-nodes start-nodes .PHONY: reset reset: destroy-network start +######################################### Gateway API ################################# +.PHONY: deploy-gateway-api +deploy-gateway-api: + #source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && deploy_haproxy_ingress + source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && deploy_envoy_gateway_api + +.PHONY: destroy-gateway-api +destroy-gateway-api: + #source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_haproxy_ingress + source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_envoy_gateway_api + ######################################### Prometheus ################################# .PHONY: fetch-prometheus-operator-bundle fetch-prometheus-operator-bundle: diff --git a/dev/gateway-api/Makefile b/dev/gateway-api/Makefile new file mode 100644 index 000000000..fe8a9cd6e --- /dev/null +++ b/dev/gateway-api/Makefile @@ -0,0 +1,43 @@ +SHELLOPTS:=$(if $(SHELLOPTS),$(SHELLOPTS):)pipefail:errexit +.ONESHELL: + +SCRIPTS_DIR=$(PWD)/../scripts +GATEWAY_API_SCRIPT="gateway.sh" + +.PHONY: deploy-gateway-api +deploy-gateway-api: + #source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && deploy_haproxy_ingress + source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && deploy_envoy_gateway_api + +.PHONY: destroy-gateway-api +destroy-gateway-api: + #source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_haproxy_ingress + source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && destroy_envoy_gateway_api + +.PHONY: deploy-fst-gateway +deploy-fst-gateway: deploy-gateway-api + kubectl apply -f ./fst-gateway.yaml + +.PHONY: destroy-fst-gateway +destroy-fst-gateway: + kubectl delete -f fst-gateway.yaml + +.PHONY: expose-envoy-gateway-svc +expose-envoy-gateway-svc: + source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && expose_envoy_gateway_svc ${PORT} ${GATEWAY_PORT} + +.PHONY: unexpose-envoy-gateway-svc +unexpose-envoy-gateway-svc: + source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && unexpose_envoy_gateway_svc + +.PHONY: test-gateway-http-route +test-gateway-http-route: + source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && test_http_route + +.PHONY: test-gateway-grpc-route +test-gateway-grpc-route: + source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && test_grpc_route + +.PHONY: test-gateway-tcp-route +test-gateway-tcp-route: + source "${SCRIPTS_DIR}/${GATEWAY_API_SCRIPT}" && test_tcp_route diff --git a/dev/gateway-api/README.md b/dev/gateway-api/README.md new file mode 100644 index 000000000..d65783469 --- /dev/null +++ b/dev/gateway-api/README.md @@ -0,0 +1,29 @@ +# Gateway API +This folder includes scripts and files to debug Gateway API. + +## Manual Test +- Deploy `fst` Gateway + - `make deploy-fst-gateway` +- Test HTTPRoute + - ` make test-gateway-http-route` and check for below line + ``` + ******************************************************** + SUCCESS: HTTPRoute debug.fst.local:8080 + ******************************************************** + ``` +- Test TCPRoute + - ` make test-gateway-tcp-route` and check for below line + ``` + ******************************************************** + SUCCESS: TCPRoute localhost:9000 + ******************************************************** + ``` +- Test GRPCRoute + - ` make test-gateway-grpc-route` and check for below line + ``` + ******************************************************** + SUCCESS: GRPCRoute debug.fst.local:9090 + ******************************************************** + ``` +- Delete `fst` Gateway + - `make destroy-fst-gateway` diff --git a/dev/gateway-api/fst-gateway.yaml b/dev/gateway-api/fst-gateway.yaml new file mode 100644 index 000000000..37cbf78ae --- /dev/null +++ b/dev/gateway-api/fst-gateway.yaml @@ -0,0 +1,31 @@ +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: GatewayClass +metadata: + name: fst +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +# controllerName: haproxy-ingress.github.io/controller +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: Gateway +metadata: + name: fst +spec: + gatewayClassName: fst + listeners: + - name: http-debug + protocol: HTTP + port: 80 + - name: tcp-debug + protocol: TCP + port: 9000 + allowedRoutes: + kinds: + - kind: TCPRoute + - name: grpc-debug + protocol: TCP + port: 9090 + allowedRoutes: + kinds: + - kind: TCPRoute # we use TCPRoute to for GRPC +--- diff --git a/dev/gateway-api/grpc-debug.yaml b/dev/gateway-api/grpc-debug.yaml new file mode 100644 index 000000000..49f0d2003 --- /dev/null +++ b/dev/gateway-api/grpc-debug.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: grpc-debug + example: grpc-routing + name: grpc-debug +spec: + selector: + matchLabels: + app: grpc-debug + replicas: 1 + template: + metadata: + labels: + app: grpc-debug + spec: + containers: + - name: grpc-debug + image: quay.io/mhausenblas/yages:0.1.0 + ports: + - containerPort: 9000 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: grpc-debug + example: grpc-routing + name: grpc-debug +spec: + type: ClusterIP + ports: + - name: grpc + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app: grpc-debug +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: grpc-debug + labels: + example: grpc-routing +spec: + parentRefs: + - name: fst + sectionName: grpc-debug + rules: + - backendRefs: + - group: "" + kind: Service + name: grpc-debug + port: 9000 + weight: 1 diff --git a/dev/gateway-api/http-debug.yaml b/dev/gateway-api/http-debug.yaml new file mode 100644 index 000000000..8fb0ddaf4 --- /dev/null +++ b/dev/gateway-api/http-debug.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: http-debug +--- +apiVersion: v1 +kind: Service +metadata: + name: http-debug + labels: + app: http-debug + service: http-debug +spec: + ports: + - name: http + port: 3000 + targetPort: 3000 + selector: + app: http-debug +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: http-debug +spec: + replicas: 1 + selector: + matchLabels: + app: http-debug + version: v1 + template: + metadata: + labels: + app: http-debug + version: v1 + spec: + serviceAccountName: http-debug + containers: + - image: gcr.io/k8s-staging-ingressconformance/echoserver:v20221109-7ee2f3e + imagePullPolicy: IfNotPresent + name: http-debug + ports: + - containerPort: 3000 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +--- +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: http-debug +spec: + parentRefs: + - name: fst + sectionName: http-debug + hostnames: + - "debug.fst.local" + rules: + - backendRefs: + - group: "" + kind: Service + name: http-debug + port: 3000 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/dev/gateway-api/tcp-debug.yaml b/dev/gateway-api/tcp-debug.yaml new file mode 100644 index 000000000..e03b45ebd --- /dev/null +++ b/dev/gateway-api/tcp-debug.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tcp-debug +--- +apiVersion: v1 +kind: Service +metadata: + name: tcp-debug + labels: + app: tcp-debug + service: tcp-debug +spec: + ports: + - name: http + port: 9000 + targetPort: 9000 + selector: + app: tcp-debug +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tcp-debug +spec: + replicas: 1 + selector: + matchLabels: + app: tcp-debug + version: v1 + template: + metadata: + labels: + app: tcp-debug + version: v1 + spec: + serviceAccountName: tcp-debug + containers: + - image: docker.io/istio/tcp-echo-server:1.2 + imagePullPolicy: IfNotPresent + name: tcp-debug + ports: + - containerPort: 9000 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +--- +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: tcp-debug +spec: + parentRefs: + - name: fst + sectionName: tcp-debug + rules: + - backendRefs: + - name: tcp-debug + port: 9000 diff --git a/dev/scripts/gateway.sh b/dev/scripts/gateway.sh new file mode 100644 index 000000000..a1fcbc161 --- /dev/null +++ b/dev/scripts/gateway.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash + +GATEWAY_API_VERSION="${GATEWAY_API_VERSION:-v0.7.1}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +readonly GATEWAY_API_DIR="${SCRIPT_DIR}/../gateway-api" +readonly SCRIPT_DIR + +function deploy_haproxy_ingress() { + deploy_gateway_api_crd + + echo "" + echo "Deploying HAProxy Ingress Controller" + echo "-----------------------------------------------------------------------------------------------------" + local helm_chart=$(helm list --all-namespaces | grep haproxy-ingress) + if [[ ! "${helm_chart}" ]]; then + helm repo add haproxy-ingress https://haproxy-ingress.github.io/charts + helm install haproxy-ingress haproxy-ingress/haproxy-ingress\ + --create-namespace --namespace haproxy-ingress \ + --version 0.14.4 + else + echo "HAProxy Ingress Controller is already installed" + echo "" + fi +} + +function destroy_haproxy_ingress() { + echo "" + echo "Uninstalling HAProxy Ingress Controller" + echo "-----------------------------------------------------------------------------------------------------" + get_gateway_status + + # Uninstall helm chart + local helm_chart=$(helm list --all-namespaces | grep haproxy-ingress) + if [[ "${helm_chart}" ]]; then + helm uninstall haproxy-ingress -n haproxy-ingress + kubectl delete ns haproxy-ingress + fi + + uninstall_crd "gateway.networking.k8s.io" + + echo "HAProxy Ingress Controller is uninstalled" + echo "" +} + +function deploy_gateway_api_crd() { + echo "" + echo "Deploying Gateway API CRD" + echo "-----------------------------------------------------------------------------------------------------" + # Expected list of CRDs + # ------------------------- + # gatewayclasses.gateway.networking.k8s.io + # gateways.gateway.networking.k8s.io + # httproutes.gateway.networking.k8s.io + # referencepolicies.gateway.networking.k8s.io + # tcproutes.gateway.networking.k8s.io + # tlsroutes.gateway.networking.k8s.io + # udproutes.gateway.networking.k8s.io + local crd_count=$(kubectl get crd | grep -c "gateway.networking.k8s.io") + + if [[ $crd_count -ne 7 ]]; then + kubectl kustomize\ + "github.com/kubernetes-sigs/gateway-api/config/crd?ref=${GATEWAY_API_VERSION}" |\ + kubectl apply -f - + else + echo "Gateway API CRD is already installed" + echo "" + fi +} + +function deploy_envoy_gateway_api() { + echo "" + echo "Installing Envoy Gateway API" + echo "-----------------------------------------------------------------------------------------------------" + local helm_chart=$(helm list --all-namespaces | grep envoy-gateway) + if [[ ! "${helm_chart}" ]]; then + helm install envoy-gateway oci://docker.io/envoyproxy/gateway-helm --version v0.0.0-latest -n envoy-gateway-system --create-namespace + kubectl wait --timeout=5m -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available + else + echo "Envoy Gateway API is already installed" + echo "" + fi +} + +function get_gateway_status() { + echo "" + helm list --all-namespaces | grep envoy-gateway + echo "-----------------------------------------------------------------------------------------------------" + kubectl get crd + echo "-----------------------------------------------------------------------------------------------------" + kubectl get gatewayclass + echo "-----------------------------------------------------------------------------------------------------" + kubectl get gateway + echo "-----------------------------------------------------------------------------------------------------" + kubectl get httproute + echo "-----------------------------------------------------------------------------------------------------" + kubectl get grpcroute + echo "-----------------------------------------------------------------------------------------------------" +} + +function destroy_envoy_gateway_api() { + echo "" + echo "Uninstalling Envoy Gateway API" + echo "-----------------------------------------------------------------------------------------------------" + get_gateway_status + + # Uninstall helm chart + local helm_chart=$(helm list --all-namespaces | grep envoy-gateway) + if [[ "${helm_chart}" ]]; then + helm uninstall envoy-gateway -n envoy-gateway-system + kubectl delete ns envoy-gateway-system + fi + + uninstall_crd "gateway.networking.k8s.io" + uninstall_crd "gateway.envoyproxy.io" + + echo "Envoy Gateway API is uninstalled" + echo "" +} + +function uninstall_crd() { + local pattern="${1}" + local crds=$(kubectl get crd | grep "${pattern}" | awk '{print $1 }') + + if [[ "${crds}" != "" ]]; then + read -a installed -d '' -r <<< "${crds}" || true + for name in "${installed[@]}"; do + echo "Uninstalling CRD: $name" + kubectl delete crd "${name}" + done + fi +} + +function expose_envoy_gateway_svc() { + local local_port="${1}" + [[ -z "${local_port}" ]] && echo "ERROR: local port is required" && return 1 + + local gateway_port="${2}" + [[ -z "${gateway_port}" ]] && echo "ERROR: gateway port is required" && return 1 + + unexpose_envoy_gateway_svc || true + + ENVOY_SERVICE=$(kubectl get svc -n envoy-gateway-system --selector=gateway.envoyproxy.io/owning-gateway-namespace=default,gateway.envoyproxy.io/owning-gateway-name=fst -o jsonpath="{.items[0].metadata.name}" ) + echo "" + echo "Exposing Envoy Gateway Service: ${ENVOY_SERVICE} on ${local_port}:${gateway_port}" + echo "-----------------------------------------------------------------------------------------------------" + kubectl port-forward "svc/${ENVOY_SERVICE}" -n envoy-gateway-system "${local_port}":"${gateway_port}" & +} + +function unexpose_envoy_gateway_svc() { + export GATEWAY_SVC_PID=$(ps aux | grep "kubectl port-forward svc/${ENVOY_SERVICE}" | sed -n 2p | awk '{ print $2 }') + [[ -z "${GATEWAY_SVC_PID}" ]] && echo "No Envoy Gateway Service PID is found" && return 0 + + if [[ "${GATEWAY_SVC_PID}" ]]; then + echo "" + echo "Un-exposing Envoy Gateway Service: ${ENVOY_SERVICE} for PID: ${GATEWAY_SVC_PID}" + echo "-----------------------------------------------------------------------------------------------------" + kill "${GATEWAY_SVC_PID}" &>/dev/null || true + fi +} + +function test_http_route() { + echo "Setup" + echo "-----------------------------------------------------------------------------------------------------" + kubectl apply -f "${GATEWAY_API_DIR}/http-debug.yaml" + kubectl wait --for=condition=Ready pods -l app=http-debug -n default + + local local_port=8080 + local gateway_port=80 + expose_envoy_gateway_svc ${local_port} ${gateway_port} || return 1 + + local route_host="debug.fst.local" + + sleep 1 + + echo "Checking ${route_host}" + echo "-----------------------------------------------------------------------------------------------------" + echo "" + + local status=$(curl --header "Host: ${route_host}" -o /dev/null -s -w "%{http_code}\n" localhost:${local_port}) + + echo "" + echo "********************************************************" + if [[ $status -eq 200 ]]; then + echo "SUCCESS: HTTPRoute ${route_host}:${gateway_port}" + else + curl --header "Host: ${route_host}" -vvv localhost:${local_port} + echo "" + echo "FAIL: HTTPRoute ${route_host}:${gateway_port}" + fi + echo "********************************************************" + echo "" + + echo "Cleanup" + echo "-----------------------------------------------------------------------------------------------------" + unexpose_envoy_gateway_svc || true + kubectl delete -f "${GATEWAY_API_DIR}/http-debug.yaml" +} + +function test_grpc_route() { + echo "Setup" + echo "-----------------------------------------------------------------------------------------------------" + kubectl apply -f "${GATEWAY_API_DIR}/grpc-debug.yaml" + kubectl wait --for=condition=Ready pods -l app=grpc-debug -n default + + local local_port=9090 + local gateway_port=9090 + expose_envoy_gateway_svc ${local_port} ${gateway_port} || return 1 + + local route_host="debug.fst.local" + + sleep 1 + + echo "Checking ${route_host}" + echo "-----------------------------------------------------------------------------------------------------" + echo "" + + grpcurl -plaintext -vv -authority=grpc-example.com 127.0.0.1:${local_port} yages.Echo/Ping + local status=$? + + echo "" + echo "********************************************************" + if [[ $status -eq 0 ]]; then + echo "SUCCESS: GRPCRoute ${route_host}:${gateway_port}" + else + echo "FAIL: GRPCRoute ${route_host}:${gateway_port}" + fi + echo "********************************************************" + echo "" + + echo "Cleanup" + echo "-----------------------------------------------------------------------------------------------------" + unexpose_envoy_gateway_svc || true + kubectl delete -f "${GATEWAY_API_DIR}/grpc-debug.yaml" +} + +function test_tcp_route() { + echo "Setup" + echo "-----------------------------------------------------------------------------------------------------" + kubectl apply -f "${GATEWAY_API_DIR}/tcp-debug.yaml" + kubectl wait --for=condition=Ready pods -l app=tcp-debug -n default + + local local_port=9000 + local gateway_port=9000 + expose_envoy_gateway_svc ${local_port} ${gateway_port} || return 1 + sleep 1 + + echo "" + echo "Checking TCP route localhost:${local_port}" + echo "-----------------------------------------------------------------------------------------------------" + echo "" + + timeout 1s bash -c "echo tcp-test | nc localhost ${local_port} >> deleteme.txt" + echo "" + echo "********************************************************" + if [[ -s deleteme.txt ]]; then + echo "SUCCESS: TCPRoute localhost:${local_port}" + else + echo "FAIL: TCPRoute localhost:${local_port}" + fi + echo "********************************************************" + echo "" + + echo "Cleanup" + echo "-----------------------------------------------------------------------------------------------------" + rm deleteme.txt + unexpose_envoy_gateway_svc || true + kubectl delete -f "${GATEWAY_API_DIR}/tcp-debug.yaml" +} diff --git a/dev/scripts/telemetry.sh b/dev/scripts/telemetry.sh index 53fd0fe93..3a2306ee6 100644 --- a/dev/scripts/telemetry.sh +++ b/dev/scripts/telemetry.sh @@ -37,7 +37,8 @@ function deploy-prometheus-operator() { kubectl create -f "${PROMETHEUS_OPERATOR_YAML}" kubectl wait --for=condition=Ready pods -l app.kubernetes.io/name=prometheus-operator -n default else - echo "Kubernetes operator CRD is already installed" + echo "Prometheus operator CRD is already installed" + echo "" fi } @@ -59,7 +60,7 @@ function deploy-prometheus() { kubectl create -f "${PROMETHEUS_RBAC_YAML}" sleep 10 kubectl create -f "${PROMETHEUS_YAML}" - echo "Waiting for prometheus to be running..." + echo "Waiting for prometheus to be active..." kubectl wait --for=condition=Ready pods -l app.kubernetes.io/name=prometheus -n default --timeout 300s }