feat: add helm chart tests for gateway api routes

charts/hedera-network/templates/rbac/pod-monitor-role.yaml

@@ -1,7 +1,8 @@
 apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
+kind: ClusterRole
 metadata:
   name: pod-monitoring-role
+  namespace: {{ .Values.namespace }}
 rules:
   - apiGroups: [ "" ]
     resources:
@@ -16,15 +17,26 @@ rules:
       - pods/exec
     verbs:
       - create
+  - apiGroups: [ "gateway.networking.k8s.io" ]
+    resources:
+      - gatewayclasses
+      - gateways
+      - httproutes
+      - tcproutes
+    verbs:
+      - get
+      - list
 ---
 apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
+kind: ClusterRoleBinding
 metadata:
   name: pod-monitoring-role-binding
+  namespace: {{ .Values.namespace }}
 subjects:
   - kind: ServiceAccount
     name: pod-monitor
+    namespace: {{ .Values.namespace }}
 roleRef:
-  kind: Role
+  kind: ClusterRole
   name: pod-monitoring-role
   apiGroup: rbac.authorization.k8s.io

charts/hedera-network/templates/rbac/service-accounts.yaml

@@ -1,4 +1,5 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: pod-monitor
+  name: pod-monitor
+  namespace: {{ .Values.namespace }}

charts/hedera-network/tests/README.md

@@ -6,6 +6,25 @@ you have the network deployed already.
 
 ## Development
 - Use the `test_basic_deployment.bats` file as the template while creating new tests.
+- In order to run and debug the tests inside the helm test container, do the following:
+
+    - Update `run` command section in `charts/hedera-network/template/tests/test-deployment.yaml` as below so that it keeps it running when we run `make helm-test`:
+  ```
+  - "/bin/bash"
+  - "-c"
+  #- "/tests/run.sh"
+  - "while true;do echo sleeping for 60s; sleep 60;done" # keep the test container running so that we can debug issues 
+  ```
+  - once the `network-test` container is running, use another terminal to shell into it using command below
+  ```
+  kubectl exec -it network-test -- bash 
+  ```
+  - Then you can run the test inside the container to debug 
+  ```
+  cd /tests && ./run.sh 
+  ```
+  - Once debug is done, you can exit and use Ctrl+C to terminate the helm-test process (you will need to delete the `network-test` container using `kubectl delete network-test`).
+  - If it looks all good, revert changes in `charts/hedera-network/template/tests/test-deployment.yaml`
 
 ## Run
 - Run `git submodule update --init` in order to install [bats](https://github.com/bats-core) for tests. 

charts/hedera-network/tests/helper.sh

@@ -82,9 +82,10 @@ function check_test_status() {
 
 function get_config_val() {
   local config_path=$1
-  ret=$(helm get values fst -a | tail -n +2 | niet "${config_path}" )
+  log_debug "Get config command: helm get values fst -a | yq '${config_path}'"
+  ret=$(helm get values fst -a | yq "${config_path}" )
   echo "${ret}"
-  log_debug "${enable_config_path} => ${ret}"
+  log_debug "${config_path} => ${ret}"
 }
 
 function get_config_val_upper() {
@@ -93,6 +94,40 @@ function get_config_val_upper() {
   echo "${config_val}"
 }
 
+function is_enabled_for_node() {
+  local node_name=$1
+  [[ -z "${node_name}" ]] && echo "ERROR: Node name is needed" && return "${EX_ERR}"
+
+  local config_path=$2
+  [[ -z "${config_path}" ]] && echo "ERROR: Config path is needed" && return "${EX_ERR}"
+
+  log_debug "Checking config '${config_path}' for node '${node_name}"
+
+  local default_config_path=".defaults${config_path}"
+  local node_config_path=".hedera.nodes[] | select(.name==\"${node_name}\") | ${config_path}"
+  local default_val=$(get_config_val_upper "${default_config_path}")
+  local node_val=$(get_config_val_upper "${node_config_path}")
+  log_debug "Default config: ${default_val}"
+  log_debug "Node config: ${node_val}"
+
+  if [ -z "${node_val}" ] || [ "${node_val}" = "FALSE" ]; then
+    echo "FALSE"
+    return
+  fi
+
+  if [ "${node_val}" = "TRUE" ]; then
+    echo "TRUE"
+    return
+  fi
+
+  if [ "${default_val}" = "TRUE" ]; then
+    echo "TRUE"
+    return
+  fi
+
+  echo "FALSE"
+}
+
 function get_sidecar_status() {
   local pod=$1
   local sidecar_name=$2
@@ -110,6 +145,7 @@ function is_sidecar_ready() {
   [[ -z "${sidecar_name}" ]] && echo "ERROR: Sidecar name is needed (is_sidecar_ready)" && return "${EX_ERR}"
 
   local sidecar_status=$(kubectl get pod "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${sidecar_name}')].ready}" | tr '[:lower:]' '[:upper:]')
+  [ -z "${sidecar_status}" ] && sidecar_status="FALSE"
   log_debug "${sidecar_name} in pod ${pod} is ready: ${sidecar_status}"
 
   [[ "${sidecar_status}" = "TRUE" ]] && return "${EX_OK}"
@@ -138,6 +174,8 @@ function is_pod_ready() {
   [[ -z "${pod}" ]] && echo "ERROR: Pod name is needed (is_pod_ready)" && return "${EX_ERR}"
 
   local pod_status=$(kubectl get pod "${pod}" -o jsonpath="{.status.conditions[?(@.type=='Ready')].status}" | tr '[:lower:]' '[:upper:]')
+  [ -z "${pod_status}" ] && pod_status="FALSE"
+
   log_debug "Pod '${pod}' is ready: ${pod_status}"
 
   [[ "${pod_status}" = "TRUE" ]] && return "${EX_OK}"
@@ -152,12 +190,13 @@ function get_pod_label() {
   [[ -z "${pod}" ]] && echo "ERROR: Label name is needed" && return "${EX_ERR}"
 
 
-  log_debug "Checking for pod ${pod}(timeout 300s)..."
-  if [ ! $(kubectl wait --for=condition=Initialized pods "${pod}" --timeout 300s) ]; then
+  log_debug "Checking for pod '${pod}'(timeout 300s)..."
+  $(kubectl wait --for=condition=Initialized pods "${pod}" --timeout 300s) > /dev/null 2>&1
+  if [ $? = 1 ]; then
     log_debug "ERROR: Pod ${pod} is not available" &&  return "${EX_ERR}"
   fi
 
-  log_debug "Checking label ${label} for pod ${pod}"
+  log_debug "Checking label '${label}' for pod '${pod}'"
   local escaped_label="${label//./\\.}"
   local label_val=$(kubectl get pod "${pod}" -o jsonpath="{.metadata.labels.${escaped_label}}" | xargs)
   log_debug "Pod '${pod}' label '${label}': ${label_val}"
@@ -174,3 +213,19 @@ function get_pod_by_label() {
   local pod_name=$(kubectl get pods -l "${label}" -o jsonpath="{.items[0].metadata.name}")
   echo "${pod_name}"
 }
+
+function is_route_accepted() {
+  local route_type=$1
+  [[ -z "${route_type}" ]] && echo "ERROR: Route type is needed" && return "${EX_ERR}"
+
+  local route_name=$2
+  [[ -z "${route_name}" ]] && echo "ERROR: Route name is needed" && return "${EX_ERR}"
+
+  local route_status=$(kubectl get "${route_type}" "${route_name}" -o jsonpath="{.status.parents[*].conditions[?(@.type=='Accepted')].status}" | tr '[:lower:]' '[:upper:]')
+  [ -z "${route_status}" ] && route_status="FALSE"
+
+  log_debug "${route_type} '${route_name}' is accepted: ${route_status}"
+
+  [[ "${route_status}" = "TRUE" ]] && return "${EX_OK}"
+  return "${EX_ERR}"
+}

charts/hedera-network/tests/test_gateway_api_deployment.bats

@@ -0,0 +1,121 @@
+# bats file_tags=deployment-test
+setup() {
+    source "$(dirname "${BATS_TEST_FILENAME}")/env.sh"
+    source "${TESTS_DIR}/load.sh"
+}
+
+@test "Check Envoy Proxy GRPC-Web routes" {
+  log_debug "----------------------------------------------------------------------------"
+  log_debug "TEST: Checking Envoy Proxy GRPC-Web Route"
+  log_debug "----------------------------------------------------------------------------"
+
+  local resp="$(get_pod_list network-node)"
+  local pods=(${resp}) # convert into an array
+  log_debug "Network nodes: ${pods[*]}"
+
+  local test_status="${FAIL}"
+  local status_val="${EX_ERR}"
+  if [[ "${#pods[@]}" -gt 0 ]]; then
+    test_status="${PASS}"
+    for pod in "${pods[@]}"; do
+      log_debug ""
+      local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
+      [[ -z "${node_name}" ]] && test_status="${FAIL}" && break
+
+      local route_name="envoy-grpc-web-route-${node_name}"
+      local is_enabled=$(is_enabled_for_node "${node_name}" ".envoyProxy.enable")
+      if [ "${is_enabled}" = "TRUE" ]; then
+        log_debug "EnvoyProxy enabled for node '${node_name}'"
+        log_debug "Checking Envoy proxy httproute '${route_name}'"
+        is_route_accepted "httproute" "${route_name}" || test_status="${FAIL}"
+        [ "${test_status}" = "FAIL" ] && break
+      else
+        log_debug "EnvoyProxy is not enabled for node '${node_name}'. Skipped route check for '${route_name}'."
+      fi
+    done
+  fi
+
+  log_debug ""
+  log_debug "[${test_status}] Envoy Proxy GRPC-Web Route Check"
+  log_debug ""
+
+  # assert success
+  [[ "${test_status}" = "${PASS}" ]]
+}
+
+@test "Check HAProxy GRPC routes" {
+  log_debug "----------------------------------------------------------------------------"
+  log_debug "TEST: Checking HAProxy GRPC Route"
+  log_debug "----------------------------------------------------------------------------"
+
+  local resp="$(get_pod_list network-node)"
+  local pods=(${resp}) # convert into an array
+  log_debug "Network nodes: ${pods[*]}"
+
+  local test_status="${FAIL}"
+  local status_val="${EX_ERR}"
+  if [[ "${#pods[@]}" -gt 0 ]]; then
+    test_status="${PASS}"
+    for pod in "${pods[@]}"; do
+      log_debug ""
+      local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
+      [[ -z "${node_name}" ]] && test_status="${FAIL}" && break
+
+      local route_name="haproxy-grpc-route-${node_name}"
+      local is_enabled=$(is_enabled_for_node "${node_name}" ".haproxy.enable")
+      if [ "${is_enabled}" = "TRUE" ]; then
+        log_debug "HAProxy enabled for node '${node_name}'"
+        log_debug "Checking HAProxy GRPC route '${route_name}'"
+        is_route_accepted "tcproute" "${route_name}" || test_status="${FAIL}"
+        [ "${test_status}" = "FAIL" ] && break
+      else
+        log_debug "HAProxy is not enabled for node '${node_name}'. Skipped route check for '${route_name}'."
+      fi
+    done
+  fi
+
+  log_debug ""
+  log_debug "[${test_status}] HAProxy GRPC Route Check"
+  log_debug ""
+
+  # assert success
+  [[ "${test_status}" = "${PASS}" ]]
+}
+
+@test "Check Network Node GRPC routes" {
+  log_debug "----------------------------------------------------------------------------"
+  log_debug "TEST: Checking Network Node GRPC Route"
+  log_debug "----------------------------------------------------------------------------"
+
+  local resp="$(get_pod_list network-node)"
+  local pods=(${resp}) # convert into an array
+  log_debug "Network nodes: ${pods[*]}"
+
+  local test_status="${FAIL}"
+  local status_val="${EX_ERR}"
+  if [[ "${#pods[@]}" -gt 0 ]]; then
+    test_status="${PASS}"
+    for pod in "${pods[@]}"; do
+      log_debug ""
+      local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
+      [[ -z "${node_name}" ]] && test_status="${FAIL}" && break
+
+      local route_name="node-grpc-route-${node_name}"
+      local is_enabled=$(is_enabled_for_node "${node_name}" ".haproxy.enable")
+      if [ "${is_enabled}" = "FALSE" ]; then
+        log_debug "Checking Node GRPC route '${route_name}'"
+        is_route_accepted "tcproute" "${route_name}" || test_status="${FAIL}"
+        [ "${test_status}" = "FAIL" ] && break
+      else
+        log_debug "HAProxy enabled for node '${node_name}'. Skipped route check for '${route_name}'."
+      fi
+    done
+  fi
+
+  log_debug ""
+  log_debug "[${test_status}] Node GRPC Route Check"
+  log_debug ""
+
+  # assert success
+  [[ "${test_status}" = "${PASS}" ]]
+}

charts/hedera-network/tests/test_proxy_deployment.bats

@@ -22,18 +22,23 @@ setup() {
       local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
       [[ -z "${node_name}" ]] && test_status="${FAIL}" && break
 
-      log_debug "Checking HAProxy for network-node '${node_name}'"
-      local haproxy_pod=$(get_pod_by_label "app=haproxy-${node_name},fullstack.hedera.com/type=haproxy")
-
-      log_debug "Checking HAProxy pod ${haproxy_pod}"
-      is_pod_ready "${haproxy_pod}" || test_status="${FAIL}"
+      local is_enabled=$(is_enabled_for_node "${node_name}" ".haproxy.enable")
+      if [ "${is_enabled}" = "TRUE" ]; then
+        log_debug "HAProxy is enabled for node '${node_name}'"
+        log_debug "Checking HAProxy for network-node '${node_name}'"
+        local haproxy_pod=$(get_pod_by_label "app=haproxy-${node_name},fullstack.hedera.com/type=haproxy")
+        log_debug "Checking HAProxy pod ${haproxy_pod}"
+        is_pod_ready "${haproxy_pod}" || test_status="${FAIL}"
+      else
+        log_debug "HAProxy is not enabled for node '${node_name}'. Skipping check"
+      fi
 
       [ "${test_status}" = "FAIL" ] && break
     done
   fi
 
   log_debug ""
-  log_debug "[${test_status}] HAProxy is running"
+  log_debug "[${test_status}] HAProxy Check"
   log_debug ""
 
   # assert success
@@ -58,18 +63,24 @@ setup() {
       local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name")
       [[ -z "${node_name}" ]] && test_status="${FAIL}" && break
 
-      log_debug "Checking Envoy proxy for network-node '${node_name}'"
-      local envoy_proxy_pod=$(get_pod_by_label "app=envoy-proxy-${node_name},fullstack.hedera.com/type=envoy-proxy")
+      local is_enabled=$(is_enabled_for_node "${node_name}" ".envoyProxy.enable")
+      if [ "${is_enabled}" = "TRUE" ]; then
+        log_debug "EnvoyProxy is enabled for node '${node_name}'"
+        log_debug "Checking Envoy proxy for network-node '${node_name}'"
+        local envoy_proxy_pod=$(get_pod_by_label "app=envoy-proxy-${node_name},fullstack.hedera.com/type=envoy-proxy")
 
-      log_debug "Checking Envoy Proxy pod ${envoy_proxy_pod}"
-      is_pod_ready "${envoy_proxy_pod}" || test_status="${FAIL}"
+        log_debug "Checking Envoy Proxy pod ${envoy_proxy_pod}"
+        is_pod_ready "${envoy_proxy_pod}" || test_status="${FAIL}"
+      else
+        log_debug "EnvoyProxy is not enabled for node '${node_name}'. Skipping check"
+      fi
 
       [ "${test_status}" = "FAIL" ] && break
     done
   fi
 
   log_debug ""
-  log_debug "[${test_status}] Envoy Proxy is running"
+  log_debug "[${test_status}] Envoy Proxy Check"
   log_debug ""
 
   # assert success