diff --git a/charts/hedera-network/templates/rbac/pod-monitor-role.yaml b/charts/hedera-network/templates/rbac/pod-monitor-role.yaml index 212072a2e..50a14d627 100644 --- a/charts/hedera-network/templates/rbac/pod-monitor-role.yaml +++ b/charts/hedera-network/templates/rbac/pod-monitor-role.yaml @@ -1,7 +1,8 @@ apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: name: pod-monitoring-role + namespace: {{ .Values.namespace }} rules: - apiGroups: [ "" ] resources: @@ -16,15 +17,26 @@ rules: - pods/exec verbs: - create + - apiGroups: [ "gateway.networking.k8s.io" ] + resources: + - gatewayclasses + - gateways + - httproutes + - tcproutes + verbs: + - get + - list --- apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +kind: ClusterRoleBinding metadata: name: pod-monitoring-role-binding + namespace: {{ .Values.namespace }} subjects: - kind: ServiceAccount name: pod-monitor + namespace: {{ .Values.namespace }} roleRef: - kind: Role + kind: ClusterRole name: pod-monitoring-role apiGroup: rbac.authorization.k8s.io diff --git a/charts/hedera-network/templates/rbac/service-accounts.yaml b/charts/hedera-network/templates/rbac/service-accounts.yaml index 82d476881..72c028175 100644 --- a/charts/hedera-network/templates/rbac/service-accounts.yaml +++ b/charts/hedera-network/templates/rbac/service-accounts.yaml @@ -1,4 +1,5 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: pod-monitor \ No newline at end of file + name: pod-monitor + namespace: {{ .Values.namespace }} diff --git a/charts/hedera-network/tests/README.md b/charts/hedera-network/tests/README.md index 9d74c0fc0..7da4919d4 100644 --- a/charts/hedera-network/tests/README.md +++ b/charts/hedera-network/tests/README.md @@ -6,6 +6,25 @@ you have the network deployed already. ## Development - Use the `test_basic_deployment.bats` file as the template while creating new tests. +- In order to run and debug the tests inside the helm test container, do the following: + + - Update `run` command section in `charts/hedera-network/template/tests/test-deployment.yaml` as below so that it keeps it running when we run `make helm-test`: + ``` + - "/bin/bash" + - "-c" + #- "/tests/run.sh" + - "while true;do echo sleeping for 60s; sleep 60;done" # keep the test container running so that we can debug issues + ``` + - once the `network-test` container is running, use another terminal to shell into it using command below + ``` + kubectl exec -it network-test -- bash + ``` + - Then you can run the test inside the container to debug + ``` + cd /tests && ./run.sh + ``` + - Once debug is done, you can exit and use Ctrl+C to terminate the helm-test process (you will need to delete the `network-test` container using `kubectl delete network-test`). + - If it looks all good, revert changes in `charts/hedera-network/template/tests/test-deployment.yaml` ## Run - Run `git submodule update --init` in order to install [bats](https://github.com/bats-core) for tests. diff --git a/charts/hedera-network/tests/helper.sh b/charts/hedera-network/tests/helper.sh index f32ea7fd7..386195f3d 100644 --- a/charts/hedera-network/tests/helper.sh +++ b/charts/hedera-network/tests/helper.sh @@ -82,9 +82,10 @@ function check_test_status() { function get_config_val() { local config_path=$1 - ret=$(helm get values fst -a | tail -n +2 | niet "${config_path}" ) + log_debug "Get config command: helm get values fst -a | yq '${config_path}'" + ret=$(helm get values fst -a | yq "${config_path}" ) echo "${ret}" - log_debug "${enable_config_path} => ${ret}" + log_debug "${config_path} => ${ret}" } function get_config_val_upper() { @@ -93,6 +94,40 @@ function get_config_val_upper() { echo "${config_val}" } +function is_enabled_for_node() { + local node_name=$1 + [[ -z "${node_name}" ]] && echo "ERROR: Node name is needed" && return "${EX_ERR}" + + local config_path=$2 + [[ -z "${config_path}" ]] && echo "ERROR: Config path is needed" && return "${EX_ERR}" + + log_debug "Checking config '${config_path}' for node '${node_name}" + + local default_config_path=".defaults${config_path}" + local node_config_path=".hedera.nodes[] | select(.name==\"${node_name}\") | ${config_path}" + local default_val=$(get_config_val_upper "${default_config_path}") + local node_val=$(get_config_val_upper "${node_config_path}") + log_debug "Default config: ${default_val}" + log_debug "Node config: ${node_val}" + + if [ -z "${node_val}" ] || [ "${node_val}" = "FALSE" ]; then + echo "FALSE" + return + fi + + if [ "${node_val}" = "TRUE" ]; then + echo "TRUE" + return + fi + + if [ "${default_val}" = "TRUE" ]; then + echo "TRUE" + return + fi + + echo "FALSE" +} + function get_sidecar_status() { local pod=$1 local sidecar_name=$2 @@ -110,6 +145,7 @@ function is_sidecar_ready() { [[ -z "${sidecar_name}" ]] && echo "ERROR: Sidecar name is needed (is_sidecar_ready)" && return "${EX_ERR}" local sidecar_status=$(kubectl get pod "${pod}" -o jsonpath="{.status.containerStatuses[?(@.name=='${sidecar_name}')].ready}" | tr '[:lower:]' '[:upper:]') + [ -z "${sidecar_status}" ] && sidecar_status="FALSE" log_debug "${sidecar_name} in pod ${pod} is ready: ${sidecar_status}" [[ "${sidecar_status}" = "TRUE" ]] && return "${EX_OK}" @@ -138,6 +174,8 @@ function is_pod_ready() { [[ -z "${pod}" ]] && echo "ERROR: Pod name is needed (is_pod_ready)" && return "${EX_ERR}" local pod_status=$(kubectl get pod "${pod}" -o jsonpath="{.status.conditions[?(@.type=='Ready')].status}" | tr '[:lower:]' '[:upper:]') + [ -z "${pod_status}" ] && pod_status="FALSE" + log_debug "Pod '${pod}' is ready: ${pod_status}" [[ "${pod_status}" = "TRUE" ]] && return "${EX_OK}" @@ -152,12 +190,13 @@ function get_pod_label() { [[ -z "${pod}" ]] && echo "ERROR: Label name is needed" && return "${EX_ERR}" - log_debug "Checking for pod ${pod}(timeout 300s)..." - if [ ! $(kubectl wait --for=condition=Initialized pods "${pod}" --timeout 300s) ]; then + log_debug "Checking for pod '${pod}'(timeout 300s)..." + $(kubectl wait --for=condition=Initialized pods "${pod}" --timeout 300s) > /dev/null 2>&1 + if [ $? = 1 ]; then log_debug "ERROR: Pod ${pod} is not available" && return "${EX_ERR}" fi - log_debug "Checking label ${label} for pod ${pod}" + log_debug "Checking label '${label}' for pod '${pod}'" local escaped_label="${label//./\\.}" local label_val=$(kubectl get pod "${pod}" -o jsonpath="{.metadata.labels.${escaped_label}}" | xargs) log_debug "Pod '${pod}' label '${label}': ${label_val}" @@ -174,3 +213,19 @@ function get_pod_by_label() { local pod_name=$(kubectl get pods -l "${label}" -o jsonpath="{.items[0].metadata.name}") echo "${pod_name}" } + +function is_route_accepted() { + local route_type=$1 + [[ -z "${route_type}" ]] && echo "ERROR: Route type is needed" && return "${EX_ERR}" + + local route_name=$2 + [[ -z "${route_name}" ]] && echo "ERROR: Route name is needed" && return "${EX_ERR}" + + local route_status=$(kubectl get "${route_type}" "${route_name}" -o jsonpath="{.status.parents[*].conditions[?(@.type=='Accepted')].status}" | tr '[:lower:]' '[:upper:]') + [ -z "${route_status}" ] && route_status="FALSE" + + log_debug "${route_type} '${route_name}' is accepted: ${route_status}" + + [[ "${route_status}" = "TRUE" ]] && return "${EX_OK}" + return "${EX_ERR}" +} \ No newline at end of file diff --git a/charts/hedera-network/tests/test_gateway_api_deployment.bats b/charts/hedera-network/tests/test_gateway_api_deployment.bats new file mode 100644 index 000000000..3db9fe4cb --- /dev/null +++ b/charts/hedera-network/tests/test_gateway_api_deployment.bats @@ -0,0 +1,121 @@ +# bats file_tags=deployment-test +setup() { + source "$(dirname "${BATS_TEST_FILENAME}")/env.sh" + source "${TESTS_DIR}/load.sh" +} + +@test "Check Envoy Proxy GRPC-Web routes" { + log_debug "----------------------------------------------------------------------------" + log_debug "TEST: Checking Envoy Proxy GRPC-Web Route" + log_debug "----------------------------------------------------------------------------" + + local resp="$(get_pod_list network-node)" + local pods=(${resp}) # convert into an array + log_debug "Network nodes: ${pods[*]}" + + local test_status="${FAIL}" + local status_val="${EX_ERR}" + if [[ "${#pods[@]}" -gt 0 ]]; then + test_status="${PASS}" + for pod in "${pods[@]}"; do + log_debug "" + local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name") + [[ -z "${node_name}" ]] && test_status="${FAIL}" && break + + local route_name="envoy-grpc-web-route-${node_name}" + local is_enabled=$(is_enabled_for_node "${node_name}" ".envoyProxy.enable") + if [ "${is_enabled}" = "TRUE" ]; then + log_debug "EnvoyProxy enabled for node '${node_name}'" + log_debug "Checking Envoy proxy httproute '${route_name}'" + is_route_accepted "httproute" "${route_name}" || test_status="${FAIL}" + [ "${test_status}" = "FAIL" ] && break + else + log_debug "EnvoyProxy is not enabled for node '${node_name}'. Skipped route check for '${route_name}'." + fi + done + fi + + log_debug "" + log_debug "[${test_status}] Envoy Proxy GRPC-Web Route Check" + log_debug "" + + # assert success + [[ "${test_status}" = "${PASS}" ]] +} + +@test "Check HAProxy GRPC routes" { + log_debug "----------------------------------------------------------------------------" + log_debug "TEST: Checking HAProxy GRPC Route" + log_debug "----------------------------------------------------------------------------" + + local resp="$(get_pod_list network-node)" + local pods=(${resp}) # convert into an array + log_debug "Network nodes: ${pods[*]}" + + local test_status="${FAIL}" + local status_val="${EX_ERR}" + if [[ "${#pods[@]}" -gt 0 ]]; then + test_status="${PASS}" + for pod in "${pods[@]}"; do + log_debug "" + local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name") + [[ -z "${node_name}" ]] && test_status="${FAIL}" && break + + local route_name="haproxy-grpc-route-${node_name}" + local is_enabled=$(is_enabled_for_node "${node_name}" ".haproxy.enable") + if [ "${is_enabled}" = "TRUE" ]; then + log_debug "HAProxy enabled for node '${node_name}'" + log_debug "Checking HAProxy GRPC route '${route_name}'" + is_route_accepted "tcproute" "${route_name}" || test_status="${FAIL}" + [ "${test_status}" = "FAIL" ] && break + else + log_debug "HAProxy is not enabled for node '${node_name}'. Skipped route check for '${route_name}'." + fi + done + fi + + log_debug "" + log_debug "[${test_status}] HAProxy GRPC Route Check" + log_debug "" + + # assert success + [[ "${test_status}" = "${PASS}" ]] +} + +@test "Check Network Node GRPC routes" { + log_debug "----------------------------------------------------------------------------" + log_debug "TEST: Checking Network Node GRPC Route" + log_debug "----------------------------------------------------------------------------" + + local resp="$(get_pod_list network-node)" + local pods=(${resp}) # convert into an array + log_debug "Network nodes: ${pods[*]}" + + local test_status="${FAIL}" + local status_val="${EX_ERR}" + if [[ "${#pods[@]}" -gt 0 ]]; then + test_status="${PASS}" + for pod in "${pods[@]}"; do + log_debug "" + local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name") + [[ -z "${node_name}" ]] && test_status="${FAIL}" && break + + local route_name="node-grpc-route-${node_name}" + local is_enabled=$(is_enabled_for_node "${node_name}" ".haproxy.enable") + if [ "${is_enabled}" = "FALSE" ]; then + log_debug "Checking Node GRPC route '${route_name}'" + is_route_accepted "tcproute" "${route_name}" || test_status="${FAIL}" + [ "${test_status}" = "FAIL" ] && break + else + log_debug "HAProxy enabled for node '${node_name}'. Skipped route check for '${route_name}'." + fi + done + fi + + log_debug "" + log_debug "[${test_status}] Node GRPC Route Check" + log_debug "" + + # assert success + [[ "${test_status}" = "${PASS}" ]] +} diff --git a/charts/hedera-network/tests/test_proxy_deployment.bats b/charts/hedera-network/tests/test_proxy_deployment.bats index c0ae4a4c7..de65494b1 100644 --- a/charts/hedera-network/tests/test_proxy_deployment.bats +++ b/charts/hedera-network/tests/test_proxy_deployment.bats @@ -22,18 +22,23 @@ setup() { local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name") [[ -z "${node_name}" ]] && test_status="${FAIL}" && break - log_debug "Checking HAProxy for network-node '${node_name}'" - local haproxy_pod=$(get_pod_by_label "app=haproxy-${node_name},fullstack.hedera.com/type=haproxy") - - log_debug "Checking HAProxy pod ${haproxy_pod}" - is_pod_ready "${haproxy_pod}" || test_status="${FAIL}" + local is_enabled=$(is_enabled_for_node "${node_name}" ".haproxy.enable") + if [ "${is_enabled}" = "TRUE" ]; then + log_debug "HAProxy is enabled for node '${node_name}'" + log_debug "Checking HAProxy for network-node '${node_name}'" + local haproxy_pod=$(get_pod_by_label "app=haproxy-${node_name},fullstack.hedera.com/type=haproxy") + log_debug "Checking HAProxy pod ${haproxy_pod}" + is_pod_ready "${haproxy_pod}" || test_status="${FAIL}" + else + log_debug "HAProxy is not enabled for node '${node_name}'. Skipping check" + fi [ "${test_status}" = "FAIL" ] && break done fi log_debug "" - log_debug "[${test_status}] HAProxy is running" + log_debug "[${test_status}] HAProxy Check" log_debug "" # assert success @@ -58,18 +63,24 @@ setup() { local node_name=$(get_pod_label "${pod}" "fullstack.hedera.com/node-name") [[ -z "${node_name}" ]] && test_status="${FAIL}" && break - log_debug "Checking Envoy proxy for network-node '${node_name}'" - local envoy_proxy_pod=$(get_pod_by_label "app=envoy-proxy-${node_name},fullstack.hedera.com/type=envoy-proxy") + local is_enabled=$(is_enabled_for_node "${node_name}" ".envoyProxy.enable") + if [ "${is_enabled}" = "TRUE" ]; then + log_debug "EnvoyProxy is enabled for node '${node_name}'" + log_debug "Checking Envoy proxy for network-node '${node_name}'" + local envoy_proxy_pod=$(get_pod_by_label "app=envoy-proxy-${node_name},fullstack.hedera.com/type=envoy-proxy") - log_debug "Checking Envoy Proxy pod ${envoy_proxy_pod}" - is_pod_ready "${envoy_proxy_pod}" || test_status="${FAIL}" + log_debug "Checking Envoy Proxy pod ${envoy_proxy_pod}" + is_pod_ready "${envoy_proxy_pod}" || test_status="${FAIL}" + else + log_debug "EnvoyProxy is not enabled for node '${node_name}'. Skipping check" + fi [ "${test_status}" = "FAIL" ] && break done fi log_debug "" - log_debug "[${test_status}] Envoy Proxy is running" + log_debug "[${test_status}] Envoy Proxy Check" log_debug "" # assert success diff --git a/charts/hedera-network/tests/test_sidecar_deployment.bats b/charts/hedera-network/tests/test_sidecar_deployment.bats index fbb9485a4..399da3d44 100644 --- a/charts/hedera-network/tests/test_sidecar_deployment.bats +++ b/charts/hedera-network/tests/test_sidecar_deployment.bats @@ -26,6 +26,7 @@ function run_default_sidecar_check() { local sidecar_exists=$(has_sidecar "${pod}" "${sidecar_name}" ) log_debug "${sidecar_name} exists in pod ${pod}: ${sidecar_exists} " + log_debug "${should_enable} ${sidecar_exists}" if [ "${should_enable}" = "TRUE" ] && [ "${sidecar_exists}" = "TRUE" ]; then is_sidecar_ready "${pod}" "${sidecar_name}" || test_status="${FAIL}" elif [[ "${should_enable}" != "${sidecar_exists}" ]]; then @@ -45,35 +46,35 @@ function run_default_sidecar_check() { @test "Check record-stream-uploader sidecar" { local sidecar_name="record-stream-uploader" - local enable_config_path="defaults.sidecars.recordStreamUploader.enable" + local enable_config_path=".defaults.sidecars.recordStreamUploader.enable" run_default_sidecar_check "${sidecar_name}" "${enable_config_path}" } @test "Check event-stream-uploader sidecar" { local sidecar_name="event-stream-uploader" - local enable_config_path="defaults.sidecars.eventStreamUploader.enable" + local enable_config_path=".defaults.sidecars.eventStreamUploader.enable" run_default_sidecar_check "${sidecar_name}" "${enable_config_path}" } @test "Check account-balance-uploader sidecar" { local sidecar_name="account-balance-uploader" - local enable_config_path="defaults.sidecars.accountBalanceUploader.enable" + local enable_config_path=".defaults.sidecars.accountBalanceUploader.enable" run_default_sidecar_check "${sidecar_name}" "${enable_config_path}" } @test "Check backup-uploader sidecar" { local sidecar_name="backup-uploader" - local enable_config_path="defaults.sidecars.backupUploader.enable" + local enable_config_path=".defaults.sidecars.backupUploader.enable" run_default_sidecar_check "${sidecar_name}" "${enable_config_path}" } @test "Check otel-collector sidecar" { local sidecar_name="otel-collector" - local enable_config_path="defaults.sidecars.otelCollector.enable" + local enable_config_path=".defaults.sidecars.otelCollector.enable" run_default_sidecar_check "${sidecar_name}" "${enable_config_path}" } diff --git a/charts/hedera-network/values.yaml b/charts/hedera-network/values.yaml index 7ce335e55..489ed4913 100644 --- a/charts/hedera-network/values.yaml +++ b/charts/hedera-network/values.yaml @@ -62,6 +62,8 @@ gatewayApi: route: hostname: "{{ .node.name }}.fst.local" +namespace: default + # default settings for a single node # This default configurations can be overridden for each node in the hedera.nodes section. defaults: diff --git a/dev/Makefile b/dev/Makefile index e1a933896..61c51ab24 100644 --- a/dev/Makefile +++ b/dev/Makefile @@ -221,7 +221,7 @@ undeploy-minio-operator: ######################################### Helm Chart Test ################################# .PHONY: helm-test helm-test: - source "${SCRIPTS_DIR}/main.sh" && run_helm_chart_tests "network-test" + source "${SCRIPTS_DIR}/main.sh" && run_helm_chart_tests "network-test" || exit 1 .PHONY: destroy-test-container destroy-test-container: @@ -234,6 +234,10 @@ destroy-test-container: local-kubectl-bats: source "${SCRIPTS_DIR}/${DOCKER_SCRIPT}" && build_kubectl_bats "${CLUSTER_NAME}" +# Here we run all steps in sequence, if any step fails, deploy-all trap the EXIT and run cleanup +.PHONY: run-deploy-seq +run-deploy-seq: setup deploy-network helm-test setup-nodes start-nodes + .PHONY: deploy-all deploy-all: # Enable cleanup_test function so that even if test fails, we cleanup the cluster. @@ -243,11 +247,7 @@ deploy-all: $(MAKE) destroy-network } trap cleanup_test EXIT # always destroy-network on exit - $(MAKE) setup - $(MAKE) deploy-network - $(MAKE) helm-test - $(MAKE) setup-nodes - $(MAKE) start-nodes + $(MAKE) run-deploy-seq .PHONY: destroy-all destroy-all: @@ -258,3 +258,8 @@ destroy-all: .PHONY: ci-test ci-test: setup-cluster local-kubectl-bats $(MAKE) deploy-all CHART_VALUES_FILES="$(PWD)/ci/ci-values.yaml" + +.PHONY: ci-deploy-network +ci-deploy-network: setup-cluster local-kubectl-bats + $(MAKE) deploy-network CHART_VALUES_FILES="$(PWD)/ci/ci-values.yaml" + $(MAKE) helm-test diff --git a/dev/scripts/main.sh b/dev/scripts/main.sh index ed6d13d2e..6763680c2 100644 --- a/dev/scripts/main.sh +++ b/dev/scripts/main.sh @@ -79,6 +79,7 @@ function run_helm_chart_tests() { kubectl logs "${test_name}" if [[ "${test_status}" != "Succeeded" ]]; then - exit 1 + echo "Returning exit code 1" + return 1 fi } \ No newline at end of file diff --git a/docker/kubectl-bats/Dockerfile b/docker/kubectl-bats/Dockerfile index 293e44273..eec460a65 100644 --- a/docker/kubectl-bats/Dockerfile +++ b/docker/kubectl-bats/Dockerfile @@ -23,12 +23,13 @@ USER root # setup bats RUN apt-get update && \ - apt-get install -y curl git vim python3 python3-pip && \ + apt-get install -y curl git vim python3 python3-pip jq && \ git clone https://github.com/bats-core/bats-core.git "${BATS_HOME}"/bats-core && \ git clone https://github.com/bats-core/bats-support.git "${BATS_HOME}"/test_helper/bats-support && \ git clone https://github.com/bats-core/bats-assert.git "${BATS_HOME}"/test_helper/bats-assert && \ - pip3 install -U niet && \ - curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash && \ + curl -L https://github.com/mikefarah/yq/releases/download/v4.34.2/yq_linux_amd64 -o /usr/bin/yq && \ + chmod +x /usr/bin/yq #switch back to user 1001 USER 1001