diff --git a/.github/workflows/build-roxie-image.yml b/.github/workflows/build-roxie-image.yml new file mode 100644 index 00000000..f0a28d8c --- /dev/null +++ b/.github/workflows/build-roxie-image.yml @@ -0,0 +1,72 @@ +name: Build Roxie Image + +on: + workflow_call: + outputs: + image: + description: "Built image with tag" + value: ${{ jobs.build-roxie-image.outputs.image }} + +env: + REGISTRY: quay.io + IMAGE_NAME: rhacs-eng/roxie + +jobs: + build-roxie-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + outputs: + image: ${{ steps.meta.outputs.tags }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Quay.io + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Get build metadata + id: build-meta + run: | + echo "version=$(make version)" >> $GITHUB_OUTPUT + echo "build_date=$(make get-build-date)" >> $GITHUB_OUTPUT + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: type=raw,value=${{ steps.build-meta.outputs.version }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + ROXIE_VERSION=${{ steps.build-meta.outputs.version }} + BUILD_DATE=${{ steps.build-meta.outputs.build_date }} + cache-from: | + type=gha + type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache + cache-to: type=gha,mode=max diff --git a/.github/workflows/create-dev-cluster.yml b/.github/workflows/create-dev-cluster.yml new file mode 100644 index 00000000..b790e721 --- /dev/null +++ b/.github/workflows/create-dev-cluster.yml @@ -0,0 +1,27 @@ +name: Create Dev Cluster + +on: + workflow_call: + inputs: + cluster-name: + required: true + type: string + outputs: + cluster-name: + description: "Name of the created cluster" + value: ${{ jobs.create-dev-cluster.outputs.cluster-name }} + +jobs: + create-dev-cluster: + runs-on: ubuntu-latest + outputs: + cluster-name: ${{ inputs.cluster-name }} + steps: + - uses: stackrox/actions/infra/create-cluster@v1 + with: + flavor: gke-default + name: ${{ inputs.cluster-name }} + args: machine-type=e2-standard-4,nodes=3,gcp-image-type=ubuntu_containerd + lifespan: "2h" + wait: true + token: ${{ secrets.INFRA_CI_TOKEN }} diff --git a/.github/workflows/delete-dev-cluster.yml b/.github/workflows/delete-dev-cluster.yml new file mode 100644 index 00000000..19f2a00f --- /dev/null +++ b/.github/workflows/delete-dev-cluster.yml @@ -0,0 +1,21 @@ +name: Delete Dev Cluster + +on: + workflow_call: + inputs: + cluster-name: + required: true + type: string + +jobs: + delete-dev-cluster: + runs-on: ubuntu-latest + env: + INFRA_TOKEN: ${{ secrets.INFRA_CI_TOKEN }} + steps: + - name: Install infractl + uses: stackrox/actions/infra/install-infractl@v1 + + - name: Delete cluster + run: | + "$HOME/.local/bin/infractl" delete "${{ inputs.cluster-name }}" diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 00000000..322defb6 --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,101 @@ +name: E2E Tests + +on: + workflow_call: + inputs: + cluster-name: + required: true + type: string + image: + required: true + type: string +env: + REGISTRY: quay.io + IMAGE_NAME: rhacs-eng/roxie + +jobs: + e2e-tests: + runs-on: ubuntu-latest + container: + image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11 + env: + CLUSTER_NAME: ${{ inputs.cluster-name }} + KUBECONFIG: /github/home/artifacts/kubeconfig + INFRA_TOKEN: ${{ secrets.INFRA_CI_TOKEN }} + INFRACTL: bin/infractl -k -e localhost:8443 + USE_GKE_GCLOUD_AUTH_PLUGIN: "True" + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Fix repository ownership + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Log in to Quay.io + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_TOKEN }} + + - name: Extract roxie binary from image + run: | + docker create --name roxie-extract "${{ inputs.image }}" + docker cp roxie-extract:/usr/local/bin/roxie "$GITHUB_WORKSPACE/roxie" + docker rm roxie-extract + + - name: Install roxie binary + run: | + cp "${GITHUB_WORKSPACE}/roxie" /usr/local/bin/roxie + chmod +x /usr/local/bin/roxie + roxie version + + - name: Install roxctl + env: + ROXCTL_VERSION: "4.10.0" + ROXCTL_SHA256: "5db647b14569465866c0162522e83393ebf02f671f4556b1b3ed551b9f8433bc" + run: | + curl -fsSLo /usr/local/bin/roxctl \ + "https://mirror.openshift.com/pub/rhacs/assets/${ROXCTL_VERSION}/bin/Linux/roxctl" + echo "${ROXCTL_SHA256} /usr/local/bin/roxctl" | sha256sum -c - + chmod +x /usr/local/bin/roxctl + roxctl version + + - name: Authenticate to GCloud + uses: google-github-actions/auth@v3 + with: + credentials_json: ${{ secrets.ROXIE_CI_AUTOMATION_GCP_SA }} + + - name: Set up Cloud SDK + uses: "google-github-actions/setup-gcloud@v3" + with: + install_components: "gke-gcloud-auth-plugin" + + - name: Download production infractl + uses: stackrox/actions/infra/install-infractl@v1 + + - name: Download artifacts + run: | + /github/home/.local/bin/infractl artifacts "$CLUSTER_NAME" -d /github/home/artifacts >> "$GITHUB_STEP_SUMMARY" + + - name: Testing cluster connection + run: | + kubectl get namespaces + + - name: Run e2e tests + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + SKIP_OLM_TESTS: "true" + run: | + make run-test-e2e + + - name: Run integration tests + env: + REGISTRY_USERNAME: ${{ secrets.QUAY_RHACS_ENG_RO_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.QUAY_RHACS_ENG_RO_PASSWORD }} + run: | + make test-integration diff --git a/.github/workflows/main-push.yml b/.github/workflows/main-push.yml new file mode 100644 index 00000000..064187c0 --- /dev/null +++ b/.github/workflows/main-push.yml @@ -0,0 +1,42 @@ +name: Main + +on: + push: + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-tests: + uses: ./.github/workflows/unit-tests.yml + + create-dev-cluster: + uses: ./.github/workflows/create-dev-cluster.yml + with: + cluster-name: infra-roxie-main-${{ github.run_number }} + secrets: inherit + + build-roxie-image: + uses: ./.github/workflows/build-roxie-image.yml + permissions: + contents: read + packages: write + secrets: inherit + + e2e-tests: + needs: [ create-dev-cluster, build-roxie-image ] + uses: ./.github/workflows/e2e-tests.yml + with: + cluster-name: ${{ needs.create-dev-cluster.outputs.cluster-name }} + image: ${{ needs.build-roxie-image.outputs.image }} + secrets: inherit + + delete-dev-cluster: + if: ${{ always() && needs.create-dev-cluster.result == 'success' }} + needs: [ create-dev-cluster, e2e-tests ] + uses: ./.github/workflows/delete-dev-cluster.yml + with: + cluster-name: ${{ needs.create-dev-cluster.outputs.cluster-name }} + secrets: inherit diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 00000000..4e27e6a4 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,42 @@ +name: PR + +on: + pull_request: + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-tests: + uses: ./.github/workflows/unit-tests.yml + + create-dev-cluster: + uses: ./.github/workflows/create-dev-cluster.yml + with: + cluster-name: infra-roxie-pr-${{ github.event.pull_request.number }} + secrets: inherit + + build-roxie-image: + uses: ./.github/workflows/build-roxie-image.yml + permissions: + contents: read + packages: write + secrets: inherit + + e2e-tests: + needs: [ create-dev-cluster, build-roxie-image ] + uses: ./.github/workflows/e2e-tests.yml + with: + cluster-name: ${{ needs.create-dev-cluster.outputs.cluster-name }} + image: ${{ needs.build-roxie-image.outputs.image }} + secrets: inherit + + delete-dev-cluster: + if: ${{ always() && needs.create-dev-cluster.result == 'success' }} + needs: [ create-dev-cluster, e2e-tests ] + uses: ./.github/workflows/delete-dev-cluster.yml + with: + cluster-name: ${{ needs.create-dev-cluster.outputs.cluster-name }} + secrets: inherit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 541f256d..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Tests - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - unit-tests: - name: Unit Tests - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - cache: true - - - name: Download dependencies - run: go mod download - - - name: Run unit tests - run: make test diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 00000000..a3be2ce5 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,28 @@ +name: Unit Tests + +on: + workflow_call: + +jobs: + unit-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache: true + + - name: Check go.mod tidiness + run: | + go mod tidy + if ! git diff --exit-code go.mod go.sum; then + echo "::error::go.mod or go.sum are not tidy. Please run 'go mod tidy' and commit the changes." + exit 1 + fi + + - name: Run unit tests + run: make test diff --git a/Makefile b/Makefile index be321545..e1b0ea1e 100644 --- a/Makefile +++ b/Makefile @@ -113,22 +113,50 @@ test-coverage: test ## Run tests with coverage report $(GOCMD) tool cover -html=coverage.out -o coverage.html @echo "โœ… Coverage report: coverage.html" -.PHONY: test-e2e -test-e2e: build ## Run end-to-end tests (requires kubectl context and cluster access) +.PHONY: run-test-e2e +run-test-e2e: ## Run end-to-end tests (requires roxie to be available in PATH, kubectl context and cluster access) @echo "๐Ÿงช Running E2E tests..." - @if [ -z "$(shell kubectl config current-context 2>/dev/null)" ]; then \ + + @echo ""; \ + echo "Checking roxie binary is available..."; \ + ROXIE=$$(command -v roxie); \ + if [ -z "$$ROXIE" ]; then \ + echo "โŒ roxie not found in PATH."; \ + exit 1; \ + fi; \ + echo "roxie found at $$ROXIE"; \ + roxie version + + @echo ""; \ + echo "Checking a kubectl cluster context is set..."; \ + CLUSTER_CONTEXT=$$(kubectl config current-context); \ + if [ -z "$$CLUSTER_CONTEXT" ]; then \ echo "โŒ No kubectl context found. Please configure kubectl first."; \ exit 1; \ - fi + fi; \ + echo "using cluster context $$CLUSTER_CONTEXT" + + @echo ""; \ + echo "Invoking go test..." $(GOTEST) -v -tags=e2e -timeout=120m -parallel=1 ./tests/e2e/... +.PHONY: test-e2e +test-e2e: build ## Run end-to-end tests (requires kubectl context and cluster access) + PATH="$$PWD:$$PATH" make run-test-e2e + .PHONY: test-integration -test-integration: build ## Run integration tests (requires kubectl context and cluster access) - @echo "๐Ÿงช Running integration tests..." - @if [ -z "$(shell kubectl config current-context 2>/dev/null)" ]; then \ +test-integration: ## Run integration tests (requires kubectl context and cluster access) + @echo ""; \ + echo "Checking a kubectl cluster context is set..."; \ + CLUSTER_CONTEXT=$$(kubectl config current-context); \ + if [ -z "$$CLUSTER_CONTEXT" ]; then \ echo "โŒ No kubectl context found. Please configure kubectl first."; \ exit 1; \ - fi + fi; \ + echo "using cluster context $$CLUSTER_CONTEXT" + + @echo ""; \ + echo "Invoking go test..." $(GOTEST) -v -tags=integration -run=_Integration$$ -timeout=120m -parallel=1 ./... .PHONY: test-all diff --git a/internal/ocihelper/ocihelper_integration_test.go b/internal/ocihelper/ocihelper_integration_test.go index 682353ac..0c320051 100644 --- a/internal/ocihelper/ocihelper_integration_test.go +++ b/internal/ocihelper/ocihelper_integration_test.go @@ -82,34 +82,34 @@ func TestExtractManifestsFromImage_Integration(t *testing.T) { t.Logf("โœ“ CSV file verified (%d bytes)", len(csvContent)) } -func TestInspectImage_Integration(t *testing.T) { +func TestVerifyImageExistence_Integration(t *testing.T) { bundleImage := "quay.io/rhacs-eng/stackrox-operator-bundle:v4.10.0" log := logger.New() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - t.Logf("Inspecting image %s", bundleImage) - err := InspectImage(ctx, log, bundleImage) + t.Logf("Verifying image %s exists", bundleImage) + err := VerifyImageExistence(ctx, log, bundleImage) if err != nil { - t.Fatalf("InspectImage failed: %v", err) + t.Fatalf("VerifyImageExistence failed: %v", err) } - t.Log("โœ“ Image inspection successful") + t.Log("โœ“ VerifyImageExistence succeeded for existing image") } -func TestInspectImage_NonExistent_Integration(t *testing.T) { +func TestVerifyImageExistence_NonExistent_Integration(t *testing.T) { nonExistentImage := "quay.io/rhacs-eng/this-image-does-not-exist:v999.999.999" log := logger.New() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - t.Logf("Inspecting non-existent image %s", nonExistentImage) - err := InspectImage(ctx, log, nonExistentImage) + t.Logf("Verifying image %s does not exist", nonExistentImage) + err := VerifyImageExistence(ctx, log, nonExistentImage) if err == nil { - t.Fatal("Expected InspectImage to fail for non-existent image, but it succeeded") + t.Fatal("Expected VerifyImageExistence to fail for non-existent image, but it succeeded") } - t.Logf("โœ“ InspectImage correctly failed for non-existent image: %v", err) + t.Log("โœ“ VerifyImageExistence correctly failed for non-existent image") } diff --git a/tests/README.md b/tests/README.md index 1b433516..588f3c47 100644 --- a/tests/README.md +++ b/tests/README.md @@ -47,7 +47,6 @@ go test -v -tags=e2e -timeout=30m -parallel=1 ./tests/e2e/... ### Environment Variables for E2E Tests - `MAIN_IMAGE_TAG` - ACS image tag to use (default: "4.8.2") -- `SKIP_OPERATOR_TESTS` - Skip operator-based tests if set - `SKIP_OLM_TESTS` - Skip OLM-specific tests if set (useful when OLM is not available) - `SKIP_IMAGE_VERIFICATION` - Skip image verification if set to "true" @@ -139,10 +138,6 @@ import ( // to prevent conflicts when modifying shared cluster resources. func TestDeployment(t *testing.T) { - if os.Getenv("SKIP_OPERATOR_TESTS") != "" { - t.Skip("Operator tests disabled") - } - runCommand(t, 30*time.Minute, nil, roxieBinary, "deploy", "central") @@ -150,16 +145,6 @@ func TestDeployment(t *testing.T) { } ``` -## Test Helpers - -The `testhelpers` package provides utilities: - -- `CreateTestLogger(t)` - Logger with captured output -- `AssertContains(t, haystack, needle)` - String contains assertion -- `AssertEqual(t, expected, actual)` - Equality assertion -- `AssertNoError(t, err)` - Error nil assertion -- `AssertError(t, err)` - Error not nil assertion - ## CI Integration The Makefile provides targets suitable for CI: @@ -169,7 +154,7 @@ The Makefile provides targets suitable for CI: make check # Run all tests -make test test-e2e +make test test-e2e test-integration # Full CI workflow make all diff --git a/tests/e2e/basic_test.go b/tests/e2e/basic_test.go index 75b1e68f..048ad36d 100644 --- a/tests/e2e/basic_test.go +++ b/tests/e2e/basic_test.go @@ -11,10 +11,6 @@ import ( // TestDeployBothSimple tests deploying both components together (simplest scenario) func TestDeployBothSimple(t *testing.T) { - if os.Getenv("SKIP_OPERATOR_TESTS") != "" { - t.Skip("SKIP_OPERATOR_TESTS is set") - } - // Create temporary envrc file envrcFile, err := os.CreateTemp(t.TempDir(), ".envrc.roxie-test-*") if err != nil { diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 5f1f1fcf..5f85da97 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -1,33 +1,11 @@ //go:build e2e -// +build e2e package e2e import ( - "bytes" - "context" - "encoding/json" "fmt" "os" - "os/exec" - "path/filepath" - "strings" "testing" - "time" -) - -const ( - // TODO(#91): this is a bit old... - defaultMainImageTag = "4.8.2" - deployTimeout = 30 * time.Minute - teardownTimeout = 10 * time.Minute -) - -var ( - commonDeployArgs = []string{"--port-forwarding", "--exposure=none", "--resources=small"} - commonDeployArgsNoPortForward = []string{"--exposure=loadbalancer", "--resources=small"} - - roxieBinary string ) func TestMain(m *testing.M) { @@ -50,14 +28,6 @@ func TestMain(m *testing.M) { } fmt.Printf("Using kubectl context: %s\n", ctx) - // Find roxie binary - roxieBinary = findRoxieBinary() - if roxieBinary == "" { - fmt.Fprintln(os.Stderr, "roxie binary not found") - os.Exit(1) - } - fmt.Printf("Using roxie binary: %s\n", roxieBinary) - // Teardown all deployments before running tests if err := teardownAllDeployments(); err != nil { fmt.Fprintf(os.Stderr, "Warning: teardown all deployments failed: %v\n", err) @@ -68,324 +38,7 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -// TODO(#91): maybe put the helper functions in a separate file? -func teardownAllDeployments() error { - fmt.Println("=== Tearing down all deployments before running tests ===") - - ctx, cancel := context.WithTimeout(context.Background(), teardownTimeout) - defer cancel() - - // Teardown standard deployments - cmd := exec.CommandContext(ctx, roxieBinary, "teardown") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("Warning: teardown command failed: %w", err) - } - - // Teardown single-namespace deployments - ctx, cancel = context.WithTimeout(context.Background(), teardownTimeout) - defer cancel() - cmd = exec.CommandContext(ctx, roxieBinary, "teardown", "--single-namespace") - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return fmt.Errorf("Warning: teardown --single-namespace command failed: %w", err) - } - - fmt.Println("=== All deployments have been torn down ===") - return nil -} - -func requireBinary(name string) error { - _, err := exec.LookPath(name) - return err -} - -func requireKubeContext() (string, error) { - cmd := exec.Command("kubectl", "config", "current-context") - output, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("no kubectl context available: %w", err) - } - - ctx := strings.TrimSpace(string(output)) - if ctx == "" { - return "", fmt.Errorf("kubectl context is empty") - } - - return ctx, nil -} - -func findRoxieBinary() string { - // Try current directory - if _, err := os.Stat("./roxie"); err == nil { - return "./roxie" - } - - // Try ../.. (from tests/e2e to repo root) - repoRoot := filepath.Join("..", "..") - roxiePath := filepath.Join(repoRoot, "roxie") - if _, err := os.Stat(roxiePath); err == nil { - return roxiePath - } - - return "" -} - -func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args ...string) { - t.Helper() - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - cmd := exec.CommandContext(ctx, args[0], args[1:]...) - - // Set environment - if env != nil { - cmd.Env = os.Environ() - for k, v := range env { - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) - } - } - - // Stream output to both test log and buffers - var stdout, stderr bytes.Buffer - cmd.Stdout = &testWriter{t: t, prefix: "[stdout] ", buf: &stdout} - cmd.Stderr = &testWriter{t: t, prefix: "[stderr] ", buf: &stderr} - - t.Logf("Running: %s", strings.Join(args, " ")) - - err := cmd.Run() - if err != nil { - t.Logf("Command failed: %v", err) - t.Fatalf("Command failed: %v", err) - } - t.Logf("Command completed successfully") -} - -// testWriter writes to both test log and a buffer -type testWriter struct { - t *testing.T - prefix string - buf *bytes.Buffer -} - -func (w *testWriter) Write(p []byte) (n int, err error) { - // Write to buffer - n, err = w.buf.Write(p) - if err != nil { - return n, err - } - - // Also log each line to test output - lines := bytes.Split(p, []byte("\n")) - for _, line := range lines { - if len(line) > 0 { - w.t.Logf("%s%s", w.prefix, string(line)) - } - } - - return n, nil -} - -func verifyNamespaceExists(t *testing.T, namespace string) { - t.Helper() - - cmd := exec.Command("kubectl", "get", "namespace", namespace) - if err := cmd.Run(); err != nil { - t.Fatalf("Namespace %s does not exist", namespace) - } -} - -func verifyNamespaceHasLabel(t *testing.T, namespace, key, value string) { - t.Helper() - - cmd := exec.Command("kubectl", "get", "namespace", namespace, - "-o", "jsonpath={.metadata.labels['"+key+"']}") - output, err := cmd.Output() - if err != nil { - t.Fatalf("Failed to get label %s from namespace %s: %v", key, namespace, err) - } - - actualValue := strings.TrimSpace(string(output)) - if actualValue != value { - t.Fatalf("Namespace %s has label %s=%s, expected %s", namespace, key, actualValue, value) - } -} - -func doesDeploymentExist(t *testing.T, namespace string, name string) bool { - t.Helper() - - cmd := exec.Command("kubectl", "-n", namespace, "get", "deployments", "-o", "name") - output, err := cmd.Output() - if err != nil { - t.Fatalf("Failed to get deployments in namespace %s: %v", namespace, err) - } - return strings.Contains(string(output), name) -} - -func loadEnvrcFile(path string) (map[string]string, error) { - content, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - env := make(map[string]string) - lines := strings.Split(string(content), "\n") - for _, line := range lines { - line = strings.TrimSpace(line) - if line == "" || !strings.HasPrefix(line, "export ") { - continue - } - - // Remove "export " prefix - line = strings.TrimPrefix(line, "export ") - - // Split on first = - parts := strings.SplitN(line, "=", 2) - if len(parts) != 2 { - continue - } - - key := strings.TrimSpace(parts[0]) - value := strings.TrimSpace(parts[1]) - - // Remove quotes - value = strings.Trim(value, "\"") - - env[key] = value - } - - return env, nil -} - -func TestDeployCentralAndSecuredCluster(t *testing.T) { - if os.Getenv("SKIP_OPERATOR_TESTS") != "" { - t.Skip("SKIP_OPERATOR_TESTS is set") - } - - // Create temporary envrc file - envrcFile, err := os.CreateTemp(t.TempDir(), ".envrc.roxie-test-*") - if err != nil { - t.Fatalf("Failed to create temp envrc: %v", err) - } - envrcPath := envrcFile.Name() - envrcFile.Close() - - // Deploy central - t.Log("=== Deploying central ===") - args := append([]string{roxieBinary, "deploy", "--early-readiness", "central", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) - runCommand(t, deployTimeout, nil, args...) - - // Load environment from envrc file for secured-cluster deployment - envrcEnv, err := loadEnvrcFile(envrcPath) - if err != nil { - t.Fatalf("Failed to load envrc file: %v", err) - } - t.Log("Loaded environment from envrc file for secured-cluster") - - t.Log("=== Deploying secured-cluster ===") - args = append([]string{roxieBinary, "deploy", "--early-readiness", "secured-cluster"}, commonDeployArgsNoPortForward...) - runCommand(t, deployTimeout, envrcEnv, args...) - - // Verify namespaces and labels - t.Log("Verifying namespace: acs-central") - verifyNamespaceExists(t, "acs-central") - verifyNamespaceHasLabel(t, "acs-central", "app.kubernetes.io/managed-by", "roxie") - - t.Log("Verifying namespace: acs-sensor") - verifyNamespaceExists(t, "acs-sensor") - verifyNamespaceHasLabel(t, "acs-sensor", "app.kubernetes.io/managed-by", "roxie") -} - -func TestTeardownCentralAndSecuredCluster(t *testing.T) { - if os.Getenv("SKIP_OPERATOR_TESTS") != "" { - t.Skip("SKIP_OPERATOR_TESTS is set") - } - - t.Log("=== Tearing down central and secured-cluster ===") - args := []string{roxieBinary, "teardown", "both"} - runCommand(t, teardownTimeout, nil, args...) - - t.Log("Verifying components are removed") - verifyCentralNotInstalled(t, "acs-central") - verifySecuredClusterNotInstalled(t, "acs-sensor") -} - -func verifyCentralInstalled(t *testing.T, namespace string) { - t.Helper() - - if !doesDeploymentExist(t, namespace, "central") { - t.Fatalf("Central is not installed in namespace %s", namespace) - } -} - -func verifySecuredClusterInstalled(t *testing.T, namespace string) { - t.Helper() - - if !doesDeploymentExist(t, namespace, "sensor") { - t.Fatalf("Secured cluster is not installed in namespace %s", namespace) - } -} - -func verifyCentralNotInstalled(t *testing.T, namespace string) { - t.Helper() - - if doesDeploymentExist(t, namespace, "central") { - t.Fatalf("Central is installed in namespace %s", namespace) - } -} - -func verifySecuredClusterNotInstalled(t *testing.T, namespace string) { - t.Helper() - - if doesDeploymentExist(t, namespace, "sensor") { - t.Fatalf("Secured cluster is installed in namespace %s", namespace) - } -} - -func TestDeployBothComponentsTogether(t *testing.T) { - if os.Getenv("SKIP_OPERATOR_TESTS") != "" { - t.Skip("SKIP_OPERATOR_TESTS is set") - } - - // Create temporary envrc file - envrcFile, err := os.CreateTemp(t.TempDir(), ".envrc.roxie-test-*") - if err != nil { - t.Fatalf("Failed to create temp envrc: %v", err) - } - envrcPath := envrcFile.Name() - envrcFile.Close() - - t.Log("=== Deploying both components ===") - // We also test --pause-reconciliation flag here. - args := append([]string{roxieBinary, "deploy", "both", "--pause-reconciliation", "--envrc", envrcPath}, commonDeployArgsNoPortForward...) - runCommand(t, deployTimeout*2, nil, args...) - - t.Log("Verifying namespace: acs-central") - verifyNamespaceExists(t, "acs-central") - verifyNamespaceHasLabel(t, "acs-central", "app.kubernetes.io/managed-by", "roxie") - - t.Log("Verifying namespace: acs-sensor") - verifyNamespaceExists(t, "acs-sensor") - verifyNamespaceHasLabel(t, "acs-sensor", "app.kubernetes.io/managed-by", "roxie") - - // Verify Central has the pause-reconcile annotation. - t.Log("Verifying pause-reconcile annotation on Central CR") - verifyAnnotation(t, "central", "stackrox-central-services", "acs-central", "stackrox.io/pause-reconcile", "true") - - // Verify SecuredCluster has the pause-reconcile annotation. - t.Log("Verifying pause-reconcile annotation on SecuredCluster CR") - verifyAnnotation(t, "securedcluster", "stackrox-secured-cluster-services", "acs-sensor", "stackrox.io/pause-reconcile", "true") - -} - func TestDeployBothComponentsTogetherInSingleNamespace(t *testing.T) { - if os.Getenv("SKIP_OPERATOR_TESTS") != "" { - t.Skip("SKIP_OPERATOR_TESTS is set") - } - // Create temporary envrc file. envrcFile, err := os.CreateTemp(t.TempDir(), ".envrc.roxie-test-*") if err != nil { @@ -408,26 +61,3 @@ func TestDeployBothComponentsTogetherInSingleNamespace(t *testing.T) { verifyCentralNotInstalled(t, "stackrox") verifySecuredClusterNotInstalled(t, "stackrox") } - -func verifyAnnotation(t *testing.T, resourceType, resourceName, namespace, annotationKey, expectedValue string) { - t.Helper() - - cmd := exec.Command("kubectl", "get", resourceType, resourceName, "-n", namespace, "-o", "jsonpath={.metadata.annotations}") - output, err := cmd.Output() - if err != nil { - t.Fatalf("Failed to get annotation %s on %s/%s in namespace %s: %v", annotationKey, resourceType, resourceName, namespace, err) - } - - annotations := make(map[string]string) - err = json.Unmarshal(output, &annotations) - if err != nil { - t.Fatalf("Failed to unmarshal JSON: %v", err) - } - - currentValue := annotations[annotationKey] - if currentValue != expectedValue { - t.Fatalf("Annotation %s on %s/%s has incorrect value. Expected: %s, Got: %s", annotationKey, resourceType, resourceName, expectedValue, currentValue) - } - - t.Logf("โœ“ Annotation %s=%s verified on %s/%s", annotationKey, expectedValue, resourceType, resourceName) -} diff --git a/tests/e2e/helpers.go b/tests/e2e/helpers.go new file mode 100644 index 00000000..a97c1653 --- /dev/null +++ b/tests/e2e/helpers.go @@ -0,0 +1,263 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + // TODO(#91): We should come up with some auto-updating of this on ACS releases. + // Don't think we should directly inject nightlies here. + defaultMainImageTag = "4.10.1" + deployTimeout = 30 * time.Minute + teardownTimeout = 10 * time.Minute +) + +var ( + commonDeployArgs = []string{"--port-forwarding", "--exposure=none", "--resources=small"} + commonDeployArgsNoPortForward = []string{"--exposure=loadbalancer", "--resources=small"} + + roxieBinary = "roxie" +) + +func teardownAllDeployments() error { + fmt.Println("=== Tearing down all deployments before running tests ===") + + ctx, cancel := context.WithTimeout(context.Background(), teardownTimeout) + defer cancel() + + // Teardown standard deployments + cmd := exec.CommandContext(ctx, roxieBinary, "teardown") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("Warning: teardown command failed: %w", err) + } + + // Teardown single-namespace deployments + ctx, cancel = context.WithTimeout(context.Background(), teardownTimeout) + defer cancel() + cmd = exec.CommandContext(ctx, roxieBinary, "teardown", "--single-namespace") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("Warning: teardown --single-namespace command failed: %w", err) + } + + fmt.Println("=== All deployments have been torn down ===") + return nil +} + +func requireBinary(name string) error { + _, err := exec.LookPath(name) + return err +} + +func requireKubeContext() (string, error) { + cmd := exec.Command("kubectl", "config", "current-context") + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("no kubectl context available: %w", err) + } + + ctx := strings.TrimSpace(string(output)) + if ctx == "" { + return "", fmt.Errorf("kubectl context is empty") + } + + return ctx, nil +} + +func runCommand(t *testing.T, timeout time.Duration, env map[string]string, args ...string) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cmd := exec.CommandContext(ctx, args[0], args[1:]...) + + // Set environment + if env != nil { + cmd.Env = os.Environ() + for k, v := range env { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + } + + // Stream output to both test log and buffers + var stdout, stderr bytes.Buffer + cmd.Stdout = &testWriter{t: t, prefix: "[stdout] ", buf: &stdout} + cmd.Stderr = &testWriter{t: t, prefix: "[stderr] ", buf: &stderr} + + t.Logf("Running: %s", strings.Join(args, " ")) + + err := cmd.Run() + if err != nil { + t.Logf("Command failed: %v", err) + t.Fatalf("Command failed: %v", err) + } + t.Logf("Command completed successfully") +} + +// testWriter writes to both test log and a buffer +type testWriter struct { + t *testing.T + prefix string + buf *bytes.Buffer +} + +func (w *testWriter) Write(p []byte) (n int, err error) { + // Write to buffer + n, err = w.buf.Write(p) + if err != nil { + return n, err + } + + // Also log each line to test output + lines := bytes.Split(p, []byte("\n")) + for _, line := range lines { + if len(line) > 0 { + w.t.Logf("%s%s", w.prefix, string(line)) + } + } + + return n, nil +} + +func verifyNamespaceExists(t *testing.T, namespace string) { + t.Helper() + + cmd := exec.Command("kubectl", "get", "namespace", namespace) + if err := cmd.Run(); err != nil { + t.Fatalf("Namespace %s does not exist", namespace) + } +} + +func verifyNamespaceHasLabel(t *testing.T, namespace, key, value string) { + t.Helper() + + cmd := exec.Command("kubectl", "get", "namespace", namespace, + "-o", "jsonpath={.metadata.labels}") + output, err := cmd.Output() + require.NoError(t, err, "Failed to retrieve labels for namespace %s", namespace) + + var labels map[string]string + err = json.Unmarshal(output, &labels) + require.NoError(t, err, "JSON unmarshalling of namespace labels failed") + + actualValue := labels[key] + assert.Equal(t, value, actualValue, "Namespace %s is missing the correct label for %s", namespace, key) +} + +func doesDeploymentExist(t *testing.T, namespace string, name string) bool { + t.Helper() + + cmd := exec.Command("kubectl", "-n", namespace, "get", "deployments", "-o", "name") + output, err := cmd.Output() + if err != nil { + t.Fatalf("Failed to get deployments in namespace %s: %v", namespace, err) + } + return strings.Contains(string(output), name) +} + +func loadEnvrcFile(path string) (map[string]string, error) { + content, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + env := make(map[string]string) + lines := strings.Split(string(content), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || !strings.HasPrefix(line, "export ") { + continue + } + + // Remove "export " prefix + line = strings.TrimPrefix(line, "export ") + + // Split on first = + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + continue + } + + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + + // Remove quotes + value = strings.Trim(value, "\"") + + env[key] = value + } + + return env, nil +} + +func verifyCentralInstalled(t *testing.T, namespace string) { + t.Helper() + + if !doesDeploymentExist(t, namespace, "central") { + t.Fatalf("Central is not installed in namespace %s", namespace) + } +} + +func verifySecuredClusterInstalled(t *testing.T, namespace string) { + t.Helper() + + if !doesDeploymentExist(t, namespace, "sensor") { + t.Fatalf("Secured cluster is not installed in namespace %s", namespace) + } +} + +func verifyCentralNotInstalled(t *testing.T, namespace string) { + t.Helper() + + if doesDeploymentExist(t, namespace, "central") { + t.Fatalf("Central is installed in namespace %s", namespace) + } +} + +func verifySecuredClusterNotInstalled(t *testing.T, namespace string) { + t.Helper() + + if doesDeploymentExist(t, namespace, "sensor") { + t.Fatalf("Secured cluster is installed in namespace %s", namespace) + } +} + +func verifyAnnotation(t *testing.T, resourceType, resourceName, namespace, annotationKey, expectedValue string) { + t.Helper() + + cmd := exec.Command("kubectl", "get", resourceType, resourceName, "-n", namespace, "-o", "jsonpath={.metadata.annotations}") + output, err := cmd.Output() + if err != nil { + t.Fatalf("Failed to get annotation %s on %s/%s in namespace %s: %v", annotationKey, resourceType, resourceName, namespace, err) + } + + annotations := make(map[string]string) + err = json.Unmarshal(output, &annotations) + if err != nil { + t.Fatalf("Failed to unmarshal JSON: %v", err) + } + + currentValue := annotations[annotationKey] + if currentValue != expectedValue { + t.Fatalf("Annotation %s on %s/%s has incorrect value. Expected: %s, Got: %s", annotationKey, resourceType, resourceName, expectedValue, currentValue) + } + + t.Logf("โœ“ Annotation %s=%s verified on %s/%s", annotationKey, expectedValue, resourceType, resourceName) +}