AE control auth + CodeRabbit #53 followup

.bazelignore

@@ -6,3 +6,7 @@ third_party/threadstacks
 tools/chef/nodes
 # To keep third party dependencies separate, privy is intentional setup as a separate bazel workspace
 src/datagen/pii/privy
+
+# adaptive_export_loadtest generator is a docker-built test tool (see its README);
+# build-agent to replace with a bazel target. Until then, keep it out of gazelle.
+src/e2e_test/adaptive_export_loadtest/tools/loadgen

.github/workflows/cli_release.yaml

@@ -15,7 +15,7 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
     permissions:
       contents: read
@@ -209,7 +209,7 @@ jobs:
   update-gh-artifacts-manifest:
     if: |
       always() && needs.create-github-release.result == 'success'
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: [get-dev-image, create-github-release]
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}

.github/workflows/cloud_release.yaml

@@ -15,7 +15,7 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
     permissions:
       contents: read

.github/workflows/mirror_demos.yaml

@@ -9,7 +9,7 @@ jobs:
     permissions:
       contents: read
       packages: write
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     steps:
     - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772  # v2
       with:

.github/workflows/mirror_deps.yaml

@@ -9,7 +9,7 @@ jobs:
     permissions:
       contents: read
       packages: write
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     steps:
     - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772  # v2
       with:

.github/workflows/mirror_releases.yaml

@@ -10,7 +10,7 @@ jobs:
     permissions:
       contents: read
       packages: write
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     steps:
     - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772  # v2
       with:

.github/workflows/operator_release.yaml

@@ -15,7 +15,7 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
     permissions:
       contents: read
@@ -140,7 +140,7 @@ jobs:
         git commit -s -m "Release Helm chart ${VERSION}"
         git push origin "gh-pages"
   update-gh-artifacts-manifest:
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: [get-dev-image, create-github-release]
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}

.github/workflows/perf_common.yaml

@@ -34,7 +34,7 @@ jobs:
       ref: ${{ inputs.ref }}
   generate-perf-matrix:
     needs: get-dev-image-with-extras
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     container:
       image: ${{ needs.get-dev-image-with-extras.outputs.image-with-tag }}
     outputs:
@@ -57,7 +57,7 @@ jobs:
         echo "matrix=${matrix}" >> $GITHUB_OUTPUT
   run-perf-eval:
     needs: [get-dev-image-with-extras, generate-perf-matrix]
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     container:
       image: ${{ needs.get-dev-image-with-extras.outputs.image-with-tag }}
     strategy:

.github/workflows/vizier_release.yaml

@@ -15,7 +15,7 @@ jobs:
       image-base-name: "dev_image_with_extras"
   build-release:
     name: Build Release
-    runs-on: oracle-16cpu-64gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: get-dev-image
     permissions:
       contents: read
@@ -140,7 +140,7 @@ jobs:
         git commit -s -m "Release Helm chart Vizier ${VERSION}"
         git push origin "gh-pages"
   update-gh-artifacts-manifest:
-    runs-on: oracle-8cpu-32gb-x86-64
+    runs-on: oracle-vm-16cpu-64gb-x86-64
     needs: [get-dev-image, create-github-release]
     container:
       image: ${{ needs.get-dev-image.outputs.image-with-tag }}

k8s/vizier/bootstrap/adaptive_export_deployment.yaml

@@ -88,13 +88,24 @@ spec:
         #   value: "changeme-ingest"
         # - name: CLICKHOUSE_DATABASE
         #   value: "forensic_db"
+        # TLS for the control surface (CONTROL_TLS=true). server.crt/key from the
+        # same service-tls-certs secret the broker/PEM use; without this the dx
+        # bearer JWT crosses the CNI in cleartext. Harmless when control is off.
+        volumeMounts:
+        - name: certs
+          mountPath: /certs
+          readOnly: true
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
             drop:
             - ALL
           seccompProfile:
             type: RuntimeDefault
+      volumes:
+      - name: certs
+        secret:
+          secretName: service-tls-certs
       securityContext:
         runAsUser: 10100
         runAsGroup: 10100

src/api/go/pxapi/opts.go

@@ -82,3 +82,17 @@ func WithDirectCredsInsecure() ClientOption {
 		c.insecureDirect = true
 	}
 }
+
+// WithDirectTLSSkipVerify is the secure-by-default option for direct (standalone /
+// node-local PEM) connections: the transport IS TLS-encrypted, but the server cert
+// is not chain/hostname-verified. Use this instead of WithDirectCredsInsecure when
+// the direct endpoint serves TLS with a self-signed / service cert whose SAN does
+// not match the node IP (e.g. vizier-pem's direct-query port served with
+// service-tls-certs, dialed at HOST_IP). Unlike WithDisableTLSVerification it does
+// NOT require a "cluster.local" address, so it works for the node-IP direct dial.
+// Bearer creds (the minted JWT) therefore ride an encrypted channel, never plaintext.
+func WithDirectTLSSkipVerify() ClientOption {
+	return func(c *Client) {
+		c.disableTLSVerification = true
+	}
+}

src/e2e_test/adaptive_export_loadtest/CONTRACTS.md

@@ -29,7 +29,7 @@ flowchart TD
       PROT["http/dns/pgsql/conn_stats/...<br/>plain MergeTree (NO dedup)"]
     end
 
-    VEC -->|"C1 ⚠️ event_time UNIT = seconds<br/>C2 ⚠️ hostname = k8s node name"| KL
+    VEC -->|"C1 ✅ event_time UNIT = nanoseconds<br/>C2 ⚠️ hostname = k8s node name"| KL
     KL -->|"C3 🔴 event_time monotone ≥ watermark<br/>C4 ⚠️ boundary dedup by content fp"| TRG
     TRG --> CTL
     CTL -->|"C5 ⚠️ anomaly_hash = f(pid,comm,pod,ns) only"| ATTR
@@ -56,7 +56,7 @@ flowchart LR
 
 | # | Contract (implied) | Enforced? | Status / fix |
 |---|---|---|---|
-| C1 | `kubescape_logs.event_time` is unix **seconds** (one unit end-to-end) | ❌ trigger auto-detects s/ms/ns; DDL `toDateTime()` assumes seconds | 🔴 **F8 root** — see C3; AE-2 standardize+normalize |
+| C1 | `kubescape_logs.event_time` is unix **nanoseconds** (one unit end-to-end) | ✅ DDL converts with `fromUnixTimestamp64Nano`; trigger keeps magnitude-normalization as a defensive net | Vector emits ns; DDL + harness aligned to ns (was the F1/F8 seconds-vs-ns root) |
 | C2 | `hostname` = the k8s **node** name (AE polls `WHERE hostname=node`) | ❌ convention only | ⚠️ fixtures must use a real node, else no AE ever reads them |
 | C3 | every new anomaly's `event_time` ≥ current watermark (monotone) | ❌ strict HWM filter | 🔴 **F8** — a larger-unit / out-of-order / future row poisons the HWM → all later rows silently dropped. **Fix (PR #53):** normalize cursor to nanos (`chNormEventTimeNanos`); AE-9: ingest-order cursor / bounded-lookback+dedup + below-watermark metric |
 | C4 | rows sharing `event_time` at the boundary are deduped by content fingerprint | ✅ `seenAtBoundary` | ok |

src/e2e_test/adaptive_export_loadtest/README.md

@@ -1,72 +1,73 @@
-# adaptive_export_loadtest
+# Adaptive Export (AE) load-test suite
 
-Load-test + e2e harness for **adaptive_export (AE)** and the dx-steered SOC chain.
-There are exactly **two ways to test**, by design — pick by what you're proving:
+A table-driven Go test suite for the AE write surface. Each experiment is a
+**fixture** (curated input); each measurement is a named **KPI** asserted with
+`testify/require`; one runner drives them against a deployed AE image on a rig.
 
-| family | needs a live SOC stack? | proves | entry point |
-|---|---|---|---|
-| **A. Fixture-isolation** | No (just ClickHouse) | AE's write behaviour is *deterministic* — injected `kubescape_logs` → exact `forensic_db` rows, across many reps | `harness/run.sh` |
-| **B. Live-attack e2e** | Yes (Pixie + kubescape + CH + AE + dx) | the real chain: attack → detection → DX-steered data-volume reduction → no-loss → NFR | `harness/poc_fire.sh` → `exp_matrix.sh` → `nfr.sh` → `exp_row_reconcile.sh` |
+It replaces the former shell harness (`harness/*.sh` + `stats.py`) — the
+experiments, the measurement scripts, and the reproducibility statistic are now
+Go fixtures, KPI helpers, and asserts under `suite/`.
 
-`event_time` is unix **SECONDS** end-to-end (the unit the soc Vector kubescape sink emits and the CH DDL TTL/PARTITION assume). Fixtures use seconds.
-
----
-
-## A. Fixture-isolation (offline AE proof — no Pixie)
-
-Injects *controlled* `kubescape_logs` trigger rows (real kubescape is **not** deployed) and a *counted* traffic band, then asserts exactly how much AE writes — so write behaviour is measured deterministically instead of lost in infra noise.
-
-```sh
-export KUBECONFIG=<kubeconfig>            # or run lab-side with CH_NO_PF=1
-bash harness/run.sh                       # full suite: ae_config → E1..E4,E6 → E5
-EXP=E1 REPS=20 OUT=/tmp/E1.csv bash harness/exp_control.sh   # one experiment
-EXP=E8 TICKS=25 INTERVAL=3 bash harness/exp_e8.sh            # sustained same-pod (F8 reproducer)
-```
-Exact reproducibility ⇔ `harness/stats.py` reports every `*_act` metric with one distinct value (std=0).
-
-**Scripts:** `run.sh` (orchestrator) · `lib.sh` (CH/kubectl helpers) · `inject.sh` (HTTP INSERT of kubescape_logs) · `ae_config.sh` (AE single-shot load-test mode) · `exp_control.sh` (E1–E4,E6) · `exp_e5.sh` (data-plane volume) · `exp_e8.sh` (sustained same-pod / F8) · `stats.py` (reproducibility verdict).
-
-## B. Live-attack e2e (the real chain, on a deployed stack)
-
-Run on a SOC stack (Pixie vizier Healthy + kubescape netStreaming + CH `forensic_db` + AE + dx). Order:
+## Layout
 
-```sh
-export KUBECONFIG=<kubeconfig>
-# 1. generate the attack signal (idempotent; verifies LDAP egress before returning)
-bash harness/poc_fire.sh
-# 2. data-volume reduction MATRIX — ALL (firehose) vs DX (steered) × {poc,argocd,react2argo}
-CONDITIONS="poc:on react2argo:on" REPS=5 bash harness/exp_matrix.sh
-# 3. NFR — throughput, AE+dx memory under load, verdict/query latency
-bash harness/nfr.sh
-# 4. no-loss — deterministic PEM↔ClickHouse row-level reconciliation for the DX arm
-bash harness/exp_row_reconcile.sh
+| Path | What |
+|------|------|
+| `suite/harness.go` | primitives: ClickHouse-over-HTTP client, kubescape-row injector, control-surface reads, kubectl helpers |
+| `suite/fixtures.go` | the experiment table (control-plane reproducibility cases) + the per-rep runner |
+| `suite/kpi.go` | KPI asserts: `RequireReproducible`, `RequireReconcile`, `RequireExact` |
+| `suite/suite_test.go` | the tests: control-plane reproducibility (live), data-plane reconcile + volume reduction (staged) |
+| `tools/loadgen/` | the counted signal generator (nested Go module) for the data-plane KPIs |
+| `k8s/` | sinks + generator pod templates |
+| `CONTRACTS.md` | the C1–C15 AE implied-contract register |
+| `fixtures/EXPERIMENTS.md` | the experiment catalog + expected outputs |
+| `FINDINGS_AND_BACKLOG.md` | observed contract violations + backlog |
+
+## KPIs
+
+| KPI | Asserts | Was |
+|-----|---------|-----|
+| **Reproducibility** | a metric is one distinct value across all reps (std = 0) | `stats.py` |
+| **Reconcile** | read == wrote == ClickHouse, per protocol table (no loss) | `exp_row_reconcile.sh` |
+| **Reduction** | firehose→steered volume delta — measured & logged, not gated | `exp_matrix.sh` |
+| **NFR / WriteDuration** | throughput/mem/latency; window stays open until `t_end` (C15) | `nfr.sh` / `exp_e8.sh` |
+
+## Running
+
+The suite is **live-only**: it drives a deployed AE image, so `go test ./...`
+skips unless enabled. Run it on the rig's dev-machine (which has Go + kubectl):
+
+```bash
+cd suite && go mod tidy            # first run only, resolves testify
+AELOAD_LIVE=1 \
+AELOAD_CH_URL=http://<clickhouse>:8123 \
+AELOAD_CH_WUSER=ingest_writer AELOAD_CH_WPASS=<pass> \
+KUBECONFIG=/path/to/kubeconfig \
+go test -v -run TestControlPlaneReproducibility ./...
 ```
 
-**Scripts:** `poc_fire.sh` (attack-signal generator, bob#140-hardened) · `exp_matrix.sh` (reduction matrix, the canonical ALL-vs-DX runner) · `nfr.sh` (throughput/mem/latency) · `exp_row_reconcile.sh` (no-loss).
-
-> The DX arm needs the load-gen pods bound to a **benign User SBoB** (`kubescape.io/managed-by: User`, `rulePolicies.R0002.processAllowed`) or benign noise gets steered and contaminates the reduction — see `biz/PoC/poc/datavolume/denoise_sbobs/`.
-
----
+Environment:
 
-## Layout
-```
-fixtures/EXPERIMENTS.md   curated kubescape_logs data-set catalog + expected outputs
-harness/                  the two families above
-k8s/                      isolated sinks + per-rep generator pod (no probes)
-tools/loadgen/            cleanloadgen + httpsink Go sources + Dockerfile
-```
-Go unit/e2e tests for AE live with the service: `src/vizier/services/adaptive_export/internal/{trigger,e2e}/*_test.go`.
+| Var | Default | Meaning |
+|-----|---------|---------|
+| `AELOAD_LIVE` | — | must be `1` to run (else skip) |
+| `AELOAD_CH_URL` | `http://127.0.0.1:8123` | ClickHouse HTTP endpoint (port-forward or in-cluster svc) |
+| `AELOAD_CH_USER` / `_PASS` | default user | read credentials |
+| `AELOAD_CH_WUSER` / `_WPASS` | `ingest_writer` | ingest (write) credentials |
+| `AELOAD_AE_NS` / `_DS` | `pl` / `adaptive-export` | AE namespace + DaemonSet |
+| `AELOAD_NODE` | first node | the node whose hostname AE polls |
 
-See `CONTRACTS.md` (AE implied contracts) and `FINDINGS_AND_BACKLOG.md` (reproduced findings incl. the F8 watermark-poison bug).
+The data-plane reconcile and volume-reduction tests are staged (they need the
+counted signal generator / a lab-owned signal) and skip with a reason until
+`AELOAD_DATAPLANE=1` / `AELOAD_REDUCTION=1` wire them in.
 
-## Validation status (honest)
+## Vocabulary
 
-| Experiment | Plane | Status |
-|---|---|---|
-| E1 single / E2 dedup / E3 fan-out / E4 boundary / E6 restart-idempotency | control | ✅ exactly reproducible (std=0) on a live rig |
-| E8 sustained same-pod | control | ✅ reproduced the F8 "writes-stop" bug + recovery |
-| E5 volume / E8 data-mode | data | ⏳ authored; pending live validation |
-| Live poc reduction / NFR / no-loss (family B) | data | ✅ validated (aeprod19 + pemdq10 + dx): #33 prefetch verdict 212→18ms; reduction ALL→DX ≫ measured |
+Fixtures carry no CVE identifiers and no adversarial verbs. The control plane is
+pure kubescape-metadata bookkeeping, so its fixtures are named by the property
+under test. Live incident signals (data-plane / reduction) are emitted by the SOC
+lab under its own naming (`java-poc`, `pathogen-ns`, `disease-*`, `Specimen`) and
+triggered here through a lab hook — no payload literals live in this tree.
 
-## Removed (consolidation 2026-06)
-Redundant variants folded into the canonical scripts above — deleted: `ae_vs_all.sh`, `vrun.sh`, `exp_poc_reps.sh`, `exp_datavolume_extreme.sh`, `exp_dx_steering_reduction.sh` (→ `exp_matrix.sh`); `exp_ae_nfr_benchmark.sh` (→ `nfr.sh`); `exp_pipeline_reconcile.sh` (→ `exp_row_reconcile.sh`); `exp_dx_validate.sh` (→ `exp_matrix.sh`); `deploy_ae.sh`, `build_gen_image.sh` (superseded by the live stack / kit).
+**External wire stays literal** and is never renamed: kubescape RuleIDs (`R0001`,
+`R0010`, …), Kubernetes API keywords, ClickHouse column/table names, and the
+`forensic_db` schema.

src/e2e_test/adaptive_export_loadtest/skaffold.yaml

@@ -0,0 +1,36 @@
+# Single pixie-native command to stand up the whole NON-Pixie e2e environment:
+#
+#   skaffold deploy -m e2e-nonpixie
+#
+# It pulls each component's Skaffold from that component's golden-source repo, so
+# there is exactly ONE source of truth per component — no manifests duplicated
+# into the pixie tree:
+#
+#   - k8sstormcenter/soc  (skaffold.yaml, module soc-stack)  = the stack:
+#         ClickHouse -> kubescape -> vector  (+ forensic_db schema, dx delivery)
+#   - k8sstormcenter/bob  (example/java-poc/skaffold.yaml, module java-poc-apps) =
+#         the sample apps: SBoBs -> workloads (java-poc chain + pathogen)
+#
+# Deploy order is: SOC stack first, then the bob apps (SBoBs before workloads).
+# Pixie itself (the `pl` namespace: vizier + adaptive_export) is deployed by
+# Pixie's OWN native Skaffold and overlaid on top — it is intentionally NOT here.
+#
+# k3s only for now. The suite's EnsureE2EStack() helper (suite/deploy.go) invokes
+# this, and can point at local checkouts via AELOAD_SOC_DIR / AELOAD_BOB_DIR.
+apiVersion: skaffold/v4beta11
+kind: Config
+metadata:
+  name: e2e-nonpixie
+requires:
+  - git:
+      repo: https://github.com/k8sstormcenter/soc
+      path: skaffold.yaml
+      ref: main
+    configs:
+      - soc-stack
+  - git:
+      repo: https://github.com/k8sstormcenter/bob
+      path: example/java-poc/skaffold.yaml
+      ref: main
+    configs:
+      - java-poc-apps