diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..17264c6d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,21 @@ +# Default reviewers for the whole repo. +# Add or change owners as the team grows; nearest match wins. + +* @nudgebee/engineering + +# eBPF probes need a closer eye — kernel quirks, license boundaries. +/ebpftracer/ @nudgebee/engineering +/ebpftracer/ebpf/ @nudgebee/engineering + +# Release surface — CI, manifests, install script, Dockerfiles. +/.github/workflows/ @nudgebee/engineering +/manifests/ @nudgebee/engineering +/install.sh @nudgebee/engineering +/Dockerfile @nudgebee/engineering +/Dockerfile.alpine @nudgebee/engineering + +# Legal / governance — changes here should be deliberate. +/LICENSE @nudgebee/engineering +/NOTICE @nudgebee/engineering +/SECURITY.md @nudgebee/engineering +/CODE_OF_CONDUCT.md @nudgebee/engineering diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 00000000..df12b8b8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,58 @@ +name: Bug report +description: The agent is doing the wrong thing or crashing +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to report a bug. The more concrete detail + you can give, the faster we can fix it. + + Please do **not** include real credentials, tokens, or internal + hostnames in this issue. Replace them with `example.com` or ``. + - type: textarea + id: what-happened + attributes: + label: What happened? + description: What did the agent do, and what did you expect instead? + placeholder: e.g. agent crashed with a panic after ~10 minutes on nodes with > 200 containers + validations: + required: true + - type: textarea + id: repro + attributes: + label: Steps to reproduce + description: Minimal repro. Workload type, kernel version, anything special about the environment. + validations: + required: true + - type: input + id: version + attributes: + label: Agent version + description: Output of `nudgebee-node-agent --version`, or the image tag. + placeholder: v1.2.3 / ghcr.io/nudgebee/node-agent:1.2.3 + validations: + required: true + - type: input + id: kernel + attributes: + label: Linux kernel version + placeholder: e.g. 5.15.0-1042-aws + validations: + required: true + - type: input + id: distro + attributes: + label: Distribution / runtime + placeholder: e.g. Ubuntu 22.04 on EKS 1.30, containerd 1.7 + - type: textarea + id: logs + attributes: + label: Relevant logs + description: Paste the relevant agent logs. Scrub any secrets first. + render: shell + - type: textarea + id: metrics + attributes: + label: Relevant metrics / Prometheus output (optional) + render: shell diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..934bb4cd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Security vulnerability + url: https://github.com/nudgebee/node-agent/security/advisories/new + about: Report security issues privately via GitHub Security Advisories or email security@nudgebee.com. Do not file a public issue. + - name: Discussion + url: https://github.com/nudgebee/node-agent/discussions + about: For questions, ideas, or anything that isn't yet a bug or a concrete feature request. diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 00000000..5a48e033 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,27 @@ +name: Feature request +description: Suggest a new capability or enhancement +labels: ["enhancement"] +body: + - type: textarea + id: problem + attributes: + label: What problem are you trying to solve? + description: Describe the use case, not the solution. We're looking for the underlying need. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: What would you like to see? + description: A concrete proposal if you have one. Mock metric names, sample configs, CLI flags — anything that helps make it real. + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives you considered + - type: textarea + id: context + attributes: + label: Additional context + description: Links to related issues, upstream coroot/coroot-node-agent discussions, blog posts, etc. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..57e63920 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,36 @@ + + +## What this PR does + + + +## Linked issue / context + + + +## How was this tested? + + + +## eBPF changes? + + + +## Checklist + +- [ ] `make lint` passes (gofmt, goimports, go vet, go mod tidy) +- [ ] `make test` passes +- [ ] CHANGELOG.md updated under `## [Unreleased]` for user-visible changes +- [ ] No real credentials, internal hostnames, or production captures in test fixtures diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..8dac6705 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,41 @@ +version: 2 +updates: + - package-ecosystem: gomod + directory: "/" + schedule: + interval: weekly + day: monday + open-pull-requests-limit: 10 + groups: + otel: + patterns: + - "go.opentelemetry.io/*" + - "github.com/agoda-com/opentelemetry-logs-go*" + k8s: + patterns: + - "k8s.io/*" + - "sigs.k8s.io/*" + prometheus: + patterns: + - "github.com/prometheus/*" + labels: + - dependencies + - go + + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + day: monday + labels: + - dependencies + - github-actions + + - package-ecosystem: docker + directory: "/" + schedule: + interval: weekly + day: monday + labels: + - dependencies + - docker diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4b772c4..0bd80c92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,12 @@ on: branches: - main +permissions: + contents: read + jobs: - GO: + go: + name: Go (build, vet, test) runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 @@ -25,5 +29,47 @@ jobs: go install golang.org/x/tools/cmd/goimports@latest files=$(goimports -l .); if [[ -n "$files" ]]; then echo "$files"; exit 1; fi - run: go vet ./... + # /containers transitively imports github.com/NVIDIA/go-nvml. The + # bindings register NVML symbols (including some only present in + # very recent libnvidia-ml.so versions, e.g. + # nvmlDeviceSetMemClkVfOffset) at process start via dlsym, which + # fails on a runner without a matching NVIDIA driver and aborts + # the test binary before any of our tests run. Skip until either + # GPU code moves behind a build tag or the runner ships a newer + # libnvidia-ml.so. - run: go test $(go list ./... | grep -v '/containers$') - run: go build -mod=readonly . + + secrets: + name: Secret scan + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # Run the gitleaks CLI directly (BSD-licensed, free for orgs). + # gitleaks/gitleaks-action requires a paid licence when run on an + # org-owned repository, which we don't need for this use case. + - name: Install gitleaks + run: | + curl -sSL https://github.com/gitleaks/gitleaks/releases/download/v8.21.2/gitleaks_8.21.2_linux_x64.tar.gz \ + | sudo tar -xz -C /usr/local/bin gitleaks + gitleaks version + # Scan only the commits introduced by this event: + # - PR: base..HEAD + # - push: before..after + # Pre-existing secrets in history are tracked separately and will + # be purged by a one-time git-filter-repo pass before the repo + # goes public. + - name: Scan PR commits + if: github.event_name == 'pull_request' + run: | + gitleaks detect --no-banner --config .gitleaks.toml \ + --redact --verbose --exit-code 1 \ + --log-opts="${{ github.event.pull_request.base.sha }}..HEAD" + - name: Scan pushed commits + if: github.event_name == 'push' + run: | + gitleaks detect --no-banner --config .gitleaks.toml \ + --redact --verbose --exit-code 1 \ + --log-opts="${{ github.event.before }}..${{ github.event.after }}" diff --git a/.gitignore b/.gitignore index c7136428..761c3a0f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,41 @@ -.idea -.vagrant +# Editors / IDEs +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Vagrant +.vagrant/ + +# Go build artifacts +/nudgebee-node-agent +/coroot-node-agent +*.exe +*.test +*.out + +# Go coverage / profiling +*.prof +*.pprof +coverage.txt +coverage.html + +# Build / dist +/dist/ +/build/ +/bin/ + +# Vendored modules (we don't vendor) +/vendor/ + +# Local env files +.env +.env.local + +# eBPF build outputs (regenerated via ebpftracer/make build) +ebpftracer/ebpf/*.o diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..31ad0cac --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,21 @@ +# gitleaks config — extends the default ruleset. +# Run locally with: gitleaks detect --config .gitleaks.toml +[extend] +useDefault = true + +[allowlist] +description = "Synthetic placeholders used in test fixtures" +regexes = [ + # AWS's documented example access key, used in ebpftracer/l7/l7_test.go + '''AKIAIOSFODNN7EXAMPLE''', + # Synthetic redaction markers, not real credentials + '''EXAMPLE_SESSION_TOKEN_REDACTED''', + '''EXAMPLE_COOKIE_REDACTED''', + '''EXAMPLE_REMOTE_WRITE_BODY_REDACTED''', + '''EXAMPLE\.JWT\.TOKEN_REDACTED''', +] +paths = [ + # GPL-2.0 text contains nothing secret but does include the FSF + # postal address regexp may flag certain text patterns + '''LICENSES/GPL-2\.0\.txt''', +] diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..6ae28c97 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,31 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- Open-source release of the Nudgebee Node Agent, forked from + [coroot/coroot-node-agent](https://github.com/coroot/coroot-node-agent). +- LLM observability pipeline: detects and parses LLM API traffic + (OpenAI, Bedrock, Google AI, Anthropic, others) and emits per-request + metrics with token counts, latency, and pricing. +- IP-to-FQDN resolver: enriches outbound flows with reverse-DNS names + via HTTP `Host` headers and DNS metadata. +- Enhanced eBPF L7 protocol detection: TLS SNI, HTTP/2, Node.js, and + improved Go TLS capture. +- Oracle Cloud instance-metadata support. +- Pressure Stall Information (PSI) cgroup metrics. + +### Changed + +- Prometheus `job` label and outbound `User-Agent` are now + `nudgebee-node-agent`. Update any dashboards or alerts that filter on + `job="coroot-node-agent"`. +- OpenTelemetry service name is now `nudgebee-node-agent`. +- Default systemd unit name is `nudgebee-node-agent.service`. +- Container image is published at `ghcr.io/nudgebee/node-agent`. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..e5245e5b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,62 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior: + +- The use of sexualized language or imagery, and sexual attention or advances +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards +and will take appropriate and fair corrective action in response to any +behavior that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies +when an individual is officially representing the community in public spaces. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +**conduct@nudgebee.com**. All complaints will be reviewed and investigated +promptly and fairly. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 401b021f..0c91320f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,36 +1,67 @@ # Contributing -Thank you for your interest in contributing to Coroot! -Below are some basic guidelines. - +Thanks for your interest in contributing. This document covers the +basics; see [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) for community +expectations and [SECURITY.md](SECURITY.md) for vulnerability reports. ## Requirements -* Linux ≥v5.1 (amd64, arm64) -* Go v1.23 +- Linux ≥ 5.1, amd64 or arm64 (the agent only builds and runs on Linux) +- Go ≥ 1.24 +- `libsystemd-dev` (for journald log reading) +- `clang`, `llvm`, `make`, `pkg-config` (only required if you change eBPF code) -## Running -```shell -sudo go run main.go -``` +The repository is a fork of +[coroot/coroot-node-agent](https://github.com/coroot/coroot-node-agent); +see [NOTICE](NOTICE) for attribution. + +## Running locally -```shell +```sh +sudo go run . curl http://127.0.0.1:80/metrics ``` -## Pull Request Checklist +The agent needs `CAP_SYS_ADMIN` / `CAP_BPF` to load eBPF programs and +host-pid + cgroup access to enumerate containers — `sudo` covers all of +this during development. -* Branch from the main branch and, if needed, rebase to the current main branch before submitting your pull request. If it doesn't merge cleanly with main you may be asked to rebase your changes. -* Commits should be as small as possible, while ensuring that each commit is correct independently (i.e., each commit should compile and pass tests). -* Add tests relevant to the fixed bug or new feature. -* Use `make lint` to run linters and ensure formatting is correct. -* Run the unit tests suite `make test`. +## Pull-request checklist +- Branch off `main`; rebase if `main` has moved before opening the PR. +- Keep commits small and self-contained — each commit should build and + pass tests on its own. +- Add tests for new functionality or bug fixes. +- `make lint` must pass (gofmt, goimports, go vet, go mod tidy). +- `make test` must pass. +- Update `CHANGELOG.md` under `## [Unreleased]` for user-visible changes. -## eBPF +## eBPF changes -If you are changing eBPF code, you need to generate the `ebpftracer/ebpf.go` file: -```shell +The compiled eBPF programs live in `ebpftracer/ebpf.go`, which is +generated. If you edit anything under `ebpftracer/ebpf/`, regenerate: + +```sh cd ebpftracer make build ``` + +Commit the regenerated `ebpf.go` along with your `.c`/`.h` changes. The +main Dockerfile only runs `go build` and will not rebuild eBPF for you. + +## Module layout + +- `cgroup/` — cgroup v1/v2 inspection and PSI +- `common/` — shared utilities including the IP→FQDN resolver +- `containers/` — container discovery, log readers, LLM detection and + parsing, per-container metric registries +- `ebpftracer/` — eBPF probe loader and L7 protocol parsers +- `node/`, `node/metadata/` — host-level metrics and cloud metadata +- `prom/` — Prometheus remote-write client +- `logs/`, `tracing/`, `profiling/` — OpenTelemetry exporters + +## License + +By submitting a contribution you agree to license your changes under +the same terms as the project: Apache-2.0 for Go code, GPL-2.0 for +eBPF C code. diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 106bd081..56993ab5 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -17,7 +17,7 @@ COPY . . # Build statically linked binary ARG VERSION=unknown -RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-extldflags='-Wl,-z,lazy' -X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o coroot-node-agent . +RUN CGO_ENABLED=1 go build -mod=readonly -ldflags "-extldflags='-Wl,-z,lazy' -X 'github.com/coroot/coroot-node-agent/flags.Version=${VERSION}'" -o nudgebee-node-agent . # Runtime stage FROM alpine:3.22 AS release-stage @@ -28,10 +28,10 @@ RUN apk add --no-cache openssl libssl3 ca-certificates WORKDIR /app # Copy the statically linked binary -COPY --from=builder /app/coroot-node-agent /usr/bin/coroot-node-agent +COPY --from=builder /app/nudgebee-node-agent /usr/bin/nudgebee-node-agent # Ensure it's executable -RUN chmod +x /usr/bin/coroot-node-agent +RUN chmod +x /usr/bin/nudgebee-node-agent # Run it -CMD ["/usr/bin/coroot-node-agent"] +CMD ["/usr/bin/nudgebee-node-agent"] diff --git a/LICENSES/GPL-2.0.txt b/LICENSES/GPL-2.0.txt new file mode 100644 index 00000000..9efa6fbc --- /dev/null +++ b/LICENSES/GPL-2.0.txt @@ -0,0 +1,338 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, see . + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Moe Ghoul, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..798c748b --- /dev/null +++ b/NOTICE @@ -0,0 +1,17 @@ +Nudgebee Node Agent +Copyright 2024-present Nudgebee, Inc. + +This product is a derivative work of coroot-node-agent +(https://github.com/coroot/coroot-node-agent), originally developed +by Coroot, Inc. (Copyright 2020-present Coroot, Inc.) and licensed +under the Apache License, Version 2.0. + +The Go code in this repository is distributed under the Apache License, +Version 2.0. See LICENSE for the full text. + +The eBPF C code under ebpftracer/ebpf/ is distributed under the +GNU General Public License, Version 2.0. See LICENSES/GPL-2.0.txt for +the full text. + +This product includes software developed by third parties; see go.sum +for the complete list of Go module dependencies and their licenses. diff --git a/README.md b/README.md index bcc26094..0c4c9e80 100644 --- a/README.md +++ b/README.md @@ -1,104 +1,116 @@ -# Coroot-node-agent +# Nudgebee Node Agent +[![Continuous Integration](https://github.com/nudgebee/node-agent/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/nudgebee/node-agent/actions/workflows/ci.yml) [![Go Report Card](https://goreportcard.com/badge/github.com/coroot/coroot-node-agent)](https://goreportcard.com/report/github.com/coroot/coroot-node-agent) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -The agent gathers metrics related to a node and the containers running on it, and it exposes them in the Prometheus format. +A per-node observability agent for Kubernetes and Linux hosts. The agent +gathers container and host metrics, logs, and L7 traffic using eBPF and +exposes them in Prometheus format. -It uses eBPF to track container related events such as TCP connects, so the minimum supported Linux kernel version is 5.1. +Minimum Linux kernel: **5.1** (eBPF CO-RE). - +> This project is a fork of +> [coroot/coroot-node-agent](https://github.com/coroot/coroot-node-agent) +> with additional features developed at Nudgebee. See [NOTICE](NOTICE) +> for attribution and [CHANGELOG.md](CHANGELOG.md) for the list of +> divergences from upstream. ## Features -### TCP connection tracing - -To provide visibility into the relationships between services, the agent traces containers TCP events, such as *connect()* and *listen()*. - -Exported metrics are useful for: -* Obtaining an actual map of inter-service communications. It doesn't require integration of distributed tracing frameworks into your code. -* Detecting connections errors from one service to another. -* Measuring network latency between containers, nodes and availability zones. - -Related blog posts: - * [Building a service map using eBPF](https://coroot.com/blog/building-a-service-map-using-ebpf) - * [How ping measures network round-trip time accurately using SO_TIMESTAMPING](https://coroot.com/blog/how-to-ping) - * [The current state of eBPF portability](https://coroot.com/blog/ebpf-portability) -### Log patterns extraction - -Log management is usually quite expensive. In most cases, you do not need to analyze each event individually. -It is enough to extract recurring patterns and the number of the related events. - -This approach drastically reduces the amount of data required for express log analysis. - -The agent discovers container logs and parses them right on the node. - -At the moment the following sources are supported: -* Direct logging to files in */var/log/* -* Journald -* Dockerd (JSON file driver) -* Containerd (CRI logs) - -To learn more about automated log clustering, check out the blog post "[Mining metrics from unstructured logs](https://coroot.com/blog/mining-logs-from-unstructured-logs)". - -### Delay accounting - -[Delay accounting](https://www.kernel.org/doc/html/latest/accounting/delay-accounting.html) allows engineers to accurately -identify situations where a container is experiencing a lack of CPU time or waiting for I/O. - -The agent gathers per-process counters through [Netlink](https://man7.org/linux/man-pages/man7/netlink.7.html) and aggregates them into per-container metrics: -* [container_resources_cpu_delay_seconds_total](https://docs.coroot.com/metrics/node-agent#container_resources_cpu_delay_seconds_total) -* [container_resources_disk_delay_seconds_total](https://docs.coroot.com/metrics/node-agent#container_resources_disk_delay_seconds_total) - - - - -Related blog posts: -* [Delay accounting: an underrated feature of the Linux kernel](https://coroot.com/blog/linux-delay-accounting) +### Inherited from upstream + +- **TCP connection tracing** — service map and inter-service latency, + derived from eBPF `connect()` / `accept()` / retransmit / RTT events. +- **Log pattern extraction** — clusters container logs into recurring + patterns at the node, drastically reducing log volume for analysis. + Reads from `/var/log/`, journald, dockerd, and containerd CRI logs. +- **Delay accounting** — per-container CPU and disk-wait metrics from + the kernel's [delay accounting](https://www.kernel.org/doc/html/latest/accounting/delay-accounting.html) + subsystem via Netlink. +- **OOM-kill events** — surfaces `container_oom_kills_total`. +- **Cloud instance metadata** — auto-detects AWS, GCP, Azure, Hetzner, + IBM Cloud, and Oracle Cloud and tags metrics with account ID, + instance type, region, AZ, lifecycle (spot/on-demand), and addresses. +- **GPU monitoring** (NVIDIA via NVML), JVM metrics (via JMX/jattach), + .NET and Node.js process detection. + +### Added in this fork + +- **[LLM observability](docs/llm-observability.md)** — detects calls + to OpenAI, Anthropic, AWS Bedrock, Google AI / Vertex, Azure OpenAI, + Cohere, and OpenAI-compatible endpoints from eBPF-traced TLS + sessions, and emits per-request metrics including model name, token + counts, latency, time-to-first-token, and estimated cost. +- **[IP-to-FQDN resolver](docs/ip-fqdn-resolver.md)** — enriches + outbound flows with hostnames and K8s workload identity by + combining Service/Pod/Node informers with the DNS cache, so the + service map shows `payments-api` instead of `10.0.5.41`. +- **Enhanced L7 protocol detection** — TLS SNI extraction, lightweight + HTTP/2 parsing with HPACK, improved Go TLS capture, Node.js TLS, + FoundationDB. +- **PSI cgroup metrics** — pressure-stall info for CPU, memory, and IO. +- **Stability fixes** — graceful shutdown, bounded caches, label + cardinality controls, panic guards in L7 parsers, OOM mitigations. +## Installation -### Out-of-memory events tracing +### Kubernetes (DaemonSet) -The [container_oom_kills_total](https://docs.coroot.com/metrics/node-agent#container_oom_kills_total) metric shows that a container has been terminated by the OOM killer. +```sh +kubectl apply -f https://raw.githubusercontent.com/nudgebee/node-agent/main/manifests/nudgebee-node-agent.yaml +``` -### Instance meta information +This creates the `nudgebee` namespace and a privileged DaemonSet that +exposes `/metrics` on port 80. -If a node is a cloud instance, the agent identifies a cloud provider and collects additional information using the related metadata services. +### systemd (bare-metal) -Supported cloud providers: [AWS](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html), [GCP](https://cloud.google.com/compute/docs/metadata/overview), [Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service?tabs=linux), [Hetzner](https://docs.hetzner.cloud/#server-metadata) +```sh +curl -fsSL https://raw.githubusercontent.com/nudgebee/node-agent/main/install.sh | sudo sh - +``` -Collected info: -* AccountID -* InstanceID -* Instance/machine type -* Region -* AvailabilityZone -* AvailabilityZoneId (AWS only) -* LifeCycle: on-demand/spot (AWS and GCP only) -* Private & Public IP addresses +Pass `-v vX.Y.Z` to pin to a specific release. The script writes a +systemd unit at `/etc/systemd/system/nudgebee-node-agent.service` and +starts it. -Related blog posts: -* [Gathering cloud instance metadata in AWS, GCP and Azure](https://coroot.com/blog/cloud-metadata) +### Container image -## Installation +Multi-arch images (linux/amd64, linux/arm64) are published to GHCR on +every tag: -Follow the Coroot [documentation](https://docs.coroot.com/) +``` +ghcr.io/nudgebee/node-agent: +ghcr.io/nudgebee/node-agent:. +ghcr.io/nudgebee/node-agent: +``` ## Metrics -The collected metrics are described [here](https://docs.coroot.com/metrics/node-agent). +The agent exposes Prometheus metrics on `:80/metrics`. Self-identifying +label: `job="nudgebee-node-agent"`. -## Coroot +- Nudgebee-specific metrics: + - [LLM observability](docs/llm-observability.md#metrics-emitted) + - [IP-to-FQDN resolver](docs/ip-fqdn-resolver.md) +- The full inherited metric catalogue is documented upstream at + [docs.coroot.com/metrics/node-agent](https://docs.coroot.com/metrics/node-agent). -The best way to turn metrics to answers about app issues is to use [Coroot](https://github.com/coroot/coroot) - a zero-instrumentation observability tool for microservice architectures. +## Contributing -A live demo of Coroot is available at [demo.coroot.com](https://demo.coroot.com) +See [CONTRIBUTING.md](CONTRIBUTING.md). Bug reports and pull requests +are welcome. -## Contributing -To start contributing, check out our [Contributing Guide](https://github.com/coroot/coroot-node-agent/blob/main/CONTRIBUTING.md). +## Security + +Please report vulnerabilities privately per [SECURITY.md](SECURITY.md). ## License -Coroot-node-agent is licensed under the [Apache License, Version 2.0](https://github.com/coroot/coroot-node-agent/blob/main/LICENSE). +This project is licensed under the [Apache License, Version 2.0](LICENSE). + +The eBPF C code in `ebpftracer/ebpf/` is licensed under the GNU General +Public License, Version 2.0; see [LICENSES/GPL-2.0.txt](LICENSES/GPL-2.0.txt). -The BPF code is licensed under the General Public License, Version 2.0. +See [NOTICE](NOTICE) for attribution to the original +coroot/coroot-node-agent project. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..ec2244d7 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,44 @@ +# Security Policy + +## Supported Versions + +We provide security updates for the latest minor release. + +| Version | Supported | +| ------- | --------- | +| latest | yes | +| older | no | + +## Reporting a Vulnerability + +If you believe you have found a security vulnerability in nudgebee-node-agent, +please report it privately. Do **not** open a public GitHub issue. + +Email: **security@nudgebee.com** + +Please include: + +- A description of the issue and its potential impact +- Steps to reproduce (proof-of-concept code is welcome) +- The version / commit you tested against +- Any suggested remediation + +We will acknowledge receipt within 3 business days, provide an initial +assessment within 10 business days, and aim to ship a fix within 90 days +of confirmation. We will credit you in the release notes unless you +request otherwise. + +## Scope + +In scope: + +- Code in this repository (Go agent and eBPF probes) +- Default configurations shipped in `manifests/` and `install.sh` +- Container images published from this repository's release workflow + +Out of scope: + +- Issues in third-party dependencies (please report upstream) +- Vulnerabilities that require root access on the host where the agent + is already running with `privileged: true` and `hostPID: true` (this + is the documented operating model) diff --git a/containers/app.go b/containers/app.go index 1403271c..bec1869f 100644 --- a/containers/app.go +++ b/containers/app.go @@ -19,6 +19,10 @@ func guessApplicationTypeByCmdline(cmdline []byte) string { } cmd := bytes.TrimSuffix(bytes.Fields(parts[0])[0], []byte{':'}) switch { + case bytes.HasSuffix(cmd, []byte("nudgebee-node-agent")): + return "nudgebee-node-agent" + case bytes.HasSuffix(cmd, []byte("nudgebee-cluster-agent")): + return "nudgebee-cluster-agent" case bytes.HasSuffix(cmd, []byte("coroot")): return "coroot-community-edition" case bytes.HasSuffix(cmd, []byte("coroot-ee")): diff --git a/containers/container.go b/containers/container.go index 7a788629..14aa8588 100644 --- a/containers/container.go +++ b/containers/container.go @@ -1771,7 +1771,7 @@ func (c *Container) runLogParser(logPath string) { for _, p := range c.processes { if p.Flags.LogMonitoringDisabled { - klog.InfoS("skipping log monitoring due to COROOT_LOG_MONITORING=disabled", "cg", c.cgroup.Id) + klog.InfoS("skipping log monitoring due to NUDGEBEE_LOG_MONITORING=disabled", "cg", c.cgroup.Id) return } } diff --git a/docs/ip-fqdn-resolver.md b/docs/ip-fqdn-resolver.md new file mode 100644 index 00000000..92b3d042 --- /dev/null +++ b/docs/ip-fqdn-resolver.md @@ -0,0 +1,85 @@ +# IP-to-FQDN resolver + +The agent enriches outbound network flows with a human-readable +destination identity. Instead of a service map full of dotted-quad +IPs, you get pod names, Kubernetes service names, deployment names, +or DNS hostnames. + +This feature is specific to the nudgebee fork and is not present in +upstream `coroot/coroot-node-agent`. + +## What it does + +For each destination IP observed via an eBPF `connect()` event, the +resolver returns a *Workload* record with: + +| Field | Example | Source | +| ----------- | ----------------------------- | ---------------------------------------------- | +| `Name` | `payments-api` / `payments-7c…-xj9` / `api.openai.com` / `10.0.5.41` | K8s API or DNS cache | +| `Namespace` | `prod` / `node` / `external` | K8s API or synthetic | +| `Kind` | `Deployment` / `Service` / `pod` / `node` / `external` | K8s API or synthetic | +| `Region` | `us-east-1` | `topology.kubernetes.io/region` node label | +| `Zone` | `us-east-1c` | `topology.kubernetes.io/zone` node label | +| `Instance` | `ip-10-0-5-41.ec2.internal` | name of the K8s Node hosting the destination | + +This record is attached as labels / span attributes to the affected +metrics and traces. + +## Resolution order + +Sources are checked in this priority: + +1. **Service ClusterIP** — IP matches a `Service.spec.clusterIPs`. + The selector is then walked to identify the backing + Deployment / ReplicaSet / StatefulSet so the result includes the + owning workload, not just the Service object. +2. **Pod IP** — IP matches a `Pod.status.podIPs`. Ownership references + are followed upward (Pod → ReplicaSet → Deployment), and ephemeral + pods (bare Pods, standalone Jobs) are aggregated by their standard + `app.kubernetes.io/*` labels to prevent label cardinality blowups + on workloads like Airflow tasks. +3. **Node IP** — IP matches a `Node.status.addresses`. +4. **eBPF DNS cache** — if `--resolve-dns` is enabled, fall back to + reverse-resolution using DNS replies the agent has captured live + on the node. No `net.LookupAddr` calls are made; only data already + observed in flight. +5. **IP literal** — last resort: `Name = ""`, kind = + `external`, namespace = `external`. + +When two sources would resolve the same IP (e.g. an in-cluster +proxy's Service IP overlapping a Pod IP), priority is deterministic: +Services > Nodes > Pods. + +## What's watched + +The resolver maintains Kubernetes informers on: Pods, Nodes, Services, +Deployments, ReplicaSets, DaemonSets, StatefulSets, Jobs, CronJobs. +Each watched object is stripped to a minimal projection (name, +namespace, labels, owner references, IPs) before being stored, which +reduces informer-cache memory roughly 5×. + +The DNS cache is an LRU bounded at 10,000 entries. The pod-IP index +is an LRU bounded at 30,000 entries. + +## Configuration + +| Flag | Default | Purpose | +| --------------------------------- | ------- | ---------------------------------------------------------------- | +| `--resolve-dns` | `true` | Enable the DNS-cache fallback for non-cluster IPs | +| `--aggregate-ephemeral-workloads` | `true` | Collapse Job/bare-Pod cardinality via `app.kubernetes.io/*` labels | + +All flags can also be set as environment variables — see +`flags/flags.go`. + +## Limitations + +- **K8s API access required.** Without RBAC permission to watch Pods, + Services, Nodes, etc., the resolver falls back to DNS-cache and + IP-literal only. The manifests in `manifests/` ship with the + minimum RBAC needed. +- **In-cluster Service IPs only.** Resolution works on ClusterIP + values; LoadBalancer/external IPs are seen as the underlying + cloud-provider IP, which then falls through to DNS. +- **CNI-overlay NAT.** If a CNI rewrites source/destination IPs in + ways the kernel doesn't surface in the connect tuple, the resolver + sees the rewritten IP and resolves against that. diff --git a/docs/llm-observability.md b/docs/llm-observability.md new file mode 100644 index 00000000..8cfacbe3 --- /dev/null +++ b/docs/llm-observability.md @@ -0,0 +1,176 @@ +# LLM observability + +The agent identifies outbound traffic to large-language-model APIs and +exposes per-request metrics — model name, token counts, latency, +time-to-first-token, error class, and estimated cost — alongside the +container, pod, and namespace the request came from. No code changes, +SDK wrappers, or proxies are required: detection happens in eBPF on +the node. + +This feature is specific to the nudgebee fork and is not present in +upstream `coroot/coroot-node-agent`. + +## Supported providers + +| Provider | Match | +| ----------------------- | ----------------------------------------------------------------------- | +| OpenAI | `api.openai.com` (and subdomains) | +| Anthropic | `api.anthropic.com`, `claude.ai` | +| Google Gemini / Vertex | `generativelanguage.googleapis.com`, `ai.googleapis.com`, `aiplatform.googleapis.com`, regional Vertex endpoints | +| AWS Bedrock | `bedrock-runtime..amazonaws.com` | +| Azure OpenAI | `.openai.azure.com` | +| Cohere | `api.cohere.com`, `api.cohere.ai` | +| OpenAI-compatible | `api.groq.com`, `api.together.xyz`, `api.fireworks.ai`, `api.deepseek.com`, `api.mistral.ai`, `api.perplexity.ai` | + +When the hostname doesn't match, the path is checked for known LLM +patterns (`/v1/messages`, `:generateContent`, `/models/gemini`, the +Bedrock `/model/.../converse` form, etc.) as a fallback. + +## How detection works + +Three signals are used, in priority order: + +1. **DNS cache** — when an in-cluster DNS reply resolves an LLM + hostname, every IP in the reply is cached. Subsequent TCP + connections to those IPs are tagged at `connect()` time. +2. **TLS SNI** — the ClientHello is parsed by an eBPF probe. The SNI + hostname matches the provider tables directly. SNI-based tagging + is the primary mechanism for everything except Google APIs (Google + shares anycast IPs across Gemini / Compute / etc., so SNI is the + only reliable disambiguator). +3. **Late tag from HTTP headers** — when an HTTP/1.1 `Host` or HTTP/2 + `:authority` is observed mid-request, a connection that wasn't + already tagged by DNS or SNI gets retroactively classified. + +For Google APIs specifically, IP caching is suppressed because the +anycast pool overlaps with non-LLM services. Detection there is +SNI + header-driven only. + +## What gets extracted + +Per request the agent records: + +- Model name (read from response body's `"model"` field, with + fallbacks to request body and URL path patterns) +- Input / output / cached-input token counts +- Tool / function-call invocation count +- Total request duration +- Time-to-first-token (streaming requests only) +- HTTP response status code +- Streaming flag +- Container ID, pod name, namespace +- W3C trace context (`traceparent`) if present + +Streaming responses (SSE / HTTP/2 DATA frames) are accumulated into a +bounded ring buffer (64 KB tail, since `usage` and `finish_reason` +typically live near the end). Completion markers (`data: [DONE]` for +OpenAI, `message_stop` for Anthropic, `finishReason` for Gemini) are +detected inline. Idle streams time out after 30 s; the hard upper +bound on a single stream is 5 minutes. + +## Metrics emitted + +All LLM metrics live under the `container_llm_` prefix. Standard +container labels (`container_id`, `namespace`, `pod`) are present +on every series; LLM-specific labels are listed per metric below. + +| Metric | Type | LLM-specific labels | +| ----------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `container_llm_requests_total` | counter | `gen_ai_operation_name`, `gen_ai_request_model`, `gen_ai_provider_name`, `server_address`, `http_response_status_code` | +| `container_llm_token_usage_total` | counter | + `gen_ai_token_type` (`input`, `output`) | +| `container_llm_cached_input_tokens_total` | counter | same as token_usage | +| `container_llm_tool_calls_total` | counter | same as requests | +| `container_llm_errors_total` | counter | + `error_type` (`rate_limit`, `timeout`, `invalid_request`, `server_error`, `auth_error`) | +| `container_llm_request_duration_seconds` | histogram | OTel GenAI buckets (0.01 – 81.92 s) | +| `container_llm_time_to_first_token_seconds` | histogram | OTel GenAI buckets (0.001 – 10 s); streaming only | +| `container_llm_tokens_per_second` | histogram | streaming only | +| `container_llm_cost_usd_total` | counter | provider+model labels | +| `node_agent_llm_sni_tags_total` | counter | `provider` — diagnostic for SNI tagging coverage | +| `node_agent_hpack_decode_errors_total` | counter | mid-stream-join indicator (see *Limitations*) | + +Label naming follows the OpenTelemetry GenAI semantic conventions +where applicable. + +## Cost estimation + +A static pricing table (`containers/llm_pricing.go`) maps +`:` to per-million-token rates (input, +output, and cached-input where applicable). Longest-prefix wins, so +`gpt-4o-2024-05-13` resolves to the `gpt-4o` row. + +Cost is computed as: + +``` +cost = (input - cached) * inputRate/1M + + output * outputRate/1M + + cached * cachedRate/1M +``` + +Coverage includes OpenAI (GPT-4, GPT-4o, o1, o3, embeddings), +Anthropic Claude 3 / 3.5, Google Gemini 1.5 / 2.x / 3.x, AWS Bedrock +(Anthropic, Nova, Llama, Mistral, Cohere), and direct Cohere. +OpenAI-compatible providers have placeholder entries and may report +zero cost depending on the model string. + +Numbers are **list prices** — no volume or contract discounts are +applied. If no row matches, cost is 0 and the request is still +counted in the other metrics. + +## Sample queries + +Cost-by-model in the last hour: + +```promql +sum by (gen_ai_request_model) ( + rate(container_llm_cost_usd_total[1h]) +) * 3600 +``` + +P95 time-to-first-token, per provider, last 5 m (streaming endpoints): + +```promql +histogram_quantile( + 0.95, + sum by (gen_ai_provider_name, le) ( + rate(container_llm_time_to_first_token_seconds_bucket[5m]) + ) +) +``` + +Rate-limit error count per pod: + +```promql +sum by (namespace, pod) ( + rate(container_llm_errors_total{error_type="rate_limit"}[5m]) +) +``` + +## Limitations + +- **HTTP/2 mid-stream-join.** If a workload's HTTP/2 connection to a + provider predates the agent attaching to that workload's TLS session, + HPACK dynamic-table state is unrecoverable and the agent will see + malformed frames for the lifetime of that connection. Mitigations: + restart the workload after rolling out the agent (forces fresh + connections), or rely on SNI-based tagging which works regardless of + HPACK state. `node_agent_hpack_decode_errors_total` is the canary — + if it climbs steadily for a particular pod, that pod likely needs a + restart. SNI tagging itself is bypassed if a sidecar proxy (Istio, + Envoy) terminates TLS upstream of the workload. +- **Service-mesh TLS termination.** Same as above — when an in-cluster + mesh sidecar handles TLS to the LLM provider, the workload itself + doesn't open the provider connection, and the agent can only attribute + the call to the sidecar process. There's no general workaround short + of instrumenting the sidecar. +- **Body capture is bounded.** eBPF payload capture is limited per + packet; very large request or response bodies may have token counts + extracted only partially. Streaming responses use a 64 KB tail + buffer that retains the end of the stream where token counts + typically land. +- **Pricing is best-effort.** Prices in `containers/llm_pricing.go` + reflect list rates at the time the entry was written and require + manual updates. Models not in the table report cost = 0. +- **Google Gemini cannot be IP-tagged.** Because Google's anycast IP + pool is shared across many services, IP-only detection would + produce false positives. Detection requires SNI (TLS) or the + `:authority` header (HTTP/2). diff --git a/ebpftracer/l7/l7_test.go b/ebpftracer/l7/l7_test.go index 1cb73e83..9c428d8b 100644 --- a/ebpftracer/l7/l7_test.go +++ b/ebpftracer/l7/l7_test.go @@ -111,7 +111,7 @@ func TestParseHost(t *testing.T) { headers := ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) - requestString = "POST / HTTP/1.1\r\nHost: ec2.us-east-1.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.50.10 (go1.21.6; linux; amd64) karpenter.sh-v0.34.0\r\nContent-Length: 422\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ASIAUCTZOIG67J6IJFOY/20240408/us-east-1/ec2/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token, Signature=3d7eadceec976dd17fbdcc07ae43a75264baf179fec70c6aa2be7411fd552e70\r\nContent-Type: application/x-www-form-urlencoded; charset=utf-8\r\nX-Amz-Date: 20240408T094330Z\r\nX-Amz-Security-Token: IQoJb3JpZ2luX2VjEML//////////wEaCXVzLWVhc3QtMSJHMEUCIB8pKvU76WFQ99xqZH8LQXhOBfCP/lhnMSkmIp1LcvLEAiEAk3XkJCPMFGQC4KqMk92boxaeNHiLvbinmW9AQgEGnBcq/gQI6///////////ARABGgwyODA1MDEzMDU3ODkiDJA/sBIYp1t0z9szDirSBPDDQ+mKiBwDvCwqZ/HH9wSh9U6WKYjh0EPX9i3cHE46oee4CAUoQwEmDy9xZp02UZcpOjiDF6iaKyOvi93+LoiwJVBNi53o/PnMoStfI4OmG4Rc61YYcPc6t6SSXeBEGCzeeesWAGBDJ6GE2PWYYQAKS3YrpxYE+UKdb4hxEBO5t8RgEO/ooiHehFC/5VSmjMeMfT/XMdmAxBplX/wCMoz2vMcpkCi3fCsX0RNjxT7bAO/D2jOUrIX8UjM38h2VGNliSfCl8B3HbCETxYiz66AhMGabknMpJ8Dcotm\x00" + requestString = "POST / HTTP/1.1\r\nHost: ec2.us-east-1.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.50.10 (go1.21.6; linux; amd64) karpenter.sh-v0.34.0\r\nContent-Length: 422\r\nAuthorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20240101/us-east-1/ec2/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token, Signature=0000000000000000000000000000000000000000000000000000000000000000\r\nContent-Type: application/x-www-form-urlencoded; charset=utf-8\r\nX-Amz-Date: 20240101T000000Z\r\nX-Amz-Security-Token: EXAMPLE_SESSION_TOKEN_REDACTED\x00" req, err = ParseHTTPRequest([]byte(requestString)) assert.Nil(t, nil, err) assert.Equal(t, "/", req.URL.Path) @@ -121,7 +121,7 @@ func TestParseHost(t *testing.T) { headers = ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) - requestString = "POST / HTTP/1.1\r\nHost: ec2.us-east-1.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.50.10 (go1.21.6; linux; amd64) karpenter.sh-v0.34.0\r\nContent-Length: 422\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ASIAUCTZOIG67J6IJFOY/20240408/us-east-1/ec2/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token, Signature=3d7eadceec976dd17fbdcc07ae43a75264baf179fec70c6aa2be7411fd552e70\r\nContent-Type: application/x-www-form-urlencoded; charset=utf-8\r\nX-Amz-Date: 20240408T094330Z\r\nX-Amz-Security-Token: IQoJb3JpZ2luX2VjEML//////////wEaCXVzLWVhc3QtMSJHMEUCIB8pKvU76WFQ99xqZH8LQXhOBfCP/lhnMSkmIp1LcvLEAiEAk3XkJCPMFGQC4KqMk92boxaeNHiLvbinmW9AQgEGnBcq/gQI6///////////ARABGgwyODA1MDEzMDU3ODkiDJA/sBIYp1t0z9szDirSBPDDQ+mKiBwDvCwqZ/HH9wSh9U6WKYjh0EPX9i3cHE46oee4CAUoQwEmDy9xZp02UZcpOjiDF6iaKyOvi93+LoiwJVBNi53o/PnMoStfI4OmG4Rc61YYcPc6t6SSXeBEGCzeeesWAGBDJ6GE2PWYYQAKS3YrpxYE+UKdb4hxEBO5t8RgEO/ooiHehFC/5VSmjMeMfT/XMdmAxBplX/wCMoz2vMcpkCi3fCsX0RNjxT7bAO/D2jOUrIX8UjM38h2VGNliSfCl8B3HbCETxYiz66AhMGabknMpJ8Dcotm\x00" + requestString = "POST / HTTP/1.1\r\nHost: ec2.us-east-1.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.50.10 (go1.21.6; linux; amd64) karpenter.sh-v0.34.0\r\nContent-Length: 422\r\nAuthorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20240101/us-east-1/ec2/aws4_request, SignedHeaders=content-length;content-type;host;x-amz-date;x-amz-security-token, Signature=0000000000000000000000000000000000000000000000000000000000000000\r\nContent-Type: application/x-www-form-urlencoded; charset=utf-8\r\nX-Amz-Date: 20240101T000000Z\r\nX-Amz-Security-Token: EXAMPLE_SESSION_TOKEN_REDACTED\x00" req, err = ParseHTTPRequest([]byte(requestString)) assert.Nil(t, nil, err) assert.Equal(t, "/", req.URL.Path) @@ -131,27 +131,27 @@ func TestParseHost(t *testing.T) { headers = ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) - requestString = "GET /api/v1/namespaces/nudgebee-agent/pods/nudgebee-agent-pgnxh/log?container=node-agent&previous=False&tailLines=1000×tamps=True HTTP/1.1\r\nHost: 10.100.0.1\r\nAccept-Encoding: identity\r\nAccept: application/json\r\nUser-Agent: OpenAPI-Generator/26.1.0/python\r\nauthorization: bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjNhZTFlNjlkN2I1ZmFkNzhhNTg2YTVjN2JmMDNjOGU3YjUzNGI4N2MifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjIl0sImV4cCI6MTc0NDEwMjY0OSwiaWF0IjoxNzEyNTY2NjQ5LCJpc3MiOiJodHRwczovL29pZGMuZWtzLnVzLWVhc3QtMS5hbWF6b25hd3MuY29tL2lkL0I4RkU4OUQ4MjdEN0Q2RTE0REIyMUMzRkIwQTk1Q0E3Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJudWRnZWJlZS1hZ2VudCIsInBvZCI6eyJuYW1lIjoibnVkZ2ViZWUtYWdlbnQtcnVubmVyLTZiZmY2NmQ3YjgtcWttbmwiLCJ1aWQiOiJiN2M4ZGM2NC04Y2Q0LTRiMWUtODg1Zi00YWMzMWE4NDYwMWIifSwic2VydmljZWFjY291bnQiOnsibmFtZSI6Im51ZGdlYmVlLWFnZW50LXJ1bm5lci1zZXJ2aWNlLWFjY291bnQiLCJ1aWQiOiJjODUzMDc2YS1hODVkLTQzYzItOWU3NS00MTk5ODIzMTVjOGIifSwid2FybmFmdGVyIjoxNzEyNTcwMjU2fSwibmJmIjoxNzEyNTY2NjQ5LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6b\x00" + requestString = "GET /api/v1/namespaces/example-ns/pods/example-pod/log?container=node-agent&previous=False&tailLines=1000×tamps=True HTTP/1.1\r\nHost: 10.0.0.1\r\nAccept-Encoding: identity\r\nAccept: application/json\r\nUser-Agent: OpenAPI-Generator/26.1.0/python\r\nauthorization: bearer EXAMPLE.JWT.TOKEN_REDACTED\x00" req, err = ParseHTTPRequest([]byte(requestString)) assert.Nil(t, nil, err) - assert.Equal(t, "/api/v1/namespaces/nudgebee-agent/pods/nudgebee-agent-pgnxh/log", req.URL.Path) - assert.Equal(t, "10.100.0.1", req.Host) + assert.Equal(t, "/api/v1/namespaces/example-ns/pods/example-pod/log", req.URL.Path) + assert.Equal(t, "10.0.0.1", req.Host) assert.Equal(t, "GET", req.Method) headers = ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) - requestString = "PUT /nudgebee-dev-loki-logs/fake/ece64b2028d30d33/18ebcc10d27%3A18ebcfd01d3%3A3377d9aa HTTP/1.1\r\nHost: s3.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.44.315 (go1.21.3; linux; amd64)\r\nContent-Length: 14183\r\nAuthorization: AWS4-HMAC-SHA256 Credential=ASIAUCTZOIG62SVE2F5J/20240408/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class, Signature=a4db153ba7127721c3a5efbbab1a0e194e473d23147a472c92ff672d9dff160c\r\nContent-Md5: VBSxwLVjCvMeC+z4V86k8A==\r\nX-Amz-Content-Sha256: fe19171172133fadb236229e71a8f60169fc5268da749b1eeeeb3396e7d90f64\r\nX-Amz-Date: 20240408T094454Z\r\nX-Amz-Security-Token: IQoJb3JpZ2luX2VjEL3//////////wEaCXVzLWVhc3QtMSJGMEQCIAvNhx/LC+nIkqUC+rRD3MOFuxr39bc+qmRwVPljywxuAiA/8bmpr+kmRy7zXBzBQ7eyayLftqLutyGrzu8TIZWVbyrGBQjm//////////8BEAEaDDI4MDUwMTMwNTc4OSIMRsng8zavN9ilo6+0KpoFc/pdelkZBjWQle/CNG4XKcnd9WFbu87JAB3xa1P97vfIBjhqpmHoPk4V38r21Vepl0NjeUnWahJA6NnOWW2mdhRv51+WGUmKSHz1j6NCRcC8axDXs65jJzF6WgqCGZGNa0aiUQP7oIGGV8\x00" + requestString = "PUT /example-bucket/fake/aaaaaaaaaaaaaaaa/bbbbbbbbbbbb%3Acccccccccccc%3Adddddddd HTTP/1.1\r\nHost: s3.amazonaws.com\r\nUser-Agent: aws-sdk-go/1.44.315 (go1.21.3; linux; amd64)\r\nContent-Length: 14183\r\nAuthorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20240101/us-east-1/s3/aws4_request, SignedHeaders=content-length;content-md5;host;x-amz-content-sha256;x-amz-date;x-amz-security-token;x-amz-storage-class, Signature=0000000000000000000000000000000000000000000000000000000000000000\r\nContent-Md5: AAAAAAAAAAAAAAAAAAAAAA==\r\nX-Amz-Content-Sha256: 0000000000000000000000000000000000000000000000000000000000000000\r\nX-Amz-Date: 20240101T000000Z\r\nX-Amz-Security-Token: EXAMPLE_SESSION_TOKEN_REDACTED\x00" req, err = ParseHTTPRequest([]byte(requestString)) assert.Nil(t, nil, err) - assert.Equal(t, "/nudgebee-dev-loki-logs/fake/ece64b2028d30d33/18ebcc10d27:18ebcfd01d3:3377d9aa", req.URL.Path) + assert.Equal(t, "/example-bucket/fake/aaaaaaaaaaaaaaaa/bbbbbbbbbbbb:cccccccccccc:dddddddd", req.URL.Path) assert.Equal(t, "s3.amazonaws.com", req.Host) assert.Equal(t, "PUT", req.Method) headers = ConvertHeadersToBase64String(req.Header) assert.NotNil(t, headers) - requestString = "POST /api/v1/write HTTP/1.1\r\nHost: vmsingle-victoria-victoria-metrics-k8s-stack.victoria.svc:8429\r\nUser-Agent: vmagent\r\nContent-Length: 282715\r\nContent-Encoding: snappy\r\nContent-Type: application/x-protobuf\r\nX-Prometheus-Remote-Write-Version: 0.1.0\r\nAccept-Encoding: gzip\r\n\r\n\x91\x9b\xdc\x01\xb0\n\xe4\f\n$\n\b__name__\x12\x18container_memory_failcnt\n$\n\t\x15\x1c\xd0\x12\x17nudgebee-agent-opencost\n;\n\x05image\x122quay.io/kubecost1\x15\n\x00-\x01*X-model:prod-1.108.0\n\x1b\n\t\x01\x87\x18space\x12\x0e6c\x00 \n/\n\x03pod\x12(6\x17\x00\x15z\xd8-55b677fb47-87mz4\n \n\x17beta_kubernetes_io_arch\x12\x05amd64\n.\n J\"\x00pinstance_type\x12\nm5ad.large\n\x1e\n\x15J0\x00\xf0Cos\x12\x05linux\n&\n\x1eeks_amazonaws_com_capacityType\x12\x04SPOT\n5\n(failure_domain_JW\x00Pregion\x12\tus-east-1\n4\n&\x867\x00\x14zone\x12\n\x155\x10a\n(\n\b\x11Ҙ\x12\x1cip-172-31-0-236.ec2.internal\n\x0e\n\x03job\x12\a!1 let\n=\n\x19k8%5\xf0--, where + # is the tag without the leading "v". Strip "v" if present. + VERSION_NO_V="${VERSION#v}" + URL="${GITHUB_URL}/download/${VERSION}/${SYSTEM_NAME}-${VERSION_NO_V}-${ARCH}" set +e case $DOWNLOADER in curl) @@ -134,9 +137,9 @@ download_binary() { setup_binary() { chmod 755 ${TMP_BIN} - info "Installing coroot-node-agent to ${BIN_DIR}/coroot-node-agent" + info "Installing ${SYSTEM_NAME} to ${BIN_DIR}/${SYSTEM_NAME}" $SUDO chown root:root ${TMP_BIN} - $SUDO mv -f ${TMP_BIN} ${BIN_DIR}/coroot-node-agent + $SUDO mv -f ${TMP_BIN} ${BIN_DIR}/${SYSTEM_NAME} } download() { @@ -158,18 +161,25 @@ set -x systemctl stop ${SYSTEM_NAME} systemctl disable ${SYSTEM_NAME} systemctl reset-failed ${SYSTEM_NAME} +# Also disable the legacy upstream unit if it's still installed; ignore failures +# so this script also works on hosts that never ran coroot-node-agent. +systemctl stop coroot-node-agent 2>/dev/null || true +systemctl disable coroot-node-agent 2>/dev/null || true +systemctl reset-failed coroot-node-agent 2>/dev/null || true systemctl daemon-reload rm -f ${FILE_SERVICE} rm -f ${FILE_ENV} +rm -f ${SYSTEMD_DIR}/coroot-node-agent.service +rm -f ${SYSTEMD_DIR}/coroot-node-agent.service.env remove_uninstall() { rm -f ${UNINSTALL_SH} } trap remove_uninstall EXIT -rm -rf /var/lib/coroot-node-agent || true -rm -f ${BIN_DIR}/coroot-node-agent +rm -rf /var/lib/${SYSTEM_NAME} /var/lib/coroot-node-agent || true +rm -f ${BIN_DIR}/${SYSTEM_NAME} ${BIN_DIR}/coroot-node-agent EOF $SUDO chmod 755 ${UNINSTALL_SH} $SUDO chown root:root ${UNINSTALL_SH} @@ -192,8 +202,8 @@ create_systemd_service_file() { info "systemd: Creating service file ${FILE_SERVICE}" $SUDO tee ${FILE_SERVICE} >/dev/null << EOF [Unit] -Description=Coroot node agent -Documentation=https://coroot.com +Description=Nudgebee node agent +Documentation=https://github.com/nudgebee/node-agent Wants=network-online.target After=network-online.target @@ -216,7 +226,7 @@ TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s -ExecStart=${BIN_DIR}/coroot-node-agent +ExecStart=${BIN_DIR}/${SYSTEM_NAME} EOF } @@ -226,7 +236,7 @@ create_service_file() { } get_installed_hashes() { - $SUDO sha256sum ${BIN_DIR}/coroot-node-agent ${FILE_SERVICE} ${FILE_ENV} 2>&1 || true + $SUDO sha256sum ${BIN_DIR}/${SYSTEM_NAME} ${FILE_SERVICE} ${FILE_ENV} 2>&1 || true } systemd_enable() { diff --git a/logs/otel.go b/logs/otel.go index d7712e9a..79213a28 100644 --- a/logs/otel.go +++ b/logs/otel.go @@ -50,14 +50,14 @@ func Init(machineId, hostname, version string) { sdk.WithResource( resource.NewWithAttributes( semconv.SchemaURL, - semconv.ServiceName("coroot-node-agent"), + semconv.ServiceName("nudgebee-node-agent"), semconv.HostName(hostname), semconv.HostID(machineId), ), ), ) otel.SetLoggerProvider(loggerProvider) - otelLogger = loggerProvider.Logger("coroot-node-agent", otelLogs.WithInstrumentationVersion(version)) + otelLogger = loggerProvider.Logger("nudgebee-node-agent", otelLogs.WithInstrumentationVersion(version)) } func OtelLogEmitter(containerId string) logparser.OnMsgCallbackF { diff --git a/manifests/coroot-node-agent.yaml b/manifests/nudgebee-node-agent.yaml similarity index 53% rename from manifests/coroot-node-agent.yaml rename to manifests/nudgebee-node-agent.yaml index 36e97441..737c3951 100644 --- a/manifests/coroot-node-agent.yaml +++ b/manifests/nudgebee-node-agent.yaml @@ -1,7 +1,13 @@ apiVersion: v1 kind: Namespace metadata: - name: coroot + name: nudgebee + labels: + # The agent requires privileged + hostPID; allow that explicitly on + # clusters that enforce Pod Security Admission (Kubernetes >= 1.25). + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged --- @@ -9,17 +15,17 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: - app: coroot-node-agent - name: coroot-node-agent - namespace: coroot + app: nudgebee-node-agent + name: nudgebee-node-agent + namespace: nudgebee spec: selector: matchLabels: - app: coroot-node-agent + app: nudgebee-node-agent template: metadata: labels: - app: coroot-node-agent + app: nudgebee-node-agent annotations: prometheus.io/scrape: 'true' prometheus.io/port: '80' @@ -28,8 +34,12 @@ spec: - operator: Exists hostPID: true containers: - - name: coroot-node-agent - image: ghcr.io/coroot/coroot-node-agent + - name: nudgebee-node-agent + # Pinning to a major-version tag keeps deploys deterministic + # within a compatible range. For production, pin to a fully + # qualified semver tag (e.g. :1.2.3). + image: ghcr.io/nudgebee/node-agent:1 + imagePullPolicy: IfNotPresent args: ["--cgroupfs-root", "/host/sys/fs/cgroup"] ports: - containerPort: 80 @@ -49,4 +59,4 @@ spec: name: cgroupfs - hostPath: path: /sys/kernel/debug - name: debugfs \ No newline at end of file + name: debugfs diff --git a/proc/flags.go b/proc/flags.go index 966bd8db..70e3c43f 100644 --- a/proc/flags.go +++ b/proc/flags.go @@ -29,15 +29,18 @@ func GetFlags(pid uint32) (Flags, error) { if len(kv) != 2 { continue } - if !strings.HasPrefix(kv[0], "COROOT_") { + // COROOT_* env vars are accepted for backwards compatibility with + // users migrating from coroot/coroot-node-agent. NUDGEBEE_* is the + // canonical prefix going forward. + if !strings.HasPrefix(kv[0], "NUDGEBEE_") && !strings.HasPrefix(kv[0], "COROOT_") { continue } switch kv[0] { - case "COROOT_EBPF_PROFILING": + case "NUDGEBEE_EBPF_PROFILING", "COROOT_EBPF_PROFILING": flags.EbpfProfilingDisabled = strings.Contains(kv[1], "disabled") - case "COROOT_LOG_MONITORING": + case "NUDGEBEE_LOG_MONITORING", "COROOT_LOG_MONITORING": flags.LogMonitoringDisabled = strings.Contains(kv[1], "disabled") - case "COROOT_EBPF_TRACES": + case "NUDGEBEE_EBPF_TRACES", "COROOT_EBPF_TRACES": flags.EbpfTracesDisabled = strings.Contains(kv[1], "disabled") } } diff --git a/prom/remote_writer.go b/prom/remote_writer.go index 4f8cf05a..4141ad8a 100644 --- a/prom/remote_writer.go +++ b/prom/remote_writer.go @@ -62,7 +62,7 @@ func StartAgent(reg *prometheus.Registry, machineId, systemUuid string) error { url: *flags.MetricsEndpoint, labels: map[string]string{ model.InstanceLabel: instance, - model.JobLabel: "coroot-node-agent", + model.JobLabel: "nudgebee-node-agent", }, httpClient: http.Client{ Timeout: RemoteWriteTimeout, @@ -143,7 +143,7 @@ func (a *Agent) send(fPath string) error { for k, v := range common.AuthHeaders() { req.Header.Set(k, v) } - req.Header.Set("User-Agent", "coroot-node-agent") + req.Header.Set("User-Agent", "nudgebee-node-agent") req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Content-Encoding", "snappy") req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") diff --git a/test_pods_metrics.sh b/test_pods_metrics.sh index c22f7b1f..62151b20 100755 --- a/test_pods_metrics.sh +++ b/test_pods_metrics.sh @@ -5,9 +5,9 @@ set -e -NAMESPACE="nudgebee-agent" -APP_LABEL="app=nudgebee-node-agent" -BASE_PORT=8090 +NAMESPACE="${NAMESPACE:-nudgebee}" +APP_LABEL="${APP_LABEL:-app=nudgebee-node-agent}" +BASE_PORT="${BASE_PORT:-8090}" echo "=== Testing Metrics Endpoint for All Nudgebee Agent Pods ===" echo "Namespace: $NAMESPACE"