From 8f38c29b837b6ded60aad73e790c13b7622afeaf Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 13 Jun 2022 08:40:51 +0100 Subject: [PATCH 01/29] X-Smart-Branch-Parent: master From 3ac69e70dea6d9b924444ae3f9b75a41c642cba3 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Thu, 21 Apr 2022 11:00:21 +0100 Subject: [PATCH 02/29] ROX-10009: modifies build-kos to use collector custom bpf probe --- kernel-modules/build/build-kos | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel-modules/build/build-kos b/kernel-modules/build/build-kos index a461c90e0d..3732ab90e3 100755 --- a/kernel-modules/build/build-kos +++ b/kernel-modules/build/build-kos @@ -89,9 +89,9 @@ build_ko() ( echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." - KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" make -j 6 -C bpf + KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" make -j 6 -C collector-bpf - collector_probe="bpf/probe.o" + collector_probe="bpf/collector-probe.o" if [[ ! -f "$collector_probe" || ! -s "$collector_probe" ]]; then echo "Empty or missing compiled bpf output" From 30b496bd695d16baaf461085d1a92ce338a079ae Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 25 Apr 2022 17:14:28 +0100 Subject: [PATCH 03/29] ROX-10009: misc collector changes to facilitate new probe --- collector/lib/SysdigService.cpp | 5 ++++- kernel-modules/build/build-kos | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/collector/lib/SysdigService.cpp b/collector/lib/SysdigService.cpp index 38d0bb5ffe..786df938fd 100644 --- a/collector/lib/SysdigService.cpp +++ b/collector/lib/SysdigService.cpp @@ -133,7 +133,10 @@ sinsp_evt* SysdigService::GetNext() { if (event->get_category() & EC_INTERNAL) return nullptr; // TODO (rc) not needed when syscall filtering implemented in the eBPF probe - if (useEbpf && !global_event_filter_[event->get_type()]) return nullptr; + // if (useEbpf && !global_event_filter_[event->get_type()]) { + // CLOG(DEBUG) << "filtering cos ebpf?"; + // return nullptr; + // } userspace_stats_.event_parse_micros[event->get_type()] += (NowMicros() - parse_start); ++userspace_stats_.nUserspaceEvents[event->get_type()]; diff --git a/kernel-modules/build/build-kos b/kernel-modules/build/build-kos index 3732ab90e3..a461c90e0d 100755 --- a/kernel-modules/build/build-kos +++ b/kernel-modules/build/build-kos @@ -89,9 +89,9 @@ build_ko() ( echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." - KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" make -j 6 -C collector-bpf + KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" make -j 6 -C bpf - collector_probe="bpf/collector-probe.o" + collector_probe="bpf/probe.o" if [[ ! -f "$collector_probe" || ! -s "$collector_probe" ]]; then echo "Empty or missing compiled bpf output" From 5142cc5f579f3029d55d6b70e7a45bdb75dba7b0 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 25 Apr 2022 17:16:35 +0100 Subject: [PATCH 04/29] ROX-10009: updates falco commit to latest working version --- falcosecurity-libs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcosecurity-libs b/falcosecurity-libs index 75dd1e36bc..b03aa42bd9 160000 --- a/falcosecurity-libs +++ b/falcosecurity-libs @@ -1 +1 @@ -Subproject commit 75dd1e36bc185337d2dc0bc83df831a533c3eaf3 +Subproject commit b03aa42bd9f6512bab0157ad7d1e4efad13b6fac From 2fcf56305d5e26bff8e68f375246ccbe85a7db84 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Tue, 26 Apr 2022 08:02:02 +0100 Subject: [PATCH 05/29] ROX-10009: bumped module version --- kernel-modules/MODULE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel-modules/MODULE_VERSION b/kernel-modules/MODULE_VERSION index 9084fa2f71..227cea2156 100644 --- a/kernel-modules/MODULE_VERSION +++ b/kernel-modules/MODULE_VERSION @@ -1 +1 @@ -1.1.0 +2.0.0 From 06cec25a602c669ee65d3affc0cf4708e16a9b43 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Tue, 3 May 2022 11:02:02 +0100 Subject: [PATCH 06/29] ROX-10009: moves custom probe into collector repo The probe is now maintained and built from collector but sets up include directories to the falco repository. Also includes a fix to RHEL-7 where the old falco probe is used due to an incomplete set of tracepoints on that platform. --- kernel-modules/build/build-kos | 20 +- kernel-modules/dockerized/Dockerfile | 1 + kernel-modules/probe/Makefile | 53 +++++ kernel-modules/probe/collector_probe.c | 280 +++++++++++++++++++++++++ 4 files changed, 348 insertions(+), 6 deletions(-) create mode 100644 kernel-modules/probe/Makefile create mode 100644 kernel-modules/probe/collector_probe.c diff --git a/kernel-modules/build/build-kos b/kernel-modules/build/build-kos index a461c90e0d..a32ddac302 100755 --- a/kernel-modules/build/build-kos +++ b/kernel-modules/build/build-kos @@ -77,6 +77,16 @@ build_ko() ( return 0 fi + # + # Unfortunately RHEL-7 does not have the necessary tracepoints that we require + # for the collector probe (specifically clone, execve, fork, and vfork) + # so we just build the falco catch-all probe for this platform + # + + echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." + KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" make -j 6 -C bpf + collector_probe="bpf/probe.o" + # Check kernel version is at least 4.14 (unless RHEL 7.6 kernel detected) else if ((bundle_version < 4 || (bundle_version == 4 && bundle_major < 14))); then @@ -85,13 +95,11 @@ build_ko() ( touch "${MODULE_BASE_DIR}/${module_version}/.collector-ebpf-${kernel_version}.unavail" return 0 fi - fi - echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." - - KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" make -j 6 -C bpf - - collector_probe="bpf/probe.o" + echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." + KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" FALCO_DIR="$(pwd)/bpf" make -j 6 -C /probe + collector_probe="/probe/probe.o" + fi if [[ ! -f "$collector_probe" || ! -s "$collector_probe" ]]; then echo "Empty or missing compiled bpf output" diff --git a/kernel-modules/dockerized/Dockerfile b/kernel-modules/dockerized/Dockerfile index b285f01c31..2bf973fc83 100644 --- a/kernel-modules/dockerized/Dockerfile +++ b/kernel-modules/dockerized/Dockerfile @@ -39,6 +39,7 @@ COPY /collector/kernel-modules/dockerized/scripts /scripts COPY /collector/kernel-modules/build/prepare-src /scripts/prepare-src.sh COPY /collector/kernel-modules/build/build-kos /scripts/ COPY /collector/kernel-modules/build/build-wrapper.sh /scripts/compile.sh +COPY /collector/kernel-modules/probe /probe FROM rhel-8-base AS patcher diff --git a/kernel-modules/probe/Makefile b/kernel-modules/probe/Makefile new file mode 100644 index 0000000000..964c03dcf7 --- /dev/null +++ b/kernel-modules/probe/Makefile @@ -0,0 +1,53 @@ +# +# Copyright (C) 2021 The Falco Authors. +# +# This file is dual licensed under either the MIT or GPL 2. See +# MIT.txt or GPL.txt for full copies of the license. +# + +always-y += probe.o +# kept for compatibility with kernels < 5.11 +always = $(always-y) + +LLC ?= llc +CLANG ?= clang + +KERNELDIR ?= /lib/modules/$(shell uname -r)/build + +ifndef FALCO_DIR +$(error FALCO_DIR must be set) +endif + +# DEBUG = -DBPF_DEBUG + +# Begin StackRox +ifeq ($(shell grep -sq "^\s*struct\s\+audit_task_info\s\+\*audit;\s*$$" $(KERNELDIR)/include/linux/sched.h ; echo $$? ), 0) + KBUILD_CPPFLAGS+= -DCOS_73_WORKAROUND +endif +# clang-7 does not support -fmacro-prefix-map +KBUILD_CPPFLAGS:=$(filter-out -fmacro-prefix-map=%,$(KBUILD_CPPFLAGS)) +# End StackRox + +all: + $(MAKE) -C $(KERNELDIR) M=$$PWD + +clean: + $(MAKE) -C $(KERNELDIR) M=$$PWD clean + @rm -f *~ + +$(obj)/probe.o: $(src)/collector_probe.c + $(CLANG) $(LINUXINCLUDE) \ + $(KBUILD_CPPFLAGS) \ + $(KBUILD_EXTRA_CPPFLAGS) \ + $(DEBUG) \ + -I $(FALCO_DIR) \ + -D__KERNEL__ \ + -DKBUILD_MODNAME=\"collector\" \ + -D__BPF_TRACING__ \ + -Wno-gnu-variable-sized-type-not-at-end \ + -Wno-address-of-packed-member \ + -fno-jump-tables \ + -fno-stack-protector \ + -Wno-tautological-compare \ + -O2 -g -emit-llvm -c $< -o $(patsubst %.o,%.ll,$@) + $(LLC) -march=bpf -filetype=obj -o $@ $(patsubst %.o,%.ll,$@) diff --git a/kernel-modules/probe/collector_probe.c b/kernel-modules/probe/collector_probe.c new file mode 100644 index 0000000000..1beae23fdd --- /dev/null +++ b/kernel-modules/probe/collector_probe.c @@ -0,0 +1,280 @@ +#include "quirks.h" + +// This definition is set when the Kernel version is newer than +// 4.17.0. It affects the way nearly everything in the falco probe +// is processed, and is a bit of a misnomer. It might be more appropriately +// called "USING_RAW_TRACEPOINTS" +// +// Since we are *not* using raw tracepoints, we unset this definition before +// including any other files, to ensure that the behaviour and structures are +// as expected for the kinds of tracepoints we are using. +#ifdef BPF_SUPPORTS_RAW_TRACEPOINTS +# undef BPF_SUPPORTS_RAW_TRACEPOINTS +#endif + +#include +#include +#include + +// Unfortunately include order is important here, so turn clang-format +// off to avoid reordering as part of reformatting. +// clang-format off +#include "../driver_config.h" +#include "../ppm_events_public.h" +#include "bpf_helpers.h" +#include "types.h" +#include "maps.h" +#include "plumbing_helpers.h" +#include "ring_helpers.h" +#include "filler_helpers.h" +#include "fillers.h" +#include "builtins.h" +// clang-format on + +static __always_inline int enter_probe(long id, struct sys_enter_args* ctx); +static __always_inline int exit_probe(long id, struct sys_exit_args* ctx); + +#define NUM_SYS_ENTER_ARGS 6 + +/** + * @brief Encapsulates the section definition and unified function signature. This is used + * to create a new section for each eBPF program, which are then processed by + * falco to attach those programs to the specified tracepoints (based on the section name) + * + * e.g. section tracepoint/syscalls/sys_enter_accept will contain the sys_enter_accept + * program, and will be attached to tracepoint/syscalls/sys_enter_accept + * + * @param prefix the kind of tracepoint to attach to. e.g. "syscall/" or "sched/" + * @param event the event to attach to. e.g. sys_enter_accept + * @param type the type name for the context argument + */ +#define PROBE_SIGNATURE(prefix, event, type) \ + __bpf_section("tracepoint/" prefix #event) int bpf_##event(struct type* ctx) + +/** + * @brief Defines the syscall-specific enter eBPF program. + * + * @param name the syscall name. e.g. accept, chdir, execve + * @param syscall_id the ID number for the syscall. e.g. __NR_accept + */ +#define _COLLECTOR_ENTER_PROBE(name, syscall_id) \ + PROBE_SIGNATURE("syscalls/", sys_enter_##name, sys_enter_args) { \ + return enter_probe(syscall_id, ctx); \ + } + +/** + * @brief Defines the syscall-specific exit eBPF program. + * + * @param name the syscall name. e.g. accept, chdir, execve + * @param syscall_id the ID number for the syscall. e.g. __NR_accept + */ +#define _COLLECTOR_EXIT_PROBE(name, syscall_id) \ + PROBE_SIGNATURE("syscalls/", sys_exit_##name, sys_exit_args) { \ + return exit_probe(syscall_id, ctx); \ + } + +/** + * @brief Brings together the enter and exit definitions, to define all programs + * for a given syscall. + * + * @param name the syscall name. e.g. accept, chdir, execve + * @param syscall_id the ID number for the syscall. e.g. __NR_accept + */ +#define COLLECTOR_PROBE(name, syscall_id) \ + _COLLECTOR_ENTER_PROBE(name, syscall_id) \ + _COLLECTOR_EXIT_PROBE(name, syscall_id) + +COLLECTOR_PROBE(chdir, __NR_chdir); +COLLECTOR_PROBE(accept, __NR_accept); +COLLECTOR_PROBE(accept4, __NR_accept4); +COLLECTOR_PROBE(clone, __NR_clone); +COLLECTOR_PROBE(close, __NR_close); +COLLECTOR_PROBE(connect, __NR_connect); +COLLECTOR_PROBE(execve, __NR_execve); +COLLECTOR_PROBE(setresgid, __NR_setresgid); +COLLECTOR_PROBE(setresuid, __NR_setresuid); +COLLECTOR_PROBE(setgid, __NR_setgid); +COLLECTOR_PROBE(setuid, __NR_setuid); +COLLECTOR_PROBE(shutdown, __NR_shutdown); +COLLECTOR_PROBE(socket, __NR_socket); +COLLECTOR_PROBE(fchdir, __NR_fchdir); +COLLECTOR_PROBE(fork, __NR_fork); +COLLECTOR_PROBE(vfork, __NR_vfork); + +/** + * @brief program for handling sched_process_fork events. As the name suggests + * they occur when a process forks, and we get information here about + * the parent and the child. This program is purely responsible for + * stashing args for the child (which are a copy of the parent's) + * + * These stashed args are used in subsequent process events where + * these args are not available (e.g. fork). + */ +PROBE_SIGNATURE("sched/", sched_process_fork, sched_process_fork_args) { + struct sysdig_bpf_settings* settings; + enum ppm_event_type evt_type; + struct sys_stash_args args; + unsigned long* argsp; + + settings = get_bpf_settings(); + if (settings == NULL) { + return 0; + } + + if (!settings->capture_enabled) { + return 0; + } + + // using the "private" version of this function so we can + // provide a pid. + argsp = __unstash_args(ctx->parent_pid); + if (argsp == NULL) { + return 0; + } + + memcpy(&args, argsp, sizeof(args)); + + __stash_args(ctx->child_pid, args.args); + return 0; +} + +/** + * @brief program for handling sched_process_exit events. As the name suggests + * they occur when a process exits. Minimal processing is performed here + * instead, we defer to the appropriate filler. + */ +PROBE_SIGNATURE("sched/", sched_process_exit, sched_process_exit_args) { + struct sysdig_bpf_settings* settings = NULL; + enum ppm_event_type evt_type = PPME_PROCEXIT_1_E; + struct task_struct* task = NULL; + unsigned int flags = 0; + + task = (struct task_struct*)bpf_get_current_task(); + + flags = _READ(task->flags); + if ((flags & PF_KTHREAD) != 0) { + // we only want to process userspace threads. + return 0; + } + + settings = get_bpf_settings(); + if (settings == NULL) { + return 0; + } + + if (!settings->capture_enabled) { + return 0; + } + + call_filler(ctx, ctx, evt_type, settings, UF_NEVER_DROP); + return 0; +} + +/** + * @brief Generic sys_enter_* program for any system call. It is responsible for + * Verifying userspace settings and early processing of the syscall event. + * + * The function will exit 0 (zero) regardless of outcome. + * + * @param id the syscall id + * @param ctx the context pointer as provided by the kernel + * + * @return 0 (regardless of outcomes) + */ +static __always_inline int enter_probe(long id, struct sys_enter_args* ctx) { + const struct syscall_evt_pair* sc_evt = NULL; + struct sysdig_bpf_settings* settings = NULL; + enum ppm_event_type evt_type = PPME_GENERIC_E; + int drop_flags = UF_ALWAYS_DROP; + struct sys_enter_args stack_ctx = {.id = id}; + + settings = get_bpf_settings(); + if (settings == NULL) { + return 0; + } + + if (!settings->capture_enabled) { + // capture_enabled is usually false when we have loaded the probe + // but not started capturing from a userspace perspective. + return 0; + } + + sc_evt = get_syscall_info(id); + if (sc_evt == NULL) { + return 0; + } + + evt_type = sc_evt->enter_event_type; + drop_flags = sc_evt->flags; + + // To satisfy some verifier requiremnts in later parts of the falco plumbing/fillers, + // it is necessary to copy the context onto the stack. + memcpy(stack_ctx.args, _READ(ctx->args), sizeof(unsigned long) * NUM_SYS_ENTER_ARGS); + + // stashing the args will copy it into a BPF map for later + // processing. This is a required step for the enter probe, + // and these args are subsequently pulled out of the map and + // written to the ring buffer. + // + // The args pointer must exist for the lifetime of this event, so we + // must not use _READ(stack_ctx.args) here. + if (stash_args(stack_ctx.args)) { + return 0; + } + + // the fillers contain syscall specific processing logic, so we simply + // call into those and let the rest of falco deal with the event. + // + // It also handles the stack context problem, so we can pass both + // pointers through without issue. + call_filler(ctx, &stack_ctx, evt_type, settings, drop_flags); + return 0; +} + +/** + * @brief Generic sys_exit_* program for any system call. It is responsible for + * Verifying userspace settings and early processing of the syscall event. + * + * The function will exit 0 (zero) regardless of outcome. + * + * @param id the syscall id + * @param ctx the eBPF context as provided by the kernel + * + * @return 0 (regardless of outcomes) + */ +static __always_inline int exit_probe(long id, struct sys_exit_args* ctx) { + const struct syscall_evt_pair* sc_evt = NULL; + struct sysdig_bpf_settings* settings = NULL; + enum ppm_event_type evt_type = PPME_GENERIC_X; + int drop_flags = UF_ALWAYS_DROP; + + settings = get_bpf_settings(); + if (settings == NULL) { + return 0; + } + + if (!settings->capture_enabled) { + // capture_enabled is usually false when we have loaded the probe + // but not started capturing from a userspace perspective. + return 0; + } + + sc_evt = get_syscall_info(id); + if (sc_evt == NULL) { + return 0; + } + + evt_type = sc_evt->exit_event_type; + drop_flags = sc_evt->flags; + + // the fillers contain syscall specific processing logic, so we simply + // call into those and let the rest of falco deal with the event. + call_filler(ctx, ctx, evt_type, settings, drop_flags); + return 0; +} + +char kernel_ver[] __bpf_section("kernel_version") = UTS_RELEASE; + +char __license[] __bpf_section("license") = "GPL"; + +char probe_ver[] __bpf_section("probe_version") = PROBE_VERSION; From 9fc80b7423a294f1792bffecfa93e59898f444d2 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Tue, 3 May 2022 11:33:15 +0100 Subject: [PATCH 07/29] ROX-10009: fixes CI kernel module builds --- .circleci/kernels/60-build-modules.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/kernels/60-build-modules.sh b/.circleci/kernels/60-build-modules.sh index a654f8731b..5d8f5109f0 100755 --- a/.circleci/kernels/60-build-modules.sh +++ b/.circleci/kernels/60-build-modules.sh @@ -17,6 +17,7 @@ for task_file in ~/kobuild-tmp/local-build-tasks.*; do -v "${HOME}/kobuild-tmp/bundles:/bundles:ro" \ -v "${WORKSPACE_ROOT}/ko-build/module-versions:/sources:ro" \ -v "${shard_output_dir}:/output" \ + -v "${SOURCE_ROOT}/kernel-modules/probe:/probe" \ --tmpfs /scratch:exec \ "build-kernel-modules-${flavor}" \ build-wrapper.sh < "$task_file" From 1b5d22c1fce40fa5fc19ac4aad40e5f6c3c081bb Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Tue, 3 May 2022 16:48:20 +0100 Subject: [PATCH 08/29] ROX-10009: updated makefile to use upstream COS/Clang fixes --- kernel-modules/probe/Makefile | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/kernel-modules/probe/Makefile b/kernel-modules/probe/Makefile index 964c03dcf7..131be3c004 100644 --- a/kernel-modules/probe/Makefile +++ b/kernel-modules/probe/Makefile @@ -20,13 +20,23 @@ endif # DEBUG = -DBPF_DEBUG -# Begin StackRox -ifeq ($(shell grep -sq "^\s*struct\s\+audit_task_info\s\+\*audit;\s*$$" $(KERNELDIR)/include/linux/sched.h ; echo $$? ), 0) - KBUILD_CPPFLAGS+= -DCOS_73_WORKAROUND +# +# https://chromium.googlesource.com/chromiumos/third_party/kernel/+/096925a44076ba5c52faa84d255a847130ff341e%5E%21/#F2 +# This commit diverged the ChromiumOS kernel from stock in the area of audit information, which this probe accesses. +# +# This enables the workaround for this divergence. +# +NEEDS_COS_73_WORKAROUND = $(shell expr `grep -sc "^\s*struct\s\+audit_task_info\s\+\*audit;\s*$$" $(KERNELDIR)/include/linux/sched.h` = 1) +ifeq ($(NEEDS_COS_73_WORKAROUND), 1) + KBUILD_CPPFLAGS += -DCOS_73_WORKAROUND +endif + +# -fmacro-prefix-map is not supported on version of clang older than 10 +# so remove it if necessary. +IS_CLANG_OLDER_THAN_10 := $(shell expr `$(CLANG) -dumpversion | cut -f1 -d.` \<= 10) +ifeq ($(IS_CLANG_OLDER_THAN_10), 1) + KBUILD_CPPFLAGS := $(filter-out -fmacro-prefix-map=%,$(KBUILD_CPPFLAGS)) endif -# clang-7 does not support -fmacro-prefix-map -KBUILD_CPPFLAGS:=$(filter-out -fmacro-prefix-map=%,$(KBUILD_CPPFLAGS)) -# End StackRox all: $(MAKE) -C $(KERNELDIR) M=$$PWD From 279f7b4dd6471d5e0106a2f7c8a0170de27079ff Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Tue, 3 May 2022 16:52:14 +0100 Subject: [PATCH 09/29] ROX-10009: added stack context to exit probe to satisfy eBPF verifier --- kernel-modules/probe/collector_probe.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel-modules/probe/collector_probe.c b/kernel-modules/probe/collector_probe.c index 1beae23fdd..beca63baf8 100644 --- a/kernel-modules/probe/collector_probe.c +++ b/kernel-modules/probe/collector_probe.c @@ -247,6 +247,7 @@ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx) { struct sysdig_bpf_settings* settings = NULL; enum ppm_event_type evt_type = PPME_GENERIC_X; int drop_flags = UF_ALWAYS_DROP; + struct sys_exit_args stack_ctx = {.id = id}; settings = get_bpf_settings(); if (settings == NULL) { @@ -267,9 +268,16 @@ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx) { evt_type = sc_evt->exit_event_type; drop_flags = sc_evt->flags; + // As with the enter probe, we copy the context onto the stack + // to satisfy verification in later parts of the codebase. + stack_ctx.ret = _READ(ctx->ret); + // the fillers contain syscall specific processing logic, so we simply // call into those and let the rest of falco deal with the event. - call_filler(ctx, ctx, evt_type, settings, drop_flags); + // + // It also handles the stack context problem, so we can pass both + // pointers through without issue. + call_filler(ctx, &stack_ctx, evt_type, settings, drop_flags); return 0; } From e5da6b41f23608d353d93dac40b7fb531d4ee6d2 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Wed, 4 May 2022 09:39:37 +0100 Subject: [PATCH 10/29] ROX-10009: reverts some attempted fixes for verifier problems --- kernel-modules/probe/Makefile | 10 ++++------ kernel-modules/probe/collector_probe.c | 10 +--------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/kernel-modules/probe/Makefile b/kernel-modules/probe/Makefile index 131be3c004..2efeb6cb09 100644 --- a/kernel-modules/probe/Makefile +++ b/kernel-modules/probe/Makefile @@ -1,9 +1,7 @@ -# -# Copyright (C) 2021 The Falco Authors. -# -# This file is dual licensed under either the MIT or GPL 2. See -# MIT.txt or GPL.txt for full copies of the license. -# +# This file is largely duplicated from the Falco version, adjusted slightly for our specific +# source file and includes. +# See here for the original: +# https://github.com/falcosecurity/libs/blob/2c8335ae64f1e775def9f9f7c97e5340319c85f0/driver/bpf/Makefile always-y += probe.o # kept for compatibility with kernels < 5.11 diff --git a/kernel-modules/probe/collector_probe.c b/kernel-modules/probe/collector_probe.c index beca63baf8..1beae23fdd 100644 --- a/kernel-modules/probe/collector_probe.c +++ b/kernel-modules/probe/collector_probe.c @@ -247,7 +247,6 @@ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx) { struct sysdig_bpf_settings* settings = NULL; enum ppm_event_type evt_type = PPME_GENERIC_X; int drop_flags = UF_ALWAYS_DROP; - struct sys_exit_args stack_ctx = {.id = id}; settings = get_bpf_settings(); if (settings == NULL) { @@ -268,16 +267,9 @@ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx) { evt_type = sc_evt->exit_event_type; drop_flags = sc_evt->flags; - // As with the enter probe, we copy the context onto the stack - // to satisfy verification in later parts of the codebase. - stack_ctx.ret = _READ(ctx->ret); - // the fillers contain syscall specific processing logic, so we simply // call into those and let the rest of falco deal with the event. - // - // It also handles the stack context problem, so we can pass both - // pointers through without issue. - call_filler(ctx, &stack_ctx, evt_type, settings, drop_flags); + call_filler(ctx, ctx, evt_type, settings, drop_flags); return 0; } From a9b6c4ee5d06c4baf51c483e88afe06964d5553c Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Wed, 4 May 2022 11:05:07 +0100 Subject: [PATCH 11/29] ROX-10009: bumps falco commit to latest probe, without duplicate collector_probe.c impl# --- kernel-modules/build/build-kos | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/kernel-modules/build/build-kos b/kernel-modules/build/build-kos index a32ddac302..e3f82f1eb4 100755 --- a/kernel-modules/build/build-kos +++ b/kernel-modules/build/build-kos @@ -68,6 +68,11 @@ build_ko() ( return 1 } + # directory in which the probe source lives. + probe_directory="/probe/" + # resulting object file from the build + collector_probe="/probe/probe.o" + # Check if this module version supports RHEL 7.6 with backported eBPF support if [[ "$rhel7_kernel_with_ebpf" == true ]]; then if ! grep -qRIs "SUPPORTS_RHEL76_EBPF" "${module_src_dir}/bpf/quirks.h"; then @@ -82,25 +87,21 @@ build_ko() ( # for the collector probe (specifically clone, execve, fork, and vfork) # so we just build the falco catch-all probe for this platform # - - echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." - KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" make -j 6 -C bpf - collector_probe="bpf/probe.o" - - # Check kernel version is at least 4.14 (unless RHEL 7.6 kernel detected) + probe_directory="$(pwd)/bpf" + collector_probe="$(pwd)/bpf/probe.o" else + # Check kernel version is at least 4.14 (unless RHEL 7.6 kernel detected) if ((bundle_version < 4 || (bundle_version == 4 && bundle_major < 14))); then echo "Kernel version ${kernel_version} does not support eBPF probe building, skipping ..." mkdir -p "${MODULE_BASE_DIR}/${module_version}" touch "${MODULE_BASE_DIR}/${module_version}/.collector-ebpf-${kernel_version}.unavail" return 0 fi - - echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." - KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" FALCO_DIR="$(pwd)/bpf" make -j 6 -C /probe - collector_probe="/probe/probe.o" fi + echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." + KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" FALCO_DIR="$(pwd)/bpf" make -j 6 -C "${probe_directory}" + if [[ ! -f "$collector_probe" || ! -s "$collector_probe" ]]; then echo "Empty or missing compiled bpf output" return 1 From 3c2c555ba128eaa31f09b88221d97045b3b37cbd Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Fri, 6 May 2022 09:52:05 +0100 Subject: [PATCH 12/29] ROX-10009: adds sys_enter/sys_exit support for RHEL-7 --- kernel-modules/build/build-kos | 11 ++--- kernel-modules/dockerized/Dockerfile | 1 + kernel-modules/probe/collector_probe.c | 60 ++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/kernel-modules/build/build-kos b/kernel-modules/build/build-kos index e3f82f1eb4..8f79b94069 100755 --- a/kernel-modules/build/build-kos +++ b/kernel-modules/build/build-kos @@ -20,7 +20,9 @@ build_ko() ( local rhel7_kernel_with_ebpf=false if ((bundle_version == 3 && bundle_major >= 10)); then + # shellcheck disable=SC2154 if [[ "$bundle_distro" == "redhat" ]]; then + # shellcheck disable=SC2154 rhel_build_id="$(echo "$bundle_uname" | awk -F'[-.]' '{ print $4 }')" if ((rhel_build_id >= 957)); then echo "Kernel ${bundle_uname} has backported eBPF support" @@ -32,6 +34,7 @@ build_ko() ( if [[ "$probe_type" == "mod" ]]; then # Attempting to run modpost will fail if it requires glibc version newer than # available in this distro. We skip building such kernel drivers for now. + # shellcheck disable=SC2154 if ((DOCKERIZED)) && ! "${kernel_build_dir}/scripts/mod/modpost"; then echo >&2 "Failed to run kbuild tools, skipping module for ${kernel_version}" return 1 @@ -81,14 +84,6 @@ build_ko() ( touch "${MODULE_BASE_DIR}/${module_version}/.collector-ebpf-${kernel_version}.unavail" return 0 fi - - # - # Unfortunately RHEL-7 does not have the necessary tracepoints that we require - # for the collector probe (specifically clone, execve, fork, and vfork) - # so we just build the falco catch-all probe for this platform - # - probe_directory="$(pwd)/bpf" - collector_probe="$(pwd)/bpf/probe.o" else # Check kernel version is at least 4.14 (unless RHEL 7.6 kernel detected) if ((bundle_version < 4 || (bundle_version == 4 && bundle_major < 14))); then diff --git a/kernel-modules/dockerized/Dockerfile b/kernel-modules/dockerized/Dockerfile index 2bf973fc83..0aea7ad7fc 100644 --- a/kernel-modules/dockerized/Dockerfile +++ b/kernel-modules/dockerized/Dockerfile @@ -107,6 +107,7 @@ COPY /collector/kernel-modules/dockerized/scripts /scripts COPY /collector/kernel-modules/build/prepare-src /scripts/prepare-src.sh COPY /collector/kernel-modules/build/build-kos /scripts/ COPY /collector/kernel-modules/build/build-wrapper.sh /scripts/compile.sh +COPY /collector/kernel-modules/probe/ /probe FROM rhel-7-base AS rhel-7-builder diff --git a/kernel-modules/probe/collector_probe.c b/kernel-modules/probe/collector_probe.c index 1beae23fdd..3c29700f6f 100644 --- a/kernel-modules/probe/collector_probe.c +++ b/kernel-modules/probe/collector_probe.c @@ -34,8 +34,15 @@ static __always_inline int enter_probe(long id, struct sys_enter_args* ctx); static __always_inline int exit_probe(long id, struct sys_exit_args* ctx); +// defines the maximum number of args to copy from the context +// to the stack. Matches up to the structure defined in types.h #define NUM_SYS_ENTER_ARGS 6 +// this can be passed to the enter or exit functions as the ID value +// to pull the ID from the context. This is specifically useful when +// we need to use the sys_enter/sys_exit tracepoints. +#define LOOKUP_SYSCALL_ID -1 + /** * @brief Encapsulates the section definition and unified function signature. This is used * to create a new section for each eBPF program, which are then processed by @@ -73,6 +80,24 @@ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx); return exit_probe(syscall_id, ctx); \ } +/** + * @brief Defines the catch-all sys_enter program for older platforms + * that do not support all the tracepoints we need. + */ +#define _COLLECTOR_SYS_ENTER_PROBE \ + PROBE_SIGNATURE("raw_syscalls/", sys_enter, sys_enter_args) { \ + return enter_probe(LOOKUP_SYSCALL_ID, ctx); \ + } + +/** + * @brief Defines the catch-all sys_exit program for older platforms + * that do not support all the tracepoints we need. + */ +#define _COLLECTOR_SYS_EXIT_PROBE \ + PROBE_SIGNATURE("raw_syscalls/", sys_exit, sys_exit_args) { \ + return exit_probe(LOOKUP_SYSCALL_ID, ctx); \ + } + /** * @brief Brings together the enter and exit definitions, to define all programs * for a given syscall. @@ -84,6 +109,19 @@ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx); _COLLECTOR_ENTER_PROBE(name, syscall_id) \ _COLLECTOR_EXIT_PROBE(name, syscall_id) +/** + * @brief brings together the sys_enter and sys_exit probes for legacy platforms + * that do not support all the necessary tracepoints. + */ +#define COLLECTOR_LEGACY_PROBE() \ + _COLLECTOR_SYS_ENTER_PROBE; \ + _COLLECTOR_SYS_EXIT_PROBE + +// this if statement relies on short circuiting to simplify the definition +// of the tracepoints. i.e. RHEL_RELEASE_VERSION will not be defined unless +// RHEL_RELEASE_CODE is defined. +#if !defined(RHEL_RELEASE_CODE) || RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8, 0) + COLLECTOR_PROBE(chdir, __NR_chdir); COLLECTOR_PROBE(accept, __NR_accept); COLLECTOR_PROBE(accept4, __NR_accept4); @@ -101,6 +139,15 @@ COLLECTOR_PROBE(fchdir, __NR_fchdir); COLLECTOR_PROBE(fork, __NR_fork); COLLECTOR_PROBE(vfork, __NR_vfork); +#else + +// Unfortunately RHEL-7 does not have the necessary tracepoints that we require +// for the collector probe (specifically clone, execve, fork, and vfork) +// so we just build the falco catch-all probe for this platform +COLLECTOR_LEGACY_PROBE(); + +#endif + /** * @brief program for handling sched_process_fork events. As the name suggests * they occur when a process forks, and we get information here about @@ -199,6 +246,13 @@ static __always_inline int enter_probe(long id, struct sys_enter_args* ctx) { return 0; } + if (id == LOOKUP_SYSCALL_ID) { + // this is to support sys_enter and sys_exit probes for older (RHEL 7) + // platforms. Just get the id from the context for this scenario. + id = bpf_syscall_get_nr(ctx); + stack_ctx.id = id; + } + sc_evt = get_syscall_info(id); if (sc_evt == NULL) { return 0; @@ -259,6 +313,12 @@ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx) { return 0; } + if (id == LOOKUP_SYSCALL_ID) { + // this is to support sys_enter and sys_exit probes for legacy (RHEL 7) + // platforms. Just get the id from the context for this scenario. + id = bpf_syscall_get_nr(ctx); + } + sc_evt = get_syscall_info(id); if (sc_evt == NULL) { return 0; From 3b1ea778ca3e089852c2929da7582b64f9b24665 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Fri, 6 May 2022 10:07:32 +0100 Subject: [PATCH 13/29] ROX-10009: ensures compatibility of enter/exit programs regardless of tracepoints they attach to --- kernel-modules/probe/collector_probe.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel-modules/probe/collector_probe.c b/kernel-modules/probe/collector_probe.c index 3c29700f6f..6cb5ae33f2 100644 --- a/kernel-modules/probe/collector_probe.c +++ b/kernel-modules/probe/collector_probe.c @@ -254,7 +254,7 @@ static __always_inline int enter_probe(long id, struct sys_enter_args* ctx) { } sc_evt = get_syscall_info(id); - if (sc_evt == NULL) { + if (sc_evt == NULL || (sc_evt->flags & UF_USED) == 0) { return 0; } @@ -320,7 +320,7 @@ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx) { } sc_evt = get_syscall_info(id); - if (sc_evt == NULL) { + if (sc_evt == NULL || (sc_evt->flags & UF_USED) == 0) { return 0; } From 8638fa7e9a40eaa4dd76b03aeb862e1281449497 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Fri, 6 May 2022 13:32:26 +0100 Subject: [PATCH 14/29] ROX-10009: updated docs about userspace event filtering --- collector/lib/SysdigService.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/collector/lib/SysdigService.cpp b/collector/lib/SysdigService.cpp index 786df938fd..1dfde03e97 100644 --- a/collector/lib/SysdigService.cpp +++ b/collector/lib/SysdigService.cpp @@ -132,11 +132,16 @@ sinsp_evt* SysdigService::GetNext() { if (event->get_category() & EC_INTERNAL) return nullptr; - // TODO (rc) not needed when syscall filtering implemented in the eBPF probe - // if (useEbpf && !global_event_filter_[event->get_type()]) { - // CLOG(DEBUG) << "filtering cos ebpf?"; - // return nullptr; - // } + // This additional userspace filter is a guard against additional events + // from the eBPF probe. This can occur when using sys_enter and sys_exit + // tracepoints rather than a targeted approach. + // + // When using targeted tracepoints this filter has no effect because + // the only events that the probe produces are of the types that we want to + // process. + if (useEbpf && !global_event_filter_[event->get_type()]) { + return nullptr; + } userspace_stats_.event_parse_micros[event->get_type()] += (NowMicros() - parse_start); ++userspace_stats_.nUserspaceEvents[event->get_type()]; From cd3d50b6482dc380d6a9a0b6d1b0a493d11c4039 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Fri, 6 May 2022 14:30:14 +0100 Subject: [PATCH 15/29] ROX-10009: adds clang-format file for ebpf probe --- kernel-modules/probe/.clang-format | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 kernel-modules/probe/.clang-format diff --git a/kernel-modules/probe/.clang-format b/kernel-modules/probe/.clang-format new file mode 100644 index 0000000000..d17114b043 --- /dev/null +++ b/kernel-modules/probe/.clang-format @@ -0,0 +1,25 @@ +BasedOnStyle: Google + +ColumnLimit: 0 + +DerivePointerAlignment: false +PointerAlignment: Left + +IndentPPDirectives: AfterHash + +IncludeBlocks: Regroup +IncludeCategories: +- Regex: '^((\"|<)(api|internalapi|storage)/)' + Priority: 45 +- Regex: '^((\"|<)(grpc|grpcpp|google)/)' + Priority: 44 +- Regex: '^((\"|<)(libsinsp|libscap)/)' + Priority: 43 +- Regex: '^<.*/.*' + Priority: 20 +- Regex: '^<[^/]*>' + Priority: 10 +- Regex: '^\"[a-z].*' + Priority: 49 +- Regex: '^\"' + Priority: 50 From 42fe11c6c0a6112cad9e8c1b085264cb437a6f74 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 16 May 2022 11:43:43 +0100 Subject: [PATCH 16/29] ROX-10009: changes to support latest falco. --- kernel-modules/probe/collector_probe.c | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/kernel-modules/probe/collector_probe.c b/kernel-modules/probe/collector_probe.c index 6cb5ae33f2..e7730c4331 100644 --- a/kernel-modules/probe/collector_probe.c +++ b/kernel-modules/probe/collector_probe.c @@ -158,7 +158,7 @@ COLLECTOR_LEGACY_PROBE(); * these args are not available (e.g. fork). */ PROBE_SIGNATURE("sched/", sched_process_fork, sched_process_fork_args) { - struct sysdig_bpf_settings* settings; + struct scap_bpf_settings* settings; enum ppm_event_type evt_type; struct sys_stash_args args; unsigned long* argsp; @@ -191,7 +191,7 @@ PROBE_SIGNATURE("sched/", sched_process_fork, sched_process_fork_args) { * instead, we defer to the appropriate filler. */ PROBE_SIGNATURE("sched/", sched_process_exit, sched_process_exit_args) { - struct sysdig_bpf_settings* settings = NULL; + struct scap_bpf_settings* settings = NULL; enum ppm_event_type evt_type = PPME_PROCEXIT_1_E; struct task_struct* task = NULL; unsigned int flags = 0; @@ -230,11 +230,15 @@ PROBE_SIGNATURE("sched/", sched_process_exit, sched_process_exit_args) { */ static __always_inline int enter_probe(long id, struct sys_enter_args* ctx) { const struct syscall_evt_pair* sc_evt = NULL; - struct sysdig_bpf_settings* settings = NULL; + struct scap_bpf_settings* settings = NULL; enum ppm_event_type evt_type = PPME_GENERIC_E; int drop_flags = UF_ALWAYS_DROP; struct sys_enter_args stack_ctx = {.id = id}; + if (bpf_in_ia32_syscall()) { + return 0; + } + settings = get_bpf_settings(); if (settings == NULL) { return 0; @@ -298,10 +302,14 @@ static __always_inline int enter_probe(long id, struct sys_enter_args* ctx) { */ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx) { const struct syscall_evt_pair* sc_evt = NULL; - struct sysdig_bpf_settings* settings = NULL; + struct scap_bpf_settings* settings = NULL; enum ppm_event_type evt_type = PPME_GENERIC_X; int drop_flags = UF_ALWAYS_DROP; + if (bpf_in_ia32_syscall()) { + return 0; + } + settings = get_bpf_settings(); if (settings == NULL) { return 0; @@ -337,4 +345,10 @@ char kernel_ver[] __bpf_section("kernel_version") = UTS_RELEASE; char __license[] __bpf_section("license") = "GPL"; -char probe_ver[] __bpf_section("probe_version") = PROBE_VERSION; +char probe_ver[] __bpf_section("probe_version") = DRIVER_VERSION; + +char probe_commit[] __bpf_section("build_commit") = DRIVER_COMMIT; + +uint64_t probe_api_ver __bpf_section("api_version") = PPM_API_CURRENT_VERSION; + +uint64_t probe_schema_ver __bpf_section("schema_version") = PPM_SCHEMA_CURRENT_VERSION; From ed22056a10c4051dbd4416233011dbaf8ace86b6 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Tue, 24 May 2022 15:38:13 +0100 Subject: [PATCH 17/29] ROX-10009: temp comment out of userspace filtering --- collector/lib/SysdigService.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/collector/lib/SysdigService.cpp b/collector/lib/SysdigService.cpp index 1dfde03e97..4942953900 100644 --- a/collector/lib/SysdigService.cpp +++ b/collector/lib/SysdigService.cpp @@ -139,9 +139,9 @@ sinsp_evt* SysdigService::GetNext() { // When using targeted tracepoints this filter has no effect because // the only events that the probe produces are of the types that we want to // process. - if (useEbpf && !global_event_filter_[event->get_type()]) { - return nullptr; - } + // if (useEbpf && !global_event_filter_[event->get_type()]) { + // return nullptr; + // } userspace_stats_.event_parse_micros[event->get_type()] += (NowMicros() - parse_start); ++userspace_stats_.nUserspaceEvents[event->get_type()]; From 5c0bf073c19f11a9ed7edac33fbcbd9af805b00f Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Wed, 25 May 2022 09:27:36 +0100 Subject: [PATCH 18/29] ROX-10009: updates to latest falco, and sets max progs to sensible limit for the custom probe --- collector/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/collector/CMakeLists.txt b/collector/CMakeLists.txt index ebe6213937..f227c3ac7d 100644 --- a/collector/CMakeLists.txt +++ b/collector/CMakeLists.txt @@ -107,5 +107,6 @@ file(STRINGS "${PROJECT_SOURCE_DIR}/../kernel-modules/MODULE_VERSION" DRIVER_VER set(DRIVER_VERSION "${DRIVER_VERSION}" CACHE STRING "Falco driver version" FORCE) set(DRIVER_DEVICE_NAME "${DRIVER_NAME}" CACHE STRING "Driver device name" FORCE) +set(SCAP_BPF_PROGS_MAX "256" CACHE STRING "Number of BPF progams supported in Falco" FORCE) add_subdirectory(${PROJECT_SOURCE_DIR}/falcosecurity-libs EXCLUDE_FROM_DEFAULT_BUILD) From e30a90aed8d5ab520413a3e3ddba7bf549dd5118 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 30 May 2022 07:57:56 +0100 Subject: [PATCH 19/29] ROX-10009: consolidates clang-format config files into repository root directory --- collector/.clang-format => .clang-format | 0 kernel-modules/probe/.clang-format | 25 ------------------------ 2 files changed, 25 deletions(-) rename collector/.clang-format => .clang-format (100%) delete mode 100644 kernel-modules/probe/.clang-format diff --git a/collector/.clang-format b/.clang-format similarity index 100% rename from collector/.clang-format rename to .clang-format diff --git a/kernel-modules/probe/.clang-format b/kernel-modules/probe/.clang-format deleted file mode 100644 index d17114b043..0000000000 --- a/kernel-modules/probe/.clang-format +++ /dev/null @@ -1,25 +0,0 @@ -BasedOnStyle: Google - -ColumnLimit: 0 - -DerivePointerAlignment: false -PointerAlignment: Left - -IndentPPDirectives: AfterHash - -IncludeBlocks: Regroup -IncludeCategories: -- Regex: '^((\"|<)(api|internalapi|storage)/)' - Priority: 45 -- Regex: '^((\"|<)(grpc|grpcpp|google)/)' - Priority: 44 -- Regex: '^((\"|<)(libsinsp|libscap)/)' - Priority: 43 -- Regex: '^<.*/.*' - Priority: 20 -- Regex: '^<[^/]*>' - Priority: 10 -- Regex: '^\"[a-z].*' - Priority: 49 -- Regex: '^\"' - Priority: 50 From 62ed696f7dd05bc9a02f3d5449dcf1d1841fbcb2 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 30 May 2022 08:00:37 +0100 Subject: [PATCH 20/29] ROX-10009: ensures that global_event_filter is only enabled on RHEL7 eBPF --- collector/lib/SysdigService.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/collector/lib/SysdigService.cpp b/collector/lib/SysdigService.cpp index 4942953900..509dc073db 100644 --- a/collector/lib/SysdigService.cpp +++ b/collector/lib/SysdigService.cpp @@ -31,6 +31,7 @@ You should have received a copy of the GNU General Public License along with thi #include "CollectorException.h" #include "EventNames.h" +#include "HostInfo.h" #include "Logging.h" #include "NetworkSignalHandler.h" #include "ProcessSignalHandler.h" @@ -132,16 +133,15 @@ sinsp_evt* SysdigService::GetNext() { if (event->get_category() & EC_INTERNAL) return nullptr; + HostInfo& host_info = HostInfo::Instance(); + // This additional userspace filter is a guard against additional events // from the eBPF probe. This can occur when using sys_enter and sys_exit - // tracepoints rather than a targeted approach. - // - // When using targeted tracepoints this filter has no effect because - // the only events that the probe produces are of the types that we want to - // process. - // if (useEbpf && !global_event_filter_[event->get_type()]) { - // return nullptr; - // } + // tracepoints rather than a targeted approach, which we currently only do + // on RHEL7 with backported eBPF + if (useEbpf && host_info.IsRHEL76() && !global_event_filter_[event->get_type()]) { + return nullptr; + } userspace_stats_.event_parse_micros[event->get_type()] += (NowMicros() - parse_start); ++userspace_stats_.nUserspaceEvents[event->get_type()]; From c4f9ef295f2b63823e10495e2e71fc6dfbad52da Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 30 May 2022 08:01:13 +0100 Subject: [PATCH 21/29] ROX-10009: minor tweak to kernel modules build to be clearer about artefacts and paths used in ebpf build --- kernel-modules/build/build-kos | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel-modules/build/build-kos b/kernel-modules/build/build-kos index 8f79b94069..e0ed3f59a5 100755 --- a/kernel-modules/build/build-kos +++ b/kernel-modules/build/build-kos @@ -71,11 +71,6 @@ build_ko() ( return 1 } - # directory in which the probe source lives. - probe_directory="/probe/" - # resulting object file from the build - collector_probe="/probe/probe.o" - # Check if this module version supports RHEL 7.6 with backported eBPF support if [[ "$rhel7_kernel_with_ebpf" == true ]]; then if ! grep -qRIs "SUPPORTS_RHEL76_EBPF" "${module_src_dir}/bpf/quirks.h"; then @@ -95,7 +90,10 @@ build_ko() ( fi echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." - KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" FALCO_DIR="$(pwd)/bpf" make -j 6 -C "${probe_directory}" + KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" FALCO_DIR="$(pwd)/bpf" make -j 6 -C "/probe/" + + # resulting object file from the build + collector_probe="/probe/probe.o" if [[ ! -f "$collector_probe" || ! -s "$collector_probe" ]]; then echo "Empty or missing compiled bpf output" From 4eeff51c3fbf3dd35bddeb9a2076c2a2ce9096fb Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 30 May 2022 16:10:39 +0100 Subject: [PATCH 22/29] ROX-10009: updated dev full build to build custom probe --- kernel-modules/dev/build-drivers.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/kernel-modules/dev/build-drivers.sh b/kernel-modules/dev/build-drivers.sh index 1a2c7734fc..fc5aa1c042 100755 --- a/kernel-modules/dev/build-drivers.sh +++ b/kernel-modules/dev/build-drivers.sh @@ -5,22 +5,23 @@ OUTPUT_DIR="/collector/kernel-modules/container/kernel-modules" package_kmod() { kernel=$1 - driver_dir=$2 + probe_object=$2 - gzip -c "${driver_dir}/build/driver/collector.ko" \ + gzip -c "${probe_object}" \ > "${OUTPUT_DIR}/collector-${kernel}.ko.gz" } package_probe() { kernel=$1 - driver_dir=$2 + probe_object=$2 - gzip -c "${driver_dir}/build/driver/bpf/probe.o" \ - > "${OUTPUT_DIR}/collector-ebpf-${kernel}.o.gz" + gzip -c "${probe_object}" \ + > "${OUTPUT_DIR}/collector-ebpf-${kernel}.ko.gz" } KERNEL_VERSION="$(uname -r)" DRIVER_DIR="/collector/falcosecurity-libs" +PROBE_DIR="/collector/kernel-modules/probe" mkdir -p "${DRIVER_DIR}/build" @@ -32,13 +33,13 @@ cmake -S ${DRIVER_DIR} \ -DBUILD_USERSPACE=OFF \ -DBUILD_DRIVER=ON \ -DENABLE_DKMS=OFF \ - -DBUILD_BPF=ON \ -B ${DRIVER_DIR}/build make -C ${DRIVER_DIR}/build/driver +make -C ${PROBE_DIR} FALCO_DIR="${DRIVER_DIR}/driver/bpf" mkdir -p "${OUTPUT_DIR}" -package_kmod "$KERNEL_VERSION" "$DRIVER_DIR" -package_probe "$KERNEL_VERSION" "$DRIVER_DIR" +package_kmod "$KERNEL_VERSION" "$DRIVER_DIR/build/driver/collector.ko" +package_probe "$KERNEL_VERSION" "$PROBE_DIR/probe.o" # No reason to leave this hanging about rm -rf "${DRIVER_DIR}/build" From 305b50434ca72212b3a126537ecf9f7a7b260a87 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Wed, 1 Jun 2022 11:05:40 +0100 Subject: [PATCH 23/29] ROX-10009: fixes copy/paste error in dev scripts that made ebpf probe .ko.gz --- kernel-modules/dev/build-drivers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel-modules/dev/build-drivers.sh b/kernel-modules/dev/build-drivers.sh index fc5aa1c042..4b3aec4be9 100755 --- a/kernel-modules/dev/build-drivers.sh +++ b/kernel-modules/dev/build-drivers.sh @@ -16,7 +16,7 @@ package_probe() { probe_object=$2 gzip -c "${probe_object}" \ - > "${OUTPUT_DIR}/collector-ebpf-${kernel}.ko.gz" + > "${OUTPUT_DIR}/collector-ebpf-${kernel}.o.gz" } KERNEL_VERSION="$(uname -r)" From 238d060193499f06f44454fd84cee0e58edd720f Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Wed, 1 Jun 2022 15:17:23 +0100 Subject: [PATCH 24/29] ROX-10009: ensures that uninteresting events are ignored by the probe --- kernel-modules/probe/collector_probe.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel-modules/probe/collector_probe.c b/kernel-modules/probe/collector_probe.c index e7730c4331..9fb707dc03 100644 --- a/kernel-modules/probe/collector_probe.c +++ b/kernel-modules/probe/collector_probe.c @@ -258,7 +258,7 @@ static __always_inline int enter_probe(long id, struct sys_enter_args* ctx) { } sc_evt = get_syscall_info(id); - if (sc_evt == NULL || (sc_evt->flags & UF_USED) == 0) { + if (sc_evt == NULL || (sc_evt->flags & UF_USED) == 0 || (sc_evt->flags & UF_UNINTERESTING) != 0) { return 0; } @@ -328,7 +328,7 @@ static __always_inline int exit_probe(long id, struct sys_exit_args* ctx) { } sc_evt = get_syscall_info(id); - if (sc_evt == NULL || (sc_evt->flags & UF_USED) == 0) { + if (sc_evt == NULL || (sc_evt->flags & UF_USED) == 0 || (sc_evt->flags & UF_UNINTERESTING) != 0) { return 0; } From 19b6c5a2a75bcc2f31cb6591ad4e08246d711d63 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Wed, 1 Jun 2022 20:27:29 +0100 Subject: [PATCH 25/29] ROX-10009: sets correct falco commit... --- falcosecurity-libs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcosecurity-libs b/falcosecurity-libs index b03aa42bd9..75dd1e36bc 160000 --- a/falcosecurity-libs +++ b/falcosecurity-libs @@ -1 +1 @@ -Subproject commit b03aa42bd9f6512bab0157ad7d1e4efad13b6fac +Subproject commit 75dd1e36bc185337d2dc0bc83df831a533c3eaf3 From c595d5a52567e496c79a9717dc3be4af0057c4de Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Thu, 2 Jun 2022 06:57:48 +0100 Subject: [PATCH 26/29] ROX-10009: temporarily bump parallelism in CI --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe8f685f51..2889a0405b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ parameters: default: "quay.io/rhacs-eng" dockerized-parallel: type: integer - default: 8 + default: 32 # TODO: revert before merging dockerized-cache-tag: type: string default: cache-v2 @@ -367,7 +367,7 @@ jobs: kernels: <<: *defaultMachine - parallelism: 8 + parallelism: 32 # TODO: revert before merging environment: - BUILD_CONTAINER_TAG: quay.io/rhacs-eng/collector-builder:kobuilder-cache - BUILD_CONTAINER_CACHE_IMAGES: quay.io/rhacs-eng/collector-builder:kobuilder-cache From e4397a67b16106b142cd7c17790659f2242b7bd9 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Thu, 2 Jun 2022 11:18:53 +0100 Subject: [PATCH 27/29] ROX-10009: fixes legacy builds by making custom probe part of the source archive --- kernel-modules/build/build-kos | 13 +++++++++---- kernel-modules/build/prepare-src | 8 ++++++++ kernel-modules/dockerized/Dockerfile | 1 - 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/kernel-modules/build/build-kos b/kernel-modules/build/build-kos index e0ed3f59a5..800dba51be 100755 --- a/kernel-modules/build/build-kos +++ b/kernel-modules/build/build-kos @@ -89,11 +89,16 @@ build_ko() ( fi fi - echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." - KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" FALCO_DIR="$(pwd)/bpf" make -j 6 -C "/probe/" + collector_probe="${module_src_dir}/bpf/probe.o" + probe_dir="${module_src_dir}/bpf" + + if [[ -d "${module_src_dir}/collector-probe" ]]; then + collector_probe="${module_src_dir}/collector-probe/probe.o" + probe_dir="${module_src_dir}/collector-probe" + fi - # resulting object file from the build - collector_probe="/probe/probe.o" + echo "Building collector eBPF probe for kernel version ${kernel_version} and module version ${module_version}." + KERNELDIR="$kernel_build_dir" BUILD_ROOT="${KERNEL_SRC_DIR}" FALCO_DIR="$(pwd)/bpf" make -j 6 -C "${probe_dir}" if [[ ! -f "$collector_probe" || ! -s "$collector_probe" ]]; then echo "Empty or missing compiled bpf output" diff --git a/kernel-modules/build/prepare-src b/kernel-modules/build/prepare-src index 5bff9479f7..cb20785481 100755 --- a/kernel-modules/build/prepare-src +++ b/kernel-modules/build/prepare-src @@ -11,6 +11,8 @@ DOCKERIZED=${DOCKERIZED:-0} DRIVER_SCRATCH="${SCRATCH_DIR}/src" BUILD_SCRATCH="${SCRATCH_DIR}/cmake-build" +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}")" &> /dev/null && pwd 2> /dev/null)" + modfiles0() ( local dir="${1:-${DRIVER_SCRATCH}/driver}" cd "$dir" @@ -88,6 +90,12 @@ cmake \ cp "${BUILD_SCRATCH}/driver/src/Makefile" "${DRIVER_SCRATCH}/driver/Makefile" cp "${BUILD_SCRATCH}/driver/src/driver_config.h" "${DRIVER_SCRATCH}/driver/driver_config.h" +if [[ -d "${SCRIPT_DIR}/../probe" ]]; then + mkdir -p "${DRIVER_SCRATCH}/driver/collector-probe" + cp "${SCRIPT_DIR}/../probe/Makefile" "${DRIVER_SCRATCH}/driver/collector-probe/Makefile" + cp "${SCRIPT_DIR}/../probe/collector_probe.c" "${DRIVER_SCRATCH}/driver/collector-probe/collector_probe.c" +fi + cd "${DRIVER_SCRATCH}/driver" module_version="$(get_module_version .)" echo "$module_version" diff --git a/kernel-modules/dockerized/Dockerfile b/kernel-modules/dockerized/Dockerfile index 0aea7ad7fc..5251d8acee 100644 --- a/kernel-modules/dockerized/Dockerfile +++ b/kernel-modules/dockerized/Dockerfile @@ -39,7 +39,6 @@ COPY /collector/kernel-modules/dockerized/scripts /scripts COPY /collector/kernel-modules/build/prepare-src /scripts/prepare-src.sh COPY /collector/kernel-modules/build/build-kos /scripts/ COPY /collector/kernel-modules/build/build-wrapper.sh /scripts/compile.sh -COPY /collector/kernel-modules/probe /probe FROM rhel-8-base AS patcher From ec935aed30163fd0355db8a8203e6125357c9466 Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Thu, 2 Jun 2022 13:19:45 +0100 Subject: [PATCH 28/29] ROX-10009: passed LEGACY_DIR to prepare-src to determine existence of custom probe code --- .circleci/prepare-kernels/10-prepare-source.sh | 1 + .circleci/prepare-kernels/20-prepare-legacy-sources.sh | 6 +++++- kernel-modules/build/prepare-src | 10 ++++------ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.circleci/prepare-kernels/10-prepare-source.sh b/.circleci/prepare-kernels/10-prepare-source.sh index d9db1e521c..694bf093fa 100755 --- a/.circleci/prepare-kernels/10-prepare-source.sh +++ b/.circleci/prepare-kernels/10-prepare-source.sh @@ -6,6 +6,7 @@ DRIVER_DIR="${SOURCE_ROOT}/falcosecurity-libs" \ SCRATCH_DIR="${HOME}/scratch" \ OUTPUT_DIR="${HOME}/kobuild-tmp/versions-src" \ M_VERSION="$(cat "${SOURCE_ROOT}/kernel-modules/MODULE_VERSION")" \ + LEGACY_DIR="${SOURCE_ROOT}" \ "${SOURCE_ROOT}/kernel-modules/build/prepare-src" versions=(~/kobuild-tmp/versions-src/*) diff --git a/.circleci/prepare-kernels/20-prepare-legacy-sources.sh b/.circleci/prepare-kernels/20-prepare-legacy-sources.sh index 4726cc7e59..c707b1e1eb 100755 --- a/.circleci/prepare-kernels/20-prepare-legacy-sources.sh +++ b/.circleci/prepare-kernels/20-prepare-legacy-sources.sh @@ -69,7 +69,11 @@ while IFS='' read -r line || [[ -n "$line" ]]; do git -C "${LEGACY_DIR}" submodule update --init "${DRIVER_REL_DIR}" mod_ver_file="${WORKSPACE_ROOT}/ko-build/released-collectors/${collector_ref}" - DRIVER_DIR="${LEGACY_DIR}/${DRIVER_REL_DIR}" \ + + DRIVER_DIR="${LEGACY_DIR}/${DRIVER_REL_DIR}" + + LEGACY_DIR="${LEGACY_DIR}" \ + DRIVER_DIR="${DRIVER_DIR}" \ SCRATCH_DIR="${HOME}/scratch" \ OUTPUT_DIR="${HOME}/kobuild-tmp/versions-src" \ M_VERSION="$(get_module_version)" \ diff --git a/kernel-modules/build/prepare-src b/kernel-modules/build/prepare-src index cb20785481..09ac214b34 100755 --- a/kernel-modules/build/prepare-src +++ b/kernel-modules/build/prepare-src @@ -2,7 +2,7 @@ set -eo pipefail -[[ -n "$SCRATCH_DIR" && -n "$DRIVER_DIR" && -n "$OUTPUT_DIR" ]] || { +[[ -n "$SCRATCH_DIR" && -n "$DRIVER_DIR" && -n "$OUTPUT_DIR" && -n "$LEGACY_DIR" ]] || { echo >&2 "SCRATCH_DIR, DRIVER_DIR and OUTPUT_DIR all need to be set" exit 1 } @@ -11,8 +11,6 @@ DOCKERIZED=${DOCKERIZED:-0} DRIVER_SCRATCH="${SCRATCH_DIR}/src" BUILD_SCRATCH="${SCRATCH_DIR}/cmake-build" -SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}")" &> /dev/null && pwd 2> /dev/null)" - modfiles0() ( local dir="${1:-${DRIVER_SCRATCH}/driver}" cd "$dir" @@ -90,10 +88,10 @@ cmake \ cp "${BUILD_SCRATCH}/driver/src/Makefile" "${DRIVER_SCRATCH}/driver/Makefile" cp "${BUILD_SCRATCH}/driver/src/driver_config.h" "${DRIVER_SCRATCH}/driver/driver_config.h" -if [[ -d "${SCRIPT_DIR}/../probe" ]]; then +if [[ -d "${LEGACY_DIR}/kernel-modules/probe" ]]; then mkdir -p "${DRIVER_SCRATCH}/driver/collector-probe" - cp "${SCRIPT_DIR}/../probe/Makefile" "${DRIVER_SCRATCH}/driver/collector-probe/Makefile" - cp "${SCRIPT_DIR}/../probe/collector_probe.c" "${DRIVER_SCRATCH}/driver/collector-probe/collector_probe.c" + cp "${LEGACY_DIR}/kernel-modules/probe/Makefile" "${DRIVER_SCRATCH}/driver/collector-probe/Makefile" + cp "${LEGACY_DIR}/kernel-modules/probe/collector_probe.c" "${DRIVER_SCRATCH}/driver/collector-probe/collector_probe.c" fi cd "${DRIVER_SCRATCH}/driver" From 1db4c345d8b81e7a372ae90e72990c0d5180b65e Mon Sep 17 00:00:00 2001 From: Giles Hutton Date: Mon, 13 Jun 2022 12:05:27 +0100 Subject: [PATCH 29/29] ROX-10009: drops parallelism back to 8 --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2889a0405b..fe8f685f51 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ parameters: default: "quay.io/rhacs-eng" dockerized-parallel: type: integer - default: 32 # TODO: revert before merging + default: 8 dockerized-cache-tag: type: string default: cache-v2 @@ -367,7 +367,7 @@ jobs: kernels: <<: *defaultMachine - parallelism: 32 # TODO: revert before merging + parallelism: 8 environment: - BUILD_CONTAINER_TAG: quay.io/rhacs-eng/collector-builder:kobuilder-cache - BUILD_CONTAINER_CACHE_IMAGES: quay.io/rhacs-eng/collector-builder:kobuilder-cache