feat: add exit code in container deletion events

container/container.go

@@ -63,6 +63,11 @@ type ContainerHandler interface {
 	// Returns the container's ip address, if available
 	GetContainerIPAddress() string
 
+	// GetExitCode returns the container's exit code if available.
+	// Returns an error if the container has not exited, exit codes are not supported
+	// for this handler type, or the container information is unavailable.
+	GetExitCode() (int, error)
+
 	// Returns whether the container still exists.
 	Exists() bool
 

container/containerd/client.go

@@ -46,6 +46,7 @@ type ContainerdClient interface {
 	LoadContainer(ctx context.Context, id string) (*containers.Container, error)
 	TaskPid(ctx context.Context, id string) (uint32, error)
 	LoadTaskProcess(ctx context.Context, id string) (*tasktypes.Process, error)
+	TaskExitStatus(ctx context.Context, id string) (uint32, error)
 	Version(ctx context.Context) (string, error)
 }
 
@@ -144,6 +145,19 @@ func (c *client) LoadTaskProcess(ctx context.Context, id string) (*tasktypes.Pro
 	return response.Process, nil
 }
 
+func (c *client) TaskExitStatus(ctx context.Context, id string) (uint32, error) {
+	response, err := c.taskService.Get(ctx, &tasksapi.GetRequest{
+		ContainerID: id,
+	})
+	if err != nil {
+		return 0, errgrpc.ToNative(err)
+	}
+	if response.Process.Status != tasktypes.Status_STOPPED {
+		return 0, fmt.Errorf("container %s has not exited (status: %v)", id, response.Process.Status)
+	}
+	return response.Process.ExitStatus, nil
+}
+
 func (c *client) Version(ctx context.Context) (string, error) {
 	response, err := c.versionService.Version(ctx, &emptypb.Empty{})
 	if err != nil {

container/containerd/client_test.go

@@ -23,9 +23,10 @@ import (
 )
 
 type containerdClientMock struct {
-	cntrs     map[string]*containers.Container
-	returnErr error
-	tasks     map[string]*task.Process
+	cntrs      map[string]*containers.Container
+	returnErr  error
+	tasks      map[string]*task.Process
+	exitStatus uint32
 }
 
 func (c *containerdClientMock) LoadContainer(ctx context.Context, id string) (*containers.Container, error) {
@@ -58,6 +59,13 @@ func (c *containerdClientMock) LoadTaskProcess(ctx context.Context, id string) (
 	return task, nil
 }
 
+func (c *containerdClientMock) TaskExitStatus(ctx context.Context, id string) (uint32, error) {
+	if c.returnErr != nil {
+		return 0, c.returnErr
+	}
+	return c.exitStatus, nil
+}
+
 func mockcontainerdClient(cntrs map[string]*containers.Container, returnErr error) ContainerdClient {
 	tasks := make(map[string]*task.Process)
 

container/containerd/handler.go

@@ -50,6 +50,7 @@ type containerdContainerHandler struct {
 	includedMetrics container.MetricSet
 
 	libcontainerHandler *containerlibcontainer.Handler
+	client              ContainerdClient
 }
 
 var _ container.ContainerHandler = &containerdContainerHandler{}
@@ -143,6 +144,7 @@ func newContainerdContainerHandler(
 		includedMetrics:     metrics,
 		reference:           containerReference,
 		libcontainerHandler: libcontainerHandler,
+		client:              client,
 	}
 	// Add the name and bare ID as aliases of the container.
 	handler.image = cntr.Image
@@ -248,3 +250,12 @@ func (h *containerdContainerHandler) GetContainerIPAddress() string {
 	// containerd doesnt take care of networking.So it doesnt maintain networking states
 	return ""
 }
+
+func (h *containerdContainerHandler) GetExitCode() (int, error) {
+	ctx := context.Background()
+	exitStatus, err := h.client.TaskExitStatus(ctx, h.reference.Id)
+	if err != nil {
+		return -1, err
+	}
+	return int(exitStatus), nil
+}

container/containerd/handler_test.go

@@ -140,3 +140,64 @@ func TestHandler(t *testing.T) {
 		}
 	}
 }
+
+func TestGetExitCode(t *testing.T) {
+	tests := []struct {
+		name         string
+		exitStatus   uint32
+		returnErr    error
+		expectErr    bool
+		errContains  string
+		expectedCode int
+	}{
+		{
+			name:         "successful exit code 0",
+			exitStatus:   0,
+			returnErr:    nil,
+			expectErr:    false,
+			expectedCode: 0,
+		},
+		{
+			name:         "successful exit code 1",
+			exitStatus:   1,
+			returnErr:    nil,
+			expectErr:    false,
+			expectedCode: 1,
+		},
+		{
+			name:         "task not stopped",
+			exitStatus:   0,
+			returnErr:    assert.AnError,
+			expectErr:    true,
+			expectedCode: -1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			as := assert.New(t)
+
+			mockClient := &containerdClientMock{
+				returnErr:  tt.returnErr,
+				exitStatus: tt.exitStatus,
+			}
+
+			h := &containerdContainerHandler{
+				client: mockClient,
+				reference: info.ContainerReference{
+					Id: "test-container-id",
+				},
+			}
+
+			code, err := h.GetExitCode()
+
+			if tt.expectErr {
+				as.Error(err)
+				as.Equal(tt.expectedCode, code)
+			} else {
+				as.NoError(err)
+				as.Equal(tt.expectedCode, code)
+			}
+		})
+	}
+}

container/crio/handler.go

@@ -360,3 +360,7 @@ func (h *crioContainerHandler) Exists() bool {
 func (h *crioContainerHandler) Type() container.ContainerType {
 	return container.ContainerTypeCrio
 }
+
+func (h *crioContainerHandler) GetExitCode() (int, error) {
+	return -1, fmt.Errorf("exit code not available from CRI-O API")
+}

container/docker/handler.go

@@ -386,3 +386,16 @@ func (h *containerHandler) Start() {
 func (h *containerHandler) Type() container.ContainerType {
 	return container.ContainerTypeDocker
 }
+
+func (h *containerHandler) GetExitCode() (int, error) {
+	ctnr, err := h.client.ContainerInspect(context.Background(), h.reference.Id)
+	if err != nil {
+		return -1, fmt.Errorf("failed to inspect container %s: %w", h.reference.Id, err)
+	}
+
+	if ctnr.State.Running {
+		return -1, fmt.Errorf("container %s is still running", h.reference.Id)
+	}
+
+	return ctnr.State.ExitCode, nil
+}

container/docker/handler_test.go

@@ -16,18 +16,30 @@
 package docker
 
 import (
+	"context"
 	"os"
 	"path"
 	"strings"
 	"testing"
 
 	"github.com/docker/docker/api/types/container"
+	dclient "github.com/docker/docker/client"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/google/cadvisor/fs"
 	info "github.com/google/cadvisor/info/v1"
 )
 
+type mockDockerClientForExitCode struct {
+	dclient.APIClient
+	inspectResp container.InspectResponse
+	inspectErr  error
+}
+
+func (m *mockDockerClientForExitCode) ContainerInspect(ctx context.Context, containerID string) (container.InspectResponse, error) {
+	return m.inspectResp, m.inspectErr
+}
+
 func TestStorageDirDetectionWithOldVersions(t *testing.T) {
 	as := assert.New(t)
 	rwLayer, err := getRwLayerID("abcd", "/", AufsStorageDriver, []int{1, 9, 0})
@@ -218,3 +230,90 @@ func TestAddDiskStats(t *testing.T) {
 	as.Equal(ioTime, fileSystem.DiskStats.IoTime, "IoTime metric should be %d but was %d", ioTime, fileSystem.DiskStats.IoTime)
 	as.Equal(weightedIoTime, fileSystem.DiskStats.WeightedIoTime, "WeightedIoTime metric should be %d but was %d", weightedIoTime, fileSystem.DiskStats.WeightedIoTime)
 }
+
+func TestGetExitCode(t *testing.T) {
+	tests := []struct {
+		name         string
+		running      bool
+		exitCode     int
+		inspectErr   error
+		expectErr    bool
+		errContains  string
+		expectedCode int
+	}{
+		{
+			name:         "successful exit code 0",
+			running:      false,
+			exitCode:     0,
+			inspectErr:   nil,
+			expectErr:    false,
+			expectedCode: 0,
+		},
+		{
+			name:         "successful exit code 1",
+			running:      false,
+			exitCode:     1,
+			inspectErr:   nil,
+			expectErr:    false,
+			expectedCode: 1,
+		},
+		{
+			name:         "container still running",
+			running:      true,
+			exitCode:     0,
+			inspectErr:   nil,
+			expectErr:    true,
+			errContains:  "still running",
+			expectedCode: -1,
+		},
+		{
+			name:         "inspect fails",
+			running:      false,
+			exitCode:     0,
+			inspectErr:   assert.AnError,
+			expectErr:    true,
+			errContains:  "failed to inspect",
+			expectedCode: -1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			as := assert.New(t)
+
+			inspectResp := container.InspectResponse{
+				ContainerJSONBase: &container.ContainerJSONBase{
+					State: &container.State{
+						Running:  tt.running,
+						ExitCode: tt.exitCode,
+					},
+				},
+			}
+
+			mockClient := &mockDockerClientForExitCode{
+				inspectResp: inspectResp,
+				inspectErr:  tt.inspectErr,
+			}
+
+			h := &containerHandler{
+				client: mockClient,
+				reference: info.ContainerReference{
+					Id: "test-container-id",
+				},
+			}
+
+			code, err := h.GetExitCode()
+
+			if tt.expectErr {
+				as.Error(err)
+				if tt.errContains != "" {
+					as.Contains(err.Error(), tt.errContains)
+				}
+				as.Equal(tt.expectedCode, code)
+			} else {
+				as.NoError(err)
+				as.Equal(tt.expectedCode, code)
+			}
+		})
+	}
+}

container/podman/handler.go

@@ -323,3 +323,20 @@ func (h *containerHandler) Start() {
 func (h *containerHandler) Type() container.ContainerType {
 	return container.ContainerTypePodman
 }
+
+func (h *containerHandler) GetExitCode() (int, error) {
+	ctnr, err := InspectContainer(h.reference.Id)
+	if err != nil {
+		return -1, fmt.Errorf("failed to inspect container %s: %w", h.reference.Id, err)
+	}
+
+	if ctnr.State == nil {
+		return -1, fmt.Errorf("container state not available for %s", h.reference.Id)
+	}
+
+	if ctnr.State.Running {
+		return -1, fmt.Errorf("container %s is still running", h.reference.Id)
+	}
+
+	return ctnr.State.ExitCode, nil
+}

container/raw/handler.go

@@ -281,6 +281,10 @@ func (h *rawContainerHandler) Type() container.ContainerType {
 	return container.ContainerTypeRaw
 }
 
+func (h *rawContainerHandler) GetExitCode() (int, error) {
+	return -1, fmt.Errorf("exit codes not applicable for raw cgroup containers")
+}
+
 type fsNamer struct {
 	fs      []fs.Fs
 	factory info.MachineInfoFactory

container/testing/mock_handler.go

@@ -101,6 +101,11 @@ func (h *MockContainerHandler) GetContainerIPAddress() string {
 	return args.Get(0).(string)
 }
 
+func (h *MockContainerHandler) GetExitCode() (int, error) {
+	args := h.Called()
+	return args.Int(0), args.Error(1)
+}
+
 type FactoryForMockContainerHandler struct {
 	Name                        string
 	PrepareContainerHandlerFunc func(name string, handler *MockContainerHandler)

info/v1/container.go

@@ -1104,6 +1104,9 @@ const (
 type EventData struct {
 	// Information about an OOM kill event.
 	OomKill *OomKillEventData `json:"oom,omitempty"`
+
+	// Information about a container deletion event.
+	ContainerDeletion *ContainerDeletionEventData `json:"container_deletion,omitempty"`
 }
 
 // Information related to an OOM kill instance
@@ -1114,3 +1117,10 @@ type OomKillEventData struct {
 	// The name of the killed process
 	ProcessName string `json:"process_name"`
 }
+
+// Information related to a container deletion event
+type ContainerDeletionEventData struct {
+	// ExitCode is the exit code of the container.
+	// A value of -1 indicates the exit code was not available or not applicable.
+	ExitCode int `json:"exit_code"`
+}