From 7e2132a06021a84265a3eac16e51c0ff474a2c0b Mon Sep 17 00:00:00 2001 From: kart2bc Date: Wed, 9 Jul 2025 10:13:10 -0400 Subject: [PATCH 1/3] Remove old property: proxy.enabled - Diego release --- containermetrics/reporters_runner_test.go | 170 +++++++----------- containermetrics/stats_reporter.go | 10 +- depot/containerstore/containerstore_test.go | 1 - depot/containerstore/proxy_config_handler.go | 15 -- .../proxy_config_handler_test.go | 153 +++++++--------- depot/transformer/transformer.go | 2 +- depot/transformer/transformer_test.go | 17 -- initializer/initializer.go | 41 ++--- initializer/initializer_test.go | 1 - resources.go | 1 - 10 files changed, 145 insertions(+), 266 deletions(-) diff --git a/containermetrics/reporters_runner_test.go b/containermetrics/reporters_runner_test.go index 291988f0..2ed4fcad 100644 --- a/containermetrics/reporters_runner_test.go +++ b/containermetrics/reporters_runner_test.go @@ -47,7 +47,6 @@ var _ = Describe("ReportersRunner", func() { testStart time.Time process ifrit.Process - enableContainerProxy bool proxyMemoryAllocationMB int reportersRunner *containermetrics.ReportersRunner statsReporter *containermetrics.StatsReporter @@ -64,14 +63,13 @@ var _ = Describe("ReportersRunner", func() { fakeExecutorClient = new(efakes.FakeClient) fakeMetronClient = new(mfakes.FakeIngressClient) - enableContainerProxy = false proxyMemoryAllocationMB = 5 metricsCache = &atomic.Value{} }) JustBeforeEach(func() { - statsReporter = containermetrics.NewStatsReporter(fakeMetronClient, enableContainerProxy, float64(proxyMemoryAllocationMB*1024*1024), metricsCache) + statsReporter = containermetrics.NewStatsReporter(fakeMetronClient, float64(proxyMemoryAllocationMB*1024*1024), metricsCache) cpuSpikeReporter := containermetrics.NewCPUSpikeReporter(fakeMetronClient) reportersRunner = containermetrics.NewReportersRunner(logger, interval, fakeClock, fakeExecutorClient, statsReporter, cpuSpikeReporter) process = ifrit.Invoke(reportersRunner) @@ -482,113 +480,77 @@ var _ = Describe("ReportersRunner", func() { )) }) - Context("when containers EnableContainerProxy is set", func() { - BeforeEach(func() { - containers := []executor.Container{ - { - Guid: "container-guid-without-index", - MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - EnableContainerProxy: true, - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index"}}, - }, + BeforeEach(func() { + containers := []executor.Container{ + { + Guid: "container-guid-without-index", + MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index"}}, }, - { - Guid: "container-guid-with-index", - MemoryLimit: megsToBytes(400 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - EnableContainerProxy: true, - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, - }, + }, + { + Guid: "container-guid-with-index", + MemoryLimit: megsToBytes(400 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, }, - { - Guid: "container-guid-without-preloaded-rootfs", - MemoryLimit: megsToBytes(200), - RunInfo: executor.RunInfo{ - EnableContainerProxy: false, - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-preloaded-rootfs"}}, - }, + }, + { + Guid: "container-guid-without-preloaded-rootfs", + MemoryLimit: megsToBytes(200), + RunInfo: executor.RunInfo{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-preloaded-rootfs"}}, }, - } - fakeExecutorClient.ListContainersReturnsOnCall(0, containers, nil) - }) - - Context("when enableContainerProxy not set", func() { - It("does not change the memory usage reported by garden", func() { - appMemory := megsToBytes(123) - expectedMemoryUsageWithoutIndex := float64(appMemory) - Eventually(sentMetrics).Should(ContainElement( - logging.ContainerMetric{ - CpuPercentage: 0.0, - MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), - DiskBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes, - DiskBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, - AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), - AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, - ContainerAge: metricsAtT0["container-guid-without-index"].ContainerMetrics.ContainerAgeInNanoseconds, - RxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.RxInBytes, - TxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.TxInBytes, - Tags: map[string]string{ - "source_id": "source-id-without-index", - "instance_id": "0", - }, - }, - )) - }) - }) + }, + } + fakeExecutorClient.ListContainersReturnsOnCall(0, containers, nil) + }) - Context("when enableContainerProxy is set", func() { - BeforeEach(func() { - enableContainerProxy = true - }) + It("emits a rescaled memory usage based on the additional memory allocation for the proxy", func() { + appMemory := megsToBytes(123) + expectedMemoryUsageWithoutIndex := float64(appMemory) * 200.0 / (200.0 + float64(proxyMemoryAllocationMB)) + expectedMemoryLimitWithoutIndex := float64(metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes) - float64(megsToBytes(proxyMemoryAllocationMB)) + Eventually(sentMetrics).Should(ContainElement( + logging.ContainerMetric{ + CpuPercentage: 0.0, + MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), + DiskBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), + DiskBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, + AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), + AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, + ContainerAge: metricsAtT0["container-guid-without-index"].ContainerMetrics.ContainerAgeInNanoseconds, + RxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.RxInBytes, + TxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.TxInBytes, + Tags: map[string]string{ + "source_id": "source-id-without-index", + "instance_id": "0", + }, + }, + )) + }) - It("emits a rescaled memory usage based on the additional memory allocation for the proxy", func() { - appMemory := megsToBytes(123) - expectedMemoryUsageWithoutIndex := float64(appMemory) * 200.0 / (200.0 + float64(proxyMemoryAllocationMB)) - expectedMemoryLimitWithoutIndex := float64(metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes) - float64(megsToBytes(proxyMemoryAllocationMB)) - Eventually(sentMetrics).Should(ContainElement( - logging.ContainerMetric{ - CpuPercentage: 0.0, - MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), - DiskBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), - DiskBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, - AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), - AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, - ContainerAge: metricsAtT0["container-guid-without-index"].ContainerMetrics.ContainerAgeInNanoseconds, - RxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.RxInBytes, - TxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.TxInBytes, - Tags: map[string]string{ - "source_id": "source-id-without-index", - "instance_id": "0", - }, + Context("when there is a container without preloaded rootfs", func() { + It("should emit memory usage that is not rescaled", func() { + Eventually(sentMetrics).Should(ContainElement( + logging.ContainerMetric{ + CpuPercentage: 0.0, + MemoryBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, + DiskBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, + MemoryBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, + DiskBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, + AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), + AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, + ContainerAge: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.ContainerAgeInNanoseconds, + RxBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.RxInBytes, + TxBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TxInBytes, + Tags: map[string]string{ + "source_id": "source-id-without-preloaded-rootfs", + "instance_id": "0", }, - )) - }) - - Context("when there is a container without preloaded rootfs", func() { - It("should emit memory usage that is not rescaled", func() { - Eventually(sentMetrics).Should(ContainElement( - logging.ContainerMetric{ - CpuPercentage: 0.0, - MemoryBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, - DiskBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, - DiskBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, - AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), - AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, - ContainerAge: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.ContainerAgeInNanoseconds, - RxBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.RxInBytes, - TxBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TxInBytes, - Tags: map[string]string{ - "source_id": "source-id-without-preloaded-rootfs", - "instance_id": "0", - }, - }, - )) - }) - }) + }, + )) }) }) diff --git a/containermetrics/stats_reporter.go b/containermetrics/stats_reporter.go index 38880c07..ca090170 100644 --- a/containermetrics/stats_reporter.go +++ b/containermetrics/stats_reporter.go @@ -19,15 +19,13 @@ type cpuInfo struct { type StatsReporter struct { cpuInfos map[string]*cpuInfo metronClient loggingclient.IngressClient - enableContainerProxy bool proxyMemoryAllocation float64 metricsCache *atomic.Value } -func NewStatsReporter(metronClient loggingclient.IngressClient, enableContainerProxy bool, proxyMemoryAllocation float64, metricsCache *atomic.Value) *StatsReporter { +func NewStatsReporter(metronClient loggingclient.IngressClient, proxyMemoryAllocation float64, metricsCache *atomic.Value) *StatsReporter { return &StatsReporter{ cpuInfos: make(map[string]*cpuInfo), - enableContainerProxy: enableContainerProxy, metronClient: metronClient, proxyMemoryAllocation: proxyMemoryAllocation, metricsCache: metricsCache, @@ -57,10 +55,8 @@ func (reporter *StatsReporter) Report(logger lager.Logger, containers []executor previousCPUInfo := cpuInfos[guid] - if reporter.enableContainerProxy && container.EnableContainerProxy { - metric.MemoryUsageInBytes = uint64(float64(metric.MemoryUsageInBytes) * reporter.scaleMemory(container)) - metric.MemoryLimitInBytes = uint64(float64(metric.MemoryLimitInBytes) - reporter.proxyMemoryAllocation) - } + metric.MemoryUsageInBytes = uint64(float64(metric.MemoryUsageInBytes) * reporter.scaleMemory(container)) + metric.MemoryLimitInBytes = uint64(float64(metric.MemoryLimitInBytes) - reporter.proxyMemoryAllocation) repMetrics, cpu := reporter.calculateAndSendMetrics(logger, metric.MetricsConfig, metric.ContainerMetrics, previousCPUInfo, timeStamp) if cpu != nil { diff --git a/depot/containerstore/containerstore_test.go b/depot/containerstore/containerstore_test.go index 0c6eb044..8d8e72d6 100644 --- a/depot/containerstore/containerstore_test.go +++ b/depot/containerstore/containerstore_test.go @@ -410,7 +410,6 @@ var _ = Describe("Container Store", func() { "some-other-key": "some-other-value", }, }, - EnableContainerProxy: true, LogRateLimitBytesPerSecond: logRateUnlimitedBytesPerSecond, VolumeMountedFiles: volumeMountedFiles, } diff --git a/depot/containerstore/proxy_config_handler.go b/depot/containerstore/proxy_config_handler.go index fbb54b33..38dfe2f5 100644 --- a/depot/containerstore/proxy_config_handler.go +++ b/depot/containerstore/proxy_config_handler.go @@ -149,9 +149,6 @@ func NewProxyConfigHandler( // This modifies the container pointer in order to create garden NetIn rules in the storenode.Create func (p *ProxyConfigHandler) ProxyPorts(logger lager.Logger, container *executor.Container) ([]executor.ProxyPortMapping, []uint16, error) { - if !container.EnableContainerProxy { - return nil, nil, nil - } proxyPortMapping := []executor.ProxyPortMapping{} @@ -202,9 +199,6 @@ func (p *ProxyConfigHandler) ProxyPorts(logger lager.Logger, container *executor } func (p *ProxyConfigHandler) CreateDir(logger lager.Logger, container executor.Container) ([]garden.BindMount, []executor.EnvironmentVariable, error) { - if !container.EnableContainerProxy { - return nil, nil, nil - } logger.Info("adding-container-proxy-bindmounts") proxyConfigDir := filepath.Join(p.containerProxyConfigPath, container.Guid) @@ -230,9 +224,6 @@ func (p *ProxyConfigHandler) CreateDir(logger lager.Logger, container executor.C } func (p *ProxyConfigHandler) RemoveDir(logger lager.Logger, container executor.Container) error { - if !container.EnableContainerProxy { - return nil - } logger.Info("removing-container-proxy-config-dir") proxyConfigDir := filepath.Join(p.containerProxyConfigPath, container.Guid) @@ -240,17 +231,11 @@ func (p *ProxyConfigHandler) RemoveDir(logger lager.Logger, container executor.C } func (p *ProxyConfigHandler) Update(credentials Credentials, container executor.Container) error { - if !container.EnableContainerProxy { - return nil - } return p.writeConfig(credentials, container) } func (p *ProxyConfigHandler) Close(invalidCredentials Credentials, container executor.Container) error { - if !container.EnableContainerProxy { - return nil - } err := p.writeConfig(invalidCredentials, container) if err != nil { diff --git a/depot/containerstore/proxy_config_handler_test.go b/depot/containerstore/proxy_config_handler_test.go index fc263517..bc770f5c 100644 --- a/depot/containerstore/proxy_config_handler_test.go +++ b/depot/containerstore/proxy_config_handler_test.go @@ -72,9 +72,7 @@ var _ = Describe("ProxyConfigHandler", func() { container = executor.Container{ Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelProcess()), InternalIP: "10.0.0.1", - RunInfo: executor.RunInfo{ - EnableContainerProxy: true, - }, + RunInfo: executor.RunInfo{}, } proxyConfigDir, err = os.MkdirTemp("", "proxymanager-config") @@ -197,53 +195,44 @@ var _ = Describe("ProxyConfigHandler", func() { }) }) - Describe("CreateDir", func() { - Context("the EnableContainerProxy is disabled on the container", func() { - BeforeEach(func() { - container.EnableContainerProxy = false - }) + It("returns an empty bind mount", func() { + mounts, _, err := proxyConfigHandler.CreateDir(logger, container) + Expect(err).NotTo(HaveOccurred()) + Expect(mounts).To(BeEmpty()) + }) + It("returns the appropriate bind mounts for container proxy", func() { + mounts, _, err := proxyConfigHandler.CreateDir(logger, container) + Expect(err).NotTo(HaveOccurred()) + Expect(mounts).To(ConsistOf([]garden.BindMount{ + { + Origin: garden.BindMountOriginHost, + SrcPath: proxyDir, + DstPath: "/etc/cf-assets/envoy", + }, + { + Origin: garden.BindMountOriginHost, + SrcPath: filepath.Join(proxyConfigDir, container.Guid), + DstPath: "/etc/cf-assets/envoy_config", + }, + })) + }) - It("returns an empty bind mount", func() { - mounts, _, err := proxyConfigHandler.CreateDir(logger, container) - Expect(err).NotTo(HaveOccurred()) - Expect(mounts).To(BeEmpty()) - }) - }) + It("makes the proxy config directory on the host", func() { + _, _, err := proxyConfigHandler.CreateDir(logger, container) + Expect(err).NotTo(HaveOccurred()) + proxyConfigDir := fmt.Sprintf("%s/%s", proxyConfigDir, container.Guid) + Expect(proxyConfigDir).To(BeADirectory()) + }) - It("returns the appropriate bind mounts for container proxy", func() { - mounts, _, err := proxyConfigHandler.CreateDir(logger, container) + Context("when the manager fails to create the proxy config directory", func() { + BeforeEach(func() { + _, err := os.Create(configPath) Expect(err).NotTo(HaveOccurred()) - Expect(mounts).To(ConsistOf([]garden.BindMount{ - { - Origin: garden.BindMountOriginHost, - SrcPath: proxyDir, - DstPath: "/etc/cf-assets/envoy", - }, - { - Origin: garden.BindMountOriginHost, - SrcPath: filepath.Join(proxyConfigDir, container.Guid), - DstPath: "/etc/cf-assets/envoy_config", - }, - })) }) - It("makes the proxy config directory on the host", func() { + It("returns an error", func() { _, _, err := proxyConfigHandler.CreateDir(logger, container) - Expect(err).NotTo(HaveOccurred()) - proxyConfigDir := fmt.Sprintf("%s/%s", proxyConfigDir, container.Guid) - Expect(proxyConfigDir).To(BeADirectory()) - }) - - Context("when the manager fails to create the proxy config directory", func() { - BeforeEach(func() { - _, err := os.Create(configPath) - Expect(err).NotTo(HaveOccurred()) - }) - - It("returns an error", func() { - _, _, err := proxyConfigHandler.CreateDir(logger, container) - Expect(err).To(HaveOccurred()) - }) + Expect(err).To(HaveOccurred()) }) }) @@ -258,16 +247,11 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(configPath).NotTo(BeADirectory()) }) - Context("the EnableContainerProxy is disabled on the container", func() { - BeforeEach(func() { - container.EnableContainerProxy = false - }) - - It("does not return an error when deleting a non existing directory", func() { - err := proxyConfigHandler.RemoveDir(logger, container) - Expect(err).NotTo(HaveOccurred()) - }) + It("does not return an error when deleting a non existing directory", func() { + err := proxyConfigHandler.RemoveDir(logger, container) + Expect(err).NotTo(HaveOccurred()) }) + }) Describe("ProxyPorts", func() { @@ -277,17 +261,11 @@ var _ = Describe("ProxyConfigHandler", func() { } }) - Context("the EnableContainerProxy is disabled on the container", func() { - BeforeEach(func() { - container.EnableContainerProxy = false - }) - - It("returns an empty proxy port mapping", func() { - ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) - Expect(err).NotTo(HaveOccurred()) - Expect(ports).To(BeEmpty()) - Expect(extraPorts).To(BeEmpty()) - }) + It("returns an empty proxy port mapping", func() { + ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) + Expect(err).NotTo(HaveOccurred()) + Expect(ports).To(BeEmpty()) + Expect(extraPorts).To(BeEmpty()) }) Context("when there is a default HTTP port (8080)", func() { @@ -429,17 +407,11 @@ var _ = Describe("ProxyConfigHandler", func() { containerProxyRequireClientCerts = true }) - Context("the EnableContainerProxy is disabled on the container", func() { - BeforeEach(func() { - container.EnableContainerProxy = false - }) - - It("does not write a envoy config file", func() { - err := proxyConfigHandler.Update(credentials, container) - Expect(err).NotTo(HaveOccurred()) + It("does not write a envoy config file", func() { + err := proxyConfigHandler.Update(credentials, container) + Expect(err).NotTo(HaveOccurred()) - Expect(proxyConfigFile).NotTo(BeAnExistingFile()) - }) + Expect(proxyConfigFile).NotTo(BeAnExistingFile()) }) Context("with containerProxyRequireClientCerts set to false", func() { @@ -969,29 +941,24 @@ var _ = Describe("ProxyConfigHandler", func() { Eventually(ch).Should(BeClosed()) }) - Context("the EnableContainerProxy is disabled on the container", func() { - BeforeEach(func() { - container.EnableContainerProxy = false - }) - - It("does not write a envoy config file", func() { - err := proxyConfigHandler.Update(credentials, container) - Expect(err).NotTo(HaveOccurred()) + It("does not write a envoy config file", func() { + err := proxyConfigHandler.Update(credentials, container) + Expect(err).NotTo(HaveOccurred()) - Expect(proxyConfigFile).NotTo(BeAnExistingFile()) - }) + Expect(proxyConfigFile).NotTo(BeAnExistingFile()) + }) - It("doesn't wait until the proxy is serving the new cert", func() { - ch := make(chan struct{}) - go func() { - defer GinkgoRecover() - proxyConfigHandler.Close(credentials, container) - close(ch) - }() + It("doesn't wait until the proxy is serving the new cert", func() { + ch := make(chan struct{}) + go func() { + defer GinkgoRecover() + proxyConfigHandler.Close(credentials, container) + close(ch) + }() - Eventually(ch).Should(BeClosed()) - }) + Eventually(ch).Should(BeClosed()) }) + }) }) diff --git a/depot/transformer/transformer.go b/depot/transformer/transformer.go index fb22c3a9..41663879 100644 --- a/depot/transformer/transformer.go +++ b/depot/transformer/transformer.go @@ -589,7 +589,7 @@ func (t *transformer) StepsRunner( longLivedAction = action } - if t.useContainerProxy && container.EnableContainerProxy { + if t.useContainerProxy { containerProxyStep := t.transformContainerProxyStep( gardenContainer, container, diff --git a/depot/transformer/transformer_test.go b/depot/transformer/transformer_test.go index 35226acd..731537ae 100644 --- a/depot/transformer/transformer_test.go +++ b/depot/transformer/transformer_test.go @@ -397,7 +397,6 @@ var _ = Describe("Transformer", func() { ContainerPort: 61001, }, }, - EnableContainerProxy: true, }, } }) @@ -552,22 +551,6 @@ var _ = Describe("Transformer", func() { }) }) - Context("when the container proxy is disabled on the container", func() { - BeforeEach(func() { - container.EnableContainerProxy = false - }) - - It("does not run the container proxy", func() { - Eventually(gardenContainer.RunCallCount).Should(Equal(2)) - paths := []string{} - for i := 0; i < gardenContainer.RunCallCount(); i++ { - spec, _ := gardenContainer.RunArgsForCall(i) - paths = append(paths, spec.Path) - } - - Expect(paths).NotTo(ContainElement("sh")) - }) - }) }) Describe("declarative healthchecks", func() { diff --git a/initializer/initializer.go b/initializer/initializer.go index 85b6c87a..cf99399b 100644 --- a/initializer/initializer.go +++ b/initializer/initializer.go @@ -101,7 +101,6 @@ type ExecutorConfig struct { DeclarativeHealthcheckPath string `json:"declarative_healthcheck_path,omitempty"` DeleteWorkPoolSize int `json:"delete_work_pool_size,omitempty"` DiskMB string `json:"disk_mb,omitempty"` - EnableContainerProxy bool `json:"enable_container_proxy,omitempty"` EnableContainerProxyHealthChecks bool `json:"enable_container_proxy_healthcheck,omitempty"` DeclarativeHealthCheckDefaultTimeout durationjson.Duration `json:"declarative_healthcheck_default_timeout,omitempty"` EnableHealtcheckMetrics bool `json:"enable_healthcheck_metrics,omitempty"` @@ -263,7 +262,6 @@ func Initialize( config.PostSetupUser, config.EnableHealtcheckMetrics, sidecarRootFSPath, - config.EnableContainerProxy, time.Duration(config.EnvoyDrainTimeout), config.EnableContainerProxyHealthChecks, time.Duration(config.ProxyHealthCheckInterval), @@ -296,23 +294,18 @@ func Initialize( driverConfig.DriverPaths = filepath.SplitList(config.VolmanDriverPaths) volmanClient, volmanDriverSyncer := vollocal.NewServer(logger, metronClient, driverConfig) - var proxyConfigHandler containerstore.ProxyManager - if config.EnableContainerProxy { - proxyConfigHandler = containerstore.NewProxyConfigHandler( - logger, - config.ContainerProxyPath, - config.ContainerProxyConfigPath, - config.ContainerProxyTrustedCACerts, - config.ContainerProxyVerifySubjectAltName, - config.ContainerProxyRequireClientCerts, - time.Duration(config.EnvoyConfigReloadDuration), - clock, - config.ContainerProxyADSServers, - config.ProxyEnableHttp2, - ) - } else { - proxyConfigHandler = containerstore.NewNoopProxyConfigHandler() - } + proxyConfigHandler := containerstore.NewProxyConfigHandler( + logger, + config.ContainerProxyPath, + config.ContainerProxyConfigPath, + config.ContainerProxyTrustedCACerts, + config.ContainerProxyVerifySubjectAltName, + config.ContainerProxyRequireClientCerts, + time.Duration(config.EnvoyConfigReloadDuration), + clock, + config.ContainerProxyADSServers, + config.ProxyEnableHttp2, + ) instanceIdentityHandler := containerstore.NewInstanceIdentityHandler( config.InstanceIdentityCredDir, @@ -387,7 +380,6 @@ func Initialize( metricsCache := &atomic.Value{} containerStatsReporter := containermetrics.NewStatsReporter( metronClient, - config.EnableContainerProxy, float64(config.ProxyMemoryAllocationMB*megabytesToBytes), metricsCache, ) @@ -568,7 +560,6 @@ func initializeTransformer( postSetupUser string, emitHealthCheckMetrics bool, declarativeHealthcheckRootFS string, - enableContainerProxy bool, drainWait time.Duration, enableProxyHealthChecks bool, proxyHealthCheckInterval time.Duration, @@ -586,12 +577,10 @@ func initializeTransformer( options = append(options, transformer.WithDeclarativeHealthcheckFailureMetrics()) } - if enableContainerProxy { - options = append(options, transformer.WithContainerProxy(drainWait)) + options = append(options, transformer.WithContainerProxy(drainWait)) - if enableProxyHealthChecks { - options = append(options, transformer.WithProxyLivenessChecks(proxyHealthCheckInterval)) - } + if enableProxyHealthChecks { + options = append(options, transformer.WithProxyLivenessChecks(proxyHealthCheckInterval)) } options = append(options, transformer.WithPostSetupHook(postSetupUser, postSetupHook)) diff --git a/initializer/initializer_test.go b/initializer/initializer_test.go index 6d7a07e6..3328af8b 100644 --- a/initializer/initializer_test.go +++ b/initializer/initializer_test.go @@ -79,7 +79,6 @@ var _ = Describe("Initializer", func() { CreateWorkPoolSize: 32, DeleteWorkPoolSize: 32, DiskMB: configuration.Automatic, - EnableContainerProxy: false, GardenAddr: "/tmp/garden.sock", GardenHealthcheckCommandRetryPause: durationjson.Duration(1 * time.Second), GardenHealthcheckEmissionInterval: durationjson.Duration(30 * time.Second), diff --git a/resources.go b/resources.go index f28a0353..4492af4e 100644 --- a/resources.go +++ b/resources.go @@ -197,7 +197,6 @@ type RunInfo struct { CertificateProperties CertificateProperties `json:"certificate_properties"` ImageUsername string `json:"image_username"` ImagePassword string `json:"image_password"` - EnableContainerProxy bool `json:"enable_container_proxy"` Sidecars []Sidecar `json:"sidecars"` LogRateLimitBytesPerSecond int64 `json:"log_rate_limit_bytes_per_second"` VolumeMountedFiles []VolumeMountedFiles `json:"volume_mounted_files,omitempty"` From 1a2f0433a6f26dfeb924386296a6be1da76ba55f Mon Sep 17 00:00:00 2001 From: kart2bc Date: Thu, 10 Jul 2025 17:49:24 -0400 Subject: [PATCH 2/3] Remove old property: proxy.enabled - Diego release --- containermetrics/reporters_runner_test.go | 246 +++++++++++------- .../proxy_config_handler_test.go | 23 +- depot/transformer/transformer_test.go | 13 +- 3 files changed, 170 insertions(+), 112 deletions(-) diff --git a/containermetrics/reporters_runner_test.go b/containermetrics/reporters_runner_test.go index 2ed4fcad..630d1c99 100644 --- a/containermetrics/reporters_runner_test.go +++ b/containermetrics/reporters_runner_test.go @@ -152,7 +152,7 @@ var _ = Describe("ReportersRunner", func() { MemoryUsageInBytes: megsToBytes(123), DiskUsageInBytes: megsToBytes(456), TimeSpentInCPU: 0, - MemoryLimitInBytes: megsToBytes(789), + MemoryLimitInBytes: megsToBytes(123), DiskLimitInBytes: megsToBytes(1024), ContainerAgeInNanoseconds: 1000, AbsoluteCPUEntitlementInNanoseconds: 0, @@ -166,7 +166,7 @@ var _ = Describe("ReportersRunner", func() { MemoryUsageInBytes: megsToBytes(123), DiskUsageInBytes: megsToBytes(456), TimeSpentInCPU: 0, - MemoryLimitInBytes: megsToBytes(789), + MemoryLimitInBytes: megsToBytes(123), DiskLimitInBytes: megsToBytes(1024), ContainerAgeInNanoseconds: 1001, AbsoluteCPUEntitlementInNanoseconds: 0, @@ -177,10 +177,10 @@ var _ = Describe("ReportersRunner", func() { "container-guid-with-index": { MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, ContainerMetrics: executor.ContainerMetrics{ - MemoryUsageInBytes: megsToBytes(321), + MemoryUsageInBytes: megsToBytes(123), DiskUsageInBytes: megsToBytes(654), TimeSpentInCPU: 0, - MemoryLimitInBytes: megsToBytes(987), + MemoryLimitInBytes: megsToBytes(123), DiskLimitInBytes: megsToBytes(2048), ContainerAgeInNanoseconds: 1002, AbsoluteCPUEntitlementInNanoseconds: 0, @@ -191,10 +191,10 @@ var _ = Describe("ReportersRunner", func() { "container-guid-without-preloaded-rootfs": { MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-preloaded-rootfs"}}, ContainerMetrics: executor.ContainerMetrics{ - MemoryUsageInBytes: megsToBytes(345), + MemoryUsageInBytes: megsToBytes(123), DiskUsageInBytes: megsToBytes(456), TimeSpentInCPU: 0, - MemoryLimitInBytes: megsToBytes(678), + MemoryLimitInBytes: megsToBytes(123), DiskLimitInBytes: megsToBytes(2048), ContainerAgeInNanoseconds: 1003, AbsoluteCPUEntitlementInNanoseconds: 0, @@ -397,15 +397,63 @@ var _ = Describe("ReportersRunner", func() { fakeExecutorClient.GetBulkMetricsReturnsOnCall(1, metricsAtT10, nil) fakeExecutorClient.GetBulkMetricsReturnsOnCall(2, metricsAtT20, nil) }) + BeforeEach(func() { + containers := []executor.Container{ + { + Guid: "container-guid-without-index", + MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index"}}, + }, + }, + { + Guid: "container-guid-with-index", + MemoryLimit: megsToBytes(400 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, + }, + }, + { + Guid: "container-guid-without-preloaded-rootfs", + MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-preloaded-rootfs"}}, + }, + }, + { + Guid: "container-guid-without-age", + MemoryLimit: megsToBytes(100 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-age"}}, + }, + }, + { + Guid: "container-guid-without-metrics-at-t0", + MemoryLimit: megsToBytes(400 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-metrics-at-t0", "instance_id": "1"}}, + }, + }, + } + fakeExecutorClient.ListContainersReturnsOnCall(0, containers, nil) + fakeExecutorClient.ListContainersReturnsOnCall(1, containers, nil) + fakeExecutorClient.ListContainersReturnsOnCall(2, containers, nil) + }) It("emits memory and disk usage for each container, but no CPU", func() { + appMemory := megsToBytes(123) + expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) + expectedMemoryUsageWithIndex, expectedMemoryLimitWithIndex := calculateMemoryUsageWithProxy(appMemory, 400.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes)) + expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) + expectedMemoryUsageWithoutAge, expectedMemoryLimitWithoutAge := calculateMemoryUsageWithProxy(appMemory, 100.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-age"].ContainerMetrics.MemoryLimitInBytes)) + Eventually(sentMetrics).Should(ConsistOf([]logging.ContainerMetric{ { CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), DiskBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), DiskBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -420,9 +468,9 @@ var _ = Describe("ReportersRunner", func() { { CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: metricsAtT0["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsageWithIndex), DiskBytes: metricsAtT0["container-guid-with-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT0["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitWithIndex), DiskBytesQuota: metricsAtT0["container-guid-with-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-with-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-with-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -437,9 +485,9 @@ var _ = Describe("ReportersRunner", func() { { CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsagePreloadedRootfs), DiskBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitPreloadedRootfs), DiskBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -454,9 +502,9 @@ var _ = Describe("ReportersRunner", func() { { CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: metricsAtT0["container-guid-without-age"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsageWithoutAge), DiskBytes: metricsAtT0["container-guid-without-age"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT0["container-guid-without-age"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitWithoutAge), DiskBytesQuota: metricsAtT0["container-guid-without-age"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-age"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-age"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -480,44 +528,17 @@ var _ = Describe("ReportersRunner", func() { )) }) - BeforeEach(func() { - containers := []executor.Container{ - { - Guid: "container-guid-without-index", - MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index"}}, - }, - }, - { - Guid: "container-guid-with-index", - MemoryLimit: megsToBytes(400 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, - }, - }, - { - Guid: "container-guid-without-preloaded-rootfs", - MemoryLimit: megsToBytes(200), - RunInfo: executor.RunInfo{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-preloaded-rootfs"}}, - }, - }, - } - fakeExecutorClient.ListContainersReturnsOnCall(0, containers, nil) - }) - It("emits a rescaled memory usage based on the additional memory allocation for the proxy", func() { appMemory := megsToBytes(123) - expectedMemoryUsageWithoutIndex := float64(appMemory) * 200.0 / (200.0 + float64(proxyMemoryAllocationMB)) - expectedMemoryLimitWithoutIndex := float64(metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes) - float64(megsToBytes(proxyMemoryAllocationMB)) + expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) + Eventually(sentMetrics).Should(ContainElement( logging.ContainerMetric{ CpuPercentage: 0.0, MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), DiskBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), - DiskBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, + DiskBytesQuota: metricsAtT0["container-guid-without-index"].DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, ContainerAge: metricsAtT0["container-guid-without-index"].ContainerMetrics.ContainerAgeInNanoseconds, @@ -533,12 +554,14 @@ var _ = Describe("ReportersRunner", func() { Context("when there is a container without preloaded rootfs", func() { It("should emit memory usage that is not rescaled", func() { + appMemory := megsToBytes(123) + expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement( logging.ContainerMetric{ CpuPercentage: 0.0, - MemoryBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsagePreloadedRootfs), DiskBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitPreloadedRootfs), DiskBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -570,12 +593,13 @@ var _ = Describe("ReportersRunner", func() { }) It("emits the new memory and disk usage, and the computed CPU percent", func() { + expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 20.0, CpuEntitlementPercentage: 40.0, - MemoryBytes: metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), DiskBytes: metricsAtT10["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), DiskBytesQuota: metricsAtT10["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT10["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT10["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -588,12 +612,13 @@ var _ = Describe("ReportersRunner", func() { }, })) + expectedMemoryUsageWithIndex, expectedMemoryLimitWithIndex := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 30.0, CpuEntitlementPercentage: 60.0, - MemoryBytes: metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsageWithIndex), DiskBytes: metricsAtT10["container-guid-with-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitWithIndex), DiskBytesQuota: metricsAtT10["container-guid-with-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT10["container-guid-with-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT10["container-guid-with-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -606,12 +631,13 @@ var _ = Describe("ReportersRunner", func() { }, })) + expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 40.0, CpuEntitlementPercentage: 80.0, - MemoryBytes: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsagePreloadedRootfs), DiskBytes: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitPreloadedRootfs), DiskBytesQuota: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -624,12 +650,13 @@ var _ = Describe("ReportersRunner", func() { }, })) + expectedMemoryUsageNoMetricsAtT0, expectedMemoryLimitNoMetricsAtT0 := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsageNoMetricsAtT0), DiskBytes: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitNoMetricsAtT0), DiskBytesQuota: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -646,63 +673,60 @@ var _ = Describe("ReportersRunner", func() { Context("Metrics", func() { It("returns the cached metrics last emitted", func() { containerMetrics := statsReporter.Metrics() - Expect(containerMetrics).To(HaveLen(6)) + Expect(containerMetrics).To(HaveLen(5)) + + expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-index", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-without-index", CPUUsageFraction: 0.2, - MemoryUsageBytes: megsToBytes(1230), + MemoryUsageBytes: uint64(expectedMemoryUsageWithoutIndex), DiskUsageBytes: 4560, - MemoryQuotaBytes: megsToBytes(7890), + MemoryQuotaBytes: uint64(expectedMemoryLimitWithoutIndex), DiskQuotaBytes: 4096, RxBytes: &two, TxBytes: &two, })) + + expectedMemoryUsageWithIndex, expectedMemoryLimitWithIndex := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes)) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-with-index", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-with-index", CPUUsageFraction: 0.3, - MemoryUsageBytes: megsToBytes(3210), + MemoryUsageBytes: uint64(expectedMemoryUsageWithIndex), DiskUsageBytes: 6540, - MemoryQuotaBytes: megsToBytes(9870), + MemoryQuotaBytes: uint64(expectedMemoryLimitWithIndex), DiskQuotaBytes: 512, RxBytes: &two, TxBytes: &two, })) - Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-source-id", &containermetrics.CachedContainerMetrics{ - MetricGUID: "", - CPUUsageFraction: 0.1, - MemoryUsageBytes: megsToBytes(123), - DiskUsageBytes: megsToBytes(456), - MemoryQuotaBytes: megsToBytes(789), - DiskQuotaBytes: megsToBytes(1024), - RxBytes: &two, - TxBytes: &two, - })) + expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-preloaded-rootfs", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-without-preloaded-rootfs", CPUUsageFraction: 0.4, - MemoryUsageBytes: megsToBytes(3450), + MemoryUsageBytes: uint64(expectedMemoryUsagePreloadedRootfs), DiskUsageBytes: 4560, - MemoryQuotaBytes: megsToBytes(6780), + MemoryQuotaBytes: uint64(expectedMemoryLimitPreloadedRootfs), DiskQuotaBytes: 2048, RxBytes: &two, TxBytes: &two, })) + expectedMemoryUsageWithoutAge, expectedMemoryLimitWithoutAge := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-age"].ContainerMetrics.MemoryUsageInBytes, 100.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-age"].ContainerMetrics.MemoryLimitInBytes)) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-age", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-without-age", CPUUsageFraction: 0.5, - MemoryUsageBytes: megsToBytes(123), + MemoryUsageBytes: uint64(expectedMemoryUsageWithoutAge), DiskUsageBytes: megsToBytes(456), - MemoryQuotaBytes: megsToBytes(789), + MemoryQuotaBytes: uint64(expectedMemoryLimitWithoutAge), DiskQuotaBytes: megsToBytes(1024), RxBytes: &two, TxBytes: &two, })) + expectedMemoryUsageNoMetricsAtT0, expectedMemoryLimitNoMetricsAtT0 := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes)) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-metrics-at-t0", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-without-metrics-at-t0", CPUUsageFraction: 0.0, - MemoryUsageBytes: megsToBytes(3210), + MemoryUsageBytes: uint64(expectedMemoryUsageNoMetricsAtT0), DiskUsageBytes: 6540, - MemoryQuotaBytes: megsToBytes(9870), + MemoryQuotaBytes: uint64(expectedMemoryLimitNoMetricsAtT0), DiskQuotaBytes: 512, RxBytes: &two, TxBytes: &two, @@ -717,12 +741,13 @@ var _ = Describe("ReportersRunner", func() { }) It("emits the new memory and disk usage, and the computed CPU percent", func() { + expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuEntitlementPercentage: 40.0, CpuPercentage: 20.0, - MemoryBytes: metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), DiskBytes: metricsAtT20["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), DiskBytesQuota: metricsAtT20["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT20["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT20["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -735,12 +760,13 @@ var _ = Describe("ReportersRunner", func() { }, })) + expectedMemoryUsageWithIndex, expectedMemoryLimitWithIndex := calculateMemoryUsageWithProxy(metricsAtT20["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuEntitlementPercentage: 60.0, CpuPercentage: 30.0, - MemoryBytes: metricsAtT20["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsageWithIndex), DiskBytes: metricsAtT20["container-guid-with-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT20["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitWithIndex), DiskBytesQuota: metricsAtT20["container-guid-with-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT20["container-guid-with-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT20["container-guid-with-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -753,12 +779,13 @@ var _ = Describe("ReportersRunner", func() { }, })) + expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuEntitlementPercentage: 80.0, CpuPercentage: 40.0, - MemoryBytes: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsagePreloadedRootfs), DiskBytes: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitPreloadedRootfs), DiskBytesQuota: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -771,12 +798,13 @@ var _ = Describe("ReportersRunner", func() { }, })) + expectedMemoryUsageNoMetricsAtT0, expectedMemoryLimitNoMetricsAtT0 := calculateMemoryUsageWithProxy(metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuEntitlementPercentage: 120.0, CpuPercentage: 60.0, - MemoryBytes: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, + MemoryBytes: uint64(expectedMemoryUsageNoMetricsAtT0), DiskBytes: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitNoMetricsAtT0), DiskBytesQuota: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -978,14 +1006,15 @@ var _ = Describe("ReportersRunner", func() { Context("Reporters reset the container list", func() { var metricsAtT0, metricsAtT10, metricsAtT20 map[string]executor.Metrics - BeforeEach(func() { containers1 := []executor.Container{ - {Guid: "container-guid"}, + {Guid: "container-guid-without-index"}, + } + containers2 := []executor.Container{ + {Guid: "container-guid-without-index"}, } - containers2 := []executor.Container{} containers3 := []executor.Container{ - {Guid: "container-guid"}, + {Guid: "container-guid-without-index"}, } fakeExecutorClient.ListContainersReturnsOnCall(0, containers1, nil) @@ -993,12 +1022,14 @@ var _ = Describe("ReportersRunner", func() { fakeExecutorClient.ListContainersReturnsOnCall(2, containers3, nil) metricsAtT0 = map[string]executor.Metrics{ - "container-guid": executor.Metrics{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, + "container-guid-without-index": executor.Metrics{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index", "instance_id": "1"}}, ContainerMetrics: executor.ContainerMetrics{ TimeSpentInCPU: 2000 * time.Nanosecond, ContainerAgeInNanoseconds: 1000, AbsoluteCPUEntitlementInNanoseconds: 1000, + MemoryUsageInBytes: megsToBytes(123), + MemoryLimitInBytes: megsToBytes(789), }, }, } @@ -1006,12 +1037,14 @@ var _ = Describe("ReportersRunner", func() { metricsAtT10 = map[string]executor.Metrics{} metricsAtT20 = map[string]executor.Metrics{ - "container-guid": executor.Metrics{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, + "container-guid-without-index": executor.Metrics{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index", "instance_id": "1"}}, ContainerMetrics: executor.ContainerMetrics{ TimeSpentInCPU: 2600 * time.Nanosecond, ContainerAgeInNanoseconds: 2000, AbsoluteCPUEntitlementInNanoseconds: 3000, + MemoryLimitInBytes: megsToBytes(789), + MemoryUsageInBytes: megsToBytes(123), }, }, } @@ -1020,7 +1053,18 @@ var _ = Describe("ReportersRunner", func() { fakeExecutorClient.GetBulkMetricsReturnsOnCall(1, metricsAtT10, nil) fakeExecutorClient.GetBulkMetricsReturnsOnCall(2, metricsAtT20, nil) }) - + BeforeEach(func() { + containers := []executor.Container{ + { + Guid: "container-guid-without-index", + MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index", "instance_id": "1"}}, + }, + }, + } + fakeExecutorClient.ListContainersReturnsOnCall(0, containers, nil) + }) It("does not send a spike metric when the container is gone", func() { fakeClock.WaitForWatcherAndIncrement(interval) fakeClock.WaitForWatcherAndIncrement(interval) @@ -1029,16 +1073,20 @@ var _ = Describe("ReportersRunner", func() { }) It("should report 0 CPU", func() { + appMemory := megsToBytes(123) + expectedMemory, expectedMemoryLimit := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) fakeClock.WaitForWatcherAndIncrement(interval) fakeClock.WaitForWatcherAndIncrement(interval) Eventually(fakeExecutorClient.GetBulkMetricsCallCount).Should(Equal(3)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 0.0, - AbsoluteCPUUsage: uint64(metricsAtT20["container-guid"].ContainerMetrics.TimeSpentInCPU), - AbsoluteCPUEntitlement: metricsAtT20["container-guid"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, - ContainerAge: metricsAtT20["container-guid"].ContainerMetrics.ContainerAgeInNanoseconds, + AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), + AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, + ContainerAge: metricsAtT0["container-guid-without-index"].ContainerMetrics.ContainerAgeInNanoseconds, + MemoryBytes: uint64(expectedMemory), + MemoryBytesQuota: uint64(expectedMemoryLimit), Tags: map[string]string{ - "source_id": "source-id-with-index", + "source_id": "source-id-without-index", "instance_id": "1", }, })) @@ -1195,3 +1243,9 @@ var _ = Describe("ReportersRunner", func() { }) }) + +func calculateMemoryUsageWithProxy(appMemory uint64, memoryLimit float64, proxyMemoryAllocationMB int, memoryLimitInBytes float64) (expectedMemoryUsage float64, expectedMemoryLimit float64) { + expectedMemoryUsage = float64(appMemory) * memoryLimit / (memoryLimit + float64(proxyMemoryAllocationMB)) + expectedMemoryLimit = float64(memoryLimitInBytes) - float64(megsToBytes(proxyMemoryAllocationMB)) + return +} diff --git a/depot/containerstore/proxy_config_handler_test.go b/depot/containerstore/proxy_config_handler_test.go index bc770f5c..5e3673a4 100644 --- a/depot/containerstore/proxy_config_handler_test.go +++ b/depot/containerstore/proxy_config_handler_test.go @@ -195,11 +195,6 @@ var _ = Describe("ProxyConfigHandler", func() { }) }) - It("returns an empty bind mount", func() { - mounts, _, err := proxyConfigHandler.CreateDir(logger, container) - Expect(err).NotTo(HaveOccurred()) - Expect(mounts).To(BeEmpty()) - }) It("returns the appropriate bind mounts for container proxy", func() { mounts, _, err := proxyConfigHandler.CreateDir(logger, container) Expect(err).NotTo(HaveOccurred()) @@ -261,11 +256,11 @@ var _ = Describe("ProxyConfigHandler", func() { } }) - It("returns an empty proxy port mapping", func() { + It("returns a non empty proxy port mapping", func() { ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) Expect(err).NotTo(HaveOccurred()) - Expect(ports).To(BeEmpty()) - Expect(extraPorts).To(BeEmpty()) + Expect(ports).NotTo(BeEmpty()) + Expect(extraPorts).NotTo(BeEmpty()) }) Context("when there is a default HTTP port (8080)", func() { @@ -407,11 +402,11 @@ var _ = Describe("ProxyConfigHandler", func() { containerProxyRequireClientCerts = true }) - It("does not write a envoy config file", func() { + It("write a envoy config file", func() { err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Expect(proxyConfigFile).NotTo(BeAnExistingFile()) + Expect(proxyConfigFile).To(BeAnExistingFile()) }) Context("with containerProxyRequireClientCerts set to false", func() { @@ -941,14 +936,14 @@ var _ = Describe("ProxyConfigHandler", func() { Eventually(ch).Should(BeClosed()) }) - It("does not write a envoy config file", func() { + It("write a envoy config file", func() { err := proxyConfigHandler.Update(credentials, container) Expect(err).NotTo(HaveOccurred()) - Expect(proxyConfigFile).NotTo(BeAnExistingFile()) + Expect(proxyConfigFile).To(BeAnExistingFile()) }) - It("doesn't wait until the proxy is serving the new cert", func() { + It("wait until the proxy is serving the new cert", func() { ch := make(chan struct{}) go func() { defer GinkgoRecover() @@ -956,7 +951,7 @@ var _ = Describe("ProxyConfigHandler", func() { close(ch) }() - Eventually(ch).Should(BeClosed()) + Eventually(ch).ShouldNot(BeClosed()) }) }) diff --git a/depot/transformer/transformer_test.go b/depot/transformer/transformer_test.go index 731537ae..b46d89f4 100644 --- a/depot/transformer/transformer_test.go +++ b/depot/transformer/transformer_test.go @@ -562,6 +562,8 @@ var _ = Describe("Transformer", func() { livenessCh chan int actionProcess *gardenfakes.FakeProcess actionCh chan int + envoyProcess *gardenfakes.FakeProcess + envoyCh chan int monitorProcess *gardenfakes.FakeProcess monitorCh chan int startupIO chan garden.ProcessIO @@ -593,6 +595,9 @@ var _ = Describe("Transformer", func() { actionCh = make(chan int, 1) actionProcess = makeProcess(actionCh) + envoyCh = make(chan int) + envoyProcess = makeProcess(envoyCh) + monitorCh = make(chan int) monitorProcess = makeProcess(monitorCh) @@ -609,6 +614,8 @@ var _ = Describe("Transformer", func() { switch spec.Path { case "/action/path": return actionProcess, nil + case "sh": + return envoyProcess, nil case filepath.Join(transformer.HealthCheckDstPath, "healthcheck"): oldCount := atomic.AddInt64(&healthcheckCallCount, 1) switch oldCount { @@ -1613,6 +1620,8 @@ var _ = Describe("Transformer", func() { switch spec.Path { case "/action/path": return actionProcess, nil + case "sh": + return envoyProcess, nil case filepath.Join(transformer.HealthCheckDstPath, "healthcheck"): oldCount := atomic.AddInt64(&healthcheckCallCount, 1) switch oldCount { @@ -1651,7 +1660,7 @@ var _ = Describe("Transformer", func() { }) It("starts the proxy liveness check", func() { - Eventually(gardenContainer.RunCallCount).Should(Equal(5)) + Eventually(gardenContainer.RunCallCount).Should(Equal(6)) var ids []string var args [][]string for i := 0; i < gardenContainer.RunCallCount(); i++ { @@ -1671,7 +1680,7 @@ var _ = Describe("Transformer", func() { Context("and proxy liveness check is disabled", func() { It("does not start the proxy liveness check", func() { - Eventually(gardenContainer.RunCallCount).Should(Equal(4)) + Eventually(gardenContainer.RunCallCount).Should(Equal(5)) var ids []string var args [][]string for i := 0; i < gardenContainer.RunCallCount(); i++ { From e9a29983d0cc8781b56b8a1846abb26dfcf41712 Mon Sep 17 00:00:00 2001 From: karthick udayakumar Date: Mon, 22 Sep 2025 13:54:09 -0400 Subject: [PATCH 3/3] restore the enablecontainerproxy spec value --- containermetrics/reporters_runner_test.go | 358 +++++++++--------- containermetrics/stats_reporter.go | 10 +- depot/containerstore/containerstore_test.go | 1 + depot/containerstore/proxy_config_handler.go | 15 + .../proxy_config_handler_test.go | 148 +++++--- depot/transformer/transformer.go | 4 +- depot/transformer/transformer_test.go | 30 +- initializer/initializer.go | 41 +- initializer/initializer_test.go | 1 + resources.go | 1 + 10 files changed, 336 insertions(+), 273 deletions(-) diff --git a/containermetrics/reporters_runner_test.go b/containermetrics/reporters_runner_test.go index 630d1c99..291988f0 100644 --- a/containermetrics/reporters_runner_test.go +++ b/containermetrics/reporters_runner_test.go @@ -47,6 +47,7 @@ var _ = Describe("ReportersRunner", func() { testStart time.Time process ifrit.Process + enableContainerProxy bool proxyMemoryAllocationMB int reportersRunner *containermetrics.ReportersRunner statsReporter *containermetrics.StatsReporter @@ -63,13 +64,14 @@ var _ = Describe("ReportersRunner", func() { fakeExecutorClient = new(efakes.FakeClient) fakeMetronClient = new(mfakes.FakeIngressClient) + enableContainerProxy = false proxyMemoryAllocationMB = 5 metricsCache = &atomic.Value{} }) JustBeforeEach(func() { - statsReporter = containermetrics.NewStatsReporter(fakeMetronClient, float64(proxyMemoryAllocationMB*1024*1024), metricsCache) + statsReporter = containermetrics.NewStatsReporter(fakeMetronClient, enableContainerProxy, float64(proxyMemoryAllocationMB*1024*1024), metricsCache) cpuSpikeReporter := containermetrics.NewCPUSpikeReporter(fakeMetronClient) reportersRunner = containermetrics.NewReportersRunner(logger, interval, fakeClock, fakeExecutorClient, statsReporter, cpuSpikeReporter) process = ifrit.Invoke(reportersRunner) @@ -152,7 +154,7 @@ var _ = Describe("ReportersRunner", func() { MemoryUsageInBytes: megsToBytes(123), DiskUsageInBytes: megsToBytes(456), TimeSpentInCPU: 0, - MemoryLimitInBytes: megsToBytes(123), + MemoryLimitInBytes: megsToBytes(789), DiskLimitInBytes: megsToBytes(1024), ContainerAgeInNanoseconds: 1000, AbsoluteCPUEntitlementInNanoseconds: 0, @@ -166,7 +168,7 @@ var _ = Describe("ReportersRunner", func() { MemoryUsageInBytes: megsToBytes(123), DiskUsageInBytes: megsToBytes(456), TimeSpentInCPU: 0, - MemoryLimitInBytes: megsToBytes(123), + MemoryLimitInBytes: megsToBytes(789), DiskLimitInBytes: megsToBytes(1024), ContainerAgeInNanoseconds: 1001, AbsoluteCPUEntitlementInNanoseconds: 0, @@ -177,10 +179,10 @@ var _ = Describe("ReportersRunner", func() { "container-guid-with-index": { MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, ContainerMetrics: executor.ContainerMetrics{ - MemoryUsageInBytes: megsToBytes(123), + MemoryUsageInBytes: megsToBytes(321), DiskUsageInBytes: megsToBytes(654), TimeSpentInCPU: 0, - MemoryLimitInBytes: megsToBytes(123), + MemoryLimitInBytes: megsToBytes(987), DiskLimitInBytes: megsToBytes(2048), ContainerAgeInNanoseconds: 1002, AbsoluteCPUEntitlementInNanoseconds: 0, @@ -191,10 +193,10 @@ var _ = Describe("ReportersRunner", func() { "container-guid-without-preloaded-rootfs": { MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-preloaded-rootfs"}}, ContainerMetrics: executor.ContainerMetrics{ - MemoryUsageInBytes: megsToBytes(123), + MemoryUsageInBytes: megsToBytes(345), DiskUsageInBytes: megsToBytes(456), TimeSpentInCPU: 0, - MemoryLimitInBytes: megsToBytes(123), + MemoryLimitInBytes: megsToBytes(678), DiskLimitInBytes: megsToBytes(2048), ContainerAgeInNanoseconds: 1003, AbsoluteCPUEntitlementInNanoseconds: 0, @@ -397,63 +399,15 @@ var _ = Describe("ReportersRunner", func() { fakeExecutorClient.GetBulkMetricsReturnsOnCall(1, metricsAtT10, nil) fakeExecutorClient.GetBulkMetricsReturnsOnCall(2, metricsAtT20, nil) }) - BeforeEach(func() { - containers := []executor.Container{ - { - Guid: "container-guid-without-index", - MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index"}}, - }, - }, - { - Guid: "container-guid-with-index", - MemoryLimit: megsToBytes(400 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, - }, - }, - { - Guid: "container-guid-without-preloaded-rootfs", - MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-preloaded-rootfs"}}, - }, - }, - { - Guid: "container-guid-without-age", - MemoryLimit: megsToBytes(100 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-age"}}, - }, - }, - { - Guid: "container-guid-without-metrics-at-t0", - MemoryLimit: megsToBytes(400 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-metrics-at-t0", "instance_id": "1"}}, - }, - }, - } - fakeExecutorClient.ListContainersReturnsOnCall(0, containers, nil) - fakeExecutorClient.ListContainersReturnsOnCall(1, containers, nil) - fakeExecutorClient.ListContainersReturnsOnCall(2, containers, nil) - }) It("emits memory and disk usage for each container, but no CPU", func() { - appMemory := megsToBytes(123) - expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) - expectedMemoryUsageWithIndex, expectedMemoryLimitWithIndex := calculateMemoryUsageWithProxy(appMemory, 400.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes)) - expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) - expectedMemoryUsageWithoutAge, expectedMemoryLimitWithoutAge := calculateMemoryUsageWithProxy(appMemory, 100.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-age"].ContainerMetrics.MemoryLimitInBytes)) - Eventually(sentMetrics).Should(ConsistOf([]logging.ContainerMetric{ { CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), + MemoryBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), + MemoryBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -468,9 +422,9 @@ var _ = Describe("ReportersRunner", func() { { CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: uint64(expectedMemoryUsageWithIndex), + MemoryBytes: metricsAtT0["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT0["container-guid-with-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitWithIndex), + MemoryBytesQuota: metricsAtT0["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT0["container-guid-with-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-with-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-with-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -485,9 +439,9 @@ var _ = Describe("ReportersRunner", func() { { CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: uint64(expectedMemoryUsagePreloadedRootfs), + MemoryBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitPreloadedRootfs), + MemoryBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -502,9 +456,9 @@ var _ = Describe("ReportersRunner", func() { { CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: uint64(expectedMemoryUsageWithoutAge), + MemoryBytes: metricsAtT0["container-guid-without-age"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT0["container-guid-without-age"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitWithoutAge), + MemoryBytesQuota: metricsAtT0["container-guid-without-age"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT0["container-guid-without-age"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-age"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-age"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -528,52 +482,113 @@ var _ = Describe("ReportersRunner", func() { )) }) - It("emits a rescaled memory usage based on the additional memory allocation for the proxy", func() { - appMemory := megsToBytes(123) - expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) - - Eventually(sentMetrics).Should(ContainElement( - logging.ContainerMetric{ - CpuPercentage: 0.0, - MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), - DiskBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), - DiskBytesQuota: metricsAtT0["container-guid-without-index"].DiskLimitInBytes, - AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), - AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, - ContainerAge: metricsAtT0["container-guid-without-index"].ContainerMetrics.ContainerAgeInNanoseconds, - RxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.RxInBytes, - TxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.TxInBytes, - Tags: map[string]string{ - "source_id": "source-id-without-index", - "instance_id": "0", + Context("when containers EnableContainerProxy is set", func() { + BeforeEach(func() { + containers := []executor.Container{ + { + Guid: "container-guid-without-index", + MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + EnableContainerProxy: true, + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index"}}, + }, }, - }, - )) - }) - - Context("when there is a container without preloaded rootfs", func() { - It("should emit memory usage that is not rescaled", func() { - appMemory := megsToBytes(123) - expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) - Eventually(sentMetrics).Should(ContainElement( - logging.ContainerMetric{ - CpuPercentage: 0.0, - MemoryBytes: uint64(expectedMemoryUsagePreloadedRootfs), - DiskBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitPreloadedRootfs), - DiskBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, - AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), - AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, - ContainerAge: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.ContainerAgeInNanoseconds, - RxBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.RxInBytes, - TxBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TxInBytes, - Tags: map[string]string{ - "source_id": "source-id-without-preloaded-rootfs", - "instance_id": "0", + { + Guid: "container-guid-with-index", + MemoryLimit: megsToBytes(400 + proxyMemoryAllocationMB), + RunInfo: executor.RunInfo{ + EnableContainerProxy: true, + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, }, }, - )) + { + Guid: "container-guid-without-preloaded-rootfs", + MemoryLimit: megsToBytes(200), + RunInfo: executor.RunInfo{ + EnableContainerProxy: false, + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-preloaded-rootfs"}}, + }, + }, + } + fakeExecutorClient.ListContainersReturnsOnCall(0, containers, nil) + }) + + Context("when enableContainerProxy not set", func() { + It("does not change the memory usage reported by garden", func() { + appMemory := megsToBytes(123) + expectedMemoryUsageWithoutIndex := float64(appMemory) + Eventually(sentMetrics).Should(ContainElement( + logging.ContainerMetric{ + CpuPercentage: 0.0, + MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), + DiskBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, + MemoryBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes, + DiskBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, + AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), + AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, + ContainerAge: metricsAtT0["container-guid-without-index"].ContainerMetrics.ContainerAgeInNanoseconds, + RxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.RxInBytes, + TxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.TxInBytes, + Tags: map[string]string{ + "source_id": "source-id-without-index", + "instance_id": "0", + }, + }, + )) + }) + }) + + Context("when enableContainerProxy is set", func() { + BeforeEach(func() { + enableContainerProxy = true + }) + + It("emits a rescaled memory usage based on the additional memory allocation for the proxy", func() { + appMemory := megsToBytes(123) + expectedMemoryUsageWithoutIndex := float64(appMemory) * 200.0 / (200.0 + float64(proxyMemoryAllocationMB)) + expectedMemoryLimitWithoutIndex := float64(metricsAtT0["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes) - float64(megsToBytes(proxyMemoryAllocationMB)) + Eventually(sentMetrics).Should(ContainElement( + logging.ContainerMetric{ + CpuPercentage: 0.0, + MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), + DiskBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, + MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), + DiskBytesQuota: metricsAtT0["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, + AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), + AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, + ContainerAge: metricsAtT0["container-guid-without-index"].ContainerMetrics.ContainerAgeInNanoseconds, + RxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.RxInBytes, + TxBytes: metricsAtT0["container-guid-without-index"].ContainerMetrics.TxInBytes, + Tags: map[string]string{ + "source_id": "source-id-without-index", + "instance_id": "0", + }, + }, + )) + }) + + Context("when there is a container without preloaded rootfs", func() { + It("should emit memory usage that is not rescaled", func() { + Eventually(sentMetrics).Should(ContainElement( + logging.ContainerMetric{ + CpuPercentage: 0.0, + MemoryBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, + DiskBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, + MemoryBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, + DiskBytesQuota: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, + AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), + AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, + ContainerAge: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.ContainerAgeInNanoseconds, + RxBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.RxInBytes, + TxBytes: metricsAtT0["container-guid-without-preloaded-rootfs"].ContainerMetrics.TxInBytes, + Tags: map[string]string{ + "source_id": "source-id-without-preloaded-rootfs", + "instance_id": "0", + }, + }, + )) + }) + }) }) }) @@ -593,13 +608,12 @@ var _ = Describe("ReportersRunner", func() { }) It("emits the new memory and disk usage, and the computed CPU percent", func() { - expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 20.0, CpuEntitlementPercentage: 40.0, - MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), + MemoryBytes: metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT10["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), + MemoryBytesQuota: metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT10["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT10["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT10["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -612,13 +626,12 @@ var _ = Describe("ReportersRunner", func() { }, })) - expectedMemoryUsageWithIndex, expectedMemoryLimitWithIndex := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 30.0, CpuEntitlementPercentage: 60.0, - MemoryBytes: uint64(expectedMemoryUsageWithIndex), + MemoryBytes: metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT10["container-guid-with-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitWithIndex), + MemoryBytesQuota: metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT10["container-guid-with-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT10["container-guid-with-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT10["container-guid-with-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -631,13 +644,12 @@ var _ = Describe("ReportersRunner", func() { }, })) - expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 40.0, CpuEntitlementPercentage: 80.0, - MemoryBytes: uint64(expectedMemoryUsagePreloadedRootfs), + MemoryBytes: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitPreloadedRootfs), + MemoryBytesQuota: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -650,13 +662,12 @@ var _ = Describe("ReportersRunner", func() { }, })) - expectedMemoryUsageNoMetricsAtT0, expectedMemoryLimitNoMetricsAtT0 := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 0.0, CpuEntitlementPercentage: 0.0, - MemoryBytes: uint64(expectedMemoryUsageNoMetricsAtT0), + MemoryBytes: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitNoMetricsAtT0), + MemoryBytesQuota: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -673,60 +684,63 @@ var _ = Describe("ReportersRunner", func() { Context("Metrics", func() { It("returns the cached metrics last emitted", func() { containerMetrics := statsReporter.Metrics() - Expect(containerMetrics).To(HaveLen(5)) - - expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) + Expect(containerMetrics).To(HaveLen(6)) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-index", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-without-index", CPUUsageFraction: 0.2, - MemoryUsageBytes: uint64(expectedMemoryUsageWithoutIndex), + MemoryUsageBytes: megsToBytes(1230), DiskUsageBytes: 4560, - MemoryQuotaBytes: uint64(expectedMemoryLimitWithoutIndex), + MemoryQuotaBytes: megsToBytes(7890), DiskQuotaBytes: 4096, RxBytes: &two, TxBytes: &two, })) - - expectedMemoryUsageWithIndex, expectedMemoryLimitWithIndex := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes)) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-with-index", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-with-index", CPUUsageFraction: 0.3, - MemoryUsageBytes: uint64(expectedMemoryUsageWithIndex), + MemoryUsageBytes: megsToBytes(3210), DiskUsageBytes: 6540, - MemoryQuotaBytes: uint64(expectedMemoryLimitWithIndex), + MemoryQuotaBytes: megsToBytes(9870), DiskQuotaBytes: 512, RxBytes: &two, TxBytes: &two, })) - expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) + Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-source-id", &containermetrics.CachedContainerMetrics{ + MetricGUID: "", + CPUUsageFraction: 0.1, + MemoryUsageBytes: megsToBytes(123), + DiskUsageBytes: megsToBytes(456), + MemoryQuotaBytes: megsToBytes(789), + DiskQuotaBytes: megsToBytes(1024), + RxBytes: &two, + TxBytes: &two, + })) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-preloaded-rootfs", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-without-preloaded-rootfs", CPUUsageFraction: 0.4, - MemoryUsageBytes: uint64(expectedMemoryUsagePreloadedRootfs), + MemoryUsageBytes: megsToBytes(3450), DiskUsageBytes: 4560, - MemoryQuotaBytes: uint64(expectedMemoryLimitPreloadedRootfs), + MemoryQuotaBytes: megsToBytes(6780), DiskQuotaBytes: 2048, RxBytes: &two, TxBytes: &two, })) - expectedMemoryUsageWithoutAge, expectedMemoryLimitWithoutAge := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-age"].ContainerMetrics.MemoryUsageInBytes, 100.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-age"].ContainerMetrics.MemoryLimitInBytes)) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-age", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-without-age", CPUUsageFraction: 0.5, - MemoryUsageBytes: uint64(expectedMemoryUsageWithoutAge), + MemoryUsageBytes: megsToBytes(123), DiskUsageBytes: megsToBytes(456), - MemoryQuotaBytes: uint64(expectedMemoryLimitWithoutAge), + MemoryQuotaBytes: megsToBytes(789), DiskQuotaBytes: megsToBytes(1024), RxBytes: &two, TxBytes: &two, })) - expectedMemoryUsageNoMetricsAtT0, expectedMemoryLimitNoMetricsAtT0 := calculateMemoryUsageWithProxy(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT10["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes)) Expect(containerMetrics).To(HaveKeyWithValue("container-guid-without-metrics-at-t0", &containermetrics.CachedContainerMetrics{ MetricGUID: "source-id-without-metrics-at-t0", CPUUsageFraction: 0.0, - MemoryUsageBytes: uint64(expectedMemoryUsageNoMetricsAtT0), + MemoryUsageBytes: megsToBytes(3210), DiskUsageBytes: 6540, - MemoryQuotaBytes: uint64(expectedMemoryLimitNoMetricsAtT0), + MemoryQuotaBytes: megsToBytes(9870), DiskQuotaBytes: 512, RxBytes: &two, TxBytes: &two, @@ -741,13 +755,12 @@ var _ = Describe("ReportersRunner", func() { }) It("emits the new memory and disk usage, and the computed CPU percent", func() { - expectedMemoryUsageWithoutIndex, expectedMemoryLimitWithoutIndex := calculateMemoryUsageWithProxy(metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuEntitlementPercentage: 40.0, CpuPercentage: 20.0, - MemoryBytes: uint64(expectedMemoryUsageWithoutIndex), + MemoryBytes: metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT20["container-guid-without-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitWithoutIndex), + MemoryBytesQuota: metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT20["container-guid-without-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT20["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT20["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -760,13 +773,12 @@ var _ = Describe("ReportersRunner", func() { }, })) - expectedMemoryUsageWithIndex, expectedMemoryLimitWithIndex := calculateMemoryUsageWithProxy(metricsAtT20["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuEntitlementPercentage: 60.0, CpuPercentage: 30.0, - MemoryBytes: uint64(expectedMemoryUsageWithIndex), + MemoryBytes: metricsAtT20["container-guid-with-index"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT20["container-guid-with-index"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitWithIndex), + MemoryBytesQuota: metricsAtT20["container-guid-with-index"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT20["container-guid-with-index"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT20["container-guid-with-index"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT20["container-guid-with-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -779,13 +791,12 @@ var _ = Describe("ReportersRunner", func() { }, })) - expectedMemoryUsagePreloadedRootfs, expectedMemoryLimitPreloadedRootfs := calculateMemoryUsageWithProxy(metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, 200.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuEntitlementPercentage: 80.0, CpuPercentage: 40.0, - MemoryBytes: uint64(expectedMemoryUsagePreloadedRootfs), + MemoryBytes: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitPreloadedRootfs), + MemoryBytesQuota: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT20["container-guid-without-preloaded-rootfs"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -798,13 +809,12 @@ var _ = Describe("ReportersRunner", func() { }, })) - expectedMemoryUsageNoMetricsAtT0, expectedMemoryLimitNoMetricsAtT0 := calculateMemoryUsageWithProxy(metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, 400.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuEntitlementPercentage: 120.0, CpuPercentage: 60.0, - MemoryBytes: uint64(expectedMemoryUsageNoMetricsAtT0), + MemoryBytes: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryUsageInBytes, DiskBytes: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.DiskUsageInBytes, - MemoryBytesQuota: uint64(expectedMemoryLimitNoMetricsAtT0), + MemoryBytesQuota: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.MemoryLimitInBytes, DiskBytesQuota: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.DiskLimitInBytes, AbsoluteCPUUsage: uint64(metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.TimeSpentInCPU), AbsoluteCPUEntitlement: metricsAtT20["container-guid-without-metrics-at-t0"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, @@ -1006,15 +1016,14 @@ var _ = Describe("ReportersRunner", func() { Context("Reporters reset the container list", func() { var metricsAtT0, metricsAtT10, metricsAtT20 map[string]executor.Metrics + BeforeEach(func() { containers1 := []executor.Container{ - {Guid: "container-guid-without-index"}, - } - containers2 := []executor.Container{ - {Guid: "container-guid-without-index"}, + {Guid: "container-guid"}, } + containers2 := []executor.Container{} containers3 := []executor.Container{ - {Guid: "container-guid-without-index"}, + {Guid: "container-guid"}, } fakeExecutorClient.ListContainersReturnsOnCall(0, containers1, nil) @@ -1022,14 +1031,12 @@ var _ = Describe("ReportersRunner", func() { fakeExecutorClient.ListContainersReturnsOnCall(2, containers3, nil) metricsAtT0 = map[string]executor.Metrics{ - "container-guid-without-index": executor.Metrics{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index", "instance_id": "1"}}, + "container-guid": executor.Metrics{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, ContainerMetrics: executor.ContainerMetrics{ TimeSpentInCPU: 2000 * time.Nanosecond, ContainerAgeInNanoseconds: 1000, AbsoluteCPUEntitlementInNanoseconds: 1000, - MemoryUsageInBytes: megsToBytes(123), - MemoryLimitInBytes: megsToBytes(789), }, }, } @@ -1037,14 +1044,12 @@ var _ = Describe("ReportersRunner", func() { metricsAtT10 = map[string]executor.Metrics{} metricsAtT20 = map[string]executor.Metrics{ - "container-guid-without-index": executor.Metrics{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index", "instance_id": "1"}}, + "container-guid": executor.Metrics{ + MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-with-index", "instance_id": "1"}}, ContainerMetrics: executor.ContainerMetrics{ TimeSpentInCPU: 2600 * time.Nanosecond, ContainerAgeInNanoseconds: 2000, AbsoluteCPUEntitlementInNanoseconds: 3000, - MemoryLimitInBytes: megsToBytes(789), - MemoryUsageInBytes: megsToBytes(123), }, }, } @@ -1053,18 +1058,7 @@ var _ = Describe("ReportersRunner", func() { fakeExecutorClient.GetBulkMetricsReturnsOnCall(1, metricsAtT10, nil) fakeExecutorClient.GetBulkMetricsReturnsOnCall(2, metricsAtT20, nil) }) - BeforeEach(func() { - containers := []executor.Container{ - { - Guid: "container-guid-without-index", - MemoryLimit: megsToBytes(200 + proxyMemoryAllocationMB), - RunInfo: executor.RunInfo{ - MetricsConfig: executor.MetricsConfig{Tags: map[string]string{"source_id": "source-id-without-index", "instance_id": "1"}}, - }, - }, - } - fakeExecutorClient.ListContainersReturnsOnCall(0, containers, nil) - }) + It("does not send a spike metric when the container is gone", func() { fakeClock.WaitForWatcherAndIncrement(interval) fakeClock.WaitForWatcherAndIncrement(interval) @@ -1073,20 +1067,16 @@ var _ = Describe("ReportersRunner", func() { }) It("should report 0 CPU", func() { - appMemory := megsToBytes(123) - expectedMemory, expectedMemoryLimit := calculateMemoryUsageWithProxy(appMemory, 200.0, proxyMemoryAllocationMB, float64(metricsAtT20["container-guid-without-index"].ContainerMetrics.MemoryLimitInBytes)) fakeClock.WaitForWatcherAndIncrement(interval) fakeClock.WaitForWatcherAndIncrement(interval) Eventually(fakeExecutorClient.GetBulkMetricsCallCount).Should(Equal(3)) Eventually(sentMetrics).Should(ContainElement(logging.ContainerMetric{ CpuPercentage: 0.0, - AbsoluteCPUUsage: uint64(metricsAtT0["container-guid-without-index"].ContainerMetrics.TimeSpentInCPU), - AbsoluteCPUEntitlement: metricsAtT0["container-guid-without-index"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, - ContainerAge: metricsAtT0["container-guid-without-index"].ContainerMetrics.ContainerAgeInNanoseconds, - MemoryBytes: uint64(expectedMemory), - MemoryBytesQuota: uint64(expectedMemoryLimit), + AbsoluteCPUUsage: uint64(metricsAtT20["container-guid"].ContainerMetrics.TimeSpentInCPU), + AbsoluteCPUEntitlement: metricsAtT20["container-guid"].ContainerMetrics.AbsoluteCPUEntitlementInNanoseconds, + ContainerAge: metricsAtT20["container-guid"].ContainerMetrics.ContainerAgeInNanoseconds, Tags: map[string]string{ - "source_id": "source-id-without-index", + "source_id": "source-id-with-index", "instance_id": "1", }, })) @@ -1243,9 +1233,3 @@ var _ = Describe("ReportersRunner", func() { }) }) - -func calculateMemoryUsageWithProxy(appMemory uint64, memoryLimit float64, proxyMemoryAllocationMB int, memoryLimitInBytes float64) (expectedMemoryUsage float64, expectedMemoryLimit float64) { - expectedMemoryUsage = float64(appMemory) * memoryLimit / (memoryLimit + float64(proxyMemoryAllocationMB)) - expectedMemoryLimit = float64(memoryLimitInBytes) - float64(megsToBytes(proxyMemoryAllocationMB)) - return -} diff --git a/containermetrics/stats_reporter.go b/containermetrics/stats_reporter.go index ca090170..38880c07 100644 --- a/containermetrics/stats_reporter.go +++ b/containermetrics/stats_reporter.go @@ -19,13 +19,15 @@ type cpuInfo struct { type StatsReporter struct { cpuInfos map[string]*cpuInfo metronClient loggingclient.IngressClient + enableContainerProxy bool proxyMemoryAllocation float64 metricsCache *atomic.Value } -func NewStatsReporter(metronClient loggingclient.IngressClient, proxyMemoryAllocation float64, metricsCache *atomic.Value) *StatsReporter { +func NewStatsReporter(metronClient loggingclient.IngressClient, enableContainerProxy bool, proxyMemoryAllocation float64, metricsCache *atomic.Value) *StatsReporter { return &StatsReporter{ cpuInfos: make(map[string]*cpuInfo), + enableContainerProxy: enableContainerProxy, metronClient: metronClient, proxyMemoryAllocation: proxyMemoryAllocation, metricsCache: metricsCache, @@ -55,8 +57,10 @@ func (reporter *StatsReporter) Report(logger lager.Logger, containers []executor previousCPUInfo := cpuInfos[guid] - metric.MemoryUsageInBytes = uint64(float64(metric.MemoryUsageInBytes) * reporter.scaleMemory(container)) - metric.MemoryLimitInBytes = uint64(float64(metric.MemoryLimitInBytes) - reporter.proxyMemoryAllocation) + if reporter.enableContainerProxy && container.EnableContainerProxy { + metric.MemoryUsageInBytes = uint64(float64(metric.MemoryUsageInBytes) * reporter.scaleMemory(container)) + metric.MemoryLimitInBytes = uint64(float64(metric.MemoryLimitInBytes) - reporter.proxyMemoryAllocation) + } repMetrics, cpu := reporter.calculateAndSendMetrics(logger, metric.MetricsConfig, metric.ContainerMetrics, previousCPUInfo, timeStamp) if cpu != nil { diff --git a/depot/containerstore/containerstore_test.go b/depot/containerstore/containerstore_test.go index 8d8e72d6..0c6eb044 100644 --- a/depot/containerstore/containerstore_test.go +++ b/depot/containerstore/containerstore_test.go @@ -410,6 +410,7 @@ var _ = Describe("Container Store", func() { "some-other-key": "some-other-value", }, }, + EnableContainerProxy: true, LogRateLimitBytesPerSecond: logRateUnlimitedBytesPerSecond, VolumeMountedFiles: volumeMountedFiles, } diff --git a/depot/containerstore/proxy_config_handler.go b/depot/containerstore/proxy_config_handler.go index 38dfe2f5..fbb54b33 100644 --- a/depot/containerstore/proxy_config_handler.go +++ b/depot/containerstore/proxy_config_handler.go @@ -149,6 +149,9 @@ func NewProxyConfigHandler( // This modifies the container pointer in order to create garden NetIn rules in the storenode.Create func (p *ProxyConfigHandler) ProxyPorts(logger lager.Logger, container *executor.Container) ([]executor.ProxyPortMapping, []uint16, error) { + if !container.EnableContainerProxy { + return nil, nil, nil + } proxyPortMapping := []executor.ProxyPortMapping{} @@ -199,6 +202,9 @@ func (p *ProxyConfigHandler) ProxyPorts(logger lager.Logger, container *executor } func (p *ProxyConfigHandler) CreateDir(logger lager.Logger, container executor.Container) ([]garden.BindMount, []executor.EnvironmentVariable, error) { + if !container.EnableContainerProxy { + return nil, nil, nil + } logger.Info("adding-container-proxy-bindmounts") proxyConfigDir := filepath.Join(p.containerProxyConfigPath, container.Guid) @@ -224,6 +230,9 @@ func (p *ProxyConfigHandler) CreateDir(logger lager.Logger, container executor.C } func (p *ProxyConfigHandler) RemoveDir(logger lager.Logger, container executor.Container) error { + if !container.EnableContainerProxy { + return nil + } logger.Info("removing-container-proxy-config-dir") proxyConfigDir := filepath.Join(p.containerProxyConfigPath, container.Guid) @@ -231,11 +240,17 @@ func (p *ProxyConfigHandler) RemoveDir(logger lager.Logger, container executor.C } func (p *ProxyConfigHandler) Update(credentials Credentials, container executor.Container) error { + if !container.EnableContainerProxy { + return nil + } return p.writeConfig(credentials, container) } func (p *ProxyConfigHandler) Close(invalidCredentials Credentials, container executor.Container) error { + if !container.EnableContainerProxy { + return nil + } err := p.writeConfig(invalidCredentials, container) if err != nil { diff --git a/depot/containerstore/proxy_config_handler_test.go b/depot/containerstore/proxy_config_handler_test.go index 5e3673a4..fc263517 100644 --- a/depot/containerstore/proxy_config_handler_test.go +++ b/depot/containerstore/proxy_config_handler_test.go @@ -72,7 +72,9 @@ var _ = Describe("ProxyConfigHandler", func() { container = executor.Container{ Guid: fmt.Sprintf("container-guid-%d", GinkgoParallelProcess()), InternalIP: "10.0.0.1", - RunInfo: executor.RunInfo{}, + RunInfo: executor.RunInfo{ + EnableContainerProxy: true, + }, } proxyConfigDir, err = os.MkdirTemp("", "proxymanager-config") @@ -195,39 +197,53 @@ var _ = Describe("ProxyConfigHandler", func() { }) }) - It("returns the appropriate bind mounts for container proxy", func() { - mounts, _, err := proxyConfigHandler.CreateDir(logger, container) - Expect(err).NotTo(HaveOccurred()) - Expect(mounts).To(ConsistOf([]garden.BindMount{ - { - Origin: garden.BindMountOriginHost, - SrcPath: proxyDir, - DstPath: "/etc/cf-assets/envoy", - }, - { - Origin: garden.BindMountOriginHost, - SrcPath: filepath.Join(proxyConfigDir, container.Guid), - DstPath: "/etc/cf-assets/envoy_config", - }, - })) - }) + Describe("CreateDir", func() { + Context("the EnableContainerProxy is disabled on the container", func() { + BeforeEach(func() { + container.EnableContainerProxy = false + }) - It("makes the proxy config directory on the host", func() { - _, _, err := proxyConfigHandler.CreateDir(logger, container) - Expect(err).NotTo(HaveOccurred()) - proxyConfigDir := fmt.Sprintf("%s/%s", proxyConfigDir, container.Guid) - Expect(proxyConfigDir).To(BeADirectory()) - }) + It("returns an empty bind mount", func() { + mounts, _, err := proxyConfigHandler.CreateDir(logger, container) + Expect(err).NotTo(HaveOccurred()) + Expect(mounts).To(BeEmpty()) + }) + }) - Context("when the manager fails to create the proxy config directory", func() { - BeforeEach(func() { - _, err := os.Create(configPath) + It("returns the appropriate bind mounts for container proxy", func() { + mounts, _, err := proxyConfigHandler.CreateDir(logger, container) Expect(err).NotTo(HaveOccurred()) + Expect(mounts).To(ConsistOf([]garden.BindMount{ + { + Origin: garden.BindMountOriginHost, + SrcPath: proxyDir, + DstPath: "/etc/cf-assets/envoy", + }, + { + Origin: garden.BindMountOriginHost, + SrcPath: filepath.Join(proxyConfigDir, container.Guid), + DstPath: "/etc/cf-assets/envoy_config", + }, + })) }) - It("returns an error", func() { + It("makes the proxy config directory on the host", func() { _, _, err := proxyConfigHandler.CreateDir(logger, container) - Expect(err).To(HaveOccurred()) + Expect(err).NotTo(HaveOccurred()) + proxyConfigDir := fmt.Sprintf("%s/%s", proxyConfigDir, container.Guid) + Expect(proxyConfigDir).To(BeADirectory()) + }) + + Context("when the manager fails to create the proxy config directory", func() { + BeforeEach(func() { + _, err := os.Create(configPath) + Expect(err).NotTo(HaveOccurred()) + }) + + It("returns an error", func() { + _, _, err := proxyConfigHandler.CreateDir(logger, container) + Expect(err).To(HaveOccurred()) + }) }) }) @@ -242,11 +258,16 @@ var _ = Describe("ProxyConfigHandler", func() { Expect(configPath).NotTo(BeADirectory()) }) - It("does not return an error when deleting a non existing directory", func() { - err := proxyConfigHandler.RemoveDir(logger, container) - Expect(err).NotTo(HaveOccurred()) - }) + Context("the EnableContainerProxy is disabled on the container", func() { + BeforeEach(func() { + container.EnableContainerProxy = false + }) + It("does not return an error when deleting a non existing directory", func() { + err := proxyConfigHandler.RemoveDir(logger, container) + Expect(err).NotTo(HaveOccurred()) + }) + }) }) Describe("ProxyPorts", func() { @@ -256,11 +277,17 @@ var _ = Describe("ProxyConfigHandler", func() { } }) - It("returns a non empty proxy port mapping", func() { - ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) - Expect(err).NotTo(HaveOccurred()) - Expect(ports).NotTo(BeEmpty()) - Expect(extraPorts).NotTo(BeEmpty()) + Context("the EnableContainerProxy is disabled on the container", func() { + BeforeEach(func() { + container.EnableContainerProxy = false + }) + + It("returns an empty proxy port mapping", func() { + ports, extraPorts, err := proxyConfigHandler.ProxyPorts(logger, &container) + Expect(err).NotTo(HaveOccurred()) + Expect(ports).To(BeEmpty()) + Expect(extraPorts).To(BeEmpty()) + }) }) Context("when there is a default HTTP port (8080)", func() { @@ -402,11 +429,17 @@ var _ = Describe("ProxyConfigHandler", func() { containerProxyRequireClientCerts = true }) - It("write a envoy config file", func() { - err := proxyConfigHandler.Update(credentials, container) - Expect(err).NotTo(HaveOccurred()) + Context("the EnableContainerProxy is disabled on the container", func() { + BeforeEach(func() { + container.EnableContainerProxy = false + }) - Expect(proxyConfigFile).To(BeAnExistingFile()) + It("does not write a envoy config file", func() { + err := proxyConfigHandler.Update(credentials, container) + Expect(err).NotTo(HaveOccurred()) + + Expect(proxyConfigFile).NotTo(BeAnExistingFile()) + }) }) Context("with containerProxyRequireClientCerts set to false", func() { @@ -936,24 +969,29 @@ var _ = Describe("ProxyConfigHandler", func() { Eventually(ch).Should(BeClosed()) }) - It("write a envoy config file", func() { - err := proxyConfigHandler.Update(credentials, container) - Expect(err).NotTo(HaveOccurred()) + Context("the EnableContainerProxy is disabled on the container", func() { + BeforeEach(func() { + container.EnableContainerProxy = false + }) - Expect(proxyConfigFile).To(BeAnExistingFile()) - }) + It("does not write a envoy config file", func() { + err := proxyConfigHandler.Update(credentials, container) + Expect(err).NotTo(HaveOccurred()) - It("wait until the proxy is serving the new cert", func() { - ch := make(chan struct{}) - go func() { - defer GinkgoRecover() - proxyConfigHandler.Close(credentials, container) - close(ch) - }() + Expect(proxyConfigFile).NotTo(BeAnExistingFile()) + }) - Eventually(ch).ShouldNot(BeClosed()) - }) + It("doesn't wait until the proxy is serving the new cert", func() { + ch := make(chan struct{}) + go func() { + defer GinkgoRecover() + proxyConfigHandler.Close(credentials, container) + close(ch) + }() + Eventually(ch).Should(BeClosed()) + }) + }) }) }) diff --git a/depot/transformer/transformer.go b/depot/transformer/transformer.go index 41663879..55f64c63 100644 --- a/depot/transformer/transformer.go +++ b/depot/transformer/transformer.go @@ -473,7 +473,7 @@ func (t *transformer) StepsRunner( var proxyStartupChecks []ifrit.Runner var proxyLivenessChecks []ifrit.Runner - if t.useContainerProxy { + if t.useContainerProxy && container.EnableContainerProxy { envoyStartupLogger := logger.Session("envoy-startup-check") envoyLivenessLogger := logger.Session("envoy-liveness-check") @@ -589,7 +589,7 @@ func (t *transformer) StepsRunner( longLivedAction = action } - if t.useContainerProxy { + if t.useContainerProxy && container.EnableContainerProxy { containerProxyStep := t.transformContainerProxyStep( gardenContainer, container, diff --git a/depot/transformer/transformer_test.go b/depot/transformer/transformer_test.go index b46d89f4..35226acd 100644 --- a/depot/transformer/transformer_test.go +++ b/depot/transformer/transformer_test.go @@ -397,6 +397,7 @@ var _ = Describe("Transformer", func() { ContainerPort: 61001, }, }, + EnableContainerProxy: true, }, } }) @@ -551,6 +552,22 @@ var _ = Describe("Transformer", func() { }) }) + Context("when the container proxy is disabled on the container", func() { + BeforeEach(func() { + container.EnableContainerProxy = false + }) + + It("does not run the container proxy", func() { + Eventually(gardenContainer.RunCallCount).Should(Equal(2)) + paths := []string{} + for i := 0; i < gardenContainer.RunCallCount(); i++ { + spec, _ := gardenContainer.RunArgsForCall(i) + paths = append(paths, spec.Path) + } + + Expect(paths).NotTo(ContainElement("sh")) + }) + }) }) Describe("declarative healthchecks", func() { @@ -562,8 +579,6 @@ var _ = Describe("Transformer", func() { livenessCh chan int actionProcess *gardenfakes.FakeProcess actionCh chan int - envoyProcess *gardenfakes.FakeProcess - envoyCh chan int monitorProcess *gardenfakes.FakeProcess monitorCh chan int startupIO chan garden.ProcessIO @@ -595,9 +610,6 @@ var _ = Describe("Transformer", func() { actionCh = make(chan int, 1) actionProcess = makeProcess(actionCh) - envoyCh = make(chan int) - envoyProcess = makeProcess(envoyCh) - monitorCh = make(chan int) monitorProcess = makeProcess(monitorCh) @@ -614,8 +626,6 @@ var _ = Describe("Transformer", func() { switch spec.Path { case "/action/path": return actionProcess, nil - case "sh": - return envoyProcess, nil case filepath.Join(transformer.HealthCheckDstPath, "healthcheck"): oldCount := atomic.AddInt64(&healthcheckCallCount, 1) switch oldCount { @@ -1620,8 +1630,6 @@ var _ = Describe("Transformer", func() { switch spec.Path { case "/action/path": return actionProcess, nil - case "sh": - return envoyProcess, nil case filepath.Join(transformer.HealthCheckDstPath, "healthcheck"): oldCount := atomic.AddInt64(&healthcheckCallCount, 1) switch oldCount { @@ -1660,7 +1668,7 @@ var _ = Describe("Transformer", func() { }) It("starts the proxy liveness check", func() { - Eventually(gardenContainer.RunCallCount).Should(Equal(6)) + Eventually(gardenContainer.RunCallCount).Should(Equal(5)) var ids []string var args [][]string for i := 0; i < gardenContainer.RunCallCount(); i++ { @@ -1680,7 +1688,7 @@ var _ = Describe("Transformer", func() { Context("and proxy liveness check is disabled", func() { It("does not start the proxy liveness check", func() { - Eventually(gardenContainer.RunCallCount).Should(Equal(5)) + Eventually(gardenContainer.RunCallCount).Should(Equal(4)) var ids []string var args [][]string for i := 0; i < gardenContainer.RunCallCount(); i++ { diff --git a/initializer/initializer.go b/initializer/initializer.go index cf99399b..85b6c87a 100644 --- a/initializer/initializer.go +++ b/initializer/initializer.go @@ -101,6 +101,7 @@ type ExecutorConfig struct { DeclarativeHealthcheckPath string `json:"declarative_healthcheck_path,omitempty"` DeleteWorkPoolSize int `json:"delete_work_pool_size,omitempty"` DiskMB string `json:"disk_mb,omitempty"` + EnableContainerProxy bool `json:"enable_container_proxy,omitempty"` EnableContainerProxyHealthChecks bool `json:"enable_container_proxy_healthcheck,omitempty"` DeclarativeHealthCheckDefaultTimeout durationjson.Duration `json:"declarative_healthcheck_default_timeout,omitempty"` EnableHealtcheckMetrics bool `json:"enable_healthcheck_metrics,omitempty"` @@ -262,6 +263,7 @@ func Initialize( config.PostSetupUser, config.EnableHealtcheckMetrics, sidecarRootFSPath, + config.EnableContainerProxy, time.Duration(config.EnvoyDrainTimeout), config.EnableContainerProxyHealthChecks, time.Duration(config.ProxyHealthCheckInterval), @@ -294,18 +296,23 @@ func Initialize( driverConfig.DriverPaths = filepath.SplitList(config.VolmanDriverPaths) volmanClient, volmanDriverSyncer := vollocal.NewServer(logger, metronClient, driverConfig) - proxyConfigHandler := containerstore.NewProxyConfigHandler( - logger, - config.ContainerProxyPath, - config.ContainerProxyConfigPath, - config.ContainerProxyTrustedCACerts, - config.ContainerProxyVerifySubjectAltName, - config.ContainerProxyRequireClientCerts, - time.Duration(config.EnvoyConfigReloadDuration), - clock, - config.ContainerProxyADSServers, - config.ProxyEnableHttp2, - ) + var proxyConfigHandler containerstore.ProxyManager + if config.EnableContainerProxy { + proxyConfigHandler = containerstore.NewProxyConfigHandler( + logger, + config.ContainerProxyPath, + config.ContainerProxyConfigPath, + config.ContainerProxyTrustedCACerts, + config.ContainerProxyVerifySubjectAltName, + config.ContainerProxyRequireClientCerts, + time.Duration(config.EnvoyConfigReloadDuration), + clock, + config.ContainerProxyADSServers, + config.ProxyEnableHttp2, + ) + } else { + proxyConfigHandler = containerstore.NewNoopProxyConfigHandler() + } instanceIdentityHandler := containerstore.NewInstanceIdentityHandler( config.InstanceIdentityCredDir, @@ -380,6 +387,7 @@ func Initialize( metricsCache := &atomic.Value{} containerStatsReporter := containermetrics.NewStatsReporter( metronClient, + config.EnableContainerProxy, float64(config.ProxyMemoryAllocationMB*megabytesToBytes), metricsCache, ) @@ -560,6 +568,7 @@ func initializeTransformer( postSetupUser string, emitHealthCheckMetrics bool, declarativeHealthcheckRootFS string, + enableContainerProxy bool, drainWait time.Duration, enableProxyHealthChecks bool, proxyHealthCheckInterval time.Duration, @@ -577,10 +586,12 @@ func initializeTransformer( options = append(options, transformer.WithDeclarativeHealthcheckFailureMetrics()) } - options = append(options, transformer.WithContainerProxy(drainWait)) + if enableContainerProxy { + options = append(options, transformer.WithContainerProxy(drainWait)) - if enableProxyHealthChecks { - options = append(options, transformer.WithProxyLivenessChecks(proxyHealthCheckInterval)) + if enableProxyHealthChecks { + options = append(options, transformer.WithProxyLivenessChecks(proxyHealthCheckInterval)) + } } options = append(options, transformer.WithPostSetupHook(postSetupUser, postSetupHook)) diff --git a/initializer/initializer_test.go b/initializer/initializer_test.go index 3328af8b..6d7a07e6 100644 --- a/initializer/initializer_test.go +++ b/initializer/initializer_test.go @@ -79,6 +79,7 @@ var _ = Describe("Initializer", func() { CreateWorkPoolSize: 32, DeleteWorkPoolSize: 32, DiskMB: configuration.Automatic, + EnableContainerProxy: false, GardenAddr: "/tmp/garden.sock", GardenHealthcheckCommandRetryPause: durationjson.Duration(1 * time.Second), GardenHealthcheckEmissionInterval: durationjson.Duration(30 * time.Second), diff --git a/resources.go b/resources.go index 4492af4e..f28a0353 100644 --- a/resources.go +++ b/resources.go @@ -197,6 +197,7 @@ type RunInfo struct { CertificateProperties CertificateProperties `json:"certificate_properties"` ImageUsername string `json:"image_username"` ImagePassword string `json:"image_password"` + EnableContainerProxy bool `json:"enable_container_proxy"` Sidecars []Sidecar `json:"sidecars"` LogRateLimitBytesPerSecond int64 `json:"log_rate_limit_bytes_per_second"` VolumeMountedFiles []VolumeMountedFiles `json:"volume_mounted_files,omitempty"`