From b931a36c64625a1269fb330df7afb386fe33b1e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Stru=C3=9F?= Date: Fri, 7 Mar 2025 14:57:51 +0100 Subject: [PATCH 1/3] Remove startupProbe in kubernetes-overlord-extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sebastian Struß --- .../apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java index 1026ccb24862..c4b7bc5e92b3 100644 --- a/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java +++ b/extensions-core/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapter.java @@ -311,6 +311,7 @@ protected Container setupMainContainer( // remove probes mainContainer.setReadinessProbe(null); mainContainer.setLivenessProbe(null); + mainContainer.setStartupProbe(null); setupPorts(mainContainer); addEnvironmentVariables(mainContainer, context, taskContents); From 65c03484c5e5eecad074211fa8dc29ec76204231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Stru=C3=9F?= Date: Tue, 11 Mar 2025 08:04:19 +0100 Subject: [PATCH 2/3] Add unit test for probes removed from podSpec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sebastian Struß --- .../taskadapter/K8sTaskAdapterTest.java | 577 +++++++++--------- .../expectedProbesRemovedOutput.yaml | 56 ++ .../src/test/resources/probesPodSpec.yaml | 26 + 3 files changed, 382 insertions(+), 277 deletions(-) create mode 100644 extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedProbesRemovedOutput.yaml create mode 100644 extensions-core/kubernetes-overlord-extensions/src/test/resources/probesPodSpec.yaml diff --git a/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java b/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java index c8c3739cf5af..5e89db02b8e8 100644 --- a/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java +++ b/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java @@ -84,8 +84,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @EnableKubernetesMockClient(crud = true) -class K8sTaskAdapterTest -{ +class K8sTaskAdapterTest { private KubernetesClient client; private final StartupLoggingConfig startupLoggingConfig; @@ -95,22 +94,21 @@ class K8sTaskAdapterTest private final TaskLogs taskLogs; - public K8sTaskAdapterTest() - { + public K8sTaskAdapterTest() { TestUtils utils = new TestUtils(); jsonMapper = utils.getTestObjectMapper(); jsonMapper.registerSubtypes( - new NamedType(ParallelIndexTuningConfig.class, "index_parallel"), - new NamedType(IndexTask.IndexTuningConfig.class, "index") + new NamedType(ParallelIndexTuningConfig.class, "index_parallel"), + new NamedType(IndexTask.IndexTuningConfig.class, "index") ); node = new DruidNode( - "test", - null, - false, - null, - null, - true, - false + "test", + null, + false, + null, + null, + true, + false ); startupLoggingConfig = new StartupLoggingConfig(); taskConfig = new TaskConfigBuilder().setBaseDir("src/test/resources").build(); @@ -118,20 +116,15 @@ public K8sTaskAdapterTest() } @Test - void testAddingLabelsAndAnnotations() throws IOException - { + void testAddingLabelsAndAnnotations() throws IOException { final PodSpec podSpec = K8sTestUtils.getDummyPodSpec(); - TestKubernetesClient testClient = new TestKubernetesClient(client) - { + TestKubernetesClient testClient = new TestKubernetesClient(client) { @SuppressWarnings("unchecked") @Override - public T executeRequest(KubernetesExecutor executor) throws KubernetesResourceNotFoundException - { - return (T) new Pod() - { + public T executeRequest(KubernetesExecutor executor) throws KubernetesResourceNotFoundException { + return (T) new Pod() { @Override - public PodSpec getSpec() - { + public PodSpec getSpec() { return podSpec; } }; @@ -139,18 +132,18 @@ public PodSpec getSpec() }; KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .withAnnotations(ImmutableMap.of("annotation_key", "annotation_value")) - .withLabels(ImmutableMap.of("label_key", "label_value")) - .build(); + .withNamespace("test") + .withAnnotations(ImmutableMap.of("annotation_key", "annotation_value")) + .withLabels(ImmutableMap.of("label_key", "label_value")) + .build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Task task = K8sTestUtils.getTask(); @@ -165,27 +158,26 @@ public PodSpec getSpec() } @Test - public void serializingAndDeserializingATask() throws IOException - { + public void serializingAndDeserializingATask() throws IOException { // given a task create a k8s job TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Task task = K8sTestUtils.getTask(); Job jobFromSpec = adapter.createJobFromPodSpec( - K8sTestUtils.getDummyPodSpec(), - task, - new PeonCommandContext(new ArrayList<>(), new ArrayList<>(), new File("/tmp/"), config.getCpuCoreInMicro()) + K8sTestUtils.getDummyPodSpec(), + task, + new PeonCommandContext(new ArrayList<>(), new ArrayList<>(), new File("/tmp/"), config.getCpuCoreInMicro()) ); client.batch().v1().jobs().inNamespace("test").create(jobFromSpec); JobList jobList = client.batch().v1().jobs().inNamespace("test").list(); @@ -203,20 +195,15 @@ public void serializingAndDeserializingATask() throws IOException } @Test - public void fromTask_dontSetTaskJSON() throws IOException - { + public void fromTask_dontSetTaskJSON() throws IOException { final PodSpec podSpec = K8sTestUtils.getDummyPodSpec(); - TestKubernetesClient testClient = new TestKubernetesClient(client) - { + TestKubernetesClient testClient = new TestKubernetesClient(client) { @SuppressWarnings("unchecked") @Override - public T executeRequest(KubernetesExecutor executor) throws KubernetesResourceNotFoundException - { - return (T) new Pod() - { + public T executeRequest(KubernetesExecutor executor) throws KubernetesResourceNotFoundException { + return (T) new Pod() { @Override - public PodSpec getSpec() - { + public PodSpec getSpec() { return podSpec; } }; @@ -224,150 +211,146 @@ public PodSpec getSpec() }; KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Task task = new NoopTask( - "id", - "id", - "datasource", - 0, - 0, - ImmutableMap.of("context", RandomStringUtils.randomAlphanumeric((int) DruidK8sConstants.MAX_ENV_VARIABLE_KBS * 20)) + "id", + "id", + "datasource", + 0, + 0, + ImmutableMap.of("context", RandomStringUtils.randomAlphanumeric((int) DruidK8sConstants.MAX_ENV_VARIABLE_KBS * 20)) ); Job job = adapter.fromTask(task); // TASK_JSON should not be set in env variables Assertions.assertFalse( - job.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0).getEnv() - .stream().anyMatch(env -> env.getName().equals(DruidK8sConstants.TASK_JSON_ENV)) + job.getSpec() + .getTemplate() + .getSpec() + .getContainers() + .get(0).getEnv() + .stream().anyMatch(env -> env.getName().equals(DruidK8sConstants.TASK_JSON_ENV)) ); // --taskId should be passed to the peon command args Assertions.assertTrue( - Arrays.stream(job.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0) - .getArgs() - .get(0).split(" ")).collect(Collectors.toSet()) - .containsAll(ImmutableList.of("--taskId", task.getId())) + Arrays.stream(job.getSpec() + .getTemplate() + .getSpec() + .getContainers() + .get(0) + .getArgs() + .get(0).split(" ")).collect(Collectors.toSet()) + .containsAll(ImmutableList.of("--taskId", task.getId())) ); } @Test - public void toTask_useTaskPayloadManager() throws IOException - { + public void toTask_useTaskPayloadManager() throws IOException { TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); Task taskInTaskPayloadManager = K8sTestUtils.getTask(); TaskLogs mockTestLogs = Mockito.mock(TaskLogs.class); Mockito.when(mockTestLogs.streamTaskPayload("ID")).thenReturn(com.google.common.base.Optional.of( - new ByteArrayInputStream(jsonMapper.writeValueAsString(taskInTaskPayloadManager).getBytes(Charset.defaultCharset())) + new ByteArrayInputStream(jsonMapper.writeValueAsString(taskInTaskPayloadManager).getBytes(Charset.defaultCharset())) )); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - mockTestLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + mockTestLogs ); Job job = new JobBuilder() - .editMetadata().withName("job").endMetadata() - .editSpec().editTemplate().editMetadata() - .addToAnnotations(DruidK8sConstants.TASK_ID, "ID") - .endMetadata().editSpec().addToContainers(new ContainerBuilder().withName("main").build()).endSpec().endTemplate().endSpec().build(); + .editMetadata().withName("job").endMetadata() + .editSpec().editTemplate().editMetadata() + .addToAnnotations(DruidK8sConstants.TASK_ID, "ID") + .endMetadata().editSpec().addToContainers(new ContainerBuilder().withName("main").build()).endSpec().endTemplate().endSpec().build(); Task taskFromJob = adapter.toTask(job); assertEquals(taskInTaskPayloadManager, taskFromJob); } @Test - public void getTaskId() - { + public void getTaskId() { TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder().build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Job job = new JobBuilder() - .editSpec().editTemplate().editMetadata() - .addToAnnotations(DruidK8sConstants.TASK_ID, "ID") - .endMetadata().endTemplate().endSpec().build(); + .editSpec().editTemplate().editMetadata() + .addToAnnotations(DruidK8sConstants.TASK_ID, "ID") + .endMetadata().endTemplate().endSpec().build(); assertEquals(new K8sTaskId("ID"), adapter.getTaskId(job)); } @Test - public void getTaskId_noAnnotations() - { + public void getTaskId_noAnnotations() { TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder().build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Job job = new JobBuilder() - .editSpec().editTemplate().editMetadata() - .endMetadata().endTemplate().endSpec() - .editMetadata().withName("job").endMetadata().build(); + .editSpec().editTemplate().editMetadata() + .endMetadata().endTemplate().endSpec() + .editMetadata().withName("job").endMetadata().build(); Assert.assertThrows(DruidException.class, () -> adapter.getTaskId(job)); } @Test - public void getTaskId_missingTaskIdAnnotation() - { + public void getTaskId_missingTaskIdAnnotation() { TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder().build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Job job = new JobBuilder() - .editSpec().editTemplate().editMetadata() - .addToAnnotations(DruidK8sConstants.TASK_GROUP_ID, "ID") - .endMetadata().endTemplate().endSpec() - .editMetadata().withName("job").endMetadata().build(); + .editSpec().editTemplate().editMetadata() + .addToAnnotations(DruidK8sConstants.TASK_GROUP_ID, "ID") + .endMetadata().endTemplate().endSpec() + .editMetadata().withName("job").endMetadata().build(); Assert.assertThrows(DruidException.class, () -> adapter.getTaskId(job)); } + @Test - void testGrabbingTheLastXmxValueFromACommand() - { + void testGrabbingTheLastXmxValueFromACommand() { List commands = Lists.newArrayList("-Xmx2g", "-Xms1g", "-Xmx4g"); Optional value = K8sTaskAdapter.getJavaOptValueBytes("-Xmx", commands); assertEquals(HumanReadableBytes.parse("4g"), value.get()); @@ -379,40 +362,38 @@ void testGrabbingTheLastXmxValueFromACommand() } @Test - void testGettingContainerSize() - { + void testGettingContainerSize() { // nothing specified no heap no dbb should be (1g + 1g) * 1.2 long expected = (long) ((HumanReadableBytes.parse("1g") + HumanReadableBytes.parse("1g")) * 1.2); PeonCommandContext context = new PeonCommandContext( - new ArrayList<>(), - new ArrayList<>(), - new File("/tmp"), - 0 + new ArrayList<>(), + new ArrayList<>(), + new File("/tmp"), + 0 ); assertEquals(expected, K8sTaskAdapter.getContainerMemory(context)); context = new PeonCommandContext( - new ArrayList<>(), - Collections.singletonList( - "-server -Xms512m -Xmx512m -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"), - new File("/tmp"), - 0 + new ArrayList<>(), + Collections.singletonList( + "-server -Xms512m -Xmx512m -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"), + new File("/tmp"), + 0 ); expected = (long) ((HumanReadableBytes.parse("512m") + HumanReadableBytes.parse("1g")) * 1.2); assertEquals(expected, K8sTaskAdapter.getContainerMemory(context)); } @Test - void testMassagingSpec() - { + void testMassagingSpec() { PodSpec spec = new PodSpec(); List containers = new ArrayList<>(); containers.add(new ContainerBuilder() - .withName("secondary").build()); + .withName("secondary").build()); containers.add(new ContainerBuilder() - .withName("sidecar").build()); + .withName("sidecar").build()); containers.add(new ContainerBuilder() - .withName("primary").build()); + .withName("primary").build()); spec.setContainers(containers); K8sTaskAdapter.massageSpec(spec, "primary"); @@ -424,16 +405,15 @@ void testMassagingSpec() } @Test - void testNoPrimaryFound() - { + void testNoPrimaryFound() { PodSpec spec = new PodSpec(); List containers = new ArrayList<>(); containers.add(new ContainerBuilder() - .withName("istio-proxy").build()); + .withName("istio-proxy").build()); containers.add(new ContainerBuilder() - .withName("main").build()); + .withName("main").build()); containers.add(new ContainerBuilder() - .withName("sidecar").build()); + .withName("sidecar").build()); spec.setContainers(containers); @@ -443,124 +423,122 @@ void testNoPrimaryFound() } @Test - void testAddingMonitors() throws IOException - { + void testAddingMonitors() throws IOException { TestKubernetesClient testClient = new TestKubernetesClient(client); PeonCommandContext context = new PeonCommandContext( - new ArrayList<>(), - new ArrayList<>(), - new File("/tmp/"), - 0 + new ArrayList<>(), + new ArrayList<>(), + new File("/tmp/"), + 0 ); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Task task = K8sTestUtils.getTask(); // no monitor in overlord, no monitor override Container container = new ContainerBuilder() - .withName("container").build(); + .withName("container").build(); adapter.addEnvironmentVariables(container, context, task.toString()); assertFalse( - container.getEnv().stream().anyMatch(x -> x.getName().equals("druid_monitoring_monitors")), - "Didn't match, envs: " + Joiner.on(',').join(container.getEnv()) + container.getEnv().stream().anyMatch(x -> x.getName().equals("druid_monitoring_monitors")), + "Didn't match, envs: " + Joiner.on(',').join(container.getEnv()) ); // we have an override, but nothing in the overlord config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .withPeonMonitors(ImmutableList.of("org.apache.druid.java.util.metrics.JvmMonitor")) - .build(); + .withNamespace("test") + .withPeonMonitors(ImmutableList.of("org.apache.druid.java.util.metrics.JvmMonitor")) + .build(); adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); adapter.addEnvironmentVariables(container, context, task.toString()); EnvVar env = container.getEnv() - .stream() - .filter(x -> x.getName().equals("druid_monitoring_monitors")) - .findFirst() - .get(); + .stream() + .filter(x -> x.getName().equals("druid_monitoring_monitors")) + .findFirst() + .get(); assertEquals(jsonMapper.writeValueAsString(config.getPeonMonitors()), env.getValue()); // we override what is in the overlord adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); container.getEnv().add(new EnvVarBuilder() - .withName("druid_monitoring_monitors") - .withValue( - "'[\"org.apache.druid.java.util.metrics.JvmMonitor\", " - + "\"org.apache.druid.server.metrics.TaskCountStatsMonitor\"]'") - .build()); + .withName("druid_monitoring_monitors") + .withValue( + "'[\"org.apache.druid.java.util.metrics.JvmMonitor\", " + + "\"org.apache.druid.server.metrics.TaskCountStatsMonitor\"]'") + .build()); adapter.addEnvironmentVariables(container, context, task.toString()); env = container.getEnv() - .stream() - .filter(x -> x.getName().equals("druid_monitoring_monitors")) - .findFirst() - .get(); + .stream() + .filter(x -> x.getName().equals("druid_monitoring_monitors")) + .findFirst() + .get(); assertEquals(jsonMapper.writeValueAsString(config.getPeonMonitors()), env.getValue()); } @Test - void testEphemeralStorageIsRespected() throws IOException - { + void testEphemeralStorageIsRespected() throws IOException { TestKubernetesClient testClient = new TestKubernetesClient(client); Pod pod = K8sTestUtils.fileToResource("ephemeralPodSpec.yaml", Pod.class); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); SingleContainerTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); NoopTask task = K8sTestUtils.createTask("id", 1); Job actual = adapter.createJobFromPodSpec( - pod.getSpec(), - task, - new PeonCommandContext( - Collections.singletonList("foo && bar"), - new ArrayList<>(), - new File("/tmp"), - config.getCpuCoreInMicro() - ) + pod.getSpec(), + task, + new PeonCommandContext( + Collections.singletonList("foo && bar"), + new ArrayList<>(), + new File("/tmp"), + config.getCpuCoreInMicro() + ) ); Job expected = K8sTestUtils.fileToResource("expectedEphemeralOutput.yaml", Job.class); // something is up with jdk 17, where if you compress with jdk < 17 and try and decompress you get different results, // this would never happen in real life, but for the jdk 17 tests this is a problem // could be related to: https://bugs.openjdk.org/browse/JDK-8081450 actual.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0) - .getEnv() - .removeIf(x -> x.getName().equals("TASK_JSON")); + .getTemplate() + .getSpec() + .getContainers() + .get(0) + .getEnv() + .removeIf(x -> x.getName().equals("TASK_JSON")); expected.getSpec() .getTemplate() .getSpec() @@ -572,50 +550,98 @@ void testEphemeralStorageIsRespected() throws IOException } @Test - void testCPUResourceIsEspected() throws IOException - { + void testProbesRemoved() throws IOException { + TestKubernetesClient testClient = new TestKubernetesClient(client); + Pod pod = K8sTestUtils.fileToResource("probesPodSpec.yaml", Pod.class); + KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() + .withNamespace("test") + .build(); + + SingleContainerTaskAdapter adapter = new SingleContainerTaskAdapter( + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs + ); + NoopTask task = K8sTestUtils.createTask("id", 1); + Job actual = adapter.createJobFromPodSpec( + pod.getSpec(), + task, + new PeonCommandContext( + Collections.singletonList("foo && bar"), + new ArrayList<>(), + new File("/tmp"), + config.getCpuCoreInMicro() + ) + ); + Job expected = K8sTestUtils.fileToResource("expectedProbesRemovedOutput.yaml", Job.class); + // something is up with jdk 17, where if you compress with jdk < 17 and try and decompress you get different results, + // this would never happen in real life, but for the jdk 17 tests this is a problem + // could be related to: https://bugs.openjdk.org/browse/JDK-8081450 + actual.getSpec() + .getTemplate() + .getSpec() + .getContainers() + .get(0) + .getEnv() + .removeIf(x -> x.getName().equals("TASK_JSON")); + expected.getSpec() + .getTemplate() + .getSpec() + .getContainers() + .get(0) + .getEnv() + .removeIf(x -> x.getName().equals("TASK_JSON")); + Assertions.assertEquals(expected, actual); + } + + @Test + void testCPUResourceIsExpected() throws IOException { TestKubernetesClient testClient = new TestKubernetesClient(client); Pod pod = K8sTestUtils.fileToResource("ephemeralPodSpec.yaml", Pod.class); List javaOpts = new ArrayList<>(); javaOpts.add("-Xms1G -Xmx2G -XX:MaxDirectMemorySize=3G"); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .withJavaOptsArray(javaOpts) - .withCpuCore(2000) - .build(); + .withNamespace("test") + .withJavaOptsArray(javaOpts) + .withCpuCore(2000) + .build(); SingleContainerTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); NoopTask task = K8sTestUtils.createTask("id", 1); Job actual = adapter.createJobFromPodSpec( - pod.getSpec(), - task, - new PeonCommandContext( - Collections.singletonList("foo && bar"), - javaOpts, - new File("/tmp"), - config.getCpuCoreInMicro() - ) + pod.getSpec(), + task, + new PeonCommandContext( + Collections.singletonList("foo && bar"), + javaOpts, + new File("/tmp"), + config.getCpuCoreInMicro() + ) ); Job expected = K8sTestUtils.fileToResource("expectedCPUResourceOutput.yaml", Job.class); // something is up with jdk 17, where if you compress with jdk < 17 and try and decompress you get different results, // this would never happen in real life, but for the jdk 17 tests this is a problem // could be related to: https://bugs.openjdk.org/browse/JDK-8081450 actual.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0) - .getEnv() - .removeIf(x -> x.getName().equals("TASK_JSON")); + .getTemplate() + .getSpec() + .getContainers() + .get(0) + .getEnv() + .removeIf(x -> x.getName().equals("TASK_JSON")); expected.getSpec() .getTemplate() .getSpec() @@ -627,16 +653,14 @@ void testCPUResourceIsEspected() throws IOException } - @Test - void testEphemeralStorage() - { + void testEphemeralStorage() { // no resources set. Container container = new ContainerBuilder().build(); ResourceRequirements result = K8sTaskAdapter.getResourceRequirements( - container.getResources(), - 100, - 1000 + container.getResources(), + 100, + 1000 ); // requests and limits will only have 2 items, cpu / memory assertEquals(2, result.getLimits().size()); @@ -647,9 +671,9 @@ void testEphemeralStorage() ImmutableMap limitMap = ImmutableMap.of("ephemeral-storage", new Quantity("10Gi")); container.setResources(new ResourceRequirementsBuilder().withRequests(requestMap).withLimits(limitMap).build()); ResourceRequirements ephemeralResult = K8sTaskAdapter.getResourceRequirements( - container.getResources(), - 100, - 1000 + container.getResources(), + 100, + 1000 ); // you will have ephemeral storage as well. assertEquals(3, ephemeralResult.getLimits().size()); @@ -666,11 +690,10 @@ void testEphemeralStorage() // we should also preserve additional properties container.getResources().setAdditionalProperty("additional", "some-value"); ResourceRequirements additionalProperties = K8sTaskAdapter.getResourceRequirements( - container.getResources(), - 100, - 1000 + container.getResources(), + 100, + 1000 ); assertEquals(1, additionalProperties.getAdditionalProperties().size()); } - -} +} \ No newline at end of file diff --git a/extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedProbesRemovedOutput.yaml b/extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedProbesRemovedOutput.yaml new file mode 100644 index 000000000000..a4dccde41633 --- /dev/null +++ b/extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedProbesRemovedOutput.yaml @@ -0,0 +1,56 @@ +apiVersion: "batch/v1" +kind: "Job" +metadata: + annotations: + task.id: "id" + tls.enabled: "false" + labels: + druid.k8s.peons: "true" + name: "id-3e70afe5cd823dfc7dd308eea616426b" +spec: + activeDeadlineSeconds: 14400 + backoffLimit: 0 + template: + metadata: + annotations: + task.id: "id" + tls.enabled: "false" + labels: + druid.k8s.peons: "true" + spec: + containers: + - args: + - "foo && bar" + command: + - "sh" + - "-c" + env: + - name: "druid_monitoring_monitors" + value: "[\"org.apache.druid.java.util.metrics.JvmMonitor\", \"org.apache.druid.server.metrics.TaskCountStatsMonitor\"\ + ]" + - name: "TASK_DIR" + value: "/tmp" + - name: "TASK_JSON" + value: "H4sIAAAAAAAAAEVOOw7CMAy9i+cOBYmlK0KItWVhNI0BSyEOToKoqt4doxZYLPv9/EbIQyRoIIhEqICd7TYquKqUePidDjN2UrSfxYEM0xKOfDdgvalr86aW0A0z9L9bSsVnc512nZkurHSTZJJQvK+gl5DpZfwIUVmU8wDNarJ0Ssu/EfCJ7PHM3tj9p9i3ltKjWKDbYsR+sU5vP86oMNUAAAA=" + - name: "JAVA_OPTS" + value: "" + - name: "druid_host" + valueFrom: + fieldRef: + fieldPath: "status.podIP" + - name: "HOSTNAME" + valueFrom: + fieldRef: + fieldPath: "metadata.name" + image: "one" + name: "main" + ports: + - containerPort: 8091 + name: "druid-tls-port" + protocol: "TCP" + - containerPort: 8100 + name: "druid-port" + protocol: "TCP" + hostname: "id-3e70afe5cd823dfc7dd308eea616426b" + restartPolicy: "Never" + ttlSecondsAfterFinished: 172800 \ No newline at end of file diff --git a/extensions-core/kubernetes-overlord-extensions/src/test/resources/probesPodSpec.yaml b/extensions-core/kubernetes-overlord-extensions/src/test/resources/probesPodSpec.yaml new file mode 100644 index 000000000000..a76979b0426e --- /dev/null +++ b/extensions-core/kubernetes-overlord-extensions/src/test/resources/probesPodSpec.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test +spec: + containers: + - command: + - sleep + - "3600" + image: one + name: primary + startupProbe: + httpGet: + port: 8100 + path: /status/health + livenessProbe: + httpGet: + port: 8100 + path: /status/health + readinessProbe: + httpGet: + port: 8100 + path: /status/health + env: + - name: "druid_monitoring_monitors" + value: '["org.apache.druid.java.util.metrics.JvmMonitor", "org.apache.druid.server.metrics.TaskCountStatsMonitor"]' From 4192a165272a994e58638796fe2ec6cc8c716d36 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Tue, 1 Apr 2025 00:49:57 -0700 Subject: [PATCH 3/3] Restore styling, add resources to expected test output. --- .../taskadapter/K8sTaskAdapterTest.java | 577 +++++++++--------- .../expectedProbesRemovedOutput.yaml | 7 + 2 files changed, 309 insertions(+), 275 deletions(-) diff --git a/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java b/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java index 5e89db02b8e8..38c7fd2612b2 100644 --- a/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java +++ b/extensions-core/kubernetes-overlord-extensions/src/test/java/org/apache/druid/k8s/overlord/taskadapter/K8sTaskAdapterTest.java @@ -84,7 +84,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @EnableKubernetesMockClient(crud = true) -class K8sTaskAdapterTest { +class K8sTaskAdapterTest +{ private KubernetesClient client; private final StartupLoggingConfig startupLoggingConfig; @@ -94,21 +95,22 @@ class K8sTaskAdapterTest { private final TaskLogs taskLogs; - public K8sTaskAdapterTest() { + public K8sTaskAdapterTest() + { TestUtils utils = new TestUtils(); jsonMapper = utils.getTestObjectMapper(); jsonMapper.registerSubtypes( - new NamedType(ParallelIndexTuningConfig.class, "index_parallel"), - new NamedType(IndexTask.IndexTuningConfig.class, "index") + new NamedType(ParallelIndexTuningConfig.class, "index_parallel"), + new NamedType(IndexTask.IndexTuningConfig.class, "index") ); node = new DruidNode( - "test", - null, - false, - null, - null, - true, - false + "test", + null, + false, + null, + null, + true, + false ); startupLoggingConfig = new StartupLoggingConfig(); taskConfig = new TaskConfigBuilder().setBaseDir("src/test/resources").build(); @@ -116,15 +118,20 @@ public K8sTaskAdapterTest() { } @Test - void testAddingLabelsAndAnnotations() throws IOException { + void testAddingLabelsAndAnnotations() throws IOException + { final PodSpec podSpec = K8sTestUtils.getDummyPodSpec(); - TestKubernetesClient testClient = new TestKubernetesClient(client) { + TestKubernetesClient testClient = new TestKubernetesClient(client) + { @SuppressWarnings("unchecked") @Override - public T executeRequest(KubernetesExecutor executor) throws KubernetesResourceNotFoundException { - return (T) new Pod() { + public T executeRequest(KubernetesExecutor executor) throws KubernetesResourceNotFoundException + { + return (T) new Pod() + { @Override - public PodSpec getSpec() { + public PodSpec getSpec() + { return podSpec; } }; @@ -132,18 +139,18 @@ public PodSpec getSpec() { }; KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .withAnnotations(ImmutableMap.of("annotation_key", "annotation_value")) - .withLabels(ImmutableMap.of("label_key", "label_value")) - .build(); + .withNamespace("test") + .withAnnotations(ImmutableMap.of("annotation_key", "annotation_value")) + .withLabels(ImmutableMap.of("label_key", "label_value")) + .build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Task task = K8sTestUtils.getTask(); @@ -158,26 +165,27 @@ public PodSpec getSpec() { } @Test - public void serializingAndDeserializingATask() throws IOException { + public void serializingAndDeserializingATask() throws IOException + { // given a task create a k8s job TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Task task = K8sTestUtils.getTask(); Job jobFromSpec = adapter.createJobFromPodSpec( - K8sTestUtils.getDummyPodSpec(), - task, - new PeonCommandContext(new ArrayList<>(), new ArrayList<>(), new File("/tmp/"), config.getCpuCoreInMicro()) + K8sTestUtils.getDummyPodSpec(), + task, + new PeonCommandContext(new ArrayList<>(), new ArrayList<>(), new File("/tmp/"), config.getCpuCoreInMicro()) ); client.batch().v1().jobs().inNamespace("test").create(jobFromSpec); JobList jobList = client.batch().v1().jobs().inNamespace("test").list(); @@ -195,15 +203,20 @@ public void serializingAndDeserializingATask() throws IOException { } @Test - public void fromTask_dontSetTaskJSON() throws IOException { + public void fromTask_dontSetTaskJSON() throws IOException + { final PodSpec podSpec = K8sTestUtils.getDummyPodSpec(); - TestKubernetesClient testClient = new TestKubernetesClient(client) { + TestKubernetesClient testClient = new TestKubernetesClient(client) + { @SuppressWarnings("unchecked") @Override - public T executeRequest(KubernetesExecutor executor) throws KubernetesResourceNotFoundException { - return (T) new Pod() { + public T executeRequest(KubernetesExecutor executor) throws KubernetesResourceNotFoundException + { + return (T) new Pod() + { @Override - public PodSpec getSpec() { + public PodSpec getSpec() + { return podSpec; } }; @@ -211,146 +224,150 @@ public PodSpec getSpec() { }; KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Task task = new NoopTask( - "id", - "id", - "datasource", - 0, - 0, - ImmutableMap.of("context", RandomStringUtils.randomAlphanumeric((int) DruidK8sConstants.MAX_ENV_VARIABLE_KBS * 20)) + "id", + "id", + "datasource", + 0, + 0, + ImmutableMap.of("context", RandomStringUtils.randomAlphanumeric((int) DruidK8sConstants.MAX_ENV_VARIABLE_KBS * 20)) ); Job job = adapter.fromTask(task); // TASK_JSON should not be set in env variables Assertions.assertFalse( - job.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0).getEnv() - .stream().anyMatch(env -> env.getName().equals(DruidK8sConstants.TASK_JSON_ENV)) + job.getSpec() + .getTemplate() + .getSpec() + .getContainers() + .get(0).getEnv() + .stream().anyMatch(env -> env.getName().equals(DruidK8sConstants.TASK_JSON_ENV)) ); // --taskId should be passed to the peon command args Assertions.assertTrue( - Arrays.stream(job.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0) - .getArgs() - .get(0).split(" ")).collect(Collectors.toSet()) - .containsAll(ImmutableList.of("--taskId", task.getId())) + Arrays.stream(job.getSpec() + .getTemplate() + .getSpec() + .getContainers() + .get(0) + .getArgs() + .get(0).split(" ")).collect(Collectors.toSet()) + .containsAll(ImmutableList.of("--taskId", task.getId())) ); } @Test - public void toTask_useTaskPayloadManager() throws IOException { + public void toTask_useTaskPayloadManager() throws IOException + { TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); Task taskInTaskPayloadManager = K8sTestUtils.getTask(); TaskLogs mockTestLogs = Mockito.mock(TaskLogs.class); Mockito.when(mockTestLogs.streamTaskPayload("ID")).thenReturn(com.google.common.base.Optional.of( - new ByteArrayInputStream(jsonMapper.writeValueAsString(taskInTaskPayloadManager).getBytes(Charset.defaultCharset())) + new ByteArrayInputStream(jsonMapper.writeValueAsString(taskInTaskPayloadManager).getBytes(Charset.defaultCharset())) )); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - mockTestLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + mockTestLogs ); Job job = new JobBuilder() - .editMetadata().withName("job").endMetadata() - .editSpec().editTemplate().editMetadata() - .addToAnnotations(DruidK8sConstants.TASK_ID, "ID") - .endMetadata().editSpec().addToContainers(new ContainerBuilder().withName("main").build()).endSpec().endTemplate().endSpec().build(); + .editMetadata().withName("job").endMetadata() + .editSpec().editTemplate().editMetadata() + .addToAnnotations(DruidK8sConstants.TASK_ID, "ID") + .endMetadata().editSpec().addToContainers(new ContainerBuilder().withName("main").build()).endSpec().endTemplate().endSpec().build(); Task taskFromJob = adapter.toTask(job); assertEquals(taskInTaskPayloadManager, taskFromJob); } @Test - public void getTaskId() { + public void getTaskId() + { TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder().build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Job job = new JobBuilder() - .editSpec().editTemplate().editMetadata() - .addToAnnotations(DruidK8sConstants.TASK_ID, "ID") - .endMetadata().endTemplate().endSpec().build(); + .editSpec().editTemplate().editMetadata() + .addToAnnotations(DruidK8sConstants.TASK_ID, "ID") + .endMetadata().endTemplate().endSpec().build(); assertEquals(new K8sTaskId("ID"), adapter.getTaskId(job)); } @Test - public void getTaskId_noAnnotations() { + public void getTaskId_noAnnotations() + { TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder().build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Job job = new JobBuilder() - .editSpec().editTemplate().editMetadata() - .endMetadata().endTemplate().endSpec() - .editMetadata().withName("job").endMetadata().build(); + .editSpec().editTemplate().editMetadata() + .endMetadata().endTemplate().endSpec() + .editMetadata().withName("job").endMetadata().build(); Assert.assertThrows(DruidException.class, () -> adapter.getTaskId(job)); } @Test - public void getTaskId_missingTaskIdAnnotation() { + public void getTaskId_missingTaskIdAnnotation() + { TestKubernetesClient testClient = new TestKubernetesClient(client); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder().build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Job job = new JobBuilder() - .editSpec().editTemplate().editMetadata() - .addToAnnotations(DruidK8sConstants.TASK_GROUP_ID, "ID") - .endMetadata().endTemplate().endSpec() - .editMetadata().withName("job").endMetadata().build(); + .editSpec().editTemplate().editMetadata() + .addToAnnotations(DruidK8sConstants.TASK_GROUP_ID, "ID") + .endMetadata().endTemplate().endSpec() + .editMetadata().withName("job").endMetadata().build(); Assert.assertThrows(DruidException.class, () -> adapter.getTaskId(job)); } - @Test - void testGrabbingTheLastXmxValueFromACommand() { + void testGrabbingTheLastXmxValueFromACommand() + { List commands = Lists.newArrayList("-Xmx2g", "-Xms1g", "-Xmx4g"); Optional value = K8sTaskAdapter.getJavaOptValueBytes("-Xmx", commands); assertEquals(HumanReadableBytes.parse("4g"), value.get()); @@ -362,38 +379,40 @@ void testGrabbingTheLastXmxValueFromACommand() { } @Test - void testGettingContainerSize() { + void testGettingContainerSize() + { // nothing specified no heap no dbb should be (1g + 1g) * 1.2 long expected = (long) ((HumanReadableBytes.parse("1g") + HumanReadableBytes.parse("1g")) * 1.2); PeonCommandContext context = new PeonCommandContext( - new ArrayList<>(), - new ArrayList<>(), - new File("/tmp"), - 0 + new ArrayList<>(), + new ArrayList<>(), + new File("/tmp"), + 0 ); assertEquals(expected, K8sTaskAdapter.getContainerMemory(context)); context = new PeonCommandContext( - new ArrayList<>(), - Collections.singletonList( - "-server -Xms512m -Xmx512m -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"), - new File("/tmp"), - 0 + new ArrayList<>(), + Collections.singletonList( + "-server -Xms512m -Xmx512m -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"), + new File("/tmp"), + 0 ); expected = (long) ((HumanReadableBytes.parse("512m") + HumanReadableBytes.parse("1g")) * 1.2); assertEquals(expected, K8sTaskAdapter.getContainerMemory(context)); } @Test - void testMassagingSpec() { + void testMassagingSpec() + { PodSpec spec = new PodSpec(); List containers = new ArrayList<>(); containers.add(new ContainerBuilder() - .withName("secondary").build()); + .withName("secondary").build()); containers.add(new ContainerBuilder() - .withName("sidecar").build()); + .withName("sidecar").build()); containers.add(new ContainerBuilder() - .withName("primary").build()); + .withName("primary").build()); spec.setContainers(containers); K8sTaskAdapter.massageSpec(spec, "primary"); @@ -405,15 +424,16 @@ void testMassagingSpec() { } @Test - void testNoPrimaryFound() { + void testNoPrimaryFound() + { PodSpec spec = new PodSpec(); List containers = new ArrayList<>(); containers.add(new ContainerBuilder() - .withName("istio-proxy").build()); + .withName("istio-proxy").build()); containers.add(new ContainerBuilder() - .withName("main").build()); + .withName("main").build()); containers.add(new ContainerBuilder() - .withName("sidecar").build()); + .withName("sidecar").build()); spec.setContainers(containers); @@ -423,122 +443,124 @@ void testNoPrimaryFound() { } @Test - void testAddingMonitors() throws IOException { + void testAddingMonitors() throws IOException + { TestKubernetesClient testClient = new TestKubernetesClient(client); PeonCommandContext context = new PeonCommandContext( - new ArrayList<>(), - new ArrayList<>(), - new File("/tmp/"), - 0 + new ArrayList<>(), + new ArrayList<>(), + new File("/tmp/"), + 0 ); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); K8sTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); Task task = K8sTestUtils.getTask(); // no monitor in overlord, no monitor override Container container = new ContainerBuilder() - .withName("container").build(); + .withName("container").build(); adapter.addEnvironmentVariables(container, context, task.toString()); assertFalse( - container.getEnv().stream().anyMatch(x -> x.getName().equals("druid_monitoring_monitors")), - "Didn't match, envs: " + Joiner.on(',').join(container.getEnv()) + container.getEnv().stream().anyMatch(x -> x.getName().equals("druid_monitoring_monitors")), + "Didn't match, envs: " + Joiner.on(',').join(container.getEnv()) ); // we have an override, but nothing in the overlord config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .withPeonMonitors(ImmutableList.of("org.apache.druid.java.util.metrics.JvmMonitor")) - .build(); + .withNamespace("test") + .withPeonMonitors(ImmutableList.of("org.apache.druid.java.util.metrics.JvmMonitor")) + .build(); adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); adapter.addEnvironmentVariables(container, context, task.toString()); EnvVar env = container.getEnv() - .stream() - .filter(x -> x.getName().equals("druid_monitoring_monitors")) - .findFirst() - .get(); + .stream() + .filter(x -> x.getName().equals("druid_monitoring_monitors")) + .findFirst() + .get(); assertEquals(jsonMapper.writeValueAsString(config.getPeonMonitors()), env.getValue()); // we override what is in the overlord adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); container.getEnv().add(new EnvVarBuilder() - .withName("druid_monitoring_monitors") - .withValue( - "'[\"org.apache.druid.java.util.metrics.JvmMonitor\", " - + "\"org.apache.druid.server.metrics.TaskCountStatsMonitor\"]'") - .build()); + .withName("druid_monitoring_monitors") + .withValue( + "'[\"org.apache.druid.java.util.metrics.JvmMonitor\", " + + "\"org.apache.druid.server.metrics.TaskCountStatsMonitor\"]'") + .build()); adapter.addEnvironmentVariables(container, context, task.toString()); env = container.getEnv() - .stream() - .filter(x -> x.getName().equals("druid_monitoring_monitors")) - .findFirst() - .get(); + .stream() + .filter(x -> x.getName().equals("druid_monitoring_monitors")) + .findFirst() + .get(); assertEquals(jsonMapper.writeValueAsString(config.getPeonMonitors()), env.getValue()); } @Test - void testEphemeralStorageIsRespected() throws IOException { + void testEphemeralStorageIsRespected() throws IOException + { TestKubernetesClient testClient = new TestKubernetesClient(client); Pod pod = K8sTestUtils.fileToResource("ephemeralPodSpec.yaml", Pod.class); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); SingleContainerTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); NoopTask task = K8sTestUtils.createTask("id", 1); Job actual = adapter.createJobFromPodSpec( - pod.getSpec(), - task, - new PeonCommandContext( - Collections.singletonList("foo && bar"), - new ArrayList<>(), - new File("/tmp"), - config.getCpuCoreInMicro() - ) + pod.getSpec(), + task, + new PeonCommandContext( + Collections.singletonList("foo && bar"), + new ArrayList<>(), + new File("/tmp"), + config.getCpuCoreInMicro() + ) ); Job expected = K8sTestUtils.fileToResource("expectedEphemeralOutput.yaml", Job.class); // something is up with jdk 17, where if you compress with jdk < 17 and try and decompress you get different results, // this would never happen in real life, but for the jdk 17 tests this is a problem // could be related to: https://bugs.openjdk.org/browse/JDK-8081450 actual.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0) - .getEnv() - .removeIf(x -> x.getName().equals("TASK_JSON")); + .getTemplate() + .getSpec() + .getContainers() + .get(0) + .getEnv() + .removeIf(x -> x.getName().equals("TASK_JSON")); expected.getSpec() .getTemplate() .getSpec() @@ -550,44 +572,45 @@ void testEphemeralStorageIsRespected() throws IOException { } @Test - void testProbesRemoved() throws IOException { + void testProbesRemoved() throws IOException + { TestKubernetesClient testClient = new TestKubernetesClient(client); Pod pod = K8sTestUtils.fileToResource("probesPodSpec.yaml", Pod.class); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .build(); + .withNamespace("test") + .build(); SingleContainerTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); NoopTask task = K8sTestUtils.createTask("id", 1); Job actual = adapter.createJobFromPodSpec( - pod.getSpec(), - task, - new PeonCommandContext( - Collections.singletonList("foo && bar"), - new ArrayList<>(), - new File("/tmp"), - config.getCpuCoreInMicro() - ) + pod.getSpec(), + task, + new PeonCommandContext( + Collections.singletonList("foo && bar"), + new ArrayList<>(), + new File("/tmp"), + config.getCpuCoreInMicro() + ) ); Job expected = K8sTestUtils.fileToResource("expectedProbesRemovedOutput.yaml", Job.class); // something is up with jdk 17, where if you compress with jdk < 17 and try and decompress you get different results, // this would never happen in real life, but for the jdk 17 tests this is a problem // could be related to: https://bugs.openjdk.org/browse/JDK-8081450 actual.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0) - .getEnv() - .removeIf(x -> x.getName().equals("TASK_JSON")); + .getTemplate() + .getSpec() + .getContainers() + .get(0) + .getEnv() + .removeIf(x -> x.getName().equals("TASK_JSON")); expected.getSpec() .getTemplate() .getSpec() @@ -599,49 +622,50 @@ void testProbesRemoved() throws IOException { } @Test - void testCPUResourceIsExpected() throws IOException { + void testCPUResourceIsEspected() throws IOException + { TestKubernetesClient testClient = new TestKubernetesClient(client); Pod pod = K8sTestUtils.fileToResource("ephemeralPodSpec.yaml", Pod.class); List javaOpts = new ArrayList<>(); javaOpts.add("-Xms1G -Xmx2G -XX:MaxDirectMemorySize=3G"); KubernetesTaskRunnerConfig config = KubernetesTaskRunnerConfig.builder() - .withNamespace("test") - .withJavaOptsArray(javaOpts) - .withCpuCore(2000) - .build(); + .withNamespace("test") + .withJavaOptsArray(javaOpts) + .withCpuCore(2000) + .build(); SingleContainerTaskAdapter adapter = new SingleContainerTaskAdapter( - testClient, - config, - taskConfig, - startupLoggingConfig, - node, - jsonMapper, - taskLogs + testClient, + config, + taskConfig, + startupLoggingConfig, + node, + jsonMapper, + taskLogs ); NoopTask task = K8sTestUtils.createTask("id", 1); Job actual = adapter.createJobFromPodSpec( - pod.getSpec(), - task, - new PeonCommandContext( - Collections.singletonList("foo && bar"), - javaOpts, - new File("/tmp"), - config.getCpuCoreInMicro() - ) + pod.getSpec(), + task, + new PeonCommandContext( + Collections.singletonList("foo && bar"), + javaOpts, + new File("/tmp"), + config.getCpuCoreInMicro() + ) ); Job expected = K8sTestUtils.fileToResource("expectedCPUResourceOutput.yaml", Job.class); // something is up with jdk 17, where if you compress with jdk < 17 and try and decompress you get different results, // this would never happen in real life, but for the jdk 17 tests this is a problem // could be related to: https://bugs.openjdk.org/browse/JDK-8081450 actual.getSpec() - .getTemplate() - .getSpec() - .getContainers() - .get(0) - .getEnv() - .removeIf(x -> x.getName().equals("TASK_JSON")); + .getTemplate() + .getSpec() + .getContainers() + .get(0) + .getEnv() + .removeIf(x -> x.getName().equals("TASK_JSON")); expected.getSpec() .getTemplate() .getSpec() @@ -653,14 +677,16 @@ void testCPUResourceIsExpected() throws IOException { } + @Test - void testEphemeralStorage() { + void testEphemeralStorage() + { // no resources set. Container container = new ContainerBuilder().build(); ResourceRequirements result = K8sTaskAdapter.getResourceRequirements( - container.getResources(), - 100, - 1000 + container.getResources(), + 100, + 1000 ); // requests and limits will only have 2 items, cpu / memory assertEquals(2, result.getLimits().size()); @@ -671,9 +697,9 @@ void testEphemeralStorage() { ImmutableMap limitMap = ImmutableMap.of("ephemeral-storage", new Quantity("10Gi")); container.setResources(new ResourceRequirementsBuilder().withRequests(requestMap).withLimits(limitMap).build()); ResourceRequirements ephemeralResult = K8sTaskAdapter.getResourceRequirements( - container.getResources(), - 100, - 1000 + container.getResources(), + 100, + 1000 ); // you will have ephemeral storage as well. assertEquals(3, ephemeralResult.getLimits().size()); @@ -690,10 +716,11 @@ void testEphemeralStorage() { // we should also preserve additional properties container.getResources().setAdditionalProperty("additional", "some-value"); ResourceRequirements additionalProperties = K8sTaskAdapter.getResourceRequirements( - container.getResources(), - 100, - 1000 + container.getResources(), + 100, + 1000 ); assertEquals(1, additionalProperties.getAdditionalProperties().size()); } -} \ No newline at end of file + +} diff --git a/extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedProbesRemovedOutput.yaml b/extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedProbesRemovedOutput.yaml index a4dccde41633..8379b2958243 100644 --- a/extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedProbesRemovedOutput.yaml +++ b/extensions-core/kubernetes-overlord-extensions/src/test/resources/expectedProbesRemovedOutput.yaml @@ -51,6 +51,13 @@ spec: - containerPort: 8100 name: "druid-port" protocol: "TCP" + resources: + limits: + cpu: "1000m" + memory: "2400000000" + requests: + cpu: "1000m" + memory: "2400000000" hostname: "id-3e70afe5cd823dfc7dd308eea616426b" restartPolicy: "Never" ttlSecondsAfterFinished: 172800 \ No newline at end of file