Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions modules/k3s/src/main/java/org/testcontainers/k3s/K3sContainer.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.model.DockerObjectAccessor;
import lombok.SneakyThrows;
import org.testcontainers.containers.BindMode;
Expand All @@ -18,13 +17,16 @@

public class K3sContainer extends GenericContainer<K3sContainer> {

private String kubeConfigYaml;
public static int KUBE_SECURE_PORT = 6443;

public static int RANCHER_WEBHOOK_PORT = 8443;


public K3sContainer(DockerImageName dockerImageName) {
super(dockerImageName);
dockerImageName.assertCompatibleWith(DockerImageName.parse("rancher/k3s"));

addExposedPorts(6443, 8443);
addExposedPorts(KUBE_SECURE_PORT, RANCHER_WEBHOOK_PORT);
setPrivilegedMode(true);
withCreateContainerCmdModifier(it -> {
DockerObjectAccessor.overrideRawValue(
Expand All @@ -47,8 +49,8 @@ public K3sContainer(DockerImageName dockerImageName) {
}

@SneakyThrows
@Override
protected void containerIsStarted(InspectContainerResponse containerInfo) {
public String getKubeConfigYaml(String networkAlias) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To avoid confusion, IMO we should rename this method to something that will tell that the returned config is for internal communication, as otherwise someone may call getKubeConfigYaml("k3s") and then expect it to work from their machine (in addition to in-network usage)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, my main thought with this PR is the risk of user confusion. This will be, I think, the first time in a module that we're outputting an address which is specifically not usable by the test host machine itself.

I think we need this method to be named to reflect that this returns a kubeconfig that is for use inside of the docker network.

We should also enhance the documentation here - both Javadocs and the md docs, so that people can understand when to use this. As @bsideup suggested, including the test into the docs would be a good way to demonstrate it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@rnorth BTW how do you feel about changing from containerIsStarted to "get file every time"? IMO getKubeConfigYaml should return pre-calculated value from containerIsStarted, while generateInternalKubeConfigYaml (or whatever the name is) can do it on demand (or maybe from the same file, just by changing the host)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree that getKubeConfigYaml being a side-effect-free simple getter would feel better and be the more intuitive API.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I think it would be nicer to keep the fetch of the kubeconfig file in containerIsStarted. It would make it clearer that the file is effectively immutable rather than something that k3s might be changing over time.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have updated the code. The documentation part is missing, I will include it on another commit.


ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());

ObjectNode rawKubeConfig = copyFileFromContainer(
Expand All @@ -61,15 +63,26 @@ protected void containerIsStarted(InspectContainerResponse containerInfo) {
throw new IllegalStateException("'/clusters/0/cluster' expected to be an object");
}
ObjectNode clusterConfig = (ObjectNode) clusterNode;

clusterConfig.replace("server", new TextNode("https://" + this.getHost() + ":" + this.getMappedPort(6443)));
String serverUrl = resolveServerUrl(networkAlias);
clusterConfig.replace("server", new TextNode(serverUrl));

rawKubeConfig.set("current-context", new TextNode("default"));

kubeConfigYaml = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(rawKubeConfig);
return objectMapper.writerWithDefaultPrettyPrinter()
.writeValueAsString(rawKubeConfig);
}

public String getKubeConfigYaml() {
return kubeConfigYaml;
return getKubeConfigYaml(this.getHost());
}

private String resolveServerUrl(String networkAlias) {
if (networkAlias.equals(this.getHost())) {
return "https://" + this.getHost() + ":" + this.getMappedPort(KUBE_SECURE_PORT);
} else if (this.getNetworkAliases().contains(networkAlias)) {
return "https://" + networkAlias + ":" + KUBE_SECURE_PORT;
} else {
throw new IllegalArgumentException(networkAlias + " is not a network alias for k3s container");
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package org.testcontainers.k3s;

import lombok.extern.slf4j.Slf4j;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.output.WaitingConsumer;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.MountableFile;

import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.TimeUnit;

import static org.testcontainers.containers.Network.newNetwork;
import static org.testcontainers.containers.output.OutputFrame.OutputType.STDOUT;

@Slf4j
public class KubectlContainerTest {

public static Network network = newNetwork();

public static K3sContainer k3s = new K3sContainer(DockerImageName.parse("rancher/k3s:v1.21.3-k3s1"))
.withNetwork(network)
.withNetworkAliases("k3s");

@BeforeClass
public static void setup() {
k3s.start();
}

@AfterClass
public static void tearDown() {
k3s.stop();
}

@Test
public void shouldExposeKubeConfigForNetworkAlias() throws Exception {

String kubeConfigYaml = k3s.getKubeConfigYaml("k3s");

Path tempFile = Files.createTempFile(null, null);
Files.write(tempFile, kubeConfigYaml.getBytes(StandardCharsets.UTF_8));

GenericContainer<?> kubectlContainer = new GenericContainer<>(DockerImageName.parse("rancher/kubectl:v1.23.3"))
.withNetwork(network)
.withCopyFileToContainer(MountableFile.forHostPath(tempFile.toAbsolutePath()), "/.kube/config")
.withCommand("get namespaces");

kubectlContainer.start();

WaitingConsumer consumer = new WaitingConsumer();
kubectlContainer.followOutput(consumer, STDOUT);

consumer.waitUntil(frame ->
frame.getUtf8String().contains("kube-system"), 30, TimeUnit.SECONDS);
}

@Test(expected = IllegalArgumentException.class)
public void shouldThrowAnExceptionForUnknownNetworkAlias() {
k3s.getKubeConfigYaml("not-set-network-alias");
}
}