diff --git a/docs/modules/elasticsearch.md b/docs/modules/elasticsearch.md index c1fb4a3f6a8..5817a21de02 100644 --- a/docs/modules/elasticsearch.md +++ b/docs/modules/elasticsearch.md @@ -30,6 +30,33 @@ You can turn on security by providing a password: [HttpClient](../../modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java) inside_block:httpClientSecuredContainer +## Kibana container + +This module also provides a `KibanaContainer` for testing with [Kibana](https://www.elastic.co/kibana). +Kibana requires a connection to Elasticsearch and `KibanaContainer` supports two modes: managed and external. + +### Managed mode + +In managed mode, `KibanaContainer` automatically connects to an `ElasticsearchContainer`: + + +[Kibana with Elasticsearch](../../modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/KibanaContainerTest.java) inside_block:managedModeCanStartAndReachElasticsearchInSameExplicitNetwork + + +When using managed mode with explicit networks, both containers must share the same `Network` instance. +Alternatively, you can omit the network configuration entirely, and `KibanaContainer` will do its best effort to create a shared, ad-hoc network automatically. + +### External mode + +In external mode, `KibanaContainer` connects to an external Elasticsearch instance via URL and using provided credentials: + + +[Kibana with external Elasticsearch](../../modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/KibanaContainerTest.java) inside_block:externalModeCanWorkWithUsernamePassword + + +For external mode with HTTPS, use `withElasticsearchCaCertificate()` to provide the CA certificate. +You can authenticate using either username/password (`withElasticsearchCredentials()`) or service account tokens (`withElasticsearchServiceAccountToken()`). + ## Adding this module to your project dependencies Add the following dependency to your `pom.xml`/`build.gradle` file: diff --git a/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java index 54f1a4c17f8..35a2fb7f707 100644 --- a/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java +++ b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/ElasticsearchContainer.java @@ -209,10 +209,99 @@ public ElasticsearchContainer withCertPath(String certPath) { return this; } + protected String getCertPath() { + return certPath; + } + public String getHttpHostAddress() { return getHost() + ":" + getMappedPort(ELASTICSEARCH_DEFAULT_PORT); } + /** + * Checks env first if this implies HTTP/HTTPS. + * Otherwise, detects the scheme used by Elasticsearch using curl + * + * @return "http" or "https" + */ + public String getHttpScheme() { + String securityEnabled = getEnvMap().get("xpack.security.enabled"); + String httpSslEnabled = getEnvMap().get("xpack.security.http.ssl.enabled"); + + // Respect explicit user config + if ("false".equalsIgnoreCase(securityEnabled) || "false".equalsIgnoreCase(httpSslEnabled)) { + return "http"; + } + if ("true".equalsIgnoreCase(httpSslEnabled)) { + return "https"; + } + + if (!isRunning()) { + throw new IllegalStateException( + "Cannot determine HTTP scheme: environment variables are not set and container is not running for curl probe" + ); + } + + ExecResult httpsResult = null; + ExecResult httpResult = null; + try { + // HTTPS probe: any HTTP response (200/401/403/...) => scheme is HTTPS. + // http_code == 000 means we didn't get an HTTP response (TLS/connect failure/timeout). + httpsResult = + execInContainer( + "curl", + "-sS", + "-k", + "--connect-timeout", + "2", + "--max-time", + "4", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "https://localhost:" + ELASTICSEARCH_DEFAULT_PORT + "/" + ); + if (httpsResult.getExitCode() == 0 && !"000".equals(httpsResult.getStdout().trim())) { + return "https"; + } + + // HTTP probe + httpResult = + execInContainer( + "curl", + "-sS", + "--connect-timeout", + "2", + "--max-time", + "4", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "http://localhost:" + ELASTICSEARCH_DEFAULT_PORT + "/" + ); + if (httpResult.getExitCode() == 0 && !"000".equals(httpResult.getStdout().trim())) { + return "http"; + } + } catch (Exception e) { + throw new RuntimeException("Failed to detect protocol via curl", e); + } + + throw new RuntimeException( + String.format( + "Failed to detect protocol via curl. Both HTTPS and HTTP probes failed. " + + "HTTPS probe - exit code: %d, stdout: %s, stderr: %s; " + + "HTTP probe - exit code: %d, stdout: %s, stderr: %s", + httpsResult.getExitCode(), + httpsResult.getStdout(), + httpsResult.getStderr(), + httpResult.getExitCode(), + httpResult.getStdout(), + httpResult.getStderr() + ) + ); + } + // The TransportClient will be removed in Elasticsearch 8. No need to expose this port anymore in the future. @Deprecated public InetSocketAddress getTcpHost() { diff --git a/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/KibanaContainer.java b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/KibanaContainer.java new file mode 100644 index 00000000000..6eca3068345 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/testcontainers/elasticsearch/KibanaContainer.java @@ -0,0 +1,613 @@ +package org.testcontainers.elasticsearch; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.dockerjava.api.command.InspectContainerResponse; +import com.github.dockerjava.api.model.ContainerNetwork; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.rnorth.ducttape.unreliables.Unreliables; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.Transferable; +import org.testcontainers.utility.Base58; +import org.testcontainers.utility.ComparableVersion; +import org.testcontainers.utility.DockerImageName; + +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Testcontainers implementation for Kibana. + * Minimum supported version: {@value MINIMUM_SUPPORTED_VERSION} + *

+ * Supports two modes: + *

+ *

+ */ +public class KibanaContainer extends GenericContainer { + + public static final String ES_CA_CERT_PATH = "/usr/share/kibana/config/certs/es-ca.crt"; + + private static final Logger log = LoggerFactory.getLogger(KibanaContainer.class); + + public static final int KIBANA_DEFAULT_PORT = 5601; + + public static final String KIBANA_SYSTEM_USER = "kibana_system"; + + public static final String MINIMUM_SUPPORTED_VERSION = "8.0.0"; + + private static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("docker.elastic.co/kibana/kibana"); + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private ElasticsearchContainer elasticsearch; + + private String elasticsearchUrl; + + private String elasticsearchUsername; + + private String elasticsearchPassword; + + private String elasticsearchServiceAccountToken; + + private byte[] elasticsearchCaCertificate; + + private Duration startupTimeout = Duration.ofSeconds(120); + + /** + * If KibanaContainer creates an ad-hoc shared network (managed mode, neither container has an explicit network), + * it owns closing it. + */ + private Network createdSharedNetwork; + + /** + * Creates a KibanaContainer in managed mode. + * Kibana automatically connects to the provided Elasticsearch container. + * + * @param elasticsearch the Elasticsearch container to connect to + */ + public KibanaContainer(ElasticsearchContainer elasticsearch) { + this(buildDockerImageName(elasticsearch)); + this.elasticsearch = elasticsearch; + dependsOn(elasticsearch); + } + + /** + * Creates a KibanaContainer in external mode. + * Use {@link #withElasticsearchUrl(String)} to configure the Elasticsearch connection. Use other methods to provide security credentials and such. + * + * @param dockerImageName the Docker image name + */ + public KibanaContainer(String dockerImageName) { + this(DockerImageName.parse(dockerImageName)); + } + + /** + * Creates a KibanaContainer in external mode. + * Use {@link #withElasticsearchUrl(String)} to configure the Elasticsearch connection. Use other methods to provide security credentials and such. + * + * @param dockerImageName the Docker image name + */ + public KibanaContainer(final DockerImageName dockerImageName) { + super(dockerImageName); + ensureCompatibleVersion(dockerImageName.getVersionPart()); + dockerImageName.assertCompatibleWith(DEFAULT_IMAGE_NAME); + + withExposedPorts(KIBANA_DEFAULT_PORT); + //we have to explicitly set wait the strategy later on in configure, once we know the security configuration + setWaitStrategy(null); + } + + /** + * Configures the Elasticsearch URL for external mode. + * + * @param elasticsearchUrl the Elasticsearch URL (e.g., "https://my.fancy.setup.elastic.cloud:9200") + * @return this container instance + * @throws IllegalStateException if already using managed mode + */ + public KibanaContainer withElasticsearchUrl(String elasticsearchUrl) { + if (elasticsearch != null) { + throw new IllegalStateException("Cannot set Elasticsearch URL when using Elasticsearch container"); + } + this.elasticsearchUrl = elasticsearchUrl; + return this; + } + + /** + * Configures Kibana to authenticate using the kibana_system user. + * + * @param password the password for the kibana_system user + * @return this container instance + */ + public KibanaContainer withElasticsearchKibanaSystemPassword(String password) { + return withElasticsearchCredentials(KIBANA_SYSTEM_USER, password); + } + + /** + * Configures Elasticsearch credentials for authentication. + * + * @param username the Elasticsearch username (cannot be 'elastic') + * @param password the password + * @return this container instance + * @throws IllegalStateException if a service account token is already configured + * @throws IllegalArgumentException if credentials are invalid + */ + public KibanaContainer withElasticsearchCredentials(String username, String password) { + if (elasticsearchServiceAccountToken != null) { + throw new IllegalStateException( + "Conflicting Elasticsearch credentials: provide either a service account token " + + "or a username/password pair, not both." + ); + } + if (StringUtils.isAnyBlank(username, password)) { + throw new IllegalArgumentException("Kibana credentials cannot be blank"); + } + if (!username.equals(username.trim()) || !password.equals(password.trim())) { + throw new IllegalArgumentException("Kibana credentials cannot have leading or trailing whitespace"); + } + if ("elastic".equals(username)) { + throw new IllegalArgumentException("Username 'elastic' is reserved for internal use by Elasticsearch"); + } + + this.elasticsearchUsername = username; + this.elasticsearchPassword = password; + return this; + } + + /** + * Configures a service account token for Elasticsearch authentication. + * + * @param token the service account token + * @return this container instance + * @throws IllegalStateException if username/password credentials are already configured + * @throws IllegalArgumentException if token is blank + */ + public KibanaContainer withElasticsearchServiceAccountToken(String token) { + if (elasticsearchUsername != null) { + throw new IllegalStateException( + "Conflicting Elasticsearch credentials: provide either a service account token " + + "or a username/password pair, not both." + ); + } + if (StringUtils.isBlank(token)) { + throw new IllegalArgumentException("Service account token cannot be empty"); + } + + if (!token.equals(token.trim())) { + throw new IllegalArgumentException("Service token cannot have leading or trailing whitespace"); + } + this.elasticsearchServiceAccountToken = token; + return this; + } + + /** + * Configures the Elasticsearch CA certificate for HTTPS connections. + * + * @param caCertificate the CA certificate in PEM format + * @return this container instance + * @throws IllegalArgumentException if certificate is empty + */ + public KibanaContainer withElasticsearchCaCertificate(byte[] caCertificate) { + if (caCertificate == null || caCertificate.length == 0) { + throw new IllegalArgumentException("Elasticsearch CA certificate cannot be empty"); + } + this.elasticsearchCaCertificate = caCertificate; + return this; + } + + @Override + protected void configure() { + super.configure(); + + addEnv("XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY", generateRandomKey(32)); + addEnv("SERVER_NAME", "kibana"); + + if (elasticsearchCaCertificate != null) { + withCopyToContainer(Transferable.of(elasticsearchCaCertificate), ES_CA_CERT_PATH); + addEnv("ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES", ES_CA_CERT_PATH); + } + if (elasticsearch != null) { + configureManagedElasticsearch(); + } else if (elasticsearchUrl != null) { + configureExternalElasticsearch(); + } else { + throw new IllegalStateException( + "Elasticsearch must be configured either via constructor KibanaContainer(elasticsearch) " + + "or via .withElasticsearchUrl() for external Elasticsearch" + ); + } + //wait strategy is set in configure, because we don't know the security configuration before + configureWaitStrategy(); + } + + @Override + protected void containerIsStarted(InspectContainerResponse containerInfo) { + super.containerIsStarted(containerInfo); + log.info("Kibana is now ready, it can be accessed at http://{}", getHttpHostAddress()); + } + + @Override + public void stop() { + super.stop(); + if (createdSharedNetwork != null) { + try { + createdSharedNetwork.close(); + } catch (Exception e) { + log.debug("Failed to close shared network", e); + } finally { + createdSharedNetwork = null; + } + } + } + + @Override + public KibanaContainer withStartupTimeout(Duration startupTimeout) { + this.startupTimeout = startupTimeout; + return this; + } + + private static DockerImageName buildDockerImageName(ElasticsearchContainer elasticsearch) { + String esVersion = DockerImageName.parse(elasticsearch.getDockerImageName()).getVersionPart(); + ensureCompatibleVersion(esVersion); + return DEFAULT_IMAGE_NAME.withTag(esVersion); + } + + private void configureExternalElasticsearch() { + addEnv("ELASTICSEARCH_HOSTS", elasticsearchUrl); + if (elasticsearchServiceAccountToken != null) { + addEnv("ELASTICSEARCH_SERVICEACCOUNTTOKEN", elasticsearchServiceAccountToken); + } else if (elasticsearchUsername != null && elasticsearchPassword != null) { + addEnv("ELASTICSEARCH_USERNAME", elasticsearchUsername); + addEnv("ELASTICSEARCH_PASSWORD", elasticsearchPassword); + } else { + log.info( + "No Elasticsearch credentials provided for external mode; Kibana will attempt to connect anonymously" + ); + } + } + + private void configureManagedElasticsearch() { + ensureCorrectNetworkSetupForManagedMode(); + + if (getNetwork() == null) { + createAdHocNetwork(); + } + + String protocol = elasticsearch.getHttpScheme(); + + String hosts = protocol + "://" + resolveExistingEsDnsNameOnNetwork(getNetwork()) + ":9200"; + addEnv("ELASTICSEARCH_HOSTS", hosts); + + if ("https".equals(protocol)) { + // In managed mode, if Elasticsearch uses HTTPS we must configure Kibana with the ES CA, unless provided by user + if (this.elasticsearchCaCertificate == null) { + byte[] ca = copyElasticsearchHttpCaCertificateOrThrow(); + + withCopyToContainer(Transferable.of(ca), ES_CA_CERT_PATH); + addEnv("ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES", ES_CA_CERT_PATH); + } + if (elasticsearch != null && !getEnvMap().containsKey("ELASTICSEARCH_SSL_VERIFICATIONMODE")) { + addEnv("ELASTICSEARCH_SSL_VERIFICATIONMODE", "certificate"); + } + } + + // Elasticsearch 8.x+ has the security enabled by default, so lack of the env var set to false means security is enabled + boolean securityDisabled = "false".equalsIgnoreCase(elasticsearch.getEnvMap().get("xpack.security.enabled")); + + if (!securityDisabled) { + // Managed mode: authenticate Kibana -> Elasticsearch using a Kibana service account token. + // This avoids any password lifecycle management for kibana_system. + String token = createKibanaServiceAccountToken(protocol); + addEnv("ELASTICSEARCH_SERVICEACCOUNTTOKEN", token); + } + } + + private byte[] copyElasticsearchHttpCaCertificateOrThrow() { + try { + return elasticsearch.copyFileFromContainer(elasticsearch.getCertPath(), IOUtils::toByteArray); + } catch (Exception e) { + throw new IllegalStateException( + "Failed to copy Elasticsearch HTTP CA certificate from '" + + elasticsearch.getCertPath() + + "'. " + + "In managed HTTPS mode, KibanaContainer requires access to the Elasticsearch HTTP CA.", + e + ); + } + } + + private void ensureCorrectNetworkSetupForManagedMode() { + Network esNetwork = elasticsearch.getNetwork(); + Network kbNetwork = this.getNetwork(); + + if ((esNetwork == null) != (kbNetwork == null)) { + throw new IllegalStateException( + "Managed mode requires either both containers share the same explicit network, " + + "or neither specifies a network (KibanaContainer will create one). " + ); + } + + // Both explicit: must be same + if (esNetwork != kbNetwork) { + throw new IllegalStateException( + "Elasticsearch and Kibana have different networks configured. " + + "In managed mode both containers must share the same explicit network instance, " + + "or neither must define a network." + ); + } + } + + private void createAdHocNetwork() { + // Fully managed: create ad-hoc network and own it. + createdSharedNetwork = Network.newNetwork(); + withNetwork(createdSharedNetwork); + + // Managed-mode safety rule: by the time Kibana is configuring itself, Elasticsearch must already be + // started (via dependsOn) + String esId = requireElasticsearchContainerId(); + + // Elasticsearch is already created/started. Attach it to the ad-hoc network. + // We don't need to provide an explicit alias - we'll use the container name for DNS resolution. + // Equivalent of https://docs.docker.com/reference/cli/docker/network/connect/ + connectRunningContainerToNetwork(esId, createdSharedNetwork); + } + + private String resolveExistingEsDnsNameOnNetwork(Network network) { + String esId = requireElasticsearchContainerId(); + + InspectContainerResponse info = DockerClientFactory.instance().client().inspectContainerCmd(esId).exec(); + + Map networks = info.getNetworkSettings().getNetworks(); + if (networks == null) { + throw new IllegalStateException("Elasticsearch container has no network configuration"); + } + + // Try to find the network endpoint - Docker may key by network name or ID + ContainerNetwork endpoint = findNetworkEndpoint(networks, network); + if (endpoint == null) { + throw new IllegalStateException( + "Elasticsearch container is not connected to the expected network. " + + "Ensure both containers use the same Network instance." + ); + } + + // Prefer user-defined network aliases (skip Testcontainers auto-generated tc-* aliases) + if (endpoint.getAliases() != null && !endpoint.getAliases().isEmpty()) { + for (String alias : endpoint.getAliases()) { + if (StringUtils.isNotBlank(alias)) { + String cleaned = alias.trim(); + // Skip Testcontainers auto-generated aliases (tc-*), prefer user-defined ones + if (!cleaned.startsWith("tc-")) { + log.info("Using Elasticsearch network alias: {}", cleaned); + return cleaned; + } + } + } + } + + // Fallback: use container name + String containerName = info.getName(); + if (containerName != null && !containerName.trim().isEmpty()) { + String dnsName = containerName.replaceFirst("^/", "").trim(); + log.info("No user-defined network alias found, using Elasticsearch container name: {}", dnsName); + return dnsName; + } + + throw new IllegalStateException( + "Cannot determine Elasticsearch DNS name. " + + "When using a custom network, set a network alias on the Elasticsearch container." + ); + } + + private ContainerNetwork findNetworkEndpoint(Map networks, Network network) { + // Try network ID first + ContainerNetwork endpoint = networks.get(network.getId()); + if (endpoint != null) { + return endpoint; + } + + // Try network name as fallback (Docker may key by name instead of ID) + try { + String networkName = DockerClientFactory + .instance() + .client() + .inspectNetworkCmd() + .withNetworkId(network.getId()) + .exec() + .getName(); + if (networkName != null) { + return networks.get(networkName); + } + } catch (Exception e) { + // Ignore and return null + } + + return null; + } + + private String requireElasticsearchContainerId() { + String id = elasticsearch.getContainerId(); + if (StringUtils.isBlank(id)) { + throw new IllegalStateException( + "Elasticsearch containerId is not available. In managed mode, Elasticsearch must be started via dependsOn(elasticsearch) " + + "before KibanaContainer is started." + ); + } + return id.trim(); + } + + private void connectRunningContainerToNetwork(String containerId, Network network) { + String networkId = network.getId(); + + try { + DockerClientFactory + .instance() + .client() + .connectToNetworkCmd() + .withContainerId(containerId) + .withNetworkId(networkId) + .exec(); + } catch (Exception e) { + throw new IllegalStateException("Failed to connect Elasticsearch container to ad-hoc shared network", e); + } + } + + private static void ensureCompatibleVersion(String esVersion) { + ComparableVersion comparableVersion = new ComparableVersion(esVersion); + if (comparableVersion.isLessThan(MINIMUM_SUPPORTED_VERSION)) { + throw new IllegalArgumentException( + String.format( + "Kibana version %s is not supported. Minimum version is %s", + comparableVersion, + MINIMUM_SUPPORTED_VERSION + ) + ); + } + } + + private String createKibanaServiceAccountToken(String protocol) { + if (elasticsearch == null) { + throw new IllegalStateException("Cannot create service account token in external mode"); + } + + String elasticPassword = elasticsearch + .getEnvMap() + .getOrDefault("ELASTIC_PASSWORD", ElasticsearchContainer.ELASTICSEARCH_DEFAULT_PASSWORD); + + // Create a unique token name to avoid collisions if the same ES container is reused. + String tokenName = "tc-kibana-" + Base58.randomString(12); + + String endpoint = protocol + "://localhost:9200/_security/service/elastic/kibana/credential/token/" + tokenName; + + return Unreliables.retryUntilSuccess( + 45, + TimeUnit.SECONDS, + () -> { + String curlTlsArgs = ""; + if ("https".equals(protocol)) { + // In managed HTTPS mode, use the Elasticsearch HTTP CA for curl. + curlTlsArgs = " --cacert '" + elasticsearch.getCertPath() + "'"; + } + + String curlCommand = String.format( + "curl -sS%s -u \"elastic:$1\" -H 'Content-Type: application/json' -X POST '%s'", + curlTlsArgs, + endpoint + ); + + Container.ExecResult result = elasticsearch.execInContainer( + "/bin/sh", + "-c", + curlCommand, + "sh", + elasticPassword + ); + + String stdout = (result.getStdout() == null) ? "" : result.getStdout(); + String stderr = (result.getStderr() == null) ? "" : result.getStderr(); + + if (result.getExitCode() != 0) { + throw new RuntimeException( + "Failed to create Kibana service account token. Exit code: " + + result.getExitCode() + + ", stdout: " + + stdout + + ", stderr: " + + stderr + ); + } + + JsonNode json = OBJECT_MAPPER.readTree(stdout); + JsonNode value = json.path("token").path("value"); + if (value.isTextual() && !value.asText().trim().isEmpty()) { + return value.asText().trim(); + } + + throw new RuntimeException("Service account token response did not contain token.value: " + stdout); + } + ); + } + + private String generateRandomKey(int length) { + return Base58.randomString(length); + } + + /** + * Returns the HTTP host address for accessing Kibana. + * + * @return the host address in the format "host:port" + */ + public String getHttpHostAddress() { + return getHost() + ":" + getMappedPort(KIBANA_DEFAULT_PORT); + } + + private void configureWaitStrategy() { + if (this.getWaitStrategy() != null) { + // the user might have set a custom wait strategy + return; + } + HttpWaitStrategy strategy = Wait + .forHttp("/api/status") + .forPort(KIBANA_DEFAULT_PORT) + .forStatusCode(200) + .forResponsePredicate(this::isKibanaReady); + + // Add authentication if we have Elasticsearch credentials available + String serviceToken = getEnvMap().get("ELASTICSEARCH_SERVICEACCOUNTTOKEN"); + String username = getEnvMap().get("ELASTICSEARCH_USERNAME"); + String password = getEnvMap().get("ELASTICSEARCH_PASSWORD"); + + if (serviceToken != null) { + strategy = strategy.withHeader("Authorization", "Bearer " + serviceToken); + } else if (username != null && password != null) { + strategy = strategy.withBasicCredentials(username, password); + } + + setWaitStrategy(strategy.withStartupTimeout(this.startupTimeout)); + } + + private boolean isKibanaReady(String body) { + try { + JsonNode json = OBJECT_MAPPER.readTree(body); + JsonNode status = json.path("status"); + + String overallLevel = status.path("overall").path("level").asText(null); + String elasticsearchLevel = status.path("core").path("elasticsearch").path("level").asText(null); + String savedObjectsLevel = status.path("core").path("savedObjects").path("level").asText(null); + + boolean overallAvailable = "available".equalsIgnoreCase(overallLevel); + boolean elasticsearchAvailable = "available".equalsIgnoreCase(elasticsearchLevel); + boolean savedObjectsAvailable = "available".equalsIgnoreCase(savedObjectsLevel); + + boolean isReady = overallAvailable && elasticsearchAvailable && savedObjectsAvailable; + + if (log.isDebugEnabled()) { + log.debug( + "Kibana status check: READY={} (overall={}, elasticsearch={}, savedObjects={})", + isReady, + overallLevel, + elasticsearchLevel, + savedObjectsLevel + ); + } + + return isReady; + } catch (Exception e) { + log.debug("Kibana status check: FAILED to parse response - {}", e.getMessage()); + return false; + } + } +} diff --git a/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java b/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java index f0219713751..52cd78b4a34 100644 --- a/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java +++ b/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/ElasticsearchContainerTest.java @@ -602,4 +602,37 @@ private void assertClusterHealthResponse(ElasticsearchContainer container) throw assertThat(response.getStatusLine().getStatusCode()).isEqualTo(200); assertThat(EntityUtils.toString(response.getEntity())).contains("cluster_name"); } + + @Test + void testGetHttpSchemeForElasticsearch7ReturnsHttp() { + try (ElasticsearchContainer container = new ElasticsearchContainer(ELASTICSEARCH_IMAGE)) { + container.start(); + assertThat(container.getHttpScheme()).isEqualTo("http"); + } + } + + @Test + void testGetHttpSchemeForElasticsearch8ReturnsHttps() { + try ( + ElasticsearchContainer container = new ElasticsearchContainer( + "docker.elastic.co/elasticsearch/elasticsearch:8.1.2" + ) + ) { + container.start(); + assertThat(container.getHttpScheme()).isEqualTo("https"); + } + } + + @Test + void testGetHttpSchemeForElasticsearch8WithSslDisabledReturnsHttp() { + try ( + ElasticsearchContainer container = new ElasticsearchContainer( + "docker.elastic.co/elasticsearch/elasticsearch:8.1.2" + ) + .withEnv("xpack.security.http.ssl.enabled", "false") + ) { + container.start(); + assertThat(container.getHttpScheme()).isEqualTo("http"); + } + } } diff --git a/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/KibanaContainerTest.java b/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/KibanaContainerTest.java new file mode 100644 index 00000000000..76919d247ad --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/testcontainers/elasticsearch/KibanaContainerTest.java @@ -0,0 +1,461 @@ +package org.testcontainers.elasticsearch; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.Transferable; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +class KibanaContainerTest { + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static final String ES_IMAGE = "docker.elastic.co/elasticsearch/elasticsearch:9.2.4"; + + @Test + void cannotCreateKibanaContainerForVersionLessThan8() { + Assertions + .assertThatThrownBy(() -> new KibanaContainer("docker.elastic.co/kibana/kibana:7.17.29")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("is not supported"); + } + + @Test + void managedModeCanStartAndReachElasticsearchInSameExplicitNetwork() throws IOException { + // managedModeCanStartAndReachElasticsearchInSameExplicitNetwork { + try ( + Network network = Network.newNetwork(); + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE).withNetwork(network); + KibanaContainer kibana = new KibanaContainer(es).withNetwork(network) + ) { + es.start(); + kibana.start(); + + String status = getKibanaStatus(kibana); + Assertions.assertThat(status).isEqualTo("available"); + } + // } + } + + @Test + void managedModeCannotStartWithOnlyESNetworkExplicit() { + Network network = Network.newNetwork(); + try ( + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE).withNetwork(network); + KibanaContainer kibana = new KibanaContainer(es) + ) { + Assertions + .assertThatThrownBy(kibana::start) + .isInstanceOf(ContainerLaunchException.class) + .satisfies(ex -> { + Assertions + .assertThat(ex.getCause()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("explicit network"); + }); + } + } + + @Test + void managedModeCannotStartWithDifferentExplicitNetworks() { + try ( + Network esNetwork = Network.newNetwork(); + Network kibanaNetwork = Network.newNetwork(); + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE).withNetwork(esNetwork); + KibanaContainer kibana = new KibanaContainer(es).withNetwork(kibanaNetwork) + ) { + Assertions + .assertThatThrownBy(kibana::start) + .isInstanceOf(ContainerLaunchException.class) + .satisfies(ex -> { + Assertions + .assertThat(ex.getCause()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("different networks"); + }); + } + } + + @Test + void managedModeUsesCustomNetworkAliasInExplicitNetwork() throws Exception { + final String customEsAlias = "my-custom-es-alias"; + + try ( + Network network = Network.newNetwork(); + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE) + .withNetwork(network) + .withNetworkAliases(customEsAlias); + KibanaContainer kibana = new KibanaContainer(es).withNetwork(network) + ) { + kibana.start(); + + Assertions.assertThat(kibana.isRunning()).isTrue(); + + // Verify Kibana uses the custom alias (not auto-generated tc-* alias) + Container.ExecResult result = kibana.execInContainer("sh", "-c", "env | grep ELASTICSEARCH_HOSTS"); + + Assertions.assertThat(result.getStdout()).contains(customEsAlias).contains(":9200"); + } + } + + @Test + void managedModeCanStartAndReachElasticsearchWithoutExplicitNetwork() throws IOException { + try ( + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE); + KibanaContainer kibana = new KibanaContainer(es) + ) { + kibana.start(); + + String status = getKibanaStatus(kibana); + Assertions.assertThat(status).isEqualTo("available"); + } + } + + @Test + void managedModeCanStartWithoutElasticsearchSecurity() throws IOException { + try ( + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE).withEnv("xpack.security.enabled", "false"); + KibanaContainer kibana = new KibanaContainer(es) + ) { + kibana.start(); + + String status = getKibanaStatus(kibana); + Assertions.assertThat(status).isEqualTo("available"); + } + } + + @Test + void managedModeCanStartWithoutElasticsearchHttps() throws IOException { + try ( + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE) + .withEnv("xpack.security.enabled", "true") + .withEnv("xpack.security.http.ssl.enabled", "false"); + KibanaContainer kibana = new KibanaContainer(es) + ) { + es.start(); + kibana.start(); + + String status = getKibanaStatus(kibana); + Assertions.assertThat(status).isEqualTo("available"); + } + } + + @Test + void externalModeFailsWithConflictingCredentials() { + Assertions + .assertThatThrownBy(() -> { + new KibanaContainer("docker.elastic.co/kibana/kibana:8.0.0") + .withElasticsearchCredentials("user", "pass") + .withElasticsearchServiceAccountToken("token"); + }) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Conflicting Elasticsearch credentials"); + } + + @Test + void managedModeFailsWhenSettingElasticsearchUrl() { + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE); + Assertions + .assertThatThrownBy(() -> { + new KibanaContainer(es).withElasticsearchUrl("http://somewhere.over.the.rainbow:9200"); + }) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Cannot set Elasticsearch URL when using Elasticsearch container"); + } + + @Test + void failsWhenNoElasticsearchConfigured() { + try (KibanaContainer kibana = new KibanaContainer("docker.elastic.co/kibana/kibana:8.0.0")) { + Assertions + .assertThatThrownBy(kibana::start) + .isInstanceOf(ContainerLaunchException.class) + .satisfies(ex -> { + Assertions + .assertThat(ex.getCause()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Elasticsearch must be configured"); + }); + } + } + + @Test + void externalModeCanWorkWithUsernamePassword() throws IOException, InterruptedException { + final String esHostname = "elasticsearch"; + + // externalModeCanWorkWithUsernamePassword { + try ( + Network network = Network.newNetwork(); + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE) + .withNetwork(network) + .withNetworkAliases(esHostname) + .withEnv("xpack.security.http.ssl.enabled", "false") + ) { + es.start(); + String kibanaSystemPassword = setKibanaSystemPassword(es); + + try ( + KibanaContainer kibana = new KibanaContainer("docker.elastic.co/kibana/kibana:9.2.2") //this minor version is intentionally below ES version + .withNetwork(network) + .withElasticsearchUrl("http://" + esHostname + ":9200") + .withElasticsearchKibanaSystemPassword(kibanaSystemPassword) + ) { + kibana.start(); + String status = getKibanaStatus(kibana); + Assertions.assertThat(status).isEqualTo("available"); + } + } + // } + } + + @Test + void externalModeCanWorkWithoutCredentials() throws IOException { + final String esHostname = "elasticsearch"; + + try ( + Network network = Network.newNetwork(); + ElasticsearchContainer es = new ElasticsearchContainer(ES_IMAGE) + .withNetwork(network) + .withNetworkAliases(esHostname) + .withEnv("xpack.security.enabled", "false") + .withEnv("xpack.security.http.ssl.enabled", "false"); + KibanaContainer kibana = new KibanaContainer("docker.elastic.co/kibana/kibana:9.2.4") + .withNetwork(network) + .withElasticsearchUrl("http://" + esHostname + ":9200") + ) { + es.start(); + kibana.start(); + String status = getKibanaStatus(kibana); + Assertions.assertThat(status).isEqualTo("available"); + } + } + + @Test + void externalModeCanStartAndReachElasticsearchWithCertAndServiceToken() throws Exception { + byte[] caCrt; + byte[] nodeCrt; + byte[] nodeKey; + + String esHostname = "elasticsearch"; + + String instancesYml = + "instances:\n" + + " - name: es01\n" + + " dns: [ \"localhost\", \"" + + esHostname + + "\", \"es01\" ]\n" + + " ip: [ \"127.0.0.1\" ]\n"; + + try ( + ElasticsearchContainer setup = new ElasticsearchContainer(ES_IMAGE) + .withEnv("discovery.type", "single-node") + .withCopyToContainer( + Transferable.of(instancesYml.getBytes(StandardCharsets.UTF_8), 0644), + "/tmp/instances.yml" + ) + ) { + setup.start(); + + // Run certutil inside the running container and write outputs inside the container FS + Container.ExecResult execResult = setup.execInContainer( + "bash", + "-lc", + "set -euo pipefail && " + + "mkdir -p /tmp/out && " + + "cd /usr/share/elasticsearch && " + + "bin/elasticsearch-certutil ca --silent --pem --out /tmp/out/ca.zip && " + + "unzip -o /tmp/out/ca.zip -d /tmp/out && " + + "bin/elasticsearch-certutil cert --silent --pem --in /tmp/instances.yml --ca-cert /tmp/out/ca/ca.crt --ca-key /tmp/out/ca/ca.key --out /tmp/out/certs.zip && " + + "unzip -o /tmp/out/certs.zip -d /tmp/out" + ); + Assertions.assertThat(execResult.getExitCode()).isEqualTo(0); + + // copy the certificates and key from the container, so we can use them later + caCrt = setup.copyFileFromContainer("/tmp/out/ca/ca.crt", IOUtils::toByteArray); + nodeCrt = setup.copyFileFromContainer("/tmp/out/es01/es01.crt", IOUtils::toByteArray); + nodeKey = setup.copyFileFromContainer("/tmp/out/es01/es01.key", IOUtils::toByteArray); + } + + try ( + Network network = Network.newNetwork(); + ElasticsearchContainer es = new ElasticsearchContainer( + "docker.elastic.co/elasticsearch/elasticsearch:9.2.4" + ) + .withNetwork(network) + .withNetworkAliases(esHostname) + ) { + applyTls(es, caCrt, nodeCrt, nodeKey); + es.start(); + String kibanaServiceAccountToken = createKibanaServiceAccountToken(es); + + try ( + KibanaContainer kibana = new KibanaContainer("docker.elastic.co/kibana/kibana:9.2.4") + // network is needed only because the ES we try to access via explicit mode is operated by non-public Docker + .withNetwork(network) + .withElasticsearchUrl("https://" + esHostname + ":9200") + .withElasticsearchServiceAccountToken(kibanaServiceAccountToken) + .withElasticsearchCaCertificate(es.caCertAsBytes().get()) + ) { + kibana.start(); + String status = getKibanaStatus(kibana); + Assertions.assertThat(status).isEqualTo("available"); + } + } + } + + private static String setKibanaSystemPassword(ElasticsearchContainer elasticsearch) throws IOException { + String kibanaPassword = "kibana-system-" + System.currentTimeMillis(); + + try (CloseableHttpClient httpClient = createHttpClient(elasticsearch)) { + String url = String.format( + "%s://%s/_security/user/kibana_system/_password", + elasticsearch.getHttpScheme(), + elasticsearch.getHttpHostAddress() + ); + HttpPost request = new HttpPost(url); + request.setHeader("Content-Type", "application/json"); + request.setEntity(new StringEntity("{\"password\":\"" + kibanaPassword + "\"}")); + + HttpResponse response = httpClient.execute(request); + int statusCode = response.getStatusLine().getStatusCode(); + String body = EntityUtils.toString(response.getEntity()); + + if (statusCode != 200) { + throw new IllegalStateException( + "Failed to set kibana_system password. HTTP " + statusCode + ", body=" + body + ); + } + + // ES 9.x returns {} on success; older versions may return {"acknowledged":true} + // Just validate that the body is valid JSON. + try { + OBJECT_MAPPER.readTree(body.isEmpty() ? "{}" : body); + } catch (IOException e) { + throw new IllegalStateException("Non-JSON response body: " + body, e); + } + + return kibanaPassword; + } + } + + private static String createKibanaServiceAccountToken(ElasticsearchContainer elasticsearch) throws IOException { + String tokenName = "kibana-token-" + System.currentTimeMillis(); + + try (CloseableHttpClient httpClient = createHttpClient(elasticsearch)) { + String url = String.format( + "%s://%s/_security/service/elastic/kibana/credential/token/%s", + elasticsearch.getHttpScheme(), + elasticsearch.getHttpHostAddress(), + tokenName + ); + HttpPost request = new HttpPost(url); + request.setHeader("Content-Type", "application/json"); + + HttpResponse response = httpClient.execute(request); + int statusCode = response.getStatusLine().getStatusCode(); + String body = EntityUtils.toString(response.getEntity()); + + if (statusCode != 200) { + throw new IllegalStateException( + "Failed to create Kibana service account token. HTTP " + statusCode + ", body=" + body + ); + } + + // Expected JSON: + // {"created":true,"token":{"name":"...","value":"AAEAA..."}} + try { + JsonNode root = OBJECT_MAPPER.readTree(body); + JsonNode tokenValue = root.path("token").path("value"); + + if (tokenValue.isMissingNode() || tokenValue.isNull()) { + throw new IllegalStateException("Token value not found in response: " + body); + } + + return tokenValue.asText(); + } catch (IOException e) { + throw new IllegalStateException("Failed to parse token response: " + body, e); + } + } + } + + private static CloseableHttpClient createHttpClient(ElasticsearchContainer elasticsearch) { + String elasticPassword = elasticsearch.getEnvMap().get("ELASTIC_PASSWORD"); + HttpClientBuilder clientBuilder = HttpClientBuilder.create(); + + if (StringUtils.isNotBlank(elasticPassword)) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials("elastic", elasticPassword) + ); + clientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + + String scheme = elasticsearch.getHttpScheme(); + if ("https".equals(scheme)) { + SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory( + elasticsearch.createSslContextFromCa() + ); + clientBuilder.setSSLSocketFactory(sslSocketFactory); + } + + return clientBuilder.build(); + } + + private static String getKibanaStatus(KibanaContainer kibana) throws IOException { + try (CloseableHttpClient httpClient = HttpClientBuilder.create().build()) { + String url = "http://" + kibana.getHttpHostAddress() + "/api/status"; + HttpResponse response = httpClient.execute(new org.apache.http.client.methods.HttpGet(url)); + int statusCode = response.getStatusLine().getStatusCode(); + String body = EntityUtils.toString(response.getEntity()); + + if (statusCode != 200) { + throw new IllegalStateException("Failed to get Kibana status. HTTP " + statusCode + ", body=" + body); + } + + JsonNode json = OBJECT_MAPPER.readTree(body); + String status = json.path("status").path("overall").path("level").asText(null); + if (status == null) { + throw new IllegalStateException("Kibana status response missing 'status.overall.level' field: " + body); + } + return status; + } + } + + private static void applyTls(ElasticsearchContainer c, byte[] caCrt, byte[] nodeCrt, byte[] nodeKey) { + final String certDir = "/usr/share/elasticsearch/config/certs"; + + // Copy provided materials + c.withCopyToContainer(Transferable.of(caCrt, 0644), certDir + "/http_ca.crt"); + c.withCopyToContainer(Transferable.of(nodeCrt, 0644), certDir + "/http.crt"); + c.withCopyToContainer(Transferable.of(nodeKey, 0644), certDir + "/http.key"); + + // Disable ES bootstrap TLS autoconfiguration + c.withEnv("xpack.security.autoconfiguration.enabled", "false"); + c.withEnv("xpack.security.enabled", "true"); + + // Configure ONLY HTTP TLS using exactly the provided files + c.withEnv("xpack.security.http.ssl.enabled", "true"); + c.withEnv("xpack.security.http.ssl.certificate_authorities", "certs/http_ca.crt"); + c.withEnv("xpack.security.http.ssl.certificate", "certs/http.crt"); + c.withEnv("xpack.security.http.ssl.key", "certs/http.key"); + } +}