diff --git a/kustomize/observability/elasticsearch/README.md b/kustomize/observability/elasticsearch/README.md new file mode 100644 index 000000000..9e0df5355 --- /dev/null +++ b/kustomize/observability/elasticsearch/README.md @@ -0,0 +1,9 @@ +# Elasticsearch Configuration Requirements + +For Elasticsearch to work properly on Talos Linux nodes, the following sysctl configuration must be applied in the Talos machine configuration: + +``` +"machine": + "sysctls": + "vm.max_map_count": 262144 +``` \ No newline at end of file diff --git a/kustomize/observability/elasticsearch/certificates.yaml b/kustomize/observability/elasticsearch/certificates.yaml new file mode 100644 index 000000000..ea2b44e4a --- /dev/null +++ b/kustomize/observability/elasticsearch/certificates.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: elasticsearch + namespace: system-observability +spec: + secretName: elasticsearch-master-certs + issuerRef: + name: private + kind: ClusterIssuer + commonName: elasticsearch + dnsNames: + - "elasticsearch-master" + - "elasticsearch-master.system-observability" + - "elasticsearch-master.system-observability.svc" + - "elasticsearch-master.system-observability.svc.${CLUSTER_DOMAIN:-cluster.local}" + - "*.elasticsearch-master.system-observability.svc.${CLUSTER_DOMAIN:-cluster.local}" + - "elasticsearch-master-headless" + - "elasticsearch-master-headless.system-observability" + - "elasticsearch-master-headless.system-observability.svc" + - "elasticsearch-master-headless.system-observability.svc.${CLUSTER_DOMAIN:-cluster.local}" + - "*.elasticsearch-master-headless.system-observability.svc.${CLUSTER_DOMAIN:-cluster.local}" + usages: + - server auth + - client auth diff --git a/kustomize/observability/elasticsearch/helm-release.yaml b/kustomize/observability/elasticsearch/helm-release.yaml new file mode 100644 index 000000000..2fd4df0e9 --- /dev/null +++ b/kustomize/observability/elasticsearch/helm-release.yaml @@ -0,0 +1,62 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: elasticsearch + namespace: system-observability +spec: + interval: 15m + chart: + spec: + chart: elasticsearch + # renovate: datasource=helm depName=elasticsearch package=elasticsearch helmRepo=https://helm.elastic.co + version: "8.5.1" + sourceRef: + kind: HelmRepository + name: elastic-elasticsearch + namespace: system-observability + values: + createCert: false + secretMounts: + - name: elasticsearch-certificates + secretName: elasticsearch-master-certs + path: /usr/share/elasticsearch/config/certificates + defaultMode: 0755 + extraEnvs: + - name: xpack.security.enabled + value: "true" + - name: xpack.security.transport.ssl.enabled + value: "true" + - name: xpack.security.http.ssl.enabled + value: "true" + - name: xpack.security.transport.ssl.verification_mode + value: "certificate" + - name: xpack.security.transport.ssl.key + value: "/usr/share/elasticsearch/config/certificates/tls.key" + - name: xpack.security.transport.ssl.certificate + value: "/usr/share/elasticsearch/config/certificates/tls.crt" + - name: xpack.security.transport.ssl.certificate_authorities + value: "/usr/share/elasticsearch/config/certificates/ca.crt" + - name: xpack.security.http.ssl.key + value: "/usr/share/elasticsearch/config/certificates/tls.key" + - name: xpack.security.http.ssl.certificate + value: "/usr/share/elasticsearch/config/certificates/tls.crt" + - name: xpack.security.http.ssl.certificate_authorities + value: "/usr/share/elasticsearch/config/certificates/ca.crt" + antiAffinity: "soft" + replicas: 1 + sysctlInitContainer: + enabled: false + esJavaOpts: "-Xmx512m -Xms512m" + resources: + requests: + cpu: "100m" + memory: "768M" + limits: + cpu: "1000m" + memory: "1024M" + volumeClaimTemplate: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "single" + resources: + requests: + storage: 500M diff --git a/kustomize/observability/elasticsearch/helm-repository.yaml b/kustomize/observability/elasticsearch/helm-repository.yaml new file mode 100644 index 000000000..4ff074ccc --- /dev/null +++ b/kustomize/observability/elasticsearch/helm-repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: elastic-elasticsearch + namespace: system-observability +spec: + interval: 1h + url: https://helm.elastic.co diff --git a/kustomize/observability/elasticsearch/kustomization.yaml b/kustomize/observability/elasticsearch/kustomization.yaml new file mode 100644 index 000000000..58f470dfe --- /dev/null +++ b/kustomize/observability/elasticsearch/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: +- helm-release.yaml +- helm-repository.yaml +- certificates.yaml diff --git a/kustomize/observability/kibana/helm-release.yaml b/kustomize/observability/kibana/helm-release.yaml new file mode 100644 index 000000000..436783fff --- /dev/null +++ b/kustomize/observability/kibana/helm-release.yaml @@ -0,0 +1,29 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: kibana + namespace: system-observability +spec: + interval: 5m + chart: + spec: + chart: kibana + # renovate: datasource=helm depName=kibana package=kibana helmRepo=https://helm.elastic.co + version: "8.5.1" + sourceRef: + kind: HelmRepository + name: elastic-kibana + namespace: system-observability + values: + kibanaConfig: + kibana.yml: | + elasticsearch.ssl.certificateAuthorities: ["/usr/share/kibana/config/certs/ca.crt"] + elasticsearch.ssl.certificate: "/usr/share/kibana/config/certs/tls.crt" + elasticsearch.ssl.key: "/usr/share/kibana/config/certs/tls.key" + resources: + requests: + cpu: "1000m" + memory: "1Gi" + limits: + cpu: "1000m" + memory: "1Gi" diff --git a/kustomize/observability/kibana/helm-repository.yaml b/kustomize/observability/kibana/helm-repository.yaml new file mode 100644 index 000000000..c1ecd24e0 --- /dev/null +++ b/kustomize/observability/kibana/helm-repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: elastic-kibana + namespace: system-observability +spec: + interval: 1h + url: https://helm.elastic.co diff --git a/kustomize/observability/kibana/ingress/ingress.yaml b/kustomize/observability/kibana/ingress/ingress.yaml new file mode 100644 index 000000000..b9f5065c7 --- /dev/null +++ b/kustomize/observability/kibana/ingress/ingress.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: kibana + namespace: system-observability + annotations: + cert-manager.io/cluster-issuer: private +spec: + ingressClassName: nginx + rules: + - host: kibana.${DOMAIN} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: kibana-kibana + port: + number: 5601 diff --git a/kustomize/observability/kibana/ingress/kustomization.yaml b/kustomize/observability/kibana/ingress/kustomization.yaml new file mode 100644 index 000000000..fb7103238 --- /dev/null +++ b/kustomize/observability/kibana/ingress/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - ingress.yaml diff --git a/kustomize/observability/kibana/kustomization.yaml b/kustomize/observability/kibana/kustomization.yaml new file mode 100644 index 000000000..84f402f63 --- /dev/null +++ b/kustomize/observability/kibana/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: +- helm-release.yaml +- helm-repository.yaml diff --git a/kustomize/telemetry/base/filebeat/certificates.yaml b/kustomize/telemetry/base/filebeat/certificates.yaml new file mode 100644 index 000000000..1491af822 --- /dev/null +++ b/kustomize/telemetry/base/filebeat/certificates.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: filebeat + namespace: system-telemetry +spec: + secretName: filebeat-tls + issuerRef: + name: private + kind: ClusterIssuer + commonName: filebeat + usages: + - client auth diff --git a/kustomize/telemetry/base/filebeat/helm-release.yaml b/kustomize/telemetry/base/filebeat/helm-release.yaml new file mode 100644 index 000000000..f059d4841 --- /dev/null +++ b/kustomize/telemetry/base/filebeat/helm-release.yaml @@ -0,0 +1,44 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: filebeat + namespace: system-telemetry +spec: + interval: 5m + chart: + spec: + chart: filebeat + # renovate: datasource=helm depName=filebeat package=filebeat helmRepo=https://helm.elastic.co + version: "8.5.1" + sourceRef: + kind: HelmRepository + name: elastic-filebeat + namespace: system-telemetry + values: + daemonset: + secretMounts: + - name: filebeat-tls + secretName: filebeat-tls + path: /usr/share/filebeat/certs/ + extraEnvs: + - name: ELASTICSEARCH_HOSTS + value: "elasticsearch-master.system-observability.svc.cluster.local" + - name: "ELASTICSEARCH_USERNAME" + valueFrom: + secretKeyRef: + name: elasticsearch-master-credentials + key: username + - name: "ELASTICSEARCH_PASSWORD" + valueFrom: + secretKeyRef: + name: elasticsearch-master-credentials + key: password + - name: "ssl.certificate_authorities" + value: "/usr/share/filebeat/certs/ca.crt" + resources: + requests: + cpu: "100m" + memory: "100M" + limits: + cpu: "500m" + memory: "300M" diff --git a/kustomize/telemetry/base/filebeat/helm-repository.yaml b/kustomize/telemetry/base/filebeat/helm-repository.yaml new file mode 100644 index 000000000..5a34c9df7 --- /dev/null +++ b/kustomize/telemetry/base/filebeat/helm-repository.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: elastic-filebeat + namespace: system-telemetry +spec: + interval: 1h + url: https://helm.elastic.co diff --git a/kustomize/telemetry/base/filebeat/kustomization.yaml b/kustomize/telemetry/base/filebeat/kustomization.yaml new file mode 100644 index 000000000..58200249a --- /dev/null +++ b/kustomize/telemetry/base/filebeat/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: +- secret-mgr.yaml +- helm-release.yaml +- helm-repository.yaml +- certificates.yaml diff --git a/kustomize/telemetry/base/filebeat/secret-mgr.yaml b/kustomize/telemetry/base/filebeat/secret-mgr.yaml new file mode 100644 index 000000000..f19f6876b --- /dev/null +++ b/kustomize/telemetry/base/filebeat/secret-mgr.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: secret-manager + namespace: system-telemetry +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: secret-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: secret-manager + namespace: system-telemetry +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: secret-manager-init-job + namespace: system-telemetry +spec: + backoffLimit: 10 + template: + spec: + containers: + - name: copy-k8s-secret + image: bitnami/kubectl:1.31.2 + command: ["/bin/sh", "-c", "--"] + args: + - | + kubectl get secret elasticsearch-master-credentials -n system-observability -o yaml \ + | yq eval 'del(.metadata.resourceVersion, .metadata.uid, .metadata.creationTimestamp, .metadata.managedFields) | .metadata.namespace = "system-telemetry"' - \ + | kubectl apply -n system-telemetry -f - + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + restartPolicy: Never + volumes: + - name: temp-volume + emptyDir: {} + serviceAccountName: secret-manager + diff --git a/terraform/cluster/talos/.terraform.lock.hcl b/terraform/cluster/talos/.terraform.lock.hcl index 11bd751e9..796e04a99 100644 --- a/terraform/cluster/talos/.terraform.lock.hcl +++ b/terraform/cluster/talos/.terraform.lock.hcl @@ -1,28 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/aws" { - version = "5.93.0" - hashes = [ - "h1:SbzGotY1leY5nnLo/PJOcwIlNTHdZpAErxJSrfr2tTg=", - "zh:00e1b15e6f02cdc788fe855232b63ccce6652930080eac3ba4b8a2e35db02b23", - "zh:3a77ee12e4f5ab2e7b320a0f507389c9171ab82c50d39ae7caa5a1fb2bd95cb3", - "zh:3e32d58e139d098d867eef37914fef01fffb08504d828e0f384c2ffc18d71f80", - "zh:41cf69a525f0fbe0fdb71d26be7ff5e20bb90ccdf5af32c83ed53f0ca2f071b5", - "zh:43055bdd0786855cf7242638a74b579f74f4f1a8e7c7e5e0e50230c8f6b908cb", - "zh:4ac4c29aa0de842ad91145c5a5fba21338531ffca13a510927d445e007a24938", - "zh:57e510498b3aeb6d6155c10fa195e1d5502e763899251057e59e73f653d1e262", - "zh:8f749645b27dba1a07d06aaf9d5596fc4213123f12f3808d68539e78ab16996e", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:aaca5934ac6273d48922ad7685c5fc2aa7ef5275346a9e70366b7a180a788d41", - "zh:b7585b720a97467302f2e29f0688a5a746778f7b73c30eb085c25831decba1e1", - "zh:c16ae0a46d796858c49a89dd90e5ca92f793e646474fadeafaf701def4a4aa83", - "zh:d66bdc9cd5108452d9dba44082e504ff5e3a3001c8f853bbcaff850cb2127a21", - "zh:ee1aec6c44b117a6c8b7159ee7dc82f1ddac6ba434b4e6c493717738326f0a99", - "zh:f0da48692e00ecacea72d7104714d9721f6be40ba094490c442bb3e68d2e2604", - ] -} - provider "registry.terraform.io/hashicorp/local" { version = "2.5.2" hashes = [ @@ -44,22 +22,21 @@ provider "registry.terraform.io/hashicorp/local" { } provider "registry.terraform.io/hashicorp/null" { - version = "3.2.3" + version = "3.2.4" hashes = [ - "h1:I0Um8UkrMUb81Fxq/dxbr3HLP2cecTH2WMJiwKSrwQY=", - "h1:obXguGZUWtNAO09f1f9Cb7hsPCOGXuGdN8bn/ohKRBQ=", - "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", - "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", - "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", - "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", - "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "h1:L5V05xwp/Gto1leRryuesxjMfgZwjb7oool4WS1UEFQ=", + "zh:59f6b52ab4ff35739647f9509ee6d93d7c032985d9f8c6237d1f8a59471bbbe2", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", - "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", - "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", - "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", - "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", - "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + "zh:795c897119ff082133150121d39ff26cb5f89a730a2c8c26f3a9c1abf81a9c43", + "zh:7b9c7b16f118fbc2b05a983817b8ce2f86df125857966ad356353baf4bff5c0a", + "zh:85e33ab43e0e1726e5f97a874b8e24820b6565ff8076523cc2922ba671492991", + "zh:9d32ac3619cfc93eb3c4f423492a8e0f79db05fec58e449dee9b2d5873d5f69f", + "zh:9e15c3c9dd8e0d1e3731841d44c34571b6c97f5b95e8296a45318b94e5287a6e", + "zh:b4c2ab35d1b7696c30b64bf2c0f3a62329107bd1a9121ce70683dec58af19615", + "zh:c43723e8cc65bcdf5e0c92581dcbbdcbdcf18b8d2037406a5f2033b1e22de442", + "zh:ceb5495d9c31bfb299d246ab333f08c7fb0d67a4f82681fbf47f2a21c3e11ab5", + "zh:e171026b3659305c558d9804062762d168f50ba02b88b231d20ec99578a6233f", + "zh:ed0fe2acdb61330b01841fa790be00ec6beaac91d41f311fb8254f74eb6a711f", ] } diff --git a/terraform/gitops/flux/.terraform.lock.hcl b/terraform/gitops/flux/.terraform.lock.hcl index d9cd36325..acc3198db 100644 --- a/terraform/gitops/flux/.terraform.lock.hcl +++ b/terraform/gitops/flux/.terraform.lock.hcl @@ -1,28 +1,6 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/hashicorp/aws" { - version = "5.93.0" - hashes = [ - "h1:SbzGotY1leY5nnLo/PJOcwIlNTHdZpAErxJSrfr2tTg=", - "zh:00e1b15e6f02cdc788fe855232b63ccce6652930080eac3ba4b8a2e35db02b23", - "zh:3a77ee12e4f5ab2e7b320a0f507389c9171ab82c50d39ae7caa5a1fb2bd95cb3", - "zh:3e32d58e139d098d867eef37914fef01fffb08504d828e0f384c2ffc18d71f80", - "zh:41cf69a525f0fbe0fdb71d26be7ff5e20bb90ccdf5af32c83ed53f0ca2f071b5", - "zh:43055bdd0786855cf7242638a74b579f74f4f1a8e7c7e5e0e50230c8f6b908cb", - "zh:4ac4c29aa0de842ad91145c5a5fba21338531ffca13a510927d445e007a24938", - "zh:57e510498b3aeb6d6155c10fa195e1d5502e763899251057e59e73f653d1e262", - "zh:8f749645b27dba1a07d06aaf9d5596fc4213123f12f3808d68539e78ab16996e", - "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:aaca5934ac6273d48922ad7685c5fc2aa7ef5275346a9e70366b7a180a788d41", - "zh:b7585b720a97467302f2e29f0688a5a746778f7b73c30eb085c25831decba1e1", - "zh:c16ae0a46d796858c49a89dd90e5ca92f793e646474fadeafaf701def4a4aa83", - "zh:d66bdc9cd5108452d9dba44082e504ff5e3a3001c8f853bbcaff850cb2127a21", - "zh:ee1aec6c44b117a6c8b7159ee7dc82f1ddac6ba434b4e6c493717738326f0a99", - "zh:f0da48692e00ecacea72d7104714d9721f6be40ba094490c442bb3e68d2e2604", - ] -} - provider "registry.terraform.io/hashicorp/helm" { version = "2.17.0" hashes = [