Merge pull request kubernetes#10544 from eparis/total-ansible

Ansible improvements to support more addons and better cert/token handling

Author: zmerlynn

contrib/ansible/group_vars/all.yml

@@ -1,6 +1,7 @@
-# Only used for the location to store flannel info in etcd, but may be used
-# for dns purposes and cluster id purposes in the future.
-cluster_name: kube.local
+# will be used as the Internal dns domain name if DNS is enabled. Services
+# will be discoverable under <service-name>.<namespace>.svc.<domainname>, e.g.
+# myservice.default.svc.cluster.local
+cluster_name: cluster.local
 
 # Account name of remote user. Ansible will use this user account to ssh into
 # the managed machines. The user must be able to use sudo without asking
@@ -43,21 +44,17 @@ flannel_prefix: 12
 # room for 4096 nodes with 254 pods per node.
 flannel_host_prefix: 24
 
+# Set to false to disable logging with elasticsearch
+cluster_logging: true
+
+# Turn to false to disable cluster monitoring with heapster and influxdb
+cluster_monitoring: true
+
 # Turn this varable to 'false' to disable whole DNS configuration.
 dns_setup: true
 # How many replicas in the Replication Controller
 dns_replicas: 1
 
-# Internal DNS domain name.
-# This domain must not be used in your network. Services will be discoverable
-# under <service-name>.<namespace>.<domainname>, e.g.
-# myservice.default.kube.local
-dns_domain: kube.local
-
-# IP address of the DNS server.
-# Kubernetes will create a pod with several containers, serving as the DNS
-# server and expose it under this IP address. The IP address must be from
-# the range specified as kube_service_addresses above.
-# And this is the IP address you should use as address of the DNS server
-# in your containers.
-dns_server: 10.254.0.10
+# There are other variable in roles/kubernetes/defaults/main.yml but changing
+# them comes with a much higher risk to your cluster. So proceed over there
+# with caution.

contrib/ansible/roles/kubernetes-addons/tasks/cluster-logging.yml

@@ -0,0 +1,14 @@
+---
+- name: LOGGING | Assures {{ kube_config_dir }}/addons/cluster-logging dir exists
+  file: path={{ kube_config_dir }}/addons/cluster-logging state=directory
+
+- name: LOGGING | Download logging files from Kubernetes repo
+  get_url:
+    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/addons/fluentd-elasticsearch/{{ item }}
+    dest="{{ kube_config_dir }}/addons/cluster-logging/"
+    force=yes
+  with_items:
+    - es-controller.yaml
+    - es-service.yaml
+    - kibana-controller.yaml
+    - kibana-service.yaml

contrib/ansible/roles/kubernetes-addons/tasks/cluster-monitoring.yml

@@ -0,0 +1,15 @@
+---
+- name: MONITORING | Assures {{ kube_config_dir }}/addons/cluster-monitoring dir exists
+  file: path={{ kube_config_dir }}/addons/cluster-monitoring state=directory
+
+- name: MONITORING | Download monitoring files from Kubernetes repo
+  get_url:
+    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/addons/cluster-monitoring/influxdb/{{ item }}
+    dest="{{ kube_config_dir }}/addons/cluster-monitoring/"
+    force=yes
+  with_items:
+    - grafana-service.yaml
+    - heapster-controller.yaml
+    - heapster-service.yaml
+    - influxdb-grafana-controller.yaml
+    - influxdb-service.yaml

contrib/ansible/roles/kubernetes-addons/tasks/main.yml

@@ -13,7 +13,12 @@
 
 - include: dns.yml
   when: dns_setup
-  tags: dns
+
+- include: cluster-monitoring.yml
+  when: cluster_monitoring
+
+- include: cluster-logging.yml
+  when: cluster_logging
 
 #- name: Get kube-addons script from Kubernetes
 #  get_url:
@@ -33,15 +38,14 @@
 - name: HACK | copy local kube-addon-update.sh
   copy: src=kube-addon-update.sh dest={{ kube_script_dir }}/kube-addon-update.sh mode=0755
 
-- name: Copy script to create known_tokens.csv
-  copy: src=kube-gen-token.sh dest={{ kube_script_dir }}/kube-gen-token.sh mode=0755
-
-- name: Run kube-gen-token script to create {{ kube_config_dir }}/known_tokens.csv
+- name: Run kube-gen-token script to create {{ kube_token_dir }}/known_tokens.csv
   command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item }}"
   environment:
-    TOKEN_DIR: "{{ kube_config_dir }}"
+    TOKEN_DIR: "{{ kube_token_dir }}"
   with_items:
     - "system:dns"
+    - "system:monitoring"
+    - "system:logging"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"
   notify:

contrib/ansible/roles/kubernetes-addons/templates/kube-addons.service.j2

@@ -3,7 +3,7 @@ Description=Kubernetes Addon Object Manager
 Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 
 [Service]
-Environment="TOKEN_DIR={{ kube_config_dir }}"
+Environment="TOKEN_DIR={{ kube_token_dir }}"
 Environment="KUBECTL_BIN=/usr/bin/kubectl"
 Environment="KUBERNETES_MASTER_NAME={{ groups['masters'][0] }}"
 ExecStart={{ kube_script_dir }}/kube-addons.sh

contrib/ansible/roles/kubernetes/defaults/main.yml

@@ -14,7 +14,26 @@ kube_config_dir: /etc/kubernetes
 # This is where all the cert scripts and certs will be located
 kube_cert_dir: "{{ kube_config_dir }}/certs"
 
+# This is where all of the bearer tokens will be stored
+kube_token_dir: "{{ kube_config_dir }}/tokens"
+
+# This is where you can drop yaml/json files and the kubelet will run those
+# pods on startup
+kube_manifest_dir: "{{ kube_config_dir }}/manifests"
 
 # This is the group that the cert creation scripts chgrp the
 # cert files to. Not really changable...
 kube_cert_group: kube-cert
+
+# Internal DNS domain name.
+# This domain must not be used in your network. Services will be discoverable
+# under <service-name>.<namespace>.<domainname>, e.g.
+# myservice.default.cluster.local
+dns_domain: "{{ cluster_name }}"
+
+# IP address of the DNS server.
+# Kubernetes will create a pod with several containers, serving as the DNS
+# server and expose it under this IP address. The IP address must be from
+# the range specified as kube_service_addresses. This magic will actually
+# pick the 10th ip address in the kube_service_addresses range and use that.
+dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(10)|ipaddr('address') }}"

…ubernetes-addons/files/kube-gen-token.sh …roles/kubernetes/files/kube-gen-token.shcontrib/ansible/roles/kubernetes-addons/files/kube-gen-token.sh renamed to contrib/ansible/roles/kubernetes/files/kube-gen-token.sh contrib/ansible/roles/kubernetes-addons/files/kube-gen-token.sh renamed to contrib/ansible/roles/kubernetes/files/kube-gen-token.sh

@@ -21,10 +21,11 @@ create_accounts=($@)
 
 touch "${token_file}"
 for account in "${create_accounts[@]}"; do
-  if grep "${account}" "${token_file}" ; then
+  if grep ",${account}," "${token_file}" ; then
     continue
   fi
   token=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
   echo "${token},${account},${account}" >> "${token_file}"
+  echo "${token}" > "${token_dir}/${account}.token"
   echo "Added ${account}"
 done

contrib/ansible/roles/kubernetes/tasks/certs.yml

@@ -0,0 +1,30 @@
+---
+- name: Copy the token gen script
+  copy:
+    src=kube-gen-token.sh
+    dest={{ kube_script_dir }}
+    mode=u+x
+
+- name: Generate tokens for master components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ "system:controller_manager", "system:scheduler" ]
+    - "{{ groups['masters'] }}"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  notify:
+    - restart daemons
+
+- name: Generate tokens for node components
+  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
+  environment:
+    TOKEN_DIR: "{{ kube_token_dir }}"
+  with_nested:
+    - [ 'system:kubelet', 'system:proxy' ]
+    - "{{ groups['nodes'] }}"
+  register: gentoken
+  changed_when: "'Added' in gentoken.stdout"
+  notify:
+    - restart daemons

contrib/ansible/roles/kubernetes/tasks/gen_tokens.yml

@@ -18,6 +18,6 @@
   notify:
     - restart daemons
 
-- include: certs.yml
+- include: secrets.yml
   tags:
-    certs
+    secrets