Yoga AUFN test environment

.gitignore

@@ -41,3 +41,11 @@ etc/kayobe/inventory/group_vars/seed/ansible-host
 
 # Ignore kolla configuration.
 etc/kolla
+
+# Ignore ceph generated config in AUFN env
+etc/kayobe/environments/aufn-ceph/kolla/config/glance/ceph.conf
+etc/kayobe/environments/aufn-ceph/kolla/config/glance/ceph.client.glance.keyring
+etc/kayobe/environments/aufn-ceph/kolla/config/cinder/ceph.conf
+etc/kayobe/environments/aufn-ceph/kolla/config/cinder/ceph.client.glance.keyring
+etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.conf
+etc/kayobe/environments/aufn-ceph/kolla/config/nova/ceph.client.glance.keyring

etc/kayobe/environments/aufn-ceph/a-universe-from-nothing.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+
+###########################################
+# STACKHPC-KAYOBE-CONFIG AUFN ENV VERSION #
+###########################################
+
+# Cheat script for a full deployment.
+# This should be used for testing only.
+
+set -eu
+
+BASE_PATH=~
+KAYOBE_BRANCH=stackhpc/yoga
+KAYOBE_CONFIG_BRANCH=stackhpc/yoga
+KAYOBE_ENVIRONMENT=aufn-ceph
+
+PELICAN_HOST="10.0.0.34 pelican pelican.service.compute.sms-lab.cloud"
+PULP_HOST="10.205.3.187 pulp-server pulp-server.internal.sms-cloud"
+
+# FIXME: Work around lack of DNS on SMS lab.
+cat << EOF | sudo tee -a /etc/hosts
+$PELICAN_HOST
+$PULP_HOST
+EOF
+
+# Install git and tmux.
+if $(which dnf 2>/dev/null >/dev/null); then
+    sudo dnf -y install git tmux python3-virtualenv
+else
+    sudo apt update
+    sudo apt -y install git tmux gcc libffi-dev python3-dev python-is-python3 python3-virtualenv
+fi
+
+# Disable the firewall.
+sudo systemctl is-enabled firewalld && sudo systemctl stop firewalld && sudo systemctl disable firewalld
+
+# Disable SELinux both immediately and permanently.
+if $(which setenforce 2>/dev/null >/dev/null); then
+    sudo setenforce 0
+    sudo sed -i 's/^SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
+fi
+
+# Prevent sudo from performing DNS queries.
+echo 'Defaults	!fqdn' | sudo tee /etc/sudoers.d/no-fqdn
+
+# Clone repositories
+cd $BASE_PATH
+mkdir -p src
+pushd src
+[[ -d kayobe ]] || git clone https://github.com/stackhpc/kayobe.git -b $KAYOBE_BRANCH
+[[ -d kayobe-config ]] || git clone https://github.com/stackhpc/stackhpc-kayobe-config kayobe-config -b $KAYOBE_CONFIG_BRANCH
+[[ -d kayobe/tenks ]] || (cd kayobe && git clone https://opendev.org/openstack/tenks.git)
+popd
+
+# Create Kayobe virtualenv
+mkdir -p venvs
+pushd venvs
+if [[ ! -d kayobe ]]; then
+    virtualenv kayobe
+fi
+# NOTE: Virtualenv's activate and deactivate scripts reference an
+# unbound variable. 
+set +u
+source kayobe/bin/activate
+set -u
+pip install -U pip
+pip install ../src/kayobe
+popd
+
+# Activate environment
+pushd $BASE_PATH/src/kayobe-config
+source kayobe-env --environment $KAYOBE_ENVIRONMENT
+
+# Configure host networking (bridge, routes & firewall)
+$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-local-networking.sh
+
+# Bootstrap the Ansible control host.
+kayobe control host bootstrap
+
+# Configure the seed hypervisor host.
+kayobe seed hypervisor host configure
+
+# Provision the seed VM.
+kayobe seed vm provision
+
+# Configure the seed host, and deploy a local registry.
+kayobe seed host configure
+
+# Deploy local pulp server as a container on the seed VM
+kayobe seed service deploy --tags seed-deploy-containers --kolla-tags none
+
+# Deploying the seed restarts networking interface, run configure-local-networking.sh again to re-add routes.
+$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-local-networking.sh
+
+# Add sms lab test pulp to /etc/hosts of seed vm's pulp container
+SEED_IP=192.168.33.5
+REMOTE_COMMAND="docker exec pulp sh -c 'echo $PULP_HOST | tee -a /etc/hosts'"
+ssh stack@$SEED_IP $REMOTE_COMMAND
+
+# Sync package & container repositories.
+kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-sync.yml
+kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-repo-publish.yml
+kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-sync.yml
+kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-container-publish.yml
+
+# Re-run full task to set up bifrost_deploy etc. using newly-populated pulp repo
+kayobe seed service deploy
+
+# NOTE: Make sure to use ./tenks, since just ‘tenks’ will install via PyPI.
+(export TENKS_CONFIG_PATH=$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/tenks.yml && \
+ export KAYOBE_CONFIG_SOURCE_PATH=$BASE_PATH/src/kayobe-config && \
+ export KAYOBE_VENV_PATH=$BASE_PATH/venvs/kayobe && \
+ cd $BASE_PATH/src/kayobe && \
+ ./dev/tenks-deploy-overcloud.sh ./tenks)
+
+# Inspect and provision the overcloud hardware:
+kayobe overcloud inventory discover
+kayobe overcloud hardware inspect
+kayobe overcloud provision
+kayobe overcloud host configure
+kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm.yml
+kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cephadm-gather-keys.yml
+kayobe overcloud container image pull
+kayobe overcloud service deploy
+source $KOLLA_CONFIG_PATH/public-openrc.sh
+kayobe overcloud post configure
+source $KOLLA_CONFIG_PATH/public-openrc.sh
+
+
+# Use openstack-config-multinode here instead of init-runonce.sh script from standard aufn
+
+#Deactivate current kayobe venv
+set +u
+deactivate
+set -u
+$KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/configure-openstack.sh $BASE_PATH
+
+# Create a test vm 
+VENV_DIR=$BASE_PATH/venvs/openstack
+if [[ ! -d $VENV_DIR ]]; then
+    python3 -m venv $VENV_DIR
+fi
+source $VENV_DIR/bin/activate
+pip install -U pip
+pip install python-openstackclient
+source $KOLLA_CONFIG_PATH/public-openrc.sh
+echo "Creating openstack key:"
+openstack keypair create --public-key ~/.ssh/id_rsa.pub mykey
+echo "Creating test vm:"
+openstack server create --key-name mykey --flavor m1.tiny --image cirros --network admin-tenant test-vm-1
+echo "Attaching floating IP:"
+openstack floating ip create external
+openstack server add floating ip test-vm-1 `openstack floating ip list -c ID  -f value`
+echo -e "Done! \nopenstack server list:"
+openstack server list

etc/kayobe/environments/aufn-ceph/cephadm.yml

@@ -0,0 +1,66 @@
+---
+###############################################################################
+# Cephadm deployment configuration.
+
+# Ceph container image.
+cephadm_image: "quay.io/ceph/ceph:v16.2.5"
+
+# List of additional cephadm commands to run before deployment
+# cephadm_commands:
+#   - "config set global osd_pool_default_size {{ [3, groups['osds'] | length] | min }}"
+#   - "config set global osd_pool_default_min_size {{ [3, groups['osds'] | length] | min }}"
+
+# Ceph OSD specification.
+cephadm_osd_spec:
+  service_type: osd
+  service_id: osd_spec_default
+  placement:
+    host_pattern: "*"
+  data_devices:
+    all: true
+
+###############################################################################
+# Ceph post-deployment configuration.
+
+# List of Ceph erasure coding profiles. See stackhpc.cephadm.ec_profiles role
+# for format.
+cephadm_ec_profiles: []
+
+# List of Ceph CRUSH rules. See stackhpc.cephadm.crush_rules role for format.
+cephadm_crush_rules: []
+
+# List of Ceph pools. See stackhpc.cephadm.pools role for format.
+cephadm_pools:
+  - name: backups
+    application: rbd
+    state: present
+  - name: images
+    application: rbd
+    state: present
+  - name: volumes
+    application: rbd
+    state: present
+  - name: vms
+    application: rbd
+    state: present
+
+# List of Cephx keys. See stackhpc.cephadm.keys role for format.
+cephadm_keys:
+  - name: client.cinder
+    caps:
+      mon: "profile rbd"
+      osd: "profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images"
+      mgr: "profile rbd pool=volumes, profile rbd pool=vms"
+    state: present
+  - name: client.cinder-backup
+    caps:
+      mon: "profile rbd"
+      osd: "profile rbd pool=volumes, profile rbd pool=backups"
+      mgr: "profile rbd pool=volumes, profile rbd pool=backups"
+    state: present
+  - name: client.glance
+    caps:
+      mon: "profile rbd"
+      osd: "profile rbd pool=images"
+      mgr: "profile rbd pool=images"
+    state: present

etc/kayobe/environments/aufn-ceph/configure-local-networking.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+# This should be run on the seed hypervisor.
+
+# IP addresses on the all-in-one Kayobe cloud network.
+# These IP addresses map to those statically configured in
+# etc/kayobe/network-allocation.yml and etc/kayobe/networks.yml.
+controller_vip=192.168.39.2
+seed_hv_ip=192.168.33.4
+
+iface=$(ip route | awk '$1 == "default" {print $5; exit}')
+
+# Private IP address by which the seed hypervisor is accessible in the cloud
+# hosting the VM.
+seed_hv_private_ip=$(ip a show dev $iface | awk '$1 == "inet" { gsub(/\/[0-9]*/,"",$2); print $2; exit }')
+
+# Forward the following ports to the controller.
+# 80: Horizon
+# 6080: VNC console
+forwarded_ports="80 6080"
+
+# Install iptables.
+if $(which dnf >/dev/null 2>&1); then
+    sudo dnf -y install iptables
+else
+    sudo apt update
+    sudo apt -y install iptables
+fi
+
+# Configure local networking.
+# Add bridges for the Kayobe networks.
+if ! sudo ip l show brprov >/dev/null 2>&1; then
+    sudo ip l add brprov type bridge
+    sudo ip l set brprov up
+    sudo ip a add $seed_hv_ip/24 dev brprov
+fi
+
+if ! sudo ip l show brcloud >/dev/null 2>&1; then
+    sudo ip l add brcloud type bridge
+    sudo ip l set brcloud up
+fi
+
+# On CentOS 8, bridges without a port are DOWN, which causes network
+# configuration to fail. Add a dummy interface and plug it into the bridge.
+for i in mgmt prov cloud; do
+    if ! sudo ip l show dummy-$i >/dev/null 2>&1; then
+      sudo ip l add dummy-$i type dummy
+    fi
+done
+
+# Configure IP routing and NAT to allow the seed VM and overcloud hosts to
+# route via this route to the outside world.
+sudo iptables -A POSTROUTING -t nat -o $iface -j MASQUERADE
+sudo sysctl -w net.ipv4.conf.all.forwarding=1
+
+# FIXME: IP MASQUERADE from control plane fails without this on Ubuntu.
+if ! $(which dnf >/dev/null 2>&1); then
+    sudo modprobe br_netfilter
+    echo 0 | sudo tee /proc/sys/net/bridge/bridge-nf-call-iptables
+fi
+
+# Configure port forwarding from the hypervisor to the Horizon GUI on the
+# controller.
+sudo iptables -A FORWARD -i $iface -o brprov -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+sudo iptables -A FORWARD -i brprov -o $iface -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+for port in $forwarded_ports; do
+  # Allow new connections.
+  sudo iptables -A FORWARD -i $iface -o brcloud -p tcp --syn --dport $port -m conntrack --ctstate NEW -j ACCEPT
+  # Destination NAT.
+  sudo iptables -t nat -A PREROUTING -i $iface -p tcp --dport $port -j DNAT --to-destination $controller_vip
+  # Source NAT.
+  sudo iptables -t nat -A POSTROUTING -o brcloud -p tcp --dport $port -d $controller_vip -j SNAT --to-source $seed_hv_private_ip
+done
+
+echo
+echo "NOTE: The network configuration applied by this script is not"
+echo "persistent across reboots."
+echo "If you reboot the system, please re-run this script."

etc/kayobe/environments/aufn-ceph/configure-openstack.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+BASE_PATH=$1
+VENV_DIR=$BASE_PATH/venvs/ansible
+cd $BASE_PATH/src/
+[[ -d openstack-config ]] || git clone https://github.com/stackhpc/openstack-config-multinode.git openstack-config
+cd openstack-config
+if [[ ! -d $VENV_DIR ]]; then
+    # Using virtualenv causes a strange bug with python3.6 where 
+    # nested virtual env creation leads to envs without pip...
+    # virtualenv $VENV_DIR
+    python3 -m venv $VENV_DIR
+fi
+
+# NOTE: Virtualenv's activate and deactivate scripts reference an unbound variable. 
+set +u
+source $VENV_DIR/bin/activate
+set -u
+
+pip install -U pip
+pip install -r requirements.txt
+ansible-galaxy role install -p ansible/roles -r requirements.yml
+ansible-galaxy collection install -p ansible/collections -r requirements.yml
+
+source $BASE_PATH/src/kayobe-config/etc/kolla/public-openrc.sh
+
+# Run script to configure openstack cloud
+tools/openstack-config 

etc/kayobe/environments/aufn-ceph/globals.yml

@@ -0,0 +1,13 @@
+---
+# Kayobe global configuration.
+
+###############################################################################
+# OS distribution.
+
+# OS distribution name. Valid options are "centos", "ubuntu". Default is
+# "centos".
+os_distribution: "{{ lookup('pipe', '. /etc/os-release && echo $ID') | trim }}"
+
+# OS release. Valid options are "8-stream" when os_distribution is "centos", or
+# "focal" when os_distribution is "ubuntu".
+#os_release:

etc/kayobe/environments/aufn-ceph/inventory/group_vars/compute/network-interfaces

@@ -0,0 +1,19 @@
+---
+###############################################################################
+# Network interface definitions for the compute group.
+
+provision_oc_interface: "{{ 'ens2' if os_distribution == 'ubuntu' else 'eth0' }}"
+# Route via the seed-hypervisor to the outside world.
+provision_oc_gateway: 192.168.33.4
+
+internal_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ internal_vlan }}"
+
+storage_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ storage_vlan }}"
+
+tunnel_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ tunnel_vlan }}"
+
+external_interface: "{{ 'ens3' if os_distribution == 'ubuntu' else 'eth1' }}.{{ external_vlan }}"
+
+###############################################################################
+# Dummy variable to allow Ansible to accept this file.
+workaround_ansible_issue_8743: yes