Huge refactoring

Split scripts and instructuins into two parts: lab preparation and
deployment.

README.md

@@ -14,13 +14,17 @@ Requirements
 How-to
 ======
 
+Vargant lab preparation
+-----------------------
+
 * Change default IP pool for vagrant networks if you want:
 
 ```bash
 export VAGRANT_POOL="10.100.0.0/16"
 ```
 
-* If you want to run OpenStack CCP (Containerised Control Plane) then you need to pull CCP repos and patches:
+* If you want to run OpenStack CCP (Containerised Control Plane) then you need
+to pull CCP repos and patches:
 
 ```bash
 pushd ccp
@@ -36,21 +40,43 @@ cd vagrant-k8s
 vagrant up
 ```
 
-* Login to master node and deploy k8s with kargo:
+Deployment on a lab
+-------------------
+
+* Login to master node and sudo to root:
 
 ```bash
 vagrant ssh $USER-k8s-01
-# Inside your master VM run this:
 sudo su -
+```
+
+* Clone this repo
+
+```bash
+git clone https://github.com/adidenko/vagrant-k8s ~/mcp
+```
+
+* Install required software and pull needed repos (modify script if you're not
+running it on Vagrant lab, you'll need to create `nodes` list manually and
+clone `microservices` and `microservices-repos` repositories, see ccp-pull.sh
+for details)
+
+```bash
+cd ~/mcp
+./bootstrap-master.sh
+```
+
+* Deploy k8s using kargo playbooks
+
+```bash
+cd ~/mcp
 ./deploy-k8s.kargo.sh
 ```
 
-* In order to deploy OpenStack CCP login to your master node and run this:
+* Deploy OpenStack CCP:
 
 ```bash
-vagrant ssh $USER-k8s-01
+cd ~/mcp
-# Inside your master VM run this:
-sudo su -
 ./deploy-ccp.sh
 ```
 

Vagrantfile

@@ -93,15 +93,16 @@ Vagrant.configure("2") do |config|
       # Provisioning
       config.vm.provision "file", source: "ssh", destination: "~/ssh"
       if master
-        config.vm.provision "deploy-k8s", type: "file", source: "deploy-k8s.kargo.sh", destination: "~/deploy-k8s.kargo.sh"
+        config.vm.provision "nodes", type: "file", source: "nodes", destination: "/var/tmp/nodes"
-        config.vm.provision "deploy-ccp", type: "file", source: "deploy-ccp.sh", destination: "~/deploy-ccp.sh"
+        config.vm.provision "microservices", type: "file",
-        config.vm.provision "custom.yaml", type: "file", source: "custom.yaml", destination: "~/custom.yaml"
+          source: "ccp/microservices",
-        config.vm.provision "playbooks", type: "file", source: "playbooks", destination: "~/playbooks"
+          destination: "/var/tmp/microservices"
-        config.vm.provision "nodes", type: "file", source: "nodes", destination: "~/nodes"
+        config.vm.provision "microservices-repos", type: "file",
-        config.vm.provision "ccp", type: "file", source: "ccp", destination: "~/ccp"
+          source: "ccp/microservices-repos",
-        config.vm.provision "bootstrap", type: "shell", path: "vagrant-scripts/bootstrap-master.sh"
+          destination: "/var/tmp/microservices-repos"
+        config.vm.provision "bootstrap", type: "shell", path: "vagrant-scripts/provision-master.sh"
       else
-        config.vm.provision "bootstrap", type: "shell", path: "vagrant-scripts/bootstrap-node.sh"
+        config.vm.provision "bootstrap", type: "shell", path: "vagrant-scripts/provision-node.sh"
       end
 
     end

bootstrap-master.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Packages
+apt-get --yes update
+apt-get --yes upgrade
+apt-get --yes install git screen vim telnet tcpdump python-setuptools gcc python-dev python-pip libssl-dev libffi-dev software-properties-common curl
+
+# Get ansible-2.1+, vanilla ubuntu-16.04 ansible (2.0.0.2) is broken due to https://github.com/ansible/ansible/issues/13876
+apt-add-repository -y ppa:ansible/ansible;apt-get update;apt-get install -y ansible
+
+# Kargo-cli
+git clone https://github.com/kubespray/kargo-cli.git /root/kargo-cli
+cd /root/kargo-cli && python setup.py install
+
+# Copy/create nodes list
+cp /var/tmp/nodes mcp/nodes
+
+# Either pull or copy microservices repos
+cp -a /vat/tmp/microservices* mcp/ccp/
+

ccp/ccp-build.sh

@@ -17,6 +17,16 @@ skip_empty = True
 EOF
 }
 
+create_resolvconf() {
+  DNS_IP=`kubectl get service/kubedns --namespace=kube-system --template={{.spec.clusterIP}}`
+  cat > /root/resolv.conf << EOF
+search openstack.svc.cluster.local svc.cluster.local cluster.local default.svc.cluster.local svc.cluster.local cluster.local
+nameserver $DNS_IP
+options attempts:2
+options ndots:5
+EOF
+}
+
 create_registry() {
   if kubectl get pods | grep registry ; then
     echo "Registry is already running"
@@ -41,6 +51,13 @@ build_images() {
   mcp-microservices --config-file /root/mcp.conf build &> /var/log/mcp-build.log
 }
 
+hack_base_image() {
+  cp /root/resolv.conf ccp/microservices-repos/ms-debian-base/docker/base/
+  sed '/COPY requirements.txt/a COPY resolv.conf /etc/resolv.conf' -i ccp/microservices-repos/ms-debian-base/docker/base/Dockerfile.j2
+}
+
 create_mcp_conf
 create_registry
+create_resolvconf
+hack_base_image
 build_images

ccp/ccp-deploy.sh

@@ -2,6 +2,17 @@
 
 set -e
 
+# FIXME: hardcoded roles
+declare -A nodes
+nodes=( \
+["node2"]="openstack-controller=true"
+["node3"]="openstack-controller=true"
+["node4"]="openstack-controller=true"
+["node5"]="openstack-compute=true"
+["node6"]="openstack-compute=true"
+["node7"]="openstack-compute=true"
+)
+
 create_network_conf() {
   kubectl get nodes -o go-template='{{range .items}}{{range .status.addresses}}{{if or (eq .type "ExternalIP") (eq .type "LegacyHostIP")}}{{.address}}{{print "\n"}}{{end}}{{end}}{{end}}'> /tmp/nodes
 #  ( echo "network:"; i=2; for ip in `cat /tmp/nodes `; do echo -e "  node$i:\n    private:\n      iface: eth2\n      address: $ip"; pip=`echo $ip | perl -pe 's/(\d+).(\d+).1/\${1}.\${2}.0/g'`; echo -e "    public:\n      iface: eth1\n      address: $pip" ; i=$(( i+=1 )) ;done ) > /root/cluster-topology.yaml
@@ -9,13 +20,12 @@ create_network_conf() {
 }
 
 assign_node_roles() {
-  # FIXME: hardcoded roles
+  for i in "${!nodes[@]}"
-  kubectl label nodes node2 openstack-controller=true
+  do
-  kubectl label nodes node3 openstack-controller=true
+    node=$i
-  kubectl label nodes node4 openstack-controller=true
+    label=${nodes[$i]}
-  kubectl label nodes node5 openstack-compute=true
+    kubectl get nodes $node --show-labels | grep -q "$label" || kubectl label nodes $node $label
-  kubectl label nodes node6 openstack-compute=true
+  done
-  kubectl label nodes node7 openstack-compute=true
 }
 
 create_network_conf

vagrant-scripts/bootstrap-master.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-echo master > /var/tmp/role
-
-# Packages
-sudo apt-get --yes update
-sudo apt-get --yes upgrade
-sudo apt-get --yes install git screen vim telnet tcpdump python-setuptools gcc python-dev python-pip libssl-dev libffi-dev software-properties-common curl
-
-# Get ansible-2.1+, vanilla ubuntu-16.04 ansible (2.0.0.2) is broken due to https://github.com/ansible/ansible/issues/13876
-sudo sh -c 'apt-add-repository -y ppa:ansible/ansible;apt-get update;apt-get install -y ansible'
-
-# Kargo-cli
-sudo git clone https://github.com/kubespray/kargo-cli.git /root/kargo-cli
-sudo sh -c 'cd /root/kargo-cli && python setup.py install'
-
-# k8s deploy script and configs
-sudo sh -c 'cp -a ~vagrant/deploy-k8s.kargo.sh /root/ && chmod 755 /root/deploy-k8s.kargo.sh'
-sudo sh -c 'cp -a ~vagrant/deploy-ccp.sh /root/ && chmod 755 /root/deploy-ccp.sh'
-sudo cp -a ~vagrant/custom.yaml /root/custom.yaml
-sudo cp -a ~vagrant/playbooks /root/playbooks
-sudo cp -a ~vagrant/ccp /root/ccp
-
-# SSH keys and config
-sudo rm -rf /root/.ssh
-sudo mv ~vagrant/ssh /root/.ssh
-sudo echo -e 'Host 10.*\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile=/dev/null' >> /root/.ssh/config
-sudo chown -R root: /root/.ssh
-
-# Copy nodes list
-sudo cp ~vagrant/nodes /root/nodes
-
-# README
-sudo echo 'cd /root/kargo ; ansible-playbook -vvv -i inv/inventory.cfg cluster.yml -u root -f 7' > /root/README

vagrant-scripts/provision-master.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+echo master > /var/tmp/role
+
+# SSH keys and config
+sudo rm -rf /root/.ssh
+sudo mv ~vagrant/ssh /root/.ssh
+sudo echo -e 'Host 10.*\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile=/dev/null' >> /root/.ssh/config
+sudo chown -R root: /root/.ssh
+
+# README
+sudo echo 'cd /root/kargo ; ansible-playbook -vvv -i inv/inventory.cfg cluster.yml -u root -f 7' > /root/README

vagrant-scripts/provision-node.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
 echo node > /var/tmp/role
 
-# Packages
+# Some debug tools
 sudo apt-get --yes update
 sudo apt-get --yes upgrade
 sudo apt-get --yes install screen vim telnet tcpdump python-pip traceroute iperf3 nmap ethtool curl
 
-# Pip
+# Pip kpm
 sudo pip install kpm
 
 # SSH