diff --git a/cluster.yml b/cluster.yml index 1b93addf4..5d3a5cdb8 100644 --- a/cluster.yml +++ b/cluster.yml @@ -6,15 +6,12 @@ - hosts: k8s-cluster roles: - - { role: etcd, tags: etcd } - { role: docker, tags: docker } + - { role: kubernetes/node, tags: node } + - { role: etcd, tags: etcd } - { role: dnsmasq, tags: dnsmasq } - { role: network_plugin, tags: ['calico', 'flannel', 'network'] } - hosts: kube-master roles: - { role: kubernetes/master, tags: master } - -- hosts: kube-node - roles: - - { role: kubernetes/node, tags: node } diff --git a/roles/apps/k8s-kube-logstash b/roles/apps/k8s-kube-logstash index 340d1a5ec..256fa156e 160000 --- a/roles/apps/k8s-kube-logstash +++ b/roles/apps/k8s-kube-logstash @@ -1 +1 @@ -Subproject commit 340d1a5ec75e7b7c43783dc7a1c02aa7d5991dbe +Subproject commit 256fa156e46d623ab0a7a60efdc7bac535cea8d7 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index eee11e2e9..0a2b3a177 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -7,9 +7,10 @@ kubectl_checksum: "01b9bea18061a27b1cf30e34fd8ab45cfc096c9a9d57d0ed21072abb40dd3 kubelet_checksum: "62191c66f2d670dd52ddf1d88ef81048977abf1ffaa95ee6333299447eb6a482" calico_version: v0.13.0 +calico_plugin_version: v0.7.0 etcd_download_url: "https://github.com/coreos/etcd/releases/download" flannel_download_url: "https://github.com/coreos/flannel/releases/download" kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64" calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download" - +calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes" diff --git a/roles/download/tasks/calico.yml b/roles/download/tasks/calico.yml index b6d506fe2..9fa82cac1 100644 --- a/roles/download/tasks/calico.yml +++ b/roles/download/tasks/calico.yml @@ -19,3 +19,9 @@ when: not c_tar.stat.exists register: dl_calico delegate_to: "{{ groups['kube-master'][0] }}" + + +- name: Download calico-kubernetes-plugin + local_action: get_url + url="{{calico_plugin_download_url}}" + dest="{{ local_release_dir }}/calico/bin/calico" diff --git a/roles/download/tasks/etcd.yml b/roles/download/tasks/etcd.yml deleted file mode 100644 index 0bf3e6c7a..000000000 --- a/roles/download/tasks/etcd.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -- name: Create etcd release directory - local_action: file - path={{ local_release_dir }}/etcd/bin - recurse=yes - state=directory - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: Check if etcd release archive has been downloaded - local_action: stat - path={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz - register: e_tar - delegate_to: "{{ groups['kube-master'][0] }}" - -# issues with get_url module and redirects, to be tested again in the near future -- name: Download etcd - local_action: shell - curl -o {{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz -Ls {{ etcd_download_url }}/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz - when: not e_tar.stat.exists - register: dl_etcd - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: Extract etcd archive - local_action: unarchive - src={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz - dest={{ local_release_dir }}/etcd copy=no - when: dl_etcd|changed - delegate_to: "{{ groups['kube-master'][0] }}" - -- name: Pick up only etcd binaries - local_action: copy - src={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/{{ item }} - dest={{ local_release_dir }}/etcd/bin - with_items: - - etcdctl - - etcd - when: dl_etcd|changed - -- name: Delete unused etcd files - local_action: file - path={{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64 state=absent - when: dl_etcd|changed diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index b4228699b..ded856ba0 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -1,5 +1,4 @@ --- - include: kubernetes.yml -- include: etcd.yml - include: calico.yml - include: flannel.yml diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml deleted file mode 100644 index 67334a353..000000000 --- a/roles/etcd/handlers/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: reload systemd - command: systemctl daemon-reload - -- name: restart reloaded-etcd2 - service: - name: etcd2 - state: restarted - -- name: restart etcd2 - command: /bin/true - notify: - - reload systemd - - restart reloaded-etcd2 diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml deleted file mode 100644 index 5d84e0bfc..000000000 --- a/roles/etcd/tasks/configure.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Copy etcd2.service systemd file - template: - src: systemd-etcd2.service.j2 - dest: /lib/systemd/system/etcd2.service - backup: yes - notify: - - restart etcd2 - -- name: Create etcd2 environment vars file - template: - src: etcd2-environment.j2 - dest: /etc/etcd2-environment - -- name: Ensure etcd2 is running - service: name=etcd2 state=started enabled=yes diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install.yml deleted file mode 100644 index 8d442e6b3..000000000 --- a/roles/etcd/tasks/install.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: Create etcd user - user: name=etcd shell=/bin/nologin home=/var/lib/etcd2 - -- name: Install etcd binaries - copy: - src={{ local_release_dir }}/etcd/bin/{{ item }} - dest={{ bin_dir }} - owner=etcd - mode=0755 - with_items: - - etcdctl - - etcd - notify: restart etcd2 - -- name: Create etcd2 binary symlink - file: src=/usr/local/bin/etcd dest=/usr/local/bin/etcd2 state=link - -- name: install required python module 'httplib2' - apt: - name: "python-httplib2" - state: present - when: inventory_hostname == groups['kube-master'][0] or inventory_hostname == groups['etcd'][0] diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index fdb17cf51..a1c77f49a 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -1,3 +1,13 @@ --- -- include: install.yml -- include: configure.yml +- name: ETCD2 | Stop etcd2 service + service: name=etcd state=stopped + ignore_errors: yes + +- name: ETCD2 | create etcd pod template + template: src=etcd-pod.yml dest=/etc/kubernetes/manifests/etcd-pod.manifest + +- name: ETCD2 | Check for etcd2 port + wait_for: + port: 2379 + delay: 5 + timeout: 30 diff --git a/roles/etcd/templates/etcd-pod.yml b/roles/etcd/templates/etcd-pod.yml new file mode 100644 index 000000000..93971bb6c --- /dev/null +++ b/roles/etcd/templates/etcd-pod.yml @@ -0,0 +1,54 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: etcd + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: etcd + image: quay.io/coreos/etcd:v2.2.2 + resources: + limits: + cpu: 100m + memory: 256M + args: +{% if inventory_hostname in groups['etcd'] %} + - --name + - etcd-{{inventory_hostname}}-master + - --advertise-client-urls + - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379" + - --listen-peer-urls + - http://0.0.0.0:2380 + - --initial-advertise-peer-urls + - http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380 + - --data-dir + - /var/etcd/data + - --initial-cluster-state + - new +{% else %} + - --proxy + - 'on' +{% endif %} + - --listen-client-urls + - "http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379,http://127.0.0.1:2379" + - --initial-cluster + - "{% for host in groups['etcd'] %}etcd-{{host}}-master=http://{{ hostvars[host]['ip'] | default( hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" + - --initial-cluster-token + - etcd-k8s-cluster + ports: + - name: etcd-client + containerPort: 2379 + hostPort: 2379 + - name: etcd-peer + containerPort: 2380 + hostPort: 2380 + volumeMounts: + - name: varetcd + mountPath: /var/etcd + readOnly: false + volumes: + - name: varetcd + hostPath: + path: /containers/pods/etcd-{{inventory_hostname}}/rootfs/var/etcd diff --git a/roles/etcd/templates/etcd2-environment.j2 b/roles/etcd/templates/etcd2-environment.j2 deleted file mode 100644 index 2c0760388..000000000 --- a/roles/etcd/templates/etcd2-environment.j2 +++ /dev/null @@ -1,20 +0,0 @@ -ETCD_DATA_DIR="/var/lib/etcd2" -{% if inventory_hostname in groups['etcd'] %} -{% set etcd = {} %} -{% for host in groups['etcd'] %} -{% if inventory_hostname == host %} -{% set _dummy = etcd.update({'name':"master"+loop.index|string}) %} -{% endif %} -{% endfor %} -ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379" -ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" -ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}master{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" -ETCD_INITIAL_CLUSTER_STATE="new" -ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd" -ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379,http://127.0.0.1:2379" -ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" -ETCD_NAME="{{ etcd.name }}" -{% else %} -ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}master{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" -ETCD_LISTEN_CLIENT_URLS="http://127.0.0.1:23799" -{% endif %} diff --git a/roles/etcd/templates/systemd-etcd2.service.j2 b/roles/etcd/templates/systemd-etcd2.service.j2 deleted file mode 100644 index ca6d37676..000000000 --- a/roles/etcd/templates/systemd-etcd2.service.j2 +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -Description=etcd2 -Conflicts=etcd.service - -[Service] -User=etcd -EnvironmentFile=/etc/etcd2-environment -{% if inventory_hostname in groups['etcd'] %} -ExecStart={{ bin_dir }}/etcd2 -{% else %} -ExecStart={{ bin_dir }}/etcd2 -proxy on -{% endif %} -Restart=always -RestartSec=10s -LimitNOFILE=40000 - -[Install] -WantedBy=multi-user.target diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index ad2739d00..0ba27a33e 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -48,6 +48,7 @@ wait_for: port: "{{kube_apiserver_insecure_port}}" delay: 10 + timeout: 60 - name: Create 'kube-system' namespace uri: diff --git a/roles/kubernetes/node/tasks/gen_tokens.yml b/roles/kubernetes/node/tasks/gen_tokens.yml index 7d1ce0156..4a60ac254 100644 --- a/roles/kubernetes/node/tasks/gen_tokens.yml +++ b/roles/kubernetes/node/tasks/gen_tokens.yml @@ -46,10 +46,3 @@ register: calico_token when: kube_network_plugin == "calico" delegate_to: "{{ groups['kube-master'][0] }}" - -- name: tokens | Add KUBE_AUTH_TOKEN for calico - lineinfile: - regexp: "^KUBE_AUTH_TOKEN=.*$" - line: "KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }}" - dest: "/etc/network-environment" - when: kube_network_plugin == "calico" diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index 6c0a75470..190b2d1c7 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -11,3 +11,16 @@ mode=0755 notify: - restart kubelet + +- name: Calico-plugin | Directory + file: path=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/ state=directory + when: kube_network_plugin == "calico" + +- name: Calico-plugin | Binary + copy: + src={{ local_release_dir }}/calico/bin/calico + dest=/usr/libexec/kubernetes/kubelet-plugins/net/exec/calico/calico + mode=0755 + when: kube_network_plugin == "calico" + notify: + - restart kubelet \ No newline at end of file diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 54217fd22..a01314f0c 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -42,6 +42,9 @@ src: manifests/kube-proxy.manifest.j2 dest: "{{ kube_manifest_dir }}/kube-proxy.manifest" +- name: Write network-environment + template: src=network-environment.j2 dest=/etc/network-environment mode=640 + - name: Enable kubelet service: name: kubelet diff --git a/roles/network_plugin/templates/network-environment.j2 b/roles/kubernetes/node/templates/network-environment.j2 similarity index 84% rename from roles/network_plugin/templates/network-environment.j2 rename to roles/kubernetes/node/templates/network-environment.j2 index 6173a7a97..20bd60311 100755 --- a/roles/network_plugin/templates/network-environment.j2 +++ b/roles/kubernetes/node/templates/network-environment.j2 @@ -13,11 +13,12 @@ KUBERNETES_MASTER={{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars # Location of etcd cluster used by Calico. By default, this uses the etcd # instance running on the Kubernetes Master -{% if inventory_hostname in groups['etcd'] %} ETCD_AUTHORITY="127.0.0.1:2379" -{% else %} -ETCD_AUTHORITY="127.0.0.1:23799" -{% endif %} +#{% if inventory_hostname in groups['etcd'] %} +#ETCD_AUTHORITY="127.0.0.1:2379" +#{% else %} +#ETCD_AUTHORITY="127.0.0.1:23799" +#{% endif %} # The kubernetes-apiserver location - used by the calico plugin {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} @@ -28,3 +29,6 @@ KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(host {% else %} FLANNEL_ETCD_PREFIX="--etcd-prefix=/{{ cluster_name }}/network" {% endif %} +{% if calico_token is defined | default('') %} +KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }} +{% endif %} diff --git a/roles/network_plugin/tasks/main.yml b/roles/network_plugin/tasks/main.yml index 68363b53c..16a80e096 100644 --- a/roles/network_plugin/tasks/main.yml +++ b/roles/network_plugin/tasks/main.yml @@ -2,10 +2,7 @@ - name: "Test if network plugin is defined" fail: msg="ERROR, One network_plugin variable must be defined (Flannel or Calico)" when: ( kube_network_plugin is defined and kube_network_plugin == "calico" and kube_network_plugin == "flannel" ) or - kube_network_plugin is not defined - -- name: Write network-environment - template: src=network-environment.j2 dest=/etc/network-environment mode=640 + kube_network_plugin is not defined - include: flannel.yml when: kube_network_plugin == "flannel"