diff --git a/.gitignore b/.gitignore index 1f9359992..86dec6fb4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.retry inventory/vagrant_ansible_inventory temp +.idea diff --git a/.travis.yml b/.travis.yml index 6c26cd110..13604765e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -109,6 +109,8 @@ before_script: - $HOME/.local/bin/ansible-playbook --version - cp tests/ansible.cfg . # - "echo $HOME/.local/bin/ansible-playbook -i inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root -e '{\"cloud_provider\": true}' $LOG_LEVEL -e kube_network_plugin=${KUBE_NETWORK_PLUGIN} setup-kubernetes/cluster.yml" + ## Configure ansible deployment logs to be collected as an artifact. Enable when GCS configured, see https://docs.travis-ci.com/user/deployment/gcs +# - $HOME/.local/bin/ansible-playbook -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/configure-logs.yaml script: - > @@ -131,6 +133,8 @@ script: - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL ## Ping the between 2 pod - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL + ## Collect env info, enable it once GCS configured, see https://docs.travis-ci.com/user/deployment/gcs +# - $HOME/.local/bin/ansible-playbook -i inventory/inventory.ini -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root scripts/collect-info.yaml after_script: - > diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..1d3b9340d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,10 @@ +# Contributing guidelines + +## How to become a contributor and submit your own code + +### Contributing A Patch + +1. Submit an issue describing your proposed change to the repo in question. +2. The [repo owners](OWNERS) will respond to your issue promptly. +3. Fork the desired repo, develop and test your code changes. +4. Submit a pull request. diff --git a/OWNERS b/OWNERS new file mode 100644 index 000000000..583a0314b --- /dev/null +++ b/OWNERS @@ -0,0 +1,6 @@ +# See the OWNERS file documentation: +# https://github.com/kubernetes/kubernetes/blob/master/docs/devel/owners.md + +owners: + - Smana + - ant31 diff --git a/README.md b/README.md index a1c9f0c1a..f050bb462 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ If you have questions, you can [invite yourself](https://slack.kubespray.io/) to To deploy the cluster you can use : -[**kargo-cli**](https://github.com/kubespray/kargo-cli)
+[**kargo-cli**](https://github.com/kubespray/kargo-cli) (deprecated, a newer [go](https://github.com/Smana/kargo-cli/tree/kargogo) version soon)
**Ansible** usual commands
**vagrant** by simply running `vagrant up` (for tests purposes)
@@ -24,7 +24,8 @@ To deploy the cluster you can use : * [CoreOS bootstrap](docs/coreos.md) * [Ansible variables](docs/ansible.md) * [Cloud providers](docs/cloud.md) -* [Openstack](docs/openstack.md) +* [OpenStack](docs/openstack.md) +* [AWS](docs/aws.md) * [Network plugins](#network-plugins) * [Roadmap](docs/roadmap.md) @@ -40,11 +41,11 @@ Supported Linux distributions Versions -------------- -[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.3.0
+[kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.4.3
[etcd](https://github.com/coreos/etcd/releases) v3.0.1
-[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.20.0
-[flanneld](https://github.com/coreos/flannel/releases) v0.5.5
-[weave](http://weave.works/) v1.5.0
+[flanneld](https://github.com/coreos/flannel/releases) v0.6.2
+[calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.22.0
+[weave](http://weave.works/) v1.6.1
[docker](https://www.docker.com/) v1.10.3
diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..900fffb39 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,9 @@ +# Release Process + +The Kargo Project is released on an as-needed basis. The process is as follows: + +1. An issue is proposing a new release with a changelog since the last release +2. At least on of the [OWNERS](OWNERS) must LGTM this release +3. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION` +4. The release issue is closed +5. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] kargo $VERSION is released` diff --git a/Vagrantfile b/Vagrantfile index ac8b91827..44f80db8c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -76,7 +76,7 @@ Vagrant.configure("2") do |config| ip = "#{$subnet}.#{i+100}" host_vars[vm_name] = { "ip" => ip, - "access_ip" => ip, + #"access_ip" => ip, "flannel_interface" => ip, "flannel_backend_type" => "host-gw", "local_release_dir" => "/vagrant/temp", @@ -100,11 +100,11 @@ Vagrant.configure("2") do |config| #ansible.tags = ['download'] ansible.groups = { # The first three nodes should be etcd servers - "etcd" => ["k8s-0[1:3]"], + "etcd" => ["#{$instance_name_prefix}-0[1:3]"], # The first two nodes should be masters - "kube-master" => ["k8s-0[1:2]"], + "kube-master" => ["#{$instance_name_prefix}-0[1:2]"], # all nodes should be kube nodes - "kube-node" => ["k8s-0[1:#{$num_instances}]"], + "kube-node" => ["#{$instance_name_prefix}-0[1:#{$num_instances}]"], "k8s-cluster:children" => ["kube-master", "kube-node"], } end diff --git a/ansible.cfg b/ansible.cfg index 2be6f4d02..f0e4ef652 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,4 +1,7 @@ [ssh_connection] pipelining=True -[defaults] +[defaults] host_key_checking=False +gathering = smart +fact_caching = jsonfile +fact_caching_connection = /tmp diff --git a/cluster.yml b/cluster.yml index 0c75b21b9..295bb668a 100644 --- a/cluster.yml +++ b/cluster.yml @@ -1,9 +1,26 @@ --- -- hosts: k8s-cluster +- hosts: all + gather_facts: false + roles: + - bootstrap-os + tags: + - bootstrap-os + + +- hosts: all + gather_facts: true + +- hosts: all roles: - { role: kubernetes/preinstall, tags: preinstall } + +- hosts: etcd:!k8s-cluster + roles: + - { role: etcd, tags: etcd } + +- hosts: k8s-cluster + roles: - { role: etcd, tags: etcd } - - { role: docker, tags: docker } - { role: kubernetes/node, tags: node } - { role: network_plugin, tags: network } @@ -17,4 +34,4 @@ - hosts: kube-master[0] roles: - - {role: kubernetes-apps, tags: apps} + - { role: kubernetes-apps, tags: apps } diff --git a/code-of-conduct.md b/code-of-conduct.md new file mode 100644 index 000000000..60270914a --- /dev/null +++ b/code-of-conduct.md @@ -0,0 +1,59 @@ +## Kubernetes Community Code of Conduct + +### Contributor Code of Conduct + +As contributors and maintainers of this project, and in the interest of fostering +an open and welcoming community, we pledge to respect all people who contribute +through reporting issues, posting feature requests, updating documentation, +submitting pull requests or patches, and other activities. + +We are committed to making participation in this project a harassment-free experience for +everyone, regardless of level of experience, gender, gender identity and expression, +sexual orientation, disability, personal appearance, body size, race, ethnicity, age, +religion, or nationality. + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery +* Personal attacks +* Trolling or insulting/derogatory comments +* Public or private harassment +* Publishing other's private information, such as physical or electronic addresses, + without explicit permission +* Other unethical or unprofessional conduct. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are not +aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers +commit themselves to fairly and consistently applying these principles to every aspect +of managing this project. Project maintainers who do not follow or enforce the Code of +Conduct may be permanently removed from the project team. + +This code of conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by +opening an issue or contacting one or more of the project maintainers. + +This Code of Conduct is adapted from the Contributor Covenant +(http://contributor-covenant.org), version 1.2.0, available at +http://contributor-covenant.org/version/1/2/0/ + +### Kubernetes Events Code of Conduct + +Kubernetes events are working conferences intended for professional networking and collaboration in the +Kubernetes community. Attendees are expected to behave according to professional standards and in accordance +with their employer's policies on appropriate workplace behavior. + +While at Kubernetes events or related social networking opportunities, attendees should not engage in +discriminatory or offensive speech or actions regarding gender, sexuality, race, or religion. Speakers should +be especially aware of these concerns. + +The Kubernetes team does not condone any statements by speakers contrary to these standards. The Kubernetes +team reserves the right to deny entrance and/or eject from an event (without refund) any individual found to +be engaging in discriminatory or offensive speech or actions. + +Please bring any concerns to to the immediate attention of Kubernetes event staff + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/code-of-conduct.md?pixel)]() diff --git a/docs/ansible.md b/docs/ansible.md index 0786a7764..101c0a075 100644 --- a/docs/ansible.md +++ b/docs/ansible.md @@ -4,14 +4,14 @@ Ansible variables Inventory ------------- -The inventory is composed of 3 groups: +The inventory is composed of 3 groups: * **kube-node** : list of kubernetes nodes where the pods will run. -* **kube-master** : list of servers where kubernetes master components (apiserver, scheduler, controller) will run. +* **kube-master** : list of servers where kubernetes master components (apiserver, scheduler, controller) will run. Note: if you want the server to act both as master and node the server must be defined on both groups _kube-master_ and _kube-node_ * **etcd**: list of server to compose the etcd server. you should have at least 3 servers for failover purposes. -Below is a complete inventory example: +Below is a complete inventory example: ``` ## Configure 'ip' variable to bind kubernetes services on a diff --git a/docs/aws.md b/docs/aws.md new file mode 100644 index 000000000..429e77a54 --- /dev/null +++ b/docs/aws.md @@ -0,0 +1,10 @@ +AWS +=============== + +To deploy kubespray on [AWS](https://aws.amazon.com/) uncomment the `cloud_provider` option in `group_vars/all.yml` and set it to `'aws'`. + +Prior to creating your instances, you **must** ensure that you have created IAM roles and policies for both "kubernetes-master" and "kubernetes-node". You can find the IAM policies [here](https://github.com/kubernetes/kubernetes/tree/master/cluster/aws/templates/iam). See the [IAM Documentation](https://aws.amazon.com/documentation/iam/) if guidance is needed on how to set these up. When you bring your instances online, associate them with the respective IAM role. Nodes that are only to be used for Etcd do not need a role. + +The next step is to make sure the hostnames in your `inventory` file are identical to your internal hostnames in AWS. This may look something like `ip-111-222-333-444.us-west-2.compute.internal`. You can then specify how Ansible connects to these instances with `ansible_ssh_host` and `ansible_ssh_user`. + +You can now create your cluster! diff --git a/docs/dns-stack.md b/docs/dns-stack.md new file mode 100644 index 000000000..e6df11b73 --- /dev/null +++ b/docs/dns-stack.md @@ -0,0 +1,92 @@ +K8s DNS stack by Kargo +====================== + +Kargo configures a [Kubernetes DNS](http://kubernetes.io/docs/admin/dns/) +[cluster add-on](http://releases.k8s.io/master/cluster/addons/README.md) +to serve as an authoritative DNS server for a given ``dns_domain`` and its +``svc, default.svc`` default subdomains (a total of ``ndots: 5`` max levels). + +Note, additional search (sub)domains may be defined in the ``searchdomains`` +and ``ndots`` vars. And additional recursive DNS resolvers in the `` upstream_dns_servers``, +``nameservers`` vars. Intranet DNS resolvers should be specified in the first +place, followed by external resolvers, for example: + +``` +skip_dnsmasq: true +nameservers: [8.8.8.8] +upstream_dns_servers: [172.18.32.6] +``` +or +``` +skip_dnsmasq: false +upstream_dns_servers: [172.18.32.6, 172.18.32.7, 8.8.8.8, 8.8.8.4] +``` +The vars are explained below as well. + +DNS configuration details +------------------------- + +Here is an approximate picture of how DNS things working and +being configured by Kargo ansible playbooks: + +![Image](figures/dns.jpeg?raw=true) + +Note that an additional dnsmasq daemon set is installed by Kargo +by default. Kubelet will configure DNS base of all pods to use the +given dnsmasq cluster IP, which is defined via the ``dns_server`` var. +The dnsmasq forwards requests for a given cluster ``dns_domain`` to +Kubedns's SkyDns service. The SkyDns server is configured to be an +authoritative DNS server for the given cluser domain (and its subdomains +up to ``ndots:5`` depth). Note: you should scale its replication controller +up, if SkyDns chokes. These two layered DNS forwarders provide HA for the +DNS cluster IP endpoint, which is a critical moving part for Kubernetes apps. + +Nameservers are as well configured in the hosts' ``/etc/resolv.conf`` files, +as the given DNS cluster IP merged with ``nameservers`` values. While the +DNS cluster IP merged with the ``upstream_dns_servers`` defines additional +nameservers for the aforementioned nsmasq daemon set running on all hosts. +This mitigates existing Linux limitation of max 3 nameservers in the +``/etc/resolv.conf`` and also brings an additional caching layer for the +clustered DNS services. + +You can skip the dnsmasq daemon set install steps by setting the +``skip_dnsmasq: true``. This may be the case, if you're fine with +the nameservers limitation. Sadly, there is no way to work around the +search domain limitations of a 256 chars and 6 domains. Thus, you can +use the ``searchdomains`` var to define no more than a three custom domains. +Remaining three slots are reserved for K8s cluster default subdomains. + +When dnsmasq skipped, Kargo redefines the DNS cluster IP to point directly +to SkyDns cluster IP ``skydns_server`` and configures Kubelet's +``--dns_cluster`` to use that IP as well. While this greatly simplifies +things, it comes by the price of limited nameservers though. As you know now, +the DNS cluster IP takes a slot in the ``/etc/resolv.conf``, thus you can +specify no more than a two nameservers for infra and/or external use. +Those may be specified either in ``nameservers`` or ``upstream_dns_servers`` +and will be merged together with the ``skydns_server`` IP into the hots' +``/etc/resolv.conf``. + +Limitations +----------- + +* Kargo has yet ways to configure Kubedns addon to forward requests SkyDns can + not answer with authority to arbitrary recursive resolvers. This task is left + for future. See [official SkyDns docs](https://github.com/skynetservices/skydns) + for details. + +* There is + [no way to specify a custom value](https://github.com/kubernetes/kubernetes/issues/33554) + for the SkyDNS ``ndots`` param via an + [option for KubeDNS](https://github.com/kubernetes/kubernetes/blob/master/cmd/kube-dns/app/options/options.go) + add-on, while SkyDNS supports it though. Thus, DNS SRV records may not work + as expected as they require the ``ndots:7``. + +* the ``searchdomains`` have a limitation of a 6 names and 256 chars + length. Due to default ``svc, default.svc`` subdomains, the actual + limits are a 4 names and 239 chars respectively. + +* the ``nameservers`` have a limitation of a 3 servers, although there + is a way to mitigate that with the ``upstream_dns_servers``, + see below. Anyway, the ``nameservers`` can take no more than a two + custom DNS servers because of one slot is reserved for a Kubernetes + cluster needs. diff --git a/docs/figures/dns.jpeg b/docs/figures/dns.jpeg new file mode 100644 index 000000000..56986252d Binary files /dev/null and b/docs/figures/dns.jpeg differ diff --git a/docs/figures/loadbalancer_localhost.png b/docs/figures/loadbalancer_localhost.png new file mode 100644 index 000000000..0732d5489 Binary files /dev/null and b/docs/figures/loadbalancer_localhost.png differ diff --git a/docs/ha-mode.md b/docs/ha-mode.md new file mode 100644 index 000000000..792c18a19 --- /dev/null +++ b/docs/ha-mode.md @@ -0,0 +1,109 @@ +HA endpoints for K8s +==================== + +The following components require a highly available endpoints: +* etcd cluster, +* kube-apiserver service instances. + +The former provides the +[etcd-proxy](https://coreos.com/etcd/docs/latest/proxy.html) service to access +the cluster members in HA fashion. + +The latter relies on a 3rd side reverse proxies, like Nginx or HAProxy, to +achieve the same goal. + +Etcd +---- + +Etcd proxies are deployed on each node in the `k8s-cluster` group. A proxy is +a separate etcd process. It has a `localhost:2379` frontend and all of the etcd +cluster members as backends. Note that the `access_ip` is used as the backend +IP, if specified. Frontend endpoints cannot be accessed externally as they are +bound to a localhost only. + +The `etcd_access_endpoint` fact provides an access pattern for clients. And the +`etcd_multiaccess` (defaults to `false`) group var controlls that behavior. +When enabled, it makes deployed components to access the etcd cluster members +directly: `http://ip1:2379, http://ip2:2379,...`. This mode assumes the clients +do a loadbalancing and handle HA for connections. Note, a pod definition of a +flannel networking plugin always uses a single `--etcd-server` endpoint! + + +Kube-apiserver +-------------- + +K8s components require a loadbalancer to access the apiservers via a reverse +proxy. Kargo includes support for an nginx-based proxy that resides on each +non-master Kubernetes node. This is referred to as localhost loadbalancing. It +is less efficient than a dedicated load balancer because it creates extra +health checks on the Kubernetes apiserver, but is more practical for scenarios +where an external LB or virtual IP management is inconvenient. + +This option is configured by the variable `loadbalancer_apiserver_localhost`. +you will need to configure your own loadbalancer to achieve HA. Note that +deploying a loadbalancer is up to a user and is not covered by ansible roles +in Kargo. By default, it only configures a non-HA endpoint, which points to +the `access_ip` or IP address of the first server node in the `kube-master` +group. It can also configure clients to use endpoints for a given loadbalancer +type. The following diagram shows how traffic to the apiserver is directed. + +![Image](figures/loadbalancer_localhost.png?raw=true) + + Note: Kubernetes master nodes still use insecure localhost access because + there are bugs in Kubernetes <1.5.0 in using TLS auth on master role + services. This makes backends receiving unencrypted traffic and may be a + security issue when interconnecting different nodes, or maybe not, if those + belong to the isolated management network without external access. + +A user may opt to use an external loadbalancer (LB) instead. An external LB +provides access for external clients, while the internal LB accepts client +connections only to the localhost, similarly to the etcd-proxy HA endpoints. +Given a frontend `VIP` address and `IP1, IP2` addresses of backends, here is +an example configuration for a HAProxy service acting as an external LB: +``` +listen kubernetes-apiserver-https + bind :8383 + option ssl-hello-chk + mode tcp + timeout client 3h + timeout server 3h + server master1 :443 + server master2 :443 + balance roundrobin +``` + +And the corresponding example global vars config: +``` +apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" +loadbalancer_apiserver: + address: + port: 8383 +``` + +This domain name, or default "lb-apiserver.kubernetes.local", will be inserted +into the `/etc/hosts` file of all servers in the `k8s-cluster` group. Note that +the HAProxy service should as well be HA and requires a VIP management, which +is out of scope of this doc. Specifying an external LB overrides any internal +localhost LB configuration. + + Note: In order to achieve HA for HAProxy instances, those must be running on + the each node in the `k8s-cluster` group as well, but require no VIP, thus + no VIP management. + +Access endpoints are evaluated automagically, as the following: + +| Endpoint type | kube-master | non-master | +|------------------------------|---------------|---------------------| +| Local LB | http://lc:p | https://lc:sp | +| External LB, no internal | https://lb:lp | https://lb:lp | +| No ext/int LB (default) | http://lc:p | https://m[0].aip:sp | + +Where: +* `m[0]` - the first node in the `kube-master` group; +* `lb` - LB FQDN, `apiserver_loadbalancer_domain_name`; +* `lc` - localhost; +* `p` - insecure port, `kube_apiserver_insecure_port` +* `sp` - secure port, `kube_apiserver_port`; +* `lp` - LB port, `loadbalancer_apiserver.port`, defers to the secure port; +* `ip` - the node IP, defers to the ansible IP; +* `aip` - `access_ip`, defers to the ip. diff --git a/docs/large-deployments.md b/docs/large-deployments.md new file mode 100644 index 000000000..2a36c3ebc --- /dev/null +++ b/docs/large-deployments.md @@ -0,0 +1,19 @@ +Large deployments of K8s +======================== + +For a large scaled deployments, consider the following configuration changes: + +* Tune [ansible settings](http://docs.ansible.com/ansible/intro_configuration.html) + for `forks` and `timeout` vars to fit large numbers of nodes being deployed. + +* Override containers' `foo_image_repo` vars to point to intranet registry. + +* Override the ``download_run_once: true`` to download binaries and container + images only once then push to nodes in batches. + +* Adjust the `retry_stagger` global var as appropriate. It should provide sane + load on a delegate (the first K8s master node) then retrying failed + push or download operations. + +For example, when deploying 200 nodes, you may want to run ansible with +``--forks=50``, ``--timeout=600`` and define the ``retry_stagger: 60``. diff --git a/docs/roadmap.md b/docs/roadmap.md index 298750493..1714ade05 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,10 @@ Kargo's roadmap ================= +### Kubeadm +- Propose kubeadm as an option in order to setup the kubernetes cluster. +That would probably improve deployment speed and certs management [#553](https://github.com/kubespray/kargo/issues/553) + ### Self deployment (pull-mode) [#320](https://github.com/kubespray/kargo/issues/320) - the playbook would install and configure docker/rkt and the etcd cluster - the following data would be inserted into etcd: certs,tokens,users,inventory,group_vars. @@ -26,13 +30,14 @@ Kargo's roadmap - single test with the Ansible version n-1 per day - Test idempotency on on single OS but for all network plugins/container engines - single test on AWS per day -- test different achitectures : +- test different achitectures : - 3 instances, 3 are members of the etcd cluster, 2 of them acting as master and node, 1 as node - 5 instances, 3 are etcd and nodes, 2 are masters only - 7 instances, 3 etcd only, 2 masters, 2 nodes - test scale up cluster: +1 etcd, +1 master, +1 node ### Lifecycle +- Adopt the kubeadm tool by delegating CM tasks it is capable to accomplish well [#553](https://github.com/kubespray/kargo/issues/553) - Drain worker node when upgrading k8s components in a worker node. [#154](https://github.com/kubespray/kargo/issues/154) - Drain worker node when shutting down/deleting an instance @@ -56,7 +61,7 @@ While waiting for the issue [kubernetes/kubernetes#18174](https://github.com/kub ### Kargo API - Perform all actions through an **API** - Store inventories / configurations of mulltiple clusters -- make sure that state of cluster is completely saved in no more than one config file beyond hosts inventory +- make sure that state of cluster is completely saved in no more than one config file beyond hosts inventory ### Addons (with kpm) Include optionals deployments to init the cluster: @@ -65,7 +70,7 @@ Include optionals deployments to init the cluster: - **Prometheus** ##### Others - + ##### Dashboards: - kubernetes-dashboard - Fabric8 diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index e82c83147..cbf2e63a2 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -1,9 +1,14 @@ +# Valid bootstrap options (required): xenial, coreos, none +bootstrap_os: none + # Directory where the binaries will be installed bin_dir: /usr/local/bin # Where the binaries will be downloaded. # Note: ensure that you've enough disk space (about 1G) local_release_dir: "/tmp/releases" +# Random shifts for retrying failed ops like pushing/downloading +retry_stagger: 5 # Uncomment this line for CoreOS only. # Directory where python binary is installed @@ -28,6 +33,8 @@ kube_users: # Kubernetes cluster name, also will be used as DNS domain cluster_name: cluster.local +# Subdomains of DNS domain to be resolved via /etc/resolv.conf +ndots: 5 # For some environments, each node has a pubilcally accessible # address and an address it should bind services to. These are @@ -51,6 +58,16 @@ cluster_name: cluster.local # but don't know about that address themselves. # access_ip: 1.1.1.1 +# Etcd access modes: +# Enable multiaccess to configure clients to access all of the etcd members directly +# as the "http://hostX:port, http://hostY:port, ..." and ignore the proxy loadbalancers. +# This may be the case if clients support and loadbalance multiple etcd servers natively. +etcd_multiaccess: false + +# Assume there are no internal loadbalancers for apiservers exist and listen on +# kube_apiserver_port (default 443) +loadbalancer_apiserver_localhost: true + # Choose network plugin (calico, weave or flannel) kube_network_plugin: flannel @@ -89,10 +106,12 @@ kube_apiserver_insecure_port: 8080 # (http) # You still must manually configure all your containers to use this DNS server, # Kubernetes won't do this for you (yet). +# Do not install additional dnsmasq +skip_dnsmasq: false # Upstream dns servers used by dnsmasq -upstream_dns_servers: - - 8.8.8.8 - - 8.8.4.4 +#upstream_dns_servers: +# - 8.8.8.8 +# - 8.8.4.4 # # # Use dns server : https://github.com/ansibl8s/k8s-skydns/blob/master/skydns-README.md dns_setup: true @@ -109,21 +128,6 @@ dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') # like you would do when using nova-client before starting the playbook. # cloud_provider: -# For multi masters architecture: -# kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer -# This domain name will be inserted into the /etc/hosts file of all servers -# configuration example with haproxy : -# listen kubernetes-apiserver-https -# bind 10.99.0.21:8383 -# option ssl-hello-chk -# mode tcp -# timeout client 3h -# timeout server 3h -# server master1 10.99.0.26:443 -# server master2 10.99.0.27:443 -# balance roundrobin -# apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local" - ## Set these proxy values in order to update docker daemon to use proxies # http_proxy: "" # https_proxy: "" @@ -136,9 +140,5 @@ dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') docker_options: "--insecure-registry={{ kube_service_addresses }}" # default packages to install within the cluster -kpm_packages: - - name: kube-system/kubedns - namespace: kube-system - variables: - cluster_ip: "{{skydns_server}}" +kpm_packages: [] # - name: kube-system/grafana diff --git a/roles/coreos-bootstrap/defaults/main.yml b/roles/bootstrap-os/defaults/main.yml similarity index 100% rename from roles/coreos-bootstrap/defaults/main.yml rename to roles/bootstrap-os/defaults/main.yml diff --git a/roles/coreos-bootstrap/files/bootstrap.sh b/roles/bootstrap-os/files/bootstrap.sh similarity index 100% rename from roles/coreos-bootstrap/files/bootstrap.sh rename to roles/bootstrap-os/files/bootstrap.sh diff --git a/roles/coreos-bootstrap/files/get-pip.py b/roles/bootstrap-os/files/get-pip.py similarity index 100% rename from roles/coreos-bootstrap/files/get-pip.py rename to roles/bootstrap-os/files/get-pip.py diff --git a/roles/coreos-bootstrap/files/runner b/roles/bootstrap-os/files/runner similarity index 100% rename from roles/coreos-bootstrap/files/runner rename to roles/bootstrap-os/files/runner diff --git a/roles/coreos-bootstrap/tasks/main.yml b/roles/bootstrap-os/tasks/bootstrap-coreos.yml similarity index 82% rename from roles/coreos-bootstrap/tasks/main.yml rename to roles/bootstrap-os/tasks/bootstrap-coreos.yml index 4d9e11ea6..a638ad82b 100644 --- a/roles/coreos-bootstrap/tasks/main.yml +++ b/roles/bootstrap-os/tasks/bootstrap-coreos.yml @@ -4,9 +4,10 @@ register: need_bootstrap ignore_errors: True + - name: Bootstrap | Run bootstrap.sh script: bootstrap.sh - when: need_bootstrap | failed + when: (need_bootstrap | failed) - set_fact: ansible_python_interpreter: "/opt/bin/python" @@ -16,23 +17,23 @@ register: need_pip ignore_errors: True changed_when: false - when: need_bootstrap | failed + when: (need_bootstrap | failed) - name: Bootstrap | Copy get-pip.py copy: src=get-pip.py dest=~/get-pip.py - when: need_pip | failed + when: (need_pip | failed) - name: Bootstrap | Install pip shell: "{{ansible_python_interpreter}} ~/get-pip.py" - when: need_pip | failed + when: (need_pip | failed) - name: Bootstrap | Remove get-pip.py file: path=~/get-pip.py state=absent - when: need_pip | failed + when: (need_pip | failed) - name: Bootstrap | Install pip launcher copy: src=runner dest=/opt/bin/pip mode=0755 - when: need_pip | failed + when: (need_pip | failed) - name: Install required python modules pip: @@ -45,4 +46,4 @@ - name: Assign inventory name to unconfigured hostnames shell: sh -c "echo \"{{inventory_hostname}}\" > /etc/hostname; hostname \"{{inventory_hostname}}\"" - when: configured_hostname.stdout == 'localhost' + when: (configured_hostname.stdout == 'localhost') diff --git a/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml b/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml new file mode 100644 index 000000000..2d3becd44 --- /dev/null +++ b/roles/bootstrap-os/tasks/bootstrap-ubuntu.yml @@ -0,0 +1,14 @@ +--- +# raw: cat /etc/issue.net | grep '{{ bootstrap_versions }}' + +- name: Bootstrap | Check if bootstrap is needed + raw: which python + register: need_bootstrap + ignore_errors: True + +- name: Bootstrap | Install python 2.x + raw: DEBIAN_FRONTEND=noninteractive apt-get install -y python-minimal + when: need_bootstrap | failed + +- set_fact: + ansible_python_interpreter: "/usr/bin/python" diff --git a/roles/bootstrap-os/tasks/main.yml b/roles/bootstrap-os/tasks/main.yml new file mode 100644 index 000000000..5d084ec74 --- /dev/null +++ b/roles/bootstrap-os/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- include: bootstrap-ubuntu.yml + when: bootstrap_os == "ubuntu" + +- include: bootstrap-coreos.yml + when: bootstrap_os == "coreos" \ No newline at end of file diff --git a/roles/coreos-bootstrap/templates/python_shim.j2 b/roles/bootstrap-os/templates/python_shim.j2 similarity index 100% rename from roles/coreos-bootstrap/templates/python_shim.j2 rename to roles/bootstrap-os/templates/python_shim.j2 diff --git a/roles/dnsmasq/defaults/main.yml b/roles/dnsmasq/defaults/main.yml new file mode 100644 index 000000000..7a1e77023 --- /dev/null +++ b/roles/dnsmasq/defaults/main.yml @@ -0,0 +1,19 @@ +--- +# Existing search/nameserver resolvconf entries will be purged and +# ensured by this additional data: + +# Max of 4 names is allowed and no more than 256 - 17 chars total +# (a 2 is reserved for the 'default.svc.' and'svc.') +#searchdomains: +# - foo.bar.lc + +# Max of 2 is allowed here (a 1 is reserved for the dns_server) +#nameservers: +# - 127.0.0.1 + +# Versions +dnsmasq_version: 2.72 + +# Images +dnsmasq_image_repo: "andyshinn/dnsmasq" +dnsmasq_image_tag: "{{ dnsmasq_version }}" \ No newline at end of file diff --git a/roles/dnsmasq/handlers/main.yml b/roles/dnsmasq/handlers/main.yml new file mode 100644 index 000000000..4bdfd10f6 --- /dev/null +++ b/roles/dnsmasq/handlers/main.yml @@ -0,0 +1,34 @@ +- name: Dnsmasq | restart network + command: /bin/true + notify: + - Dnsmasq | reload network + - Dnsmasq | update resolvconf + when: ansible_os_family != "CoreOS" + +- name: Dnsmasq | reload network + service: + name: >- + {% if ansible_os_family == "RedHat" -%} + network + {%- elif ansible_os_family == "Debian" -%} + networking + {%- endif %} + state: restarted + when: ansible_os_family != "RedHat" and ansible_os_family != "CoreOS" + +- name: Dnsmasq | update resolvconf + command: /bin/true + notify: + - Dnsmasq | reload resolvconf + - Dnsmasq | reload kubelet + +- name: Dnsmasq | reload resolvconf + command: /sbin/resolvconf -u + ignore_errors: true + +- name: Dnsmasq | reload kubelet + service: + name: kubelet + state: restarted + when: "{{ inventory_hostname in groups['kube-master'] }}" + ignore_errors: true diff --git a/roles/dnsmasq/library/kube.py b/roles/dnsmasq/library/kube.py index aab92a733..2922c6212 100644 --- a/roles/dnsmasq/library/kube.py +++ b/roles/dnsmasq/library/kube.py @@ -44,12 +44,6 @@ options: default: null description: - The url for the API server that commands are executed against. - api_version: - required: false - choices: ['v1', 'v1beta3'] - default: v1 - description: - - The API version associated with cluster. force: required: false default: false @@ -105,10 +99,6 @@ class KubeManager(object): if self.kubectl is None: self.kubectl = module.get_bin_path('kubectl', True) self.base_cmd = [self.kubectl] - self.api_version = module.params.get('api_version') - - if self.api_version: - self.base_cmd.append('--api-version=' + self.api_version) if module.params.get('server'): self.base_cmd.append('--server=' + module.params.get('server')) @@ -164,8 +154,6 @@ class KubeManager(object): return [] cmd = ['replace'] - if self.api_version != 'v1': - cmd = ['update'] if self.force: cmd.append('--force') @@ -271,7 +259,6 @@ def main(): label=dict(), server=dict(), kubectl=dict(), - api_version=dict(default='v1', choices=['v1', 'v1beta3']), force=dict(default=False, type='bool'), all=dict(default=False, type='bool'), log_level=dict(default=0, type='int'), diff --git a/roles/dnsmasq/tasks/dnsmasq.yml b/roles/dnsmasq/tasks/dnsmasq.yml new file mode 100644 index 000000000..bc37580de --- /dev/null +++ b/roles/dnsmasq/tasks/dnsmasq.yml @@ -0,0 +1,58 @@ +--- +- name: ensure dnsmasq.d directory exists + file: + path: /etc/dnsmasq.d + state: directory + +- name: ensure dnsmasq.d-available directory exists + file: + path: /etc/dnsmasq.d-available + state: directory + +- name: Write dnsmasq configuration + template: + src: 01-kube-dns.conf.j2 + dest: /etc/dnsmasq.d-available/01-kube-dns.conf + mode: 0755 + backup: yes + +- name: Stat dnsmasq configuration + stat: path=/etc/dnsmasq.d/01-kube-dns.conf + register: sym + +- name: Move previous configuration + command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak + changed_when: False + when: sym.stat.islnk is defined and sym.stat.islnk == False + +- name: Enable dnsmasq configuration + file: + src: /etc/dnsmasq.d-available/01-kube-dns.conf + dest: /etc/dnsmasq.d/01-kube-dns.conf + state: link + +- name: Create dnsmasq manifests + template: src={{item.file}} dest=/etc/kubernetes/{{item.file}} + with_items: + - {file: dnsmasq-ds.yml, type: ds} + - {file: dnsmasq-svc.yml, type: svc} + register: manifests + when: inventory_hostname == groups['kube-master'][0] + +- name: Start Resources + kube: + name: dnsmasq + namespace: kube-system + kubectl: "{{bin_dir}}/kubectl" + resource: "{{item.item.type}}" + filename: /etc/kubernetes/{{item.item.file}} + state: "{{item.changed | ternary('latest','present') }}" + with_items: "{{ manifests.results }}" + when: inventory_hostname == groups['kube-master'][0] + +- name: Check for dnsmasq port (pulling image and running container) + wait_for: + host: "{{dns_server}}" + port: 53 + delay: 5 + when: inventory_hostname == groups['kube-node'][0] diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml index 87a89e293..46c1604f6 100644 --- a/roles/dnsmasq/tasks/main.yml +++ b/roles/dnsmasq/tasks/main.yml @@ -1,114 +1,5 @@ --- -- name: ensure dnsmasq.d directory exists - file: - path: /etc/dnsmasq.d - state: directory +- include: dnsmasq.yml + when: "{{ not skip_dnsmasq|bool }}" -- name: ensure dnsmasq.d-available directory exists - file: - path: /etc/dnsmasq.d-available - state: directory - -- name: Write dnsmasq configuration - template: - src: 01-kube-dns.conf.j2 - dest: /etc/dnsmasq.d-available/01-kube-dns.conf - mode: 0755 - backup: yes - -- name: Stat dnsmasq configuration - stat: path=/etc/dnsmasq.d/01-kube-dns.conf - register: sym - -- name: Move previous configuration - command: mv /etc/dnsmasq.d/01-kube-dns.conf /etc/dnsmasq.d-available/01-kube-dns.conf.bak - changed_when: False - when: sym.stat.islnk is defined and sym.stat.islnk == False - -- name: Enable dnsmasq configuration - file: - src: /etc/dnsmasq.d-available/01-kube-dns.conf - dest: /etc/dnsmasq.d/01-kube-dns.conf - state: link - -- name: Create dnsmasq manifests - template: src={{item.file}} dest=/etc/kubernetes/{{item.file}} - with_items: - - {file: dnsmasq-ds.yml, type: ds} - - {file: dnsmasq-svc.yml, type: svc} - register: manifests - when: inventory_hostname == groups['kube-master'][0] - -- name: Start Resources - kube: - name: dnsmasq - namespace: kube-system - kubectl: "{{bin_dir}}/kubectl" - resource: "{{item.item.type}}" - filename: /etc/kubernetes/{{item.item.file}} - state: "{{item.changed | ternary('latest','present') }}" - with_items: "{{ manifests.results }}" - when: inventory_hostname == groups['kube-master'][0] - -- name: Check for dnsmasq port (pulling image and running container) - wait_for: - host: "{{dns_server}}" - port: 53 - delay: 5 - when: inventory_hostname == groups['kube-master'][0] - - -- name: check resolvconf - stat: path=/etc/resolvconf/resolv.conf.d/head - register: resolvconf - -- name: target resolv.conf file - set_fact: - resolvconffile: >- - {%- if resolvconf.stat.exists == True -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%} - -- name: Add search resolv.conf - lineinfile: - line: "search {{ [ 'default.svc.' + dns_domain, 'svc.' + dns_domain, dns_domain ] | join(' ') }}" - dest: "{{resolvconffile}}" - state: present - insertbefore: BOF - backup: yes - follow: yes - -- name: Add local dnsmasq to resolv.conf - lineinfile: - line: "nameserver {{dns_server}}" - dest: "{{resolvconffile}}" - state: present - insertafter: "^search.*$" - backup: yes - follow: yes - -- name: Add options to resolv.conf - lineinfile: - line: options {{ item }} - dest: "{{resolvconffile}}" - state: present - regexp: "^options.*{{ item }}$" - insertafter: EOF - backup: yes - follow: yes - with_items: - - timeout:2 - - attempts:2 - -- name: disable resolv.conf modification by dhclient - copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate mode=0755 backup=yes - when: ansible_os_family == "Debian" - -- name: disable resolv.conf modification by dhclient - copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x backup=yes - when: ansible_os_family == "RedHat" - -- name: update resolvconf - command: resolvconf -u - changed_when: False - when: resolvconf.stat.exists == True - -- meta: flush_handlers +- include: resolvconf.yml diff --git a/roles/dnsmasq/tasks/resolvconf.yml b/roles/dnsmasq/tasks/resolvconf.yml new file mode 100644 index 000000000..c018e9771 --- /dev/null +++ b/roles/dnsmasq/tasks/resolvconf.yml @@ -0,0 +1,101 @@ +--- +- name: check resolvconf + shell: which resolvconf + register: resolvconf + ignore_errors: yes + +- name: target resolv.conf file + set_fact: + resolvconffile: >- + {%- if resolvconf.rc == 0 -%}/etc/resolvconf/resolv.conf.d/head{%- else -%}/etc/resolv.conf{%- endif -%} + +- name: generate search domains to resolvconf + set_fact: + searchentries: + "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }}" + +- name: pick dnsmasq cluster IP + set_fact: + dnsmasq_server: >- + {%- if skip_dnsmasq|bool -%}{{ [ skydns_server ] + upstream_dns_servers|default([]) }}{%- else -%}{{ [ dns_server ] }}{%- endif -%} + +- name: generate nameservers to resolvconf + set_fact: + nameserverentries: + "{{ dnsmasq_server|default([]) + nameservers|default([]) }}" + +- name: Remove search and nameserver options from resolvconf head + lineinfile: + dest: /etc/resolvconf/resolv.conf.d/head + state: absent + regexp: "^{{ item }}.*$" + backup: yes + follow: yes + with_items: + - search + - nameserver + when: resolvconf.rc == 0 + notify: Dnsmasq | update resolvconf + +- name: Add search domains to resolv.conf + lineinfile: + line: "search {{searchentries}}" + dest: "{{resolvconffile}}" + state: present + insertbefore: BOF + backup: yes + follow: yes + notify: Dnsmasq | update resolvconf + +- name: Add nameservers to resolv.conf + blockinfile: + dest: "{{resolvconffile}}" + block: |- + {% for item in nameserverentries -%} + nameserver {{ item }} + {% endfor %} + state: present + insertafter: "^search.*$" + create: yes + backup: yes + follow: yes + marker: "# Ansible nameservers {mark}" + notify: Dnsmasq | update resolvconf + +- name: Add options to resolv.conf + lineinfile: + line: options {{ item }} + dest: "{{resolvconffile}}" + state: present + regexp: "^options.*{{ item }}$" + insertafter: EOF + backup: yes + follow: yes + with_items: + - ndots:{{ ndots }} + - timeout:2 + - attempts:2 + notify: Dnsmasq | update resolvconf + +- name: Remove search and nameserver options from resolvconf base + lineinfile: + dest: /etc/resolvconf/resolv.conf.d/base + state: absent + regexp: "^{{ item }}.*$" + backup: yes + follow: yes + with_items: + - search + - nameserver + when: resolvconf.rc == 0 + notify: Dnsmasq | update resolvconf + +- name: disable resolv.conf modification by dhclient + copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient-enter-hooks.d/znodnsupdate mode=0755 + notify: Dnsmasq | restart network + when: ansible_os_family == "Debian" + +- name: disable resolv.conf modification by dhclient + copy: src=dhclient_nodnsupdate dest=/etc/dhcp/dhclient.d/nodnsupdate mode=u+x + notify: Dnsmasq | restart network + when: ansible_os_family == "RedHat" diff --git a/roles/dnsmasq/templates/01-kube-dns.conf.j2 b/roles/dnsmasq/templates/01-kube-dns.conf.j2 index 5ade101ce..4d73eebdb 100644 --- a/roles/dnsmasq/templates/01-kube-dns.conf.j2 +++ b/roles/dnsmasq/templates/01-kube-dns.conf.j2 @@ -4,17 +4,26 @@ listen-address=0.0.0.0 addn-hosts=/etc/hosts -bogus-priv +strict-order +# Forward k8s domain to kube-dns +server=/{{ dns_domain }}/{{ skydns_server }} #Set upstream dns servers {% if upstream_dns_servers is defined %} {% for srv in upstream_dns_servers %} server={{ srv }} {% endfor %} +{% elif cloud_provider is defined and cloud_provider == "gce" %} +server=169.254.169.254 {% else %} server=8.8.8.8 server=8.8.4.4 {% endif %} -# Forward k8s domain to kube-dns -server=/{{ dns_domain }}/{{ skydns_server }} +bogus-priv +no-resolv +no-negcache +cache-size=1000 +max-cache-ttl=10 +max-ttl=20 +log-facility=- diff --git a/roles/dnsmasq/templates/dnsmasq-ds.yml b/roles/dnsmasq/templates/dnsmasq-ds.yml index 44c046f18..49223124e 100644 --- a/roles/dnsmasq/templates/dnsmasq-ds.yml +++ b/roles/dnsmasq/templates/dnsmasq-ds.yml @@ -14,7 +14,7 @@ spec: spec: containers: - name: dnsmasq - image: andyshinn/dnsmasq:2.72 + image: "{{ dnsmasq_image_repo }}:{{ dnsmasq_image_tag }}" command: - dnsmasq args: @@ -25,7 +25,7 @@ spec: capabilities: add: - NET_ADMIN - imagePullPolicy: Always + imagePullPolicy: IfNotPresent resources: limits: cpu: 100m @@ -50,3 +50,4 @@ spec: - name: etcdnsmasqdavailable hostPath: path: /etc/dnsmasq.d-available + dnsPolicy: Default # Don't use cluster DNS. diff --git a/roles/docker/handlers/main.yml b/roles/docker/handlers/main.yml index 4153d123d..6f54f33d5 100644 --- a/roles/docker/handlers/main.yml +++ b/roles/docker/handlers/main.yml @@ -2,14 +2,26 @@ - name: restart docker command: /bin/true notify: - - reload systemd - - reload docker + - Docker | reload systemd + - Docker | reload docker + - Docker | pause while Docker restarts + - Docker | wait for docker -- name : reload systemd +- name : Docker | reload systemd shell: systemctl daemon-reload when: ansible_service_mgr == "systemd" -- name: reload docker +- name: Docker | reload docker service: name: docker state: restarted + +- name: Docker | pause while Docker restarts + pause: seconds=10 prompt="Waiting for docker restart" + +- name: Docker | wait for docker + command: /usr/bin/docker images + register: docker_ready + retries: 10 + delay: 5 + until: docker_ready.rc == 0 diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 03db20c75..1d237f5e9 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -27,6 +27,10 @@ id: "{{item}}" keyserver: "{{docker_repo_key_info.keyserver}}" state: present + register: keyserver_task_result + until: keyserver_task_result|success + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ docker_repo_key_info.repo_keys }}" when: ansible_os_family != "CoreOS" @@ -51,6 +55,10 @@ pkg: "{{item.name}}" force: "{{item.force|default(omit)}}" state: present + register: docker_task_result + until: docker_task_result|success + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" with_items: "{{ docker_package_info.pkgs }}" when: (ansible_os_family != "CoreOS") and (docker_package_info.pkgs|length > 0) @@ -59,6 +67,14 @@ when: ansible_service_mgr == "systemd" and (http_proxy is defined or https_proxy is defined or no_proxy is defined) +- name: Write docker.service systemd file + template: + src: systemd-docker.service.j2 + dest: /etc/systemd/system/docker.service + register: docker_service_file + notify: restart docker + when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS" + - meta: flush_handlers - name: ensure docker service is started and enabled diff --git a/roles/network_plugin/calico/templates/systemd-docker.service b/roles/docker/templates/systemd-docker.service.j2 similarity index 83% rename from roles/network_plugin/calico/templates/systemd-docker.service rename to roles/docker/templates/systemd-docker.service.j2 index 01383d772..b19b1caaf 100644 --- a/roles/network_plugin/calico/templates/systemd-docker.service +++ b/roles/docker/templates/systemd-docker.service.j2 @@ -2,11 +2,11 @@ Description=Docker Application Container Engine Documentation=http://docs.docker.com {% if ansible_os_family == "RedHat" %} -After=network.target +After=network.target docker-storage-setup.service Wants=docker-storage-setup.service {% elif ansible_os_family == "Debian" %} After=network.target docker.socket -Requires=docker.socket +Wants=docker.socket {% endif %} [Service] @@ -20,17 +20,21 @@ EnvironmentFile=-/etc/sysconfig/docker-storage EnvironmentFile=-/etc/default/docker {% endif %} Environment=GOTRACEBACK=crash +ExecReload=/bin/kill -s HUP $MAINPID +Delegate=yes +KillMode=process ExecStart=/usr/bin/docker daemon \ $OPTIONS \ $DOCKER_STORAGE_OPTIONS \ $DOCKER_NETWORK_OPTIONS \ $INSECURE_REGISTRY \ $DOCKER_OPTS +TasksMax=infinity LimitNOFILE=1048576 LimitNPROC=1048576 LimitCORE=infinity -MountFlags=slave TimeoutStartSec=1min +Restart=on-abnormal [Install] WantedBy=multi-user.target diff --git a/roles/docker/vars/debian.yml b/roles/docker/vars/debian.yml index ce8b52263..192517097 100644 --- a/roles/docker/vars/debian.yml +++ b/roles/docker/vars/debian.yml @@ -5,6 +5,8 @@ docker_versioned_pkg: latest: docker-engine 1.9: docker-engine=1.9.1-0~{{ ansible_distribution_release|lower }} 1.10: docker-engine=1.10.3-0~{{ ansible_distribution_release|lower }} + 1.11: docker-engine=1.11.2-0~{{ ansible_distribution_release|lower }} + 1.12: docker-engine=1.12.1-0~{{ ansible_distribution_release|lower }} docker_package_info: pkg_mgr: apt diff --git a/roles/docker/vars/fedora.yml b/roles/docker/vars/fedora.yml index d1693fab7..3fd06b4b3 100644 --- a/roles/docker/vars/fedora.yml +++ b/roles/docker/vars/fedora.yml @@ -4,6 +4,8 @@ docker_versioned_pkg: latest: docker 1.9: docker-1:1.9.1 1.10: docker-1:1.10.1 + 1.11: docker-1:1.11.2 + 1.12: docker-1:1.12.1 docker_package_info: pkg_mgr: dnf diff --git a/roles/docker/vars/ubuntu-16.04.yml b/roles/docker/vars/ubuntu-16.04.yml index e5b84702d..208fe5edc 100644 --- a/roles/docker/vars/ubuntu-16.04.yml +++ b/roles/docker/vars/ubuntu-16.04.yml @@ -2,10 +2,11 @@ docker_version: 1.11 docker_kernel_min_version: '3.2' -# https://apt.dockerproject.org/repo/dists/ubuntu-trusty/main/filelist +# https://apt.dockerproject.org/repo/dists/ubuntu-xenial/main/filelist docker_versioned_pkg: latest: docker-engine 1.11: docker-engine=1.11.1-0~{{ ansible_distribution_release|lower }} + 1.12: docker-engine=1.12.1-0~{{ ansible_distribution_release|lower }} docker_package_info: pkg_mgr: apt diff --git a/roles/docker/vars/ubuntu.yml b/roles/docker/vars/ubuntu.yml index 18da1b85e..1bf8a44a0 100644 --- a/roles/docker/vars/ubuntu.yml +++ b/roles/docker/vars/ubuntu.yml @@ -6,6 +6,8 @@ docker_versioned_pkg: latest: docker-engine 1.9: docker-engine=1.9.0-0~{{ ansible_distribution_release|lower }} 1.10: docker-engine=1.10.3-0~{{ ansible_distribution_release|lower }} + 1.11: docker-engine=1.11.1-0~{{ ansible_distribution_release|lower }} + 1.12: docker-engine=1.12.1-0~{{ ansible_distribution_release|lower }} docker_package_info: pkg_mgr: apt diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index c402b1efa..cbe053fa0 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -5,30 +5,47 @@ local_release_dir: /tmp download_run_once: False # Versions -include_vars: kube_versions.yml +kube_version: v1.4.3 -etcd_version: v3.0.1 +etcd_version: v3.0.6 #TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults # after migration to container download -calico_version: v0.20.0 -calico_cni_version: v1.3.1 -weave_version: v1.5.0 +calico_version: v0.22.0 +calico_cni_version: v1.4.2 +weave_version: v1.6.1 +flannel_version: v0.6.2 +flannel_server_helper_version: 0.1 +pod_infra_version: 3.0 # Download URL's -kubelet_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-kubelet" -apiserver_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-apiserver" -kubectl_download_url: "https://storage.googleapis.com/kargo/{{kube_version}}_kubernetes-kubectl" - etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd" calico_cni_download_url: "https://storage.googleapis.com/kargo/{{calico_cni_version}}_calico-cni-plugin" calico_cni_ipam_download_url: "https://storage.googleapis.com/kargo/{{calico_cni_version}}_calico-cni-plugin-ipam" weave_download_url: "https://storage.googleapis.com/kargo/{{weave_version}}_weave" # Checksums -calico_cni_checksum: "ac05cb9254b5aaa5822cf10325983431bd25489147f2edf9dec7e43d99c43e77" -calico_cni_ipam_checksum: "3df6951a30749c279229e7e318e74ac4e41263996125be65257db7cd25097273" -weave_checksum: "28d2c4e2b1ad8600da69882501eba697679aea10a5e61c769aa3a9ee72b0d89a" -etcd_checksum: "7e5d8db2b8a7cec7a93e531c8ae0f3108c66c7d896a2fb6d8768c067923ce0aa" +calico_cni_checksum: "9cab29764681e9d80da826e4b2cd10841cc01a749e0018867d96dd76a4691548" +calico_cni_ipam_checksum: "09d076b15b791956efee91646e47fdfdcf382db16082cef4f542a9fff7bae172" +weave_checksum: "9bf9d6e5a839e7bcbb28cc00c7acae9d09284faa3e7a3720ca9c2b9e93c68580" +etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485" + +# Containers +# Possible values: host, docker +etcd_deployment_type: "docker" +etcd_image_repo: "quay.io/coreos/etcd" +etcd_image_tag: "{{ etcd_version }}" +flannel_server_helper_image_repo: "gcr.io/google_containers/flannel-server-helper" +flannel_server_helper_image_tag: "{{ flannel_server_helper_version }}" +flannel_image_repo: "quay.io/coreos/flannel" +flannel_image_tag: "{{ flannel_version }}" +calicoctl_image_repo: "calico/ctl" +calicoctl_image_tag: "{{ calico_version }}" +calico_node_image_repo: "calico/node" +calico_node_image_tag: "{{ calico_version }}" +hyperkube_image_repo: "quay.io/coreos/hyperkube" +hyperkube_image_tag: "{{ kube_version }}_coreos.0" +pod_infra_image_repo: "gcr.io/google_containers/pause-amd64" +pod_infra_image_tag: "{{ pod_infra_version }}" downloads: calico_cni_plugin: @@ -39,6 +56,7 @@ downloads: url: "{{ calico_cni_download_url }}" owner: "root" mode: "0755" + enabled: "{{ kube_network_plugin == 'calico' }}" calico_cni_plugin_ipam: dest: calico/bin/calico-ipam version: "{{calico_cni_version}}" @@ -47,6 +65,7 @@ downloads: url: "{{ calico_cni_ipam_download_url }}" owner: "root" mode: "0755" + enabled: "{{ kube_network_plugin == 'calico' }}" weave: dest: weave/bin/weave version: "{{weave_version}}" @@ -55,6 +74,7 @@ downloads: sha256: "{{ weave_checksum }}" owner: "root" mode: "0755" + enabled: "{{ kube_network_plugin == 'weave' }}" etcd: version: "{{etcd_version}}" dest: "etcd/etcd-{{ etcd_version }}-linux-amd64.tar.gz" @@ -64,32 +84,42 @@ downloads: unarchive: true owner: "etcd" mode: "0755" - kubernetes_kubelet: - version: "{{kube_version}}" - dest: kubernetes/bin/kubelet - sha256: "{{vars['kube_checksum'][kube_version]['kubelet']}}" - source_url: "{{ kubelet_download_url }}" - url: "{{ kubelet_download_url }}" - owner: "kube" - mode: "0755" - kubernetes_kubectl: - dest: kubernetes/bin/kubectl - version: "{{kube_version}}" - sha256: "{{vars['kube_checksum'][kube_version]['kubectl']}}" - source_url: "{{ kubectl_download_url }}" - url: "{{ kubectl_download_url }}" - owner: "kube" - mode: "0755" - kubernetes_apiserver: - dest: kubernetes/bin/kube-apiserver - version: "{{kube_version}}" - sha256: "{{vars['kube_checksum'][kube_version]['kube_apiserver']}}" - source_url: "{{ apiserver_download_url }}" - url: "{{ apiserver_download_url }}" - owner: "kube" - mode: "0755" + container: "{{ etcd_deployment_type == 'docker' }}" + repo: "{{ etcd_image_repo }}" + tag: "{{ etcd_image_tag }}" + hyperkube: + container: true + repo: "{{ hyperkube_image_repo }}" + tag: "{{ hyperkube_image_tag }}" + flannel: + container: true + repo: "{{ flannel_image_repo }}" + tag: "{{ flannel_image_tag }}" + enabled: "{{ kube_network_plugin == 'flannel' }}" + flannel_server_helper: + container: true + repo: "{{ flannel_server_helper_image_repo }}" + tag: "{{ flannel_server_helper_image_tag }}" + enabled: "{{ kube_network_plugin == 'flannel' }}" + calicoctl: + container: true + repo: "{{ calicoctl_image_repo }}" + tag: "{{ calicoctl_image_tag }}" + enabled: "{{ kube_network_plugin == 'calico' }}" + calico_node: + container: true + repo: "{{ calico_node_image_repo }}" + tag: "{{ calico_node_image_tag }}" + enabled: "{{ kube_network_plugin == 'calico' }}" + pod_infra: + container: true + repo: "{{ pod_infra_image_repo }}" + tag: "{{ pod_infra_image_tag }}" download: + container: "{{ file.container|default('false') }}" + repo: "{{ file.repo|default(None) }}" + tag: "{{ file.tag|default(None) }}" enabled: "{{ file.enabled|default('true') }}" dest: "{{ file.dest|default(None) }}" version: "{{ file.version|default(None) }}" diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 40c52f5ea..e715f380d 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -1,15 +1,12 @@ --- -- include_vars: kube_versions.yml - - name: downloading... debug: msg: "{{ download.url }}" - when: "{{ download.enabled|bool }}" + when: "{{ download.enabled|bool and not download.container|bool }}" - name: Create dest directories file: path={{local_release_dir}}/{{download.dest|dirname}} state=directory recurse=yes - when: "{{ download.enabled|bool }}" - run_once: "{{ download_run_once|bool }}" + when: "{{ download.enabled|bool and not download.container|bool }}" - name: Download items get_url: @@ -18,8 +15,11 @@ sha256sum: "{{download.sha256 | default(omit)}}" owner: "{{ download.owner|default(omit) }}" mode: "{{ download.mode|default(omit) }}" - when: "{{ download.enabled|bool }}" - run_once: "{{ download_run_once|bool }}" + register: get_url_result + until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: "{{ download.enabled|bool and not download.container|bool }}" - name: Extract archives unarchive: @@ -28,8 +28,7 @@ owner: "{{ download.owner|default(omit) }}" mode: "{{ download.mode|default(omit) }}" copy: no - when: "{{ download.enabled|bool }} and ({{download.unarchive is defined and download.unarchive == True}})" - run_once: "{{ download_run_once|bool }}" + when: "{{ download.enabled|bool and not download.container|bool and download.unarchive is defined and download.unarchive == True }}" - name: Fix permissions file: @@ -37,5 +36,59 @@ path: "{{local_release_dir}}/{{download.dest}}" owner: "{{ download.owner|default(omit) }}" mode: "{{ download.mode|default(omit) }}" - when: "{{ download.enabled|bool }} and ({{download.unarchive is not defined or download.unarchive == False}})" + when: "{{ download.enabled|bool and not download.container|bool and (download.unarchive is not defined or download.unarchive == False) }}" + +- name: pulling... + debug: + msg: "{{ download.repo }}:{{ download.tag }}" + when: "{{ download.enabled|bool and download.container|bool }}" + +- name: Create dest directory for saved/loaded container images + file: path="{{local_release_dir}}/containers" state=directory recurse=yes + when: "{{ download.enabled|bool and download.container|bool }}" + +#NOTE(bogdando) this brings no docker-py deps for nodes +- name: Download containers + command: "/usr/bin/docker pull {{ download.repo }}:{{ download.tag }}" + register: pull_task_result + until: pull_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: "{{ download.enabled|bool and download.container|bool }}" + delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else inventory_hostname }}" run_once: "{{ download_run_once|bool }}" + +- set_fact: + fname: "{{local_release_dir}}/containers/{{download.repo|regex_replace('/|\0|:', '_')}}:{{download.tag|regex_replace('/|\0|:', '_')}}.tar" + +- name: "Set default value for 'container_changed' to false" + set_fact: + container_changed: false + +- name: "Update the 'container_changed' fact" + set_fact: + container_changed: "{{ not 'up to date' in pull_task_result.stdout }}" + when: "{{ download.enabled|bool and download.container|bool }}" + delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else inventory_hostname }}" + run_once: "{{ download_run_once|bool }}" + +- name: Download | save container images + shell: docker save "{{ download.repo }}:{{ download.tag }}" > "{{ fname }}" + delegate_to: "{{groups['kube-master'][0]}}" + run_once: true + when: ansible_os_family != "CoreOS" and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool + +- name: Download | get container images + synchronize: + src: "{{ fname }}" + dest: "{{local_release_dir}}/containers" + mode: push + register: get_task + until: get_task|success + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool + +- name: Download | load container images + shell: docker load < "{{ fname }}" + when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool and download.enabled|bool and download.container|bool and container_changed|bool diff --git a/roles/download/vars/kube_versions.yml b/roles/download/vars/kube_versions.yml deleted file mode 100644 index 5b5f64a42..000000000 --- a/roles/download/vars/kube_versions.yml +++ /dev/null @@ -1,22 +0,0 @@ -kube_checksum: - v1.2.2: - kube_apiserver: eb1bfd8b877052cbd1991b8c429a1d06661f4cb019905e20e128174f724e16de - kubectl: 473e6924569fba30d4a50cecdc2cae5f31d97d1f662463e85b74a472105dcff4 - kubelet: f16827dc7e7c82f0e215f0fc73eb01e2dfe91a2ec83f9cbcaf8d37c91b64fd3b - v1.2.3: - kube_apiserver_checksum: ebaeeeb72cb29b358337b330617a96355ff2d08a5a523fc1a81beba36cc9d6f9 - kubectl_checksum: 394853edd409a721bcafe4f1360009ef9f845050719fe7d6fc7176f45cc92a8c - kubelet_checksum: 633bb41c51c5c0df0645dd60ba82b12eba39d009eb87bae9227de7d9a89c0797 - v1.2.4: - kube_apiserver: 6ac99b36b02968459e026fcfc234207c66064b5e11816b69dd8fc234b2ffec1e - kubectl: dac61fbd506f7a17540feca691cd8a9d9d628d59661eebce788a50511f578897 - kubelet: 4adaf40592248eef6fd4fa126464915ea41e624a70dc77178089760ed235e341 - v1.2.5: - kube_apiserver: fbe8296ad4b194c06f6802a126d35cd2887dc1aded308d4da2b580f270412b33 - kubectl: 5526a496a84701015485e32c86486e2f23599f7a865164f546e619c6a62f7f19 - kubelet: cd15b929f0190876216f397c2c6e7aa8c08d3b047fd90b4980cd68c8f4896211 - v1.3.0: - kube_apiserver: 431cd312984a29f45590138e990d5c4d537b069b71f2587a72414fabc4fcffdd - kubectl: f40b2d0ff33984e663a0dea4916f1cb9041abecc09b11f9372cdb8049ded95dc - kubelet: bd5f10ccb95fe6e95ddf7ad8a119195c27cb2bce4be6f80c1810ff1a2111496d -kube_version: v1.3.0 diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 63f485719..02234a2fe 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -1,10 +1,2 @@ --- -etcd_version: v3.0.1 etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/" - -# Possible values: host, docker -etcd_deployment_type: "host" - - -etcd_image_repo: "quay.io/coreos/etcd" -etcd_image_tag: "{{ etcd_version }}" diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml index 334a4c396..693754a06 100644 --- a/roles/etcd/handlers/main.yml +++ b/roles/etcd/handlers/main.yml @@ -2,21 +2,36 @@ - name: restart etcd command: /bin/true notify: - - reload systemd - - start etcd + - etcd | reload systemd - reload etcd + - wait for etcd up -- name: reload systemd +- name: restart etcd-proxy + command: /bin/true + notify: + - etcd | reload systemd + - reload etcd-proxy + - wait for etcd up + +- name: etcd | reload systemd command: systemctl daemon-reload when: ansible_service_mgr == "systemd" -- name: start etcd - service: - name: etcd - enabled: yes - state: started +- name: wait for etcd up + uri: url="http://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health" + register: result + until: result.status == 200 + retries: 10 + delay: 5 - name: reload etcd service: name: etcd - state: "{{ 'restarted' if etcd_deployment_type == 'host' else 'reloaded' }}" + state: restarted + when: is_etcd_master + +- name: reload etcd-proxy + service: + name: etcd-proxy + state: restarted + when: is_etcd_proxy diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index 84d029d58..b55966a99 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -3,8 +3,7 @@ dependencies: - role: adduser user: "{{ addusers.etcd }}" when: ansible_os_family != 'CoreOS' + - role: docker + when: (ansible_os_family != "CoreOS" and etcd_deployment_type == "docker" or inventory_hostname in groups['k8s-cluster']) - role: download file: "{{ downloads.etcd }}" - when: etcd_deployment_type == "host" - - role: docker - when: (ansible_os_family != "CoreOS" and etcd_deployment_type == "docker") diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml index 818559239..514a79d73 100644 --- a/roles/etcd/tasks/configure.yml +++ b/roles/etcd/tasks/configure.yml @@ -1,6 +1,6 @@ --- - name: Configure | Check if member is in cluster - shell: "etcdctl --peers={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}" + shell: "{{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}" register: etcd_member_in_cluster ignore_errors: true changed_when: false @@ -8,14 +8,14 @@ - name: Configure | Add member to the cluster if it is not there when: is_etcd_master and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0 - shell: "etcdctl --peers={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}" + shell: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}" - name: Configure | Copy etcd.service systemd file template: src: "etcd-{{ etcd_deployment_type }}.service.j2" dest: /etc/systemd/system/etcd.service backup: yes - when: ansible_service_mgr == "systemd" + when: ansible_service_mgr == "systemd" and is_etcd_master notify: restart etcd - name: Configure | Write etcd initd script @@ -24,5 +24,21 @@ dest: /etc/init.d/etcd owner: root mode: 0755 - when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" + when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" and is_etcd_master notify: restart etcd + +- name: Configure | Copy etcd-proxy.service systemd file + template: + src: "etcd-proxy-{{ etcd_deployment_type }}.service.j2" + dest: /etc/systemd/system/etcd-proxy.service + backup: yes + when: ansible_service_mgr == "systemd" and is_etcd_proxy + notify: restart etcd-proxy +- name: Configure | Write etcd-proxy initd script + template: + src: "deb-etcd-proxy-{{ etcd_deployment_type }}.initd.j2" + dest: /etc/init.d/etcd-proxy + owner: root + mode: 0755 + when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" and is_etcd_proxy + notify: restart etcd-proxy diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install.yml index 26271943a..aa7f32ca3 100644 --- a/roles/etcd/tasks/install.yml +++ b/roles/etcd/tasks/install.yml @@ -17,6 +17,10 @@ /usr/bin/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl && /usr/bin/docker rm -f etcdctl-binarycopy" when: etcd_deployment_type == "docker" + register: etcd_task_result + until: etcd_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" changed_when: false #Plan B: looks nicer, but requires docker-py on all hosts: diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index 060d3708b..3ecaa00e6 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -1,23 +1,35 @@ --- -- include: set_facts.yml - include: install.yml - include: set_cluster_health.yml - include: configure.yml - include: refresh_config.yml -- name: Restart etcd if binary changed - command: /bin/true - notify: restart etcd - when: etcd_deployment_type == "host" and etcd_copy.stdout_lines - -# Reload systemd before starting service -- meta: flush_handlers - - name: Ensure etcd is running service: name: etcd state: started enabled: yes + when: is_etcd_master + +- name: Ensure etcd-proxy is running + service: + name: etcd-proxy + state: started + enabled: yes + when: is_etcd_proxy + +- name: Restart etcd if binary changed + command: /bin/true + notify: restart etcd + when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_master + +- name: Restart etcd-proxy if binary changed + command: /bin/true + notify: restart etcd-proxy + when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_proxy + +# Reload systemd before starting service +- meta: flush_handlers # After etcd cluster is assembled, make sure that # initial state of the cluster is in `existing` diff --git a/roles/etcd/tasks/refresh_config.yml b/roles/etcd/tasks/refresh_config.yml index 701a1d149..178466153 100644 --- a/roles/etcd/tasks/refresh_config.yml +++ b/roles/etcd/tasks/refresh_config.yml @@ -4,3 +4,11 @@ src: etcd.j2 dest: /etc/etcd.env notify: restart etcd + when: is_etcd_master + +- name: Refresh config | Create etcd-proxy config file + template: + src: etcd-proxy.j2 + dest: /etc/etcd-proxy.env + notify: restart etcd-proxy + when: is_etcd_proxy diff --git a/roles/etcd/tasks/set_cluster_health.yml b/roles/etcd/tasks/set_cluster_health.yml index be0d938dd..1a27e4dcf 100644 --- a/roles/etcd/tasks/set_cluster_health.yml +++ b/roles/etcd/tasks/set_cluster_health.yml @@ -1,6 +1,6 @@ --- - name: Configure | Check if cluster is healthy - shell: "etcdctl --peers={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" + shell: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" register: etcd_cluster_is_healthy ignore_errors: true changed_when: false diff --git a/roles/etcd/tasks/set_facts.yml b/roles/etcd/tasks/set_facts.yml deleted file mode 100644 index 415ed85c6..000000000 --- a/roles/etcd/tasks/set_facts.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -- set_fact: etcd_access_address="{{ access_ip | default(ip | default(ansible_default_ipv4['address'])) }}" -- set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380" -- set_fact: etcd_client_url="http://{{ etcd_access_address }}:2379" -- set_fact: - etcd_access_addresses: |- - {% for item in groups['etcd'] -%} - http://{{ hostvars[item].etcd_access_address }}:2379{% if not loop.last %},{% endif %} - {%- endfor %} -- set_fact: - etcd_member_name: |- - {% for host in groups['etcd'] %} - {% if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %} - {% endfor %} -- set_fact: - is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}" -- set_fact: - etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=") -- set_fact: - etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" diff --git a/roles/etcd/templates/deb-etcd-docker.initd.j2 b/roles/etcd/templates/deb-etcd-docker.initd.j2 index 6e5a8cb03..a83aae184 100644 --- a/roles/etcd/templates/deb-etcd-docker.initd.j2 +++ b/roles/etcd/templates/deb-etcd-docker.initd.j2 @@ -16,34 +16,25 @@ PATH=/sbin:/usr/sbin:/bin/:/usr/bin DESC="etcd k/v store" NAME=etcd DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker -{% if is_etcd_master %} -DAEMON_ARGS='--restart=always --env-file=/etc/etcd.env \ +DAEMON_EXEC=`basename $DAEMON` +DAEMON_ARGS="run --restart=always --env-file=/etc/etcd.env \ --net=host \ -v /usr/share/ca-certificates/:/etc/ssl/certs:ro \ -v /var/lib/etcd:/var/lib/etcd:rw \ ---name={{ etcd_member_name | default("etcd-proxy") }} \ +--name={{ etcd_member_name | default("etcd") }} \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ {% if etcd_after_v3 %} -{{ etcd_container_bin_dir }}etcd \ -{% endif %} -{% if is_etcd_master %} ---proxy off -{% else %} ---proxy on -{% endif %}' - +{{ etcd_container_bin_dir }}etcd +{% endif %}" SCRIPTNAME=/etc/init.d/$NAME -DAEMON_USER=etcd +DAEMON_USER=root STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}" PID=/var/run/etcd.pid # Exit if the binary is not present [ -x "$DAEMON" ] || exit 0 -# Read configuration variable file if it is present -[ -f /etc/etcd.env ] && . /etc/etcd.env - # Define LSB log_* functions. # Depend on lsb-base (>= 3.2-14) to ensure that this file is present # and status_of_proc is working. @@ -58,6 +49,8 @@ do_status() # do_start() { + {{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_member_name | default("etcd-proxy") }} &>/dev/null || true + sleep 1 start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON -- \ $DAEMON_ARGS \ || return 2 @@ -68,7 +61,7 @@ do_start() # do_stop() { - start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $NAME + start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $DAEMON_EXEC RETVAL="$?" sleep 1 diff --git a/roles/etcd/templates/deb-etcd-host.initd.j2 b/roles/etcd/templates/deb-etcd-host.initd.j2 index ccecdce4e..b27c0f49a 100644 --- a/roles/etcd/templates/deb-etcd-host.initd.j2 +++ b/roles/etcd/templates/deb-etcd-host.initd.j2 @@ -16,11 +16,6 @@ PATH=/sbin:/usr/sbin:/bin:/usr/bin DESC="etcd k/v store" NAME=etcd DAEMON={{ bin_dir }}/etcd -{% if is_etcd_master %} -DAEMON_ARGS="" -{% else %} -DAEMON_ARGS="--proxy on" -{% endif %} SCRIPTNAME=/etc/init.d/$NAME DAEMON_USER=etcd STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}" diff --git a/roles/etcd/templates/deb-etcd-proxy-docker.initd.j2 b/roles/etcd/templates/deb-etcd-proxy-docker.initd.j2 new file mode 100644 index 000000000..ad0338a09 --- /dev/null +++ b/roles/etcd/templates/deb-etcd-proxy-docker.initd.j2 @@ -0,0 +1,120 @@ +#!/bin/sh +set -a + +### BEGIN INIT INFO +# Provides: etcd-proxy +# Required-Start: $local_fs $network $syslog +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: etcd-proxy +# Description: +# etcd-proxy is a proxy for etcd: distributed, consistent key-value store for shared configuration and service discovery +### END INIT INFO + +PATH=/sbin:/usr/sbin:/bin/:/usr/bin +DESC="etcd-proxy" +NAME=etcd-proxy +DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker +DAEMON_EXEC=`basename $DAEMON` +DAEMON_ARGS="run --restart=always --env-file=/etc/etcd-proxy.env \ +--net=host \ +--stop-signal=SIGKILL \ +-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \ +--name={{ etcd_proxy_member_name | default("etcd-proxy") }} \ +{{ etcd_image_repo }}:{{ etcd_image_tag }} \ +{% if etcd_after_v3 %} +{{ etcd_container_bin_dir }}etcd +{% endif %}" + + +SCRIPTNAME=/etc/init.d/$NAME +DAEMON_USER=root +STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}" +PID=/var/run/etcd-proxy.pid + +# Exit if the binary is not present +[ -x "$DAEMON" ] || exit 0 + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +do_status() +{ + status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $? +} + +# Function that starts the daemon/service +# +do_start() +{ + {{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_proxy_member_name | default("etcd-proxy") }} &>/dev/null || true + sleep 1 + start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $DAEMON_EXEC + RETVAL="$?" + + sleep 1 + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + if do_stop; then + log_end_msg 0 + else + log_failure_msg "Can't stop etcd-proxy" + log_end_msg 1 + fi + ;; + status) + if do_status; then + log_end_msg 0 + else + log_failure_msg "etcd-proxy is not running" + log_end_msg 1 + fi + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + if do_stop; then + if do_start; then + log_end_msg 0 + exit 0 + else + rc="$?" + fi + else + rc="$?" + fi + log_failure_msg "Can't restart etcd-proxy" + log_end_msg ${rc} + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + diff --git a/roles/etcd/templates/deb-etcd-proxy-host.initd.j2 b/roles/etcd/templates/deb-etcd-proxy-host.initd.j2 new file mode 100644 index 000000000..d0858bb2f --- /dev/null +++ b/roles/etcd/templates/deb-etcd-proxy-host.initd.j2 @@ -0,0 +1,110 @@ +#!/bin/sh +set -a + +### BEGIN INIT INFO +# Provides: etcd-proxy +# Required-Start: $local_fs $network $syslog +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: etcd-proxy +# Description: +# etcd-proxy is a proxy for etcd: distributed, consistent key-value store for shared configuration and service discovery +### END INIT INFO + +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="etcd-proxy" +NAME=etcd-proxy +DAEMON={{ bin_dir }}/etcd +DAEMON_ARGS="" +SCRIPTNAME=/etc/init.d/$NAME +DAEMON_USER=etcd +STOP_SCHEDULE="${STOP_SCHEDULE:-QUIT/5/TERM/5/KILL/5}" +PID=/var/run/etcd-proxy.pid + +# Exit if the binary is not present +[ -x "$DAEMON" ] || exit 0 + +# Read configuration variable file if it is present +[ -f /etc/etcd-proxy.env ] && . /etc/etcd-proxy.env + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +do_status() +{ + status_of_proc -p $PID "$DAEMON" "$NAME" && exit 0 || exit $? +} + +# Function that starts the daemon/service +# +do_start() +{ + start-stop-daemon --background --start --quiet --make-pidfile --pidfile $PID --user $DAEMON_USER --exec $DAEMON -- \ + $DAEMON_ARGS \ + || return 2 +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + start-stop-daemon --stop --quiet --retry=$STOP_SCHEDULE --pidfile $PID --name $NAME + RETVAL="$?" + + sleep 1 + return "$RETVAL" +} + + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + if do_stop; then + log_end_msg 0 + else + log_failure_msg "Can't stop etcd-proxy" + log_end_msg 1 + fi + ;; + status) + if do_status; then + log_end_msg 0 + else + log_failure_msg "etcd-proxy is not running" + log_end_msg 1 + fi + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + if do_stop; then + if do_start; then + log_end_msg 0 + exit 0 + else + rc="$?" + fi + else + rc="$?" + fi + log_failure_msg "Can't restart etcd-proxy" + log_end_msg ${rc} + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac + diff --git a/roles/etcd/templates/etcd-docker.service.j2 b/roles/etcd/templates/etcd-docker.service.j2 index 623ec70b9..a37759fec 100644 --- a/roles/etcd/templates/etcd-docker.service.j2 +++ b/roles/etcd/templates/etcd-docker.service.j2 @@ -1,6 +1,6 @@ [Unit] Description=etcd docker wrapper -Requires=docker.service +Wants=docker.socket After=docker.service [Service] @@ -13,21 +13,16 @@ ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always --net=host \ -v /usr/share/ca-certificates/:/etc/ssl/certs:ro \ -v /var/lib/etcd:/var/lib/etcd:rw \ ---name={{ etcd_member_name | default("etcd-proxy") }} \ +--name={{ etcd_member_name | default("etcd") }} \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ {% if etcd_after_v3 %} -{{ etcd_container_bin_dir }}etcd \ -{% endif %} -{% if is_etcd_master %} ---proxy off -{% else %} ---proxy on +{{ etcd_container_bin_dir }}etcd {% endif %} ExecStartPre=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_member_name | default("etcd-proxy") }} ExecReload={{ docker_bin_dir | default("/usr/bin") }}/docker restart {{ etcd_member_name | default("etcd-proxy") }} ExecStop={{ docker_bin_dir | default("/usr/bin") }}/docker stop {{ etcd_member_name | default("etcd-proxy") }} Restart=always -RestartSec=10s +RestartSec=15s [Install] WantedBy=multi-user.target diff --git a/roles/etcd/templates/etcd-host.service.j2 b/roles/etcd/templates/etcd-host.service.j2 index 19cc0386e..8a91fab92 100644 --- a/roles/etcd/templates/etcd-host.service.j2 +++ b/roles/etcd/templates/etcd-host.service.j2 @@ -1,15 +1,13 @@ [Unit] Description=etcd - +After=network.target [Service] +Type=notify User=etcd EnvironmentFile=/etc/etcd.env -{% if inventory_hostname in groups['etcd'] %} ExecStart={{ bin_dir }}/etcd -{% else %} -ExecStart={{ bin_dir }}/etcd -proxy on -{% endif %} +NotifyAccess=all Restart=always RestartSec=10s LimitNOFILE=40000 diff --git a/roles/etcd/templates/etcd-proxy-docker.service.j2 b/roles/etcd/templates/etcd-proxy-docker.service.j2 new file mode 100644 index 000000000..bf70f0e7f --- /dev/null +++ b/roles/etcd/templates/etcd-proxy-docker.service.j2 @@ -0,0 +1,28 @@ +[Unit] +Description=etcd-proxy docker wrapper +Wants=docker.socket +After=docker.service + +[Service] +User=root +PermissionsStartOnly=true +ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always \ +--env-file=/etc/etcd-proxy.env \ +{# TODO(mattymo): Allow docker IP binding and disable in envfile + -p 2380:2380 -p 2379:2379 #} +--net=host \ +--stop-signal=SIGKILL \ +-v /usr/share/ca-certificates/:/etc/ssl/certs:ro \ +--name={{ etcd_proxy_member_name | default("etcd-proxy") }} \ +{{ etcd_image_repo }}:{{ etcd_image_tag }} \ +{% if etcd_after_v3 %} +{{ etcd_container_bin_dir }}etcd +{% endif %} +ExecStartPre=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_proxy_member_name | default("etcd-proxy") }} +ExecReload={{ docker_bin_dir | default("/usr/bin") }}/docker restart {{ etcd_proxy_member_name | default("etcd-proxy") }} +ExecStop={{ docker_bin_dir | default("/usr/bin") }}/docker stop {{ etcd_proxy_member_name | default("etcd-proxy") }} +Restart=always +RestartSec=15s + +[Install] +WantedBy=multi-user.target diff --git a/roles/etcd/templates/etcd-proxy-host.service.j2 b/roles/etcd/templates/etcd-proxy-host.service.j2 new file mode 100644 index 000000000..4ea5f7bc9 --- /dev/null +++ b/roles/etcd/templates/etcd-proxy-host.service.j2 @@ -0,0 +1,19 @@ +[Unit] +Description=etcd-proxy +After=network.target + +[Service] +Type=notify +User=etcd +PermissionsStartOnly=true +EnvironmentFile=/etc/etcd-proxy.env +ExecStart={{ bin_dir }}/etcd +ExecStartPre=/bin/mkdir -p /var/lib/etcd-proxy +ExecStartPre=/bin/chown -R etcd: /var/lib/etcd-proxy +NotifyAccess=all +Restart=always +RestartSec=10s +LimitNOFILE=40000 + +[Install] +WantedBy=multi-user.target diff --git a/roles/etcd/templates/etcd-proxy.j2 b/roles/etcd/templates/etcd-proxy.j2 new file mode 100644 index 000000000..0a1492a37 --- /dev/null +++ b/roles/etcd/templates/etcd-proxy.j2 @@ -0,0 +1,5 @@ +ETCD_DATA_DIR=/var/lib/etcd-proxy +ETCD_PROXY=on +ETCD_LISTEN_CLIENT_URLS={{ etcd_access_endpoint }} +ETCD_NAME={{ etcd_proxy_member_name | default("etcd-proxy") }} +ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }} diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 index 94d9e8121..b82116612 100644 --- a/roles/etcd/templates/etcd.j2 +++ b/roles/etcd/templates/etcd.j2 @@ -1,14 +1,16 @@ ETCD_DATA_DIR=/var/lib/etcd -{% if is_etcd_master %} -ETCD_ADVERTISE_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2379 -ETCD_INITIAL_ADVERTISE_PEER_URLS=http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2380 +ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }} +ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }} ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %} +{% if not is_etcd_proxy %} +ETCD_LISTEN_CLIENT_URLS=http://{{ etcd_address }}:2379,http://127.0.0.1:2379 +{% else %} +ETCD_LISTEN_CLIENT_URLS=http://{{ etcd_address }}:2379 +{% endif %} ETCD_ELECTION_TIMEOUT=10000 ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd -ETCD_LISTEN_PEER_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:2380 +ETCD_LISTEN_PEER_URLS=http://{{ etcd_address }}:2380 ETCD_NAME={{ etcd_member_name }} -{% endif %} -ETCD_INITIAL_CLUSTER={% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %} - -ETCD_LISTEN_CLIENT_URLS=http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}:2379,http://127.0.0.1:2379 +ETCD_PROXY=off +ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }} diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml new file mode 100644 index 000000000..b1086aa0d --- /dev/null +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -0,0 +1,12 @@ +# Versions +kubedns_version: 1.7 +kubednsmasq_version: 1.3 +exechealthz_version: 1.1 + +# Images +kubedns_image_repo: "gcr.io/google_containers/kubedns-amd64" +kubedns_image_tag: "{{ kubedns_version }}" +kubednsmasq_image_repo: "gcr.io/google_containers/kube-dnsmasq-amd64" +kubednsmasq_image_tag: "{{ kubednsmasq_version }}" +exechealthz_image_repo: "gcr.io/google_containers/exechealthz-amd64" +exechealthz_image_tag: "{{ exechealthz_version }}" \ No newline at end of file diff --git a/roles/kubernetes-apps/ansible/library/kube.py b/roles/kubernetes-apps/ansible/library/kube.py new file mode 100644 index 000000000..2922c6212 --- /dev/null +++ b/roles/kubernetes-apps/ansible/library/kube.py @@ -0,0 +1,305 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +DOCUMENTATION = """ +--- +module: kube +short_description: Manage Kubernetes Cluster +description: + - Create, replace, remove, and stop resources within a Kubernetes Cluster +version_added: "2.0" +options: + name: + required: false + default: null + description: + - The name associated with resource + filename: + required: false + default: null + description: + - The path and filename of the resource(s) definition file. + kubectl: + required: false + default: null + description: + - The path to the kubectl bin + namespace: + required: false + default: null + description: + - The namespace associated with the resource(s) + resource: + required: false + default: null + description: + - The resource to perform an action on. pods (po), replicationControllers (rc), services (svc) + label: + required: false + default: null + description: + - The labels used to filter specific resources. + server: + required: false + default: null + description: + - The url for the API server that commands are executed against. + force: + required: false + default: false + description: + - A flag to indicate to force delete, replace, or stop. + all: + required: false + default: false + description: + - A flag to indicate delete all, stop all, or all namespaces when checking exists. + log_level: + required: false + default: 0 + description: + - Indicates the level of verbosity of logging by kubectl. + state: + required: false + choices: ['present', 'absent', 'latest', 'reloaded', 'stopped'] + default: present + description: + - present handles checking existence or creating if definition file provided, + absent handles deleting resource(s) based on other options, + latest handles creating ore updating based on existence, + reloaded handles updating resource(s) definition using definition file, + stopped handles stopping resource(s) based on other options. +requirements: + - kubectl +author: "Kenny Jones (@kenjones-cisco)" +""" + +EXAMPLES = """ +- name: test nginx is present + kube: name=nginx resource=rc state=present + +- name: test nginx is stopped + kube: name=nginx resource=rc state=stopped + +- name: test nginx is absent + kube: name=nginx resource=rc state=absent + +- name: test nginx is present + kube: filename=/tmp/nginx.yml +""" + + +class KubeManager(object): + + def __init__(self, module): + + self.module = module + + self.kubectl = module.params.get('kubectl') + if self.kubectl is None: + self.kubectl = module.get_bin_path('kubectl', True) + self.base_cmd = [self.kubectl] + + if module.params.get('server'): + self.base_cmd.append('--server=' + module.params.get('server')) + + if module.params.get('log_level'): + self.base_cmd.append('--v=' + str(module.params.get('log_level'))) + + if module.params.get('namespace'): + self.base_cmd.append('--namespace=' + module.params.get('namespace')) + + self.all = module.params.get('all') + self.force = module.params.get('force') + self.name = module.params.get('name') + self.filename = module.params.get('filename') + self.resource = module.params.get('resource') + self.label = module.params.get('label') + + def _execute(self, cmd): + args = self.base_cmd + cmd + try: + rc, out, err = self.module.run_command(args) + if rc != 0: + self.module.fail_json( + msg='error running kubectl (%s) command (rc=%d): %s' % (' '.join(args), rc, out or err)) + except Exception as exc: + self.module.fail_json( + msg='error running kubectl (%s) command: %s' % (' '.join(args), str(exc))) + return out.splitlines() + + def _execute_nofail(self, cmd): + args = self.base_cmd + cmd + rc, out, err = self.module.run_command(args) + if rc != 0: + return None + return out.splitlines() + + def create(self, check=True): + if check and self.exists(): + return [] + + cmd = ['create'] + + if not self.filename: + self.module.fail_json(msg='filename required to create') + + cmd.append('--filename=' + self.filename) + + return self._execute(cmd) + + def replace(self): + + if not self.force and not self.exists(): + return [] + + cmd = ['replace'] + + if self.force: + cmd.append('--force') + + if not self.filename: + self.module.fail_json(msg='filename required to reload') + + cmd.append('--filename=' + self.filename) + + return self._execute(cmd) + + def delete(self): + + if not self.force and not self.exists(): + return [] + + cmd = ['delete'] + + if self.filename: + cmd.append('--filename=' + self.filename) + else: + if not self.resource: + self.module.fail_json(msg='resource required to delete without filename') + + cmd.append(self.resource) + + if self.name: + cmd.append(self.name) + + if self.label: + cmd.append('--selector=' + self.label) + + if self.all: + cmd.append('--all') + + if self.force: + cmd.append('--ignore-not-found') + + return self._execute(cmd) + + def exists(self): + cmd = ['get'] + + if not self.resource: + return False + + cmd.append(self.resource) + + if self.name: + cmd.append(self.name) + + cmd.append('--no-headers') + + if self.label: + cmd.append('--selector=' + self.label) + + if self.all: + cmd.append('--all-namespaces') + + result = self._execute_nofail(cmd) + if not result: + return False + return True + + def stop(self): + + if not self.force and not self.exists(): + return [] + + cmd = ['stop'] + + if self.filename: + cmd.append('--filename=' + self.filename) + else: + if not self.resource: + self.module.fail_json(msg='resource required to stop without filename') + + cmd.append(self.resource) + + if self.name: + cmd.append(self.name) + + if self.label: + cmd.append('--selector=' + self.label) + + if self.all: + cmd.append('--all') + + if self.force: + cmd.append('--ignore-not-found') + + return self._execute(cmd) + + +def main(): + + module = AnsibleModule( + argument_spec=dict( + name=dict(), + filename=dict(), + namespace=dict(), + resource=dict(), + label=dict(), + server=dict(), + kubectl=dict(), + force=dict(default=False, type='bool'), + all=dict(default=False, type='bool'), + log_level=dict(default=0, type='int'), + state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped']), + ) + ) + + changed = False + + manager = KubeManager(module) + state = module.params.get('state') + + if state == 'present': + result = manager.create() + + elif state == 'absent': + result = manager.delete() + + elif state == 'reloaded': + result = manager.replace() + + elif state == 'stopped': + result = manager.stop() + + elif state == 'latest': + if manager.exists(): + manager.force = True + result = manager.replace() + else: + result = manager.create(check=False) + + else: + module.fail_json(msg='Unrecognized state %s.' % state) + + if result: + changed = True + module.exit_json(changed=changed, + msg='success: %s' % (' '.join(result)) + ) + + +from ansible.module_utils.basic import * # noqa +if __name__ == '__main__': + main() diff --git a/roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml b/roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml new file mode 100644 index 000000000..f4ac65aeb --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/calico-policy-controller.yml @@ -0,0 +1,10 @@ +- name: Write calico-policy-controller yaml + template: src=calico-policy-controller.yml.j2 dest=/etc/kubernetes/calico-policy-controller.yml + when: inventory_hostname == groups['kube-master'][0] + + +- name: Start of Calico policy controller + kube: + kubectl: "{{bin_dir}}/kubectl" + filename: /etc/kubernetes/calico-policy-controller.yml + when: inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ansible/tasks/main.yaml b/roles/kubernetes-apps/ansible/tasks/main.yaml new file mode 100644 index 000000000..f31eb442b --- /dev/null +++ b/roles/kubernetes-apps/ansible/tasks/main.yaml @@ -0,0 +1,23 @@ +--- +- name: Kubernetes Apps | Lay Down KubeDNS Template + template: src={{item.file}} dest=/etc/kubernetes/{{item.file}} + with_items: + - {file: kubedns-rc.yml, type: rc} + - {file: kubedns-svc.yml, type: svc} + register: manifests + when: inventory_hostname == groups['kube-master'][0] + +- name: Kubernetes Apps | Start Resources + kube: + name: kubedns + namespace: kube-system + kubectl: "{{bin_dir}}/kubectl" + resource: "{{item.item.type}}" + filename: /etc/kubernetes/{{item.item.file}} + state: "{{item.changed | ternary('latest','present') }}" + with_items: "{{ manifests.results }}" + when: inventory_hostname == groups['kube-master'][0] + + +- include: tasks/calico-policy-controller.yml + when: enable_network_policy is defined and enable_network_policy == True diff --git a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 new file mode 100644 index 000000000..7c0a21cfa --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 @@ -0,0 +1,40 @@ +apiVersion: extensions/v1beta1 +kind: ReplicaSet +metadata: + name: calico-policy-controller + namespace: kube-system + labels: + k8s-app: calico-policy + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + matchLabels: + kubernetes.io/cluster-service: "true" + k8s-app: calico-policy + template: + metadata: + name: calico-policy-controller + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + k8s-app: calico-policy + spec: + hostNetwork: true + containers: + - name: calico-policy-controller + image: calico/kube-policy-controller:latest + env: + - name: ETCD_ENDPOINTS + value: "{{ etcd_endpoint }}" + # Location of the Kubernetes API - this shouldn't need to be + # changed so long as it is used in conjunction with + # CONFIGURE_ETC_HOSTS="true". + - name: K8S_API + value: "https://kubernetes.default:443" + # Configure /etc/hosts within the container to resolve + # the kubernetes.default Service to the correct clusterIP + # using the environment provided by the kubelet. + # This removes the need for KubeDNS to resolve the Service. + - name: CONFIGURE_ETC_HOSTS + value: "true" diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml b/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml new file mode 100644 index 000000000..ed38d671d --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml @@ -0,0 +1,100 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: kubedns + namespace: kube-system + labels: + k8s-app: kubedns + version: v19 + kubernetes.io/cluster-service: "true" +spec: + replicas: 1 + selector: + k8s-app: kubedns + version: v19 + template: + metadata: + labels: + k8s-app: kubedns + version: v19 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: kubedns + image: "{{ kubedns_image_repo }}:{{ kubedns_image_tag }}" + resources: + # TODO: Set memory limits when we've profiled the container for large + # clusters, then set request = limit to keep this container in + # guaranteed class. Currently, this container falls into the + # "burstable" category so the kubelet doesn't backoff from restarting it. + limits: + cpu: 100m + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + readinessProbe: + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + # we poll on pod startup for the Kubernetes master service and + # only setup the /readiness HTTP server once that's available. + initialDelaySeconds: 30 + timeoutSeconds: 5 + args: + # command = "/kube-dns" + - --domain={{ dns_domain }}. + - --dns-port=10053 + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - name: dnsmasq + image: "{{ kubednsmasq_image_repo }}:{{ kubednsmasq_image_tag }}" + args: + - --log-facility=- + - --cache-size=1000 + - --no-resolv + - --server=127.0.0.1#10053 + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - name: healthz + image: "{{ exechealthz_image_repo }}:{{ exechealthz_image_tag }}" + resources: + # keep request = limit to keep this container in guaranteed class + limits: + cpu: 10m + memory: 50Mi + requests: + cpu: 10m + # Note that this container shouldn't really need 50Mi of memory. The + # limits are set higher than expected pending investigation on #29688. + # The extra memory was stolen from the kubedns container to keep the + # net memory requested by the pod constant. + memory: 50Mi + args: + - -cmd=nslookup kubernetes.default.svc.{{ dns_domain }} 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc.{{ dns_domain }} 127.0.0.1:10053 >/dev/null + - -port=8080 + - -quiet + ports: + - containerPort: 8080 + protocol: TCP + dnsPolicy: Default # Don't use cluster DNS. diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-svc.yml b/roles/kubernetes-apps/ansible/templates/kubedns-svc.yml new file mode 100644 index 000000000..2e21bc9e6 --- /dev/null +++ b/roles/kubernetes-apps/ansible/templates/kubedns-svc.yml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubedns + namespace: kube-system + labels: + k8s-app: kubedns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "kubedns" +spec: + selector: + k8s-app: kubedns + clusterIP: {{ skydns_server }} + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/roles/kubernetes-apps/defaults/main.yaml b/roles/kubernetes-apps/defaults/main.yaml deleted file mode 100644 index d95001316..000000000 --- a/roles/kubernetes-apps/defaults/main.yaml +++ /dev/null @@ -1,4 +0,0 @@ ---- -kpm_registry: "https://api.kpm.sh" -kpm_namespace: "default" -kpm_packages: [] \ No newline at end of file diff --git a/roles/kubernetes-apps/helm/.gitkeep b/roles/kubernetes-apps/helm/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/roles/kubernetes-apps/helm/tasks/main.yml b/roles/kubernetes-apps/helm/tasks/main.yml new file mode 100644 index 000000000..9fd691ddd --- /dev/null +++ b/roles/kubernetes-apps/helm/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- debug: msg="No helm charts" diff --git a/library/kpm.py b/roles/kubernetes-apps/kpm/library/kpm.py similarity index 100% rename from library/kpm.py rename to roles/kubernetes-apps/kpm/library/kpm.py diff --git a/roles/kubernetes-apps/tasks/main.yaml b/roles/kubernetes-apps/kpm/tasks/main.yaml similarity index 74% rename from roles/kubernetes-apps/tasks/main.yaml rename to roles/kubernetes-apps/kpm/tasks/main.yaml index c30a73d24..7e88cc30d 100644 --- a/roles/kubernetes-apps/tasks/main.yaml +++ b/roles/kubernetes-apps/kpm/tasks/main.yaml @@ -1,14 +1,9 @@ -- name: Install pip - action: - module: "{{ ansible_pkg_mgr }}" - name: "python-pip" - state: latest - when: ansible_os_family != "CoreOS" and kpm_packages | length > 0 - +--- - name: install kpm pip: name: "kpm" - state: "latest" + state: "present" + version: "0.16.1" when: kpm_packages | length > 0 - name: manage kubernetes applications diff --git a/roles/kubernetes-apps/meta/main.yaml b/roles/kubernetes-apps/meta/main.yaml new file mode 100644 index 000000000..496517d72 --- /dev/null +++ b/roles/kubernetes-apps/meta/main.yaml @@ -0,0 +1,3 @@ +dependencies: + - {role: kubernetes-apps/ansible, tags: apps} + - {role: kubernetes-apps/kpm, tags: [apps, kpm]} diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml new file mode 100644 index 000000000..ee32ccf57 --- /dev/null +++ b/roles/kubernetes/master/defaults/main.yml @@ -0,0 +1,30 @@ +# This is where all the cert scripts and certs will be located +kube_cert_dir: "{{ kube_config_dir }}/ssl" + +# This is where all of the bearer tokens will be stored +kube_token_dir: "{{ kube_config_dir }}/tokens" + +# This is where to save basic auth file +kube_users_dir: "{{ kube_config_dir }}/users" + +# An experimental dev/test only dynamic volumes provisioner, +# for PetSets. Works for kube>=v1.3 only. +kube_hostpath_dynamic_provisioner: "false" + +# This is where you can drop yaml/json files and the kubelet will run those +# pods on startup +kube_manifest_dir: "{{ kube_config_dir }}/manifests" + +# This directory is where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernets. +# This puts them in a sane location. +# Editting this value will almost surely break something. Don't +# change it. Things like the systemd scripts are hard coded to +# look in here. Don't do it. +kube_config_dir: /etc/kubernetes + +# change to 0.0.0.0 to enable insecure access from anywhere (not recommended) +kube_apiserver_insecure_bind_address: 127.0.0.1 + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml index a4082887b..3d69cba7d 100644 --- a/roles/kubernetes/master/handlers/main.yml +++ b/roles/kubernetes/master/handlers/main.yml @@ -1,4 +1,44 @@ --- -- name: restart kube-apiserver - set_fact: - restart_apimaster: True +- name: Master | restart kubelet + command: /bin/true + notify: + - Master | reload systemd + - Master | reload kubelet + - Master | wait for master static pods + +- name: Master | wait for master static pods + command: /bin/true + notify: + - Master | wait for the apiserver to be running + - Master | wait for kube-scheduler + - Master | wait for kube-controller-manager + +- name: Master | reload systemd + command: systemctl daemon-reload + when: ansible_service_mgr == "systemd" + +- name: Master | reload kubelet + service: + name: kubelet + state: restarted + +- name: Master | wait for kube-scheduler + uri: url=http://localhost:10251/healthz + register: scheduler_result + until: scheduler_result.status == 200 + retries: 15 + delay: 5 + +- name: Master | wait for kube-controller-manager + uri: url=http://localhost:10252/healthz + register: controller_manager_result + until: controller_manager_result.status == 200 + retries: 15 + delay: 5 + +- name: Master | wait for the apiserver to be running + uri: url=http://localhost:8080/healthz + register: result + until: result.status == 200 + retries: 10 + delay: 6 diff --git a/roles/kubernetes/master/meta/main.yml b/roles/kubernetes/master/meta/main.yml index 11f02f99d..f4da42e39 100644 --- a/roles/kubernetes/master/meta/main.yml +++ b/roles/kubernetes/master/meta/main.yml @@ -1,8 +1,4 @@ --- dependencies: - role: download - file: "{{ downloads.kubernetes_kubectl }}" - - role: download - file: "{{ downloads.kubernetes_apiserver }}" - - { role: etcd } - - { role: kubernetes/node } + file: "{{ downloads.hyperkube }}" diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index deaf017f3..419be1f5a 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -1,53 +1,27 @@ --- +- include: pre-upgrade.yml + - name: Copy kubectl bash completion copy: src: kubectl_bash_completion.sh dest: /etc/bash_completion.d/kubectl.sh when: ansible_os_family in ["Debian","RedHat"] -- name: Copy kube-apiserver binary - command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kube-apiserver" "{{ bin_dir }}/kube-apiserver" - register: kube_apiserver_copy +- name: Copy kubectl from hyperkube container + command: "/usr/bin/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl" + register: kube_task_result + until: kube_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" changed_when: false -- name: Copy kubectl binary - command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kubectl" "{{ bin_dir }}/kubectl" - changed_when: false - -- name: install | Write kube-apiserver systemd init file +- name: Write kube-apiserver manifest template: - src: "kube-apiserver.service.j2" - dest: "/etc/systemd/system/kube-apiserver.service" - backup: yes - when: ansible_service_mgr == "systemd" - notify: restart kube-apiserver - -- name: install | Write kube-apiserver initd script - template: - src: "deb-kube-apiserver.initd.j2" - dest: "/etc/init.d/kube-apiserver" - owner: root - mode: 0755 - backup: yes - when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian" - -- name: Write kube-apiserver config file - template: - src: "kube-apiserver.j2" - dest: "{{ kube_config_dir }}/kube-apiserver.env" - backup: yes - notify: restart kube-apiserver - -- name: Allow apiserver to bind on both secure and insecure ports - shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver - changed_when: false + src: manifests/kube-apiserver.manifest.j2 + dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest" + notify: Master | wait for the apiserver to be running - meta: flush_handlers - -- include: start.yml - with_items: "{{ groups['kube-master'] }}" - when: "{{ hostvars[item].inventory_hostname == inventory_hostname }}" - # Create kube-system namespace - name: copy 'kube-system' namespace manifest copy: src=namespace.yml dest=/etc/kubernetes/kube-system-ns.yml @@ -61,29 +35,20 @@ failed_when: False run_once: yes -- name: wait for the apiserver to be running - wait_for: - port: "{{kube_apiserver_insecure_port}}" - timeout: 60 - - name: Create 'kube-system' namespace command: "{{ bin_dir }}/kubectl create -f /etc/kubernetes/kube-system-ns.yml" changed_when: False when: kubesystem|failed and inventory_hostname == groups['kube-master'][0] -# Write manifests +# Write other manifests - name: Write kube-controller-manager manifest template: src: manifests/kube-controller-manager.manifest.j2 dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest" + notify: Master | wait for kube-controller-manager - name: Write kube-scheduler manifest template: src: manifests/kube-scheduler.manifest.j2 dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest" - -- name: restart kubelet - service: - name: kubelet - state: restarted - changed_when: false + notify: Master | wait for kube-scheduler diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml new file mode 100644 index 000000000..3b9f26de1 --- /dev/null +++ b/roles/kubernetes/master/tasks/pre-upgrade.yml @@ -0,0 +1,25 @@ +--- +- name: "Pre-upgrade | check for kube-apiserver unit file" + stat: + path: /etc/systemd/system/kube-apiserver.service + register: kube_apiserver_service_file + +- name: "Pre-upgrade | check for kube-apiserver init script" + stat: + path: /etc/init.d/kube-apiserver + register: kube_apiserver_init_script + +- name: "Pre-upgrade | stop kube-apiserver if service defined" + service: + name: kube-apiserver + state: stopped + when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False)) + +- name: "Pre-upgrade | remove kube-apiserver service definition" + file: + path: "{{ item }}" + state: absent + when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False)) + with_items: + - /etc/systemd/system/kube-apiserver.service + - /etc/init.d/kube-apiserver diff --git a/roles/kubernetes/master/tasks/start.yml b/roles/kubernetes/master/tasks/start.yml deleted file mode 100644 index 9cd247c42..000000000 --- a/roles/kubernetes/master/tasks/start.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Pause - pause: seconds=10 - -- name: reload systemd - command: systemctl daemon-reload - when: ansible_service_mgr == "systemd" and restart_apimaster is defined and restart_apimaster == True - -- name: reload kube-apiserver - service: - name: kube-apiserver - state: restarted - enabled: yes - when: ( restart_apimaster is defined and restart_apimaster == True) or - secret_changed | default(false) - -- name: Enable apiserver - service: - name: kube-apiserver - enabled: yes - state: started - when: restart_apimaster is not defined or restart_apimaster == False diff --git a/roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 b/roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 deleted file mode 100644 index 576c70128..000000000 --- a/roles/kubernetes/master/templates/deb-kube-apiserver.initd.j2 +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash -# -### BEGIN INIT INFO -# Provides: kube-apiserver -# Required-Start: $local_fs $network $syslog -# Required-Stop: -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: The Kubernetes apiserver -# Description: -# The Kubernetes apiserver. -### END INIT INFO - - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin -DESC="The Kubernetes apiserver" -NAME=kube-apiserver -DAEMON={{ bin_dir }}/kube-apiserver -DAEMON_LOG_FILE=/var/log/$NAME.log -PIDFILE=/var/run/$NAME.pid -SCRIPTNAME=/etc/init.d/$NAME -DAEMON_USER=root - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/kubernetes/$NAME.env ] && . /etc/kubernetes/$NAME.env - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.2-14) to ensure that this file is present -# and status_of_proc is working. -. /lib/lsb/init-functions - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - start-stop-daemon --start --quiet --background --no-close \ - --make-pidfile --pidfile $PIDFILE \ - --exec $DAEMON -c $DAEMON_USER --test > /dev/null \ - || return 1 - start-stop-daemon --start --quiet --background --no-close \ - --make-pidfile --pidfile $PIDFILE \ - --exec $DAEMON -c $DAEMON_USER -- \ - $DAEMON_ARGS >> $DAEMON_LOG_FILE 2>&1 \ - || return 2 -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE --name $NAME - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - # Many daemons don't delete their pidfiles when they exit. - rm -f $PIDFILE - return "$RETVAL" -} - - -case "$1" in - start) - log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) log_end_msg 0 || exit 0 ;; - 2) log_end_msg 1 || exit 1 ;; - esac - ;; - stop) - log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) log_end_msg 0 ;; - 2) exit 1 ;; - esac - ;; - status) - status_of_proc -p $PIDFILE "$DAEMON" "$NAME" && exit 0 || exit $? - ;; - - restart|force-reload) - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - case "$?" in - 0|1) - do_start - case "$?" in - 0) log_end_msg 0 ;; - 1) log_end_msg 1 ;; # Old process is still running - *) log_end_msg 1 ;; # Failed to start - esac - ;; - *) - # Failed to stop - log_end_msg 1 - ;; - esac - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac diff --git a/roles/kubernetes/master/templates/kube-apiserver.j2 b/roles/kubernetes/master/templates/kube-apiserver.j2 deleted file mode 100644 index 07ed59cc6..000000000 --- a/roles/kubernetes/master/templates/kube-apiserver.j2 +++ /dev/null @@ -1,58 +0,0 @@ -### -# kubernetes system config -# -# The following values are used to configure the kube-apiserver - -{% if ansible_service_mgr in ["sysvinit","upstart"] %} -# Logging directory -KUBE_LOGGING="--log-dir={{ kube_log_dir }} --logtostderr=true" -{% else %} -# logging to stderr means we get it in the systemd journal -KUBE_LOGGING="--logtostderr=true" -{% endif %} - -# Apiserver Log level, 0 is debug -KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}" - -# Should this cluster be allowed to run privileged docker containers -KUBE_ALLOW_PRIV="--allow_privileged=true" - -# The port on the local server to listen on. -KUBE_API_PORT="--insecure-port={{kube_apiserver_insecure_port}} --secure-port={{ kube_apiserver_port }}" - -# Insecure API address (default is localhost) -KUBE_API_INSECURE_BIND="--insecure-bind-address={{ kube_apiserver_insecure_bind_address | default('127.0.0.1') }}" - -# Address range to use for services -KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}" - -# Location of the etcd cluster -KUBE_ETCD_SERVERS="--etcd_servers={% for host in groups['etcd'] %}http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %}" - -# Bind address for secure endpoint -KUBE_API_ADDRESS="--bind-address={{ ip | default(ansible_default_ipv4.address) }}" - -# default admission control policies -KUBE_ADMISSION_CONTROL="--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota" - -# RUNTIME API CONFIGURATION (e.g. enable extensions) -KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in kube_api_runtime_config %}--runtime-config={{ conf }} {% endfor %}{% endif %}" - -# TLS CONFIGURATION -KUBE_TLS_CONFIG="--tls_cert_file={{ kube_cert_dir }}/apiserver.pem --tls_private_key_file={{ kube_cert_dir }}/apiserver-key.pem --client_ca_file={{ kube_cert_dir }}/ca.pem" - -# Add you own! -KUBE_API_ARGS="--token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/apiserver-key.pem --advertise-address={{ ip | default(ansible_default_ipv4.address) }}" - -{% if cloud_provider is defined and cloud_provider == "openstack" %} -KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" -{% else %} -{# TODO: gce and aws don't need the cloud provider to be set? #} -KUBELET_CLOUDPROVIDER="" -{% endif %} - -{% if ansible_service_mgr in ["sysvinit","upstart"] %} -DAEMON_ARGS="$KUBE_LOGGING $KUBE_LOG_LEVEL $KUBE_ALLOW_PRIV $KUBE_API_PORT $KUBE_API_INSECURE_BIND \ -$KUBE_SERVICE_ADDRESSES $KUBE_ETCD_SERVERS $KUBE_ADMISSION_CONTROL $KUBE_RUNTIME_CONFIG \ -$KUBE_TLS_CONFIG $KUBE_API_ARGS $KUBELET_CLOUDPROVIDER" -{% endif %} diff --git a/roles/kubernetes/master/templates/kube-apiserver.service.j2 b/roles/kubernetes/master/templates/kube-apiserver.service.j2 deleted file mode 100644 index 699797171..000000000 --- a/roles/kubernetes/master/templates/kube-apiserver.service.j2 +++ /dev/null @@ -1,30 +0,0 @@ -[Unit] -Description=Kubernetes API Server -Documentation=https://github.com/GoogleCloudPlatform/kubernetes -Requires=etcd.service -After=etcd.service - -[Service] -EnvironmentFile=/etc/kubernetes/kube-apiserver.env -User=kube -ExecStart={{ bin_dir }}/kube-apiserver \ - $KUBE_LOGTOSTDERR \ - $KUBE_LOG_LEVEL \ - $KUBE_ETCD_SERVERS \ - $KUBE_API_ADDRESS \ - $KUBE_API_PORT \ - $KUBE_API_INSECURE_BIND \ - $KUBELET_PORT \ - $KUBE_ALLOW_PRIV \ - $KUBE_SERVICE_ADDRESSES \ - $KUBE_ADMISSION_CONTROL \ - $KUBE_RUNTIME_CONFIG \ - $KUBE_TLS_CONFIG \ - $KUBE_API_ARGS \ - $KUBELET_CLOUDPROVIDER -Restart=on-failure -Type=notify -LimitNOFILE=65536 - -[Install] -WantedBy=multi-user.target diff --git a/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 b/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 index 5cc74cf9e..a9800d3ac 100644 --- a/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 +++ b/roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2 @@ -5,7 +5,7 @@ preferences: {} clusters: - cluster: certificate-authority-data: {{ kube_node_cert|b64encode }} - server: https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }} + server: {{ kube_apiserver_endpoint }} name: {{ cluster_name }} contexts: - context: diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 48b013e6c..ddd6f2085 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -2,6 +2,7 @@ apiVersion: v1 kind: Pod metadata: name: kube-apiserver + namespace: kube-system spec: hostNetwork: true containers: @@ -11,13 +12,17 @@ spec: - /hyperkube - apiserver - --advertise-address={{ ip | default(ansible_default_ipv4.address) }} - - --etcd-servers={% for srv in groups['etcd'] %}http://{{ hostvars[srv]['access_ip'] | default(hostvars[srv]['ip']|default(hostvars[srv]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %}{% endfor %} - - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota + - --etcd-servers={{ etcd_access_endpoint }} + - --etcd-quorum-read=true + - --insecure-bind-address={{ kube_apiserver_insecure_bind_address }} + - --apiserver-count={{ kube_apiserver_count }} + - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,ServiceAccount,ResourceQuota - --service-cluster-ip-range={{ kube_service_addresses }} - --client-ca-file={{ kube_cert_dir }}/ca.pem - --basic-auth-file={{ kube_users_dir }}/known_users.csv - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem + - --token-auth-file={{ kube_token_dir }}/known_tokens.csv - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem - --secure-port={{ kube_apiserver_port }} - --insecure-port={{ kube_apiserver_insecure_port }} @@ -26,16 +31,18 @@ spec: - --runtime-config={{ conf }} {% endfor %} {% endif %} - - --token-auth-file={{ kube_token_dir }}/known_tokens.csv +{% if enable_network_policy is defined and enable_network_policy == True %} + - --runtime-config=extensions/v1beta1/networkpolicies=true +{% endif %} - --v={{ kube_log_level | default('2') }} - --allow-privileged=true - ports: - - containerPort: {{ kube_apiserver_port }} - hostPort: {{ kube_apiserver_port }} - name: https - - containerPort: {{ kube_apiserver_insecure_port }} - hostPort: {{ kube_apiserver_insecure_port }} - name: local +{% if cloud_provider is defined and cloud_provider == "openstack" %} + - --cloud-provider={{ cloud_provider }} + - --cloud-config={{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider == "aws" %} + - --cloud-provider={{ cloud_provider }} +{% endif %} + - 2>&1 >> {{ kube_log_dir }}/kube-apiserver.log volumeMounts: - mountPath: {{ kube_config_dir }} name: kubernetes-config @@ -43,6 +50,8 @@ spec: - mountPath: /etc/ssl/certs name: ssl-certs-host readOnly: true + - mountPath: /var/log/ + name: logfile volumes: - hostPath: path: {{ kube_config_dir }} @@ -50,3 +59,6 @@ spec: - hostPath: path: /etc/ssl/certs/ name: ssl-certs-host + - hostPath: + path: /var/log/ + name: logfile diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 29c56e6a8..3a9e1ef1b 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -11,15 +11,17 @@ spec: command: - /hyperkube - controller-manager - - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}} + - --master={{ kube_apiserver_endpoint }} - --leader-elect=true - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem - --root-ca-file={{ kube_cert_dir }}/ca.pem - --enable-hostpath-provisioner={{ kube_hostpath_dynamic_provisioner }} - --v={{ kube_log_level | default('2') }} {% if cloud_provider is defined and cloud_provider == "openstack" %} - - --cloud-provider=openstack + - --cloud-provider={{cloud_provider}} - --cloud-config={{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider == "aws" %} + - --cloud-provider={{cloud_provider}} {% endif %} livenessProbe: httpGet: @@ -32,9 +34,6 @@ spec: - mountPath: {{ kube_cert_dir }} name: ssl-certs-kubernetes readOnly: true - - mountPath: /etc/ssl/certs - name: ssl-certs-host - readOnly: true {% if cloud_provider is defined and cloud_provider == "openstack" %} - mountPath: {{ kube_config_dir }}/cloud_config name: cloudconfig @@ -44,9 +43,6 @@ spec: - hostPath: path: {{ kube_cert_dir }} name: ssl-certs-kubernetes - - hostPath: - path: /etc/ssl/certs/ - name: ssl-certs-host {% if cloud_provider is defined and cloud_provider == "openstack" %} - hostPath: path: {{ kube_config_dir }}/cloud_config diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index f642fbf70..024ddbfaa 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -12,7 +12,7 @@ spec: - /hyperkube - scheduler - --leader-elect=true - - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}} + - --master={{ kube_apiserver_endpoint }} - --v={{ kube_log_level | default('2') }} livenessProbe: httpGet: diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 2997946f5..8c4ce38a5 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -1,7 +1,29 @@ -# This directory is where all the additional scripts go -# that Kubernetes normally puts in /srv/kubernetes. -# This puts them in a sane location -kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" +# This is where all the cert scripts and certs will be located +kube_cert_dir: "{{ kube_config_dir }}/ssl" + +# change to 0.0.0.0 to enable insecure access from anywhere (not recommended) +kube_apiserver_insecure_bind_address: 127.0.0.1 + +# This is where you can drop yaml/json files and the kubelet will run those +# pods on startup +kube_manifest_dir: "{{ kube_config_dir }}/manifests" + +dns_domain: "{{ cluster_name }}" + +# resolv.conf to base dns config +kube_resolv_conf: "/etc/resolv.conf" + +kube_proxy_mode: iptables + +# If using the pure iptables proxy, SNAT everything +kube_proxy_masquerade_all: true + +# kube_api_runtime_config: +# - extensions/v1beta1/daemonsets=true +# - extensions/v1beta1/deployments=true + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" # This directory is where all the additional config stuff goes # the kubernetes normally puts in /srv/kubernets. @@ -11,40 +33,5 @@ kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" # look in here. Don't do it. kube_config_dir: /etc/kubernetes -# This is where all the cert scripts and certs will be located -kube_cert_dir: "{{ kube_config_dir }}/ssl" - -# This is where all of the bearer tokens will be stored -kube_token_dir: "{{ kube_config_dir }}/tokens" - -# This is where to save basic auth file -kube_users_dir: "{{ kube_config_dir }}/users" - -# This is where you can drop yaml/json files and the kubelet will run those -# pods on startup -kube_manifest_dir: "{{ kube_config_dir }}/manifests" - -# Logging directory (sysvinit systems) -kube_log_dir: "/var/log/kubernetes" - -dns_domain: "{{ cluster_name }}" - -kube_proxy_mode: iptables - -# An experimental dev/test only dynamic volumes provisioner, -# for PetSets. Works for kube>=v1.3 only. -kube_hostpath_dynamic_provisioner: "false" - -hyperkube_image_repo: "quay.io/coreos/hyperkube" -hyperkube_image_tag: "{{ kube_version }}_coreos.0" - -# IP address of the DNS server. -# Kubernetes will create a pod with several containers, serving as the DNS -# server and expose it under this IP address. The IP address must be from -# the range specified as kube_service_addresses. This magic will actually -# pick the 10th ip address in the kube_service_addresses range and use that. -dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}" - -# kube_api_runtime_config: -# - extensions/v1beta1/daemonsets=true -# - extensions/v1beta1/deployments=true +nginx_image_repo: nginx +nginx_image_tag: 1.11.4-alpine diff --git a/roles/kubernetes/node/handlers/main.yml b/roles/kubernetes/node/handlers/main.yml index 6be44279d..5991bebf3 100644 --- a/roles/kubernetes/node/handlers/main.yml +++ b/roles/kubernetes/node/handlers/main.yml @@ -2,14 +2,14 @@ - name: restart kubelet command: /bin/true notify: - - reload systemd - - reload kubelet + - Kubelet | reload systemd + - Kubelet | reload kubelet -- name: reload systemd +- name: Kubelet | reload systemd command: systemctl daemon-reload when: ansible_service_mgr == "systemd" -- name: reload kubelet +- name: Kubelet | reload kubelet service: name: kubelet state: restarted diff --git a/roles/kubernetes/node/meta/main.yml b/roles/kubernetes/node/meta/main.yml index a277c7d8a..9c52b2d80 100644 --- a/roles/kubernetes/node/meta/main.yml +++ b/roles/kubernetes/node/meta/main.yml @@ -1,5 +1,7 @@ --- dependencies: - role: download - file: "{{ downloads.kubernetes_kubelet }}" + file: "{{ downloads.hyperkube }}" + - role: download + file: "{{ downloads.pod_infra }}" - role: kubernetes/secrets diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index 4fabf1c88..6b5fe5bb4 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -14,7 +14,6 @@ when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "RedHat" notify: restart kubelet -- name: install | Install kubelet binary - command: rsync -piu "{{ local_release_dir }}/kubernetes/bin/kubelet" "{{ bin_dir }}/kubelet" - register: kubelet_copy - changed_when: false +- name: install | Install kubelet launch script + template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes + notify: restart kubelet diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 803c9251b..a8cb6ce5a 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -1,6 +1,9 @@ --- - include: install.yml +- include: nginx-proxy.yml + when: is_kube_master == false and loadbalancer_apiserver_localhost|default(false) + - name: Write Calico cni config template: src: "cni-calico.conf.j2" @@ -23,11 +26,6 @@ src: manifests/kube-proxy.manifest.j2 dest: "{{ kube_manifest_dir }}/kube-proxy.manifest" -- name: Restart kubelet if binary changed - command: /bin/true - notify: restart kubelet - when: kubelet_copy.stdout_lines - # reload-systemd - meta: flush_handlers diff --git a/roles/kubernetes/node/tasks/nginx-proxy.yml b/roles/kubernetes/node/tasks/nginx-proxy.yml new file mode 100644 index 000000000..056c55a93 --- /dev/null +++ b/roles/kubernetes/node/tasks/nginx-proxy.yml @@ -0,0 +1,9 @@ +--- +- name: nginx-proxy | Write static pod + template: src=manifests/nginx-proxy.manifest.j2 dest=/etc/kubernetes/manifests/nginx-proxy.yml + +- name: nginx-proxy | Make nginx directory + file: path=/etc/nginx state=directory mode=0700 owner=root + +- name: nginx-proxy | Write nginx-proxy configuration + template: src=nginx.conf.j2 dest="/etc/nginx/nginx.conf" owner=root mode=0755 backup=yes diff --git a/roles/kubernetes/node/templates/cni-calico.conf.j2 b/roles/kubernetes/node/templates/cni-calico.conf.j2 index c11067965..4615cdabd 100644 --- a/roles/kubernetes/node/templates/cni-calico.conf.j2 +++ b/roles/kubernetes/node/templates/cni-calico.conf.j2 @@ -1,9 +1,16 @@ { "name": "calico-k8s-network", "type": "calico", - "etcd_authority": "127.0.0.1:2379", "log_level": "info", "ipam": { "type": "calico-ipam" + }, +{% if enable_network_policy is defined and enable_network_policy == True %} + "policy": { + "type": "k8s" + }, +{% endif %} + "kubernetes": { + "kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml" } } diff --git a/roles/kubernetes/node/templates/deb-kubelet.initd.j2 b/roles/kubernetes/node/templates/deb-kubelet.initd.j2 index 65fd537f0..5d5184efe 100644 --- a/roles/kubernetes/node/templates/deb-kubelet.initd.j2 +++ b/roles/kubernetes/node/templates/deb-kubelet.initd.j2 @@ -1,7 +1,7 @@ #!/bin/bash # ### BEGIN INIT INFO -# Provides: kubelet +# Provides: kubelet # Required-Start: $local_fs $network $syslog # Required-Stop: # Default-Start: 2 3 4 5 @@ -39,6 +39,8 @@ DAEMON_USER=root # do_start() { + /usr/bin/docker rm -f kubelet &>/dev/null || true + sleep 1 # Return # 0 if daemon has been started # 1 if daemon was already running diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2 new file mode 100644 index 000000000..2fcc7307f --- /dev/null +++ b/roles/kubernetes/node/templates/kubelet-container.j2 @@ -0,0 +1,15 @@ +#!/bin/bash +/usr/bin/docker run --privileged --rm \ +--net=host --pid=host --name=kubelet \ +-v /etc/cni:/etc/cni:ro \ +-v /opt/cni:/opt/cni:ro \ +-v /etc/kubernetes:/etc/kubernetes \ +-v /sys:/sys \ +-v /dev:/dev \ +-v /var/lib/docker:/var/lib/docker \ +-v /var/run:/var/run \ +-v /var/lib/kubelet:/var/lib/kubelet \ +{{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \ +nsenter --target=1 --mount --wd=. -- \ +./hyperkube kubelet \ +$@ diff --git a/roles/kubernetes/node/templates/kubelet.j2 b/roles/kubernetes/node/templates/kubelet.j2 index d96f7ff6e..53f2915d9 100644 --- a/roles/kubernetes/node/templates/kubelet.j2 +++ b/roles/kubernetes/node/templates/kubelet.j2 @@ -6,36 +6,38 @@ KUBE_LOGGING="--log-dir={{ kube_log_dir }} --logtostderr=true" KUBE_LOGGING="--logtostderr=true" {% endif %} KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}" -KUBE_ALLOW_PRIV="--allow_privileged=true" {% if inventory_hostname in groups['kube-node'] %} -KUBELET_API_SERVER="--api_servers={% for host in groups['kube-master'] %}https://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}{% if not loop.last %},{% endif %}{% endfor %}" +KUBELET_API_SERVER="--api_servers={{ kube_apiserver_endpoint }}" {% endif %} # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces) KUBELET_ADDRESS="--address={{ ip | default("0.0.0.0") }}" # The port for the info server to serve on # KUBELET_PORT="--port=10250" # You may leave this blank to use the actual hostname -KUBELET_HOSTNAME="--hostname_override={{ inventory_hostname }}" +KUBELET_HOSTNAME="--hostname-override={{ inventory_hostname }}" {% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} KUBELET_REGISTER_NODE="--register-node=false" {% endif %} # location of the api-server -{% if dns_setup %} -KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }}" +{% if dns_setup|bool and skip_dnsmasq|bool %} +KUBELET_ARGS="--cluster_dns={{ skydns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" +{% elif dns_setup|bool %} +KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} --kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --config={{ kube_manifest_dir }} --resolv-conf={{ kube_resolv_conf }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" {% else %} -KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}" +KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }}" {% endif %} {% if kube_network_plugin is defined and kube_network_plugin in ["calico", "weave"] %} -KUBELET_NETWORK_PLUGIN="--network_plugin=cni --network-plugin-dir=/etc/cni/net.d" +KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d" {% elif kube_network_plugin is defined and kube_network_plugin == "weave" %} DOCKER_SOCKET="--docker-endpoint=unix:/var/run/weave/weave.sock" {% endif %} # Should this cluster be allowed to run privileged docker containers -KUBE_ALLOW_PRIV="--allow_privileged=true" +KUBE_ALLOW_PRIV="--allow-privileged=true" {% if cloud_provider is defined and cloud_provider == "openstack" %} KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" +{% elif cloud_provider is defined and cloud_provider == "aws" %} +KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }}" {% else %} -{# TODO: gce and aws don't need the cloud provider to be set? #} KUBELET_CLOUDPROVIDER="" {% endif %} {% if ansible_service_mgr in ["sysvinit","upstart"] %} diff --git a/roles/kubernetes/node/templates/kubelet.service.j2 b/roles/kubernetes/node/templates/kubelet.service.j2 index 9fa47bf13..ad62d8562 100644 --- a/roles/kubernetes/node/templates/kubelet.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.service.j2 @@ -2,9 +2,11 @@ Description=Kubernetes Kubelet Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes {% if kube_network_plugin is defined and kube_network_plugin == "calico" %} -After=docker.service calico-node.service +After=docker.service docker.socket calico-node.service +Wants=docker.socket calico-node.service {% else %} After=docker.service +Wants=docker.socket {% endif %} [Service] @@ -22,7 +24,10 @@ ExecStart={{ bin_dir }}/kubelet \ $KUBELET_REGISTER_NODE \ $KUBELET_NETWORK_PLUGIN \ $KUBELET_CLOUDPROVIDER -Restart=on-failure +ExecStartPre=-/usr/bin/docker rm -f kubelet +ExecReload=/usr/bin/docker restart kubelet +Restart=always +RestartSec=10s [Install] WantedBy=multi-user.target diff --git a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 index d094766b5..7abffe053 100644 --- a/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 +++ b/roles/kubernetes/node/templates/manifests/kube-proxy.manifest.j2 @@ -12,18 +12,16 @@ spec: - /hyperkube - proxy - --v={{ kube_log_level | default('2') }} -{% if inventory_hostname in groups['kube-master'] %} - - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}} -{% else %} -{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} - - --master=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }} -{% else %} - - --master=https://{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }} -{% endif%} + - --master={{ kube_apiserver_endpoint }} +{% if not is_kube_master %} - --kubeconfig=/etc/kubernetes/node-kubeconfig.yaml {% endif %} - --bind-address={{ ip | default(ansible_default_ipv4.address) }} + - --cluster-cidr={{ kube_pods_subnet }} - --proxy-mode={{ kube_proxy_mode }} +{% if kube_proxy_masquerade_all and kube_proxy_mode == "iptables" %} + - --masquerade-all +{% endif %} securityContext: privileged: true volumeMounts: diff --git a/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 b/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 new file mode 100644 index 000000000..50e054268 --- /dev/null +++ b/roles/kubernetes/node/templates/manifests/nginx-proxy.manifest.j2 @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-proxy + namespace: kube-system +spec: + hostNetwork: true + containers: + - name: nginx-proxy + image: {{ nginx_image_repo }}:{{ nginx_image_tag }} + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/nginx + name: etc-nginx + readOnly: true + volumes: + - name: etc-nginx + hostPath: + path: /etc/nginx diff --git a/roles/kubernetes/node/templates/nginx.conf.j2 b/roles/kubernetes/node/templates/nginx.conf.j2 new file mode 100644 index 000000000..edcee08a9 --- /dev/null +++ b/roles/kubernetes/node/templates/nginx.conf.j2 @@ -0,0 +1,26 @@ +error_log stderr notice; + +worker_processes auto; +events { + multi_accept on; + use epoll; + worker_connections 1024; +} + +stream { + upstream kube_apiserver { + least_conn; + {% for host in groups['kube-master'] -%} + server {{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:{{ kube_apiserver_port }}; + {% endfor %} + } + + server { + listen {{ kube_apiserver_port }}; + proxy_pass kube_apiserver; + proxy_timeout 3s; + proxy_connect_timeout 1s; + + } + +} diff --git a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 index d21b8eef3..e1593303d 100644 --- a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 +++ b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 @@ -4,6 +4,7 @@ clusters: - name: local cluster: certificate-authority: {{ kube_cert_dir }}/ca.pem + server: {{ kube_apiserver_endpoint }} users: - name: kubelet user: diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index 49f9b7c0e..3eae9757d 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -1,13 +1,40 @@ --- run_gitinfos: false +# This directory is where all the additional scripts go +# that Kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location +kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" + +# This directory is where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernets. +# This puts them in a sane location. +# Editting this value will almost surely break something. Don't +# change it. Things like the systemd scripts are hard coded to +# look in here. Don't do it. +kube_config_dir: /etc/kubernetes + +# Logging directory (sysvinit systems) +kube_log_dir: "/var/log/kubernetes" + +# This is where you can drop yaml/json files and the kubelet will run those +# pods on startup +kube_manifest_dir: "{{ kube_config_dir }}/manifests" + +epel_rpm_download_url: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" + common_required_pkgs: - python-httplib2 - openssl - curl - rsync - bash-completion + - socat +# Set to true if your network does not support IPv6 +# This maybe necessary for pulling Docker images from +# GCE docker repository +disable_ipv6_dns: false # For the openstack integration kubelet will need credentials to access diff --git a/roles/kubernetes/preinstall/gen-gitinfos.sh b/roles/kubernetes/preinstall/gen-gitinfos.sh index 34b977f7e..bfab5a4be 100755 --- a/roles/kubernetes/preinstall/gen-gitinfos.sh +++ b/roles/kubernetes/preinstall/gen-gitinfos.sh @@ -54,7 +54,7 @@ EOF printf "unstaged changes=\"/etc/.git-ansible.diff\"" fi - if [ ${current_commit} == ${latest_tag_commit} ]; then + if [ "${current_commit}" = "${latest_tag_commit}" ]; then printf "\ncurrent_commit_tag=\"${latest_tag}\"" else printf "\nlast tag was "$(git describe --tags | awk -F- '{print $2}')" commits ago =\"" diff --git a/roles/kubernetes/preinstall/tasks/etchosts.yml b/roles/kubernetes/preinstall/tasks/etchosts.yml index dd8562b8c..6f21ffa8f 100644 --- a/roles/kubernetes/preinstall/tasks/etchosts.yml +++ b/roles/kubernetes/preinstall/tasks/etchosts.yml @@ -1,14 +1,15 @@ --- - name: Hosts | populate inventory into hosts file - lineinfile: + blockinfile: dest: /etc/hosts - regexp: "^{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}$" - line: "{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }}" + block: |- + {% for item in groups['all'] -%} + {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item].ansible_default_ipv4.address)) }} {{ item }} + {% endfor %} state: present create: yes backup: yes - when: hostvars[item].ansible_default_ipv4.address is defined - with_items: "{{ groups['all'] }}" + marker: "# Ansible inventory hosts {mark}" - name: Hosts | populate kubernetes loadbalancer address into hosts file lineinfile: diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 95cd134ce..49e69a907 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -1,7 +1,21 @@ --- +- name: Force binaries directory for CoreOS + set_fact: + bin_dir: "/opt/bin" + when: ansible_os_family == "CoreOS" + + +- name: 'GIT | Install script for collecting git info' + template: + src: "{{ role_path }}/gen-gitinfos.sh" + dest: "{{ bin_dir }}/gen-gitinfos.sh" + mode: a+rwx + - include: gitinfos.yml when: run_gitinfos +- include: set_facts.yml + - name: gather os specific variables include_vars: "{{ item }}" with_first_found: @@ -16,35 +30,33 @@ - ../vars skip: true -- name: Force binaries directory for CoreOS - set_fact: - bin_dir: "/opt/bin" - when: ansible_os_family == "CoreOS" - - name: Create kubernetes config directory file: path: "{{ kube_config_dir }}" state: directory owner: kube + when: "{{ inventory_hostname in groups['k8s-cluster'] }}" - name: Create kubernetes script directory file: path: "{{ kube_script_dir }}" state: directory owner: kube + when: "{{ inventory_hostname in groups['k8s-cluster'] }}" - name: Create kubernetes manifests directory file: path: "{{ kube_manifest_dir }}" state: directory owner: kube + when: "{{ inventory_hostname in groups['k8s-cluster'] }}" - name: Create kubernetes logs directory file: path: "{{ kube_log_dir }}" state: directory owner: kube - when: ansible_service_mgr in ["sysvinit","upstart"] + when: ansible_service_mgr in ["sysvinit","upstart"] and "{{ inventory_hostname in groups['k8s-cluster'] }}" - name: check cloud_provider value fail: @@ -62,7 +74,7 @@ with_items: - "/etc/cni/net.d" - "/opt/cni/bin" - when: kube_network_plugin in ["calico", "weave"] + when: kube_network_plugin in ["calico", "weave"] and "{{ inventory_hostname in groups['k8s-cluster'] }}" - name: Update package management cache (YUM) yum: update_cache=yes name='*' @@ -79,7 +91,7 @@ changed_when: False - name: Install epel-release on RedHat/CentOS - shell: rpm -qa | grep epel-release || rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + shell: rpm -qa | grep epel-release || rpm -ivh {{ epel_rpm_download_url }} when: ansible_distribution in ["CentOS","RedHat"] and ansible_distribution_major_version >= 7 changed_when: False @@ -89,9 +101,21 @@ module: "{{ ansible_pkg_mgr }}" name: "{{ item }}" state: latest + register: pkgs_task_result + until: pkgs_task_result|success + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" with_items: "{{required_pkgs | default([]) | union(common_required_pkgs|default([]))}}" when: ansible_os_family != "CoreOS" +- name: Disable IPv6 DNS lookup + lineinfile: + dest: /etc/gai.conf + line: "precedence ::ffff:0:0/96 100" + state: present + backup: yes + when: disable_ipv6_dns and ansible_os_family != "CoreOS" + # Todo : selinux configuration - name: Set selinux policy to permissive selinux: policy=targeted state=permissive diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml new file mode 100644 index 000000000..2dd947dda --- /dev/null +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -0,0 +1,58 @@ +--- +- set_fact: kube_apiserver_count="{{ groups['kube-master'] | length }}" +- set_fact: kube_apiserver_address="{{ ip | default(ansible_default_ipv4['address']) }}" +- set_fact: kube_apiserver_access_address="{{ access_ip | default(kube_apiserver_address) }}" +- set_fact: is_kube_master="{{ inventory_hostname in groups['kube-master'] }}" +- set_fact: first_kube_master="{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}" +- set_fact: + loadbalancer_apiserver_localhost: false + when: loadbalancer_apiserver is defined +- set_fact: + kube_apiserver_endpoint: |- + {% if not is_kube_master and loadbalancer_apiserver_localhost -%} + https://localhost:{{ kube_apiserver_port }} + {%- elif is_kube_master and loadbalancer_apiserver is not defined -%} + http://127.0.0.1:{{ kube_apiserver_insecure_port }} + {%- else -%} + {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%} + https://{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} + {%- else -%} + https://{{ first_kube_master }}:{{ kube_apiserver_port }} + {%- endif -%} + {%- endif %} + +- set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}" +- set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}" +- set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380" +- set_fact: etcd_client_url="http://{{ etcd_access_address }}:2379" +- set_fact: etcd_authority="127.0.0.1:2379" +- set_fact: etcd_endpoint="http://{{ etcd_authority }}" +- set_fact: + etcd_access_addresses: |- + {% for item in groups['etcd'] -%} + http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2379{% if not loop.last %},{% endif %} + {%- endfor %} +- set_fact: etcd_access_endpoint="{% if etcd_multiaccess %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}" +- set_fact: + etcd_member_name: |- + {% for host in groups['etcd'] %} + {% if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %} + {% endfor %} +- set_fact: + etcd_peer_addresses: |- + {% for item in groups['etcd'] -%} + {{ "etcd"+loop.index|string }}=http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %} + {%- endfor %} +- set_fact: + etcd_proxy_member_name: |- + {% for host in groups['k8s-cluster'] %} + {% if inventory_hostname == host %}{{"etcd-proxy"+loop.index|string }}{% endif %} + {% endfor %} +- set_fact: + is_etcd_proxy: "{{ inventory_hostname in groups['k8s-cluster'] }}" +- set_fact: + is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}" +- set_fact: + etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=") +- set_fact: + etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" diff --git a/roles/kubernetes/secrets/defaults/main.yml b/roles/kubernetes/secrets/defaults/main.yml new file mode 100644 index 000000000..c6011a9bf --- /dev/null +++ b/roles/kubernetes/secrets/defaults/main.yml @@ -0,0 +1,21 @@ +# This is where all the cert scripts and certs will be located +kube_cert_dir: "{{ kube_config_dir }}/ssl" + +# This is where all of the bearer tokens will be stored +kube_token_dir: "{{ kube_config_dir }}/tokens" + +# This is where to save basic auth file +kube_users_dir: "{{ kube_config_dir }}/users" + +# This directory is where all the additional config stuff goes +# the kubernetes normally puts in /srv/kubernets. +# This puts them in a sane location. +# Editting this value will almost surely break something. Don't +# change it. Things like the systemd scripts are hard coded to +# look in here. Don't do it. +kube_config_dir: /etc/kubernetes + +# This directory is where all the additional scripts go +# that Kubernetes normally puts in /srv/kubernetes. +# This puts them in a sane location +kube_script_dir: "{{ bin_dir }}/kubernetes-scripts" diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index a2f698541..f90fb7e8b 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -26,8 +26,8 @@ Usage : $(basename $0) -f [-d ] -h | --help : Show this message -f | --config : Openssl configuration file -d | --ssldir : Directory where the certificates will be installed - - ex : + + ex : $(basename $0) -f openssl.conf -d /srv/ssl EOF } @@ -37,7 +37,7 @@ while (($#)); do case "$1" in -h | --help) usage; exit 0;; -f | --config) CONFIG=${2}; shift 2;; - -d | --ssldir) SSLDIR="${2}"; shift 2;; + -d | --ssldir) SSLDIR="${2}"; shift 2;; *) usage echo "ERROR : Unknown option" @@ -68,6 +68,7 @@ openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1 openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1 openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1 +cat ca.pem >> apiserver.pem # Nodes and Admin for i in node admin; do diff --git a/roles/kubernetes/secrets/tasks/check-tokens.yml b/roles/kubernetes/secrets/tasks/check-tokens.yml index 1ecaa7006..14cfbb124 100644 --- a/roles/kubernetes/secrets/tasks/check-tokens.yml +++ b/roles/kubernetes/secrets/tasks/check-tokens.yml @@ -27,7 +27,7 @@ sync_tokens: true when: >- {%- set tokens = {'sync': False} -%} - {%- for server in groups['kube-master'] + {%- for server in groups['kube-master'] | intersect(play_hosts) if (not hostvars[server].known_tokens.stat.exists) or (hostvars[server].known_tokens.stat.checksum != known_tokens_master.stat.checksum|default('')) -%} {%- set _ = tokens.update({'sync': True}) -%} diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs.yml index 7178bce0c..6057c0676 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs.yml @@ -65,3 +65,30 @@ shell: chmod 0600 {{ kube_cert_dir}}/*key.pem when: inventory_hostname in groups['kube-master'] changed_when: false + +- name: Gen_certs | target ca-certificates directory + set_fact: + ca_cert_dir: |- + {% if ansible_os_family == "Debian" -%} + /usr/local/share/ca-certificates + {%- elif ansible_os_family == "RedHat" -%} + /etc/pki/ca-trust/source/anchors + {%- elif ansible_os_family == "CoreOS" -%} + /etc/ssl/certs + {%- endif %} + +- name: Gen_certs | add CA to trusted CA dir + copy: + src: "{{ kube_cert_dir }}/ca.pem" + dest: "{{ ca_cert_dir }}/kube-ca.crt" + remote_src: true + register: kube_ca_cert + +- name: Gen_certs | update ca-certificates (Debian/Ubuntu/CoreOS) + command: update-ca-certificates + when: kube_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"] + +- name: Gen_certs | update ca-certificatesa (RedHat) + command: update-ca-trust extract + when: kube_ca_cert.changed and ansible_os_family == "RedHat" + diff --git a/roles/kubernetes/secrets/templates/openssl.conf.j2 b/roles/kubernetes/secrets/templates/openssl.conf.j2 index fa00163a3..ac94b6800 100644 --- a/roles/kubernetes/secrets/templates/openssl.conf.j2 +++ b/roles/kubernetes/secrets/templates/openssl.conf.j2 @@ -11,12 +11,18 @@ DNS.1 = kubernetes DNS.2 = kubernetes.default DNS.3 = kubernetes.default.svc DNS.4 = kubernetes.default.svc.{{ dns_domain }} +DNS.5 = localhost +{% for host in groups['kube-master'] %} +DNS.{{ 5 + loop.index }} = {{ host }} +{% endfor %} {% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} -DNS.5 = {{ apiserver_loadbalancer_domain_name }} +{% set idx = groups['kube-master'] | length | int + 5 %} +DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }} {% endif %} {% for host in groups['kube-master'] %} IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} {% endfor %} {% set idx = groups['kube-master'] | length | int * 2 + 1 %} -IP.{{ idx | string }} = {{ kube_apiserver_ip }} +IP.{{ idx }} = {{ kube_apiserver_ip }} +IP.{{ idx + 1 }} = 127.0.0.1 diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index e55b3ab4b..aec7a5e15 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -2,7 +2,9 @@ # Enables Internet connectivity from containers nat_outgoing: true -# cloud_provider can only be set to 'gce' or 'aws' -# cloud_provider: -calicoctl_image_repo: calico/ctl -calicoctl_image_tag: "{{ calico_version }}" +# Use IP-over-IP encapsulation across hosts +ipip: false + +# Set to true if you want your calico cni binaries to overwrite the +# ones from hyperkube while leaving other cni plugins intact. +overwrite_hyperkube_cni: true diff --git a/roles/network_plugin/calico/handlers/main.yml b/roles/network_plugin/calico/handlers/main.yml index 1e8d56dbe..d4059c412 100644 --- a/roles/network_plugin/calico/handlers/main.yml +++ b/roles/network_plugin/calico/handlers/main.yml @@ -2,15 +2,15 @@ - name: restart calico-node command: /bin/true notify: - - reload systemd - - reload calico-node + - Calico | reload systemd + - Calico | reload calico-node -- name : reload systemd +- name : Calico | reload systemd shell: systemctl daemon-reload when: ansible_service_mgr == "systemd" -- name: reload calico-node +- name: Calico | reload calico-node service: name: calico-node state: restarted - sleep: 10 \ No newline at end of file + sleep: 10 diff --git a/roles/network_plugin/calico/meta/main.yml b/roles/network_plugin/calico/meta/main.yml index 92ab5391b..c13e976d3 100644 --- a/roles/network_plugin/calico/meta/main.yml +++ b/roles/network_plugin/calico/meta/main.yml @@ -4,3 +4,9 @@ dependencies: file: "{{ downloads.calico_cni_plugin }}" - role: download file: "{{ downloads.calico_cni_plugin_ipam }}" + - role: download + file: "{{ downloads.calico_node }}" + - role: download + file: "{{ downloads.calicoctl }}" + - role: download + file: "{{ downloads.hyperkube }}" diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index a8185d4f2..46f729883 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -10,13 +10,6 @@ - restart docker when: ansible_os_family != "CoreOS" -- name: Calico | Write docker.service systemd file - template: - src: systemd-docker.service - dest: /lib/systemd/system/docker.service - notify: restart docker - when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS" - - meta: flush_handlers - name: Calico | Install calicoctl container script @@ -29,48 +22,68 @@ changed_when: false notify: restart calico-node -- name: Calico | Install calico cni bin - command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico" +- name: Calico | Copy cni plugins from hyperkube + command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/" + register: cni_task_result + until: cni_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" changed_when: false +- name: Calico | Install calico cni bin + command: rsync -pi "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico" + changed_when: false + when: "{{ overwrite_hyperkube_cni|bool }}" + - name: Calico | Install calico-ipam cni bin - command: rsync -piu "{{ local_release_dir }}/calico/bin/calico" "/opt/cni/bin/calico-ipam" + command: rsync -pi "{{ local_release_dir }}/calico/bin/calico-ipam" "/opt/cni/bin/calico-ipam" changed_when: false + when: "{{ overwrite_hyperkube_cni|bool }}" - name: Calico | wait for etcd - wait_for: - port: 2379 + uri: url=http://localhost:2379/health + register: result + until: result.status == 200 + retries: 10 + delay: 5 when: inventory_hostname in groups['kube-master'] - name: Calico | Check if calico network pool has already been configured uri: - url: "http://127.0.0.1:2379/v2/keys/calico/v1/ipam/v4/pool" + url: "{{ etcd_endpoint }}/v2/keys/calico/v1/ipam/v4/pool" return_content: yes status_code: 200,404 register: calico_conf run_once: true -- name: Calico | Configure calico network pool - command: "{{ bin_dir }}/calicoctl pool add {{ kube_pods_subnet }}" +- name: Calico | Define ipip pool argument run_once: true - when: calico_conf.status == 404 and cloud_provider is not defined - and not nat_outgoing|default(false) or - (nat_outgoing|default(false) and peer_with_router|default(false)) + set_fact: + ipip_arg: "--ipip" + when: cloud_provider is defined or ipip|default(false) -- name: Calico | Configure calico network pool for cloud - command: "{{ bin_dir }}/calicoctl pool add {{ kube_pods_subnet }} --ipip --nat-outgoing" +- name: Calico | Define nat-outgoing pool argument run_once: true - when: calico_conf.status == 404 and cloud_provider is defined + set_fact: + nat_arg: "--nat-outgoing" + when: nat_outgoing|default(false) and not peer_with_router|default(false) -- name: Calico | Configure calico network pool with nat outgoing - command: "{{ bin_dir}}/calicoctl pool add {{ kube_pods_subnet }} --nat-outgoing" +- name: Calico | Define calico pool task name run_once: true - when: calico_conf.status == 404 and cloud_provider is not defined - and nat_outgoing|default(false) and not peer_with_router|default(false) + set_fact: + pool_task_name: "with options {{ ipip_arg|default('') }} {{ nat_arg|default('') }}" + when: ipip_arg|default(false) or nat_arg|default(false) + +- name: Calico | Configure calico network pool {{ pool_task_name|default('') }} + command: "{{ bin_dir}}/calicoctl pool add {{ kube_pods_subnet }} {{ ipip_arg|default('') }} {{ nat_arg|default('') }}" + environment: + NO_DEFAULT_POOLS: true + run_once: true + when: calico_conf.status == 404 or "nodes" not in calico_conf.content - name: Calico | Get calico configuration from etcd uri: - url: "http://127.0.0.1:2379/v2/keys/calico/v1/ipam/v4/pool" + url: "{{ etcd_endpoint }}/v2/keys/calico/v1/ipam/v4/pool" return_content: yes register: calico_pools run_once: true diff --git a/roles/network_plugin/calico/templates/calico-node.service.j2 b/roles/network_plugin/calico/templates/calico-node.service.j2 index 8c8af6971..a7f7e4bab 100644 --- a/roles/network_plugin/calico/templates/calico-node.service.j2 +++ b/roles/network_plugin/calico/templates/calico-node.service.j2 @@ -1,19 +1,19 @@ [Unit] Description=Calico per-node agent Documentation=https://github.com/projectcalico/calico-docker -Requires=docker.service -After=docker.service etcd.service +After=docker.service docker.socket etcd-proxy.service +Wants=docker.socket etcd-proxy.service [Service] User=root PermissionsStartOnly=true {% if inventory_hostname in groups['kube-node'] and peer_with_router|default(false)%} -ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false +ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }} {% else %} -ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false +ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }} {% endif %} Restart=always -Restart=10 +RestartSec=10s [Install] WantedBy=multi-user.target diff --git a/roles/network_plugin/calico/templates/calicoctl-container.j2 b/roles/network_plugin/calico/templates/calicoctl-container.j2 index 9436a50e7..466f1df93 100644 --- a/roles/network_plugin/calico/templates/calicoctl-container.j2 +++ b/roles/network_plugin/calico/templates/calicoctl-container.j2 @@ -1,6 +1,6 @@ #!/bin/bash /usr/bin/docker run --privileged --rm \ ---net=host -e ETCD_AUTHORITY=127.0.0.1:2379 \ +--net=host --pid=host -e ETCD_AUTHORITY={{ etcd_authority }} \ -v /usr/bin/docker:/usr/bin/docker \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/run/calico:/var/run/calico \ diff --git a/roles/network_plugin/calico/templates/network-environment.j2 b/roles/network_plugin/calico/templates/network-environment.j2 index 9a588cfc4..086803d1b 100644 --- a/roles/network_plugin/calico/templates/network-environment.j2 +++ b/roles/network_plugin/calico/templates/network-environment.j2 @@ -3,7 +3,7 @@ DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }} # The Kubernetes master IP -KUBERNETES_MASTER={{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }} +KUBERNETES_MASTER={{ first_kube_master }} # IP and port of etcd instance used by Calico -ETCD_AUTHORITY=127.0.0.1:2379 +ETCD_AUTHORITY={{ etcd_authority }} diff --git a/roles/network_plugin/flannel/handlers/main.yml b/roles/network_plugin/flannel/handlers/main.yml index cb3986312..a503569f6 100644 --- a/roles/network_plugin/flannel/handlers/main.yml +++ b/roles/network_plugin/flannel/handlers/main.yml @@ -4,23 +4,7 @@ ignore_errors: yes notify: restart docker -- name: restart docker - command: /bin/true - notify: - - reload systemd - - reload docker - - reload kubelet - -- name : reload systemd - shell: systemctl daemon-reload - when: ansible_service_mgr == "systemd" - -- name: reload docker - service: - name: docker - state: restarted - -- name: reload kubelet +- name: Flannel | reload kubelet service: name: kubelet state: restarted diff --git a/roles/network_plugin/flannel/meta/main.yml b/roles/network_plugin/flannel/meta/main.yml new file mode 100644 index 000000000..935d9c3bb --- /dev/null +++ b/roles/network_plugin/flannel/meta/main.yml @@ -0,0 +1,6 @@ +--- +dependencies: + - role: download + file: "{{ downloads.flannel_server_helper }}" + - role: download + file: "{{ downloads.flannel }}" diff --git a/roles/network_plugin/flannel/tasks/main.yml b/roles/network_plugin/flannel/tasks/main.yml index 55c47d211..a6fa183ef 100644 --- a/roles/network_plugin/flannel/tasks/main.yml +++ b/roles/network_plugin/flannel/tasks/main.yml @@ -42,19 +42,18 @@ mode: 0644 notify: - restart docker + when: ansible_os_family != "CoreOS" -- name: Flannel | Create docker config symlink for CoreOS - file: - src: "/etc/default/docker" - dest: "/run/flannel_docker_opts.env" - state: link +- name: Flannel | Create docker service path for CoreOS + file: path=/etc/systemd/system/docker.service.d state=directory when: ansible_os_family == "CoreOS" -- name: Flannel | Write docker.service systemd file +- name: Flannel | Create docker dropin for CoreOS template: - src: systemd-docker.service - dest: /lib/systemd/system/docker.service - notify: restart docker - when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS" + src: docker-systemd + dest: "/etc/systemd/system/docker.service.d/flannel-options.conf" + notify: + - restart docker + when: ansible_os_family == "CoreOS" - meta: flush_handlers diff --git a/roles/network_plugin/flannel/templates/docker-systemd b/roles/network_plugin/flannel/templates/docker-systemd new file mode 100644 index 000000000..8d7d6ad83 --- /dev/null +++ b/roles/network_plugin/flannel/templates/docker-systemd @@ -0,0 +1,2 @@ +[Service] +Environment="DOCKER_OPTS=--bip={{ flannel_subnet }} --mtu={{ flannel_mtu }} {% if docker_options is defined %}{{ docker_options }}{% endif %}" diff --git a/roles/network_plugin/flannel/templates/flannel-pod.yml b/roles/network_plugin/flannel/templates/flannel-pod.yml index 62f18f500..15523bdde 100644 --- a/roles/network_plugin/flannel/templates/flannel-pod.yml +++ b/roles/network_plugin/flannel/templates/flannel-pod.yml @@ -17,21 +17,21 @@ path: "/etc/flannel-network.json" containers: - name: "flannel-server-helper" - image: "gcr.io/google_containers/flannel-server-helper:0.1" + image: "{{ flannel_server_helper_image_repo }}:{{ flannel_server_helper_image_tag }}" args: - "--network-config=/etc/flannel-network.json" - "--etcd-prefix=/{{ cluster_name }}/network" - - "--etcd-server=http://{{ groups['etcd'][0] }}:2379" + - "--etcd-server={{ etcd_endpoint }}" volumeMounts: - name: "networkconfig" mountPath: "/etc/flannel-network.json" imagePullPolicy: "Always" - name: "flannel-container" - image: "quay.io/coreos/flannel:0.5.5" + image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}" command: - "/bin/sh" - "-c" - - "/opt/bin/flanneld -etcd-endpoints {% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %} -etcd-prefix /{{ cluster_name }}/network {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}" + - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}" ports: - hostPort: 10253 containerPort: 10253 diff --git a/roles/network_plugin/flannel/templates/systemd-docker.service b/roles/network_plugin/flannel/templates/systemd-docker.service deleted file mode 100644 index 3275c6e24..000000000 --- a/roles/network_plugin/flannel/templates/systemd-docker.service +++ /dev/null @@ -1,28 +0,0 @@ -[Unit] -Description=Docker Application Container Engine -Documentation=http://docs.docker.com -{% if ansible_os_family == "RedHat" %} -After=network.target -Wants=docker-storage-setup.service -{% elif ansible_os_family == "Debian" %} -After=network.target docker.socket -Requires=docker.socket -{% endif %} - -[Service] -Type=notify -EnvironmentFile=-/etc/default/docker -Environment=GOTRACEBACK=crash -ExecStart=/usr/bin/docker daemon \ - $OPTIONS \ - $DOCKER_STORAGE_OPTIONS \ - $DOCKER_NETWORK_OPTIONS \ - $INSECURE_REGISTRY -LimitNOFILE=1048576 -LimitNPROC=1048576 -LimitCORE=infinity -MountFlags=slave -TimeoutStartSec=1min - -[Install] -WantedBy=multi-user.target diff --git a/roles/network_plugin/weave/handlers/main.yml b/roles/network_plugin/weave/handlers/main.yml index 3bb21c346..e821e989b 100644 --- a/roles/network_plugin/weave/handlers/main.yml +++ b/roles/network_plugin/weave/handlers/main.yml @@ -1,37 +1,26 @@ --- -- name: restart docker - command: /bin/true - notify: - - reload systemd - - reload docker - -- name : reload systemd - shell: systemctl daemon-reload - when: ansible_service_mgr == "systemd" - - name: restart weave command: /bin/true notify: - - reload systemd + - Weave | reload systemd - reload weave +- name : Weave | reload systemd + shell: systemctl daemon-reload + when: ansible_service_mgr == "systemd" + - name: restart weaveproxy command: /bin/true notify: - - reload systemd + - Weave | reload systemd - reload weaveproxy - name: restart weaveexpose command: /bin/true notify: - - reload systemd + - Weave | reload systemd - reload weaveexpose -- name: reload docker - service: - name: docker - state: restarted - - name: reload weave service: name: weave diff --git a/roles/network_plugin/weave/tasks/main.yml b/roles/network_plugin/weave/tasks/main.yml index 2cd7b2a60..59cc1bf37 100644 --- a/roles/network_plugin/weave/tasks/main.yml +++ b/roles/network_plugin/weave/tasks/main.yml @@ -9,14 +9,13 @@ notify: - restart docker -- name: Write docker.service systemd file - template: - src: systemd-docker.service - dest: /lib/systemd/system/docker.service - notify: restart docker - when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS" - -- meta: flush_handlers +- name: Weave | Copy cni plugins from hyperkube + command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/" + register: cni_task_result + until: cni_task_result.rc == 0 + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + changed_when: false - name: Weave | Install weave command: rsync -piu "{{ local_release_dir }}/weave/bin/weave" "{{ bin_dir }}/weave" @@ -37,22 +36,24 @@ group: root mode: 0644 notify: - - restart systemd-weave + - restart weave - name: Weave | Write weave systemd init file template: src=weave.service.j2 dest=/etc/systemd/system/weave.service when: ansible_service_mgr == "systemd" - notify: restart systemd-weave + notify: restart weave - name: Weave | Write weaveproxy systemd init file template: src=weaveproxy.service.j2 dest=/etc/systemd/system/weaveproxy.service when: ansible_service_mgr == "systemd" - notify: restart systemd-weaveproxy + notify: restart weaveproxy - name: Weave | Write weaveexpose systemd init file template: src=weaveexpose.service.j2 dest=/etc/systemd/system/weaveexpose.service when: ansible_service_mgr == "systemd" - notify: restart systemd-weaveexpose + notify: restart weaveexpose + +- meta: flush_handlers - name: Weave | Enable weave service: name=weave enabled=yes state=started diff --git a/roles/network_plugin/weave/templates/systemd-docker.service b/roles/network_plugin/weave/templates/systemd-docker.service deleted file mode 100644 index 3275c6e24..000000000 --- a/roles/network_plugin/weave/templates/systemd-docker.service +++ /dev/null @@ -1,28 +0,0 @@ -[Unit] -Description=Docker Application Container Engine -Documentation=http://docs.docker.com -{% if ansible_os_family == "RedHat" %} -After=network.target -Wants=docker-storage-setup.service -{% elif ansible_os_family == "Debian" %} -After=network.target docker.socket -Requires=docker.socket -{% endif %} - -[Service] -Type=notify -EnvironmentFile=-/etc/default/docker -Environment=GOTRACEBACK=crash -ExecStart=/usr/bin/docker daemon \ - $OPTIONS \ - $DOCKER_STORAGE_OPTIONS \ - $DOCKER_NETWORK_OPTIONS \ - $INSECURE_REGISTRY -LimitNOFILE=1048576 -LimitNPROC=1048576 -LimitCORE=infinity -MountFlags=slave -TimeoutStartSec=1min - -[Install] -WantedBy=multi-user.target diff --git a/roles/network_plugin/weave/templates/weave.service.j2 b/roles/network_plugin/weave/templates/weave.service.j2 index a4e9e8d8e..e901c34e7 100644 --- a/roles/network_plugin/weave/templates/weave.service.j2 +++ b/roles/network_plugin/weave/templates/weave.service.j2 @@ -1,8 +1,8 @@ [Unit] Description=Weave Network Documentation=http://docs.weave.works/weave/latest_release/ -Requires=docker.service -After=docker.service +Wants=docker.socket +After=docker.service docker.socket [Service] EnvironmentFile=-/etc/weave.env diff --git a/roles/network_plugin/weave/templates/weaveexpose.service.j2 b/roles/network_plugin/weave/templates/weaveexpose.service.j2 index 03446ee0f..f9931696e 100644 --- a/roles/network_plugin/weave/templates/weaveexpose.service.j2 +++ b/roles/network_plugin/weave/templates/weaveexpose.service.j2 @@ -1,9 +1,7 @@ [Unit] Documentation=http://docs.weave.works/ -Requires=docker.service -Requires=weave.service -After=weave.service -After=docker.service +Wants=docker.socket weave.service +After=docker.service docker.socket weave.service [Service] Type=oneshot diff --git a/roles/network_plugin/weave/templates/weaveproxy.service.j2 b/roles/network_plugin/weave/templates/weaveproxy.service.j2 index fe5032893..29197296f 100644 --- a/roles/network_plugin/weave/templates/weaveproxy.service.j2 +++ b/roles/network_plugin/weave/templates/weaveproxy.service.j2 @@ -1,8 +1,8 @@ [Unit] Description=Weave proxy for Docker API Documentation=http://docs.weave.works/ -Requires=docker.service -After=docker.service +Wants=docker.socket +After=docker.service docker.socket [Service] EnvironmentFile=-/etc/weave.%H.env diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml index 2eb76bb16..0774d324c 100644 --- a/roles/uploads/defaults/main.yml +++ b/roles/uploads/defaults/main.yml @@ -2,25 +2,24 @@ local_release_dir: /tmp # Versions -include_vars: kube_versions.yml +kube_version: v1.4.3 -etcd_version: v3.0.1 -calico_version: v0.20.0 -calico_cni_version: v1.3.1 -weave_version: v1.5.0 +etcd_version: v3.0.6 +calico_version: v0.22.0 +calico_cni_version: v1.4.2 +weave_version: v1.6.1 # Download URL's -kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64" etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz" calico_cni_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico" calico_cni_ipam_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico-ipam" weave_download_url: "https://github.com/weaveworks/weave/releases/download/{{weave_version}}/weave" # Checksums -calico_cni_checksum: "ac05cb9254b5aaa5822cf10325983431bd25489147f2edf9dec7e43d99c43e77" -calico_cni_ipam_checksum: "3df6951a30749c279229e7e318e74ac4e41263996125be65257db7cd25097273" -weave_checksum: "28d2c4e2b1ad8600da69882501eba697679aea10a5e61c769aa3a9ee72b0d89a" -etcd_checksum: "7e5d8db2b8a7cec7a93e531c8ae0f3108c66c7d896a2fb6d8768c067923ce0aa" +calico_cni_checksum: "9cab29764681e9d80da826e4b2cd10841cc01a749e0018867d96dd76a4691548" +calico_cni_ipam_checksum: "09d076b15b791956efee91646e47fdfdcf382db16082cef4f542a9fff7bae172" +weave_checksum: "9bf9d6e5a839e7bcbb28cc00c7acae9d09284faa3e7a3720ca9c2b9e93c68580" +etcd_checksum: "385afd518f93e3005510b7aaa04d38ee4a39f06f5152cd33bb86d4f0c94c7485" downloads: - name: calico-cni-plugin @@ -59,30 +58,3 @@ downloads: unarchive: true owner: "etcd" mode: "0755" - - - name: kubernetes-kubelet - version: "{{kube_version}}" - dest: kubernetes/bin/kubelet - sha256: "{{vars['kube_checksum'][kube_version]['kubelet']}}" - source_url: "{{ kube_download_url }}/kubelet" - url: "{{ kube_download_url }}/kubelet" - owner: "kube" - mode: "0755" - - - name: kubernetes-kubectl - dest: kubernetes/bin/kubectl - version: "{{kube_version}}" - sha256: "{{vars['kube_checksum'][kube_version]['kubectl']}}" - source_url: "{{ kube_download_url }}/kubectl" - url: "{{ kube_download_url }}/kubectl" - owner: "kube" - mode: "0755" - - - name: kubernetes-apiserver - dest: kubernetes/bin/kube-apiserver - version: "{{kube_version}}" - sha256: "{{vars['kube_checksum'][kube_version]['kube_apiserver']}}" - source_url: "{{ kube_download_url }}/kube-apiserver" - url: "{{ kube_download_url }}/kube-apiserver" - owner: "kube" - mode: "0755" diff --git a/roles/uploads/tasks/main.yml b/roles/uploads/tasks/main.yml index 68fcd4320..2d6000599 100644 --- a/roles/uploads/tasks/main.yml +++ b/roles/uploads/tasks/main.yml @@ -1,6 +1,4 @@ --- -- include_vars: "kube_versions.yml" - - name: Create dest directories file: path={{local_release_dir}}/{{item.dest|dirname}} state=directory recurse=yes with_items: '{{downloads}}' diff --git a/roles/uploads/vars/kube_versions.yml b/roles/uploads/vars/kube_versions.yml deleted file mode 100644 index 5b5f64a42..000000000 --- a/roles/uploads/vars/kube_versions.yml +++ /dev/null @@ -1,22 +0,0 @@ -kube_checksum: - v1.2.2: - kube_apiserver: eb1bfd8b877052cbd1991b8c429a1d06661f4cb019905e20e128174f724e16de - kubectl: 473e6924569fba30d4a50cecdc2cae5f31d97d1f662463e85b74a472105dcff4 - kubelet: f16827dc7e7c82f0e215f0fc73eb01e2dfe91a2ec83f9cbcaf8d37c91b64fd3b - v1.2.3: - kube_apiserver_checksum: ebaeeeb72cb29b358337b330617a96355ff2d08a5a523fc1a81beba36cc9d6f9 - kubectl_checksum: 394853edd409a721bcafe4f1360009ef9f845050719fe7d6fc7176f45cc92a8c - kubelet_checksum: 633bb41c51c5c0df0645dd60ba82b12eba39d009eb87bae9227de7d9a89c0797 - v1.2.4: - kube_apiserver: 6ac99b36b02968459e026fcfc234207c66064b5e11816b69dd8fc234b2ffec1e - kubectl: dac61fbd506f7a17540feca691cd8a9d9d628d59661eebce788a50511f578897 - kubelet: 4adaf40592248eef6fd4fa126464915ea41e624a70dc77178089760ed235e341 - v1.2.5: - kube_apiserver: fbe8296ad4b194c06f6802a126d35cd2887dc1aded308d4da2b580f270412b33 - kubectl: 5526a496a84701015485e32c86486e2f23599f7a865164f546e619c6a62f7f19 - kubelet: cd15b929f0190876216f397c2c6e7aa8c08d3b047fd90b4980cd68c8f4896211 - v1.3.0: - kube_apiserver: 431cd312984a29f45590138e990d5c4d537b069b71f2587a72414fabc4fcffdd - kubectl: f40b2d0ff33984e663a0dea4916f1cb9041abecc09b11f9372cdb8049ded95dc - kubelet: bd5f10ccb95fe6e95ddf7ad8a119195c27cb2bce4be6f80c1810ff1a2111496d -kube_version: v1.3.0 diff --git a/scripts/collect-info.yaml b/scripts/collect-info.yaml new file mode 100644 index 000000000..67d4c8b35 --- /dev/null +++ b/scripts/collect-info.yaml @@ -0,0 +1,70 @@ +--- +- hosts: all + become: true + gather_facts: no + + vars: + debug: false + commands: + - name: git_info + cmd: find . -type d -name .git -execdir sh -c 'gen-gitinfos.sh global|head -12' \; + - name: timedate_info + cmd: timedatectl status + - name: space_info + cmd: df -h + - name: kernel_info + cmd: uname -r + - name: distro_info + cmd: cat /etc/issue.net + - name: docker_info + cmd: docker info + - name: ip_info + cmd: ip -4 -o a + - name: route_info + cmd: ip ro + - name: proc_info + cmd: ps auxf | grep -v ]$ + - name: systemctl_info + cmd: systemctl status + - name: systemctl_failed_info + cmd: systemctl --state=failed --no-pager + - name: k8s_info + cmd: kubectl get all --all-namespaces -o wide + - name: errors_info + cmd: journalctl -p err --utc --no-pager + + logs: + - /var/log/ansible.log + - /var/log/ansible/ansible.log + - /var/log/syslog + - /var/log/daemon.log + - /var/log/kern.log + - inventory/inventory.ini + - cluster.yml + + tasks: + - name: Storing commands output + shell: "{{ item.cmd }} 2>&1 | tee {{ item.name }}" + register: output + ignore_errors: true + with_items: "{{commands}}" + + - debug: var=item + with_items: output.results + when: debug + + - name: Fetch results + fetch: src={{ item.name }} dest=/tmp/collect-info/commands + with_items: "{{commands}}" + + - name: Fetch logs + fetch: src={{ item }} dest=/tmp/collect-info/logs + with_items: "{{logs}}" + + - name: Pack results and logs + local_action: shell GZIP=-9 tar --remove-files -cvzf logs.tar.gz -C /tmp collect-info + run_once: true + + - name: Clean up collected command outputs + file: path={{ item.name }} state=absent + with_items: "{{commands}}" diff --git a/scripts/configure-logs.yaml b/scripts/configure-logs.yaml new file mode 100644 index 000000000..d093e9279 --- /dev/null +++ b/scripts/configure-logs.yaml @@ -0,0 +1,39 @@ +--- +- hosts: localhost + become: true + gather_facts: no + + vars: + log_path: /var/log/ansible/ + conf_file: /etc/ansible/ansible.cfg + human_readable_plugin: false + callback_plugin_path: /usr/share/ansible/plugins/callback + + tasks: + - name: LOGS | ensure log path + file: path="{{log_path}}" state=directory owner={{ansible_ssh_user}} + + - name: LOGS | ensure plugin path + file: path="{{callback_plugin_path}}" state=directory owner={{ansible_ssh_user}} + when: human_readable_plugin + + - name: LOGS | get plugin + git: repo=https://gist.github.com/cd706de198c85a8255f6.git dest=/tmp/cd706de198c85a8255f6 + when: human_readable_plugin + + - name: LOGS | install plugin + copy: src=/tmp/cd706de198c85a8255f6/human_log.py dest="{{callback_plugin_path}}" + when: human_readable_plugin + + - name: LOGS | config + lineinfile: + line: "log_path={{log_path}}/ansible.log" + regexp: "^#log_path|^log_path" + dest: "{{conf_file}}" + + - name: LOGS | callback plugin + lineinfile: + line: "callback_plugins={{callback_plugin_path}}" + regexp: "^#callback_plugins|^callback_plugins" + dest: "{{conf_file}}" + when: human_readable_plugin diff --git a/tests/ansible.cfg b/tests/ansible.cfg index 2be6f4d02..f0e4ef652 100644 --- a/tests/ansible.cfg +++ b/tests/ansible.cfg @@ -1,4 +1,7 @@ [ssh_connection] pipelining=True -[defaults] +[defaults] host_key_checking=False +gathering = smart +fact_caching = jsonfile +fact_caching_connection = /tmp diff --git a/coreos-bootstrap.yml b/ubuntu-bootstrap.yml similarity index 67% rename from coreos-bootstrap.yml rename to ubuntu-bootstrap.yml index 88fcb888f..b6adf783d 100644 --- a/coreos-bootstrap.yml +++ b/ubuntu-bootstrap.yml @@ -2,4 +2,4 @@ - hosts: all gather_facts: False roles: - - coreos-bootstrap + - ubuntu-bootstrap