From 009d2ffc6c7c5fafeb950293e96a60b6a9b986de Mon Sep 17 00:00:00 2001 From: rongzhang Date: Sat, 8 Sep 2018 18:13:33 +0800 Subject: [PATCH 1/7] Add insecure_registry config to docker options --- inventory/sample/group_vars/all/docker.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/inventory/sample/group_vars/all/docker.yml b/inventory/sample/group_vars/all/docker.yml index 3fb169e33..fcc628b7c 100644 --- a/inventory/sample/group_vars/all/docker.yml +++ b/inventory/sample/group_vars/all/docker.yml @@ -15,10 +15,17 @@ docker_daemon_graph: "/var/lib/docker" ## A string of extra options to pass to the docker daemon. ## This string should be exactly as you wish it to appear. -## An obvious use case is allowing insecure-registry access -## to self hosted registries like so: +## An obvious use case is allowing insecure-registry access to self hosted registries. +## Can be ipddress and domain_name. +## example define 172.19.16.11 or mirror.registry.io +#insecure_registries: +# - mirror.registry.io +# - 172.19.16.11 docker_options: >- - --insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }} {{ docker_log_opts }} + {%- if insecure_registries is defined %} + {{ insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} + {%- endif -%} + --graph={{ docker_daemon_graph }} {{ docker_log_opts }} {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %} --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current --default-runtime=docker-runc --exec-opt native.cgroupdriver=systemd From c41ca22a78b68db627c9f01debd8e1ca6369843c Mon Sep 17 00:00:00 2001 From: rongzhang Date: Sun, 9 Sep 2018 00:21:33 +0800 Subject: [PATCH 2/7] Planning the configuration of docker parameters --- inventory/sample/group_vars/all/docker.yml | 42 +++++++++++---------- roles/docker/defaults/main.yml | 3 -- roles/kubespray-defaults/defaults/main.yaml | 26 ------------- 3 files changed, 23 insertions(+), 48 deletions(-) diff --git a/inventory/sample/group_vars/all/docker.yml b/inventory/sample/group_vars/all/docker.yml index fcc628b7c..cea860c36 100644 --- a/inventory/sample/group_vars/all/docker.yml +++ b/inventory/sample/group_vars/all/docker.yml @@ -1,42 +1,46 @@ ## Uncomment this if you want to force overlay/overlay2 as docker storage driver ## Please note that overlay2 is only supported on newer kernels - #docker_storage_options: -s overlay2 ## Uncomment this if you have more than 3 nameservers, then we'll only use the first 3. - -#docker_dns_servers_strict: false +docker_dns_servers_strict: false # Path used to store Docker data docker_daemon_graph: "/var/lib/docker" ## Used to set docker daemon iptables options to true -#docker_iptables_enabled: "true" +docker_iptables_enabled: "false" + +# Docker log options +# Rotate container stderr/stdout logs at 50m and keep last 5 +docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" + +# define docker bin_dir +docker_bin_dir: "/usr/bin" -## A string of extra options to pass to the docker daemon. -## This string should be exactly as you wish it to appear. ## An obvious use case is allowing insecure-registry access to self hosted registries. ## Can be ipddress and domain_name. ## example define 172.19.16.11 or mirror.registry.io #insecure_registries: # - mirror.registry.io # - 172.19.16.11 -docker_options: >- - {%- if insecure_registries is defined %} - {{ insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} - {%- endif -%} - --graph={{ docker_daemon_graph }} {{ docker_log_opts }} - {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %} - --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current - --default-runtime=docker-runc --exec-opt native.cgroupdriver=systemd - --userland-proxy-path=/usr/libexec/docker/docker-proxy-current --signature-verification=false - {%- endif -%} - -docker_bin_dir: "/usr/bin" ## If non-empty will override default system MounFlags value. ## This option takes a mount propagation flag: shared, slave ## or private, which control whether mounts in the file system ## namespace set up for docker will receive or propagate mounts ## and unmounts. Leave empty for system default -docker_mount_flags: +#docker_mount_flags: + +## A string of extra options to pass to the docker daemon. +## This string should be exactly as you wish it to appear. +docker_options: >- + {%- if insecure_registries is defined -%} + {{ insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} + {%- endif %} + --graph={{ docker_daemon_graph }} {{ docker_log_opts }} + {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %} + --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current + --default-runtime=docker-runc --exec-opt native.cgroupdriver=systemd + --userland-proxy-path=/usr/libexec/docker/docker-proxy-current --signature-verification=false + {%- endif -%} diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml index 4a3b24f80..2d6681b34 100644 --- a/roles/docker/defaults/main.yml +++ b/roles/docker/defaults/main.yml @@ -40,6 +40,3 @@ dockerproject_rh_repo_base_url: 'https://yum.dockerproject.org/repo/main/centos/ dockerproject_rh_repo_gpgkey: 'https://yum.dockerproject.org/gpg' dockerproject_apt_repo_base_url: 'https://apt.dockerproject.org/repo' dockerproject_apt_repo_gpgkey: 'https://apt.dockerproject.org/gpg' - -# Used to set docker daemon iptables options -docker_iptables_enabled: "false" diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 970d7eecf..af886af91 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -142,32 +142,6 @@ kube_api_aggregator_routing: false # Container for runtime container_manager: docker -# Path used to store Docker data -docker_daemon_graph: "/var/lib/docker" - -# Docker log options -# Rotate container stderr/stdout logs at 50m and keep last 5 -docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" - -## A string of extra options to pass to the docker daemon. -## This string should be exactly as you wish it to appear. -## An obvious use case is allowing insecure-registry access -## to self hosted registries like so: -docker_options: > - --insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }} {{ docker_log_opts }} - {% if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %} - --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current - --default-runtime=docker-runc --exec-opt native.cgroupdriver=systemd - --userland-proxy-path=/usr/libexec/docker/docker-proxy-current --signature-verification=false - {% endif %} - -## If non-empty will override default system MounFlags value. -## This option takes a mount propagation flag: shared, slave -## or private, which control whether mounts in the file system -## namespace set up for docker will receive or propagate mounts -## and unmounts. Leave empty for system default -docker_mount_flags: - # Settings for containerized control plane (etcd/kubelet/secrets) etcd_deployment_type: docker kubelet_deployment_type: docker From cb133cba68440de7124fdc5d58674419e8779c04 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Sun, 9 Sep 2018 01:20:48 +0800 Subject: [PATCH 3/7] Add registry_mirrors config to docker options --- inventory/sample/group_vars/all/docker.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/inventory/sample/group_vars/all/docker.yml b/inventory/sample/group_vars/all/docker.yml index cea860c36..b32b55253 100644 --- a/inventory/sample/group_vars/all/docker.yml +++ b/inventory/sample/group_vars/all/docker.yml @@ -25,6 +25,11 @@ docker_bin_dir: "/usr/bin" # - mirror.registry.io # - 172.19.16.11 +## Add other registry,example China registry mirror. +#registry_mirrors: +# - https://registry.docker-cn.com +# - https://mirror.aliyuncs.com + ## If non-empty will override default system MounFlags value. ## This option takes a mount propagation flag: shared, slave ## or private, which control whether mounts in the file system @@ -38,6 +43,9 @@ docker_options: >- {%- if insecure_registries is defined -%} {{ insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} {%- endif %} + {% if registry_mirrors is defined -%} + {{ registry_mirrors | map('regex_replace', '^(.*)$', '--registry-mirror=\1' ) | list | join(' ') }} + {%- endif %} --graph={{ docker_daemon_graph }} {{ docker_log_opts }} {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %} --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current From 20caaf9d1f79fdd684c7f816bfbef2fef77d2738 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Sun, 9 Sep 2018 02:09:02 +0800 Subject: [PATCH 4/7] Delete gitignore file --- roles/docker/.gitignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 roles/docker/.gitignore diff --git a/roles/docker/.gitignore b/roles/docker/.gitignore deleted file mode 100644 index e11a4750e..000000000 --- a/roles/docker/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.*.swp -.vagrant From b249b06036b70aae7cb133d08f02d099a2c7a838 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Sun, 9 Sep 2018 10:15:33 +0800 Subject: [PATCH 5/7] Move docker options to kubespray-defaults --- inventory/sample/group_vars/all/docker.yml | 7 +++ roles/kubespray-defaults/defaults/main.yaml | 62 +++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/inventory/sample/group_vars/all/docker.yml b/inventory/sample/group_vars/all/docker.yml index b32b55253..4f8969bd7 100644 --- a/inventory/sample/group_vars/all/docker.yml +++ b/inventory/sample/group_vars/all/docker.yml @@ -2,6 +2,13 @@ ## Please note that overlay2 is only supported on newer kernels #docker_storage_options: -s overlay2 +## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7. +docker_container_storage_setup: false + +## It must be define a disk path for docker_container_storage_setup_devs. +## Otherwise docker-storage-setup will be executed incorrectly. +#docker_container_storage_setup_devs: /dev/vdb + ## Uncomment this if you have more than 3 nameservers, then we'll only use the first 3. docker_dns_servers_strict: false diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index af886af91..b6514aad7 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -142,6 +142,68 @@ kube_api_aggregator_routing: false # Container for runtime container_manager: docker +## Uncomment this if you want to force overlay/overlay2 as docker storage driver +## Please note that overlay2 is only supported on newer kernels +#docker_storage_options: -s overlay2 + +## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7. +docker_container_storage_setup: false + +## It must be define a disk path for docker_container_storage_setup_devs. +## Otherwise docker-storage-setup will be executed incorrectly. +#docker_container_storage_setup_devs: /dev/vdb + +## Uncomment this if you have more than 3 nameservers, then we'll only use the first 3. +docker_dns_servers_strict: false + +# Path used to store Docker data +docker_daemon_graph: "/var/lib/docker" + +## Used to set docker daemon iptables options to true +docker_iptables_enabled: "false" + +# Docker log options +# Rotate container stderr/stdout logs at 50m and keep last 5 +docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" + +# define docker bin_dir +docker_bin_dir: "/usr/bin" + +## An obvious use case is allowing insecure-registry access to self hosted registries. +## Can be ipddress and domain_name. +## example define 172.19.16.11 or mirror.registry.io +#insecure_registries: +# - mirror.registry.io +# - 172.19.16.11 + +## Add other registry,example China registry mirror. +#registry_mirrors: +# - https://registry.docker-cn.com +# - https://mirror.aliyuncs.com + +## If non-empty will override default system MounFlags value. +## This option takes a mount propagation flag: shared, slave +## or private, which control whether mounts in the file system +## namespace set up for docker will receive or propagate mounts +## and unmounts. Leave empty for system default +#docker_mount_flags: + +## A string of extra options to pass to the docker daemon. +## This string should be exactly as you wish it to appear. +docker_options: >- + {%- if insecure_registries is defined -%} + {{ insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} + {%- endif %} + {% if registry_mirrors is defined -%} + {{ registry_mirrors | map('regex_replace', '^(.*)$', '--registry-mirror=\1' ) | list | join(' ') }} + {%- endif %} + --graph={{ docker_daemon_graph }} {{ docker_log_opts }} + {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %} + --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current + --default-runtime=docker-runc --exec-opt native.cgroupdriver=systemd + --userland-proxy-path=/usr/libexec/docker/docker-proxy-current --signature-verification=false + {%- endif -%} + # Settings for containerized control plane (etcd/kubelet/secrets) etcd_deployment_type: docker kubelet_deployment_type: docker From 51794e4c139f5d92b737edc22d8b1de76f82a932 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Sun, 9 Sep 2018 10:51:35 +0800 Subject: [PATCH 6/7] Deploying k8s clusters in a private environment --- roles/download/defaults/main.yml | 7 +++++++ roles/kubespray-defaults/defaults/main.yaml | 13 +++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 4437861ae..d6c772a6d 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -61,6 +61,13 @@ kubeadm_checksum: 6b17720a65b8ff46efe92a5544f149c39a221910d89939838d75581d4e6924 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 # Containers +# In some cases, we need a way to set --registry-mirror or --insecure-registry for docker, +# it helps a lot for local private development or bare metal environment. +# So you need define --registry-mirror or --insecure-registry, and modify the following url address. +# example: +# You need to deploy kubernetes cluster on local private development. +# Also provide the address of your own private registry. +# And use --insecure-registry options for docker etcd_image_repo: "quay.io/coreos/etcd" etcd_image_tag: "{{ etcd_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}" flannel_image_repo: "quay.io/coreos/flannel" diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index b6514aad7..6288811eb 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -144,14 +144,14 @@ container_manager: docker ## Uncomment this if you want to force overlay/overlay2 as docker storage driver ## Please note that overlay2 is only supported on newer kernels -#docker_storage_options: -s overlay2 +# docker_storage_options: -s overlay2 ## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7. docker_container_storage_setup: false ## It must be define a disk path for docker_container_storage_setup_devs. ## Otherwise docker-storage-setup will be executed incorrectly. -#docker_container_storage_setup_devs: /dev/vdb +# docker_container_storage_setup_devs: /dev/vdb ## Uncomment this if you have more than 3 nameservers, then we'll only use the first 3. docker_dns_servers_strict: false @@ -166,18 +166,15 @@ docker_iptables_enabled: "false" # Rotate container stderr/stdout logs at 50m and keep last 5 docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" -# define docker bin_dir -docker_bin_dir: "/usr/bin" - ## An obvious use case is allowing insecure-registry access to self hosted registries. ## Can be ipddress and domain_name. ## example define 172.19.16.11 or mirror.registry.io -#insecure_registries: +# insecure_registries: # - mirror.registry.io # - 172.19.16.11 ## Add other registry,example China registry mirror. -#registry_mirrors: +# registry_mirrors: # - https://registry.docker-cn.com # - https://mirror.aliyuncs.com @@ -186,7 +183,7 @@ docker_bin_dir: "/usr/bin" ## or private, which control whether mounts in the file system ## namespace set up for docker will receive or propagate mounts ## and unmounts. Leave empty for system default -#docker_mount_flags: +# docker_mount_flags: ## A string of extra options to pass to the docker daemon. ## This string should be exactly as you wish it to appear. From f557b544898db9e4a5dee9b55ced6f9fc2331d19 Mon Sep 17 00:00:00 2001 From: rongzhang Date: Mon, 10 Sep 2018 18:05:49 +0800 Subject: [PATCH 7/7] Add `docker_` to values --- inventory/sample/group_vars/all/docker.yml | 12 ++++++------ roles/kubespray-defaults/defaults/main.yaml | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/inventory/sample/group_vars/all/docker.yml b/inventory/sample/group_vars/all/docker.yml index 4f8969bd7..c1a1dd85a 100644 --- a/inventory/sample/group_vars/all/docker.yml +++ b/inventory/sample/group_vars/all/docker.yml @@ -28,12 +28,12 @@ docker_bin_dir: "/usr/bin" ## An obvious use case is allowing insecure-registry access to self hosted registries. ## Can be ipddress and domain_name. ## example define 172.19.16.11 or mirror.registry.io -#insecure_registries: +#docker_insecure_registries: # - mirror.registry.io # - 172.19.16.11 ## Add other registry,example China registry mirror. -#registry_mirrors: +#docker_registry_mirrors: # - https://registry.docker-cn.com # - https://mirror.aliyuncs.com @@ -47,11 +47,11 @@ docker_bin_dir: "/usr/bin" ## A string of extra options to pass to the docker daemon. ## This string should be exactly as you wish it to appear. docker_options: >- - {%- if insecure_registries is defined -%} - {{ insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} + {%- if docker_insecure_registries is defined -%} + {{ docker_insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} {%- endif %} - {% if registry_mirrors is defined -%} - {{ registry_mirrors | map('regex_replace', '^(.*)$', '--registry-mirror=\1' ) | list | join(' ') }} + {% if docker_registry_mirrors is defined -%} + {{ docker_registry_mirrors | map('regex_replace', '^(.*)$', '--registry-mirror=\1' ) | list | join(' ') }} {%- endif %} --graph={{ docker_daemon_graph }} {{ docker_log_opts }} {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %} diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 6288811eb..0a3c4c523 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -169,12 +169,12 @@ docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" ## An obvious use case is allowing insecure-registry access to self hosted registries. ## Can be ipddress and domain_name. ## example define 172.19.16.11 or mirror.registry.io -# insecure_registries: +# docker_insecure_registries: # - mirror.registry.io # - 172.19.16.11 ## Add other registry,example China registry mirror. -# registry_mirrors: +# docker_registry_mirrors: # - https://registry.docker-cn.com # - https://mirror.aliyuncs.com @@ -188,11 +188,11 @@ docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" ## A string of extra options to pass to the docker daemon. ## This string should be exactly as you wish it to appear. docker_options: >- - {%- if insecure_registries is defined -%} - {{ insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} + {%- if docker_insecure_registries is defined -%} + {{ docker_insecure_registries | map('regex_replace', '^(.*)$', '--insecure-registry=\1' ) | list | join(' ') }} {%- endif %} - {% if registry_mirrors is defined -%} - {{ registry_mirrors | map('regex_replace', '^(.*)$', '--registry-mirror=\1' ) | list | join(' ') }} + {% if docker_registry_mirrors is defined -%} + {{ docker_registry_mirrors | map('regex_replace', '^(.*)$', '--registry-mirror=\1' ) | list | join(' ') }} {%- endif %} --graph={{ docker_daemon_graph }} {{ docker_log_opts }} {%- if ansible_architecture == "aarch64" and ansible_os_family == "RedHat" %}