fix issues

contrib/terraform/terraform.py

@@ -273,6 +273,7 @@ def openstack_host(resource, module_name):
         'access_ip_v4': raw_attrs['access_ip_v4'],
         'access_ip_v6': raw_attrs['access_ip_v6'],
         'access_ip': raw_attrs['access_ip_v4'],
+        'access_ip6': raw_attrs['access_ip_v6'],
         'ip': raw_attrs['network.0.fixed_ip_v4'],
         'flavor': parse_dict(raw_attrs, 'flavor',
                              sep='_'),

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -120,7 +120,7 @@ kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112
 kube_network_node_prefix_ipv6: 120
 
 # The port the API Server will be listening on.
-kube_apiserver_ip: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}"
+kube_apiserver_ip: "{{ (kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses) | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}"
 kube_apiserver_port: 6443  # (https)
 
 # Kube-proxy proxyMode configuration.
@@ -218,8 +218,8 @@ resolvconf_mode: host_resolvconf
 # Deploy netchecker app to verify DNS resolve as an HTTP service
 deploy_netchecker: false
 # Ip address of the kubernetes skydns service
-skydns_server: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}"
-skydns_server_secondary: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}"
+skydns_server: "{{ (kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses) | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}"
+skydns_server_secondary: "{{ (kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses) | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
 
 ## Container runtime

roles/container-engine/cri-dockerd/templates/cri-dockerd.service.j2

@@ -7,7 +7,7 @@ Requires=cri-dockerd.socket
 
 [Service]
 Type=notify
-ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} --log-level {{ cri_dockerd_log_level }} {% if enable_dual_stack_networks %}--ipv6-dual-stack=True{% endif %}
+ExecStart={{ bin_dir }}/cri-dockerd --container-runtime-endpoint {{ cri_socket }} --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --network-plugin=cni --pod-cidr={{ kube_pods_subnet_range }} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_version }} --log-level {{ cri_dockerd_log_level }} {% if enable_dual_stack_networks %}--ipv6-dual-stack=True{% endif %}
 
 ExecReload=/bin/kill -s HUP $MAINPID
 TimeoutSec=0

roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -25,16 +25,17 @@
 
 - name: Kubeadm | aggregate all SANs
   set_fact:
-    apiserver_sans: "{{ (sans_base + groups['kube_control_plane'] + sans_apiserver_ip + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_address + sans_override + sans_hostname + sans_fqdn + sans_kube_vip_address) | unique }}"
+    apiserver_sans: "{{ (sans_base + groups['kube_control_plane'] + sans_lb + sans_lb_ip + sans_supp + sans_access_ip + sans_ip + sans_address + sans_override + sans_hostname + sans_fqdn + sans_kube_vip_address) | unique }}"
   vars:
     sans_base:
       - "kubernetes"
       - "kubernetes.default"
       - "kubernetes.default.svc"
       - "kubernetes.default.svc.{{ dns_domain }}"
+      - "{{ kube_apiserver_ip }}"
       - "localhost"
       - "127.0.0.1"
-    sans_apiserver_ip: "{{ [kube_apiserver_ip] if not enable_ipv6only_stack_networks else [] }}"
+      - "::1"
     sans_lb: "{{ [apiserver_loadbalancer_domain_name] if apiserver_loadbalancer_domain_name is defined else [] }}"
     sans_lb_ip: "{{ [loadbalancer_apiserver.address] if loadbalancer_apiserver is defined and loadbalancer_apiserver.address is defined else [] }}"
     sans_supp: "{{ supplementary_addresses_in_ssl_keys if supplementary_addresses_in_ssl_keys is defined else [] }}"

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta3.yaml.j2

@@ -94,9 +94,9 @@ dns:
   imageTag: {{ coredns_image_tag }}
 networking:
   dnsDomain: {{ dns_domain }}
-  serviceSubnet: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+  serviceSubnet: "{{ kube_service_addresses_range }}"
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-  podSubnet: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+  podSubnet: "{{ kube_pods_subnet_range }}"
 {% endif %}
 {% if kubeadm_feature_gates %}
 featureGates:
@@ -143,7 +143,7 @@ apiServer:
     etcd-servers-overrides: "/events#{{ etcd_events_access_addresses_semicolon }}"
 {% endif %}
     service-node-port-range: {{ kube_apiserver_node_port_range }}
-    service-cluster-ip-range: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+    service-cluster-ip-range: "{{ kube_service_addresses_range }}"
     kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
     profiling: "{{ kube_profiling }}"
     request-timeout: "{{ kube_apiserver_request_timeout }}"
@@ -285,17 +285,17 @@ apiServer:
 {% endif %}
   certSANs:
 {% for san in apiserver_sans %}
-  - "{{ san }}"
+  - {{ san }}
 {% endfor %}
   timeoutForControlPlane: 5m0s
 controllerManager:
   extraArgs:
     node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
     node-monitor-period: {{ kube_controller_node_monitor_period }}
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-    cluster-cidr: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+    cluster-cidr: "{{ kube_pods_subnet_range }}"
 {% endif %}
-    service-cluster-ip-range: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+    service-cluster-ip-range: "{{ kube_service_addresses_range }}"
 {% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
     allocate-node-cidrs: "false"
 {% elif enable_ipv6only_stack_networks %}
@@ -383,7 +383,7 @@ clientConnection:
   kubeconfig: {{ kube_proxy_client_kubeconfig }}
   qps: {{ kube_proxy_client_qps }}
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-clusterCIDR: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+clusterCIDR: "{{ kube_pods_subnet_range }}"
 {% endif %}
 configSyncPeriod: {{ kube_proxy_config_sync_period }}
 conntrack:

roles/kubernetes/control-plane/templates/kubeadm-config.v1beta4.yaml.j2

@@ -106,9 +106,9 @@ dns:
   imageTag: {{ coredns_image_tag }}
 networking:
   dnsDomain: {{ dns_domain }}
-  serviceSubnet: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+  serviceSubnet: "{{ kube_service_addresses_range }}"
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-  podSubnet: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+  podSubnet: "{{ kube_pods_subnet_range }}"
 {% endif %}
 {% if kubeadm_feature_gates %}
 featureGates:
@@ -169,7 +169,7 @@ apiServer:
   - name: service-node-port-range
     value: "{{ kube_apiserver_node_port_range }}"
   - name: service-cluster-ip-range
-    value: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+    value: "{{ kube_service_addresses_range }}"
   - name: kubelet-preferred-address-types
     value: "{{ kubelet_preferred_address_types }}"
   - name: profiling
@@ -341,7 +341,7 @@ apiServer:
 {% endif %}
   certSANs:
 {% for san in apiserver_sans %}
-  - "{{ san }}"
+  - {{ san }}
 {% endfor %}
 controllerManager:
   extraArgs:
@@ -351,10 +351,10 @@ controllerManager:
     value: "{{ kube_controller_node_monitor_period }}"
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
   - name: cluster-cidr
-    value: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+    value: "{{ kube_pods_subnet_range }}"
 {% endif %}
   - name: service-cluster-ip-range
-    value: "{{ [kube_service_addresses if not enable_ipv6only_stack_networks, kube_service_addresses_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+    value: "{{ kube_service_addresses_range }}"
 {% if kube_network_plugin is defined and kube_network_plugin == "calico" and not calico_ipam_host_local %}
   - name: allocate-node-cidrs
     value: "false"
@@ -479,7 +479,7 @@ clientConnection:
   kubeconfig: {{ kube_proxy_client_kubeconfig }}
   qps: {{ kube_proxy_client_qps }}
 {% if kube_network_plugin is defined and kube_network_plugin not in ["kube-ovn"] %}
-clusterCIDR: "{{ [kube_pods_subnet if not enable_ipv6only_stack_networks, kube_pods_subnet_ipv6 if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}{{ '' }}"
+clusterCIDR: "{{ kube_pods_subnet_range }}"
 {% endif %}
 configSyncPeriod: {{ kube_proxy_config_sync_period }}
 conntrack:

roles/kubernetes/preinstall/tasks/0090-etchosts.yml

@@ -3,12 +3,13 @@
   set_fact:
     etc_hosts_inventory_block: |-
       {% for item in (groups['k8s_cluster'] + groups['etcd'] | default([]) + groups['calico_rr'] | default([])) | unique -%}
-      {% if enable_ipv6only_stack_networks -%}
-      {{ hostvars[item]['access_ip_v6'] | default(hostvars[item]['ip6'] | default(hostvars[item]['ansible_default_ipv6']['address'])) }}
+      {% if 'access_ip' in hostvars[item] or 'ip' in hostvars[item] or 'ansible_default_ipv4' in hostvars[item] -%}
+      {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}
       {%- if ('ansible_hostname' in hostvars[item] and item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }} {% else %} {{ item }}.{{ dns_domain }} {{ item }} {% endif %}
 
-      {% else %}
-      {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}
+      {% endif %}
+      {% if ('access_ip6' in hostvars[item] or 'ip6' in hostvars[item] or 'ansible_default_ipv6' in hostvars[item]) and (enable_ipv6only_stack_networks or enable_dual_stack_networks) -%}
+      {{ hostvars[item]['access_ip6'] | default(hostvars[item]['ip6'] | default(hostvars[item]['ansible_default_ipv6']['address'])) }}
       {%- if ('ansible_hostname' in hostvars[item] and item != hostvars[item]['ansible_hostname']) %} {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }} {{ hostvars[item]['ansible_hostname'] }} {% else %} {{ item }}.{{ dns_domain }} {{ item }} {% endif %}
 
       {% endif %}

roles/kubespray-defaults/defaults/main/main.yml

@@ -131,8 +131,8 @@ resolvconf_mode: host_resolvconf
 # Deploy netchecker app to verify DNS resolve as an HTTP service
 deploy_netchecker: false
 # Ip address of the kubernetes DNS service (called skydns for historical reasons)
-skydns_server: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}"
-skydns_server_secondary: "{{ kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}"
+skydns_server: "{{ (kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses) | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(3) | ansible.utils.ipaddr('address') }}"
+skydns_server_secondary: "{{ (kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses) | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(4) | ansible.utils.ipaddr('address') }}"
 dns_domain: "{{ cluster_name }}"
 docker_dns_search_domains:
 - 'default.svc.{{ dns_domain }}'
@@ -246,11 +246,37 @@ kube_pods_subnet_ipv6: fd85:ee78:d8a6:8607::1:0000/112
 # This provides room for 254 pods per node.
 kube_network_node_prefix_ipv6: 120
 
+# Configure all of service addresses in one variable.
+# Merge all of different stack.
+kube_service_addresses_range: >-
+  {%- if enable_ipv6only_stack_networks -%}
+  {{ kube_service_addresses_ipv6 }}
+  {%- elif enable_dual_stack_networks -%}
+  {{ kube_service_addresses }},{{ kube_service_addresses_ipv6 }}
+  {%- else -%}
+  {{ kube_service_addresses }}
+  {%- endif -%}
+
+# Configure all of pods subnets in one variable.
+# Merge all of different stack.
+kube_pods_subnet_range: >-
+  {%- if enable_ipv6only_stack_networks -%}
+  {{ kube_pods_subnet_ipv6 }}
+  {%- elif enable_dual_stack_networks -%}
+  {{ kube_pods_subnet }},{{ kube_pods_subnet_ipv6 }}
+  {%- else -%}
+  {{ kube_pods_subnet }}
+  {%- endif -%}
+
+# Configure variable with default stack for ip.
+# IPv6 get more priority only with enable_ipv6only_stack_networks options.
+default_net_mode: "{{ '6' if enable_ipv6only_stack_networks else '' }}"
+
 # The virtual cluster IP, real host IPs and ports the API Server will be
 # listening on.
 # NOTE: loadbalancer_apiserver_localhost somewhat alters the final API enpdoint
 # access IP value (automatically evaluated below)
-kube_apiserver_ip: "{{ kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}"
+kube_apiserver_ip: "{{ (kube_service_addresses_ipv6 if enable_ipv6only_stack_networks else kube_service_addresses) | ansible.utils.ipaddr('net') | ansible.utils.ipaddr(1) | ansible.utils.ipaddr('address') }}"
 
 # NOTE: If you specific address/interface and use loadbalancer_apiserver_localhost
 # loadbalancer_apiserver_localhost (nginx/haproxy) will deploy on control plane nodes on 127.0.0.1:{{ loadbalancer_apiserver_port | default(kube_apiserver_port) }} too.
@@ -551,9 +577,9 @@ ssl_ca_dirs: |-
 
 # Vars for pointing to kubernetes api endpoints
 kube_apiserver_count: "{{ groups['kube_control_plane'] | length }}"
-kube_apiserver_address: "{{ ip6 | default(hostvars[inventory_hostname]['fallback_ip6']) if enable_ipv6only_stack_networks else ip | default(hostvars[inventory_hostname]['fallback_ip']) }}"
+kube_apiserver_address: "{{ vars['ip' + default_net_mode] | default(hostvars[inventory_hostname]['fallback_ip' + default_net_mode]) }}"
 kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}"
-first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['access_ip'] | default(hostvars[groups['kube_control_plane'][0]]['ip'] | default(hostvars[groups['kube_control_plane'][0]]['fallback_ip'])) if not enable_ipv6only_stack_networks else hostvars[groups['kube_control_plane'][0]]['access_ip_v6'] | default(hostvars[groups['kube_control_plane'][0]]['ip6'] | default(hostvars[groups['kube_control_plane'][0]]['fallback_ip6'])) }}"
+first_kube_control_plane_address: "{{ hostvars[groups['kube_control_plane'][0]]['access_ip' + default_net_mode] | default(hostvars[groups['kube_control_plane'][0]]['ip' + default_net_mode] | default(hostvars[groups['kube_control_plane'][0]]['fallback_ip' + default_net_mode])) }}"
 loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}"
 loadbalancer_apiserver_type: "nginx"
 # applied if only external loadbalancer_apiserver is defined, otherwise ignored

roles/kubespray-defaults/tasks/no_proxy.yml

@@ -23,7 +23,7 @@
       {%- if additional_no_proxy is defined -%}
       {{ additional_no_proxy }},
       {%- endif -%}
-      127.0.0.1,localhost,{{ kube_service_addresses }},{{ kube_pods_subnet }},svc,svc.{{ dns_domain }}
+      127.0.0.1,::1,localhost,{{ kube_service_addresses_range }},{{ kube_pods_subnet }},svc,svc.{{ dns_domain }}
   delegate_to: localhost
   connection: local
   delegate_facts: true

roles/network_plugin/calico/tasks/install.yml

@@ -350,7 +350,7 @@
               {% if not calico_no_global_as_num | default(false) %}"asNumber": {{ global_as_num }},{% endif %}
               "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled | default('true') }} ,
               {% if calico_advertise_cluster_ips | default(false) %}
-              "serviceClusterIPs": [{{ ['{"cidr": "' + kube_service_addresses + '"}' if not enable_ipv6only_stack_networks, '{"cidr": "' + kube_service_addresses_ipv6 + '"}' if (enable_dual_stack_networks or enable_ipv6only_stack_networks)] | reject('match', '^$') | join(',') }}],{% endif %}
+              "serviceClusterIPs": [{% for cidr in kube_service_addresses_range.split(",") %}{{ "," if not loop.first }}{"cidr": "{{ cidr }}"}{% endfor %}],{% endif %}
               {% if calico_advertise_service_loadbalancer_ips | length > 0  %}"serviceLoadBalancerIPs": {{ _service_loadbalancer_ips }},{% endif %}
               "serviceExternalIPs": {{ _service_external_ips | default([]) }}
             }