Merge "Add nspawn container driver"

This commit is contained in:
Zuul 2018-02-14 07:07:59 +00:00 committed by Gerrit Code Review
commit 25b671e344
23 changed files with 513 additions and 25 deletions

View File

@ -194,3 +194,14 @@
scm: git
src: https://github.com/logan2211/ansible-haproxy-endpoints
version: 49901861b16b8afaa9bccdbc649ac956610ff22b
# Once the initial commit for nspawn has been merged we can work on porting
# these roles over into the openstack-ansible namspace.
- name: nspawn_container_create
src: https://github.com/cloudnull/ansible-nspawn_container_create
scm: git
version: master
- name: nspawn_host
src: https://github.com/cloudnull/ansible-nspawn_host
scm: git
version: master

View File

@ -120,6 +120,9 @@ global_overrides:
shared-infra_hosts:
aio1:
ip: 172.29.236.100
container_vars:
# Optional | container_tech for a target host, default is "lxc".
container_tech: "{{ container_tech }}"
repo-infra_hosts:
aio1:

View File

@ -325,6 +325,37 @@
# infra3:
# ip: 172.29.236.103
#
# List of target hosts on which to deploy shared infrastructure services
# and define the the container_tech for a specific infra node. If this setting
# is omitted the inventory generation system will default to "lxc". Accpetable
# options are "lxc" and "nspawn".
#
# Level: <value> (required, string)
# Hostname of a target host.
#
# Option: ip (required, string)
# IP address of this target host, typically the IP address assigned to
# the management bridge.
#
# Level: container_vars (required)
# Contains storage options for this target host.
#
# Example:
#
# Define three shared infrastructure hosts with different "container_tech":
#
# shared-infra_hosts:
# infra1:
# ip: 172.29.236.101
# container_vars:
# container_tech: nspawn
# infra2:
# ip: 172.29.236.102
# container_vars:
# container_tech: lxc
# infra3:
# ip: 172.29.236.103
#
# --------
#
# Level: repo-infra_hosts (required)
@ -764,4 +795,3 @@
# address is ``193.0.14.129``. To change this default,
# set the ``keepalived_ping_address`` variable in the
# ``user_variables.yml`` file.

View File

@ -81,8 +81,14 @@ global_overrides:
shared-infra_hosts:
infra1:
ip: 172.29.236.11
container_vars:
# Optional | Example setting the container_tech for a target host.
container_tech: lxc
infra2:
ip: 172.29.236.12
container_vars:
# Optional | Example setting the container_tech for a target host.
container_tech: nspawn
infra3:
ip: 172.29.236.13

View File

@ -66,7 +66,8 @@ service_region: RegionOne
## OpenStack Domain
openstack_domain: openstack.local
lxc_container_domain: "{{ openstack_domain }}"
lxc_container_domain: "{{ container_domain }}"
container_domain: "{{ openstack_domain }}"
## DHCP Domain Name
dhcp_domain: openstacklocal

View File

@ -28,4 +28,3 @@ lxc_container_wait_params:
delay: 3
# Wait 60 seconds for the container to respond
timeout: 60

View File

@ -114,8 +114,9 @@ class MissingStaticRouteInfo(Exception):
class LxcHostsDefined(Exception):
def __init__(self):
self.message = ("The group 'lxc_hosts' must not be defined in config;"
" it will be dynamically generated.")
self.message = ("The group 'lxc_hosts' or 'nspawn_hosts' must not"
" be defined in config; it will be dynamically "
" generated.")
def __str__(self):
return self.message
@ -754,9 +755,10 @@ def populate_lxc_hosts(inventory):
:param inventory: The dictionary containing the Ansible inventory
"""
host_nodes = _find_lxc_hosts(inventory)
inventory['lxc_hosts'] = {'hosts': host_nodes}
logger.debug("Created lxc_hosts group.")
lxc_host_nodes, nspawn_host_nodes = _find_lxc_hosts(inventory)
inventory['nspawn_hosts'] = {'hosts': nspawn_host_nodes}
inventory['lxc_hosts'] = {'hosts': lxc_host_nodes}
logger.debug("Created lxc_hosts and nspawn_hosts group.")
def _find_lxc_hosts(inventory):
@ -773,16 +775,33 @@ def _find_lxc_hosts(inventory):
:returns: List of hostnames that are LXC hosts
:rtype: list
"""
host_nodes = []
lxc_host_nodes = []
nspawn_host_nodes = []
for host, hostvars in inventory['_meta']['hostvars'].items():
physical_host = hostvars.get('physical_host', None)
container_tech = hostvars.get('container_tech', 'lxc')
hostvars['container_tech'] = container_tech
# We want this node's "parent", so append the physical host
if not host == physical_host:
appended = du.append_if(array=host_nodes, item=physical_host)
if container_tech == 'lxc':
appended = du.append_if(
array=lxc_host_nodes,
item=physical_host
)
elif container_tech == 'nspawn':
appended = du.append_if(
array=nspawn_host_nodes,
item=physical_host
)
else:
appended = None
if appended:
logger.debug("%s added to lxc_hosts group", physical_host)
return host_nodes
logger.debug("%s added to lxc_hosts and nspawn_hosts group",
physical_host)
return lxc_host_nodes, nspawn_host_nodes
def _ensure_inventory_uptodate(inventory, container_skel):
@ -908,7 +927,9 @@ def _check_multiple_ips_to_host(config):
def _check_lxc_hosts(config):
if 'lxc_hosts' in config.keys():
raise LxcHostsDefined()
logger.debug("lxc_hosts group not defined")
elif 'nspawn_hosts' in config.keys():
raise LxcHostsDefined()
logger.debug("lxc_hosts or nspawn_hosts group not defined")
def _check_group_branches(config, physical_skel):

View File

@ -0,0 +1,134 @@
---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Pull systemd version
command: "systemctl --version"
changed_when: false
register: systemd_version
delegate_to: "{{ physical_host }}"
tags:
- skip_ansible_lint
- always
- name: Set facts
set_fact:
nspawn_systemd_version: "{{ systemd_version.stdout_lines[0].split()[-1] }}"
tags:
- always
- name: Escape quote container name
command: "systemd-escape {{ inventory_hostname }}"
changed_when: false
register: systemd_escape
delegate_to: "{{ physical_host }}"
tags:
- skip_ansible_lint
- always
- name: Ensure mount directories exists (container)
file:
path: "{{ item['mount_path'] }}"
state: "directory"
with_items:
- "{{ list_of_bind_mounts | default([]) }}"
delegate_to: "{{ physical_host }}"
when:
- not is_metal | bool
tags:
- common-nspawn
- name: Ensure mount directories exists (physical host)
file:
path: "{{ item['bind_dir_path'] }}"
state: "directory"
with_items:
- "{{ list_of_bind_mounts | default([]) }}"
when:
- not is_metal | bool
tags:
- common-nspawn
- name: Create container bind mount config
lineinfile:
dest: "/etc/systemd/nspawn/{{ inventory_hostname }}.nspawn"
line: "Bind={{ item['mount_path'] }}:{{ item['bind_dir_path'] }}"
insertafter: "^Bind"
backup: "true"
with_items:
- "{{ list_of_bind_mounts | default([]) }}"
delegate_to: "{{ physical_host }}"
register: _ec
when:
- not is_metal | bool
- nspawn_systemd_version | int > 219
tags:
- common-nspawn
- name: Create container bind mount config (old)
block:
- name: Get ExecStart from config
shell: >-
grep -w '^ExecStart=/usr/bin/systemd-nspawn'
/etc/systemd/system/systemd-nspawn@$(/usr/bin/systemd-escape {{ inventory_hostname }}).service
delegate_to: "{{ physical_host }}"
register: _ec_old_start
changed_when: false
- name: set flag fact
set_fact:
nspawn_flags: "{{ _ec_old_start.stdout.split('ExecStart=/usr/bin/systemd-nspawn')[-1] }}"
nspawn_extra_flags: "{% for item in list_of_bind_mounts %} --bind={{ item['mount_path'] }}:{{ item['bind_dir_path'] }}{% endfor %}"
- name: set flag list
set_fact:
nspawn_flag_list: "{{ nspawn_flags.split() | union(nspawn_extra_flags.split()) | unique }}"
- name: Add line in container start config
lineinfile:
dest: "/etc/systemd/system/systemd-nspawn@{{ systemd_escape.stdout }}.service"
line: "ExecStart=/usr/bin/systemd-nspawn {{ nspawn_flag_list | join(' ') }}"
regexp: "^ExecStart"
backup: "true"
delegate_to: "{{ physical_host }}"
register: _ec
when:
- not is_metal | bool
- list_of_bind_mounts | default([])
- nspawn_systemd_version | int < 220
tags:
- common-nspawn
- name: Restart container
systemd:
name: "systemd-nspawn@{{ systemd_escape.stdout }}"
state: restarted
register: _container_restart
until: _container_restart | success
retries: 3
delay: 5
delegate_to: "{{ physical_host }}"
when:
- _ec | changed
tags:
- common-nspawn
- name: Wait for container connectivity
wait_for_connection:
delay: 3
timeout: 60
when:
- _container_restart | changed
tags:
- common-nspawn

View File

@ -13,5 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- include: "containers-{{ container_tech | default('lxc') }}-host.yml"
- include: "containers-{{ container_tech | default('lxc') }}-create.yml"
- include: "containers-lxc-host.yml"
- include: "containers-lxc-create.yml"
- include: "containers-nspawn-host.yml"
- include: "containers-nspawn-create.yml"

View File

@ -14,11 +14,25 @@
# limitations under the License.
- name: Gather lxc container host facts
hosts: "{{ lxc_host_group | default('lxc_hosts')}}"
hosts: "{{ lxc_host_group | default('lxc_hosts') }}"
gather_facts: "{{ osa_gather_facts | default(True) }}"
- name: Set lxc containers group
hosts: "{{ container_group | default('all_containers') }}"
gather_facts: false
tasks:
- name: Add hosts to dynamic inventory group
group_by:
key: lxc_containers
parents: all_lxc_containers
when:
- container_tech == 'lxc'
tags:
- always
- lxc-containers-create
- name: Create container(s)
hosts: "{{ container_group|default('all_containers') }}"
hosts: all_lxc_containers
gather_facts: false
max_fail_percentage: 20
user: root
@ -42,7 +56,7 @@
# TODO(evrardjp): Remove host_need_pip in the future
# when the process building the repo is done before this step.
- name: Configure containers default software, but don't run pip yet
hosts: "{{ container_group|default('all_containers') }}"
hosts: all_lxc_containers
gather_facts: true
user: root
roles:

View File

@ -13,8 +13,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Set lxc containers group
hosts: "{{ container_group | default('all_containers') }}"
gather_facts: false
tasks:
- name: Add hosts to dynamic inventory group
group_by:
key: lxc_containers
parents: all_lxc_containers
when:
- container_tech == 'lxc'
tags:
- always
- lxc-containers-create
- name: Destroy lxc containers
hosts: "{{ container_group|default('all_containers') }}"
hosts: all_lxc_containers
gather_facts: false
max_fail_percentage: 20
user: root

View File

@ -0,0 +1,57 @@
---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather nspawn container host facts
hosts: "{{ nspawn_host_group | default('nspawn_hosts') }}"
gather_facts: true
- name: Set nspawn containers group
hosts: "{{ container_group | default('all_containers') }}"
gather_facts: false
tasks:
- name: Add hosts to dynamic inventory group
group_by:
key: nspawn_containers
parents: all_nspawn_containers
when:
- container_tech == 'nspawn'
tags:
- always
- nspawn-containers-create
- name: Create container(s)
hosts: all_nspawn_containers
gather_facts: false
user: root
roles:
- role: "nspawn_container_create"
environment: "{{ deployment_environment_variables | default({}) }}"
tags:
- nspawn-containers-create
# TODO(evrardjp): Remove host_need_pip in the future
# when the process building the repo is done before this step.
- name: Configure containers default software, but don't run pip yet
hosts: all_nspawn_containers
gather_facts: true
user: root
roles:
- role: "openstack_hosts"
is_container: true
vars:
host_need_pip: False
environment: "{{ deployment_environment_variables | default({}) }}"
tags:
- nspawn-containers-create

View File

@ -0,0 +1,103 @@
---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Set nspawn containers group
hosts: "{{ container_group | default('all_containers') }}"
gather_facts: false
tasks:
- name: Add hosts to dynamic inventory group
group_by:
key: nspawn_containers
parents: all_nspawn_containers
when:
- container_tech == 'nspawn'
tags:
- always
- nspawn-containers-destroy
- name: Destroy nspawn containers
hosts: all_nspawn_containers
gather_facts: false
max_fail_percentage: 20
user: root
tasks:
- name: Get container status
command: machinectl status "{{ inventory_hostname }}"
register: machinectl_status
failed_when: false
delegate_to: "{{ physical_host }}"
- name: Get container image status
command: machinectl image-status "{{ inventory_hostname }}"
register: machinectl_image_status
failed_when: false
delegate_to: "{{ physical_host }}"
- name: Escape quote container name
command: "systemd-escape {{ inventory_hostname }}"
changed_when: false
register: systemd_escape
delegate_to: "{{ physical_host }}"
- name: Disable container
systemd:
name: "systemd-nspawn@{{ systemd_escape.stdout }}"
state: stopped
enabled: false
failed_when: false
delegate_to: "{{ physical_host }}"
when:
- force_containers_destroy | bool
- name: Halt container
command: "machinectl poweroff {{ inventory_hostname }}"
failed_when: false
delegate_to: "{{ physical_host }}"
when:
- machinectl_status.rc == 0
- force_containers_destroy | bool
- name: Remove container
command: "machinectl remove {{ inventory_hostname }}"
delegate_to: "{{ physical_host }}"
when:
- machinectl_image_status.rc == 0
- force_containers_destroy | bool
- name: Destroy container data
file:
path: "{{ item }}"
state: "absent"
with_items:
- "/openstack/{{ container_name }}"
- "/openstack/backup/{{ container_name }}"
- "/openstack/log/{{ container_name }}"
delegate_to: "{{ physical_host }}"
when:
- force_containers_destroy | bool
- force_containers_data_destroy | bool
vars_prompt:
- name: "force_containers_destroy"
prompt: "Are you sure you want to destroy the nspawn containers?"
default: "no"
private: no
when: force_containers_destroy is undefined
- name: "force_containers_data_destroy"
prompt: "Are you sure you want to destroy the nspawn container data?"
default: "no"
private: no
when: force_containers_data_destroy is undefined
tags:
- nspawn-containers-destroy

View File

@ -0,0 +1,24 @@
---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Additional nspawn host setup
hosts: "{{ nspawn_host_group | default('nspawn_hosts') }}"
gather_facts: true
user: root
roles:
- role: "nspawn_host"
environment: "{{ deployment_environment_variables | default({}) }}"
tags:
- nspawn-hosts

View File

@ -0,0 +1,16 @@
---
features:
- Deployers can now set the ``container_tech`` to **nspawn** when deploying
OSA within containers. When making the decision to deploy container types
the deployer only needs to define the desired ``container_tech`` and
continue the deployment as normal.
- The addition of the ``container_tech`` option and the inclusion of
**nspawn** support deployers now have the availability to define a desired
containerization strategy globally or on specific hosts.
- When using the **nspawn** driver containers will connect to the system
bridges using a MACVLAN, more on this type of network setup can be seen
`here <https://www.systutorials.com/docs/linux/man/8-ip-link/>`_.
- When using the **nspawn** driver container networking is managed by
systemd-networkd both on the host and within the container. This gives us a
single interface to manage regardless of distro and allows systemd to
efficiently manage the resources.

View File

@ -47,6 +47,7 @@ export ANSIBLE_HOST_KEY_CHECKING="${ANSIBLE_HOST_KEY_CHECKING:-False}"
export ANSIBLE_TIMEOUT="${ANSIBLE_TIMEOUT:-5}"
export ANSIBLE_TRANSPORT="${ANSIBLE_TRANSPORT:-ssh}"
export ANSIBLE_SSH_PIPELINING="${ANSIBLE_SSH_PIPELINING:-True}"
export ANSIBLE_SSH_RETRIES="${ANSIBLE_SSH_RETRIES:-3}"
export ANSIBLE_PIPELINING="${ANSIBLE_SSH_PIPELINING}"
export ANSIBLE_STRATEGY_PLUGINS="${ANSIBLE_STRATEGY_PLUGINS:-/etc/ansible/roles/plugins/strategy}"

View File

@ -261,6 +261,20 @@ function get_instance_info {
lxc-checkconfig > \
"/openstack/log/instance-info/host_lxc_config_info_${TS}.log" || true
fi
if [ "$(which machinectl)" ]; then
machinectl list > \
"/openstack/log/instance-info/host_nspawn_container_info_${TS}.log" || true
machinectl list-images > \
"/openstack/log/instance-info/host_nspawn_container_image_info_${TS}.log" || true
fi
if [ "$(which networkctl)" ]; then
networkctl list > \
"/openstack/log/instance-info/host_netowrkd_list_${TS}.log" || true
networkctl status >> \
"/openstack/log/instance-info/host_netowrkd_status_${TS}.log" || true
networkctl lldp >> \
"/openstack/log/instance-info/host_netowrkd_lldp_${TS}.log" || true
fi
(iptables -vnL && iptables -t nat -vnL && iptables -t mangle -vnL) > \
"/openstack/log/instance-info/host_firewall_info_${TS}.log" || true
if [ "$(which ansible)" ]; then
@ -271,6 +285,14 @@ function get_instance_info {
get_repos_info > \
"/openstack/log/instance-info/host_repo_info_${TS}.log" || true
for i in nspawn-macvlan.service nspawn-networking.slice nspawn.slice; do
systemctl status ${i} > "/openstack/log/instance-info/${i}_${TS}.log" || true
journalctl -u ${i} >> "/openstack/log/instance-info/${i}_${TS}.log" || true
done
ip route get 1 > "/openstack/log/instance-info/routes_${TS}.log" || true
ip link show > "/openstack/log/instance-info/links_${TS}.log" || true
determine_distro
case ${DISTRO_ID} in
centos|rhel|fedora|opensuse)

View File

@ -194,9 +194,6 @@ bridge_iptables_rules: |
up /sbin/iptables -t nat -A POSTROUTING -o {{ bootstrap_host_public_interface }} -j MASQUERADE
down /sbin/iptables -t nat -D POSTROUTING -o {{ bootstrap_host_public_interface }} -j MASQUERADE
# Set the container technology in service. Options are lxc.
container_tech: "lxc"
## Extra storage
# An AIO may optionally be built using a second storage device. If a
# secondary disk device to use is not specified, then the AIO will be
@ -239,3 +236,6 @@ bootstrap_host_apt_components:
# By default the address will be set to the ipv4 address of the
# host's network interface that has the default route on it.
#bootstrap_host_public_address: 0.0.0.0
# Set the container technology in service. Options are nspawn and lxc.
container_tech: "{{ ('nspawn' in bootstrap_host_scenario) | ternary('nspawn', 'lxc') }}"

View File

@ -204,6 +204,3 @@ nova_service_negate:
{% if _pypi_mirror is defined and _pypi_mirror.stdout is defined %}
repo_nginx_pypi_upstream: "{{ _pypi_mirror.stdout | netloc }}"
{% endif %}
# Set the container tech. Options are "lxc"
container_tech: "{{ container_tech }}"

View File

@ -265,6 +265,7 @@ class TestAnsibleInventoryFormatConstraints(unittest.TestCase):
'mano_all',
'mano_containers',
'mano_hosts',
'nspawn_hosts',
'octavia-infra_hosts',
'octavia_all',
'octavia-api',

View File

@ -33,6 +33,17 @@ confd_overrides:
- name: neutron.yml.aio
- name: nova.yml.aio
- name: swift.yml.aio
aio_nspawn:
- name: haproxy.yml.aio
- name: cinder.yml.aio
- name: designate.yml.aio
- name: glance.yml.aio
- name: heat.yml.aio
- name: horizon.yml.aio
- name: keystone.yml.aio
- name: neutron.yml.aio
- name: nova.yml.aio
- name: swift.yml.aio
ceph:
- name: haproxy.yml.aio
- name: ceph.yml.aio

View File

@ -79,6 +79,13 @@
action: upgrade
scenario: aio
- job:
name: openstack-ansible-deploy-aio_nspawn-ubuntu-xenial
parent: openstack-ansible-deploy-aio_lxc-ubuntu-xenial
voting: false
vars:
scenario: aio_nspawn
- job:
name: openstack-ansible-upgrade-ceph-ubuntu-xenial
parent: openstack-ansible-deploy-aio_lxc-ubuntu-xenial
@ -123,6 +130,12 @@
action: upgrade
scenario: aio
- job:
name: openstack-ansible-deploy-aio_nspawn-centos-7
parent: openstack-ansible-deploy-aio_lxc-centos-7
vars:
scenario: aio_nspawn
- job:
name: openstack-ansible-upgrade-ceph-centos-7
parent: openstack-ansible-deploy-aio_lxc-centos-7
@ -153,6 +166,12 @@
action: upgrade
scenario: aio
- job:
name: openstack-ansible-deploy-aio_nspawn-opensuse-423
parent: openstack-ansible-deploy-aio_lxc-opensuse-423
vars:
scenario: aio_nspawn
# NOTE(cloudnull): META JOB MAP
# In order to cater for the possibility that an external job was dependent on the old job name

View File

@ -24,10 +24,12 @@
- openstack-ansible-deploy-ceph-ubuntu-xenial
- openstack-ansible-deploy-ceph-opensuse-423
- openstack-ansible-deploy-aio_metal-ubuntu-xenial
- openstack-ansible-deploy-aio_nspawn-ubuntu-xenial
experimental:
jobs:
- openstack-ansible-deploy-octavia-ubuntu-xenial
- openstack-ansible-deploy_with_ansible_devel-aio-ubuntu-xenial
- openstack-ansible-deploy-aio_nspawn-centos-7
gate:
jobs:
- openstack-ansible-linters