diff --git a/.zuul.yaml b/.zuul.yaml index 5c2f462af9..724fd1e9db 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -2232,7 +2232,7 @@ - playbooks/roles/configure-kubectl/ - playbooks/roles/configure-openstacksdk/ - playbooks/roles/install-docker/ - - playbooks/roles/nodepool- + - playbooks/roles/nodepool - playbooks/templates/clouds/nodepool_ - job: diff --git a/inventory/groups.yaml b/inventory/groups.yaml index 76cca8d14c..72c6fad00c 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -101,8 +101,7 @@ groups: nodepool-builder_opendev: - nb[0-9]*.opendev.org nodepool-launcher: - - nl[0-9]*.openstack.org - - nl[0-8]*.opendev.org + - nl[0-9]*.open*.org ns: - ns[0-9]*.open*.org openstackid-dev: @@ -137,7 +136,6 @@ groups: - mirror-update[0-9]*.openstack.org - mirror[0-9]*.openstack.org - nb[0-9]*.openstack.org - - nl[0-9]*.openstack.org - openstackid-dev*.openstack.org - openstackid.org - openstackid[0-9]*.openstack.org @@ -177,7 +175,6 @@ groups: - mirror-update[0-9]*.openstack.org - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|linaro-us).*\.openstack\.org - ^nb(?!03)[0-9]*\.openstack\.org - - nl[0-9]*.open*.org - openstackid[0-9]*.openstack.org - openstackid-dev[0-9]*.openstack.org - paste[0-9]*.open*.org diff --git a/manifests/site.pp b/manifests/site.pp index 2cd0bcb957..3a4f7e6210 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -369,28 +369,6 @@ node /^survey\d+\.open.*\.org$/ { } } -# Node-OS: xenial -node /^nl\d+\.open.*\.org$/ { - $group = 'nodepool' - - # NOTE(ianw) From 09-2018 (https://review.opendev.org/#/c/598329/) - # the cloud credentials are deployed with ansible via the - # configure-openstacksdk role and are no longer configured here - - class { 'openstack_project::server': } - - include openstack_project - - class { '::openstackci::nodepool_launcher': - nodepool_ssh_private_key => hiera('zuul_worker_ssh_private_key_contents'), - project_config_repo => 'https://opendev.org/openstack/project-config', - statsd_host => 'graphite.opendev.org', - revision => 'master', - python_version => 3, - enable_webapp => true, - } -} - # Node-OS: xenial node /^nb\d+\.open.*\.org$/ { $group = 'nodepool' diff --git a/playbooks/group_vars/nodepool-launcher.yaml b/playbooks/group_vars/nodepool-launcher.yaml index 81cac6a1aa..4174245222 100644 --- a/playbooks/group_vars/nodepool-launcher.yaml +++ b/playbooks/group_vars/nodepool-launcher.yaml @@ -1,4 +1,4 @@ -openstacksdk_config_dir: /home/nodepool/.config/openstack +openstacksdk_config_dir: /etc/openstack openstacksdk_config_owner: nodepool openstacksdk_config_group: nodepool openstacksdk_config_template: clouds/nodepool_clouds.yaml.j2 diff --git a/playbooks/roles/nodepool-builder/templates/docker-compose.yaml.j2 b/playbooks/roles/nodepool-builder/templates/docker-compose.yaml.j2 index de3efc9d69..3594256ed4 100644 --- a/playbooks/roles/nodepool-builder/templates/docker-compose.yaml.j2 +++ b/playbooks/roles/nodepool-builder/templates/docker-compose.yaml.j2 @@ -10,6 +10,8 @@ services: environment: - DEBUG=1 + - STATSD_HOST=graphite.opendev.org + - STATSD_PORT=8125 volumes: # nodepool config diff --git a/playbooks/roles/nodepool-launcher/README.rst b/playbooks/roles/nodepool-launcher/README.rst new file mode 100644 index 0000000000..774c054dc0 --- /dev/null +++ b/playbooks/roles/nodepool-launcher/README.rst @@ -0,0 +1 @@ +Deploy nodepool launchers diff --git a/playbooks/roles/nodepool-launcher/defaults/main.yaml b/playbooks/roles/nodepool-launcher/defaults/main.yaml new file mode 100644 index 0000000000..bb2ef3adab --- /dev/null +++ b/playbooks/roles/nodepool-launcher/defaults/main.yaml @@ -0,0 +1 @@ +nodepool_launcher_start: false diff --git a/playbooks/roles/nodepool-launcher/files/logging.conf b/playbooks/roles/nodepool-launcher/files/logging.conf new file mode 100644 index 0000000000..9ba2de41a2 --- /dev/null +++ b/playbooks/roles/nodepool-launcher/files/logging.conf @@ -0,0 +1,54 @@ +[loggers] +keys=root,nodepool,requests,openstack,kazoo + +[handlers] +keys=console,debug,normal + +[formatters] +keys=simple + +[logger_root] +level=WARNING +handlers=console + +[logger_requests] +level=WARNING +handlers=debug,normal +qualname=requests + +[logger_openstack] +level=WARNING +handlers=debug,normal +qualname=openstack + +[logger_kazoo] +level=INFO +handlers=debug,normal +qualname=kazoo + +[logger_nodepool] +level=DEBUG +handlers=debug,normal +qualname=nodepool + +[handler_console] +level=WARNING +class=StreamHandler +formatter=simple +args=(sys.stdout,) + +[handler_debug] +level=DEBUG +class=logging.handlers.TimedRotatingFileHandler +formatter=simple +args=('/var/log/nodepool/launcher-debug.log', 'H', 8, 30,) + +[handler_normal] +level=INFO +class=logging.handlers.TimedRotatingFileHandler +formatter=simple +args=('/var/log/nodepool/nodepool-launcher.log', 'H', 8, 30,) + +[formatter_simple] +format=%(asctime)s %(levelname)s %(name)s: %(message)s +datefmt= diff --git a/playbooks/roles/nodepool-launcher/handlers/main.yaml b/playbooks/roles/nodepool-launcher/handlers/main.yaml new file mode 100644 index 0000000000..7e06db9757 --- /dev/null +++ b/playbooks/roles/nodepool-launcher/handlers/main.yaml @@ -0,0 +1,4 @@ +- name: nodepool-launcher Reload apache2 + service: + name: apache2 + state: reloaded diff --git a/playbooks/roles/nodepool-launcher/tasks/main.yaml b/playbooks/roles/nodepool-launcher/tasks/main.yaml new file mode 100644 index 0000000000..9b0fee0062 --- /dev/null +++ b/playbooks/roles/nodepool-launcher/tasks/main.yaml @@ -0,0 +1,55 @@ +- name: Copy logging config + copy: + src: logging.conf + dest: /etc/nodepool/launcher-logging.conf + +- name: Install apache2 + apt: + name: + - apache2 + - apache2-utils + state: present + +- name: Apache modules + apache2_module: + state: present + name: "{{ item }}" + loop: + - rewrite + - proxy + - proxy_http + +- name: Copy apache config + template: + src: launcher.vhost.j2 + dest: /etc/apache2/sites-enabled/000-default.conf + owner: root + group: root + mode: 0644 + notify: nodepool-launcher Reload apache2 + +# Do this until 719589/ passes +- name: Install docker-compose + pip: + name: docker-compose + state: present + executable: pip3 + +- name: Ensure docker compose dir + file: + state: directory + path: /etc/nodepool-docker + +- name: Copy docker compose file + template: + src: docker-compose.yaml.j2 + dest: /etc/nodepool-docker/docker-compose.yaml + +- name: Run docker-compose pull + shell: + cmd: docker-compose pull + chdir: /etc/nodepool-docker/ + +- name: Start nodepool launcher + include_tasks: start.yaml + when: nodepool_launcher_start | bool diff --git a/playbooks/roles/nodepool-launcher/tasks/start.yaml b/playbooks/roles/nodepool-launcher/tasks/start.yaml new file mode 100644 index 0000000000..02b8bd3905 --- /dev/null +++ b/playbooks/roles/nodepool-launcher/tasks/start.yaml @@ -0,0 +1,8 @@ +- name: Run docker-compose up + shell: + cmd: docker-compose up -d + chdir: /etc/nodepool-docker/ + +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f diff --git a/playbooks/roles/nodepool-launcher/templates/docker-compose.yaml.j2 b/playbooks/roles/nodepool-launcher/templates/docker-compose.yaml.j2 new file mode 100644 index 0000000000..95c8d8fce7 --- /dev/null +++ b/playbooks/roles/nodepool-launcher/templates/docker-compose.yaml.j2 @@ -0,0 +1,20 @@ +version: '2' +services: + nodepool-launcher: + image: docker.io/zuul/nodepool-launcher:{{ nodepool_launcher_container_tag|default('latest') }} + user: nodepool + network_mode: host + restart: always + + environment: + - DEBUG=1 + - STATSD_HOST=graphite.opendev.org + - STATSD_PORT=8125 + + volumes: + # nodepool config + - /etc/nodepool:/etc/nodepool:ro + # openstacksdk config + - /etc/openstack:/etc/openstack:ro + # logs + - /var/log/nodepool:/var/log/nodepool:rw diff --git a/playbooks/roles/nodepool-launcher/templates/launcher.vhost.j2 b/playbooks/roles/nodepool-launcher/templates/launcher.vhost.j2 new file mode 100644 index 0000000000..bc8440196f --- /dev/null +++ b/playbooks/roles/nodepool-launcher/templates/launcher.vhost.j2 @@ -0,0 +1,19 @@ + + ServerName {{ inventory_hostname }} + + ErrorLog ${APACHE_LOG_DIR}/nodepool-error.log + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/nodepool-access.log combined + ServerSignature Off + + + SetOutputFilter DEFLATE + + + RewriteEngine on + RewriteRule ^/image-list$ http://127.0.0.1:8005/image-list [P] + RewriteRule ^/dib-image-list$ http://127.0.0.1:8005/dib-image-list [P] + RewriteRule ^/image-list.json$ http://127.0.0.1:8005/image-list.json [P] + RewriteRule ^/dib-image-list.json$ http://127.0.0.1:8005/dib-image-list.json [P] + + diff --git a/playbooks/service-nodepool.yaml b/playbooks/service-nodepool.yaml index 4eacc6b3a5..84ab9a3a88 100644 --- a/playbooks/service-nodepool.yaml +++ b/playbooks/service-nodepool.yaml @@ -1,11 +1,3 @@ -- hosts: nodepool-launcher:nodepool-builder:!disabled - name: "Base: configure OpenStackSDK on nodepool legacy hosts" - strategy: free - roles: - - nodepool-base-legacy - - configure-openstacksdk - - configure-kubectl - - hosts: nodepool-builder_opendev:!disabled name: "Configure nodepool builders" strategy: free @@ -19,10 +11,19 @@ name: "run puppet on all older servers" strategy: free roles: + - nodepool-base-legacy + - configure-openstacksdk + - configure-kubectl - puppet-install - disable-puppet-agent - puppet -# TODO(ianw) 2020-03-03 : watch this space... -#- hosts: nodepool-launcher_opendev:!disabled -# name: "Configure nodepool launchers" +- hosts: nodepool-launcher:!disabled + name: "Configure nodepool launchers" + strategy: free + roles: + - install-docker + - nodepool-base + - configure-openstacksdk + - configure-kubectl + - nodepool-launcher diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 08fef58db4..0e613d9e41 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -53,6 +53,7 @@ - group_vars/gitea-lb.yaml - group_vars/letsencrypt.yaml - group_vars/meetpad.yaml + - group_vars/nodepool-launcher.yaml - group_vars/registry.yaml - group_vars/review.yaml - group_vars/review-dev.yaml diff --git a/playbooks/zuul/templates/group_vars/nodepool-launcher.yaml.j2 b/playbooks/zuul/templates/group_vars/nodepool-launcher.yaml.j2 new file mode 100644 index 0000000000..893b3b4c4c --- /dev/null +++ b/playbooks/zuul/templates/group_vars/nodepool-launcher.yaml.j2 @@ -0,0 +1 @@ +nodepool_launcher_start: true diff --git a/testinfra/test_nodepool.py b/testinfra/test_nodepool.py index d07579ffdc..50d1e34d37 100644 --- a/testinfra/test_nodepool.py +++ b/testinfra/test_nodepool.py @@ -19,7 +19,7 @@ testinfra_hosts = ['nl01.openstack.org', 'nb01.openstack.org', def test_clouds_yaml(host): - if host.backend.get_hostname().endswith('openstack.org'): + if host.backend.get_hostname() == 'nb01.openstack.org': cfg_file = '/home/nodepool/.config/openstack/clouds.yaml' else: cfg_file = '/etc/openstack/clouds.yaml' @@ -30,13 +30,20 @@ def test_clouds_yaml(host): assert b'password' in clouds_yaml.content def test_kube_config(host): - if not host.backend.get_hostname().endswith('openstack.org'): + if not host.backend.get_hostname().startswith('nl'): pytest.skip() kubeconfig = host.file('/home/nodepool/.kube/config') assert kubeconfig.exists assert b'nodepool_k8s_key' in kubeconfig.content +def test_launcher_container_running(host): + if host.backend.get_hostname() != 'nl01.openstack.org': + pytest.skip() + + cmd = host.run("docker ps -a --format '{{ .Names }}'") + assert 'nodepool-docker_nodepool-launcher_1' in cmd.stdout + def test_builder_container_running(host): if host.backend.get_hostname() != 'nb04.opendev.org': pytest.skip()