opendev/system-config
Code Issues Proposed changes

Merge "Remove zuul-launcher support"

Browse Source
This commit is contained in:
Zuul 2017-11-30 18:37:43 +00:00 committed by Gerrit Code Review
commit 87644f4369
8 changed files with 0 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
doc/source/zuul.rst
View File

@ -16,7 +16,6 @@ At a Glance
* http://zuul.openstack.org
* http://zuul-dev.openstack.org
* zm*.openstack.org
* zl*.openstack.org
:Puppet:
* https://git.openstack.org/cgit/openstack-infra/puppet-zuul/tree/
* :file:`modules/openstack_project/manifests/zuul_prod.pp`
@ -162,18 +161,3 @@ necessary to accommodate load. If you remove a merger, be sure to
leave Apache running for several hours until the last job that may
have been launched with instructions to fetch from that merger has
completed.
Launchers
---------
We use an Ansible based launcher in Zuul to actually run jobs. This
component runs on a horizontally scalable set of servers named
zl*.openstack.org. It reads job configuration from Jenkins Job
Builder files in the project-config repository and translates that
into Ansible playbooks which it runs on our workers. Our jobs are
configured to upload as much information as possible along with their
logs, but if there is an error which can not be diagnosed in that
manner, Ansible logs are available in the launcher-debug log file on
the launcher host. You may use the Zuul build UUID to track
assignment of a given job from the Zuul scheduler to the Zuul launcher
used by that job.

7
hiera/common.yaml
View File

@ -488,13 +488,6 @@ cacti_hosts:
- ze08.openstack.org
- ze09.openstack.org
- ze10.openstack.org
- zlstatic01.openstack.org
- zl01.openstack.org
- zl02.openstack.org
- zl03.openstack.org
- zl04.openstack.org
- zl05.openstack.org
- zl06.openstack.org
- zm01.openstack.org
- zm02.openstack.org
- zm03.openstack.org

63
manifests/site.pp
View File

@ -1277,7 +1277,6 @@ node 'zuulv3-dev.openstack.org' {
manage_common_zuul => false,
}
# TODO(pabelanger): Add zuul_scheduler support
# TODO(pabelanger): Add zuul_launcher support
}
# Node-OS: xenial
@ -1380,13 +1379,6 @@ node 'zuulv3.openstack.org' {
node 'zuul.openstack.org' {
$gearman_workers = [
'nodepool.openstack.org',
'zlstatic01.openstack.org',
'zl01.openstack.org',
'zl02.openstack.org',
'zl03.openstack.org',
'zl04.openstack.org',
'zl05.openstack.org',
'zl06.openstack.org',
]
$iptables_rules = regsubst ($gearman_workers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 4730 -s \1 -j ACCEPT')
@ -1412,61 +1404,6 @@ node 'zuul.openstack.org' {
}
}
# Node-OS: trusty
node /^zlstatic\d+\.openstack\.org$/ {
$group = "zuul-merger"
$zmq_event_receivers = ['logstash.openstack.org',
'nodepool.openstack.org']
$zmq_iptables_rule = regsubst($zmq_event_receivers,
'^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 8888 -s \1 -j ACCEPT')
$iptables_rule = flatten([$zmq_iptables_rule])
class { 'openstack_project::server':
iptables_rules6 => $iptables_rule,
iptables_rules4 => $iptables_rule,
sysadmins => hiera('sysadmins', []),
afs => true,
}
class { 'openstack_project::zuul_launcher':
gearman_server => 'zuul.openstack.org',
gerrit_server => 'review.openstack.org',
gerrit_user => 'jenkins',
gerrit_ssh_host_key => hiera('gerrit_ssh_rsa_pubkey_contents'),
zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'),
project_config_repo => 'https://git.openstack.org/openstack-infra/project-config',
sysadmins => hiera('sysadmins', []),
sites => hiera('zuul_sites', []),
nodes => hiera('zuul_nodes', []),
accept_nodes => false,
}
}
# Node-OS: trusty
node /^zl\d+\.openstack\.org$/ {
$group = "zuul-merger"
$zmq_event_receivers = ['logstash.openstack.org',
'nodepool.openstack.org']
$zmq_iptables_rule = regsubst($zmq_event_receivers,
'^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 8888 -s \1 -j ACCEPT')
$iptables_rule = flatten([$zmq_iptables_rule])
class { 'openstack_project::server':
iptables_rules6 => $iptables_rule,
iptables_rules4 => $iptables_rule,
sysadmins => hiera('sysadmins', []),
afs => true,
}
class { 'openstack_project::zuul_launcher':
gearman_server => 'zuul.openstack.org',
gerrit_server => 'review.openstack.org',
gerrit_user => 'jenkins',
gerrit_ssh_host_key => hiera('gerrit_ssh_rsa_pubkey_contents'),
zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'),
project_config_repo => 'https://git.openstack.org/openstack-infra/project-config',
sysadmins => hiera('sysadmins', []),
sites => hiera('zuul_sites', []),
zuul_launcher_keytab => hiera('zuul_launcher_keytab'),
}
}
# Node-OS: xenial
node /^zm\d+.openstack\.org$/ {
$group = "zuul-merger"

107
modules/openstack_project/manifests/zuul_launcher.pp
View File

@ -1,107 +0,0 @@
# == Class: openstack_project::zuul_launcher
#
class openstack_project::zuul_launcher(
$vhost_name = $::fqdn,
$gearman_server = '127.0.0.1',
$gerrit_server = '',
$gerrit_user = '',
$gerrit_ssh_host_key = '',
$zuul_ssh_private_key = '',
$url_pattern = '',
$zuul_url = '',
$status_url = 'http://status.openstack.org/zuul/',
$swift_authurl = '',
$swift_auth_version = '',
$swift_user = '',
$swift_key = '',
$swift_tenant_name = '',
$swift_region_name = '',
$swift_default_container = '',
$swift_default_logserver_prefix = '',
$swift_default_expiry = 7200,
$proxy_ssl_cert_file_contents = '',
$proxy_ssl_key_file_contents = '',
$proxy_ssl_chain_file_contents = '',
$sysadmins = [],
$statsd_host = '',
$project_config_repo = '',
$project_config_base = '',
$git_email = 'jenkins@openstack.org',
$git_name = 'OpenStack Jenkins',
$workspace_root = '/home/jenkins/workspace',
$worker_private_key_file = '/var/lib/zuul/ssh/id_rsa',
$worker_username = 'jenkins',
$sites = [],
$nodes = [],
$accept_nodes = '',
$zuul_launcher_keytab = '',
) {
class { '::project_config':
url => $project_config_repo,
base => $project_config_base,
}
file { '/etc/zuul-launcher.keytab':
owner => 'zuul',
group => 'zuul',
mode => '0400',
content => $zuul_launcher_keytab,
}
file { '/etc/jenkins_jobs':
ensure => directory,
}
file { '/etc/jenkins_jobs/config':
ensure => directory,
owner => 'root',
group => 'root',
mode => '0755',
recurse => true,
purge => true,
force => true,
source => $::project_config::jenkins_job_builder_config_dir,
require => [File['/etc/jenkins_jobs'],
$::project_config::config_dir],
notify => Exec['zuul-launcher-reload'],
}
file { '/home/zuul/.ssh':
ensure => directory,
owner => 'zuul',
group => 'zuul',
mode => '0700',
require => User['zuul'],
}
file { '/home/zuul/.ssh/config':
ensure => present,
source => 'puppet:///modules/openstack_project/zuul/launcher_ssh_config',
owner => 'zuul',
group => 'zuul',
require => File['/home/zuul/.ssh'],
}
class { '::zuul':
vhost_name => $vhost_name,
gearman_server => $gearman_server,
gerrit_server => $gerrit_server,
gerrit_user => $gerrit_user,
zuul_ssh_private_key => $zuul_ssh_private_key,
zuul_url => $zuul_url,
git_email => $git_email,
git_name => $git_name,
revision => $revision,
git_source_repo => $git_source_repo,
jenkins_jobs => '/etc/jenkins_jobs/config',
workspace_root => $workspace_root,
worker_private_key_file => $worker_private_key_file,
worker_username => $worker_username,
sites => $sites,
nodes => $nodes,
accept_nodes => $accept_nodes,
}
class { 'zuul::launcher': }
}

23
playbooks/hard_restart_zuul_launchers.yaml
View File

@ -1,23 +0,0 @@
---
- hosts: 'zlstatic0*.openstack.org:zl0*.openstack.org'
# Do the entire play completely for one host at a time
serial: 1
any_errors_fatal: true
tasks:
- name: Hard stop zuul-launcher
shell: 'zuul-launcher stop'
become: true
become_user: zuul
- name: Wait for (3h10m) to stop zuul-launcher
wait_for:
path: /var/run/zuul-launcher/zuul-launcher.pid
state: absent
timeout: 11400
- name: Restart zuul-launcher
service:
name: zuul-launcher
state: restarted
become: true
become_user: root

23
playbooks/restart_zuul_launchers.yaml
View File

@ -1,23 +0,0 @@
---
- hosts: 'zlstatic0*.openstack.org:zl0*.openstack.org'
# Do the entire play completely for one host at a time
serial: 1
any_errors_fatal: true
tasks:
- name: Graceful stop zuul-launcher
shell: 'zuul-launcher graceful'
become: true
become_user: zuul
- name: Wait for (3h10m) to stop zuul-launcher
wait_for:
path: /var/run/zuul-launcher/zuul-launcher.pid
state: absent
timeout: 11400
- name: Restart zuul-launcher
service:
name: zuul-launcher
state: restarted
become: true
become_user: root

9
start_all_zuul.yaml
View File

@ -9,15 +9,6 @@
become: true
become_user: root
- hosts: 'zlstatic0*.openstack.org:zl0*.openstack.org'
tasks:
- name: Start zuul-launcher
service:
name: zuul-launcher
state: started
become: true
become_user: root
- hosts: 'zm0*.openstack.org'
tasks:
- name: Start zuul-merger

13
stop_all_zuul.yaml
View File

@ -14,19 +14,6 @@
state: absent
timeout: 11400
- hosts: 'zlstatic0*.openstack.org:zl0*.openstack.org'
tasks:
- name: Hard stop zuul-launcher
shell: 'zuul-launcher stop'
become: true
become_user: zuul
- name: Wait for (3h10m) to stop zuul-launcher
wait_for:
path: /var/run/zuul-launcher/zuul-launcher.pid
state: absent
timeout: 11400
- hosts: 'zm0*.openstack.org'
tasks:
- name: Hard stop zuul-merger