From 745cc1829023dd113be8c2dc6b5e62871e760054 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 9 Oct 2012 20:28:07 +0000 Subject: [PATCH] Add a launch script. Add a script to launch new OpenStack project servers. Change-Id: I9f12ac0b7e38592128de1d6b999a5d540d621514 Reviewed-on: https://review.openstack.org/14246 Reviewed-by: Clark Boylan Approved: Monty Taylor Reviewed-by: Monty Taylor Tested-by: Jenkins --- launch/README | 41 +++++++++ launch/launch-node.py | 190 ++++++++++++++++++++++++++++++++++++++++++ launch/sshclient.py | 51 ++++++++++++ launch/utils.py | 188 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 470 insertions(+) create mode 100644 launch/README create mode 100755 launch/launch-node.py create mode 100644 launch/sshclient.py create mode 100644 launch/utils.py diff --git a/launch/README b/launch/README new file mode 100644 index 0000000000..82eb5c0666 --- /dev/null +++ b/launch/README @@ -0,0 +1,41 @@ +To launch a node in the OpenStack CI account (production servers):: + + . openstackci-rs-nova.sh + +To launch a node in the OpenStack Jenkins account (slave nodes):: + + . openstackjenkins-rs-nova.sh + +Then:: + + puppet cert generate servername.openstack.org + ./launch-node.py servername.openstack.org --cert servername.openstack.org.pem + +If you are launching a replacement server, you may skip the generate +step and specify the name of an existing puppet cert (as long as the +private key is on this host). + +The server name and cert names may be different. + +Manually add the hostname to DNS (the launch script does not do so +automatically). + +DNS +=== + +There are no scripts to handle DNS at the moment due to a lack of +library support for the new Rackspace Cloud DNS (with IPv6). To +manually update DNS, you will need the hostname, v4 and v6 addresses +of the host, as well as the UUID. The environment variables used in +the URL should be satisfied by sourcing the "openstackci-rs-nova.sh" +script (or jenkins, as appropriate). + + . ~/rackdns-venv/bin/activate + . openstackci-rs-nova.sh + + rackdns rdns-create --name HOSTNAME.openstack.org --data IPV6ADDR --server-href https://$OS_REGION_NAME.servers.api.rackspacecloud.com/v2/$OS_TENANT_NAME/servers/UUID --ttl 300 + rackdns rdns-create --name HOSTNAME.openstack.org --data IPV4ADDR --server-href https://$OS_REGION_NAME.servers.api.rackspacecloud.com/v2/$OS_TENANT_NAME/servers/UUID --ttl 300 + + . openstack-rs-nova.sh + rackdns record-create --name HOSTNAME.openstack.org --type AAAA --data IPV6ADDR --ttl 300 openstack.org + rackdns record-create --name HOSTNAME.openstack.org --type A --data IPV4ADDR --ttl 300 openstack.org diff --git a/launch/launch-node.py b/launch/launch-node.py new file mode 100755 index 0000000000..92c58978c5 --- /dev/null +++ b/launch/launch-node.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python + +# Launch a new OpenStack project infrastructure node. + +# Copyright (C) 2011-2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os +import commands +import time +import subprocess +import traceback +import socket +import argparse +import utils + +NOVA_USERNAME=os.environ['OS_USERNAME'] +NOVA_PASSWORD=os.environ['OS_PASSWORD'] +NOVA_URL=os.environ['OS_AUTH_URL'] +NOVA_PROJECT_ID=os.environ['OS_TENANT_NAME'] +NOVA_REGION_NAME=os.environ['OS_REGION_NAME'] + +def get_client(): + args = [NOVA_USERNAME, NOVA_PASSWORD, NOVA_PROJECT_ID, NOVA_URL] + kwargs = {} + kwargs['region_name'] = NOVA_REGION_NAME + kwargs['service_type'] = 'compute' + from novaclient.v1_1.client import Client + client = Client(*args, **kwargs) + return client + +def bootstrap_server(server, admin_pass, key, cert): + client = server.manager.api + ip = utils.get_public_ip(server) + if not ip: + raise Exception("Unable to find public ip of server") + + ssh_kwargs = {} + if key: + ssh_kwargs['pkey'] = key + else: + ssh_kwargs['password'] = admin_pass + + for username in ['root', 'ubuntu']: + ssh_client = utils.ssh_connect(ip, username, ssh_kwargs, timeout=600) + if ssh_client: break + + if not ssh_client: + raise Exception("Unable to log in via SSH") + + if username != 'root': + ssh_client.ssh("sudo cp ~/.ssh/authorized_keys" + " ~root/.ssh/authorized_keys") + ssh_client.ssh("sudo chmod 644 ~root/.ssh/authorized_keys") + ssh_client.ssh("sudo chown root.root ~root/.ssh/authorized_keys") + + ssh_client = utils.ssh_connect(ip, 'root', ssh_kwargs, timeout=600) + + ssh_client.ssh("apt-get update") + ssh_client.ssh("DEBIAN_FRONTEND=noninteractive apt-get --option" + " 'Dpkg::Options::=--force-confold'" + " --assume-yes upgrade") + ssh_client.ssh("apt-get install -y --force-yes puppet") + + certname = cert[:0-len('.pem')] + + ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/certs") + ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/private_keys") + ssh_client.ssh("mkdir -p /var/lib/puppet/ssl/public_keys") + ssh_client.ssh("chown -R puppet:root /var/lib/puppet/ssl") + ssh_client.ssh("chmod 0771 /var/lib/puppet/ssl") + ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/certs") + ssh_client.ssh("chmod 0750 /var/lib/puppet/ssl/private_keys") + ssh_client.ssh("chmod 0755 /var/lib/puppet/ssl/public_keys") + + for ssldir in ['/var/lib/puppet/ssl/certs/', + '/var/lib/puppet/ssl/private_keys/', + '/var/lib/puppet/ssl/public_keys/']: + ssh_client.scp(os.path.join(ssldir, cert), + os.path.join(ssldir, cert)) + + ssh_client.scp("/var/lib/puppet/ssl/crl.pem", + "/var/lib/puppet/ssl/crl.pem") + ssh_client.scp("/var/lib/puppet/ssl/certs/ca.pem", + "/var/lib/puppet/ssl/certs/ca.pem") + + ssh_client.ssh("puppet agent " + "--server ci-puppetmaster.openstack.org " + "--no-daemonize --verbose --onetime " + "--certname %s" % certname) + +def build_server(client, name, image, flavor, cert): + key = None + server = None + + create_kwargs = dict(image=image, flavor=flavor, name=name) + + key_name = 'launch-%i' % (time.time()) + if 'os-keypairs' in utils.get_extensions(client): + print "Adding keypair" + key, kp = utils.add_keypair(client, key_name) + create_kwargs['key_name'] = key_name + try: + server = client.servers.create(**create_kwargs) + except Exception, real_error: + try: + kp.delete() + except Exception, delete_error: + print "Exception encountered deleting keypair:" + traceback.print_exc() + raise + + try: + admin_pass = server.adminPass + server = utils.wait_for_resource(server) + bootstrap_server(server, admin_pass, key, cert) + if key: + kp.delete() + except Exception, real_error: + try: + utils.delete_server(server) + except Exception, delete_error: + print "Exception encountered deleting server:" + traceback.print_exc() + # Raise the important exception that started this + raise + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("name", help="server name") + parser.add_argument("--ram", dest="ram", default=1024, type=int, + help="minimum amount of ram") + parser.add_argument("--image", dest="image", + default="Ubuntu 12.04 LTS (Precise Pangolin)", + help="image name") + parser.add_argument("--cert", dest="cert", required=True, + help="name of signed puppet certificate file (e.g., " + "hostname.example.com.pem)") + options = parser.parse_args() + + client = get_client() + + if not os.path.exists(os.path.join("/var/lib/puppet/ssl/private_keys", + options.cert)): + raise Exception("Please specify the name of a signed puppet cert.") + + flavors = [f for f in client.flavors.list() if f.ram >= options.ram] + flavors.sort(lambda a, b: cmp(a.ram, b.ram)) + flavor = flavors[0] + print "Found flavor", flavor + + images = [i for i in client.images.list() + if (options.image.lower() in i.name.lower() and + not i.name.endswith('(Kernel)') and + not i.name.endswith('(Ramdisk)'))] + + if len(images) > 1: + print "Ambiguous image name; matches:" + for i in images: + print i.name + sys.exit(1) + + if len(images) == 0: + print "Unable to find matching image; image list:" + for i in client.images.list(): + print i.name + sys.exit(1) + + image = images[0] + print "Found image", image + + build_server(client, options.name, image, flavor, options.cert) + +if __name__ == '__main__': + main() diff --git a/launch/sshclient.py b/launch/sshclient.py new file mode 100644 index 0000000000..a4c552049c --- /dev/null +++ b/launch/sshclient.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python + +# Update the base image that is used for devstack VMs. + +# Copyright (C) 2011-2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import paramiko +import sys + +class SSHClient(object): + def __init__(self, ip, username, password=None, pkey=None): + client = paramiko.SSHClient() + client.load_system_host_keys() + client.set_missing_host_key_policy(paramiko.WarningPolicy()) + client.connect(ip, username=username, password=password, pkey=pkey) + self.client = client + + def ssh(self, command, error_ok=False): + stdin, stdout, stderr = self.client.exec_command(command) + print command + output = '' + for x in stdout: + output += x + sys.stdout.write(x) + ret = stdout.channel.recv_exit_status() + print stderr.read() + if (not error_ok) and ret: + raise Exception("Unable to %s" % command) + return ret, output + + def scp(self, source, dest): + print 'copy', source, dest + ftp = self.client.open_sftp() + ftp.put(source, dest) + ftp.close() + + diff --git a/launch/utils.py b/launch/utils.py new file mode 100644 index 0000000000..0eaa178a2b --- /dev/null +++ b/launch/utils.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +# Update the base image that is used for devstack VMs. + +# Copyright (C) 2011-2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +import novaclient +from novaclient.v1_1 import client as Client11 +try: + from v1_0 import client as Client10 +except: + pass +import time +import os +import traceback +import paramiko +import socket +from sshclient import SSHClient + + +def iterate_timeout(max_seconds, purpose): + start = time.time() + count = 0 + while (time.time() < start + max_seconds): + count += 1 + yield count + time.sleep(2) + raise Exception("Timeout waiting for %s" % purpose) + + +def get_client(provider): + args = [provider.nova_username, provider.nova_api_key, + provider.nova_project_id, provider.nova_auth_url] + kwargs = {} + if provider.nova_service_type: + kwargs['service_type'] = provider.nova_service_type + if provider.nova_service_name: + kwargs['service_name'] = provider.nova_service_name + if provider.nova_service_region: + kwargs['region_name'] = provider.nova_service_region + if provider.nova_api_version == '1.0': + Client = Client10.Client + elif provider.nova_api_version == '1.1': + Client = Client11.Client + else: + raise Exception("API version not supported") + if provider.nova_rax_auth: + os.environ['NOVA_RAX_AUTH'] = '1' + client = Client(*args, **kwargs) + return client + +extension_cache = {} +def get_extensions(client): + global extension_cache + cache = extension_cache.get(client) + if cache: + return cache + try: + resp, body = client.client.get('/extensions') + extensions = [x['alias'] for x in body['extensions']] + except novaclient.exceptions.NotFound: + extensions = [] + extension_cache[client] = extensions + return extensions + +def get_flavor(client, min_ram): + flavors = [f for f in client.flavors.list() if f.ram >= min_ram] + flavors.sort(lambda a, b: cmp(a.ram, b.ram)) + return flavors[0] + +def get_public_ip(server, version=4): + if 'os-floating-ips' in get_extensions(server.manager.api): + print 'using floating ips' + for addr in server.manager.api.floating_ips.list(): + print 'checking addr', addr + if addr.instance_id == server.id: + print 'found addr', addr + return addr.ip + print 'no floating ip, addresses:' + print server.addresses + for addr in server.addresses.get('public', []): + if type(addr) == type(u''): # Rackspace/openstack 1.0 + return addr + if addr['version'] == version: #Rackspace/openstack 1.1 + return addr['addr'] + for addr in server.addresses.get('private', []): + if addr['version'] == version and not addr['addr'].startswith('10.'): #HPcloud + return addr['addr'] + return None + +def add_public_ip(server): + ip = server.manager.api.floating_ips.create() + print "created floating ip", ip + server.add_floating_ip(ip) + for count in iterate_timeout(600, "ip to be added"): + try: + newip = ip.manager.get(ip.id) + except: + print "Unable to get ip details, will retry" + traceback.print_exc() + time.sleep(5) + continue + + if newip.instance_id == server.id: + print 'ip has been added' + return + +def add_keypair(client, name): + key = paramiko.RSAKey.generate(2048) + public_key = key.get_name() + ' ' + key.get_base64() + kp = client.keypairs.create(name, public_key) + return key, kp + +def wait_for_resource(wait_resource): + last_progress = None + last_status = None + # It can take a _very_ long time for Rackspace 1.0 to save an image + for count in iterate_timeout(21600, "waiting for %s" % wait_resource): + try: + resource = wait_resource.manager.get(wait_resource.id) + except: + print "Unable to list resources, will retry" + traceback.print_exc() + time.sleep(5) + continue + + # In Rackspace v1.0, there is no progress attribute while queued + if hasattr(resource, 'progress'): + if last_progress != resource.progress or last_status != resource.status: + print resource.status, resource.progress + last_progress = resource.progress + elif last_status != resource.status: + print resource.status + last_status = resource.status + if resource.status == 'ACTIVE': + return resource + +def ssh_connect(ip, username, connect_kwargs={}, timeout=60): + # HPcloud may return errno 111 for about 30 seconds after adding the IP + for count in iterate_timeout(timeout, "ssh access"): + try: + client = SSHClient(ip, username, **connect_kwargs) + break + except socket.error, e: + print "While testing ssh access:", e + time.sleep(5) + + ret, out = client.ssh("echo access okay") + if "access okay" in out: + return client + return None + +def delete_server(server): + try: + if 'os-floating-ips' in get_extensions(server.manager.api): + for addr in server.manager.api.floating_ips.list(): + if addr.instance_id == server.id: + server.remove_floating_ip(addr) + addr.delete() + except: + print "Unable to remove floating IP" + traceback.print_exc() + + try: + if 'os-keypairs' in get_extensions(server.manager.api): + for kp in server.manager.api.keypairs.list(): + if kp.name == server.key_name: + kp.delete() + except: + print "Unable to delete keypair" + traceback.print_exc() + + print "Deleting server", server.id + server.delete()