From ec21da0c338a4dadf96135f2ed9ea44b63ccfe61 Mon Sep 17 00:00:00 2001 From: Tim Kuhlman Date: Wed, 14 May 2014 10:46:45 -0600 Subject: [PATCH 1/3] Initial set of baremetal scripts --- .gitignore | 1 + utils/README.md | 1 + utils/baremetal.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++ utils/fabfile.py | 1 + 4 files changed, 91 insertions(+) create mode 100644 utils/README.md create mode 100644 utils/baremetal.py create mode 100644 utils/fabfile.py diff --git a/.gitignore b/.gitignore index 16b15da..f946b43 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ templates/packer_cache .pydevproject .idea *.deb +*.pyc diff --git a/utils/README.md b/utils/README.md new file mode 100644 index 0000000..8127dae --- /dev/null +++ b/utils/README.md @@ -0,0 +1 @@ +Various utilities for leveraging mini-mon. These are written as [fabric](http://docs.fabfile.org/) scripts diff --git a/utils/baremetal.py b/utils/baremetal.py new file mode 100644 index 0000000..210640e --- /dev/null +++ b/utils/baremetal.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# +# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Fabric Tasks for installing mini-mon on baremetal +These tasks were developed for hLinux but will likely work on any decently up to date debian based distro +""" + +from fabric.api import * + + +@task +def chef_solo(chef_dir='/vagrant', run_list='role[Mini-Mon'): + """Runs chef-solo + This assumes chef solo and other dependencies are setup. + """ + # Setup solo.rb + solo_content = """ + cookbook_path "{dir}/cookbooks" + role_path "{dir}/roles" + data_bag_path "{dir}/data_bags" + """.format(dir=chef_dir) + sudo('echo %s > %s/solo.rb' % (solo_content, chef_dir)) + + # Setup node.json + node_json = '{ "run_list": "%s" }' % run_list + sudo('echo "%s" > %s/node.json' % (node_json, chef_dir)) + + # Run chef-solo + sudo('chef-solo -c {dir}/solo.rb -j {dir}/node.json'.format(dir=chef_dir)) + + +@task(default=True) +def install(install_dir='/vagrant'): # /vagrant to match assumptions in mini-mon even though vagrant is not used here + """Installs the latest mini-mon and bits necessary to run chef-solo and runs chef-solo on the box. + """ + execute(install_deps) + + #Clone mini-mon + sudo('git clone https://github.com/hpcloud-mon/mon-vagrant.git %s' % install_dir) + # currently the one-vm setup is on a branch + with cd(install_dir): + sudo('git checkout feature/one-vm') + + # download cookbooks + with cd(install_dir): + sudo('berks vendor cookbooks') + + # the vertica packages from my.vertica.com are needed, this assumes they are one level up from where this script is + put('../vertica*.deb', install_dir, use_sudo=True) + + execute(chef_solo) + + +@task +def install_berkshelf(): + """Installs berkshelf.""" + # todo check for ruby 1.9.2 or greater + # If needed use rvm to get a newer ruby - curl -sSL https://get.rvm.io | bash -s stable --ruby + sudo('gem install berkshelf') + + +@task +def install_chef(): + """Installs chef via omnibus.""" + sudo('apt-get install -y curl') + sudo('curl -L https://www.opscode.com/chef/install.sh | bash') + + +@task +def install_deps(): + """Install all dependencies needed for running chef-solo""" + with settings(hide('running', 'output')): + sudo('apt-get install -y git') + execute(install_chef) + execute(install_berkshelf) diff --git a/utils/fabfile.py b/utils/fabfile.py new file mode 100644 index 0000000..2228b29 --- /dev/null +++ b/utils/fabfile.py @@ -0,0 +1 @@ +import baremetal From 766fc2ccb96e7e447de11ce6113454b9857acc93 Mon Sep 17 00:00:00 2001 From: Tim Kuhlman Date: Wed, 14 May 2014 16:38:13 -0600 Subject: [PATCH 2/3] Initial proxy support and fixes from trying to run it on hLinux --- utils/baremetal.py | 108 +++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/utils/baremetal.py b/utils/baremetal.py index 210640e..da0e6de 100644 --- a/utils/baremetal.py +++ b/utils/baremetal.py @@ -22,67 +22,111 @@ from fabric.api import * @task -def chef_solo(chef_dir='/vagrant', run_list='role[Mini-Mon'): +def chef_solo(chef_dir='/vagrant', run_list='role[Mini-Mon]', proxy=None): """Runs chef-solo This assumes chef solo and other dependencies are setup. """ # Setup solo.rb - solo_content = """ - cookbook_path "{dir}/cookbooks" - role_path "{dir}/roles" - data_bag_path "{dir}/data_bags" - """.format(dir=chef_dir) - sudo('echo %s > %s/solo.rb' % (solo_content, chef_dir)) + solo_content = '''cookbook_path "{dir}/cookbooks" +role_path "{dir}/roles" +data_bag_path "{dir}/data_bags"'''.format(dir=chef_dir) + sudo("echo '%s' > %s/solo.rb" % (solo_content, chef_dir)) # Setup node.json node_json = '{ "run_list": "%s" }' % run_list - sudo('echo "%s" > %s/node.json' % (node_json, chef_dir)) + sudo("echo '%s' > %s/node.json" % (node_json, chef_dir)) # Run chef-solo - sudo('chef-solo -c {dir}/solo.rb -j {dir}/node.json'.format(dir=chef_dir)) + # todo - proxy hell defeats chef at this point because some components like pip need it but others like apt choke + with prefix(proxy_string(proxy)): + sudo('chef-solo -c {dir}/solo.rb -j {dir}/node.json'.format(dir=chef_dir)) @task(default=True) -def install(install_dir='/vagrant'): # /vagrant to match assumptions in mini-mon even though vagrant is not used here +def install(install_dir='/vagrant', proxy=None): """Installs the latest mini-mon and bits necessary to run chef-solo and runs chef-solo on the box. + proxy is an optional proxy url used for http and https, it is not used for apt as that is assumed to be + correctly setup. + install_dir defaults to /vagrant to match assumptions from mini-mon even though vagrant is not used here """ - execute(install_deps) + if proxy is not None: + abort('Proxy support is incomplete.') + execute(install_deps, proxy) #Clone mini-mon - sudo('git clone https://github.com/hpcloud-mon/mon-vagrant.git %s' % install_dir) - # currently the one-vm setup is on a branch - with cd(install_dir): - sudo('git checkout feature/one-vm') + with prefix(proxy_string(proxy)): + # Update the install dir if it already has code, otherwise check out + with settings(hide('running', 'output', 'warnings'), warn_only=True): + install_dir_check = run('ls %s' % install_dir) - # download cookbooks - with cd(install_dir): - sudo('berks vendor cookbooks') + if install_dir_check.succeeded: + with cd(install_dir): + sudo('git pull -f origin master') + else: + sudo('git clone https://github.com/hpcloud-mon/mon-vagrant.git %s' % install_dir) - # the vertica packages from my.vertica.com are needed, this assumes they are one level up from where this script is - put('../vertica*.deb', install_dir, use_sudo=True) + # currently the one-vm setup is on a branch + with cd(install_dir): + sudo('git checkout feature/one-vm') - execute(chef_solo) + # download cookbooks + with cd(install_dir): + sudo('berks vendor cookbooks') + + # the vertica packages from my.vertica.com are needed, this assumes they are one level up from cwd + put('../vertica*.deb', install_dir, use_sudo=True) + + execute(chef_solo, proxy) @task -def install_berkshelf(): +def install_berkshelf(proxy=None): """Installs berkshelf.""" - # todo check for ruby 1.9.2 or greater - # If needed use rvm to get a newer ruby - curl -sSL https://get.rvm.io | bash -s stable --ruby - sudo('gem install berkshelf') + # check for ruby 1.9.2 or greater and if needed install + with settings(hide('running', 'output', 'warnings'), warn_only=True): + ruby_check = run('ruby -v') + if ruby_check.failed: + # Install both ruby and tools needed to build gems + sudo('apt-get install -y ruby ruby-hitimes build-essential') + else: + # A better semantic version check like semantic_version module provides would be nice + version_parts = ruby_check.split()[1].split('.') + if int(version_parts[0]) < 2 and int(version_parts[1]) < 9: + abort('Ruby reports version %s, > 1.9.2 is needed' % ruby_check) + + with prefix(proxy_string(proxy)): + sudo('gem install berkshelf') @task -def install_chef(): +def install_chef(proxy=None): """Installs chef via omnibus.""" sudo('apt-get install -y curl') - sudo('curl -L https://www.opscode.com/chef/install.sh | bash') + + # If chef is already installed continue + with settings(hide('running', 'output', 'warnings'), warn_only=True): + chef_check = run('chef-solo -v') + if chef_check.succeeded: + return + + # Run the omnibus installer + with prefix(proxy_string(proxy)): + sudo('curl -s -S -L https://www.opscode.com/chef/install.sh | bash') @task -def install_deps(): +def install_deps(proxy=None): """Install all dependencies needed for running chef-solo""" - with settings(hide('running', 'output')): - sudo('apt-get install -y git') - execute(install_chef) - execute(install_berkshelf) + sudo('apt-get install -y git') + execute(install_chef, proxy) + execute(install_berkshelf, proxy) + + +def proxy_string(proxy): + """Return a string using the given proxy url. + An example proxy to pass in myproxy.me.com:8080 + """ + if proxy is None: + return "no_proxy=" # I have to return a noop string as the prefix context manager will add it to the cmd + else: + return "export http_proxy='http://{proxy}' && export https_proxy='http://{proxy}'".format(proxy=proxy) From 5c1e2a3a985e0c2b7a3ddbdde771b25ac3c93c7d Mon Sep 17 00:00:00 2001 From: Tim Kuhlman Date: Wed, 14 May 2014 16:59:47 -0600 Subject: [PATCH 3/3] Added apt-transport-https before defining the https based apt repo --- metadata.rb | 2 +- recipes/default.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/metadata.rb b/metadata.rb index b21dd44..dc3fa72 100644 --- a/metadata.rb +++ b/metadata.rb @@ -1,5 +1,5 @@ name "mini-mon" -maintainer "SOM Team" +maintainer "Mon Team" maintainer_email "hpcs-mon@hp.com" license "All rights reserved" description "Base setup for all vagrant boxes" diff --git a/recipes/default.rb b/recipes/default.rb index 195baf1..bddebf1 100644 --- a/recipes/default.rb +++ b/recipes/default.rb @@ -20,6 +20,10 @@ rb.run_action(:create) # Run during compile time so that apt::cacher-client has # Add in the cacher-client, it will do something or nothing depending on the value of node[:apt][:cacher_ipaddress] include_recipe('apt::cacher-client') +# Some barebones distros do not yet have this installed +package 'apt-transport-https' do + action :install +end apt_repository 'dev' do uri 'https://region-a.geo-1.objects.hpcloudsvc.com/v1/46995959297574/mini-mon/public_repo' arch 'amd64'