fix merge errors

README

@@ -15,3 +15,6 @@ To disect it in detail: visit http://github.com/nova/cc
 To taunt it with its weaknesses: use http://github.com/nova/cc/issues
 
 To hack at it: read HACKING
+
+To watch it: http://test.novacc.org/waterfall
+

bin/dhcpleasor.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+# Copyright 2010 Anso Labs, LLC
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+dhcpleasor.py
+
+Handle lease database updates from DHCP servers.
+"""
+
+import sys
+import os
+import logging
+sys.path.append(os.path.abspath(os.path.join(__file__, "../../")))
+
+logging.debug(sys.path)
+import getopt
+from os import environ
+from nova.compute import linux_net
+from nova.compute import network
+from nova import rpc
+
+from nova import flags
+FLAGS = flags.FLAGS
+
+
+def add_lease(mac, ip, hostname, interface):
+    if FLAGS.fake_rabbit:
+        network.lease_ip(ip)
+    else:
+        rpc.cast(FLAGS.cloud_topic, {"method": "lease_ip",
+                "args" : {"address": ip}})
+
+def old_lease(mac, ip, hostname, interface):
+    logging.debug("Adopted old lease or got a change of mac/hostname")
+
+def del_lease(mac, ip, hostname, interface):
+    if FLAGS.fake_rabbit:
+        network.release_ip(ip)
+    else:
+        rpc.cast(FLAGS.cloud_topic, {"method": "release_ip",
+                "args" : {"address": ip}})
+
+def init_leases(interface):
+    net = network.get_network_by_interface(interface)
+    res = ""
+    for host_name in net.hosts:
+        res += "%s\n" % linux_net.hostDHCP(net, host_name, net.hosts[host_name])
+    return res
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+    interface = environ.get('DNSMASQ_INTERFACE', 'br0')
+    if int(environ.get('TESTING', '0')):
+        FLAGS.fake_rabbit = True
+        FLAGS.redis_db = 8
+        FLAGS.network_size = 32
+        FLAGS.fake_libvirt=True
+        FLAGS.fake_network=True
+        FLAGS.fake_users = True
+    action = argv[1]
+    if action in ['add','del','old']:
+        mac = argv[2]
+        ip = argv[3]
+        hostname = argv[4]
+        logging.debug("Called %s for mac %s with ip %s and hostname %s on interface %s" % (action, mac, ip, hostname, interface))
+        globals()[action+'_lease'](mac, ip, hostname, interface)
+    else:
+        print init_leases(interface)
+    exit(0)
+
+if __name__ == "__main__":
+    sys.exit(main())

debian/changelog

@@ -1,3 +1,54 @@
+nova (0.2.3-1) UNRELEASED; urgency=low
+
+  * Relax the Twisted dependency to python-twisted-core (rather than the
+    full stack).
+  * Move nova related configuration files into /etc/nova/.
+  * Add a dependency on nginx from nova-objectsstore and install a
+    suitable configuration file.
+  * Ship the CA directory in nova-common.
+  * Add a default flag file for nova-manage to help it find the CA.
+  * If set, pass KernelId and RamdiskId from RunInstances call to the
+    target compute node.
+  * Added --network_path setting to nova-compute's flagfile.
+  * Move templates from python directories to /usr/share/nova.
+  * Add debian/nova-common.dirs to create
+    var/lib/nova/{buckets,CA,images,instances,keys,networks}
+  * Don't pass --daemonize=1 to nova-compute. It's already daemonising
+    by default.
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Mon, 14 Jul 2010 12:00:00 -0700
+
+nova (0.2.2-10) UNRELEASED; urgency=low
+
+  * Fixed extra space in vblade-persist
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Mon, 13 Jul 2010 19:00:00 -0700
+
+nova (0.2.2-9) UNRELEASED; urgency=low
+
+  * Fixed invalid dn bug in ldap for adding roles
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Mon, 12 Jul 2010 15:20:00 -0700
+
+nova (0.2.2-8) UNRELEASED; urgency=low
+
+  * Added a missing comma
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Mon, 08 Jul 2010 10:05:00 -0700
+
+nova (0.2.2-7) UNRELEASED; urgency=low
+
+  * Missing files from twisted patch
+  * License upedates
+  * Reformatting/cleanup
+  * Users/ldap bugfixes
+  * Merge fixes
+  * Documentation updates
+  * Vpn key creation fix
+  * Multiple shelves for volumes
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Wed, 07 Jul 2010 18:45:00 -0700
+
 nova (0.2.2-6) UNRELEASED; urgency=low
 
   * Fix to make Key Injection work again

debian/control

@@ -3,36 +3,112 @@ Section: net
 Priority: extra
 Maintainer: Jesse Andrews <jesse@ansolabs.com>
 Build-Depends: debhelper (>= 7)
-Build-Depends-Indep: python-support
+Build-Depends-Indep: python-support, python-setuptools
 Standards-Version: 3.8.4
 XS-Python-Version: 2.6
 
 Package: nova-common
 Architecture: all
-Depends: ${python:Depends}, aoetools, vlan, python-ipy, python-boto, python-m2crypto, python-pycurl, python-twisted (>= 10.0.0-2ubuntu2nebula1), python-daemon, python-redis, python-carrot, python-lockfile, python-gflags, python-tornado, ${misc:Depends}
+Depends: ${python:Depends}, aoetools, vlan, python-ipy, python-boto, python-m2crypto, python-pycurl, python-twisted-core, python-daemon, python-redis, python-carrot, python-lockfile, python-gflags, python-tornado, ${misc:Depends}
 Provides: ${python:Provides}
-Conflicts: nova
-Description: Nova is a cloud
+Description: Nova Cloud Computing - common files
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This package contains things that are needed by all parts of Nova.
 
 Package: nova-compute
 Architecture: all
-Depends: nova-common (= ${binary:Version}), kpartx, kvm, python-libvirt, libvirt-bin (>= 0.8.1), ${python:Depends}, ${misc:Depends}
-Description: Nova compute
+Depends: nova-common (= ${binary:Version}), kpartx, kvm, python-libvirt, libvirt-bin (>= 0.7.5), curl, ${python:Depends}, ${misc:Depends}
+Description: Nova Cloud Computing - compute node
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This is the package you will install on the nodes that will run your
+ virtual machines.
 
 Package: nova-volume
 Architecture: all
 Depends: nova-common (= ${binary:Version}), vblade, vblade-persist, ${python:Depends}, ${misc:Depends}
-Description: Nova volume
+Description: Nova Cloud Computing - storage
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This is the package you will install on your storage nodes.
 
 Package: nova-api
 Architecture: all
 Depends: nova-common (= ${binary:Version}), ${python:Depends}, ${misc:Depends}
-Description: Nova api
+Description: Nova Cloud Computing - API frontend 
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This package provides the API frontend.
 
 Package: nova-objectstore
 Architecture: all
-Depends: nova-common (= ${binary:Version}), ${python:Depends}, ${misc:Depends}
-Description: Nova object store
+Depends: nova-common (= ${binary:Version}), nginx, ${python:Depends}, ${misc:Depends}
+Description: Nova Cloud Computing - object store
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This is the package you will install on the nodes that will contain your
+ object store.
 
 Package: nova-instancemonitor
 Architecture: all
@@ -42,4 +118,19 @@ Description: Nova instance monitor
 Package: nova-tools
 Architecture: all
 Depends: python-boto, ${python:Depends}, ${misc:Depends}
-Description: CLI tools to access nova
+Description: Nova Cloud Computing - management tools
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This package contains admin tools for Nova.

debian/nova-api.conf

@@ -0,0 +1,5 @@
+--daemonize=1
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--fake_users=1
+--datastore_path=/var/lib/nova/keeper

debian/nova-api.init

@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-api
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-api.conf"
 PIDFILE=/var/run/nova-api.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-api; then
   . /etc/default/nova-api

debian/nova-api.install

@@ -1 +1,2 @@
 bin/nova-api usr/bin
+debian/nova-api.conf etc/nova

debian/nova-common.dirs

@@ -0,0 +1,11 @@
+etc/nova
+var/lib/nova/buckets
+var/lib/nova/CA
+var/lib/nova/CA/INTER
+var/lib/nova/CA/newcerts
+var/lib/nova/CA/private
+var/lib/nova/CA/reqs
+var/lib/nova/images
+var/lib/nova/instances
+var/lib/nova/keys
+var/lib/nova/networks

debian/nova-common.install

@@ -1,5 +1,10 @@
 bin/nova-manage usr/bin
-nova/auth/novarc.template             usr/lib/pymodules/python2.6/nova/auth
-nova/cloudpipe/client.ovpn.template   usr/lib/pymodules/python2.6/nova/cloudpipe
-nova/compute/libvirt.xml.template     usr/lib/pymodules/python2.6/nova/compute
+debian/nova-manage.conf etc/nova
+nova/auth/novarc.template             usr/share/nova
+nova/cloudpipe/client.ovpn.template   usr/share/nova
+nova/compute/libvirt.xml.template     usr/share/nova
+nova/compute/interfaces.template      usr/share/nova
 usr/lib/python*/*-packages/nova/*
+CA/openssl.cnf.tmpl                   var/lib/nova/CA
+CA/geninter.sh                        var/lib/nova/CA
+CA/genrootca.sh                       var/lib/nova/CA

debian/nova-compute.conf

@@ -0,0 +1,10 @@
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--datastore_path=/var/lib/nova/keeper
+--instances_path=/var/lib/nova/instances
+--networks_path=/var/lib/nova/networks
+--simple_network_template=/usr/share/nova/interfaces.template
+--libvirt_xml_template=/usr/share/nova/libvirt.xml.template
+--vpn_client_template=/usr/share/nova/client.ovpn.template
+--credentials_template=/usr/share/nova/novarc.template
+--fake_users=1

debian/nova-compute.init

@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-compute
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-compute.conf"
 PIDFILE=/var/run/nova-compute.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-compute; then
   . /etc/default/nova-compute

debian/nova-compute.install

@@ -1 +1,2 @@
 bin/nova-compute usr/bin
+debian/nova-compute.conf etc/nova

debian/nova-manage.conf

@@ -0,0 +1,4 @@
+--ca_path=/var/lib/nova/CA
+--credentials_template=/usr/share/nova/novarc.template
+--keys_path=/var/lib/nova/keys
+--vpn_client_template=/usr/share/nova/client.ovpn.template

debian/nova-objectstore.conf

@@ -0,0 +1,7 @@
+--daemonize=1
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--datastore_path=/var/lib/nova/keeper
+--fake_users=1
+--images_path=/var/lib/nova/images
+--buckets_path=/var/lib/nova/buckets

debian/nova-objectstore.init

@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-objectstore
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-objectstore.conf"
 PIDFILE=/var/run/nova-objectstore.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-objectstore; then
   . /etc/default/nova-objectstore

debian/nova-objectstore.install

@@ -1 +1,3 @@
 bin/nova-objectstore usr/bin
+debian/nova-objectstore.conf etc/nova
+debian/nova-objectstore.nginx.conf etc/nginx/sites-available

debian/nova-objectstore.links

@@ -0,0 +1 @@
+/etc/nginx/sites-available/nova-objectstore.nginx.conf /etc/nginx/sites-enabled/nova-objectstore.nginx.conf

debian/nova-objectstore.nginx.conf

@@ -0,0 +1,17 @@
+server {
+  listen 3333 default;
+  server_name localhost;
+  client_max_body_size 10m;
+
+  access_log /var/log/nginx/localhost.access.log;
+
+  location ~ /_images/.+ {
+    root /var/lib/nova/images;
+    rewrite ^/_images/(.*)$ /$1 break;
+  }
+
+  location / {
+    proxy_pass http://localhost:3334/;
+  }
+}
+

debian/nova-volume.conf

@@ -0,0 +1,7 @@
+--daemonize=1
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--datastore_path=/var/lib/nova/keeper
+--fake_users=1
+--images_path=/var/lib/nova/images
+--buckets_path=/var/lib/nova/buckets

debian/nova-volume.init

@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-volume
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-volume.conf"
 PIDFILE=/var/run/nova-volume.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-volume; then
   . /etc/default/nova-volume

debian/nova-volume.install

@@ -1 +1,2 @@
 bin/nova-volume usr/bin
+debian/nova-volume.conf etc/nova

docs/conf.py

@@ -16,7 +16,7 @@ import sys, os
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.append(os.path.abspath('.'))
+sys.path.append(os.path.abspath('/Users/jmckenty/Projects/cc'))
 sys.path.append([os.path.abspath('../nova'),os.path.abspath('../'),os.path.abspath('../vendor')])
 from nova import vendor
 

docs/getting.started.rst

@@ -1,22 +1,3 @@
-..
-      Copyright 2010 United States Government as represented by the
-      Administrator of the National Aeronautics and Space Administration. 
-      All Rights Reserved.
-
-      Copyright 2010 Anso Labs, LLC
-
-      Licensed under the Apache License, Version 2.0 (the "License"); you may
-      not use this file except in compliance with the License. You may obtain
-      a copy of the License at
-
-          http://www.apache.org/licenses/LICENSE-2.0
-
-      Unless required by applicable law or agreed to in writing, software
-      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-      License for the specific language governing permissions and limitations
-      under the License.
-
 Getting Started with Nova
 =========================
 
@@ -62,7 +43,7 @@ Installation
 ::
 
     # system libraries and tools
-    apt-get install -y aoetools vlan
+    apt-get install -y aoetools vlan curl
     modprobe aoe
 
     # python libraries
@@ -81,6 +62,7 @@ Installation
     # ON THE COMPUTE NODE:
     apt-get install -y python-libvirt
     apt-get install -y kpartx kvm libvirt-bin
+    modprobe kvm
 
     # optional packages
     apt-get install -y euca2ools
@@ -111,7 +93,7 @@ ON CLOUD CONTROLLER
 
     location ~ /_images/.+ {
       root NOVA_PATH/images;
-      rewrite ^/_images/(.*)\$ /\$1 break;
+      rewrite ^/_images/(.*)$ /$1 break;
     }
 
     location / {
@@ -128,6 +110,7 @@ ON VOLUME NODE
     # This creates a 1GB file to create volumes out of
     dd if=/dev/zero of=MY_FILE_PATH bs=100M count=10
     losetup --show -f MY_FILE_PATH
+    # replace loop0 below with whatever losetup returns
     echo "--storage_dev=/dev/loop0" >> NOVA_PATH/bin/nova.conf
 
 Running

run_tests.py

@@ -46,13 +46,13 @@ import sys
 from nova import vendor
 from twisted.scripts import trial as trial_script
 
+from nova import datastore
 from nova import flags
 from nova import twistd
 
 from nova.tests.access_unittest import *
 from nova.tests.api_unittest import *
 from nova.tests.cloud_unittest import *
-from nova.tests.keeper_unittest import *
 from nova.tests.model_unittest import *
 from nova.tests.network_unittest import *
 from nova.tests.node_unittest import *
@@ -60,7 +60,6 @@ from nova.tests.objectstore_unittest import *
 from nova.tests.process_unittest import *
 from nova.tests.storage_unittest import *
 from nova.tests.users_unittest import *
-from nova.tests.datastore_unittest import *
 from nova.tests.validator_unittest import *