x/solar
Code Issues Proposed changes

Merge pull request #64 from xarses/various-fixes

Browse Source 
Various fixes
This commit is contained in:
Łukasz Oleś 2015-05-29 17:37:43 +02:00
commit a9f105e98d
18 changed files with 208 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Vagrantfile vendored
View File

@ -33,6 +33,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
index = i + 1
ip_index = i + 2
config.vm.define "solar-dev#{index}" do |config|
config.vm.provision "shell", inline: init_script, privileged: true
config.vm.network "private_network", ip: "10.0.0.#{ip_index}"
config.vm.host_name = "solar-dev#{index}"

15
cli.py
View File

@ -66,7 +66,15 @@ def init_cli_resource():
@click.argument('path')
@click.option('--all/--one', default=False)
@click.option('--tag', default=None)
def show(tag, all, path):
@click.option('--use-json/--no-use-json', default=False)
def show(use_json, tag, all, path):
import json
import six
printer = lambda r: six.print_(r)
if use_json:
printer = lambda r: six.print_(json.dumps(r.to_dict()))
if all or tag:
for name, resource in xr.load_all(path).items():
show = True
@ -75,10 +83,9 @@ def init_cli_resource():
show = False
if show:
print resource
print
printer(resource)
else:
print xr.load(path)
printer(xr.load(path))
resource.add_command(show)

233
example.py
View File

@ -1,124 +1,155 @@
import shutil
import os
import click
import requests
import sys
import time
from solar.core import actions
from solar.core import resource
from solar.core import signals
from solar.core import validation
from solar.interfaces.db import get_db
db = get_db()
db.clear()
signals.Connections.clear()
if os.path.exists('rs'):
shutil.rmtree('rs')
os.mkdir('rs')
node1 = resource.create('node1', 'resources/ro_node/', {'ip':'10.0.0.3', 'ssh_key' : '/vagrant/.vagrant/machines/solar-dev2/virtualbox/private_key', 'ssh_user':'vagrant'})
node2 = resource.create('node2', 'resources/ro_node/', {'ip':'10.0.0.4', 'ssh_key' : '/vagrant/.vagrant/machines/solar-dev3/virtualbox/private_key', 'ssh_user':'vagrant'})
mariadb_service1 = resource.create('mariadb_service1', 'resources/mariadb_service', {'image':'mariadb', 'root_password' : 'mariadb', 'port' : '3306', 'ip': '', 'ssh_user': '', 'ssh_key': ''})
keystone_db = resource.create('keystone_db', 'resources/mariadb_db/', {'db_name':'keystone_db', 'login_password':'', 'login_user':'root', 'login_port': '', 'ip':'', 'ssh_user':'', 'ssh_key':''})
keystone_db_user = resource.create('keystone_db_user', 'resources/mariadb_user/', {'new_user_name' : 'keystone', 'new_user_password' : 'keystone', 'db_name':'', 'login_password':'', 'login_user':'root', 'login_port': '', 'ip':'', 'ssh_user':'', 'ssh_key':''})
keystone_config1 = resource.create('keystone_config1', 'resources/keystone_config/', {'config_dir' : '/etc/solar/keystone', 'ip':'', 'ssh_user':'', 'ssh_key':'', 'admin_token':'admin', 'db_password':'', 'db_name':'', 'db_user':'', 'db_host':''})
keystone_service1 = resource.create('keystone_service1', 'resources/keystone_service/', {'port':'5001', 'admin_port':'35357', 'image': '', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':''})
keystone_config2 = resource.create('keystone_config2', 'resources/keystone_config/', {'config_dir' : '/etc/solar/keystone', 'ip':'', 'ssh_user':'', 'ssh_key':'', 'admin_token':'admin', 'db_password':'', 'db_name':'', 'db_user':'', 'db_host':''})
keystone_service2 = resource.create('keystone_service2', 'resources/keystone_service/', {'port':'5002', 'admin_port':'35357', 'image': '', 'ip':'', 'ssh_key':'', 'ssh_user':'', 'config_dir':''})
@click.group()
def main():
pass
haproxy_keystone_config = resource.create('haproxy_keystone1_config', 'resources/haproxy_keystone_config/', {'name':'keystone_config', 'listen_port':'5000', 'servers':[], 'ports':[]})
haproxy_config = resource.create('haproxy_config', 'resources/haproxy', {'ip':'', 'ssh_key':'', 'ssh_user':'', 'configs_names':[], 'configs_ports':[], 'listen_ports':[], 'configs':[], 'config_dir': ''})
haproxy_service = resource.create('haproxy_service', 'resources/docker_container/', {'image' : 'tutum/haproxy', 'ports': [], 'host_binds': [], 'volume_binds':[], 'ip':'', 'ssh_key':'', 'ssh_user':''})
@click.command()
def deploy():
db = get_db()
db.clear()
signals.Connections.clear()
node1 = resource.create('node1', 'resources/ro_node/', {'ip': '10.0.0.3', 'ssh_key': '/vagrant/.vagrant/machines/solar-dev2/virtualbox/private_key', 'ssh_user': 'vagrant'})
node2 = resource.create('node2', 'resources/ro_node/', {'ip': '10.0.0.4', 'ssh_key': '/vagrant/.vagrant/machines/solar-dev3/virtualbox/private_key', 'ssh_user': 'vagrant'})
mariadb_service1 = resource.create('mariadb_service1', 'resources/mariadb_service', {'image': 'mariadb', 'root_password': 'mariadb', 'port': 3306, 'ip': '', 'ssh_user': '', 'ssh_key': ''})
keystone_db = resource.create('keystone_db', 'resources/mariadb_db/', {'db_name': 'keystone_db', 'login_password': '', 'login_user': 'root', 'login_port': '', 'ip': '', 'ssh_user': '', 'ssh_key': ''})
keystone_db_user = resource.create('keystone_db_user', 'resources/mariadb_user/', {'new_user_name': 'keystone', 'new_user_password': 'keystone', 'db_name': '', 'login_password': '', 'login_user': 'root', 'login_port': '', 'ip': '', 'ssh_user': '', 'ssh_key': ''})
keystone_config1 = resource.create('keystone_config1', 'resources/keystone_config/', {'config_dir': '/etc/solar/keystone', 'ip': '', 'ssh_user': '', 'ssh_key': '', 'admin_token': 'admin', 'db_password': '', 'db_name': '', 'db_user': '', 'db_host': '', 'db_port': ''})
keystone_service1 = resource.create('keystone_service1', 'resources/keystone_service/', {'port': 5001, 'admin_port': 35357, 'image': '', 'ip': '', 'ssh_key': '', 'ssh_user': '', 'config_dir': ''})
keystone_config2 = resource.create('keystone_config2', 'resources/keystone_config/', {'config_dir': '/etc/solar/keystone', 'ip': '', 'ssh_user': '', 'ssh_key': '', 'admin_token': 'admin', 'db_password': '', 'db_name': '', 'db_user': '', 'db_host': '', 'db_port': ''})
keystone_service2 = resource.create('keystone_service2', 'resources/keystone_service/', {'port': 5002, 'admin_port': 35357, 'image': '', 'ip': '', 'ssh_key': '', 'ssh_user': '', 'config_dir': ''})
####
# connections
####
#mariadb
signals.connect(node1, mariadb_service1)
#keystone db
signals.connect(node1, keystone_db)
signals.connect(mariadb_service1, keystone_db, {'root_password':'login_password', 'port':'login_port'})
# keystone_db_user
signals.connect(node1, keystone_db_user)
signals.connect(mariadb_service1, keystone_db_user, {'root_password':'login_password', 'port':'login_port'})
signals.connect(keystone_db, keystone_db_user, {'db_name':'db_name'})
signals.connect(node1, keystone_config1)
signals.connect(mariadb_service1, keystone_config1, {'ip':'db_host'})
signals.connect(keystone_db_user, keystone_config1, {'db_name':'db_name', 'new_user_name':'db_user', 'new_user_password':'db_password'})
signals.connect(node1, keystone_service1)
signals.connect(keystone_config1, keystone_service1, {'config_dir': 'config_dir'})
signals.connect(node2, keystone_config2)
signals.connect(mariadb_service1, keystone_config2, {'ip':'db_host'})
signals.connect(keystone_db_user, keystone_config2, {'db_name':'db_name', 'new_user_name':'db_user', 'new_user_password':'db_password'})
signals.connect(node2, keystone_service2)
signals.connect(keystone_config2, keystone_service2, {'config_dir': 'config_dir'})
signals.connect(keystone_service1, haproxy_keystone_config, {'ip':'servers', 'port':'ports'})
signals.connect(keystone_service2, haproxy_keystone_config, {'ip':'servers', 'port':'ports'})
signals.connect(node2, haproxy_config)
signals.connect(haproxy_keystone_config, haproxy_config, {'listen_port': 'listen_ports', 'name':'configs_names', 'ports' : 'configs_ports', 'servers':'configs'})
signals.connect(node2, haproxy_service)
signals.connect(haproxy_config, haproxy_service, {'listen_ports':'ports', 'config_dir':'host_binds'})
haproxy_keystone_config = resource.create('haproxy_keystone1_config', 'resources/haproxy_keystone_config/', {'name': 'keystone_config', 'listen_port':5000, 'servers':[], 'ports':[]})
haproxy_config = resource.create('haproxy_config', 'resources/haproxy', {'ip': '', 'ssh_key': '', 'ssh_user': '', 'configs_names':[], 'configs_ports':[], 'listen_ports':[], 'configs':[], 'config_dir': ''})
haproxy_service = resource.create('haproxy_service', 'resources/docker_container/', {'image': 'tutum/haproxy', 'ports': [], 'host_binds': [], 'volume_binds':[], 'ip': '', 'ssh_key': '', 'ssh_user': ''})
from solar.core import validation
####
# connections
####
for r in [node1,
node2,
mariadb_service1,
keystone_db,
keystone_db_user,
keystone_config1,
keystone_service1,
keystone_config2,
keystone_service2,
haproxy_keystone_config,
haproxy_config,
haproxy_service]:
validation.validate_resource(r)
# mariadb
signals.connect(node1, mariadb_service1)
# keystone db
signals.connect(node1, keystone_db)
signals.connect(mariadb_service1, keystone_db, {'root_password': 'login_password', 'port': 'login_port'})
# keystone_db_user
signals.connect(node1, keystone_db_user)
signals.connect(mariadb_service1, keystone_db_user, {'root_password': 'login_password', 'port': 'login_port'})
signals.connect(keystone_db, keystone_db_user, {'db_name': 'db_name'})
signals.connect(node1, keystone_config1)
signals.connect(mariadb_service1, keystone_config1, {'ip': 'db_host', 'port': 'db_port'})
signals.connect(keystone_db_user, keystone_config1, {'db_name': 'db_name', 'new_user_name': 'db_user', 'new_user_password': 'db_password'})
signals.connect(node1, keystone_service1)
signals.connect(keystone_config1, keystone_service1, {'config_dir': 'config_dir'})
signals.connect(node2, keystone_config2)
signals.connect(mariadb_service1, keystone_config2, {'ip': 'db_host', 'port': 'db_port'})
signals.connect(keystone_db_user, keystone_config2, {'db_name': 'db_name', 'new_user_name': 'db_user', 'new_user_password': 'db_password'})
signals.connect(node2, keystone_service2)
signals.connect(keystone_config2, keystone_service2, {'config_dir': 'config_dir'})
signals.connect(keystone_service1, haproxy_keystone_config, {'ip': 'servers', 'port': 'ports'})
signals.connect(keystone_service2, haproxy_keystone_config, {'ip': 'servers', 'port': 'ports'})
signals.connect(node2, haproxy_config)
signals.connect(haproxy_keystone_config, haproxy_config, {'listen_port': 'listen_ports', 'name': 'configs_names', 'ports': 'configs_ports', 'servers': 'configs'})
signals.connect(node2, haproxy_service)
signals.connect(haproxy_config, haproxy_service, {'listen_ports': 'ports', 'config_dir': 'host_binds'})
#run
from solar.core import actions
has_errors = False
for r in [node1,
node2,
mariadb_service1,
keystone_db,
keystone_db_user,
keystone_config1,
keystone_service1,
keystone_config2,
keystone_service2,
haproxy_keystone_config,
haproxy_config,
haproxy_service]:
errors = validation.validate_resource(r)
if errors:
has_errors = True
print 'ERROR: %s: %s' % (r.name, errors)
actions.resource_action(mariadb_service1, 'run')
time.sleep(10)
actions.resource_action(keystone_db, 'run')
actions.resource_action(keystone_db_user, 'run')
actions.resource_action(keystone_config1, 'run')
actions.resource_action(keystone_service1, 'run')
actions.resource_action(keystone_config2, 'run')
actions.resource_action(keystone_service2, 'run')
actions.resource_action(haproxy_config, 'run')
actions.resource_action(haproxy_service, 'run')
if has_errors:
sys.exit(1)
#remove
#actions.resource_action(haproxy_service, 'remove')
#actions.resource_action(haproxy_config, 'remove')
#actions.resource_action(keystone_service1, 'remove')
#actions.resource_action(keystone_config1, 'remove')
#actions.resource_action(keystone_db_user, 'remove')
#actions.resource_action(keystone_db, 'remove')
#actions.resource_action(mariadb_service1, 'remove')
# run
actions.resource_action(mariadb_service1, 'run')
time.sleep(10)
actions.resource_action(keystone_db, 'run')
actions.resource_action(keystone_db_user, 'run')
actions.resource_action(keystone_config1, 'run')
actions.resource_action(keystone_service1, 'run')
actions.resource_action(keystone_config2, 'run')
actions.resource_action(keystone_service2, 'run')
actions.resource_action(haproxy_config, 'run')
actions.resource_action(haproxy_service, 'run')
# test working configuration
requests.get('http://%s:%s' % (keystone_service1.args['ip'].value, keystone_service1.args['port'].value))
requests.get('http://%s:%s' % (keystone_service2.args['ip'].value, keystone_service2.args['port'].value))
requests.get('http://%s:%s' % (haproxy_service.args['ip'].value, haproxy_service.args['ports'].value[0]['value'][0]['value']))
# test working configuration
requests.get('http://%s:%s' % (keystone_service1.args['ip'].value, keystone_service1.args['port'].value))
requests.get('http://%s:%s' % (keystone_service2.args['ip'].value, keystone_service2.args['port'].value))
requests.get('http://%s:%s' % (haproxy_service.args['ip'].value, haproxy_service.args['ports'].value[0]['value'][0]['value']))
@click.command()
def undeploy():
db = get_db()
resources = map(resource.wrap_resource, db.get_list('resource'))
resources = {r.name: r for r in resources}
actions.resource_action(resources['haproxy_service'], 'remove')
actions.resource_action(resources['haproxy_config'], 'remove')
actions.resource_action(resources['keystone_service2'], 'remove')
actions.resource_action(resources['keystone_config2'], 'remove')
actions.resource_action(resources['keystone_service1'], 'remove')
actions.resource_action(resources['keystone_config1'], 'remove')
actions.resource_action(resources['keystone_db_user'], 'remove')
actions.resource_action(resources['keystone_db'], 'remove')
actions.resource_action(resources['mariadb_service1'], 'remove')
db.clear()
signals.Connections.clear()
main.add_command(deploy)
main.add_command(undeploy)
if __name__ == '__main__':
main()

4
main.yml
View File

@ -7,8 +7,8 @@
- apt: name=vim state=present
- apt: name=tmux state=present
- apt: name=htop state=present
- apt: name=python-virtualenv state=present
- apt: name=virtualenvwrapper state=present
#- apt: name=python-virtualenv state=present
#- apt: name=virtualenvwrapper state=present
- apt: name=ipython state=present
- apt: name=python-pudb state=present
- apt: name=python-pip state=present

6
resources/data_container/actions/run.yml
View File

@ -6,10 +6,13 @@
image: {{ image }}
state: running
net: host
{% if ports.value %}
ports:
{% for port in ports.value %}
- {{ port['value'] }}:{{ port['value'] }}
{% endfor %}
{% endif %}
{% if host_binds.value %}
volumes:
# TODO: host_binds might need more work
# Currently it's not that trivial to pass custom src: dst here
@ -17,4 +20,5 @@
# so we mount it to the same directory as on host
{% for bind in host_binds.value %}
- {{ bind['value']['src'] }}:{{ bind['value']['dst'] }}:{{ bind['value'].get('mode', 'ro') }}
{% endfor %}
{% endfor %}
{% endif %}

4
resources/docker_container/actions/run.yml
View File

@ -7,12 +7,15 @@
image: {{ image }}
state: running
net: host
{% if ports.value %}
ports:
{% for port in ports.value %}
{% for p in port['value'] %}
- {{ p['value'] }}:{{ p['value'] }}
{% endfor %}
{% endfor %}
{% endif %}
{% if host_binds.value %}
volumes:
# TODO: host_binds might need more work
# Currently it's not that trivial to pass custom src: dst here
@ -21,3 +24,4 @@
{% for bind in host_binds.value %}
- {{ bind['value']['src'] }}:{{ bind['value']['dst'] }}:{{ bind['value'].get('mode', 'ro') }}
{% endfor %}
{% endif %}

4
resources/docker_container/meta.yaml
View File

@ -9,10 +9,10 @@ input:
schema: str!
value:
ports:
schema: [int]
schema: [{value: [{value: int}]}]
value: []
host_binds:
schema: [[int]]
schema: [{value: {src: str, dst: str}}]
value: []
volume_binds:
schema: [{src: str, dst: str}]

10
resources/haproxy/meta.yaml
View File

@ -3,22 +3,22 @@ handler: ansible
version: 1.0.0
input:
ip:
schema: int!
schema: str!
value:
config_dir:
schema: {src: str!, dst: str!}
value: {src: /etc/solar/haproxy, dst: /etc/haproxy}
listen_ports:
schema: [int]
schema: [{value: int}]
value: []
configs:
schema: [[str]]
schema: [{value: [{value: str}]}]
value: []
configs_names:
schema: [str]
schema: [{value: str}]
value: []
configs_ports:
schema: [[int]]
schema: [{value: [{value: int}]}]
value: []
ssh_user:
schema: str!

4
resources/haproxy_keystone_config/meta.yaml
View File

@ -9,10 +9,10 @@ input:
schema: int!
value: 9999
ports:
schema: [int]
schema: [{value: int}]
value:
servers:
schema: [str]
schema: [{value: str}]
value:
tags: [resources/haproxy, resource/haproxy_keystone_config]

2
resources/keystone_config/meta.yaml
View File

@ -19,7 +19,7 @@ input:
schema: str!
value:
db_port:
schema: str!
schema: int!
value:
db_name:
schema: str!

6
resources/mariadb_db/actions/run.yml
View File

@ -3,9 +3,9 @@
tasks:
- name: mariadb db
mysql_db:
name: {{db_name}}
name: {{ db_name }}
state: present
login_user: root
login_password: {{login_password}}
login_port: {{login_port}}
login_password: {{ login_password }}
login_port: {{ login_port }}
login_host: 127.0.0.1

4
resources/mariadb_service/meta.yaml
View File

@ -9,10 +9,10 @@ input:
schema: str!
value: password
port:
schema: str!
schema: int!
value: 3306
ip:
schema: int!
schema: str!
value:
ssh_key:
schema: str!

10
resources/mariadb_user/actions/run.yml
View File

@ -3,12 +3,12 @@
tasks:
- name: mariadb user
mysql_user:
name: {{new_user_name}}
password: {{new_user_password}}
priv: {{db_name}}.*:ALL
name: {{ new_user_name }}
password: {{ new_user_password }}
priv: {{ db_name }}.*:ALL
host: '%'
state: present
login_user: root
login_password: {{login_password}}
login_port: {{login_port}}
login_password: {{ login_password }}
login_port: {{ login_port }}
login_host: 127.0.0.1

12
solar/solar/core/connections.py
View File

@ -1,16 +1,4 @@
import copy
import json
from itertools import imap, ifilter
import networkx as nx
import jinja2
import mock
from jinja2 import Template
def depends_on(init_value, value=None, tags=None):
if tags is None:
tags = []

13
solar/solar/core/handlers/ansible.py
View File

@ -22,6 +22,13 @@ class Ansible(BaseHandler):
# 'host': ''}
def _create_inventory(self, r):
directory = self.dirs[r.name]
inventory_path = os.path.join(directory, 'inventory')
with open(inventory_path, 'w') as inv:
inv.write(self._render_inventory(r))
return inventory_path
def _render_inventory(self, r):
inventory = '{0} ansible_ssh_host={1} ansible_connection=ssh ansible_ssh_user={2} ansible_ssh_private_key_file={3}'
host, user, ssh_key = r.args['ip'].value, r.args['ssh_user'].value, r.args['ssh_key'].value
print host
@ -29,11 +36,7 @@ class Ansible(BaseHandler):
print ssh_key
inventory = inventory.format(host, host, user, ssh_key)
print inventory
directory = self.dirs[r.name]
inventory_path = os.path.join(directory, 'inventory')
with open(inventory_path, 'w') as inv:
inv.write(inventory)
return inventory_path
return inventory
def _create_playbook(self, resource, action):
return self._compile_action_file(resource, action)

24
solar/solar/core/handlers/base.py
View File

@ -24,22 +24,24 @@ class BaseHandler(object):
shutil.rmtree(self.dst)
def _compile_action_file(self, resource, action):
action_file = resource.metadata['actions'][action]
action_file = os.path.join(resource.metadata['actions_path'], action_file)
dir_path = self.dirs[resource.name]
dest_file = tempfile.mkstemp(text=True, prefix=action, dir=dir_path)[1]
args = self._make_args(resource)
self._compile_file(action_file, dest_file, args)
with open(dest_file, 'w') as f:
f.write(self._render_action(resource, action))
return dest_file
def _compile_file(self, template, dest_file, args):
print 'Rendering', template, args
with open(template) as f:
tpl = Template(f.read())
tpl = tpl.render(args, zip=zip)
def _render_action(self, resource, action):
print 'Rendering %s %s' % (resource.name, action)
with open(dest_file, 'w') as g:
g.write(tpl)
action_file = resource.metadata['actions'][action]
action_file = os.path.join(resource.metadata['actions_path'], action_file)
args = self._make_args(resource)
with open(action_file) as f:
tpl = Template(f.read())
return tpl.render(str=str, zip=zip, **args)
def _make_args(self, resource):
args = {'name': resource.name}

2
solar/solar/core/observer.py
View File

@ -23,7 +23,7 @@ class BaseObserver(object):
return '[{}:{}] {}'.format(self.attached_to.name, self.name, self.value)
def __unicode__(self):
return self.value
return unicode(self.value)
def __eq__(self, other):
if isinstance(other, BaseObserver):

19
solar/solar/core/resource.py
View File

@ -1,8 +1,6 @@
# -*- coding: utf-8 -*-
import copy
import json
import os
import shutil
from copy import deepcopy
@ -45,11 +43,16 @@ class Resource(object):
self.tags = tags or []
def __repr__(self):
return ("Resource(name='{0}', metadata={1}, args={2}, "
"tags={3})").format(self.name,
json.dumps(self.metadata),
json.dumps(self.args_show()),
self.tags)
return ("Resource(name='{name}', metadata={metadata}, args={args}, "
"tags={tags})").format(**self.to_dict)
def to_dict(self):
return {
'name': self.name,
'metadata': self.metadata,
'args': self.args_show(),
'tags': self.tags,
}
def args_show(self):
def formatter(v):
@ -144,7 +147,7 @@ def create(name, base_path, args, tags=[], connections={}):
def wrap_resource(raw_resource):
name = raw_resource['id']
args = raw_resource['input']
args = {k: v['value'] for k, v in raw_resource['input'].items()}
tags = raw_resource.get('tags', [])
return Resource(name, raw_resource, args, tags=tags)