x/solar
Code Issues Proposed changes

Use resource as primary citizen in inventory

Browse Source
This commit is contained in:
Dmitry Shulyak 2015-04-17 12:46:52 +03:00 committed by Evgeniy L
parent a4d97e7171
commit cd10acc11a
10 changed files with 116 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
examples/nodes_list.yaml
View File

@ -1,9 +1,9 @@
- id: 6176aaa2-d97f-11e4-8dbe-080027c2ffdb
- id: node_1
ip: 10.0.0.2
ssh_user: vagrant
ssh_private_key_path: /vagrant/tmp/keys/ssh_private
- id: cc48cf72-df88-11e4-9f5b-080027c2ffdb
- id: node_2
ip: 10.0.0.3
ssh_user: vagrant
ssh_private_key_path: /vagrant/tmp/keys/ssh_private

2
examples/resources/docker.yml
View File

@ -1,4 +1,4 @@
id: docker
id: docker_1
type: resource
handler: ansible
version: v1

13
examples/resources/docker2.yml Normal file
View File

@ -0,0 +1,13 @@
id: docker_2
type: resource
handler: ansible
version: v1
actions:
run: simple/docker/run.yml
remove: simple/docker/remove.yml
input:
base_image: ubuntu
node:
link: node_2
tags: [service/docker]

2
examples/resources/mariadb.yml
View File

@ -2,11 +2,13 @@ id: mariadb
type: resource
handler: ansible
version: v1
actions:
run: simple/mariadb/run.yml
remove: simple/mariadb/remove.yml
wait: simple/mariadb/wait.yml
users: simple/mariadb/users.yml
input:
name: mariadb-test
image: kollaglue/fedora-rdo-mariadb-app

2
examples/resources/simple/docker/remove.yml
View File

@ -1,4 +1,4 @@
- hosts: [docker]
- hosts: [service/docker]
sudo: yes
tasks:
- shell: apt-get remove -y lxc-docker

2
examples/resources/simple/docker/run.yml
View File

@ -1,5 +1,5 @@
- hosts: [docker]
- hosts: [service/docker]
sudo: yes
tasks:
- shell: docker --version

26
solar/solar/cli.py
View File

@ -25,8 +25,9 @@ import textwrap
import yaml
from solar import extensions
from solar.interfaces.db import get_db
from solar import utils
from solar.core import data
from solar.interfaces.db import get_db
# NOTE: these are extensions, they shouldn't be imported here
# Maybe each extension can also extend the CLI with parsers
@ -78,6 +79,9 @@ class Cmd(object):
group.add_argument('-t', '--tags', nargs='+', default=['env/test_env'])
group.add_argument('-i', '--id', default=utils.generate_uuid())
parser = self.subparser.add_parser('data')
parser.set_defaults(func=getattr(self, 'data'))
def profile(self, args):
if args.create:
params = {'tags': args.tags, 'id': args.id}
@ -99,6 +103,26 @@ class Cmd(object):
def discover(self, args):
Discovery({'id': 'discovery'}).discover()
def data(self, args):
resources = [
{'id': 'service/1',
'tags': ['service'],
'input': {'node': {'link':'node/1'}}},
{'id': 'service/2',
'tags': ['service'],
'input': {'node': {'link':'node/2'}}},
{'id': 'node/1',
'input':{'host_ip': '10.0.0.2'},
'tags': ['service']},
{'id': 'node/2',
'input': {'host_ip': '10.0.0.3'},
'tags': ['service']}]
dg = data.DataGraph(resources)
pprint.pprint(dg.resolve())
def main():

134
solar/solar/core/data.py
View File

@ -2,115 +2,53 @@
import copy
from pprint import pprint
class Data(object):
def __init__(self, nodes, resources):
self.resources = [Resource(r) for r in resources]
self.nodes = [Node(n) for n in nodes]
self.raw = {'hosts': []}
for node in self.nodes:
self.raw['hosts'].append(node.data)
for res in self.resources:
if not res.tags:
continue
if res.tags <= node.tags:
node.data[res.uid] = res.data
def get(self):
# recursively go over variables
for node in self.nodes:
for res in self.resources:
if not res.tags:
continue
if res.tags <= node.tags:
for link in res.links:
link.resolve(node.data)
return self.raw
class Link(object):
"""Represents reference to another resource."""
def __init__(self, path, parent):
self.parent = parent.split('.')
self.path = path
def resolve(self, glob_data):
data = glob_data
for item in self.parent:
data = data[item]
if isinstance(data, Link):
value = data.resolve(glob_data)
else:
value = data
self.set_value(glob_data, value)
def set_value(self, glob_data, value):
data = glob_data
for item in self.path[:-1]:
data = data[item]
data[self.path[-1]] = value
def __repr__(self):
return 'Link(path={0},parent={1})'.format(
self.path, self.parent)
class Node(object):
def __init__(self, config):
self.uid = config['id']
self.config = config
self.tags = set(config.get('tags', ()))
self.data = copy.deepcopy(self.config)
def __repr__(self):
return 'Node(uid={0},tags={1})'.format(self.uid, self.tags)
import networkx as nx
class Resource(object):
def __init__(self, config):
self.uid = config['id']
self.config = config
self.values = config['input']
self.tags = set(config.get('tags', ()))
self.links = []
@property
def data(self):
_data = {}
for key, value in self.config.get('values', {}).items():
path = [self.uid, key]
if isinstance(value, dict):
if 'link' in value:
link = Link(path, value['link'])
_data[key] = link
self.links.append(link)
else:
_data[key] = value
return _data
def __repr__(self):
return 'Resource(uid={0},tags={1})'.format(self.uid, self.tags)
def __hash__(self):
return hash(self.uid)
if __name__ == '__main__':
@property
def links(self):
for item, value in self.values.items():
if 'link' in value:
yield value['link'], {'parent': item}
nodes = [
{'id': 'node_1', 'host_ip': '10.0.0.2', 'tags': ['service']},
{'id': 'node_2', 'host_ip': '10.0.0.3', 'tags': ['service']}]
resources = [
{'id': 'service',
'tags': ['service'],
'values': {'listen': {'link':'host_ip'}}}]
inv = Data(nodes, resources)
pprint(inv.get())
class DataGraph(nx.DiGraph):
node_klass = Resource
def __init__(self, resources=(), *args, **kwargs):
super(DataGraph, self).__init__(*args, **kwargs)
for res in resources:
init_res = self.node_klass(res)
self.add_node(init_res.uid, res=init_res)
for link, attrs in init_res.links:
self.add_edge(init_res.uid, link, **attrs)
def resolve(self):
data = {}
for item in nx.topological_sort(self.reverse()):
res = self.node[item]['res']
res_data = copy.deepcopy(res.values)
res_data['tags'] = res.tags
for child, parent, attrs in self.edges(item, data=True):
res_data[attrs['parent']] = data[parent]
data[item] = res_data
return data

72
solar/solar/extensions/modules/ansible.py
View File

@ -5,20 +5,22 @@ import yaml
from solar import utils
from solar.extensions import base
from solar.core import data
from jinja2 import Template
ANSIBLE_INVENTORY = """
{% for node in nodes %}
{{node.name}} ansible_ssh_host={{node.ip}} ansible_connection=ssh ansible_ssh_user={{node.ssh_user}} ansible_ssh_private_key_file={{node.ssh_private_key_path}}
{% for key, res in resources.items() %}
{% if res.node %}
{{key}} ansible_ssh_host={{res.node.ip}} ansible_connection=ssh ansible_ssh_user={{res.node.ssh_user}} ansible_ssh_private_key_file={{res.node.ssh_private_key_path}}
{% endif %}
{% endfor %}
{% for key, group in groups.items() %}
[{{key}}]
{% for item in group %}
{{item}}
{% endfor %}
{% for res in resources %}
[{{ res.id }}]
{% for node in nodes_mapping[res.id] %}
{{node['name']}}
{% endfor %}
{% endfor %}
"""
@ -74,13 +76,10 @@ class AnsibleOrchestration(base.BaseExtension):
return resources
@property
def inventory(self):
def inventory(self, **kwargs):
temp = Template(ANSIBLE_INVENTORY)
return temp.render(
nodes_mapping=self._make_nodes_services_mapping(),
resources=self.resources,
nodes=self.nodes)
return temp.render(**kwargs)
def _make_nodes_services_mapping(self):
mapping = {}
@ -89,27 +88,6 @@ class AnsibleOrchestration(base.BaseExtension):
return mapping
def _get_nodes_for_resource(self, resource):
resource_tags = set(resource['tags'])
nodes = []
for node in self.nodes:
if resource_tags <= set(node['tags']):
nodes.append(node)
return nodes
@property
def vars(self):
result = {}
for res in self.resources:
compiled = Template(
utils.yaml_dump({res['id']: res.get('input', {})}))
compiled = yaml.load(compiled.render(**result))
result.update(compiled)
return result
def prepare_from_profile(self, profile_action):
@ -159,9 +137,27 @@ class AnsibleOrchestration(base.BaseExtension):
return result
def configure(self, profile_action='run', actions=None):
utils.create_dir(BASE_PATH + '/group_vars')
utils.write_to_file(self.inventory, BASE_PATH + '/hosts')
utils.yaml_dump_to(self.vars, BASE_PATH + '/group_vars/all')
dg = data.DataGraph(self.nodes + self.resources)
resolved = dg.resolve()
groups = {}
for key, resource in resolved.items():
if resource.get('node'):
for tag in resource.get('tags', []):
groups.setdefault(tag, [])
groups[tag].append(key)
utils.create_dir('tmp/group_vars')
utils.create_dir('tmp/host_vars')
utils.write_to_file(
self.inventory(
resources=resolved, groups=groups), 'tmp/hosts')
for item, value in resolved.items():
utils.yaml_dump_to(
value, 'tmp/host_vars/{0}'.format(item))
if actions:
prepared = self.prepare_many(actions)

2
solar/solar/extensions/modules/discovery.py
View File

@ -45,7 +45,7 @@ class Discovery(base.BaseExtension):
node_resource['type'] = 'resource'
node_resource['version'] = self.VERSION
node_resource['tags'] = node['tags']
node_resource['output'] = node
node_resource['input'] = node
node_resource['ip'] = node['ip']
node_resource['ssh_user'] = node['ssh_user']
node_resource['ssh_private_key_path'] = node['ssh_private_key_path']