Cleanup of repository
- Remove not necessary code - Fix import in some files - Use only one utils file - Get rid from two config files
This commit is contained in:
Dmitry Shulyak
Evgeniy L
parent
ef186d1c8d
commit
e3e7d7bad1
14
config.yaml
14
config.yaml
@ -1 +1,15 @@
|
||||
clients-data-file: /vagrant/clients.json
|
||||
|
||||
tmp: /vagrant/tmp
|
||||
|
||||
examples-dir: /vagrant/examples
|
||||
|
||||
extensions-dir: /vagrant/solar/solar/extensions
|
||||
|
||||
file-system-db:
|
||||
resources-path: ./schema/resources
|
||||
storage-path: /vagrant/tmp/storage
|
||||
|
||||
template-dir: /vagrant/templates
|
||||
resources-files-mask: /vagrant/x/resources/*/*.yaml
|
||||
resource-instances-path: /vagrant/tmp/resource-instances
|
||||
|
||||
13
config.yml
13
config.yml
@ -1,13 +0,0 @@
|
||||
tmp: /vagrant/tmp
|
||||
|
||||
examples-dir: /vagrant/examples
|
||||
|
||||
extensions-dir: /vagrant/solar/solar/extensions
|
||||
|
||||
file-system-db:
|
||||
resources-path: ./schema/resources
|
||||
storage-path: /vagrant/tmp/storage
|
||||
|
||||
template-dir: /vagrant/templates
|
||||
resources-files-mask: /vagrant/resources/*/*.yaml
|
||||
resource-instances-path: /vagrant/tmp/resource-instances
|
||||
@ -27,7 +27,6 @@ import yaml
|
||||
|
||||
from solar import extensions
|
||||
from solar import utils
|
||||
from solar.core import data
|
||||
from solar.core.resource import assign_resources_to_nodes
|
||||
from solar.core.resource import connect_resources
|
||||
from solar.core.tags_set_parser import Expression
|
||||
@ -60,20 +59,6 @@ class Cmd(object):
|
||||
parser = self.subparser.add_parser('discover')
|
||||
parser.set_defaults(func=getattr(self, 'discover'))
|
||||
|
||||
# Perform configuration
|
||||
parser = self.subparser.add_parser('configure')
|
||||
parser.set_defaults(func=getattr(self, 'configure'))
|
||||
parser.add_argument(
|
||||
'-p',
|
||||
'--profile')
|
||||
parser.add_argument(
|
||||
'-a',
|
||||
'--actions',
|
||||
nargs='+')
|
||||
parser.add_argument(
|
||||
'-pa',
|
||||
'--profile_action')
|
||||
|
||||
# Profile actions
|
||||
parser = self.subparser.add_parser('profile')
|
||||
parser.set_defaults(func=getattr(self, 'profile'))
|
||||
@ -106,13 +91,6 @@ class Cmd(object):
|
||||
else:
|
||||
pprint.pprint(self.db.get_list('profiles'))
|
||||
|
||||
def configure(self, args):
|
||||
profile = self.db.get_record('profiles', args.profile)
|
||||
extensions.find_by_provider_from_profile(
|
||||
profile, 'configure').configure(
|
||||
actions=args.actions,
|
||||
profile_action=args.profile_action)
|
||||
|
||||
def discover(self, args):
|
||||
Discovery({'id': 'discovery'}).discover()
|
||||
|
||||
|
||||
@ -1,194 +0,0 @@
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
from itertools import imap, ifilter
|
||||
from pprint import pprint
|
||||
|
||||
import networkx as nx
|
||||
import jinja2
|
||||
import mock
|
||||
|
||||
from jinja2 import Template
|
||||
|
||||
|
||||
class Node(object):
|
||||
|
||||
def __init__(self, config):
|
||||
self.uid = config['id']
|
||||
self.tags = set(config.get('tags', ()))
|
||||
self.config = copy.deepcopy(config)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Node(uid={0},tags={1})'.format(self.uid, self.tags)
|
||||
|
||||
|
||||
class Resource(object):
|
||||
|
||||
def __init__(self, config):
|
||||
self.uid = config['id']
|
||||
self.values = config['input']
|
||||
self.tags = set(config.get('tags', ()))
|
||||
|
||||
def __repr__(self):
|
||||
return 'Resource(uid={0},tags={1})'.format(self.uid, self.tags)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.uid)
|
||||
|
||||
def depends_on(self, value=None, tags=None):
|
||||
if tags is None:
|
||||
tags = []
|
||||
|
||||
if value is None:
|
||||
value = self.values
|
||||
|
||||
called_with_tags = []
|
||||
|
||||
if isinstance(value, dict):
|
||||
if value.get('first_with_tags'):
|
||||
called_with_tags.extend(value.get('first_with_tags'))
|
||||
elif value.get('with_tags'):
|
||||
called_with_tags.extend(value.get('with_tags'))
|
||||
|
||||
for k, v in value.items():
|
||||
self.depends_on(value=v, tags=tags)
|
||||
elif isinstance(value, list):
|
||||
for e in value:
|
||||
self.depends_on(value=e, tags=tags)
|
||||
elif isinstance(value, str):
|
||||
return value
|
||||
|
||||
tags.extend(called_with_tags)
|
||||
|
||||
return tags
|
||||
|
||||
|
||||
class DataGraph(nx.DiGraph):
|
||||
|
||||
node_klass = Resource
|
||||
|
||||
def __init__(self, resources=(), nodes=(), *args, **kwargs):
|
||||
super(DataGraph, self).__init__(*args, **kwargs)
|
||||
self.resources = map(lambda r: self.node_klass(r), resources)
|
||||
self.nodes = map(lambda n: Node(n), nodes)
|
||||
self.init_edges()
|
||||
|
||||
def init_edges(self):
|
||||
for res in self.resources:
|
||||
self.add_node(res.uid, res=res)
|
||||
|
||||
for dep_res in self.resources_with_tags(res.depends_on()):
|
||||
self.add_node(dep_res.uid, res=dep_res)
|
||||
self.add_edge(res.uid, dep_res.uid, parent=res.uid)
|
||||
|
||||
def resources_with_tags(self, tags):
|
||||
"""Filter all resources which have tags
|
||||
"""
|
||||
return ifilter(lambda r: r.tags & set(tags), self.resources)
|
||||
|
||||
def merge_nodes_resources(self):
|
||||
"""Each node has a list of resources
|
||||
"""
|
||||
merged = {}
|
||||
for node in self.nodes:
|
||||
merged.setdefault(node.uid, {})
|
||||
merged[node.uid]['resources'] = list(self.resources_with_tags(node.tags))
|
||||
merged[node.uid]['node'] = node
|
||||
|
||||
return merged
|
||||
|
||||
def get_node(self, uid):
|
||||
return filter(lambda n: n.uid == uid, self.nodes)[0]
|
||||
|
||||
def resolve(self):
|
||||
rendered_accum = {}
|
||||
render_order = nx.topological_sort(self.reverse())
|
||||
|
||||
# Use provided order to render resources
|
||||
for resource_id in render_order:
|
||||
# Iterate over all resources which are assigned for node
|
||||
for node_id, node_data in self.merge_nodes_resources().items():
|
||||
# Render resources which should be rendered regarding to order
|
||||
for resource in filter(lambda r: r.uid == resource_id, node_data['resources']):
|
||||
# Create render context
|
||||
ctx = {
|
||||
'this': resource.values,
|
||||
}
|
||||
ctx['this']['node'] = self.get_node(node_id).config
|
||||
|
||||
rendered = self.render(resource.values, ctx, rendered_accum)
|
||||
|
||||
rendered['tags'] = resource.tags
|
||||
rendered_accum['{0}-{1}'.format(node_id, resource.uid)] = rendered
|
||||
|
||||
return rendered_accum
|
||||
|
||||
def render(self, value, context, previous_render):
|
||||
|
||||
if isinstance(value, dict):
|
||||
# Handle iterators
|
||||
if value.get('first_with_tags') or value.get('with_tags'):
|
||||
|
||||
if len(value.keys()) != 2:
|
||||
raise Exception("Iterator should have two elements '{0}'".format(value))
|
||||
|
||||
def with_tags(*args):
|
||||
resources_with_tags = filter(
|
||||
lambda n: n[1]['tags'] & set(list(*args)),
|
||||
previous_render.items())
|
||||
|
||||
return map(lambda n: n[1], resources_with_tags)
|
||||
|
||||
method_name = 'with_tags'
|
||||
if value.get('first_with_tags'):
|
||||
method_name = 'first_with_tags'
|
||||
|
||||
iter_key = (set(value.keys()) - set([method_name])).pop()
|
||||
|
||||
items = []
|
||||
if isinstance(value[method_name], list):
|
||||
items = with_tags(value[method_name])
|
||||
elif isinstance(value[method_name], str):
|
||||
items = with_tags([value[method_name]])
|
||||
else:
|
||||
raise Exception('Cannot iterate over dict "{0}"'.format(value))
|
||||
|
||||
# first_with_tags returns a single object, hence we should
|
||||
# render a single object
|
||||
if value.get('first_with_tags'):
|
||||
if len(items) > 1:
|
||||
raise Exception('Returns too many objects, check that '
|
||||
'tags assigned properly "{0}"'.format(value))
|
||||
|
||||
iter_ctx = copy.deepcopy(context)
|
||||
iter_ctx[iter_key] = items[0]
|
||||
return self.render(value[iter_key], iter_ctx, previous_render)
|
||||
|
||||
# If it's with_tags construction, than it returns a list
|
||||
# we should render each element and return a list of rendered
|
||||
# elements
|
||||
result_list = []
|
||||
for item in items:
|
||||
iter_ctx = copy.deepcopy(context)
|
||||
iter_ctx[iter_key] = item
|
||||
result_list.append(self.render(value[iter_key], iter_ctx, previous_render))
|
||||
|
||||
return result_list
|
||||
|
||||
else:
|
||||
# Handle usual data
|
||||
result_dict = {}
|
||||
for k, v in value.items():
|
||||
result_dict[k] = self.render(v, context, previous_render)
|
||||
|
||||
return result_dict
|
||||
elif isinstance(value, list):
|
||||
return map(lambda v: self.render(v, context, previous_render), value)
|
||||
elif isinstance(value, str):
|
||||
env = Template(value)
|
||||
return env.render(**context)
|
||||
else:
|
||||
# If non of above return value, e.g. if there is
|
||||
# interger, float etc
|
||||
return value
|
||||
@ -12,7 +12,7 @@ from solar.core import actions
|
||||
from solar.core import db
|
||||
from solar.core import observer
|
||||
from solar.core import signals
|
||||
from solar.core import utils
|
||||
from solar import utils
|
||||
from solar.core import validation
|
||||
|
||||
from solar.core.connections import ResourcesConnectionGraph
|
||||
|
||||
@ -6,7 +6,7 @@ import os
|
||||
|
||||
import db
|
||||
|
||||
from solar.core import utils
|
||||
from solar import utils
|
||||
|
||||
|
||||
CLIENTS_CONFIG_KEY = 'clients-data-file'
|
||||
|
||||
@ -1,42 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import yaml
|
||||
|
||||
|
||||
def ext_encoder(fpath):
|
||||
ext = os.path.splitext(os.path.basename(fpath))[1].strip('.')
|
||||
if ext in ['json']:
|
||||
return json
|
||||
elif ext in ['yaml', 'yml']:
|
||||
return yaml
|
||||
|
||||
raise Exception('Unknown extension {}'.format(ext))
|
||||
|
||||
|
||||
def load_file(fpath):
|
||||
encoder = ext_encoder(fpath)
|
||||
|
||||
try:
|
||||
with open(fpath) as f:
|
||||
return encoder.load(f)
|
||||
except IOError:
|
||||
return {}
|
||||
|
||||
|
||||
def read_config():
|
||||
CONFIG_FILE = os.environ.get('CONFIG_FILE') or '/vagrant/config.yaml'
|
||||
return load_file(CONFIG_FILE)
|
||||
|
||||
|
||||
def read_config_file(key):
|
||||
fpath = read_config()[key]
|
||||
|
||||
return load_file(fpath)
|
||||
|
||||
|
||||
def save_to_config_file(key, data):
|
||||
fpath = read_config()[key]
|
||||
|
||||
with open(fpath, 'w') as f:
|
||||
encoder = ext_encoder(fpath)
|
||||
encoder.dump(data, f)
|
||||
@ -1,166 +0,0 @@
|
||||
import os
|
||||
|
||||
import subprocess
|
||||
import yaml
|
||||
|
||||
from solar import utils
|
||||
from solar.extensions import base
|
||||
from solar.core import data
|
||||
|
||||
from jinja2 import Template
|
||||
|
||||
|
||||
ANSIBLE_INVENTORY = """
|
||||
{% for key, res in resources.items() %}
|
||||
{% if res.node %}
|
||||
{{key}} ansible_ssh_host={{res.node.ip}} ansible_connection=ssh ansible_ssh_user={{res.node.ssh_user}} ansible_ssh_private_key_file={{res.node.ssh_private_key_path}}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% for key, group in groups.items() %}
|
||||
[{{key}}]
|
||||
{% for item in group %}
|
||||
{{item}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
"""
|
||||
|
||||
BASE_PATH=utils.read_config()['tmp']
|
||||
|
||||
def playbook(resource_path, playbook_name):
|
||||
resource_dir = os.path.dirname(resource_path)
|
||||
return {'include': '{0}'.format(
|
||||
os.path.join(resource_dir, playbook_name))}
|
||||
|
||||
|
||||
class AnsibleOrchestration(base.BaseExtension):
|
||||
|
||||
ID = 'ansible'
|
||||
VERSION = '1.0.0'
|
||||
PROVIDES = ['configure']
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(AnsibleOrchestration, self).__init__(*args, **kwargs)
|
||||
|
||||
self.nodes = self._get_nodes()
|
||||
self.resources = self._get_resources_for_nodes(self.nodes)
|
||||
|
||||
def _get_nodes(self):
|
||||
nodes = []
|
||||
for node in self.core.get_data('nodes_resources'):
|
||||
if self.profile.tags <= set(node.get('tags', [])):
|
||||
nodes.append(node)
|
||||
|
||||
return nodes
|
||||
|
||||
def _get_resources_for_nodes(self, nodes):
|
||||
"""Retrieves resources which required for nodes deployment"""
|
||||
resources = []
|
||||
|
||||
for node in nodes:
|
||||
node_tags = set(node.get('tags', []))
|
||||
result_resources = self._get_resources_with_tags(node_tags)
|
||||
resources.extend(result_resources)
|
||||
|
||||
return dict((r['id'], r) for r in resources).values()
|
||||
|
||||
def _get_resources_with_tags(self, tags):
|
||||
resources = []
|
||||
for resource in self.core.get_data('resources'):
|
||||
resource_tags = set(resource.get('tags', []))
|
||||
# If resource without tags, it means that it should
|
||||
# not be assigned to any node
|
||||
if not resource_tags:
|
||||
continue
|
||||
if resource_tags & tags:
|
||||
resources.append(resource)
|
||||
|
||||
return resources
|
||||
|
||||
def inventory(self, **kwargs):
|
||||
temp = Template(ANSIBLE_INVENTORY)
|
||||
return temp.render(**kwargs)
|
||||
|
||||
def prepare_from_profile(self, profile_action):
|
||||
|
||||
paths = self.profile.get(profile_action)
|
||||
if paths is None:
|
||||
raise Exception('Action %s not supported', profile_action)
|
||||
|
||||
return self.prepare_many(paths)
|
||||
|
||||
def prepare_many(self, paths):
|
||||
|
||||
ansible_actions = []
|
||||
|
||||
for path in paths:
|
||||
ansible_actions.extend(self.prepare_one(path))
|
||||
|
||||
return ansible_actions
|
||||
|
||||
def prepare_one(self, path):
|
||||
"""
|
||||
:param path: docker.actions.run or openstack.action
|
||||
"""
|
||||
steps = path.split('.')
|
||||
|
||||
if len(steps) < 2:
|
||||
raise Exception('Path %s is not valid,'
|
||||
' should be atleast 2 items', path)
|
||||
|
||||
# NOTE(dshulyak)
|
||||
# it is not obvious - but we dont need to run same playbook
|
||||
# for several times
|
||||
resources = filter(lambda r: r['class'] == steps[0], self.resources)
|
||||
resource = resources[0]
|
||||
|
||||
action = resource
|
||||
for step in steps[1:]:
|
||||
action = action[step]
|
||||
|
||||
result = []
|
||||
if isinstance(action, list):
|
||||
for item in action:
|
||||
result.append(playbook(resource['parent_path'], item))
|
||||
else:
|
||||
result.append(playbook(resource['parent_path'], action))
|
||||
|
||||
return result
|
||||
|
||||
def configure(self, profile_action='run', actions=None):
|
||||
dg = data.DataGraph(self.resources, self.nodes)
|
||||
resolved = dg.resolve()
|
||||
|
||||
groups = {}
|
||||
|
||||
for key, resource in resolved.items():
|
||||
if resource.get('node'):
|
||||
for tag in resource.get('tags', []):
|
||||
groups.setdefault(tag, [])
|
||||
groups[tag].append(key)
|
||||
|
||||
utils.create_dir('tmp/group_vars')
|
||||
utils.create_dir('tmp/host_vars')
|
||||
utils.write_to_file(
|
||||
self.inventory(
|
||||
resources=resolved, groups=groups), 'tmp/hosts')
|
||||
|
||||
for item, value in resolved.items():
|
||||
utils.yaml_dump_to(
|
||||
value, 'tmp/host_vars/{0}'.format(item))
|
||||
|
||||
if actions:
|
||||
prepared = self.prepare_many(actions)
|
||||
elif profile_action:
|
||||
prepared = self.prepare_from_profile(profile_action)
|
||||
else:
|
||||
raise Exception('Either profile_action '
|
||||
'or actions should be provided.')
|
||||
|
||||
utils.yaml_dump_to(prepared, BASE_PATH + '/main.yml')
|
||||
|
||||
sub = subprocess.Popen(
|
||||
['ansible-playbook', '-v', '-i',
|
||||
BASE_PATH + '/hosts',
|
||||
BASE_PATH + '/main.yml'],
|
||||
env=dict(os.environ, ANSIBLE_HOST_KEY_CHECKING='False'))
|
||||
out, err = sub.communicate()
|
||||
@ -1,4 +1,5 @@
|
||||
import io
|
||||
import json
|
||||
import glob
|
||||
import yaml
|
||||
import logging
|
||||
@ -61,5 +62,40 @@ def render_template(template_path, params):
|
||||
return temp.render(**params)
|
||||
|
||||
|
||||
def ext_encoder(fpath):
|
||||
ext = os.path.splitext(os.path.basename(fpath))[1].strip('.')
|
||||
if ext in ['json']:
|
||||
return json
|
||||
elif ext in ['yaml', 'yml']:
|
||||
return yaml
|
||||
|
||||
raise Exception('Unknown extension {}'.format(ext))
|
||||
|
||||
|
||||
def load_file(fpath):
|
||||
encoder = ext_encoder(fpath)
|
||||
|
||||
try:
|
||||
with open(fpath) as f:
|
||||
return encoder.load(f)
|
||||
except IOError:
|
||||
return {}
|
||||
|
||||
|
||||
def read_config():
|
||||
return yaml_load('/vagrant/config.yml')
|
||||
CONFIG_FILE = os.environ.get('CONFIG_FILE') or '/vagrant/config.yaml'
|
||||
return load_file(CONFIG_FILE)
|
||||
|
||||
|
||||
def read_config_file(key):
|
||||
fpath = read_config()[key]
|
||||
|
||||
return load_file(fpath)
|
||||
|
||||
|
||||
def save_to_config_file(key, data):
|
||||
fpath = read_config()[key]
|
||||
|
||||
with open(fpath, 'w') as f:
|
||||
encoder = ext_encoder(fpath)
|
||||
encoder.dump(data, f)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user