diff --git a/.gitignore b/.gitignore index 043c504a..596446eb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ tmp/ #vim *.swp + +state/ +clients.json +rs/ diff --git a/cli.py b/cli.py old mode 100644 new mode 100755 index ccc30c8b..37e3e117 --- a/cli.py +++ b/cli.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python import click import json #import matplotlib @@ -8,9 +9,10 @@ import os import subprocess from solar.core import actions as xa -from solar.core import deployment as xd from solar.core import resource as xr from solar.core import signals as xs +from solar import operations +from solar import state @click.group() @@ -137,6 +139,48 @@ def init_cli_connect(): cli.add_command(disconnect) +def init_changes(): + @click.group() + def changes(): + pass + + cli.add_command(changes) + + @click.command() + def stage(): + log = operations.stage_changes() + print log.show() + + changes.add_command(stage) + + @click.command() + def commit(): + operations.commit_changes() + + changes.add_command(commit) + + @click.command() + @click.option('--limit', default=5) + def history(limit): + print state.CL().show() + + changes.add_command(history) + + @click.command() + @click.option('--last', is_flag=True, default=False) + @click.option('--all', is_flag=True, default=False) + @click.option('--uid', default=None) + def rollback(last, all, uid): + if last: + print operations.rollback_last() + elif all: + print operations.rollback_all() + elif uid: + print operations.rollback_uid(uid) + + changes.add_command(rollback) + + def init_cli_connections(): @click.group() def connections(): @@ -185,5 +229,6 @@ if __name__ == '__main__': init_cli_connect() init_cli_connections() init_cli_deployment_config() + init_changes() cli() diff --git a/config.yaml b/config.yaml index d52c879b..3af564ca 100644 --- a/config.yaml +++ b/config.yaml @@ -10,5 +10,9 @@ file-system-db: storage-path: /vagrant/tmp/storage template-dir: /vagrant/templates + resources-files-mask: /vagrant/resources/*/*.yaml node_resource_template: /vagrant/resources/ro_node/ + +state: /vagrant/state/ + diff --git a/requirements.txt b/requirements.txt index a13f9914..de631f81 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ PyYAML==3.11 jsonschema==2.4.0 requests==2.7.0 mock +dictdiffer==0.4.0 +enum34==1.0.4 diff --git a/resources/keystone_role/actions/remove.yml b/resources/keystone_role/actions/remove.yml index 386d38ea..2a719f43 100644 --- a/resources/keystone_role/actions/remove.yml +++ b/resources/keystone_role/actions/remove.yml @@ -2,4 +2,5 @@ sudo: yes tasks: - name: keystone role - keystone_user: endpoint=http://{{keystone_host}}:{{keystone_port}}/v2.0/ token={{admin_token}} user={{user_name}} tenant={{tenant_name}} role={{role_name}} state=absent + #TODO: not implemented in module + pause: seconds=1 diff --git a/resources/keystone_service_endpoint/actions/remove.yaml b/resources/keystone_service_endpoint/actions/remove.yaml index 0b30ff48..67218324 100644 --- a/resources/keystone_service_endpoint/actions/remove.yaml +++ b/resources/keystone_service_endpoint/actions/remove.yaml @@ -1,5 +1,6 @@ -#todo - hosts: [{{ ip }}] sudo: yes tasks: - - shell: echo 1 + - name: keystone service and endpoint + #TODO: not implemented in module + pause: seconds=1 diff --git a/resources/keystone_user/actions/remove.yml b/resources/keystone_user/actions/remove.yml index a56289db..8ab1004f 100644 --- a/resources/keystone_user/actions/remove.yml +++ b/resources/keystone_user/actions/remove.yml @@ -2,4 +2,5 @@ sudo: yes tasks: - name: keystone user - keystone_user: endpoint=http://{{keystone_host}}:{{keystone_port}}/v2.0/ token={{admin_token}} user={{user_name}} password={{user_password}} tenant={{tenant_name}} state=absent + #TODO: not implemented in module + pause: seconds=1 diff --git a/resources/rabbitmq_service/actions/run.yml b/resources/rabbitmq_service/actions/run.yml index f928c291..66d36c77 100644 --- a/resources/rabbitmq_service/actions/run.yml +++ b/resources/rabbitmq_service/actions/run.yml @@ -15,3 +15,8 @@ - {{ management_port }}:15672 env: RABBITMQ_NODENAME: {{container_name}} + - shell: docker exec -t {{ name }} rabbitmqctl list_users + register: result + until: result.rc == 0 + retries: 20 + delay: 0.5 diff --git a/solar/solar/core/resource.py b/solar/solar/core/resource.py index 824b29d1..ca5f44eb 100644 --- a/solar/solar/core/resource.py +++ b/solar/solar/core/resource.py @@ -127,7 +127,7 @@ def create(name, base_path, args, tags=[], connections={}): base_meta_file = os.path.join(base_path, 'meta.yaml') actions_path = os.path.join(base_path, 'actions') - meta = yaml.load(open(base_meta_file).read()) + meta = utils.yaml_load(base_meta_file) meta['id'] = name meta['version'] = '1.0.0' meta['actions'] = {} diff --git a/solar/solar/core/signals.py b/solar/solar/core/signals.py index 79bff180..5afbb72a 100644 --- a/solar/solar/core/signals.py +++ b/solar/solar/core/signals.py @@ -93,8 +93,7 @@ def guess_mapping(emitter, receiver): def connect(emitter, receiver, mapping=None): - guessed = guess_mapping(emitter, receiver) - mapping = mapping or guessed + mapping = mapping or guess_mapping(emitter, receiver) for src, dst in mapping.items(): # Disconnect all receiver inputs diff --git a/solar/solar/operations.py b/solar/solar/operations.py new file mode 100644 index 00000000..e1c43842 --- /dev/null +++ b/solar/solar/operations.py @@ -0,0 +1,154 @@ + + +from solar import state +from solar.core import signals +from solar.core import resource +from solar import utils +from solar.interfaces.db import get_db +from solar.core import actions + +db = get_db() + +from dictdiffer import diff, patch, revert +import networkx as nx + + +def guess_action(from_, to): + # TODO(dshulyak) it should be more flexible + if not from_: + return 'run' + elif not to: + return 'remove' + else: + # it should be update + return 'update' + + +def connections(res, graph): + result = [] + for pred in graph.predecessors(res.name): + for num, edge in graph.get_edge_data(pred, res.name).items(): + if 'label' in edge: + if ':' in edge['label']: + parent, child = edge['label'].split(':') + mapping = [parent, child] + else: + mapping = [edge['label'], edge['label']] + else: + mapping = None + result.append([pred, res.name, mapping]) + return result + + +def to_dict(resource, graph): + return {'uid': resource.name, + 'tags': resource.tags, + 'args': resource.args_dict(), + 'connections': connections(resource, graph)} + + +def stage_changes(): + resources = resource.load_all() + conn_graph = signals.detailed_connection_graph() + + commited = state.CD() + log = state.SL() + action = None + + for res_uid in nx.topological_sort(conn_graph): + commited_data = commited.get(res_uid, {}) + staged_data = to_dict(resources[res_uid], conn_graph) + + if 'connections' in commited_data: + commited_data['connections'].sort() + staged_data['connections'].sort() + if 'tags' in commited_data: + commited_data['tags'].sort() + staged_data['tags'].sort() + + df = list(diff(commited_data, staged_data)) + if df: + + log_item = state.LogItem( + utils.generate_uuid(), + res_uid, + df, + guess_action(commited_data, staged_data)) + log.add(log_item) + return log + + +def commit_changes(): + # just shortcut to test stuff + commited = state.CD() + history = state.CL() + staged = state.SL() + resources = resource.load_all() + + while staged: + l = staged.popleft() + wrapper = resources[l.res] + + staged_data = patch(l.diff, commited.get(l.res, {})) + + # TODO(dshulyak) think about this hack for update + if l.action == 'update': + commited_args = commited[l.res]['args'] + wrapper.update(commited_args) + actions.resource_action(wrapper, 'remove') + + wrapper.update(staged_data.get('args', {})) + actions.resource_action(wrapper, 'run') + else: + actions.resource_action(wrapper, l.action) + + commited[l.res] = staged_data + l.state = state.STATES.success + history.add(l) + + +def rollback(log_item): + log = state.SL() + + resources = resource.load_all() + commited = state.CD()[log_item.res] + + staged = revert(log_item.diff, commited) + + for e, r, mapping in commited.get('connections', ()): + signals.disconnect(resources[e], resources[r]) + + for e, r, mapping in staged.get('connections', ()): + signals.connect(resources[e], resources[r], dict([mapping])) + + df = list(diff(commited, staged)) + + log_item = state.LogItem( + utils.generate_uuid(), + log_item.res, df, guess_action(commited, staged)) + log.add(log_item) + + res = resource.wrap_resource(db.get_resource(log_item.res)) + res.update(staged.get('args', {})) + res.save() + + return log + + +def rollback_uid(uid): + item = next(l for l in state.CL() if l.uuid == uid) + return rollback(item) + + +def rollback_last(): + l = state.CL().items[-1] + return rollback(l) + + +def rollback_all(): + cl = state.CL() + + while cl: + rollback(cl.pop()) + + diff --git a/solar/solar/state.py b/solar/solar/state.py new file mode 100644 index 00000000..a997e011 --- /dev/null +++ b/solar/solar/state.py @@ -0,0 +1,132 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import collections +from collections import deque +from functools import partial + +from solar import utils + +from enum import Enum + + +STATES = Enum('States', 'pending inprogress error success') + + +def state_file(filename): + filepath = os.path.join(utils.read_config()['state'], filename) + if 'log' in filename: + return Log(filepath) + elif 'data' in filename: + return Data(filepath) + + +CD = partial(state_file, 'commited_data') +SD = partial(state_file, 'staged_data') +SL = partial(state_file, 'stage_log') +IL = partial(state_file, 'inprogress_log') +CL = partial(state_file, 'commit_log') + + +class LogItem(object): + + def __init__(self, uid, res_uid, diff, action, state=None): + self.uid = uid + self.res = res_uid + self.diff = diff + self.state = state or STATES.pending + self.action = action + + def to_yaml(self): + return utils.yaml_dump(self.to_dict()) + + def to_dict(self): + return {'uid': self.uid, + 'res': self.res, + 'diff': self.diff, + 'state': self.state.name, + 'action': self.action} + + def __str__(self): + return self.to_yaml() + + def __repr__(self): + return self.to_yaml() + + +class Log(object): + + def __init__(self, path): + self.path = path + items = utils.yaml_load(path) or [] + self.items = deque([LogItem( + l['uid'], l['res'], + l['diff'], l['action'], + getattr(STATES, l['state'])) for l in items]) + + def sync(self): + utils.yaml_dump_to([i.to_dict() for i in self.items], self.path) + + def add(self, logitem): + self.items.append(logitem) + self.sync() + + def popleft(self): + item = self.items.popleft() + self.sync() + return item + + def pop(self): + item = self.items.pop() + self.sync() + return item + + def show(self, verbose=False): + return ['L(uuid={0}, res={1}, aciton={2})'.format( + l.uid, l.res, l.action) for l in self.items] + + def __repr__(self): + return 'Log({0})'.format(self.path) + + def __iter__(self): + return iter(self.items) + + def __nonzero__(self): + return bool(self.items) + + +class Data(collections.MutableMapping): + + def __init__(self, path): + self.path = path + self.store = utils.yaml_load(path) or {} + + def __getitem__(self, key): + return self.store[key] + + def __setitem__(self, key, value): + self.store[key] = value + utils.yaml_dump_to(self.store, self.path) + + def __delitem__(self, key): + self.store.pop(key) + utils.yaml_dump_to(self.store, self.path) + + def __iter__(self): + return iter(self.store) + + def __len__(self): + return len(self.store) + diff --git a/state/commit_log b/state/commit_log new file mode 100644 index 00000000..e69de29b diff --git a/state/commited_data b/state/commited_data new file mode 100644 index 00000000..e69de29b diff --git a/state/inprogress_log b/state/inprogress_log new file mode 100644 index 00000000..e69de29b diff --git a/state/stage_log b/state/stage_log new file mode 100644 index 00000000..e69de29b diff --git a/state/staged_data b/state/staged_data new file mode 100644 index 00000000..e69de29b