Merge branch 'master' into glance

.gitignore

@@ -10,3 +10,7 @@ tmp/
 
 #vim
 *.swp
+
+state/
+clients.json
+rs/

cli.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 import click
 import json
 #import matplotlib
@@ -8,9 +9,10 @@ import os
 import subprocess
 
 from solar.core import actions as xa
-from solar.core import deployment as xd
 from solar.core import resource as xr
 from solar.core import signals as xs
+from solar import operations
+from solar import state
 
 
 @click.group()
@@ -137,6 +139,48 @@ def init_cli_connect():
     cli.add_command(disconnect)
 
 
+def init_changes():
+    @click.group()
+    def changes():
+        pass
+
+    cli.add_command(changes)
+
+    @click.command()
+    def stage():
+        log = operations.stage_changes()
+        print log.show()
+
+    changes.add_command(stage)
+
+    @click.command()
+    def commit():
+        operations.commit_changes()
+
+    changes.add_command(commit)
+
+    @click.command()
+    @click.option('--limit', default=5)
+    def history(limit):
+        print state.CL().show()
+
+    changes.add_command(history)
+
+    @click.command()
+    @click.option('--last', is_flag=True, default=False)
+    @click.option('--all', is_flag=True, default=False)
+    @click.option('--uid', default=None)
+    def rollback(last, all, uid):
+        if last:
+            print operations.rollback_last()
+        elif all:
+            print operations.rollback_all()
+        elif uid:
+            print operations.rollback_uid(uid)
+
+    changes.add_command(rollback)
+
+
 def init_cli_connections():
     @click.group()
     def connections():
@@ -185,5 +229,6 @@ if __name__ == '__main__':
     init_cli_connect()
     init_cli_connections()
     init_cli_deployment_config()
+    init_changes()
 
     cli()

config.yaml

@@ -10,5 +10,9 @@ file-system-db:
   storage-path: /vagrant/tmp/storage
 
 template-dir: /vagrant/templates
+
 resources-files-mask: /vagrant/resources/*/*.yaml
 node_resource_template: /vagrant/resources/ro_node/
+
+state: /vagrant/state/
+

requirements.txt

@@ -5,3 +5,5 @@ PyYAML==3.11
 jsonschema==2.4.0
 requests==2.7.0
 mock
+dictdiffer==0.4.0
+enum34==1.0.4

resources/keystone_role/actions/remove.yml

@@ -2,4 +2,5 @@
   sudo: yes
   tasks:
     - name: keystone role
-      keystone_user: endpoint=http://{{keystone_host}}:{{keystone_port}}/v2.0/ token={{admin_token}} user={{user_name}} tenant={{tenant_name}} role={{role_name}} state=absent
+    #TODO: not implemented in module
+      pause: seconds=1

resources/keystone_service_endpoint/actions/remove.yaml

@@ -1,5 +1,6 @@
-#todo
 - hosts: [{{ ip }}]
   sudo: yes
   tasks:
-    - shell: echo 1
+    - name: keystone service and endpoint
+    #TODO: not implemented in module
+      pause: seconds=1

resources/keystone_user/actions/remove.yml

@@ -2,4 +2,5 @@
   sudo: yes
   tasks:
     - name: keystone user
-      keystone_user: endpoint=http://{{keystone_host}}:{{keystone_port}}/v2.0/ token={{admin_token}} user={{user_name}} password={{user_password}} tenant={{tenant_name}} state=absent
+    #TODO: not implemented in module
+      pause: seconds=1

resources/rabbitmq_service/actions/run.yml

@@ -15,3 +15,8 @@
         - {{ management_port }}:15672
         env:
             RABBITMQ_NODENAME: {{container_name}}
+    - shell: docker exec -t {{ name }} rabbitmqctl list_users
+      register: result
+      until: result.rc == 0
+      retries: 20
+      delay: 0.5

solar/solar/core/resource.py

@@ -127,7 +127,7 @@ def create(name, base_path, args, tags=[], connections={}):
     base_meta_file = os.path.join(base_path, 'meta.yaml')
     actions_path = os.path.join(base_path, 'actions')
 
-    meta = yaml.load(open(base_meta_file).read())
+    meta = utils.yaml_load(base_meta_file)
     meta['id'] = name
     meta['version'] = '1.0.0'
     meta['actions'] = {}

solar/solar/core/signals.py

@@ -93,8 +93,7 @@ def guess_mapping(emitter, receiver):
 
 
 def connect(emitter, receiver, mapping=None):
-    guessed = guess_mapping(emitter, receiver)
-    mapping = mapping or guessed
+    mapping = mapping or guess_mapping(emitter, receiver)
 
     for src, dst in mapping.items():
         # Disconnect all receiver inputs

solar/solar/operations.py

@@ -0,0 +1,154 @@
+
+
+from solar import state
+from solar.core import signals
+from solar.core import resource
+from solar import utils
+from solar.interfaces.db import get_db
+from solar.core import actions
+
+db = get_db()
+
+from dictdiffer import diff, patch, revert
+import networkx as nx
+
+
+def guess_action(from_, to):
+    # TODO(dshulyak) it should be more flexible
+    if not from_:
+        return 'run'
+    elif not to:
+        return 'remove'
+    else:
+        # it should be update
+        return 'update'
+
+
+def connections(res, graph):
+    result = []
+    for pred in graph.predecessors(res.name):
+        for num, edge in graph.get_edge_data(pred, res.name).items():
+            if 'label' in edge:
+                if ':' in edge['label']:
+                    parent, child = edge['label'].split(':')
+                    mapping = [parent, child]
+                else:
+                    mapping = [edge['label'], edge['label']]
+            else:
+                mapping = None
+            result.append([pred, res.name, mapping])
+    return result
+
+
+def to_dict(resource, graph):
+    return {'uid': resource.name,
+            'tags': resource.tags,
+            'args': resource.args_dict(),
+            'connections': connections(resource, graph)}
+
+
+def stage_changes():
+    resources = resource.load_all()
+    conn_graph = signals.detailed_connection_graph()
+
+    commited = state.CD()
+    log = state.SL()
+    action = None
+
+    for res_uid in nx.topological_sort(conn_graph):
+        commited_data = commited.get(res_uid, {})
+        staged_data = to_dict(resources[res_uid], conn_graph)
+
+        if 'connections' in commited_data:
+            commited_data['connections'].sort()
+            staged_data['connections'].sort()
+        if 'tags' in commited_data:
+            commited_data['tags'].sort()
+            staged_data['tags'].sort()
+
+        df = list(diff(commited_data, staged_data))
+        if df:
+
+            log_item = state.LogItem(
+                utils.generate_uuid(),
+                res_uid,
+                df,
+                guess_action(commited_data, staged_data))
+            log.add(log_item)
+    return log
+
+
+def commit_changes():
+    # just shortcut to test stuff
+    commited = state.CD()
+    history = state.CL()
+    staged = state.SL()
+    resources = resource.load_all()
+
+    while staged:
+        l = staged.popleft()
+        wrapper = resources[l.res]
+
+        staged_data = patch(l.diff, commited.get(l.res, {}))
+
+        # TODO(dshulyak) think about this hack for update
+        if l.action == 'update':
+            commited_args = commited[l.res]['args']
+            wrapper.update(commited_args)
+            actions.resource_action(wrapper, 'remove')
+
+            wrapper.update(staged_data.get('args', {}))
+            actions.resource_action(wrapper, 'run')
+        else:
+            actions.resource_action(wrapper, l.action)
+
+        commited[l.res] = staged_data
+        l.state = state.STATES.success
+        history.add(l)
+
+
+def rollback(log_item):
+    log = state.SL()
+
+    resources = resource.load_all()
+    commited = state.CD()[log_item.res]
+
+    staged = revert(log_item.diff, commited)
+
+    for e, r, mapping in commited.get('connections', ()):
+        signals.disconnect(resources[e], resources[r])
+
+    for e, r, mapping in staged.get('connections', ()):
+        signals.connect(resources[e], resources[r], dict([mapping]))
+
+    df = list(diff(commited, staged))
+
+    log_item = state.LogItem(
+        utils.generate_uuid(),
+        log_item.res, df, guess_action(commited, staged))
+    log.add(log_item)
+
+    res = resource.wrap_resource(db.get_resource(log_item.res))
+    res.update(staged.get('args', {}))
+    res.save()
+
+    return log
+
+
+def rollback_uid(uid):
+    item = next(l for l in state.CL() if l.uuid == uid)
+    return rollback(item)
+
+
+def rollback_last():
+    l = state.CL().items[-1]
+    return rollback(l)
+
+
+def rollback_all():
+    cl = state.CL()
+
+    while cl:
+        rollback(cl.pop())
+
+

solar/solar/state.py

@@ -0,0 +1,132 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import collections
+from collections import deque
+from functools import partial
+
+from solar import utils
+
+from enum import Enum
+
+
+STATES = Enum('States', 'pending inprogress error success')
+
+
+def state_file(filename):
+    filepath = os.path.join(utils.read_config()['state'], filename)
+    if 'log' in filename:
+        return Log(filepath)
+    elif 'data' in filename:
+        return Data(filepath)
+
+
+CD = partial(state_file, 'commited_data')
+SD = partial(state_file, 'staged_data')
+SL = partial(state_file, 'stage_log')
+IL = partial(state_file, 'inprogress_log')
+CL = partial(state_file, 'commit_log')
+
+
+class LogItem(object):
+
+    def __init__(self, uid, res_uid, diff, action, state=None):
+        self.uid = uid
+        self.res = res_uid
+        self.diff = diff
+        self.state = state or STATES.pending
+        self.action = action
+
+    def to_yaml(self):
+        return utils.yaml_dump(self.to_dict())
+
+    def to_dict(self):
+        return {'uid': self.uid,
+                'res': self.res,
+                'diff': self.diff,
+                'state': self.state.name,
+                'action': self.action}
+
+    def __str__(self):
+        return self.to_yaml()
+
+    def __repr__(self):
+        return self.to_yaml()
+
+
+class Log(object):
+
+    def __init__(self, path):
+        self.path = path
+        items = utils.yaml_load(path) or []
+        self.items = deque([LogItem(
+            l['uid'], l['res'],
+            l['diff'], l['action'],
+            getattr(STATES, l['state'])) for l in items])
+
+    def sync(self):
+        utils.yaml_dump_to([i.to_dict() for i in self.items], self.path)
+
+    def add(self, logitem):
+        self.items.append(logitem)
+        self.sync()
+
+    def popleft(self):
+        item = self.items.popleft()
+        self.sync()
+        return item
+
+    def pop(self):
+        item = self.items.pop()
+        self.sync()
+        return item
+
+    def show(self, verbose=False):
+        return ['L(uuid={0}, res={1}, aciton={2})'.format(
+            l.uid, l.res, l.action) for l in self.items]
+
+    def __repr__(self):
+        return 'Log({0})'.format(self.path)
+
+    def __iter__(self):
+        return iter(self.items)
+
+    def __nonzero__(self):
+        return bool(self.items)
+
+
+class Data(collections.MutableMapping):
+
+    def __init__(self, path):
+        self.path = path
+        self.store = utils.yaml_load(path) or {}
+
+    def __getitem__(self, key):
+        return self.store[key]
+
+    def __setitem__(self, key, value):
+        self.store[key] = value
+        utils.yaml_dump_to(self.store, self.path)
+
+    def __delitem__(self, key):
+        self.store.pop(key)
+        utils.yaml_dump_to(self.store, self.path)
+
+    def __iter__(self):
+        return iter(self.store)
+
+    def __len__(self):
+        return len(self.store)
+