Merge branch 'master' into glance-perf

This commit is contained in:
Przemyslaw Kaminski 2015-06-03 11:38:58 +02:00
commit 2abd8a6b40
39 changed files with 380 additions and 87 deletions

View File

@ -32,8 +32,8 @@ def deploy():
openstack_rabbitmq_user = resource.create('openstack_rabbitmq_user', 'resources/rabbitmq_user/', {'ssh_user': '', 'ip': '', 'ssh_key': '', 'vhost_name': '', 'user_name': 'openstack', 'password': 'openstack_password', 'container_name': ''})
mariadb_service1 = resource.create('mariadb_service1', 'resources/mariadb_service', {'image': 'mariadb', 'root_password': 'mariadb', 'port': 3306, 'ip': '', 'ssh_user': '', 'ssh_key': ''})
keystone_db = resource.create('keystone_db', 'resources/mariadb_db/', {'db_name': 'keystone_db', 'login_password': '', 'login_user': 'root', 'login_port': '', 'ip': '', 'ssh_user': '', 'ssh_key': ''})
keystone_db_user = resource.create('keystone_db_user', 'resources/mariadb_user/', {'new_user_name': 'keystone', 'new_user_password': 'keystone', 'db_name': '', 'login_password': '', 'login_user': 'root', 'login_port': '', 'ip': '', 'ssh_user': '', 'ssh_key': ''})
keystone_db = resource.create('keystone_db', 'resources/mariadb_keystone_db/', {'db_name': 'keystone_db', 'login_password': '', 'login_user': 'root', 'login_port': '', 'ip': '', 'ssh_user': '', 'ssh_key': ''})
keystone_db_user = resource.create('keystone_db_user', 'resources/mariadb_keystone_user/', {'new_user_name': 'keystone', 'new_user_password': 'keystone', 'db_name': '', 'login_password': '', 'login_user': 'root', 'login_port': '', 'ip': '', 'ssh_user': '', 'ssh_key': ''})
keystone_config1 = resource.create('keystone_config1', 'resources/keystone_config/', {'config_dir': '/etc/solar/keystone', 'ip': '', 'ssh_user': '', 'ssh_key': '', 'admin_token': 'admin', 'db_password': '', 'db_name': '', 'db_user': '', 'db_host': '', 'db_port': ''})
keystone_service1 = resource.create('keystone_service1', 'resources/keystone_service/', {'port': 5001, 'admin_port': 35357, 'image': '', 'ip': '', 'ssh_key': '', 'ssh_user': '', 'config_dir': ''})
@ -41,9 +41,9 @@ def deploy():
keystone_config2 = resource.create('keystone_config2', 'resources/keystone_config/', {'config_dir': '/etc/solar/keystone', 'ip': '', 'ssh_user': '', 'ssh_key': '', 'admin_token': 'admin', 'db_password': '', 'db_name': '', 'db_user': '', 'db_host': '', 'db_port': ''})
keystone_service2 = resource.create('keystone_service2', 'resources/keystone_service/', {'port': 5002, 'admin_port': 35358, 'image': '', 'ip': '', 'ssh_key': '', 'ssh_user': '', 'config_dir': ''})
haproxy_keystone_config = resource.create('haproxy_keystone_config', 'resources/haproxy_keystone_config/', {'name': 'keystone_config', 'listen_port': 5000, 'servers': [], 'ports':[]})
haproxy_config = resource.create('haproxy_config', 'resources/haproxy', {'ip': '', 'ssh_key': '', 'ssh_user': '', 'configs_names': [], 'configs_ports': [], 'listen_ports': [], 'configs':[], 'config_dir': ''})
haproxy_service = resource.create('haproxy_service', 'resources/docker_container/', {'image': 'tutum/haproxy', 'ports': [], 'host_binds': [], 'volume_binds': [], 'ip': '', 'ssh_key': '', 'ssh_user': ''})
haproxy_keystone_config = resource.create('haproxy_keystone1_config', 'resources/haproxy_keystone_config/', {'name': 'keystone_config', 'listen_port':5000, 'servers':[], 'ports':[]})
haproxy_config = resource.create('haproxy_config', 'resources/haproxy_config', {'ip': '', 'ssh_key': '', 'ssh_user': '', 'configs_names':[], 'configs_ports':[], 'listen_ports':[], 'configs':[], 'config_dir': ''})
haproxy_service = resource.create('haproxy_service', 'resources/docker_container/', {'image': 'tutum/haproxy', 'ports': [], 'host_binds': [], 'volume_binds':[], 'ip': '', 'ssh_key': '', 'ssh_user': ''})
glance_db = resource.create('glance_db', 'resources/mariadb_db/', {'db_name': 'glance_db', 'login_password': '', 'login_user': 'root', 'login_port': '', 'ip': '', 'ssh_user': '', 'ssh_key': ''})
glance_db_user = resource.create('glance_db_user', 'resources/mariadb_user/', {'new_user_name': 'glance', 'new_user_password': 'glance', 'db_name': '', 'login_password': '', 'login_user': 'root', 'login_port': '', 'ip': '', 'ssh_user': '', 'ssh_key': ''})

27
example.sh Executable file
View File

@ -0,0 +1,27 @@
#!/bin/bash
set -eux
rm -rf /tmp/tmp*
rm /vagrant/tmp/storage/* || true
rm /vagrant/tmp/connections.yaml || true
echo > /vagrant/state/commit_log || true
echo > /vagrant/state/commited_data || true
echo > /vagrant/state/stage_log || true
find /vagrant/solar/solar -name '*.pyc' -delete || true
sudo docker stop $(sudo docker ps -q) || true
sudo docker rm $(sudo docker ps -qa) || true
solar profile -c -t env/test_env -i prf1
solar discover
solar assign -n 'node/node_2 | node/node_1' -r 'resources/docker'
solar assign -n 'node/node_1' -r 'resources/mariadb'
solar assign -n 'node/node_1' -r 'resources/keystone'
solar assign -n 'node/node_1' -r 'resources/haproxy'
solar assign -n 'node/node_1' -r 'resources/rabbitmq'
solar connect --profile prf1
./cli.py changes stage
./cli.py changes commit

View File

@ -1,5 +1,5 @@
- hosts: [resource/docker]
- hosts: [{{ ip }}]
sudo: yes
tasks:
- shell: docker --version

View File

@ -4,9 +4,13 @@ version: 1.0.0
input:
ip:
schema: str!
value:
ssh_user:
schema: str!
value:
ssh_key:
schema: str!
value:
input-types:
tags: [resource/docker]
tags: [resources/docker]

View File

@ -1,4 +1,4 @@
id: haproxy
id: haproxy_config
handler: ansible
version: 1.0.0
input:
@ -27,4 +27,4 @@ input:
schema: str!
value:
tags: [resources/haproxy]
tags: [resources/haproxy, resource/haproxy_config]

View File

@ -10,9 +10,9 @@ input:
value: 9999
ports:
schema: [{value: int}]
value:
value: []
servers:
schema: [{value: str}]
value:
value: []
tags: [resources/haproxy, resource/haproxy_keystone_config]

View File

@ -0,0 +1,6 @@
- hosts: [{{ ip }}]
sudo: yes
tasks:
- shell: docker stop {{ name }}
- shell: docker rm {{ name }}

View File

@ -0,0 +1,27 @@
- hosts: [{{ ip }}]
sudo: yes
tasks:
- docker:
name: {{ name }}
image: {{ image }}
state: running
net: host
{% if ports.value %}
ports:
{% for port in ports.value %}
{% for p in port['value'] %}
- {{ p['value'] }}:{{ p['value'] }}
{% endfor %}
{% endfor %}
{% endif %}
{% if host_binds.value %}
volumes:
# TODO: host_binds might need more work
# Currently it's not that trivial to pass custom src: dst here
# (when a config variable is passed here from other resource)
# so we mount it to the same directory as on host
{% for bind in host_binds.value %}
- {{ bind['value']['src'] }}:{{ bind['value']['dst'] }}:{{ bind['value'].get('mode', 'ro') }}
{% endfor %}
{% endif %}

View File

@ -0,0 +1,27 @@
id: haproxy_service
handler: ansible
version: 1.0.0
input:
ip:
schema: str!
value:
image:
schema: str!
value: tutum/haproxy
ports:
schema: [{value: [{value: int}]}]
value: []
host_binds:
schema: [{value: {src: str, dst: str}}]
value: []
volume_binds:
schema: [{src: str, dst: str}]
value: []
ssh_user:
schema: str!
value:
ssh_key:
schema: str!
value:
tags: [resources/haproxy, resource/haproxy_service]

View File

@ -8,7 +8,7 @@ input:
value: /etc/solar/keystone
admin_token:
schema: str!
value:
value: admin
db_password:
schema: str!
value: password

View File

@ -1,4 +1,4 @@
id: keystone_user
id: keystone_role
handler: ansible
version: 1.0.0
input:
@ -13,13 +13,13 @@ input:
value:
user_name:
schema: str!
value:
value: admin
tenant_name:
schema: str!
value:
role_name:
schema: str!
value:
value: admin
ip:
schema: str!
value:
@ -30,4 +30,4 @@ input:
schema: str!
value:
tags: [resource/keystone_user, resources/keystone]
tags: [resource/keystone_role, resources/keystone]

View File

@ -15,3 +15,6 @@
- {{ admin_port }}:35357
volumes:
- {{ config_dir }}:/etc/keystone
- name: wait for keystone
wait_for: host={{ip}} port={{port}} timeout=20

View File

@ -1,4 +1,4 @@
id: keystone
id: keystone_service
handler: ansible
version: 1.0.0
input:
@ -10,10 +10,10 @@ input:
value: /etc/solar/keystone
port:
schema: int!
value:
value: 5000
admin_port:
schema: int!
value:
value: 35357
ip:
schema: str!
value:

View File

@ -1,4 +1,4 @@
id: keystone_user
id: keystone_service_endpoint
handler: ansible
version: 1.0.0
input:
@ -19,19 +19,22 @@ input:
value:
name:
schema: str!
value:
value: keystone
type:
schema: str!
value:
value: identity
description:
schema: str!
value:
value: OpenStack Identity Service
publicurl:
schema: str!
value:
value: http://{{ip}}:{{port}}/v2.0
internalurl:
schema: str!
value:
value: http://{{ip}}:{{port}}/v2.0
adminurl:
schema: str!
value: http://{{ip}}:{{admin_port}}/v2.0
adminurl:
schema: str!
value:
@ -45,5 +48,4 @@ input:
schema: str!
value:
tags: [resource/keystone_tenant, resources/keystone]
tags: [resource/keystone_service_endpoint, resources/keystone]

View File

@ -1,4 +1,4 @@
id: keystone_user
id: keystone_tenant
handler: ansible
version: 1.0.0
input:
@ -13,7 +13,7 @@ input:
value:
tenant_name:
schema: str!
value:
value: admin
ip:
schema: str!
value:

View File

@ -13,10 +13,10 @@ input:
value:
user_name:
schema: str!
value:
value: admin
user_password:
schema: str!
value:
value: admin
tenant_name:
schema: str!
value:

View File

@ -1,4 +1,4 @@
id: mariadb_table
id: mariadb_db
handler: ansible
version: 1.0.0
actions:
@ -8,15 +8,15 @@ input:
db_name:
schema: str!
value:
login_user:
schema: str!
value:
login_password:
schema: str!
value:
login_port:
schema: int!
value:
login_user:
schema: str!
value:
ip:
schema: str!
value:
@ -26,5 +26,3 @@ input:
ssh_user:
schema: str!
value:
tags: [resource/mariadb_table, resources/mariadb]

View File

@ -0,0 +1,11 @@
- hosts: [{{ ip }}]
sudo: yes
tasks:
- name: mariadb db
mysql_db:
name: {{db_name}}
state: absent
login_user: root
login_password: {{login_password}}
login_port: {{login_port}}
login_host: 127.0.0.1

View File

@ -0,0 +1,11 @@
- hosts: [{{ ip }}]
sudo: yes
tasks:
- name: mariadb db
mysql_db:
name: {{ db_name }}
state: present
login_user: root
login_password: {{ login_password }}
login_port: {{ login_port }}
login_host: 127.0.0.1

View File

@ -0,0 +1,30 @@
id: mariadb_keystone_db
handler: ansible
version: 1.0.0
actions:
run: run.yml
remove: remove.yml
input:
db_name:
schema: str!
value: keystone_db
login_user:
schema: str!
value: root
login_password:
schema: str!
value:
login_port:
schema: int!
value:
ip:
schema: str!
value:
ssh_key:
schema: str!
value:
ssh_user:
schema: str!
value:
tags: [resource/mariadb_keystone_db, resources/mariadb]

View File

@ -0,0 +1,11 @@
- hosts: [{{ ip }}]
sudo: yes
tasks:
- name: mariadb user
mysql_user:
name: {{new_user_name}}
state: absent
login_user: root
login_password: {{login_password}}
login_port: {{login_port}}
login_host: 127.0.0.1

View File

@ -0,0 +1,14 @@
- hosts: [{{ ip }}]
sudo: yes
tasks:
- name: mariadb user
mysql_user:
name: {{ new_user_name }}
password: {{ new_user_password }}
priv: {{ db_name }}.*:ALL
host: '%'
state: present
login_user: root
login_password: {{ login_password }}
login_port: {{ login_port }}
login_host: 127.0.0.1

View File

@ -0,0 +1,37 @@
id: mariadb_keystone_user
handler: ansible
version: 1.0.0
actions:
run: run.yml
remove: remove.yml
input:
new_user_password:
schema: str!
value: keystone
new_user_name:
schema: str!
value: keystone
db_name:
schema: str!
value:
login_password:
schema: str!
value:
login_port:
schema: int!
value:
login_user:
schema: str!
value:
ip:
schema: str!
value:
ssh_key:
schema: str!
value:
ssh_user:
schema: str!
value:
tags: [resource/mariadb_keystone_user, resources/mariadb]

View File

@ -10,8 +10,9 @@
- {{ port }}:3306
env:
MYSQL_ROOT_PASSWORD: {{ root_password }}
- shell: docker exec -t {{ name }} mysql -p{{ root_password }} -uroot -e "select 1"
- shell: docker exec -t {{ name }} mysql -p{{ root_password }} -uroot -e "SELECT 1"
register: result
until: result.rc == 0
retries: 20
retries: 30
delay: 0.5

View File

@ -1,4 +1,4 @@
id: mariadb
id: mariadb_service
handler: ansible
version: 1.0.0
input:
@ -21,4 +21,4 @@ input:
schema: str!
value:
tags: [resource/mariadb, resources/mariadb]
tags: [resource/mariadb_service, resources/mariadb]

View File

@ -32,6 +32,3 @@ input:
ssh_user:
schema: str!
value:
tags: [resource/mariadb_user, resources/mariadb]

View File

@ -15,7 +15,7 @@
- {{ management_port }}:15672
env:
RABBITMQ_NODENAME: {{container_name}}
- shell: docker exec -t {{ name }} rabbitmqctl list_users
- shell: docker exec -t {{ container_name }} rabbitmqctl list_users
register: result
until: result.rc == 0
retries: 20

View File

@ -7,13 +7,13 @@ input:
value: rabbitmq:3-management
container_name:
schema: str!
value:
value: rabbitmq_service
management_port:
schema: str!
value:
value: 15672
port:
schema: str!
value:
value: 5672
ip:
schema: str!
value:
@ -23,3 +23,5 @@ input:
ssh_user:
schema: str!
value:
tags: [resources/rabbitmq, resource/rabbitmq_service]

View File

@ -1,4 +1,4 @@
id: keystone_user
id: rabbitmq_user
handler: ansible
version: 1.0.0
input:
@ -7,10 +7,10 @@ input:
value:
user_name:
schema: str!
value:
value: openstack
password:
schema: str!
value:
value: openstack_password
vhost_name:
schema: str!
value:
@ -23,3 +23,5 @@ input:
ssh_user:
schema: str!
value:
tags: [resources/rabbitmq, resource/rabbitmq_user]

View File

@ -7,7 +7,7 @@ input:
value:
vhost_name:
schema: str!
value:
value: openstack
ip:
schema: str!
value:
@ -18,4 +18,4 @@ input:
schema: str!
value:
tags: [resource/rabbitmq]
tags: [resources/rabbitmq, resource/rabbitmq_vhost]

View File

@ -74,6 +74,12 @@ class Cmd(object):
parser.add_argument('-n', '--nodes')
parser.add_argument('-r', '--resources')
# Run action on tags
parser = self.subparser.add_parser('run')
parser.set_defaults(func=getattr(self, 'run'))
parser.add_argument('-t', '--tags')
parser.add_argument('-a', '--action')
# Perform resources connection
parser = self.subparser.add_parser('connect')
parser.set_defaults(func=getattr(self, 'connect'))
@ -81,6 +87,18 @@ class Cmd(object):
'-p',
'--profile')
def run(self, args):
from solar.core import actions
from solar.core import signals
resources = filter(
lambda r: Expression(args.tags, r.get('tags', [])).evaluate(),
self.db.get_list('resource'))
for resource in resources:
resource_obj = self.db.get_obj_resource(resource['id'])
actions.resource_action(resource_obj, args.action)
def profile(self, args):
if args.create:
params = {'tags': args.tags, 'id': args.id}
@ -107,6 +125,7 @@ class Cmd(object):
lambda r: Expression(args.resources, r.get('tags', [])).evaluate(),
self._get_resources_list())
print("For {0} nodes assign {1} resources".format(len(nodes), len(resources)))
assign_resources_to_nodes(resources, nodes)
def _get_resources_list(self):

View File

@ -36,8 +36,8 @@ class ResourcesConnectionGraph(object):
def iter_connections(self):
for connection in self.connections:
connections_from = self.resources_with_tags(depends_on(connection))
connections_to = self.resources_with_tags(connection['for_resources'])
connections_to = self.resources_with_tags(depends_on(connection))
connections_from = self.resources_with_tags(connection['for_resources'])
mapping = self.make_mapping(connection)
for connection_from in connections_from:

View File

@ -19,11 +19,6 @@ class Ansible(BaseHandler):
print e.output
raise
#def _get_connection(self, resource):
# return {'ssh_user': '',
# 'ssh_key': '',
# 'host': ''}
def _create_inventory(self, r):
directory = self.dirs[r.name]
inventory_path = os.path.join(directory, 'inventory')

View File

@ -170,7 +170,7 @@ def assign_resources_to_nodes(resources, nodes):
res = deepcopy(resource)
res['tags'] = list(set(node.get('tags', [])) |
set(resource.get('tags', [])))
resource_uuid = solar.utils.generate_uuid()
resource_uuid = '{0}-{1}'.format(res['id'], solar.utils.generate_uuid())
# We should not generate here any uuid's, because
# a single node should be represented with a single
# resource

View File

@ -141,4 +141,3 @@ class Data(collections.MutableMapping):
def __len__(self):
return len(self.store)

View File

@ -10,33 +10,103 @@ extensions:
tags: {{tags}}
connections:
- for_resources: [resource/keystone_config]
filter_resources: [resource/mariadb]
# keystone
- for_resources: [resource/mariadb_service]
filter_resources: [resource/mariadb_keystone_db, resource/mariadb_keystone_user]
mapping:
root_password: login_password
port: login_port
- for_resources: [resource/mariadb_keystone_db]
filter_resources: [resource/mariadb_keystone_user]
mapping:
db_name: db_name
- for_resources: [resource/mariadb_service]
filter_resources: [resource/keystone_config]
mapping:
ip: db_host
root_password: db_password
port: db_port
- for_resources: [resource/keystone_service]
filter_resources: [resources/haproxy]
- for_resources: [resource/mariadb_keystone_user]
filter_resources: [resource/keystone_config]
mapping:
servers: ip
ports: admin_port
db_name: db_name
new_user_name: db_user
new_user_password: db_password
# NOTE(dshulyak) it is mandatory for some profiles to user graph based
# api to provide order of execution for different events
run:
- containers.actions.clean_list
- docker.actions.run
- mariadb.actions.run
- mariadb.actions.wait
- mariadb.actions.users
- keystone.actions.run
- haproxy.actions.run
- containers.actions.ensure
- for_resources: [resource/keystone_config]
filter_resources: [resource/keystone_service]
mapping:
config_dir: config_dir
remove:
- haproxy.actions.remove
- keystone.actions.remove
- mariadb.actions.remove
- docker.actions.remove
# haproxy
- for_resources: [resource/keystone_service]
filter_resources: [resource/haproxy_keystone_config]
mapping:
ip: servers
port: ports
- for_resources: [resource/haproxy_keystone_config]
filter_resources: [resource/haproxy_config]
mapping:
listen_port: listen_ports
name: configs_names
ports: configs_ports
servers: configs
- for_resources: [resource/haproxy_config]
filter_resources: [resource/haproxy_service]
mapping:
listen_ports: ports
config_dir: host_binds
# keystone data
- for_resources: [resource/keystone_config]
filter_resources: [resource/keystone_tenant]
mapping: {}
- for_resources: [resource/keystone_service]
filter_resources: [resource/keystone_tenant]
mapping:
admin_port: keystone_port
ip: keystone_host
- for_resources: [resource/keystone_tenant]
filter_resources: [resource/keystone_user]
mapping: {}
- for_resources: [resource/keystone_user]
filter_resources: [resource/keystone_role]
mapping: {}
- for_resources: [resource/keystone_service]
filter_resources: [resource/keystone_service_endpoint]
mapping:
admin_port: admin_port
ip: keystone_host
port: port
- for_resources: [resource/keystone_config]
filter_resources: [resource/keystone_service_endpoint]
mapping:
admin_token: admin_token
- for_resources: [resource/keystone_service]
filter_resources: [resource/keystone_service_endpoint]
mapping:
admin_port: keystone_port
# rabbitmq
- for_resources: [resource/rabbitmq_service]
filter_resources: [resource/rabbitmq_vhost]
mapping: {}
- for_resources: [resource/rabbitmq_service]
filter_resources: [resource/rabbitmq_user]
mapping: {}
- for_resources: [resource/rabbitmq_vhost]
filter_resources: [resource/rabbitmq_user]
mapping:
vhost_name: vhost_name