diff --git a/compass/api/api.py b/compass/api/api.py index 2a861523..026b7209 100644 --- a/compass/api/api.py +++ b/compass/api/api.py @@ -124,8 +124,35 @@ def _get_request_data_as_list(): return [] -def _get_request_args(): - return dict(request.args) +def _bool_converter(value): + if not value: + return True + if value in ['False', 'false', '0']: + return False + return True + + +def _int_converter(value): + try: + return int(value) + except Exception: + raise exception_handler.BadRequest( + '%r type is not int' % value + ) + + +def _get_request_args(**kwargs): + args = dict(request.args) + logging.debug('origin request args: %s', args) + for key, value in args.items(): + if key in kwargs: + converter = kwargs[key] + if isinstance(value, list): + args[key] = [converter(item) for item in value] + else: + args[key] = converter(value) + logging.debug('request args: %s', args) + return args def _group_data_action(data, **data_callbacks): @@ -232,7 +259,10 @@ def logout(): @login_required def list_users(): """list users.""" - data = _get_request_args() + data = _get_request_args( + is_admin=_bool_converter, + active=_bool_converter + ) return utils.make_json_response( 200, user_api.list_users(current_user, **data) ) @@ -748,7 +778,7 @@ def _filter_location(data): @login_required def list_switch_machines(switch_id): """Get switch machines.""" - data = _get_request_args() + data = _get_request_args(vlans=_int_converter) _filter_port(data) _filter_general(data, 'vlans') _filter_tag(data) @@ -766,7 +796,7 @@ def list_switch_machines(switch_id): @login_required def list_switch_machines_hosts(switch_id): """Get switch machines or hosts.""" - data = _get_request_args() + data = _get_request_args(vlans=_int_converter, os_id=_int_converter) _filter_port(data) _filter_general(data, 'vlans') _filter_tag(data) @@ -929,7 +959,7 @@ def take_machine_action(machine_id): @login_required def list_switchmachines(): """List switch machines.""" - data = _get_request_args() + data = _get_request_args(vlans=_int_converter) _filter_ip(data) _filter_port(data) _filter_general(data, 'vlans') @@ -948,7 +978,7 @@ def list_switchmachines(): @login_required def list_switchmachines_hosts(): """List switch machines or hosts.""" - data = _get_request_args() + data = _get_request_args(vlans=_int_converter, os_id=_int_converter) _filter_ip(data) _filter_port(data) _filter_general(data, 'vlans') @@ -1276,7 +1306,7 @@ def list_clusters(): @login_required def show_cluster(cluster_id): """Get cluster.""" - data = _get_request_args() + data = _get_request_args(adapter_id=_int_converter) return utils.make_json_response( 200, cluster_api.get_cluster( @@ -1828,11 +1858,11 @@ def show_host(host_id): @login_required def list_machines_or_hosts(): """Get host.""" - data = _get_request_args() + data = _get_request_args(os_id=_int_converter) _filter_tag(data) _filter_location(data) _filter_general(data, 'os_name') - _filter_general(data, 'os_name') + _filter_general(data, 'os_id') return utils.make_json_response( 200, _reformat_host(host_api.list_machines_or_hosts( @@ -1979,7 +2009,10 @@ def list_host_networks(host_id): @login_required def list_hostnetworks(): """list host networks.""" - data = _get_request_args() + data = _get_request_args( + is_mgmt=_bool_converter, + is_promiscuous=_bool_converter + ) return utils.make_json_response( 200, host_api.list_hostnetworks(current_user, **data) ) diff --git a/compass/apiclient/restful.py b/compass/apiclient/restful.py index e14890a1..bae38922 100644 --- a/compass/apiclient/restful.py +++ b/compass/apiclient/restful.py @@ -544,12 +544,13 @@ class Client(object): return self._get('/clusters/%s' % cluster_id) def add_cluster(self, name, adapter_id, os_id, - flavor_id, raw_data=None): + flavor_id=None, raw_data=None): data = {} if raw_data: data = raw_data else: - data['flavor_id'] = flavor_id + if flavor_id: + data['flavor_id'] = flavor_id data['name'] = name data['adapter_id'] = adapter_id data['os_id'] = os_id diff --git a/compass/db/api/cluster.py b/compass/db/api/cluster.py index 626a947d..bf175b69 100644 --- a/compass/db/api/cluster.py +++ b/compass/db/api/cluster.py @@ -96,6 +96,7 @@ RESP_REVIEW_FIELDS = [ RESP_DEPLOY_FIELDS = [ 'status', 'cluster', 'clusterhosts' ] +IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] ADDED_FIELDS = ['name', 'adapter_id', 'os_id'] OPTIONAL_ADDED_FIELDS = ['flavor_id'] UPDATED_FIELDS = ['name', 'reinstall_distributed_system'] @@ -145,13 +146,6 @@ UPDATED_CLUSTERHOST_LOG_FIELDS = [ ] -def _check_roles(roles): - if not roles: - raise exception.InvalidParameter( - 'roles %s is empty' % roles - ) - - @utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -234,7 +228,8 @@ def is_cluster_editable( @utils.supported_filters( ADDED_FIELDS, - optional_support_keys=OPTIONAL_ADDED_FIELDS + optional_support_keys=OPTIONAL_ADDED_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @utils.input_validates(name=utils.check_name) @database.run_in_session() @@ -255,7 +250,10 @@ def add_cluster( ) -@utils.supported_filters(optional_support_keys=UPDATED_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates(name=utils.check_name) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -373,7 +371,8 @@ def _update_cluster_config(session, updater, cluster, **kwargs): package_config='deployed_package_config' ) @utils.supported_filters( - optional_support_keys=UPDATED_DEPLOYED_CONFIG_FIELDS + optional_support_keys=UPDATED_DEPLOYED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -398,7 +397,10 @@ def update_cluster_deployed_config( os_config='put_os_config', package_config='put_package_config' ) -@utils.supported_filters(optional_support_keys=UPDATED_CONFIG_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @database.run_in_session() def update_cluster_config(session, updater, cluster_id, **kwargs): """Update cluster config.""" @@ -430,7 +432,10 @@ def update_cluster_config(session, updater, cluster_id, **kwargs): os_config='patched_os_config', package_config='patched_package_config' ) -@utils.supported_filters(optional_support_keys=PATCHED_CONFIG_FIELDS) +@utils.supported_filters( + optional_support_keys=PATCHED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @database.run_in_session() def patch_cluster_config(session, updater, cluster_id, **kwargs): """patch cluster config.""" @@ -477,9 +482,10 @@ def del_cluster_config(session, deleter, cluster_id): @utils.supported_filters( ADDED_HOST_FIELDS, - optional_support_keys=(UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS) + optional_support_keys=(UPDATED_HOST_FIELDS + UPDATED_CLUSTERHOST_FIELDS), + ignore_support_keys=IGNORE_FIELDS ) -@utils.input_validates(name=utils.check_name, roles=_check_roles) +@utils.input_validates(name=utils.check_name) def add_clusterhost_internal( session, cluster, exception_when_existing=False, @@ -538,6 +544,16 @@ def add_clusterhost_internal( creator=cluster.creator, **host_dict ) + + if 'roles' in kwargs: + roles = kwargs['roles'] + if not roles: + flavor = cluster.flavor + if flavor and flavor.flavor_roles: + raise exception.InvalidParameter( + 'roles %s is empty' % roles + ) + return utils.add_db_object( session, models.ClusterHost, exception_when_existing, cluster.id, host.id, **clusterhost_dict @@ -652,7 +668,6 @@ def add_cluster_host( ) -@utils.input_validates(roles=_check_roles) @user_api.check_user_permission_in_session( permission.PERMISSION_UPDATE_CLUSTER_HOSTS ) @@ -675,6 +690,14 @@ def _update_clusterhost(session, updater, clusterhost, **kwargs): role, cluster_roles ) ) + if 'roles' in kwargs: + roles = kwargs['roles'] + flavor = clusterhost.cluster.flavor + if not roles: + if flavor and flavor.flavor_roles: + raise exception.InvalidParameter( + 'roles %s is empty' % roles + ) @utils.input_validates( roles=roles_validates, @@ -692,7 +715,8 @@ def _update_clusterhost(session, updater, clusterhost, **kwargs): @utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_FIELDS + optional_support_keys=UPDATED_CLUSTERHOST_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() def update_cluster_host( @@ -707,7 +731,8 @@ def update_cluster_host( @utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_FIELDS + optional_support_keys=UPDATED_CLUSTERHOST_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() def update_clusterhost( @@ -725,7 +750,8 @@ def update_clusterhost( roles='patched_roles' ) @utils.supported_filters( - optional_support_keys=PATCHED_CLUSTERHOST_FIELDS + optional_support_keys=PATCHED_CLUSTERHOST_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() def patch_cluster_host( @@ -743,7 +769,8 @@ def patch_cluster_host( roles='patched_roles' ) @utils.supported_filters( - optional_support_keys=PATCHED_CLUSTERHOST_FIELDS + optional_support_keys=PATCHED_CLUSTERHOST_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() def patch_clusterhost( @@ -907,7 +934,7 @@ def _update_clusterhost_deployed_config( ): from compass.db.api import host as host_api ignore_keys = [] - if host_api.is_host_editable( + if not host_api.is_host_editable( session, clusterhost.host, updater, exception_when_not_editable=False ): @@ -1017,7 +1044,7 @@ def update_clusterhost_deployed_config( def _patch_clusterhost_config(session, updater, clusterhost, **kwargs): from compass.db.api import host as host_api ignore_keys = [] - if host_api.is_host_editable( + if not host_api.is_host_editable( session, clusterhost.host, updater, exception_when_not_editable=False ): @@ -1096,7 +1123,7 @@ def _delete_clusterhost_config( ): from compass.db.api import host as host_api ignore_keys = [] - if host_api.is_host_editable( + if not host_api.is_host_editable( session, clusterhost.host, deleter, exception_when_not_editable=False ): @@ -1189,9 +1216,11 @@ def update_cluster_hosts( def validate_clusterhost(session, clusterhost): roles = clusterhost.roles if not roles: - raise exception.InvalidParameter( - 'empty roles for clusterhost %s' % clusterhost.name - ) + flavor = clusterhost.cluster.flavor + if flavor and flavor.flavor_roles: + raise exception.InvalidParameter( + 'empty roles for clusterhost %s' % clusterhost.name + ) def validate_cluster(session, cluster): @@ -1199,10 +1228,14 @@ def validate_cluster(session, cluster): raise exception.InvalidParameter( '%s does not have any hosts' % cluster.name ) - cluster_roles = [ - flavor_role.role - for flavor_role in cluster.flavor.flavor_roles - ] + flavor = cluster.flavor + if flavor: + cluster_roles = [ + flavor_role.role + for flavor_role in flavor.flavor_roles + ] + else: + cluster_roles = [] necessary_roles = set([ role.name for role in cluster_roles if not role.optional ]) @@ -1437,7 +1470,8 @@ def get_clusterhost_self_state( @utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS + optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -1457,7 +1491,8 @@ def update_cluster_host_state( @utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS + optional_support_keys=UPDATED_CLUSTERHOST_STATE_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -1477,7 +1512,8 @@ def update_clusterhost_state( @utils.supported_filters( - optional_support_keys=UPDATED_CLUSTER_STATE_FIELDS + optional_support_keys=UPDATED_CLUSTER_STATE_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -1545,7 +1581,8 @@ def get_clusterhost_log_history( @utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS + optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() @utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) @@ -1561,7 +1598,8 @@ def update_cluster_host_log_history( @utils.supported_filters( - optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS + optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() @utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) @@ -1578,7 +1616,8 @@ def update_clusterhost_log_history( @utils.supported_filters( ADDED_CLUSTERHOST_LOG_FIELDS, - optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS + optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() @utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) @@ -1595,7 +1634,8 @@ def add_clusterhost_log_history( @utils.supported_filters( ADDED_CLUSTERHOST_LOG_FIELDS, - optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS + optional_support_keys=UPDATED_CLUSTERHOST_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() @utils.wrap_to_dict(RESP_CLUSTERHOST_LOG_FIELDS) diff --git a/compass/db/api/host.py b/compass/db/api/host.py index 8a527c2d..5345cb55 100644 --- a/compass/db/api/host.py +++ b/compass/db/api/host.py @@ -79,8 +79,8 @@ UPDATED_NETWORK_FIELDS = [ 'interface', 'ip', 'subnet_id', 'subnet', 'is_mgmt', 'is_promiscuous' ] -IGNORED_NETWORK_FIELDS = [ - 'interface' +IGNORE_FIELDS = [ + 'id', 'created_at', 'updated_at' ] RESP_STATE_FIELDS = [ 'id', 'state', 'percentage', 'message', 'severity' @@ -259,7 +259,10 @@ def validate_host(session, host): ) -@utils.supported_filters(optional_support_keys=UPDATED_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates(name=utils.check_name) @utils.wrap_to_dict(RESP_FIELDS) def _update_host(session, updater, host_id, **kwargs): @@ -352,7 +355,10 @@ def get_host_deployed_config(session, getter, host_id, **kwargs): @utils.replace_filters( os_config='deployed_os_config' ) -@utils.supported_filters(UPDATED_DEPLOYED_CONFIG_FIELDS) +@utils.supported_filters( + UPDATED_DEPLOYED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_ADD_HOST_CONFIG @@ -381,7 +387,10 @@ def _update_host_config(session, updater, host, **kwargs): @utils.replace_filters( os_config='put_os_config' ) -@utils.supported_filters(UPDATED_CONFIG_FIELDS) +@utils.supported_filters( + UPDATED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @database.run_in_session() def update_host_config(session, updater, host_id, **kwargs): host = utils.get_db_object( @@ -407,7 +416,10 @@ def update_host_config(session, updater, host_id, **kwargs): @utils.replace_filters( os_config='patched_os_config' ) -@utils.supported_filters(PATCHED_CONFIG_FIELDS) +@utils.supported_filters( + PATCHED_CONFIG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @database.run_in_session() def patch_host_config(session, updater, host_id, **kwargs): host = utils.get_db_object( @@ -517,7 +529,9 @@ def get_hostnetwork(session, getter, host_network_id, **kwargs): @utils.supported_filters( - ADDED_NETWORK_FIELDS, optional_support_keys=OPTIONAL_ADDED_NETWORK_FIELDS + ADDED_NETWORK_FIELDS, + optional_support_keys=OPTIONAL_ADDED_NETWORK_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @utils.input_validates( ip=utils.check_ip @@ -650,7 +664,7 @@ def _update_host_network( @utils.supported_filters( optional_support_keys=UPDATED_NETWORK_FIELDS, - ignore_support_keys=IGNORED_NETWORK_FIELDS + ignore_support_keys=IGNORE_FIELDS ) @utils.input_validates( ip=utils.check_ip @@ -677,7 +691,7 @@ def update_host_network( @utils.supported_filters( optional_support_keys=UPDATED_NETWORK_FIELDS, - ignore_support_keys=IGNORED_NETWORK_FIELDS + ignore_support_keys=IGNORE_FIELDS ) @utils.input_validates( ip=utils.check_ip @@ -743,7 +757,10 @@ def get_host_state(session, getter, host_id, **kwargs): ).state_dict() -@utils.supported_filters(optional_support_keys=UPDATED_STATE_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_STATE_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_UPDATE_HOST_STATE @@ -778,7 +795,10 @@ def get_host_log_history(session, getter, host_id, filename, **kwargs): ) -@utils.supported_filters(optional_support_keys=UPDATED_LOG_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @database.run_in_session() @utils.wrap_to_dict(RESP_LOG_FIELDS) def update_host_log_history(session, updater, host_id, filename, **kwargs): @@ -791,7 +811,9 @@ def update_host_log_history(session, updater, host_id, filename, **kwargs): @utils.supported_filters( ADDED_LOG_FIELDS, - optional_support_keys=UPDATED_LOG_FIELDS) + optional_support_keys=UPDATED_LOG_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @database.run_in_session() @utils.wrap_to_dict(RESP_LOG_FIELDS) def add_host_log_history( diff --git a/compass/db/api/machine.py b/compass/db/api/machine.py index a7295843..b94c63f4 100644 --- a/compass/db/api/machine.py +++ b/compass/db/api/machine.py @@ -27,6 +27,7 @@ from compass.utils import util SUPPORTED_FIELDS = ['mac', 'tag', 'location'] +IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] UPDATED_FIELDS = ['ipmi_credentials', 'tag', 'location'] PATCHED_FIELDS = [ 'patched_ipmi_credentials', 'patched_tag', @@ -89,7 +90,10 @@ def _update_machine(session, updater, machine_id, **kwargs): return utils.update_db_object(session, machine, **kwargs) -@utils.supported_filters(optional_support_keys=UPDATED_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates(ipmi_credentials=utils.check_ipmi_credentials) @database.run_in_session() def update_machine(session, updater, machine_id, **kwargs): @@ -103,7 +107,10 @@ def update_machine(session, updater, machine_id, **kwargs): tag='patched_tag', location='patched_location' ) -@utils.supported_filters(optional_support_keys=PATCHED_FIELDS) +@utils.supported_filters( + optional_support_keys=PATCHED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @database.run_in_session() @utils.output_validates(ipmi_credentials=utils.check_ipmi_credentials) def patch_machine(session, updater, machine_id, **kwargs): diff --git a/compass/db/api/network.py b/compass/db/api/network.py index 4f70a4da..3eedb26f 100644 --- a/compass/db/api/network.py +++ b/compass/db/api/network.py @@ -30,13 +30,10 @@ RESP_FIELDS = [ ] ADDED_FIELDS = ['subnet'] OPTIONAL_ADDED_FIELDS = ['name'] -IGNORE_ADDED_FIELDS = [ +IGNORE_FIELDS = [ 'id', 'created_at', 'updated_at' ] UPDATED_FIELDS = ['subnet', 'name'] -IGNORE_UPDATED_FIELDS = [ - 'id', 'created_at', 'updated_at' -] def _check_subnet(subnet): @@ -80,7 +77,7 @@ def get_subnet( @utils.supported_filters( ADDED_FIELDS, optional_support_keys=OPTIONAL_ADDED_FIELDS, - ignore_support_keys=IGNORE_ADDED_FIELDS + ignore_support_keys=IGNORE_FIELDS ) @utils.input_validates(subnet=_check_subnet) @database.run_in_session() @@ -101,7 +98,7 @@ def add_subnet( @utils.supported_filters( optional_support_keys=UPDATED_FIELDS, - ignore_support_keys=IGNORE_UPDATED_FIELDS + ignore_support_keys=IGNORE_FIELDS ) @utils.input_validates(subnet=_check_subnet) @database.run_in_session() diff --git a/compass/db/api/switch.py b/compass/db/api/switch.py index 045a179a..f21c550a 100644 --- a/compass/db/api/switch.py +++ b/compass/db/api/switch.py @@ -42,19 +42,24 @@ SUPPORTED_MACHINES_HOSTS_FIELDS = [ 'port', 'vlans', 'mac', 'tag', 'location', 'os_name', 'os_id' ] +IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] ADDED_FIELDS = ['ip'] -OPTIONAL_ADDED_FIELDS = ['credentials', 'vendor', 'state', 'err_msg'] -UPDATED_FIELDS = ['credentials', 'vendor', 'state', 'err_msg'] -PATCHED_FIELDS = ['patched_credentials'] -UPDATED_FILTERS_FIELDS = ['filters'] +OPTIONAL_ADDED_FIELDS = [ + 'credentials', 'vendor', 'state', 'err_msg', 'filters' +] +UPDATED_FIELDS = [ + 'ip', 'credentials', 'vendor', 'state', + 'err_msg', 'put_filters' +] +PATCHED_FIELDS = ['patched_credentials', 'patched_filters'] +UPDATED_FILTERS_FIELDS = ['put_filters'] PATCHED_FILTERS_FIELDS = ['patched_filters'] ADDED_MACHINES_FIELDS = ['mac', 'port'] OPTIONAL_ADDED_MACHINES_FIELDS = [ 'vlans', 'ipmi_credentials', 'tag', 'location' ] -CHECK_FILTER_FIELDS = ['filter_name', 'filter_type'] OPTIONAL_CHECK_FILTER_FIELDS = [ - 'ports', 'port_prefix', 'port_suffix', + 'filter_type', 'ports', 'port_prefix', 'port_suffix', 'port_start', 'port_end' ] ADDED_SWITCH_MACHINES_FIELDS = ['port', 'vlans'] @@ -70,7 +75,7 @@ PATCHED_MACHINES_FIELDS = [ PATCHED_SWITCH_MACHINES_FIELDS = ['patched_vlans'] RESP_FIELDS = [ 'id', 'ip', 'credentials', 'vendor', 'state', 'err_msg', - 'created_at', 'updated_at' + 'filters', 'created_at', 'updated_at' ] RESP_FILTERS_FIELDS = [ 'id', 'ip', 'filters', 'created_at', 'updated_at' @@ -90,8 +95,7 @@ RESP_MACHINES_HOSTS_FIELDS = [ 'ipmi_credentials', 'tag', 'location', 'name', 'hostname', 'os_name', 'os_id', 'owner', 'os_installer', 'reinstall_os', 'os_installed', - 'clusters', - 'created_at', 'updated_at' + 'clusters', 'created_at', 'updated_at' ] RESP_CLUSTER_FIELDS = [ 'name', 'id' @@ -99,35 +103,19 @@ RESP_CLUSTER_FIELDS = [ def _check_filters(switch_filters): + logging.debug('check filters: %s', switch_filters) + switch_filters = models.Switch.parse_filters(switch_filters) for switch_filter in switch_filters: - if not isinstance(switch_filter, dict): - raise exception.InvalidParameter( - 'filter %s is not dict' % switch_filter - ) _check_filter_internal(**switch_filter) @utils.supported_filters( - CHECK_FILTER_FIELDS, optional_support_keys=OPTIONAL_CHECK_FILTER_FIELDS + optional_support_keys=OPTIONAL_CHECK_FILTER_FIELDS ) def _check_filter_internal( - filter_name, filter_type, **switch_filter + **switch_filter ): - if filter_type not in ['allow', 'deny']: - raise exception.InvalidParameter( - 'filter_type should be `allow` or `deny` in %s' % switch_filter - ) - if 'ports' in switch_filter: - if not isinstance(switch_filter['ports'], list): - raise exception.InvalidParameter( - '`ports` is not list in filter %s' % switch_filter - ) - for key in ['port_start', 'port_end']: - if key in switch_filter: - if not isinstance(switch_filter[key], int): - raise exception.InvalidParameter( - '`key` is not int in filer %s' % switch_filter - ) + pass def _check_vlans(vlans): @@ -145,7 +133,7 @@ def add_switch_internal( with session.begin(subtransactions=True): return utils.add_db_object( session, models.Switch, exception_when_existing, ip_int, - **kwargs + filters=filters, **kwargs ) @@ -211,11 +199,13 @@ def del_switch(session, deleter, switch_id, **kwargs): @utils.supported_filters( ADDED_FIELDS, - optional_support_keys=OPTIONAL_ADDED_FIELDS + optional_support_keys=OPTIONAL_ADDED_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @utils.input_validates( ip=utils.check_ip, - credentials=utils.check_switch_credentials + credentials=utils.check_switch_credentials, + filters=_check_filters ) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -253,9 +243,16 @@ def _update_switch(session, updater, switch_id, **kwargs): return utils.update_db_object(session, switch, **kwargs) -@utils.supported_filters(optional_support_keys=UPDATED_FIELDS) +@utils.replace_filters( + filters='put_filters' +) +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates( - credentials=utils.check_switch_credentials + credentials=utils.check_switch_credentials, + put_filters=_check_filters ) @database.run_in_session() def update_switch(session, updater, switch_id, **kwargs): @@ -263,9 +260,16 @@ def update_switch(session, updater, switch_id, **kwargs): @utils.replace_filters( - credentials='patched_credentials' + credentials='patched_credentials', + filters='patched_filters' +) +@utils.supported_filters( + optional_support_keys=PATCHED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates( + patched_filters=_check_filters ) -@utils.supported_filters(optional_support_keys=PATCHED_FIELDS) @database.run_in_session() @utils.output_validates( credentials=utils.check_switch_credentials @@ -302,8 +306,14 @@ def get_switch_filters( ) -@utils.supported_filters(optional_support_keys=UPDATED_FILTERS_FIELDS) -@utils.input_validates(filters=_check_filters) +@utils.replace_filters( + filters='put_filters' +) +@utils.supported_filters( + optional_support_keys=UPDATED_FILTERS_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) +@utils.input_validates(put_filters=_check_filters) @database.run_in_session() @user_api.check_user_permission_in_session( permission.PERMISSION_UPDATE_SWITCH_FILTERS @@ -318,7 +328,10 @@ def update_switch_filters(session, updater, switch_id, **kwargs): @utils.replace_filters( filters='patched_filters' ) -@utils.supported_filters(optional_support_keys=PATCHED_FILTERS_FIELDS) +@utils.supported_filters( + optional_support_keys=PATCHED_FILTERS_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates(patched_filters=_check_filters) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -511,7 +524,8 @@ def list_switchmachines_hosts(session, lister, **filters): @utils.supported_filters( ADDED_MACHINES_FIELDS, - optional_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS + optional_support_keys=OPTIONAL_ADDED_MACHINES_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @utils.input_validates(mac=utils.check_mac, vlans=_check_vlans) @database.run_in_session() @@ -634,7 +648,10 @@ def update_switch_machine_internal( ) -@utils.supported_filters(optional_support_keys=UPDATED_MACHINES_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_MACHINES_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates(vlans=_check_vlans) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -653,7 +670,10 @@ def update_switch_machine(session, updater, switch_id, machine_id, **kwargs): ) -@utils.supported_filters(optional_support_keys=UPDATED_MACHINES_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_MACHINES_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates(vlans=_check_vlans) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -678,7 +698,10 @@ def update_switchmachine(session, updater, switch_machine_id, **kwargs): tag='patched_tag', location='patched_location' ) -@utils.supported_filters(optional_support_keys=PATCHED_MACHINES_FIELDS) +@utils.supported_filters( + optional_support_keys=PATCHED_MACHINES_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates(patched_vlans=_check_vlans) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -703,7 +726,10 @@ def patch_switch_machine(session, updater, switch_id, machine_id, **kwargs): tag='patched_tag', location='patched_location' ) -@utils.supported_filters(optional_support_keys=PATCHED_MACHINES_FIELDS) +@utils.supported_filters( + optional_support_keys=PATCHED_MACHINES_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates(patched_vlans=_check_vlans) @database.run_in_session() @user_api.check_user_permission_in_session( @@ -754,7 +780,8 @@ def del_switchmachine(session, deleter, switch_machine_id, **kwargs): @utils.supported_filters( ['machine_id'], - optional_support_keys=UPDATED_SWITCH_MACHINES_FIELDS + optional_support_keys=UPDATED_SWITCH_MACHINES_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) def _update_machine_internal(session, switch_id, machine_id, **kwargs): utils.add_db_object( diff --git a/compass/db/api/user.py b/compass/db/api/user.py index b1e2e35f..ff7e9d13 100644 --- a/compass/db/api/user.py +++ b/compass/db/api/user.py @@ -31,6 +31,7 @@ SUPPORTED_FIELDS = ['email', 'is_admin', 'active'] PERMISSION_SUPPORTED_FIELDS = ['name'] SELF_UPDATED_FIELDS = ['email', 'firstname', 'lastname', 'password'] ADMIN_UPDATED_FIELDS = ['is_admin', 'active'] +IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] UPDATED_FIELDS = [ 'email', 'firstname', 'lastname', 'password', 'is_admin', 'active' ] @@ -321,7 +322,9 @@ def list_users(session, lister, **filters): @utils.input_validates(email=_check_email) @utils.supported_filters( - ADDED_FIELDS, optional_support_keys=OPTIONAL_ADDED_FIELDS + ADDED_FIELDS, + optional_support_keys=OPTIONAL_ADDED_FIELDS, + ignore_support_keys=IGNORE_FIELDS ) @check_user_admin() @database.run_in_session() @@ -347,7 +350,10 @@ def del_user(session, deleter, user_id, **kwargs): return utils.del_db_object(session, user) -@utils.supported_filters(optional_support_keys=UPDATED_FIELDS) +@utils.supported_filters( + optional_support_keys=UPDATED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @utils.input_validates(email=_check_email) @database.run_in_session() @utils.wrap_to_dict(RESP_FIELDS) @@ -414,7 +420,10 @@ def del_permission(session, deleter, user_id, permission_id, **kwargs): return utils.del_db_object(session, user_permission) -@utils.supported_filters(PERMISSION_ADDED_FIELDS) +@utils.supported_filters( + PERMISSION_ADDED_FIELDS, + ignore_support_keys=IGNORE_FIELDS +) @check_user_admin() @database.run_in_session() @utils.wrap_to_dict(PERMISSION_RESP_FIELDS) diff --git a/compass/db/models.py b/compass/db/models.py index 29c8dd12..2f56822b 100644 --- a/compass/db/models.py +++ b/compass/db/models.py @@ -1146,11 +1146,13 @@ class Cluster(BASE, TimestampMixin, HelperMixin): distributed_system.name ) ) - flavor = self.flavor - if not flavor: + flavor = self.flavor + if not flavor: + if distributed_system: raise exception.InvalidParameter( 'flavor is not set in cluster %s' % self.id ) + else: flavor_adapter_id = flavor.adapter_id adapter_id = self.adapter_id logging.info( @@ -1497,39 +1499,57 @@ class SwitchMachine(BASE, HelperMixin, TimestampMixin): def filtered(self): filters = self.switch.filters port = self.port + unmatched_allowed = True + ports_pattern = re.compile(r'(\D*)(\d+)-(\d+)(\D*)') + port_pattern = re.compile(r'(\D*)(\d+)(\D*)') + port_match = port_pattern.match(port) + if port_match: + port_prefix = port_match.group(1) + port_number = int(port_match.group(2)) + port_suffix = port_match.group(3) + else: + port_prefix = '' + port_number = 0 + port_suffix = '' for port_filter in filters: - logging.debug('apply filter %s on port %s', port_filter, port) - denied = port_filter['filter_type'] != 'allow' + filter_type = port_filter.get('filter_type', 'allow') + denied = filter_type != 'allow' + unmatched_allowed = denied if 'ports' in port_filter: if port in port_filter['ports']: - logging.debug('port %s is allowed? %s', port, not denied) - return denied - port_prefix = port_filter.get('port_prefix', '') - port_suffix = port_filter.get('port_suffix', '') - pattern = re.compile(r'%s(\d+)%s' % (port_prefix, port_suffix)) - match = pattern.match(port) - if match: - logging.debug( - 'port %s matches pattern %s', - port, pattern.pattern - ) - port_number = match.group(1) - if ( - 'port_start' not in port_filter or - port_number >= port_filter['port_start'] - ) and ( - 'port_end' not in port_filter or - port_number <= port_filter['port_end'] - ): - logging.debug('port %s is allowed? %s', port, not denied) return denied + if port_match: + for port_or_ports in port_filter['ports']: + ports_match = ports_pattern.match(port_or_ports) + if ports_match: + filter_port_prefix = ports_match.group(1) + filter_port_start = int(ports_match.group(2)) + filter_port_end = int(ports_match.group(3)) + filter_port_suffix = ports_match.group(4) + if ( + filter_port_prefix == port_prefix and + filter_port_suffix == port_suffix and + filter_port_start <= port_number and + port_number <= filter_port_end + ): + return denied else: - logging.debug( - 'port %s does not match pattern %s', - port, pattern.pattern - ) - logging.debug('port %s is allowed', port) - return False + filter_port_prefix = port_filter.get('port_prefix', '') + filter_port_suffix = port_filter.get('port_suffix', '') + if ( + port_match and + port_prefix == filter_port_prefix and + port_suffix == filter_port_suffix + ): + if ( + 'port_start' not in port_filter or + port_number >= port_filter['port_start'] + ) and ( + 'port_end' not in port_filter or + port_number <= port_filter['port_end'] + ): + return denied + return not unmatched_allowed def to_dict(self): dict_info = self.machine.to_dict() @@ -1632,7 +1652,7 @@ class Switch(BASE, HelperMixin, TimestampMixin): 'repolling', 'error', 'under_monitoring', name='switch_state'), ColumnDefault('initialized')) - filters = Column(JSONEncoded, default=[]) + _filters = Column('filters', JSONEncoded, default=[]) switch_machines = relationship( SwitchMachine, passive_deletes=True, passive_updates=True, @@ -1640,6 +1660,117 @@ class Switch(BASE, HelperMixin, TimestampMixin): backref=backref('switch') ) + @classmethod + def parse_filters(cls, filters): + if isinstance(filters, basestring): + filters = filters.replace('\r\n', '\n').replace('\n', ';') + filters = [ + switch_filter for switch_filter in filters.split(';') + if switch_filter + ] + if not isinstance(filters, list): + filters = [filters] + switch_filters = [] + for switch_filter in filters: + if not switch_filter: + continue + if isinstance(switch_filter, basestring): + filter_dict = {} + filter_items = [ + item for item in switch_filter.split() if item + ] + if filter_items[0] in ['allow', 'deny']: + filter_dict['filter_type'] = filter_items[0] + filter_items = filter_items[1:] + elif filter_items[0] not in [ + 'ports', 'port_prefix', 'port_suffix', + 'port_start', 'port_end' + ]: + raise exception.InvalidParameter( + 'unrecognized filter type %s' % filter_items[0] + ) + while filter_items: + if len(filter_items) >= 2: + filter_dict[filter_items[0]] = filter_items[1] + filter_items = filter_items[2:] + else: + filter_dict[filter_items[0]] = '' + filter_items = filter_items[1:] + switch_filter = filter_dict + if not isinstance(switch_filter, dict): + raise exception.InvalidParameter( + 'filter %s is not dict' % switch_filter + ) + if 'filter_type' in switch_filter: + if switch_filter['filter_type'] not in ['allow', 'deny']: + raise exception.InvalidParameter( + 'filter_type should be `allow` or `deny` in %s' % ( + switch_filter + ) + ) + if 'ports' in switch_filter: + if isinstance(switch_filter['ports'], basestring): + switch_filter['ports'] = [ + port_or_ports + for port_or_ports in switch_filter['ports'].split(',') + if port_or_ports + ] + if not isinstance(switch_filter['ports'], list): + raise exception.InvalidParameter( + '`ports` type is not list in filter %s' % switch_filter + ) + for port_or_ports in switch_filter['ports']: + if not isinstance(port_or_ports, basestring): + raise exception.InvalidParameter( + '%s type is not basestring in `ports` %s' % ( + port_or_ports, switch_filter['ports'] + ) + ) + for key in ['port_start', 'port_end']: + if key in switch_filter: + if isinstance(switch_filter[key], basestring): + if switch_filter[key].isdigit(): + switch_filter[key] = int(switch_filter[key]) + if not isinstance(switch_filter[key], int): + raise exception.InvalidParameter( + '`%s` type is not int in filer %s' % ( + key, switch_filter + ) + ) + switch_filters.append(switch_filter) + return switch_filters + + @classmethod + def format_filters(cls, filters): + filter_strs = [] + for switch_filter in filters: + filter_properties = [] + filter_properties.append( + switch_filter.get('filter_type', 'allow') + ) + if 'ports' in switch_filter: + filter_properties.append( + 'ports ' + ','.join(switch_filter['ports']) + ) + if 'port_prefix' in switch_filter: + filter_properties.append( + 'port_prefix ' + switch_filter['port_prefix'] + ) + if 'port_suffix' in switch_filter: + filter_properties.append( + 'port_suffix ' + switch_filter['port_suffix'] + ) + if 'port_start' in switch_filter: + filter_properties.append( + 'port_start ' + str(switch_filter['port_start']) + ) + if 'port_end' in switch_filter: + filter_properties.append( + 'port_end ' + str(switch_filter['port_end']) + ) + filter_strs.append(' '.join(filter_properties)) + return ';'.join(filter_strs) + def __init__(self, ip_int, **kwargs): self.ip_int = ip_int super(Switch, self).__init__(**kwargs) @@ -1660,29 +1791,41 @@ class Switch(BASE, HelperMixin, TimestampMixin): def patched_credentials(self, value): self.credentials = util.merge_dict(dict(self.credentials), value) + @property + def filters(self): + return self._filters + + @filters.setter + def filters(self, value): + if not value: + return + self._filters = self.parse_filters(value) + + @property + def put_filters(self): + return self._filters + + @put_filters.setter + def put_filters(self, value): + if not value: + return + self._filters = self.parse_filters(value) + @property def patched_filters(self): - return self.filters + return self._filters @patched_filters.setter def patched_filters(self, value): if not value: return filters = list(self.filters) - for item in value: - found_filter = False - for switch_filter in filters: - if switch_filter['filter_name'] == item['filter_name']: - switch_filter.update(item) - found_filter = True - break - if not found_filter: - filters.append(item) - self.filters = filters + self.filters = self.parse_filters(value) + filters def to_dict(self): dict_info = super(Switch, self).to_dict() dict_info['ip'] = self.ip + dict_info['filters'] = self.format_filters(self._filters) return dict_info @@ -1904,7 +2047,9 @@ class AdapterFlavor(BASE, HelperMixin): name = Column(String(80), unique=True) display_name = Column(String(80)) template = Column(String(80)) - _ordered_flavor_roles = Column(JSONEncoded, default=[]) + _ordered_flavor_roles = Column( + 'ordered_flavor_roles', JSONEncoded, default=[] + ) flavor_roles = relationship( AdapterFlavorRole, diff --git a/compass/tests/db/api/test_switch.py b/compass/tests/db/api/test_switch.py index cbe325d0..db7a6b71 100644 --- a/compass/tests/db/api/test_switch.py +++ b/compass/tests/db/api/test_switch.py @@ -237,7 +237,6 @@ class TestUpdateSwitchFilters(BaseTest): 1, filters=[ { - 'filter_name': 'test', 'filter_type': 'allow' } ] @@ -247,7 +246,6 @@ class TestUpdateSwitchFilters(BaseTest): 1 ) expected = { - 'filter_name': 'test', 'filter_type': 'allow' } self.assertTrue( @@ -271,7 +269,6 @@ class TestPatchSwitchFilter(BaseTest): 1, patched_filters=[ { - 'filter_name': 'test', 'filter_type': 'allow' } ] @@ -281,7 +278,6 @@ class TestPatchSwitchFilter(BaseTest): 1 ) expected = { - 'filter_name': 'test', 'filter_type': 'allow' } self.assertTrue(