Add keystone util functions

Change-Id: Ide5756565268aa934e305d2ecb76e49449bd85e5

distil/collector/ceilometer.py

@@ -14,12 +14,10 @@
 
 from oslo_config import cfg
 from oslo_log import log as logging
-from ceilometerclient import client
-from keystoneclient.auth.identity import v3
-from keystoneclient import session
 
 from distil.collector import base
 from distil import constants
+from distil.utils import keystone
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
@@ -29,21 +27,7 @@ class CeilometerCollector(base.BaseCollector):
     def __init__(self, *arg, **kwargs):
         super(CeilometerCollector, self).__init__(*arg, **kwargs)
 
-        auth = v3.Password(
+        self.cclient = keystone.get_ceilometer_client()
-            auth_url=CONF.keystone_authtoken.auth_url,
-            username=CONF.keystone_authtoken.username,
-            password=CONF.keystone_authtoken.password,
-            project_name=CONF.keystone_authtoken.project_name,
-            user_domain_name=CONF.keystone_authtoken.user_domain_name,
-            project_domain_name=CONF.keystone_authtoken.project_domain_name
-        )
-        sess = session.Session(auth=auth, verify=False)
-
-        self.cclient = client.get_client(
-            '2',
-            session=sess,
-            region_name=CONF.keystone_authtoken.region_name
-        )
 
     def get_meter(self, project_id, meter, start, end):
         """Get samples of a particular meter.

distil/service/collector.py

@@ -21,17 +21,38 @@ from oslo_service import threadgroup
 from stevedore import driver
 
 from distil.db import api as db_api
+from distil import exceptions
+from distil.utils import keystone
 
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 
 
+def filter_projects(projects):
+    p_filtered = list()
+
+    if CONF.collector.include_tenants:
+        p_filtered = [p for p in projects if
+                      p['name'] in CONF.collector.include_tenants]
+    elif CONF.collector.ignore_tenants:
+        p_filtered = [p for p in projects if
+                      p['name'] not in CONF.collector.ignore_tenants]
+    else:
+        p_filtered = projects
+
+    LOG.info("After filtering, %s project(s) left." % len(p_filtered))
+
+    return p_filtered
+
+
 class CollectorService(service.Service):
     def __init__(self):
         super(CollectorService, self).__init__()
 
         self.thread_grp = None
 
+        self.validate_config()
+
         collector_args = {}
         self.collector = driver.DriverManager(
             'distil.collector',
@@ -39,6 +60,16 @@ class CollectorService(service.Service):
             invoke_on_load=True,
             invoke_kwds=collector_args).driver
 
+    def validate_config(self):
+        include_tenants = set(CONF.collector.include_tenants)
+        ignore_tenants = set(CONF.collector.ignore_tenants)
+
+        if include_tenants & ignore_tenants:
+            raise exceptions.InvalidConfig(
+                "Duplicate tenants config in include_tenants and "
+                "ignore_tenants."
+            )
+
     def start(self):
         LOG.info("Starting collector service...")
 
@@ -61,14 +92,10 @@ class CollectorService(service.Service):
         super(CollectorService, self).reset()
         logging.setup(CONF, 'distil-collector')
 
-    def _get_projects(self):
-        return [{'id': '35b17138-b364-4e6a-a131-8f3099c5be68', 'name': 'fake',
-                 'description': 'fake_project'}]
-
     def collect_usage(self):
         LOG.info("Begin to collect usage...")
 
-        projects = self._get_projects()
+        projects = filter_projects(keystone.get_projects())
         end = datetime.utcnow().replace(minute=0, second=0, microsecond=0)
 
         for project in projects:

distil/utils/keystone.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2014 Catalyst IT Ltd
+# Copyright 2016 Catalyst IT Ltd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -12,34 +12,49 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import requests
+from ceilometerclient import client as c_client
-import json
+from keystoneauth1.identity import v3
-import urllib
+from keystoneauth1 import session
+from keystoneclient.v3 import client as ks_client
+from oslo_config import cfg
 
-from keystoneclient.v2_0 import client as keystone_client
+CONF = cfg.CONF
+KS_SESSION = None
 
 
-class KeystoneClient(keystone_client.Client):
+def _get_keystone_session():
+    global KS_SESSION
 
-    def tenant_by_name(self, name):
+    if not KS_SESSION:
-        authenticator = self.auth_url
+        auth = v3.Password(
-        url = "%(url)s/tenants?%(query)s" % {
+            auth_url=CONF.keystone_authtoken.auth_url,
-            "url": authenticator,
+            username=CONF.keystone_authtoken.username,
-            "query": urllib.urlencode({"name": name})
+            password=CONF.keystone_authtoken.password,
-        }
+            project_name=CONF.keystone_authtoken.project_name,
-        r = requests.get(url, headers={
+            user_domain_name=CONF.keystone_authtoken.user_domain_name,
-            "X-Auth-Token": self.auth_token,
+            project_domain_name=CONF.keystone_authtoken.project_domain_name,
-            "Content-Type": "application/json"
+        )
-        })
+        KS_SESSION = session.Session(auth=auth, verify=False)
-        if r.ok:
-            data = json.loads(r.text)
-            assert data
-            return data
-        else:
-            if r.status_code == 404:
-                raise
 
-    def get_ceilometer_endpoint(self):
+    return KS_SESSION
-        endpoint = self.service_catalog.url_for(service_type="metering",
+
-                                                endpoint_type="adminURL")
+
-        return endpoint
+def get_keystone_client():
+    sess = _get_keystone_session()
+    return ks_client.Client(session=sess)
+
+
+def get_ceilometer_client():
+    sess = _get_keystone_session()
+
+    return c_client.get_client(
+        '2',
+        session=sess,
+        region_name=CONF.keystone_authtoken.region_name
+    )
+
+
+def get_projects():
+    keystone = get_keystone_client()
+
+    return [obj.to_dict() for obj in keystone.projects.list()]

requirements.txt

@@ -10,9 +10,10 @@ six>=1.9.0 # MIT
 odoorpc==0.4.2
 SQLAlchemy<1.1.0,>=1.0.10 # MIT
 keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0
+keystoneauth1>=2.1.0  # Apache-2.0
 
 python-cinderclient>=1.6.0 # Apache-2.0
-python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0
+python-keystoneclient>=1.7.0,!=1.8.0,!=2.1.0  # Apache-2.0
 python-manilaclient>=1.3.0 # Apache-2.0
 python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
 python-swiftclient>=2.2.0 # Apache-2.0