x/surveil
Code Issues Proposed changes

Add help to surveil-api command

Browse Source 
Change-Id: I2c3b94b8d36adb2a87a63f7746be3959e320583e
This commit is contained in:
Thibault Cohen 2015-07-27 17:04:38 -04:00 committed by Alexandre Viau
parent c3366e70fb
commit 39e5f67b4c
7 changed files with 243 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
doc/source/tutorials/developing_the_api.rst
View File

@ -30,12 +30,48 @@ Disabling permissions
Depending on what you are working on, it might be practical to disable permissions. This can be done by editing the ``policy.json`` file found at ``etc/surveil/policy.json``.
For example, you could modify the following line: ::
For example, you could modify the following lines: ::
"admin_required": "role:admin or is_admin:1",
"surveil_required": "role:surveil or rule:admin_required",
"surveil:admin": "rule:admin_required",
"surveil:authenticated": "rule:surveil_required",
by: ::
"surveil:admin": "rule:pass",
"admin_required": "@",
"surveil_required": "@",
"surveil:admin": "@",
"surveil:authenticated": "@",
This will modify permissions so that all API calls that require the ``admin`` rule now pass without any verification.
Developping the API without docker
----------------------------------
You can get development environment without docker
::
git clone https://review.openstack.org/stackforge/surveil
cd surveil
virtualenv env
source env/bin/activate
pip install -r requirements.txt
python setup.py develop
python setup.py install_data
surveil-api -p env/etc/surveil/config.py -a env/etc/surveil/api_paste.ini -c env/etc/surveil/surveil.cfg -r
Edit your config files
::
vim env/etc/surveil/config.py
vim env/etc/surveil/surveil.cfg
vim env/etc/surveil/policy.json
vim env/etc/surveil/api_paste.ini
Don't forget to start your databases (MongoDB and InfluxDB)

18
surveil/api/config.py → etc/surveil/config.py
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from six.moves import configparser
import pecan
from surveil.api import hooks
@ -22,20 +22,16 @@ server = {
'host': '0.0.0.0'
}
config = configparser.ConfigParser()
config.read("/etc/surveil/surveil.cfg")
surveil_api_config = {
"mongodb_uri": config.get("surveil", "mongodb_uri"),
"ws_arbiter_url": config.get("surveil", "ws_arbiter_url"),
"influxdb_uri": config.get("surveil", "influxdb_uri")
authentication = {
'config_file': 'policy.json',
}
app_hooks = [
hooks.DBHook(
surveil_api_config['mongodb_uri'],
surveil_api_config['ws_arbiter_url'],
surveil_api_config['influxdb_uri']
pecan.conf.surveil_api_config['mongodb_uri'],
pecan.conf.surveil_api_config['ws_arbiter_url'],
pecan.conf.surveil_api_config['influxdb_uri']
)
]

4
setup.cfg
View File

@ -8,10 +8,12 @@ description-file =
[files]
packages =
surveil
data_files =
etc = etc/*
[entry_points]
console_scripts =
surveil-api = surveil.cmd.api:main
surveil-api = surveil.cmd.api:SurveilCommandRunner.handle_command_line
surveil-init = surveil.cmd.init:main
surveil-pack-upload = surveil.cmd.pack_upload:main
surveil-os-discovery = surveil.cmd.os_discovery:main

128
surveil/api/app.py
View File

@ -12,24 +12,22 @@
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
from __future__ import unicode_literals
import os
import subprocess
import sys
import threading
import time
from paste import deploy
import pecan
from pecan import commands
from pecan import configuration
from six.moves import configparser
def get_config_filename():
abspath = os.path.abspath(__file__)
path = os.path.dirname(abspath)
filename = "config.py"
return os.path.join(path, filename)
def get_pecan_config():
# Set up the pecan configuration
return pecan.configuration.conf_from_file(get_config_filename())
global_pecan_config_file = None
def setup_app(pecan_config):
@ -44,22 +42,116 @@ def setup_app(pecan_config):
return app
def load_app():
return deploy.loadapp('config:/etc/surveil/api_paste.ini')
def app_factory(global_config, **local_conf):
return VersionSelectorApplication()
global global_pecan_config_file
return VersionSelectorApplication(global_pecan_config_file)
class VersionSelectorApplication(object):
def __init__(self):
pc = get_pecan_config()
def __init__(self, pecan_conf_file):
pc = pecan.configuration.conf_from_file(pecan_conf_file)
self.v1 = setup_app(pecan_config=pc)
self.v2 = setup_app(pecan_config=pc)
self.config = pc
def __call__(self, environ, start_response):
if environ['PATH_INFO'].startswith('/v1/'):
return self.v1(environ, start_response)
return self.v2(environ, start_response)
class SurveilCommand(commands.ServeCommand):
def run(self, args):
global global_pecan_config_file
global_pecan_config_file = args.pecan_config
super(commands.ServeCommand, self).run(args)
app = self.load_app()
self.args = args
self.serve(app, app.config)
def load_app(self):
config = configparser.ConfigParser()
config.read(self.args.config_file)
surveil_cfg = {"surveil_api_config":
dict([i for i in config.items("surveil")])}
configuration.set_config(surveil_cfg, overwrite=True)
configuration.set_config(self.args.pecan_config, overwrite=False)
app = deploy.loadapp('config:%s' % self.args.api_paste_config)
app.config = configuration._runtime_conf
return app
def create_subprocess(self):
self.server_process = subprocess.Popen(
[arg for arg in sys.argv if arg not in ['--reload', '-r']],
stdout=sys.stdout, stderr=sys.stderr
)
# TODO(future useless): delete this function when
# https://review.openstack.org/#/c/206213/
# will be released
def watch_and_spawn(self, conf):
import watchdog.events as events
import watchdog.observers as observers
print('Monitoring for changes...',
file=sys.stderr)
self.create_subprocess()
parent = self
class AggressiveEventHandler(events.FileSystemEventHandler):
def __init__(self):
self.wait = False
def should_reload(self, event):
for t in (
events.FileSystemMovedEvent,
events.FileModifiedEvent,
events.DirModifiedEvent
):
if isinstance(event, t):
return True
return False
def ignore_events_one_sec(self):
if not self.wait:
self.wait = True
t = threading.Thread(target=self.wait_one_sec)
t.start()
def wait_one_sec(self):
time.sleep(1)
self.wait = False
def on_modified(self, event):
if self.should_reload(event) and not self.wait:
print("Some source files have been modified",
file=sys.stderr)
print("Restarting server...",
file=sys.stderr)
parent.server_process.kill()
self.ignore_events_one_sec()
parent.create_subprocess()
paths = self.paths_to_monitor(conf)
event_handler = AggressiveEventHandler()
for path, recurse in paths:
observer = observers.Observer()
observer.schedule(
event_handler,
path=path,
recursive=recurse
)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass

24
surveil/api/rbac.py
View File

@ -17,18 +17,32 @@
"""Access Control Lists (ACL's) control access the API server."""
import os
from oslo_config import cfg
from oslo_policy import policy
import pecan
import surveil.api.app as app
_ENFORCER = None
policy_opts = [cfg.StrOpt('project', default='surveil')]
policy_opts = [
cfg.StrOpt('config_dir', default='/etc/surveil/'),
cfg.StrOpt('config_file', default='policy.json'),
cfg.StrOpt('project', default='surveil')
]
if app.global_pecan_config_file:
pecan_config_dir = os.path.dirname(app.global_pecan_config_file)
config_dir = cfg.StrOpt('config_dir', default=pecan_config_dir)
else:
config_dir = cfg.StrOpt('config_dir', default="")
if hasattr(pecan.conf, 'authentication'):
config_file_name = pecan.conf.authentication.get('config_file',
'policy.json')
config_file = cfg.StrOpt('config_file', default=config_file_name)
else:
config_file = cfg.StrOpt('config_file', default='policy.json')
policy_opts.append(config_dir)
policy_opts.append(config_file)
CONF = cfg.CONF

134
surveil/cmd/api.py
View File

@ -16,16 +16,14 @@
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import os
import subprocess
import sys
import threading
import time
from wsgiref import simple_server
from oslo_config import cfg
from pecan.commands import base
import surveil.api.app as app
from surveil.api import app
CONF = cfg.CONF
@ -40,100 +38,50 @@ OPTS = [
CONF.register_opts(OPTS)
class ServerManager:
class SurveilCommandRunner(base.CommandRunner):
def __init__(self):
self.config = {}
self.config_file = ""
self.server_process = None
self.should_run = True
super(SurveilCommandRunner, self).__init__()
self.parser = argparse.ArgumentParser(description='Surveil API server')
self.parser.add_argument('--reload', '-r', action='store_true',
help='Automatically reload as code changes')
self.parser.add_argument('--pecan_config', '-p',
default='/etc/surveil/config.py',
help='Pecan config file (config.py)')
self.parser.add_argument('--api_paste_config', '-a',
default='/etc/surveil/api_paste.ini',
help='API Paste config file (api_paste.ini)')
self.parser.add_argument('--config_file', '-c',
default='/etc/surveil/surveil.cfg',
help='Pecan config file (surveil.cfg)')
def run(self, pecan_config, config_file):
self.config = pecan_config
self.config_file = config_file
def run(self, args):
namespace = self.parser.parse_args(args)
# Get absolute paths
namespace.pecan_config = os.path.join(os.getcwd(),
namespace.pecan_config)
namespace.api_paste_config = os.path.join(os.getcwd(),
namespace.api_paste_config)
namespace.config_file = os.path.join(os.getcwd(),
namespace.config_file)
if '--reload' in sys.argv:
self.watch_and_spawn()
else:
self.start_server()
def create_subprocess(self):
self.server_process = subprocess.Popen(['surveil-api'])
def start_server(self):
pecan_app = app.load_app()
host, port = self.config.server.host, self.config.server.port
srv = simple_server.make_server(host, port, pecan_app)
srv.serve_forever()
def watch_and_spawn(self):
import watchdog.events as events
import watchdog.observers as observers
print('Monitoring for changes...',
# Check conf files exist
if not os.path.isfile(namespace.pecan_config):
print("Bad config file: %s" % namespace.pecan_config,
file=sys.stderr)
self.create_subprocess()
parent = self
class AggressiveEventHandler(events.FileSystemEventHandler):
def __init__(self):
self.wait = False
def should_reload(self, event):
for t in (
events.FileSystemMovedEvent,
events.FileModifiedEvent,
events.DirModifiedEvent
):
if isinstance(event, t):
return True
return False
def ignore_events_one_sec(self):
if not self.wait:
self.wait = True
t = threading.Thread(target=self.wait_one_sec)
t.start()
def wait_one_sec(self):
time.sleep(1)
self.wait = False
def on_modified(self, event):
if self.should_reload(event) and not self.wait:
print("Some source files have been modified",
sys.exit(1)
if not os.path.isfile(namespace.api_paste_config):
print("Bad config file: %s" % namespace.api_paste_config,
file=sys.stderr)
print("Restarting server...",
sys.exit(2)
if not os.path.isfile(namespace.config_file):
print("Bad config file: %s" % namespace.config_file,
file=sys.stderr)
parent.server_process.kill()
self.ignore_events_one_sec()
parent.create_subprocess()
sys.exit(1)
path = self.path_to_monitor()
app.SurveilCommand().run(namespace)
event_handler = AggressiveEventHandler()
observer = observers.Observer()
observer.schedule(
event_handler,
path=path,
recursive=True
)
observer.start()
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
def path_to_monitor(self):
module = __import__('surveil')
return os.path.dirname(module.__file__)
def main():
srv = ServerManager()
srv.run(app.get_pecan_config(), app.get_config_filename())
@classmethod
def handle_command_line(cls): # pragma: nocover
runner = SurveilCommandRunner()
runner.run(sys.argv[1:])

24
surveil/cmd/init.py
View File

@ -18,10 +18,12 @@ import optparse
import sys
import influxdb
import pecan
from pecan import configuration
import pymongo
from six.moves import configparser
import surveilclient.client as sc
from surveil.api import config
from surveil.cmd import pack_upload
@ -47,18 +49,34 @@ def main():
dest='packs',
help="Upload/Update configuration packs to MongoDB",
action='store_true')
parser.add_option('--pecan_config', '-P',
default='/etc/surveil/config.py',
dest='pecan_config',
help='Pecan config file (config.py)')
parser.add_option('--config_file', '-c',
default='/etc/surveil/surveil.cfg',
dest='config_file',
help='Pecan config file (surveil.cfg)')
opts, _ = parser.parse_args(sys.argv)
surveil_api_url = 'http://localhost:5311/v2'
surveil_auth_url = 'http://localhost:5311/v2/auth'
surveil_api_version = '2_0'
config = configparser.ConfigParser()
config.read(opts.config_file)
surveil_cfg = {"surveil_api_config":
dict([i for i in config.items("surveil")])}
configuration.set_config(surveil_cfg, overwrite=True)
configuration.set_config(opts.pecan_config, overwrite=False)
cli_surveil = sc.Client(surveil_api_url,
auth_url=surveil_auth_url,
version=surveil_api_version)
# Create a basic config in mongodb
mongo = pymongo.MongoClient(config.surveil_api_config['mongodb_uri'])
mongo = pymongo.MongoClient(pecan.conf.surveil_api_config['mongodb_uri'])
if opts.mongodb is True:
# Drop the current shinken config
@ -69,7 +87,7 @@ def main():
print("Pre-creating InfluxDB database...")
# Create the InfluxDB database
influx_client = influxdb.InfluxDBClient.from_DSN(
config.surveil_api_config['influxdb_uri']
pecan.conf.surveil_api_config['influxdb_uri']
)
databases = influx_client.get_list_database()