opendev/grafyaml
Code Issues Proposed changes

Add a slug to better track dashboards

Browse Source 
Currently grafana will generate a slug based on the title name. For
now, we can use an existing python library to do this.  Ideally, we'll
need to make sure the slugs generated between both are kept in sync.

How hard can that be?

Also move our validation logic into parser.py, since they are closely
related.

Change-Id: Id43f51d3f9db9131c25a8130f957572344a7d8f1
Signed-off-by: Paul Belanger <pabelanger@redhat.com>
This commit is contained in:
Paul Belanger 2015-09-21 16:41:21 -04:00
parent aa97ba048d
commit 42ef61dde0
17 changed files with 310 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
grafana_dashboards/builder.py
View File

@ -16,7 +16,6 @@ from oslo_config import cfg
from grafana_dashboards.grafana import Grafana
from grafana_dashboards.parser import YamlParser
from grafana_dashboards.schema.dashboard import Dashboard
grafana_opts = [
cfg.StrOpt(
@ -42,7 +41,8 @@ class Builder(object):
self.parser = YamlParser()
def update_dashboard(self, path):
data = self.parser.load(path)
schema = Dashboard()
result = schema.validate(data)
self.grafana.create_dashboard(result, overwrite=True)
self.parser.parse(path)
dashboards = self.parser.data.get('dashboard', {})
for item in dashboards:
data = self.parser.get_dashboard(item)
self.grafana.create_dashboard(data, overwrite=True)

7
grafana_dashboards/grafana.py
View File

@ -32,10 +32,13 @@ class Grafana(object):
})
def create_dashboard(self, data, overwrite=False):
data['overwrite'] = overwrite
dashboard = {
'dashboard': data,
'overwrite': overwrite,
}
headers = {
'Content-Type': 'application/json',
}
res = self.session.post(
self.url, data=json.dumps(data), headers=headers)
self.url, data=json.dumps(dashboard), headers=headers)
res.raise_for_status()

35
grafana_dashboards/parser.py
View File

@ -12,10 +12,41 @@
# License for the specific language governing permissions and limitations
# under the License.
import io
import yaml
from slugify import slugify
from grafana_dashboards.schema.dashboard import Dashboard
class YamlParser(object):
def load(self, path):
return yaml.safe_load(open(path))
def __init__(self):
self.data = {}
def get_dashboard(self, slug):
return self.data.get('dashboard', {}).get(slug, None)
def parse(self, fn):
with io.open(fn, 'r', encoding='utf-8') as fp:
self.parse_fp(fp)
def parse_fp(self, fp):
data = yaml.safe_load(fp)
result = self.validate(data)
for item in result.items():
group = self.data.get(item[0], {})
# Create slug to make it easier to find dashboards.
title = item[1]['title']
slug = slugify(title)
if slug in group:
raise Exception(
"Duplicate dashboard found in '{0}: '{1}' "
"already defined".format(fp.name, title))
group[slug] = item[1]
self.data[item[0]] = group
def validate(self, data):
schema = Dashboard()
return schema.validate(data)

1
requirements.txt
View File

@ -4,5 +4,6 @@
oslo.config>=1.11.0
oslo.log>=1.0.0,<1.1.0
python-slugify
PyYAML>=3.1.0
voluptuous>=0.7

8
tests/base.py
View File

@ -23,7 +23,6 @@ import re
import testtools
from grafana_dashboards.parser import YamlParser
from grafana_dashboards.schema import dashboard
def get_scenarios(fixtures_path, in_ext='yaml', out_ext='json'):
@ -50,7 +49,6 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='json'):
class TestCase(object):
"""Test case base class for all unit tests."""
parser = YamlParser()
def _read_raw_content(self):
# if None assume empty file
@ -62,11 +60,11 @@ class TestCase(object):
return content
def test_yaml_snippet(self):
parser = YamlParser()
expected_json = self._read_raw_content()
yaml_content = self.parser.load(self.in_filename)
parser.parse(self.in_filename)
valid_yaml = parser.data
schema = dashboard.Dashboard()
valid_yaml = schema.validate(yaml_content)
pretty_json = json.dumps(
valid_yaml, indent=4, separators=(',', ': '), sort_keys=True)

2
tests/fixtures/parser/dashboard-0001.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
dashboard:
title: New dashboard

2
tests/fixtures/parser/dashboard-0002.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
dashboard:
title: foobar

5
tests/fixtures/parser/dashboard-0003.yaml vendored Normal file
View File

@ -0,0 +1,5 @@
dashboard:
title: New dashboard
rows:
- title: New row
height: 250px

6
tests/schema/fixtures/dashboard-0001.json
View File

@ -1,6 +1,8 @@
{
"dashboard": {
"rows": [],
"title": "New dashboard"
"new-dashboard": {
"rows": [],
"title": "New dashboard"
}
}
}

24
tests/schema/fixtures/dashboard-0002.json
View File

@ -1,15 +1,17 @@
{
"dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
"new-dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
}
}
}

48
tests/schema/fixtures/dashboard-0003.json
View File

@ -1,27 +1,29 @@
{
"dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"editable": true,
"error": false,
"limit": 10,
"mode": "starred",
"query": "",
"span": 12,
"tag": "",
"title": "Starred Dashboards",
"type": "dashlist"
}
],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
"new-dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"editable": true,
"error": false,
"limit": 10,
"mode": "starred",
"query": "",
"span": 12,
"tag": "",
"title": "Starred Dashboards",
"type": "dashlist"
}
],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
}
}
}

44
tests/schema/fixtures/dashboard-0004.json
View File

@ -1,25 +1,27 @@
{
"dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"content": "Some example text is required.",
"editable": true,
"error": false,
"mode": "markdown",
"span": 12,
"title": "no title (click here)",
"type": "text"
}
],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
"new-dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"content": "Some example text is required.",
"editable": true,
"error": false,
"mode": "markdown",
"span": 12,
"title": "no title (click here)",
"type": "text"
}
],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
}
}
}

64
tests/schema/fixtures/dashboard-0005.json
View File

@ -1,35 +1,37 @@
{
"dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"bars": false,
"editable": true,
"error": false,
"fill": 1,
"lines": true,
"linewidth": 2,
"percentage": false,
"pointradius": 5,
"points": false,
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [],
"title": "no title (click here)",
"type": "graph",
"x-axis": true,
"y-axis": true
}
],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
"new-dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"bars": false,
"editable": true,
"error": false,
"fill": 1,
"lines": true,
"linewidth": 2,
"percentage": false,
"pointradius": 5,
"points": false,
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [],
"title": "no title (click here)",
"type": "graph",
"x-axis": true,
"y-axis": true
}
],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
}
}
}

64
tests/schema/fixtures/dashboard-0006.json
View File

@ -1,35 +1,37 @@
{
"dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"colorBackground": false,
"colorValue": false,
"editable": true,
"error": false,
"maxDataPoints": 100,
"span": 12,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"targets": [],
"thresholds": "",
"title": "no title (click here)",
"type": "singlestat",
"valueName": "avg"
}
],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
"new-dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"colorBackground": false,
"colorValue": false,
"editable": true,
"error": false,
"maxDataPoints": 100,
"span": 12,
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": false
},
"targets": [],
"thresholds": "",
"title": "no title (click here)",
"type": "singlestat",
"valueName": "avg"
}
],
"showTitle": false,
"title": "New row"
}
],
"title": "New dashboard"
}
}
}

84
tests/schema/fixtures/dashboard-0007.json
View File

@ -1,45 +1,47 @@
{
"dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"editable": true,
"error": false,
"limit": 10,
"mode": "starred",
"query": "",
"span": 12,
"tag": "",
"title": "Starred Dashboards",
"type": "dashlist"
}
],
"showTitle": false,
"title": "foo"
},
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"content": "Some example text is required.",
"editable": true,
"error": false,
"mode": "markdown",
"span": 12,
"title": "no title (click here)",
"type": "text"
}
],
"showTitle": false,
"title": "bar"
}
],
"title": "New dashboard"
"new-dashboard": {
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"editable": true,
"error": false,
"limit": 10,
"mode": "starred",
"query": "",
"span": 12,
"tag": "",
"title": "Starred Dashboards",
"type": "dashlist"
}
],
"showTitle": false,
"title": "foo"
},
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"content": "Some example text is required.",
"editable": true,
"error": false,
"mode": "markdown",
"span": 12,
"title": "no title (click here)",
"type": "text"
}
],
"showTitle": false,
"title": "bar"
}
],
"title": "New dashboard"
}
}
}

78
tests/test_parser.py Normal file
View File

@ -0,0 +1,78 @@
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from testtools import TestCase
from grafana_dashboards import parser
class TestCaseParser(TestCase):
def setUp(self):
super(TestCaseParser, self).setUp()
self.parser = parser.YamlParser()
def test_get_dashboard_empty(self):
self._get_empty_dashboard('foobar')
def test_parse_multiple(self):
path = os.path.join(
os.path.dirname(__file__), 'fixtures/parser/dashboard-0001.yaml')
self.parser.parse(path)
dashboard = {
'foobar': {'rows': [], 'title': 'foobar'},
'new-dashboard': {'rows': [], 'title': 'New dashboard'},
}
# Get parsed dashboard
res = self.parser.get_dashboard('new-dashboard')
self.assertEqual(res, dashboard['new-dashboard'])
# Check for a dashboard that does not exist
self._get_empty_dashboard('foobar')
# Parse another file to ensure we are appending data.
path = os.path.join(
os.path.dirname(__file__), 'fixtures/parser/dashboard-0002.yaml')
self.parser.parse(path)
res = self.parser.get_dashboard('foobar')
self.assertEqual(res, dashboard['foobar'])
# Ensure our first dashboard still exists.
res = self.parser.get_dashboard('new-dashboard')
self.assertEqual(res, dashboard['new-dashboard'])
def test_parse_duplicate(self):
path = os.path.join(
os.path.dirname(__file__), 'fixtures/parser/dashboard-0001.yaml')
self.parser.parse(path)
dashboard = {
'new-dashboard': {'rows': [], 'title': 'New dashboard'},
}
# Get parsed dashboard
res = self.parser.get_dashboard('new-dashboard')
self.assertEqual(res, dashboard['new-dashboard'])
path = os.path.join(
os.path.dirname(__file__), 'fixtures/parser/dashboard-0003.yaml')
# Fail to parse duplicate dashboard
self.assertRaises(Exception, self.parser.parse, path)
def _get_empty_dashboard(self, name):
res = self.parser.get_dashboard(name)
self.assertEqual(res, None)

3
tox.ini
View File

@ -35,8 +35,9 @@ commands = oslo_debug_helper {posargs}
[flake8]
# E123, E125 skipped as they are invalid PEP-8.
# H202 skip until we actually write our own exceptions
show-source = True
ignore = E123,E125
ignore = E123,E125,H202
builtins = _
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build