Merge "Graphite container deployment"

inventory/service/group_vars/graphite_opendev.org

@@ -0,0 +1,14 @@
+iptables_extra_allowed_hosts:
+  - hostname: bridge.openstack.org
+    port: 8125
+    protocol: udp
+  - hostname: opendev.org
+    port: 8125
+    protocol: udp
+
+iptables_extra_allowed_groups:
+  - {'protocol': 'udp', 'port': '8125', 'group': 'firehose'}
+  - {'protocol': 'udp', 'port': '8125', 'group': 'mirror-update'}
+  - {'protocol': 'udp', 'port': '8125', 'group': 'logstash'}
+  - {'protocol': 'udp', 'port': '8125', 'group': 'nodepool'}
+  - {'protocol': 'udp', 'port': '8125', 'group': 'zuul'}

inventory/service/groups.yaml

@@ -65,6 +65,10 @@ groups:
     - grafana[0-9]*.opendev.org
   graphite:
     - graphite*.open*.org
+  # NOTE(ianw) : to be cleaned up once the half-puppet
+  # graphite01.opendev.org is gone.
+  graphite_opendev:
+    - graphite02.opendev.org
   health:
     - health[0-9]*.openstack.org
   jvb:
@@ -74,7 +78,7 @@ groups:
   letsencrypt:
     - etherpad[0-9]*.opendev.org
     - gitea[0-9]*.opendev.org
-    - graphite01.opendev.org
+    - graphite[0-9]*.opendev.org
     - grafana[0-9]*.opendev.org
     - insecure-ci-registry[0-9]*.opendev.org
     - meetpad[0-9]*.opendev.org
@@ -130,7 +134,10 @@ groups:
     - ethercalc[0-9]*.open*.org
     - firehose[0-9]*.open*.org
     - grafana[0-9]*.open*.org
-    - graphite*.open*.org
+    # TODO(ianw) : this is a weird one we half-converted and moved
+    # into opendev.org in the early days of opendev.  remove when
+    # graphite02 up.
+    - graphite01.opendev.org
     - health[0-9]*.openstack.org
     - kdc[0-9]*.open*.org
     - lists*.katacontainers.io
@@ -167,7 +174,7 @@ groups:
     - ethercalc[0-9]*.open*.org
     - firehose[0-9]*.open*.org
     - grafana[0-9]*.open*.org
-    - graphite[0-9]*.open*.org
+    - graphite01.opendev.org
     - health[0-9]*.openstack.org
     - kdc[0-9]*.open*.org
     - lists*.katacontainers.io

inventory/service/host_vars/graphite02.opendev.org.yaml

@@ -0,0 +1,4 @@
+letsencrypt_certs:
+  graphite02-main:
+    - graphite02.opendev.org
+    - graphite.opendev.org

playbooks/roles/graphite/README.rst

@@ -0,0 +1 @@
+Run Graphite

playbooks/roles/graphite/handlers/main.yaml

@@ -0,0 +1,4 @@
+- name: graphite Reload apache2
+  service:
+    name: apache2
+    state: reloaded

playbooks/roles/graphite/tasks/main.yaml

@@ -0,0 +1,50 @@
+- name: Ensure docker-compose directory exists
+  file:
+    state: directory
+    path: /etc/graphite-docker
+
+- name: Write settings file
+  template:
+    src: docker-compose.yaml.j2
+    dest: /etc/graphite-docker/docker-compose.yaml
+
+- name: Write nginx override config
+  template:
+    src: graphite-statsd.conf.j2
+    dest: /etc/graphite-docker/graphite-statsd.conf
+
+- name: Write statsd override config
+  template:
+    src: statsd.js.j2
+    dest: /etc/graphite-docker/statsd.js
+
+- name: Ensure storage directory exists
+  file:
+    state: directory
+    path: /opt/graphite/storage
+
+- name: Ensure log directory exists
+  file:
+    state: directory
+    path: /var/log/graphite
+
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/graphite-docker/
+
+- name: Run docker-compose up
+  shell:
+    cmd: docker-compose up -d
+    chdir: /etc/graphite-docker/
+
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f
+
+# This is handy to have on the host for checking stat ingestion
+- name: Install netcat
+  package:
+    name:
+      - netcat
+    state: present

playbooks/roles/graphite/templates/docker-compose.yaml.j2

@@ -0,0 +1,15 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  graphite:
+    restart: always
+    image: docker.io/graphiteapp/graphite-statsd
+    network_mode: host
+    volumes:
+      - /etc/graphite-docker/graphite-statsd.conf:/etc/nginx/sites-enabled/graphite-statsd.conf
+      - /etc/graphite-docker/statsd.js:/opt/statsd/config/udp.js
+      - /etc/letsencrypt-certs:/etc/letsencrypt-certs
+      - /opt/graphite/storage:/opt/graphite/storage
+      - /var/log/graphite:/var/log/

playbooks/roles/graphite/templates/graphite-statsd.conf.j2

@@ -0,0 +1,57 @@
+server {
+  listen 80 default_server;
+  listen [::]:80 default_server;
+  server_name _;
+
+  return 301 https://$host$request_uri;
+}
+
+server {
+  listen 443 ssl;
+  listen [::]:443 ssl;
+  server_name {{ inventory_hostname }};
+
+  ssl_certificate /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer;
+  ssl_certificate_key /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key;
+  root /opt/graphite/static;
+  index index.html;
+
+  location /nginx_status {
+    stub_status on;
+    access_log   off;
+    allow 127.0.0.1;
+    deny all;
+  }
+
+  # No remote login
+  location /admin {
+    allow 127.0.0.1;
+    deny all;
+  }
+
+  location /media {
+    # django admin static files
+    alias /usr/local/lib/python3.6/dist-packages/django/contrib/admin/media/;
+  }
+
+  location /admin/auth/admin {
+    alias /usr/local/lib/python3.6/dist-packages/django/contrib/admin/static/admin;
+  }
+
+  location /admin/auth/user/admin {
+    alias /usr/local/lib/python3.6/dist-packages/django/contrib/admin/static/admin;
+  }
+
+  location / {
+    proxy_pass http://127.0.0.1:8080;
+    proxy_set_header  Host      $http_host;
+    proxy_set_header  X-Real-IP $remote_addr;
+    proxy_set_header  X-Forwarded-For $proxy_add_x_forwarded_for;
+
+    add_header 'Access-Control-Allow-Origin' '*';
+    add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+    add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type';
+    add_header 'Access-Control-Allow-Credentials' 'true';
+  }
+
+}

playbooks/roles/graphite/templates/statsd.js.j2

@@ -0,0 +1,9 @@
+{
+    "graphiteHost": "127.0.0.1",
+    "graphitePort": 2003,
+    "port": 8125,
+    "flushInterval": 10000,
+    "servers": [
+	{ server: "./servers/udp", address: "::", port: 8125, address_ipv6: true }
+    ]
+}

playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml

@@ -18,8 +18,11 @@ results:
     - puppet
     - puppet4
 
-  graphite.opendev.org:
+  graphite01.opendev.org:
+    - puppet
+    - puppet4
     - graphite
+    - letsencrypt
     - puppet
     - webservers
 

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -12,6 +12,9 @@
 # https://docs.ansible.com/ansible/latest/porting_guides/porting_guide_2.8.html#imports-as-handlers
 
 - name: letsencrypt updated graphite01-main
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_graphite.yaml
+
+- name: letsencrypt updated graphite02-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
 
 - name: letsencrypt updated tarballs-main

playbooks/roles/letsencrypt-create-certs/handlers/restart_graphite.yaml

@@ -0,0 +1,12 @@
+- name: Check for running nginx
+  command: pgrep -f nginx
+  ignore_errors: yes
+  register: nginx_pids
+
+- name: Restart graphite container
+  when: nginx_pids.rc == 0
+  block:
+    - name: Restart nginx
+      shell:
+        cmd: docker-compose restart graphite
+        chdir: /etc/graphite-docker

playbooks/service-graphite.yaml

@@ -0,0 +1,6 @@
+- hosts: "graphite_opendev:!disabled"
+  name: "Base: configure graphite"
+  roles:
+    - iptables
+    - install-docker
+    - graphite

testinfra/test_graphite.py

@@ -0,0 +1,58 @@
+# Copyright 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import json
+import logging
+import ssl
+import urllib.request
+
+testinfra_hosts = ['graphite02.opendev.org']
+
+
+def test_graphite_container_web_listening(host):
+    graphite_http = host.socket("tcp://127.0.0.1:80")
+    assert graphite_http.is_listening
+
+    graphite_https = host.socket("tcp://127.0.0.1:443")
+    assert graphite_https.is_listening
+
+def test_graphite(host):
+    cmd = host.run('curl --insecure '
+                   '--resolve graphite.opendev.org:443:127.0.0.1 '
+                   'https://graphite.opendev.org')
+    assert '<title>Graphite Browser</title>' in cmd.stdout
+
+def test_graphite_data(host):
+    # seed some data; send it over ipv6
+    cmd = ('timeout 20 bash -c '
+           '\'while true; do echo -n "example:$((RANDOM % 100))|c" '
+           '| nc -6 -w 1 -u localhost 8125; done\'')
+    host.run(cmd)
+
+    url='render?from=-10mins&until=now&target=stats.example&format=json'
+
+    # Assert we see some non-null values for this stat
+    # multi-node-hosts-file has setup graphite02.opendev.org to
+    # resolve from hosts.
+    found_value = False
+    with urllib.request.urlopen('https://graphite02.opendev.org/%s' % (url),
+                                context=ssl._create_unverified_context()) \
+                                as req:
+        data = json.loads(req.read().decode())
+        logging.debug('got: %s' % data)
+        datapoints = (data[0]['datapoints'])
+        for p in datapoints:
+            if p[0] != None:
+                found_value = True
+
+    assert found_value

zuul.d/infra-prod.yaml

@@ -526,6 +526,22 @@
       - playbooks/roles/logrotate
       - playbooks/roles/iptables/
 
+- job:
+    name: infra-prod-service-graphite
+    parent: infra-prod-service-base
+    description: Run service-graphite.yaml playbook.
+    vars:
+      playbook_name: service-graphite.yaml
+    files:
+      - inventory/
+      - playbooks/service-graphite.yaml
+      - inventory/service/host_vars/graphite02.opendev.org.yaml
+      - inventory/service/group_vars/graphite
+      - playbooks/roles/install-docker/
+      - playbooks/roles/pip3/
+      - playbooks/roles/graphite/
+      - playbooks/roles/iptables/
+
 # Run AFS changes separately so we can make sure to only do one at a time
 # (turns out quorum is nice to have)
 - job:

zuul.d/project.yaml

@@ -48,6 +48,7 @@
               - name: opendev-buildset-registry
               - name: system-config-build-image-grafana
                 soft: true
+        - system-config-run-graphite
         - system-config-run-review:
             dependencies:
               - name: opendev-buildset-registry
@@ -132,6 +133,7 @@
               - name: opendev-buildset-registry
               - name: system-config-upload-image-grafana
                 soft: true
+        - system-config-run-graphite
         - system-config-run-review:
             dependencies:
               - name: opendev-buildset-registry
@@ -227,6 +229,7 @@
                 soft: true
               - name: system-config-promote-image-grafana
                 soft: true
+        - infra-prod-service-graphite
         - infra-prod-service-meetpad
         - infra-prod-service-mirror-update
         - infra-prod-service-mirror

zuul.d/system-config-run.yaml

@@ -572,6 +572,39 @@
       - docker/grafana/
       - testinfra/test_grafana.py
 
+- job:
+    name: system-config-run-graphite
+    parent: system-config-run
+    description: |
+      Run the playbook for the graphite servers.
+    timeout: 3600
+    required-projects:
+      - opendev/system-config
+    nodeset:
+      nodes:
+        - name: bridge.openstack.org
+          label: ubuntu-bionic
+        # NOTE(ianw): 01 is a half-puppet opendev.org
+        # server
+        - name: graphite02.opendev.org
+          label: ubuntu-focal
+    vars:
+      run_playbooks:
+        - playbooks/letsencrypt.yaml
+        - playbooks/service-graphite.yaml
+    host-vars:
+      graphite02.opendev.org:
+        host_copy_output:
+          '/var/log/graphite': logs
+    files:
+      - playbooks/bridge.yaml
+      - playbooks/letsencrypt.yaml
+      - playbooks/service-graphite.yaml
+      - playbooks/roles/graphite
+      - playbooks/roles/install-docker/
+      - playbooks/roles/pip3/
+      - testinfra/test_graphite.py
+
 - job:
     name: system-config-run-meetpad
     parent: system-config-run-containers