From ccd3ac23440970b849fe7f9f4f7239ad5d9ae22a Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 18 May 2020 12:51:59 +1000 Subject: [PATCH] Add tool to export Rackspace DNS domains to bind format This exports Rackspace DNS domains to bind format for backup and migration purposes. This installs a small tool to query and export all the domains we can see via the Racksapce DNS API. Because we don't want to publish the backups (it's the equivalent of a zone xfer) it is run on, and logs output to, bridge.openstack.org from cron once a day. Change-Id: I50fd33f5f3d6440a8f20d6fec63507cb883f2d56 --- playbooks/roles/rax-dns-backup/README.rst | 4 + .../roles/rax-dns-backup/files/rax-dns-backup | 240 ++++++++++++++++++ .../roles/rax-dns-backup/tasks/main.yaml | 38 +++ .../templates/rax-dns-auth.conf.j2 | 4 + playbooks/service-bridge.yaml | 5 + .../host_vars/bridge.openstack.org.yaml.j2 | 4 + testinfra/test_bridge.py | 11 + 7 files changed, 306 insertions(+) create mode 100644 playbooks/roles/rax-dns-backup/README.rst create mode 100755 playbooks/roles/rax-dns-backup/files/rax-dns-backup create mode 100644 playbooks/roles/rax-dns-backup/tasks/main.yaml create mode 100644 playbooks/roles/rax-dns-backup/templates/rax-dns-auth.conf.j2 diff --git a/playbooks/roles/rax-dns-backup/README.rst b/playbooks/roles/rax-dns-backup/README.rst new file mode 100644 index 0000000000..5344597dc9 --- /dev/null +++ b/playbooks/roles/rax-dns-backup/README.rst @@ -0,0 +1,4 @@ +Backup Rackspace managed DNS domain names + +Export a bind file for each of the domains used in the Rackspace +managed DNS-as-a-service. diff --git a/playbooks/roles/rax-dns-backup/files/rax-dns-backup b/playbooks/roles/rax-dns-backup/files/rax-dns-backup new file mode 100755 index 0000000000..3feb937278 --- /dev/null +++ b/playbooks/roles/rax-dns-backup/files/rax-dns-backup @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# +# Export domains for a given user/project +# +# Set auth values in the environment, or in a .ini file specified with +# --config: +# +# RACKSPACE_USERNAME = used to login to web +# RACKSPACE_PROJECT_ID = listed on account info +# RACKSPACE_API_KEY = listed in the account details page +# +# By default exports all domains, filter the list with --domains= +# + +import argparse +import configparser +import collections +import datetime +import glob +import logging +import os +import requests +import sys +import time + +RACKSPACE_IDENTITY_ENDPOINT='https://identity.api.rackspacecloud.com/v2.0/tokens' +RACKSPACE_DNS_ENDPOINT="https://dns.api.rackspacecloud.com/v1.0" + +RACKSPACE_PROJECT_ID=os.environ.get('RACKSPACE_PROJECT_ID', None) +RACKSPACE_USERNAME=os.environ.get('RACKSPACE_USERNAME', None) +RACKSPACE_API_KEY=os.environ.get('RACKSPACE_API_KEY', None) + +def get_auth_token(session): + # Get auth token + data = {'auth': + { + 'RAX-KSKEY:apiKeyCredentials': + { + 'username': RACKSPACE_USERNAME, + 'apiKey': RACKSPACE_API_KEY + } + } + } + token_response = session.post(url=RACKSPACE_IDENTITY_ENDPOINT, json=data) + token = token_response.json()['access']['token']['id'] + + return token + +def get_domain_list(session, token): + # List all domains + domain_list_url = "%s/%s/domains" % (RACKSPACE_DNS_ENDPOINT, + RACKSPACE_PROJECT_ID) + headers = { + 'Accept': 'application/json', + 'X-Auth-Token': token, + 'X-Project-Id': RACKSPACE_PROJECT_ID, + 'Content-Type': 'application/json' + } + domain_list_response = session.get(url=domain_list_url, headers=headers) + return domain_list_response.json()['domains'] + +def get_domain_id(session, token, domain): + # Find domain id + domain_url = "%s/%s/domains/search" % (RACKSPACE_DNS_ENDPOINT, + RACKSPACE_PROJECT_ID) + headers = { + 'Accept': 'application/json', + 'X-Auth-Token': token, + 'X-Project-Id': RACKSPACE_PROJECT_ID, + 'Content-Type': 'application/json' + } + + query = {'name': domain} + domain_response = session.get(url=domain_url, params=query, headers=headers) + domains = domain_response.json() + + for d in domains['domains']: + if d['name'] == domain: + return d + + logging.error("Did not find domain: %s" % domain) + sys.exit(1) + +def do_bind_export(session, token, domain_id, outfile): + # export to file + headers = { + 'Accept': 'application/json', + 'X-Auth-Token': token, + 'X-Project-Id': RACKSPACE_PROJECT_ID, + 'Content-Type': 'application/json' + } + + # Run export + export_url = '%s/%s/domains/%s/export' % (RACKSPACE_DNS_ENDPOINT, + RACKSPACE_PROJECT_ID, + domain_id) + + # We get a callback URL; we should loop around and correctly + # detect the completed status and timeout and whatnot. But we + # just sleep and that's enough. + export_response = session.get(url=export_url, headers=headers) + if export_response.status_code != 202: + logging.error("Didn't get export callback?") + sys.exit(1) + r = export_response.json() + callback_url = r['callbackUrl'] + time.sleep(2) + + query = {'showDetails': 'true'} + final_response = session.get(callback_url, params=query, headers=headers) + + bind_output = final_response.json()['response']['contents'] + + output = [] + for line in bind_output.split('\n'): + if line == '': + continue + fields = line.split(' ') + output.append(fields) + + # find padding space for the first column + max_first = max([len(x[0]) for x in output]) + + # create a dict keyed by domain with each record + out_dict = collections.defaultdict(list) + for domain in output: + out_dict[domain[0]].append(domain[1:]) + + outstr = '' + + # first output SOA then get rid of it + outstr += ("%-*s\t%s\n\n" % + (max_first+1, '@', '\t'.join(out_dict['@'][0]) )) + del(out_dict['@']) + + # print out the rest of the entries, with individual records + # sorted and grouped + for domain in sorted(out_dict): + records = out_dict[domain] + # sort records by type + records.sort(key=lambda x: x[1]) + for record in records: + outstr += ("%-*s\t%s\n" % (max_first+1, domain, '\t'.join(record) )) + outstr += '\n' + + with open(outfile, 'w') as f: + f.write(outstr) + + +def main(): + parser = argparse.ArgumentParser(description='Dump Rackspace DNS domains') + parser.add_argument('--domains', dest='domains', + help='Comma separated list of domains to export') + parser.add_argument('--output-dir', dest='output_dir', + default='/var/lib/rax-dns-backup') + parser.add_argument('--config', dest='config', + default='/etc/rax-dns-auth.conf') + parser.add_argument('--keep', dest='keep', type=int, default=30) + parser.add_argument('--debug', dest='debug', action='store_true') + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + if args.debug: + logging.getLogger().setLevel(logging.DEBUG) + requests_log = logging.getLogger("requests.packages.urllib3") + requests_log.setLevel(logging.DEBUG) + requests_log.propogate = True + + logging.debug("Starting") + + try: + logging.info("Reading config file %s" % args.config) + config = configparser.ConfigParser() + config.read(args.config) + global RACKSPACE_PROJECT_ID + global RACKSPACE_USERNAME + global RACKSPACE_API_KEY + RACKSPACE_PROJECT_ID = config['DEFAULT']['RACKSPACE_PROJECT_ID'] + RACKSPACE_USERNAME = config['DEFAULT']['RACKSPACE_USERNAME'] + RACKSPACE_API_KEY = config['DEFAULT']['RACKSPACE_API_KEY'] + except: + logging.info("Skipping config read") + + if (not RACKSPACE_PROJECT_ID) or \ + (not RACKSPACE_USERNAME) or \ + (not RACKSPACE_API_KEY): + logging.error("Must set auth variables!") + sys.exit(1) + + if not os.path.isdir(args.output_dir): + logging.error("Output directory does not exist") + sys.exit(1) + + session = requests.Session() + token = get_auth_token(session) + + if args.domains: + to_dump = [] + domains = args.domains.split(',') + for domain in domains: + logging.debug("Looking up domain: %s" % domain) + to_dump.append(get_domain_id(session, token, domain)) + else: + to_dump = get_domain_list(session, token) + + date_suffix = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.db") + + for domain in to_dump: + outfile = os.path.join( + args.output_dir, "%s_%s" % (domain['name'], date_suffix)) + logging.info("Dumping %s to %s" % (domain['name'], outfile)) + + do_bind_export(session, token, domain['id'], outfile) + + # cleanup old runs + old_files = glob.glob(os.path.join(args.output_dir, + '%s_*.db' % domain['name'])) + old_files.sort() + for f in old_files[:-args.keep]: + logging.info("Cleaning up old output: %s" % f) + os.unlink(f) + +if __name__ == "__main__": + main() + diff --git a/playbooks/roles/rax-dns-backup/tasks/main.yaml b/playbooks/roles/rax-dns-backup/tasks/main.yaml new file mode 100644 index 0000000000..37684ebdc6 --- /dev/null +++ b/playbooks/roles/rax-dns-backup/tasks/main.yaml @@ -0,0 +1,38 @@ +- name: Ensure configuration file + template: + src: rax-dns-auth.conf.j2 + dest: /etc/rax-dns-auth.conf + owner: root + group: root + mode: 0600 + +- name: Ensure output directory + file: + state: directory + path: /var/lib/rax-dns-backup + owner: root + group: root + mode: 0644 + +- name: Install backup tool + copy: + content: rax-dns-backup + dest: /usr/local/bin/rax-dns-backup + owner: root + group: root + mode: 0755 + +- name: Install cron job + cron: + name: 'Backup Rackspace DNS' + state: present + job: '/usr/local/bin/rax-dns-backup 2>&1 > /var/log/rax-dns-backup.log' + hour: '2' + minute: '0' + day: '*' + +- name: Install logrotate + include_role: + name: logrotate + vars: + logrotate_file_name: '/var/log/rax-dns-backup.log' diff --git a/playbooks/roles/rax-dns-backup/templates/rax-dns-auth.conf.j2 b/playbooks/roles/rax-dns-backup/templates/rax-dns-auth.conf.j2 new file mode 100644 index 0000000000..43458ce013 --- /dev/null +++ b/playbooks/roles/rax-dns-backup/templates/rax-dns-auth.conf.j2 @@ -0,0 +1,4 @@ +[DEFAULT] +RACKSPACE_USERNAME={{ rackspace_dns_username }} +RACKSPACE_PROJECT_ID={{ rackspace_dns_project_id }} +RACKSPACE_API_KEY={{ rackspace_dns_api_key }} diff --git a/playbooks/service-bridge.yaml b/playbooks/service-bridge.yaml index 74b746dde9..72e0c694b1 100644 --- a/playbooks/service-bridge.yaml +++ b/playbooks/service-bridge.yaml @@ -23,7 +23,12 @@ name: configure-openstacksdk vars: openstacksdk_config_template: clouds/bridge_all_clouds.yaml.j2 + - name: Get rid of all-clouds.yaml file: state: absent path: '/etc/openstack/all-clouds.yaml' + + - name: Install rackspace DNS backup tool + include_role: + name: rax-dns-backup diff --git a/playbooks/zuul/templates/host_vars/bridge.openstack.org.yaml.j2 b/playbooks/zuul/templates/host_vars/bridge.openstack.org.yaml.j2 index 18c272ec5c..61ec3c3597 100644 --- a/playbooks/zuul/templates/host_vars/bridge.openstack.org.yaml.j2 +++ b/playbooks/zuul/templates/host_vars/bridge.openstack.org.yaml.j2 @@ -67,3 +67,7 @@ gitea_kube_key: Z2l0ZWFfazhzX2tleQ== ansible_cron_disable_job: true cloud_launcher_disable_job: true extra_users: [] + +rackspace_dns_username: user +rackspace_dns_project_id: 1234 +rackspace_dns_api_key: apikey diff --git a/testinfra/test_bridge.py b/testinfra/test_bridge.py index 61fff1dc8a..f72326edb2 100644 --- a/testinfra/test_bridge.py +++ b/testinfra/test_bridge.py @@ -91,3 +91,14 @@ def test_zuul_authorized_keys(host): assert len(keys) >= 2 for key in keys: assert 'ssh-rsa' in key + + +def test_rax_dns_backup(host): + config_file = host.file('/etc/rax-dns-auth.conf') + assert config_file.exists + + tool_file = host.file('/usr/local/bin/rax-dns-backup') + assert tool_file.exists + + output_dir = host.file('/var/lib/rax-dns-backup') + assert output_dir.exists