From bf54ed0b03846da8654f4ac7281d2e89df59bb89 Mon Sep 17 00:00:00 2001 From: Tobias Henkel Date: Wed, 10 Feb 2021 20:00:04 +0100 Subject: [PATCH] Initial import of zuul storage proxy Change-Id: I75f3910095b669e502127b3870dba9616da0b3c8 --- .zuul.yaml | 14 +++ Dockerfile | 13 +++ requirements.txt | 2 + setup.cfg | 8 ++ setup.py | 23 ++++ test-requirements.txt | 1 + tox.ini | 18 +++ zuul_storage_proxy/__init__.py | 207 +++++++++++++++++++++++++++++++++ 8 files changed, 286 insertions(+) create mode 100644 .zuul.yaml create mode 100644 Dockerfile create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 test-requirements.txt create mode 100644 tox.ini create mode 100644 zuul_storage_proxy/__init__.py diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..abd09f2 --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,14 @@ +- project: + check: + jobs: + - tox-linters: + vars: + tox_install_bindep: false + gate: + jobs: + - tox-linters: + vars: + tox_install_bindep: false + release: + jobs: + - zuul-release-python diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7a689dc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.6-slim + +RUN python -m venv /opt/swift-proxy + +RUN mkdir -p /opt/swift-proxy-src + +COPY . /opt/swift-proxy-src/ + +RUN PBR_VERSION=1.0.0 /opt/swift-proxy/bin/pip install /opt/swift-proxy-src + +EXPOSE 8000 + +CMD /opt/swift-proxy/bin/gunicorn -k eventlet -t 3600 --workers 10 swift_proxy:proxy diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ac8f473 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +gunicorn[eventlet] +openstacksdk diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3f8b672 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,8 @@ +[metadata] +name = zuul-storage-proxy +requires-python = >=3.6 +requires-dist = + setuptools + +[pbr] +warnerrors = True diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..034c1fa --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +setuptools.setup( + setup_requires=['pbr'], + version='1.0.0', + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..3930480 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +flake8 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..194f3f2 --- /dev/null +++ b/tox.ini @@ -0,0 +1,18 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = linters + +[testenv] +basepython = python3 + +install_command = pip install {opts} {packages} +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + +[testenv:linters] +commands = flake8 {posargs} + +[flake8] +show-source = True +exclude = .venv*,.tox,dist,doc,build,*.egg,external diff --git a/zuul_storage_proxy/__init__.py b/zuul_storage_proxy/__init__.py new file mode 100644 index 0000000..7dad075 --- /dev/null +++ b/zuul_storage_proxy/__init__.py @@ -0,0 +1,207 @@ +# Copyright 2020 BMW Group +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import openstack +import os + +NOT_FOUND = b''' + +404 Not Found + +

Not Found

+

The requested URL was not found on this server.

+ +''' + +METHOD_NOT_ALLOWED = b''' + +405 Method Not Allowed + +

Method Not Allowed

+

The requested method is not supported.

+ +''' + +FORBIDDEN = b''' + +403 Forbidden + +

Forbidden

+

Overwriting files is not allowed.

+ +''' + +BACKEND_CONNECTION = b''' + +503 + +

Backend problem

+

Backend connection failed.

+ +''' + + +def redirect_directory(start_response, path): + # We need to redirect with the last element plus a terminating '/' + last_element = os.path.basename(path.rstrip('/')) + redirect = last_element + '/' + + start_response( + '301 Moved Permanently', + [('Location', redirect)] + ) + return iter([b'']) + + +def data_generator(data): + chunk_size = 16384 + buf = data.read(chunk_size) + while buf: + yield buf + buf = data.read(chunk_size) + + +def handle_get(start_response, clouds, container, path): + failures = [] + for cloud in clouds: + with cloud.object_store.get( + '%s/%s' % (container, path), + stream=True) as response: + response_headers = {k: v for k, v in response.headers.items()} + + if response.status_code == 404: + failures.append((404, '')) + continue + elif response.status_code != 200: + failures.append((response.status_code, response.reason)) + continue + + data = b'Status code %s' % str(response.status_code).encode() + start_response( + "{} {}".format(response.status_code, response.reason), + [('Content-Length', str(len(data)))] + ) + yield data + return + + if response_headers['Content-Type'] == 'application/directory': + # We got a directory so redirect it to path/index.html + return redirect_directory(start_response, path) + + start_response( + "{} {}".format(response.status_code, response.reason), + response_headers.items()) + + # We want to forward the compressed data stream here so use the raw + # response stream. + while response.raw: + data = response.raw.read(16384) + yield data + if len(data) == 0: + break + return + + # When hitting a special failure in one cloud return this, + # otherwise return 404 + for status_code, reason in failures: + if status_code == 404: + # handle later + continue + + # We hit a special failure, report this + data = b'Status code %s' % str(status_code).encode() + start_response( + "{} {}".format(status_code, reason), + [('Content-Length', str(len(data)))] + ) + yield data + return + + # No special failure, return 404 + start_response( + '404 Not Found', [] + ) + yield NOT_FOUND + return + + +def swift_proxy(environ, start_response, clouds, container_prefix): + path = environ['PATH_INFO'].lstrip('/') + method = environ['REQUEST_METHOD'] + + # If the path ends with a / this is a directory and we want to deliver the + # index.html instead. + if path.endswith('/'): + path += 'index.html' + + components = path.split('/', 1) + + if not path: + start_response( + '404 Not Found', [] + ) + yield NOT_FOUND + return + + if len(components) < 2: + # no path inside tenant given, redirect to root index + return redirect_directory(start_response, path) + + tenant = components[0] + path = components[1] + container = '-'.join([container_prefix, tenant]) + + print('%s request %s/%s' % (method, container, path)) + try: + if method == 'GET': + for chunk in handle_get(start_response, clouds, container, path): + yield chunk + else: + start_response( + '405 Method Not Allowed', [] + ) + yield METHOD_NOT_ALLOWED + except Exception as e: + start_response( + '503 Backend connection failed', [] + ) + yield '{}\n'.format(e).encode() + + +class CloudCache(object): + + def __init__(self, app): + self.log = logging.getLogger('middleware') + self.app = app + + if 'CLOUD_NAME' in os.environ: + cloud_names = [os.environ['CLOUD_NAME']] + else: + cloud_names = os.environ['CLOUD_NAMES'].split(',') + + self.clouds = [] + for cloud_name in cloud_names: + self.log.warning('Using cloud %s', cloud_name) + self.clouds.append(openstack.connect(cloud=cloud_name)) + self.container_prefix = os.environ['CONTAINER_PREFIX'] + self.log.warning('Using container prefix %s', self.container_prefix) + + def __call__(self, environ, start_response): + for chunk in self.app(environ, start_response, self.clouds, + self.container_prefix): + yield chunk + + +proxy = CloudCache(swift_proxy)