zuul/zuul-storage-proxy
Code Issues Proposed changes

Initial import of zuul storage proxy

Browse Source 
Change-Id: I75f3910095b669e502127b3870dba9616da0b3c8
This commit is contained in:
Tobias Henkel 2021-02-10 20:00:04 +01:00
parent c6ec750c40
commit bf54ed0b03
No known key found for this signature in database
GPG Key ID: 03750DEC158E5FA2
8 changed files with 286 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.zuul.yaml Normal file
View File

@ -0,0 +1,14 @@
- project:
check:
jobs:
- tox-linters:
vars:
tox_install_bindep: false
gate:
jobs:
- tox-linters:
vars:
tox_install_bindep: false
release:
jobs:
- zuul-release-python

13
Dockerfile Normal file
View File

@ -0,0 +1,13 @@
FROM python:3.6-slim
RUN python -m venv /opt/swift-proxy
RUN mkdir -p /opt/swift-proxy-src
COPY . /opt/swift-proxy-src/
RUN PBR_VERSION=1.0.0 /opt/swift-proxy/bin/pip install /opt/swift-proxy-src
EXPOSE 8000
CMD /opt/swift-proxy/bin/gunicorn -k eventlet -t 3600 --workers 10 swift_proxy:proxy

2
requirements.txt Normal file
View File

@ -0,0 +1,2 @@
gunicorn[eventlet]
openstacksdk

8
setup.cfg Normal file
View File

@ -0,0 +1,8 @@
[metadata]
name = zuul-storage-proxy
requires-python = >=3.6
requires-dist =
setuptools
[pbr]
warnerrors = True

23
setup.py Normal file
View File

@ -0,0 +1,23 @@
#!/usr/bin/env python
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
setuptools.setup(
setup_requires=['pbr'],
version='1.0.0',
pbr=True)

1
test-requirements.txt Normal file
View File

@ -0,0 +1 @@
flake8

18
tox.ini Normal file
View File

@ -0,0 +1,18 @@
[tox]
minversion = 1.6
skipsdist = True
envlist = linters
[testenv]
basepython = python3
install_command = pip install {opts} {packages}
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
[testenv:linters]
commands = flake8 {posargs}
[flake8]
show-source = True
exclude = .venv*,.tox,dist,doc,build,*.egg,external

207
zuul_storage_proxy/__init__.py Normal file
View File

@ -0,0 +1,207 @@
# Copyright 2020 BMW Group
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import openstack
import os
NOT_FOUND = b'''<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>404 Not Found</title>
</head><body>
<h1>Not Found</h1>
<p>The requested URL was not found on this server.</p>
</body></html>
'''
METHOD_NOT_ALLOWED = b'''<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>405 Method Not Allowed</title>
</head><body>
<h1>Method Not Allowed</h1>
<p>The requested method is not supported.</p>
</body></html>
'''
FORBIDDEN = b'''<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>403 Forbidden</title>
</head><body>
<h1>Forbidden</h1>
<p>Overwriting files is not allowed.</p>
</body></html>
'''
BACKEND_CONNECTION = b'''<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html><head>
<title>503</title>
</head><body>
<h1>Backend problem</h1>
<p>Backend connection failed.</p>
</body></html>
'''
def redirect_directory(start_response, path):
# We need to redirect with the last element plus a terminating '/'
last_element = os.path.basename(path.rstrip('/'))
redirect = last_element + '/'
start_response(
'301 Moved Permanently',
[('Location', redirect)]
)
return iter([b''])
def data_generator(data):
chunk_size = 16384
buf = data.read(chunk_size)
while buf:
yield buf
buf = data.read(chunk_size)
def handle_get(start_response, clouds, container, path):
failures = []
for cloud in clouds:
with cloud.object_store.get(
'%s/%s' % (container, path),
stream=True) as response:
response_headers = {k: v for k, v in response.headers.items()}
if response.status_code == 404:
failures.append((404, ''))
continue
elif response.status_code != 200:
failures.append((response.status_code, response.reason))
continue
data = b'Status code %s' % str(response.status_code).encode()
start_response(
"{} {}".format(response.status_code, response.reason),
[('Content-Length', str(len(data)))]
)
yield data
return
if response_headers['Content-Type'] == 'application/directory':
# We got a directory so redirect it to path/index.html
return redirect_directory(start_response, path)
start_response(
"{} {}".format(response.status_code, response.reason),
response_headers.items())
# We want to forward the compressed data stream here so use the raw
# response stream.
while response.raw:
data = response.raw.read(16384)
yield data
if len(data) == 0:
break
return
# When hitting a special failure in one cloud return this,
# otherwise return 404
for status_code, reason in failures:
if status_code == 404:
# handle later
continue
# We hit a special failure, report this
data = b'Status code %s' % str(status_code).encode()
start_response(
"{} {}".format(status_code, reason),
[('Content-Length', str(len(data)))]
)
yield data
return
# No special failure, return 404
start_response(
'404 Not Found', []
)
yield NOT_FOUND
return
def swift_proxy(environ, start_response, clouds, container_prefix):
path = environ['PATH_INFO'].lstrip('/')
method = environ['REQUEST_METHOD']
# If the path ends with a / this is a directory and we want to deliver the
# index.html instead.
if path.endswith('/'):
path += 'index.html'
components = path.split('/', 1)
if not path:
start_response(
'404 Not Found', []
)
yield NOT_FOUND
return
if len(components) < 2:
# no path inside tenant given, redirect to root index
return redirect_directory(start_response, path)
tenant = components[0]
path = components[1]
container = '-'.join([container_prefix, tenant])
print('%s request %s/%s' % (method, container, path))
try:
if method == 'GET':
for chunk in handle_get(start_response, clouds, container, path):
yield chunk
else:
start_response(
'405 Method Not Allowed', []
)
yield METHOD_NOT_ALLOWED
except Exception as e:
start_response(
'503 Backend connection failed', []
)
yield '{}\n'.format(e).encode()
class CloudCache(object):
def __init__(self, app):
self.log = logging.getLogger('middleware')
self.app = app
if 'CLOUD_NAME' in os.environ:
cloud_names = [os.environ['CLOUD_NAME']]
else:
cloud_names = os.environ['CLOUD_NAMES'].split(',')
self.clouds = []
for cloud_name in cloud_names:
self.log.warning('Using cloud %s', cloud_name)
self.clouds.append(openstack.connect(cloud=cloud_name))
self.container_prefix = os.environ['CONTAINER_PREFIX']
self.log.warning('Using container prefix %s', self.container_prefix)
def __call__(self, environ, start_response):
for chunk in self.app(environ, start_response, self.clouds,
self.container_prefix):
yield chunk
proxy = CloudCache(swift_proxy)