zuul/zuul-registry
Code Issues Proposed changes

Implement namespacing

Browse Source 
This allows the user to configure the registry to treat the first
component of the repository as a namespace.  Namespaces are
completely separate from each other -- they have their own upload
areas, and can have different content under the same repository
name.

In truth, we could omit this entirely and just use longer repository
names.  But treating it this way may have advantages later if we
implement more of the API (e.g., for listing repositories).  Or it
may allow us to colocate a buildset registry and another registry
on the same storage.  Or we may, in the future, choose a different
method for accessing the different namespaces (e.g., listening on
a different port).  Or it may be that we find some containers programs
don't handle long repository names (I haven't seen that yet, but I
haven't tested all of them).  At any rate, the design was there from
the start, and it's not that much code to implement it.

To use this as a buildset registry, you must first choose whether
to support docker or OCI tooling.  If using, docker, only docker.io
can be shadowed.  If using OCI, docker.io and all other registries
may be shadowed.

Docker:
* Do not enable the namespaced option.
* Add to /etc/docker/daemon.json:
  "registry-mirrors": ["https://localhost:9000"]
* Push and pull content into the registry as normal.

OCI:
* Enable the namespaced option.
* Add to /etc/containers/registries.conf:
  [[registry]]
  prefix = "docker.io"
  location = "docker.io"

  [[registry.mirror]]
  location = "localhost:9000/docker.io"

  [[registry.mirror]]
  location = "docker.io"
* Add similar entries for other registries
* When pushing an image into the registry that shadows docker.io,
  use a command like:
    skopeo copy containers-storage:docker.io/test/registry \
      docker://localhost:9000/docker.io/test/registry
* Then a "podman pull docker.io/test/registry" will pull the
  image as expected.

Change-Id: I88273a4a3e56971d2e2847fcd638d86d6bc491fc
This commit is contained in:
James E. Blair 2019-10-09 09:06:03 -07:00
parent 56968a9c80
commit a2dcc167a2
1 changed files with 35 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
zuul_registry/main.py
View File

@ -21,7 +21,6 @@ import logging
import cherrypy import cherrypy
import hashlib import hashlib
import json import json
import urllib
import yaml import yaml
from . import filesystem from . import filesystem
@ -140,30 +139,21 @@ class RegistryAPI:
https://docs.docker.com/registry/spec/api/ https://docs.docker.com/registry/spec/api/
""" """
log = logging.getLogger("registry.api") log = logging.getLogger("registry.api")
DEFAULT_NAMESPACE = '_local'
def __init__(self, store, authz): def __init__(self, store, namespaced, authz):
self.storage = store self.storage = store
self.authz = authz self.authz = authz
self.shadow = None self.namespaced = namespaced
def get_namespace(self): def get_namespace(self, repository):
if not self.shadow: if not self.namespaced:
return '_local' return (self.DEFAULT_NAMESPACE, repository)
return cherrypy.request.headers['Host'] parts = repository.split('/')
return (parts[0], '/'.join(parts[1:]))
def not_found(self): def not_found(self):
if not self.shadow:
raise cherrypy.HTTPError(404) raise cherrypy.HTTPError(404)
# TODO: Proxy the request (this is where we implement the
# buildset registry functionality).
host = cherrypy.request.headers['Host']
method = cherrypy.request.method
path = cherrypy.request.path_info
url = self.shadow.get(host)
if not url:
raise cherrypy.HTTPError(404)
url = urllib.parse.urljoin(url, path)
self.log.debug("Proxy request %s %s", method, url)
@cherrypy.expose @cherrypy.expose
@cherrypy.tools.json_out(content_type='application/json; charset=utf-8') @cherrypy.tools.json_out(content_type='application/json; charset=utf-8')
@ -175,8 +165,8 @@ class RegistryAPI:
@cherrypy.expose @cherrypy.expose
def head_blob(self, repository, digest): def head_blob(self, repository, digest):
namespace = self.get_namespace() namespace, repository = self.get_namespace(repository)
self.log.info('Head blob %s %s', repository, digest) self.log.info('Head blob %s %s %s', namespace, repository, digest)
size = self.storage.blob_size(namespace, digest) size = self.storage.blob_size(namespace, digest)
if size is None: if size is None:
return self.not_found() return self.not_found()
@ -188,8 +178,8 @@ class RegistryAPI:
@cherrypy.expose @cherrypy.expose
@cherrypy.config(**{'response.stream': True}) @cherrypy.config(**{'response.stream': True})
def get_blob(self, repository, digest): def get_blob(self, repository, digest):
namespace = self.get_namespace() namespace, repository = self.get_namespace(repository)
self.log.info('Get blob %s %s', repository, digest) self.log.info('Get blob %s %s %s', namespace, repository, digest)
size, data_iter = self.storage.stream_blob(namespace, digest) size, data_iter = self.storage.stream_blob(namespace, digest)
if data_iter is None: if data_iter is None:
return self.not_found() return self.not_found()
@ -202,14 +192,15 @@ class RegistryAPI:
@cherrypy.expose @cherrypy.expose
@cherrypy.config(**{'tools.check_auth.level': Authorization.WRITE}) @cherrypy.config(**{'tools.check_auth.level': Authorization.WRITE})
def start_upload(self, repository, digest=None): def start_upload(self, repository, digest=None):
namespace = self.get_namespace() orig_repository = repository
namespace, repository = self.get_namespace(repository)
method = cherrypy.request.method method = cherrypy.request.method
uuid = self.storage.start_upload(namespace) uuid = self.storage.start_upload(namespace)
self.log.info('Start upload %s %s uuid %s digest %s', self.log.info('Start upload %s %s %s uuid %s digest %s',
method, repository, uuid, digest) method, namespace, repository, uuid, digest)
res = cherrypy.response res = cherrypy.response
res.headers['Location'] = '/v2/%s/blobs/uploads/%s' % ( res.headers['Location'] = '/v2/%s/blobs/uploads/%s' % (
repository, uuid) orig_repository, uuid)
res.headers['Docker-Upload-UUID'] = uuid res.headers['Docker-Upload-UUID'] = uuid
res.headers['Range'] = '0-0' res.headers['Range'] = '0-0'
res.status = '202 Accepted' res.status = '202 Accepted'
@ -217,13 +208,14 @@ class RegistryAPI:
@cherrypy.expose @cherrypy.expose
@cherrypy.config(**{'tools.check_auth.level': Authorization.WRITE}) @cherrypy.config(**{'tools.check_auth.level': Authorization.WRITE})
def upload_chunk(self, repository, uuid): def upload_chunk(self, repository, uuid):
self.log.info('Upload chunk %s %s', repository, uuid) orig_repository = repository
namespace = self.get_namespace() namespace, repository = self.get_namespace(repository)
self.log.info('Upload chunk %s %s %s', namespace, repository, uuid)
old_length, new_length = self.storage.upload_chunk( old_length, new_length = self.storage.upload_chunk(
namespace, uuid, cherrypy.request.body) namespace, uuid, cherrypy.request.body)
res = cherrypy.response res = cherrypy.response
res.headers['Location'] = '/v2/%s/blobs/uploads/%s' % ( res.headers['Location'] = '/v2/%s/blobs/uploads/%s' % (
repository, uuid) orig_repository, uuid)
res.headers['Docker-Upload-UUID'] = uuid res.headers['Docker-Upload-UUID'] = uuid
res.headers['Range'] = '0-%s' % (new_length,) res.headers['Range'] = '0-%s' % (new_length,)
res.status = '204 No Content' res.status = '204 No Content'
@ -233,13 +225,14 @@ class RegistryAPI:
@cherrypy.expose @cherrypy.expose
@cherrypy.config(**{'tools.check_auth.level': Authorization.WRITE}) @cherrypy.config(**{'tools.check_auth.level': Authorization.WRITE})
def finish_upload(self, repository, uuid, digest): def finish_upload(self, repository, uuid, digest):
self.log.info('Finish upload %s %s', repository, uuid) orig_repository = repository
namespace = self.get_namespace() namespace, repository = self.get_namespace(repository)
self.log.info('Finish upload %s %s %s', namespace, repository, uuid)
old_length, new_length = self.storage.upload_chunk( old_length, new_length = self.storage.upload_chunk(
namespace, uuid, cherrypy.request.body) namespace, uuid, cherrypy.request.body)
self.storage.store_upload(namespace, uuid, digest) self.storage.store_upload(namespace, uuid, digest)
res = cherrypy.response res = cherrypy.response
res.headers['Location'] = '/v2/%s/blobs/%s' % (repository, digest) res.headers['Location'] = '/v2/%s/blobs/%s' % (orig_repository, digest)
res.headers['Docker-Content-Digest'] = digest res.headers['Docker-Content-Digest'] = digest
res.headers['Content-Range'] = '%s-%s' % (old_length, new_length) res.headers['Content-Range'] = '%s-%s' % (old_length, new_length)
res.status = '201 Created' res.status = '201 Created'
@ -247,12 +240,13 @@ class RegistryAPI:
@cherrypy.expose @cherrypy.expose
@cherrypy.config(**{'tools.check_auth.level': Authorization.WRITE}) @cherrypy.config(**{'tools.check_auth.level': Authorization.WRITE})
def put_manifest(self, repository, ref): def put_manifest(self, repository, ref):
namespace = self.get_namespace() namespace, repository = self.get_namespace(repository)
body = cherrypy.request.body.read() body = cherrypy.request.body.read()
hasher = hashlib.sha256() hasher = hashlib.sha256()
hasher.update(body) hasher.update(body)
digest = 'sha256:' + hasher.hexdigest() digest = 'sha256:' + hasher.hexdigest()
self.log.info('Put manifest %s %s digest %s', repository, ref, digest) self.log.info('Put manifest %s %s %s digest %s',
namespace, repository, ref, digest)
self.storage.put_blob(namespace, digest, body) self.storage.put_blob(namespace, digest, body)
manifest = self.storage.get_manifest(namespace, repository, ref) manifest = self.storage.get_manifest(namespace, repository, ref)
if manifest is None: if manifest is None:
@ -269,10 +263,10 @@ class RegistryAPI:
@cherrypy.expose @cherrypy.expose
def get_manifest(self, repository, ref): def get_manifest(self, repository, ref):
namespace = self.get_namespace() namespace, repository = self.get_namespace(repository)
headers = cherrypy.request.headers headers = cherrypy.request.headers
res = cherrypy.response res = cherrypy.response
self.log.info('Get manifest %s %s', repository, ref) self.log.info('Get manifest %s %s %s', namespace, repository, ref)
if ref.startswith('sha256:'): if ref.startswith('sha256:'):
manifest = self.storage.get_blob(namespace, ref) manifest = self.storage.get_blob(namespace, ref)
if manifest is None: if manifest is None:
@ -321,8 +315,11 @@ class RegistryServer:
self.conf['public-url']) self.conf['public-url'])
route_map = cherrypy.dispatch.RoutesDispatcher() route_map = cherrypy.dispatch.RoutesDispatcher()
api = RegistryAPI(self.store, authz) api = RegistryAPI(self.store,
self.conf.get('namespaced', False),
authz)
cherrypy.tools.check_auth = authz cherrypy.tools.check_auth = authz
route_map.connect('api', '/v2/', route_map.connect('api', '/v2/',
controller=api, action='version_check') controller=api, action='version_check')
route_map.connect('api', '/v2/{repository:.*}/blobs/uploads/', route_map.connect('api', '/v2/{repository:.*}/blobs/uploads/',