From a602f0a77905e2d04b03055ab23cfeae6dd2e2f3 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Tue, 2 Jan 2024 11:17:59 -0800 Subject: [PATCH] Replace rehash with sha256 This is a cython implementation of sha256 where we can get and set the state for resumable hashing. The rehash library uses openssl but does not support openssl 3.x and is therefore becomming more difficult to use. Change-Id: I5ece8a7532421d76f5d9507131abd5f86c806158 --- bindep.txt | 12 ++++++-- requirements.txt | 6 +++- zuul_registry/hasher.py | 65 ++++++++++++++++++++++++++++++++++++++++ zuul_registry/storage.py | 15 ++++------ 4 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 zuul_registry/hasher.py diff --git a/bindep.txt b/bindep.txt index a4176d7..9e9a0c1 100644 --- a/bindep.txt +++ b/bindep.txt @@ -1,7 +1,11 @@ musl-dev [compile test platform:apk] make [compile test platform:apk] linux-headers [compile test platform:apk] -gcc [compile test] +# Git only needed for temporary sha256 git repo +git +# gcc normally only needed for compile time, but temporarily runtime +# for sha256 +gcc g++ [compile test platform:apk platform:dpkg] gcc-c++ [compile test platform:rpm] libssl-dev [compile test platform:dpkg] @@ -9,5 +13,7 @@ openssl-devel [compile test platform:rpm] libressl-dev [compile test platform:apk] libffi-dev [compile test platform:dpkg platform:apk] libffi-devel [compile test platform:rpm] -python3-dev [compile test platform:dpkg platform:apk] -python3-devel [compile test platform:rpm] +# python3-dev normally only needed for compile time, but temporarily runtime +# for sha256 +python3-dev [platform:dpkg platform:apk] +python3-devel [platform:rpm] diff --git a/requirements.txt b/requirements.txt index 37e8500..302ccea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,9 @@ routes requests openstacksdk python-dateutil -rehash +# Temporary until upstream releases support for py 3.11: +git+https://github.com/jeblair/sha256 +# cython and wheel temporary since for the temp sha256 repo install phase +cython +wheel pyjwt>=2.0.0,<3.0.0 diff --git a/zuul_registry/hasher.py b/zuul_registry/hasher.py new file mode 100644 index 0000000..f9acddd --- /dev/null +++ b/zuul_registry/hasher.py @@ -0,0 +1,65 @@ +# Copyright 2024 Acme Gating, LLC +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . + +import base64 + +import sha256 + + +class ResumableSha256: + def __init__(self): + self.hasher = sha256.sha256() + self.buffer = b'' + + def update(self, data): + if self.buffer is None: + raise Exception("Unable to update: hash is complete") + if self.buffer: + data = self.buffer + data + self.buffer = b'' + extra_len = len(data) % 64 + if extra_len: + self.buffer = data[-extra_len:] + data = data[:-extra_len] + self.hasher.update(data) + + def get_state(self): + hstate = self.hasher.state + return { + 'hash': base64.encodebytes(hstate[0]).decode('ascii'), + 'counter': hstate[1], + 'buffer': base64.encodebytes(self.buffer).decode('ascii'), + } + + def set_state(self, state): + hstate = ( + base64.decodebytes(state['hash'].encode('ascii')), + state['counter'], + ) + self.hasher.state = hstate + self.buffer = base64.decodebytes(state['buffer'].encode('ascii')) + + def finish(self): + if self.buffer: + self.hasher.update(self.buffer) + self.buffer = None + + def digest(self): + self.finish() + return self.hasher.digest() + + def hexdigest(self): + self.finish() + return self.hasher.hexdigest() diff --git a/zuul_registry/storage.py b/zuul_registry/storage.py index f944b35..1fef65b 100644 --- a/zuul_registry/storage.py +++ b/zuul_registry/storage.py @@ -13,17 +13,17 @@ # You should have received a copy of the GNU General Public License # along with this software. If not, see . -import base64 import json import logging import os import queue -import rehash import hashlib import threading import time from uuid import uuid4 +from . import hasher + class UploadRecord: """Information about an upload. @@ -49,7 +49,7 @@ class UploadRecord: def __init__(self): self.chunks = [] - self.hasher = rehash.sha256() + self.hasher = hasher.ResumableSha256() @property def count(self): @@ -66,15 +66,10 @@ class UploadRecord: def load(self, data): data = json.loads(data.decode('utf8')) self.chunks = data['chunks'] - hash_state = data['hash_state'] - hash_state['md_data'] = base64.decodebytes( - hash_state['md_data'].encode('ascii')) - self.hasher.__setstate__(hash_state) + self.hasher.set_state(data['hash_state']) def dump(self): - hash_state = self.hasher.__getstate__() - hash_state['md_data'] = base64.encodebytes( - hash_state['md_data']).decode('ascii') + hash_state = self.hasher.get_state() data = dict(chunks=self.chunks, hash_state=hash_state) return json.dumps(data).encode('utf8')