From 4a04383076bca84c23b0d963a4ca25b4a826c5b0 Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Tue, 8 Nov 2022 13:07:41 +0000 Subject: [PATCH] pypi-upload: support twine --skip-existing option Warehouse (the software implementing the PyPI service) expressly disallows reuploading any file which is already present, and returns an error if you attempt to do so. Under normal circumstances this is desirable feedback, but if you're using this role to upload a batch of files and encounter a network or service error partway through, you can be left in a position where it's impossible to rerun the job for those same artifacts later in order to correct the issue. Add a pypi_twine_skip_existing boolean toggle to the pypi-upload role, which will allow callers to indicate to twine that errors from PyPI indicating a file is already present are to be treated as a non-fatal condition so that it can proceed with uploading any which do exist. Set its default to false, preserving the existing behavior for the sake of backward compatibility. In addition to the previously stated use case, this also makes it possible to build different architecture-specific wheels in separate jobs without needing to worry about deciding which one will include the sdist, since they can all try to upload it safely. Change-Id: I66a8ffce47eb5e856c3b481c20841b92e060b532 --- roles/upload-pypi/README.rst | 5 +++++ roles/upload-pypi/defaults/main.yaml | 1 + roles/upload-pypi/tasks/main.yaml | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/roles/upload-pypi/README.rst b/roles/upload-pypi/README.rst index 6986adcdd..a3b2ba2a5 100644 --- a/roles/upload-pypi/README.rst +++ b/roles/upload-pypi/README.rst @@ -43,6 +43,11 @@ Upload python packages to PyPI Path to twine executable. +.. zuul:rolevar:: pypi_twine_skip_existing + :default: false + + Skip uploading any file which already exists, rather than failing. + .. zuul:rolevar:: pypi_register_first :default: false diff --git a/roles/upload-pypi/defaults/main.yaml b/roles/upload-pypi/defaults/main.yaml index 32c2c523e..cd780287d 100644 --- a/roles/upload-pypi/defaults/main.yaml +++ b/roles/upload-pypi/defaults/main.yaml @@ -3,4 +3,5 @@ pypi_path: "src/{{ zuul.project.canonical_name }}/dist" pypi_repository: "{{ pypi_info.repository | default('pypi') }}" pypi_repository_url: "{{ pypi_info.repository_url | default(None) }}" pypi_twine_executable: twine +pypi_twine_skip_existing: false pypi_register_first: false diff --git a/roles/upload-pypi/tasks/main.yaml b/roles/upload-pypi/tasks/main.yaml index 10e67f25a..3e0444c06 100644 --- a/roles/upload-pypi/tasks/main.yaml +++ b/roles/upload-pypi/tasks/main.yaml @@ -40,7 +40,7 @@ when: found_tarballs.files == [] - name: Upload wheels and sdist tarballs with twine - command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} -r {{ pypi_repository }} {{ found_wheels.files | map(attribute='path') | join(' ') }} {{ found_tarballs.files | map(attribute='path') | join(' ') }}" + command: "{{ pypi_twine_executable }} upload --config-file {{ _pypirc_tmp.path }} {% if pypi_twine_skip_existing %}--skip-existing{% endif %} -r {{ pypi_repository }} {{ found_wheels.files | map(attribute='path') | join(' ') }} {{ found_tarballs.files | map(attribute='path') | join(' ') }}" - name: Delete .pypirc configuration file file: