diff --git a/charts/development-pipeline/config_map.yaml.example b/charts/development-pipeline/config_map.yaml.example index 80a98b55..41f9c9a6 100644 --- a/charts/development-pipeline/config_map.yaml.example +++ b/charts/development-pipeline/config_map.yaml.example @@ -8,84 +8,87 @@ data: "use_existing_cluster": "true" } image.json: | - { - "build_from_source": false, - "image_name": "mongodb", - "project": "mongodb", - "repo": "mongodb", - "tag": "1.built", - "image_fullname": "cnf/mongodb:1.0", - "path": "tools/gate/jarvis/5G-SA-core/mongodb/images/mongodb", - "build": { - "repo": "docker.io/bitnami/mongodb", - "tag": "4.4.3-debian-10-r42" - } - } - image_build_from_source.json: | - { - "build_from_source": true, - "image_name": "mongodb", - "project": "mongodb", - "repo": "mongodb", - "tag": "1.built", - "image_fullname": "cnf/mongodb:1.0", - "path": "tools/gate/jarvis/5G-SA-core/mongodb/images/mongodb", - "build": { - "repo": "https://review.opendev.org/airship/charts", - "target_loc": "/src/checkout/mongodb", - "refspec": "master", - "version": "refs/changes/*:refs/changes/*" - } - } + [ + { + "build_from_source": false, + "image_name": "mongodb-sharded", + "project": "mongodb", + "repo": "mongodb-sharded", + "tag": "4.4.3", + "image_fullname": "mongodb/mongodb-sharded:4.4.3", + "path": "", + "build": { + "repo": "docker.io/bitnami/mongodb-sharded", + "tag": "4.4.3-debian-10-r44" + } + }, + { + "build_from_source": true, + "image_name": "mongodb", + "project": "mongodb", + "repo": "mongodb", + "tag": "1.built", + "image_fullname": "mongodb/mongodb:1.0", + "path": "tools/gate/jarvis/5G-SA-core/mongodb/images/mongodb", + "build": { + "repo": "https://review.opendev.org/airship/charts", + "target_loc": "/src/checkout/mongodb", + "refspec": "master", + "version": "refs/changes/*:refs/changes/*" + } + } + ] chart.json: | - { - "chart_name": "mongodb", - "project": "mongodb", - "repo": "mongodb", - "version": "10.6.2", - "build_from_source": false, - "path": "", - "build": { - "repo": "https://charts.bitnami.com/bitnami", - "target_loc": "/src/fetch/charts" + [ + { + "chart_name": "mongodb-sharded", + "project": "mongodb", + "repo": "mongodb-sharded", + "version": "3.3.3", + "build_from_source": false, + "path": "", + "build": { + "repo": "https://charts.bitnami.com/bitnami", + "local_repo": "bitnami_mongodb", + "target_loc": "/src/fetch/charts" + }, + "namespace": "development-pipeline", + "release_name": "mongodb-bitnami", + "images": { + "applications": { + "mongodb-sharded": { + "tag": "1.0", + "name": "mongodb", + "repo": "mongodb" + } + } + } }, - "namespace": "development-pipeline", - "release_name": "mongodb", - "images": { - "applications": { - "mongodb": { - "tag": "1.0", - "name": "mongodb", - "repo": "mongodb" + { + "chart_name": "mongodb", + "project": "mongodb", + "repo": "mongodb", + "version": "0.1.0", + "build_from_source": true, + "path": "tools/gate/jarvis/5G-SA-core/mongodb/charts", + "build": { + "repo": "https://review.opendev.org/airship/charts", + "target_loc": "/src/checkout/airship/charts", + "refspec": "master" + }, + "namespace": "development-pipeline", + "release_name": "mongodb", + "images": { + "applications": { + "mongodb": { + "tag": "1.0", + "name": "mongodb", + "repo": "mongodb" + } } } } - } - chart_build_from_source.json: | - { - "chart_name": "mongodb", - "project": "mongodb", - "repo": "mongodb", - "version": "0.1.0", - "build_from_source": true, - "path": "tools/gate/jarvis/5G-SA-core/mongodb/charts", - "build": { - "repo": "https://review.opendev.org/airship/charts", - "target_loc": "/src/checkout/airship/charts", - "refspec": "master" - }, - "namespace": "development-pipeline", - "release_name": "mongodb", - "images": { - "applications": { - "mongodb": { - "tag": "1.0", - "name": "mongodb", - "repo": "mongodb" - } - } - } - } + ] default.json: | { "proxy": { @@ -97,7 +100,10 @@ data: "chart_registry_url": "harbor-core.jarvis.local/chartrepo", "chart_repository": "jarvis-harbor", "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds" + "harbor_secret_mounted_path": "/workspace/helm-creds", + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { diff --git a/charts/development-pipeline/templates/task-chart.yaml b/charts/development-pipeline/templates/task-chart.yaml index e5e33a85..8bf8d3c8 100644 --- a/charts/development-pipeline/templates/task-chart.yaml +++ b/charts/development-pipeline/templates/task-chart.yaml @@ -14,7 +14,7 @@ spec: image: {{ $.Values.tasks.chart.buildChartImage }} script: | #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.git.gitPlaybook }} -i hosts -e '{"stage":"clone"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + ansible-playbook -vvv {{ $.Values.tasks.git.gitPlaybook }} -i hosts -e '{"stage":"clone"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - name: set-chart-output image: {{ $.Values.tasks.chart.buildChartImage }} @@ -26,7 +26,7 @@ spec: image: {{ $.Values.tasks.chart.buildChartImage }} script: | #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"lint"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"lint"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - name: set-chart-output-after-lint-dryrun image: {{ $.Values.tasks.chart.buildChartImage }} @@ -35,10 +35,11 @@ spec: cat "$(workspaces.development_pipeline_data.path)/chart.json" - name: package-chart + # Mount all secrets we want to use for external repos to fetch charts image: {{ $.Values.tasks.chart.buildChartImage }} script: | #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"package"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"package"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - name: set-chart-output-after-packaging image: {{ $.Values.tasks.chart.buildChartImage }} @@ -57,7 +58,7 @@ spec: script: | #!/usr/bin/env sh update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"publish"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"publish"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - name: set-chart-output-after-publish image: {{ $.Values.tasks.chart.buildChartImage }} diff --git a/charts/development-pipeline/templates/task-cleanup.yaml b/charts/development-pipeline/templates/task-cleanup.yaml index d8d470e5..50af8b40 100644 --- a/charts/development-pipeline/templates/task-cleanup.yaml +++ b/charts/development-pipeline/templates/task-cleanup.yaml @@ -19,10 +19,10 @@ spec: #!/bin/sh ansible-playbook -vvv "{{ $.Values.tasks.cleanup.cleanupPlaybook }}" -i hosts \ -e @"$(workspaces.development_pipeline_data.path)/default.json" \ - -e @"$(workspaces.development_pipeline_data.path)/chart.json" \ - -e @"$(workspaces.development_pipeline_data.path)/image.json" \ + -e 'loop_chart_source="$(workspaces.development_pipeline_data.path)/chart.json"' \ + -e 'loop_image_source="$(workspaces.development_pipeline_data.path)/image.json"' \ -e @"$(workspaces.development_pipeline_data.path)/cluster.json" volumes: - name: kubeconfig secret: - secretName: kubeconfig-secret \ No newline at end of file + secretName: kubeconfig-secret diff --git a/charts/development-pipeline/templates/task-functional.yaml b/charts/development-pipeline/templates/task-functional.yaml index 2edb3c3b..c3d5f6ca 100644 --- a/charts/development-pipeline/templates/task-functional.yaml +++ b/charts/development-pipeline/templates/task-functional.yaml @@ -23,13 +23,13 @@ spec: script: | #!/bin/sh update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts -e '{"stage":"deploy"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts -e '{"stage":"deploy"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - name: run-helm-tests image: {{ $.Values.tasks.functional.functionalTestImage }} script: | #!/bin/sh - ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts -e '{"stage":"test"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts -e '{"stage":"test"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' volumes: - name: helm-publish-creds secret: diff --git a/charts/development-pipeline/templates/task-image.yaml b/charts/development-pipeline/templates/task-image.yaml index c47c0bb1..bcec3ac5 100644 --- a/charts/development-pipeline/templates/task-image.yaml +++ b/charts/development-pipeline/templates/task-image.yaml @@ -17,7 +17,7 @@ spec: name: dind-certs script: | #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.git.gitPlaybook }} -i hosts -e '{"stage":"clone"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" + ansible-playbook -vvv {{ $.Values.tasks.git.gitPlaybook }} -i hosts -e '{"stage":"clone"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' - name: set-image-output image: {{ $.Values.tasks.image.buildImage }} @@ -42,7 +42,7 @@ spec: value: /certs/client script: | #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"build"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" + ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"build"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' - name: set-image-build-output-after-build image: {{ $.Values.tasks.image.buildImage }} @@ -70,7 +70,7 @@ spec: value: /certs/client script: | #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"push"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" + ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"push"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' - name: set-image-output-after-publish-scan image: {{ $.Values.tasks.image.buildImage }} @@ -85,7 +85,7 @@ spec: name: dind-certs script: | #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"scan_results"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" + ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"scan_results"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' - name: set-image-output-set-scan-results image: {{ $.Values.tasks.image.buildImage }} diff --git a/charts/development-pipeline/templates/task-promote.yaml b/charts/development-pipeline/templates/task-promote.yaml index 03c94001..182dcfc9 100644 --- a/charts/development-pipeline/templates/task-promote.yaml +++ b/charts/development-pipeline/templates/task-promote.yaml @@ -37,8 +37,8 @@ spec: #!/usr/bin/env sh set -ex update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.promote.promotePlaybook }} -i hosts -e '{"stage":"promote_image"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" - ansible-playbook -vvv {{ $.Values.tasks.promote.promotePlaybook }} -i hosts -e '{"stage":"promote_chart"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + ansible-playbook -vvv {{ $.Values.tasks.promote.promotePlaybook }} -i hosts -e '{"stage":"promote_image"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' + ansible-playbook -vvv {{ $.Values.tasks.promote.promotePlaybook }} -i hosts -e '{"stage":"promote_chart"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' sidecars: - image: {{ $.Values.tasks.image.sidecarServer }} name: server diff --git a/charts/development-pipeline/templates/task-setup-config.yaml b/charts/development-pipeline/templates/task-setup-config.yaml index 58b207be..91e4af67 100644 --- a/charts/development-pipeline/templates/task-setup-config.yaml +++ b/charts/development-pipeline/templates/task-setup-config.yaml @@ -35,12 +35,14 @@ spec: script: | #!/usr/bin/env sh cp "$(workspaces.k8s_cluster_data.path)/image.json" "$(workspaces.development_pipeline_data.path)/image.json" + jq 'if type=="array" then . else [.] end' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" echo "Set temporary image name to test/scan-image:$(context.taskRun.uid)" - jq '.image_fullname="test/scan-image:$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" + jq 'reduce range(0, length) as $d (.; (.[$d].image_fullname="test/scan-image:"+($d|tostring)+"$(context.taskRun.uid)"))' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" echo "Set tag to context $(context.taskRun.uid)" - jq '.tag="$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" + jq 'reduce range(0, length) as $d (.; (.[$d].tag=($d|tostring)+"$(context.taskRun.uid)"))' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" echo "Set target location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)" - jq '.build.target_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" + jq 'reduce range(0, length) as $d (.; (.[$d].build.target_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)/"+.[$d].build.repo+"/"+.[$d].build.refspec))' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" + cat "$(workspaces.development_pipeline_data.path)/image.json" --- apiVersion: tekton.dev/v1beta1 kind: Task @@ -59,11 +61,12 @@ spec: script: | #!/usr/bin/env sh cp "$(workspaces.k8s_cluster_data.path)/chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" + jq 'if type=="array" then . else [.] end' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" echo "Set tag to context $(context.taskRun.uid)" - jq '.tag="$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" - mkdir "$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)" + jq 'reduce range(0, length) as $d (.; (.[$d].tag="$(context.taskRun.uid)"))' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" echo "Set target location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)" - jq '.build.target_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" + jq 'reduce range(0, length) as $d (.; (.[$d].build.target_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)/"+.[$d].build.repo+"/"+.[$d].build.refspec))' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" + cat "$(workspaces.development_pipeline_data.path)/chart.json" --- apiVersion: tekton.dev/v1beta1 kind: Task @@ -81,4 +84,4 @@ spec: image: {{ $.Values.tasks.setup.setupConfigImage }} script: | #!/usr/bin/env sh - cp "$(workspaces.k8s_cluster_data.path)/cleanup.json" "$(workspaces.development_pipeline_data.path)/cleanup.json" \ No newline at end of file + cp "$(workspaces.k8s_cluster_data.path)/cleanup.json" "$(workspaces.development_pipeline_data.path)/cleanup.json" diff --git a/tools/images/standard-container/assets/playbooks/charts-microflow.yaml b/tools/images/standard-container/assets/playbooks/charts-microflow.yaml index bc3ce40a..d25f23d8 100644 --- a/tools/images/standard-container/assets/playbooks/charts-microflow.yaml +++ b/tools/images/standard-container/assets/playbooks/charts-microflow.yaml @@ -1,4 +1,32 @@ --- - hosts: localhost - roles: - - { role: charts, vars: { stage: "default" } } + tasks: + - name: Get list of charts + shell: "jq -c '.[]' {{ loop_source }}" + register: data + + - set_fact: + chart_async_batch_size: 20 + stage: "default" + + - name: Get list of remote repos + when: (stage == "package") + shell: "jq -c '.[] | select(.build_from_source==false) | {repo: .build.repo, secret: .build.secret, name: .build.local_repo}' {{ loop_source }} | sort | uniq" + register: external_repos + + - name: Get list of internal repos + when: (stage == "publish") + shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name: (.project+\"-staging\")}' {{ loop_source }} | sort | uniq" + register: internal_repos + + - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml + when: (stage == "package") or (stage == "publish") + vars: + repo: "{{ item }}" + loop: "{{ external_repos.stdout_lines if stage == 'package' else internal_repos.stdout_lines}}" + + - include_tasks: ./roles/charts/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(chart_async_batch_size) | list }}" + loop_control: + loop_var: "charts" diff --git a/tools/images/standard-container/assets/playbooks/cleanup.yaml b/tools/images/standard-container/assets/playbooks/cleanup.yaml index 718ceacd..913bb6dd 100644 --- a/tools/images/standard-container/assets/playbooks/cleanup.yaml +++ b/tools/images/standard-container/assets/playbooks/cleanup.yaml @@ -1,5 +1,16 @@ - hosts: localhost become: yes tasks: - - name: Clean up release - include_tasks: ./roles/cleanup/tasks/cleanup.yaml \ No newline at end of file + - name: Get list of charts + shell: "jq -c '.[]' {{ loop_chart_source }}" + register: data + + - set_fact: + chart_async_batch_size: 20 + stage: "default" + + - vars: + charts: "{{ item }}" + include_tasks: ./roles/cleanup/tasks/cleanup.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(chart_async_batch_size) | list }}" diff --git a/tools/images/standard-container/assets/playbooks/functional-microflow.yaml b/tools/images/standard-container/assets/playbooks/functional-microflow.yaml index 5f548258..34934f28 100644 --- a/tools/images/standard-container/assets/playbooks/functional-microflow.yaml +++ b/tools/images/standard-container/assets/playbooks/functional-microflow.yaml @@ -1,4 +1,27 @@ --- - hosts: localhost - roles: - - { role: functional, vars: { stage: "default" } } \ No newline at end of file + tasks: + - name: Get list of charts + shell: "jq -c '.[]' {{ loop_source }}" + register: data + + - set_fact: + chart_async_batch_size: 20 + stage: "default" + + - name: Get list of internal repos + when: (stage == 'deploy') + shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name: (.project+\"-staging\")}' {{ loop_source }} | sort | uniq" + register: internal_repos + + - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml + when: (stage == 'deploy') + loop: "{{ internal_repos.stdout_lines }}" + vars: + repo: "{{ item }}" + + - include_tasks: ./roles/functional/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(chart_async_batch_size) | list }}" + loop_control: + loop_var: "charts" diff --git a/tools/images/standard-container/assets/playbooks/git-microflow.yaml b/tools/images/standard-container/assets/playbooks/git-microflow.yaml index c47bcb8d..6ce1f224 100644 --- a/tools/images/standard-container/assets/playbooks/git-microflow.yaml +++ b/tools/images/standard-container/assets/playbooks/git-microflow.yaml @@ -1,4 +1,17 @@ --- - hosts: localhost - roles: - - { role: git, vars: { stage: "default" } } \ No newline at end of file + tasks: + - name: Get list of repos to clone + shell: "jq -c '.[] | select(.build_from_source) | {build: .build}' {{ loop_source }} | sort | uniq" + register: data + + - set_fact: + clone_async_batch_size: 20 + stage: "default" + + - name: Run git clone task + include_tasks: ./roles/git/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(clone_async_batch_size) | list }}" + loop_control: + loop_var: "repos" diff --git a/tools/images/standard-container/assets/playbooks/images-microflow.yaml b/tools/images/standard-container/assets/playbooks/images-microflow.yaml index 99cea768..6ab275e6 100644 --- a/tools/images/standard-container/assets/playbooks/images-microflow.yaml +++ b/tools/images/standard-container/assets/playbooks/images-microflow.yaml @@ -1,4 +1,16 @@ --- - hosts: localhost - roles: - - { role: images, vars: { stage: "default" } } \ No newline at end of file + tasks: + - name: Get list of images + shell: "jq -c '.[]' {{ loop_source }}" + register: data + + - set_fact: + image_async_batch_size: 2 + stage: "default" + + - include_tasks: ./roles/images/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(image_async_batch_size) | list }}" + loop_control: + loop_var: "images" diff --git a/tools/images/standard-container/assets/playbooks/promote-microflow.yaml b/tools/images/standard-container/assets/playbooks/promote-microflow.yaml index b4bd1659..b1f939e5 100644 --- a/tools/images/standard-container/assets/playbooks/promote-microflow.yaml +++ b/tools/images/standard-container/assets/playbooks/promote-microflow.yaml @@ -1,4 +1,32 @@ ---- -- hosts: localhost - roles: - - { role: promote, vars: { stage: "default" } } \ No newline at end of file +--- +- hosts: localhost + tasks: + - name: Get list of objects + shell: "jq -c '.[]' {{ loop_source }}" + register: data + + - set_fact: + async_batch_size: 2 + stage: "default" + + - name: Get list of internal staging repos + when: ( stage == "promote_chart") + shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name: (.project+\"-staging\")}' {{ loop_source }} | sort | uniq" + register: staging_internal_repos + + - name: Get list of internal stable repos + when: ( stage == "promote_chart") + shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project), secret: \"{{ harbor_secret_mounted_path }}\", name: .project}' {{ loop_source }} | sort | uniq" + register: internal_repos + + - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml + when: ( stage == "promote_chart") + loop: "{{ staging_internal_repos.stdout_lines + internal_repos.stdout_lines}}" + vars: + repo: "{{ item }}" + + - include_tasks: ./roles/promote/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(async_batch_size) | list }}" + loop_control: + loop_var: "items" diff --git a/tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml index 48bdd5fa..ec097920 100644 --- a/tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml +++ b/tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml @@ -1,60 +1,118 @@ +- set_fact: + _async: "{{ chart_async | default(600) }}" + _delay: "{{ chart_check_status_delay | default(5) }}" + - name: Helm Lint and Dry-Run when: - stage == "lint" - - build_from_source block: - - name: Helm lint "{{ chart_name }}" - command: "helm lint {{ chart_name }}" + - name: Helm lint + command: "helm lint {{ chart.chart_name }}" args: - chdir: "{{ build.target_loc }}/{{ path }}" + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', True) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: lint_async_results - - name: Helm Dry-run "{{ chart_name }}" - command: "helm install --dry-run {{ chart_name }} {{ chart_name }}" + - name: Wait for helm lint results + vars: + async_results: "{{ lint_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + + - name: Helm Dry-run + command: "helm install --dry-run {{ chart.chart_name }} {{ chart.chart_name }}" args: - chdir: "{{ build.target_loc }}/{{ path }}" + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', True) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: dry_run_async_results + + - name: Wait for helm dry-run results + vars: + async_results: "{{ dry_run_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml become: true - name: Helm Package when: - stage == "package" - - build_from_source block: - name: Package Helm Chart - shell: helm package "{{ chart_name }}" + shell: helm package "{{ chart.chart_name }}" args: - chdir: "{{ build.target_loc }}/{{ path }}" + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', True) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: package_async_results + + - name: Wait for helm package results + vars: + async_results: "{{ package_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml - name: Fetch Package when: - stage == "package" - - not build_from_source block: - - name: Add remote helm repo - shell: helm repo add remote_{{ chart_name }} {{ build.repo }} && helm repo update + - name: Create target location for fetching charts + file: + state: directory + path: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', False) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: create_target_folder_async_results - - name: Fetch {{ chart_name }}-{{ version }} - shell: helm fetch remote_{{ chart_name }}/{{ chart_name }} --version {{ version }} + - name: Wait for target location setup results + vars: + async_results: "{{ create_target_folder_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + + - name: Fetch Helm charts + shell: helm repo update && helm fetch {{ chart.build.local_repo }}/{{ chart.chart_name }} --version {{ chart.version }} args: - chdir: "{{ build.target_loc }}/{{ path }}" + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', False) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: fetch_async_results + + - name: Wait for helm fetch results + vars: + async_results: "{{ fetch_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + - name: Helm Publish when: ( stage == "publish") block: - name: Install Plugin shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true - - - name: Get harbor username - shell: cat {{ harbor_secret_mounted_path }}/username - register: harbor_username - - - name: Get harbor password - shell: cat {{ harbor_secret_mounted_path }}/password - register: harbor_password - - - name: Add Harbor Helm repository and Test repository - shell: helm repo add "{{ chart_repository }}-staging" "https://{{ docker_registry }}/chartrepo/{{ project }}-staging" --username={{ harbor_username.stdout }} --password={{ harbor_password.stdout }} - - - name: Push chart "{{ chart_name }}" to Harbor staging registry - command: helm push "{{ chart_name }}-{{ version }}".tgz "{{ chart_repository }}-staging" + - name: Push chart to Harbor staging registry + command: helm push "{{ chart.chart_name }}-{{ chart.version }}".tgz "{{ chart.project }}-staging" args: - chdir: "{{ build.target_loc }}/{{ path }}" + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: push_async_results + + - name: Wait for helm push results + vars: + async_results: "{{ push_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml diff --git a/tools/images/standard-container/assets/playbooks/roles/charts/tasks/setup_helm_repo.yaml b/tools/images/standard-container/assets/playbooks/roles/charts/tasks/setup_helm_repo.yaml new file mode 100644 index 00000000..f1d34c16 --- /dev/null +++ b/tools/images/standard-container/assets/playbooks/roles/charts/tasks/setup_helm_repo.yaml @@ -0,0 +1,17 @@ +- name: Get harbor username + when: repo.secret + shell: cat {{ repo.secret }}/username + register: repo_username + +- name: Get harbor password + when: repo.secret + shell: cat {{ repo.secret }}/password + register: repo_password + +- name: Add Harbor Helm repository and Test repository + when: repo.secret + shell: helm repo add "{{ repo.name }}" "{{ repo.repo }}" --username={{ repo_username.stdout }} --password={{ repo_password.stdout }} + +- name: Add Harbor Helm repository and Test repository + when: repo.secret is none + shell: helm repo add "{{ repo.name }}" "{{ repo.repo }}" diff --git a/tools/images/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml b/tools/images/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml index ca9a0680..ef49bab2 100644 --- a/tools/images/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml +++ b/tools/images/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml @@ -1,3 +1,18 @@ #Remove any resources deployed +- set_fact: + _async: "{{ chart_async | default(600) }}" + _delay: "{{ chart_check_status_delay | default(5) }}" + - name: Remove test release - shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ release_name }}" -n "{{ namespace }}" + shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" -n "{{ chart.namespace }}" + loop: "{{ charts }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: cleanup_async_results + +- name: Wait for helm delete results + vars: + async_results: "{{ cleanup_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml diff --git a/tools/images/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml b/tools/images/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml new file mode 100644 index 00000000..46d5d06a --- /dev/null +++ b/tools/images/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml @@ -0,0 +1,14 @@ +- set_fact: + _delay: "{{ _delay |default(5) }}" + _retries: "{{ _retries |default(30) }}" + +- name: Check sync status + async_status: + jid: "{{ async_result_item.ansible_job_id }}" + loop: "{{ async_results.results }}" + loop_control: + loop_var: "async_result_item" + register: async_poll_results + until: async_poll_results.finished + delay: "{{ _delay }}" + retries: "{{ _retries }}" diff --git a/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml index c06c6c2c..4bb86052 100644 --- a/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml +++ b/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml @@ -1,27 +1,30 @@ -- name: Functional Deploy - when: ("{{ stage }}" == "deploy") - block: - #Deploy CNF - - name: Get harbor username - shell: cat {{ harbor_secret_mounted_path }}/username - register: harbor_username - - - name: Get harbor password - shell: cat {{ harbor_secret_mounted_path }}/password - register: harbor_password - - #TODO dex-aio doesn't install, look into another test chart - - name: Add Harbor Helm repository and Test repository - shell: helm repo add "{{ chart_repository }}-staging" "https://{{ docker_registry }}/chartrepo/{{ project }}-staging" --username={{ harbor_username.stdout }} --password={{ harbor_password.stdout }} - - - name: Deploy chart "{{ chart_name }}" - #shell: echo "Deploy after mongodb" - shell: helm upgrade --install --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ release_name }}" "{{ chart_repository }}-staging/{{ chart_name }}" --version="{{ version }}" --namespace="{{ namespace }}" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" --create-namespace - become: true - -- name: Functional Test - when: ( stage == "test") - block: - #Test Deployed CNF - - name: Chart currently has no Helm Tests, echo for now - shell: echo "There are no helm tests yet" +- set_fact: + _async: "{{ chart_async | default(600) }}" + _delay: "{{ chart_check_status_delay | default(5) }}" + +- name: Functional Deploy + when: ("{{ stage }}" == "deploy") + block: + #Deploy CNF + + - name: Deploy charts + shell: helm upgrade --install --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" "{{ chart.project }}-staging/{{ chart.chart_name }}" --version="{{ chart.version }}" --namespace="{{ chart.namespace }}" + loop: "{{ charts }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: install_async_results + + - name: Wait for helm install results + vars: + async_results: "{{ install_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true + +- name: Functional Test + when: ( stage == "test") + block: + #Test Deployed CNF + - name: Chart currently has no Helm Tests, echo for now + shell: echo "There are no helm tests yet" diff --git a/tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml index e122a0d5..2af0aec1 100644 --- a/tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml +++ b/tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml @@ -1,11 +1,26 @@ #Build docker image using Makefile given git repository location to clone code from -- name: Clone repository +- set_fact: + _async: "{{ git_async | default(600) }}" + _delay: "{{ git_check_status_delay | default(3) }}" + +- name: Run async git clone + git: + repo: "{{ repo.build.repo }}" + dest: "{{ repo.build.target_loc }}" + version: "{{ repo.build.refspec }}" + refspec: "refs/changes/*:refs/changes/*" + async: "{{ _async }}" + poll: 0 + loop: "{{ repos }}" + loop_control: + loop_var: "repo" + register: git_async_results + when: + - stage == "clone" + +- name: Wait for git clone results + vars: + async_results: "{{ git_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml when: - stage == "clone" - - build_from_source - block: - - git: - repo: "{{ build.repo }}" - dest: "{{ build.target_loc }}" - version: "{{ build.refspec }}" - refspec: "refs/changes/*:refs/changes/*" diff --git a/tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml index a9fa20b3..c694d146 100644 --- a/tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml +++ b/tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml @@ -1,63 +1,109 @@ - -- name: Image Build - when: - - stage == "build" - - build_from_source - block: - #Build docker image using Makefile given git repository location to clone code from - - name: Build Docker Image for "{{ image_name }}" - shell: docker build -t "{{ image_fullname }}" . - args: - chdir: "{{ build.target_loc }}/{{ path }}" - become: true - -- name: Image Pull - when: - - stage == "build" - - not build_from_source - block: - #Build docker image using Makefile given git repository location to clone code from - - name: Pull Docker image "{{ build.repo }}:{{ build.tag }}" - docker_image: - pull: true - repository: "{{ image_fullname }}" - name: "{{ build.repo }}:{{ build.tag }}" - become: true - -- name: Tag and Push Image - when: ( stage == "push") - block: - - name: Tag and push to internal test repository for vulnerability scanning - docker_image: - push: true - name: "{{ image_fullname }}" - repository: "{{ docker_registry }}/{{ project }}-staging/{{ repo }}" - tag: "{{ tag }}" - -- name: Get Scan Results - when: ( stage == "scan_results") - block: - #Scan results may take some time, putting in some retries and a delay to determine if scan results get finished - - name: output the request - shell: echo "https://{{ docker_registry }}/api/v2.0/projects/{{ project }}-staging/repositories/{{ repo | replace('/','%2F') }}/artifacts/{{ tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" - - - name: Get Scan Results - uri: - validate_certs: false - url: "https://{{ docker_registry }}/api/v2.0/projects/{{ project }}-staging/repositories/{{ repo | replace('/','%2F') }}/artifacts/{{ tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" - method: GET - body_format: "json" - headers: - accept: "application/json" - X-Request-Id: "12345" - #Change to encoded from configmap - authorization: "Basic YWRtaW46SGFyYm9yMTIzNDU=" - register: result - until: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].scan_status == "Success" - retries: 5 - delay: 30 - - - name: Check Scan Results Summary for High and Critical CVE - set_fact: - image_status: "Vulnerable" - when: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].severity in ("High","Critical") +- set_fact: + _async: "{{ image_async | default(1200) }}" + _delay: "{{ image_check_status_delay | default(20) }}" + +- name: Image Build + when: + - stage == "build" + block: + #Build docker image using Makefile given git repository location to clone code from + - name: Build Docker Image + shell: docker build -t "{{ image.image_fullname }}" . + args: + chdir: "{{ image.build.target_loc }}/{{ image.path }}" + loop: "{{ images | selectattr('build_from_source', 'equalto', True) | list }}" + loop_control: + pause: 5 + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: build_async_results + + - name: Wait for Build Docker Image results + vars: + async_results: "{{ build_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true + +- name: Image Pull + when: + - stage == "build" + block: + #Build docker image using Makefile given git repository location to clone code from + - name: Pull Docker image + docker_image: + pull: true + repository: "{{ image.image_fullname }}" + name: "{{ image.build.repo }}:{{ image.build.tag }}" + loop: "{{ images | selectattr('build_from_source', 'equalto', False) | list }}" + loop_control: + pause: 5 + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: pull_async_results + + - name: Wait for Docker Pull Image results + vars: + async_results: "{{ pull_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true + +- name: Tag and Push Image + when: ( stage == "push") + block: + - name: Tag and push to internal test repository for vulnerability scanning + docker_image: + push: true + name: "{{ image.image_fullname }}" + repository: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}" + tag: "{{ image.tag }}" + loop: "{{ images }}" + loop_control: + pause: 5 + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: tag_async_results + + - name: Wait for Docker Tag Images results + vars: + async_results: "{{ tag_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + +- name: Get Scan Results + when: ( stage == "scan_results") + block: + #Scan results may take some time, putting in some retries and a delay to determine if scan results get finished + - name: output the request + shell: echo "https://{{ docker_registry }}/api/v2.0/projects/{{ image.project }}-staging/repositories/{{ image.repo | replace('/','%2F') }}/artifacts/{{ image.tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" + loop: "{{ images }}" + loop_control: + loop_var: "image" + + - name: Get Scan Results + uri: + validate_certs: false + url: "https://{{ docker_registry }}/api/v2.0/projects/{{ image.project }}-staging/repositories/{{ image.repo | replace('/','%2F') }}/artifacts/{{ image.tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" + method: GET + body_format: "json" + headers: + accept: "application/json" + X-Request-Id: "12345" + #Change to encoded from configmap + authorization: "Basic YWRtaW46SGFyYm9yMTIzNDU=" + loop: "{{ images }}" + loop_control: + loop_var: "image" + register: result + until: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].scan_status == "Success" + retries: 5 + delay: 30 + + - name: Check Scan Results Summary for High and Critical CVE + set_fact: + image_status: "Vulnerable" + when: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].severity in ("High","Critical") + loop: "{{ result.results }}" + loop_control: + loop_var: "result" diff --git a/tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml index 26c7865e..a0421c54 100644 --- a/tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml +++ b/tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml @@ -1,41 +1,81 @@ -- name: Promote Chart - when: ( stage == "promote_chart") - block: - - name: Install Plugin - shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true - - - name: Get harbor username - shell: cat {{ harbor_secret_mounted_path }}/username - register: harbor_username - - - name: Get harbor password - shell: cat {{ harbor_secret_mounted_path }}/password - register: harbor_password - - - name: Install Plugin - shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push - - - name: Add Harbor Helm repository and Test repository - shell: helm repo add "{{ chart_repository }}-staging" "https://{{ chart_registry_url }}/{{ project }}-staging" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" && helm repo add "{{ chart_repository }}" "https://{{ chart_registry_url }}/{{ project }}" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" - - - name: Pull down Helm Chart - shell: helm pull "{{ chart_repository }}-staging/{{ chart_name }}" --version="{{ version }}" - - - name: Push chart "{{ chart_name }}" to Helm registry - command: helm push "{{ chart_name }}-{{ version }}".tgz "{{ chart_repository }}" - become: true - -- name: Promote Image - when: ( stage == "promote_image") - block: - - name: Tag and push to promotion repository - docker_image: - pull: true - name: "{{ docker_registry }}/{{ project }}-staging/{{ repo }}:{{ tag }}" - - - name: Tag and push to promotion repository - docker_image: - push: true - name: "{{ docker_registry }}/{{ project }}-staging/{{ repo }}" - repository: "{{ docker_registry }}/{{ project }}/{{ repo }}" - tag: "{{ tag }}" +- set_fact: + _async: "{{ chart_async | default(600) }}" + _delay: "{{ chart_check_status_delay | default(5) }}" + when: ( stage == "promote_chart") + +- set_fact: + _async: "{{ image_async | default(1200) }}" + _delay: "{{ image_check_status_delay | default(20) }}" + when: ( stage == "promote_image") + +- name: Promote Chart + when: ( stage == "promote_chart") + block: + - name: Install Plugin + shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true + + - name: Pull down Helm Chart + shell: helm pull "{{ chart.project }}-staging/{{ chart.chart_name }}" --version="{{ chart.version }}" + loop: "{{ items }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: pull_async_results + + - name: Wait for helm pull results + vars: + async_results: "{{ pull_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + + - name: Push charts to Helm registry + command: helm push "{{ chart.chart_name }}-{{ chart.version }}".tgz "{{ chart.project }}" + loop: "{{ items }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: push_async_results + + - name: Wait for helm push results + vars: + async_results: "{{ push_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true + +- name: Promote Image + when: ( stage == "promote_image") + block: + - name: Pull image from staging repo + docker_image: + pull: true + name: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" + loop: "{{ items }}" + loop_control: + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: image_pull_async_results + + - name: Wait for docker pull results + vars: + async_results: "{{ image_pull_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + + - name: Tag and push to promotion repository + docker_image: + push: true + name: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}" + repository: "{{ docker_registry }}/{{ image.project }}/{{ image.repo }}" + tag: "{{ image.tag }}" + loop: "{{ items }}" + loop_control: + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: image_push_async_results + + - name: Wait for docker push results + vars: + async_results: "{{ image_push_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml