From beefa8dd30ecd5d7474a5d492a1c98d45b447f05 Mon Sep 17 00:00:00 2001 From: FLETCHER Date: Thu, 18 Feb 2021 15:48:19 -0600 Subject: [PATCH] feat(entrypoint) Create entrypoints This patchset moves all of the Tekton task script sections to use a defined entrypoint instead of an ansible-playbook command. This is a step to keeping all business logic out of the tekton tasks, i.e. the tekton tasks should work with any standard container given, not just one using Ansible. Change-Id: I5e106a8a75b79c0c2948cda638fbe532fd12fae3 --- charts/development-pipeline/.helmignore | 23 --- charts/development-pipeline/Chart.yaml | 6 - .../config_map.yaml.example | 125 ------------- .../pipelinerun-validation.yaml | 21 --- .../templates/pipeline.yaml | 128 ------------- .../development-pipeline/templates/role.yaml | 29 --- .../templates/rolebinding.yaml | 14 -- .../templates/serviceaccount.yaml | 7 - .../templates/task-chart.yaml | 74 -------- .../templates/task-cleanup.yaml | 28 --- .../templates/task-deployment-manifests.yaml | 16 -- .../templates/task-functional.yaml | 51 ------ .../templates/task-image.yaml | 127 ------------- .../templates/task-k8s-cluster.yaml | 23 --- .../templates/task-promote.yaml | 77 -------- .../templates/task-setup-config.yaml | 87 --------- charts/development-pipeline/values.yaml | 45 ----- .../templates/ClusterRole-el.yaml | 10 +- .../templates/Task-createProjectAccess.yaml | 13 +- tools/gate/jarvis/400-deploy-harbor.sh | 6 + tools/gate/jarvis/5G-SA-core/amf/.gitreview | 4 + .../templates/config_map.yaml | 98 +++++----- tools/gate/jarvis/5G-SA-core/ausf/.gitreview | 4 + .../templates/config_map.yaml | 98 +++++----- .../gate/jarvis/5G-SA-core/mongodb/.gitreview | 4 + .../templates/config_map.yaml | 94 +++++----- tools/gate/jarvis/5G-SA-core/nrf/.gitreview | 4 + .../templates/config_map.yaml | 98 +++++----- tools/gate/jarvis/5G-SA-core/pcf/.gitreview | 4 + .../templates/config_map.yaml | 98 +++++----- tools/gate/jarvis/5G-SA-core/smf/.gitreview | 4 + .../templates/config_map.yaml | 98 +++++----- tools/gate/jarvis/5G-SA-core/udm/.gitreview | 4 + .../templates/config_map.yaml | 98 +++++----- tools/gate/jarvis/5G-SA-core/udr/.gitreview | 4 + .../templates/config_map.yaml | 25 ++- tools/gate/jarvis/650-temporary-setup.sh | 3 +- .../gate/jarvis/800-deploy-jarvis-projects.sh | 103 ++++++----- .../pipelinerun-validation.yaml | 42 ++--- .../templates/pipeline.yaml | 17 +- .../development-pipeline/templates/role.yaml | 29 --- .../templates/rolebinding.yaml | 14 -- .../templates/serviceaccount.yaml | 7 - .../templates/task-chart.yaml | 24 +-- .../templates/task-cleanup.yaml | 9 +- .../templates/task-deployment-manifests.yaml | 2 +- .../templates/task-functional.yaml | 7 +- .../templates/task-image.yaml | 27 +-- .../templates/task-k8s-cluster.yaml | 4 +- .../templates/task-promote.yaml | 7 +- .../templates/task-setup-config.yaml | 26 +-- .../jarvis/development-pipeline/values.yaml | 22 +-- .../gate/jarvis/standard-container/Dockerfile | 6 +- .../assets/jarvis/build_docker_image.sh | 8 + .../assets/jarvis/cleanup.sh | 9 + .../assets/jarvis/deployment_manifests.sh | 6 + .../assets/jarvis/functional_deploy.sh | 10 ++ .../assets/jarvis/functional_test.sh | 9 + .../assets/jarvis/get_image_scan_results.sh | 8 + .../assets/jarvis/get_kubeconfig.sh | 8 + .../assets/jarvis/git_clone_chart_repo.sh | 10 ++ .../assets/jarvis/git_clone_image_repo.sh | 10 ++ .../assets/jarvis/lint_dryrun_chart.sh | 8 + .../assets/jarvis/package_chart.sh | 8 + .../assets/jarvis/promote_artifacts.sh | 13 ++ .../assets/jarvis/publish_chart.sh | 9 + .../assets/jarvis/push_docker_image.sh | 8 + .../assets/jarvis/setup_chart_config.sh | 13 ++ .../assets/jarvis/setup_cleanup_config.sh | 5 + .../assets/jarvis/setup_cluster_config.sh | 7 + .../assets/jarvis/setup_image_config.sh | 19 ++ .../assets/playbooks/charts-microflow.yaml | 36 +++- .../assets/playbooks/cleanup.yaml | 15 +- .../playbooks/functional-microflow.yaml | 31 +++- .../assets/playbooks/git-microflow.yaml | 17 +- .../assets/playbooks/images-microflow.yaml | 20 ++- .../assets/playbooks/promote-microflow.yaml | 36 +++- .../playbooks/roles/charts/tasks/main.yaml | 154 ++++++++++++---- .../roles/charts/tasks/setup_helm_repo.yaml | 0 .../roles/cleanup/tasks/cleanup.yaml | 17 +- .../roles/common/tasks/check_sync_status.yaml | 2 +- .../roles/functional/tasks/main.yaml | 52 +++--- .../playbooks/roles/git/tasks/main.yaml | 33 +++- .../playbooks/roles/images/tasks/main.yaml | 168 +++++++++++++----- .../kubernetes/tasks/get-kubeconfig.yaml | 7 +- .../playbooks/roles/promote/tasks/main.yaml | 111 +++++++++--- tools/images/standard-container/Dockerfile | 54 ------ .../standard-container/assets/entrypoint.sh | 3 - .../assets/playbooks/charts-microflow.yaml | 32 ---- .../assets/playbooks/cleanup.yaml | 16 -- .../playbooks/deployment-manifests.yaml | 5 - .../playbooks/functional-microflow.yaml | 27 --- .../assets/playbooks/get-kubeconfig.yaml | 5 - .../assets/playbooks/git-microflow.yaml | 17 -- .../standard-container/assets/playbooks/hosts | 1 - .../assets/playbooks/images-microflow.yaml | 16 -- .../assets/playbooks/promote-microflow.yaml | 32 ---- .../playbooks/roles/charts/tasks/main.yaml | 118 ------------ .../roles/cleanup/tasks/cleanup.yaml | 18 -- .../tasks/validate-config.yaml | 4 - .../functional/tasks/functional-test.yaml | 4 - .../roles/functional/tasks/main.yaml | 57 ------ .../playbooks/roles/git/tasks/main.yaml | 26 --- .../playbooks/roles/images/tasks/main.yaml | 109 ------------ .../kubernetes/tasks/get-kubeconfig.yaml | 10 -- .../playbooks/roles/promote/tasks/main.yaml | 81 --------- .../assets/setup-cluster-config.sh | 5 - .../assets/test_helm_repo_basic_auth.yaml | 9 - 108 files changed, 1305 insertions(+), 2209 deletions(-) delete mode 100644 charts/development-pipeline/.helmignore delete mode 100644 charts/development-pipeline/Chart.yaml delete mode 100644 charts/development-pipeline/config_map.yaml.example delete mode 100644 charts/development-pipeline/pipelinerun-validation.yaml delete mode 100644 charts/development-pipeline/templates/pipeline.yaml delete mode 100644 charts/development-pipeline/templates/role.yaml delete mode 100644 charts/development-pipeline/templates/rolebinding.yaml delete mode 100644 charts/development-pipeline/templates/serviceaccount.yaml delete mode 100644 charts/development-pipeline/templates/task-chart.yaml delete mode 100644 charts/development-pipeline/templates/task-cleanup.yaml delete mode 100644 charts/development-pipeline/templates/task-deployment-manifests.yaml delete mode 100644 charts/development-pipeline/templates/task-functional.yaml delete mode 100644 charts/development-pipeline/templates/task-image.yaml delete mode 100644 charts/development-pipeline/templates/task-k8s-cluster.yaml delete mode 100644 charts/development-pipeline/templates/task-promote.yaml delete mode 100644 charts/development-pipeline/templates/task-setup-config.yaml delete mode 100644 charts/development-pipeline/values.yaml create mode 100644 tools/gate/jarvis/5G-SA-core/amf/.gitreview create mode 100644 tools/gate/jarvis/5G-SA-core/ausf/.gitreview create mode 100644 tools/gate/jarvis/5G-SA-core/mongodb/.gitreview create mode 100644 tools/gate/jarvis/5G-SA-core/nrf/.gitreview create mode 100644 tools/gate/jarvis/5G-SA-core/pcf/.gitreview create mode 100644 tools/gate/jarvis/5G-SA-core/smf/.gitreview create mode 100644 tools/gate/jarvis/5G-SA-core/udm/.gitreview create mode 100644 tools/gate/jarvis/5G-SA-core/udr/.gitreview delete mode 100644 tools/gate/jarvis/development-pipeline/templates/role.yaml delete mode 100644 tools/gate/jarvis/development-pipeline/templates/rolebinding.yaml delete mode 100644 tools/gate/jarvis/development-pipeline/templates/serviceaccount.yaml create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/build_docker_image.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/cleanup.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/deployment_manifests.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/functional_deploy.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/functional_test.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/get_image_scan_results.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/get_kubeconfig.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/git_clone_chart_repo.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/git_clone_image_repo.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/lint_dryrun_chart.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/package_chart.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/promote_artifacts.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/publish_chart.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/push_docker_image.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/setup_chart_config.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/setup_cleanup_config.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/setup_cluster_config.sh create mode 100644 tools/gate/jarvis/standard-container/assets/jarvis/setup_image_config.sh rename tools/{images => gate/jarvis}/standard-container/assets/playbooks/roles/charts/tasks/setup_helm_repo.yaml (100%) rename tools/{images => gate/jarvis}/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml (88%) delete mode 100644 tools/images/standard-container/Dockerfile delete mode 100755 tools/images/standard-container/assets/entrypoint.sh delete mode 100644 tools/images/standard-container/assets/playbooks/charts-microflow.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/cleanup.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/deployment-manifests.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/functional-microflow.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/get-kubeconfig.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/git-microflow.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/hosts delete mode 100644 tools/images/standard-container/assets/playbooks/images-microflow.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/promote-microflow.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/roles/deployment-manifests/tasks/validate-config.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/roles/functional/tasks/functional-test.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/roles/kubernetes/tasks/get-kubeconfig.yaml delete mode 100644 tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml delete mode 100644 tools/images/standard-container/assets/setup-cluster-config.sh delete mode 100644 tools/images/standard-container/assets/test_helm_repo_basic_auth.yaml diff --git a/charts/development-pipeline/.helmignore b/charts/development-pipeline/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/charts/development-pipeline/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/charts/development-pipeline/Chart.yaml b/charts/development-pipeline/Chart.yaml deleted file mode 100644 index f2e87a90..00000000 --- a/charts/development-pipeline/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: development-pipeline -description: A Helm chart for Tekton Development pipeline -type: application -version: 0.1.0 -appVersion: 1.16.0 diff --git a/charts/development-pipeline/config_map.yaml.example b/charts/development-pipeline/config_map.yaml.example deleted file mode 100644 index 1f825819..00000000 --- a/charts/development-pipeline/config_map.yaml.example +++ /dev/null @@ -1,125 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: deployment-flow -data: - cluster.json: | - { - "use_existing_cluster": "true" - } - image.json: | - [ - { - "build_from_source": false, - "image_name": "mongodb-sharded", - "project": "mongodb", - "repo": "mongodb-sharded", - "tag": "4.4.3", - "image_fullname": "mongodb/mongodb-sharded:4.4.3", - "path": "", - "build": { - "repo": "docker.io/bitnami/mongodb-sharded", - "tag": "4.4.3-debian-10-r44" - } - }, - { - "build_from_source": true, - "image_name": "mongodb", - "project": "mongodb", - "repo": "mongodb", - "tag": "1.built", - "image_fullname": "mongodb/mongodb:1.0", - "path": "tools/gate/jarvis/5G-SA-core/mongodb/images/mongodb", - "build": { - "repo": "https://review.opendev.org/airship/charts", - "target_loc": "/src/checkout/mongodb", - "refspec": "master", - "version": "refs/changes/*:refs/changes/*" - } - } - ] - chart.json: | - [ - { - "chart_name": "mongodb-sharded", - "project": "mongodb", - "repo": "mongodb-sharded", - "version": "3.3.3", - "build_from_source": false, - "path": "", - "build": { - "repo": "https://charts.bitnami.com/bitnami", - "local_repo": "bitnami_mongodb", - "target_loc": "/src/fetch/charts" - }, - "namespace": "development-pipeline", - "release_name": "mongodb-bitnami", - "sources": { - "image_map": { - "tag": ["releases", "image", "tag"], - "tmp_name": ["releases", "image", "repository"], - "tmp_repo": ["releases", "image", "registry"] - } - }, - "releases": { - "image": { - "registry": "docker.io", - "repository": "bitnami/mongodb-sharded", - "tag": "4.4.3-debian-10-r44" - } - } - }, - { - "chart_name": "mongodb", - "project": "mongodb", - "repo": "mongodb", - "version": "0.1.0", - "build_from_source": true, - "path": "tools/gate/jarvis/5G-SA-core/mongodb/charts", - "build": { - "repo": "https://review.opendev.org/airship/charts", - "target_loc": "/src/checkout/airship/charts", - "refspec": "master" - }, - "namespace": "development-pipeline", - "release_name": "mongodb", - "sources": { - "image_map": { - "tag": ["releases", "images", "applications", "mongodb", "tag"], - "tmp_name": ["releases", "images", "applications", "mongodb", "name"], - "tmp_repo": ["releases", "images", "applications", "mongodb", "repo"] - } - }, - "releases": { - "images": { - "applications": { - "mongodb": { - "tag": "1.0", - "name": "mongodb", - "repo": "mongodb" - } - } - } - } - } - ] - default.json: | - { - "proxy": { - "http": "", - "https": "", - "noproxy": "", - "enabled": false - }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds", - "clone_async_batch_size": 20, - "chart_async_batch_size": 20, - "image_async_batch_size": 2 - } - cleanup.json: | - { - "remove_artifacts": "true" - } diff --git a/charts/development-pipeline/pipelinerun-validation.yaml b/charts/development-pipeline/pipelinerun-validation.yaml deleted file mode 100644 index 3acb55ed..00000000 --- a/charts/development-pipeline/pipelinerun-validation.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: PipelineRun -metadata: - generateName: development-pipeline-run -spec: - pipelineRef: - name: development-pipeline - serviceAccountName: sa-development-pipeline - workspaces: - - name: k8s_cluster_data - configMap: - name: deployment-flow - - name: development_pipeline_data - volumeClaimTemplate: - spec: - storageClassName: standard - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/charts/development-pipeline/templates/pipeline.yaml b/charts/development-pipeline/templates/pipeline.yaml deleted file mode 100644 index 9138394a..00000000 --- a/charts/development-pipeline/templates/pipeline.yaml +++ /dev/null @@ -1,128 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Pipeline -metadata: - name: development-pipeline - namespace: {{ $.Release.Namespace }} -spec: - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - tasks: - - name: microflow-setup-cluster-config - taskRef: - name: setup-cluster-config - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - - - name: microflow-setup-image-config - taskRef: - name: setup-image-config - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - - - name: microflow-setup-chart-config - taskRef: - name: setup-chart-config - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - - - name: microflow-setup-cleanup-config - taskRef: - name: setup-cleanup-config - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - - - name: microflow-k8s - runAfter: - - microflow-setup-cluster-config - taskRef: - name: k8s-cluster - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - - - name: microflow-images - runAfter: - - microflow-setup-cluster-config - - microflow-setup-image-config - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - taskRef: - name: image-tasks - - - name: microflow-charts - runAfter: - - microflow-setup-cluster-config - - microflow-setup-chart-config - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - taskRef: - name: chart-tasks - - - name: microflow-deployment-manifests - runAfter: - - microflow-setup-image-config - - microflow-setup-chart-config - - microflow-setup-cluster-config - - microflow-setup-cleanup-config - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - taskRef: - name: deployment-manifests - - - name: microflow-functional - runAfter: - - microflow-deployment-manifests - - microflow-k8s - - microflow-images - - microflow-charts - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - taskRef: - name: functional - - - name: microflow-promote-artifacts - runAfter: - - microflow-functional - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - taskRef: - name: promote - finally: - - name: microflow-cleanup - workspaces: - - name: k8s_cluster_data - workspace: k8s_cluster_data - - name: development_pipeline_data - workspace: development_pipeline_data - taskRef: - name: cleanup diff --git a/charts/development-pipeline/templates/role.yaml b/charts/development-pipeline/templates/role.yaml deleted file mode 100644 index 2cda2b23..00000000 --- a/charts/development-pipeline/templates/role.yaml +++ /dev/null @@ -1,29 +0,0 @@ -{{- if $.Values.role.create }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ $.Values.role.name }} - namespace: {{ $.Release.Namespace }} -rules: - # EventListeners need to be able to fetch all namespaced resources - - apiGroups: ["triggers.tekton.dev"] - resources: ["eventlisteners", "triggerbindings", "triggertemplates", "triggers"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch"] - # Permissions to create resources in associated TriggerTemplates - - apiGroups: [""] - resources: ["serviceaccounts"] - verbs: ["impersonate", "get"] - # Permissions to execute helm dry-run - - apiGroups: [""] - resources: ["secrets", "services"] - verbs: ["get"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["get"] - - apiGroups: ["rbac.authorization.k8s.io"] - resources: ["roles", "rolebindings"] - verbs: ["get"] -{{- end }} diff --git a/charts/development-pipeline/templates/rolebinding.yaml b/charts/development-pipeline/templates/rolebinding.yaml deleted file mode 100644 index 0987692f..00000000 --- a/charts/development-pipeline/templates/rolebinding.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if and ($.Values.serviceAccount.create) ($.Values.role.bind) }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ $.Values.role.name }} - namespace: {{ $.Release.Namespace }} -subjects: - - kind: ServiceAccount - name: {{ $.Values.serviceAccount.name }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ $.Values.role.name }} -{{- end }} diff --git a/charts/development-pipeline/templates/serviceaccount.yaml b/charts/development-pipeline/templates/serviceaccount.yaml deleted file mode 100644 index 77714b5a..00000000 --- a/charts/development-pipeline/templates/serviceaccount.yaml +++ /dev/null @@ -1,7 +0,0 @@ -{{- if $.Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ $.Values.serviceAccount.name }} - namespace: {{ $.Release.Namespace }} -{{- end }} diff --git a/charts/development-pipeline/templates/task-chart.yaml b/charts/development-pipeline/templates/task-chart.yaml deleted file mode 100644 index 8bf8d3c8..00000000 --- a/charts/development-pipeline/templates/task-chart.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: chart-tasks - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task builds charts if source is provided - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: clone - image: {{ $.Values.tasks.chart.buildChartImage }} - script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.git.gitPlaybook }} -i hosts -e '{"stage":"clone"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - - - name: set-chart-output - image: {{ $.Values.tasks.chart.buildChartImage }} - script: | - #!/usr/bin/env sh - cat "$(workspaces.development_pipeline_data.path)/chart.json" - - - name: lint-chart - image: {{ $.Values.tasks.chart.buildChartImage }} - script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"lint"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - - - name: set-chart-output-after-lint-dryrun - image: {{ $.Values.tasks.chart.buildChartImage }} - script: | - #!/usr/bin/env sh - cat "$(workspaces.development_pipeline_data.path)/chart.json" - - - name: package-chart - # Mount all secrets we want to use for external repos to fetch charts - image: {{ $.Values.tasks.chart.buildChartImage }} - script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"package"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - - - name: set-chart-output-after-packaging - image: {{ $.Values.tasks.chart.buildChartImage }} - script: | - #!/usr/bin/env sh - cat "$(workspaces.development_pipeline_data.path)/chart.json" - - - name: publish-chart - volumeMounts: - - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt - name: harbor-ca - subPath: harbor-ca - - mountPath: /workspace/helm-creds - name: helm-publish-creds - image: {{ $.Values.tasks.chart.buildChartImage }} - script: | - #!/usr/bin/env sh - update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"publish"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - - - name: set-chart-output-after-publish - image: {{ $.Values.tasks.chart.buildChartImage }} - script: | - #!/usr/bin/env sh - cat "$(workspaces.development_pipeline_data.path)/chart.json" - volumes: - - name: helm-publish-creds - secret: - secretName: harbor-basic-auth - - name: harbor-ca - secret: - secretName: harbor-ca diff --git a/charts/development-pipeline/templates/task-cleanup.yaml b/charts/development-pipeline/templates/task-cleanup.yaml deleted file mode 100644 index 50af8b40..00000000 --- a/charts/development-pipeline/templates/task-cleanup.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: cleanup - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task gathers logs and cleans up the environment - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: cleanup - image: {{ $.Values.tasks.cleanup.cleanupImage }} - volumeMounts: - - mountPath: $(workspaces.development_pipeline_data.path)/config - name: kubeconfig - script: | - #!/bin/sh - ansible-playbook -vvv "{{ $.Values.tasks.cleanup.cleanupPlaybook }}" -i hosts \ - -e @"$(workspaces.development_pipeline_data.path)/default.json" \ - -e 'loop_chart_source="$(workspaces.development_pipeline_data.path)/chart.json"' \ - -e 'loop_image_source="$(workspaces.development_pipeline_data.path)/image.json"' \ - -e @"$(workspaces.development_pipeline_data.path)/cluster.json" - volumes: - - name: kubeconfig - secret: - secretName: kubeconfig-secret diff --git a/charts/development-pipeline/templates/task-deployment-manifests.yaml b/charts/development-pipeline/templates/task-deployment-manifests.yaml deleted file mode 100644 index 2031495f..00000000 --- a/charts/development-pipeline/templates/task-deployment-manifests.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: deployment-manifests - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task will do any validation of manifests required to deploy the CNF - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: example-validation-step - image: {{ $.Values.tasks.deploymentManifests.deploymentManifestsImage }} - script: | - ansible-playbook -vvv "{{ $.Values.tasks.deploymentManifests.deploymentManifestsPlaybook }}" -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" diff --git a/charts/development-pipeline/templates/task-functional.yaml b/charts/development-pipeline/templates/task-functional.yaml deleted file mode 100644 index 5fabfe2c..00000000 --- a/charts/development-pipeline/templates/task-functional.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: functional - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task will deploy the CNF and run any tests specified - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: deploy-helm-charts - image: {{ $.Values.tasks.functional.functionalDeployImage }} - volumeMounts: - - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt - name: harbor-ca - subPath: harbor-ca - - mountPath: /workspace/helm-creds - name: helm-publish-creds - - mountPath: $(workspaces.development_pipeline_data.path)/config - name: kubeconfig - script: | - #!/bin/sh - update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts \ - -e '{"stage":"deploy"}' \ - -e @"$(workspaces.development_pipeline_data.path)/default.json" \ - -e @"$(workspaces.development_pipeline_data.path)/cluster.json" \ - -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' \ - -e 'datapath="$(workspaces.development_pipeline_data.path)"' - - - name: run-helm-tests - image: {{ $.Values.tasks.functional.functionalTestImage }} - script: | - #!/bin/sh - ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts \ - -e '{"stage":"test"}' \ - -e @"$(workspaces.development_pipeline_data.path)/default.json" \ - -e @"$(workspaces.development_pipeline_data.path)/cluster.json" \ - -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - volumes: - - name: helm-publish-creds - secret: - secretName: harbor-basic-auth - - name: harbor-ca - secret: - secretName: harbor-ca - - name: kubeconfig - secret: - secretName: kubeconfig-secret diff --git a/charts/development-pipeline/templates/task-image.yaml b/charts/development-pipeline/templates/task-image.yaml deleted file mode 100644 index bcec3ac5..00000000 --- a/charts/development-pipeline/templates/task-image.yaml +++ /dev/null @@ -1,127 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: image-tasks - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task builds images if source is provided - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: clone - image: {{ $.Values.tasks.image.buildImage }} - volumeMounts: - - mountPath: /certs/client - name: dind-certs - script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.git.gitPlaybook }} -i hosts -e '{"stage":"clone"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' - - - name: set-image-output - image: {{ $.Values.tasks.image.buildImage }} - script: | - #!/usr/bin/env sh - cat "$(workspaces.development_pipeline_data.path)/image.json" - - - name: docker-build - image: {{ $.Values.tasks.image.buildImage }} - volumeMounts: - - mountPath: /certs/client - name: dind-certs - env: - # Connect to the sidecar over TCP, with TLS. - - name: DOCKER_HOST - value: tcp://localhost:2376 - # Verify TLS. - - name: DOCKER_TLS_VERIFY - value: '1' - # Use the certs generated by the sidecar daemon. - - name: DOCKER_CERT_PATH - value: /certs/client - script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"build"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' - - - name: set-image-build-output-after-build - image: {{ $.Values.tasks.image.buildImage }} - script: | - #!/usr/bin/env sh - cat "$(workspaces.development_pipeline_data.path)/image.json" - - - name: publish-and-scan-image - image: {{ $.Values.tasks.image.buildImage }} - volumeMounts: - - mountPath: /certs/client - name: dind-certs - - mountPath: /tekton/home/.docker/config.json - name: image-push-creds - subPath: .dockerconfigjson - env: - # Connect to the sidecar over TCP, with TLS. - - name: DOCKER_HOST - value: tcp://localhost:2376 - # Verify TLS. - - name: DOCKER_TLS_VERIFY - value: '1' - # Use the certs generated by the sidecar daemon. - - name: DOCKER_CERT_PATH - value: /certs/client - script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"push"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' - - - name: set-image-output-after-publish-scan - image: {{ $.Values.tasks.image.buildImage }} - script: | - #!/usr/bin/env sh - cat "$(workspaces.development_pipeline_data.path)/image.json" - - - name: get-scan-results - image: {{ $.Values.tasks.image.buildImage }} - volumeMounts: - - mountPath: /certs/client - name: dind-certs - script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"scan_results"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' - - - name: set-image-output-set-scan-results - image: {{ $.Values.tasks.image.buildImage }} - script: | - #!/usr/bin/env sh - cat "$(workspaces.development_pipeline_data.path)/image.json" - sidecars: - - image: {{ $.Values.tasks.image.sidecarServer }} - name: server - args: - - --storage-driver=vfs - - --userland-proxy=false - - --debug - - --insecure-registry={{ $.Values.tasks.image.insecureRegistry }} - ##TODO: Get rid of privileged true - securityContext: - privileged: true - env: - # Write generated certs to the path shared with the client. - - name: DOCKER_TLS_CERTDIR - value: /certs - volumeMounts: - - mountPath: /certs/client - name: dind-certs - # Wait for the dind daemon to generate the certs it will share with the - # client. - readinessProbe: - periodSeconds: 1 - exec: - command: ['ls', '/certs/client/ca.pem'] - volumes: - - name: dind-certs - emptyDir: {} - - name: image-push-creds - secret: - secretName: harbor-docker-auth - - name: helm-publish-creds - secret: - secretName: harbor-basic-auth diff --git a/charts/development-pipeline/templates/task-k8s-cluster.yaml b/charts/development-pipeline/templates/task-k8s-cluster.yaml deleted file mode 100644 index bd78249c..00000000 --- a/charts/development-pipeline/templates/task-k8s-cluster.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: k8s-cluster - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task will create a k8s cluster if needed or verify that an existing cluster is reachable - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: generate-kubeconfig - image: {{ $.Values.tasks.kubernetes.createClusterImage }} - script: | - ansible-playbook -vvv "{{ $.Values.tasks.kubernetes.getKubeconfigPlaybook }}" -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" - volumeMounts: - - mountPath: $(workspaces.development_pipeline_data.path)/config - name: kubeconfig - volumes: - - name: kubeconfig - secret: - secretName: kubeconfig-secret diff --git a/charts/development-pipeline/templates/task-promote.yaml b/charts/development-pipeline/templates/task-promote.yaml deleted file mode 100644 index 182dcfc9..00000000 --- a/charts/development-pipeline/templates/task-promote.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: promote - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task will promote images and chart into a non-test repository - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: promote-artifacts - image: {{ $.Values.tasks.promote.promoteImage }} - env: - # Connect to the sidecar over TCP, with TLS. - - name: DOCKER_HOST - value: tcp://localhost:2376 - # Verify TLS. - - name: DOCKER_TLS_VERIFY - value: '1' - # Use the certs generated by the sidecar daemon. - - name: DOCKER_CERT_PATH - value: /certs/client - volumeMounts: - - mountPath: /tekton/home/.docker/config.json - name: image-push-creds - subPath: .dockerconfigjson - - mountPath: /certs/client - name: dind-certs - - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt - name: harbor-ca - subPath: harbor-ca - - mountPath: /workspace/helm-creds - name: helm-publish-creds - script: | - #!/usr/bin/env sh - set -ex - update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.promote.promotePlaybook }} -i hosts -e '{"stage":"promote_image"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/image.json"' - ansible-playbook -vvv {{ $.Values.tasks.promote.promotePlaybook }} -i hosts -e '{"stage":"promote_chart"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e 'loop_source="$(workspaces.development_pipeline_data.path)/chart.json"' - sidecars: - - image: {{ $.Values.tasks.image.sidecarServer }} - name: server - args: - - --storage-driver=vfs - - --userland-proxy=false - - --debug - - --insecure-registry={{ $.Values.tasks.image.insecureRegistry }} - ##TODO: Get rid of privileged true - securityContext: - privileged: true - env: - # Write generated certs to the path shared with the client. - - name: DOCKER_TLS_CERTDIR - value: /certs - volumeMounts: - - mountPath: /certs/client - name: dind-certs - # Wait for the dind daemon to generate the certs it will share with the - # client. - readinessProbe: - periodSeconds: 1 - exec: - command: ['ls', '/certs/client/ca.pem'] - volumes: - - name: dind-certs - emptyDir: {} - - name: image-push-creds - secret: - secretName: harbor-docker-auth - - name: helm-publish-creds - secret: - secretName: harbor-basic-auth - - name: harbor-ca - secret: - secretName: harbor-ca diff --git a/charts/development-pipeline/templates/task-setup-config.yaml b/charts/development-pipeline/templates/task-setup-config.yaml deleted file mode 100644 index 91e4af67..00000000 --- a/charts/development-pipeline/templates/task-setup-config.yaml +++ /dev/null @@ -1,87 +0,0 @@ -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: setup-cluster-config - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task sets configurations that will be used as overrides to the Ansible tasks. - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: setup-cluster-config - image: {{ $.Values.tasks.setup.setupConfigImage }} - script: | - #!/usr/bin/env sh - cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json" - cp "$(workspaces.k8s_cluster_data.path)/cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json" - jq '.cluster_kubeconfig_path="$(workspaces.development_pipeline_data.path)/config"' "$(workspaces.development_pipeline_data.path)/cluster.json" > "$(workspaces.development_pipeline_data.path)/temp_cluster.json" && mv "$(workspaces.development_pipeline_data.path)/temp_cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json" ---- -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: setup-image-config - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task sets configurations that will be used as overrides to the Ansible tasks. - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: setup-image-config - image: {{ $.Values.tasks.setup.setupConfigImage }} - script: | - #!/usr/bin/env sh - cp "$(workspaces.k8s_cluster_data.path)/image.json" "$(workspaces.development_pipeline_data.path)/image.json" - jq 'if type=="array" then . else [.] end' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" - echo "Set temporary image name to test/scan-image:$(context.taskRun.uid)" - jq 'reduce range(0, length) as $d (.; (.[$d].image_fullname="test/scan-image:"+($d|tostring)+"$(context.taskRun.uid)"))' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" - echo "Set tag to context $(context.taskRun.uid)" - jq 'reduce range(0, length) as $d (.; (.[$d].tag=($d|tostring)+"$(context.taskRun.uid)"))' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" - echo "Set target location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)" - jq 'reduce range(0, length) as $d (.; (.[$d].build.target_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)/"+.[$d].build.repo+"/"+.[$d].build.refspec))' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" - cat "$(workspaces.development_pipeline_data.path)/image.json" ---- -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: setup-chart-config - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task sets configurations that will be used as overrides to the Ansible tasks. - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: setup-chart-config - image: {{ $.Values.tasks.setup.setupConfigImage }} - script: | - #!/usr/bin/env sh - cp "$(workspaces.k8s_cluster_data.path)/chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" - jq 'if type=="array" then . else [.] end' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" - echo "Set tag to context $(context.taskRun.uid)" - jq 'reduce range(0, length) as $d (.; (.[$d].tag="$(context.taskRun.uid)"))' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" - echo "Set target location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)" - jq 'reduce range(0, length) as $d (.; (.[$d].build.target_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)/"+.[$d].build.repo+"/"+.[$d].build.refspec))' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" - cat "$(workspaces.development_pipeline_data.path)/chart.json" ---- -apiVersion: tekton.dev/v1beta1 -kind: Task -metadata: - name: setup-cleanup-config - namespace: {{ $.Release.Namespace }} -spec: - description: >- - This task sets configurations that will be used as overrides to the Ansible tasks. - workspaces: - - name: k8s_cluster_data - - name: development_pipeline_data - steps: - - name: setup-cleanup-config - image: {{ $.Values.tasks.setup.setupConfigImage }} - script: | - #!/usr/bin/env sh - cp "$(workspaces.k8s_cluster_data.path)/cleanup.json" "$(workspaces.development_pipeline_data.path)/cleanup.json" diff --git a/charts/development-pipeline/values.yaml b/charts/development-pipeline/values.yaml deleted file mode 100644 index e53895f8..00000000 --- a/charts/development-pipeline/values.yaml +++ /dev/null @@ -1,45 +0,0 @@ -serviceAccount: - create: true - name: sa-development-pipeline - -role: - bind: true - name: development-pipeline - create: true - -pvc: - storageClass: standard - size: 1Gi - -tasks: - kubernetes: - createClusterImage: &base_image standard-container:1.0 - createClusterPlaybook: /playbooks/create-cluster.yaml - getKubeconfigPlaybook: /playbooks/get-kubeconfig.yaml - validateClusterPlaybook: /playbooks/validate-cluster.yaml - setup: - setupConfigImage: *base_image - git: - gitPlaybook: /playbooks/git-microflow.yaml - image: - buildImage: *base_image - sidecarServer: docker:19-dind - insecureRegistry: harbor-core.jarvis.local - imagePlaybook: /playbooks/images-microflow.yaml - name: standard-container.yaml - chart: - buildChartImage: *base_image - chartPlaybook: /playbooks/charts-microflow.yaml - deploymentManifests: - deploymentManifestsImage: *base_image - deploymentManifestsPlaybook: /playbooks/deployment-manifests.yaml - promote: - promoteImage: *base_image - promotePlaybook: /playbooks/promote-microflow.yaml - functional: - functionalDeployImage: *base_image - functionalTestImage: *base_image - functionalPlaybook: /playbooks/functional-microflow.yaml - cleanup: - cleanupImage: *base_image - cleanupPlaybook: /playbooks/cleanup.yaml diff --git a/charts/jarvis-system/templates/ClusterRole-el.yaml b/charts/jarvis-system/templates/ClusterRole-el.yaml index 405d5490..57635469 100644 --- a/charts/jarvis-system/templates/ClusterRole-el.yaml +++ b/charts/jarvis-system/templates/ClusterRole-el.yaml @@ -12,21 +12,21 @@ rules: - apiGroups: [""] # allow namespaces to be retrieved to validate we haven't already created it already resources: ["namespaces"] - verbs: ["list", "get", "create"] + verbs: ["list", "get", "create", "delete"] - apiGroups: ["rbac.authorization.k8s.io"] # allow roles to be retrieved to validate we haven't already created it already resources: ["roles"] - verbs: ["list", "get", "create"] + verbs: ["list", "get", "create", "delete"] - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create"] # Permissions to create resources in associated TriggerTemplates - apiGroups: ["tekton.dev"] resources: ["pipelineruns", "pipelineresources", "taskruns", "pipelines","tasks"] - verbs: ["create", "get", "list"] + verbs: ["create", "get", "list", "delete"] - apiGroups: [""] resources: ["serviceaccounts"] - verbs: ["impersonate", "get", "create"] + verbs: ["impersonate", "get", "create", "delete"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "create"] @@ -38,7 +38,7 @@ rules: verbs: ["get"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["rolebindings"] - verbs: ["get", "create"] + verbs: ["get", "create", "delete"] ... {{- end -}} {{- include "helpers.template.overlay" ( dict "Global" $ "template_definition" "ClusterRole-el" ) }} diff --git a/charts/jarvis-system/templates/Task-createProjectAccess.yaml b/charts/jarvis-system/templates/Task-createProjectAccess.yaml index 24eda169..52e8795c 100644 --- a/charts/jarvis-system/templates/Task-createProjectAccess.yaml +++ b/charts/jarvis-system/templates/Task-createProjectAccess.yaml @@ -16,6 +16,8 @@ spec: results: - name: commit description: The precise commit SHA that was fetched by this Task + - name: namespace + description: The namespace created for this pipelinerun steps: - name: create-namespace image: {{ include "helpers.pod.container.image" ( dict "Global" $ "Application" "task_create_namespace" ) }} @@ -28,7 +30,11 @@ spec: echo "Created namespace jarvis-$(params.changeNumber)-$(params.patchSetNumber)" else echo "Namespace already exists, delete all resources for re-run." - kubectl delete all --all -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) + kubectl delete pr -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) --all + helm delete development-pipeline -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) + kubectl delete role -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) + kubectl delete sa -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) + kubectl delete rolebinding -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) fi } create_namespace @@ -208,7 +214,10 @@ spec: -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) \ ./development-pipeline \ --set "git_repo=$(params.repoRoot)/$(params.project)" \ - --set "refspec=$change_ref" + --set "refspec=$change_ref" \ + --set "namespace=jarvis-$(params.changeNumber)-$(params.patchSetNumber)" + + echo -n "jarvis-$(params.changeNumber)-$(params.patchSetNumber)" > $(results.namespace.path) - name: create-pipelinerun image: {{ include "helpers.pod.container.image" ( dict "Global" $ "Application" "task_secrets" ) }} script: | diff --git a/tools/gate/jarvis/400-deploy-harbor.sh b/tools/gate/jarvis/400-deploy-harbor.sh index 4d4932c8..8130ad26 100755 --- a/tools/gate/jarvis/400-deploy-harbor.sh +++ b/tools/gate/jarvis/400-deploy-harbor.sh @@ -73,6 +73,12 @@ EOF sudo -E docker rmi harbor-core.jarvis.local/library/busybox:latest sudo -E docker pull harbor-core.jarvis.local/library/busybox:latest sudo -E docker trust inspect --pretty harbor-core.jarvis.local/library/busybox:latest + + #Required for pipelines + sudo docker pull docker.io/library/ubuntu:focal + sudo docker tag docker.io/library/ubuntu:focal harbor-core.jarvis.local/library/ubuntu:focal + sudo -E notary init -p harbor-core.jarvis.local/library/ubuntu:focal + sudo -E docker push harbor-core.jarvis.local/library/ubuntu:focal } validate diff --git a/tools/gate/jarvis/5G-SA-core/amf/.gitreview b/tools/gate/jarvis/5G-SA-core/amf/.gitreview new file mode 100644 index 00000000..6e4adb16 --- /dev/null +++ b/tools/gate/jarvis/5G-SA-core/amf/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.jarvis.local +port=29418 +project=amf.git \ No newline at end of file diff --git a/tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml index 60394a8a..6d70693d 100644 --- a/tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml @@ -5,51 +5,56 @@ metadata: data: cluster.json: | { + "namespace": "{{ $.Values.namespace }}", "use_existing_cluster": "true" } image.json: | - { - "build_from_source": true, - "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", - "image_name": "amf", - "project": "amf", - "repo": "amf", - "tag": "1.built", - "image_fullname": "cnf/amf:1.0", - "path": "images/amf", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/amf", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - } - } + [ + { + "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", + "build_from_source": true, + "image_name": "amf", + "project": "amf", + "repo": "amf", + "image_fullname": "amf/amf:1.0", + "path": "images/amf", + "namespace": "{{ $.Values.namespace }}", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/amf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + } + } + ] chart.json: | - { - "chart_name": "amf", - "project": "amf", - "repo": "amf", - "version": "0.1.0", - "build_from_source": true, - "path": "charts", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/airship/charts", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - }, - "namespace": "development-pipeline", - "release_name": "amf", - "images": { - "applications": { - "amf": { - "tag": "1.0", - "name": "amf", - "repo": "amf" + [ + { + "chart_name": "amf", + "project": "amf", + "repo": "amf", + "version": "0.1.0", + "build_from_source": true, + "path": "charts", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/amf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + }, + "namespace": "{{ $.Values.namespace }}", + "release_name": "amf", + "images": { + "applications": { + "amf": { + "tag": "1.0", + "name": "amf", + "repo": "amf" + } } } } - } + ] default.json: | { "proxy": { @@ -58,10 +63,19 @@ data: "noproxy": "", "enabled": false }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds" + "docker_content_trust": { + "DOCKER_CONTENT_TRUST": 1, + "DOCKER_CONTENT_TRUST_SERVER": "https://harbor-notary.jarvis.local", + "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "passphrase", + "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "passphrase" + }, + "chart_registry_url": "harbor-core.jarvis.local/chartrepo", + "chart_repository": "jarvis-harbor", + "docker_registry": "harbor-core.jarvis.local", + "harbor_secret_mounted_path": "/workspace/helm-creds", + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { diff --git a/tools/gate/jarvis/5G-SA-core/ausf/.gitreview b/tools/gate/jarvis/5G-SA-core/ausf/.gitreview new file mode 100644 index 00000000..3d3a005e --- /dev/null +++ b/tools/gate/jarvis/5G-SA-core/ausf/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.jarvis.local +port=29418 +project=ausf.git \ No newline at end of file diff --git a/tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml index 187ecfed..233733ae 100644 --- a/tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml @@ -5,51 +5,56 @@ metadata: data: cluster.json: | { + "namespace": "{{ $.Values.namespace }}", "use_existing_cluster": "true" } image.json: | - { - "build_from_source": true, - "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", - "image_name": "ausf", - "project": "ausf", - "repo": "ausf", - "tag": "1.built", - "image_fullname": "cnf/ausf:1.0", - "path": "images/ausf", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/ausf", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - } - } + [ + { + "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", + "build_from_source": true, + "image_name": "ausf", + "project": "ausf", + "repo": "ausf", + "image_fullname": "ausf/ausf:1.0", + "path": "images/ausf", + "namespace": "{{ $.Values.namespace }}", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/ausf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + } + } + ] chart.json: | - { - "chart_name": "ausf", - "project": "ausf", - "repo": "ausf", - "version": "0.1.0", - "build_from_source": true, - "path": "charts", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/airship/charts", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - }, - "namespace": "development-pipeline", - "release_name": "ausf", - "images": { - "applications": { - "ausf": { - "tag": "1.0", - "name": "ausf", - "repo": "ausf" + [ + { + "chart_name": "ausf", + "project": "ausf", + "repo": "ausf", + "version": "0.1.0", + "build_from_source": true, + "path": "charts", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/ausf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + }, + "namespace": "{{ $.Values.namespace }}", + "release_name": "ausf", + "images": { + "applications": { + "ausf": { + "tag": "1.0", + "name": "ausf", + "repo": "ausf" + } } } } - } + ] default.json: | { "proxy": { @@ -58,10 +63,19 @@ data: "noproxy": "", "enabled": false }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds" + "docker_content_trust": { + "DOCKER_CONTENT_TRUST": 1, + "DOCKER_CONTENT_TRUST_SERVER": "https://harbor-notary.jarvis.local", + "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "passphrase", + "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "passphrase" + }, + "chart_registry_url": "harbor-core.jarvis.local/chartrepo", + "chart_repository": "jarvis-harbor", + "docker_registry": "harbor-core.jarvis.local", + "harbor_secret_mounted_path": "/workspace/helm-creds" + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { diff --git a/tools/gate/jarvis/5G-SA-core/mongodb/.gitreview b/tools/gate/jarvis/5G-SA-core/mongodb/.gitreview new file mode 100644 index 00000000..4323ae36 --- /dev/null +++ b/tools/gate/jarvis/5G-SA-core/mongodb/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.jarvis.local +port=29418 +project=mongodb.git \ No newline at end of file diff --git a/tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml index 8107daf1..dae7c6bb 100644 --- a/tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml @@ -8,48 +8,49 @@ data: "use_existing_cluster": "true" } image.json: | - { - "build_from_source": true, - "image_name": "mongodb", - "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", - "project": "mongodb", - "repo": "mongodb", - "tag": "1.built", - "image_fullname": "cnf/mongodb:1.0", - "path": "images/mongodb", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/mongodb", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - } - } + [ + { + "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", + "build_from_source": false, + "image_name": "mongodb-sharded", + "project": "mongodb", + "repo": "mongodb-sharded", + "tag": "4.4.3", + "image_fullname": "mongodb/mongodb-sharded:4.4.3", + "path": "", + "build": { + "git_repo": "docker.io/bitnami/mongodb-sharded", + "tag": "4.4.3-debian-10-r44" + } + } + ] chart.json: | - { - "chart_name": "mongodb", - "project": "mongodb", - "repo": "mongodb", - "version": "0.1.0", - "build_from_source": true, - "path": "charts", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/airship/charts", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - }, - "namespace": "development-pipeline", - "release_name": "mongodb", - "images": { - "applications": { - "mongodb": { - "tag": "1.0", - "name": "mongodb", - "repo": "mongodb" + [ + { + "chart_name": "mongodb-sharded", + "project": "mongodb", + "repo": "mongodb-sharded", + "version": "3.3.3", + "build_from_source": false, + "path": "", + "build": { + "name": "https://charts.bitnami.com/bitnami", + "local_repo": "bitnami_mongodb", + "target_loc": "/src/fetch/charts" + }, + "namespace": "development-pipeline", + "release_name": "mongodb-bitnami", + "images": { + "applications": { + "mongodb-sharded": { + "tag": "1.0", + "name": "mongodb", + "repo": "mongodb" + } } } } - } + ] default.json: | { "proxy": { @@ -58,10 +59,19 @@ data: "noproxy": "", "enabled": false }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds" + "docker_content_trust": { + "DOCKER_CONTENT_TRUST": 1, + "DOCKER_CONTENT_TRUST_SERVER": "https://harbor-notary.jarvis.local", + "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "passphrase", + "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "passphrase" + }, + "chart_registry_url": "harbor-core.jarvis.local/chartrepo", + "chart_repository": "jarvis-harbor", + "docker_registry": "harbor-core.jarvis.local", + "harbor_secret_mounted_path": "/workspace/helm-creds" + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { diff --git a/tools/gate/jarvis/5G-SA-core/nrf/.gitreview b/tools/gate/jarvis/5G-SA-core/nrf/.gitreview new file mode 100644 index 00000000..f6c75ec7 --- /dev/null +++ b/tools/gate/jarvis/5G-SA-core/nrf/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.jarvis.local +port=29418 +project=nrf.git \ No newline at end of file diff --git a/tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml index f52a1549..0718b33f 100644 --- a/tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml @@ -5,51 +5,56 @@ metadata: data: cluster.json: | { + "namespace": "{{ $.Values.namespace }}", "use_existing_cluster": "true" } image.json: | - { - "build_from_source": true, - "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", - "image_name": "nrf", - "project": "nrf", - "repo": "nrf", - "tag": "1.built", - "image_fullname": "cnf/nrf:1.0", - "path": "images/nrf", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/nrf", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - } - } + [ + { + "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", + "build_from_source": true, + "image_name": "nrf", + "project": "nrf", + "repo": "nrf", + "image_fullname": "nrf/nrf:1.0", + "path": "images/nrf", + "namespace": "{{ $.Values.namespace }}", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/nrf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + } + } + ] chart.json: | - { - "chart_name": "nrf", - "project": "nrf", - "repo": "nrf", - "version": "0.1.0", - "build_from_source": true, - "path": "charts", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/airship/charts", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - }, - "namespace": "development-pipeline", - "release_name": "nrf", - "images": { - "applications": { - "nrf": { - "tag": "1.0", - "name": "nrf", - "repo": "nrf" + [ + { + "chart_name": "nrf", + "project": "nrf", + "repo": "nrf", + "version": "0.1.0", + "build_from_source": true, + "path": "charts", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/nrf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + }, + "namespace": "{{ $.Values.namespace }}", + "release_name": "nrf", + "images": { + "applications": { + "nrf": { + "tag": "1.0", + "name": "nrf", + "repo": "nrf" + } } } } - } + ] default.json: | { "proxy": { @@ -58,10 +63,19 @@ data: "noproxy": "", "enabled": false }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds" + "docker_content_trust": { + "DOCKER_CONTENT_TRUST": 1, + "DOCKER_CONTENT_TRUST_SERVER": "https://harbor-notary.jarvis.local", + "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "passphrase", + "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "passphrase" + }, + "chart_registry_url": "harbor-core.jarvis.local/chartrepo", + "chart_repository": "jarvis-harbor", + "docker_registry": "harbor-core.jarvis.local", + "harbor_secret_mounted_path": "/workspace/helm-creds" + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { diff --git a/tools/gate/jarvis/5G-SA-core/pcf/.gitreview b/tools/gate/jarvis/5G-SA-core/pcf/.gitreview new file mode 100644 index 00000000..ef34ef41 --- /dev/null +++ b/tools/gate/jarvis/5G-SA-core/pcf/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.jarvis.local +port=29418 +project=pcf.git \ No newline at end of file diff --git a/tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml index 8784470f..3fab9178 100644 --- a/tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml @@ -5,51 +5,56 @@ metadata: data: cluster.json: | { + "namespace": "{{ $.Values.namespace }}", "use_existing_cluster": "true" } image.json: | - { - "build_from_source": true, - "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", - "image_name": "pcf", - "project": "pcf", - "repo": "pcf", - "tag": "1.built", - "image_fullname": "cnf/pcf:1.0", - "path": "images/pcf", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/pcf", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - } - } + [ + { + "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", + "build_from_source": true, + "image_name": "pcf", + "project": "pcf", + "repo": "pcf", + "image_fullname": "pcf/pcf:1.0", + "path": "images/pcf", + "namespace": "{{ $.Values.namespace }}", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/pcf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + } + } + ] chart.json: | - { - "chart_name": "pcf", - "project": "pcf", - "repo": "pcf", - "version": "0.1.0", - "build_from_source": true, - "path": "charts", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/airship/charts", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - }, - "namespace": "development-pipeline", - "release_name": "pcf", - "images": { - "applications": { - "pcf": { - "tag": "1.0", - "name": "pcf", - "repo": "pcf" + [ + { + "chart_name": "pcf", + "project": "pcf", + "repo": "pcf", + "version": "0.1.0", + "build_from_source": true, + "path": "charts", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/pcf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + }, + "namespace": "{{ $.Values.namespace }}", + "release_name": "pcf", + "images": { + "applications": { + "pcf": { + "tag": "1.0", + "name": "pcf", + "repo": "pcf" + } } } } - } + ] default.json: | { "proxy": { @@ -58,10 +63,19 @@ data: "noproxy": "", "enabled": false }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds" + "docker_content_trust": { + "DOCKER_CONTENT_TRUST": 1, + "DOCKER_CONTENT_TRUST_SERVER": "https://harbor-notary.jarvis.local", + "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "passphrase", + "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "passphrase" + }, + "chart_registry_url": "harbor-core.jarvis.local/chartrepo", + "chart_repository": "jarvis-harbor", + "docker_registry": "harbor-core.jarvis.local", + "harbor_secret_mounted_path": "/workspace/helm-creds" + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { diff --git a/tools/gate/jarvis/5G-SA-core/smf/.gitreview b/tools/gate/jarvis/5G-SA-core/smf/.gitreview new file mode 100644 index 00000000..e9a8c0c3 --- /dev/null +++ b/tools/gate/jarvis/5G-SA-core/smf/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.jarvis.local +port=29418 +project=smf.git \ No newline at end of file diff --git a/tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml index 4a0e13b4..989515f9 100644 --- a/tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml @@ -5,51 +5,56 @@ metadata: data: cluster.json: | { + "namespace": "{{ $.Values.namespace }}", "use_existing_cluster": "true" } image.json: | - { - "build_from_source": true, - "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", - "image_name": "smf", - "project": "smf", - "repo": "smf", - "tag": "1.built", - "image_fullname": "cnf/smf:1.0", - "path": "images/smf", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/smf", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - } - } + [ + { + "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", + "build_from_source": true, + "image_name": "smf", + "project": "smf", + "repo": "smf", + "image_fullname": "smf/smf:1.0", + "path": "images/smf", + "namespace": "{{ $.Values.namespace }}", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/smf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + } + } + ] chart.json: | - { - "chart_name": "smf", - "project": "smf", - "repo": "smf", - "version": "0.1.0", - "build_from_source": true, - "path": "charts", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/airship/charts", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - }, - "namespace": "development-pipeline", - "release_name": "smf", - "images": { - "applications": { - "smf": { - "tag": "1.0", - "name": "smf", - "repo": "smf" + [ + { + "chart_name": "smf", + "project": "smf", + "repo": "smf", + "version": "0.1.0", + "build_from_source": true, + "path": "charts", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/smf", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + }, + "namespace": "{{ $.Values.namespace }}", + "release_name": "smf", + "images": { + "applications": { + "smf": { + "tag": "1.0", + "name": "smf", + "repo": "smf" + } } } } - } + ] default.json: | { "proxy": { @@ -58,10 +63,19 @@ data: "noproxy": "", "enabled": false }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds" + "docker_content_trust": { + "DOCKER_CONTENT_TRUST": 1, + "DOCKER_CONTENT_TRUST_SERVER": "https://harbor-notary.jarvis.local", + "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "passphrase", + "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "passphrase" + }, + "chart_registry_url": "harbor-core.jarvis.local/chartrepo", + "chart_repository": "jarvis-harbor", + "docker_registry": "harbor-core.jarvis.local", + "harbor_secret_mounted_path": "/workspace/helm-creds" + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { diff --git a/tools/gate/jarvis/5G-SA-core/udm/.gitreview b/tools/gate/jarvis/5G-SA-core/udm/.gitreview new file mode 100644 index 00000000..45f158d2 --- /dev/null +++ b/tools/gate/jarvis/5G-SA-core/udm/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.jarvis.local +port=29418 +project=udm.git \ No newline at end of file diff --git a/tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml index 8f584665..55e7f307 100644 --- a/tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml @@ -5,51 +5,56 @@ metadata: data: cluster.json: | { + "namespace": "{{ $.Values.namespace }}", "use_existing_cluster": "true" } image.json: | - { - "build_from_source": true, - "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", - "image_name": "udm", - "project": "udm", - "repo": "udm", - "tag": "1.built", - "image_fullname": "cnf/udm:1.0", - "path": "images/udm", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/udm", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - } - } + [ + { + "image_from": "harbor-core.jarvis.local/library/ubuntu:focal", + "build_from_source": true, + "image_name": "udm", + "project": "udm", + "repo": "udm", + "image_fullname": "udm/udm:1.0", + "path": "images/udm", + "namespace": "{{ $.Values.namespace }}", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/udm", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + } + } + ] chart.json: | - { - "chart_name": "udm", - "project": "udm", - "repo": "udm", - "version": "0.1.0", - "build_from_source": true, - "path": "charts", - "build": { - "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/airship/charts", - "refspec": "{{ $.Values.refspec }}", - "version": "refs/changes/*:refs/changes/*" - }, - "namespace": "development-pipeline", - "release_name": "udm", - "images": { - "applications": { - "udm": { - "tag": "1.0", - "name": "udm", - "repo": "udm" + [ + { + "chart_name": "udm", + "project": "udm", + "repo": "udm", + "version": "0.1.0", + "build_from_source": true, + "path": "charts", + "build": { + "git_repo": "{{ $.Values.git_repo }}", + "target_loc": "/src/checkout/udm", + "refspec": "{{ $.Values.refspec }}", + "version": "refs/changes/*:refs/changes/*" + }, + "namespace": "{{ $.Values.namespace }}", + "release_name": "udm", + "images": { + "applications": { + "udm": { + "tag": "1.0", + "name": "udm", + "repo": "udm" + } } } } - } + ] default.json: | { "proxy": { @@ -58,10 +63,19 @@ data: "noproxy": "", "enabled": false }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds" + "docker_content_trust": { + "DOCKER_CONTENT_TRUST": 1, + "DOCKER_CONTENT_TRUST_SERVER": "https://harbor-notary.jarvis.local", + "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "passphrase", + "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "passphrase" + }, + "chart_registry_url": "harbor-core.jarvis.local/chartrepo", + "chart_repository": "jarvis-harbor", + "docker_registry": "harbor-core.jarvis.local", + "harbor_secret_mounted_path": "/workspace/helm-creds" + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { diff --git a/tools/gate/jarvis/5G-SA-core/udr/.gitreview b/tools/gate/jarvis/5G-SA-core/udr/.gitreview new file mode 100644 index 00000000..7aa9bebe --- /dev/null +++ b/tools/gate/jarvis/5G-SA-core/udr/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=gerrit.jarvis.local +port=29418 +project=udr.git \ No newline at end of file diff --git a/tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml index d4fadbf7..94c0bbf7 100644 --- a/tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml @@ -5,6 +5,7 @@ metadata: data: cluster.json: | { + "namespace": "{{ $.Values.namespace }}", "use_existing_cluster": "true" } image.json: | @@ -17,9 +18,10 @@ data: "tag": "1.built", "image_fullname": "cnf/udr:1.0", "path": "images/udr", + "namespace": "{{ $.Values.namespace }}", "build": { "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/udr", + "target_loc": "/src/checkout/udr", "refspec": "{{ $.Values.refspec }}", "version": "refs/changes/*:refs/changes/*" } @@ -32,13 +34,13 @@ data: "version": "0.1.0", "build_from_source": true, "path": "charts", + "namespace": "{{ $.Values.namespace }}", "build": { "git_repo": "{{ $.Values.git_repo }}", - "checkout_loc": "/src/checkout/airship/charts", + "target_loc": "/src/checkout/udr", "refspec": "{{ $.Values.refspec }}", "version": "refs/changes/*:refs/changes/*" }, - "namespace": "development-pipeline", "release_name": "udr", "images": { "applications": { @@ -58,10 +60,19 @@ data: "noproxy": "", "enabled": false }, - "chart_registry_url": "harbor-core.jarvis.local/chartrepo", - "chart_repository": "jarvis-harbor", - "docker_registry": "harbor-core.jarvis.local", - "harbor_secret_mounted_path": "/workspace/helm-creds" + "docker_content_trust": { + "DOCKER_CONTENT_TRUST": 1, + "DOCKER_CONTENT_TRUST_SERVER": "https://harbor-notary.jarvis.local", + "DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE": "passphrase", + "DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE": "passphrase" + }, + "chart_registry_url": "harbor-core.jarvis.local/chartrepo", + "chart_repository": "jarvis-harbor", + "docker_registry": "harbor-core.jarvis.local", + "harbor_secret_mounted_path": "/workspace/helm-creds", + "clone_async_batch_size": 20, + "chart_async_batch_size": 20, + "image_async_batch_size": 2 } cleanup.json: | { diff --git a/tools/gate/jarvis/650-temporary-setup.sh b/tools/gate/jarvis/650-temporary-setup.sh index 570ed90a..363302d4 100755 --- a/tools/gate/jarvis/650-temporary-setup.sh +++ b/tools/gate/jarvis/650-temporary-setup.sh @@ -1,5 +1,6 @@ #!/bin/bash set -ex + ldap_username="jarvis" ldap_password="password" ldap_email="jarvis@cluster.local" @@ -14,4 +15,4 @@ kubectl create secret generic harbor-basic-auth --from-literal=username=$ldap_us kubectl create secret docker-registry harbor-docker-auth --docker-username=$ldap_username --docker-password=$ldap_password --docker-email=$ldap_email --docker-server=$harbor_core -n development-pipeline || true cd ./tools/gate/jarvis/standard-container -sudo docker build -t standard-container:1.0 . +sudo docker build -t standard-container:1.0 . \ No newline at end of file diff --git a/tools/gate/jarvis/800-deploy-jarvis-projects.sh b/tools/gate/jarvis/800-deploy-jarvis-projects.sh index f154f20e..cff319d9 100755 --- a/tools/gate/jarvis/800-deploy-jarvis-projects.sh +++ b/tools/gate/jarvis/800-deploy-jarvis-projects.sh @@ -28,7 +28,6 @@ for jarvis_project in `find ./tools/gate/jarvis/5G-SA-core -maxdepth 1 -mindepth fi # shellcheck disable=SC2046 - # Copy development-pipeline to be helm upgrade \ --create-namespace \ --install \ @@ -50,15 +49,11 @@ for jarvis_project in `find ./tools/gate/jarvis/5G-SA-core -maxdepth 1 -mindepth jarvis_sanity_repo=$(mktemp -d) git clone ssh://${ldap_username}@gerrit.jarvis.local:29418/${jarvis_project}.git "${jarvis_sanity_repo}" pushd "${jarvis_sanity_repo}" - tee .gitreview <- @@ -11,40 +11,40 @@ spec: - name: development_pipeline_data steps: - name: clone + image: {{ $.Values.tasks.chart.buildChartImage }} volumeMounts: - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt name: harbor-ca subPath: harbor-ca - image: {{ $.Values.tasks.chart.buildChartImage }} script: | - #!/usr/bin/env sh - update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.git.gitPlaybook }} -i hosts -e '{"stage":"clone"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + /jarvis/git_clone_chart_repo.sh - name: set-chart-output image: {{ $.Values.tasks.chart.buildChartImage }} script: | #!/usr/bin/env sh cat "$(workspaces.development_pipeline_data.path)/chart.json" + - name: lint-chart image: {{ $.Values.tasks.chart.buildChartImage }} script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"lint"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + /jarvis/lint_dryrun_chart.sh - name: set-chart-output-after-lint-dryrun image: {{ $.Values.tasks.chart.buildChartImage }} script: | #!/usr/bin/env sh cat "$(workspaces.development_pipeline_data.path)/chart.json" + - name: package-chart + # Mount all secrets we want to use for external repos to fetch charts image: {{ $.Values.tasks.chart.buildChartImage }} script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"package"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + /jarvis/package_chart.sh - name: set-chart-output-after-packaging image: {{ $.Values.tasks.chart.buildChartImage }} script: | #!/usr/bin/env sh cat "$(workspaces.development_pipeline_data.path)/chart.json" + - name: publish-chart volumeMounts: - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt @@ -54,9 +54,9 @@ spec: name: helm-publish-creds image: {{ $.Values.tasks.chart.buildChartImage }} script: | - #!/usr/bin/env sh - update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.chart.chartPlaybook }} -i hosts -e '{"stage":"publish"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + ls -ltr + + /jarvis/publish_chart.sh - name: set-chart-output-after-publish image: {{ $.Values.tasks.chart.buildChartImage }} script: | diff --git a/tools/gate/jarvis/development-pipeline/templates/task-cleanup.yaml b/tools/gate/jarvis/development-pipeline/templates/task-cleanup.yaml index d8d470e5..32681c02 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-cleanup.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-cleanup.yaml @@ -16,13 +16,8 @@ spec: - mountPath: $(workspaces.development_pipeline_data.path)/config name: kubeconfig script: | - #!/bin/sh - ansible-playbook -vvv "{{ $.Values.tasks.cleanup.cleanupPlaybook }}" -i hosts \ - -e @"$(workspaces.development_pipeline_data.path)/default.json" \ - -e @"$(workspaces.development_pipeline_data.path)/chart.json" \ - -e @"$(workspaces.development_pipeline_data.path)/image.json" \ - -e @"$(workspaces.development_pipeline_data.path)/cluster.json" + /jarvis/cleanup.sh volumes: - name: kubeconfig secret: - secretName: kubeconfig-secret \ No newline at end of file + secretName: kubeconfig-secret diff --git a/tools/gate/jarvis/development-pipeline/templates/task-deployment-manifests.yaml b/tools/gate/jarvis/development-pipeline/templates/task-deployment-manifests.yaml index 2031495f..0ec31a32 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-deployment-manifests.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-deployment-manifests.yaml @@ -13,4 +13,4 @@ spec: - name: example-validation-step image: {{ $.Values.tasks.deploymentManifests.deploymentManifestsImage }} script: | - ansible-playbook -vvv "{{ $.Values.tasks.deploymentManifests.deploymentManifestsPlaybook }}" -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" + /jarvis/deployment_manifests.sh \ No newline at end of file diff --git a/tools/gate/jarvis/development-pipeline/templates/task-functional.yaml b/tools/gate/jarvis/development-pipeline/templates/task-functional.yaml index c8a90cae..977b4b8c 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-functional.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-functional.yaml @@ -21,14 +21,11 @@ spec: - mountPath: $(workspaces.development_pipeline_data.path)/config name: kubeconfig script: | - #!/bin/sh - update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts -e '{"stage":"deploy"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + /jarvis/functional_deploy.sh - name: run-helm-tests image: {{ $.Values.tasks.functional.functionalTestImage }} script: | - #!/bin/sh - ansible-playbook -vvv {{ $.Values.tasks.functional.functionalPlaybook }} -i hosts -e '{"stage":"test"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + /jarvis/functional_test.sh volumes: - name: helm-publish-creds secret: diff --git a/tools/gate/jarvis/development-pipeline/templates/task-image.yaml b/tools/gate/jarvis/development-pipeline/templates/task-image.yaml index bb648ac6..e6188f33 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-image.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-image.yaml @@ -1,7 +1,7 @@ apiVersion: tekton.dev/v1beta1 kind: Task metadata: - name: build-images + name: image-tasks namespace: {{ $.Release.Namespace }} spec: description: >- @@ -19,14 +19,14 @@ spec: name: harbor-ca subPath: harbor-ca script: | - #!/usr/bin/env sh - update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.git.gitPlaybook }} -i hosts -e '{"stage":"clone"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" + /jarvis/git_clone_image_repo.sh + - name: set-image-output image: {{ $.Values.tasks.image.buildImage }} script: | #!/usr/bin/env sh cat "$(workspaces.development_pipeline_data.path)/image.json" + - name: docker-build image: {{ $.Values.tasks.image.buildImage }} volumeMounts: @@ -42,14 +42,17 @@ spec: # Use the certs generated by the sidecar daemon. - name: DOCKER_CERT_PATH value: /certs/client + script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"build"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" + update-ca-certificates + /jarvis/build_docker_image.sh + - name: set-image-build-output-after-build image: {{ $.Values.tasks.image.buildImage }} script: | #!/usr/bin/env sh cat "$(workspaces.development_pipeline_data.path)/image.json" + - name: publish-and-scan-image image: {{ $.Values.tasks.image.buildImage }} volumeMounts: @@ -69,21 +72,21 @@ spec: - name: DOCKER_CERT_PATH value: /certs/client script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"push"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" + /jarvis/push_docker_image.sh + - name: set-image-output-after-publish-scan image: {{ $.Values.tasks.image.buildImage }} script: | #!/usr/bin/env sh cat "$(workspaces.development_pipeline_data.path)/image.json" + - name: get-scan-results image: {{ $.Values.tasks.image.buildImage }} volumeMounts: - mountPath: /certs/client name: dind-certs script: | - #!/usr/bin/env sh - ansible-playbook -vvv {{ $.Values.tasks.image.imagePlaybook }} -i hosts -e '{"stage":"scan_results"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" + /jarvis/get_image_scan_results.sh - name: set-image-output-set-scan-results image: {{ $.Values.tasks.image.buildImage }} script: | @@ -97,6 +100,7 @@ spec: - --userland-proxy=false - --debug - --insecure-registry={{ $.Values.tasks.image.insecureRegistry }} + ##TODO: Get rid of privileged true securityContext: privileged: true env: @@ -108,6 +112,9 @@ spec: name: dind-certs - mountPath: /var/lib/docker name: var-lib-docker + - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt + name: harbor-ca + subPath: harbor-ca # Wait for the dind daemon to generate the certs it will share with the # client. readinessProbe: diff --git a/tools/gate/jarvis/development-pipeline/templates/task-k8s-cluster.yaml b/tools/gate/jarvis/development-pipeline/templates/task-k8s-cluster.yaml index bd78249c..b871ee1c 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-k8s-cluster.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-k8s-cluster.yaml @@ -10,10 +10,10 @@ spec: - name: k8s_cluster_data - name: development_pipeline_data steps: - - name: generate-kubeconfig + - name: get-kubeconfig image: {{ $.Values.tasks.kubernetes.createClusterImage }} script: | - ansible-playbook -vvv "{{ $.Values.tasks.kubernetes.getKubeconfigPlaybook }}" -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" + /jarvis/get_kubeconfig.sh volumeMounts: - mountPath: $(workspaces.development_pipeline_data.path)/config name: kubeconfig diff --git a/tools/gate/jarvis/development-pipeline/templates/task-promote.yaml b/tools/gate/jarvis/development-pipeline/templates/task-promote.yaml index dd3bf307..6f227166 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-promote.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-promote.yaml @@ -34,11 +34,7 @@ spec: - mountPath: /workspace/helm-creds name: helm-publish-creds script: | - #!/usr/bin/env sh - set -ex - update-ca-certificates - ansible-playbook -vvv {{ $.Values.tasks.promote.promotePlaybook }} -i hosts -e '{"stage":"promote_image"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json" - ansible-playbook -vvv {{ $.Values.tasks.promote.promotePlaybook }} -i hosts -e '{"stage":"promote_chart"}' -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json" + /jarvis/promote_artifacts.sh sidecars: - image: {{ $.Values.tasks.image.sidecarServer }} name: server @@ -47,6 +43,7 @@ spec: - --userland-proxy=false - --debug - --insecure-registry={{ $.Values.tasks.image.insecureRegistry }} + ##TODO: Get rid of privileged true securityContext: privileged: true env: diff --git a/tools/gate/jarvis/development-pipeline/templates/task-setup-config.yaml b/tools/gate/jarvis/development-pipeline/templates/task-setup-config.yaml index b177bbec..c87148eb 100644 --- a/tools/gate/jarvis/development-pipeline/templates/task-setup-config.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/task-setup-config.yaml @@ -13,10 +13,8 @@ spec: - name: setup-cluster-config image: {{ $.Values.tasks.setup.setupConfigImage }} script: | - #!/usr/bin/env sh - cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json" - cp "$(workspaces.k8s_cluster_data.path)/cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json" - jq '.cluster_kubeconfig_path="$(workspaces.development_pipeline_data.path)/config"' "$(workspaces.development_pipeline_data.path)/cluster.json" > "$(workspaces.development_pipeline_data.path)/temp_cluster.json" && mv "$(workspaces.development_pipeline_data.path)/temp_cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json" + ls -ltr /opt/ + /jarvis/setup_cluster_config.sh --- apiVersion: tekton.dev/v1beta1 kind: Task @@ -33,14 +31,7 @@ spec: - name: setup-image-config image: {{ $.Values.tasks.setup.setupConfigImage }} script: | - #!/usr/bin/env sh - cp "$(workspaces.k8s_cluster_data.path)/image.json" "$(workspaces.development_pipeline_data.path)/image.json" - echo "Set temporary image name to test/scan-image:$(context.taskRun.uid)" - jq '.image_fullname="test/scan-image:$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" - echo "Set tag to context $(context.taskRun.uid)" - jq '.tag="$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" - echo "Set checkout location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)" - jq '.build.checkout_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json" + /jarvis/setup_image_config.sh $(context.taskRun.uid) --- apiVersion: tekton.dev/v1beta1 kind: Task @@ -57,13 +48,7 @@ spec: - name: setup-chart-config image: {{ $.Values.tasks.setup.setupConfigImage }} script: | - #!/usr/bin/env sh - cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json" - cp "$(workspaces.k8s_cluster_data.path)/chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" - echo "Set tag to context $(context.taskRun.uid)" - jq '.tag="$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" - echo "Set checkout location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)" - jq '.build.checkout_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json" + /jarvis/setup_chart_config.sh $(context.taskRun.uid) --- apiVersion: tekton.dev/v1beta1 kind: Task @@ -80,5 +65,4 @@ spec: - name: setup-cleanup-config image: {{ $.Values.tasks.setup.setupConfigImage }} script: | - #!/usr/bin/env sh - cp "$(workspaces.k8s_cluster_data.path)/cleanup.json" "$(workspaces.development_pipeline_data.path)/cleanup.json" \ No newline at end of file + /jarvis/setup_cleanup_config.sh $(context.taskRun.uid) diff --git a/tools/gate/jarvis/development-pipeline/values.yaml b/tools/gate/jarvis/development-pipeline/values.yaml index 07a04233..c15d49df 100644 --- a/tools/gate/jarvis/development-pipeline/values.yaml +++ b/tools/gate/jarvis/development-pipeline/values.yaml @@ -1,11 +1,3 @@ -serviceAccount: - create: false - name: sa-development-pipeline - -role: - bind: true - name: development-pipeline - create: true pvc: storageClass: standard @@ -14,32 +6,20 @@ pvc: tasks: kubernetes: createClusterImage: &base_image standard-container:1.0 - createClusterPlaybook: /playbooks/create-cluster.yaml - getKubeconfigPlaybook: /playbooks/get-kubeconfig.yaml - validateClusterPlaybook: /playbooks/validate-cluster.yaml setup: setupConfigImage: *base_image - git: - gitPlaybook: /playbooks/git-microflow.yaml image: buildImage: *base_image - sidecarServer: docker:19-dind insecureRegistry: harbor-core.jarvis.local - imagePlaybook: /playbooks/images-microflow.yaml - name: standard-container.yaml + sidecarServer: docker:19-dind chart: buildChartImage: *base_image - chartPlaybook: /playbooks/charts-microflow.yaml deploymentManifests: deploymentManifestsImage: *base_image - deploymentManifestsPlaybook: /playbooks/deployment-manifests.yaml promote: promoteImage: *base_image - promotePlaybook: /playbooks/promote-microflow.yaml functional: functionalDeployImage: *base_image functionalTestImage: *base_image - functionalPlaybook: /playbooks/functional-microflow.yaml cleanup: cleanupImage: *base_image - cleanupPlaybook: /playbooks/cleanup.yaml diff --git a/tools/gate/jarvis/standard-container/Dockerfile b/tools/gate/jarvis/standard-container/Dockerfile index e4335382..ee15dd48 100644 --- a/tools/gate/jarvis/standard-container/Dockerfile +++ b/tools/gate/jarvis/standard-container/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=ubuntu:20.04 +ARG BASE_IMAGE=harbor-core.jarvis.local/library/ubuntu:focal FROM ${BASE_IMAGE} SHELL ["bash", "-exc"] @@ -27,7 +27,7 @@ RUN apt-get update ;\ apt-get install -y --no-install-recommends \ docker-ce-cli ;\ pip3 install --upgrade wheel ;\ - pip3 install ansible ;\ + pip3 install ansible==2.10.7 ;\ ansible-galaxy collection install community.kubernetes ;\ pip3 install docker ;\ # Install kubectl @@ -50,4 +50,4 @@ COPY assets /opt/assets/ RUN cp -ravf /opt/assets/* / ;\ rm -rf /opt/assets -ENTRYPOINT /entrypoint.sh +ENTRYPOINT /git_clone.sh diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/build_docker_image.sh b/tools/gate/jarvis/standard-container/assets/jarvis/build_docker_image.sh new file mode 100644 index 00000000..ba60d697 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/build_docker_image.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex + +ansible-playbook -vvv /playbooks/images-microflow.yaml -i hosts \ + -e '{"stage":"build"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/image.json"' \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/cleanup.sh b/tools/gate/jarvis/standard-container/assets/jarvis/cleanup.sh new file mode 100644 index 00000000..19a7f3a7 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/cleanup.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -ex + +ansible-playbook -vvv "/playbooks/cleanup.yaml" -i hosts \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_chart_source="/workspace/development_pipeline_data/chart.json"' \ + -e 'loop_image_source="/workspace/development_pipeline_data/image.json"' \ + -e @"/workspace/development_pipeline_data/cluster.json" \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/deployment_manifests.sh b/tools/gate/jarvis/standard-container/assets/jarvis/deployment_manifests.sh new file mode 100644 index 00000000..a9699e3b --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/deployment_manifests.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -ex + +ansible-playbook -vvv "/playbooks/deployment-manifests.yaml" -i hosts \ + -e @"/workspace/development_pipeline_data/default.json" diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/functional_deploy.sh b/tools/gate/jarvis/standard-container/assets/jarvis/functional_deploy.sh new file mode 100644 index 00000000..ac69ab1c --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/functional_deploy.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +update-ca-certificates +ansible-playbook -vvv /playbooks/functional-microflow.yaml -i hosts \ + -e '{"stage":"deploy"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e @"/workspace/development_pipeline_data/cluster.json" \ + -e 'loop_source="/workspace/development_pipeline_data/chart.json"' diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/functional_test.sh b/tools/gate/jarvis/standard-container/assets/jarvis/functional_test.sh new file mode 100644 index 00000000..cbb14b0e --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/functional_test.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -ex + +ansible-playbook -vvv /playbooks/functional-microflow.yaml -i hosts \ + -e '{"stage":"test"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e @"/workspace/development_pipeline_data/cluster.json" \ + -e 'loop_source="/workspace/development_pipeline_data/chart.json"' diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/get_image_scan_results.sh b/tools/gate/jarvis/standard-container/assets/jarvis/get_image_scan_results.sh new file mode 100644 index 00000000..db072e4e --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/get_image_scan_results.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex + +ansible-playbook -vvv /playbooks/images-microflow.yaml -i hosts \ + -e '{"stage":"scan_results"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/image.json"' \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/get_kubeconfig.sh b/tools/gate/jarvis/standard-container/assets/jarvis/get_kubeconfig.sh new file mode 100644 index 00000000..fe8d2b59 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/get_kubeconfig.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex +ls -ltr /workspace/development_pipeline_data +pwd +ansible-playbook -vvv "/playbooks/get-kubeconfig.yaml" -i hosts \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e @"/workspace/development_pipeline_data/cluster.json" \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/git_clone_chart_repo.sh b/tools/gate/jarvis/standard-container/assets/jarvis/git_clone_chart_repo.sh new file mode 100644 index 00000000..ff775a03 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/git_clone_chart_repo.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +update-ca-certificates + +ansible-playbook -vvv /playbooks/git-microflow.yaml -i hosts \ + -e '{"stage":"clone"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/chart.json"' \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/git_clone_image_repo.sh b/tools/gate/jarvis/standard-container/assets/jarvis/git_clone_image_repo.sh new file mode 100644 index 00000000..e7b25465 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/git_clone_image_repo.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +update-ca-certificates + +ansible-playbook -vvv /playbooks/git-microflow.yaml -i hosts \ + -e '{"stage":"clone"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/image.json"' \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/lint_dryrun_chart.sh b/tools/gate/jarvis/standard-container/assets/jarvis/lint_dryrun_chart.sh new file mode 100644 index 00000000..e2a72df3 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/lint_dryrun_chart.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex + +ansible-playbook -vvv /playbooks/charts-microflow.yaml -i hosts \ + -e '{"stage":"lint"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/chart.json"' diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/package_chart.sh b/tools/gate/jarvis/standard-container/assets/jarvis/package_chart.sh new file mode 100644 index 00000000..bc1ef232 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/package_chart.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex + +ansible-playbook -vvv /playbooks/charts-microflow.yaml -i hosts \ + -e '{"stage":"package"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/chart.json"' diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/promote_artifacts.sh b/tools/gate/jarvis/standard-container/assets/jarvis/promote_artifacts.sh new file mode 100644 index 00000000..6d303da3 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/promote_artifacts.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -ex +update-ca-certificates +ansible-playbook -vvv /playbooks/promote-microflow.yaml -i hosts \ + -e '{"stage":"promote_image"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/image.json"' + +ansible-playbook -vvv /playbooks/promote-microflow.yaml -i hosts \ + -e '{"stage":"promote_chart"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/chart.json"' diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/publish_chart.sh b/tools/gate/jarvis/standard-container/assets/jarvis/publish_chart.sh new file mode 100644 index 00000000..fd427864 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/publish_chart.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -ex + +update-ca-certificates +ansible-playbook -vvv /playbooks/charts-microflow.yaml -i hosts \ + -e '{"stage":"publish"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/chart.json"' diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/push_docker_image.sh b/tools/gate/jarvis/standard-container/assets/jarvis/push_docker_image.sh new file mode 100644 index 00000000..49e79c35 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/push_docker_image.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex + +ansible-playbook -vvv /playbooks/images-microflow.yaml -i hosts \ + -e '{"stage":"push"}' \ + -e @"/workspace/development_pipeline_data/default.json" \ + -e 'loop_source="/workspace/development_pipeline_data/image.json"' \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/setup_chart_config.sh b/tools/gate/jarvis/standard-container/assets/jarvis/setup_chart_config.sh new file mode 100644 index 00000000..58207159 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/setup_chart_config.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -ex + +: "${CONTEXT_UID:="$1"}" + +cp "/workspace/k8s_cluster_data/chart.json" "/workspace/development_pipeline_data/chart.json" +jq 'if type=="array" then . else [.] end' "/workspace/development_pipeline_data/chart.json" > "/workspace/development_pipeline_data/temp_chart.json" && mv "/workspace/development_pipeline_data/temp_chart.json" "/workspace/development_pipeline_data/chart.json" +echo "Set tag to context ${CONTEXT_UID}" +jq "reduce range(0, length) as \$d (.; (.[\$d].tag=\"${CONTEXT_UID}\"))" "/workspace/development_pipeline_data/chart.json" > "/workspace/development_pipeline_data/temp_chart.json" && mv "/workspace/development_pipeline_data/temp_chart.json" "/workspace/development_pipeline_data/chart.json" +echo "Set target location for git repository to /workspace/development_pipeline_data/${CONTEXT_UID}" +jq "reduce range(0, length) as \$d (.; (.[\$d].build.target_loc=\"/workspace/development_pipeline_data/${CONTEXT_UID}/\"+.[\$d].build.repo+\"/\"+.[\$d].build.refspec))" "/workspace/development_pipeline_data/chart.json" > "/workspace/development_pipeline_data/temp_chart.json" && mv "/workspace/development_pipeline_data/temp_chart.json" "/workspace/development_pipeline_data/chart.json" +cat "/workspace/development_pipeline_data/chart.json" \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/setup_cleanup_config.sh b/tools/gate/jarvis/standard-container/assets/jarvis/setup_cleanup_config.sh new file mode 100644 index 00000000..5cf95068 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/setup_cleanup_config.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -ex + +cp "/workspace/k8s_cluster_data/cleanup.json" "/workspace/development_pipeline_data/cleanup.json" \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/setup_cluster_config.sh b/tools/gate/jarvis/standard-container/assets/jarvis/setup_cluster_config.sh new file mode 100644 index 00000000..8f8ada6b --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/setup_cluster_config.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -ex + +cp "/workspace/k8s_cluster_data/default.json" "/workspace/development_pipeline_data/default.json" +cp "/workspace/k8s_cluster_data/cluster.json" "/workspace/development_pipeline_data/cluster.json" +jq '.cluster_kubeconfig_path="/workspace/development_pipeline_data/config"' "/workspace/development_pipeline_data/cluster.json" > "/workspace/development_pipeline_data/temp_cluster.json" && mv "/workspace/development_pipeline_data/temp_cluster.json" "/workspace/development_pipeline_data/cluster.json" \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/setup_image_config.sh b/tools/gate/jarvis/standard-container/assets/jarvis/setup_image_config.sh new file mode 100644 index 00000000..1e15bf91 --- /dev/null +++ b/tools/gate/jarvis/standard-container/assets/jarvis/setup_image_config.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -ex + +: "${CONTEXT_UID:=$1}" + +cp "/workspace/k8s_cluster_data/image.json" "/workspace/development_pipeline_data/image.json" +jq 'if type=="array" then . else [.] end' "/workspace/development_pipeline_data/image.json" > "/workspace/development_pipeline_data/temp_image.json" && mv "/workspace/development_pipeline_data/temp_image.json" "/workspace/development_pipeline_data/image.json" + +echo "Set image_fullname" +jq "reduce range(0, length) as \$d (.; (.[\$d].image_fullname=\"test/scan-image:\"+(\$d|tostring)+\"${CONTEXT_UID}\"))" "/workspace/development_pipeline_data/image.json" > "/workspace/development_pipeline_data/temp_image.json" && mv "/workspace/development_pipeline_data/temp_image.json" "/workspace/development_pipeline_data/image.json" + +echo "Set tag to context ${CONTEXT_UID}" +jq "reduce range(0, length) as \$d (.; (.[\$d].tag=\"${CONTEXT_UID}\"))" "/workspace/development_pipeline_data/image.json" > "/workspace/development_pipeline_data/temp_image.json" && mv "/workspace/development_pipeline_data/temp_image.json" "/workspace/development_pipeline_data/image.json" + +echo "Set target location for git repository to /workspace/development_pipeline_data/${CONTEXT_UID}" +jq "reduce range(0, length) as \$d (.; (.[\$d].build.target_loc=\"/workspace/development_pipeline_data/${CONTEXT_UID}\"+.[\$d].build.repo+\"/\"+.[\$d].build.refspec))" "/workspace/development_pipeline_data/image.json" > "/workspace/development_pipeline_data/temp_image.json" && mv "/workspace/development_pipeline_data/temp_image.json" "/workspace/development_pipeline_data/image.json" + +cat "/workspace/development_pipeline_data/image.json" \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/charts-microflow.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/charts-microflow.yaml index 61e6ee97..32489596 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/charts-microflow.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/charts-microflow.yaml @@ -1,4 +1,32 @@ ---- -- hosts: localhost - roles: - - { role: charts, vars: { stage: "default" } } +--- +- hosts: localhost + tasks: + - name: Get list of charts + shell: "jq -c '.[]' {{ loop_source }}" + register: data + + - set_fact: + chart_async_batch_size: 20 + stage: "default" + + - name: Get list of remote repos + when: (stage == "package") + shell: "jq -c '.[] | select(.build_from_source==false) | {repo: .build.name, secret: .build.secret, name: .build.local_repo}' {{ loop_source }} | sort | uniq" + register: external_repos + + - name: Get list of internal repos + when: (stage == "publish") + shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name: (.project+\"-staging\")}' {{ loop_source }} | sort | uniq" + register: internal_repos + + - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml + when: (stage == "package") or (stage == "publish") + vars: + repo: "{{ item }}" + loop: "{{ external_repos.stdout_lines if stage == 'package' else internal_repos.stdout_lines}}" + + - include_tasks: ./roles/charts/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(chart_async_batch_size) | list }}" + loop_control: + loop_var: "charts" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/cleanup.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/cleanup.yaml index 718ceacd..913bb6dd 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/cleanup.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/cleanup.yaml @@ -1,5 +1,16 @@ - hosts: localhost become: yes tasks: - - name: Clean up release - include_tasks: ./roles/cleanup/tasks/cleanup.yaml \ No newline at end of file + - name: Get list of charts + shell: "jq -c '.[]' {{ loop_chart_source }}" + register: data + + - set_fact: + chart_async_batch_size: 20 + stage: "default" + + - vars: + charts: "{{ item }}" + include_tasks: ./roles/cleanup/tasks/cleanup.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(chart_async_batch_size) | list }}" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/functional-microflow.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/functional-microflow.yaml index b13d4fa6..b25d1061 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/functional-microflow.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/functional-microflow.yaml @@ -1,4 +1,27 @@ ---- -- hosts: localhost - roles: - - { role: functional, vars: { stage: "default" } } \ No newline at end of file +--- +- hosts: localhost + tasks: + - name: Get list of charts + shell: "jq -c '.[]' {{ loop_source }}" + register: data + + - set_fact: + chart_async_batch_size: 20 + stage: "{{ stage }}" + + - name: Get list of internal repos + when: (stage == 'deploy') + shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name: (.project+\"-staging\")}' {{ loop_source }} | sort | uniq" + register: internal_repos + + - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml + when: (stage == 'deploy') + loop: "{{ internal_repos.stdout_lines }}" + vars: + repo: "{{ item }}" + + - include_tasks: ./roles/functional/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(chart_async_batch_size) | list }}" + loop_control: + loop_var: "charts" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/git-microflow.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/git-microflow.yaml index c47bcb8d..6ce1f224 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/git-microflow.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/git-microflow.yaml @@ -1,4 +1,17 @@ --- - hosts: localhost - roles: - - { role: git, vars: { stage: "default" } } \ No newline at end of file + tasks: + - name: Get list of repos to clone + shell: "jq -c '.[] | select(.build_from_source) | {build: .build}' {{ loop_source }} | sort | uniq" + register: data + + - set_fact: + clone_async_batch_size: 20 + stage: "default" + + - name: Run git clone task + include_tasks: ./roles/git/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(clone_async_batch_size) | list }}" + loop_control: + loop_var: "repos" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/images-microflow.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/images-microflow.yaml index fc3ddf4c..76fdb5d2 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/images-microflow.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/images-microflow.yaml @@ -1,4 +1,16 @@ ---- -- hosts: localhost - roles: - - { role: images, vars: { stage: "default" } } \ No newline at end of file +--- +- hosts: localhost + tasks: + - name: Get list of images + shell: "jq -c '.[]' {{ loop_source }}" + register: data + + - set_fact: + image_async_batch_size: 2 + stage: "{{ stage }}" + + - include_tasks: ./roles/images/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(image_async_batch_size) | list }}" + loop_control: + loop_var: "images" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/promote-microflow.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/promote-microflow.yaml index b4bd1659..b1f939e5 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/promote-microflow.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/promote-microflow.yaml @@ -1,4 +1,32 @@ ---- -- hosts: localhost - roles: - - { role: promote, vars: { stage: "default" } } \ No newline at end of file +--- +- hosts: localhost + tasks: + - name: Get list of objects + shell: "jq -c '.[]' {{ loop_source }}" + register: data + + - set_fact: + async_batch_size: 2 + stage: "default" + + - name: Get list of internal staging repos + when: ( stage == "promote_chart") + shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name: (.project+\"-staging\")}' {{ loop_source }} | sort | uniq" + register: staging_internal_repos + + - name: Get list of internal stable repos + when: ( stage == "promote_chart") + shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project), secret: \"{{ harbor_secret_mounted_path }}\", name: .project}' {{ loop_source }} | sort | uniq" + register: internal_repos + + - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml + when: ( stage == "promote_chart") + loop: "{{ staging_internal_repos.stdout_lines + internal_repos.stdout_lines}}" + vars: + repo: "{{ item }}" + + - include_tasks: ./roles/promote/tasks/main.yaml + when: data.stdout_lines |length > 0 + loop: "{{ data.stdout_lines | map('from_json') | batch(async_batch_size) | list }}" + loop_control: + loop_var: "items" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/main.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/main.yaml index 8baca73c..ec097920 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/main.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/main.yaml @@ -1,36 +1,118 @@ -- name: Helm Lint - block: - - name: Helm lint "{{ chart_name }}" - command: "helm lint {{ chart_name }}" - args: - chdir: "{{ build.checkout_loc }}/{{ path }}" - - name: Helm Dry-run "{{ chart_name }}" - command: "helm install --dry-run {{ chart_name }} {{ chart_name }}" - args: - chdir: "{{ build.checkout_loc }}/{{ path }}" - when: ( stage == "lint") - become: true -- name: Helm Package - block: - - name: Package Helm Chart - shell: helm package "{{ chart_name }}" - args: - chdir: "{{ build.checkout_loc }}/{{ path }}" - when: ( stage == "package") -- name: Helm Publish - block: - - name: Install Plugin - shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true - - name: Get harbor username - shell: cat {{ harbor_secret_mounted_path }}/username - register: harbor_username - - name: Get harbor password - shell: cat {{ harbor_secret_mounted_path }}/password - register: harbor_password - - name: Add Harbor Helm repository and Test repository - shell: helm repo add "{{ chart_repository }}-staging" "https://{{ docker_registry }}/chartrepo/{{ chart_name }}-staging" --username={{ harbor_username.stdout }} --password={{ harbor_password.stdout }} - - name: Push chart "{{ chart_name }}" to Harbor staging registry - command: helm push "{{ chart_name }}-{{ version }}".tgz "{{ chart_repository }}-staging" - args: - chdir: "{{ build.checkout_loc }}/{{ path }}" - when: ( stage == "publish") +- set_fact: + _async: "{{ chart_async | default(600) }}" + _delay: "{{ chart_check_status_delay | default(5) }}" + +- name: Helm Lint and Dry-Run + when: + - stage == "lint" + block: + - name: Helm lint + command: "helm lint {{ chart.chart_name }}" + args: + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', True) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: lint_async_results + + - name: Wait for helm lint results + vars: + async_results: "{{ lint_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + + - name: Helm Dry-run + command: "helm install --dry-run {{ chart.chart_name }} {{ chart.chart_name }}" + args: + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', True) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: dry_run_async_results + + - name: Wait for helm dry-run results + vars: + async_results: "{{ dry_run_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true + +- name: Helm Package + when: + - stage == "package" + block: + - name: Package Helm Chart + shell: helm package "{{ chart.chart_name }}" + args: + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', True) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: package_async_results + + - name: Wait for helm package results + vars: + async_results: "{{ package_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + +- name: Fetch Package + when: + - stage == "package" + block: + - name: Create target location for fetching charts + file: + state: directory + path: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', False) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: create_target_folder_async_results + + - name: Wait for target location setup results + vars: + async_results: "{{ create_target_folder_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + + - name: Fetch Helm charts + shell: helm repo update && helm fetch {{ chart.build.local_repo }}/{{ chart.chart_name }} --version {{ chart.version }} + args: + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts | selectattr('build_from_source', 'equalto', False) | list }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: fetch_async_results + + - name: Wait for helm fetch results + vars: + async_results: "{{ fetch_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + + +- name: Helm Publish + when: ( stage == "publish") + block: + - name: Install Plugin + shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true + - name: Push chart to Harbor staging registry + command: helm push "{{ chart.chart_name }}-{{ chart.version }}".tgz "{{ chart.project }}-staging" + args: + chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" + loop: "{{ charts }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: push_async_results + + - name: Wait for helm push results + vars: + async_results: "{{ push_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml diff --git a/tools/images/standard-container/assets/playbooks/roles/charts/tasks/setup_helm_repo.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/setup_helm_repo.yaml similarity index 100% rename from tools/images/standard-container/assets/playbooks/roles/charts/tasks/setup_helm_repo.yaml rename to tools/gate/jarvis/standard-container/assets/playbooks/roles/charts/tasks/setup_helm_repo.yaml diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml index ca9a0680..ef49bab2 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml @@ -1,3 +1,18 @@ #Remove any resources deployed +- set_fact: + _async: "{{ chart_async | default(600) }}" + _delay: "{{ chart_check_status_delay | default(5) }}" + - name: Remove test release - shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ release_name }}" -n "{{ namespace }}" + shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" -n "{{ chart.namespace }}" + loop: "{{ charts }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: cleanup_async_results + +- name: Wait for helm delete results + vars: + async_results: "{{ cleanup_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml diff --git a/tools/images/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml similarity index 88% rename from tools/images/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml rename to tools/gate/jarvis/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml index 46d5d06a..aa625866 100644 --- a/tools/images/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/common/tasks/check_sync_status.yaml @@ -1,6 +1,6 @@ - set_fact: _delay: "{{ _delay |default(5) }}" - _retries: "{{ _retries |default(30) }}" + _retries: "{{ _retries |default(300) }}" - name: Check sync status async_status: diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/functional/tasks/main.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/functional/tasks/main.yaml index 7c31d737..4bb86052 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/functional/tasks/main.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/functional/tasks/main.yaml @@ -1,22 +1,30 @@ -- name: Functional Deploy - block: - #Deploy CNF - - name: Get harbor username - shell: cat {{ harbor_secret_mounted_path }}/username - register: harbor_username - - name: Get harbor password - shell: cat {{ harbor_secret_mounted_path }}/password - register: harbor_password - #TODO dex-aio doesn't install, look into another test chart - - name: Add Harbor Helm repository and Test repository - shell: helm repo add "{{ chart_repository }}-staging" "https://{{ docker_registry }}/chartrepo/{{ project }}-staging" --username={{ harbor_username.stdout }} --password={{ harbor_password.stdout }} - - name: Deploy chart "{{ chart_name }}" - shell: helm upgrade --install --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ release_name }}" "{{ chart_repository }}-staging/{{ chart_name }}" --version="{{ version }}" --namespace="{{ namespace }}" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" --create-namespace - when: ("{{ stage }}" == "deploy") - become: true -- name: Functional Test - block: - #Test Deployed CNF - - name: Chart currently has no Helm Tests, echo for now - shell: echo "There are no helm tests yet" - when: ( stage == "test") +- set_fact: + _async: "{{ chart_async | default(600) }}" + _delay: "{{ chart_check_status_delay | default(5) }}" + +- name: Functional Deploy + when: ("{{ stage }}" == "deploy") + block: + #Deploy CNF + + - name: Deploy charts + shell: helm upgrade --install --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" "{{ chart.project }}-staging/{{ chart.chart_name }}" --version="{{ chart.version }}" --namespace="{{ chart.namespace }}" + loop: "{{ charts }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: install_async_results + + - name: Wait for helm install results + vars: + async_results: "{{ install_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true + +- name: Functional Test + when: ( stage == "test") + block: + #Test Deployed CNF + - name: Chart currently has no Helm Tests, echo for now + shell: echo "There are no helm tests yet" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/git/tasks/main.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/git/tasks/main.yaml index d9c4dc58..456f9122 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/git/tasks/main.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/git/tasks/main.yaml @@ -1,9 +1,26 @@ #Build docker image using Makefile given git repository location to clone code from -- name: Clone repository - block: - - git: - repo: "{{ build.git_repo }}" - dest: "{{ build.checkout_loc }}" - version: "{{ build.refspec }}" - refspec: "refs/changes/*:refs/changes/*" - when: ( stage == "clone") +- set_fact: + _async: "{{ git_async | default(600) }}" + _delay: "{{ git_check_status_delay | default(3) }}" + +- name: Run async git clone + git: + repo: "{{ repo.build.git_repo }}" + dest: "{{ repo.build.target_loc }}" + version: "{{ repo.build.refspec }}" + refspec: "refs/changes/*:refs/changes/*" + async: "{{ _async }}" + poll: 0 + loop: "{{ repos }}" + loop_control: + loop_var: "repo" + register: git_async_results + when: + - stage == "clone" + +- name: Wait for git clone results + vars: + async_results: "{{ git_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + when: + - stage == "clone" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/images/tasks/main.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/images/tasks/main.yaml index 1a9ae12d..c5d2aab3 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/images/tasks/main.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/images/tasks/main.yaml @@ -1,45 +1,123 @@ - -- name: Image Build - block: - #Build docker image using Makefile given git repository location to clone code from - - name: Build Docker Image for "{{ image_name }}" - shell: docker build -t "{{ image_fullname }}" . - args: - chdir: "{{ build.checkout_loc }}/{{ path }}" - when: ("{{ stage }}" == "build") - become: true -- name: Tag and Push Image - block: - - name: Tag image to Harbor url - shell: docker tag "{{ image_fullname }}" "{{ docker_registry }}/{{ project }}-staging/{{ repo }}:{{ tag }}" - - name: Push image to Harbor - shell: docker push "{{ docker_registry }}/{{ project }}-staging/{{ repo }}:{{ tag }}" - when: ( stage == "push") -- name: Get Scan Results - block: - #Scan results may take some time, putting in some retries and a delay to determine if scan results get finished - - name: output the request - shell: echo "https://{{ docker_registry }}/api/v2.0/projects/{{ project }}-staging/repositories/{{ repo | replace('/','%2F') }}/artifacts/{{ tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" - - - name: Get Scan Results - uri: - validate_certs: false - url: "https://{{ docker_registry }}/api/v2.0/projects/{{ project }}-staging/repositories/{{ repo | replace('/','%2F') }}/artifacts/{{ tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" - method: GET - body_format: "json" - headers: - accept: "application/json" - X-Request-Id: "12345" - #Change to encoded from configmap - authorization: "Basic YWRtaW46SGFyYm9yMTIzNDU=" - register: result - until: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].scan_status == "Success" - retries: 5 - delay: 30 - - - name: Check Scan Results Summary for High and Critical CVE - #shell: echo '{{ result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"] }}' - set_fact: - image_status: "Vulnerable" - when: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].severity in ("High","Critical") - when: ( stage == "scan_results") \ No newline at end of file +- set_fact: + _async: "{{ image_async | default(1800) }}" + _delay: "{{ image_check_status_delay | default(20) }}" + +- name: Image Build + when: + - stage == "build" + block: + - name: Build Docker Image + shell: | + docker build -t "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" . + args: + chdir: "{{ image.build.target_loc }}/{{ image.path }}" + loop: "{{ images | selectattr('build_from_source', 'equalto', True) | list }}" + loop_control: + pause: 5 + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: build_async_results + + - name: Wait for Build Docker Image results + vars: + async_results: "{{ build_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + +- name: Push Built Image + when: ( stage == "push" ) + block: + - name: Docker Tag + shell: | + docker push "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" + loop: "{{ images | selectattr('build_from_source', 'equalto', True) | list }}" + loop_control: + pause: 5 + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: push_async_results + + - name: Wait for Docker Push results + vars: + async_results: "{{ push_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true + +- name: Image Pull + when: + - stage == "build" + block: + #Build docker image using Makefile given git repository location to clone code from + - name: Pull Docker image + shell: | + docker pull "{{ image.build.git_repo }}:{{ image.build.tag }}" + loop: "{{ images | selectattr('build_from_source', 'equalto', False) | list }}" + loop_control: + pause: 5 + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: pull_async_results + - name: Wait for Docker Pull Image results + vars: + async_results: "{{ pull_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true + +- name: Tag and Push Image + when: ( stage == "push") + block: + - name: Docker Tag + shell: | + docker tag "{{ image.build.git_repo }}:{{ image.build.tag }}" "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" + docker push "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" + loop: "{{ images | selectattr('build_from_source', 'equalto', False) | list }}" + loop_control: + pause: 5 + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: tag_async_results + + - name: Wait for Docker Tag and Push results + vars: + async_results: "{{ tag_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + +- name: Get Scan Results + when: ( stage == "scan_results") + block: + #Scan results may take some time, putting in some retries and a delay to determine if scan results get finished + - name: output the request + shell: echo "https://{{ docker_registry }}/api/v2.0/projects/{{ image.project }}-staging/repositories/{{ image.repo | replace('/','%2F') }}/artifacts/{{ image.tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" + loop: "{{ images }}" + loop_control: + loop_var: "image" + + - name: Get Scan Results + uri: + validate_certs: false + url: "https://{{ docker_registry }}/api/v2.0/projects/{{ image.project }}-staging/repositories/{{ image.repo | replace('/','%2F') }}/artifacts/{{ image.tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" + method: GET + body_format: "json" + headers: + accept: "application/json" + X-Request-Id: "12345" + #TODO Change to encoded from configmap + authorization: "Basic YWRtaW46SGFyYm9yMTIzNDU=" + loop: "{{ images }}" + loop_control: + loop_var: "image" + register: result + until: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].scan_status == "Success" + retries: 5 + delay: 30 + + - name: Check Scan Results Summary for High and Critical CVE + set_fact: + image_status: "Vulnerable" + when: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].severity in ("High","Critical") + loop: "{{ result.results }}" + loop_control: + loop_var: "result" diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/kubernetes/tasks/get-kubeconfig.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/kubernetes/tasks/get-kubeconfig.yaml index f1db7c17..d43947c2 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/kubernetes/tasks/get-kubeconfig.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/kubernetes/tasks/get-kubeconfig.yaml @@ -1,10 +1,7 @@ #Validate Kubernetes cluster is accessible. - set_fact: existing_cluster: "{{ use_existing_cluster }}" -- name: Get kubeconfig - shell: echo "Retrieve kubeconfig from mounted secret" - when: existing_cluster - name: Validate kubeconfig - shell: kubectl --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" get pods -n development-pipeline + shell: kubectl --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" get pods -n "{{ namespace }}" - name: Find existing service accounts - shell: kubectl --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" get serviceaccounts -n development-pipeline \ No newline at end of file + shell: kubectl --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" get serviceaccounts -n "{{ namespace }}" \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/promote/tasks/main.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/promote/tasks/main.yaml index 3a47c5d7..adcc5b07 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/promote/tasks/main.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/promote/tasks/main.yaml @@ -1,29 +1,82 @@ -- name: Promote Chart - block: - - name: Install Plugin - shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true - - name: Get harbor username - shell: cat {{ harbor_secret_mounted_path }}/username - register: harbor_username - - name: Get harbor password - shell: cat {{ harbor_secret_mounted_path }}/password - register: harbor_password - - name: Install Plugin - shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push - - name: Add Harbor Helm repository and Test repository - shell: helm repo add "{{ chart_repository }}-staging" "https://{{ chart_registry_url }}/{{ chart_name }}-staging" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" && helm repo add "{{ chart_repository }}" "https://{{ chart_registry_url }}/{{ chart_name }}" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" - - name: Pull down Helm Chart - shell: helm pull "{{ chart_repository }}-staging/{{ chart_name }}" --version="{{ version }}" - - name: Push chart "{{ chart_name }}" to Helm registry - command: helm push "{{ chart_name }}-{{ version }}".tgz "{{ chart_repository }}" - when: ( stage == "promote_chart") - become: true -- name: Promote Image - block: - - name: Pull image locally - shell: docker pull "{{ docker_registry }}/{{ project }}-staging/{{ repo }}:{{ tag }}" - - name: Tag image - shell: docker tag "{{ docker_registry }}/{{ project }}-staging/{{ repo }}:{{ tag }}" "{{ docker_registry }}/{{ project }}/{{ repo }}:{{ tag }}" - - name: Push to non-staging Harbor Project - shell: docker push "{{ docker_registry }}/{{ project }}/{{ repo }}:{{ tag }}" - when: ( stage == "promote_image") +- set_fact: + _async: "{{ chart_async | default(600) }}" + _delay: "{{ chart_check_status_delay | default(5) }}" + when: ( stage == "promote_chart") + +- set_fact: + _async: "{{ image_async | default(1200) }}" + _delay: "{{ image_check_status_delay | default(20) }}" + when: ( stage == "promote_image") + +- name: Promote Chart + when: ( stage == "promote_chart") + block: + - name: Install Plugin + shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true + + - name: Pull down Helm Chart + shell: helm pull "{{ chart.project }}-staging/{{ chart.chart_name }}" --version="{{ chart.version }}" + loop: "{{ items }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: pull_async_results + + - name: Wait for helm pull results + vars: + async_results: "{{ pull_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + + - name: Push charts to Helm registry + command: helm push "{{ chart.chart_name }}-{{ chart.version }}".tgz "{{ chart.project }}" + loop: "{{ items }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: push_async_results + + - name: Wait for helm push results + vars: + async_results: "{{ push_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true + +- name: Promote Image + when: ( stage == "promote_image") + block: + - name: Pull image from staging repo + docker_image: + pull: true + name: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" + loop: "{{ items }}" + loop_control: + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: image_pull_async_results + + - name: Wait for docker pull results + vars: + async_results: "{{ image_pull_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + + - name: Tag and push to promotion repository + docker_image: + push: true + name: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}" + repository: "{{ docker_registry }}/{{ image.project }}/{{ image.repo }}" + tag: "{{ image.tag }}" + loop: "{{ items }}" + loop_control: + loop_var: "image" + async: "{{ _async }}" + poll: 0 + register: image_push_async_results + environment: "{{ docker_content_trust }}" + + - name: Wait for docker push results + vars: + async_results: "{{ image_push_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml diff --git a/tools/images/standard-container/Dockerfile b/tools/images/standard-container/Dockerfile deleted file mode 100644 index 047f8a4d..00000000 --- a/tools/images/standard-container/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -ARG BASE_IMAGE -FROM ${BASE_IMAGE} - -SHELL ["bash", "-exc"] -ENV DEBIAN_FRONTEND noninteractive - -# Update distro and install ansible -RUN apt-get update ;\ - apt-get dist-upgrade -y ;\ - apt-get install -y \ - python3-minimal \ - python3-pip \ - python3-setuptools \ - make \ - sudo \ - git \ - jq \ - curl \ - git-review \ - apt-transport-https \ - ca-certificates \ - gnupg-agent \ - software-properties-common \ - moreutils \ - gettext-base ;\ - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - ;\ - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" ;\ - apt-get install -y --no-install-recommends \ - docker-ce-cli ;\ - pip3 install --upgrade wheel ;\ - pip3 install ansible==2.10.7 ;\ - ansible-galaxy collection install community.kubernetes ;\ - pip3 install docker ;\ - # Install kubectl - apt-get install -y --no-install-recommends \ - apt-transport-https \ - gnupg2 ;\ - curl -o /usr/bin/kubectl -L "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" ;\ - chmod +x /usr/bin/kubectl ;\ - # Install Helm - curl -fsSL -o helm-install.tar.gz https://get.helm.sh/helm-v3.4.2-linux-amd64.tar.gz \ - && tar -xvf helm-install.tar.gz \ - && rm helm-install.tar.gz \ - && mv linux-amd64/helm /usr/local/bin/helm \ - && chmod +x /usr/local/bin/helm \ - && helm --help ;\ - helm plugin install https://github.com/chartmuseum/helm-push ;\ - rm -rf /var/lib/apt/lists/* - -COPY assets /opt/assets/ -RUN cp -ravf /opt/assets/* / ;\ - rm -rf /opt/assets - -ENTRYPOINT /entrypoint.sh diff --git a/tools/images/standard-container/assets/entrypoint.sh b/tools/images/standard-container/assets/entrypoint.sh deleted file mode 100755 index dd64608b..00000000 --- a/tools/images/standard-container/assets/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -ansible --version diff --git a/tools/images/standard-container/assets/playbooks/charts-microflow.yaml b/tools/images/standard-container/assets/playbooks/charts-microflow.yaml deleted file mode 100644 index d25f23d8..00000000 --- a/tools/images/standard-container/assets/playbooks/charts-microflow.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -- hosts: localhost - tasks: - - name: Get list of charts - shell: "jq -c '.[]' {{ loop_source }}" - register: data - - - set_fact: - chart_async_batch_size: 20 - stage: "default" - - - name: Get list of remote repos - when: (stage == "package") - shell: "jq -c '.[] | select(.build_from_source==false) | {repo: .build.repo, secret: .build.secret, name: .build.local_repo}' {{ loop_source }} | sort | uniq" - register: external_repos - - - name: Get list of internal repos - when: (stage == "publish") - shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name: (.project+\"-staging\")}' {{ loop_source }} | sort | uniq" - register: internal_repos - - - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml - when: (stage == "package") or (stage == "publish") - vars: - repo: "{{ item }}" - loop: "{{ external_repos.stdout_lines if stage == 'package' else internal_repos.stdout_lines}}" - - - include_tasks: ./roles/charts/tasks/main.yaml - when: data.stdout_lines |length > 0 - loop: "{{ data.stdout_lines | map('from_json') | batch(chart_async_batch_size) | list }}" - loop_control: - loop_var: "charts" diff --git a/tools/images/standard-container/assets/playbooks/cleanup.yaml b/tools/images/standard-container/assets/playbooks/cleanup.yaml deleted file mode 100644 index 913bb6dd..00000000 --- a/tools/images/standard-container/assets/playbooks/cleanup.yaml +++ /dev/null @@ -1,16 +0,0 @@ -- hosts: localhost - become: yes - tasks: - - name: Get list of charts - shell: "jq -c '.[]' {{ loop_chart_source }}" - register: data - - - set_fact: - chart_async_batch_size: 20 - stage: "default" - - - vars: - charts: "{{ item }}" - include_tasks: ./roles/cleanup/tasks/cleanup.yaml - when: data.stdout_lines |length > 0 - loop: "{{ data.stdout_lines | map('from_json') | batch(chart_async_batch_size) | list }}" diff --git a/tools/images/standard-container/assets/playbooks/deployment-manifests.yaml b/tools/images/standard-container/assets/playbooks/deployment-manifests.yaml deleted file mode 100644 index 78a0d982..00000000 --- a/tools/images/standard-container/assets/playbooks/deployment-manifests.yaml +++ /dev/null @@ -1,5 +0,0 @@ -- hosts: localhost - become: yes - tasks: - - name: validate-deployment-manifests-config - include_tasks: ./roles/deployment-manifests/tasks/validate-config.yaml \ No newline at end of file diff --git a/tools/images/standard-container/assets/playbooks/functional-microflow.yaml b/tools/images/standard-container/assets/playbooks/functional-microflow.yaml deleted file mode 100644 index 34934f28..00000000 --- a/tools/images/standard-container/assets/playbooks/functional-microflow.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- hosts: localhost - tasks: - - name: Get list of charts - shell: "jq -c '.[]' {{ loop_source }}" - register: data - - - set_fact: - chart_async_batch_size: 20 - stage: "default" - - - name: Get list of internal repos - when: (stage == 'deploy') - shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name: (.project+\"-staging\")}' {{ loop_source }} | sort | uniq" - register: internal_repos - - - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml - when: (stage == 'deploy') - loop: "{{ internal_repos.stdout_lines }}" - vars: - repo: "{{ item }}" - - - include_tasks: ./roles/functional/tasks/main.yaml - when: data.stdout_lines |length > 0 - loop: "{{ data.stdout_lines | map('from_json') | batch(chart_async_batch_size) | list }}" - loop_control: - loop_var: "charts" diff --git a/tools/images/standard-container/assets/playbooks/get-kubeconfig.yaml b/tools/images/standard-container/assets/playbooks/get-kubeconfig.yaml deleted file mode 100644 index 66c6b354..00000000 --- a/tools/images/standard-container/assets/playbooks/get-kubeconfig.yaml +++ /dev/null @@ -1,5 +0,0 @@ -- hosts: localhost - become: yes - tasks: - - name: Get kubeconfig for Kubernetes cluster to deploy CNF - include_tasks: ./roles/kubernetes/tasks/get-kubeconfig.yaml diff --git a/tools/images/standard-container/assets/playbooks/git-microflow.yaml b/tools/images/standard-container/assets/playbooks/git-microflow.yaml deleted file mode 100644 index 6ce1f224..00000000 --- a/tools/images/standard-container/assets/playbooks/git-microflow.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- hosts: localhost - tasks: - - name: Get list of repos to clone - shell: "jq -c '.[] | select(.build_from_source) | {build: .build}' {{ loop_source }} | sort | uniq" - register: data - - - set_fact: - clone_async_batch_size: 20 - stage: "default" - - - name: Run git clone task - include_tasks: ./roles/git/tasks/main.yaml - when: data.stdout_lines |length > 0 - loop: "{{ data.stdout_lines | map('from_json') | batch(clone_async_batch_size) | list }}" - loop_control: - loop_var: "repos" diff --git a/tools/images/standard-container/assets/playbooks/hosts b/tools/images/standard-container/assets/playbooks/hosts deleted file mode 100644 index 2302edae..00000000 --- a/tools/images/standard-container/assets/playbooks/hosts +++ /dev/null @@ -1 +0,0 @@ -localhost ansible_connection=local diff --git a/tools/images/standard-container/assets/playbooks/images-microflow.yaml b/tools/images/standard-container/assets/playbooks/images-microflow.yaml deleted file mode 100644 index 6ab275e6..00000000 --- a/tools/images/standard-container/assets/playbooks/images-microflow.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- hosts: localhost - tasks: - - name: Get list of images - shell: "jq -c '.[]' {{ loop_source }}" - register: data - - - set_fact: - image_async_batch_size: 2 - stage: "default" - - - include_tasks: ./roles/images/tasks/main.yaml - when: data.stdout_lines |length > 0 - loop: "{{ data.stdout_lines | map('from_json') | batch(image_async_batch_size) | list }}" - loop_control: - loop_var: "images" diff --git a/tools/images/standard-container/assets/playbooks/promote-microflow.yaml b/tools/images/standard-container/assets/playbooks/promote-microflow.yaml deleted file mode 100644 index b1f939e5..00000000 --- a/tools/images/standard-container/assets/playbooks/promote-microflow.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -- hosts: localhost - tasks: - - name: Get list of objects - shell: "jq -c '.[]' {{ loop_source }}" - register: data - - - set_fact: - async_batch_size: 2 - stage: "default" - - - name: Get list of internal staging repos - when: ( stage == "promote_chart") - shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name: (.project+\"-staging\")}' {{ loop_source }} | sort | uniq" - register: staging_internal_repos - - - name: Get list of internal stable repos - when: ( stage == "promote_chart") - shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project), secret: \"{{ harbor_secret_mounted_path }}\", name: .project}' {{ loop_source }} | sort | uniq" - register: internal_repos - - - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml - when: ( stage == "promote_chart") - loop: "{{ staging_internal_repos.stdout_lines + internal_repos.stdout_lines}}" - vars: - repo: "{{ item }}" - - - include_tasks: ./roles/promote/tasks/main.yaml - when: data.stdout_lines |length > 0 - loop: "{{ data.stdout_lines | map('from_json') | batch(async_batch_size) | list }}" - loop_control: - loop_var: "items" diff --git a/tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml deleted file mode 100644 index ec097920..00000000 --- a/tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml +++ /dev/null @@ -1,118 +0,0 @@ -- set_fact: - _async: "{{ chart_async | default(600) }}" - _delay: "{{ chart_check_status_delay | default(5) }}" - -- name: Helm Lint and Dry-Run - when: - - stage == "lint" - block: - - name: Helm lint - command: "helm lint {{ chart.chart_name }}" - args: - chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" - loop: "{{ charts | selectattr('build_from_source', 'equalto', True) | list }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: lint_async_results - - - name: Wait for helm lint results - vars: - async_results: "{{ lint_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - - - name: Helm Dry-run - command: "helm install --dry-run {{ chart.chart_name }} {{ chart.chart_name }}" - args: - chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" - loop: "{{ charts | selectattr('build_from_source', 'equalto', True) | list }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: dry_run_async_results - - - name: Wait for helm dry-run results - vars: - async_results: "{{ dry_run_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - become: true - -- name: Helm Package - when: - - stage == "package" - block: - - name: Package Helm Chart - shell: helm package "{{ chart.chart_name }}" - args: - chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" - loop: "{{ charts | selectattr('build_from_source', 'equalto', True) | list }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: package_async_results - - - name: Wait for helm package results - vars: - async_results: "{{ package_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - -- name: Fetch Package - when: - - stage == "package" - block: - - name: Create target location for fetching charts - file: - state: directory - path: "{{ chart.build.target_loc }}/{{ chart.path }}" - loop: "{{ charts | selectattr('build_from_source', 'equalto', False) | list }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: create_target_folder_async_results - - - name: Wait for target location setup results - vars: - async_results: "{{ create_target_folder_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - - - name: Fetch Helm charts - shell: helm repo update && helm fetch {{ chart.build.local_repo }}/{{ chart.chart_name }} --version {{ chart.version }} - args: - chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" - loop: "{{ charts | selectattr('build_from_source', 'equalto', False) | list }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: fetch_async_results - - - name: Wait for helm fetch results - vars: - async_results: "{{ fetch_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - - -- name: Helm Publish - when: ( stage == "publish") - block: - - name: Install Plugin - shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true - - name: Push chart to Harbor staging registry - command: helm push "{{ chart.chart_name }}-{{ chart.version }}".tgz "{{ chart.project }}-staging" - args: - chdir: "{{ chart.build.target_loc }}/{{ chart.path }}" - loop: "{{ charts }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: push_async_results - - - name: Wait for helm push results - vars: - async_results: "{{ push_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml diff --git a/tools/images/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml b/tools/images/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml deleted file mode 100644 index ef49bab2..00000000 --- a/tools/images/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml +++ /dev/null @@ -1,18 +0,0 @@ -#Remove any resources deployed -- set_fact: - _async: "{{ chart_async | default(600) }}" - _delay: "{{ chart_check_status_delay | default(5) }}" - -- name: Remove test release - shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" -n "{{ chart.namespace }}" - loop: "{{ charts }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: cleanup_async_results - -- name: Wait for helm delete results - vars: - async_results: "{{ cleanup_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml diff --git a/tools/images/standard-container/assets/playbooks/roles/deployment-manifests/tasks/validate-config.yaml b/tools/images/standard-container/assets/playbooks/roles/deployment-manifests/tasks/validate-config.yaml deleted file mode 100644 index d272b852..00000000 --- a/tools/images/standard-container/assets/playbooks/roles/deployment-manifests/tasks/validate-config.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- name: This task is to validate deployment manifests - command: echo "It can do anything you want, just put it right here." - args: - chdir: "/workspace" \ No newline at end of file diff --git a/tools/images/standard-container/assets/playbooks/roles/functional/tasks/functional-test.yaml b/tools/images/standard-container/assets/playbooks/roles/functional/tasks/functional-test.yaml deleted file mode 100644 index a2c58955..00000000 --- a/tools/images/standard-container/assets/playbooks/roles/functional/tasks/functional-test.yaml +++ /dev/null @@ -1,4 +0,0 @@ -#Test Deployed CNF -- name: Chart has no Helm Tests, echo for now - shell: echo "There are no helm tests yet" -# shell: echo 'helm test --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ name }}"' \ No newline at end of file diff --git a/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml deleted file mode 100644 index ff00b7a0..00000000 --- a/tools/images/standard-container/assets/playbooks/roles/functional/tasks/main.yaml +++ /dev/null @@ -1,57 +0,0 @@ -- set_fact: - _async: "{{ chart_async | default(600) }}" - _delay: "{{ chart_check_status_delay | default(5) }}" - -- name: Functional Deploy - when: ("{{ stage }}" == "deploy") - block: - #Deploy CNF - - name: Deploy charts - shell: | - set -xe ; - CHARTNAME={{ chart.chart_name }} ; - REPO={{ docker_registry }} ; - NAME={{ chart.project }}-staging/{{ chart.chart_name }} ; - - jq -c --arg c $CHARTNAME --arg v $REPO 'map(if .repo == $c then .["tmp_repo"] = $v else . end)' {{ datapath }}/image.json | sponge {{ datapath }}/image.json ; - jq -c --arg c $CHARTNAME --arg v $NAME 'map(if .repo == $c then .["tmp_name"] = $v else . end)' {{ datapath }}/image.json | sponge {{ datapath }}/image.json ; - - M=$(jq -c --arg c $CHARTNAME 'map(select(.chart_name == $c)) | first | .sources.image_map' {{ datapath }}/chart.json) ; - if [ "$M" != "null" ] && [ -n "$M" ] ; then \ - echo $M | jq -r 'keys[]' | while IFS= read -r k; do \ - v=$(echo $M | jq -c --arg k $k '.[$k]') ;\ - filter=".$k" ;\ - value=$(jq -c --arg c $CHARTNAME 'map(select(.image_name == $c)) | first' {{ datapath}}/image.json | jq -r $filter) ;\ - jq -r --arg c $CHARTNAME --argjson k $v --arg v $value 'map(if .chart_name == $c then getpath($k) = $v else . end)' {{ datapath }}/chart.json | sponge {{ datapath }}/chart.json ;\ - done ; \ - fi ; - jq -c --arg c $CHARTNAME 'map(select(.chart_name == $c)) | first | .releases' \ - {{ datapath }}/chart.json > {{ datapath }}/{{ chart.chart_name }}-overrides.json ; - cat {{ datapath }}/{{ chart.chart_name }}-overrides.json ; - helm upgrade --install \ - --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" \ - "{{ chart.release_name }}" "{{ chart.project }}-staging/{{ chart.chart_name }}" \ - --version="{{ chart.version }}" \ - --values="{{ datapath }}/{{ chart.chart_name }}-overrides.json" \ - --namespace="{{ chart.namespace }}" - args: - executable: /bin/bash - loop: "{{ charts }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: install_async_results - - - name: Wait for helm install results - vars: - async_results: "{{ install_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - become: true - -- name: Functional Test - when: ( stage == "test") - block: - #Test Deployed CNF - - name: Chart currently has no Helm Tests, echo for now - shell: echo "There are no helm tests yet" diff --git a/tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml deleted file mode 100644 index 2af0aec1..00000000 --- a/tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml +++ /dev/null @@ -1,26 +0,0 @@ -#Build docker image using Makefile given git repository location to clone code from -- set_fact: - _async: "{{ git_async | default(600) }}" - _delay: "{{ git_check_status_delay | default(3) }}" - -- name: Run async git clone - git: - repo: "{{ repo.build.repo }}" - dest: "{{ repo.build.target_loc }}" - version: "{{ repo.build.refspec }}" - refspec: "refs/changes/*:refs/changes/*" - async: "{{ _async }}" - poll: 0 - loop: "{{ repos }}" - loop_control: - loop_var: "repo" - register: git_async_results - when: - - stage == "clone" - -- name: Wait for git clone results - vars: - async_results: "{{ git_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - when: - - stage == "clone" diff --git a/tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml deleted file mode 100644 index c694d146..00000000 --- a/tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml +++ /dev/null @@ -1,109 +0,0 @@ -- set_fact: - _async: "{{ image_async | default(1200) }}" - _delay: "{{ image_check_status_delay | default(20) }}" - -- name: Image Build - when: - - stage == "build" - block: - #Build docker image using Makefile given git repository location to clone code from - - name: Build Docker Image - shell: docker build -t "{{ image.image_fullname }}" . - args: - chdir: "{{ image.build.target_loc }}/{{ image.path }}" - loop: "{{ images | selectattr('build_from_source', 'equalto', True) | list }}" - loop_control: - pause: 5 - loop_var: "image" - async: "{{ _async }}" - poll: 0 - register: build_async_results - - - name: Wait for Build Docker Image results - vars: - async_results: "{{ build_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - become: true - -- name: Image Pull - when: - - stage == "build" - block: - #Build docker image using Makefile given git repository location to clone code from - - name: Pull Docker image - docker_image: - pull: true - repository: "{{ image.image_fullname }}" - name: "{{ image.build.repo }}:{{ image.build.tag }}" - loop: "{{ images | selectattr('build_from_source', 'equalto', False) | list }}" - loop_control: - pause: 5 - loop_var: "image" - async: "{{ _async }}" - poll: 0 - register: pull_async_results - - - name: Wait for Docker Pull Image results - vars: - async_results: "{{ pull_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - become: true - -- name: Tag and Push Image - when: ( stage == "push") - block: - - name: Tag and push to internal test repository for vulnerability scanning - docker_image: - push: true - name: "{{ image.image_fullname }}" - repository: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}" - tag: "{{ image.tag }}" - loop: "{{ images }}" - loop_control: - pause: 5 - loop_var: "image" - async: "{{ _async }}" - poll: 0 - register: tag_async_results - - - name: Wait for Docker Tag Images results - vars: - async_results: "{{ tag_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - -- name: Get Scan Results - when: ( stage == "scan_results") - block: - #Scan results may take some time, putting in some retries and a delay to determine if scan results get finished - - name: output the request - shell: echo "https://{{ docker_registry }}/api/v2.0/projects/{{ image.project }}-staging/repositories/{{ image.repo | replace('/','%2F') }}/artifacts/{{ image.tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" - loop: "{{ images }}" - loop_control: - loop_var: "image" - - - name: Get Scan Results - uri: - validate_certs: false - url: "https://{{ docker_registry }}/api/v2.0/projects/{{ image.project }}-staging/repositories/{{ image.repo | replace('/','%2F') }}/artifacts/{{ image.tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false" - method: GET - body_format: "json" - headers: - accept: "application/json" - X-Request-Id: "12345" - #Change to encoded from configmap - authorization: "Basic YWRtaW46SGFyYm9yMTIzNDU=" - loop: "{{ images }}" - loop_control: - loop_var: "image" - register: result - until: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].scan_status == "Success" - retries: 5 - delay: 30 - - - name: Check Scan Results Summary for High and Critical CVE - set_fact: - image_status: "Vulnerable" - when: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].severity in ("High","Critical") - loop: "{{ result.results }}" - loop_control: - loop_var: "result" diff --git a/tools/images/standard-container/assets/playbooks/roles/kubernetes/tasks/get-kubeconfig.yaml b/tools/images/standard-container/assets/playbooks/roles/kubernetes/tasks/get-kubeconfig.yaml deleted file mode 100644 index f1db7c17..00000000 --- a/tools/images/standard-container/assets/playbooks/roles/kubernetes/tasks/get-kubeconfig.yaml +++ /dev/null @@ -1,10 +0,0 @@ -#Validate Kubernetes cluster is accessible. -- set_fact: - existing_cluster: "{{ use_existing_cluster }}" -- name: Get kubeconfig - shell: echo "Retrieve kubeconfig from mounted secret" - when: existing_cluster -- name: Validate kubeconfig - shell: kubectl --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" get pods -n development-pipeline -- name: Find existing service accounts - shell: kubectl --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" get serviceaccounts -n development-pipeline \ No newline at end of file diff --git a/tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml b/tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml deleted file mode 100644 index a0421c54..00000000 --- a/tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml +++ /dev/null @@ -1,81 +0,0 @@ -- set_fact: - _async: "{{ chart_async | default(600) }}" - _delay: "{{ chart_check_status_delay | default(5) }}" - when: ( stage == "promote_chart") - -- set_fact: - _async: "{{ image_async | default(1200) }}" - _delay: "{{ image_check_status_delay | default(20) }}" - when: ( stage == "promote_image") - -- name: Promote Chart - when: ( stage == "promote_chart") - block: - - name: Install Plugin - shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true - - - name: Pull down Helm Chart - shell: helm pull "{{ chart.project }}-staging/{{ chart.chart_name }}" --version="{{ chart.version }}" - loop: "{{ items }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: pull_async_results - - - name: Wait for helm pull results - vars: - async_results: "{{ pull_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - - - name: Push charts to Helm registry - command: helm push "{{ chart.chart_name }}-{{ chart.version }}".tgz "{{ chart.project }}" - loop: "{{ items }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: push_async_results - - - name: Wait for helm push results - vars: - async_results: "{{ push_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - become: true - -- name: Promote Image - when: ( stage == "promote_image") - block: - - name: Pull image from staging repo - docker_image: - pull: true - name: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" - loop: "{{ items }}" - loop_control: - loop_var: "image" - async: "{{ _async }}" - poll: 0 - register: image_pull_async_results - - - name: Wait for docker pull results - vars: - async_results: "{{ image_pull_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml - - - name: Tag and push to promotion repository - docker_image: - push: true - name: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}" - repository: "{{ docker_registry }}/{{ image.project }}/{{ image.repo }}" - tag: "{{ image.tag }}" - loop: "{{ items }}" - loop_control: - loop_var: "image" - async: "{{ _async }}" - poll: 0 - register: image_push_async_results - - - name: Wait for docker push results - vars: - async_results: "{{ image_push_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml diff --git a/tools/images/standard-container/assets/setup-cluster-config.sh b/tools/images/standard-container/assets/setup-cluster-config.sh deleted file mode 100644 index 3c9d13f0..00000000 --- a/tools/images/standard-container/assets/setup-cluster-config.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json" -cp "$(workspaces.k8s_cluster_data.path)/cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json" -jq '.cluster_kubeconfig_path="$(workspaces.development_pipeline_data.path)/config"' "$(workspaces.development_pipeline_data.path)/cluster.json" > "$(workspaces.development_pipeline_data.path)/temp_cluster.json" && mv "$(workspaces.development_pipeline_data.path)/temp_cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json" diff --git a/tools/images/standard-container/assets/test_helm_repo_basic_auth.yaml b/tools/images/standard-container/assets/test_helm_repo_basic_auth.yaml deleted file mode 100644 index e0af040d..00000000 --- a/tools/images/standard-container/assets/test_helm_repo_basic_auth.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: harbor-helm-creds - namespace: development-pipeline -type: kubernetes.io/basic-auth -stringData: - username: admin - password: Harbor12345 \ No newline at end of file