Support import chart/image in development pipeline

Change-Id: I071d5d0c112b7dfdcc01c0b2b3d7500ef54ea2a2
2021-02-05 18:19:31 -06:00 · 2021-02-05 18:19:31 -06:00 · 417340ab3d
commit 417340ab3d
parent 0cef6abcbe
7 changed files with 96 additions and 22 deletions
--- a/charts/development-pipeline/config_map.yaml.example
+++ b/charts/development-pipeline/config_map.yaml.example
@ -8,6 +8,20 @@ data:
     "use_existing_cluster": "true"
    }
  image.json: |
+    {
+     "build_from_source": false,
+     "image_name": "mongodb",
+     "project": "mongodb",
+     "repo": "mongodb",
+     "tag": "1.built",
+     "image_fullname": "cnf/mongodb:1.0",
+     "path": "tools/gate/jarvis/5G-SA-core/mongodb/images/mongodb",
+     "build": {
+        "repo": "docker.io/bitnami/mongodb",
+        "tag": "4.4.3-debian-10-r42"
+     }
+    }
+  image_build_from_source.json: |
    {
     "build_from_source": true,
     "image_name": "mongodb",
@ -17,13 +31,37 @@ data:
     "image_fullname": "cnf/mongodb:1.0",
     "path": "tools/gate/jarvis/5G-SA-core/mongodb/images/mongodb",
     "build": {
-        "git_repo": "https://review.opendev.org/airship/charts",
-        "checkout_loc": "/src/checkout/mongodb",
+        "repo": "https://review.opendev.org/airship/charts",
+        "target_loc": "/src/checkout/mongodb",
        "refspec": "master",
        "version": "refs/changes/*:refs/changes/*"
     }
    }
  chart.json: |
+    {
+      "chart_name": "mongodb",
+      "project": "mongodb",
+      "repo": "mongodb",
+      "version": "10.6.2",
+      "build_from_source": false,
+      "path": "",
+      "build": {
+        "repo": "https://charts.bitnami.com/bitnami",
+        "target_loc": "/src/fetch/charts"
+      },
+      "namespace": "development-pipeline",
+      "release_name": "mongodb",
+      "images": {
+        "applications": {
+          "mongodb": {
+            "tag": "1.0",
+            "name": "mongodb",
+            "repo": "mongodb"
+          }
+        }
+      }
+    }
+  chart_build_from_source.json: |
    {
      "chart_name": "mongodb",
      "project": "mongodb",
@ -32,8 +70,8 @@ data:
      "build_from_source": true,
      "path": "tools/gate/jarvis/5G-SA-core/mongodb/charts",
      "build": {
-        "git_repo": "https://review.opendev.org/airship/charts",
-        "checkout_loc": "/src/checkout/airship/charts",
+        "repo": "https://review.opendev.org/airship/charts",
+        "target_loc": "/src/checkout/airship/charts",
        "refspec": "master"
      },
      "namespace": "development-pipeline",
--- a/charts/development-pipeline/templates/pipeline.yaml
+++ b/charts/development-pipeline/templates/pipeline.yaml
@ -57,6 +57,7 @@ spec:

    - name: microflow-images
      runAfter:
+        - microflow-setup-cluster-config
        - microflow-setup-image-config
      workspaces:
        - name: k8s_cluster_data
@ -68,6 +69,7 @@ spec:

    - name: microflow-charts
      runAfter:
+        - microflow-setup-cluster-config
        - microflow-setup-chart-config
      workspaces:
        - name: k8s_cluster_data
--- a/charts/development-pipeline/templates/task-setup-config.yaml
+++ b/charts/development-pipeline/templates/task-setup-config.yaml
@ -39,8 +39,8 @@ spec:
        jq '.image_fullname="test/scan-image:$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json"
        echo "Set tag to context $(context.taskRun.uid)"
        jq '.tag="$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json"
-        echo "Set checkout location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"
-        jq '.build.checkout_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json"
+        echo "Set target location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"
+        jq '.build.target_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json"
 ---
 apiVersion: tekton.dev/v1beta1
 kind: Task
@ -58,12 +58,12 @@ spec:
      image: {{ $.Values.tasks.setup.setupConfigImage }}
      script: |
        #!/usr/bin/env sh
-        cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json"
        cp "$(workspaces.k8s_cluster_data.path)/chart.json" "$(workspaces.development_pipeline_data.path)/chart.json"
        echo "Set tag to context $(context.taskRun.uid)"
        jq '.tag="$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json"
-        echo "Set checkout location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"
-        jq '.build.checkout_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json"
+        mkdir "$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"
+        echo "Set target location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"
+        jq '.build.target_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json"
 ---
 apiVersion: tekton.dev/v1beta1
 kind: Task
--- a/tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml
+++ b/tools/images/standard-container/assets/playbooks/roles/charts/tasks/main.yaml
@ -1,24 +1,41 @@
 - name: Helm Lint and Dry-Run
-  when: ( stage == "lint")
+  when:
+    - stage == "lint"
+    - build_from_source
  block:
    - name: Helm lint "{{ chart_name }}"
      command: "helm lint {{ chart_name }}"
      args:
-        chdir: "{{ build.checkout_loc }}/{{ path }}"
+        chdir: "{{ build.target_loc }}/{{ path }}"

    - name: Helm Dry-run "{{ chart_name }}"
      command: "helm install --dry-run {{ chart_name }} {{ chart_name }}"
      args:
-        chdir: "{{ build.checkout_loc }}/{{ path }}"
+        chdir: "{{ build.target_loc }}/{{ path }}"
  become: true

 - name: Helm Package
-  when: ( stage == "package")
+  when:
+    - stage == "package"
+    - build_from_source
  block:
    - name: Package Helm Chart
      shell: helm package "{{ chart_name }}"
      args:
-        chdir: "{{ build.checkout_loc }}/{{ path }}"
+        chdir: "{{ build.target_loc }}/{{ path }}"
+
+- name: Fetch Package
+  when:
+    - stage == "package"
+    - not build_from_source
+  block:
+    - name: Add remote helm repo
+      shell: helm repo add remote_{{ chart_name }} {{ build.repo }} && helm repo update
+
+    - name: Fetch {{ chart_name }}-{{ version }}
+      shell: helm fetch remote_{{ chart_name }}/{{ chart_name }} --version {{ version }}
+      args:
+        chdir: "{{ build.target_loc }}/{{ path }}"

 - name: Helm Publish
  when: ( stage == "publish")
@ -35,9 +52,9 @@
      register: harbor_password

    - name: Add Harbor Helm repository and Test repository
-      shell: helm repo add "{{ chart_repository }}-staging" "https://{{ docker_registry }}/chartrepo/{{ chart_name }}-staging" --username={{ harbor_username.stdout }} --password={{ harbor_password.stdout }}
+      shell: helm repo add "{{ chart_repository }}-staging" "https://{{ docker_registry }}/chartrepo/{{ project }}-staging" --username={{ harbor_username.stdout }} --password={{ harbor_password.stdout }}

    - name: Push chart "{{ chart_name }}" to Harbor staging registry
      command: helm push "{{ chart_name }}-{{ version }}".tgz "{{ chart_repository }}-staging"
      args:
-        chdir: "{{ build.checkout_loc }}/{{ path }}"
+        chdir: "{{ build.target_loc }}/{{ path }}"
--- a/tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml
+++ b/tools/images/standard-container/assets/playbooks/roles/git/tasks/main.yaml
@ -1,9 +1,11 @@
 #Build docker image using Makefile given git repository location to clone code from
 - name: Clone repository
-  when: ( stage == "clone")
+  when:
+    - stage == "clone"
+    - build_from_source
  block:
    - git:
-        repo: "{{ build.git_repo }}"
-        dest: "{{ build.checkout_loc }}"
+        repo: "{{ build.repo }}"
+        dest: "{{ build.target_loc }}"
        version: "{{ build.refspec }}"
        refspec: "refs/changes/*:refs/changes/*"
--- a/tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml
+++ b/tools/images/standard-container/assets/playbooks/roles/images/tasks/main.yaml
@ -1,12 +1,27 @@

 - name: Image Build
-  when: ("{{ stage }}" == "build")
+  when:
+    -  stage == "build"
+    - build_from_source
  block:
    #Build docker image using Makefile given git repository location to clone code from
    - name: Build Docker Image for "{{ image_name }}"
      shell: docker build -t "{{ image_fullname }}" .
      args:
-        chdir: "{{ build.checkout_loc }}/{{ path }}"
+        chdir: "{{ build.target_loc }}/{{ path }}"
+  become: true
+
+- name: Image Pull
+  when:
+    - stage == "build"
+    - not build_from_source
+  block:
+    #Build docker image using Makefile given git repository location to clone code from
+    - name: Pull Docker image "{{ build.repo }}:{{ build.tag }}"
+      docker_image:
+        pull: true
+        repository: "{{ image_fullname }}"
+        name: "{{ build.repo }}:{{ build.tag }}"
  become: true

 - name: Tag and Push Image
--- a/tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml
+++ b/tools/images/standard-container/assets/playbooks/roles/promote/tasks/main.yaml
@ -16,7 +16,7 @@
      shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push

    - name: Add Harbor Helm repository and Test repository
-      shell: helm repo add "{{ chart_repository }}-staging" "https://{{ chart_registry_url }}/{{ chart_name }}-staging" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" && helm repo add "{{ chart_repository }}" "https://{{ chart_registry_url }}/{{ chart_name }}" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}"
+      shell: helm repo add "{{ chart_repository }}-staging" "https://{{ chart_registry_url }}/{{ project }}-staging" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" && helm repo add "{{ chart_repository }}" "https://{{ chart_registry_url }}/{{ project }}" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}"

    - name: Pull down Helm Chart
      shell: helm pull "{{ chart_repository }}-staging/{{ chart_name }}" --version="{{ version }}"