Merge "feat(chart) Development Pipeline and Standard Container"

charts/development-pipeline/config_map.yaml.example

@@ -2,85 +2,66 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: deployment-flow
-  namespace: default
 data:
-  credentials: "Some creds"
-  kubeconfig.json: "Default kubeconfig.json"
-  cluster_exists: "true"
-  production: "false"
-  development.yaml: |
-    proxy:
-      http: ""
-      https: ""
-      noproxy: ""
-      enabled: false
-    docker_registry: "harbor-core.jarvis.local"
-    images:
-      - build_from_source: true
-        project: "test"
-        repo: "scratch"
-        tag: "1.built"
-        build:
-          git_repo: "https://review.opendev.org/airship/charts"
-          checkout_loc: "/src/checkout/scratch"
-          refspec: "refs/changes/41/770141/7"
-          version: "refs/changes/*:refs/changes/*"
-        makefile:
-          path: "tools/images"
-          target: "build"
-          file: "Makefile"
-          image_name: "scratch"
-          image_base: "scratch"
-      - build_from_source: true
-        repo: "microflow/standard-container"
-        project: "test"
-        tag: "1.built"
-        build:
-          git_repo: "https://review.opendev.org/airship/charts"
-          checkout_loc: "/src/checkout/standard-container"
-          refspec: "refs/changes/41/770141/7"
-          version: "refs/changes/*:refs/changes/*"
-        makefile:
-          path: "tools/images"
-          target: "build"
-          file: "Makefile"
-          image_name: "standard-container"
-          image_base: "ubuntu:20.04"
-      - build_from_source: false
-        remote_registry: "docker.io"
-        remote_repo: "testing2016/jrunner"
-        remote_tag: "2.0"
-        repo: "microflow/standard-container"
-        project: "test"
-        tag: "1.existing"
-    charts:
-      - name: "tekton-pipelines"
-        build_from_source: true
-        build:
-          git_repo: "https://review.opendev.org/airship/charts"
-          checkout_loc: "/src/checkout/airship/charts"
-          refspec: "master"
-        makefile:
-          path: "charts"
-          target: "tekton-pipelines"
-          file: "Makefile"
-      - name: "tekton-triggers"
-        build_from_source: true
-        build:
-          git_repo: "https://review.opendev.org/airship/charts"
-          checkout_loc: "/src/checkout/airship/charts"
-          refspec: "master"
-        makefile:
-          path: "charts"
-          target: "tekton-triggers"
-          file: "Makefile"
-      - name: "tekton-dashboard"
-        build_from_source: true
-        build:
-          git_repo: "https://review.opendev.org/airship/charts"
-          checkout_loc: "/src/checkout/airship/charts"
-          refspec: "master"
-        makefile:
-          path: "charts"
-          target: "tekton-dashboard"
-          file: "Makefile"
+  cluster.json: |
+    {
+     "use_existing_cluster": "true"
+    }
+  image.json: |
+    {
+     "build_from_source": true,
+     "image_name": "mongodb",
+     "project": "mongodb",
+     "repo": "mongodb",
+     "tag": "1.built",
+     "image_fullname": "cnf/mongodb:1.0",
+     "path": "tools/gate/jarvis/5G-SA-core/mongodb/images/mongodb",
+     "build": {
+        "git_repo": "https://review.opendev.org/airship/charts",
+        "checkout_loc": "/src/checkout/mongodb",
+        "refspec": "master",
+        "version": "refs/changes/*:refs/changes/*"
+     }
+    }
+  chart.json: |
+    {
+      "chart_name": "mongodb",
+      "project": "mongodb",
+      "repo": "mongodb",
+      "version": "0.1.0",
+      "build_from_source": true,
+      "path": "tools/gate/jarvis/5G-SA-core/mongodb/charts",
+      "build": {
+        "git_repo": "https://review.opendev.org/airship/charts",
+        "checkout_loc": "/src/checkout/airship/charts",
+        "refspec": "master"
+      },
+      "namespace": "development-pipeline",
+      "release_name": "mongodb",
+      "images": {
+        "applications": {
+          "mongodb": {
+            "tag": "1.0",
+            "name": "mongodb",
+            "repo": "mongodb"
+          }
+        }
+      }
+    }
+  default.json: |
+    {
+      "proxy": {
+        "http": "",
+        "https": "",
+        "noproxy": "",
+        "enabled": false
+      },
+     "chart_registry_url": "harbor-core.jarvis.local/chartrepo",
+     "chart_repository": "jarvis-harbor",
+     "docker_registry": "harbor-core.jarvis.local",
+     "harbor_secret_mounted_path": "/workspace/helm-creds"
+    }
+  cleanup.json: |
+    {
+     "remove_artifacts": "true"
+    }

charts/development-pipeline/pipelinerun-validation.yaml

@@ -0,0 +1,21 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  generateName: development-pipeline-run
+spec:
+  pipelineRef:
+    name: development-pipeline
+  serviceAccountName: sa-development-pipeline
+  workspaces:
+    - name: k8s_cluster_data
+      configMap:
+        name: deployment-flow
+    - name: development_pipeline_data
+      volumeClaimTemplate:
+        spec:
+          storageClassName: standard
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 1Gi

charts/development-pipeline/templates/cluster_role.yaml

@@ -1,11 +0,0 @@
-{{- if $.Values.clusterRole.create }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: {{ $.Values.clusterRole.name }}
-rules:
-  # EventListeners need to be able to fetch any clustertriggerbindings
-   - apiGroups: ["triggers.tekton.dev"]
-     resources: ["clustertriggerbindings"]
-     verbs: ["get", "list", "watch"]
-{{- end }}

charts/development-pipeline/templates/clusterrolebinding.yaml

@@ -1,14 +0,0 @@
-{{- if and ($.Values.serviceAccount.create) ($.Values.clusterRole.bind) }}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: {{ $.Values.clusterRole.name }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ $.Values.serviceAccount.name }}
-    namespace: {{ $.Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ $.Values.clusterRole.name }}
-{{- end }}

charts/development-pipeline/templates/eventlistener.yaml

@@ -1,13 +0,0 @@
-apiVersion: triggers.tekton.dev/v1alpha1
-kind: EventListener
-metadata:
-  name: development-listener
-  namespace: {{ $.Release.Namespace }}
-spec:
-  serviceAccountName:  {{ $.Values.serviceAccount.name }}
-  triggers:
-    - name: development-trigger
-      bindings:
-        - ref: development-pipeline-binding
-      template:
-        name: development-pipeline-template

charts/development-pipeline/templates/ingress.yaml

@@ -1,14 +0,0 @@
-apiVersion: networking.k8s.io/v1beta1
-kind: Ingress
-metadata:
-  name: ingress-development-listener
-  namespace: {{ .Release.Namespace }}
-spec:
-  rules:
-    - host: el-development.{{- .Release.Namespace }}.tekton.jarvis.local
-      http:
-        paths:
-          - path: /
-            backend:
-              serviceName: el-development-listener
-              servicePort: 8080

charts/development-pipeline/templates/pipeline.yaml

@@ -8,7 +8,41 @@ spec:
     - name: k8s_cluster_data
     - name: development_pipeline_data
   tasks:
-    - name: deploy-k8s
+    - name: microflow-setup-cluster-config
+      taskRef:
+        name: setup-cluster-config
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+    - name: microflow-setup-image-config
+      taskRef:
+        name: setup-image-config
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+    - name: microflow-setup-chart-config
+      taskRef:
+        name: setup-chart-config
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+    - name: microflow-setup-cleanup-config
+      taskRef:
+        name: setup-cleanup-config
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+    - name: microflow-k8s
+      runAfter:
+        - microflow-setup-cluster-config
       taskRef:
         name: k8s-cluster
       workspaces:
@@ -16,7 +50,9 @@ spec:
           workspace: k8s_cluster_data
         - name: development_pipeline_data
           workspace: development_pipeline_data
-    - name: build-image
+    - name: microflow-images
+      runAfter:
+        - microflow-setup-image-config
       workspaces:
         - name: k8s_cluster_data
           workspace: k8s_cluster_data
@@ -24,7 +60,9 @@ spec:
           workspace: development_pipeline_data
       taskRef:
         name: build-images
-    - name: build-chart
+    - name: microflow-charts
+      runAfter:
+        - microflow-setup-chart-config
       workspaces:
         - name: k8s_cluster_data
           workspace: k8s_cluster_data
@@ -32,17 +70,20 @@ spec:
           workspace: development_pipeline_data
       taskRef:
         name: build-charts
-    - name: deploy-artifacts
-      runAfter: [deploy-k8s,build-image,build-chart]
+    - name: microflow-deployment-manifests
       workspaces:
         - name: k8s_cluster_data
           workspace: k8s_cluster_data
         - name: development_pipeline_data
           workspace: development_pipeline_data
       taskRef:
-        name: deployment
-    - name: functional-testing
-      runAfter: [deploy-artifacts]
+        name: deployment-manifests
+    - name: microflow-functional
+      runAfter:
+        - microflow-deployment-manifests
+        - microflow-k8s
+        - microflow-images
+        - microflow-charts
       workspaces:
         - name: k8s_cluster_data
           workspace: k8s_cluster_data
@@ -50,8 +91,9 @@ spec:
           workspace: development_pipeline_data
       taskRef:
         name: functional
-    - name: promote-artifacts
-      runAfter: [functional-testing]
+    - name: microflow-promote-artifacts
+      runAfter:
+        - microflow-functional
       workspaces:
         - name: k8s_cluster_data
           workspace: k8s_cluster_data
@@ -60,7 +102,7 @@ spec:
       taskRef:
         name: promote
   finally:
-    - name: cleanup
+    - name: microflow-cleanup
       workspaces:
         - name: k8s_cluster_data
           workspace: k8s_cluster_data

charts/development-pipeline/templates/role.yaml

@@ -13,10 +13,17 @@ rules:
     resources: ["configmaps"]
     verbs: ["get", "list", "watch"]
    # Permissions to create resources in associated TriggerTemplates
-  - apiGroups: ["tekton.dev"]
-    resources: ["pipelineruns", "taskruns"]
-    verbs: ["create"]
   - apiGroups: [""]
     resources: ["serviceaccounts"]
-    verbs: ["impersonate"]
+    verbs: ["impersonate", "get"]
+   # Permissions to execute helm dry-run
+  - apiGroups: [""]
+    resources: ["secrets", "services"]
+    verbs: ["get"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources: ["roles", "rolebindings"]
+    verbs: ["get"]
 {{- end }}

charts/development-pipeline/templates/task-chart.yaml

@@ -10,9 +10,57 @@ spec:
     - name: k8s_cluster_data
     - name: development_pipeline_data
   steps:
-    - name: build-chart
+    - name: clone
       image: {{ $.Values.tasks.chart.buildChartImage }}
       script: |
         #!/usr/bin/env sh
-        ansible-playbook -vvv {{ $.Values.tasks.chart.buildPlaybook }}  -i hosts -e @"$(workspaces.k8s_cluster_data.path)/development.yaml"
-        # TODO copy JSON to shared workspace to make it available for other tasks
+        ansible-playbook -vvv  {{ $.Values.tasks.chart.clonePlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json"
+    - name: set-chart-output
+      image: {{ $.Values.tasks.chart.buildChartImage }}
+      script: |
+        #!/usr/bin/env sh
+        cat "$(workspaces.development_pipeline_data.path)/chart.json"
+    - name: lint-chart
+      image: {{ $.Values.tasks.chart.buildChartImage }}
+      script: |
+        #!/usr/bin/env sh
+        ansible-playbook -vvv {{ $.Values.tasks.chart.lintdryrunPlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json"
+    - name: set-chart-output-after-lint-dryrun
+      image: {{ $.Values.tasks.chart.buildChartImage }}
+      script: |
+        #!/usr/bin/env sh
+        cat "$(workspaces.development_pipeline_data.path)/chart.json"
+    - name: package-chart
+      image: {{ $.Values.tasks.chart.buildChartImage }}
+      script: |
+        #!/usr/bin/env sh
+        ansible-playbook -vvv {{ $.Values.tasks.chart.packagePlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json"
+    - name: set-chart-output-after-packaging
+      image: {{ $.Values.tasks.chart.buildChartImage }}
+      script: |
+        #!/usr/bin/env sh
+        cat "$(workspaces.development_pipeline_data.path)/chart.json"
+    - name: publish-chart
+      volumeMounts:
+        - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt
+          name: harbor-ca
+          subPath: harbor-ca
+        - mountPath: /workspace/helm-creds
+          name: helm-publish-creds
+      image: {{ $.Values.tasks.chart.buildChartImage }}
+      script: |
+        #!/usr/bin/env sh
+        update-ca-certificates
+        ansible-playbook -vvv {{ $.Values.tasks.chart.publishPlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json"
+    - name: set-chart-output-after-publish
+      image: {{ $.Values.tasks.chart.buildChartImage }}
+      script: |
+        #!/usr/bin/env sh
+        cat "$(workspaces.development_pipeline_data.path)/chart.json"
+  volumes:
+    - name: helm-publish-creds
+      secret:
+        secretName: harbor-basic-auth
+    - name: harbor-ca
+      secret:
+        secretName: harbor-ca

charts/development-pipeline/templates/task-cleanup.yaml

@@ -10,28 +10,19 @@ spec:
     - name: k8s_cluster_data
     - name: development_pipeline_data
   steps:
-    - name: gather-logs
-      image: {{ $.Values.tasks.cleanup.cleanupImage }}
-      script: |
-        #!/bin/sh
-        # TODO
-        echo "gather logs"
-        echo "publish logs"
-        echo "notify"
     - name: cleanup
       image: {{ $.Values.tasks.cleanup.cleanupImage }}
+      volumeMounts:
+        - mountPath: $(workspaces.development_pipeline_data.path)/config
+          name:  kubeconfig
       script: |
         #!/bin/sh
-        if grep -i "true" "$(workspaces.k8s_cluster_data.path)/production"; then
-            echo "No cleanup is required. CI was set to false."
-            exit 0
-        fi
-        if grep "true" "$(workspaces.k8s_cluster_data.path)/cluster_exists"; then
-            # TODO
-            echo "Cleanup artifacts"
-        else
-          # TODO
-          echo "Teardown k8s cluster"
-        fi
-        # TODO Delete sensitive data from shared workspace
-        rm "$(workspaces.development_pipeline_data.path)/kubeconfig.json"
+        ansible-playbook -vvv "{{ $.Values.tasks.cleanup.cleanupPlaybook }}" -i hosts \
+        -e @"$(workspaces.development_pipeline_data.path)/default.json" \
+        -e @"$(workspaces.development_pipeline_data.path)/chart.json" \
+        -e @"$(workspaces.development_pipeline_data.path)/image.json" \
+        -e @"$(workspaces.development_pipeline_data.path)/cluster.json"
+  volumes:
+    - name: kubeconfig
+      secret:
+        secretName: kubeconfig-secret

charts/development-pipeline/templates/task-deployment-manifests.yaml

@@ -0,0 +1,16 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: deployment-manifests
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task will do any validation of manifests required to deploy the CNF
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: example-validation-step
+      image: {{ $.Values.tasks.deploymentManifests.deploymentManifestsImage }}
+      script: |
+        ansible-playbook -vvv "{{ $.Values.tasks.deploymentManifests.deploymentManifestsPlaybook }}" -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json"

charts/development-pipeline/templates/task-deployment.yaml

@@ -1,19 +0,0 @@
-apiVersion: tekton.dev/v1beta1
-kind: Task
-metadata:
-  name: deployment
-  namespace: {{ $.Release.Namespace }}
-spec:
-  description: >-
-    This task deploys artifacts from produced by previous tasks
-  workspaces:
-    - name: k8s_cluster_data
-    - name: development_pipeline_data
-  steps:
-    - name: deploy-artifacts
-      image: {{ $.Values.tasks.deployment.deployImage }}
-      script: |
-        #!/usr/bin/env sh
-        # Artifacts details are stored in JSON files as an output of previous tasks.
-        # TODO copy logs and scan results to shared workspace
-        echo "Deploying artifacts"

charts/development-pipeline/templates/task-functional.yaml

@@ -5,21 +5,37 @@ metadata:
   namespace: {{ $.Release.Namespace }}
 spec:
   description: >-
-    This task will validate the k8s cluster using kubeconfig from k8s-cluster task
+    This task will deploy the CNF and run any tests specified
   workspaces:
     - name: k8s_cluster_data
     - name: development_pipeline_data
   steps:
-    - name: run-functional-tests
-      image: {{ $.Values.tasks.functional.validateImage }}
+    - name: deploy-helm-charts
+      image: {{ $.Values.tasks.functional.functionalDeployImage }}
+      volumeMounts:
+        - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt
+          name: harbor-ca
+          subPath: harbor-ca
+        - mountPath: /workspace/helm-creds
+          name: helm-publish-creds
+        - mountPath: $(workspaces.development_pipeline_data.path)/config
+          name:  kubeconfig
       script: |
         #!/bin/sh
-        if [ -f "$(workspaces.development_pipeline_data.path)/kubeconfig.json" ] ; then
-          echo "Kubeconfig is found, starting validation"
-        else
-          echo "Error: missing kubeconfig.json file"
-          exit 1
-        fi
-        echo "Validating cluster"
-        echo "ansible-playbook {{ $.Values.tasks.functional.playbook }}"
-        cat "$(workspaces.development_pipeline_data.path)/kubeconfig.json"
+        update-ca-certificates
+        ansible-playbook -vvv "{{ $.Values.tasks.functional.functionalDeployPlaybook }}"  -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json"
+    - name: run-helm-tests
+      image: {{ $.Values.tasks.functional.functionalTestImage }}
+      script: |
+        #!/bin/sh
+        ansible-playbook -vvv "{{ $.Values.tasks.functional.functionalTestPlaybook }}"  -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json"
+  volumes:
+    - name: helm-publish-creds
+      secret:
+        secretName: harbor-basic-auth
+    - name: harbor-ca
+      secret:
+        secretName: harbor-ca
+    - name: kubeconfig
+      secret:
+        secretName: kubeconfig-secret

charts/development-pipeline/templates/task-image.yaml

@@ -10,7 +10,23 @@ spec:
     - name: k8s_cluster_data
     - name: development_pipeline_data
   steps:
-    - name: build-image
+    - name: clone
+      image: {{ $.Values.tasks.image.buildImage }}
+      volumeMounts:
+        - mountPath: /certs/client
+          name: dind-certs
+      script: |
+        #!/usr/bin/env sh
+        ansible-playbook -vvv  {{ $.Values.tasks.image.clonePlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json"
+        #docker images
+        # TODO copy JSON file to shared workspace to make it available for other tasks
+        # TODO copy logs and scan results to shared location
+    - name: set-image-output
+      image: {{ $.Values.tasks.image.buildImage }}
+      script: |
+        #!/usr/bin/env sh
+        cat "$(workspaces.development_pipeline_data.path)/image.json"
+    - name: docker-build
       image: {{ $.Values.tasks.image.buildImage }}
       volumeMounts:
         - mountPath: /certs/client
@@ -27,10 +43,54 @@ spec:
         value: /certs/client
       script: |
         #!/usr/bin/env sh
-        ansible-playbook -vvv  {{ $.Values.tasks.image.buildPlaybook }} -i hosts -e @"$(workspaces.k8s_cluster_data.path)/development.yaml"
+        ansible-playbook -vvv  {{ $.Values.tasks.image.buildPlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json"
         docker images
         # TODO copy JSON file to shared workspace to make it available for other tasks
         # TODO copy logs and scan results to shared location
+    - name: set-image-build-output-after-build
+      image: {{ $.Values.tasks.image.buildImage }}
+      script: |
+        #!/usr/bin/env sh
+        cat "$(workspaces.development_pipeline_data.path)/image.json"
+    - name: publish-and-scan-image
+      image: {{ $.Values.tasks.image.buildImage }}
+      volumeMounts:
+        - mountPath: /certs/client
+          name: dind-certs
+        - mountPath: /tekton/home/.docker/config.json
+          name: image-push-creds
+          subPath: .dockerconfigjson
+      env:
+      # Connect to the sidecar over TCP, with TLS.
+      - name: DOCKER_HOST
+        value: tcp://localhost:2376
+      # Verify TLS.
+      - name: DOCKER_TLS_VERIFY
+        value: '1'
+      # Use the certs generated by the sidecar daemon.
+      - name: DOCKER_CERT_PATH
+        value: /certs/client
+      script: |
+        #!/usr/bin/env sh
+        ansible-playbook -vvv  {{ $.Values.tasks.image.pushPlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json"
+    - name: set-image-output-after-publish-scan
+      image: {{ $.Values.tasks.image.buildImage }}
+      script: |
+        #!/usr/bin/env sh
+        cat "$(workspaces.development_pipeline_data.path)/image.json"
+    - name: get-scan-results
+      image: {{ $.Values.tasks.image.buildImage }}
+      volumeMounts:
+        - mountPath: /certs/client
+          name: dind-certs
+      script: |
+        #!/usr/bin/env sh
+        ansible-playbook -vvv  {{ $.Values.tasks.image.getScanResultsPlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json"
+    - name: set-image-output-set-scan-results
+      image: {{ $.Values.tasks.image.buildImage }}
+      script: |
+        #!/usr/bin/env sh
+        cat "$(workspaces.development_pipeline_data.path)/image.json"
   sidecars:
     - image: {{ $.Values.tasks.image.sidecarServer }}
       name: server
@@ -57,3 +117,9 @@ spec:
   volumes:
     - name: dind-certs
       emptyDir: {}
+    - name: image-push-creds
+      secret:
+        secretName: harbor-docker-auth
+    - name: helm-publish-creds
+      secret:
+        secretName: harbor-basic-auth

charts/development-pipeline/templates/task-k8s-cluster.yaml

@@ -5,26 +5,19 @@ metadata:
   namespace: {{ $.Release.Namespace }}
 spec:
   description: >-
-    This task will create a k8s cluster if needed and store kubeconfig in shared workspace
+    This task will create a k8s cluster if needed or verify that an existing cluster is reachable
   workspaces:
     - name: k8s_cluster_data
     - name: development_pipeline_data
   steps:
     - name: generate-kubeconfig
-      image: {{ $.Values.tasks.cluster.createClusterImage }}
+      image: {{ $.Values.tasks.kubernetes.createClusterImage }}
       script: |
-        #!/usr/bin/env sh
-        # Use provided kubeconfig for existing cluster
-        if grep -i "true" "$(workspaces.k8s_cluster_data.path)/cluster_exists"; then
-          echo "Use existing cluster."
-          cp "$(workspaces.k8s_cluster_data.path)/kubeconfig.json" "$(workspaces.development_pipeline_data.path)/kubeconfig.json"
-        # Deploy cluster and copy kubeconfig
-        else
-          echo "Create a k8s cluster"
-          # TODO
-          echo "ansible-playbook -vvv {{ $.Values.tasks.cluster.createClusterPlaybook }}  -i hosts -e @\"$(workspaces.k8s_cluster_data.path)/development.yaml\""
-          # TODO copy generated kubeconfig to shared wokspace
-          echo "New cluster's kubeconfig data" > "$(workspaces.development_pipeline_data.path)/kubeconfig.json"
-        fi
-        echo "ansible-playbook -vvv {{ $.Values.tasks.cluster.validateClusterPlaybook }}  -i hosts -e @\"$(workspaces.k8s_cluster_data.path)/development.yaml\""
-        # TODO copy JSON file to shared workspace to make it available for other tasks
+          ansible-playbook -vvv "{{ $.Values.tasks.kubernetes.getKubeconfigPlaybook }}"  -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/cluster.json"
+      volumeMounts:
+        - mountPath: $(workspaces.development_pipeline_data.path)/config
+          name:  kubeconfig
+  volumes:
+    - name: kubeconfig
+      secret:
+        secretName: kubeconfig-secret

charts/development-pipeline/templates/task-promote.yaml

@@ -5,14 +5,72 @@ metadata:
   namespace: {{ $.Release.Namespace }}
 spec:
   description: >-
-    This task builds charts if source is provided
+    This task will promote images and chart into a non-test repository
   workspaces:
     - name: k8s_cluster_data
     - name: development_pipeline_data
   steps:
-    - name: build-chart
-      image: {{ $.Values.tasks.promote.promoteArtifacts }}
+    - name: promote-artifacts
+      image: {{ $.Values.tasks.promote.promoteImage }}
+      env:
+      # Connect to the sidecar over TCP, with TLS.
+      - name: DOCKER_HOST
+        value: tcp://localhost:2376
+      # Verify TLS.
+      - name: DOCKER_TLS_VERIFY
+        value: '1'
+      # Use the certs generated by the sidecar daemon.
+      - name: DOCKER_CERT_PATH
+        value: /certs/client
+      volumeMounts:
+        - mountPath: /tekton/home/.docker/config.json
+          name: image-push-creds
+          subPath: .dockerconfigjson
+        - mountPath: /certs/client
+          name: dind-certs
+        - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt
+          name: harbor-ca
+          subPath: harbor-ca
+        - mountPath: /workspace/helm-creds
+          name: helm-publish-creds
       script: |
         #!/usr/bin/env sh
-        # TODO call playbook to promote artifacts generated in previous tasks
-        echo "Promote artifacts"
+        set -ex
+        update-ca-certificates
+        ansible-playbook -vvv  {{ $.Values.tasks.promote.promoteImagePlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/image.json"
+        ansible-playbook -vvv  {{ $.Values.tasks.promote.promoteChartPlaybook }} -i hosts -e @"$(workspaces.development_pipeline_data.path)/default.json" -e @"$(workspaces.development_pipeline_data.path)/chart.json"
+  sidecars:
+    - image: {{ $.Values.tasks.image.sidecarServer }}
+      name: server
+      args:
+        - --storage-driver=vfs
+        - --userland-proxy=false
+        - --debug
+        - --insecure-registry={{ $.Values.tasks.image.insecureRegistry }}
+      securityContext:
+        privileged: true
+      env:
+      # Write generated certs to the path shared with the client.
+      - name: DOCKER_TLS_CERTDIR
+        value: /certs
+      volumeMounts:
+        - mountPath: /certs/client
+          name: dind-certs
+      # Wait for the dind daemon to generate the certs it will share with the
+      # client.
+      readinessProbe:
+        periodSeconds: 1
+        exec:
+          command: ['ls', '/certs/client/ca.pem']
+  volumes:
+    - name: dind-certs
+      emptyDir: {}
+    - name: image-push-creds
+      secret:
+        secretName: harbor-docker-auth
+    - name: helm-publish-creds
+      secret:
+        secretName: harbor-basic-auth
+    - name: harbor-ca
+      secret:
+        secretName: harbor-ca

charts/development-pipeline/templates/task-setup-config.yaml

@@ -0,0 +1,84 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: setup-cluster-config
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task sets configurations that will be used as overrides to the Ansible tasks.
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: setup-cluster-config
+      image: {{ $.Values.tasks.setup.setupConfigImage }}
+      script: |
+        #!/usr/bin/env sh
+        cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json"
+        cp "$(workspaces.k8s_cluster_data.path)/cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json"
+        jq '.cluster_kubeconfig_path="$(workspaces.development_pipeline_data.path)/config"' "$(workspaces.development_pipeline_data.path)/cluster.json" > "$(workspaces.development_pipeline_data.path)/temp_cluster.json" && mv "$(workspaces.development_pipeline_data.path)/temp_cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json"
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: setup-image-config
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task sets configurations that will be used as overrides to the Ansible tasks.
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: setup-image-config
+      image: {{ $.Values.tasks.setup.setupConfigImage }}
+      script: |
+        #!/usr/bin/env sh
+        cp "$(workspaces.k8s_cluster_data.path)/image.json" "$(workspaces.development_pipeline_data.path)/image.json"
+        echo "Set temporary image name to test/scan-image:$(context.taskRun.uid)"
+        jq '.image_fullname="test/scan-image:$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json"
+        echo "Set tag to context $(context.taskRun.uid)"
+        jq '.tag="$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json"
+        echo "Set checkout location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"
+        jq '.build.checkout_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/image.json" > "$(workspaces.development_pipeline_data.path)/temp_image.json" && mv "$(workspaces.development_pipeline_data.path)/temp_image.json" "$(workspaces.development_pipeline_data.path)/image.json"
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: setup-chart-config
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task sets configurations that will be used as overrides to the Ansible tasks.
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: setup-chart-config
+      image: {{ $.Values.tasks.setup.setupConfigImage }}
+      script: |
+        #!/usr/bin/env sh
+        cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json"
+        cp "$(workspaces.k8s_cluster_data.path)/chart.json" "$(workspaces.development_pipeline_data.path)/chart.json"
+        echo "Set tag to context $(context.taskRun.uid)"
+        jq '.tag="$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json"
+        echo "Set checkout location for git repository to $(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"
+        jq '.build.checkout_loc="$(workspaces.development_pipeline_data.path)/$(context.taskRun.uid)"' "$(workspaces.development_pipeline_data.path)/chart.json" > "$(workspaces.development_pipeline_data.path)/temp_chart.json" && mv "$(workspaces.development_pipeline_data.path)/temp_chart.json" "$(workspaces.development_pipeline_data.path)/chart.json"
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: setup-cleanup-config
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task sets configurations that will be used as overrides to the Ansible tasks.
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: setup-cleanup-config
+      image: {{ $.Values.tasks.setup.setupConfigImage }}
+      script: |
+        #!/usr/bin/env sh
+        cp "$(workspaces.k8s_cluster_data.path)/cleanup.json" "$(workspaces.development_pipeline_data.path)/cleanup.json"

charts/development-pipeline/templates/triggerbindings.yaml

@@ -1,5 +0,0 @@
-apiVersion: triggers.tekton.dev/v1alpha1
-kind: TriggerBinding
-metadata:
-  name: development-pipeline-binding
-  namespace: {{ $.Release.Namespace }}

charts/development-pipeline/templates/triggertemplate.yaml

@@ -1,28 +0,0 @@
-apiVersion: triggers.tekton.dev/v1alpha1
-kind: TriggerTemplate
-metadata:
-  name: development-pipeline-template
-  namespace: {{ $.Release.Namespace }}
-spec:
-  resourcetemplates:
-    - apiVersion: tekton.dev/v1beta1
-      kind: PipelineRun
-      metadata:
-        generateName: development-pipeline-run
-      spec:
-        pipelineRef:
-          name: development-pipeline
-        serviceAccountName: {{ $.Values.serviceAccount.name }}
-        workspaces:
-          - name: k8s_cluster_data
-            configMap:
-              name: {{ $.Values.trigger.configMap }}
-          - name: development_pipeline_data
-            volumeClaimTemplate:
-              spec:
-                storageClassName: {{ $.Values.pvc.storageClass }}
-                accessModes:
-                  - ReadWriteOnce
-                resources:
-                  requests:
-                    storage: {{ $.Values.pvc.size }}

charts/development-pipeline/values.yaml

@@ -7,37 +7,45 @@ role:
   name: development-pipeline
   create: true
 
-clusterRole:
-  bind: true
-  name: development-pipeline
-  create: true
-
 pvc:
   storageClass: standard
   size: 1Gi
 
 tasks:
-  cluster:
-    createClusterImage: &base_image testing2016/standard-container:1.0
-    createClusterPlaybook: /playbooks/create_cluster.yaml
-    validateClusterPlaybook: /playbooks/validate_cluster.yaml
+  kubernetes:
+    createClusterImage: &base_image standard-container:1.0
+    createClusterPlaybook: /playbooks/create-cluster.yaml
+    getKubeconfigPlaybook: /playbooks/get-kubeconfig.yaml
+    validateClusterPlaybook: /playbooks/validate-cluster.yaml
+  setup:
+    setupConfigImage: *base_image
   image:
     buildImage: *base_image
     sidecarServer: docker:19-dind
-    buildPlaybook: /playbooks/build_image.yaml
     insecureRegistry: harbor-core.jarvis.local
+    clonePlaybook: /playbooks/clone.yaml
+    buildPlaybook: /playbooks/build-image.yaml
+    pushPlaybook: /playbooks/tag-push-image.yaml
+    getScanResultsPlaybook: /playbooks/get-scan-results.yaml
+    name: standard-container.yaml
   chart:
     buildChartImage: *base_image
-    buildPlaybook: /playbooks/helm_chart.yaml
-  deployment:
-    deployImage: *base_image
+    clonePlaybook: /playbooks/clone.yaml
+    packagePlaybook: /playbooks/package-chart.yaml
+    lintdryrunPlaybook: /playbooks/lint-dryrun-chart.yaml
+    publishPlaybook: /playbooks/publish-chart.yaml
+  deploymentManifests:
+    deploymentManifestsImage: *base_image
+    deploymentManifestsPlaybook: /playbooks/deployment-manifests.yaml
   promote:
-    promoteArtifacts: *base_image
+    promoteImage: *base_image
+    promoteImagePlaybook: /playbooks/promote-image.yaml
+    promoteChartPlaybook: /playbooks/promote-chart.yaml
   functional:
-    validateImage: *base_image
-    playbook: /playbooks/functional.yaml
+    functionalDeployImage: *base_image
+    functionalTestImage: *base_image
+    functionalDeployPlaybook: /playbooks/functional-deploy.yaml
+    functionalTestPlaybook: /playbooks/functional-test.yaml
   cleanup:
     cleanupImage: *base_image
-
-trigger:
-  configMap: deployment-flow
+    cleanupPlaybook: /playbooks/cleanup.yaml

tools/deployment/common/wait-for-pipelinerun.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+set -ex
+
+# Default wait timeout is 1000 seconds
+end=$(date +%s)
+timeout=${3:-1000}
+end=$((end + timeout))
+
+while true; do
+    pipelinerunstatus="$(kubectl get pipelinerun -n $1 $(kubectl get pipelinerun -n $1 -o name | awk -F '/' "/$2/ { print \$NF; exit }") | tail -1 | awk '{ print $2 }')"
+    [ "${pipelinerunstatus}" == "True" ] && break
+    [ "${pipelinerunstatus}" == "False" ] && exit 1
+    sleep 5
+    now=$(date +%s)
+    if  [ $now -gt $end ] ; then
+        echo "Pipelinerun failed to complete after $timeout seconds"
+        echo
+        kubectl get pipelinerun --namespace $1 -o wide
+        echo "Some pipelineruns are not complete"
+        exit 1
+    fi
+done

tools/deployment/vagrant/Vagrantfile

@@ -40,6 +40,7 @@ Vagrant.configure("2") do |config|
     ./tools/gate/jarvis/400-deploy-harbor.sh
     ./tools/gate/jarvis/500-deploy-gerrit.sh
     ./tools/gate/jarvis/600-deploy-tekton.sh
+    ./tools/gate/jarvis/650-temporary-setup.sh
     ./tools/gate/jarvis/700-deploy-jarvis-system.sh
     ./tools/gate/jarvis/800-deploy-jarvis-projects.sh
   SHELL

tools/gate/jarvis/400-deploy-harbor.sh

@@ -30,15 +30,6 @@ function validate() {
     local chart_dir
     chart_dir="$(mktemp -d)"
     helm pull jarvis-harbor/library/chartmuseum --destination "${chart_dir}"
-
-    #TODO(staceyF) Put this into appropriate jarvis-system tasks
-    kubectl create ns development-pipeline
-    kubectl create secret generic harbor-ca --from-file=harbor-ca=/etc/jarvis/certs/ca/ca.pem -n development-pipeline
-    #NOTE Will not be required once Harbor is backed by LDAP
-    kubectl create secret generic harbor-basic-auth --from-literal=username='admin' --from-literal=password='Harbor12345' -n development-pipeline
-    kubectl create secret docker-registry harbor-docker-auth --docker-username=admin --docker-password=Harbor12345 --docker-email=example@gmail.com --docker-server=harbor-core.jarvis.local -n development-pipeline
-    #TODO(staceyF) Put this into appropriate jarvis-project tasks
-    curl -X POST "https://harbor-core.jarvis.local/api/v2.0/projects" -H "accept: application/json" -H "X-Request-Id: 12345" -H "authorization: Basic YWRtaW46SGFyYm9yMTIzNDU=" -H "Content-Type: application/json" -d "{ \"project_name\": \"test\", \"public\": true, \"metadata\": { \"auto_scan\": \"true\" }}"
     # Tests that we can upload an image
     sudo -E docker login harbor-core.jarvis.local --username admin --password Harbor12345
     sudo -E docker pull debian:buster-slim

tools/gate/jarvis/650-temporary-setup.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -ex
+
+#TODO(staceyF) Put this into appropriate jarvis-system tasks
+kubectl create ns development-pipeline || true
+kubectl create secret generic harbor-ca --from-file=harbor-ca=/etc/jarvis/certs/ca/ca.pem -n development-pipeline || true
+kubectl create secret generic kubeconfig-secret --from-file=kubeconfig=$HOME/.kube/config -n development-pipeline || true
+#NOTE Will not be required once Harbor is backed by LDAP
+kubectl create secret generic harbor-basic-auth --from-literal=username='admin' --from-literal=password='Harbor12345' -n development-pipeline || true
+kubectl create secret docker-registry harbor-docker-auth --docker-username=admin --docker-password=Harbor12345 --docker-email=example@gmail.com --docker-server=harbor-core.jarvis.local -n development-pipeline || true
+#TODO(staceyF) Put this into appropriate jarvis-project tasks
+curl -X POST "https://harbor-core.jarvis.local/api/v2.0/projects" -H "accept: application/json" -H "X-Request-Id: 12345" -H "authorization: Basic YWRtaW46SGFyYm9yMTIzNDU=" -H "Content-Type: application/json" -d "{ \"project_name\": \"mongodb-staging\", \"public\": true, \"metadata\": { \"auto_scan\": \"true\" }}" || true
+curl -X POST "https://harbor-core.jarvis.local/api/v2.0/projects" -H "accept: application/json" -H "X-Request-Id: 12345" -H "authorization: Basic YWRtaW46SGFyYm9yMTIzNDU=" -H "Content-Type: application/json" -d "{ \"project_name\": \"mongodb\", \"public\": true, \"metadata\": { \"auto_scan\": \"true\" }}" || true
+
+#NOTE This is temporary to trigger and validate that the development-pipeline is working prior to being refactored.
+
+cd ./tools/images
+sudo make build IMAGE_FULLNAME=standard-container:1.0
+
+cd ../../charts
+helm upgrade --install development-pipeline -n development-pipeline ./development-pipeline
+
+kubectl apply -n development-pipeline -f ./development-pipeline/config_map.yaml.example
+
+kubectl create -n development-pipeline -f ./development-pipeline/pipelinerun-validation.yaml
+
+../tools/deployment/common/wait-for-pipelinerun.sh development-pipeline development-pipeline

tools/images/Makefile

@@ -0,0 +1,28 @@
+BUILD_DIR         := $(shell mktemp -d)
+IMAGE_BASE        ?= ubuntu:20.04
+IMAGE_PREFIX      ?= test
+IMAGE_REGISTRY    ?= core.harbor.domain
+IMAGE_TAG         ?= 1.0
+IMAGE_NAME        ?= standard-container
+PROXY             ?= http://proxy.foo.com:8000
+NO_PROXY          ?= localhost,127.0.0.1,.svc.cluster.local
+USE_PROXY         ?= false
+# use this variable for image labels added in internal build process
+LABEL             ?= org.attcomdev.build=community
+COMMIT            ?= $(shell git rev-parse HEAD)
+DISTRO            ?= ubuntu_bionic
+IMAGE_FULLNAME    := ${IMAGE_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}
+
+.PHONY: help
+
+SHELL:=/bin/bash
+.ONESHELL:
+
+help: ## This help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+build: ## Build the containers.
+	docker build --tag $(IMAGE_FULLNAME) --build-arg BASE_IMAGE=$(IMAGE_BASE) ./$(IMAGE_NAME)
+
+push: build ## Build and push the containers
+	docker push $(IMAGE_FULLNAME)

tools/images/scratch/Dockerfile

@@ -0,0 +1,7 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+SHELL ["bash", "-exc"]
+ENV DEBIAN_FRONTEND noninteractive
+
+ENTRYPOINT /entrypoint.sh

tools/images/standard-container/Dockerfile

@@ -0,0 +1,53 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+SHELL ["bash", "-exc"]
+ENV DEBIAN_FRONTEND noninteractive
+
+# Update distro and install ansible
+RUN apt-get update ;\
+    apt-get dist-upgrade -y ;\
+    apt-get install -y \
+        python3-minimal \
+        python3-pip \
+        python3-setuptools \
+        make \
+        sudo \
+        git \
+        jq \
+        curl \
+        git-review \
+        apt-transport-https \
+        ca-certificates \
+        gnupg-agent \
+        software-properties-common \
+        gettext-base ;\
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - ;\
+    add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" ;\
+    apt-get install -y --no-install-recommends \
+        docker-ce-cli ;\
+    pip3 install --upgrade wheel ;\
+    pip3 install ansible ;\
+    ansible-galaxy collection install community.kubernetes ;\
+    pip3 install docker ;\
+    # Install kubectl
+    apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        gnupg2 ;\
+    curl -o /usr/bin/kubectl -L "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" ;\
+    chmod +x /usr/bin/kubectl ;\
+    # Install Helm
+    curl -fsSL -o helm-install.tar.gz https://get.helm.sh/helm-v3.4.2-linux-amd64.tar.gz \
+    && tar -xvf helm-install.tar.gz \
+    && rm helm-install.tar.gz \
+    && mv linux-amd64/helm /usr/local/bin/helm \
+    && chmod +x /usr/local/bin/helm \
+    && helm --help ;\
+    helm plugin install https://github.com/chartmuseum/helm-push ;\
+    rm -rf /var/lib/apt/lists/*
+
+COPY assets /opt/assets/
+RUN cp -ravf /opt/assets/* / ;\
+    rm -rf /opt/assets
+
+ENTRYPOINT /entrypoint.sh

tools/images/standard-container/assets/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+ansible --version

tools/images/standard-container/assets/playbooks/build-image.yaml

@@ -0,0 +1,8 @@
+- hosts: localhost
+  become: yes
+  vars:
+    image_status: "Success"
+  tasks:
+    - name: Build image
+      include_tasks: ./roles/images/tasks/build-image.yaml
+      when: "{{ build_from_source }}"

tools/images/standard-container/assets/playbooks/cleanup.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: Clean up release
+      include_tasks: ./roles/cleanup/tasks/cleanup.yaml

tools/images/standard-container/assets/playbooks/clone.yaml

@@ -0,0 +1,8 @@
+- hosts: localhost
+  become: yes
+  vars:
+    image_status: "Success"
+  tasks:
+    - name: Run git clone task
+      include_tasks: ./roles/common/tasks/git-clone.yaml
+      when: "{{ build_from_source }}"

tools/images/standard-container/assets/playbooks/deployment-manifests.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: validate-deployment-manifests-config
+      include_tasks: ./roles/deployment-manifests/tasks/validate-config.yaml

tools/images/standard-container/assets/playbooks/functional-deploy.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: Deploy CNF
+      include_tasks: ./roles/functional/tasks/functional-deploy.yaml

tools/images/standard-container/assets/playbooks/functional-test.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: Test deployed CNF
+      include_tasks: ./roles/functional/tasks/functional-test.yaml

tools/images/standard-container/assets/playbooks/get-kubeconfig.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: Get kubeconfig for Kubernetes cluster to deploy CNF
+      include_tasks: ./roles/kubernetes/tasks/get-kubeconfig.yaml

tools/images/standard-container/assets/playbooks/get-scan-results.yaml

@@ -0,0 +1,7 @@
+- hosts: localhost
+  become: yes
+  vars:
+    image_status: "Success"
+  tasks:
+    - name: Get Scan Results
+      include_tasks: ./roles/images/tasks/get-scan-results.yaml

tools/images/standard-container/assets/playbooks/group_vars/main.yaml

@@ -0,0 +1,6 @@
+proxy:
+  http: ""
+  https: ""
+  noproxy: ""
+  enabled: false
+docker_registry: "harbor-core.jarvis.local"

tools/images/standard-container/assets/playbooks/hosts

@@ -0,0 +1 @@
+localhost ansible_connection=local

tools/images/standard-container/assets/playbooks/lint-dryrun-chart.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: Chart lint and dry-run
+      include_tasks: ./roles/charts/tasks/lint-dryrun-chart.yaml

tools/images/standard-container/assets/playbooks/package-chart.yaml

@@ -0,0 +1,6 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: Package Helm Chart
+      include_tasks: ./roles/charts/tasks/package-chart.yaml
+      when: "{{ build_from_source }}"

tools/images/standard-container/assets/playbooks/promote-chart.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: Promote charts after testing is successful
+      include_tasks: ./roles/promote/tasks/promote-chart.yaml

tools/images/standard-container/assets/playbooks/promote-image.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: Promote images after testing is successful
+      include_tasks: ./roles/promote/tasks/promote-image.yaml

tools/images/standard-container/assets/playbooks/publish-chart.yaml

@@ -0,0 +1,5 @@
+- hosts: localhost
+  become: yes
+  tasks:
+    - name: Publish Helm Chart
+      include_tasks: ./roles/charts/tasks/publish-chart.yaml

tools/images/standard-container/assets/playbooks/roles/charts/tasks/download-chart.yaml

@@ -0,0 +1,6 @@
+#Helm Chart is upstream already packaged and versioned
+- name: Existing helm_chart
+  block:
+    - name: Download helm_chart
+      get_url:
+        name: "{{ remote_url }}"

tools/images/standard-container/assets/playbooks/roles/charts/tasks/lint-dryrun-chart.yaml

@@ -0,0 +1,9 @@
+- name: Helm lint "{{ chart_name }}"
+  command: "helm lint {{ chart_name }}"
+  args:
+    chdir: "{{ build.checkout_loc }}/{{ path }}"
+
+- name: Helm Dry-run "{{ chart_name }}"
+  command: "helm install --dry-run {{ chart_name }} {{ chart_name }}"
+  args:
+    chdir: "{{ build.checkout_loc }}/{{ path }}"

tools/images/standard-container/assets/playbooks/roles/charts/tasks/package-chart.yaml

@@ -0,0 +1,4 @@
+- name: Package Helm Chart
+  shell: helm package "{{ chart_name }}"
+  args:
+    chdir: "{{ build.checkout_loc }}/{{ path }}"

tools/images/standard-container/assets/playbooks/roles/charts/tasks/publish-chart.yaml

@@ -0,0 +1,15 @@
+# TODO: Bring in secrets securely via K8s
+- name: Install Plugin
+  shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true
+- name: Get harbor username
+  shell: cat {{ harbor_secret_mounted_path }}/username
+  register: harbor_username
+- name: Get harbor password
+  shell: cat {{ harbor_secret_mounted_path }}/password
+  register: harbor_password
+- name: Add Harbor Helm repository and Test repository
+  shell: helm repo add "{{ chart_repository }}-staging" "https://{{ docker_registry }}/chartrepo/{{ chart_name }}-staging" --username={{ harbor_username.stdout }} --password={{ harbor_password.stdout }}
+- name: Push chart "{{ chart_name }}" to Harbor staging registry
+  command: helm push "{{ chart_name }}-{{ version }}".tgz "{{ chart_repository }}-staging"
+  args:
+    chdir: "{{ build.checkout_loc }}/{{ path }}"

tools/images/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml

@@ -0,0 +1,3 @@
+#Remove any resources deployed
+- name: Remove test release
+  shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ release_name }}" -n "{{ namespace }}"

tools/images/standard-container/assets/playbooks/roles/common/tasks/git-clone.yaml

@@ -0,0 +1,6 @@
+#Build docker image using Makefile given git repository location to clone code from
+- git:
+    repo: "{{ build.git_repo }}"
+    dest: "{{ build.checkout_loc }}"
+    version: "{{ build.refspec }}"
+    refspec: "refs/changes/*:refs/changes/*"

tools/images/standard-container/assets/playbooks/roles/deployment-manifests/tasks/validate-config.yaml

@@ -0,0 +1,4 @@
+- name: This task is to validate deployment manifests
+  command: echo "It can do anything you want, just put it right here."
+  args:
+    chdir: "/workspace"

tools/images/standard-container/assets/playbooks/roles/functional/tasks/functional-deploy.yaml

@@ -0,0 +1,13 @@
+#Deploy CNF
+- name: Get harbor username
+  shell: cat {{ harbor_secret_mounted_path }}/username
+  register: harbor_username
+- name: Get harbor password
+  shell: cat {{ harbor_secret_mounted_path }}/password
+  register: harbor_password
+#TODO dex-aio doesn't install, look into another test chart
+- name: Add Harbor Helm repository and Test repository
+  shell: helm repo add "{{ chart_repository }}-staging" "https://{{ docker_registry }}/chartrepo/{{ project }}-staging" --username={{ harbor_username.stdout }} --password={{ harbor_password.stdout }}
+- name: Deploy chart "{{ chart_name }}"
+  #shell: echo "Deploy after mongodb"
+  shell: helm upgrade --install --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ release_name }}" "{{ chart_repository }}-staging/{{ chart_name }}" --version="{{ version }}" --namespace="{{ namespace }}" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" --create-namespace

tools/images/standard-container/assets/playbooks/roles/functional/tasks/functional-test.yaml

@@ -0,0 +1,4 @@
+#Test Deployed CNF
+- name: Chart has no Helm Tests, echo for now
+  shell: echo "There are no helm tests yet"
+#  shell: echo 'helm test --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ name }}"'

tools/images/standard-container/assets/playbooks/roles/images/tasks/build-image.yaml

@@ -0,0 +1,5 @@
+#Build docker image using Makefile given git repository location to clone code from
+- name: Build Docker Image for "{{ image_name }}"
+  shell: docker build -t "{{ image_fullname }}" .
+  args:
+    chdir: "{{ build.checkout_loc }}/{{ path }}"

tools/images/standard-container/assets/playbooks/roles/images/tasks/get-scan-results.yaml

@@ -0,0 +1,25 @@
+#Scan results may take some time, putting in some retries and a delay to determine if scan results get finished
+- name: output the request
+  shell: echo "https://{{ docker_registry }}/api/v2.0/projects/{{ project }}-staging/repositories/{{ repo | replace('/','%2F') }}/artifacts/{{ tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false"
+
+- name: Get Scan Results
+  uri:
+    validate_certs: false
+    url: "https://{{ docker_registry }}/api/v2.0/projects/{{ project }}-staging/repositories/{{ repo | replace('/','%2F') }}/artifacts/{{ tag }}?page=1&page_size=10&with_tag=true&with_label=false&with_scan_overview=true&with_signature=false&with_immutable_status=false"
+    method: GET
+    body_format: "json"
+    headers:
+      accept: "application/json"
+      X-Request-Id: "12345"
+      #Change to encoded from configmap
+      authorization: "Basic YWRtaW46SGFyYm9yMTIzNDU="
+  register: result
+  until: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].scan_status == "Success"
+  retries: 5
+  delay: 30
+
+- name: Check Scan Results Summary for High and Critical CVE
+  #shell: echo '{{ result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"] }}'
+  set_fact:
+    image_status: "Vulnerable"
+  when: result.json.scan_overview["application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0"].severity in ("High","Critical")

tools/images/standard-container/assets/playbooks/roles/images/tasks/pull-image.yaml

@@ -0,0 +1,4 @@
+- name: Tag and push to promotion repository
+  docker_image:
+    pull: true
+    name: "{{ docker_registry }}/{{ project }}-staging/{{ project }}/{{ repo }}:{{ tag }}"

tools/images/standard-container/assets/playbooks/roles/images/tasks/tag-push-image.yaml

@@ -0,0 +1,6 @@
+- name: Tag and push to internal test repository for vulnerability scanning
+  docker_image:
+    push: true
+    name: "{{ image_fullname }}"
+    repository: "{{ docker_registry }}/{{ project }}-staging/{{ repo }}"
+    tag: "{{ tag }}"

tools/images/standard-container/assets/playbooks/roles/images/vars/main.yaml

@@ -0,0 +1,15 @@
+build_from_source: true
+project: "test"
+repo: "scratch"
+tag: "1.built"
+build:
+  git_repo: "https://review.opendev.org/airship/charts"
+  checkout_loc: "/src/checkout/scratch"
+  refspec: "refs/changes/41/770141/7"
+  version: "refs/changes/*:refs/changes/*"
+makefile:
+  path: "tools/images"
+  target: "build"
+  file: "Makefile"
+  image_name: "scratch"
+  image_base: "scratch"

tools/images/standard-container/assets/playbooks/roles/images/vars/scratch.yaml

@@ -0,0 +1,16 @@
+build_from_source: true
+project: "test"
+repo: "scratch"
+tag: "1.built"
+build:
+  git_repo: "https://review.opendev.org/airship/charts"
+  checkout_loc: "/src/checkout/scratch"
+  refspec: "refs/changes/41/770141/7"
+  version: "refs/changes/*:refs/changes/*"
+makefile:
+  path: "tools/images"
+  target: "build"
+  file: "Makefile"
+  image_name: "scratch"
+  image_base: "scratch"
+

tools/images/standard-container/assets/playbooks/roles/images/vars/standard-container.yaml

@@ -0,0 +1,15 @@
+build_from_source: true
+repo: "microflow/standard-container"
+project: "test"
+tag: "1.built"
+build:
+  git_repo: "https://review.opendev.org/airship/charts"
+  checkout_loc: "/src/checkout/standard-container"
+  refspec: "refs/changes/41/770141/7"
+  version: "refs/changes/*:refs/changes/*"
+makefile:
+  path: "tools/images"
+  target: "build"
+  file: "Makefile"
+  image_name: "standard-container"
+  image_base: "ubuntu:20.04"

tools/images/standard-container/assets/playbooks/roles/images/vars/upstream-container.yaml

@@ -0,0 +1,7 @@
+build_from_source: false
+remote_registry: "docker.io"
+remote_repo: "testing2016/jrunner"
+remote_tag: "2.0"
+repo: "microflow/standard-container"
+project: "test"
+tag: "1.existing"

tools/images/standard-container/assets/playbooks/roles/kubernetes/tasks/get-kubeconfig.yaml

@@ -0,0 +1,10 @@
+#Validate Kubernetes cluster is accessible.
+- set_fact:
+    existing_cluster: "{{ use_existing_cluster }}"
+- name: Get kubeconfig
+  shell: echo "Retrieve kubeconfig from mounted secret"
+  when: existing_cluster
+- name: Validate kubeconfig
+  shell: kubectl --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" get pods -n development-pipeline
+- name: Find existing service accounts
+  shell: kubectl --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" get serviceaccounts -n development-pipeline

tools/images/standard-container/assets/playbooks/roles/promote/tasks/promote-chart.yaml

@@ -0,0 +1,16 @@
+- name: Install Plugin
+  shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push || true
+- name: Get harbor username
+  shell: cat {{ harbor_secret_mounted_path }}/username
+  register: harbor_username
+- name: Get harbor password
+  shell: cat {{ harbor_secret_mounted_path }}/password
+  register: harbor_password
+- name: Install Plugin
+  shell: helm plugin update push || helm plugin install https://github.com/chartmuseum/helm-push
+- name: Add Harbor Helm repository and Test repository
+  shell: helm repo add "{{ chart_repository }}-staging" "https://{{ chart_registry_url }}/{{ chart_name }}-staging" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}" && helm repo add "{{ chart_repository }}" "https://{{ chart_registry_url }}/{{ chart_name }}" --username="{{ harbor_username.stdout }}" --password="{{ harbor_password.stdout }}"
+- name: Pull down Helm Chart
+  shell: helm pull "{{ chart_repository }}-staging/{{ chart_name }}" --version="{{ version }}"
+- name: Push chart "{{ chart_name }}" to Helm registry
+  command: helm push "{{ chart_name }}-{{ version }}".tgz "{{ chart_repository }}"

tools/images/standard-container/assets/playbooks/roles/promote/tasks/promote-image.yaml

@@ -0,0 +1,10 @@
+- name: Tag and push to promotion repository
+  docker_image:
+    pull: true
+    name: "{{ docker_registry }}/{{ project }}-staging/{{ repo }}:{{ tag }}"
+- name: Tag and push to promotion repository
+  docker_image:
+    push: true
+    name: "{{ docker_registry }}/{{ project }}-staging/{{ repo }}"
+    repository: "{{ docker_registry }}/{{ project }}/{{ repo }}"
+    tag: "{{ tag }}"

tools/images/standard-container/assets/playbooks/tag-push-image.yaml

@@ -0,0 +1,7 @@
+- hosts: localhost
+  become: yes
+  vars:
+    image_status: "Success"
+  tasks:
+    - name: Push Image for scanning to Docker Repository
+      include_tasks: ./roles/images/tasks/tag-push-image.yaml

tools/images/standard-container/assets/setup-cluster-config.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json"
+cp "$(workspaces.k8s_cluster_data.path)/cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json"
+jq '.cluster_kubeconfig_path="$(workspaces.development_pipeline_data.path)/config"' "$(workspaces.development_pipeline_data.path)/cluster.json" > "$(workspaces.development_pipeline_data.path)/temp_cluster.json" && mv "$(workspaces.development_pipeline_data.path)/temp_cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json"

tools/images/standard-container/assets/test_helm_repo_basic_auth.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: harbor-helm-creds
+  namespace: development-pipeline
+type: kubernetes.io/basic-auth
+stringData:
+  username: admin
+  password: Harbor12345

zuul.d/jobs.yaml

@@ -38,6 +38,7 @@
         - ./tools/gate/jarvis/400-deploy-harbor.sh
         - ./tools/gate/jarvis/500-deploy-gerrit.sh
         - ./tools/gate/jarvis/600-deploy-tekton.sh
+        - ./tools/gate/jarvis/650-temporary-setup.sh
         - ./tools/gate/jarvis/700-deploy-jarvis-system.sh
         - ./tools/gate/jarvis/800-deploy-jarvis-projects.sh