Add development pipeline

Change-Id: I2e773092fafefa1f9c6666ad7883766e24206de3

charts/development-pipeline/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/development-pipeline/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: development-pipeline
+description: A Helm chart for Tekton Development pipeline
+type: application
+version: 0.1.0
+appVersion: 1.16.0

charts/development-pipeline/config_map.yaml.example

@@ -0,0 +1,86 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: deployment-flow
+  namespace: default
+data:
+  credentials: "Some creds"
+  kubeconfig.json: "Default kubeconfig.json"
+  cluster_exists: "true"
+  production: "false"
+  development.yaml: |
+    proxy:
+      http: ""
+      https: ""
+      noproxy: ""
+      enabled: false
+    docker_registry: "harbor-core.jarvis.local"
+    images:
+      - build_from_source: true
+        project: "test"
+        repo: "scratch"
+        tag: "1.built"
+        build:
+          git_repo: "https://review.opendev.org/airship/charts"
+          checkout_loc: "/src/checkout/scratch"
+          refspec: "refs/changes/41/770141/7"
+          version: "refs/changes/*:refs/changes/*"
+        makefile:
+          path: "tools/images"
+          target: "build"
+          file: "Makefile"
+          image_name: "scratch"
+          image_base: "scratch"
+      - build_from_source: true
+        repo: "microflow/standard-container"
+        project: "test"
+        tag: "1.built"
+        build:
+          git_repo: "https://review.opendev.org/airship/charts"
+          checkout_loc: "/src/checkout/standard-container"
+          refspec: "refs/changes/41/770141/7"
+          version: "refs/changes/*:refs/changes/*"
+        makefile:
+          path: "tools/images"
+          target: "build"
+          file: "Makefile"
+          image_name: "standard-container"
+          image_base: "ubuntu:20.04"
+      - build_from_source: false
+        remote_registry: "docker.io"
+        remote_repo: "testing2016/jrunner"
+        remote_tag: "2.0"
+        repo: "microflow/standard-container"
+        project: "test"
+        tag: "1.existing"
+    charts:
+      - name: "tekton-pipelines"
+        build_from_source: true
+        build:
+          git_repo: "https://review.opendev.org/airship/charts"
+          checkout_loc: "/src/checkout/airship/charts"
+          refspec: "master"
+        makefile:
+          path: "charts"
+          target: "tekton-pipelines"
+          file: "Makefile"
+      - name: "tekton-triggers"
+        build_from_source: true
+        build:
+          git_repo: "https://review.opendev.org/airship/charts"
+          checkout_loc: "/src/checkout/airship/charts"
+          refspec: "master"
+        makefile:
+          path: "charts"
+          target: "tekton-triggers"
+          file: "Makefile"
+      - name: "tekton-dashboard"
+        build_from_source: true
+        build:
+          git_repo: "https://review.opendev.org/airship/charts"
+          checkout_loc: "/src/checkout/airship/charts"
+          refspec: "master"
+        makefile:
+          path: "charts"
+          target: "tekton-dashboard"
+          file: "Makefile"

charts/development-pipeline/templates/cluster_role.yaml

@@ -0,0 +1,11 @@
+{{- if $.Values.clusterRole.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ $.Values.clusterRole.name }}
+rules:
+  # EventListeners need to be able to fetch any clustertriggerbindings
+   - apiGroups: ["triggers.tekton.dev"]
+     resources: ["clustertriggerbindings"]
+     verbs: ["get", "list", "watch"]
+{{- end }}

charts/development-pipeline/templates/clusterrolebinding.yaml

@@ -0,0 +1,14 @@
+{{- if and ($.Values.serviceAccount.create) ($.Values.clusterRole.bind) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ $.Values.clusterRole.name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $.Values.serviceAccount.name }}
+    namespace: {{ $.Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ $.Values.clusterRole.name }}
+{{- end }}

charts/development-pipeline/templates/eventlistener.yaml

@@ -0,0 +1,13 @@
+apiVersion: triggers.tekton.dev/v1alpha1
+kind: EventListener
+metadata:
+  name: development-listener
+  namespace: {{ $.Release.Namespace }}
+spec:
+  serviceAccountName:  {{ $.Values.serviceAccount.name }}
+  triggers:
+    - name: development-trigger
+      bindings:
+        - ref: development-pipeline-binding
+      template:
+        name: development-pipeline-template

charts/development-pipeline/templates/ingress.yaml

@@ -0,0 +1,14 @@
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: ingress-development-listener
+  namespace: {{ .Release.Namespace }}
+spec:
+  rules:
+    - host: el-development.{{- .Release.Namespace }}.tekton.jarvis.local
+      http:
+        paths:
+          - path: /
+            backend:
+              serviceName: el-development-listener
+              servicePort: 8080

charts/development-pipeline/templates/pipeline.yaml

@@ -0,0 +1,70 @@
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: development-pipeline
+  namespace: {{ $.Release.Namespace }}
+spec:
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  tasks:
+    - name: deploy-k8s
+      taskRef:
+        name: k8s-cluster
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+    - name: build-image
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+      taskRef:
+        name: build-images
+    - name: build-chart
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+      taskRef:
+        name: build-charts
+    - name: deploy-artifacts
+      runAfter: [deploy-k8s,build-image,build-chart]
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+      taskRef:
+        name: deployment
+    - name: functional-testing
+      runAfter: [deploy-artifacts]
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+      taskRef:
+        name: functional
+    - name: promote-artifacts
+      runAfter: [functional-testing]
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+      taskRef:
+        name: promote
+  finally:
+    - name: cleanup
+      workspaces:
+        - name: k8s_cluster_data
+          workspace: k8s_cluster_data
+        - name: development_pipeline_data
+          workspace: development_pipeline_data
+      taskRef:
+        name: cleanup

charts/development-pipeline/templates/role.yaml

@@ -0,0 +1,22 @@
+{{- if $.Values.role.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ $.Values.role.name }}
+  namespace: {{ $.Release.Namespace }}
+rules:
+  # EventListeners need to be able to fetch all namespaced resources
+  - apiGroups: ["triggers.tekton.dev"]
+    resources: ["eventlisteners", "triggerbindings", "triggertemplates", "triggers"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch"]
+   # Permissions to create resources in associated TriggerTemplates
+  - apiGroups: ["tekton.dev"]
+    resources: ["pipelineruns", "taskruns"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["serviceaccounts"]
+    verbs: ["impersonate"]
+{{- end }}

charts/development-pipeline/templates/rolebinding.yaml

@@ -0,0 +1,14 @@
+{{- if and ($.Values.serviceAccount.create) ($.Values.role.bind) }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ $.Values.role.name }}
+  namespace: {{ $.Release.Namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ $.Values.serviceAccount.name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ $.Values.role.name }}
+{{- end }}

charts/development-pipeline/templates/serviceaccount.yaml

@@ -0,0 +1,7 @@
+{{- if $.Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name:  {{ $.Values.serviceAccount.name }}
+  namespace: {{ $.Release.Namespace }}
+{{- end }}

charts/development-pipeline/templates/task-chart.yaml

@@ -0,0 +1,18 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: build-charts
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task builds charts if source is provided
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: build-chart
+      image: {{ $.Values.tasks.chart.buildChartImage }}
+      script: |
+        #!/usr/bin/env sh
+        ansible-playbook -vvv {{ $.Values.tasks.chart.buildPlaybook }}  -i hosts -e @"$(workspaces.k8s_cluster_data.path)/development.yaml"
+        # TODO copy JSON to shared workspace to make it available for other tasks

charts/development-pipeline/templates/task-cleanup.yaml

@@ -0,0 +1,37 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: cleanup
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task gathers logs and cleans up the environment
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: gather-logs
+      image: {{ $.Values.tasks.cleanup.cleanupImage }}
+      script: |
+        #!/bin/sh
+        # TODO
+        echo "gather logs"
+        echo "publish logs"
+        echo "notify"
+    - name: cleanup
+      image: {{ $.Values.tasks.cleanup.cleanupImage }}
+      script: |
+        #!/bin/sh
+        if grep -i "true" "$(workspaces.k8s_cluster_data.path)/production"; then
+            echo "No cleanup is required. CI was set to false."
+            exit 0
+        fi
+        if grep "true" "$(workspaces.k8s_cluster_data.path)/cluster_exists"; then
+            # TODO
+            echo "Cleanup artifacts"
+        else
+          # TODO
+          echo "Teardown k8s cluster"
+        fi
+        # TODO Delete sensitive data from shared workspace
+        rm "$(workspaces.development_pipeline_data.path)/kubeconfig.json"

charts/development-pipeline/templates/task-deployment.yaml

@@ -0,0 +1,19 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: deployment
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task deploys artifacts from produced by previous tasks
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: deploy-artifacts
+      image: {{ $.Values.tasks.deployment.deployImage }}
+      script: |
+        #!/usr/bin/env sh
+        # Artifacts details are stored in JSON files as an output of previous tasks.
+        # TODO copy logs and scan results to shared workspace
+        echo "Deploying artifacts"

charts/development-pipeline/templates/task-functional.yaml

@@ -0,0 +1,25 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: functional
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task will validate the k8s cluster using kubeconfig from k8s-cluster task
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: run-functional-tests
+      image: {{ $.Values.tasks.functional.validateImage }}
+      script: |
+        #!/bin/sh
+        if [ -f "$(workspaces.development_pipeline_data.path)/kubeconfig.json" ] ; then
+          echo "Kubeconfig is found, starting validation"
+        else
+          echo "Error: missing kubeconfig.json file"
+          exit 1
+        fi
+        echo "Validating cluster"
+        echo "ansible-playbook {{ $.Values.tasks.functional.playbook }}"
+        cat "$(workspaces.development_pipeline_data.path)/kubeconfig.json"

charts/development-pipeline/templates/task-image.yaml

@@ -0,0 +1,59 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: build-images
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task builds images if source is provided
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: build-image
+      image: {{ $.Values.tasks.image.buildImage }}
+      volumeMounts:
+        - mountPath: /certs/client
+          name: dind-certs
+      env:
+      # Connect to the sidecar over TCP, with TLS.
+      - name: DOCKER_HOST
+        value: tcp://localhost:2376
+      # Verify TLS.
+      - name: DOCKER_TLS_VERIFY
+        value: '1'
+      # Use the certs generated by the sidecar daemon.
+      - name: DOCKER_CERT_PATH
+        value: /certs/client
+      script: |
+        #!/usr/bin/env sh
+        ansible-playbook -vvv  {{ $.Values.tasks.image.buildPlaybook }} -i hosts -e @"$(workspaces.k8s_cluster_data.path)/development.yaml"
+        docker images
+        # TODO copy JSON file to shared workspace to make it available for other tasks
+        # TODO copy logs and scan results to shared location
+  sidecars:
+    - image: {{ $.Values.tasks.image.sidecarServer }}
+      name: server
+      args:
+        - --storage-driver=vfs
+        - --userland-proxy=false
+        - --debug
+        - --insecure-registry={{ $.Values.tasks.image.insecureRegistry }}
+      securityContext:
+        privileged: true
+      env:
+      # Write generated certs to the path shared with the client.
+      - name: DOCKER_TLS_CERTDIR
+        value: /certs
+      volumeMounts:
+        - mountPath: /certs/client
+          name: dind-certs
+      # Wait for the dind daemon to generate the certs it will share with the
+      # client.
+      readinessProbe:
+        periodSeconds: 1
+        exec:
+          command: ['ls', '/certs/client/ca.pem']
+  volumes:
+    - name: dind-certs
+      emptyDir: {}

charts/development-pipeline/templates/task-k8s-cluster.yaml

@@ -0,0 +1,30 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: k8s-cluster
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task will create a k8s cluster if needed and store kubeconfig in shared workspace
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: generate-kubeconfig
+      image: {{ $.Values.tasks.cluster.createClusterImage }}
+      script: |
+        #!/usr/bin/env sh
+        # Use provided kubeconfig for existing cluster
+        if grep -i "true" "$(workspaces.k8s_cluster_data.path)/cluster_exists"; then
+          echo "Use existing cluster."
+          cp "$(workspaces.k8s_cluster_data.path)/kubeconfig.json" "$(workspaces.development_pipeline_data.path)/kubeconfig.json"
+        # Deploy cluster and copy kubeconfig
+        else
+          echo "Create a k8s cluster"
+          # TODO
+          echo "ansible-playbook -vvv {{ $.Values.tasks.cluster.createClusterPlaybook }}  -i hosts -e @\"$(workspaces.k8s_cluster_data.path)/development.yaml\""
+          # TODO copy generated kubeconfig to shared wokspace
+          echo "New cluster's kubeconfig data" > "$(workspaces.development_pipeline_data.path)/kubeconfig.json"
+        fi
+        echo "ansible-playbook -vvv {{ $.Values.tasks.cluster.validateClusterPlaybook }}  -i hosts -e @\"$(workspaces.k8s_cluster_data.path)/development.yaml\""
+        # TODO copy JSON file to shared workspace to make it available for other tasks

charts/development-pipeline/templates/task-promote.yaml

@@ -0,0 +1,18 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: promote
+  namespace: {{ $.Release.Namespace }}
+spec:
+  description: >-
+    This task builds charts if source is provided
+  workspaces:
+    - name: k8s_cluster_data
+    - name: development_pipeline_data
+  steps:
+    - name: build-chart
+      image: {{ $.Values.tasks.promote.promoteArtifacts }}
+      script: |
+        #!/usr/bin/env sh
+        # TODO call playbook to promote artifacts generated in previous tasks
+        echo "Promote artifacts"

charts/development-pipeline/templates/triggerbindings.yaml

@@ -0,0 +1,5 @@
+apiVersion: triggers.tekton.dev/v1alpha1
+kind: TriggerBinding
+metadata:
+  name: development-pipeline-binding
+  namespace: {{ $.Release.Namespace }}

charts/development-pipeline/templates/triggertemplate.yaml

@@ -0,0 +1,28 @@
+apiVersion: triggers.tekton.dev/v1alpha1
+kind: TriggerTemplate
+metadata:
+  name: development-pipeline-template
+  namespace: {{ $.Release.Namespace }}
+spec:
+  resourcetemplates:
+    - apiVersion: tekton.dev/v1beta1
+      kind: PipelineRun
+      metadata:
+        generateName: development-pipeline-run
+      spec:
+        pipelineRef:
+          name: development-pipeline
+        serviceAccountName: {{ $.Values.serviceAccount.name }}
+        workspaces:
+          - name: k8s_cluster_data
+            configMap:
+              name: {{ $.Values.trigger.configMap }}
+          - name: development_pipeline_data
+            volumeClaimTemplate:
+              spec:
+                storageClassName: {{ $.Values.pvc.storageClass }}
+                accessModes:
+                  - ReadWriteOnce
+                resources:
+                  requests:
+                    storage: {{ $.Values.pvc.size }}

charts/development-pipeline/values.yaml

@@ -0,0 +1,43 @@
+serviceAccount:
+  create: true
+  name: sa-development-pipeline
+
+role:
+  bind: true
+  name: development-pipeline
+  create: true
+
+clusterRole:
+  bind: true
+  name: development-pipeline
+  create: true
+
+pvc:
+  storageClass: standard
+  size: 1Gi
+
+tasks:
+  cluster:
+    createClusterImage: &base_image testing2016/standard-container:1.0
+    createClusterPlaybook: /playbooks/create_cluster.yaml
+    validateClusterPlaybook: /playbooks/validate_cluster.yaml
+  image:
+    buildImage: *base_image
+    sidecarServer: docker:19-dind
+    buildPlaybook: /playbooks/build_image.yaml
+    insecureRegistry: harbor-core.jarvis.local
+  chart:
+    buildChartImage: *base_image
+    buildPlaybook: /playbooks/helm_chart.yaml
+  deployment:
+    deployImage: *base_image
+  promote:
+    promoteArtifacts: *base_image
+  functional:
+    validateImage: *base_image
+    playbook: /playbooks/functional.yaml
+  cleanup:
+    cleanupImage: *base_image
+
+trigger:
+  configMap: deployment-flow