Merge "Single node dev deploy gate for osh-infra"

This commit is contained in:
Zuul 2018-05-10 18:55:04 +00:00 committed by Gerrit Code Review
commit f86fa5ae6c
23 changed files with 682 additions and 0 deletions

View File

@ -23,6 +23,8 @@
voting: true voting: true
- openstack-helm-infra-fedora: - openstack-helm-infra-fedora:
voting: false voting: false
- openstack-helm-infra-dev-deploy:
voting: false
gate: gate:
jobs: jobs:
- openstack-helm-infra-linter - openstack-helm-infra-linter
@ -194,6 +196,14 @@
parent: openstack-helm-infra parent: openstack-helm-infra
nodeset: openstack-helm-fedora nodeset: openstack-helm-fedora
- job:
name: openstack-helm-infra-dev-deploy
timeout: 7200
pre-run: playbooks/osh-infra-upgrade-host.yaml
run: playbooks/osh-infra-dev-deploy.yaml
post-run: playbooks/osh-infra-collect-logs.yaml
nodeset: openstack-helm-single-node
- job: - job:
name: openstack-helm-infra-five-ubuntu name: openstack-helm-infra-five-ubuntu
parent: openstack-helm-infra parent: openstack-helm-infra

View File

@ -0,0 +1,100 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: primary
tasks:
- name: Deploy Required packages
shell: |
set -xe;
./tools/deployment/developer/000-install-packages.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Kubernetes
shell: |
set -xe;
./tools/deployment/developer/005-deploy-k8s.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Registry NFS, Redis, and Docker Registry
shell: |
set -xe;
./tools/deployment/developer/010-deploy-docker-registry.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy NFS for Logging, Monitoring and Alerting Components
shell: |
set -xe;
./tools/deployment/developer/020-lma-nfs-provisioner.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Prometheus
shell: |
set -xe;
./tools/deployment/developer/030-prometheus.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Alertmanager
shell: |
set -xe;
./tools/deployment/developer/040-alertmanager.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Kube-State-Metrics
shell: |
set -xe;
./tools/deployment/developer/050-kube-state-metrics.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Node Exporter
shell: |
set -xe;
./tools/deployment/developer/060-node-exporter.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Prometheus OpenStack Exporter
shell: |
set -xe;
./tools/deployment/developer/070-openstack-exporter.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Grafana
shell: |
set -xe;
./tools/deployment/developer/080-grafana.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Nagios
shell: |
set -xe;
./tools/deployment/developer/090-nagios.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Elasticsearch
shell: |
set -xe;
./tools/deployment/developer/100-elasticsearch.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Fluent-Logging
shell: |
set -xe;
./tools/deployment/developer/110-fluent-logging.sh
args:
chdir: "{{ zuul.project.src_dir }}"
- name: Deploy Kibana
shell: |
set -xe;
./tools/deployment/developer/120-kibana.sh
args:
chdir: "{{ zuul.project.src_dir }}"

View File

@ -0,0 +1,60 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint charts for deploying the docker registry
make pull-images nfs-provisioner
make pull-images redis
make pull-images registry
#NOTE: Deploy nfs for the docker registry
tee /tmp/docker-registry-nfs-provisioner.yaml << EOF
labels:
node_selector_key: openstack-helm-node-class
node_selector_value: primary
storageclass:
name: openstack-helm-bootstrap
EOF
helm upgrade --install docker-registry-nfs-provisioner \
./nfs-provisioner --namespace=docker-registry \
--values=/tmp/docker-registry-nfs-provisioner.yaml
#NOTE: Deploy redis for the docker registry
helm upgrade --install docker-registry-redis ./redis \
--namespace=docker-registry \
--set labels.node_selector_key=openstack-helm-node-class \
--set labels.node_selector_value=primary
#NOTE: Deploy the docker registry
tee /tmp/docker-registry.yaml << EOF
labels:
node_selector_key: openstack-helm-node-class
node_selector_value: primary
volume:
class_name: openstack-helm-bootstrap
EOF
helm upgrade --install docker-registry ./registry \
--namespace=docker-registry \
--values=/tmp/docker-registry.yaml
#NOTE: Wait for deployments
./tools/deployment/common/wait-for-pods.sh docker-registry
#NOTE: Validate Deployment info
helm status docker-registry-nfs-provisioner
helm status docker-registry-redis
helm status docker-registry

View File

@ -0,0 +1,35 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Deploy nfs instance for logging, monitoring and alerting components
tee /tmp/lma-nfs-provisioner.yaml << EOF
labels:
node_selector_key: openstack-control-plane
node_selector_value: enabled
storageclass:
name: openstack-helm-lma-nfs
EOF
helm upgrade --install lma-nfs-provisioner \
./nfs-provisioner --namespace=openstack \
--values=/tmp/lma-nfs-provisioner.yaml
#NOTE: Wait for deployment
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status lma-nfs-provisioner

View File

@ -0,0 +1,30 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images prometheus-kube-state-metrics
#NOTE: Deploy command
helm upgrade --install prometheus-kube-state-metrics \
./prometheus-kube-state-metrics --namespace=kube-system
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh kube-system
#NOTE: Validate Deployment info
helm status prometheus-kube-state-metrics

View File

@ -0,0 +1,30 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images prometheus-node-exporter
#NOTE: Deploy command
helm upgrade --install prometheus-node-exporter \
./prometheus-node-exporter --namespace=kube-system
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh kube-system
#NOTE: Validate Deployment info
helm status prometheus-node-exporter

View File

@ -0,0 +1,41 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images prometheus-openstack-exporter
#NOTE: Deploy command
tee /tmp/prometheus-openstack-exporter.yaml << EOF
manifests:
job_ks_user: false
dependencies:
static:
prometheus_openstack_exporter:
jobs: null
services: null
EOF
helm upgrade --install prometheus-openstack-exporter \
./prometheus-openstack-exporter \
--namespace=openstack \
--values=/tmp/prometheus-openstack-exporter.yaml
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status prometheus-openstack-exporter

View File

@ -0,0 +1,32 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images kibana
#NOTE: Deploy command
helm upgrade --install kibana ./kibana \
--namespace=openstack \
--set network.kibana.node_port.enabled=true \
--set network.kibana.ingress.public=false
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status kibana

View File

@ -0,0 +1,43 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -e
# From Kolla-Kubernetes, orginal authors Kevin Fox & Serguei Bezverkhi
# Default wait timeout is 600 seconds
end=$(date +%s)
if ! [ -z $2 ]; then
end=$((end + $2))
else
end=$((end + 900))
fi
while true; do
kubectl get pods --namespace=$1 -o json | jq -r \
'.items[].status.phase' | grep Pending > /dev/null && \
PENDING=True || PENDING=False
query='.items[]|select(.status.phase=="Running")'
query="$query|.status.containerStatuses[].ready"
kubectl get pods --namespace=$1 -o json | jq -r "$query" | \
grep false > /dev/null && READY="False" || READY="True"
kubectl get jobs -o json --namespace=$1 | jq -r \
'.items[] | .spec.completions == .status.succeeded' | \
grep false > /dev/null && JOBR="False" || JOBR="True"
[ $PENDING == "False" -a $READY == "True" -a $JOBR == "True" ] && \
break || true
sleep 5
now=$(date +%s)
[ $now -gt $end ] && echo containers failed to start. && \
kubectl get pods --namespace $1 -o wide && exit -1
done

View File

@ -0,0 +1,25 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
sudo apt-get update
sudo apt-get install --no-install-recommends -y \
ca-certificates \
git \
make \
nmap \
curl

View File

@ -0,0 +1,20 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
make dev-deploy setup-host
make dev-deploy k8s

View File

@ -0,0 +1 @@
../common/010-deploy-docker-registry.sh

View File

@ -0,0 +1 @@
../common/020-lma-nfs-provisioner.sh

View File

@ -0,0 +1,41 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images prometheus
#NOTE: Deploy command
tee /tmp/prometheus.yaml << EOF
storage:
storage_class: openstack-helm-lma-nfs
network:
prometheus:
ingress:
public: false
node_port:
enabled: true
EOF
helm upgrade --install prometheus ./prometheus \
--namespace=openstack \
--values=/tmp/prometheus.yaml
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status prometheus

View File

@ -0,0 +1,44 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images alertmanager
#NOTE: Deploy command
tee /tmp/prometheus-alertmanager.yaml << EOF
pod:
replicas:
alertmanager: 1
storage:
storage_class: openstack-helm-lma-nfs
network:
alertmanager:
ingress:
public: false
node_port:
enabled: true
EOF
helm upgrade --install prometheus-alertmanager ./prometheus-alertmanager \
--namespace=openstack \
--values=/tmp/prometheus-alertmanager.yaml
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status prometheus-alertmanager

View File

@ -0,0 +1 @@
../common/050-kube-state-metrics.sh

View File

@ -0,0 +1 @@
../common/060-node-exporter.sh

View File

@ -0,0 +1 @@
../common/070-openstack-exporter.sh

View File

@ -0,0 +1,59 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images grafana
#NOTE: Deploy command
tee /tmp/grafana.yaml << EOF
dependencies:
static:
grafana:
jobs: null
services: null
manifests:
ingress: false
job_db_init: false
job_db_init_session: false
job_db_session_sync: false
secret_db: false
secret_db_session: false
service_ingress: false
conf:
grafana:
database:
type: sqlite3
session:
provider: file
provider_config: sessions
network:
grafana:
ingress:
public: false
node_port:
enabled: true
EOF
helm upgrade --install grafana ./grafana \
--namespace=openstack \
--values=/tmp/grafana.yaml
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status grafana

View File

@ -0,0 +1,32 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images nagios
#NOTE: Deploy command
helm upgrade --install nagios ./nagios \
--namespace=openstack \
--set network.nagios.ingress.public=false \
--set network.nagios.node_port.enabled=true
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status nagios

View File

@ -0,0 +1,42 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images elasticsearch
#NOTE: Deploy command
tee /tmp/elasticsearch.yaml << EOF
storage:
elasticsearch:
storage_class: openstack-helm-lma-nfs
monitoring:
prometheus:
enabled: true
pod:
replicas:
data: 1
EOF
helm upgrade --install elasticsearch ./elasticsearch \
--namespace=openstack \
--values=/tmp/elasticsearch.yaml
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status elasticsearch

View File

@ -0,0 +1,32 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
set -xe
#NOTE: Pull images and lint chart
make pull-images fluent-logging
#NOTE: Deploy command
helm upgrade --install fluent-logging ./fluent-logging \
--namespace=openstack \
--set monitoring.prometheus.enabled=true \
--set pod.replicas.fluentd=1
#NOTE: Wait for deploy
./tools/deployment/common/wait-for-pods.sh openstack
#NOTE: Validate Deployment info
helm status fluent-logging

View File

@ -0,0 +1 @@
../common/120-kibana.sh