diff --git a/doc/source/index.rst b/doc/source/index.rst index b991d22ca..8dc939311 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -6,7 +6,6 @@ Contents: .. toctree:: :maxdepth: 2 - install/index contributor/contributing testing/index monitoring/index diff --git a/doc/source/install/index.rst b/doc/source/install/index.rst deleted file mode 100644 index 7843482d9..000000000 --- a/doc/source/install/index.rst +++ /dev/null @@ -1,9 +0,0 @@ -Installation -============ - -Contents: - -.. toctree:: - :maxdepth: 2 - - multinode diff --git a/doc/source/install/multinode.rst b/doc/source/install/multinode.rst deleted file mode 100644 index a7a544a8f..000000000 --- a/doc/source/install/multinode.rst +++ /dev/null @@ -1,237 +0,0 @@ -====================== -Development Deployment -====================== - -Deploy Local Docker Registry -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/010-deploy-docker-registry.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/010-deploy-docker-registry.sh - -Deploy Cluster and Namespace Ingress Controllers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/common/ingress.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/020-ingress.sh - -Deploy Ceph -^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/030-ceph.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/030-ceph.sh - -Activate the OSH-Infra namespace to be able to use Ceph -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/035-ceph-ns-activate.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/035-ceph-ns-activate.sh - -Deploy LDAP -^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/040-ldap.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/040-ldap.sh - -Deploy MariaDB -^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/045-mariadb.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/045-mariadb.sh - -Deploy Prometheus -^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/050-prometheus.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/050-prometheus.sh - -Deploy Alertmanager -^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/060-alertmanager.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/060-alertmanager.sh - -Deploy Kube-State-Metrics -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/070-kube-state-metrics.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/070-kube-state-metrics.sh - -Deploy Node Exporter -^^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/080-node-exporter.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/080-node-exporter.sh - -Deploy Process Exporter -^^^^^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/085-process-exporter.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/085-process-exporter.sh - -Deploy OpenStack Exporter -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/090-openstack-exporter.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/090-openstack-exporter.sh - -Deploy Grafana -^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/100-grafana.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/100-grafana.sh - -Deploy Nagios -^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/110-nagios.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/110-nagios.sh - -Deploy Rados Gateway for OSH-Infra -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/115-radosgw-osh-infra.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/115-radosgw-osh-infra.sh - -Deploy Elasticsearch -^^^^^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/120-elasticsearch.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/120-elasticsearch.sh - -Deploy Fluentbit -^^^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/125-fluentbit.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/125-fluentbit.sh - -Deploy Fluentd -^^^^^^^^^^^^^^ - -.. literalinclude:: ../../../tools/deployment/multinode/130-fluentd.sh - :language: shell - :lines: 1,17- - -Alternatively, this step can be performed by running the script directly: - -.. code-block:: shell - - ./tools/deployment/multinode/130-fluentd.sh diff --git a/tools/deployment/apparmor/000-install-packages.sh b/tools/deployment/apparmor/000-install-packages.sh deleted file mode 120000 index d702c4899..000000000 --- a/tools/deployment/apparmor/000-install-packages.sh +++ /dev/null @@ -1 +0,0 @@ -../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/001-setup-apparmor-profiles.sh b/tools/deployment/apparmor/001-setup-apparmor-profiles.sh deleted file mode 120000 index 543e2fc9d..000000000 --- a/tools/deployment/apparmor/001-setup-apparmor-profiles.sh +++ /dev/null @@ -1 +0,0 @@ -../common/001-setup-apparmor-profiles.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/005-deploy-k8s.sh b/tools/deployment/apparmor/005-deploy-k8s.sh deleted file mode 120000 index 003bfbb8e..000000000 --- a/tools/deployment/apparmor/005-deploy-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/020-ceph.sh b/tools/deployment/apparmor/020-ceph.sh deleted file mode 120000 index 1ab828eed..000000000 --- a/tools/deployment/apparmor/020-ceph.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-logging/020-ceph.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/025-ceph-ns-activate.sh b/tools/deployment/apparmor/025-ceph-ns-activate.sh deleted file mode 120000 index 10e71eedb..000000000 --- a/tools/deployment/apparmor/025-ceph-ns-activate.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-logging/025-ceph-ns-activate.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/030-mariadb.sh b/tools/deployment/apparmor/030-mariadb.sh deleted file mode 100755 index b53fb698d..000000000 --- a/tools/deployment/apparmor/030-mariadb.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make mariadb - -: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB:="$(./tools/deployment/common/get-values-overrides.sh mariadb)"} - -#NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} -helm upgrade --install mariadb ./mariadb \ - --namespace=osh-infra \ - --set monitoring.prometheus.enabled=true \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB} - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra - -# Delete the test pod if it still exists -kubectl delete pods -l application=mariadb,release_group=mariadb,component=test --namespace=osh-infra --ignore-not-found -#NOTE: Validate the deployment -helm test mariadb --namespace osh-infra diff --git a/tools/deployment/apparmor/040-memcached.sh b/tools/deployment/apparmor/040-memcached.sh deleted file mode 100755 index 5a05c67d1..000000000 --- a/tools/deployment/apparmor/040-memcached.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -namespace="osh-infra" -: ${OSH_INFRA_EXTRA_HELM_ARGS_MEMCACHED:="$(./tools/deployment/common/get-values-overrides.sh memcached)"} - -# NOTE: Lint and package chart -make memcached - -tee /tmp/memcached.yaml < $unsorted_process_file -sort --numeric-sort $unsorted_process_file > $sorted_process_file - -# The last/latest process in the list will actually be the "ls" command above, -# which isn't running any more, so remove it. -sed -i '$ d' $sorted_process_file - -while IFS='' read -r process || [[ -n "$process" ]]; do - echo "Process ID: $process" - proc_name=`kubectl -n $namespace exec $pod -- cat /proc/$process/status | grep "Name:" | awk -F' ' '{print $2}'` - echo "Process Name: $proc_name" - profile=`kubectl -n $namespace exec $pod -- cat /proc/$process/attr/current` - echo "Profile running: $profile" - if test "$profile" != "$expected_profile" - then - if test "$proc_name" == "pause" - then - echo "Root process (pause) can run docker-default, it's ok." - else - echo "$profile is the WRONG PROFILE!!" - return 1 - fi - fi -done < $sorted_process_file diff --git a/tools/deployment/apparmor/050-libvirt.sh b/tools/deployment/apparmor/050-libvirt.sh deleted file mode 100755 index 700fc8758..000000000 --- a/tools/deployment/apparmor/050-libvirt.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -set -xe - -#NOTE: Lint and package chart -make libvirt - -tee /tmp/libvirt.yaml < - @{LIBVIRT}="libvirt" - profile my-apparmor-v1 flags=(attach_disconnected) { - #include - #include - - capability kill, - capability audit_write, - capability audit_control, - capability net_admin, - capability net_raw, - capability setgid, - capability sys_admin, - capability sys_module, - capability sys_ptrace, - capability sys_pacct, - capability sys_nice, - capability sys_chroot, - capability setuid, - capability dac_override, - capability dac_read_search, - capability fowner, - capability chown, - capability setpcap, - capability mknod, - capability fsetid, - capability audit_write, - capability ipc_lock, - - # Needed for vfio - capability sys_resource, - - mount options=(rw,rslave) -> /, - mount options=(rw, nosuid) -> /{var/,}run/libvirt/qemu/*.dev/, - - mount options=(rw, move) /dev/ -> /{var/,}run/libvirt/qemu/*.dev/, - mount options=(rw, move) /dev/hugepages/ -> /{var/,}run/libvirt/qemu/*.hugepages/, - mount options=(rw, move) /dev/mqueue/ -> /{var/,}run/libvirt/qemu/*.mqueue/, - mount options=(rw, move) /dev/pts/ -> /{var/,}run/libvirt/qemu/*.pts/, - mount options=(rw, move) /dev/shm/ -> /{var/,}run/libvirt/qemu/*.shm/, - - mount options=(rw, move) /{var/,}run/libvirt/qemu/*.dev/ -> /dev/, - mount options=(rw, move) /{var/,}run/libvirt/qemu/*.hugepages/ -> /dev/hugepages/, - mount options=(rw, move) /{var/,}run/libvirt/qemu/*.mqueue/ -> /dev/mqueue/, - mount options=(rw, move) /{var/,}run/libvirt/qemu/*.pts/ -> /dev/pts/, - mount options=(rw, move) /{var/,}run/libvirt/qemu/*.shm/ -> /dev/shm/, - - network inet stream, - network inet dgram, - network inet6 stream, - network inet6 dgram, - network netlink raw, - network packet dgram, - network packet raw, - - # for --p2p migrations - unix (send, receive) type=stream addr=none peer=(label=unconfined addr=none), - - ptrace (trace) peer=unconfined, - ptrace (trace) peer=/usr/sbin/libvirtd, - ptrace (trace) peer=/usr/sbin/dnsmasq, - ptrace (trace) peer=libvirt-*, - - signal (send) peer=/usr/sbin/dnsmasq, - signal (read, send) peer=libvirt-*, - signal (send) set=("kill", "term") peer=unconfined, - - # For communication/control to qemu-bridge-helper - unix (send, receive) type=stream addr=none peer=(label=/usr/sbin/libvirtd//qemu_bridge_helper), - signal (send) set=("term") peer=/usr/sbin/libvirtd//qemu_bridge_helper, - - # Very lenient profile for libvirtd since we want to first focus on confining - # the guests. Guests will have a very restricted profile. - / r, - /** rwmkl, - - /bin/* PUx, - /sbin/* PUx, - /usr/bin/* PUx, - /usr/sbin/virtlogd pix, - /usr/sbin/* PUx, - /{usr/,}lib/udev/scsi_id PUx, - /usr/{lib,lib64}/xen-common/bin/xen-toolstack PUx, - /usr/{lib,lib64}/xen/bin/* Ux, - /usr/lib/xen-*/bin/libxl-save-helper PUx, - - # Required by nwfilter_ebiptables_driver.c:ebiptablesWriteToTempFile() to - # read and run an ebtables script. - /var/lib/libvirt/virtd* ixr, - - # force the use of virt-aa-helper - audit deny /{usr/,}sbin/apparmor_parser rwxl, - audit deny /etc/apparmor.d/libvirt/** wxl, - audit deny /sys/kernel/security/apparmor/features rwxl, - audit deny /sys/kernel/security/apparmor/matching rwxl, - audit deny /sys/kernel/security/apparmor/.* rwxl, - /sys/kernel/security/apparmor/profiles r, - /usr/{lib,lib64}/libvirt/* PUxr, - /usr/{lib,lib64}/libvirt/libvirt_parthelper ix, - /usr/{lib,lib64}/libvirt/libvirt_iohelper ix, - /etc/libvirt/hooks/** rmix, - /etc/xen/scripts/** rmix, - - # allow changing to our UUID-based named profiles - change_profile -> @{LIBVIRT}-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*-[0-9a-f]*, - - /usr/{lib,lib64,lib/qemu,libexec}/qemu-bridge-helper Cx -> qemu_bridge_helper, - # child profile for bridge helper process - profile qemu_bridge_helper { - #include - - capability setuid, - capability setgid, - capability setpcap, - capability net_admin, - - network inet stream, - - # For communication/control from libvirtd - unix (send, receive) type=stream addr=none peer=(label=/usr/sbin/libvirtd), - signal (receive) set=("term") peer=/usr/sbin/libvirtd, - - /dev/net/tun rw, - /etc/qemu/** r, - owner @{PROC}/*/status r, - - /usr/{lib,lib64,lib/qemu,libexec}/qemu-bridge-helper rmix, - } - } -EOF - -#NOTE: Deploy command -: ${OSH_EXTRA_HELM_ARGS_LIBVIRT:="$(./tools/deployment/common/get-values-overrides.sh libvirt)"} - -helm upgrade --install libvirt ./libvirt \ - --namespace=openstack \ - --values=/tmp/libvirt.yaml \ - --set network.backend="null" \ - ${OSH_EXTRA_HELM_ARGS} \ - ${OSH_EXTRA_HELM_ARGS_LIBVIRT} - -#NOTE: Validate Deployment info -./tools/deployment/common/wait-for-pods.sh openstack diff --git a/tools/deployment/apparmor/050-prometheus-alertmanager.sh b/tools/deployment/apparmor/050-prometheus-alertmanager.sh deleted file mode 100755 index 12bcecc8e..000000000 --- a/tools/deployment/apparmor/050-prometheus-alertmanager.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make prometheus-alertmanager - -: ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS_ALERTMANAGER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-alertmanager)"} - -#NOTE: Deploy command -helm upgrade --install prometheus-alertmanager ./prometheus-alertmanager \ - --namespace=osh-infra \ - --set pod.replicas.alertmanager=1 \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS_ALERTMANAGER} - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/apparmor/055-prometheus.sh b/tools/deployment/apparmor/055-prometheus.sh deleted file mode 120000 index 9e44b15f8..000000000 --- a/tools/deployment/apparmor/055-prometheus.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/050-prometheus.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/060-prometheus-node-exporter.sh b/tools/deployment/apparmor/060-prometheus-node-exporter.sh deleted file mode 120000 index 4104e88c9..000000000 --- a/tools/deployment/apparmor/060-prometheus-node-exporter.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/080-node-exporter.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/065-prometheus-openstack-exporter.sh b/tools/deployment/apparmor/065-prometheus-openstack-exporter.sh deleted file mode 100755 index 4d6ed1ceb..000000000 --- a/tools/deployment/apparmor/065-prometheus-openstack-exporter.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make prometheus-openstack-exporter - -: ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS_OPENSTACK_EXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-openstack-exporter)"} - -#NOTE: Deploy command -helm upgrade --install prometheus-openstack-exporter \ - ./prometheus-openstack-exporter \ - --namespace=openstack \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS_OPENSTACK_EXPORTER} - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh openstack diff --git a/tools/deployment/apparmor/070-prometheus-blackbox-exporter.sh b/tools/deployment/apparmor/070-prometheus-blackbox-exporter.sh deleted file mode 100755 index a80e515a0..000000000 --- a/tools/deployment/apparmor/070-prometheus-blackbox-exporter.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make prometheus-blackbox-exporter - -: ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS_BLACKBOX_EXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-blackbox-exporter)"} - -#NOTE: Deploy command -helm upgrade --install prometheus-blackbox-exporter \ - ./prometheus-blackbox-exporter \ - --namespace=openstack \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS_BLACKBOX_EXPORTER} - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh openstack diff --git a/tools/deployment/apparmor/075-prometheus-process-exporter.sh b/tools/deployment/apparmor/075-prometheus-process-exporter.sh deleted file mode 120000 index dc2a7b056..000000000 --- a/tools/deployment/apparmor/075-prometheus-process-exporter.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/090-process-exporter.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/080-grafana.sh b/tools/deployment/apparmor/080-grafana.sh deleted file mode 120000 index 60dc21427..000000000 --- a/tools/deployment/apparmor/080-grafana.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/110-grafana.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/085-rabbitmq.sh b/tools/deployment/apparmor/085-rabbitmq.sh deleted file mode 100755 index c21698c46..000000000 --- a/tools/deployment/apparmor/085-rabbitmq.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make rabbitmq - -: ${OSH_INFRA_EXTRA_HELM_ARGS_RABBITMQ:="$(./tools/deployment/common/get-values-overrides.sh rabbitmq)"} - -#NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} -helm upgrade --install rabbitmq ./rabbitmq \ - --namespace=osh-infra \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_RABBITMQ} - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/apparmor/090-elasticsearch.sh b/tools/deployment/apparmor/090-elasticsearch.sh deleted file mode 100755 index c3ffeb992..000000000 --- a/tools/deployment/apparmor/090-elasticsearch.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make elasticsearch - -#NOTE: Deploy command -tee /tmp/elasticsearch.yaml << EOF -dependencies: - static: - tests: - jobs: null -storage: - data: - enabled: false - master: - enabled: false -pod: - mandatory_access_control: - type: apparmor - elasticsearch-master: - elasticsearch-master: runtime/default - elasticsearch-data: - elasticsearch-data: runtime/default - elasticsearch-client: - elasticsearch-client: runtime/default - replicas: - client: 1 - data: 1 - master: 2 -conf: - curator: - schedule: "0 */6 * * *" - action_file: - actions: - 1: - action: delete_indices - description: >- - "Delete indices older than 365 days" - options: - timeout_override: - continue_if_exception: False - ignore_empty_list: True - disable_action: True - filters: - - filtertype: pattern - kind: prefix - value: logstash- - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 365 - -EOF -helm upgrade --install elasticsearch ./elasticsearch \ - --namespace=osh-infra \ - --values=/tmp/elasticsearch.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra - -# Delete the test pod if it still exists -kubectl delete pods -l application=elasticsearch,release_group=elasticsearch,component=test --namespace=osh-infra --ignore-not-found -helm test elasticsearch --namespace osh-infra diff --git a/tools/deployment/apparmor/095-nagios.sh b/tools/deployment/apparmor/095-nagios.sh deleted file mode 120000 index 5371752a3..000000000 --- a/tools/deployment/apparmor/095-nagios.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/120-nagios.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/100-fluentbit.sh b/tools/deployment/apparmor/100-fluentbit.sh deleted file mode 100755 index dca71cc07..000000000 --- a/tools/deployment/apparmor/100-fluentbit.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -set -xe - -#NOTE: Lint and package chart -make fluentbit - -tee /tmp/fluentbit.yaml < - bind 0.0.0.0 - port 24220 - @type monitor_agent - - - - - time_format %Y-%m-%dT%H:%M:%S.%NZ - @type json - - path /var/log/containers/*.log - read_from_head true - tag kubernetes.* - @type tail - - - - @type kubernetes_metadata - - - - bind 0.0.0.0 - port "#{ENV['FLUENTD_PORT']}" - @type forward - - - - @type null - - - - - chunk_limit_size 500K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 16 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - include_tag_key true - logstash_format true - logstash_prefix libvirt - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 500K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 16 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - include_tag_key true - logstash_format true - logstash_prefix qemu - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 500K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 16 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - include_tag_key true - logstash_format true - logstash_prefix journal - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 500K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 16 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - include_tag_key true - logstash_format true - logstash_prefix kernel - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 500K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 16 - retry_forever false - retry_max_interval 30 - - flush_interval 15s - host "#{ENV['ELASTICSEARCH_HOST']}" - include_tag_key true - logstash_format true - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - type_name fluent - user "#{ENV['ELASTICSEARCH_USERNAME']}" - -EOF - -#NOTE: Deploy command -helm upgrade --install fluentd-daemonset ./fluentd \ - --namespace=osh-infra \ - --values=/tmp/fluentd-daemonset.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra - -# Delete the test pod if it still exists -kubectl delete pods -l application=fluentd,release_group=fluentd-daemonset,component=test --namespace=osh-infra --ignore-not-found -helm test fluentd-daemonset --namespace osh-infra diff --git a/tools/deployment/apparmor/115-node-problem-detector.sh b/tools/deployment/apparmor/115-node-problem-detector.sh deleted file mode 100644 index 885a5b468..000000000 --- a/tools/deployment/apparmor/115-node-problem-detector.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/075-node-problem-detector.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/120-openvswitch.sh b/tools/deployment/apparmor/120-openvswitch.sh deleted file mode 120000 index 0f4158528..000000000 --- a/tools/deployment/apparmor/120-openvswitch.sh +++ /dev/null @@ -1 +0,0 @@ -../openstack-support/060-openvswitch.sh \ No newline at end of file diff --git a/tools/deployment/apparmor/140-ceph-radosgateway.sh b/tools/deployment/apparmor/140-ceph-radosgateway.sh deleted file mode 100755 index f0f82cc0e..000000000 --- a/tools/deployment/apparmor/140-ceph-radosgateway.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe -: ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_RGW:="$(./tools/deployment/common/get-values-overrides.sh ceph-rgw)"} - -#NOTE: Lint and package chart -: ${OSH_INFRA_PATH:="../openstack-helm-infra"} -make -C ${OSH_INFRA_PATH} ceph-rgw - -#NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} -tee /tmp/radosgw-openstack.yaml < /etc/systemd/system/loops-setup.service -[Unit] -Description=Setup loop devices -DefaultDependencies=no -Conflicts=umount.target -Before=local-fs.target -After=systemd-udevd.service -Requires=systemd-udevd.service - -[Service] -Type=oneshot -ExecStart=/sbin/losetup $osd_data_device '${CEPH_LOOPBACK_PATH}/$namespace/ceph-osd-data-loopbackfile.img' -ExecStart=/sbin/losetup $osd_wal_db_device '${CEPH_LOOPBACK_PATH}/$namespace/ceph-osd-db-wal-loopbackfile.img' -ExecStop=/sbin/losetup -d $osd_data_device -ExecStop=/sbin/losetup -d $osd_wal_db_device -TimeoutSec=60 -RemainAfterExit=yes - -[Install] -WantedBy=local-fs.target -Also=systemd-udevd.service -EOF" - - sudo systemctl daemon-reload - sudo systemctl start loops-setup - sudo systemctl status loops-setup - sudo systemctl enable loops-setup - # let's verify the devices - sudo losetup -a - if losetup |grep -i $osd_data_device; then - echo "ceph osd data disk got created successfully" - else - echo "could not find ceph osd data disk so exiting" - exit 1 - fi - if losetup |grep -i $osd_wal_db_device; then - echo "ceph osd wal/db disk got created successfully" - else - echo "could not find ceph osd wal/db disk so exiting" - exit 1 - fi -} - -while [[ "$#" > 0 ]]; do case $1 in - -d|--ceph-osd-data) OSD_DATA_DEVICE="$2"; shift;shift;; - -w|--ceph-osd-dbwal) OSD_DB_WAL_DEVICE="$2";shift;shift;; - -v|--verbose) VERBOSE=1;shift;; - *) echo "Unknown parameter passed: $1"; shift;; -esac; done - -# verify params -if [ -z "$OSD_DATA_DEVICE" ]; then - OSD_DATA_DEVICE=/dev/loop0 - echo "Ceph osd data device is not set so using ${OSD_DATA_DEVICE}" -else - ceph_osd_disk_name=`basename "$OSD_DATA_DEVICE"` - if losetup -a|grep $ceph_osd_disk_name; then - echo "Ceph osd data device is already in use, please double check and correct the device name" - exit 1 - fi -fi - -if [ -z "$OSD_DB_WAL_DEVICE" ]; then - OSD_DB_WAL_DEVICE=/dev/loop1 - echo "Ceph osd db/wal device is not set so using ${OSD_DB_WAL_DEVICE}" -else - ceph_dbwal_disk_name=`basename "$OSD_DB_WAL_DEVICE"` - if losetup -a|grep $ceph_dbwal_disk_name; then - echo "Ceph osd dbwal device is already in use, please double check and correct the device name" - exit 1 - fi -fi - -: "${CEPH_NAMESPACE:="ceph"}" -# setup loopback devices for ceph osds -setup_loopback_devices $OSD_DATA_DEVICE $OSD_DB_WAL_DEVICE diff --git a/tools/deployment/mariadb-operator-cluster/090-mariadb-backup-test.sh b/tools/deployment/db/mariadb-backup.sh similarity index 71% rename from tools/deployment/mariadb-operator-cluster/090-mariadb-backup-test.sh rename to tools/deployment/db/mariadb-backup.sh index cd99e05e6..80775bbc8 100755 --- a/tools/deployment/mariadb-operator-cluster/090-mariadb-backup-test.sh +++ b/tools/deployment/db/mariadb-backup.sh @@ -14,27 +14,20 @@ set -xe -#NOTE: Lint and package chart -make mariadb-backup - -: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_BACKUP:="$(./tools/deployment/common/get-values-overrides.sh mariadb-backup)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_BACKUP:="$(helm osh get-values-overrides -c mariadb-backup ${FEATURES})"} #NOTE: Deploy command -# Deploying downscaled cluster -: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} helm upgrade --install mariadb-backup ./mariadb-backup \ --namespace=openstack \ --wait \ --timeout 900s \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_BACKUP} - -./tools/deployment/common/wait-for-pods.sh openstack - +helm osh wait-for-pods openstack kubectl create job --from=cronjob/mariadb-backup mariadb-backup-manual-001 -n openstack -./tools/deployment/common/wait-for-pods.sh openstack +helm osh wait-for-pods openstack kubectl logs jobs/mariadb-backup-manual-001 -n openstack diff --git a/tools/deployment/mariadb-operator-cluster/045-mariadb-operator-cluster.sh b/tools/deployment/db/mariadb-operator-cluster.sh similarity index 82% rename from tools/deployment/mariadb-operator-cluster/045-mariadb-operator-cluster.sh rename to tools/deployment/db/mariadb-operator-cluster.sh index e50b6dbac..bd1e54b12 100755 --- a/tools/deployment/mariadb-operator-cluster/045-mariadb-operator-cluster.sh +++ b/tools/deployment/db/mariadb-operator-cluster.sh @@ -19,16 +19,12 @@ set -xe # install mariadb-operator helm repo add mariadb-operator https://mariadb-operator.github.io/mariadb-operator -helm install mariadb-operator mariadb-operator/mariadb-operator --version ${MARIADB_OPERATOR_RELEASE} -n mariadb-operator +helm upgrade --install mariadb-operator mariadb-operator/mariadb-operator --version ${MARIADB_OPERATOR_RELEASE} -n mariadb-operator #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh mariadb-operator +helm osh wait-for-pods mariadb-operator - -#NOTE: Lint and package chart -make mariadb-cluster - -: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER:="$(./tools/deployment/common/get-values-overrides.sh mariadb-cluster)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER:="$(helm osh get-values-overrides -c mariadb-cluster ${FEATURES})"} #NOTE: Deploy command # Deploying downscaled cluster @@ -41,9 +37,10 @@ helm upgrade --install mariadb-cluster ./mariadb-cluster \ ${OSH_INFRA_EXTRA_HELM_ARGS} \ ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER} +sleep 30 #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh openstack +helm osh wait-for-pods openstack kubectl get pods --namespace=openstack -o wide @@ -61,7 +58,7 @@ helm upgrade --install mariadb-cluster ./mariadb-cluster \ ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh openstack +helm osh wait-for-pods openstack kubectl get pods --namespace=openstack -o wide diff --git a/tools/deployment/osh-infra-monitoring/045-mariadb.sh b/tools/deployment/db/mariadb.sh similarity index 79% rename from tools/deployment/osh-infra-monitoring/045-mariadb.sh rename to tools/deployment/db/mariadb.sh index 2a0e08d04..091647cb2 100755 --- a/tools/deployment/osh-infra-monitoring/045-mariadb.sh +++ b/tools/deployment/db/mariadb.sh @@ -14,21 +14,17 @@ set -xe -#NOTE: Lint and package chart -make mariadb - -: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB:="$(./tools/deployment/common/get-values-overrides.sh mariadb)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB:="$(helm osh get-values-overrides -c mariadb ${FEATURES})"} #NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} helm upgrade --install mariadb ./mariadb \ --namespace=osh-infra \ --set monitoring.prometheus.enabled=true \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra # Delete the test pod if it still exists kubectl delete pods -l application=mariadb,release_group=mariadb,component=test --namespace=osh-infra --ignore-not-found diff --git a/tools/deployment/common/postgresql.sh b/tools/deployment/db/postgresql.sh similarity index 82% rename from tools/deployment/common/postgresql.sh rename to tools/deployment/db/postgresql.sh index ffb685f78..0b156d8ba 100755 --- a/tools/deployment/common/postgresql.sh +++ b/tools/deployment/db/postgresql.sh @@ -14,12 +14,9 @@ set -xe -#NOTE: Lint and package chart -make postgresql - #NOTE: Deploy command : ${OSH_INFRA_EXTRA_HELM_ARGS:=""} -: ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL:="$(./tools/deployment/common/get-values-overrides.sh postgresql)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL:="$(helm osh get-values-overrides -c postgresql ${FEATURES})"} helm upgrade --install postgresql ./postgresql \ --namespace=osh-infra \ @@ -31,4 +28,4 @@ helm upgrade --install postgresql ./postgresql \ ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/elastic-beats/005-deploy-k8s.sh b/tools/deployment/elastic-beats/005-deploy-k8s.sh deleted file mode 120000 index 003bfbb8e..000000000 --- a/tools/deployment/elastic-beats/005-deploy-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/030-ceph.sh b/tools/deployment/elastic-beats/030-ceph.sh deleted file mode 120000 index 9f7b38835..000000000 --- a/tools/deployment/elastic-beats/030-ceph.sh +++ /dev/null @@ -1 +0,0 @@ -../multinode/030-ceph.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/035-ceph-ns-activate.sh b/tools/deployment/elastic-beats/035-ceph-ns-activate.sh deleted file mode 120000 index f6c0f5f2e..000000000 --- a/tools/deployment/elastic-beats/035-ceph-ns-activate.sh +++ /dev/null @@ -1 +0,0 @@ -../multinode/035-ceph-ns-activate.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/040-ldap.sh b/tools/deployment/elastic-beats/040-ldap.sh deleted file mode 120000 index 4ed4b9d4b..000000000 --- a/tools/deployment/elastic-beats/040-ldap.sh +++ /dev/null @@ -1 +0,0 @@ -../common/040-ldap.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/050-elasticsearch.sh b/tools/deployment/elastic-beats/050-elasticsearch.sh deleted file mode 100755 index 0862aeaaf..000000000 --- a/tools/deployment/elastic-beats/050-elasticsearch.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make elasticsearch - -#NOTE: Deploy command -tee /tmp/elasticsearch.yaml << EOF -manifests: - cron_curator: false - configmap_bin_curator: false - configmap_etc_curator: false -images: - tags: - elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_1_0-20191115 -storage: - data: - requests: - storage: 20Gi - master: - requests: - storage: 5Gi -jobs: - verify_repositories: - cron: "*/10 * * * *" -monitoring: - prometheus: - enabled: false -pod: - replicas: - client: 1 - data: 1 - master: 2 -conf: - elasticsearch: - config: - xpack: - security: - enabled: false - ilm: - enabled: false - -EOF -helm upgrade --install elasticsearch ./elasticsearch \ - --namespace=osh-infra \ - --values=/tmp/elasticsearch.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/elastic-beats/060-kibana.sh b/tools/deployment/elastic-beats/060-kibana.sh deleted file mode 100755 index 677e4b915..000000000 --- a/tools/deployment/elastic-beats/060-kibana.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make kibana - -: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(./tools/deployment/common/get-values-overrides.sh kibana)"} - -tee /tmp/kibana.yaml << EOF -images: - tags: - kibana: docker.elastic.co/kibana/kibana:7.1.0 -conf: - kibana: - xpack: - security: - enabled: false - spaces: - enabled: false - apm: - enabled: false - graph: - enabled: false - ml: - enabled: false - monitoring: - enabled: false - reporting: - enabled: false - canvas: - enabled: false -EOF - -#NOTE: Deploy command -helm upgrade --install kibana ./kibana \ - --namespace=osh-infra \ - --values=/tmp/kibana.yaml - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA} - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/elastic-beats/070-kube-state-metrics.sh b/tools/deployment/elastic-beats/070-kube-state-metrics.sh deleted file mode 120000 index 2a18ebb8b..000000000 --- a/tools/deployment/elastic-beats/070-kube-state-metrics.sh +++ /dev/null @@ -1 +0,0 @@ -../common/070-kube-state-metrics.sh \ No newline at end of file diff --git a/tools/deployment/elastic-beats/080-elastic-metricbeat.sh b/tools/deployment/elastic-beats/080-elastic-metricbeat.sh deleted file mode 100755 index 2e0820cf2..000000000 --- a/tools/deployment/elastic-beats/080-elastic-metricbeat.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make elastic-metricbeat - -tee /tmp/metricbeat.yaml << EOF -images: - tags: - metricbeat: docker.elastic.co/beats/metricbeat:7.1.0 -conf: - metricbeat: - setup: - ilm: - enabled: false -endpoints: - elasticsearch: - namespace: osh-infra - kibana: - namespace: osh-infra -EOF - -#NOTE: Deploy command -helm upgrade --install elastic-metricbeat ./elastic-metricbeat \ - --namespace=kube-system \ - --values=/tmp/metricbeat.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/090-elastic-filebeat.sh b/tools/deployment/elastic-beats/090-elastic-filebeat.sh deleted file mode 100755 index 44c5e5086..000000000 --- a/tools/deployment/elastic-beats/090-elastic-filebeat.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make elastic-filebeat - -tee /tmp/filebeat.yaml << EOF -images: - tags: - filebeat: docker.elastic.co/beats/filebeat:7.1.0 -conf: - filebeat: - setup: - ilm: - enabled: false -endpoints: - elasticsearch: - namespace: osh-infra - kibana: - namespace: osh-infra -EOF - -#NOTE: Deploy command -helm upgrade --install elastic-filebeat ./elastic-filebeat \ - --namespace=kube-system \ - --values=/tmp/filebeat.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/elastic-beats/100-elastic-packetbeat.sh b/tools/deployment/elastic-beats/100-elastic-packetbeat.sh deleted file mode 100755 index 43ba1acb9..000000000 --- a/tools/deployment/elastic-beats/100-elastic-packetbeat.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make elastic-packetbeat - -tee /tmp/packetbeat.yaml << EOF -images: - tags: - filebeat: docker.elastic.co/beats/packetbeat:7.1.0 -conf: - packetbeat: - setup: - ilm: - enabled: false -endpoints: - elasticsearch: - namespace: osh-infra - kibana: - namespace: osh-infra -EOF - -#NOTE: Deploy command -helm upgrade --install elastic-packetbeat ./elastic-packetbeat \ - --namespace=kube-system \ - --values=/tmp/packetbeat.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system diff --git a/tools/deployment/federated-monitoring/000-install-packages.sh b/tools/deployment/federated-monitoring/000-install-packages.sh deleted file mode 120000 index d702c4899..000000000 --- a/tools/deployment/federated-monitoring/000-install-packages.sh +++ /dev/null @@ -1 +0,0 @@ -../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/federated-monitoring/005-deploy-k8s.sh b/tools/deployment/federated-monitoring/005-deploy-k8s.sh deleted file mode 120000 index 003bfbb8e..000000000 --- a/tools/deployment/federated-monitoring/005-deploy-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/federated-monitoring/020-nfs-provisioner.sh b/tools/deployment/federated-monitoring/020-nfs-provisioner.sh deleted file mode 120000 index 2d0231b7f..000000000 --- a/tools/deployment/federated-monitoring/020-nfs-provisioner.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/030-nfs-provisioner.sh \ No newline at end of file diff --git a/tools/deployment/federated-monitoring/030-ldap.sh b/tools/deployment/federated-monitoring/030-ldap.sh deleted file mode 120000 index 4ed4b9d4b..000000000 --- a/tools/deployment/federated-monitoring/030-ldap.sh +++ /dev/null @@ -1 +0,0 @@ -../common/040-ldap.sh \ No newline at end of file diff --git a/tools/deployment/federated-monitoring/040-kube-state-metrics.sh b/tools/deployment/federated-monitoring/040-kube-state-metrics.sh deleted file mode 120000 index 2a18ebb8b..000000000 --- a/tools/deployment/federated-monitoring/040-kube-state-metrics.sh +++ /dev/null @@ -1 +0,0 @@ -../common/070-kube-state-metrics.sh \ No newline at end of file diff --git a/tools/deployment/federated-monitoring/050-node-exporter.sh b/tools/deployment/federated-monitoring/050-node-exporter.sh deleted file mode 120000 index 412748a74..000000000 --- a/tools/deployment/federated-monitoring/050-node-exporter.sh +++ /dev/null @@ -1 +0,0 @@ -../common/080-node-exporter.sh \ No newline at end of file diff --git a/tools/deployment/federated-monitoring/060-prometheus.sh b/tools/deployment/federated-monitoring/060-prometheus.sh deleted file mode 100755 index e05668346..000000000 --- a/tools/deployment/federated-monitoring/060-prometheus.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make prometheus - -tee /tmp/prometheus-one.yaml << EOF -endpoints: - monitoring: - hosts: - default: prom-metrics-one - public: prometheus-one -manifests: - network_policy: false -EOF - -tee /tmp/prometheus-two.yaml << EOF -endpoints: - monitoring: - hosts: - default: prom-metrics-two - public: prometheus-two -manifests: - network_policy: false -EOF - -tee /tmp/prometheus-three.yaml << EOF -endpoints: - monitoring: - hosts: - default: prom-metrics-three - public: prometheus-three -manifests: - network_policy: false -EOF -#NOTE: Deploy command -for release in prometheus-one prometheus-two prometheus-three; do - rules_overrides="" - for rules_file in $(ls ./prometheus/values_overrides); do - rules_overrides="$rules_overrides --values=./prometheus/values_overrides/$rules_file" - done - helm upgrade --install prometheus-$release ./prometheus \ - --namespace=osh-infra \ - --values=/tmp/$release.yaml \ - $rules_overrides - #NOTE: Wait for deploy - ./tools/deployment/common/wait-for-pods.sh osh-infra - - # Delete the test pod if it still exists - kubectl delete pods -l application=prometheus,release_group=prometheus-$release,component=test --namespace=osh-infra --ignore-not-found - helm test prometheus-$release --namespace osh-infra -done diff --git a/tools/deployment/federated-monitoring/070-federated-prometheus.sh b/tools/deployment/federated-monitoring/070-federated-prometheus.sh deleted file mode 100755 index b1c8591ac..000000000 --- a/tools/deployment/federated-monitoring/070-federated-prometheus.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -tee /tmp/federated-prometheus.yaml << EOF -endpoints: - monitoring: - hosts: - default: prom-metrics-federate - public: prometheus-federate -manifests: - network_policy: false -conf: - prometheus: - scrape_configs: - template: | - global: - scrape_interval: 60s - evaluation_interval: 60s - scrape_configs: - - job_name: 'federate' - scrape_interval: 15s - - honor_labels: true - metrics_path: '/federate' - - params: - 'match[]': - - '{__name__=~".+"}' - - static_configs: - - targets: - - 'prometheus-one.osh-infra.svc.cluster.local:80' - - 'prometheus-two.osh-infra.svc.cluster.local:80' - - 'prometheus-three.osh-infra.svc.cluster.local:80' -EOF - -#NOTE: Lint and package chart -make prometheus - -#NOTE: Deploy command -helm upgrade --install federated-prometheus ./prometheus \ - --namespace=osh-infra \ - --values=/tmp/federated-prometheus.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra - -# Delete the test pod if it still exists -kubectl delete pods -l application=prometheus,release_group=federated-prometheus,component=test --namespace=osh-infra --ignore-not-found -helm test federated-prometheus --namespace osh-infra diff --git a/tools/deployment/federated-monitoring/080-mariadb.sh b/tools/deployment/federated-monitoring/080-mariadb.sh deleted file mode 120000 index 880f9f76c..000000000 --- a/tools/deployment/federated-monitoring/080-mariadb.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/045-mariadb.sh \ No newline at end of file diff --git a/tools/deployment/federated-monitoring/090-grafana.sh b/tools/deployment/federated-monitoring/090-grafana.sh deleted file mode 100755 index cfe61666f..000000000 --- a/tools/deployment/federated-monitoring/090-grafana.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make grafana - -tee /tmp/grafana.yaml << EOF -endpoints: - monitoring_one: - name: prometheus-one - namespace: osh-infra - auth: - user: - username: admin - password: changeme - hosts: - default: prom-metrics-one - public: prometheus-one - host_fqdn_override: - default: null - path: - default: null - scheme: - default: http - port: - api: - default: 80 - public: 80 - monitoring_two: - name: prometheus-two - namespace: osh-infra - auth: - user: - username: admin - password: changeme - hosts: - default: prom-metrics-two - public: prometheus-two - host_fqdn_override: - default: null - path: - default: null - scheme: - default: http - port: - api: - default: 80 - public: 80 - monitoring_three: - name: prometheus-three - namespace: osh-infra - auth: - user: - username: admin - password: changeme - hosts: - default: prom-metrics-three - public: prometheus-three - host_fqdn_override: - default: null - path: - default: null - scheme: - default: http - port: - api: - default: 80 - public: 80 - monitoring_federated: - name: prometheus-federate - namespace: osh-infra - auth: - user: - username: admin - password: changeme - hosts: - default: prom-metrics-federate - public: prometheus-federate - host_fqdn_override: - default: null - path: - default: null - scheme: - default: http - port: - api: - default: 80 - public: 80 -conf: - provisioning: - datasources: - template: | - apiVersion: 1 - datasources: - - name: prometheus-one - type: prometheus - access: proxy - orgId: 1 - editable: false - basicAuth: true - basicAuthUser: admin - secureJsonData: - basicAuthPassword: changeme - url: {{ tuple "monitoring_one" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} - - name: prometheus-two - type: prometheus - access: proxy - orgId: 1 - editable: false - basicAuth: true - basicAuthUser: admin - secureJsonData: - basicAuthPassword: changeme - url: {{ tuple "monitoring_two" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} - - name: prometheus-three - type: prometheus - access: proxy - orgId: 1 - editable: false - basicAuth: true - basicAuthUser: admin - secureJsonData: - basicAuthPassword: changeme - url: {{ tuple "monitoring_three" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} - - name: prometheus-federated - type: prometheus - access: proxy - orgId: 1 - editable: false - basicAuth: true - basicAuthUser: admin - secureJsonData: - basicAuthPassword: changeme - url: {{ tuple "monitoring_federated" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} - -EOF - -#NOTE: Deploy command -helm upgrade --install grafana ./grafana \ - --namespace=osh-infra \ - --values=/tmp/grafana.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra - -# Delete the test pod if it still exists -kubectl delete pods -l application=grafana,release_group=grafana,component=test --namespace=osh-infra --ignore-not-found - -helm test grafana --namespace osh-infra - -echo "Get list of all configured datasources in Grafana" -curl -u admin:password http://grafana.osh-infra.svc.cluster.local/api/datasources | jq -r . diff --git a/tools/deployment/federated-monitoring/100-prometheus-selenium.sh b/tools/deployment/federated-monitoring/100-prometheus-selenium.sh deleted file mode 100755 index 545397f52..000000000 --- a/tools/deployment/federated-monitoring/100-prometheus-selenium.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -xe - -export CHROMEDRIVER="${CHROMEDRIVER:="/etc/selenium/chromedriver"}" -export ARTIFACTS_DIR="${ARTIFACTS_DIR:="/tmp/artifacts/"}" - -export PROMETHEUS_USER="admin" -export PROMETHEUS_PASSWORD="changeme" - -export PROMETHEUS_URI="prometheus-one.osh-infra.svc.cluster.local" -python3 tools/gate/selenium/prometheusSelenium.py -mv ${ARTIFACTS_DIR}/Prometheus_Command_Line_Flags.png ${ARTIFACTS_DIR}/Prometheus_One_Command_Line_Flags.png -mv ${ARTIFACTS_DIR}Prometheus_Dashboard.png ${ARTIFACTS_DIR}/Prometheus_One_Dashboard.png -mv ${ARTIFACTS_DIR}/Prometheus_Runtime_Info.png ${ARTIFACTS_DIR}/Prometheus_One_Runtime_Info.png - -export PROMETHEUS_URI="prometheus-two.osh-infra.svc.cluster.local" -python3 tools/gate/selenium/prometheusSelenium.py -mv ${ARTIFACTS_DIR}/Prometheus_Command_Line_Flags.png ${ARTIFACTS_DIR}/Prometheus_Two_Command_Line_Flags.png -mv ${ARTIFACTS_DIR}/Prometheus_Dashboard.png ${ARTIFACTS_DIR}/Prometheus_Two_Dashboard.png -mv ${ARTIFACTS_DIR}/Prometheus_Runtime_Info.png ${ARTIFACTS_DIR}/Prometheus_Two_Runtime_Info.png - -export PROMETHEUS_URI="prometheus-three.osh-infra.svc.cluster.local" -python3 tools/gate/selenium/prometheusSelenium.py -mv ${ARTIFACTS_DIR}/Prometheus_Command_Line_Flags.png ${ARTIFACTS_DIR}/Prometheus_Three_Command_Line_Flags.png -mv ${ARTIFACTS_DIR}/Prometheus_Dashboard.png ${ARTIFACTS_DIR}/Prometheus_Three_Dashboard.png -mv ${ARTIFACTS_DIR}/Prometheus_Runtime_Info.png ${ARTIFACTS_DIR}/Prometheus_Three_Runtime_Info.png - -export PROMETHEUS_URI="prometheus-federate.osh-infra.svc.cluster.local" -python3 tools/gate/selenium/prometheusSelenium.py -mv ${ARTIFACTS_DIR}/Prometheus_Command_Line_Flags.png ${ARTIFACTS_DIR}/Prometheus_Federated_Command_Line_Flags.png -mv ${ARTIFACTS_DIR}/Prometheus_Dashboard.png ${ARTIFACTS_DIR}/Prometheus_Federated_Dashboard.png -mv ${ARTIFACTS_DIR}/Prometheus_Runtime_Info.png ${ARTIFACTS_DIR}/Prometheus_Federated_Runtime_Info.png diff --git a/tools/deployment/keystone-auth/010-setup-client.sh b/tools/deployment/keystone-auth/010-setup-client.sh deleted file mode 100755 index 21b71d5cb..000000000 --- a/tools/deployment/keystone-auth/010-setup-client.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Move into openstack-helm root dir & Run client setup script -cd ${OSH_PATH:-"../openstack-helm/"}; ./tools/deployment/developer/nfs/020-setup-client.sh; cd - diff --git a/tools/deployment/keystone-auth/030-nfs-provisioner.sh b/tools/deployment/keystone-auth/030-nfs-provisioner.sh deleted file mode 120000 index 2d0231b7f..000000000 --- a/tools/deployment/keystone-auth/030-nfs-provisioner.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/030-nfs-provisioner.sh \ No newline at end of file diff --git a/tools/deployment/keystone-auth/040-rabbitmq.sh b/tools/deployment/keystone-auth/040-rabbitmq.sh deleted file mode 120000 index 497e38873..000000000 --- a/tools/deployment/keystone-auth/040-rabbitmq.sh +++ /dev/null @@ -1 +0,0 @@ -../openstack-support/030-rabbitmq.sh \ No newline at end of file diff --git a/tools/deployment/keystone-auth/050-memcached.sh b/tools/deployment/keystone-auth/050-memcached.sh deleted file mode 120000 index 706eb90b3..000000000 --- a/tools/deployment/keystone-auth/050-memcached.sh +++ /dev/null @@ -1 +0,0 @@ -../openstack-support/040-memcached.sh \ No newline at end of file diff --git a/tools/deployment/keystone-auth/060-mariadb.sh b/tools/deployment/keystone-auth/060-mariadb.sh deleted file mode 100755 index 9187c56c3..000000000 --- a/tools/deployment/keystone-auth/060-mariadb.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB:="$(./tools/deployment/common/get-values-overrides.sh mariadb)"} - -#NOTE: Lint and package chart -make mariadb - -#NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} -helm upgrade --install mariadb ./mariadb \ - --namespace=openstack \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB} - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh openstack - -# Delete the test pod if it still exists -kubectl delete pods -l application=mariadb,release_group=mariadb,component=test --namespace=openstack --ignore-not-found -#NOTE: Validate the deployment -helm test mariadb --namespace openstack diff --git a/tools/deployment/keystone-auth/080-check.sh b/tools/deployment/keystone-auth/080-check.sh deleted file mode 100755 index 34f231495..000000000 --- a/tools/deployment/keystone-auth/080-check.sh +++ /dev/null @@ -1,153 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -export OS_CLOUD=openstack_helm -function keystone_token () { - openstack token issue -f value -c id -} - -function report_failed_policy () { - echo "$1 was $2 to perform $3, which contradicts current policy" - exit 1 -} - -function test_user_is_authorized () { - TOKEN=$(keystone_token) - if ! kubectl --kubeconfig /tmp/kubeconfig.yaml --token $TOKEN $1 ; then - report_failed_policy "$OS_USERNAME" "not allowed" "$1" - fi -} - -function test_user_is_unauthorized () { - TOKEN=$(keystone_token) - if ! kubectl --kubeconfig /tmp/kubeconfig.yaml --token $TOKEN $1 ; then - echo "Denied, as expected by policy" - else - report_failed_policy "$OS_USERNAME" "allowed" "$1" - fi -} - -sudo cp -va $HOME/.kube/config /tmp/kubeconfig.yaml -sudo kubectl --kubeconfig /tmp/kubeconfig.yaml config unset users.kubernetes-admin - -# Test -# This issues token with admin role -TOKEN=$(keystone_token) -kubectl --kubeconfig /tmp/kubeconfig.yaml --token $TOKEN get pods -kubectl --kubeconfig /tmp/kubeconfig.yaml --token $TOKEN get pods -n openstack -kubectl --kubeconfig /tmp/kubeconfig.yaml --token $TOKEN get secrets -n openstack - -# This is used to grab a pod name for the following tests -TEST_POD="$(kubectl get pods -n openstack | awk 'NR==2{print $1}')" - -# create users -openstack user create --or-show --password password admin_k8cluster_user -openstack user create --or-show --password password admin_k8cluster_edit_user -openstack user create --or-show --password password admin_k8cluster_view_user - -# create project -openstack project create --or-show openstack-system -openstack project create --or-show demoProject - -# create roles -openstack role create --or-show openstackRole -openstack role create --or-show kube-system-admin -openstack role create --or-show admin_k8cluster -openstack role create --or-show admin_k8cluster_editor -openstack role create --or-show admin_k8cluster_viewer - -# assign user role to project -openstack role add --project openstack-system --user bob --project-domain default --user-domain ldapdomain openstackRole -openstack role add --project demoProject --user alice --project-domain default --user-domain ldapdomain kube-system-admin -openstack role add --project demoProject --user admin_k8cluster_user --project-domain default --user-domain default admin_k8cluster -openstack role add --project demoProject --user admin_k8cluster_edit_user --project-domain default --user-domain default admin_k8cluster_editor -openstack role add --project demoProject --user admin_k8cluster_view_user --project-domain default --user-domain default admin_k8cluster_viewer - -unset OS_CLOUD -export OS_AUTH_URL="http://keystone.openstack.svc.cluster.local/v3" -export OS_IDENTITY_API_VERSION="3" -export OS_PROJECT_NAME="openstack-system" -export OS_PASSWORD="password" -export OS_USERNAME="bob" -export OS_USER_DOMAIN_NAME="ldapdomain" - -# Create files for secret generation -echo -n 'admin' > /tmp/user.txt -echo -n 'password' > /tmp/pass.txt - -# See this does fail as the policy does not allow for a non-admin user -TOKEN=$(keystone_token) -test_user_is_unauthorized "get pods" - -export OS_USERNAME="alice" -export OS_PROJECT_NAME="demoProject" -test_user_is_unauthorized "get pods -n openstack" - -export OS_USER_DOMAIN_NAME="default" - -#admin_k8cluser_user -export OS_USERNAME="admin_k8cluster_user" -RESOURCES=("pods" "configmaps" "endpoints" "persistentvolumeclaims" \ - "replicationcontrollers" "secrets" "serviceaccounts" \ - "services" "events" "limitranges" "namespace" \ - "replicationcontrollers" "resourcequotas" "daemonsets" \ - "deployments" "replicasets" "statefulsets" "jobs" \ - "cronjobs" "poddisruptionbudgets" "serviceaccounts" \ - "networkpolicies" "horizontalpodautoscalers") -for r in "${RESOURCES[@]}" ; do - test_user_is_authorized "get $r" -done - -test_user_is_authorized "create secret generic test-secret --from-file=/tmp/user.txt --from-file=/tmp/pass.txt" -test_user_is_authorized "delete secret test-secret" - -#admin_k8cluster_edit_user -export OS_USERNAME="admin_k8cluster_edit_user" -RESOURCES=("pods" "configmaps" "endpoints" "persistentvolumeclaims" \ - "replicationcontrollers" "secrets" "serviceaccounts" \ - "services" "events" "limitranges" "namespace" \ - "replicationcontrollers" "resourcequotas" "daemonsets" \ - "deployments" "replicasets" "statefulsets" "jobs" \ - "cronjobs" "poddisruptionbudgets" "serviceaccounts" \ - "networkpolicies" "horizontalpodautoscalers") -for r in "${RESOURCES[@]}" ; do - test_user_is_authorized "get $r" -done - -test_user_is_authorized "create secret generic test-secret --from-file=/tmp/user.txt --from-file=/tmp/pass.txt" -test_user_is_authorized "delete secret test-secret" -test_user_is_authorized "logs -n openstack $TEST_POD --tail=5" - -test_user_is_unauthorized "create namespace test" - - -#admin_k8cluster_view_user -export OS_USERNAME="admin_k8cluster_view_user" -RESOURCES=("pods" "configmaps" "endpoints" "persistentvolumeclaims" \ - "replicationcontrollers" "services" "serviceaccounts" \ - "replicationcontrollers" "resourcequotas" "namespaces" \ - "daemonsets" "deployments" "replicasets" "statefulsets" \ - "poddisruptionbudgets" "networkpolicies") -for r in "${RESOURCES[@]}" ; do - test_user_is_authorized "get $r" -done - -test_user_is_authorized "logs -n openstack $TEST_POD --tail=5" - -test_user_is_unauthorized "delete pod $TEST_POD -n openstack" -test_user_is_unauthorized "create namespace test" -test_user_is_unauthorized "get secrets" -test_user_is_unauthorized "create secret generic test-secret --from-file=/tmp/user.txt --from-file=/tmp/pass.txt" diff --git a/tools/deployment/osh-infra-logging/050-elasticsearch.sh b/tools/deployment/logging/elasticsearch.sh similarity index 96% rename from tools/deployment/osh-infra-logging/050-elasticsearch.sh rename to tools/deployment/logging/elasticsearch.sh index 4c4019869..6198d9ec4 100755 --- a/tools/deployment/osh-infra-logging/050-elasticsearch.sh +++ b/tools/deployment/logging/elasticsearch.sh @@ -14,9 +14,6 @@ set -xe -#NOTE: Lint and package chart -make elasticsearch - #NOTE: Deploy command tee /tmp/elasticsearch.yaml << EOF jobs: @@ -167,7 +164,7 @@ manifests: object_bucket_claim: true EOF -: ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH:="$(./tools/deployment/common/get-values-overrides.sh elasticsearch)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH:="$(helm osh get-values-overrides -c elasticsearch ${FEATURES})"} helm upgrade --install elasticsearch ./elasticsearch \ --namespace=osh-infra \ @@ -176,7 +173,7 @@ helm upgrade --install elasticsearch ./elasticsearch \ ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra # Delete the test pod if it still exists kubectl delete pods -l application=elasticsearch,release_group=elasticsearch,component=test --namespace=osh-infra --ignore-not-found diff --git a/tools/deployment/common/fluentbit.sh b/tools/deployment/logging/fluentbit.sh similarity index 75% rename from tools/deployment/common/fluentbit.sh rename to tools/deployment/logging/fluentbit.sh index 2a15ba0e6..deb36f737 100755 --- a/tools/deployment/common/fluentbit.sh +++ b/tools/deployment/logging/fluentbit.sh @@ -14,16 +14,12 @@ set -xe -#NOTE: Lint and package chart -make fluentbit - -: ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTBIT:="$(./tools/deployment/common/get-values-overrides.sh fluentbit)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTBIT:="$(helm osh get-values-overrides -c fluentbit ${FEATURES})"} helm upgrade --install fluentbit ./fluentbit \ --namespace=osh-infra \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTBIT} - #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/common/fluentd.sh b/tools/deployment/logging/fluentd.sh similarity index 95% rename from tools/deployment/common/fluentd.sh rename to tools/deployment/logging/fluentd.sh index 7bf34b75d..fbf43b292 100755 --- a/tools/deployment/common/fluentd.sh +++ b/tools/deployment/logging/fluentd.sh @@ -14,9 +14,7 @@ set -xe -#NOTE: Lint and package chart -make fluentd -: ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTD:="$(./tools/deployment/common/get-values-overrides.sh fluentd)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTD:="$(helm osh get-values-overrides -c fluentd ${FEATURES})"} tee /tmp/fluentd.yaml << EOF pod: @@ -185,4 +183,4 @@ helm upgrade --install fluentd ./fluentd \ ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTD} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/osh-infra-logging/070-kibana.sh b/tools/deployment/logging/kibana.sh similarity index 81% rename from tools/deployment/osh-infra-logging/070-kibana.sh rename to tools/deployment/logging/kibana.sh index ac3d48568..3e0e384d1 100755 --- a/tools/deployment/osh-infra-logging/070-kibana.sh +++ b/tools/deployment/logging/kibana.sh @@ -14,10 +14,7 @@ set -xe -#NOTE: Lint and package chart -make kibana - -: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(./tools/deployment/common/get-values-overrides.sh kibana)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(helm osh get-values-overrides -c kibana ${FEATURES})"} #NOTE: Deploy command helm upgrade --install kibana ./kibana \ @@ -27,4 +24,4 @@ helm upgrade --install kibana ./kibana \ ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/mariadb-operator-cluster/000-prepare-k8s.sh b/tools/deployment/mariadb-operator-cluster/000-prepare-k8s.sh deleted file mode 120000 index aa9807064..000000000 --- a/tools/deployment/mariadb-operator-cluster/000-prepare-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../common/prepare-k8s.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/010-deploy-docker-registry.sh b/tools/deployment/mariadb-operator-cluster/010-deploy-docker-registry.sh deleted file mode 120000 index b1dde55a7..000000000 --- a/tools/deployment/mariadb-operator-cluster/010-deploy-docker-registry.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/010-deploy-docker-registry.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/012-setup-client.sh b/tools/deployment/mariadb-operator-cluster/012-setup-client.sh deleted file mode 120000 index b2416e5e9..000000000 --- a/tools/deployment/mariadb-operator-cluster/012-setup-client.sh +++ /dev/null @@ -1 +0,0 @@ -../common/setup-client.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/030-nfs-provisioner.sh b/tools/deployment/mariadb-operator-cluster/030-nfs-provisioner.sh deleted file mode 120000 index 2d0231b7f..000000000 --- a/tools/deployment/mariadb-operator-cluster/030-nfs-provisioner.sh +++ /dev/null @@ -1 +0,0 @@ -../osh-infra-monitoring/030-nfs-provisioner.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/040-rabbitmq.sh b/tools/deployment/mariadb-operator-cluster/040-rabbitmq.sh deleted file mode 120000 index a5eca6ee5..000000000 --- a/tools/deployment/mariadb-operator-cluster/040-rabbitmq.sh +++ /dev/null @@ -1 +0,0 @@ -../keystone-auth/040-rabbitmq.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/050-memcached.sh b/tools/deployment/mariadb-operator-cluster/050-memcached.sh deleted file mode 120000 index 3c3fa1821..000000000 --- a/tools/deployment/mariadb-operator-cluster/050-memcached.sh +++ /dev/null @@ -1 +0,0 @@ -../keystone-auth/050-memcached.sh \ No newline at end of file diff --git a/tools/deployment/mariadb-operator-cluster/070-keystone.sh b/tools/deployment/mariadb-operator-cluster/070-keystone.sh deleted file mode 100755 index ceefa831e..000000000 --- a/tools/deployment/mariadb-operator-cluster/070-keystone.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -: ${OSH_PATH:="../openstack-helm"} -: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} -: ${OSH_EXTRA_HELM_ARGS:=""} -: ${OSH_EXTRA_HELM_ARGS_KEYSTONE:="$(HELM_CHART_ROOT_PATH=${OSH_PATH} ./tools/deployment/common/get-values-overrides.sh keystone)"} - -# Install LDAP -make ldap -helm upgrade --install ldap ./ldap \ - --namespace=openstack \ - --set pod.replicas.server=1 \ - --set bootstrap.enabled=true \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_LDAP} - -# Install Keystone -cd ${OSH_PATH} -make keystone -cd - -helm upgrade --install keystone ${OSH_PATH}/keystone \ - --namespace=openstack \ - --values=${OSH_PATH}/keystone/values_overrides/ldap.yaml \ - --set network.api.ingress.classes.namespace=nginx \ - --set endpoints.oslo_db.hosts.default=mariadb-server-primary \ - ${OSH_EXTRA_HELM_ARGS} \ - ${OSH_EXTRA_HELM_ARGS_KEYSTONE} - -./tools/deployment/common/wait-for-pods.sh openstack - -# Testing basic functionality -export OS_CLOUD=openstack_helm -sleep 30 #NOTE(portdirect): Wait for ingress controller to update rules and restart Nginx -openstack endpoint list diff --git a/tools/deployment/osh-infra-monitoring/060-alertmanager.sh b/tools/deployment/monitoring/alertmanager.sh similarity index 87% rename from tools/deployment/osh-infra-monitoring/060-alertmanager.sh rename to tools/deployment/monitoring/alertmanager.sh index 5da7b2fa6..02d3c61f3 100755 --- a/tools/deployment/osh-infra-monitoring/060-alertmanager.sh +++ b/tools/deployment/monitoring/alertmanager.sh @@ -14,13 +14,10 @@ set -xe -#NOTE: Lint and package chart -make prometheus-alertmanager - #NOTE: Deploy command helm upgrade --install prometheus-alertmanager ./prometheus-alertmanager \ --namespace=osh-infra \ --set pod.replicas.alertmanager=1 #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/common/blackbox-exporter.sh b/tools/deployment/monitoring/blackbox-exporter.sh similarity index 86% rename from tools/deployment/common/blackbox-exporter.sh rename to tools/deployment/monitoring/blackbox-exporter.sh index 4ed1b44d9..97b17acb6 100755 --- a/tools/deployment/common/blackbox-exporter.sh +++ b/tools/deployment/monitoring/blackbox-exporter.sh @@ -14,12 +14,9 @@ set -xe -#NOTE: Lint and package chart -make prometheus-blackbox-exporter - #NOTE: Deploy command helm upgrade --install prometheus-blackbox-exporter \ ./prometheus-blackbox-exporter --namespace=osh-infra #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/osh-infra-monitoring/110-grafana.sh b/tools/deployment/monitoring/grafana.sh similarity index 70% rename from tools/deployment/osh-infra-monitoring/110-grafana.sh rename to tools/deployment/monitoring/grafana.sh index 54556391b..975f0acab 100755 --- a/tools/deployment/osh-infra-monitoring/110-grafana.sh +++ b/tools/deployment/monitoring/grafana.sh @@ -14,20 +14,17 @@ set -xe -#NOTE: Lint and package chart -make grafana - -FEATURE_GATES="calico,ceph,containers,coredns,elasticsearch,kubernetes,nginx,nodes,openstack,prometheus,home_dashboard,persistentvolume,apparmor" -: ${OSH_INFRA_EXTRA_HELM_ARGS_GRAFANA:="$({ ./tools/deployment/common/get-values-overrides.sh grafana;} 2> /dev/null)"} +FEATURE_GATES="calico ceph containers coredns elasticsearch kubernetes nginx nodes openstack prometheus home_dashboard persistentvolume apparmor" +: ${OSH_INFRA_EXTRA_HELM_ARGS_GRAFANA:=$(helm osh get-values-overrides -c grafana ${FEATURE_GATES} ${FEATURES} 2>/dev/null)} #NOTE: Deploy command helm upgrade --install grafana ./grafana \ --namespace=osh-infra \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ ${OSH_INFRA_EXTRA_HELM_ARGS_GRAFANA} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra # Delete the test pod if it still exists kubectl delete pods -l application=grafana,release_group=grafana,component=test --namespace=osh-infra --ignore-not-found diff --git a/tools/deployment/common/070-kube-state-metrics.sh b/tools/deployment/monitoring/kube-state-metrics.sh similarity index 75% rename from tools/deployment/common/070-kube-state-metrics.sh rename to tools/deployment/monitoring/kube-state-metrics.sh index 35c8e2645..411cf1e86 100755 --- a/tools/deployment/common/070-kube-state-metrics.sh +++ b/tools/deployment/monitoring/kube-state-metrics.sh @@ -14,15 +14,12 @@ set -xe -#NOTE: Lint and package chart -make prometheus-kube-state-metrics - #NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS_KUBE_STATE_METRICS:="$(./tools/deployment/common/get-values-overrides.sh prometheus-kube-state-metrics)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_KUBE_STATE_METRICS:="$(helm osh get-values-overrides -c prometheus-kube-state-metrics ${FEATURES})"} helm upgrade --install prometheus-kube-state-metrics \ ./prometheus-kube-state-metrics --namespace=kube-system \ ${OSH_INFRA_EXTRA_HELM_ARGS_KUBE_STATE_METRICS} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system +helm osh wait-for-pods kube-system diff --git a/tools/deployment/mariadb-operator-cluster/095-mariadb-prometheus-mysql-exporter.sh b/tools/deployment/monitoring/mysql-exporter.sh similarity index 71% rename from tools/deployment/mariadb-operator-cluster/095-mariadb-prometheus-mysql-exporter.sh rename to tools/deployment/monitoring/mysql-exporter.sh index ba03e36be..3f63bf998 100755 --- a/tools/deployment/mariadb-operator-cluster/095-mariadb-prometheus-mysql-exporter.sh +++ b/tools/deployment/monitoring/mysql-exporter.sh @@ -14,23 +14,18 @@ set -xe -#NOTE: Lint and package chart -make prometheus-mysql-exporter - -: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_MYSQL_EXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-mysql-exporter)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_MYSQL_EXPORTER:="$(helm osh get-values-overrides -c prometheus-mysql-exporter ${FEATURES})"} #NOTE: Deploy command -# Deploying downscaled cluster -: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} helm upgrade --install prometheus-mysql-exporter ./prometheus-mysql-exporter \ --namespace=openstack \ --wait \ --timeout 900s \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_MYSQL_EXPORTER} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh openstack +helm osh wait-for-pods openstack kubectl get pods --namespace=openstack -o wide diff --git a/tools/deployment/osh-infra-monitoring-tls/120-nagios.sh b/tools/deployment/monitoring/nagios.sh similarity index 80% rename from tools/deployment/osh-infra-monitoring-tls/120-nagios.sh rename to tools/deployment/monitoring/nagios.sh index a41de6a54..444339d8f 100755 --- a/tools/deployment/osh-infra-monitoring-tls/120-nagios.sh +++ b/tools/deployment/monitoring/nagios.sh @@ -14,19 +14,16 @@ set -xe -#NOTE: Lint and package chart -make nagios - -: ${OSH_INFRA_EXTRA_HELM_ARGS_NAGIOS:="$(./tools/deployment/common/get-values-overrides.sh nagios)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_NAGIOS:="$(helm osh get-values-overrides -c nagios ${FEATURES})"} #NOTE: Deploy command helm upgrade --install nagios ./nagios \ --namespace=osh-infra \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ ${OSH_INFRA_EXTRA_HELM_ARGS_NAGIOS} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra # Delete the test pod if it still exists kubectl delete pods -l application=nagios,release_group=nagios,component=test --namespace=osh-infra --ignore-not-found diff --git a/tools/deployment/common/080-node-exporter.sh b/tools/deployment/monitoring/node-exporter.sh similarity index 76% rename from tools/deployment/common/080-node-exporter.sh rename to tools/deployment/monitoring/node-exporter.sh index 5527a9db8..6657b1bf7 100755 --- a/tools/deployment/common/080-node-exporter.sh +++ b/tools/deployment/monitoring/node-exporter.sh @@ -14,15 +14,12 @@ set -xe -#NOTE: Lint and package chart -make prometheus-node-exporter - #NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS_NODE_EXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-node-exporter)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_NODE_EXPORTER:="$(helm osh get-values-overrides -c prometheus-node-exporter ${FEATURES})"} helm upgrade --install prometheus-node-exporter \ ./prometheus-node-exporter --namespace=kube-system \ ${OSH_INFRA_EXTRA_HELM_ARGS_NODE_EXPORTER} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system +helm osh wait-for-pods kube-system diff --git a/tools/deployment/common/node-problem-detector.sh b/tools/deployment/monitoring/node-problem-detector.sh similarity index 88% rename from tools/deployment/common/node-problem-detector.sh rename to tools/deployment/monitoring/node-problem-detector.sh index 7bbd114e4..7799d7e18 100755 --- a/tools/deployment/common/node-problem-detector.sh +++ b/tools/deployment/monitoring/node-problem-detector.sh @@ -13,9 +13,6 @@ set -xe -#NOTE: Lint and package chart -make kubernetes-node-problem-detector - #NOTE: Deploy command tee /tmp/kubernetes-node-problem-detector.yaml << EOF monitoring: @@ -32,4 +29,4 @@ helm upgrade --install kubernetes-node-problem-detector \ --values=/tmp/kubernetes-node-problem-detector.yaml #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system +helm osh wait-for-pods kube-system diff --git a/tools/deployment/common/openstack-exporter.sh b/tools/deployment/monitoring/openstack-exporter.sh similarity index 80% rename from tools/deployment/common/openstack-exporter.sh rename to tools/deployment/monitoring/openstack-exporter.sh index b55ab1c39..0c57c3cfd 100755 --- a/tools/deployment/common/openstack-exporter.sh +++ b/tools/deployment/monitoring/openstack-exporter.sh @@ -14,11 +14,8 @@ set -xe -#NOTE: Lint and package chart -make prometheus-openstack-exporter - #NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS_OS_EXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-openstack-exporter)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_OS_EXPORTER:="$(helm osh get-values-overrides -c prometheus-openstack-exporter ${FEATURES})"} tee /tmp/prometheus-openstack-exporter.yaml << EOF manifests: @@ -37,4 +34,4 @@ helm upgrade --install prometheus-openstack-exporter \ ${OSH_INFRA_EXTRA_HELM_ARGS_OS_EXPORTER} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh openstack +helm osh wait-for-pods openstack diff --git a/tools/deployment/common/090-process-exporter.sh b/tools/deployment/monitoring/process-exporter.sh similarity index 76% rename from tools/deployment/common/090-process-exporter.sh rename to tools/deployment/monitoring/process-exporter.sh index 167930de5..a78820847 100755 --- a/tools/deployment/common/090-process-exporter.sh +++ b/tools/deployment/monitoring/process-exporter.sh @@ -14,15 +14,12 @@ set -xe -#NOTE: Lint and package chart -make prometheus-process-exporter - #NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS_PROCESS_EXPORTER:="$(./tools/deployment/common/get-values-overrides.sh prometheus-process-exporter)"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_PROCESS_EXPORTER:="$(helm osh get-values-overrides -c prometheus-process-exporter ${FEATURES})"} helm upgrade --install prometheus-process-exporter \ ./prometheus-process-exporter --namespace=kube-system \ ${OSH_INFRA_EXTRA_HELM_ARGS_PROCESS_EXPORTER} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh kube-system +helm osh wait-for-pods kube-system diff --git a/tools/deployment/osh-infra-local-storage/040-prometheus.sh b/tools/deployment/monitoring/prometheus.sh similarity index 77% rename from tools/deployment/osh-infra-local-storage/040-prometheus.sh rename to tools/deployment/monitoring/prometheus.sh index caf52624e..ce0ebb62a 100755 --- a/tools/deployment/osh-infra-local-storage/040-prometheus.sh +++ b/tools/deployment/monitoring/prometheus.sh @@ -14,20 +14,17 @@ set -xe -#NOTE: Lint and package chart -make prometheus +FEATURE_GATES="alertmanager ceph elasticsearch kubernetes nodes openstack postgresql apparmor" +: ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS:="$(helm osh get-values-overrides -c prometheus ${FEATURE_GATES} ${FEATURES})"} #NOTE: Deploy command -: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} -: ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS:="$(./tools/deployment/common/get-values-overrides.sh prometheus)"} - helm upgrade --install prometheus ./prometheus \ --namespace=osh-infra \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS} #NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra +helm osh wait-for-pods osh-infra # Delete the test pod if it still exists kubectl delete pods -l application=prometheus,release_group=prometheus,component=test --namespace=osh-infra --ignore-not-found diff --git a/tools/deployment/multinode/010-deploy-docker-registry.sh b/tools/deployment/multinode/010-deploy-docker-registry.sh deleted file mode 120000 index 7360ae428..000000000 --- a/tools/deployment/multinode/010-deploy-docker-registry.sh +++ /dev/null @@ -1 +0,0 @@ -../common/010-deploy-docker-registry.sh \ No newline at end of file diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh deleted file mode 100755 index 04a41d44b..000000000 --- a/tools/deployment/multinode/030-ceph.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -# setup loopback devices for ceph -free_loop_devices=( $(ls -1 /dev/loop[0-7] | while read loopdev; do losetup | grep -q $loopdev || echo $loopdev; done) ) -./tools/deployment/common/setup-ceph-loopback-device.sh \ - --ceph-osd-data ${CEPH_OSD_DATA_DEVICE:=${free_loop_devices[0]}} \ - --ceph-osd-dbwal ${CEPH_OSD_DB_WAL_DEVICE:=${free_loop_devices[1]}} - -#NOTE: Lint and package chart -make ceph-mon -make ceph-osd -make ceph-client -make ceph-provisioners - -#NOTE: Deploy command -[ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt -CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -CEPH_CLUSTER_NETWORK="${CEPH_PUBLIC_NETWORK}" -CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" - -#NOTE(portdirect): to use RBD devices with kernels < 4.5 this should be set to 'hammer' -. /etc/os-release -if [ "x${ID}" == "xcentos" ] || \ - ([ "x${ID}" == "xubuntu" ] && \ - dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then - CRUSH_TUNABLES=hammer -else - CRUSH_TUNABLES=null -fi - -NUMBER_OF_OSDS="$(kubectl get nodes -l ceph-osd=enabled --no-headers | wc -l)" -tee /tmp/ceph.yaml << EOF -endpoints: - identity: - namespace: openstack - object_store: - namespace: ceph - ceph_mon: - namespace: ceph -network: - public: ${CEPH_PUBLIC_NETWORK} - cluster: ${CEPH_CLUSTER_NETWORK} -deployment: - storage_secrets: true - ceph: true - csi_rbd_provisioner: true - client_secrets: false - rgw_keystone_user_and_endpoints: false -bootstrap: - enabled: true -conf: - ceph: - global: - fsid: ${CEPH_FS_ID} - mon_allow_pool_size_one: true - rgw_ks: - enabled: true - pool: - crush: - tunables: ${CRUSH_TUNABLES} - target: - osd: ${NUMBER_OF_OSDS} - pg_per_osd: 100 - storage: - osd: - - data: - type: bluestore - location: ${CEPH_OSD_DATA_DEVICE} - block_db: - location: ${CEPH_OSD_DB_WAL_DEVICE} - size: "5GB" - block_wal: - location: ${CEPH_OSD_DB_WAL_DEVICE} - size: "2GB" - -jobs: - ceph_defragosds: - # Execute every 15 minutes for gates - cron: "*/15 * * * *" - history: - # Number of successful job to keep - successJob: 1 - # Number of failed job to keep - failJob: 1 - concurrency: - # Skip new job if previous job still active - execPolicy: Forbid - startingDeadlineSecs: 60 -storageclass: - cephfs: - provision_storage_class: false -manifests: - cronjob_defragosds: true - job_cephfs_client_key: false -EOF - -for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do - helm upgrade --install ${CHART} ./${CHART} \ - --namespace=ceph \ - --values=/tmp/ceph.yaml \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_DEPLOY:-$(./tools/deployment/common/get-values-overrides.sh ${CHART})} - - #NOTE: Wait for deploy - ./tools/deployment/common/wait-for-pods.sh ceph 1200 - - #NOTE: Validate deploy - MON_POD=$(kubectl get pods \ - --namespace=ceph \ - --selector="application=ceph" \ - --selector="component=mon" \ - --no-headers | awk '{ print $1; exit }') - kubectl exec -n ceph ${MON_POD} -- ceph -s -done - -# Delete the test pod if it still exists -kubectl delete pods -l application=ceph-osd,release_group=ceph-osd,component=test --namespace=ceph --ignore-not-found -helm test ceph-osd --namespace ceph --timeout 900s - -# Delete the test pod if it still exists -kubectl delete pods -l application=ceph-client,release_group=ceph-client,component=test --namespace=ceph --ignore-not-found -helm test ceph-client --namespace ceph --timeout 900s diff --git a/tools/deployment/multinode/035-ceph-ns-activate.sh b/tools/deployment/multinode/035-ceph-ns-activate.sh deleted file mode 100755 index 389899e3a..000000000 --- a/tools/deployment/multinode/035-ceph-ns-activate.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Deploy command -CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -CEPH_CLUSTER_NETWORK="${CEPH_PUBLIC_NETWORK}" -tee /tmp/ceph-osh-infra-config.yaml <- - "Delete indices older than 365 days" - options: - timeout_override: - continue_if_exception: False - ignore_empty_list: True - disable_action: True - filters: - - filtertype: pattern - kind: prefix - value: logstash- - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 365 -monitoring: - prometheus: - enabled: true - -EOF -helm upgrade --install elasticsearch ./elasticsearch \ - --namespace=osh-infra \ - --values=/tmp/elasticsearch.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra - -# Delete the test pod if it still exists -kubectl delete pods -l application=elasticsearch,release_group=elasticsearch,component=test --namespace=osh-infra --ignore-not-found -#NOTE: Run helm tests -helm test elasticsearch --namespace osh-infra diff --git a/tools/deployment/multinode/125-fluentbit.sh b/tools/deployment/multinode/125-fluentbit.sh deleted file mode 120000 index 0ed92806a..000000000 --- a/tools/deployment/multinode/125-fluentbit.sh +++ /dev/null @@ -1 +0,0 @@ -../common/fluentbit.sh \ No newline at end of file diff --git a/tools/deployment/multinode/130-fluentd.sh b/tools/deployment/multinode/130-fluentd.sh deleted file mode 120000 index c4b76c18c..000000000 --- a/tools/deployment/multinode/130-fluentd.sh +++ /dev/null @@ -1 +0,0 @@ -../common/fluentd.sh \ No newline at end of file diff --git a/tools/deployment/multinode/140-kibana.sh b/tools/deployment/multinode/140-kibana.sh deleted file mode 100755 index 7366dbc3d..000000000 --- a/tools/deployment/multinode/140-kibana.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make kibana - -#NOTE: Deploy command -helm upgrade --install kibana ./kibana \ - --namespace=osh-infra \ - --set pod.replicas.kibana=2 - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/multinode/150-falco.sh b/tools/deployment/multinode/150-falco.sh deleted file mode 120000 index d1264fb7b..000000000 --- a/tools/deployment/multinode/150-falco.sh +++ /dev/null @@ -1 +0,0 @@ -../common/150-falco.sh \ No newline at end of file diff --git a/tools/deployment/multinode/170-postgresql.sh b/tools/deployment/multinode/170-postgresql.sh deleted file mode 120000 index dad2d5019..000000000 --- a/tools/deployment/multinode/170-postgresql.sh +++ /dev/null @@ -1 +0,0 @@ -../common/postgresql.sh \ No newline at end of file diff --git a/tools/deployment/multinode/600-grafana-selenium.sh b/tools/deployment/multinode/600-grafana-selenium.sh deleted file mode 120000 index ca1714bb5..000000000 --- a/tools/deployment/multinode/600-grafana-selenium.sh +++ /dev/null @@ -1 +0,0 @@ -../common/grafana-selenium.sh \ No newline at end of file diff --git a/tools/deployment/multinode/610-nagios-selenium.sh b/tools/deployment/multinode/610-nagios-selenium.sh deleted file mode 120000 index a4f66c4ea..000000000 --- a/tools/deployment/multinode/610-nagios-selenium.sh +++ /dev/null @@ -1 +0,0 @@ -../common/nagios-selenium.sh \ No newline at end of file diff --git a/tools/deployment/multinode/620-prometheus-selenium.sh b/tools/deployment/multinode/620-prometheus-selenium.sh deleted file mode 120000 index aeb8622ba..000000000 --- a/tools/deployment/multinode/620-prometheus-selenium.sh +++ /dev/null @@ -1 +0,0 @@ -../common/prometheus-selenium.sh \ No newline at end of file diff --git a/tools/deployment/multinode/630-kibana-selenium.sh b/tools/deployment/multinode/630-kibana-selenium.sh deleted file mode 120000 index d5114e2cc..000000000 --- a/tools/deployment/multinode/630-kibana-selenium.sh +++ /dev/null @@ -1 +0,0 @@ -../common/kibana-selenium.sh \ No newline at end of file diff --git a/tools/deployment/multinode/kube-node-subnet.sh b/tools/deployment/multinode/kube-node-subnet.sh deleted file mode 100755 index 08f069a87..000000000 --- a/tools/deployment/multinode/kube-node-subnet.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -set -e - -UTILS_IMAGE=docker.io/openstackhelm/gate-utils:v0.1.0 -NODE_IPS=$(mktemp) -kubectl get nodes -o json | jq -r '.items[].status.addresses[] | select(.type=="InternalIP").address' | sort -V > $NODE_IPS -function run_and_log_ipcalc { - POD_NAME="tmp-$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-z | head -c 5; echo)" - kubectl run ${POD_NAME} \ - --generator=run-pod/v1 \ - --wait \ - --image ${UTILS_IMAGE} \ - --restart=Never \ - ipcalc -- "$1" - end=$(($(date +%s) + 900)) - until kubectl get pod/${POD_NAME} -o go-template='{{.status.phase}}' | grep -q Succeeded; do - now=$(date +%s) - [ $now -gt $end ] && echo containers failed to start. && \ - kubectl get pod/${POD_NAME} -o wide && exit 1 - done - kubectl logs pod/${POD_NAME} - kubectl delete pod/${POD_NAME} -} -FIRST_IP_SUBNET=$(run_and_log_ipcalc "$(head -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }') -LAST_IP_SUBNET=$(run_and_log_ipcalc "$(tail -n 1 ${NODE_IPS})/24" | awk '/^Network/ { print $2 }') -rm -f $NODE_IPS -function ip_diff { - echo $(($(echo $LAST_IP_SUBNET | awk -F '.' "{ print \$$1}") - $(echo $FIRST_IP_SUBNET | awk -F '.' "{ print \$$1}"))) -} -for X in {1..4}; do - if ! [ "$(ip_diff ${X})" -eq "0" ]; then - SUBMASK=$((((${X} - 1 )) * 8)) - break - elif [ ${X} -eq "4" ]; then - SUBMASK=24 - fi -done -echo ${FIRST_IP_SUBNET%/*}/${SUBMASK} diff --git a/tools/deployment/network-policy/000-install-packages.sh b/tools/deployment/network-policy/000-install-packages.sh deleted file mode 120000 index d702c4899..000000000 --- a/tools/deployment/network-policy/000-install-packages.sh +++ /dev/null @@ -1 +0,0 @@ -../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/network-policy/005-deploy-k8s.sh b/tools/deployment/network-policy/005-deploy-k8s.sh deleted file mode 120000 index 003bfbb8e..000000000 --- a/tools/deployment/network-policy/005-deploy-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/network-policy/020-nfs-provisioner.sh b/tools/deployment/network-policy/020-nfs-provisioner.sh deleted file mode 120000 index f7ec8c7ca..000000000 --- a/tools/deployment/network-policy/020-nfs-provisioner.sh +++ /dev/null @@ -1 +0,0 @@ -../common/030-nfs-provisioner.sh \ No newline at end of file diff --git a/tools/deployment/network-policy/039-lockdown.sh b/tools/deployment/network-policy/039-lockdown.sh deleted file mode 100755 index daf077963..000000000 --- a/tools/deployment/network-policy/039-lockdown.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -set -xe - -#NOTE: Lint and package chart -make lockdown - -#NOTE: Deploy command -helm upgrade --install lockdown ./lockdown \ - --namespace=osh-infra - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh openstack diff --git a/tools/deployment/network-policy/040-ldap.sh b/tools/deployment/network-policy/040-ldap.sh deleted file mode 100755 index 3dad60dac..000000000 --- a/tools/deployment/network-policy/040-ldap.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Pull images and lint chart -make ldap - -tee /tmp/ldap.yaml <- - "Delete indices older than 365 days" - options: - timeout_override: - continue_if_exception: False - ignore_empty_list: True - disable_action: True - filters: - - filtertype: pattern - kind: prefix - value: logstash- - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 365 -monitoring: - prometheus: - enabled: true -manifests: - network_policy: true - monitoring: - prometheus: - network_policy_exporter: true -EOF - -helm upgrade --install elasticsearch ./elasticsearch \ - --namespace=osh-infra \ - --values=/tmp/elasticsearch.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/network-policy/125-fluentbit.sh b/tools/deployment/network-policy/125-fluentbit.sh deleted file mode 120000 index 0ed92806a..000000000 --- a/tools/deployment/network-policy/125-fluentbit.sh +++ /dev/null @@ -1 +0,0 @@ -../common/fluentbit.sh \ No newline at end of file diff --git a/tools/deployment/network-policy/130-fluentd-daemonset.sh b/tools/deployment/network-policy/130-fluentd-daemonset.sh deleted file mode 100755 index dad5c0936..000000000 --- a/tools/deployment/network-policy/130-fluentd-daemonset.sh +++ /dev/null @@ -1,314 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make fluentd - -tee /tmp/fluentd-daemonset.yaml << EOF -endpoints: - fluentd: - hosts: - default: fluentd-daemonset - prometheus_fluentd_exporter: - hosts: - default: fluentd-daemonset-exporter -monitoring: - prometheus: - enabled: true -pod: - env: - fluentd: - vars: - MY_TEST_VAR: FOO - secrets: - MY_TEST_SECRET: BAR - security_context: - fluentd: - pod: - runAsUser: 0 -deployment: - type: DaemonSet -conf: - fluentd: - template: | - - bind 0.0.0.0 - port 24220 - @type monitor_agent - - - - bind 0.0.0.0 - port "#{ENV['FLUENTD_PORT']}" - @type forward - - - - - time_format %Y-%m-%dT%H:%M:%S.%NZ - @type json - - path /var/log/containers/*.log - read_from_head true - tag kubernetes.* - @type tail - - - - @type tail - tag ceph.* - path /var/log/ceph/*/*.log - read_from_head true - - @type none - - - - - @type tail - tag libvirt.* - path /var/log/libvirt/**.log - read_from_head true - - @type none - - - - - @type tail - tag kernel - path /var/log/kern.log - read_from_head true - - @type none - - - - - @type tail - tag auth - path /var/log/auth.log - read_from_head true - - @type none - - - - - @type systemd - tag journal.* - path /var/log/journal - matches [{ "_SYSTEMD_UNIT": "docker.service" }] - read_from_head true - - - fields_strip_underscores true - fields_lowercase true - - - - - @type systemd - tag journal.* - path /var/log/journal - matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] - read_from_head true - - - fields_strip_underscores true - fields_lowercase true - - - - - @type kubernetes_metadata - - - - @type record_transformer - - hostname "#{ENV['NODE_NAME']}" - fluentd_pod "#{ENV['POD_NAME']}" - - - - - @type record_transformer - - hostname "#{ENV['NODE_NAME']}" - fluentd_pod "#{ENV['POD_NAME']}" - - - - - @type record_transformer - - hostname "#{ENV['NODE_NAME']}" - fluentd_pod "#{ENV['POD_NAME']}" - - - - - @type record_transformer - - hostname "#{ENV['NODE_NAME']}" - fluentd_pod "#{ENV['POD_NAME']}" - - - - - @type null - - - - - chunk_limit_size 512K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 32 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - logstash_prefix libvirt - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 512K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 32 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - logstash_prefix ceph - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 512K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 32 - retry_forever false - disable_chunk_backup true - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - logstash_prefix kernel - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 512K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 32 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - logstash_prefix auth - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 512K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 32 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - logstash_prefix journal - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - - - - - chunk_limit_size 512K - flush_interval 5s - flush_thread_count 8 - queue_limit_length 32 - retry_forever false - retry_max_interval 30 - - host "#{ENV['ELASTICSEARCH_HOST']}" - reload_connections false - reconnect_on_error true - reload_on_failure true - include_tag_key true - logstash_format true - password "#{ENV['ELASTICSEARCH_PASSWORD']}" - port "#{ENV['ELASTICSEARCH_PORT']}" - @type elasticsearch - user "#{ENV['ELASTICSEARCH_USERNAME']}" - -EOF -helm upgrade --install fluentd-daemonset ./fluentd \ - --namespace=osh-infra \ - --values=/tmp/fluentd-daemonset.yaml \ - --set manifests.network_policy=true \ - --set manifests.monitoring.prometheus.network_policy_exporter=true - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/network-policy/140-kibana.sh b/tools/deployment/network-policy/140-kibana.sh deleted file mode 100755 index 56dbd0a5c..000000000 --- a/tools/deployment/network-policy/140-kibana.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make kibana - -#NOTE: Deploy command -tee /tmp/kibana.yaml << EOF -network_policy: - kibana: - ingress: - - from: - - podSelector: - matchLabels: - application: elasticsearch - - podSelector: - matchLabels: - application: kibana - - podSelector: - matchLabels: - application: ingress - ports: - - protocol: TCP - port: 80 - - protocol: TCP - port: 443 - - protocol: TCP - port: 5601 -manifests: - network_policy: true -EOF -helm upgrade --install kibana ./kibana \ - --namespace=osh-infra \ - --values=/tmp/kibana.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/network-policy/901-test-networkpolicy.sh b/tools/deployment/network-policy/901-test-networkpolicy.sh deleted file mode 100755 index b5dfe4e32..000000000 --- a/tools/deployment/network-policy/901-test-networkpolicy.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -# test_netpol(namespace, application label, component label, target_host, expected_result{fail,success}) -function test_netpol { - NS=$1 - APPLICATION=$2 - COMPONENT=$3 - HOST=$4 - STATUS=$5 - echo Testing connection from component:$COMPONENT, application:$APPLICATION to host $HOST with namespace $NS - POD=$(kubectl -n $NS get pod -l application=$APPLICATION,component=$COMPONENT | grep Running | cut -f 1 -d " " | head -n 1) - PID=$(sudo docker inspect --format '{{ .State.Pid }}' $(kubectl get pods --namespace $NS $POD -o jsonpath='{.status.containerStatuses[0].containerID}' | cut -c 10-21)) - if [ "x${STATUS}" == "xfail" ]; then - if ! sudo nsenter -t $PID -n wget -r -nd --delete-after --timeout=5 --tries=1 $HOST ; then - if [[ "$?" == 6 ]]; then - exit 1 - else - echo "Connection timed out; as expected by policy." - fi - else - exit 1 - fi - else - if sudo nsenter -t $PID -n wget -r -nd --delete-after --timeout=10 --tries=1 $HOST; then - echo "Connection successful; as expected by policy" - # NOTE(srwilkers): If wget returns error code 6 (invalid credentials), we should consider it - # a success - elif [[ "$?" == 6 ]]; then - echo "Connection successful; as expected by policy" - else - exit 1 - fi - fi -} - -# Doing negative tests -# NOTE(gagehugo): Uncomment these once the proper netpol rules are made -#test_netpol osh-infra mariadb server elasticsearch.osh-infra.svc.cluster.local fail -#test_netpol osh-infra mariadb server nagios.osh-infra.svc.cluster.local fail -#test_netpol osh-infra mariadb server prometheus.osh-infra.svc.cluster.local fail -#test_netpol osh-infra mariadb server nagios.osh-infra.svc.cluster.local fail -#test_netpol osh-infra mariadb server openstack-metrics.openstack.svc.cluster.local:9103 fail -#test_netpol osh-infra mariadb server kibana.osh-infra.svc.cluster.local fail -#test_netpol osh-infra mariadb server fluentd-logging.osh-infra.svc.cluster.local:24224 fail -#test_netpol osh-infra fluentbit daemon prometheus.osh-infra.svc.cluster.local fail - -# Doing positive tests -test_netpol osh-infra grafana dashboard mariadb.osh-infra.svc.cluster.local:3306 success -test_netpol osh-infra elasticsearch client kibana-dash.osh-infra.svc.cluster.local success -test_netpol osh-infra fluentd internal elasticsearch-logging.osh-infra.svc.cluster.local success -test_netpol osh-infra prometheus api fluentd-exporter.osh-infra.svc.cluster.local:9309/metrics success -test_netpol osh-infra prometheus api elasticsearch-exporter.osh-infra.svc.cluster.local:9108/metrics success diff --git a/tools/deployment/network-policy/openstack-exporter.sh b/tools/deployment/network-policy/openstack-exporter.sh deleted file mode 100755 index 691cc0f05..000000000 --- a/tools/deployment/network-policy/openstack-exporter.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make prometheus-openstack-exporter - -tee /tmp/prometheus-openstack-exporter.yaml << EOF -manifests: - job_ks_user: false - network_policy: true -dependencies: - static: - prometheus_openstack_exporter: - jobs: null - services: null -network_policy: - prometheus-openstack-exporter: - ingress: - - from: - - podSelector: - matchLabels: - application: prometheus-openstack-exporter - - namespaceSelector: - matchLabels: - name: osh-infra - podSelector: - matchLabels: - application: prometheus - ports: - - protocol: TCP - port: 80 - - protocol: TCP - port: 9103 -EOF - -#NOTE: Deploy command -helm upgrade --install prometheus-openstack-exporter \ - ./prometheus-openstack-exporter \ - --namespace=openstack \ - --values=/tmp/prometheus-openstack-exporter.yaml - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh openstack diff --git a/tools/deployment/openstack-support-rook/000-install-packages.sh b/tools/deployment/openstack-support-rook/000-install-packages.sh deleted file mode 120000 index d702c4899..000000000 --- a/tools/deployment/openstack-support-rook/000-install-packages.sh +++ /dev/null @@ -1 +0,0 @@ -../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/openstack-support-rook/000-prepare-k8s.sh b/tools/deployment/openstack-support-rook/000-prepare-k8s.sh deleted file mode 120000 index aa9807064..000000000 --- a/tools/deployment/openstack-support-rook/000-prepare-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../common/prepare-k8s.sh \ No newline at end of file diff --git a/tools/deployment/openstack-support-rook/005-deploy-k8s.sh b/tools/deployment/openstack-support-rook/005-deploy-k8s.sh deleted file mode 120000 index 003bfbb8e..000000000 --- a/tools/deployment/openstack-support-rook/005-deploy-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/openstack-support-rook/050-libvirt.sh b/tools/deployment/openstack-support-rook/050-libvirt.sh deleted file mode 100755 index bb6296337..000000000 --- a/tools/deployment/openstack-support-rook/050-libvirt.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -set -xe - -: ${OSH_INFRA_EXTRA_HELM_ARGS_LIBVIRT:="$(./tools/deployment/common/get-values-overrides.sh libvirt)"} - -#NOTE: Lint and package chart -make libvirt - -#NOTE: Deploy command -helm upgrade --install libvirt ./libvirt \ - --namespace=openstack \ - --set network.backend="null" \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_LIBVIRT} - -#NOTE: Please be aware that a network backend might affect -#The loadability of this, as some need to be asynchronously -#loaded. See also: -#https://github.com/openstack/openstack-helm-infra/blob/b69584bd658ae5cb6744e499975f9c5a505774e5/libvirt/values.yaml#L151-L172 -if [[ "${WAIT_FOR_PODS:=True}" == "True" ]]; then - ./tools/deployment/common/wait-for-pods.sh openstack -fi diff --git a/tools/deployment/openstack-support-rook/051-libvirt-ssl.sh b/tools/deployment/openstack-support-rook/051-libvirt-ssl.sh deleted file mode 100755 index 281a21985..000000000 --- a/tools/deployment/openstack-support-rook/051-libvirt-ssl.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -set -xe - -: ${OSH_INFRA_EXTRA_HELM_ARGS_LIBVIRT:="$(./tools/deployment/common/get-values-overrides.sh libvirt)"} - -CERT_DIR=$(mktemp -d) -cd ${CERT_DIR} -openssl req -x509 -new -nodes -days 1 -newkey rsa:2048 -keyout cacert.key -out cacert.pem -subj "/CN=libvirt.org" -openssl req -newkey rsa:2048 -days 1 -nodes -keyout client-key.pem -out client-req.pem -subj "/CN=libvirt.org" -openssl rsa -in client-key.pem -out client-key.pem -openssl x509 -req -in client-req.pem -days 1 \ - -CA cacert.pem -CAkey cacert.key -set_serial 01 \ - -out client-cert.pem -openssl req -newkey rsa:2048 -days 1 -nodes -keyout server-key.pem -out server-req.pem -subj "/CN=libvirt.org" -openssl rsa -in server-key.pem -out server-key.pem -openssl x509 -req -in server-req.pem -days 1 \ - -CA cacert.pem -CAkey cacert.key -set_serial 01 \ - -out server-cert.pem -cd - - -cat < /tmp/ceph-fs-uuid.txt -CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" -#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this -# should be set to 'hammer' -. /etc/os-release -if [ "x${ID}" == "xcentos" ] || \ - ([ "x${ID}" == "xubuntu" ] && \ - dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then - CRUSH_TUNABLES=hammer -else - CRUSH_TUNABLES=null -fi -tee /tmp/ceph.yaml <- - "Delete indices older than 365 days" - options: - timeout_override: - continue_if_exception: False - ignore_empty_list: True - disable_action: False - filters: - - filtertype: pattern - kind: prefix - value: logstash- - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 365 - 2: - action: snapshot - description: >- - "Snapshot all indices older than 365 days" - options: - repository: logstash_snapshots - name: "snapshot-%Y-.%m.%d" - wait_for_completion: True - max_wait: 36000 - wait_interval: 30 - ignore_empty_list: True - continue_if_exception: False - disable_action: False - filters: - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 365 - 3: - action: delete_snapshots - description: >- - "Delete index snapshots older than 365 days" - options: - repository: logstash_snapshots - timeout_override: 1200 - retry_interval: 120 - retry_count: 5 - ignore_empty_list: True - continue_if_exception: False - disable_action: False - filters: - - filtertype: pattern - kind: prefix - value: snapshot- - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 365 - -EOF - -: ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH:="$(./tools/deployment/common/get-values-overrides.sh elasticsearch)"} - -helm upgrade --install elasticsearch ./elasticsearch \ - --namespace=osh-infra \ - --values=/tmp/elasticsearch.yaml\ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH} - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra - -# Delete the test pod if it still exists -kubectl delete pods -l application=elasticsearch,release_group=elasticsearch,component=test --namespace=osh-infra --ignore-not-found -helm test elasticsearch --namespace osh-infra diff --git a/tools/deployment/osh-infra-logging-tls/060-fluentd.sh b/tools/deployment/osh-infra-logging-tls/060-fluentd.sh deleted file mode 120000 index c4b76c18c..000000000 --- a/tools/deployment/osh-infra-logging-tls/060-fluentd.sh +++ /dev/null @@ -1 +0,0 @@ -../common/fluentd.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging-tls/070-kibana.sh b/tools/deployment/osh-infra-logging-tls/070-kibana.sh deleted file mode 100755 index 2d80a3938..000000000 --- a/tools/deployment/osh-infra-logging-tls/070-kibana.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make kibana - -: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(./tools/deployment/common/get-values-overrides.sh kibana)"} - -#NOTE: Deploy command -: ${OSH_EXTRA_HELM_ARGS:=""} -helm upgrade --install kibana ./kibana \ - --namespace=osh-infra \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA} - -#NOTE: Wait for deploy -./tools/deployment/common/wait-for-pods.sh osh-infra diff --git a/tools/deployment/osh-infra-logging-tls/600-kibana-selenium.sh b/tools/deployment/osh-infra-logging-tls/600-kibana-selenium.sh deleted file mode 120000 index d5114e2cc..000000000 --- a/tools/deployment/osh-infra-logging-tls/600-kibana-selenium.sh +++ /dev/null @@ -1 +0,0 @@ -../common/kibana-selenium.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/000-install-packages.sh b/tools/deployment/osh-infra-logging/000-install-packages.sh deleted file mode 120000 index d702c4899..000000000 --- a/tools/deployment/osh-infra-logging/000-install-packages.sh +++ /dev/null @@ -1 +0,0 @@ -../common/000-install-packages.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/000-prepare-k8s.sh b/tools/deployment/osh-infra-logging/000-prepare-k8s.sh deleted file mode 120000 index aa9807064..000000000 --- a/tools/deployment/osh-infra-logging/000-prepare-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../common/prepare-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/005-deploy-k8s.sh b/tools/deployment/osh-infra-logging/005-deploy-k8s.sh deleted file mode 120000 index 003bfbb8e..000000000 --- a/tools/deployment/osh-infra-logging/005-deploy-k8s.sh +++ /dev/null @@ -1 +0,0 @@ -../../gate/deploy-k8s.sh \ No newline at end of file diff --git a/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh b/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh deleted file mode 100755 index c276a178e..000000000 --- a/tools/deployment/osh-infra-logging/025-ceph-ns-activate.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Lint and package chart -make ceph-provisioners - -#NOTE: Deploy command -: ${OSH_EXTRA_HELM_ARGS:=""} -tee /tmp/ceph-osh-infra-config.yaml <> /tmp/nodes.json - -# Use jq to find the names of the nodes to relabel by slicing the output at the -# number identified above -export RELABEL_NODES=$(cat /tmp/nodes.json | jq -r '.items[0:(env.NUM_RELABEL|tonumber)] | .[].metadata.name') - -# Relabel the nodes appropriately -for node in $RELABEL_NODES; do - for ceph_label in ceph-mon ceph-osd ceph-mds ceph-rgw ceph-mgr; do - kubectl label node $node $ceph_label-; - kubectl label node $node $ceph_label-tenant=enabled; - done; - kubectl label node $node tenant-ceph-control-plane=enabled; -done; diff --git a/tools/deployment/tenant-ceph/030-ceph.sh b/tools/deployment/tenant-ceph/030-ceph.sh deleted file mode 100755 index fb0cb58d9..000000000 --- a/tools/deployment/tenant-ceph/030-ceph.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -# setup loopback devices for ceph -free_loop_devices=( $(ls -1 /dev/loop[0-7] | while read loopdev; do losetup | grep -q $loopdev || echo $loopdev; done) ) -./tools/deployment/common/setup-ceph-loopback-device.sh \ - --ceph-osd-data ${CEPH_OSD_DATA_DEVICE:=${free_loop_devices[0]}} \ - --ceph-osd-dbwal ${CEPH_OSD_DB_WAL_DEVICE:=${free_loop_devices[1]}} - -#NOTE: Deploy command -[ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt -CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" -#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this -# should be set to 'hammer' -. /etc/os-release -if [ "x${ID}" == "xcentos" ] || \ - ([ "x${ID}" == "xubuntu" ] && \ - dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then - CRUSH_TUNABLES=hammer -else - CRUSH_TUNABLES=null -fi -if [ "x${ID}" == "xcentos" ]; then - CRUSH_TUNABLES=hammer -fi -tee /tmp/ceph.yaml << EOF -endpoints: - ceph_mon: - namespace: ceph - port: - mon: - default: 6789 - ceph_mgr: - namespace: ceph - port: - mgr: - default: 7000 - metrics: - default: 9283 -network: - public: ${CEPH_PUBLIC_NETWORK} - cluster: ${CEPH_CLUSTER_NETWORK} -deployment: - storage_secrets: true - ceph: true - csi_rbd_provisioner: true - client_secrets: false - rgw_keystone_user_and_endpoints: false -jobs: - ceph_defragosds: - # Execute every 15 minutes for gates - cron: "*/15 * * * *" - history: - # Number of successful job to keep - successJob: 1 - # Number of failed job to keep - failJob: 1 - concurrency: - # Skip new job if previous job still active - execPolicy: Forbid - startingDeadlineSecs: 60 -manifests: - deployment_mds: false - cronjob_defragosds: true - job_cephfs_client_key: false -bootstrap: - enabled: true -conf: - ceph: - global: - fsid: ${CEPH_FS_ID} - mon_allow_pool_size_one: true - mon: - mon_clock_drift_allowed: 2.0 - rgw_ks: - enabled: true - pool: - crush: - tunables: ${CRUSH_TUNABLES} - target: - # NOTE(portdirect): 5 nodes, with one osd per node - osd: 3 - pg_per_osd: 100 - storage: - osd: - - data: - type: bluestore - location: ${CEPH_OSD_DATA_DEVICE} - block_db: - location: ${CEPH_OSD_DB_WAL_DEVICE} - size: "5GB" - block_wal: - location: ${CEPH_OSD_DB_WAL_DEVICE} - size: "2GB" -storageclass: - csi_rbd: - ceph_configmap_name: ceph-etc - rbd: - provision_storage_class: false - cephfs: - provision_storage_class: false -ceph_mgr_modules_config: - prometheus: - server_port: 9283 -monitoring: - prometheus: - enabled: true - ceph_mgr: - port: 9283 -EOF - -for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do - helm upgrade --install ${CHART} ./${CHART} \ - --namespace=ceph \ - --values=/tmp/ceph.yaml \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_DEPLOY:-$(./tools/deployment/common/get-values-overrides.sh ${CHART})} - - #NOTE: Wait for deploy - ./tools/deployment/common/wait-for-pods.sh ceph 1200 - - #NOTE: Validate deploy - MON_POD=$(kubectl get pods \ - --namespace=ceph \ - --selector="application=ceph" \ - --selector="component=mon" \ - --no-headers | awk '{ print $1; exit }') - kubectl exec -n ceph ${MON_POD} -- ceph -s -done - -# Delete the test pod if it still exists -kubectl delete pods -l application=ceph-osd,release_group=ceph-osd,component=test --namespace=ceph --ignore-not-found -helm test ceph-osd --namespace ceph --timeout 900s -# Delete the test pod if it still exists -kubectl delete pods -l application=ceph-client,release_group=ceph-client,component=test --namespace=ceph --ignore-not-found -helm test ceph-client --namespace ceph --timeout 900s diff --git a/tools/deployment/tenant-ceph/035-ceph-ns-activate.sh b/tools/deployment/tenant-ceph/035-ceph-ns-activate.sh deleted file mode 120000 index f6c0f5f2e..000000000 --- a/tools/deployment/tenant-ceph/035-ceph-ns-activate.sh +++ /dev/null @@ -1 +0,0 @@ -../multinode/035-ceph-ns-activate.sh \ No newline at end of file diff --git a/tools/deployment/tenant-ceph/040-tenant-ceph.sh b/tools/deployment/tenant-ceph/040-tenant-ceph.sh deleted file mode 100755 index 45aff5cc0..000000000 --- a/tools/deployment/tenant-ceph/040-tenant-ceph.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -# setup loopback devices for ceph -free_loop_devices=( $(ls -1 /dev/loop[0-7] | while read loopdev; do losetup | grep -q $loopdev || echo $loopdev; done) ) -export CEPH_NAMESPACE="tenant-ceph" -./tools/deployment/common/setup-ceph-loopback-device.sh \ - --ceph-osd-data ${CEPH_OSD_DATA_DEVICE:=${free_loop_devices[0]}} \ - --ceph-osd-dbwal ${CEPH_OSD_DB_WAL_DEVICE:=${free_loop_devices[1]}} - -# setup loopback devices for ceph osds -setup_loopback_devices $OSD_DATA_DEVICE $OSD_DB_WAL_DEVICE - -#NOTE: Deploy command -[ -s /tmp/tenant-ceph-fs-uuid.txt ] || uuidgen > /tmp/tenant-ceph-fs-uuid.txt -CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -TENANT_CEPH_FS_ID="$(cat /tmp/tenant-ceph-fs-uuid.txt)" -#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this -# should be set to 'hammer' -. /etc/os-release -if [ "x${ID}" == "xubuntu" ] && \ - [ "$(uname -r | awk -F "." '{ print $2 }')" -lt "5" ]; then - CRUSH_TUNABLES=hammer -else - CRUSH_TUNABLES=null -fi -if [ "x${ID}" == "xcentos" ]; then - CRUSH_TUNABLES=hammer -fi -tee /tmp/tenant-ceph.yaml << EOF -endpoints: - identity: - namespace: openstack - object_store: - namespace: openstack - ceph_mon: - namespace: tenant-ceph - port: - mon: - default: 6790 - ceph_mgr: - namespace: tenant-ceph - port: - mgr: - default: 7001 - metrics: - default: 9284 -network: - public: ${CEPH_PUBLIC_NETWORK} - cluster: ${CEPH_CLUSTER_NETWORK} -deployment: - storage_secrets: true - ceph: true - csi_rbd_provisioner: false - client_secrets: false - rgw_keystone_user_and_endpoints: false -labels: - mon: - node_selector_key: ceph-mon-tenant - osd: - node_selector_key: ceph-osd-tenant - rgw: - node_selector_key: ceph-rgw-tenant - mgr: - node_selector_key: ceph-mgr-tenant - job: - node_selector_key: tenant-ceph-control-plane -storageclass: - rbd: - ceph_configmap_name: tenant-ceph-etc - provision_storage_class: false - metadata: - name: tenant-rbd - parameters: - adminSecretName: pvc-tenant-ceph-conf-combined-storageclass - adminSecretNamespace: tenant-ceph - userSecretName: pvc-tenant-ceph-client-key - cephfs: - provision_storage_class: false - metadata: - name: cephfs - parameters: - adminSecretName: pvc-tenant-ceph-conf-combined-storageclass - adminSecretNamespace: tenant-ceph - userSecretName: pvc-tenant-ceph-cephfs-client-key -bootstrap: - enabled: true -jobs: - ceph_defragosds: - # Execute every 15 minutes for gates - cron: "*/15 * * * *" - history: - # Number of successful job to keep - successJob: 1 - # Number of failed job to keep - failJob: 1 - concurrency: - # Skip new job if previous job still active - execPolicy: Forbid - startingDeadlineSecs: 60 -manifests: - deployment_mds: false - cronjob_defragosds: true - job_cephfs_client_key: false -ceph_mgr_modules_config: - prometheus: - server_port: 9284 -monitoring: - prometheus: - enabled: true - ceph_mgr: - port: 9284 -conf: - ceph: - global: - fsid: ${TENANT_CEPH_FS_ID} - mon_allow_pool_size_one: true - rgw_ks: - enabled: true - pool: - crush: - tunables: ${CRUSH_TUNABLES} - target: - osd: 2 - pg_per_osd: 100 - storage: - osd: - - data: - type: bluestore - location: ${CEPH_OSD_DATA_DEVICE} - block_db: - location: ${CEPH_OSD_DB_WAL_DEVICE} - size: "5GB" - block_wal: - location: ${CEPH_OSD_DB_WAL_DEVICE} - size: "2GB" - mon: - directory: /var/lib/openstack-helm/tenant-ceph/mon -deploy: - tool: "ceph-volume" -EOF - -for CHART in ceph-mon ceph-osd ceph-client; do - helm upgrade --install tenant-${CHART} ./${CHART} \ - --namespace=tenant-ceph \ - --values=/tmp/tenant-ceph.yaml \ - ${OSH_INFRA_EXTRA_HELM_ARGS} \ - ${OSH_INFRA_EXTRA_HELM_ARGS_CEPH_DEPLOY:-$(./tools/deployment/common/get-values-overrides.sh ${CHART})} - - #NOTE: Wait for deploy - ./tools/deployment/common/wait-for-pods.sh tenant-ceph 1200 - - #NOTE: Validate deploy - MON_POD=$(kubectl get pods \ - --namespace=tenant-ceph \ - --selector="application=ceph" \ - --selector="component=mon" \ - --no-headers | awk '{ print $1; exit }') - kubectl exec -n tenant-ceph ${MON_POD} -- ceph -s -done - -helm test tenant-ceph-osd --namespace tenant-ceph --timeout 900s -helm test tenant-ceph-client --namespace tenant-ceph --timeout 900s diff --git a/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh b/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh deleted file mode 100755 index 29ff4b761..000000000 --- a/tools/deployment/tenant-ceph/045-tenant-ceph-ns-activate.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -set -xe - -#NOTE: Deploy command -CEPH_PUBLIC_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -CEPH_CLUSTER_NETWORK="$(./tools/deployment/multinode/kube-node-subnet.sh)" -tee /tmp/tenant-ceph-openstack-config.yaml <