Update openstack requests to go through ingress

The requests to Openstack services hang/fail sometimes
due to message loss when connecting to internal service
endpoints. This issue was observed before and fixed in
commit https://review.opendev.org/#/c/683818/ by setting
net.ipv4.tcp_tw_reuse to 0, however, it's still being
seen on recent STX loads.

It has been tested and proved that requests go through
ingress pod do not have the issue. This commit updates
helm charts and manifest to make all requests sending
to openstack services go to ingress pod and then ingress
would forward requests to the corresponding api service.

Changes included:
- update helm-toolkit manifest job-ks-endpoint.yaml
  to provide an ability to conditionally configure
  all types of openstack endpoints with public endpoint
  url when endpoints.identity.force_public_endpoint is
  true. Same update for keystone and keystone-api-proxy.
  With the update, for example, the admin,internal
  and public endpoints for neutron will be
  neutron.openstack.svc.cluster.local:80
- update armada manifest to make neccessary overrides
  in openstack service configuration file to make
  communications between services go through ingress

Change-Id: Icd646dd07d544da420a75f920bd7c3e8fc931327
Closes-Bug: 1880777
Signed-off-by: Angie Wang <angie.wang@windriver.com>

openstack-helm-infra/centos/openstack-helm-infra.spec

@@ -27,6 +27,7 @@ Patch09: 0009-Enable-override-of-mariadb-server-probe-parameters.patch
 Patch10: 0010-Mariadb-use-utf8_general_ci-collation-as-default.patch
 Patch11: 0011-Add-mariadb-database-config-override-to-support-ipv6.patch
 Patch12: 0012-enable-Values.conf.database.config_override-for-mari.patch
+Patch13: 0013-Allow-set-public-endpoint-url-for-all-openstack-types.patch
 
 BuildRequires: helm
 BuildRequires: chartmuseum
@@ -48,6 +49,7 @@ Openstack Helm Infra charts
 %patch10 -p1
 %patch11 -p1
 %patch12 -p1
+%patch13 -p1
 
 %build
 # Host a server for the charts

openstack-helm-infra/files/0013-Allow-set-public-endpoint-url-for-all-openstack-types.patch

@@ -0,0 +1,29 @@
+From 7fd9d60e124868f277a1f2b0e86296dc50dd07ec Mon Sep 17 00:00:00 2001
+From: Angie Wang <angie.wang@windriver.com>
+Date: Mon, 31 Aug 2020 23:51:06 -0400
+Subject: [PATCH 1/1] Allow set public endpoint url for all openstack
+ endpoint types
+
+---
+ helm-toolkit/templates/manifests/_job-ks-endpoints.tpl | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+index 02f2013..39766f4 100644
+--- a/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
++++ b/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl
+@@ -84,7 +84,11 @@ spec:
+             - name: OS_SERVICE_TYPE
+               value: {{ $osServiceType | quote }}
+             - name: OS_SERVICE_ENDPOINT
++              {{- if and (hasKey $envAll.Values.endpoints.identity "force_public_endpoint") $envAll.Values.endpoints.identity.force_public_endpoint }}
++              value: {{ tuple $osServiceType "public" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
++              {{- else }}
+               value: {{ tuple $osServiceType $osServiceEndPoint "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
++              {{- end }}
+ {{- end }}
+ {{- end }}
+       volumes:
+-- 
+1.8.3.1
+

openstack-helm/centos/openstack-helm.spec

@@ -26,6 +26,7 @@ Patch04: 0004-Nova-chart-Support-ephemeral-pool-creation.patch
 Patch05: 0005-Nova-Add-support-for-disabling-Readiness-Liveness-pr.patch
 Patch06: 0006-Support-ingress-creation-for-keystone-admin-endpoint.patch
 Patch07: 0007-Allow-more-generic-overrides-for-placeme.patch
+Patch08: 0008-Allow-set-public-endpoint-url-for-keystone-endpoints.patch
 
 BuildRequires: helm
 BuildRequires: openstack-helm-infra
@@ -44,6 +45,7 @@ Openstack Helm charts
 %patch05 -p1
 %patch06 -p1
 %patch07 -p1
+%patch08 -p1
 
 %build
 # Stage helm-toolkit in the local repo

openstack-helm/files/0008-Allow-set-public-endpoint-url-for-keystone-endpoints.patch

@@ -0,0 +1,28 @@
+From b272e8ff3a78f38ab82df7995233705611e99f81 Mon Sep 17 00:00:00 2001
+From: Angie Wang <angie.wang@windriver.com>
+Date: Tue, 1 Sep 2020 00:00:22 -0400
+Subject: [PATCH 1/1] Allow set public endpoint url for keystone endpoints
+
+---
+ keystone/templates/job-db-sync.yaml | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/keystone/templates/job-db-sync.yaml b/keystone/templates/job-db-sync.yaml
+index 56a39b8..3d0681e 100644
+--- a/keystone/templates/job-db-sync.yaml
++++ b/keystone/templates/job-db-sync.yaml
+@@ -20,7 +20,11 @@ env:
+   - name: OS_BOOTSTRAP_ADMIN_URL
+     value: {{ tuple "identity" "admin" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+   - name: OS_BOOTSTRAP_INTERNAL_URL
++    {{- if and (hasKey $envAll.Values.endpoints.identity "force_public_endpoint") $envAll.Values.endpoints.identity.force_public_endpoint }}
++    value: {{ tuple "identity" "public" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
++    {{- else }}
+     value: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
++    {{- end }}
+   - name: OS_BOOTSTRAP_PUBLIC_URL
+     value: {{ tuple "identity" "public" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
+   - name: OPENSTACK_CONFIG_FILE
+-- 
+1.8.3.1
+

stx-openstack-helm/stx-openstack-helm/helm-charts/keystone-api-proxy/templates/job-ks-endpoints.yaml

@@ -61,7 +61,11 @@ spec:
             - name: OS_SERVICE_TYPE
               value: {{ $osServiceType | quote }}
             - name: OS_SERVICE_ENDPOINT
+              {{- if and (hasKey $envAll.Values.endpoints.identity "force_public_endpoint") $envAll.Values.endpoints.identity.force_public_endpoint }}
+              value: {{ tuple $osRealServiceType "public" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+              {{- else }}
               value: {{ tuple $osRealServiceType $osServiceEndPoint "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+              {{- end }}
             - name: OS_SERVICE_REGION
               value: {{ $osServiceRegion | quote }}
 {{- end }}

stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml

@@ -385,6 +385,7 @@ data:
       identity:
         name: keystone
         namespace: openstack
+        force_public_endpoint: true
     labels:
       api:
         node_selector_key: openstack-control-plane
@@ -487,6 +488,9 @@ data:
             release_group: osh-openstack-keystone-api-proxy
             component: test
   values:
+    endpoints:
+      identity:
+        force_public_endpoint: true
     images:
       tags:
         keystone_api_proxy: docker.io/starlingx/stx-keystone-api-proxy:master-centos-stable-latest
@@ -495,6 +499,8 @@ data:
       keystone_api_proxy:
         identity:
           bind_host: "::"
+          remote_host: keystone.openstack.svc.cluster.local
+          remote_port: 80
   source:
     type: tar
     location: http://172.17.0.1/helm_charts/keystone-api-proxy-0.1.0.tgz
@@ -561,10 +567,15 @@ data:
       oslo_messaging:
         statefulset:
           name: osh-openstack-rabbitmq-rabbitmq
+      identity:
+        force_public_endpoint: true
     conf:
       barbican:
         barbican_api:
           bind_host: "::"
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
   source:
     type: tar
     location: http://172.17.0.1/helm_charts/starlingx/barbican-0.1.0.tgz
@@ -720,10 +731,21 @@ data:
       oslo_messaging:
         statefulset:
           name: osh-openstack-rabbitmq-rabbitmq
+      identity:
+        force_public_endpoint: true
     conf:
       glance:
         DEFAULT:
           bind_host: "::"
+          registry_host: glance-reg.openstack.svc.cluster.local
+          registry_port: 80
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+      glance_registry:
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
   source:
     type: tar
     location: http://172.17.0.1/helm_charts/starlingx/glance-0.1.0.tgz
@@ -808,13 +830,18 @@ data:
       oslo_messaging:
         statefulset:
           name: osh-openstack-rabbitmq-rabbitmq
+      identity:
+        force_public_endpoint: true
     conf:
       cinder:
         DEFAULT:
           backup_driver: cinder.backup.drivers.ceph.CephBackupDriver
           osapi_volume_listen: "::"
+          glance_api_servers: http://glance.openstack.svc.cluster.local:80/
         keystone_authtoken:
           interface: internal
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
     storage: rbd
   source:
     type: tar
@@ -1081,6 +1108,8 @@ data:
           my_ip: "::"
         keystone_authtoken:
           interface: internal
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
         vnc:
           novncproxy_host: "::"
           vncserver_listen: "::"
@@ -1109,8 +1138,17 @@ data:
           idle_timeout: 60
           max_overflow: 64
           max_pool_size: 1
+        glance:
+          api_servers: http://glance.openstack.svc.cluster.local:80/
+        ironic:
+          api_endpoint: http://ironic.openstack.svc.cluster.local:80/
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+        placement:
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
         neutron:
           default_floating_pool: public
+          url: http://neutron.openstack.svc.cluster.local:80/
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
         notifications:
           notification_format: unversioned
         filter_scheduler:
@@ -1138,6 +1176,7 @@ data:
           discover_hosts_in_cells_interval: 30
           periodic_task_interval: -1
         service_user:
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
           send_service_user_token: true
         metrics:
           required: false
@@ -1183,6 +1222,9 @@ data:
           labels:
             release_group: osh-openstack-placement
   values:
+    endpoints:
+      identity:
+        force_public_endpoint: true
     labels:
       placement:
         node_selector_key: openstack-control-plane
@@ -1214,6 +1256,9 @@ data:
       placement:
         DEFAULT:
           log_config_append: /etc/placement/logging.conf
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
       wsgi_placement: |
         Listen :::{{ tuple "placement" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
         LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
@@ -1284,6 +1329,9 @@ data:
       tags:
         nova_api_proxy: docker.io/starlingx/stx-nova-api-proxy:master-centos-stable-latest
         ks_endpoints: docker.io/starlingx/stx-heat:master-centos-stable-latest
+    endpoints:
+      identity:
+        force_public_endpoint: true
     pod:
       affinity:
         anti:
@@ -1296,6 +1344,8 @@ data:
           nfvi_compute_listen: "::"
         keystone_authtoken:
           interface: internal
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
   source:
     type: tar
     location: http://172.17.0.1/helm_charts/starlingx/nova-api-proxy-0.1.0.tgz
@@ -1332,6 +1382,8 @@ data:
       oslo_messaging:
         statefulset:
           name: osh-openstack-rabbitmq-rabbitmq
+      identity:
+        force_public_endpoint: true
     pod:
       replicas:
         server: 2
@@ -1516,6 +1568,11 @@ data:
           root_helper: sudo
         vhost:
           vhost_user_enabled: true
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+        nova:
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
       dhcp_agent:
         DEFAULT:
           enable_isolated_metadata: true
@@ -1636,6 +1693,18 @@ data:
           enabled_raid_interfaces: no-raid
           enabled_storage_interfaces: cinder,noop
           enabled_vendor_interfaces: ipmitool,no-vendor
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+        glance:
+          glance_host: glance.openstack.svc.cluster.local
+          glance_port: 80
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+        inspector:
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+        neutron:
+          url: http://neutron.openstack.svc.cluster.local:80/
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
         api:
           port: 6385
         pxe:
@@ -1654,6 +1723,8 @@ data:
         port:
           pxe_http:
             default: 28080
+      identity:
+        force_public_endpoint: true
   source:
     type: tar
     location: http://172.17.0.1/helm_charts/starlingx/ironic-0.1.0.tgz
@@ -1697,6 +1768,8 @@ data:
       oslo_cache:
         hosts:
           default: heat-memcached
+      identity:
+        force_public_endpoint: true
     labels:
       api:
         node_selector_key: openstack-control-plane
@@ -1735,6 +1808,13 @@ data:
         stacks:global_index: rule:context_is_admin
         software_configs:global_index: rule:context_is_admin
       heat:
+        clients_keystone:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+        trustee:
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
         heat_api:
           bind_host: "::"
         heat_api_cloudwatch:
@@ -1814,11 +1894,19 @@ data:
       oslo_messaging:
         statefulset:
           name: osh-openstack-rabbitmq-rabbitmq
+      identity:
+        force_public_endpoint: true
     jobs:
       alarms_cleaner:
         # daily at the 35 minute mark
         cron: "35 */24 * * *"
     conf:
+      aodh:
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+        service_credentials:
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
       wsgi_aodh: |
         Listen :::{{ tuple "alarming" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
 
@@ -1900,6 +1988,8 @@ data:
           driver: mariadb
         keystone_authtoken:
           interface: internal
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
       apache: |
         Listen :::{{ tuple "metric" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
 
@@ -1996,6 +2086,8 @@ data:
       oslo_cache:
         hosts:
           default: memcached
+      identity:
+        force_public_endpoint: true
   source:
     type: tar
     location: http://172.17.0.1/helm_charts/starlingx/gnocchi-0.1.0.tgz
@@ -2032,6 +2124,9 @@ data:
             release_group: osh-openstack-panko
             component: test
   values:
+    endpoints:
+      identity:
+        force_public_endpoint: true
     pod:
       user:
         panko:
@@ -2054,6 +2149,10 @@ data:
         panko_events_cleaner: docker.io/starlingx/stx-panko:master-centos-stable-latest
         test: null
     conf:
+      panko:
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
       paste:
         composite:panko+noauth:
           use: egg:Paste#urlmap
@@ -2210,6 +2309,8 @@ data:
       secret_db: false
       secret_mongodb: false
     endpoints:
+      identity:
+        force_public_endpoint: true
       oslo_messaging:
         statefulset:
           name: osh-openstack-rabbitmq-rabbitmq
@@ -2233,6 +2334,15 @@ data:
             public: 80
     conf:
       ceilometer:
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+        service_credentials:
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
+        dispatcher_gnocchi:
+          url: http://gnocchi.openstack.svc.cluster.local:80/
+        api:
+          aodh_url: http://aodh.openstack.svc.cluster.local:80/
         cache:
           expiration_time: 86400
         compute:
@@ -2724,6 +2834,9 @@ data:
             release_group: osh-openstack-fm-rest-api
             component: ks-endpoints
   values:
+    endpoints:
+      identity:
+        force_public_endpoint: true
     pod:
       affinity:
         anti:
@@ -2749,6 +2862,9 @@ data:
       fm:
         api:
           bind_host: "::"
+        keystone_authtoken:
+          auth_uri: http://keystone.openstack.svc.cluster.local:80/v3
+          auth_url: http://keystone.openstack.svc.cluster.local:80/v3
   source:
     type: tar
     location: http://172.17.0.1/helm_charts/starlingx/fm-rest-api-0.1.0.tgz