Merge "Run iptables in service playbooks instead of base"

2020-06-04 21:05:04 +00:00 · 2020-06-04 21:05:04 +00:00 · 075c4035b3
commit 075c4035b3
parent 8cc2c77bd8 d93a661ae4
37 changed files with 123 additions and 19 deletions
--- a/playbooks/base.yaml
+++ b/playbooks/base.yaml
@ -8,4 +8,4 @@
    - base/unbound
    - base/exim
    - base/snmpd
-    - base/iptables
+    - iptables
--- a/playbooks/roles/base/iptables/README.rst
+++ b/playbooks/roles/base/iptables/README.rst
--- a/playbooks/roles/base/iptables/defaults/main.yaml
+++ b/playbooks/roles/base/iptables/defaults/main.yaml
--- a/playbooks/roles/base/iptables/handlers/main.yaml
+++ b/playbooks/roles/base/iptables/handlers/main.yaml
--- a/playbooks/roles/base/iptables/tasks/RedHat.yaml
+++ b/playbooks/roles/base/iptables/tasks/RedHat.yaml
--- a/playbooks/roles/base/iptables/tasks/main.yaml
+++ b/playbooks/roles/base/iptables/tasks/main.yaml
--- a/playbooks/roles/base/iptables/templates/rules.v4.j2
+++ b/playbooks/roles/base/iptables/templates/rules.v4.j2
--- a/playbooks/roles/base/iptables/templates/rules.v6.j2
+++ b/playbooks/roles/base/iptables/templates/rules.v6.j2
--- a/playbooks/roles/base/iptables/vars/Debian.yaml
+++ b/playbooks/roles/base/iptables/vars/Debian.yaml
--- a/playbooks/roles/base/iptables/vars/RedHat.yaml
+++ b/playbooks/roles/base/iptables/vars/RedHat.yaml
--- a/playbooks/roles/base/iptables/vars/Ubuntu.trusty.yaml
+++ b/playbooks/roles/base/iptables/vars/Ubuntu.trusty.yaml
--- a/playbooks/roles/run-puppet/tasks/main.yaml
+++ b/playbooks/roles/run-puppet/tasks/main.yaml
@ -1,4 +1,6 @@
 # Use include_role instead of roles: so that we can late-bind the roles list
+- include_role:
+    name: iptables
 - include_role:
    name: install-ansible-roles
 - include_role:
--- a/playbooks/service-backup.yaml
+++ b/playbooks/service-backup.yaml
@ -3,8 +3,10 @@
 - hosts: "backup:!disabled"
  name: "Base: Generate backup users and keys"
  roles:
+    - iptables
    - backup
 - hosts: "backup-server:!disabled"
  name: "Generate bup configuration"
  roles:
+    - iptables
    - backup-server
--- a/playbooks/service-bridge.yaml
+++ b/playbooks/service-bridge.yaml
@ -1,6 +1,7 @@
 - hosts: bridge.openstack.org:!disabled
  name: "Bridge: configure the bastion host"
  roles:
+    - iptables
    - edit-secrets-script
    - install-docker
  tasks:
--- a/playbooks/service-codesearch.yaml
+++ b/playbooks/service-codesearch.yaml
@ -2,6 +2,7 @@
  name: "codesearch: run puppet on codesearch"
  strategy: free
  roles:
+    - iptables
    - sync-project-config
    - name: run-puppet
      manifest: /opt/system-config/production/manifests/codesearch.pp
--- a/playbooks/service-eavesdrop.yaml
+++ b/playbooks/service-eavesdrop.yaml
@ -2,6 +2,7 @@
  name: "eavesdrop: run puppet on eavesdrop"
  strategy: free
  roles:
+    - iptables
    - zuul-user
    - sync-project-config
    - install-docker
--- a/playbooks/service-etherpad.yaml
+++ b/playbooks/service-etherpad.yaml
@ -1,5 +1,6 @@
 - hosts: "etherpad01.opendev.org:!disabled"
  name: "Base: configure etherpad"
  roles:
+    - iptables
    - install-docker
    - etherpad
--- a/playbooks/service-gitea-lb.yaml
+++ b/playbooks/service-gitea-lb.yaml
@ -1,5 +1,6 @@
 - hosts: "gitea-lb:!disabled"
  name: "Base: configure gitea load balancer"
  roles:
+    - iptables
    - install-docker
    - haproxy
--- a/playbooks/service-gitea.yaml
+++ b/playbooks/service-gitea.yaml
@ -2,5 +2,6 @@
  name: "Base: configure gitea"
  serial: 1
  roles:
+    - iptables
    - install-docker
    - gitea
--- a/playbooks/service-letsencrypt.yaml
+++ b/playbooks/service-letsencrypt.yaml
@ -5,7 +5,7 @@
  roles:
    - install-certcheck
 - hosts: "letsencrypt:!disabled"
-  name: "Base: deploy and renew certificates"
+  name: "Deploy and renew certificates"
  roles:
    - letsencrypt-acme-sh-install
    - letsencrypt-request-certs
--- a/playbooks/service-meetpad.yaml
+++ b/playbooks/service-meetpad.yaml
@ -1,12 +1,14 @@
 - hosts: "meetpad:!disabled"
  name: "Configure meetpad"
  roles:
+    - iptables
    - install-docker
    - jitsi-meet

 - hosts: "jvb:!disabled"
  name: "Configure extra jitsi video bridges"
  roles:
+    - iptables
    - install-docker
    - role: jitsi-meet
      docker_compose_file: jvb-docker-compose.yaml
--- a/playbooks/service-mirror-update.yaml
+++ b/playbooks/service-mirror-update.yaml
@ -1,6 +1,7 @@
 - hosts: "mirror-update:!disabled"
  name: "Configure mirror-update"
  roles:
+    - role: iptables
    - role: kerberos-client
      kerberos_realm: 'OPENSTACK.ORG'
      kerberos_admin_server: 'kdc.openstack.org'
--- a/playbooks/service-mirror.yaml
+++ b/playbooks/service-mirror.yaml
@ -1,6 +1,7 @@
 - hosts: "mirror:!disabled"
  name: "Configure per region opendev mirrors"
  roles:
+    - role: iptables
    - role: kerberos-client
      kerberos_realm: 'OPENSTACK.ORG'
      kerberos_admin_server: 'kdc.openstack.org'
--- a/playbooks/service-nameserver.yaml
+++ b/playbooks/service-nameserver.yaml
@ -1,10 +1,12 @@
 - hosts: adns:!disabled
  name: "Base: configure adns server"
  roles:
+    - iptables
    - master-nameserver

 - hosts: "ns1.opendev.org:ns2.opendev.org:!disabled"
  name: "Base: configure authoritative nameservers"
  roles:
+    - iptables
    - nameserver

--- a/playbooks/service-nodepool.yaml
+++ b/playbooks/service-nodepool.yaml
@ -2,6 +2,7 @@
  name: "Configure nodepool builders"
  strategy: free
  roles:
+    - iptables
    - install-docker
    - nodepool-base
    - configure-openstacksdk
@ -11,6 +12,7 @@
  name: "run puppet on all older servers"
  strategy: free
  roles:
+    - iptables
    - nodepool-base-legacy
    - configure-openstacksdk
    - configure-kubectl
@ -20,6 +22,7 @@
  name: "Configure nodepool launchers"
  strategy: free
  roles:
+    - iptables
    - install-docker
    - nodepool-base
    - configure-openstacksdk
--- a/playbooks/service-registry.yaml
+++ b/playbooks/service-registry.yaml
@ -1,5 +1,6 @@
 - hosts: "registry:!disabled"
  name: "Base: configure registry"
  roles:
+    - iptables
    - install-docker
    - registry
--- a/playbooks/service-review-dev.yaml
+++ b/playbooks/service-review-dev.yaml
@ -1,6 +1,7 @@
 - hosts: "review-dev:!disabled"
  name: "Configure gerrit on review-dev"
  roles:
+    - iptables
    - install-docker
    - role: gerrit
      gerrit_ssh_rsa_key_contents: "{{ gerrit_dev_ssh_rsa_key_contents }}"
--- a/playbooks/service-review.yaml
+++ b/playbooks/service-review.yaml
@ -1,5 +1,6 @@
 - hosts: "review:!disabled"
  name: "Configure gerrit"
  roles:
+    - iptables
    - install-docker
    - gerrit
--- a/playbooks/service-static.yaml
+++ b/playbooks/service-static.yaml
@ -1,6 +1,7 @@
 - hosts: "static:!disabled"
  name: "Static webserver"
  roles:
+    - role: iptables
    - role: kerberos-client
      kerberos_realm: 'OPENSTACK.ORG'
      kerberos_admin_server: 'kdc.openstack.org'
--- a/playbooks/service-zookeeper.yaml
+++ b/playbooks/service-zookeeper.yaml
@ -12,5 +12,6 @@
  name: "Configure Zookeeper"
  serial: 1
  roles:
+    - iptables
    - install-docker
    - zookeeper
--- a/playbooks/service-zuul-preview.yaml
+++ b/playbooks/service-zuul-preview.yaml
@ -1,5 +1,6 @@
 - hosts: "zuul-preview:!disabled"
  name: "Base: configure zuul-preview"
  roles:
+    - iptables
    - install-docker
    - zuul-preview
--- a/playbooks/service-zuul.yaml
+++ b/playbooks/service-zuul.yaml
@ -11,6 +11,7 @@
 - hosts: "zuul:!disabled"
  name: "Configure zuul servers"
  roles:
+    - iptables
    - install-docker
    - zuul

--- a/testinfra/test_base.py
+++ b/testinfra/test_base.py
@ -14,7 +14,6 @@

 import socket

-
 testinfra_hosts = ['all']


--- a/testinfra/test_zuul.py
+++ b/testinfra/test_zuul.py
@ -0,0 +1,73 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import socket
+
+
+testinfra_hosts = [
+    'ze01.opendev.org',
+    'zm01.openstack.org',
+    'zuul01.openstack.org',
+]
+
+
+def get_ips(value, family=None):
+    ret = set()
+    try:
+        addr_info = socket.getaddrinfo(value, None, family)
+    except socket.gaierror:
+        return ret
+    for addr in addr_info:
+        ret.add(addr[4][0])
+    return ret
+
+
+def test_iptables(host):
+    rules = host.iptables.rules()
+    rules = [x.strip() for x in rules]
+
+    needed_rules = [
+        '-P INPUT ACCEPT',
+        '-P FORWARD DROP',
+        '-P OUTPUT ACCEPT',
+        '-N openstack-INPUT',
+        '-A INPUT -j openstack-INPUT',
+        '-A openstack-INPUT -i lo -j ACCEPT',
+        '-A openstack-INPUT -p icmp -m icmp --icmp-type any -j ACCEPT',
+        '-A openstack-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT',
+        '-A openstack-INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT',
+        '-A openstack-INPUT -j REJECT --reject-with icmp-host-prohibited'
+    ]
+    for rule in needed_rules:
+        assert rule in rules
+
+    # Make sure that the gearman port is open to executors on the scheduler
+    if host.backend.get_hostname() == 'zuul01.openstack.org':
+        for ip in get_ips('ze01.opendev.org', socket.AF_INET):
+            zuul = ('-A openstack-INPUT -s %s/32 -p tcp -m state --state NEW'
+                    ' -m tcp --dport 4730 -j ACCEPT' % ip)
+            assert zuul in rules
+
+    # Ensure all IPv4+6 addresses for cacti are allowed
+    for ip in get_ips('cacti.openstack.org', socket.AF_INET):
+        snmp = ('-A openstack-INPUT -s %s/32 -p udp -m udp'
+                ' --dport 161 -j ACCEPT' % ip)
+        assert snmp in rules
+
+    # TODO(ianw) add ip6tables support to testinfra iptables module
+    ip6rules = host.check_output('ip6tables -S')
+    for ip in get_ips('cacti.openstack.org', socket.AF_INET6):
+        snmp = ('-A openstack-INPUT -s %s/128 -p udp -m udp'
+                ' --dport 161 -j ACCEPT' % ip)
+        assert snmp in ip6rules
--- a/zuul.d/infra-prod.yaml
+++ b/zuul.d/infra-prod.yaml
@ -69,8 +69,6 @@
    dependencies:
      - name: infra-prod-install-ansible
        soft: true
-      - name: infra-prod-base
-        soft: true
    files:
      - inventory/
      - playbooks/service-letsencrypt.yaml
@ -105,8 +103,6 @@
    dependencies:
      - name: infra-prod-install-ansible
        soft: true
-      - name: infra-prod-base
-        soft: true
      - name: infra-prod-service-letsencrypt
        soft: true

@ -120,6 +116,7 @@
      - inventory/
      - playbooks/service-bridge.yaml
      - playbooks/host_vars/bridge.openstack.org.yaml
+      - playbooks/roles/iptables/
      - playbooks/roles/logrotate/
      - playbooks/roles/edit-secrets-script/
      - playbooks/roles/install-kubectl/
@ -138,6 +135,7 @@
      - playbooks/service-gitea-lb.yaml
      - playbooks/group_vars/gitea-lb.yaml
      - playbooks/roles/pip3/
+      - playbooks/roles/iptables/
      - playbooks/roles/install-docker/
      - playbooks/roles/haproxy/

@ -157,6 +155,7 @@
      - playbooks/group_vars/ns.yaml
      - playbooks/roles/master-nameserver/
      - playbooks/roles/nameserver/
+      - playbooks/roles/iptables/

 - job:
    name: infra-prod-service-nodepool
@ -179,6 +178,7 @@
      - playbooks/roles/configure-kubectl/
      - playbooks/roles/configure-openstacksdk/
      - playbooks/roles/install-docker/
+      - playbooks/roles/iptables/
      - playbooks/roles/nodepool
      - playbooks/templates/clouds/nodepool_

@ -197,6 +197,7 @@
      - playbooks/roles/pip3/
      - playbooks/roles/etherpad
      - playbooks/roles/logrotate
+      - playbooks/roles/iptables/

 - job:
    name: infra-prod-service-meetpad
@ -205,8 +206,6 @@
    dependencies:
      - name: infra-prod-install-ansible
        soft: true
-      - name: infra-prod-base
-        soft: true
      - name: infra-prod-service-letsencrypt
        soft: true
      - name: system-config-promote-image-jitsi-meet
@ -220,6 +219,7 @@
      - playbooks/group_vars/meetpad.yaml
      - playbooks/roles/pip3/
      - playbooks/roles/install-docker/
+      - playbooks/roles/iptables/
      - playbooks/roles/jitsi-meet/

 - job:
@ -234,6 +234,7 @@
      - playbooks/roles/kerberos-client/
      - playbooks/roles/openafs-client/
      - playbooks/roles/mirror-update/
+      - playbooks/roles/iptables/
      - playbooks/roles/logrotate/

 - job:
@ -251,6 +252,7 @@
      - playbooks/roles/mirror/
      - playbooks/roles/afs-release/
      - playbooks/roles/afsmon/
+      - playbooks/roles/iptables/
      - playbooks/roles/logrotate/

 - job:
@ -264,6 +266,7 @@
      - playbooks/service-static.yaml
      - playbooks/host_vars/static01.opendev.org.yaml
      - playbooks/group_vars/static.yaml
+      - playbooks/roles/iptables/
      - playbooks/roles/kerberos-client/
      - playbooks/roles/openafs-client/
      - playbooks/roles/static/
@ -280,6 +283,7 @@
      - playbooks/service-backup.yaml
      - playbooks/roles/backup/
      - playbooks/roles/backup-server/
+      - playbooks/roles/iptables/

 - job:
    name: infra-prod-service-registry
@ -293,6 +297,7 @@
      - playbooks/group_vars/registry.yaml
      - playbooks/roles/pip3/
      - playbooks/roles/install-docker/
+      - playbooks/roles/iptables/
      - playbooks/roles/registry/

 - job:
@ -307,6 +312,7 @@
      - playbooks/group_vars/zuul-preview.yaml
      - playbooks/roles/pip3/
      - playbooks/roles/install-docker/
+      - playbooks/roles/iptables/
      - playbooks/roles/zuul-preview/

 - job:
@ -321,6 +327,7 @@
      - ^playbooks/host_vars/zk\d+\..*
      - playbooks/roles/pip3/
      - playbooks/roles/install-docker/
+      - playbooks/roles/iptables/
      - playbooks/roles/zookeeper/

 - job:
@ -337,8 +344,6 @@
    dependencies:
      - name: infra-prod-install-ansible
        soft: true
-      - name: infra-prod-base
-        soft: true
      - name: infra-prod-service-letsencrypt
        soft: true
      - name: infra-prod-manage-projects
@ -352,6 +357,7 @@
      - playbooks/host_vars/zk\d+
      - playbooks/host_vars/zuul01.openstack.org
      - playbooks/roles/install-docker/
+      - playbooks/roles/iptables/
      - playbooks/roles/zookeeper/
      - playbooks/roles/zuul

@ -364,8 +370,6 @@
    dependencies: &infra_prod_service_review_deps
      - name: infra-prod-install-ansible
        soft: true
-      - name: infra-prod-base
-        soft: true
      - name: infra-prod-service-letsencrypt
        soft: true
      - name: system-config-promote-image-gerrit-2.13
@ -377,6 +381,7 @@
      - playbooks/host_vars/review01.openstack.org.yaml
      - playbooks/roles/pip3/
      - playbooks/roles/install-docker/
+      - playbooks/roles/iptables/
      - playbooks/roles/gerrit/

 - job:
@ -393,6 +398,7 @@
      - playbooks/host_vars/review-dev01.opendev.org.yaml
      - playbooks/roles/pip3/
      - playbooks/roles/install-docker/
+      - playbooks/roles/iptables/
      - playbooks/roles/gerrit/

 - job:
@ -404,8 +410,6 @@
    dependencies:
      - name: infra-prod-install-ansible
        soft: true
-      - name: infra-prod-base
-        soft: true
      - name: infra-prod-service-letsencrypt
        soft: true
      - name: system-config-promote-image-gitea-init
@ -420,6 +424,7 @@
      - playbooks/roles/install-docker/
      - playbooks/roles/pip3/
      - playbooks/roles/gitea/
+      - playbooks/roles/iptables/
      - playbooks/roles/logrotate/
      - docker/gitea/
      - docker/gitea-init/
@ -443,6 +448,7 @@
      - playbooks/group_vars/puppet.yaml
      - playbooks/roles/run-puppet/
      - playbooks/roles/install-ansible-roles/
+      - playbooks/roles/iptables/
      - playbooks/roles/sync-project-config
      - playbooks/roles/puppet-install/
      - playbooks/roles/disable-puppet-agent/
@ -461,8 +467,6 @@
    dependencies:
      - name: infra-prod-install-ansible
        soft: true
-      - name: infra-prod-base
-        soft: true
      - name: infra-prod-service-letsencrypt
        soft: true
      - name: system-config-promote-image-accessbot
@ -479,6 +483,7 @@
      - playbooks/roles/install-ansible-roles/
      - playbooks/roles/zuul-user
      - playbooks/roles/install-docker
+      - playbooks/roles/iptables/
      - playbooks/roles/puppet-install/
      - playbooks/roles/disable-puppet-agent/
      - playbooks/roles/accessbot
@ -526,6 +531,7 @@
      - playbooks/roles/install-ansible-roles/
      - playbooks/roles/puppet-install/
      - playbooks/roles/disable-puppet-agent/
+      - playbooks/roles/iptables/
      - playbooks/roles/vos-release/
      - modules/
      - manifests/
@ -551,6 +557,7 @@
      - playbooks/roles/install-ansible-roles/
      - playbooks/roles/puppet-install/
      - playbooks/roles/disable-puppet-agent/
+      - playbooks/roles/iptables/
      - modules/
      - manifests/

--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@ -204,8 +204,6 @@
            dependencies:
              - name: infra-prod-install-ansible
                soft: true
-              - name: infra-prod-base
-                soft: true
              - name: infra-prod-service-letsencrypt
                soft: true
              - name: system-config-promote-image-etherpad
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@ -374,6 +374,7 @@
 - job:
    name: system-config-run-mirror-x86
    parent: system-config-run-mirror-base
+    timeout: 3600
    nodeset:
      nodes:
        - name: bridge.openstack.org