Test the rename_repos playbook

It would be useful to test our rename playbook against gitea and gerrit
when we make changes to these related playbooks, roles, and docker
images. To do this we need to converge our test and production setups
for gerrit a bit more. We create an openstack-project-creator account in
the test gerrit to match prod and we have rename_repos.yaml talk to
localhost for gerrit ssh commands.

With that done we can run the rename_repos.yaml playbook from
test-gitea.yaml and test-gerrit.yaml to help ensure the playbook
functions as expected against these services.

Co-Authored-By: Ian Wienand <iwienand@redhat.com>
Change-Id: I49ffaf86828e87705da303f40ad4a86be030c709

playbooks/rename_repos.yaml

@@ -130,13 +130,20 @@
 
     - include_vars: "{{ repolist }}"
 
-    # TODO: Replace this with a wait_for?
-    - pause:
-        prompt: "Make sure that Gerrit ssh api is accepting requests. Use Enter or ^C c to continue. ^C a to abort."
+    - name: Wait for gerrit ssh to be listening
+      wait_for:
+        host: localhost
+        port: 29418
+        # Far longer than necessary
+        timeout: 1800
+
+    - name: Sleep a bit longer just to be sure it is up
+      wait_for:
+        timeout: 30
 
     - name: Set base ssh command fact
       set_fact:
-        gerrit_ssh_command: ssh -p 29418 -i /home/gerrit2/review_site/etc/ssh_project_rsa_key openstack-project-creator@review.opendev.org gerrit
+        gerrit_ssh_command: ssh -p 29418 -i /home/gerrit2/review_site/etc/ssh_project_rsa_key openstack-project-creator@localhost gerrit
 
     - name: Rename groups in gerrit
       shell: "{{ gerrit_ssh_command }} rename-group {{ item.old }} {{ item.new }}"

playbooks/test-gitea.yaml

@@ -21,8 +21,16 @@
         cd /home/zuul/src/opendev.org/opendev/system-config
         git remote add test-gitea https://gerrit:{{ gitea_gerrit_password }}@localhost:3000/opendev/system-config
         GIT_SSL_NO_VERIFY=1 git push -f test-gitea master
-    # This is conveniently left here so that it can be uncommented in order to
-    # autohold the system-config-run-gitea job in zuul.
-    #- name: Force a failure for human intervention
-    #  fail:
-    #    msg: Failing so that we can check on things with a hold
+
+- name: Run rename playbook
+  import_playbook: rename_repos.yaml
+  vars:
+    repolist: /home/zuul/src/opendev.org/opendev/system-config/playbooks/zuul/test_gitea_renames.yaml
+
+# This is conveniently left here so that it can be uncommented in order to
+# autohold the system-config-run-gitea job in zuul.
+#- hosts: bridge.openstack.org
+#  tasks:
+#    - name: Force a failure for human intervention
+#      fail:
+#        msg: Failing so that we can check on things with a hold

playbooks/test-review.yaml

@@ -12,6 +12,22 @@
       delay: 1
       retries: 120
 
+    - name: Create openstack-project-creator user
+      uri:
+        url: http://localhost:8081/a/accounts/openstack-project-creator
+        method: PUT
+        user: admin
+        password: secret
+        body_format: json
+        body:
+          username: 'openstack-project-creator'
+          name: 'Project Creator'
+          email: 'project.creator@example.com'
+          http_password: 'secret'
+          groups:
+            - Administrators
+        status_code: 201
+
     - name: Create CI group
       uri:
         url: 'http://localhost:8081/a/groups/CI-tools'
@@ -105,6 +121,10 @@
 
           popd
 
+          # openstack-project-creator bootstrapping
+          curl -X POST --user "openstack-project-creator:secret" -H "Content-Type: text/plain" -d@/home/gerrit2/review_site/etc/ssh_project_rsa_key.pub http://localhost:8081/a/accounts/openstack-project-creator/sshkeys
+          ssh-keyscan -p 29418 localhost >> /home/gerrit2/.ssh/known_hosts
+
     # This is helpful on a held node when you're trying to fix/enhance
     # the Zuul summary plugin.  You can build it locally, scp the new
     # .jar to /tmp and run this to deploy your changes.
@@ -133,7 +153,7 @@
 
     - name: Create project project in Gerrit
       uri:
-        url: http://localhost:8081/a/projects/x%2Ftest-project
+        url: http://localhost:8081/a/projects/y%2Ftest-project
         method: PUT
         user: admin
         password: secret
@@ -151,11 +171,11 @@
           [gerrit]
           host=localhost
           port=29418
-          project=x/test-project
+          project=y/test-project
           EOF
           git add .gitreview
           git commit -m "Initial commit"
-          git remote add gerrit http://admin:secret@localhost:8081/x/test-project
+          git remote add gerrit http://admin:secret@localhost:8081/y/test-project
           git push -f --set-upstream gerrit +HEAD:master
 
           curl -Lo .git/hooks/commit-msg http://localhost:8081/tools/hooks/commit-msg
@@ -238,3 +258,37 @@
       include_role:
         name: run-selenium
 
+- name: Run rename playbook
+  import_playbook: rename_repos.yaml
+  vars:
+    repolist: /home/zuul/src/opendev.org/opendev/system-config/playbooks/zuul/test_gerrit_renames.yaml
+
+- hosts: "review"
+  tasks:
+    - name: Get group UUID of renamed group
+      uri:
+        url: 'http://localhost:8081/a/groups/CI-tools-updated'
+        method: GET
+        user: admin
+        password: secret
+        status_code: 200
+        return_content: yes
+      register: group_renamed_raw
+
+    - name: Debug serialized json
+      debug:
+        var: group_renamed_raw
+
+    - name: Deserialize returned data to internal variable.
+      set_fact:
+        # The first 4 bytes of the returned data are )]}' which is
+        # invalid json.
+        group_renamed_json: '{{ group_renamed_raw.content | regex_replace("^....", "") | from_json }}'
+
+    - name: Debug deserialized json
+      debug:
+        var: group_renamed_json
+
+    - name: Check renamed UUID is consistent
+      assert:
+        that: group_renamed_json['id'] == group_json['id']

playbooks/zuul/test_gerrit_renames.yaml

@@ -0,0 +1,6 @@
+repos:
+  - old: y/test-project
+    new: x/test-project
+gerrit_groups:
+  - old: CI-tools
+    new: CI-tools-updated

playbooks/zuul/test_gitea_renames.yaml

@@ -0,0 +1,6 @@
+repos:
+  - old: openstack/diskimage-builder
+    new: opendev/diskimage-builder
+gerrit_groups:
+  - old: CI-tools
+    new: CI-tools-updated

testinfra/test_gitea.py

@@ -61,3 +61,12 @@ def test_ondisk_logs(host):
     gitea_ssh_log = host.file('/var/log/containers/docker-gitea-ssh.log')
     assert gitea_ssh_log.exists
     assert gitea_ssh_log.contains("Server listening on :: port 222.")
+
+def test_project_clone(host):
+    # Note this tests the result of a project rename in gitea as well.
+    cmd = host.run(
+        'GIT_SSL_NO_VERIFY=1 '
+        'git clone https://localhost:3081/opendev/diskimage-builder '
+        '/tmp/diskimage-builder')
+    assert "Cloning into '/tmp/diskimage-builder'..." in cmd.stderr
+    assert cmd.succeeded

zuul.d/system-config-run.yaml

@@ -568,6 +568,7 @@
       - playbooks/service-gitea.yaml
       - playbooks/manage-projects.yaml
       - playbooks/test-gitea.yaml
+      - playbooks/rename_repos.yaml
       - inventory/service/group_vars/gitea.yaml
       - inventory/service/group_vars/gitea-lb.yaml
       - inventory/service/host_vars/gitea
@@ -864,6 +865,7 @@
       - playbooks/letsencrypt.yaml
       - playbooks/service-review.*.yaml
       - playbooks/test-review.yaml
+      - playbooks/rename_repos.yaml
       - inventory/service/group_vars/review.yaml
       - inventory/service/host_vars/review\d+.openstack.org.yaml
       - playbooks/roles/pip3/