Base 2.13 image on gerrit-base

We install jeepyb and launchpadlib in gerrit-base. Those are
important. We also need to add cgi for gitweb.

The gerrit init command does two things that we don't actually
want it to do at runtime. It extracts the plugins into the
plugins dir, and it downloads the right database library.

We can extract the plugins for it during image creation, and
then we can also download the plugin it would have downloaded.

We can also download the mysql library for it:

https://gerrit.googlesource.com/gerrit/+/refs/heads/stable-2.13/gerrit-pgm/src/main/resources/com/google/gerrit/pgm/init/libraries.config

Finally, we tell it to not download or expand anything during
init, because we're running in a container and next time we run
the process that dir isn't going to be there.

Our gerrit integration tests don't depend on our gerrit image builds.
Put in image depends between run-review and gerrit builds.

We also need to depend directly on opendev-buildset-registry.

Add java.security.egd setting to java invocation

This tells java to be secure.

https://stackoverflow.com/questions/58991966/what-java-security-egd-option-is-for

Add support for setting heap limit properly

The gerrit init script does this based on the value in
container.javaOptions. We could, but then we'd have to
run an entrypoint script. Instead, set the value via
the JAVA_OPTIONS env var setting based on a value from
ansible.

Finally, make gerrit-master image build non-voting

It looks like there might be a real issue, but debugging that
is not important for us at this moment.

Depends-On: https://review.opendev.org/714216
Change-Id: I01e94c10f470fb3c8ddfce7b0e201357e5050679

.zuul.yaml

@@ -177,6 +177,8 @@
     name: system-config-build-image-gerrit-2.13
     description: Build a gerrit 2.13 image.
     parent: system-config-build-image
+    requires: gerrit-base-container-image
+    provides: gerrit-2.13-container-image
     vars: &gerrit_vars_2_13
       docker_images:
         # The 2.13 image doesn't build from source, but from existing war file
@@ -186,12 +188,15 @@
           tags:
             - 2.13
     files: &gerrit_files_2_13
+      - docker/gerrit/base/.*
       - docker/gerrit/2.13/.*
 
 - job:
     name: system-config-upload-image-gerrit-2.13
     description: Build and upload a gerrit 2.13 image.
     parent: system-config-upload-image
+    requires: gerrit-base-container-image
+    provides: gerrit-2.13-container-image
     vars: *gerrit_vars_2_13
     files: *gerrit_files_2_13
 
@@ -210,6 +215,7 @@
     pre-run: playbooks/zuul/gerrit/repos.yaml
     run: playbooks/zuul/gerrit/run.yaml
     requires: gerrit-base-container-image
+    provides: gerrit-2.15-container-image
     required-projects: &gerrit_projects_2_15
       - name: gerrit.googlesource.com/gerrit
         override-checkout: stable-2.15
@@ -251,6 +257,7 @@
     pre-run: playbooks/zuul/gerrit/repos.yaml
     run: playbooks/zuul/gerrit/run.yaml
     requires: gerrit-base-container-image
+    provides: gerrit-2.15-container-image
     required-projects: *gerrit_projects_2_15
     vars: *gerrit_vars_2_15
     files: *gerrit_files_2_15
@@ -270,6 +277,7 @@
     pre-run: playbooks/zuul/gerrit/repos.yaml
     run: playbooks/zuul/gerrit/run.yaml
     requires: gerrit-base-container-image
+    provides: gerrit-2.16-container-image
     required-projects: &gerrit_projects_2_16
       - name: gerrit.googlesource.com/gerrit
         override-checkout: stable-2.16
@@ -315,6 +323,7 @@
     pre-run: playbooks/zuul/gerrit/repos.yaml
     run: playbooks/zuul/gerrit/run.yaml
     requires: gerrit-base-container-image
+    provides: gerrit-2.16-container-image
     required-projects: *gerrit_projects_2_16
     vars: *gerrit_vars_2_16
     files: *gerrit_files_2_16
@@ -334,6 +343,7 @@
     pre-run: playbooks/zuul/gerrit/repos.yaml
     run: playbooks/zuul/gerrit/run.yaml
     requires: gerrit-base-container-image
+    provides: gerrit-3.0-container-image
     required-projects: &gerrit_projects_3_0
       - name: gerrit.googlesource.com/gerrit
         override-checkout: stable-3.0
@@ -391,6 +401,7 @@
     pre-run: playbooks/zuul/gerrit/repos.yaml
     run: playbooks/zuul/gerrit/run.yaml
     requires: gerrit-base-container-image
+    provides: gerrit-3.0-container-image
     required-projects: *gerrit_projects_3_0
     vars: *gerrit_vars_3_0
     files: *gerrit_files_3_0
@@ -415,6 +426,7 @@
       - playbooks/zuul/gerrit/submodules.yaml
     run: playbooks/zuul/gerrit/run.yaml
     requires: gerrit-base-container-image
+    provides: gerrit-master-container-image
     required-projects: &gerrit_projects_master
       - opendev/system-config
       - gerrit.googlesource.com/jgit
@@ -473,6 +485,7 @@
       - playbooks/zuul/gerrit/submodules.yaml
     run: playbooks/zuul/gerrit/run.yaml
     requires: gerrit-base-container-image
+    provides: gerrit-master-container-image
     required-projects: *gerrit_projects_master
     vars: *gerrit_vars_master
     files: *gerrit_files_master
@@ -1087,6 +1100,7 @@
     parent: system-config-run
     description: |
       Run the playbook for gerrit (in a container).
+    requires: gerrit-2.13-container-image
     nodeset:
       nodes:
         - name: bridge.openstack.org
@@ -1325,7 +1339,11 @@
                 soft: true
               - name: system-config-build-image-haproxy-statsd
                 soft: true
-        - system-config-run-review
+        - system-config-run-review:
+            dependencies:
+              - name: opendev-buildset-registry
+              - name: system-config-build-image-gerrit-2.13
+                soft: true
         - system-config-run-zuul-preview
         - system-config-run-letsencrypt
         - system-config-build-image-bazel
@@ -1340,29 +1358,41 @@
         - system-config-build-image-gitea
         - system-config-build-image-gerrit-base:
             dependencies:
+              - name: opendev-buildset-registry
               - name: system-config-build-image-python-builder
                 soft: true
-        - system-config-build-image-gerrit-2.13
+        - system-config-build-image-gerrit-2.13:
+            dependencies:
+              - name: opendev-buildset-registry
+              - name: system-config-build-image-python-builder
+                soft: true
+              - name: system-config-build-image-gerrit-base
+                soft: true
         - system-config-build-image-gerrit-2.15:
             dependencies:
+              - name: opendev-buildset-registry
               - name: system-config-build-image-bazel
                 soft: true
               - name: system-config-build-image-gerrit-base
                 soft: true
         - system-config-build-image-gerrit-2.16:
             dependencies:
+              - name: opendev-buildset-registry
               - name: system-config-build-image-bazel
                 soft: true
               - name: system-config-build-image-gerrit-base
                 soft: true
         - system-config-build-image-gerrit-3.0:
             dependencies:
+              - name: opendev-buildset-registry
               - name: system-config-build-image-bazel
                 soft: true
               - name: system-config-build-image-gerrit-base
                 soft: true
         - system-config-build-image-gerrit-master:
+            voting: false
             dependencies:
+              - name: opendev-buildset-registry
               - name: system-config-build-image-bazel
                 soft: true
               - name: system-config-build-image-gerrit-base
@@ -1398,7 +1428,11 @@
                 soft: true
               - name: system-config-upload-image-haproxy-statsd
                 soft: true
-        - system-config-run-review
+        - system-config-run-review:
+            dependencies:
+              - name: opendev-buildset-registry
+              - name: system-config-upload-image-gerrit-2.13
+                soft: true
         - system-config-run-zuul-preview
         - system-config-run-letsencrypt
         - system-config-upload-image-bazel
@@ -1413,29 +1447,33 @@
         - system-config-upload-image-gitea
         - system-config-upload-image-gerrit-base:
             dependencies:
+              - name: opendev-buildset-registry
               - name: system-config-upload-image-python-builder
                 soft: true
-        - system-config-upload-image-gerrit-2.13
+        - system-config-upload-image-gerrit-2.13:
+            dependencies:
+              - name: opendev-buildset-registry
+              - name: system-config-upload-image-python-builder
+                soft: true
+              - name: system-config-upload-image-gerrit-base
+                soft: true
         - system-config-upload-image-gerrit-2.15:
             dependencies:
+              - name: opendev-buildset-registry
               - name: system-config-upload-image-bazel
                 soft: true
               - name: system-config-upload-image-gerrit-base
                 soft: true
         - system-config-upload-image-gerrit-2.16:
             dependencies:
+              - name: opendev-buildset-registry
               - name: system-config-upload-image-bazel
                 soft: true
               - name: system-config-upload-image-gerrit-base
                 soft: true
         - system-config-upload-image-gerrit-3.0:
             dependencies:
-              - name: system-config-upload-image-bazel
+              - name: opendev-buildset-registry
-                soft: true
-              - name: system-config-upload-image-gerrit-base
-                soft: true
-        - system-config-upload-image-gerrit-master:
-            dependencies:
               - name: system-config-upload-image-bazel
                 soft: true
               - name: system-config-upload-image-gerrit-base
@@ -1457,7 +1495,6 @@
         - system-config-promote-image-gerrit-2.15
         - system-config-promote-image-gerrit-2.16
         - system-config-promote-image-gerrit-3.0
-        - system-config-promote-image-gerrit-master
         - system-config-promote-image-haproxy-statsd
         - system-config-promote-image-python-base
         - system-config-promote-image-python-builder

docker/gerrit/2.13/Dockerfile

@@ -13,56 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM docker.io/library/openjdk:8
+FROM docker.io/opendevorg/gerrit-base
-
-# It's not 100% clear that unzip and libmysql-java are needed
-RUN apt-get update \
-  && apt-get install -y dumb-init wget unzip libmysql-java \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/*
-
-# 3000 is what the existing opendev gerrit2 user is
-RUN addgroup gerrit --gid 3000 --system \
-  && adduser \
-    --uid 3000 \
-    --system \
-    --home /var/gerrit \
-    --shell /bin/bash \
-    --ingroup gerrit \
-    gerrit
-
-USER gerrit
 
 # Download the gerrit war
-RUN mkdir /var/gerrit/bin \
+RUN wget https://tarballs.openstack.org/gerrit/gerrit-v2.13.12.11.1707fec.war -O /var/gerrit/bin/gerrit.war
-  && mkdir /var/gerrit/hooks \
-  && mkdir /var/gerrit/static \
-  && wget https://tarballs.openstack.org/gerrit/gerrit-v2.13.12.11.1707fec.war -O /var/gerrit/bin/gerrit.war
 
 # Install plugins
 RUN mkdir /var/gerrit/plugins && \
   wget https://tarballs.openstack.org/ci/gerrit/plugins/javamelody/javamelody-v2.13.3.e4233d6.jar -O /var/gerrit/plugins/javamelody.jar && \
-  wget https://tarballs.openstack.org/ci/gerrit/plugins/its-storyboard/its-storyboard-805f9ac.jar -O /var/gerrit/plugins/its-storyboard.jar
+  wget https://tarballs.openstack.org/ci/gerrit/plugins/its-storyboard/its-storyboard-805f9ac.jar -O /var/gerrit/plugins/its-storyboard.jar && \
+  unzip -jo /var/gerrit/bin/gerrit.war WEB-INF/plugins/* -d /var/gerrit/plugins
 
-# Force gerrit to use bouncycastle for security things.
+# Gerrit 2.13 needs bouncy castle
-# Also use the distro-provided mysql-connector.
+RUN \
-RUN mkdir /var/gerrit/lib && \
-  unzip -jo /var/gerrit/bin/gerrit.war WEB-INF/plugins/* -d /var/gerrit/plugins && \
   wget https://repo1.maven.org/maven2/org/bouncycastle/bcprov-jdk15on/1.52/bcprov-jdk15on-1.52.jar -O /var/gerrit/lib/bcprov-1.52.jar && \
-  wget https://repo1.maven.org/maven2/org/bouncycastle/bcpkix-jdk15on/1.52/bcpkix-jdk15on-1.52.jar -O /var/gerrit/lib/bcpkix-1.52.jar && \
+  wget https://repo1.maven.org/maven2/org/bouncycastle/bcpkix-jdk15on/1.52/bcpkix-jdk15on-1.52.jar -O /var/gerrit/lib/bcpkix-1.52.jar
-  ln -s /usr/share/java/mysql-connector-java.jar /var/gerrit/lib/mysql-connector-java.jar
-
-# Allow incoming traffic
-EXPOSE 29418 8080
-
-VOLUME /var/gerrit/git /var/gerrit/index /var/gerrit/cache /var/gerrit/db /var/gerrit/etc /var/log/gerrit /var/gerrit/tmp
-
-RUN ln -s /var/log/gerrit /var/gerrit/logs
-
-# container.javaOptions
-# Also include container.heapLimit - but with -Xmx prefixing it
-ENV JAVA_OPTIONS ""
-
-# Ulimits should be set on command line or in docker-compose.yaml
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-CMD /usr/local/openjdk-8/bin/java ${JAVA_OPTIONS} -jar /var/gerrit/bin/gerrit.war daemon -d /var/gerrit

docker/gerrit/base/Dockerfile

@@ -20,8 +20,10 @@ RUN assemble
 
 FROM docker.io/library/openjdk:8
 
+# libcgi-pm-perl is for gitweb
 RUN apt-get update \
   && apt-get install -y dumb-init python3-launchpadlib python3-distutils \
+      wget unzip libcgi-pm-perl \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
   && curl https://bootstrap.pypa.io/get-pip.py > /tmp/get-pip.py \
@@ -46,6 +48,11 @@ RUN mkdir /var/gerrit/bin \
   && mkdir /var/gerrit/hooks \
   && mkdir /var/gerrit/static
 
+# Force gerrit to use bouncycastle for security things.
+# Download mysql-connector so that gerrit doens't download it during init.
+RUN mkdir /var/gerrit/lib && \
+  wget https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.43/mysql-connector-java-5.1.43.jar -O /var/gerrit/lib/mysql-connector-java.jar
+
 # Allow incoming traffic
 EXPOSE 29418 8080
 
@@ -59,4 +66,5 @@ ENV JAVA_OPTIONS ""
 
 # Ulimits should be set on command line or in docker-compose.yaml
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-CMD /usr/local/openjdk-8/bin/java ${JAVA_OPTIONS} -jar /var/gerrit/bin/gerrit.war daemon -d /var/gerrit
+# The /dev/./urandom is not a typo. https://stackoverflow.com/questions/58991966/what-java-security-egd-option-is-for
+CMD /usr/local/openjdk-8/bin/java -Djava.security.egd=file:/dev/./urandom ${JAVA_OPTIONS} -jar /var/gerrit/bin/gerrit.war daemon -d /var/gerrit

docker/gerrit/bazel/Dockerfile

@@ -16,3 +16,7 @@
 FROM docker.io/opendevorg/gerrit-base
 
 COPY release.war /var/gerrit/bin/gerrit.war
+
+# Install plugins
+RUN mkdir /var/gerrit/plugins && \
+  unzip -jo /var/gerrit/bin/gerrit.war WEB-INF/plugins/* -d /var/gerrit/plugins

playbooks/host_vars/review01.openstack.org.yaml

@@ -73,6 +73,7 @@ gerrit_replication:
 gerrit_storyboard_url: https://storyboard.openstack.org
 gerrit_vhost_name: review.opendev.org
 gerrit_redirect_vhost: review.openstack.org
+gerrit_heap_limit: 48g
 letsencrypt_certs:
   review01-opendev-org-main:
     - review.opendev.org

playbooks/roles/gerrit/tasks/start.yaml

@@ -2,7 +2,7 @@
   shell:
     cmd: >
       docker-compose run shell
-      java -jar /var/gerrit/bin/gerrit.war init -d /var/gerrit -b --no-auto-start --install-all-plugins
+      java -jar /var/gerrit/bin/gerrit.war init -d /var/gerrit -b --no-auto-start --skip-plugins --skip-all-downloads
     chdir: /etc/gerrit-compose/
   when: gerrit_run_init | bool
 

playbooks/roles/gerrit/templates/docker-compose.yaml.j2

@@ -9,6 +9,10 @@ services:
 {% for volume in gerrit_container_volumes %}
       - {{ volume }}
 {% endfor %}
+{% if gerrit_heap_limit is defined %}
+    environment:
+      JAVA_OPTIONS: "-Xmx{{ gerrit_heap_limit }}"
+{% endif %}
   # Utility "service" to allow us to run ad-hoc commands
   shell:
     image: {{ gerrit_container_image }}

playbooks/roles/gerrit/templates/gerrit.config

@@ -35,7 +35,9 @@
 [container]
 	user = gerrit2
 	startupTimeout = 300
-	heapLimit = 48g
+{% if gerrit_heap_limit is defined %}
+	heapLimit = {{ gerrit_heap_limit }}
+{% endif %}
 [gc]
 [core]
 	packedGitOpenFiles = 4096