Clark Boylan dbe477b205 Increase gitea db connection limit
By default our mariadb database for gitea nodes limits itself to a
maximum of 100 connections. We've seen errors like this:

 ...eb/routing/logger.go:102:func1() [I] router: completed POST /openstack/requirements/git-upload-pack for 127.0.0.1:50562, 500 Internal Server Error in 2.6ms @ context/user.go:17(web.gitHTTPRouters.UserAssignmentWeb)
 ...ules/context/repo.go:467:RepoAssignment() [E] GetUserByName: Error 1040: Too many connections

And after reading gitea's source code this appears to be related to user
lookups to determine if the user making a request against a repo owns
the repo. To do this gitea does a db request to lookup the user from the
request and when this hits the connection limit it bubbles up the mysql
error 1040: Too may connections error.

This problem seems infrequent so we double the limit to 200 which is
both much larger but still a reasonable number.

We also modify the test that checks for gitea server errors without an
http 500 return code to avoid it matching this change improperly. This
was happening because the commit message ends up in the rendered pages
for system-config in the test gitea.

Change-Id: If8c72ab277e88ae09a44a64a1571f94e43df23f8
2024-02-05 10:40:03 -08:00

249 lines
7.9 KiB
YAML

- name: Ensure docker-compose directory exists
file:
state: directory
path: /etc/gitea-docker
mode: 0700
- name: Write docker-compose file
template:
src: docker-compose.yaml.j2
dest: /etc/gitea-docker/docker-compose.yaml
mode: 0600
- name: Ensure gitea volume directories exists
file:
state: directory
path: "/var/gitea/{{ item }}"
owner: 1000
group: 1000
loop:
- conf
- data
- logs
- certs
- db
- name: Write app.ini
template:
src: app.ini.j2
dest: /var/gitea/conf/app.ini
- name: Write mariadb conn limit config file
copy:
src: 99-max_conn_my.cnf
dest: /var/gitea/conf/99-max_conn_my.cnf
owner: root
group: root
mode: 0644
- name: Install distro packages
package:
name:
- docker-compose
- python3-requests
state: present
- name: Install reverse proxy
include_tasks: proxy.yaml
- name: Run docker-compose pull
shell:
cmd: docker-compose pull
chdir: /etc/gitea-docker/
register: docker_compose_pull
- name: Stop/Start gitea safely for Gerrit replication
when: "'downloaded newer image' in docker_compose_pull.stderr"
block:
- name: Run docker-compose stop
shell:
cmd: docker-compose stop --timeout 60
chdir: /etc/gitea-docker/
- name: Run docker-compose up mariadb gitea-web
shell:
cmd: docker-compose up -d --timeout 60 mariadb gitea-web
chdir: /etc/gitea-docker/
# We wait here for the main gitea service to start before starting
# the ssh service. This is friendly to gerrit replication.
- name: Wait until the web service is sufficiently up to start ssh
uri:
url: "https://localhost:3000/api/v1/users/root"
validate_certs: false
status_code: 200, 404
register: root_user_check
delay: 1
retries: 300
until: root_user_check and root_user_check.status in (200, 404)
- name: Run docker-compose up gitea-ssh
shell:
cmd: docker-compose up -d --timeout 60 gitea-ssh
chdir: /etc/gitea-docker/
- name: Run docker prune to cleanup unneeded images
shell:
# Keep images around for 3 days before pruning them. Allows for
# easy rollback if necessary. Note "3d" seems to be rejected by
# docker but 72 hours == 3 days.
cmd: docker image prune -f --filter "until=72h"
# User management outside of service bringup to avoid confusion between
# the two stages.
- name: Check if root user exists
uri:
url: "https://localhost:3000/api/v1/users/root"
validate_certs: false
status_code: 200, 404
register: root_user_check
delay: 1
retries: 300
until: root_user_check and root_user_check.status in (200, 404)
- name: Create root user
when: root_user_check.status==404
block:
- name: Create root user
command: "docker exec -t gitea-docker_gitea-web_1 gitea admin user create --name root --password {{ gitea_root_password }} --email {{ gitea_root_email }} --admin"
no_log: "{{ gitea_no_log }}"
- name: Check if gerrit user exists
uri:
url: "https://localhost:3000/api/v1/users/gerrit"
validate_certs: false
status_code: 200, 404
register: gerrit_user_check
- name: Create gerrit user
when: gerrit_user_check.status==404
no_log: true
uri:
url: "https://localhost:3000/api/v1/admin/users"
validate_certs: false
method: POST
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
status_code: 201
body_format: json
body:
email: "gerrit@review.opendev.org"
full_name: Gerrit
login_name: gerrit
must_change_password: false
password: "{{ gitea_gerrit_password }}"
send_notify: false
source_id: 0
username: gerrit
- name: List keys to determine which updates are necessary.
uri:
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
url: "https://localhost:3000/api/v1/users/gerrit/keys"
validate_certs: false
status_code: 200
register: gerrit_key_check
no_log: true
# We want to allow for multiple keys in order to do key rotations.
# Check if both keys are present. If a key is not present then add it
# to Gitea. Keep in mind the two keys may be the same in which case
# we can skip the second key creation. Finally clean up any keys
# that don't match the two keys. This allows us to do key rotations.
- name: Determine if key A and key B are already present
set_fact:
key_A_present: "{{ gerrit_key_check.json | selectattr('key', 'equalto', gitea_gerrit_public_key_A ) | list | length > 0 }}"
key_B_present: "{{ gerrit_key_check.json | selectattr('key', 'equalto', gitea_gerrit_public_key_B ) | list | length > 0 }}"
- name: Add gerrit ssh key A
when: not key_A_present
no_log: true
uri:
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
url: "https://localhost:3000/api/v1/admin/users/gerrit/keys"
validate_certs: false
method: POST
status_code: 201
body_format: json
body:
key: "{{ gitea_gerrit_public_key_A }}"
read_only: false
title: "Gerrit replication key A"
- name: Add gerrit ssh key B
when: not key_B_present and gitea_gerrit_public_key_A != gitea_gerrit_public_key_B
no_log: true
uri:
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
url: "https://localhost:3000/api/v1/admin/users/gerrit/keys"
validate_certs: false
method: POST
status_code: 201
body_format: json
body:
key: "{{ gitea_gerrit_public_key_B }}"
read_only: false
title: "Gerrit replication key B"
- name: List keys again to ensure key ids are correct for deletion.
uri:
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
url: "https://localhost:3000/api/v1/users/gerrit/keys"
validate_certs: false
status_code: 200
register: gerrit_key_check
no_log: true
- name: Delete old gerrit ssh keys
when: existing_pubkey.key != gitea_gerrit_public_key_A and existing_pubkey.key != gitea_gerrit_public_key_B
no_log: true
uri:
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
url: "https://localhost:3000/api/v1/user/keys/{{ existing_pubkey.id }}"
validate_certs: false
method: DELETE
status_code: 204
loop: "{{ gerrit_key_check.json }}"
loop_control:
loop_var: existing_pubkey
- name: Set up cron job to pack git refs
cron:
name: pack-git-refs
state: present
job: "docker exec -t gitea-docker_gitea-web_1 find /data/git/repositories/ -maxdepth 2 -name *.git -type d -execdir git --git-dir={} gc --quiet \\;"
minute: '{{ 59 | random(seed=inventory_hostname) }}'
hour: '{{ 23 | random(seed=inventory_hostname) }}'
weekday: '*/2'
- name: Create db backup dest
file:
state: directory
path: /var/backups/gitea-mariadb
mode: 0700
owner: root
group: root
- name: Set up cron job to backup the database
cron:
name: gitea-db-backup
state: present
user: root
job: >
/usr/local/bin/docker-compose -f /etc/gitea-docker/docker-compose.yaml exec -T mariadb
bash -c '/usr/bin/mysqldump --opt --databases gitea --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' |
gzip -9 > /var/backups/gitea-mariadb/gitea-mariadb.sql.gz
minute: 42
hour: 4
- name: Rotate db backups
include_role:
name: logrotate
vars:
logrotate_file_name: /var/backups/gitea-mariadb/gitea-mariadb.sql.gz
logrotate_compress: false
- name: Setup db backup streaming job
block:
- name: Create backup streaming config dir
file:
path: /etc/borg-streams
state: directory
- name: Create db streaming file
copy:
content: >-
/usr/local/bin/docker-compose -f /etc/gitea-docker/docker-compose.yaml exec -T mariadb
bash -c '/usr/bin/mysqldump --skip-extended-insert --databases gitea --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"'
dest: /etc/borg-streams/mysql