openstack/trove
Code Issues Proposed changes

Merge "Upgrade Postgresql support to v9.6"

Browse Source
This commit is contained in:
Zuul 2018-01-20 05:09:53 +00:00 committed by Gerrit Code Review
commit 91cb668adb
7 changed files with 43 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
integration/scripts/files/elements/fedora-postgresql/install.d/10-postgresql
View File

@ -4,7 +4,7 @@ set -e
set -o xtrace set -o xtrace
cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_ cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_
# See 'http://www.postgresql.org/docs/9.3/static/kernel-resources.html' # See 'http://www.postgresql.org/docs/9.6/static/kernel-resources.html'
# for best practices. # for best practices.
# It is recommended to disable memory overcommit, # It is recommended to disable memory overcommit,
# but the Python interpreter may require it on smaller flavors. # but the Python interpreter may require it on smaller flavors.
@ -16,9 +16,9 @@ _EOF_
cat > "/etc/rc.local" << _EOF_ cat > "/etc/rc.local" << _EOF_
#!/bin/bash #!/bin/bash
# See 'http://www.postgresql.org/docs/9.3/static/kernel-resources.html' # See 'http://www.postgresql.org/docs/9.6/static/kernel-resources.html'
# Disable Linux kernel transparent huge pages. This feature is not supported by # Disable Linux kernel transparent huge pages. This feature is not supported by
# by Postgres 9.3 and may negatively impact performance of the database. # by Postgres 9.6 and may negatively impact performance of the database.
if test -f /sys/kernel/mm/redhat_transparent_hugepage/defrag; then if test -f /sys/kernel/mm/redhat_transparent_hugepage/defrag; then
echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag
fi fi
@ -30,56 +30,32 @@ exit \$?
_EOF_ _EOF_
dnf install -y http://yum.postgresql.org/9.4/fedora/fedora-22-x86_64/pgdg-fedora94-9.4-4.noarch.rpm dnf install -y https://yum.postgresql.org/9.6/fedora/fedora-27-x86_64/pgdg-fedora96-9.6-4.noarch.rpm
dnf install -y postgresql94-server postgresql94-contrib postgresql94-devel gcc
###########################################
# Hack alert:
# For Postgresql 9.4, pg_rewind is not in the main source tree and
# no packages exist in the repos, so it must be compiled manually
# and installed on the image until we can move to 9.5
# See README at
# https://github.com/vmware/pg_rewind/tree/REL9_4_STABLE
dev_pkgs="readline-devel zlib-devel krb5-devel openssl-devel pam-devel libxml2-devel libxslt-devel"
yum install -y $dev_pkgs
# We need pg_config to be accessible on the path
mkdir -p /tmp/build
cd /tmp/build
git clone https://github.com/vmware/pg_rewind.git --branch REL9_4_STABLE
git clone https://github.com/postgres/postgres.git --branch REL9_4_STABLE
ln -s /usr/pgsql-9.4/bin/pg_config /usr/bin/pg_config
cd pg_rewind
make USE_PGXS=1 top_srcdir=/tmp/build/postgres install
ln -s /usr/pgsql-9.4/bin/pg_rewind /usr/bin/pg_rewind
# Cleanup
cd
rm -rf /tmp/build
yum remove -y $dev_pkgs
dnf install -y postgresql96-server postgresql96-contrib postgresql96-devel gcc
# Though /var/lib/pgsql is the preferred directory, need to move it as # Though /var/lib/pgsql is the preferred directory, need to move it as
# this is where the volume will be mounted # this is where the volume will be mounted
su - postgres -c "/usr/pgsql-9.4/bin/initdb /var/lib/pgsql/9.4/data" su - postgres -c "/usr/pgsql-9.6/bin/initdb /var/lib/pgsql/9.6/data"
mv /var/lib/pgsql /var/lib/postgresql mv /var/lib/pgsql /var/lib/postgresql
mv /lib/systemd/system/postgresql-9.4.service /lib/systemd/system/postgresql.service # The `postgresql96-setup` use postgresql-${MAJORVERSION} as the service name, so rename
# postgresql-9.6.service to postgresql.service is not a good idea.
#mv /lib/systemd/system/postgresql-9.6.service /lib/systemd/system/postgresql.service
sed -i 's/PGDATA=\/var\/lib\/pgsql\/9.4\/data/PGDATA=\/var\/lib\/postgresql\/9.4\/data/' /lib/systemd/system/postgresql.service sed -i 's/PGDATA=\/var\/lib\/pgsql\/9.6\/data/PGDATA=\/var\/lib\/postgresql\/9.6\/data/' /lib/systemd/system/postgresql-9.6.service
# Create a volatile directory for runtime files. # Set the right log location for PGUPLOG and PGLOG.
echo "d /var/run/postgresql/ 0755 postgres postgres" > /lib/tmpfiles.d/postgresql.conf sed -i 's/\/var\/lib\/pgsql/\/var\/lib\/postgresql/' /usr/pgsql-9.6/bin/postgresql96-setup
# Add all postgresql related command(include pg_rewind) to the OS path.
export PATH=$PATH:/usr/pgsql-9.6/bin
# Install the native Python client. # Install the native Python client.
dnf install -y postgresql-devel python-devel dnf install -y postgresql-devel python-devel
pip install psycopg2 pip install psycopg2
systemctl enable postgresql.service # Run initdb before `systemctl start`
systemctl start postgresql.service /usr/pgsql-9.6/bin/postgresql96-setup initdb
systemctl enable postgresql-9.6.service
systemctl start postgresql-9.6.service

55
integration/scripts/files/elements/ubuntu-postgresql/install.d/30-postgresql
View File

@ -6,21 +6,20 @@ set -o xtrace
export DEBIAN_FRONTEND=noninteractive export DEBIAN_FRONTEND=noninteractive
cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_ cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_
# See 'http://www.postgresql.org/docs/9.4/static/kernel-resources.html' # See 'http://www.postgresql.org/docs/9.6/static/kernel-resources.html'
# for best practices. # for best practices.
# It is recommended to disable memory overcommit, # It is recommended to disable memory overcommit,
# but the Python interpreter may require it on smaller flavors. # but the Python interpreter may require it on smaller flavors.
# We therefore stick with the heuristic overcommit setting. # We therefore stick with the heuristic overcommit setting.
vm.overcommit_memory=0 vm.overcommit_memory=0
vm.nr_hugepages=64
_EOF_ _EOF_
cat > "/etc/rc.local" << _EOF_ cat > "/etc/rc.local" << _EOF_
#!/bin/bash #!/bin/bash
# See 'http://www.postgresql.org/docs/9.4/static/kernel-resources.html' # See 'http://www.postgresql.org/docs/9.6/static/kernel-resources.html'
# Postgres 9.4 added support for THP. Using huge pages reduces overhead when # Postgres 9.6 added support for THP. Using huge pages reduces overhead when
# using large contiguous chunks of memory, like PostgreSQL does. # using large contiguous chunks of memory, like PostgreSQL does.
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
echo never > /sys/kernel/mm/transparent_hugepage/defrag echo never > /sys/kernel/mm/transparent_hugepage/defrag
@ -33,49 +32,17 @@ exit \$?
_EOF_ _EOF_
apt-get --allow-unauthenticated -y install postgresql-9.4 postgresql-contrib-9.4 postgresql-server-dev-9.4 apt-get --allow-unauthenticated -y install postgresql-9.6 postgresql-contrib-9.6 postgresql-server-dev-9.6
########################################### sed -i "s/external_pid_file =.*/external_pid_file = '\/var\/run\/postgresql\/postgresql.pid'/" /etc/postgresql/9.6/main/postgresql.conf
# Hack alert:
# For Postgresql 9.4, pg_rewind is not in the main source tree and
# no packages exist in the repos, so it must be compiled manually
# and installed on the image until we can move to 9.5
# See README at
# https://github.com/vmware/pg_rewind/tree/REL9_4_STABLE
tmpdir=/tmp/build # The script will create a postgresql-9.6 image for user to use.
mkdir -p $tmpdir # From Postgresql >9.4, pg_rewind is in the main source tree. But, the default location is
cd $tmpdir # /usr/lib/postgresql/<version>/bin/pg_rewind. So, there need create a symlink from that
git clone https://github.com/postgres/postgres.git --branch REL9_4_STABLE # location to /usr/bin
cd postgres/contrib ln -s /usr/lib/postgresql/9.6/bin/pg_rewind /usr/bin/pg_rewind
git clone https://github.com/vmware/pg_rewind.git --branch REL9_4_STABLE
dev_pkgs="libreadline-dev libkrb5-dev libssl-dev libpam-dev libxml2-dev libxslt-dev libedit-dev libselinux1-dev bison flex"
apt-get --allow-unauthenticated install $dev_pkgs -y
# Unfortunately, on ubuntu, was not able to get pg_rewind to build
# outside of the pgsql source tree. Configure and compile postgres
# but only call make install against the contrib/pg_rewind directory
# so that support library is accessible to the server
cd $tmpdir/postgres
./configure
make
cd contrib/pg_rewind
make install
# Make the pg_rewind binary and the library used by the
# pg_rewind stored procedures accessible
ln -s /usr/local/pgsql/bin/pg_rewind /usr/bin/pg_rewind
ln -s /usr/local/pgsql/lib/pg_rewind_support.so /usr/lib/postgresql/9.4/lib/pg_rewind_support.so
cd
rm -rf $tmpdir
apt-get remove -y $dev_pkgs
# End hack
################################
# Install the native Python client. # Install the native Python client.
apt-get --allow-unauthenticated -y install libpq-dev apt-get --allow-unauthenticated -y install libpq-dev python-dev
pip2 install psycopg2 pip2 install psycopg2

4
integration/scripts/trovestack
View File

@ -583,8 +583,8 @@ function cmd_set_datastore() {
PACKAGES=${PACKAGES:-"couchbase-server"} PACKAGES=${PACKAGES:-"couchbase-server"}
VERSION="2.2.0" VERSION="2.2.0"
elif [ "$DATASTORE_TYPE" == "postgresql" ]; then elif [ "$DATASTORE_TYPE" == "postgresql" ]; then
PACKAGES=${PACKAGES:-"postgresql-9.4"} PACKAGES=${PACKAGES:-"postgresql-9.6"}
VERSION="9.4" VERSION="9.6"
elif [ "$DATASTORE_TYPE" == "couchdb" ]; then elif [ "$DATASTORE_TYPE" == "couchdb" ]; then
PACKAGES=${PACKAGES:-"couchdb"} PACKAGES=${PACKAGES:-"couchdb"}
VERSION="1.6.1" VERSION="1.6.1"

2
trove/guestagent/datastore/experimental/postgresql/pgsql_query.py
View File

@ -100,7 +100,7 @@ class UserQuery(object):
create_clause = "CREATE USER \"{name}\"".format(name=name) create_clause = "CREATE USER \"{name}\"".format(name=name)
with_clause = cls._build_with_clause( with_clause = cls._build_with_clause(
password, encrypt_password, *options) password, encrypt_password, *options)
return ''.join([create_clause, with_clause]) return ' '.join([create_clause, with_clause])
@classmethod @classmethod
def _build_with_clause(cls, password, encrypt_password=None, *options): def _build_with_clause(cls, password, encrypt_password=None, *options):

10
trove/guestagent/datastore/experimental/postgresql/service.py
View File

@ -280,16 +280,16 @@ class PgSqlApp(object):
arch_cmd = "'test ! -f {wal_arch}/%f && cp %p {wal_arch}/%f'".format( arch_cmd = "'test ! -f {wal_arch}/%f && cp %p {wal_arch}/%f'".format(
wal_arch=wal_arch_loc wal_arch=wal_arch_loc
) )
# Only support pg version > 9.6, wal_level set to replica, and
# remove parameter "checkpoint_segments".
opts = { opts = {
'wal_level': 'hot_standby', 'wal_level': 'replica',
'archive_mode': 'on', 'archive_mode': 'on',
'max_wal_senders': 8, 'max_wal_senders': 8,
'checkpoint_segments': 8, 'wal_log_hints': 'on',
'wal_keep_segments': 8, 'wal_keep_segments': 8,
'archive_command': arch_cmd 'archive_command': arch_cmd
} }
if not self.pg_version[1] in ('9.3'):
opts['wal_log_hints'] = 'on'
self.configuration_manager.apply_system_override( self.configuration_manager.apply_system_override(
opts, BACKUP_CFG_OVERRIDE) opts, BACKUP_CFG_OVERRIDE)
@ -594,7 +594,7 @@ class PgSqlAdmin(object):
# Default set of options of an administrative account. # Default set of options of an administrative account.
ADMIN_OPTIONS = ( ADMIN_OPTIONS = (
'SUPERUSER', 'CREATEDB', 'CREATEROLE', 'INHERIT', 'REPLICATION', 'SUPERUSER', 'CREATEDB', 'CREATEROLE', 'INHERIT', 'REPLICATION',
'LOGIN' 'BYPASSRLS', 'LOGIN'
) )
def __init__(self, user): def __init__(self, user):

8
trove/guestagent/strategies/replication/experimental/postgresql_impl.py
View File

@ -283,11 +283,11 @@ class PostgresqlReplicationStreaming(base.Replication):
group=service.pgsql_owner, as_root=True) group=service.pgsql_owner, as_root=True)
def enable_hot_standby(self, service): def enable_hot_standby(self, service):
# Only support pg version > 9.6, wal_level set to replica, and
# remove parameter "checkpoint_segments".
opts = {'hot_standby': 'on', opts = {'hot_standby': 'on',
'wal_level': 'hot_standby'} 'wal_level': 'replica',
# wal_log_hints for pg_rewind is only supported in 9.4+ 'wal_log_hints': 'on'}
if service.pg_version[1] in ('9.4', '9.5'):
opts['wal_log_hints'] = 'on'
service.configuration_manager.\ service.configuration_manager.\
apply_system_override(opts, SLAVE_STANDBY_OVERRIDE) apply_system_override(opts, SLAVE_STANDBY_OVERRIDE)

5
trove/templates/postgresql/config.template
View File

@ -218,11 +218,6 @@ wal_level = minimal # minimal, archive, hot_standby, or logical
# - Checkpoints - # - Checkpoints -
checkpoint_segments = {{ (shared_buffers_mb[0] / 16 + 1)|int }} # in logfile segments, min 1, 16MB each
# Each segment is normally 16MB long.
# The number of segments should be enough to
# span the 'shared_buffers' size.
# We set the default to (shared_buffers / 16 + 1).
#checkpoint_timeout = 5min # range 30s-1h #checkpoint_timeout = 5min # range 30s-1h
#checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 #checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0
#checkpoint_warning = 30s # 0 disables #checkpoint_warning = 30s # 0 disables