rsync-mirrors: drop rsync -t and make flags consistent
After investigating why every release seems to take hours, we noticed that even in zero-delta updates where nothing has changed upstream, we'd get a utimensat() call that updated the modified time nanoseconds portion; e.g. lstat("Everything/x86_64/Packages/a/arch-install-scripts-23-1.fc31.noarch.rpm", {st_mode=S_IFREG|0644, st_size=28004, ...}) = 0 utimensat(AT_FDCWD, "Everything/x86_64/Packages/a/arch-install-scripts-23-1.fc31.noarch.rpm", [UTIME_NOW, {tv_sec=1585676005, tv_nsec=6041000} /* 2020-03-31T17:33:25.006041000+0000 */], AT_SYMLINK_NOFOLLOW) = 0 This does not apply; in fact openafs uses the ns field as some sort of generation counter [1]. This update is enough to convince openafs the file has changed and it needs to be resynced, meaning that basically every rsync run results in a full release. This unnecessary update been fixed with [2] but is only in rsync 3.1.3+; our bionic host is currently 3.1.2. Dropping "-t" from the rsync commands avoids transferring modification times and should avoid this problem. While looking, "-D" turns on "--devices" and "--specials" to transfer block devices and named sockets/fifos. Turn this off. Also remove "-p" if it was present. We already did this for centos with Ib5db052cdd23e39aecbeead15cf08d4bd7fcab38 and Fedora with Id24196791f80cd99fe8a330fb2c7c6d893fc9995, where odd upstream permissions such as setgid on directories can't be synced to afs. Consistently remove it. Also switch back the fedora updates to just "-v"; we had it at "-i" for debugging. [1] http://eavesdrop.openstack.org/irclogs/%23opendev/%23opendev.2020-06-15.log.html#t2020-06-15T02:58:08-2 [2] https://git.samba.org/?p=rsync.git;a=patch;h=0f8e9e2d8638e47d646a6baba694b303ac84e695 Change-Id: I78f3d4990b168c71185eb1c4900ceeaca4b6a16f
This commit is contained in:
parent
afd3abd87a
commit
4918594aa4
@ -38,7 +38,7 @@ if ! [ -f $BASE/8 ]; then
|
|||||||
fi
|
fi
|
||||||
date --iso-8601=ns
|
date --iso-8601=ns
|
||||||
echo "Running Centos 8 rsync..."
|
echo "Running Centos 8 rsync..."
|
||||||
$K5START rsync -rtDvz \
|
$K5START rsync -rlvz \
|
||||||
--delete \
|
--delete \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="isos" \
|
--exclude="isos" \
|
||||||
@ -75,7 +75,7 @@ for REPO in $REPOS; do
|
|||||||
|
|
||||||
date --iso-8601=ns
|
date --iso-8601=ns
|
||||||
echo "Running rsync..."
|
echo "Running rsync..."
|
||||||
$K5START rsync -rtDvz \
|
$K5START rsync -rlvz \
|
||||||
--delete \
|
--delete \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="atomic" \
|
--exclude="atomic" \
|
||||||
|
@ -42,7 +42,7 @@ fi
|
|||||||
|
|
||||||
date --iso-8601=ns
|
date --iso-8601=ns
|
||||||
echo "Running rsync..."
|
echo "Running rsync..."
|
||||||
$K5START rsync -rlptDvz \
|
$K5START rsync -rlvz \
|
||||||
--delete \
|
--delete \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="SRPMS" \
|
--exclude="SRPMS" \
|
||||||
@ -72,7 +72,7 @@ date --iso-8601=ns
|
|||||||
echo "Running rsync..."
|
echo "Running rsync..."
|
||||||
# --no-perms because some of the top-level directories are 02755
|
# --no-perms because some of the top-level directories are 02755
|
||||||
# (setgid) and we can't set that on AFS
|
# (setgid) and we can't set that on AFS
|
||||||
$K5START rsync -rlptDvz --no-perms \
|
$K5START rsync -rlvz --no-perms \
|
||||||
--delete \
|
--delete \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="Everything/SRPMS" \
|
--exclude="Everything/SRPMS" \
|
||||||
|
@ -52,7 +52,7 @@ for REPO in releases/30 releases/31; do
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
echo_ts "Running rsync for $REPO..."
|
echo_ts "Running rsync for $REPO..."
|
||||||
$K5START rsync -rltDiz \
|
$K5START rsync -rlvz \
|
||||||
--delete \
|
--delete \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="Cloud/x86_64/images/*.box" \
|
--exclude="Cloud/x86_64/images/*.box" \
|
||||||
@ -77,7 +77,7 @@ for REPO in updates/30 updates/31; do
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
echo_ts "Running rsync for $REPO..."
|
echo_ts "Running rsync for $REPO..."
|
||||||
$K5START rsync -rltDiz \
|
$K5START rsync -rlvz \
|
||||||
--delete \
|
--delete \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="aarch64/" \
|
--exclude="aarch64/" \
|
||||||
@ -98,7 +98,7 @@ if ! [ -f $BASE/atomic ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
echo_ts "Running rsync atomic..."
|
echo_ts "Running rsync atomic..."
|
||||||
$K5START rsync -rltDiz \
|
$K5START rsync -rlvz \
|
||||||
--delete \
|
--delete \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="testing/" \
|
--exclude="testing/" \
|
||||||
|
@ -62,7 +62,7 @@ for DISTVER in 15.1; do
|
|||||||
|
|
||||||
date --iso-8601=ns
|
date --iso-8601=ns
|
||||||
echo "Running rsync distribution $DISTVER ..."
|
echo "Running rsync distribution $DISTVER ..."
|
||||||
$K5START rsync -rlptDvz \
|
$K5START rsync -rlvz \
|
||||||
--delete --stats \
|
--delete --stats \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="iso" \
|
--exclude="iso" \
|
||||||
@ -76,7 +76,7 @@ for DISTVER in 15.1; do
|
|||||||
|
|
||||||
date --iso-8601=ns
|
date --iso-8601=ns
|
||||||
echo "Running rsync updates $DISTVER ..."
|
echo "Running rsync updates $DISTVER ..."
|
||||||
$K5START rsync -rlptDvz \
|
$K5START rsync -rlvz \
|
||||||
--delete --stats \
|
--delete --stats \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="src/" \
|
--exclude="src/" \
|
||||||
@ -92,7 +92,7 @@ for obs_repo in ${OBS_REPOS[@]}; do
|
|||||||
$K5START mkdir -p $BASE/$REPO
|
$K5START mkdir -p $BASE/$REPO
|
||||||
fi
|
fi
|
||||||
echo "Running rsync ${obs_repo} ..."
|
echo "Running rsync ${obs_repo} ..."
|
||||||
$K5START rsync -rlptDvz \
|
$K5START rsync -rlvz \
|
||||||
--delete --stats \
|
--delete --stats \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="src/" \
|
--exclude="src/" \
|
||||||
@ -109,7 +109,7 @@ fi
|
|||||||
date --iso-8601=ns
|
date --iso-8601=ns
|
||||||
echo "Running rsync distribution $REPO ..."
|
echo "Running rsync distribution $REPO ..."
|
||||||
# TW is large and can have failures, be more resilient
|
# TW is large and can have failures, be more resilient
|
||||||
$K5START rsync -rlptDvz \
|
$K5START rsync -rlvz \
|
||||||
--delete --stats \
|
--delete --stats \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="i586" \
|
--exclude="i586" \
|
||||||
@ -123,7 +123,7 @@ fi
|
|||||||
|
|
||||||
date --iso-8601=ns
|
date --iso-8601=ns
|
||||||
echo "Running rsync distribution $REPO ..."
|
echo "Running rsync distribution $REPO ..."
|
||||||
$K5START rsync -rlptDvz \
|
$K5START rsync -rlvz \
|
||||||
--timeout=600 \
|
--timeout=600 \
|
||||||
--delete --stats \
|
--delete --stats \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
|
@ -16,7 +16,7 @@
|
|||||||
DEST="/afs/.openstack.org/mirror/logs/"
|
DEST="/afs/.openstack.org/mirror/logs/"
|
||||||
|
|
||||||
K5START="k5start -t -f /etc/logs.keytab service/logs-mirror --"
|
K5START="k5start -t -f /etc/logs.keytab service/logs-mirror --"
|
||||||
RSYNC="rsync -avz --no-perms --no-owner --no-group"
|
RSYNC="rsync -rliz"
|
||||||
|
|
||||||
$K5START $RSYNC /var/log/rsync-mirrors $DEST
|
$K5START $RSYNC /var/log/rsync-mirrors $DEST
|
||||||
$K5START $RSYNC /var/log/afs-release $DEST
|
$K5START $RSYNC /var/log/afs-release $DEST
|
||||||
|
@ -36,7 +36,7 @@ date --iso-8601=ns
|
|||||||
echo "Running rsync..."
|
echo "Running rsync..."
|
||||||
|
|
||||||
# We don't need cisco-wrlinux arch in OpenStack Infra.
|
# We don't need cisco-wrlinux arch in OpenStack Infra.
|
||||||
$K5START rsync -rlptDvz \
|
$K5START rsync -rlvz \
|
||||||
--delete \
|
--delete \
|
||||||
--delete-excluded \
|
--delete-excluded \
|
||||||
--exclude="cisco-wrlinux" \
|
--exclude="cisco-wrlinux" \
|
||||||
|
Loading…
Reference in New Issue
Block a user