From 6f4cb962a7a762d3223f996cf63cd1e0ccd83a79 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Sun, 1 Oct 2017 09:41:54 -0700 Subject: [PATCH] Handle z-c shim copies across filesystems The zuulv3 zuul-cloner shim was using cp -l which uses hardlinks to "copy" files. You cannot hard link across filesystem boundaries and there is at least one case where jobs copy using z-c shim across filesystem boundaries (tripleo-ci jobs). Handle this by listing all of the filesystem mounts, finding the longest filepath prefix that our source and destination dirs match and comparing if they are on the same filesystem to determine if we can hardlink or not. Change-Id: I7cf211fc43df1b86c54fcc1b62e4321a658d9c84 --- .../templates/zuul-cloner-shim.py.j2 | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/roles/fetch-zuul-cloner/templates/zuul-cloner-shim.py.j2 b/roles/fetch-zuul-cloner/templates/zuul-cloner-shim.py.j2 index 0641fefba..f0a1e44a8 100644 --- a/roles/fetch-zuul-cloner/templates/zuul-cloner-shim.py.j2 +++ b/roles/fetch-zuul-cloner/templates/zuul-cloner-shim.py.j2 @@ -124,6 +124,19 @@ def readCloneMap(clone_map): return clone_map +def get_fs(mounts, path): + longest_match = '' + real_path = os.path.realpath(path) + for mount_path in mounts.keys(): + if (real_path.startswith(mount_path) and + len(mount_path) > len(longest_match)): + longest_match = mount_path + if longest_match: + return mounts[longest_match] + else: + return None + + def main(): args = parseArgs() @@ -134,6 +147,12 @@ def main(): mapper = CloneMapper(clone_map, args.projects) dests = mapper.expand(workspace=args.workspace) + mounts = {} + with open('/proc/mounts') as f: + for line in f.readlines(): + fs, mount_path = line.split()[0:2] + mounts[mount_path] = fs + for project in args.projects: src = os.path.join(os.path.expanduser(REPO_SRC_DIR), project) if not os.path.exists(src): @@ -167,13 +186,19 @@ def main(): print("Creating %s" % d) os.makedirs(d) + src_fs = get_fs(mounts, src) + dst_fs = get_fs(mounts, dst) + # Create hard link copy of the source directory # note: don't use "-a" here as that implies "--preserve=all" # which overwrites the permissions of dst from src ... this is # fatal to ssh if dst is a home directory and we make it # world-accessable. This should leave dst alone - cmd = "cp -dRl %s/. %s" % (src, dst) + if src_fs == dst_fs: + cmd = "cp -dRl %s/. %s" % (src, dst) + else: + cmd = "cp -dR %s/. %s" % (src, dst) print("%s" % cmd) if os.system(cmd): print("Error executing: %s" % cmd)