diff --git a/roles/generate-zuul-manifest/library/test-fixtures/.gitattributes b/roles/generate-zuul-manifest/library/test-fixtures/.gitattributes new file mode 100644 index 000000000..2325d0821 --- /dev/null +++ b/roles/generate-zuul-manifest/library/test-fixtures/.gitattributes @@ -0,0 +1 @@ +*.* eol=lf \ No newline at end of file diff --git a/roles/test-upload-logs-swift/library/filefixture.py b/roles/test-upload-logs-swift/library/filefixture.py new file mode 100644 index 000000000..615b26ab1 --- /dev/null +++ b/roles/test-upload-logs-swift/library/filefixture.py @@ -0,0 +1,139 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Handle file name special characters in a file tree. + +All files stored in a filetree can be renamed to urlencoded filenames. +A file tree can also be copied to a temporary location with file names +decoded, to be used in tests with special characters that are not always +possible to store on all file systems. + +""" + +from __future__ import print_function + +import os +try: + from urllib.parse import quote as urlib_quote + from urllib.parse import unquote as urlib_unquote +except ImportError: + from urllib import quote as urlib_quote + from urllib import unquote as urlib_unquote +import argparse +import fixtures +import tempfile +import shutil + + +FIXTURE_DIR = os.path.join(os.path.dirname(__file__), + 'test-fixtures') + +SAFE_CHARS = "\\/" + + +def portable_makedirs_exist_ok(path): + try: + os.makedirs(path, exist_ok=True) + except TypeError as err: + if "unexpected keyword argument" not in str(err): + raise err + if not os.path.exists(path): + try: + os.makedirs(path) + except OSError as err: + if "File exists" not in err: + raise err + + +def urlencode_filetree(): + for root, _, files in os.walk(FIXTURE_DIR): + for filename in files: + os.rename( + os.path.join(root, filename), + os.path.join( + root, urlib_quote(urlib_unquote(filename), SAFE_CHARS) + ) + ) + + +def populate_filetree(dst_dir=None): + + if not os.path.exists(FIXTURE_DIR): + return None + + if not dst_dir: + dst_dir = tempfile.mkdtemp() + + portable_makedirs_exist_ok(dst_dir) + + for root, dirs, files in os.walk(FIXTURE_DIR): + dst_root = root.replace(FIXTURE_DIR, dst_dir, 1) + for directory in dirs: + portable_makedirs_exist_ok(os.path.join(dst_root, directory)) + for filename in files: + try: + shutil.copyfile( + os.path.join(root, filename), + os.path.join(dst_root, urlib_unquote(filename)) + ) + except IOError as err: + print( + "\nFile {}".format( + os.path.join(dst_root, urlib_unquote(filename)) + ), + "\nnot possible to write to disk,", + "\npossibly due to filename not being valid on Windows?\n" + ) + shutil.rmtree(dst_dir) + raise err + + return dst_dir + + +class FileFixture(fixtures.Fixture): + + def _setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(self.local_clean_up) + populate_filetree(self.root) + # There is no cleanup action, as the filetree is left intact for other + # tests to use + + def local_clean_up(self): + shutil.rmtree(self.root) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(__doc__) + parser.add_argument( + '--populate', + help="Causes files in {}".format(FIXTURE_DIR) + + "to be copied with decoded file name to a tmp dir" + + "Overrides --encode", + action='store_true' + ) + parser.add_argument( + '--encode', + help="Causes files under {} to be renamed with urlencoding.".format( + FIXTURE_DIR + ) + "DEFAULT behaviour, overridden by --populate", + action='store_true' + ) + args = parser.parse_args() + + if args.populate: + print(populate_filetree()) + else: + urlencode_filetree() diff --git a/roles/test-upload-logs-swift/library/test-fixtures/.gitattributes b/roles/test-upload-logs-swift/library/test-fixtures/.gitattributes new file mode 100644 index 000000000..2325d0821 --- /dev/null +++ b/roles/test-upload-logs-swift/library/test-fixtures/.gitattributes @@ -0,0 +1 @@ +*.* eol=lf \ No newline at end of file diff --git a/roles/test-upload-logs-swift/library/test-fixtures/logs/Ꮓບບξ-unicode.txt b/roles/test-upload-logs-swift/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt similarity index 100% rename from roles/test-upload-logs-swift/library/test-fixtures/logs/Ꮓບບξ-unicode.txt rename to roles/test-upload-logs-swift/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt diff --git a/roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo::3.txt b/roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt similarity index 100% rename from roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo::3.txt rename to roles/test-upload-logs-swift/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt diff --git a/roles/test-upload-logs-swift/library/test_zuul_swift_upload.py b/roles/test-upload-logs-swift/library/test_zuul_swift_upload.py index cf088987a..2e0c32a23 100644 --- a/roles/test-upload-logs-swift/library/test_zuul_swift_upload.py +++ b/roles/test-upload-logs-swift/library/test_zuul_swift_upload.py @@ -32,7 +32,7 @@ except ImportError: import requests from bs4 import BeautifulSoup from .zuul_swift_upload import FileList, Indexer, FileDetail, Uploader - +from .filefixture import FileFixture FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'test-fixtures') @@ -50,8 +50,11 @@ class SymlinkFixture(fixtures.Fixture): ] def _setUp(self): + self.file_fixture = FileFixture() + self.file_fixture.setUp() + self.addCleanup(self.file_fixture.cleanUp) for (src, target) in self.links: - path = os.path.join(FIXTURE_DIR, 'links', src) + path = os.path.join(self.file_fixture.root, 'links', src) os.symlink(target, path) self.addCleanup(os.unlink, path) @@ -88,66 +91,69 @@ class TestFileList(testtools.TestCase): '''Test a single directory with a trailing slash''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ]) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs/')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ]) def test_single_dir(self): '''Test a single directory without a trailing slash''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('logs', 'application/directory', None), - ('logs/controller', 'application/directory', None), - ('logs/zuul-info', 'application/directory', None), - ('logs/job-output.json', 'application/json', None), - (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('logs/controller/subdir', 'application/directory', None), - ('logs/controller/compressed.gz', 'text/plain', 'gzip'), - ('logs/controller/cpu-load.svg', 'image/svg+xml', None), - ('logs/controller/journal.xz', 'text/plain', 'xz'), - ('logs/controller/service_log.txt', 'text/plain', None), - ('logs/controller/syslog', 'text/plain', None), - ('logs/controller/subdir/foo::3.txt', 'text/plain', None), - ('logs/controller/subdir/subdir.txt', 'text/plain', None), - ('logs/zuul-info/inventory.yaml', 'text/plain', None), - ('logs/zuul-info/zuul-info.controller.txt', - 'text/plain', None), - ]) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('logs', 'application/directory', None), + ('logs/controller', 'application/directory', None), + ('logs/zuul-info', 'application/directory', None), + ('logs/job-output.json', 'application/json', None), + (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('logs/controller/subdir', 'application/directory', None), + ('logs/controller/compressed.gz', 'text/plain', 'gzip'), + ('logs/controller/cpu-load.svg', 'image/svg+xml', None), + ('logs/controller/journal.xz', 'text/plain', 'xz'), + ('logs/controller/service_log.txt', 'text/plain', None), + ('logs/controller/syslog', 'text/plain', None), + ('logs/controller/subdir/foo::3.txt', 'text/plain', None), + ('logs/controller/subdir/subdir.txt', 'text/plain', None), + ('logs/zuul-info/inventory.yaml', 'text/plain', None), + ('logs/zuul-info/zuul-info.controller.txt', + 'text/plain', None), + ]) def test_single_file(self): '''Test a single file''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, - 'logs/zuul-info/inventory.yaml')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('inventory.yaml', 'text/plain', None), - ]) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, + 'logs/zuul-info/inventory.yaml')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('inventory.yaml', 'text/plain', None), + ]) def test_symlinks(self): '''Test symlinks''' with FileList() as fl: - self.useFixture(SymlinkFixture()) - fl.add(os.path.join(FIXTURE_DIR, 'links/')) + symlink_fixture = self.useFixture(SymlinkFixture()) + fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/')) self.assert_files(fl, [ ('', 'application/directory', None), ('controller', 'application/directory', None), @@ -165,7 +171,8 @@ class TestFileList(testtools.TestCase): def test_index_files(self): '''Test index generation''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) + symlink_fixture = self.useFixture(SymlinkFixture()) + fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs')) ix = Indexer(fl) ix.make_indexes() @@ -223,32 +230,33 @@ class TestFileList(testtools.TestCase): def test_index_files_trailing_slash(self): '''Test index generation with a trailing slash''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes() + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs/')) + ix = Indexer(fl) + ix.make_indexes() - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('index.html', 'text/html', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/index.html', 'text/html', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('controller/subdir/index.html', 'text/html', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ('zuul-info/index.html', 'text/html', None), + ]) top_index = self.find_file(fl, 'index.html') page = open(top_index.full_path).read() @@ -280,134 +288,145 @@ class TestFileList(testtools.TestCase): def test_topdir_parent_link(self): '''Test index generation creates topdir parent link''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=True, - create_topdir_parent_link=True) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs/')) + ix = Indexer(fl) + ix.make_indexes( + create_parent_links=True, + create_topdir_parent_link=True) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('index.html', 'text/html', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/index.html', 'text/html', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('controller/subdir/index.html', 'text/html', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ('zuul-info/index.html', 'text/html', None), + ]) - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] + top_index = self.find_file(fl, 'index.html') + page = open(top_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] - self.assertEqual(len(rows), 5) + self.assertEqual(len(rows), 5) - self.assertEqual(rows[0].find('a').get('href'), '../') - self.assertEqual(rows[0].find('a').text, '../') + self.assertEqual(rows[0].find('a').get('href'), '../') + self.assertEqual(rows[0].find('a').text, '../') - self.assertEqual(rows[1].find('a').get('href'), 'controller/') - self.assertEqual(rows[1].find('a').text, 'controller/') + self.assertEqual(rows[1].find('a').get('href'), 'controller/') + self.assertEqual(rows[1].find('a').text, 'controller/') - self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/') - self.assertEqual(rows[2].find('a').text, 'zuul-info/') + self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/') + self.assertEqual(rows[2].find('a').text, 'zuul-info/') - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../') - self.assertEqual(rows[0].find('a').text, '../') + subdir_index = self.find_file( + fl, 'controller/subdir/index.html' + ) + page = open(subdir_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + self.assertEqual(rows[0].find('a').get('href'), '../') + self.assertEqual(rows[0].find('a').text, '../') - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') + # Test proper escaping of files with funny names + self.assertEqual( + rows[1].find('a').get('href'), 'foo%3A%3A3.txt' + ) + self.assertEqual(rows[1].find('a').text, 'foo::3.txt') + # Test files without escaping + self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') + self.assertEqual(rows[2].find('a').text, 'subdir.txt') def test_no_parent_links(self): '''Test index generation creates topdir parent link''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=False, - create_topdir_parent_link=False) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs/')) + ix = Indexer(fl) + ix.make_indexes( + create_parent_links=False, + create_topdir_parent_link=False) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('index.html', 'text/html', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/index.html', 'text/html', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('controller/subdir/index.html', 'text/html', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ('zuul-info/index.html', 'text/html', None), + ]) - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] + top_index = self.find_file(fl, 'index.html') + page = open(top_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] - self.assertEqual(len(rows), 4) + self.assertEqual(len(rows), 4) - self.assertEqual(rows[0].find('a').get('href'), 'controller/') - self.assertEqual(rows[0].find('a').text, 'controller/') + self.assertEqual(rows[0].find('a').get('href'), 'controller/') + self.assertEqual(rows[0].find('a').text, 'controller/') - self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/') - self.assertEqual(rows[1].find('a').text, 'zuul-info/') + self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/') + self.assertEqual(rows[1].find('a').text, 'zuul-info/') - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] + subdir_index = self.find_file( + fl, 'controller/subdir/index.html' + ) + page = open(subdir_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] - # Test proper escaping of files with funny names - self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[0].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[1].find('a').text, 'subdir.txt') + # Test proper escaping of files with funny names + self.assertEqual( + rows[0].find('a').get('href'), 'foo%3A%3A3.txt' + ) + self.assertEqual(rows[0].find('a').text, 'foo::3.txt') + # Test files without escaping + self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') + self.assertEqual(rows[1].find('a').text, 'subdir.txt') class TestFileDetail(testtools.TestCase): def test_get_file_detail(self): '''Test files info''' - path = os.path.join(FIXTURE_DIR, 'logs/job-output.json') - file_detail = FileDetail(path, '') - path_stat = os.stat(path) - self.assertEqual( - time.gmtime(path_stat[stat.ST_MTIME]), - file_detail.last_modified) - self.assertEqual(16, file_detail.size) + with FileFixture() as file_fixture: + path = os.path.join(file_fixture.root, 'logs/job-output.json') + file_detail = FileDetail(path, '') + path_stat = os.stat(path) + self.assertEqual( + time.gmtime(path_stat[stat.ST_MTIME]), + file_detail.last_modified) + self.assertEqual(16, file_detail.size) def test_get_file_detail_missing_file(self): '''Test files that go missing during a walk''' @@ -447,26 +466,29 @@ class TestUpload(testtools.TestCase): ) # Get some test files to upload - files = [ - FileDetail( - os.path.join(FIXTURE_DIR, "logs/job-output.json"), - "job-output.json", - ), - FileDetail( - os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"), - "inventory.yaml", - ), - ] - - expected_failures = [ - { - "file": "job-output.json", - "error": ( - "Error posting file after multiple attempts: " - "Failed for a reason" + with FileFixture() as file_fixture: + files = [ + FileDetail( + os.path.join(file_fixture.root, "logs/job-output.json"), + "job-output.json", ), - }, - ] + FileDetail( + os.path.join( + file_fixture.root, "logs/zuul-info/inventory.yaml" + ), + "inventory.yaml", + ), + ] - failures = uploader.upload(files) - self.assertEqual(expected_failures, failures) + expected_failures = [ + { + "file": "job-output.json", + "error": ( + "Error posting file after multiple attempts: " + "Failed for a reason" + ), + }, + ] + + failures = uploader.upload(files) + self.assertEqual(expected_failures, failures) diff --git a/roles/upload-logs-base/library/filefixture.py b/roles/upload-logs-base/library/filefixture.py new file mode 100644 index 000000000..615b26ab1 --- /dev/null +++ b/roles/upload-logs-base/library/filefixture.py @@ -0,0 +1,139 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Handle file name special characters in a file tree. + +All files stored in a filetree can be renamed to urlencoded filenames. +A file tree can also be copied to a temporary location with file names +decoded, to be used in tests with special characters that are not always +possible to store on all file systems. + +""" + +from __future__ import print_function + +import os +try: + from urllib.parse import quote as urlib_quote + from urllib.parse import unquote as urlib_unquote +except ImportError: + from urllib import quote as urlib_quote + from urllib import unquote as urlib_unquote +import argparse +import fixtures +import tempfile +import shutil + + +FIXTURE_DIR = os.path.join(os.path.dirname(__file__), + 'test-fixtures') + +SAFE_CHARS = "\\/" + + +def portable_makedirs_exist_ok(path): + try: + os.makedirs(path, exist_ok=True) + except TypeError as err: + if "unexpected keyword argument" not in str(err): + raise err + if not os.path.exists(path): + try: + os.makedirs(path) + except OSError as err: + if "File exists" not in err: + raise err + + +def urlencode_filetree(): + for root, _, files in os.walk(FIXTURE_DIR): + for filename in files: + os.rename( + os.path.join(root, filename), + os.path.join( + root, urlib_quote(urlib_unquote(filename), SAFE_CHARS) + ) + ) + + +def populate_filetree(dst_dir=None): + + if not os.path.exists(FIXTURE_DIR): + return None + + if not dst_dir: + dst_dir = tempfile.mkdtemp() + + portable_makedirs_exist_ok(dst_dir) + + for root, dirs, files in os.walk(FIXTURE_DIR): + dst_root = root.replace(FIXTURE_DIR, dst_dir, 1) + for directory in dirs: + portable_makedirs_exist_ok(os.path.join(dst_root, directory)) + for filename in files: + try: + shutil.copyfile( + os.path.join(root, filename), + os.path.join(dst_root, urlib_unquote(filename)) + ) + except IOError as err: + print( + "\nFile {}".format( + os.path.join(dst_root, urlib_unquote(filename)) + ), + "\nnot possible to write to disk,", + "\npossibly due to filename not being valid on Windows?\n" + ) + shutil.rmtree(dst_dir) + raise err + + return dst_dir + + +class FileFixture(fixtures.Fixture): + + def _setUp(self): + self.root = tempfile.mkdtemp() + self.addCleanup(self.local_clean_up) + populate_filetree(self.root) + # There is no cleanup action, as the filetree is left intact for other + # tests to use + + def local_clean_up(self): + shutil.rmtree(self.root) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(__doc__) + parser.add_argument( + '--populate', + help="Causes files in {}".format(FIXTURE_DIR) + + "to be copied with decoded file name to a tmp dir" + + "Overrides --encode", + action='store_true' + ) + parser.add_argument( + '--encode', + help="Causes files under {} to be renamed with urlencoding.".format( + FIXTURE_DIR + ) + "DEFAULT behaviour, overridden by --populate", + action='store_true' + ) + args = parser.parse_args() + + if args.populate: + print(populate_filetree()) + else: + urlencode_filetree() diff --git a/roles/upload-logs-base/library/test-fixtures/.gitattributes b/roles/upload-logs-base/library/test-fixtures/.gitattributes new file mode 100644 index 000000000..2325d0821 --- /dev/null +++ b/roles/upload-logs-base/library/test-fixtures/.gitattributes @@ -0,0 +1 @@ +*.* eol=lf \ No newline at end of file diff --git a/roles/upload-logs-base/library/test-fixtures/logs/Ꮓບບξ-unicode.txt b/roles/upload-logs-base/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt similarity index 100% rename from roles/upload-logs-base/library/test-fixtures/logs/Ꮓບບξ-unicode.txt rename to roles/upload-logs-base/library/test-fixtures/logs/%E1%8F%83%E0%BA%9A%E0%BA%9A%CE%BE-unicode.txt diff --git a/roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo::3.txt b/roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt similarity index 100% rename from roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo::3.txt rename to roles/upload-logs-base/library/test-fixtures/logs/controller/subdir/foo%3A%3A3.txt diff --git a/roles/upload-logs-base/library/test_index.py b/roles/upload-logs-base/library/test_index.py index 69832d473..40df94777 100644 --- a/roles/upload-logs-base/library/test_index.py +++ b/roles/upload-logs-base/library/test_index.py @@ -33,6 +33,7 @@ import requests from bs4 import BeautifulSoup from .zuul_swift_upload import Uploader from ..module_utils.zuul_jobs.upload_utils import FileList, Indexer, FileDetail +from .filefixture import FileFixture FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'test-fixtures') @@ -50,8 +51,11 @@ class SymlinkFixture(fixtures.Fixture): ] def _setUp(self): + self.file_fixture = FileFixture() + self.file_fixture.setUp() + self.addCleanup(self.file_fixture.cleanUp) for (src, target) in self.links: - path = os.path.join(FIXTURE_DIR, 'links', src) + path = os.path.join(self.file_fixture.root, 'links', src) os.symlink(target, path) self.addCleanup(os.unlink, path) @@ -88,66 +92,69 @@ class TestFileList(testtools.TestCase): '''Test a single directory with a trailing slash''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ]) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs/')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ]) def test_single_dir(self): '''Test a single directory without a trailing slash''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('logs', 'application/directory', None), - ('logs/controller', 'application/directory', None), - ('logs/zuul-info', 'application/directory', None), - ('logs/job-output.json', 'application/json', None), - (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('logs/controller/subdir', 'application/directory', None), - ('logs/controller/compressed.gz', 'text/plain', 'gzip'), - ('logs/controller/cpu-load.svg', 'image/svg+xml', None), - ('logs/controller/journal.xz', 'text/plain', 'xz'), - ('logs/controller/service_log.txt', 'text/plain', None), - ('logs/controller/syslog', 'text/plain', None), - ('logs/controller/subdir/foo::3.txt', 'text/plain', None), - ('logs/controller/subdir/subdir.txt', 'text/plain', None), - ('logs/zuul-info/inventory.yaml', 'text/plain', None), - ('logs/zuul-info/zuul-info.controller.txt', - 'text/plain', None), - ]) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('logs', 'application/directory', None), + ('logs/controller', 'application/directory', None), + ('logs/zuul-info', 'application/directory', None), + ('logs/job-output.json', 'application/json', None), + (u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('logs/controller/subdir', 'application/directory', None), + ('logs/controller/compressed.gz', 'text/plain', 'gzip'), + ('logs/controller/cpu-load.svg', 'image/svg+xml', None), + ('logs/controller/journal.xz', 'text/plain', 'xz'), + ('logs/controller/service_log.txt', 'text/plain', None), + ('logs/controller/syslog', 'text/plain', None), + ('logs/controller/subdir/foo::3.txt', 'text/plain', None), + ('logs/controller/subdir/subdir.txt', 'text/plain', None), + ('logs/zuul-info/inventory.yaml', 'text/plain', None), + ('logs/zuul-info/zuul-info.controller.txt', + 'text/plain', None), + ]) def test_single_file(self): '''Test a single file''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, - 'logs/zuul-info/inventory.yaml')) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('inventory.yaml', 'text/plain', None), - ]) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, + 'logs/zuul-info/inventory.yaml')) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('inventory.yaml', 'text/plain', None), + ]) def test_symlinks(self): '''Test symlinks''' with FileList() as fl: - self.useFixture(SymlinkFixture()) - fl.add(os.path.join(FIXTURE_DIR, 'links/')) + symlink_fixture = self.useFixture(SymlinkFixture()) + fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/')) self.assert_files(fl, [ ('', 'application/directory', None), ('controller', 'application/directory', None), @@ -165,7 +172,8 @@ class TestFileList(testtools.TestCase): def test_index_files(self): '''Test index generation''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs')) + symlink_fixture = self.useFixture(SymlinkFixture()) + fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs')) ix = Indexer(fl) ix.make_indexes() @@ -223,32 +231,33 @@ class TestFileList(testtools.TestCase): def test_index_files_trailing_slash(self): '''Test index generation with a trailing slash''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes() + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs/')) + ix = Indexer(fl) + ix.make_indexes() - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('index.html', 'text/html', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/index.html', 'text/html', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('controller/subdir/index.html', 'text/html', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ('zuul-info/index.html', 'text/html', None), + ]) top_index = self.find_file(fl, 'index.html') page = open(top_index.full_path).read() @@ -282,141 +291,154 @@ class TestFileList(testtools.TestCase): def test_topdir_parent_link(self): '''Test index generation creates topdir parent link''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=True, - create_topdir_parent_link=True) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs/')) + ix = Indexer(fl) + ix.make_indexes( + create_parent_links=True, + create_topdir_parent_link=True) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('index.html', 'text/html', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/index.html', 'text/html', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('controller/subdir/index.html', 'text/html', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ('zuul-info/index.html', 'text/html', None), + ]) - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] + top_index = self.find_file(fl, 'index.html') + page = open(top_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] - self.assertEqual(len(rows), 5) + self.assertEqual(len(rows), 5) - self.assertEqual(rows[0].find('a').get('href'), - '../index.html') - self.assertEqual(rows[0].find('a').text, '../') + self.assertEqual(rows[0].find('a').get('href'), + '../index.html') + self.assertEqual(rows[0].find('a').text, '../') - self.assertEqual(rows[1].find('a').get('href'), - 'controller/index.html') - self.assertEqual(rows[1].find('a').text, 'controller/') + self.assertEqual(rows[1].find('a').get('href'), + 'controller/index.html') + self.assertEqual(rows[1].find('a').text, 'controller/') - self.assertEqual(rows[2].find('a').get('href'), - 'zuul-info/index.html') - self.assertEqual(rows[2].find('a').text, 'zuul-info/') + self.assertEqual(rows[2].find('a').get('href'), + 'zuul-info/index.html') + self.assertEqual(rows[2].find('a').text, 'zuul-info/') - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] - self.assertEqual(rows[0].find('a').get('href'), '../index.html') - self.assertEqual(rows[0].find('a').text, '../') + subdir_index = self.find_file( + fl, 'controller/subdir/index.html' + ) + page = open(subdir_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] + self.assertEqual( + rows[0].find('a').get('href'), '../index.html' + ) + self.assertEqual(rows[0].find('a').text, '../') - # Test proper escaping of files with funny names - self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[1].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[2].find('a').text, 'subdir.txt') + # Test proper escaping of files with funny names + self.assertEqual( + rows[1].find('a').get('href'), 'foo%3A%3A3.txt' + ) + self.assertEqual(rows[1].find('a').text, 'foo::3.txt') + # Test files without escaping + self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt') + self.assertEqual(rows[2].find('a').text, 'subdir.txt') def test_no_parent_links(self): '''Test index generation creates topdir parent link''' with FileList() as fl: - fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl) - ix.make_indexes( - create_parent_links=False, - create_topdir_parent_link=False) + with FileFixture() as file_fixture: + fl.add(os.path.join(file_fixture.root, 'logs/')) + ix = Indexer(fl) + ix.make_indexes( + create_parent_links=False, + create_topdir_parent_link=False) - self.assert_files(fl, [ - ('', 'application/directory', None), - ('controller', 'application/directory', None), - ('zuul-info', 'application/directory', None), - ('job-output.json', 'application/json', None), - (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', - 'text/plain', None), - ('index.html', 'text/html', None), - ('controller/subdir', 'application/directory', None), - ('controller/compressed.gz', 'text/plain', 'gzip'), - ('controller/cpu-load.svg', 'image/svg+xml', None), - ('controller/journal.xz', 'text/plain', 'xz'), - ('controller/service_log.txt', 'text/plain', None), - ('controller/syslog', 'text/plain', None), - ('controller/index.html', 'text/html', None), - ('controller/subdir/foo::3.txt', 'text/plain', None), - ('controller/subdir/subdir.txt', 'text/plain', None), - ('controller/subdir/index.html', 'text/html', None), - ('zuul-info/inventory.yaml', 'text/plain', None), - ('zuul-info/zuul-info.controller.txt', 'text/plain', None), - ('zuul-info/index.html', 'text/html', None), - ]) + self.assert_files(fl, [ + ('', 'application/directory', None), + ('controller', 'application/directory', None), + ('zuul-info', 'application/directory', None), + ('job-output.json', 'application/json', None), + (u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt', + 'text/plain', None), + ('index.html', 'text/html', None), + ('controller/subdir', 'application/directory', None), + ('controller/compressed.gz', 'text/plain', 'gzip'), + ('controller/cpu-load.svg', 'image/svg+xml', None), + ('controller/journal.xz', 'text/plain', 'xz'), + ('controller/service_log.txt', 'text/plain', None), + ('controller/syslog', 'text/plain', None), + ('controller/index.html', 'text/html', None), + ('controller/subdir/foo::3.txt', 'text/plain', None), + ('controller/subdir/subdir.txt', 'text/plain', None), + ('controller/subdir/index.html', 'text/html', None), + ('zuul-info/inventory.yaml', 'text/plain', None), + ('zuul-info/zuul-info.controller.txt', 'text/plain', None), + ('zuul-info/index.html', 'text/html', None), + ]) - top_index = self.find_file(fl, 'index.html') - page = open(top_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] + top_index = self.find_file(fl, 'index.html') + page = open(top_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] - self.assertEqual(len(rows), 4) + self.assertEqual(len(rows), 4) - self.assertEqual(rows[0].find('a').get('href'), - 'controller/index.html') - self.assertEqual(rows[0].find('a').text, - 'controller/') + self.assertEqual(rows[0].find('a').get('href'), + 'controller/index.html') + self.assertEqual(rows[0].find('a').text, + 'controller/') - self.assertEqual(rows[1].find('a').get('href'), - 'zuul-info/index.html') - self.assertEqual(rows[1].find('a').text, - 'zuul-info/') + self.assertEqual(rows[1].find('a').get('href'), + 'zuul-info/index.html') + self.assertEqual(rows[1].find('a').text, + 'zuul-info/') - subdir_index = self.find_file(fl, 'controller/subdir/index.html') - page = open(subdir_index.full_path).read() - page = BeautifulSoup(page, 'html.parser') - rows = page.find_all('tr')[1:] + subdir_index = self.find_file( + fl, 'controller/subdir/index.html' + ) + page = open(subdir_index.full_path).read() + page = BeautifulSoup(page, 'html.parser') + rows = page.find_all('tr')[1:] - # Test proper escaping of files with funny names - self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt') - self.assertEqual(rows[0].find('a').text, 'foo::3.txt') - # Test files without escaping - self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') - self.assertEqual(rows[1].find('a').text, 'subdir.txt') + # Test proper escaping of files with funny names + self.assertEqual( + rows[0].find('a').get('href'), 'foo%3A%3A3.txt' + ) + self.assertEqual(rows[0].find('a').text, 'foo::3.txt') + # Test files without escaping + self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt') + self.assertEqual(rows[1].find('a').text, 'subdir.txt') class TestFileDetail(testtools.TestCase): def test_get_file_detail(self): '''Test files info''' - path = os.path.join(FIXTURE_DIR, 'logs/job-output.json') - file_detail = FileDetail(path, '') - path_stat = os.stat(path) - self.assertEqual( - time.gmtime(path_stat[stat.ST_MTIME]), - file_detail.last_modified) - self.assertEqual(16, file_detail.size) + with FileFixture() as file_fixture: + path = os.path.join(file_fixture.root, 'logs/job-output.json') + file_detail = FileDetail(path, '') + path_stat = os.stat(path) + self.assertEqual( + time.gmtime(path_stat[stat.ST_MTIME]), + file_detail.last_modified) + self.assertEqual(16, file_detail.size) def test_get_file_detail_missing_file(self): '''Test files that go missing during a walk''' @@ -456,26 +478,29 @@ class TestUpload(testtools.TestCase): ) # Get some test files to upload - files = [ - FileDetail( - os.path.join(FIXTURE_DIR, "logs/job-output.json"), - "job-output.json", - ), - FileDetail( - os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"), - "inventory.yaml", - ), - ] - - expected_failures = [ - { - "file": "job-output.json", - "error": ( - "Error posting file after multiple attempts: " - "Failed for a reason" + with FileFixture() as file_fixture: + files = [ + FileDetail( + os.path.join(file_fixture.root, "logs/job-output.json"), + "job-output.json", ), - }, - ] + FileDetail( + os.path.join( + file_fixture.root, "logs/zuul-info/inventory.yaml" + ), + "inventory.yaml", + ), + ] - failures = uploader.upload(files) - self.assertEqual(expected_failures, failures) + expected_failures = [ + { + "file": "job-output.json", + "error": ( + "Error posting file after multiple attempts: " + "Failed for a reason" + ), + }, + ] + + failures = uploader.upload(files) + self.assertEqual(expected_failures, failures)