Merge "Use urlencoded filenames in test fixtures"
This commit is contained in:
Zuul
Gerrit Code Review
commit
5bb91e867b
1
roles/generate-zuul-manifest/library/test-fixtures/.gitattributes
vendored
Normal file
1
roles/generate-zuul-manifest/library/test-fixtures/.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.* eol=lf
|
||||
139
roles/test-upload-logs-swift/library/filefixture.py
Normal file
139
roles/test-upload-logs-swift/library/filefixture.py
Normal file
@ -0,0 +1,139 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Handle file name special characters in a file tree.
|
||||
|
||||
All files stored in a filetree can be renamed to urlencoded filenames.
|
||||
A file tree can also be copied to a temporary location with file names
|
||||
decoded, to be used in tests with special characters that are not always
|
||||
possible to store on all file systems.
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
try:
|
||||
from urllib.parse import quote as urlib_quote
|
||||
from urllib.parse import unquote as urlib_unquote
|
||||
except ImportError:
|
||||
from urllib import quote as urlib_quote
|
||||
from urllib import unquote as urlib_unquote
|
||||
import argparse
|
||||
import fixtures
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
|
||||
'test-fixtures')
|
||||
|
||||
SAFE_CHARS = "\\/"
|
||||
|
||||
|
||||
def portable_makedirs_exist_ok(path):
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
except TypeError as err:
|
||||
if "unexpected keyword argument" not in str(err):
|
||||
raise err
|
||||
if not os.path.exists(path):
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as err:
|
||||
if "File exists" not in err:
|
||||
raise err
|
||||
|
||||
|
||||
def urlencode_filetree():
|
||||
for root, _, files in os.walk(FIXTURE_DIR):
|
||||
for filename in files:
|
||||
os.rename(
|
||||
os.path.join(root, filename),
|
||||
os.path.join(
|
||||
root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def populate_filetree(dst_dir=None):
|
||||
|
||||
if not os.path.exists(FIXTURE_DIR):
|
||||
return None
|
||||
|
||||
if not dst_dir:
|
||||
dst_dir = tempfile.mkdtemp()
|
||||
|
||||
portable_makedirs_exist_ok(dst_dir)
|
||||
|
||||
for root, dirs, files in os.walk(FIXTURE_DIR):
|
||||
dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
|
||||
for directory in dirs:
|
||||
portable_makedirs_exist_ok(os.path.join(dst_root, directory))
|
||||
for filename in files:
|
||||
try:
|
||||
shutil.copyfile(
|
||||
os.path.join(root, filename),
|
||||
os.path.join(dst_root, urlib_unquote(filename))
|
||||
)
|
||||
except IOError as err:
|
||||
print(
|
||||
"\nFile {}".format(
|
||||
os.path.join(dst_root, urlib_unquote(filename))
|
||||
),
|
||||
"\nnot possible to write to disk,",
|
||||
"\npossibly due to filename not being valid on Windows?\n"
|
||||
)
|
||||
shutil.rmtree(dst_dir)
|
||||
raise err
|
||||
|
||||
return dst_dir
|
||||
|
||||
|
||||
class FileFixture(fixtures.Fixture):
|
||||
|
||||
def _setUp(self):
|
||||
self.root = tempfile.mkdtemp()
|
||||
self.addCleanup(self.local_clean_up)
|
||||
populate_filetree(self.root)
|
||||
# There is no cleanup action, as the filetree is left intact for other
|
||||
# tests to use
|
||||
|
||||
def local_clean_up(self):
|
||||
shutil.rmtree(self.root)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument(
|
||||
'--populate',
|
||||
help="Causes files in {}".format(FIXTURE_DIR) +
|
||||
"to be copied with decoded file name to a tmp dir" +
|
||||
"Overrides --encode",
|
||||
action='store_true'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--encode',
|
||||
help="Causes files under {} to be renamed with urlencoding.".format(
|
||||
FIXTURE_DIR
|
||||
) + "DEFAULT behaviour, overridden by --populate",
|
||||
action='store_true'
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.populate:
|
||||
print(populate_filetree())
|
||||
else:
|
||||
urlencode_filetree()
|
||||
1
roles/test-upload-logs-swift/library/test-fixtures/.gitattributes
vendored
Normal file
1
roles/test-upload-logs-swift/library/test-fixtures/.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.* eol=lf
|
||||
@ -32,7 +32,7 @@ except ImportError:
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from .zuul_swift_upload import FileList, Indexer, FileDetail, Uploader
|
||||
|
||||
from .filefixture import FileFixture
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
|
||||
'test-fixtures')
|
||||
@ -50,8 +50,11 @@ class SymlinkFixture(fixtures.Fixture):
|
||||
]
|
||||
|
||||
def _setUp(self):
|
||||
self.file_fixture = FileFixture()
|
||||
self.file_fixture.setUp()
|
||||
self.addCleanup(self.file_fixture.cleanUp)
|
||||
for (src, target) in self.links:
|
||||
path = os.path.join(FIXTURE_DIR, 'links', src)
|
||||
path = os.path.join(self.file_fixture.root, 'links', src)
|
||||
os.symlink(target, path)
|
||||
self.addCleanup(os.unlink, path)
|
||||
|
||||
@ -88,66 +91,69 @@ class TestFileList(testtools.TestCase):
|
||||
'''Test a single directory with a trailing slash'''
|
||||
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_dir(self):
|
||||
'''Test a single directory without a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_file(self):
|
||||
'''Test a single file'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR,
|
||||
'logs/zuul-info/inventory.yaml'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('inventory.yaml', 'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root,
|
||||
'logs/zuul-info/inventory.yaml'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('inventory.yaml', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_symlinks(self):
|
||||
'''Test symlinks'''
|
||||
with FileList() as fl:
|
||||
self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'links/'))
|
||||
symlink_fixture = self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
@ -165,7 +171,8 @@ class TestFileList(testtools.TestCase):
|
||||
def test_index_files(self):
|
||||
'''Test index generation'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
symlink_fixture = self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
@ -223,32 +230,33 @@ class TestFileList(testtools.TestCase):
|
||||
def test_index_files_trailing_slash(self):
|
||||
'''Test index generation with a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
@ -280,134 +288,145 @@ class TestFileList(testtools.TestCase):
|
||||
def test_topdir_parent_link(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=True,
|
||||
create_topdir_parent_link=True)
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=True,
|
||||
create_topdir_parent_link=True)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 5)
|
||||
self.assertEqual(len(rows), 5)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[1].find('a').text, 'controller/')
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[1].find('a').text, 'controller/')
|
||||
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
subdir_index = self.find_file(
|
||||
fl, 'controller/subdir/index.html'
|
||||
)
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(
|
||||
rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
|
||||
)
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
|
||||
def test_no_parent_links(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=False,
|
||||
create_topdir_parent_link=False)
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=False,
|
||||
create_topdir_parent_link=False)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 4)
|
||||
self.assertEqual(len(rows), 4)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[0].find('a').text, 'controller/')
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'controller/')
|
||||
self.assertEqual(rows[0].find('a').text, 'controller/')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').text, 'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').text, 'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
subdir_index = self.find_file(
|
||||
fl, 'controller/subdir/index.html'
|
||||
)
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(
|
||||
rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
|
||||
)
|
||||
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
|
||||
|
||||
|
||||
class TestFileDetail(testtools.TestCase):
|
||||
|
||||
def test_get_file_detail(self):
|
||||
'''Test files info'''
|
||||
path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
|
||||
file_detail = FileDetail(path, '')
|
||||
path_stat = os.stat(path)
|
||||
self.assertEqual(
|
||||
time.gmtime(path_stat[stat.ST_MTIME]),
|
||||
file_detail.last_modified)
|
||||
self.assertEqual(16, file_detail.size)
|
||||
with FileFixture() as file_fixture:
|
||||
path = os.path.join(file_fixture.root, 'logs/job-output.json')
|
||||
file_detail = FileDetail(path, '')
|
||||
path_stat = os.stat(path)
|
||||
self.assertEqual(
|
||||
time.gmtime(path_stat[stat.ST_MTIME]),
|
||||
file_detail.last_modified)
|
||||
self.assertEqual(16, file_detail.size)
|
||||
|
||||
def test_get_file_detail_missing_file(self):
|
||||
'''Test files that go missing during a walk'''
|
||||
@ -447,26 +466,29 @@ class TestUpload(testtools.TestCase):
|
||||
)
|
||||
|
||||
# Get some test files to upload
|
||||
files = [
|
||||
FileDetail(
|
||||
os.path.join(FIXTURE_DIR, "logs/job-output.json"),
|
||||
"job-output.json",
|
||||
),
|
||||
FileDetail(
|
||||
os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
|
||||
"inventory.yaml",
|
||||
),
|
||||
]
|
||||
|
||||
expected_failures = [
|
||||
{
|
||||
"file": "job-output.json",
|
||||
"error": (
|
||||
"Error posting file after multiple attempts: "
|
||||
"Failed for a reason"
|
||||
with FileFixture() as file_fixture:
|
||||
files = [
|
||||
FileDetail(
|
||||
os.path.join(file_fixture.root, "logs/job-output.json"),
|
||||
"job-output.json",
|
||||
),
|
||||
},
|
||||
]
|
||||
FileDetail(
|
||||
os.path.join(
|
||||
file_fixture.root, "logs/zuul-info/inventory.yaml"
|
||||
),
|
||||
"inventory.yaml",
|
||||
),
|
||||
]
|
||||
|
||||
failures = uploader.upload(files)
|
||||
self.assertEqual(expected_failures, failures)
|
||||
expected_failures = [
|
||||
{
|
||||
"file": "job-output.json",
|
||||
"error": (
|
||||
"Error posting file after multiple attempts: "
|
||||
"Failed for a reason"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
failures = uploader.upload(files)
|
||||
self.assertEqual(expected_failures, failures)
|
||||
|
||||
139
roles/upload-logs-base/library/filefixture.py
Normal file
139
roles/upload-logs-base/library/filefixture.py
Normal file
@ -0,0 +1,139 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
#
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Handle file name special characters in a file tree.
|
||||
|
||||
All files stored in a filetree can be renamed to urlencoded filenames.
|
||||
A file tree can also be copied to a temporary location with file names
|
||||
decoded, to be used in tests with special characters that are not always
|
||||
possible to store on all file systems.
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
try:
|
||||
from urllib.parse import quote as urlib_quote
|
||||
from urllib.parse import unquote as urlib_unquote
|
||||
except ImportError:
|
||||
from urllib import quote as urlib_quote
|
||||
from urllib import unquote as urlib_unquote
|
||||
import argparse
|
||||
import fixtures
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
|
||||
'test-fixtures')
|
||||
|
||||
SAFE_CHARS = "\\/"
|
||||
|
||||
|
||||
def portable_makedirs_exist_ok(path):
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
except TypeError as err:
|
||||
if "unexpected keyword argument" not in str(err):
|
||||
raise err
|
||||
if not os.path.exists(path):
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as err:
|
||||
if "File exists" not in err:
|
||||
raise err
|
||||
|
||||
|
||||
def urlencode_filetree():
|
||||
for root, _, files in os.walk(FIXTURE_DIR):
|
||||
for filename in files:
|
||||
os.rename(
|
||||
os.path.join(root, filename),
|
||||
os.path.join(
|
||||
root, urlib_quote(urlib_unquote(filename), SAFE_CHARS)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def populate_filetree(dst_dir=None):
|
||||
|
||||
if not os.path.exists(FIXTURE_DIR):
|
||||
return None
|
||||
|
||||
if not dst_dir:
|
||||
dst_dir = tempfile.mkdtemp()
|
||||
|
||||
portable_makedirs_exist_ok(dst_dir)
|
||||
|
||||
for root, dirs, files in os.walk(FIXTURE_DIR):
|
||||
dst_root = root.replace(FIXTURE_DIR, dst_dir, 1)
|
||||
for directory in dirs:
|
||||
portable_makedirs_exist_ok(os.path.join(dst_root, directory))
|
||||
for filename in files:
|
||||
try:
|
||||
shutil.copyfile(
|
||||
os.path.join(root, filename),
|
||||
os.path.join(dst_root, urlib_unquote(filename))
|
||||
)
|
||||
except IOError as err:
|
||||
print(
|
||||
"\nFile {}".format(
|
||||
os.path.join(dst_root, urlib_unquote(filename))
|
||||
),
|
||||
"\nnot possible to write to disk,",
|
||||
"\npossibly due to filename not being valid on Windows?\n"
|
||||
)
|
||||
shutil.rmtree(dst_dir)
|
||||
raise err
|
||||
|
||||
return dst_dir
|
||||
|
||||
|
||||
class FileFixture(fixtures.Fixture):
|
||||
|
||||
def _setUp(self):
|
||||
self.root = tempfile.mkdtemp()
|
||||
self.addCleanup(self.local_clean_up)
|
||||
populate_filetree(self.root)
|
||||
# There is no cleanup action, as the filetree is left intact for other
|
||||
# tests to use
|
||||
|
||||
def local_clean_up(self):
|
||||
shutil.rmtree(self.root)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = argparse.ArgumentParser(__doc__)
|
||||
parser.add_argument(
|
||||
'--populate',
|
||||
help="Causes files in {}".format(FIXTURE_DIR) +
|
||||
"to be copied with decoded file name to a tmp dir" +
|
||||
"Overrides --encode",
|
||||
action='store_true'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--encode',
|
||||
help="Causes files under {} to be renamed with urlencoding.".format(
|
||||
FIXTURE_DIR
|
||||
) + "DEFAULT behaviour, overridden by --populate",
|
||||
action='store_true'
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.populate:
|
||||
print(populate_filetree())
|
||||
else:
|
||||
urlencode_filetree()
|
||||
1
roles/upload-logs-base/library/test-fixtures/.gitattributes
vendored
Normal file
1
roles/upload-logs-base/library/test-fixtures/.gitattributes
vendored
Normal file
@ -0,0 +1 @@
|
||||
*.* eol=lf
|
||||
@ -33,6 +33,7 @@ import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from .zuul_swift_upload import Uploader
|
||||
from ..module_utils.zuul_jobs.upload_utils import FileList, Indexer, FileDetail
|
||||
from .filefixture import FileFixture
|
||||
|
||||
FIXTURE_DIR = os.path.join(os.path.dirname(__file__),
|
||||
'test-fixtures')
|
||||
@ -50,8 +51,11 @@ class SymlinkFixture(fixtures.Fixture):
|
||||
]
|
||||
|
||||
def _setUp(self):
|
||||
self.file_fixture = FileFixture()
|
||||
self.file_fixture.setUp()
|
||||
self.addCleanup(self.file_fixture.cleanUp)
|
||||
for (src, target) in self.links:
|
||||
path = os.path.join(FIXTURE_DIR, 'links', src)
|
||||
path = os.path.join(self.file_fixture.root, 'links', src)
|
||||
os.symlink(target, path)
|
||||
self.addCleanup(os.unlink, path)
|
||||
|
||||
@ -88,66 +92,69 @@ class TestFileList(testtools.TestCase):
|
||||
'''Test a single directory with a trailing slash'''
|
||||
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_dir(self):
|
||||
'''Test a single directory without a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('logs', 'application/directory', None),
|
||||
('logs/controller', 'application/directory', None),
|
||||
('logs/zuul-info', 'application/directory', None),
|
||||
('logs/job-output.json', 'application/json', None),
|
||||
(u'logs/\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('logs/controller/subdir', 'application/directory', None),
|
||||
('logs/controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('logs/controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('logs/controller/journal.xz', 'text/plain', 'xz'),
|
||||
('logs/controller/service_log.txt', 'text/plain', None),
|
||||
('logs/controller/syslog', 'text/plain', None),
|
||||
('logs/controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('logs/controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('logs/zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('logs/zuul-info/zuul-info.controller.txt',
|
||||
'text/plain', None),
|
||||
])
|
||||
|
||||
def test_single_file(self):
|
||||
'''Test a single file'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR,
|
||||
'logs/zuul-info/inventory.yaml'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('inventory.yaml', 'text/plain', None),
|
||||
])
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root,
|
||||
'logs/zuul-info/inventory.yaml'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('inventory.yaml', 'text/plain', None),
|
||||
])
|
||||
|
||||
def test_symlinks(self):
|
||||
'''Test symlinks'''
|
||||
with FileList() as fl:
|
||||
self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'links/'))
|
||||
symlink_fixture = self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'links/'))
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
@ -165,7 +172,8 @@ class TestFileList(testtools.TestCase):
|
||||
def test_index_files(self):
|
||||
'''Test index generation'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs'))
|
||||
symlink_fixture = self.useFixture(SymlinkFixture())
|
||||
fl.add(os.path.join(symlink_fixture.file_fixture.root, 'logs'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
@ -223,32 +231,33 @@ class TestFileList(testtools.TestCase):
|
||||
def test_index_files_trailing_slash(self):
|
||||
'''Test index generation with a trailing slash'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes()
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
@ -282,141 +291,154 @@ class TestFileList(testtools.TestCase):
|
||||
def test_topdir_parent_link(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=True,
|
||||
create_topdir_parent_link=True)
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=True,
|
||||
create_topdir_parent_link=True)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 5)
|
||||
self.assertEqual(len(rows), 5)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'),
|
||||
'../index.html')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
self.assertEqual(rows[0].find('a').get('href'),
|
||||
'../index.html')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'),
|
||||
'controller/index.html')
|
||||
self.assertEqual(rows[1].find('a').text, 'controller/')
|
||||
self.assertEqual(rows[1].find('a').get('href'),
|
||||
'controller/index.html')
|
||||
self.assertEqual(rows[1].find('a').text, 'controller/')
|
||||
|
||||
self.assertEqual(rows[2].find('a').get('href'),
|
||||
'zuul-info/index.html')
|
||||
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
|
||||
self.assertEqual(rows[2].find('a').get('href'),
|
||||
'zuul-info/index.html')
|
||||
self.assertEqual(rows[2].find('a').text, 'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(rows[0].find('a').get('href'), '../index.html')
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
subdir_index = self.find_file(
|
||||
fl, 'controller/subdir/index.html'
|
||||
)
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
self.assertEqual(
|
||||
rows[0].find('a').get('href'), '../index.html'
|
||||
)
|
||||
self.assertEqual(rows[0].find('a').text, '../')
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(
|
||||
rows[1].find('a').get('href'), 'foo%3A%3A3.txt'
|
||||
)
|
||||
self.assertEqual(rows[1].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[2].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[2].find('a').text, 'subdir.txt')
|
||||
|
||||
def test_no_parent_links(self):
|
||||
'''Test index generation creates topdir parent link'''
|
||||
with FileList() as fl:
|
||||
fl.add(os.path.join(FIXTURE_DIR, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=False,
|
||||
create_topdir_parent_link=False)
|
||||
with FileFixture() as file_fixture:
|
||||
fl.add(os.path.join(file_fixture.root, 'logs/'))
|
||||
ix = Indexer(fl)
|
||||
ix.make_indexes(
|
||||
create_parent_links=False,
|
||||
create_topdir_parent_link=False)
|
||||
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
self.assert_files(fl, [
|
||||
('', 'application/directory', None),
|
||||
('controller', 'application/directory', None),
|
||||
('zuul-info', 'application/directory', None),
|
||||
('job-output.json', 'application/json', None),
|
||||
(u'\u13c3\u0e9a\u0e9a\u03be-unicode.txt',
|
||||
'text/plain', None),
|
||||
('index.html', 'text/html', None),
|
||||
('controller/subdir', 'application/directory', None),
|
||||
('controller/compressed.gz', 'text/plain', 'gzip'),
|
||||
('controller/cpu-load.svg', 'image/svg+xml', None),
|
||||
('controller/journal.xz', 'text/plain', 'xz'),
|
||||
('controller/service_log.txt', 'text/plain', None),
|
||||
('controller/syslog', 'text/plain', None),
|
||||
('controller/index.html', 'text/html', None),
|
||||
('controller/subdir/foo::3.txt', 'text/plain', None),
|
||||
('controller/subdir/subdir.txt', 'text/plain', None),
|
||||
('controller/subdir/index.html', 'text/html', None),
|
||||
('zuul-info/inventory.yaml', 'text/plain', None),
|
||||
('zuul-info/zuul-info.controller.txt', 'text/plain', None),
|
||||
('zuul-info/index.html', 'text/html', None),
|
||||
])
|
||||
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
top_index = self.find_file(fl, 'index.html')
|
||||
page = open(top_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
self.assertEqual(len(rows), 4)
|
||||
self.assertEqual(len(rows), 4)
|
||||
|
||||
self.assertEqual(rows[0].find('a').get('href'),
|
||||
'controller/index.html')
|
||||
self.assertEqual(rows[0].find('a').text,
|
||||
'controller/')
|
||||
self.assertEqual(rows[0].find('a').get('href'),
|
||||
'controller/index.html')
|
||||
self.assertEqual(rows[0].find('a').text,
|
||||
'controller/')
|
||||
|
||||
self.assertEqual(rows[1].find('a').get('href'),
|
||||
'zuul-info/index.html')
|
||||
self.assertEqual(rows[1].find('a').text,
|
||||
'zuul-info/')
|
||||
self.assertEqual(rows[1].find('a').get('href'),
|
||||
'zuul-info/index.html')
|
||||
self.assertEqual(rows[1].find('a').text,
|
||||
'zuul-info/')
|
||||
|
||||
subdir_index = self.find_file(fl, 'controller/subdir/index.html')
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
subdir_index = self.find_file(
|
||||
fl, 'controller/subdir/index.html'
|
||||
)
|
||||
page = open(subdir_index.full_path).read()
|
||||
page = BeautifulSoup(page, 'html.parser')
|
||||
rows = page.find_all('tr')[1:]
|
||||
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(rows[0].find('a').get('href'), 'foo%3A%3A3.txt')
|
||||
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
|
||||
# Test proper escaping of files with funny names
|
||||
self.assertEqual(
|
||||
rows[0].find('a').get('href'), 'foo%3A%3A3.txt'
|
||||
)
|
||||
self.assertEqual(rows[0].find('a').text, 'foo::3.txt')
|
||||
# Test files without escaping
|
||||
self.assertEqual(rows[1].find('a').get('href'), 'subdir.txt')
|
||||
self.assertEqual(rows[1].find('a').text, 'subdir.txt')
|
||||
|
||||
|
||||
class TestFileDetail(testtools.TestCase):
|
||||
|
||||
def test_get_file_detail(self):
|
||||
'''Test files info'''
|
||||
path = os.path.join(FIXTURE_DIR, 'logs/job-output.json')
|
||||
file_detail = FileDetail(path, '')
|
||||
path_stat = os.stat(path)
|
||||
self.assertEqual(
|
||||
time.gmtime(path_stat[stat.ST_MTIME]),
|
||||
file_detail.last_modified)
|
||||
self.assertEqual(16, file_detail.size)
|
||||
with FileFixture() as file_fixture:
|
||||
path = os.path.join(file_fixture.root, 'logs/job-output.json')
|
||||
file_detail = FileDetail(path, '')
|
||||
path_stat = os.stat(path)
|
||||
self.assertEqual(
|
||||
time.gmtime(path_stat[stat.ST_MTIME]),
|
||||
file_detail.last_modified)
|
||||
self.assertEqual(16, file_detail.size)
|
||||
|
||||
def test_get_file_detail_missing_file(self):
|
||||
'''Test files that go missing during a walk'''
|
||||
@ -456,26 +478,29 @@ class TestUpload(testtools.TestCase):
|
||||
)
|
||||
|
||||
# Get some test files to upload
|
||||
files = [
|
||||
FileDetail(
|
||||
os.path.join(FIXTURE_DIR, "logs/job-output.json"),
|
||||
"job-output.json",
|
||||
),
|
||||
FileDetail(
|
||||
os.path.join(FIXTURE_DIR, "logs/zuul-info/inventory.yaml"),
|
||||
"inventory.yaml",
|
||||
),
|
||||
]
|
||||
|
||||
expected_failures = [
|
||||
{
|
||||
"file": "job-output.json",
|
||||
"error": (
|
||||
"Error posting file after multiple attempts: "
|
||||
"Failed for a reason"
|
||||
with FileFixture() as file_fixture:
|
||||
files = [
|
||||
FileDetail(
|
||||
os.path.join(file_fixture.root, "logs/job-output.json"),
|
||||
"job-output.json",
|
||||
),
|
||||
},
|
||||
]
|
||||
FileDetail(
|
||||
os.path.join(
|
||||
file_fixture.root, "logs/zuul-info/inventory.yaml"
|
||||
),
|
||||
"inventory.yaml",
|
||||
),
|
||||
]
|
||||
|
||||
failures = uploader.upload(files)
|
||||
self.assertEqual(expected_failures, failures)
|
||||
expected_failures = [
|
||||
{
|
||||
"file": "job-output.json",
|
||||
"error": (
|
||||
"Error posting file after multiple attempts: "
|
||||
"Failed for a reason"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
failures = uploader.upload(files)
|
||||
self.assertEqual(expected_failures, failures)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user