Add binary parameter to execute and ssh_execute
Add an optional binary parameter to execute() and ssh_execute() functions of oslo_concurrency.processutils. If binary is True, stdout and stderr are returned as byte strings on Python 2 and Python 3. When Nova will be ported to Python 3, binary=True will be needed to get binary outputs (like an encryption key, see the bug #1410348). Add new tests. This changeset does not change the default behaviour. Change-Id: I283cadf04781942b65f211237070cc2354939cdb Related-Bug: 1410348
This commit is contained in:
Victor Stinner
parent
c05c8c37be
commit
4d7dce2ea8
@ -144,6 +144,9 @@ def execute(*cmd, **kwargs):
|
||||
last attempt, and LOG_ALL_ERRORS requires
|
||||
logging on each occurence of an error.
|
||||
:type log_errors: integer.
|
||||
:param binary: On Python 3, return stdout and stderr as bytes if
|
||||
binary is True, as Unicode otherwise.
|
||||
:type binary: boolean
|
||||
:returns: (stdout, stderr) from process execution
|
||||
:raises: :class:`UnknownArgumentError` on
|
||||
receiving unknown arguments
|
||||
@ -163,6 +166,7 @@ def execute(*cmd, **kwargs):
|
||||
shell = kwargs.pop('shell', False)
|
||||
loglevel = kwargs.pop('loglevel', logging.DEBUG)
|
||||
log_errors = kwargs.pop('log_errors', None)
|
||||
binary = kwargs.pop('binary', False)
|
||||
|
||||
if isinstance(check_exit_code, bool):
|
||||
ignore_exit_code = not check_exit_code
|
||||
@ -223,22 +227,26 @@ def execute(*cmd, **kwargs):
|
||||
end_time = time.time() - start_time
|
||||
LOG.log(loglevel, 'CMD "%s" returned: %s in %0.3fs' %
|
||||
(sanitized_cmd, _returncode, end_time))
|
||||
if result is not None and six.PY3:
|
||||
(stdout, stderr) = result
|
||||
# Decode from the locale using using the surrogateescape error
|
||||
# handler (decoding cannot fail)
|
||||
stdout = os.fsdecode(stdout)
|
||||
stderr = os.fsdecode(stderr)
|
||||
result = (stdout, stderr)
|
||||
if not ignore_exit_code and _returncode not in check_exit_code:
|
||||
(stdout, stderr) = result
|
||||
if six.PY3:
|
||||
stdout = os.fsdecode(stdout)
|
||||
stderr = os.fsdecode(stderr)
|
||||
sanitized_stdout = strutils.mask_password(stdout)
|
||||
sanitized_stderr = strutils.mask_password(stderr)
|
||||
raise ProcessExecutionError(exit_code=_returncode,
|
||||
stdout=sanitized_stdout,
|
||||
stderr=sanitized_stderr,
|
||||
cmd=sanitized_cmd)
|
||||
return result
|
||||
if six.PY3 and not binary and result is not None:
|
||||
(stdout, stderr) = result
|
||||
# Decode from the locale using using the surrogateescape error
|
||||
# handler (decoding cannot fail)
|
||||
stdout = os.fsdecode(stdout)
|
||||
stderr = os.fsdecode(stderr)
|
||||
return (stdout, stderr)
|
||||
else:
|
||||
return result
|
||||
|
||||
except (ProcessExecutionError, OSError) as err:
|
||||
# if we want to always log the errors or if this is
|
||||
@ -308,7 +316,8 @@ def trycmd(*args, **kwargs):
|
||||
|
||||
|
||||
def ssh_execute(ssh, cmd, process_input=None,
|
||||
addl_env=None, check_exit_code=True):
|
||||
addl_env=None, check_exit_code=True,
|
||||
binary=False):
|
||||
sanitized_cmd = strutils.mask_password(cmd)
|
||||
LOG.debug('Running cmd (SSH): %s', sanitized_cmd)
|
||||
if addl_env:
|
||||
@ -332,7 +341,8 @@ def ssh_execute(ssh, cmd, process_input=None,
|
||||
|
||||
if six.PY3:
|
||||
# Decode from the locale using using the surrogateescape error handler
|
||||
# (decoding cannot fail)
|
||||
# (decoding cannot fail). Decode even if binary is True because
|
||||
# mask_password() requires Unicode on Python 3
|
||||
stdout = os.fsdecode(stdout)
|
||||
stderr = os.fsdecode(stderr)
|
||||
stdout = strutils.mask_password(stdout)
|
||||
@ -347,6 +357,21 @@ def ssh_execute(ssh, cmd, process_input=None,
|
||||
stderr=stderr,
|
||||
cmd=sanitized_cmd)
|
||||
|
||||
if binary:
|
||||
if six.PY2:
|
||||
# On Python 2, stdout is a bytes string if mask_password() failed
|
||||
# to decode it, or an Unicode string otherwise. Encode to the
|
||||
# default encoding (ASCII) because mask_password() decodes from
|
||||
# the same encoding.
|
||||
if isinstance(stdout, unicode):
|
||||
stdout = stdout.encode()
|
||||
if isinstance(stderr, unicode):
|
||||
stderr = stderr.encode()
|
||||
else:
|
||||
# fsencode() is the reverse operation of fsdecode()
|
||||
stdout = os.fsencode(stdout)
|
||||
stderr = os.fsencode(stderr)
|
||||
|
||||
return (stdout, stderr)
|
||||
|
||||
|
||||
|
||||
@ -312,6 +312,17 @@ grep foo
|
||||
|
||||
self.assertIn('SUPER_UNIQUE_VAR=The answer is 42', out)
|
||||
|
||||
def test_binary(self):
|
||||
env_vars = {'SUPER_UNIQUE_VAR': 'The answer is 42'}
|
||||
|
||||
out, err = processutils.execute('/usr/bin/env',
|
||||
env_variables=env_vars,
|
||||
binary=True)
|
||||
self.assertIsInstance(out, bytes)
|
||||
self.assertIsInstance(err, bytes)
|
||||
|
||||
self.assertIn(b'SUPER_UNIQUE_VAR=The answer is 42', out)
|
||||
|
||||
def test_exception_and_masking(self):
|
||||
tmpfilename = self.create_tempfiles(
|
||||
[["test_exceptions_and_masking",
|
||||
@ -338,7 +349,8 @@ grep foo
|
||||
'something']))
|
||||
self.assertNotIn('secret', str(err))
|
||||
|
||||
def check_undecodable_bytes(self, out_bytes, err_bytes, exitcode=0):
|
||||
def execute_undecodable_bytes(self, out_bytes, err_bytes,
|
||||
exitcode=0, binary=False):
|
||||
if six.PY3:
|
||||
code = ';'.join(('import sys',
|
||||
'sys.stdout.buffer.write(%a)' % out_bytes,
|
||||
@ -354,37 +366,55 @@ grep foo
|
||||
'sys.stderr.flush()',
|
||||
'sys.exit(%s)' % exitcode))
|
||||
|
||||
return processutils.execute(sys.executable, '-c', code)
|
||||
return processutils.execute(sys.executable, '-c', code, binary=binary)
|
||||
|
||||
def test_undecodable_bytes(self):
|
||||
def check_undecodable_bytes(self, binary):
|
||||
out_bytes = b'out: ' + UNDECODABLE_BYTES
|
||||
err_bytes = b'err: ' + UNDECODABLE_BYTES
|
||||
out, err = self.check_undecodable_bytes(out_bytes, err_bytes)
|
||||
if six.PY3:
|
||||
out, err = self.execute_undecodable_bytes(out_bytes, err_bytes,
|
||||
binary=binary)
|
||||
if six.PY3 and not binary:
|
||||
self.assertEqual(out, os.fsdecode(out_bytes))
|
||||
self.assertEqual(err, os.fsdecode(err_bytes))
|
||||
else:
|
||||
self.assertEqual(out, out_bytes)
|
||||
self.assertEqual(err, err_bytes)
|
||||
|
||||
def test_undecodable_bytes_error(self):
|
||||
def test_undecodable_bytes(self):
|
||||
self.check_undecodable_bytes(False)
|
||||
|
||||
def test_binary_undecodable_bytes(self):
|
||||
self.check_undecodable_bytes(True)
|
||||
|
||||
def check_undecodable_bytes_error(self, binary):
|
||||
out_bytes = b'out: password="secret1" ' + UNDECODABLE_BYTES
|
||||
err_bytes = b'err: password="secret2" ' + UNDECODABLE_BYTES
|
||||
exc = self.assertRaises(processutils.ProcessExecutionError,
|
||||
self.check_undecodable_bytes,
|
||||
out_bytes, err_bytes, exitcode=1)
|
||||
self.execute_undecodable_bytes,
|
||||
out_bytes, err_bytes, exitcode=1,
|
||||
binary=binary)
|
||||
|
||||
out = exc.stdout
|
||||
err = exc.stderr
|
||||
out_bytes = b'out: password="***" ' + UNDECODABLE_BYTES
|
||||
err_bytes = b'err: password="***" ' + UNDECODABLE_BYTES
|
||||
if six.PY3:
|
||||
# On Python 3, stdout and stderr attributes of
|
||||
# ProcessExecutionError must always be Unicode
|
||||
self.assertEqual(out, os.fsdecode(out_bytes))
|
||||
self.assertEqual(err, os.fsdecode(err_bytes))
|
||||
else:
|
||||
# On Python 2, stdout and stderr attributes of
|
||||
# ProcessExecutionError must always be bytes
|
||||
self.assertEqual(out, out_bytes)
|
||||
self.assertEqual(err, err_bytes)
|
||||
|
||||
def test_undecodable_bytes_error(self):
|
||||
self.check_undecodable_bytes_error(False)
|
||||
|
||||
def test_binary_undecodable_bytes_error(self):
|
||||
self.check_undecodable_bytes_error(True)
|
||||
|
||||
|
||||
class ProcessExecutionErrorLoggingTest(test_base.BaseTestCase):
|
||||
def setUp(self):
|
||||
@ -527,40 +557,65 @@ class SshExecuteTestCase(test_base.BaseTestCase):
|
||||
self.assertIsInstance(out, six.text_type)
|
||||
self.assertIsInstance(err, six.text_type)
|
||||
|
||||
def test_undecodable_bytes(self):
|
||||
def test_binary(self):
|
||||
o, e = processutils.ssh_execute(FakeSshConnection(0), 'ls',
|
||||
binary=True)
|
||||
self.assertEqual(b'stdout', o)
|
||||
self.assertEqual(b'stderr', e)
|
||||
self.assertIsInstance(o, bytes)
|
||||
self.assertIsInstance(e, bytes)
|
||||
|
||||
def check_undecodable_bytes(self, binary):
|
||||
out_bytes = b'out: ' + UNDECODABLE_BYTES
|
||||
err_bytes = b'err: ' + UNDECODABLE_BYTES
|
||||
conn = FakeSshConnection(0, out=out_bytes, err=err_bytes)
|
||||
|
||||
out, err = processutils.ssh_execute(conn, 'ls')
|
||||
if six.PY3:
|
||||
out, err = processutils.ssh_execute(conn, 'ls', binary=binary)
|
||||
if six.PY3 and not binary:
|
||||
self.assertEqual(out, os.fsdecode(out_bytes))
|
||||
self.assertEqual(err, os.fsdecode(err_bytes))
|
||||
else:
|
||||
self.assertEqual(out, out_bytes)
|
||||
self.assertEqual(err, err_bytes)
|
||||
|
||||
def test_undecodable_bytes_error(self):
|
||||
def test_undecodable_bytes(self):
|
||||
self.check_undecodable_bytes(False)
|
||||
|
||||
def test_binary_undecodable_bytes(self):
|
||||
self.check_undecodable_bytes(True)
|
||||
|
||||
def check_undecodable_bytes_error(self, binary):
|
||||
out_bytes = b'out: password="secret1" ' + UNDECODABLE_BYTES
|
||||
err_bytes = b'err: password="secret2" ' + UNDECODABLE_BYTES
|
||||
conn = FakeSshConnection(1, out=out_bytes, err=err_bytes)
|
||||
|
||||
out_bytes = b'out: password="***" ' + UNDECODABLE_BYTES
|
||||
err_bytes = b'err: password="***" ' + UNDECODABLE_BYTES
|
||||
|
||||
exc = self.assertRaises(processutils.ProcessExecutionError,
|
||||
processutils.ssh_execute,
|
||||
conn, 'ls',
|
||||
check_exit_code=True)
|
||||
binary=binary, check_exit_code=True)
|
||||
|
||||
out = exc.stdout
|
||||
err = exc.stderr
|
||||
out_bytes = b'out: password="***" ' + UNDECODABLE_BYTES
|
||||
err_bytes = b'err: password="***" ' + UNDECODABLE_BYTES
|
||||
if six.PY3:
|
||||
# On Python 3, stdout and stderr attributes of
|
||||
# ProcessExecutionError must always be Unicode
|
||||
self.assertEqual(out, os.fsdecode(out_bytes))
|
||||
self.assertEqual(err, os.fsdecode(err_bytes))
|
||||
else:
|
||||
# On Python 2, stdout and stderr attributes of
|
||||
# ProcessExecutionError must always be bytes
|
||||
self.assertEqual(out, out_bytes)
|
||||
self.assertEqual(err, err_bytes)
|
||||
|
||||
def test_undecodable_bytes_error(self):
|
||||
self.check_undecodable_bytes_error(False)
|
||||
|
||||
def test_binary_undecodable_bytes_error(self):
|
||||
self.check_undecodable_bytes_error(True)
|
||||
|
||||
def test_fails(self):
|
||||
self.assertRaises(processutils.ProcessExecutionError,
|
||||
processutils.ssh_execute, FakeSshConnection(1), 'ls')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user