commit: f1367a2eeb0c911a743b0da5f37a7e9f7de76488 Author: Zac Medico <zmedico <AT> gentoo <DOT> org> AuthorDate: Wed Feb 14 20:22:03 2018 +0000 Commit: Zac Medico <zmedico <AT> gentoo <DOT> org> CommitDate: Sat Feb 17 18:25:45 2018 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=f1367a2e
filter-bash-environment.py: use buffered input, raw bytes (bug 647654) Use sys.stdin.buffer instead of sys.stdin.buffer.raw, for buffered input. Also use raw bytes instead of unicode strings, in order to avoid making assumptions about character encodings, and also to avoid overhead from unicode decoding/encoding. Since the % operator does not support bytes operands in python3.4, use the + operator to format strings of bytes. Bug: https://bugs.gentoo.org/647654 bin/filter-bash-environment.py | 47 +++++------ pym/portage/tests/bin/test_filter_bash_env.py | 115 ++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 25 deletions(-) diff --git a/bin/filter-bash-environment.py b/bin/filter-bash-environment.py index a4cdc5429..045ea6f52 100755 --- a/bin/filter-bash-environment.py +++ b/bin/filter-bash-environment.py @@ -2,21 +2,19 @@ # Copyright 1999-2014 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import codecs -import io import os import re import sys -here_doc_re = re.compile(r'.*\s<<[-]?(\w+)$') -func_start_re = re.compile(r'^[-\w]+\s*\(\)\s*$') -func_end_re = re.compile(r'^\}$') +here_doc_re = re.compile(br'.*\s<<[-]?(\w+)$') +func_start_re = re.compile(br'^[-\w]+\s*\(\)\s*$') +func_end_re = re.compile(br'^\}$') -var_assign_re = re.compile(r'(^|^declare\s+-\S+\s+|^declare\s+|^export\s+)([^=\s]+)=("|\')?.*$') -close_quote_re = re.compile(r'(\\"|"|\')\s*$') -readonly_re = re.compile(r'^declare\s+-(\S*)r(\S*)\s+') +var_assign_re = re.compile(br'(^|^declare\s+-\S+\s+|^declare\s+|^export\s+)([^=\s]+)=("|\')?.*$') +close_quote_re = re.compile(br'(\\"|"|\')\s*$') +readonly_re = re.compile(br'^declare\s+-(\S*)r(\S*)\s+') # declare without assignment -var_declare_re = re.compile(r'^declare(\s+-\S+)?\s+([^=\s]+)\s*$') +var_declare_re = re.compile(br'^declare(\s+-\S+)?\s+([^=\s]+)\s*$') def have_end_quote(quote, line): """ @@ -32,16 +30,16 @@ def have_end_quote(quote, line): def filter_declare_readonly_opt(line): readonly_match = readonly_re.match(line) if readonly_match is not None: - declare_opts = '' + declare_opts = b'' for i in (1, 2): group = readonly_match.group(i) if group is not None: declare_opts += group if declare_opts: - line = 'declare -%s %s' % \ - (declare_opts, line[readonly_match.end():]) + line = b'declare -' + declare_opts + \ + b' ' + line[readonly_match.end():] else: - line = 'declare ' + line[readonly_match.end():] + line = b'declare ' + line[readonly_match.end():] return line def filter_bash_environment(pattern, file_in, file_out): @@ -57,7 +55,7 @@ def filter_bash_environment(pattern, file_in, file_out): for line in file_in: if multi_line_quote is not None: if not multi_line_quote_filter: - file_out.write(line.replace("\1", "")) + file_out.write(line.replace(b"\1", b"")) if have_end_quote(multi_line_quote, line): multi_line_quote = None multi_line_quote_filter = None @@ -78,7 +76,7 @@ def filter_bash_environment(pattern, file_in, file_out): multi_line_quote_filter = filter_this if not filter_this: line = filter_declare_readonly_opt(line) - file_out.write(line.replace("\1", "")) + file_out.write(line.replace(b"\1", b"")) continue else: declare_match = var_declare_re.match(line) @@ -98,7 +96,7 @@ def filter_bash_environment(pattern, file_in, file_out): continue here_doc = here_doc_re.match(line) if here_doc is not None: - here_doc_delim = re.compile("^%s$" % here_doc.group(1)) + here_doc_delim = re.compile(b'^' + here_doc.group(1) + b'$') file_out.write(line) continue # Note: here-documents are handled before functions since otherwise @@ -141,18 +139,17 @@ if __name__ == "__main__": file_in = sys.stdin file_out = sys.stdout if sys.hexversion >= 0x3000000: - file_in = codecs.iterdecode(sys.stdin.buffer.raw, - 'utf_8', errors='replace') - file_out = io.TextIOWrapper(sys.stdout.buffer, - 'utf_8', errors='backslashreplace') - - var_pattern = args[0].split() + file_in = sys.stdin.buffer + file_out = sys.stdout.buffer + var_pattern = os.fsencode(args[0]).split() + else: + var_pattern = args[0].split() # Filter invalid variable names that are not supported by bash. - var_pattern.append(r'\d.*') - var_pattern.append(r'.*\W.*') + var_pattern.append(br'\d.*') + var_pattern.append(br'.*\W.*') - var_pattern = "^(%s)$" % "|".join(var_pattern) + var_pattern = b'^(' + b'|'.join(var_pattern) + b')$' filter_bash_environment( re.compile(var_pattern), file_in, file_out) file_out.flush() diff --git a/pym/portage/tests/bin/test_filter_bash_env.py b/pym/portage/tests/bin/test_filter_bash_env.py new file mode 100644 index 000000000..10847bb7d --- /dev/null +++ b/pym/portage/tests/bin/test_filter_bash_env.py @@ -0,0 +1,115 @@ +# Copyright 2018 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import difflib +import os +import subprocess + +import portage +from portage.const import PORTAGE_BIN_PATH +from portage.tests import TestCase + + +class TestFilterBashEnv(TestCase): + def testTestFilterBashEnv(self): + + test_cases = ( + ( + 'RDEPEND BASH.* _EPATCH_ECLASS', + b'''declare -ir BASHPID="28997" +declare -rx A="portage-2.3.24.tar.bz2" +declare -- DESKTOP_DATABASE_DIR="/usr/share/applications" +declare PDEPEND=" + !build? ( + >=net-misc/rsync-2.6.4 + userland_GNU? ( >=sys-apps/coreutils-6.4 ) + ) " +declare RDEPEND=" + >=app-arch/tar-1.27 + dev-lang/python-exec:2" +declare -x PF="portage-2.3.24" +declare -a PYTHON_COMPAT=([0]="pypy" [1]="python3_4" [2]="python3_5" [3]="python3_6" [4]="python2_7") +declare -- _EPATCH_ECLASS="1" +declare -- _EUTILS_ECLASS="1" +declare -- f +get_libdir () +{ + local CONF_LIBDIR; + if [ -n "${CONF_LIBDIR_OVERRIDE}" ]; then + echo ${CONF_LIBDIR_OVERRIDE}; + else + get_abi_LIBDIR; + fi +} +make_wrapper () +{ + cat <<-EOF +export ${var}="\${${var}}:${EPREFIX}${libdir}" +EOF +} +use_if_iuse () +{ + in_iuse $1 || return 1; + use $1 +} +''', + b'''declare -x A="portage-2.3.24.tar.bz2" +declare -- DESKTOP_DATABASE_DIR="/usr/share/applications" +declare PDEPEND=" + !build? ( + >=net-misc/rsync-2.6.4 + userland_GNU? ( >=sys-apps/coreutils-6.4 ) + ) " +declare -x PF="portage-2.3.24" +declare -a PYTHON_COMPAT=([0]="pypy" [1]="python3_4" [2]="python3_5" [3]="python3_6" [4]="python2_7") +declare -- _EUTILS_ECLASS="1" +declare -- f +get_libdir () +{ + local CONF_LIBDIR; + if [ -n "${CONF_LIBDIR_OVERRIDE}" ]; then + echo ${CONF_LIBDIR_OVERRIDE}; + else + get_abi_LIBDIR; + fi +} +make_wrapper () +{ + cat <<-EOF +export ${var}="\${${var}}:${EPREFIX}${libdir}" +EOF +} +use_if_iuse () +{ + in_iuse $1 || return 1; + use $1 +} +'''), + ) + + for filter_vars, env_in, env_out in test_cases: + proc = None + try: + proc = subprocess.Popen( + [ + portage._python_interpreter, + os.path.join(PORTAGE_BIN_PATH, 'filter-bash-environment.py'), + filter_vars, + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + proc.stdin.write(env_in) + proc.stdin.close() + result = proc.stdout.read() + finally: + if proc is not None: + proc.stdin.close() + proc.wait() + proc.stdout.close() + + diff = list(difflib.unified_diff( + env_out.decode('utf_8').splitlines(), + result.decode('utf_8').splitlines())) + + self.assertEqual(diff, [])
