From 5260345d0d7acefb91590212d8d629abe9eb1538 Mon Sep 17 00:00:00 2001
From: Mike Gilbert <floppym@gentoo.org>
Date: Fri, 5 Jan 2018 13:40:40 -0500
Subject: [PATCH 06/16] h2py: use binary I/O to avoid encoding issues

https://bugs.python.org/issue13032
---
 Tools/scripts/h2py.py | 67 ++++++++++++++++++++++---------------------
 1 file changed, 34 insertions(+), 33 deletions(-)

diff --git a/Tools/scripts/h2py.py b/Tools/scripts/h2py.py
index ea37c04d4c..aabea34697 100755
--- a/Tools/scripts/h2py.py
+++ b/Tools/scripts/h2py.py
@@ -23,36 +23,36 @@
 
 import sys, re, getopt, os
 
-p_define = re.compile(r'^[\t ]*#[\t ]*define[\t ]+([a-zA-Z0-9_]+)[\t ]+')
+p_define = re.compile(br'^[\t ]*#[\t ]*define[\t ]+([a-zA-Z0-9_]+)[\t ]+')
 
 p_macro = re.compile(
-  r'^[\t ]*#[\t ]*define[\t ]+'
-  r'([a-zA-Z0-9_]+)\(([_a-zA-Z][_a-zA-Z0-9]*)\)[\t ]+')
+  br'^[\t ]*#[\t ]*define[\t ]+'
+  br'([a-zA-Z0-9_]+)\(([_a-zA-Z][_a-zA-Z0-9]*)\)[\t ]+')
 
-p_include = re.compile(r'^[\t ]*#[\t ]*include[\t ]+<([^>\n]+)>')
+p_include = re.compile(br'^[\t ]*#[\t ]*include[\t ]+<([^>\n]+)>')
 
-p_comment = re.compile(r'/\*([^*]+|\*+[^/])*(\*+/)?')
-p_cpp_comment = re.compile('//.*')
+p_comment = re.compile(br'/\*([^*]+|\*+[^/])*(\*+/)?')
+p_cpp_comment = re.compile(b'//.*')
 
 ignores = [p_comment, p_cpp_comment]
 
-p_char = re.compile(r"'(\\.[^\\]*|[^\\])'")
+p_char = re.compile(br"'(\\.[^\\]*|[^\\])'")
 
-p_hex = re.compile(r"0x([0-9a-fA-F]+)L?")
+p_hex = re.compile(br"0x([0-9a-fA-F]+)L?")
 
 filedict = {}
 importable = {}
 
 try:
-    searchdirs=os.environ['include'].split(';')
+    searchdirs=os.environb[b'include'].split(b';')
 except KeyError:
     try:
-        searchdirs=os.environ['INCLUDE'].split(';')
+        searchdirs=os.environb[b'INCLUDE'].split(b';')
     except KeyError:
-        searchdirs=['/usr/include']
+        searchdirs=[b'/usr/include']
         try:
-            searchdirs.insert(0, os.path.join('/usr/include',
-                                              os.environ['MULTIARCH']))
+            searchdirs.insert(0, os.path.join(b'/usr/include',
+                                              os.environb[b'MULTIARCH']))
         except KeyError:
             pass
 
@@ -61,22 +61,23 @@ def main():
     opts, args = getopt.getopt(sys.argv[1:], 'i:')
     for o, a in opts:
         if o == '-i':
-            ignores.append(re.compile(a))
+            ignores.append(re.compile(a.encode()))
     if not args:
         args = ['-']
     for filename in args:
         if filename == '-':
             sys.stdout.write('# Generated by h2py from stdin\n')
-            process(sys.stdin, sys.stdout)
+            process(sys.stdin.buffer, sys.stdout.buffer)
         else:
-            with open(filename) as fp:
+            filename = filename.encode()
+            with open(filename, 'rb') as fp:
                 outfile = os.path.basename(filename)
-                i = outfile.rfind('.')
+                i = outfile.rfind(b'.')
                 if i > 0: outfile = outfile[:i]
                 modname = outfile.upper()
-                outfile = modname + '.py'
-                with open(outfile, 'w') as outfp:
-                    outfp.write('# Generated by h2py from %s\n' % filename)
+                outfile = modname + b'.py'
+                with open(outfile, 'wb') as outfp:
+                    outfp.write(b'# Generated by h2py from %s\n' % filename)
                     filedict = {}
                     for dir in searchdirs:
                         if filename[:len(dir)] == dir:
@@ -88,9 +89,9 @@ def main():
 def pytify(body):
     # replace ignored patterns by spaces
     for p in ignores:
-        body = p.sub(' ', body)
+        body = p.sub(b' ', body)
     # replace char literals by ord(...)
-    body = p_char.sub("ord('\\1')", body)
+    body = p_char.sub(b"ord('\\1')", body)
     # Compute negative hexadecimal constants
     start = 0
     UMAX = 2*(sys.maxsize+1)
@@ -101,7 +102,7 @@ def pytify(body):
         val = int(body[slice(*m.span(1))], 16)
         if val > sys.maxsize:
             val -= UMAX
-            body = body[:s] + "(" + str(val) + ")" + body[e:]
+            body = body[:s] + b"(" + str(val).encode() + b")" + body[e:]
         start = s + 1
     return body
 
@@ -114,7 +115,7 @@ def process(fp, outfp, env = {}):
         match = p_define.match(line)
         if match:
             # gobble up continuation lines
-            while line[-2:] == '\\\n':
+            while line[-2:] == b'\\\n':
                 nextline = fp.readline()
                 if not nextline: break
                 lineno = lineno + 1
@@ -123,11 +124,11 @@ def process(fp, outfp, env = {}):
             body = line[match.end():]
             body = pytify(body)
             ok = 0
-            stmt = '%s = %s\n' % (name, body.strip())
+            stmt = name + b' = ' + body.strip() + b'\n'
             try:
                 exec(stmt, env)
             except:
-                sys.stderr.write('Skipping: %s' % stmt)
+                sys.stderr.buffer.write(b'Skipping: ' + stmt)
             else:
                 outfp.write(stmt)
         match = p_macro.match(line)
@@ -135,11 +136,11 @@ def process(fp, outfp, env = {}):
             macro, arg = match.group(1, 2)
             body = line[match.end():]
             body = pytify(body)
-            stmt = 'def %s(%s): return %s\n' % (macro, arg, body)
+            stmt = b'def ' + macro + b'(' + arg + b'): return ' + body + b'\n'
             try:
                 exec(stmt, env)
             except:
-                sys.stderr.write('Skipping: %s' % stmt)
+                sys.stderr.buffer.write(b'Skipping: ' + stmt)
             else:
                 outfp.write(stmt)
         match = p_include.match(line)
@@ -148,24 +149,24 @@ def process(fp, outfp, env = {}):
             a, b = regs[1]
             filename = line[a:b]
             if filename in importable:
-                outfp.write('from %s import *\n' % importable[filename])
+                outfp.write(b'from ' + importable[filename] + b' import *\n')
             elif filename not in filedict:
                 filedict[filename] = None
                 inclfp = None
                 for dir in searchdirs:
                     try:
-                        inclfp = open(dir + '/' + filename)
+                        inclfp = open(dir + b'/' + filename, 'rb')
                         break
                     except IOError:
                         pass
                 if inclfp:
                     with inclfp:
                         outfp.write(
-                                '\n# Included from %s\n' % filename)
+                                b'\n# Included from %s\n' % filename)
                         process(inclfp, outfp, env)
                 else:
-                    sys.stderr.write('Warning - could not find file %s\n' %
-                                     filename)
+                    sys.stderr.buffer.write(b'Warning - could not find file ' +
+                                            filename + b'\n')
 
 if __name__ == '__main__':
     main()
-- 
2.33.0

