diff --git a/lslopt/lslbasefuncs.py b/lslopt/lslbasefuncs.py index 7a66015..5738521 100644 --- a/lslopt/lslbasefuncs.py +++ b/lslopt/lslbasefuncs.py @@ -568,42 +568,42 @@ def InternalUTF8toString(s): # U+10FFFF are not supported. Both things complicate the alg a bit. ret = u'' - partialchar = b'' + partialchar = bytearray(b'') pending = 0 - for c in s: - o = ord(c) + for o in s: if partialchar: + c = partialchar[0] if len(partialchar) == 1 else None if 0x80 <= o < 0xC0 and ( - partialchar[1:2] - or b'\xC2' <= partialchar < b'\xF4' and partialchar not in b'\xE0\xED\xF0' - or partialchar == b'\xE0' and o >= 0xA0 - or partialchar == b'\xED' and o < 0xA0 - or partialchar == b'\xF0' and o >= 0x90 - or partialchar == b'\xF4' and o < 0x90 + c is None + or 0xC2 <= c < 0xF4 and c not in (0xE0, 0xED, 0xF0) + or c == 0xE0 and o >= 0xA0 + or c == 0xED and o < 0xA0 + or c == 0xF0 and o >= 0x90 + or c == 0xF4 and o < 0x90 ): - partialchar += c + partialchar.append(o) pending -= 1 if pending == 0: ret += partialchar.decode('utf8') - partialchar = b'' - c = c + partialchar = bytearray(b'') + o = o # NOTE: Without the above line, the following one hits a bug in # python-coverage. It IS executed but not detected. continue if lslcommon.LSO: raise ELSONotSupported(u"Byte strings not supported") ret += u'?' * len(partialchar) - partialchar = b'' + partialchar = bytearray(b'') # fall through to process current character if o >= 0xC2 and o <= 0xF4: - partialchar = c + partialchar = bytearray((o,)) pending = 1 if o < 0xE0 else 2 if o < 0xF0 else 3 elif o >= 0x80: if lslcommon.LSO: raise ELSONotSupported(u"Byte strings not supported") ret += u'?' else: - ret += c.decode('utf8') + ret += unichr(o) if partialchar: if lslcommon.LSO: @@ -997,9 +997,9 @@ def llBase64ToInteger(s): if len(s) < 3: # not computable deterministically raise ELSLCantCompute - s = (s + b'\0')[:4] - i = ord(s[0]) if s[0] < b'\x80' else ord(s[0])-256 - return (i<<24)+(ord(s[1])<<16)+(ord(s[2])<<8)+ord(s[3]) + s = bytearray(s + b'\0')[:4] + i = s[0] if s[0] < 128 else s[0]-256 + return (i<<24)+(s[1]<<16)+(s[2]<<8)+s[3] b64tos_re = re.compile( b'(' @@ -1056,7 +1056,7 @@ def llBase64ToString(s): match = b64tos_re.search(byteseq, pos) - return InternalUTF8toString(bytes(byteseq)) + return InternalUTF8toString(byteseq) def llCSV2List(s): s = fs(s) @@ -1119,13 +1119,14 @@ def llDumpList2String(lst, sep): def llEscapeURL(s): s = fs(s) - s = s.encode('utf8') # get bytes + s = bytearray(s.encode('utf8')) ret = u'' for c in s: - if b'A' <= c <= b'Z' or b'a' <= c <= b'z' or b'0' <= c <= b'9': - ret += c.encode('utf8') + # 0x30='0', 0x39='9', 0x41='A', 0x5A='Z', 0x61='a', 0x7A='z' + if 0x30 <= c <= 0x39 or 0x41 <= c <= 0x5A or 0x61 <= c <= 0x7A: + ret += unichr(c) else: - ret += u'%%%02X' % ord(c) + ret += u'%%%02X' % c return ret def llEuler2Rot(v): @@ -1239,8 +1240,8 @@ def llInsertString(s, pos, src): def llIntegerToBase64(x): x = fi(x) - return b64encode(chr((x>>24)&255) + chr((x>>16)&255) + chr((x>>8)&255) - + chr(x&255)).decode('utf8') + return (b64encode(bytearray(((x>>24)&255, (x>>16)&255, (x>>8)&255, x&255))) + .decode('utf8')) def llLinear2sRGB(v): v = v2f(v) @@ -1327,9 +1328,9 @@ def llList2ListStrided(lst, start, end, stride): start = 0 end = L-1 # start is rounded up to ceil(start/stride)*stride - start = ((start+stride-1)/stride)*stride + start = ((start+stride-1)//stride)*stride # end is rounded down to floor(start/stride)*stride - end = (end/stride)*stride + end = (end//stride)*stride return lst[start:end+1:stride] @@ -1588,7 +1589,8 @@ def llLog10(f): def llMD5String(s, salt): s = fs(s) salt = fi(salt) - return hashlib.md5(zstr(s).encode('utf8') + b':' + bytes(salt)).hexdigest().decode('utf8') + return str2u(hashlib.md5(zstr(s).encode('utf8') + b':' + + unicode(salt).encode('utf8')).hexdigest(), 'utf8') def llModPow(base, exp, mod): base = fi(base) @@ -1768,7 +1770,7 @@ def llRound(f): def llSHA1String(s): s = fs(s) - return hashlib.sha1(s.encode('utf8')).hexdigest().decode('utf8') + return str2u(hashlib.sha1(s.encode('utf8')).hexdigest(), 'utf8') def llSin(f): f = ff(f) @@ -1842,7 +1844,7 @@ def llToUpper(s): def llUnescapeURL(s): s = fs(s) - ret = b'' + ret = bytearray(b'') L = len(s) i = 0 while i < L: @@ -1862,13 +1864,13 @@ def llUnescapeURL(s): v = int(c, 16)<<4 c = s[i] # Second digit if c == u'%': - ret += chr(v) + ret.append(v) i += 1 continue i += 1 if u'0' <= c <= u'9' or u'A' <= c <= u'F' or u'a' <= c <= u'f': v += int(c, 16) - ret += chr(v) + ret.append(v) return InternalUTF8toString(ret) def llVecDist(v1, v2): @@ -1914,12 +1916,12 @@ def llXorBase64(s, xor): L2 = 2 xor = u'AA' - s = b64decode(s + u'=' * (-L1 & 3)) - xor = b64decode(xor + u'=' * (-L2 & 3)) + s = bytearray(b64decode(s + u'=' * (-L1 & 3))) + xor = bytearray(b64decode(xor + u'=' * (-L2 & 3))) L2 = len(xor) i = 0 - ret = b'' + ret = bytearray(b'') Bug3763 = 3763 in Bugs # BUG-3763 consists of the binary string having an extra NULL every time after the second repetition of @@ -1927,13 +1929,13 @@ def llXorBase64(s, xor): # b'12345678901234567890', the XOR binary string behaves as if it was b'pqrpqr\0pqr\0pqr\0pqr\0pq'. # We emulate that by adding the zero and increasing the length the first time. for c in s: - ret += chr(ord(c) ^ ord(xor[i])) + ret.append(c ^ xor[i]) i += 1 if i >= L2: i = 0 if Bug3763: Bug3763 = False - xor = xor + b'\x00' + xor.append(0) L2 += 1 return b64encode(ret).decode('utf8') @@ -2004,16 +2006,16 @@ def llXorBase64StringsCorrect(s, xor): L2 = 2 xor = u'AA' - s = b64decode(s + u'=' * (-L1 & 3)) - xor = b64decode(xor + u'=' * (-L2 & 3)) + b'\x00' + s = bytearray(b64decode(s + u'=' * (-L1 & 3))) + xor = bytearray(b64decode(xor + u'=' * (-L2 & 3)) + b'\x00') i = 0 - ret = b'' + ret = bytearray(b'') for c in s: - ret += chr(ord(c) ^ ord(xor[i])) + ret.append(c ^ xor[i]) i += 1 - if xor[i] == b'\x00': + if xor[i] == 0: i = 0 return b64encode(ret).decode('utf8') diff --git a/lslopt/lslfuncopt.py b/lslopt/lslfuncopt.py index 78d5cae..d6cba62 100644 --- a/lslopt/lslfuncopt.py +++ b/lslopt/lslfuncopt.py @@ -21,6 +21,7 @@ from lslopt import lslcommon from lslopt.lslcommon import Key, Vector, Quaternion, nr from lslopt import lslfuncs +from strutil import unicode SensorFunctions = {'llSensor', 'llSensorRepeat'} # not sure about llRemoteDataReply but let's fall on the safe side diff --git a/lslopt/lslloadlib.py b/lslopt/lslloadlib.py index 58fecb6..3d60ba1 100644 --- a/lslopt/lslloadlib.py +++ b/lslopt/lslloadlib.py @@ -64,7 +64,7 @@ def LoadLibrary(builtins = None, fndata = None): ubuiltins = str2u(builtins, sys.getfilesystemencoding()) except UnicodeDecodeError: # This is just a guess at the filename encoding. - ubuiltins = builtins.decode('iso-8859-15') + ubuiltins = str2u(builtins, 'iso-8859-15') while True: linenum += 1 line = f.readline() @@ -98,7 +98,7 @@ def LoadLibrary(builtins = None, fndata = None): for arg in arglist: argtyp = parse_arg_re.search(arg).group(1) if argtyp not in types: - uargtyp = argtyp.decode('utf8') + uargtyp = str2u(argtyp, 'utf8') warning(u"Invalid type in %s, line %d: %s" % (ubuiltins, linenum, uargtyp)) del uargtyp @@ -110,7 +110,7 @@ def LoadLibrary(builtins = None, fndata = None): name = match.group(2) if typ == 'event': if name in events: - uname = name.decode('utf8') + uname = str2u(name, 'utf8') warning(u"Event at line %d was already defined in %s," u" overwriting: %s" % (linenum, ubuiltins, uname)) @@ -121,7 +121,7 @@ def LoadLibrary(builtins = None, fndata = None): # they are implemented in lslfuncs.*, they get a # reference to the implementation; otherwise None. if name in functions: - uname = name.decode('utf8') + uname = str2u(name, 'utf8') warning(u"Function at line %d was already defined" u" in %s, overwriting: %s" % (linenum, ubuiltins, uname)) @@ -135,13 +135,13 @@ def LoadLibrary(builtins = None, fndata = None): # constant name = match.group(5) if name in constants: - uname = name.decode('utf8') + uname = str2u(name, 'utf8') warning(u"Global at line %d was already defined in %s," u" overwriting: %s" % (linenum, ubuiltins, uname)) del uname typ = match.group(4) if typ not in types: - utyp = typ.decode('utf8') + utyp = str2u(typ, 'utf8') warning(u"Invalid type in %s, line %d: %s" % (ubuiltins, linenum, utyp)) del utyp @@ -253,7 +253,7 @@ def LoadLibrary(builtins = None, fndata = None): ufndata = str2u(fndata, sys.getfilesystemencoding()) except UnicodeDecodeError: # This is just a guess at the filename encoding. - ufndata = fndata.decode('iso-8859-15') + ufndata = str2u(fndata, 'iso-8859-15') while True: linenum += 1 line = f.readline() @@ -448,7 +448,7 @@ def LoadLibrary(builtins = None, fndata = None): if functions[i]['min'] > functions[i]['max']: warning(u"Library data: Function %s has min > max:" u" min=%s max=%s, removing both." - % (ui, repr(functions[i]['min']).decode('utf8'), + % (ui, str2u(repr(functions[i]['min']), 'utf8'), repr(functions[i]['max']))) del functions[i]['min'], functions[i]['max'] if 'SEF' in functions[i] and 'delay' in functions[i]: diff --git a/lslopt/lsloutput.py b/lslopt/lsloutput.py index 38c5e67..33d08d4 100644 --- a/lslopt/lsloutput.py +++ b/lslopt/lsloutput.py @@ -63,8 +63,9 @@ class outscript(object): " spaces by the viewer when copy-pasting the code" " (disable this warning by disabling the 'warntabs'" " option).") - return pfx + '"' + any2str(value, 'utf8').replace('\\','\\\\') \ - .replace('"','\\"').replace('\n','\\n') + '"' + sfx + return (pfx + '"' + u2str(unicode(value), 'utf8') + .replace('\\','\\\\').replace('"','\\"').replace('\n','\\n') + + '"' + sfx) if tvalue == int: if value < 0 and not self.globalmode and self.optsigns: #return '0x%X' % (value + 4294967296) diff --git a/lslopt/lslrenamer.py b/lslopt/lslrenamer.py index 32946c8..fa2d80f 100644 --- a/lslopt/lslrenamer.py +++ b/lslopt/lslrenamer.py @@ -175,7 +175,7 @@ class renamer(object): # Skip globals continue InParams = False - for name,sym in table.iteritems(): + for name,sym in table.items(): if name == -1: continue if sym['Kind'] != 'v': assert sym['Kind'] == 'l' diff --git a/main.py b/main.py index 256452c..d913877 100755 --- a/main.py +++ b/main.py @@ -38,8 +38,8 @@ VERSION = '0.3.0beta' def ReportError(script, e): - linestart = script.rfind(b'\n', 0, e.errorpos) + 1 - lineend = script.find(b'\n', e.errorpos) + linestart = script.rfind('\n', 0, e.errorpos) + 1 + lineend = script.find('\n', e.errorpos) if lineend == -1: lineend = len(script) # may hit EOF # When the encoding of stderr is unknown (e.g. when redirected to a file), @@ -53,13 +53,13 @@ def ReportError(script, e): # 1. Trim the UTF-8 line. err_frag = script[linestart:e.errorpos] # 2. Convert to Unicode; encode in the target encoding with replacing. - err_frag = err_frag.decode('utf8').encode(enc, 'backslashreplace') + err_frag = str2u(err_frag, 'utf8').encode(enc, 'backslashreplace') # 3. Collect our prize: the length of that in characters. cno = len(err_frag.decode(enc)) # Write the whole line in the target encoding. - err_line = script[linestart:lineend] + b'\n' - werr(err_line.decode('utf8')) + err_line = script[linestart:lineend] + '\n' + werr(err_line) werr(" " * cno + "^\n") werr(e.args[0] + u"\n") @@ -412,7 +412,7 @@ def main(argv): 'libdata=', 'postarg=')) except getopt.GetoptError as e: Usage(argv[0]) - werr(u"\nError: %s\n" % str(e).decode('utf8', 'replace')) + werr(u"\nError: %s\n" % str2u(str(e), 'utf8')) return 1 outfile = '-' diff --git a/run-tests.py b/run-tests.py index c420c41..a668493 100755 --- a/run-tests.py +++ b/run-tests.py @@ -486,7 +486,8 @@ class UnitTestCoverage(UnitTestCase): msg = str(e) finally: lslcommon.IsCalc = save_IsCalc - self.assertEqual(msg, u"Value of unknown type in Value2LSL: 'ab'") + self.assertEqual(msg, u"Value of unknown type in Value2LSL: 'ab'" + if python2 else u"Value of unknown type in Value2LSL: b'ab'") del msg # Extended assignment in output script = [nr(nt='EXPR', t='integer', ch=[