3


 \k��@s:
dd
lm
Z

ddlmZmZmZmZmZmZmZm	Z	m
Z
mZmZm
ZmZmZmZ
ddlmZ
ddlZddlmZmZ
ddlmZmZ
ddlZddlZdd�ZGd	d
�d
e�Zdd�ZGd
d�de�ZGdd�de�Z Gdd�de�Z!Gdd�de�Z"Gdd�de�Z#Gdd�de�Z$Gdd�de�Z%e&dk�
r6ej'�
dS)�)
�support)�tokenize�	_tokenize�
untokenize�NUMBER�NAME�OP�STRING�	ENDMARKER�ENCODING�tok_name�detect_encoding�open�Untokenizer�generate_tokens�NEWLINE)
�BytesION)�TestCase�mock)�VALID_UNDERSCORE_LITERALS�INVALID_UNDERSCORE_LITERALSc

Cs�g}t|
j
��
}|
ddk}xh|D]`\}}}}}	|tkr:P|rT|tkrT|d|krTq"t|}|jd|d�d|d�d|�d|���

q"W|S)	N�
z
r
z    Z10�
 Z13���)�len�
splitlinesr
rr�append)
Ztoken_generatorZ
source_string�resultZ	num_linesZmissing_trailing_nl�type�token�start�end�line�r#�%/usr/lib64/python3.6/test_tokenize.py�stringify_tokens_from_sources







*r%c@s�eZ
dZd
d�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�Zdd�Zdd�Zdd �Zd!d"�Zd#d$�Zd%d&�Zd'd(�Zd)S)*�TokenizeTestcCs<t|
j
d
�
�
}tt|j�
|
�}|j|dg
|j�j��
dS)Nzutf-8z*    ENCODING   'utf-8'       (0, 0) (0, 0))r�encoder%r�readline�assertEqual�rstripr)�self�
s�expected�
frr#r#r$�check_tokenize*s



zTokenizeTest.check_tokenizec
CsDtd
j
d�
�
}
tt|
j�
�
}|j|djt�
|j|djt�
dS)N�
xzutf-8�r���r)	rr'�listrr(r)rrr
)r+r.�tokensr#r#r$�test_implicit_newline4s


z"TokenizeTest.test_implicit_newlinec
	CsT|jd
d�
|jdd�
d}
t
|
�
j}|jtd��
xt|�
D]}q>WWdQRXdS)Nz1 + 1z�    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z.if False:
    # NL
    True = False # NEWLINE
a;    NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'True'        (3, 4) (3, 8)
    OP         '='           (3, 9) (3, 10)
    NAME       'False'       (3, 11) (3, 16)
    COMMENT    '# NEWLINE'   (3, 17) (3, 26)
    NEWLINE    '\n'          (3, 26) (3, 27)
    DEDENT     ''            (4, 0) (4, 0)
    sdef k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation level)r/rr(�assertRaisesRegex�IndentationErrorr)r+Zindent_error_filer(�tokr#r#r$�
test_basic<s






zTokenizeTest.test_basicc

Csp|jd
d�
|jdd�
|jdd�
|jdd�
|jd	d
�
|jdd�
|jd
d�
|jdd�
|jdd�
dS)Nz0xff <= 255z�    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    z0b10 <= 255z�    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    z0o123 <= 0O123z�    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    z1234567 > ~0x15z�    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    z2134568 != 1231515z�    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    z(-124561-1) & 200000000ac
    OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    z0xdeadbeef != -1z�    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    z0xdeadc0de & 12345z�    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    z0xFF & 0x15 | 1234z�    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    )
r/)
r+r#r#r$�test_int_s$




	


zTokenizeTest.test_intc

Cs4|jd
d�
|jdd�
|jdd�
|jdd�
dS)	Nzx = 0z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xfffffffffff' (1, 4) (1, 17)
    zx = 123141242151251616110z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151251616110' (1, 4) (1, 25)
    zx = -15921590215012591z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '15921590215012591' (1, 5) (1, 22)
    )
r/)
r+r#r#r$�	test_long�s


zTokenizeTest.test_longc

CsX|jd
d�
|jdd�
|jdd�
|jdd�
|jd	d
�
|jdd�
|jd
d�
dS)Nzx = 3.14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    zx = 314159.z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    zx = .314159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    zx = 3e14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    z	x = 3E123z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    z
x+y = 3e-1230z�    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    zx = 3.14e159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    )
r/)
r+r#r#r$�
test_float�s





zTokenizeTest.test_floatc
CsRd
d�}
x&tD]}d|krq|j
|
|�
|�
qWxtD]}|j|
|�
|�
q6WdS)Nc
Ss<t|j
d
�
�
}
x(t|
j�
D]\}}}}}|tkr|SqWdS)Nzutf-8z
invalid token)rr'rr(r)r,r.Ztoktyperr r!r"r#r#r$�number_token�s





z;TokenizeTest.test_underscore_literals.<locals>.number_token�
()rr)rZassertNotEqual)r+r=Zlitr#r#r$�test_underscore_literals�s






z%TokenizeTest.test_underscore_literalsc

Cs<
|jd
d�
|jdd�
|jdd�
|jdd�
|jd	d
�
|jdd�
|jd
d�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd �
|jd!d"�
|jd#d$�
|jd%d&�
|jd'd(�
|jd)d*�
|jd+d,�
|jd-d.�
|jd/d0�
|jd1d2�
|jd3d4�
dS)5Nzx = ''; y = ""a5
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    zx = '"'; y = "'"a6
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    zx = "doesn't "shrink", does it"z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    zx = 'abc' + 'ABC'z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    zy = "ABC" + "ABC"z�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    z%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a�
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    z%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a�
    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    zu'abc' + U'abc'z�    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    zu"abc" + U"abc"z�    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    zb'abc' + B'abc'z�    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    zb"abc" + B"abc"z�    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    z%br'abc' + bR'abc' + Br'abc' + BR'abc'a;
    STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    z%br"abc" + bR"abc" + Br"abc" + BR"abc"a;
    STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    z%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;
    STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    z%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;
    STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    z"a\
de\
fg"z3    STRING     '"a\\\nde\\\nfg"' (1, 0) (3, 3)
    zu"a\
de"z/    STRING     'u"a\\\nde"'  (1, 0) (2, 3)
    zrb"a\
d"z/    STRING     'rb"a\\\nd"'  (1, 0) (2, 2)
    z
"""a\
b"""z0    STRING     '"""a\\\nb"""' (1, 0) (2, 4)
    zu"""a\
b"""z1    STRING     'u"""a\\\nb"""' (1, 0) (2, 4)
    zrb"""a\
b\
c"""z7    STRING     'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
    zf"abc"z/    STRING     'f"abc"'      (1, 0) (1, 6)
    z	fR"a{b}c"z/    STRING     'fR"a{b}c"'   (1, 0) (1, 9)
    z
f"""abc"""z0    STRING     'f"""abc"""'  (1, 0) (1, 10)
    zf"abc\
def"z1    STRING     'f"abc\\\ndef"' (1, 0) (2, 4)
    zRf"abc\
def"z2    STRING     'Rf"abc\\\ndef"' (1, 0) (2, 4)
    )
r/)
r+r#r#r$�test_string�sx





























zTokenizeTest.test_stringc

Cs|jd
d�
|jdd�
dS)Nz!def d22(a, b, c=2, d=2, *k): passa    NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    zdef d01v_(a=1, *k, **w): passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    )
r/)
r+r#r#r$�
test_function�
s

zTokenizeTest.test_functionc

Cs|jd
d�
dS)NzXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa�    NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    )
r/)
r+r#r#r$�test_comparison�
s"zTokenizeTest.test_comparisonc

Cs|jd
d�
dS)Nzx = 1 << 1 >> 5a6
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    )
r/)
r+r#r#r$�
test_shift�
szTokenizeTest.test_shiftc

Cs|jd
d�
dS)Nz%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    )
r/)
r+r#r#r$�
test_additive�
szTokenizeTest.test_additivec

Cs|jd
d�
dS)Nzx = 1//1*1/5*12%0x12@42a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    )
r/)
r+r#r#r$�test_multiplicative�
sz TokenizeTest.test_multiplicativec

Cs|jd
d�
|jdd�
dS)Nz~1 ^ 1 & 1 |1 ^ -1a�
    OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    z-1*1/1+1*1//1 - ---1**1aH    OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    )
r/)
r+r#r#r$�
test_unarys
zTokenizeTest.test_unaryc

Cs|jd
d�
dS)Nz/import sys, time
x = sys.modules['time'].time()a�    NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    )
r/)
r+r#r#r$�
test_selector8szTokenizeTest.test_selectorc

Cs|jd
d�
dS)Nz @staticmethod
def foo(x,y): passa    OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    )
r/)
r+r#r#r$�test_methodNs
zTokenizeTest.test_methodc

Cs|jd
d�
dS)Nzdef f():
	if x
        	passa_    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    )
r/)
r+r#r#r$�	test_tabs_szTokenizeTest.test_tabsc

Cs|jd
d�
dS)Nu!Örter = 'places'
grün = 'green'u7
    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    )
r/)
r+r#r#r$�test_non_ascii_identifierstsz'TokenizeTest.test_non_ascii_identifiersc

Cs|jd
d�
dS)Nu#Örter = u'places'
grün = U'green'u7
    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    )
r/)
r+r#r#r$�test_unicode�szTokenizeTest.test_unicodec

Cs
|jd
d�
|jdd�
|jdd�
|jdd�
|jd	d
�
|jdd�
|jd
d�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd�
|jdd �
|jd!d"�
|jd#d$�
|jd%d&�
|jd'd(�
|jd)d*�
|jd+d,�
dS)-Nz	async = 1z�    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    zasync\zZ    ERRORTOKEN '\\'          (1, 5) (1, 6)
    NAME       'async'       (1, 0) (1, 5)
    za = (async = 1)a8
    NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    zasync()z�    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    zclass async(Bar):passa<
    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    zclass async:passz�    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    z	await = 1z�    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    z	foo.asyncz�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    zasync for a in b: passa;
    NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    zasync with a as b: passa<
    NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    z	async.fooz�    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    �asyncz/    NAME       'async'       (1, 0) (1, 5)
    zasync
#comment
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    zasync
...
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    zasync
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    z
foo.async + 1z�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    zasync def foo(): passa;
    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    zNasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    AWAIT      'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    AWAIT      'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    AWAIT      'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    z)async def foo():
  async for i in 1: passa�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    ASYNC      'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    zasync def foo(async): awaitah
    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    ASYNC      'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    AWAIT      'await'       (1, 22) (1, 27)
    z@def f():

  def baz(): pass
  async def bar(): pass

  await = 2a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    zFasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    AWAIT      'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    )
r/)
r+r#r#r$�
test_async�s`#	zTokenizeTest.test_asyncN)�__name__�
__module__�__qualname__r/r5r9r:r;r<r?r@rArBrCrDrErFrGrHrIrJrKrMr#r#r#r$r&#s*
#9(')&%r&c
Cs�g}
tt
|jd
�
�
j�
}x\|D]T\}}}}}|tkrdd|krd|
jtdftdftt	|�
ftdfg�

q|
j
||f�

qWt|
�
jd
�
S)Nzutf-8�
.�Decimalr>�
))
rrr'r(r�extendrrr	�reprrr�decode)r,r�
g�toknum�tokval�
_r#r#r$�decistmt�s










r[c@seZ
dZd
d�ZdS)�TestMiscc
CsPd
dlm
}

d}|jt|�
d�
|jtt|�
�
d�
|jtt|�
�
|
d�
�
dS)Nr
)
rRz+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)�decimalrRr)r[ZassertRegexrU�eval)r+rRr,r#r#r$�
test_decistmt�s




zTestMisc.test_decistmtN)rNrOrPr_r#r#r#r$r\�sr\c@s@eZ
dZd
Zdd�Zdd�Zdd�Zdd	�Zd
d�Zdd
�Z	dS)�TestTokenizerAdheresToPep0263zU
    Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
    cCs,tj
jtj
jt�
|
�}tj|t|d
��
dS)N�rb)�os�path�join�dirname�__file__�
TestRoundtrip�check_roundtripr)r+�filenamercr#r#r$�	_testFile�s

z'TestTokenizerAdheresToPep0263._testFilec
Csd
}
|j|
�

dS)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt)
rj)r+r.r#r#r$�'test_utf8_coding_cookie_and_no_utf8_bom�s

zETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomc
Csd
}
|jt
|j|
�
dS)a�

        As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
        allowed encoding for the comment is 'utf-8'.  The text file used in
        this test starts with a BOM signature, but specifies latin1 as the
        coding, so verify that a SyntaxError is raised, which matches the
        behaviour of the interpreter when it encounters a similar condition.
        z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN)�assertRaises�SyntaxErrorrj)r+r.r#r#r$�&test_latin1_coding_cookie_and_utf8_bom�s
zDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomc
Csd
}
|j|
�

dS)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt)
rj)r+r.r#r#r$�"test_no_coding_cookie_and_utf8_bom�s

z@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomc
Csd
}
|j|
�

dS)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt)
rj)r+r.r#r#r$�$test_utf8_coding_cookie_and_utf8_bom�s

zBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomc

Cs$|jt
|jd
�
|jt
|jd�
dS)Nz
bad_coding.pyzbad_coding2.py)rlrmrj)
r+r#r#r$�test_bad_coding_cookie�s

z4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN)
rNrOrP�__doc__rjrkrnrorprqr#r#r#r$r`�sr`c@seZ
dZd
d�Zdd�ZdS)�
Test_Tokenizec
sPd
}
|
jd�
�
d���
fdd�}t
t|dd��
dd
�}dg
}|j||d�
dS)N�"ЉЊЈЁЂ"zutf-8Fc
s�sd
��
SdSdS)NT�r#r#)�firstr"r#r$r(�s

zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline)
�encodingrr1�r
�zbytes not decoded with encodingr2�rr
�rry)rxrtrzr{rt)r'r3rr))r+�literalr(r4�expected_tokensr#)rvr"r$�.test__tokenize_decodes_with_specified_encoding�s




	


z<Test_Tokenize.test__tokenize_decodes_with_specified_encodingc
sFd
�
d���
fdd�}
tt
|
dd��
dd�}dg
}|j||d�
dS)N�"ЉЊЈЁЂ"Fc
s�sd
��
SdSdS)NTrur#r#)rvr|r#r$r(s

zQTest_Tokenize.test__tokenize_does_not_decode_with_encoding_none.<locals>.readline)
rwr1rxrr
ryz*string not tokenized when encoding is Noner2�rr
�rry)rxrr�r�r)r3rr))r+r(r4r}r#)rvr|r$�1test__tokenize_does_not_decode_with_encoding_nones


	


z?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN)rNrOrPr~r�r#r#r#r$rs�srsc@s�eZ
dZd
d�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�Zdd�Zdd�Zdd �Zd!d"�Zd#d$�Zd%d&�Zd'd(�Zd)S)*�TestDetectEncodingcsd
���
fdd�}|S)Nr
c
s$�t�
�
krt
�
�
�}�d
7�|S)Nr)r�
StopIteration)
r")�index�linesr#r$r(s




z1TestDetectEncoding.get_readline.<locals>.readliner#)r+r�r(r#)r�r�r$�get_readlines

zTestDetectEncoding.get_readlinec
Cs>d}
t|j
|
�
�
\}}|j|d�
|j|t|
dd��
�
dS)N�# something
�print(something)
�do_something(else)
zutf-8r1)r�r�r�)r
r�r)r3)r+r�rw�consumed_linesr#r#r$�test_no_bom_no_encoding_cookie"s



z1TestDetectEncoding.test_no_bom_no_encoding_cookiec
Cs6d}
t|j
|
�
�
\}}|j|d�
|j|ddg�
dS)N�# something
�print(something)
�do_something(else)
z	utf-8-sigs# something
)r�r�r�)r
r�r))r+r�rwr�r#r#r$�test_bom_no_cookie,s




z%TestDetectEncoding.test_bom_no_cookiec
Cs4d}
t|j
|
�
�
\}}|j|d�
|j|d
g
�
dS)N�# -*- coding: latin-1 -*-
�print(something)
�do_something(else)
z
iso-8859-1)r�r�r�)r
r�r))r+r�rwr�r#r#r$�test_cookie_first_line_no_bom7s



z0TestDetectEncoding.test_cookie_first_line_no_bomc
Cs4d}
t|j
|
�
�
\}}|j|d�
|j|dg
�
dS)N�# coding=utf-8
�print(something)
�do_something(else)
z	utf-8-sigs# coding=utf-8
)r�r�r�)r
r�r))r+r�rwr�r#r#r$�&test_matched_bom_and_cookie_first_lineAs



z9TestDetectEncoding.test_matched_bom_and_cookie_first_linec
Cs d}
|j|
�
}|j
tt|�
dS)N�## vim: set fileencoding=ascii :
�print(something)
�do_something(else)
)r�r�r�)r�rlrmr
)r+r�r(r#r#r$�<test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorKs




zOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorc
Cs:d}
t|j
|
�
�
\}}|j|d�
d
dg}|j||�
dS)N�
#! something
� # vim: set fileencoding=ascii :
�print(something)
�do_something(else)
�ascii)r�r�r�r�)r
r�r))r+r�rwr�r-r#r#r$�test_cookie_second_line_no_bomTs





z1TestDetectEncoding.test_cookie_second_line_no_bomc
Cs6d}
t|j
|
�
�
\}}|j|d�
|j|ddg�
dS)N�#! something
�f# coding=utf-8
�print(something)
�do_something(else)
z	utf-8-sigs
#! something
)r�r�r�r�)r
r�r))r+r�rwr�r#r#r$�'test_matched_bom_and_cookie_second_line`s





z:TestDetectEncoding.test_matched_bom_and_cookie_second_linec
Cs d}
|j|
�
}|j
tt|�
dS)N�#! something
� # vim: set fileencoding=ascii :
�print(something)
�do_something(else)
)r�r�r�r�)r�rlrmr
)r+r�r(r#r#r$�=test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorls




zPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorc
Cs8d}
t|j
|
�
�
\}}|j|d�
d
g
}|j||�
dS)N�print('£')
�%# vim: set fileencoding=iso8859-15 :
�
print('€')
zutf-8)r�r�r�)r
r�r))r+r�rwr�r-r#r#r$�/test_cookie_second_line_noncommented_first_linevs




zBTestDetectEncoding.test_cookie_second_line_noncommented_first_linec
Cs:d}
t|j
|
�
�
\}}|j|d�
d
dg}|j||�
dS)N�
#print('£')
�%# vim: set fileencoding=iso8859-15 :
�
print('€')
z
iso8859-15)r�r�r�)r
r�r))r+r�rwr�r-r#r#r$�,test_cookie_second_line_commented_first_line�s




z?TestDetectEncoding.test_cookie_second_line_commented_first_linec
Cs:d}
t|j
|
�
�
\}}|j|d�
d
dg}|j||�
dS)N�

�%# vim: set fileencoding=iso8859-15 :
�
print('€')
z
iso8859-15)r�r�r�)r
r�r))r+r�rwr�r-r#r#r$�(test_cookie_second_line_empty_first_line�s




z;TestDetectEncoding.test_cookie_second_line_empty_first_linec
	Csld}
xb|
D]Z}xTdD]L}|jd|�}d	d
|j
d�
dd
df}|j|�
}t|�
\}}|j|d�
qWq
WdS)N�latin-1�
iso-8859-1�iso-latin-1�latin-1-unix�iso-8859-1-unix�iso-latin-1-mac�
-rZs#!/usr/bin/python
s
# coding: r�r�sprint(things)
sdo_something += 4
)r�r�r�r�r�r�)r�rZ)�replacer'r�r
r))	r+�	encodingsrw�rep�encr��rl�foundr�r#r#r$�test_latin1_normalization�s













z,TestDetectEncoding.test_latin1_normalizationc
Cs d}
|j|
�
}|j
tt|�
dS)N�
print("�"))
r�)r�rlrmr
)r+r�r(r#r#r$�test_syntaxerror_latin1�s

z*TestDetectEncoding.test_syntaxerror_latin1c
	Csjd}
x`|
D]X}xRdD]J}|jd|�}dd|j
d�
d	d
f}|j|�
}t|�
\}}|j|d
�
qWq
WdS)
N�utf-8�	utf-8-mac�
utf-8-unixr�rZs#!/usr/bin/python
s
# coding: r�r�s1 + 3
)r�r�r�)r�rZ)r�r'r�r
r))	r+r�rwr�r�r�r�r�r�r#r#r$�test_utf8_normalization�s











z*TestDetectEncoding.test_utf8_normalizationc
Cs�|jd�
}
t
|
�
\}}|j|d�
|j|d
g
�
t
|jf�
�
\}}|j|d�
|j|g�
|jd�
}
t
|
�
\}}|j|d�
|j|d
g
�
|jd	�
}
t
|
�
\}}|j|d�
|j|g�
|jd
�
}
|jtt
|
�
dS)N�print(something)
zutf-8�print(something)
z	utf-8-sig��# coding: bad
)
r�)
r�)
r�)
r�)r�r
r)rlrm)r+r(rwr�r#r#r$�test_short_files�s"
















z#TestDetectEncoding.test_short_filesc
Cs4|jd�
}
t
|
�
\}}|j|d�
|j|d
g
�
dS)N�print("#coding=fake")zutf-8)
r�)r�r
r))r+r(rwr�r#r#r$�test_false_encoding�s



z&TestDetectEncoding.test_false_encodingc
Cs�tj
d
}
|jtj|
�
xrdD]j}t|
d|d��"}td||d�
td|d�
WdQRXt|
�
�"}|j|j|�
|j|j	d	�
WdQRXqWt|
dd
d��}td|d�
WdQRXt|
�
�"}|j|jd
�
|j|j	d	�
WdQRXdS)Nz.py�iso-8859-15�utf-8�
w)
rwz# coding: %s)
�fileuprint('euro:€')�
rz	utf-8-sig)r�r�)
rZTESTFNZ
addCleanup�unlinkr�print�
tokenize_openr)rw�mode)r+rirw�fpr#r#r$�	test_open�s















zTestDetectEncoding.test_openc
s~d
}
d�G�f
dd�d�}|jt
�
�
|�|
�}|`t|j�

WdQRX|jt
dj|
�
��
|�|
�}t|j�

WdQRXdS)NZsome_file_path�
print("�")cs eZ
dZd
d�Z�f
dd�ZdS)z;TestDetectEncoding.test_filename_in_exception.<locals>.BunkcSs||_|
|_
d
|_dS)Nr
)�nameZ_lines�_index)r+r�rcr#r#r$�__init__�s


zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__c
s.|jt
��
krt�
�|j}
|jd
7_|
S)Nr)r�rr�)r+r")
r�r#r$r(�s






zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readlineN)rNrOrPr�r(r#)
r�r#r$�Bunk�s
r�z.*{})
r�)rlrmr�r
r(r6�format)r+rcr�Zinsr#)
r�r$�test_filename_in_exception�s







z-TestDetectEncoding.test_filename_in_exceptionc
Cs@td
�
}
t
jd|
d��
|jttd�
WdQRX|j|
j�

dS)Ns#coding:xxxztokenize._builtin_open)
Zreturn_valueZfoobar)rrZpatchrlrmr�Z
assertTrue�closed)r+�
mr#r#r$�test_open_errors


z"TestDetectEncoding.test_open_errorN)rNrOrPr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r#r#r#r$r�s(


	


r�c@s4eZ
dZd
d�Zdd�Zdd�Zdd�Zd	d
�ZdS)�TestTokenizec
s�d
dl}
t
��
d��
f
dd�}�f
dd�}d
��f
dd�}|
j}|
j}||
_||
_z(t|�
}|jt|�
dd	d
ddd
g�
Wd||
_||
_X|j��
�
dS)Nr
c

s�d
dgfS)Nsfirstssecondr#)
r()
rwr#r$�mock_detect_encodings
z8TestTokenize.test_tokenize.<locals>.mock_detect_encodingcs(|
�g}x|�}|r |j|�

q
|SdS)
N)
r)r(rw�outZ	next_line)
�
encoding_usedr#r$�mock__tokenizes







z2TestTokenize.test_tokenize.<locals>.mock__tokenizecs �d
7��dkrdSt��
j
�S)Nr�ru)�strr'r#)
�counterr#r$�
mock_readline(s


z1TestTokenize.test_tokenize.<locals>.mock_readlinesfirstssecond�
1�
2�
3�
4)r�objectr
rr)r3)r+Ztokenize_moduler�r�r�Zorig_detect_encodingZorig__tokenize�resultsr#)r�rwr�r$�
test_tokenizes$













zTestTokenize.test_tokenizec
Csjg}
x"td
�
D]}|
j
dj|d�
�

qW|
j
d�

dj|
�
}
ttt|
jd�
�
j�
�
}|j	|dj
d�
dS)	Ni�
zdef i{i}(): return {i})
�
iZOK�

zutf-8rx���)�rangerr�rdr3rrr'r(r)�string)r+Zbufr�Ztoksr#r#r$�test_oneline_defs=s







zTestTokenize.test_oneline_defscGs�tt
t|
jd
�
�
j�
�
}t|�
}|jt|�
d|�
|jtj|dj	tjt
�
x6t|�
D]*}|jtj||dj	tj||�
q\W|jtj|d|j	tjtj�
|jtj|d|j	tjtj
�
dS)Nzutf-8rxr
rr1)r3rrr'r(rr)rr�
exact_typerr�rr
)r+ZopstrZoptypesr4Znum_optypesr�r#r#r$�assertExactTypeEqualIs











z!TestTokenize.assertExactTypeEqualc


Cs�|jd
t
jt
j�
|jdt
jt
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
j	�
|jdt
j
�
|jdt
j�
|jd	t
j�
|jd
t
j
�
|jdt
j�
|jdt
j�
|jd
t
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
jt
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
j�
|jdt
j �
|jdt
j!�
|jdt
j"�
|jdt
j#�
|jd t
j$�
|jd!t
j%�
|jd"t
j&�
|jd"t
j&�
|jd#t
j'�
|jd$t
j(�
|jd%t
j)�
|jd&t
j*�
|jd't
j+�
|jd(t
j,�
|jd)t
j-�
|jd*t.t
jt/t
j	t.t
jt/t
jt.t
jt/�
|jd+t
jt
j/t
jt
j/t
jt
j/t
j�
|jd,t
jt
jt
j.t
jt
j/t
j�
dS)-Nz()z[]�
:�
,�
;�
+r��
*�
/�
|�
&�
<�
>�
=rQ�
%z{}z==z!=z<=z>=�
~�
^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=�
@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1))0r
r�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL�	LESSEQUAL�GREATEREQUAL�TILDE�
CIRCUMFLEX�	LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR�	PLUSEQUAL�MINEQUAL�	STAREQUAL�
SLASHEQUAL�PERCENTEQUAL�
AMPEREQUAL�	VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL�DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUALrr)
r+r#r#r$�test_exact_typeWsv

























































zTestTokenize.test_exact_typec

Cs|jd
t
j�
dS)Nz@          )r
rr:
)
r+r#r#r$�%test_pathological_trailing_whitespace�sz2TestTokenize.test_pathological_trailing_whitespaceN)rNrOrPr�r�r
r<
r=
r#r#r#r$r�s
(>r�c@s$eZ
dZd
d�Zdd�Zdd�ZdS)�UntokenizeTestc
CsZt�}
d
|
_
d
|
_|jt�
�}|
jd�

WdQRX|j|jjdd�
|jt|
jd�
dS)Nr1rrxr
z'start (1,3) precedes previous end (2,2))rrx)r1r)	r�prev_row�prev_colrl�
ValueError�add_whitespacer)Z	exception�args)r+�
u�cmr#r#r$�test_bad_input_order�s





z#UntokenizeTest.test_bad_input_orderc
Csft�}
d
|
_
d
|
_g|
_|
jd	�

|j|
jdg
�
d|
_
|
jd
�

|j|
jdddg�
tj|d�
dS)Nrr1r
z\
�z\
\
z    za
  b
    c
  \
  c
)r1r
)rG
rG
)rr?
r@
r4rB
r)rgrh)r+rD
r#r#r$�test_backslash_continuation�s










z*UntokenizeTest.test_backslash_continuationc
Cs�t�}
t
d
f}tdf|g}|
j|tg�
�
|j|
jdg
�
t�}
|j|
jt|g
�
�
d�
t�}
|j|
jt|�
�
d�
|j|
jd�
|jtt|�
�
d�
dS)NZHellozutf-8zHello sHello )	rrr�compat�iterr)r4rrw)r+rD
rr4r#r#r$�test_iter_compat�s










zUntokenizeTest.test_iter_compatN)rNrOrPrF
rH
rK
r#r#r#r$r>
�s
r>
c@sLeZ
dZd
d�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dS)rgcCs�t|
t
�r|
jd
�
}n|
j�}|
j�
t|jdd�
�
j}tt	|�
�
}dd�|D�
}t
|�
}t|jdd�
�
j}dd�t	|�
D�
}|j||�
t
|�
}	t|	jdd�
�
j}
dd�t	|
�
D�
}|j||�
dS)	a)
        Test roundtrip for `untokenize`. `f` is an open file or a string.
        The source code in f is tokenized to both 5- and 2-tuples.
        Both sequences are converted back to source code via
        tokenize.untokenize(), and the latter tokenized again to 2-tuples.
        The test fails if the 3 pair tokenizations do not match.

        When untokenize bugs are fixed, untokenize with 5-tuples should
        reproduce code that does not contain a backslash continuation
        following spaces.  A proper test should test this.
        zutf-8T)
�keependsc
Ssg|]}
|
dd
��qS)Nr1r#)�.0r8r#r#r$�
<listcomp>�sz1TestRoundtrip.check_roundtrip.<locals>.<listcomp>c
Ssg|]}
|
dd
��qS)Nr1r#)rM
r8r#r#r$rN
�sc
Ssg|]}
|
dd
��qS)Nr1r#)rM
r8r#r#r$rN
�sN)�
isinstancer�r'�read�closerJ
r�__next__r3rrr))r+r.�coder(Ztokens5Ztokens2Zbytes_from2Z	readline2Z
tokens2_from2Zbytes_from5Z	readline5Z
tokens2_from5r#r#r$rh�s











zTestRoundtrip.check_roundtripc
	Cst|jd
�

|jd�

|jd�

t
jd�
}
t|
d��}|j|�

WdQRX|jd�

|jd�

|jd�

|jd	�

dS)
Nzif x == 1:
    print(x)
z # This is a comment
# This also
zif x == 1 : 
  print(x)
ztokenize_tests.txtrazoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
z�class Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
)rhr�findfiler)r+�fnr.r#r#r$�test_roundtrip�s









zTestRoundtrip.test_roundtripc

Cs|jd
�

dS)Nzna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
)
rh)
r+r#r#r$�test_continuationszTestRoundtrip.test_continuationc

Cs|jd
�

|jd�

dS)Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0)
rh)
r+r#r#r$rH
s
z)TestRoundtrip.test_backslash_continuationc

Cs|jd
�

dS)Nz'' '')
rh)
r+r#r#r$�test_string_concatenation"sz'TestRoundtrip.test_string_concatenationc
Cs�d
dl}
d
dl
}tjd�
}tjj|�
p*tj}|
jtjj|d��
}|j	tjj|d��

x$dD]}|j	tjj|d�|�

qZWtj
d
�
s�|j|d�}xD|D]<}t|d��(}|j
|d�
�
|j|�

WdQRXWdQRXq�WdS)Nr
ztokenize_tests.txtztest*.pyztest_unicode_identifiers.py�buffer�builtin�fileio�inspectrb�platform�sysz
test_%s.pyZcpu�
ra)
r�)rY
rZ
r[
r\
rbr]
r^
)�glob�randomrrT
rbrcre�curdirrd�removeZis_resource_enabledZsamplerZsubTestrh)r+r`
ra
rU
ZtempdirZ	testfilesr.Ztestfiler#r#r$�test_random_files&s












zTestRoundtrip.test_random_filescCs,t|
t
�r|
jd
�
}
ttt|
�
j�
�
jd
�
S)Nzutf-8)rO
r�r'rrrr(rV)r+rS
r#r#r$�	roundtripAs




zTestRoundtrip.roundtripc
Cs6d
}
|j|
�
j
d�
}|j|d|d�
|j|
�

dS)z�
        Ensure that although whitespace might be mutated in a roundtrip,
        the semantic meaning of the indentation remains consistent.
        zif False:
	x=3
	x=3
r�rr1N)re
�splitr)rh)r+rS
Z	codelinesr#r#r$�#test_indentation_semantics_retainedFs


z1TestRoundtrip.test_indentation_semantics_retainedN)rNrOrPrhrV
rW
rH
rX
rd
re
rg
r#r#r#r$rg�s '
	rg�__main__)(Ztestrrrrrrrr	r
rrr
rr�rrr�iorZunittestrrZtest.test_grammarrrrbrr%r&r[r\r`rsr�r�r>
rgrN�mainr#r#r#r$�<module>
s<
D



%&)