B

��g���@s:
dd
lm
Z

ddlmZmZmZmZmZmZmZm	Z	m
Z
mZmZm
ZmZmZmZ
ddlmZ
ddlZddlmZmZ
ddlmZmZ
ddlZddlZdd�ZGd	d
�d
e�Zdd�ZGd
d�de�ZGdd�de�Z Gdd�de�Z!Gdd�de�Z"Gdd�de�Z#Gdd�de�Z$Gdd�de�Z%e&dk�
r6e�'�
dS)�)
�support)�tokenize�	_tokenize�
untokenize�NUMBER�NAME�OP�STRING�	ENDMARKER�ENCODING�tok_name�detect_encoding�open�Untokenizer�generate_tokens�NEWLINE)
�BytesION)�TestCase�mock)�VALID_UNDERSCORE_LITERALS�INVALID_UNDERSCORE_LITERALSc
Cs�g}t|
�
��
}|
d
dk}xh|D]`\}}}}}	|tkr:P|rT|tkrT|d|krTq"t|}|�d|d�d|d�d|�d|���

q"W|S)N���z
r
z    Z10�
 Z13)�len�
splitlinesr
rr�append)
Ztoken_generatorZ
source_string�resultZ	num_linesZmissing_trailing_nl�type�token�start�end�line�r"�./usr/local/lib/python3.7/test/test_tokenize.py�stringify_tokens_from_sources







*r$c@s�eZ
dZd
d�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�Zdd�Zdd�Zdd �Zd!d"�Zd#d$�Zd%d&�Zd'd(�Zd)S)*�TokenizeTestcCs<t|
�
d
�
�
}tt|j�
|
�}|�|dg
|�����
dS)Nzutf-8z*    ENCODING   'utf-8'       (0, 0) (0, 0))r�encoder$r�readline�assertEqual�rstripr)�self�
s�expected�
frr"r"r#�check_tokenize*s



zTokenizeTest.check_tokenizec
CsDtd
�
d�
�
}
tt|
j�
�
}|�|djt�
|�|djt�
dS)N�
xzutf-8���r)	rr&�listrr'r(rrr
)r*r-�tokensr"r"r#�test_implicit_newline4s


z"TokenizeTest.test_implicit_newlinec
	CsT|�d
d�
|�dd�
d}
t
|
�
j}|�td��
xt|�
D]}q>WWdQRXdS)Nz1 + 1z�    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z3if False:
    # NL
    
    True = False # NEWLINE
af    NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NL         '\n'          (3, 4) (3, 5)
    INDENT     '    '        (4, 0) (4, 4)
    NAME       'True'        (4, 4) (4, 8)
    OP         '='           (4, 9) (4, 10)
    NAME       'False'       (4, 11) (4, 16)
    COMMENT    '# NEWLINE'   (4, 17) (4, 26)
    NEWLINE    '\n'          (4, 26) (4, 27)
    DEDENT     ''            (5, 0) (5, 0)
    sdef k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation level)r.rr'�assertRaisesRegex�IndentationErrorr)r*Zindent_error_filer'�tokr"r"r#�
test_basic<s






zTokenizeTest.test_basicc

Csp|�d
d�
|�dd�
|�dd�
|�dd�
|�d	d
�
|�dd�
|�d
d�
|�dd�
|�dd�
dS)Nz0xff <= 255z�    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    z0b10 <= 255z�    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    z0o123 <= 0O123z�    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    z1234567 > ~0x15z�    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    z2134568 != 1231515z�    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    z(-124561-1) & 200000000ac
    OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    z0xdeadbeef != -1z�    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    z0xdeadc0de & 12345z�    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    z0xFF & 0x15 | 1234z�    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    )
r.)
r*r"r"r#�test_intas$




	


zTokenizeTest.test_intc

Cs4|�d
d�
|�dd�
|�dd�
|�dd�
dS)	Nzx = 0z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xfffffffffff' (1, 4) (1, 17)
    zx = 123141242151251616110z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151251616110' (1, 4) (1, 25)
    zx = -15921590215012591z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '15921590215012591' (1, 5) (1, 22)
    )
r.)
r*r"r"r#�	test_long�s


zTokenizeTest.test_longc

CsX|�d
d�
|�dd�
|�dd�
|�dd�
|�d	d
�
|�dd�
|�d
d�
dS)Nzx = 3.14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    zx = 314159.z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    zx = .314159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    zx = 3e14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    z	x = 3E123z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    z
x+y = 3e-1230z�    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    zx = 3.14e159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    )
r.)
r*r"r"r#�
test_float�s





zTokenizeTest.test_floatc
CsRd
d�}
x&tD]}d|krq|�
|
|�
|�
qWxtD]}|�|
|�
|�
q6WdS)Nc
Ss<t|�
d
�
�
}
x(t|
j�
D]\}}}}}|tkr|SqWdS)Nzutf-8z
invalid token)rr&rr'r)r+r-Ztoktyperrr r!r"r"r#�number_token�s





z;TokenizeTest.test_underscore_literals.<locals>.number_token�
()rr(rZassertNotEqual)r*r;Zlitr"r"r#�test_underscore_literals�s






z%TokenizeTest.test_underscore_literalsc

Cs<
|�d
d�
|�dd�
|�dd�
|�dd�
|�d	d
�
|�dd�
|�d
d�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd �
|�d!d"�
|�d#d$�
|�d%d&�
|�d'd(�
|�d)d*�
|�d+d,�
|�d-d.�
|�d/d0�
|�d1d2�
|�d3d4�
dS)5Nzx = ''; y = ""a5
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    zx = '"'; y = "'"a6
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    zx = "doesn't "shrink", does it"z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    zx = 'abc' + 'ABC'z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    zy = "ABC" + "ABC"z�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    z%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a�
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    z%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a�
    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    zu'abc' + U'abc'z�    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    zu"abc" + U"abc"z�    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    zb'abc' + B'abc'z�    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    zb"abc" + B"abc"z�    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    z%br'abc' + bR'abc' + Br'abc' + BR'abc'a;
    STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    z%br"abc" + bR"abc" + Br"abc" + BR"abc"a;
    STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    z%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;
    STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    z%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;
    STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    z"a\
de\
fg"z3    STRING     '"a\\\nde\\\nfg"' (1, 0) (3, 3)
    zu"a\
de"z/    STRING     'u"a\\\nde"'  (1, 0) (2, 3)
    zrb"a\
d"z/    STRING     'rb"a\\\nd"'  (1, 0) (2, 2)
    z
"""a\
b"""z0    STRING     '"""a\\\nb"""' (1, 0) (2, 4)
    zu"""a\
b"""z1    STRING     'u"""a\\\nb"""' (1, 0) (2, 4)
    zrb"""a\
b\
c"""z7    STRING     'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
    zf"abc"z/    STRING     'f"abc"'      (1, 0) (1, 6)
    z	fR"a{b}c"z/    STRING     'fR"a{b}c"'   (1, 0) (1, 9)
    z
f"""abc"""z0    STRING     'f"""abc"""'  (1, 0) (1, 10)
    zf"abc\
def"z1    STRING     'f"abc\\\ndef"' (1, 0) (2, 4)
    zRf"abc\
def"z2    STRING     'Rf"abc\\\ndef"' (1, 0) (2, 4)
    )
r.)
r*r"r"r#�test_string�sx





























zTokenizeTest.test_stringc

Cs(|�d
d�
|�dd�
|�dd�
dS)Nz!def d22(a, b, c=2, d=2, *k): passa    NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    zdef d01v_(a=1, *k, **w): passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    z&def d23(a: str, b: int=3) -> int: passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    OP         ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    OP         '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    OP         ')'           (1, 24) (1, 25)
    OP         '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    OP         ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    )
r.)
r*r"r"r#�
test_function�
s


zTokenizeTest.test_functionc

Cs|�d
d�
dS)NzXif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa�    NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    )
r.)
r*r"r"r#�test_comparison�
s"zTokenizeTest.test_comparisonc

Cs|�d
d�
dS)Nzx = 1 << 1 >> 5a6
    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    )
r.)
r*r"r"r#�
test_shift�
szTokenizeTest.test_shiftc

Cs|�d
d�
dS)Nz%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    )
r.)
r*r"r"r#�
test_additive�
szTokenizeTest.test_additivec

Cs|�d
d�
dS)Nzx = 1//1*1/5*12%0x12@42a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    )
r.)
r*r"r"r#�test_multiplicativesz TokenizeTest.test_multiplicativec

Cs|�d
d�
|�dd�
dS)Nz~1 ^ 1 & 1 |1 ^ -1a�
    OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    z-1*1/1+1*1//1 - ---1**1aH    OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    )
r.)
r*r"r"r#�
test_unary(s
zTokenizeTest.test_unaryc

Cs|�d
d�
dS)Nz/import sys, time
x = sys.modules['time'].time()a�    NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    )
r.)
r*r"r"r#�
test_selectorMszTokenizeTest.test_selectorc

Cs|�d
d�
dS)Nz @staticmethod
def foo(x,y): passa    OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    )
r.)
r*r"r"r#�test_methodcs
zTokenizeTest.test_methodc

Cs|�d
d�
dS)Nzdef f():
	if x
        	passa_    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    )
r.)
r*r"r"r#�	test_tabstszTokenizeTest.test_tabsc

Cs|�d
d�
dS)Nu!Örter = 'places'
grün = 'green'u7
    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    )
r.)
r*r"r"r#�test_non_ascii_identifiers�sz'TokenizeTest.test_non_ascii_identifiersc

Cs|�d
d�
dS)Nu#Örter = u'places'
grün = U'green'u7
    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    )
r.)
r*r"r"r#�test_unicode�szTokenizeTest.test_unicodec

Cs
|�d
d�
|�dd�
|�dd�
|�dd�
|�d	d
�
|�dd�
|�d
d�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd�
|�dd �
|�d!d"�
|�d#d$�
|�d%d&�
|�d'd(�
|�d)d*�
dS)+Nz	async = 1z�    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    za = (async = 1)a8
    NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    zasync()z�    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    zclass async(Bar):passa<
    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    zclass async:passz�    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    z	await = 1z�    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    z	foo.asyncz�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    zasync for a in b: passa;
    NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    zasync with a as b: passa<
    NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    z	async.fooz�    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    �asyncz/    NAME       'async'       (1, 0) (1, 5)
    zasync
#comment
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    zasync
...
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    zasync
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    z
foo.async + 1z�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    zasync def foo(): passa;
    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    zNasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a�    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    z)async def foo():
  async for i in 1: passa�    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    zasync def foo(async): awaitah
    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    z@def f():

  def baz(): pass
  async def bar(): pass

  await = 2a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    zFasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    )
r.)
r*r"r"r#�
test_async�s\#	zTokenizeTest.test_asyncN)�__name__�
__module__�__qualname__r.r3r7r8r9r:r=r>r?r@rArBrCrDrErFrGrHrIrKr"r"r"r#r%#s*
%9('<&%r%c
Cs�g}
tt
|�d
�
�
j�
}x\|D]T\}}}}}|tkrdd|krd|
�tdftdftt	|�
ftdfg�

q|
�
||f�

qWt|
�
�d
�
S)Nzutf-8�
.�Decimalr<�
))
rrr&r'r�extendrrr	�reprrr�decode)r+r�
g�toknum�tokval�
_r"r"r#�decistmt�s










rYc@seZ
dZd
d�ZdS)�TestMiscc
CsPd
dlm
}

d}|�t|�
d�
|�tt|�
�
d�
|�tt|�
�
|
d�
�
dS)Nr
)
rPz+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)�decimalrPr(rYZassertRegexrS�eval)r*rPr+r"r"r#�
test_decistmt�s




zTestMisc.test_decistmtN)rLrMrNr]r"r"r"r#rZ�srZc@s<eZ
dZd
d�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
S)�TestTokenizerAdheresToPep0263cCs,tj
�tj
�t�
|
�}t�|t|d
��
dS)N�rb)�os�path�join�dirname�__file__�
TestRoundtrip�check_roundtripr)r*�filenamerar"r"r#�	_testFile�s

z'TestTokenizerAdheresToPep0263._testFilec
Csd
}
|�|
�

dS)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt)
rh)r*r-r"r"r#�'test_utf8_coding_cookie_and_no_utf8_bom�s

zETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bomc
Csd
}
|�t
|j|
�
dS)Nz8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txt)�assertRaises�SyntaxErrorrh)r*r-r"r"r#�&test_latin1_coding_cookie_and_utf8_bom�s
zDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bomc
Csd
}
|�|
�

dS)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txt)
rh)r*r-r"r"r#�"test_no_coding_cookie_and_utf8_bom�s

z@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bomc
Csd
}
|�|
�

dS)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txt)
rh)r*r-r"r"r#�$test_utf8_coding_cookie_and_utf8_bom�s

zBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bomc

Cs$|�t
|jd
�
|�t
|jd�
dS)Nz
bad_coding.pyzbad_coding2.py)rjrkrh)
r*r"r"r#�test_bad_coding_cookie�s

z4TestTokenizerAdheresToPep0263.test_bad_coding_cookieN)	rLrMrNrhrirlrmrnror"r"r"r#r^�sr^c@seZ
dZd
d�Zdd�ZdS)�
Test_Tokenizec
sPd
}
|
�d�
�
d���
fdd�}t
t|dd��
dd�}d	g
}|�||d
�
dS)Nu"ЉЊЈЁЂ"zutf-8Fc
s�sd
��
SdSdS)NT�r"r")�firstr!r"r#r's

zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline)
�encoding�
r0)�u"ЉЊЈЁЂ")rtr
)rt�u"ЉЊЈЁЂ"zbytes not decoded with encoding)r&r1rr()r*�literalr'r2�expected_tokensr")rrr!r#�.test__tokenize_decodes_with_specified_encoding
s




	


z<Test_Tokenize.test__tokenize_decodes_with_specified_encodingc
sFd
�
d���
fdd�}
tt
|
dd��
dd�}dg
}|�||d�
dS)	Nu"ЉЊЈЁЂ"Fc
s�sd
��
SdSdS)NTrqr"r")rrrwr"r#r's

zQTest_Tokenize.test__tokenize_does_not_decode_with_encoding_none.<locals>.readline)
rsr0)ruu"ЉЊЈЁЂ")rtr
)rtrvu"ЉЊЈЁЂ"z*string not tokenized when encoding is None)r1rr()r*r'r2rxr")rrrwr#�1test__tokenize_does_not_decode_with_encoding_nones


	


z?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_noneN)rLrMrNryrzr"r"r"r#rp�srpc@s�eZ
dZd
d�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dd�Zdd�Zdd�Z
dd�Zdd�Zdd�Zdd�Zdd �Zd!d"�Zd#d$�Zd%d&�Zd'd(�Zd)S)*�TestDetectEncodingcsd
���
fdd�}|S)Nr
c
s$�t�
�
krt
�
�
�}�d
7�|S)Nrt)r�
StopIteration)
r!)�index�linesr"r#r')s




z1TestDetectEncoding.get_readline.<locals>.readliner")r*r~r'r")r}r~r#�get_readline's

zTestDetectEncoding.get_readlinec
Cs>d
}
t|�
|
�
�
\}}|�|d�
|�|t|
dd��
�
dS)N)s# something
sprint(something)
sdo_something(else)
zutf-8�)r
rr(r1)r*r~rs�consumed_linesr"r"r#�test_no_bom_no_encoding_cookie2s

z1TestDetectEncoding.test_no_bom_no_encoding_cookiec
Cs6d
}
t|�
|
�
�
\}}|�|d�
|�|ddg�
dS)N)s# something
sprint(something)
sdo_something(else)
z	utf-8-sigs# something
sprint(something)
)r
rr()r*r~rsr�r"r"r#�test_bom_no_cookie<s



z%TestDetectEncoding.test_bom_no_cookiec
Cs4d
}
t|�
|
�
�
\}}|�|d�
|�|dg
�
dS)N)s# -*- coding: latin-1 -*-
sprint(something)
sdo_something(else)
z
iso-8859-1s# -*- coding: latin-1 -*-
)r
rr()r*r~rsr�r"r"r#�test_cookie_first_line_no_bomGs

z0TestDetectEncoding.test_cookie_first_line_no_bomc
Cs4d
}
t|�
|
�
�
\}}|�|d�
|�|dg
�
dS)N)s# coding=utf-8
sprint(something)
sdo_something(else)
z	utf-8-sigs# coding=utf-8
)r
rr()r*r~rsr�r"r"r#�&test_matched_bom_and_cookie_first_lineQs

z9TestDetectEncoding.test_matched_bom_and_cookie_first_linec
Cs d
}
|�|
�
}|�
tt|�
dS)N)s## vim: set fileencoding=ascii :
sprint(something)
sdo_something(else)
)rrjrkr
)r*r~r'r"r"r#�<test_mismatched_bom_and_cookie_first_line_raises_syntaxerror[s

zOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorc
Cs:d
}
t|�
|
�
�
\}}|�|d�
ddg}|�||�
dS)N)s
#! something
s # vim: set fileencoding=ascii :
sprint(something)
sdo_something(else)
�asciis
#! something
s # vim: set fileencoding=ascii :
)r
rr()r*r~rsr�r,r"r"r#�test_cookie_second_line_no_bomds



z1TestDetectEncoding.test_cookie_second_line_no_bomc
Cs6d
}
t|�
|
�
�
\}}|�|d�
|�|ddg�
dS)N)s#! something
sf# coding=utf-8
sprint(something)
sdo_something(else)
z	utf-8-sigs
#! something
sf# coding=utf-8
)r
rr()r*r~rsr�r"r"r#�'test_matched_bom_and_cookie_second_lineps



z:TestDetectEncoding.test_matched_bom_and_cookie_second_linec
Cs d
}
|�|
�
}|�
tt|�
dS)N)s#! something
s # vim: set fileencoding=ascii :
sprint(something)
sdo_something(else)
)rrjrkr
)r*r~r'r"r"r#�=test_mismatched_bom_and_cookie_second_line_raises_syntaxerror|s

zPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorc
Cs8d
}
t|�
|
�
�
\}}|�|d�
dg
}|�||�
dS)N)sprint('£')
s%# vim: set fileencoding=iso8859-15 :
s
print('€')
zutf-8sprint('£')
)r
rr()r*r~rsr�r,r"r"r#�/test_cookie_second_line_noncommented_first_line�s



zBTestDetectEncoding.test_cookie_second_line_noncommented_first_linec
Cs:d
}
t|�
|
�
�
\}}|�|d�
ddg}|�||�
dS)N)s
#print('£')
s%# vim: set fileencoding=iso8859-15 :
s
print('€')
z
iso8859-15s
#print('£')
s%# vim: set fileencoding=iso8859-15 :
)r
rr()r*r~rsr�r,r"r"r#�,test_cookie_second_line_commented_first_line�s



z?TestDetectEncoding.test_cookie_second_line_commented_first_linec
Cs:d
}
t|�
|
�
�
\}}|�|d�
ddg}|�||�
dS)N)�

s%# vim: set fileencoding=iso8859-15 :
s
print('€')
z
iso8859-15r�s%# vim: set fileencoding=iso8859-15 :
)r
rr()r*r~rsr�r,r"r"r#�(test_cookie_second_line_empty_first_line�s



z;TestDetectEncoding.test_cookie_second_line_empty_first_linec
	Csld
}
xb|
D]Z}xTdD]L}|�d|�}dd|�
d�
ddd	f}|�|�
}t|�
\}}|�|d
�
qWq
WdS)N)zlatin-1z
iso-8859-1ziso-latin-1zlatin-1-unixziso-8859-1-unixziso-latin-1-mac)�
-rXr�s#!/usr/bin/python
s
# coding: r�r�sprint(things)
sdo_something += 4
z
iso-8859-1)�replacer&rr
r()	r*�	encodingsrs�rep�encr~�rl�foundr�r"r"r#�test_latin1_normalization�s











z,TestDetectEncoding.test_latin1_normalizationc
Cs d
}
|�|
�
}|�
tt|�
dS)N)
s
print("�"))rrjrkr
)r*r~r'r"r"r#�test_syntaxerror_latin1�s

z*TestDetectEncoding.test_syntaxerror_latin1c
	Csjd
}
x`|
D]X}xRdD]J}|�d|�}dd|�
d�
ddf}|�|�
}t|�
\}}|�|d	�
qWq
WdS)
N)zutf-8z	utf-8-macz
utf-8-unix)r�rXr�s#!/usr/bin/python
s
# coding: r�r�s1 + 3
zutf-8)r�r&rr
r()	r*r�rsr�r�r~r�r�r�r"r"r#�test_utf8_normalization�s











z*TestDetectEncoding.test_utf8_normalizationc
Cs�|�d
�
}
t
|
�
\}}|�|d�
|�|dg
�
t
|�d�
�
\}}|�|d�
|�|g�
|�d�
}
t
|
�
\}}|�|d�
|�|dg
�
|�d�
}
t
|
�
\}}|�|d�
|�|g�
|�d�
}
|�tt
|
�
dS)	N)
sprint(something)
zutf-8sprint(something)
r")
sprint(something)
z	utf-8-sig)
s)
s# coding: bad
)rr
r(rjrk)r*r'rsr�r"r"r#�test_short_files�s"
















z#TestDetectEncoding.test_short_filesc
Cs4|�d
�
}
t
|
�
\}}|�|d�
|�|dg
�
dS)N)
sprint("#coding=fake")zutf-8sprint("#coding=fake"))rr
r()r*r'rsr�r"r"r#�test_false_encoding�s



z&TestDetectEncoding.test_false_encodingc

Cs�tj
d
}
|�tj|
�
xrdD]j}t|
d|d��"}td||d�
td|d�
WdQRXt|
�
�"}|�|j|�
|�|j	d�
WdQRXqWt|
dd	d��}td|d�
WdQRXt|
�
�"}|�|jd	�
|�|j	d�
WdQRXdS)
Nz.py)ziso-8859-15zutf-8�
w)
rsz# coding: %s)
�fileuprint('euro:€')�
rz	utf-8-sig)
rZTESTFNZ
addCleanup�unlinkr�print�
tokenize_openr(rs�mode)r*rgrs�fpr"r"r#�	test_open�s















zTestDetectEncoding.test_openc
	s~d
}
d�G�f
dd�d�}|�t
�
�
|�|
�}|`t|j�

WdQRX|�t
d�|
�
��
|�|
�}t|j�

WdQRXdS)NZsome_file_path)
s
print("�")cs eZ
dZd
d�Z�f
dd�ZdS)z;TestDetectEncoding.test_filename_in_exception.<locals>.BunkcSs||_|
|_
d
|_dS)Nr
)�nameZ_lines�_index)r*r~rar"r"r#�__init__s


zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__c
s.|jt
��
krt�
�|j}
|jd
7_|
S)Nrt)r�rr|)r*r!)
r~r"r#r's






zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readlineN)rLrMrNr�r'r")
r~r"r#�Bunks
r�z.*{})rjrkr�r
r'r4�format)r*rar�Zinsr")
r~r#�test_filename_in_exception�s







z-TestDetectEncoding.test_filename_in_exceptionc
	Cs@td
�
}
t
jd|
d��
|�ttd�
WdQRX|�|
j�

dS)Ns#coding:xxxztokenize._builtin_open)
Zreturn_valueZfoobar)rrZpatchrjrkr�Z
assertTrue�closed)r*�
mr"r"r#�test_open_errors


z"TestDetectEncoding.test_open_errorN)rLrMrNrr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r"r"r"r#r{%s(


	


r{c@s4eZ
dZd
d�Zdd�Zdd�Zdd�Zd	d
�ZdS)�TestTokenizec
	s�d
dl}
t
��
d��
f
dd�}�f
dd�}d
��f
dd�}|
j}|
j}||
_||
_z(t|�
}|�t|�
dd	d
ddd
g�
Wd||
_||
_X|���
�
dS)Nr
c

s�d
dgfS)Nsfirstssecondr")
r')
rsr"r#�mock_detect_encoding)s
z8TestTokenize.test_tokenize.<locals>.mock_detect_encodingcs(|
�g}x|�}|r |�|�

q
|SdS)
N)
r)r'rs�outZ	next_line)
�
encoding_usedr"r#�mock__tokenize,s







z2TestTokenize.test_tokenize.<locals>.mock__tokenizecs �d
7��dkrdSt��
�
�S)Nrt�rq)�strr&r")
�counterr"r#�
mock_readline8s


z1TestTokenize.test_tokenize.<locals>.mock_readlinesfirstssecond�
1�
2�
3�
4)r�objectr
rr(r1)r*Ztokenize_moduler�r�r�Zorig_detect_encodingZorig__tokenize�resultsr")r�rsr�r#�
test_tokenize%s$













zTestTokenize.test_tokenizec
Csjg}
x"td
�
D]}|
�
dj|d�
�

qW|
�
d�

d�|
�
}
ttt|
�d�
�
j�
�
}|�	|dj
d�
dS)Ni�
zdef i{i}(): return {i})
�
iZOK�

zutf-8���)�rangerr�rbr1rrr&r'r(�string)r*�bufr�Ztoksr"r"r#�test_oneline_defsMs







zTestTokenize.test_oneline_defscGs�tt
t|
�d
�
�
j�
�
}t|�
}|�t|�
d|�
|�t|djtt	�
x2t
|�
D]&}|�t||djt||�
qXW|�t|d|jttj�
|�t|d|jttj
�
dS)Nzutf-8rur
rtr�)r1rrr&r'rr(r�
exact_typerr�rrr
)r*ZopstrZoptypesr2Znum_optypesr�r"r"r#�assertExactTypeEqualYs












z!TestTokenize.assertExactTypeEqualc

Cs�|�d
t
jt
j�
|�dt
jt
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
j	�
|�dt
j
�
|�dt
j�
|�d	t
j�
|�d
t
j
�
|�dt
j�
|�dt
j�
|�d
t
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
jt
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
j�
|�dt
j �
|�dt
j!�
|�dt
j"�
|�dt
j#�
|�d t
j$�
|�d!t
j%�
|�d"t
j&�
|�d"t
j&�
|�d#t
j'�
|�d$t
j(�
|�d%t
j)�
|�d&t
j*�
|�d't
j+�
|�d(t
j,�
|�d)t
j-�
|�d*t
j.�
|�d+t
j/�
|�d,t0t
jt1t
j	t0t
jt1t
jt0t
jt1�
|�d-t
jt
j1t
jt
j1t
jt
j1t
j�
|�d.t
jt
jt
j0t
jt
j1t
j�
dS)/Nz()z[]�
:�
,�
;�
+r��
*�
/�
|�
&�
<�
>�
=rO�
%z{}z==z!=z<=z>=�
~�
^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z...z->�
@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1))2r�r�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL�	LESSEQUAL�GREATEREQUAL�TILDE�
CIRCUMFLEX�	LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR�	PLUSEQUAL�MINEQUAL�	STAREQUAL�
SLASHEQUAL�PERCENTEQUAL�
AMPEREQUAL�	VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL�DOUBLESLASH�DOUBLESLASHEQUAL�ELLIPSIS�RARROW�AT�ATEQUALrr)
r*r"r"r#�test_exact_typegsz



























































zTestTokenize.test_exact_typec

Cs|�d
t
j�
dS)Nz@          )r�rr

)
r*r"r"r#�%test_pathological_trailing_whitespace�sz2TestTokenize.test_pathological_trailing_whitespaceN)rLrMrNr�r�r�r
r
r"r"r"r#r�#s
(@r�c@s$eZ
dZd
d�Zdd�Zdd�ZdS)�UntokenizeTestc
	CsZt�}
d
|
_
d
|
_|�t�
�}|
�d�

WdQRX|�|jjdd�
|�t|
jd�
dS)Nr�)rtrur
z'start (1,3) precedes previous end (2,2))r�rt)	r�prev_row�prev_colrj�
ValueError�add_whitespacer(Z	exception�args)r*�
u�cmr"r"r#�test_bad_input_order�s





z#UntokenizeTest.test_bad_input_orderc
Csft�}
d
|
_
d
|
_g|
_|
�d�

|�|
jdg
�
d|
_
|
�d�

|�|
jdddg�
t�|d�
dS)	Nrt)r�r
z\
r�)�r
z\
\
z    za
  b
    c
  \
  c
)rr
r
r2r	
r(rerf)r*r
r"r"r#�test_backslash_continuation�s










z*UntokenizeTest.test_backslash_continuationc
Cs�t�}
t
d
f}tdf|g}|
�|tg�
�
|�|
jdg
�
t�}
|�|
�t|g
�
�
d�
t�}
|�|
�t|�
�
d�
|�|
jd�
|�tt|�
�
d�
dS)NZHellozutf-8zHello sHello )	rrr�compat�iterr(r2rrs)r*r
rr2r"r"r#�test_iter_compat�s










zUntokenizeTest.test_iter_compatN)rLrMrNr

r
r
r"r"r"r#r
�s
r
c@sLeZ
dZd
d�Zdd�Zdd�Zdd�Zd	d
�Zdd�Zd
d�Z	dd�Z
dS)recCs�t|
t
�r|
�d
�
}n|
��}|
��
t|jdd�
�
j}tt	|�
�
}dd�|D�
}t
|�
}t|jdd�
�
j}dd�t	|�
D�
}|�||�
t
|�
}	t|	jdd�
�
j}
dd�t	|
�
D�
}|�||�
dS)Nzutf-8T)
�keependsc
Ssg|]}
|
dd
��qS)Nr�r")�.0r6r"r"r#�
<listcomp>�sz1TestRoundtrip.check_roundtrip.<locals>.<listcomp>c
Ssg|]}
|
dd
��qS)Nr�r")r
r6r"r"r#r
�sc
Ssg|]}
|
dd
��qS)Nr�r")r
r6r"r"r#r
�s)�
isinstancer�r&�read�closer
r�__next__r1rrr()r*r-�coder'Ztokens5Ztokens2Zbytes_from2Z	readline2Z
tokens2_from2Zbytes_from5Z	readline5Z
tokens2_from5r"r"r#rf�s











zTestRoundtrip.check_roundtripc
	Cst|�d
�

|�d�

|�d�

t
�d�
}
t|
d��}|�|�

WdQRX|�d�

|�d�

|�d�

|�d	�

dS)
Nzif x == 1:
    print(x)
z # This is a comment
# This also
zif x == 1 : 
  print(x)
ztokenize_tests.txtr_zoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
z�class Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
)rfr�findfiler)r*�fnr-r"r"r#�test_roundtrip�s









zTestRoundtrip.test_roundtripc

Cs|�d
�

dS)Nzna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
)
rf)
r*r"r"r#�test_continuationszTestRoundtrip.test_continuationc

Cs|�d
�

|�d�

dS)Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0)
rf)
r*r"r"r#r
+s
z)TestRoundtrip.test_backslash_continuationc

Cs|�d
�

dS)Nz'' '')
rf)
r*r"r"r#�test_string_concatenation4sz'TestRoundtrip.test_string_concatenationc
Cs�d
dl}
d
dl
}t�d�
}tj�|�
p*tj}|
�tj�|d��
}|�	tj�|d��

x$dD]}|�	tj�|d�|�

qZWt�
d�
s�|�|d�}xD|D]<}t|d	��(}|j
|d
�
�
|�|�

WdQRXWdQRXq�WdS)Nr
ztokenize_tests.txtztest*.pyztest_unicode_identifiers.py)�bufferZbuiltinZfileio�inspectr`�platform�sysz
test_%s.pyZcpu�
r_)
r�)�glob�randomrr
r`rarc�curdirrb�removeZis_resource_enabledZsamplerZsubTestrf)r*r%
r&
r
ZtempdirZ	testfilesr-Ztestfiler"r"r#�test_random_files8s












zTestRoundtrip.test_random_filescCs,t|
t
�r|
�d
�
}
ttt|
�
j�
�
�d
�
S)Nzutf-8)r
r�r&rrrr'rT)r*r
r"r"r#�	roundtripSs




zTestRoundtrip.roundtripc
Cs6d
}
|�|
�
�
d�
}|�|d|d�
|�|
�

dS)Nzif False:
	x=3
	x=3
r�rtr�)r*
�splitr(rf)r*r
Z	codelinesr"r"r#�#test_indentation_semantics_retainedXs


z1TestRoundtrip.test_indentation_semantics_retainedN)rLrMrNrfr
r
r
r
r)
r*
r,
r"r"r"r#re�s '
	re�__main__)(Ztestrrrrrrrr	r
rrr
rr�rrr�iorZunittestrrZtest.test_grammarrrr`rr$r%rYrZr^rpr{r�r
rerL�mainr"r"r"r#�<module>
s<
D



%&
)