ansible-later/env_27/lib/python2.7/site-packages/coverage/phystokens.py
Robert Kaussow 10aaa8e7e3 fix pytest
2019-04-11 15:56:20 +02:00

299 lines
9.9 KiB
Python

# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt
"""Better tokenizing for coverage.py."""
import codecs
import keyword
import re
import sys
import token
import tokenize
from coverage import env
from coverage.backward import iternext, unicode_class
from coverage.misc import contract
def phys_tokens(toks):
"""Return all physical tokens, even line continuations.
tokenize.generate_tokens() doesn't return a token for the backslash that
continues lines. This wrapper provides those tokens so that we can
re-create a faithful representation of the original source.
Returns the same values as generate_tokens()
"""
last_line = None
last_lineno = -1
last_ttype = None
for ttype, ttext, (slineno, scol), (elineno, ecol), ltext in toks:
if last_lineno != elineno:
if last_line and last_line.endswith("\\\n"):
# We are at the beginning of a new line, and the last line
# ended with a backslash. We probably have to inject a
# backslash token into the stream. Unfortunately, there's more
# to figure out. This code::
#
# usage = """\
# HEY THERE
# """
#
# triggers this condition, but the token text is::
#
# '"""\\\nHEY THERE\n"""'
#
# so we need to figure out if the backslash is already in the
# string token or not.
inject_backslash = True
if last_ttype == tokenize.COMMENT:
# Comments like this \
# should never result in a new token.
inject_backslash = False
elif ttype == token.STRING:
if "\n" in ttext and ttext.split('\n', 1)[0][-1] == '\\':
# It's a multi-line string and the first line ends with
# a backslash, so we don't need to inject another.
inject_backslash = False
if inject_backslash:
# Figure out what column the backslash is in.
ccol = len(last_line.split("\n")[-2]) - 1
# Yield the token, with a fake token type.
yield (
99999, "\\\n",
(slineno, ccol), (slineno, ccol+2),
last_line
)
last_line = ltext
last_ttype = ttype
yield ttype, ttext, (slineno, scol), (elineno, ecol), ltext
last_lineno = elineno
@contract(source='unicode')
def source_token_lines(source):
"""Generate a series of lines, one for each line in `source`.
Each line is a list of pairs, each pair is a token::
[('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ]
Each pair has a token class, and the token text.
If you concatenate all the token texts, and then join them with newlines,
you should have your original `source` back, with two differences:
trailing whitespace is not preserved, and a final line with no newline
is indistinguishable from a final line with a newline.
"""
ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])
line = []
col = 0
source = source.expandtabs(8).replace('\r\n', '\n')
tokgen = generate_tokens(source)
for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):
mark_start = True
for part in re.split('(\n)', ttext):
if part == '\n':
yield line
line = []
col = 0
mark_end = False
elif part == '':
mark_end = False
elif ttype in ws_tokens:
mark_end = False
else:
if mark_start and scol > col:
line.append(("ws", u" " * (scol - col)))
mark_start = False
tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3]
if ttype == token.NAME and keyword.iskeyword(ttext):
tok_class = "key"
line.append((tok_class, part))
mark_end = True
scol = 0
if mark_end:
col = ecol
if line:
yield line
class CachedTokenizer(object):
"""A one-element cache around tokenize.generate_tokens.
When reporting, coverage.py tokenizes files twice, once to find the
structure of the file, and once to syntax-color it. Tokenizing is
expensive, and easily cached.
This is a one-element cache so that our twice-in-a-row tokenizing doesn't
actually tokenize twice.
"""
def __init__(self):
self.last_text = None
self.last_tokens = None
@contract(text='unicode')
def generate_tokens(self, text):
"""A stand-in for `tokenize.generate_tokens`."""
if text != self.last_text:
self.last_text = text
readline = iternext(text.splitlines(True))
self.last_tokens = list(tokenize.generate_tokens(readline))
return self.last_tokens
# Create our generate_tokens cache as a callable replacement function.
generate_tokens = CachedTokenizer().generate_tokens
COOKIE_RE = re.compile(r"^[ \t]*#.*coding[:=][ \t]*([-\w.]+)", flags=re.MULTILINE)
@contract(source='bytes')
def _source_encoding_py2(source):
"""Determine the encoding for `source`, according to PEP 263.
`source` is a byte string, the text of the program.
Returns a string, the name of the encoding.
"""
assert isinstance(source, bytes)
# Do this so the detect_encode code we copied will work.
readline = iternext(source.splitlines(True))
# This is mostly code adapted from Py3.2's tokenize module.
def _get_normal_name(orig_enc):
"""Imitates get_normal_name in tokenizer.c."""
# Only care about the first 12 characters.
enc = orig_enc[:12].lower().replace("_", "-")
if re.match(r"^utf-8($|-)", enc):
return "utf-8"
if re.match(r"^(latin-1|iso-8859-1|iso-latin-1)($|-)", enc):
return "iso-8859-1"
return orig_enc
# From detect_encode():
# It detects the encoding from the presence of a UTF-8 BOM or an encoding
# cookie as specified in PEP-0263. If both a BOM and a cookie are present,
# but disagree, a SyntaxError will be raised. If the encoding cookie is an
# invalid charset, raise a SyntaxError. Note that if a UTF-8 BOM is found,
# 'utf-8-sig' is returned.
# If no encoding is specified, then the default will be returned.
default = 'ascii'
bom_found = False
encoding = None
def read_or_stop():
"""Get the next source line, or ''."""
try:
return readline()
except StopIteration:
return ''
def find_cookie(line):
"""Find an encoding cookie in `line`."""
try:
line_string = line.decode('ascii')
except UnicodeDecodeError:
return None
matches = COOKIE_RE.findall(line_string)
if not matches:
return None
encoding = _get_normal_name(matches[0])
try:
codec = codecs.lookup(encoding)
except LookupError:
# This behavior mimics the Python interpreter
raise SyntaxError("unknown encoding: " + encoding)
if bom_found:
# codecs in 2.3 were raw tuples of functions, assume the best.
codec_name = getattr(codec, 'name', encoding)
if codec_name != 'utf-8':
# This behavior mimics the Python interpreter
raise SyntaxError('encoding problem: utf-8')
encoding += '-sig'
return encoding
first = read_or_stop()
if first.startswith(codecs.BOM_UTF8):
bom_found = True
first = first[3:]
default = 'utf-8-sig'
if not first:
return default
encoding = find_cookie(first)
if encoding:
return encoding
second = read_or_stop()
if not second:
return default
encoding = find_cookie(second)
if encoding:
return encoding
return default
@contract(source='bytes')
def _source_encoding_py3(source):
"""Determine the encoding for `source`, according to PEP 263.
`source` is a byte string: the text of the program.
Returns a string, the name of the encoding.
"""
readline = iternext(source.splitlines(True))
return tokenize.detect_encoding(readline)[0]
if env.PY3:
source_encoding = _source_encoding_py3
else:
source_encoding = _source_encoding_py2
@contract(source='unicode')
def compile_unicode(source, filename, mode):
"""Just like the `compile` builtin, but works on any Unicode string.
Python 2's compile() builtin has a stupid restriction: if the source string
is Unicode, then it may not have a encoding declaration in it. Why not?
Who knows! It also decodes to utf8, and then tries to interpret those utf8
bytes according to the encoding declaration. Why? Who knows!
This function neuters the coding declaration, and compiles it.
"""
source = neuter_encoding_declaration(source)
if env.PY2 and isinstance(filename, unicode_class):
filename = filename.encode(sys.getfilesystemencoding(), "replace")
code = compile(source, filename, mode)
return code
@contract(source='unicode', returns='unicode')
def neuter_encoding_declaration(source):
"""Return `source`, with any encoding declaration neutered."""
if COOKIE_RE.search(source):
source_lines = source.splitlines(True)
for lineno in range(min(2, len(source_lines))):
source_lines[lineno] = COOKIE_RE.sub("# (deleted declaration)", source_lines[lineno])
source = "".join(source_lines)
return source