352 lines
10 KiB
Python
352 lines
10 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
"""
|
||
|
|
pygments.util
|
||
|
|
~~~~~~~~~~~~~
|
||
|
|
|
||
|
|
Utility functions.
|
||
|
|
|
||
|
|
:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
|
||
|
|
:license: BSD, see LICENSE for details.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
from io import TextIOWrapper
|
||
|
|
|
||
|
|
|
||
|
|
split_path_re = re.compile(r'[/\\ ]')
|
||
|
|
doctype_lookup_re = re.compile(r'''
|
||
|
|
(<\?.*?\?>)?\s*
|
||
|
|
<!DOCTYPE\s+(
|
||
|
|
[a-zA-Z_][a-zA-Z0-9]*
|
||
|
|
(?: \s+ # optional in HTML5
|
||
|
|
[a-zA-Z_][a-zA-Z0-9]*\s+
|
||
|
|
"[^"]*")?
|
||
|
|
)
|
||
|
|
[^>]*>
|
||
|
|
''', re.DOTALL | re.MULTILINE | re.VERBOSE)
|
||
|
|
tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>',
|
||
|
|
re.UNICODE | re.IGNORECASE | re.DOTALL | re.MULTILINE)
|
||
|
|
xml_decl_re = re.compile(r'\s*<\?xml[^>]*\?>', re.I)
|
||
|
|
|
||
|
|
|
||
|
|
class ClassNotFound(ValueError):
|
||
|
|
"""Raised if one of the lookup functions didn't find a matching class."""
|
||
|
|
|
||
|
|
|
||
|
|
class OptionError(Exception):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
def get_choice_opt(options, optname, allowed, default=None, normcase=False):
|
||
|
|
string = options.get(optname, default)
|
||
|
|
if normcase:
|
||
|
|
string = string.lower()
|
||
|
|
if string not in allowed:
|
||
|
|
raise OptionError('Value for option %s must be one of %s' %
|
||
|
|
(optname, ', '.join(map(str, allowed))))
|
||
|
|
return string
|
||
|
|
|
||
|
|
|
||
|
|
def get_bool_opt(options, optname, default=None):
|
||
|
|
string = options.get(optname, default)
|
||
|
|
if isinstance(string, bool):
|
||
|
|
return string
|
||
|
|
elif isinstance(string, int):
|
||
|
|
return bool(string)
|
||
|
|
elif not isinstance(string, str):
|
||
|
|
raise OptionError('Invalid type %r for option %s; use '
|
||
|
|
'1/0, yes/no, true/false, on/off' % (
|
||
|
|
string, optname))
|
||
|
|
elif string.lower() in ('1', 'yes', 'true', 'on'):
|
||
|
|
return True
|
||
|
|
elif string.lower() in ('0', 'no', 'false', 'off'):
|
||
|
|
return False
|
||
|
|
else:
|
||
|
|
raise OptionError('Invalid value %r for option %s; use '
|
||
|
|
'1/0, yes/no, true/false, on/off' % (
|
||
|
|
string, optname))
|
||
|
|
|
||
|
|
|
||
|
|
def get_int_opt(options, optname, default=None):
|
||
|
|
string = options.get(optname, default)
|
||
|
|
try:
|
||
|
|
return int(string)
|
||
|
|
except TypeError:
|
||
|
|
raise OptionError('Invalid type %r for option %s; you '
|
||
|
|
'must give an integer value' % (
|
||
|
|
string, optname))
|
||
|
|
except ValueError:
|
||
|
|
raise OptionError('Invalid value %r for option %s; you '
|
||
|
|
'must give an integer value' % (
|
||
|
|
string, optname))
|
||
|
|
|
||
|
|
|
||
|
|
def get_list_opt(options, optname, default=None):
|
||
|
|
val = options.get(optname, default)
|
||
|
|
if isinstance(val, str):
|
||
|
|
return val.split()
|
||
|
|
elif isinstance(val, (list, tuple)):
|
||
|
|
return list(val)
|
||
|
|
else:
|
||
|
|
raise OptionError('Invalid type %r for option %s; you '
|
||
|
|
'must give a list value' % (
|
||
|
|
val, optname))
|
||
|
|
|
||
|
|
|
||
|
|
def docstring_headline(obj):
|
||
|
|
if not obj.__doc__:
|
||
|
|
return ''
|
||
|
|
res = []
|
||
|
|
for line in obj.__doc__.strip().splitlines():
|
||
|
|
if line.strip():
|
||
|
|
res.append(" " + line.strip())
|
||
|
|
else:
|
||
|
|
break
|
||
|
|
return ''.join(res).lstrip()
|
||
|
|
|
||
|
|
|
||
|
|
def make_analysator(f):
|
||
|
|
"""Return a static text analyser function that returns float values."""
|
||
|
|
def text_analyse(text):
|
||
|
|
try:
|
||
|
|
rv = f(text)
|
||
|
|
except Exception:
|
||
|
|
return 0.0
|
||
|
|
if not rv:
|
||
|
|
return 0.0
|
||
|
|
try:
|
||
|
|
return min(1.0, max(0.0, float(rv)))
|
||
|
|
except (ValueError, TypeError):
|
||
|
|
return 0.0
|
||
|
|
text_analyse.__doc__ = f.__doc__
|
||
|
|
return staticmethod(text_analyse)
|
||
|
|
|
||
|
|
|
||
|
|
def shebang_matches(text, regex):
|
||
|
|
r"""Check if the given regular expression matches the last part of the
|
||
|
|
shebang if one exists.
|
||
|
|
|
||
|
|
>>> from pygments.util import shebang_matches
|
||
|
|
>>> shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?')
|
||
|
|
True
|
||
|
|
>>> shebang_matches('#!/usr/bin/python2.4', r'python(2\.\d)?')
|
||
|
|
True
|
||
|
|
>>> shebang_matches('#!/usr/bin/python-ruby', r'python(2\.\d)?')
|
||
|
|
False
|
||
|
|
>>> shebang_matches('#!/usr/bin/python/ruby', r'python(2\.\d)?')
|
||
|
|
False
|
||
|
|
>>> shebang_matches('#!/usr/bin/startsomethingwith python',
|
||
|
|
... r'python(2\.\d)?')
|
||
|
|
True
|
||
|
|
|
||
|
|
It also checks for common windows executable file extensions::
|
||
|
|
|
||
|
|
>>> shebang_matches('#!C:\\Python2.4\\Python.exe', r'python(2\.\d)?')
|
||
|
|
True
|
||
|
|
|
||
|
|
Parameters (``'-f'`` or ``'--foo'`` are ignored so ``'perl'`` does
|
||
|
|
the same as ``'perl -e'``)
|
||
|
|
|
||
|
|
Note that this method automatically searches the whole string (eg:
|
||
|
|
the regular expression is wrapped in ``'^$'``)
|
||
|
|
"""
|
||
|
|
index = text.find('\n')
|
||
|
|
if index >= 0:
|
||
|
|
first_line = text[:index].lower()
|
||
|
|
else:
|
||
|
|
first_line = text.lower()
|
||
|
|
if first_line.startswith('#!'):
|
||
|
|
try:
|
||
|
|
found = [x for x in split_path_re.split(first_line[2:].strip())
|
||
|
|
if x and not x.startswith('-')][-1]
|
||
|
|
except IndexError:
|
||
|
|
return False
|
||
|
|
regex = re.compile(r'^%s(\.(exe|cmd|bat|bin))?$' % regex, re.IGNORECASE)
|
||
|
|
if regex.search(found) is not None:
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def doctype_matches(text, regex):
|
||
|
|
"""Check if the doctype matches a regular expression (if present).
|
||
|
|
|
||
|
|
Note that this method only checks the first part of a DOCTYPE.
|
||
|
|
eg: 'html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"'
|
||
|
|
"""
|
||
|
|
m = doctype_lookup_re.search(text)
|
||
|
|
if m is None:
|
||
|
|
return False
|
||
|
|
doctype = m.group(2)
|
||
|
|
return re.compile(regex, re.I).match(doctype.strip()) is not None
|
||
|
|
|
||
|
|
|
||
|
|
def html_doctype_matches(text):
|
||
|
|
"""Check if the file looks like it has a html doctype."""
|
||
|
|
return doctype_matches(text, r'html')
|
||
|
|
|
||
|
|
|
||
|
|
_looks_like_xml_cache = {}
|
||
|
|
|
||
|
|
|
||
|
|
def looks_like_xml(text):
|
||
|
|
"""Check if a doctype exists or if we have some tags."""
|
||
|
|
if xml_decl_re.match(text):
|
||
|
|
return True
|
||
|
|
key = hash(text)
|
||
|
|
try:
|
||
|
|
return _looks_like_xml_cache[key]
|
||
|
|
except KeyError:
|
||
|
|
m = doctype_lookup_re.search(text)
|
||
|
|
if m is not None:
|
||
|
|
return True
|
||
|
|
rv = tag_re.search(text[:1000]) is not None
|
||
|
|
_looks_like_xml_cache[key] = rv
|
||
|
|
return rv
|
||
|
|
|
||
|
|
|
||
|
|
# Python narrow build compatibility
|
||
|
|
|
||
|
|
def _surrogatepair(c):
|
||
|
|
# Given a unicode character code
|
||
|
|
# with length greater than 16 bits,
|
||
|
|
# return the two 16 bit surrogate pair.
|
||
|
|
# From example D28 of:
|
||
|
|
# http://www.unicode.org/book/ch03.pdf
|
||
|
|
return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff)))
|
||
|
|
|
||
|
|
|
||
|
|
def unirange(a, b):
|
||
|
|
"""Returns a regular expression string to match the given non-BMP range."""
|
||
|
|
if b < a:
|
||
|
|
raise ValueError("Bad character range")
|
||
|
|
if a < 0x10000 or b < 0x10000:
|
||
|
|
raise ValueError("unirange is only defined for non-BMP ranges")
|
||
|
|
|
||
|
|
if sys.maxunicode > 0xffff:
|
||
|
|
# wide build
|
||
|
|
return u'[%s-%s]' % (chr(a), chr(b))
|
||
|
|
else:
|
||
|
|
# narrow build stores surrogates, and the 're' module handles them
|
||
|
|
# (incorrectly) as characters. Since there is still ordering among
|
||
|
|
# these characters, expand the range to one that it understands. Some
|
||
|
|
# background in http://bugs.python.org/issue3665 and
|
||
|
|
# http://bugs.python.org/issue12749
|
||
|
|
#
|
||
|
|
# Additionally, the lower constants are using chr rather than
|
||
|
|
# literals because jython [which uses the wide path] can't load this
|
||
|
|
# file if they are literals.
|
||
|
|
ah, al = _surrogatepair(a)
|
||
|
|
bh, bl = _surrogatepair(b)
|
||
|
|
if ah == bh:
|
||
|
|
return u'(?:%s[%s-%s])' % (chr(ah), chr(al), chr(bl))
|
||
|
|
else:
|
||
|
|
buf = []
|
||
|
|
buf.append(u'%s[%s-%s]' % (chr(ah), chr(al),
|
||
|
|
ah == bh and chr(bl) or chr(0xdfff)))
|
||
|
|
if ah - bh > 1:
|
||
|
|
buf.append(u'[%s-%s][%s-%s]' %
|
||
|
|
chr(ah+1), chr(bh-1), chr(0xdc00), chr(0xdfff))
|
||
|
|
if ah != bh:
|
||
|
|
buf.append(u'%s[%s-%s]' %
|
||
|
|
(chr(bh), chr(0xdc00), chr(bl)))
|
||
|
|
|
||
|
|
return u'(?:' + u'|'.join(buf) + u')'
|
||
|
|
|
||
|
|
|
||
|
|
def format_lines(var_name, seq, raw=False, indent_level=0):
|
||
|
|
"""Formats a sequence of strings for output."""
|
||
|
|
lines = []
|
||
|
|
base_indent = ' ' * indent_level * 4
|
||
|
|
inner_indent = ' ' * (indent_level + 1) * 4
|
||
|
|
lines.append(base_indent + var_name + ' = (')
|
||
|
|
if raw:
|
||
|
|
# These should be preformatted reprs of, say, tuples.
|
||
|
|
for i in seq:
|
||
|
|
lines.append(inner_indent + i + ',')
|
||
|
|
else:
|
||
|
|
for i in seq:
|
||
|
|
# Force use of single quotes
|
||
|
|
r = repr(i + '"')
|
||
|
|
lines.append(inner_indent + r[:-2] + r[-1] + ',')
|
||
|
|
lines.append(base_indent + ')')
|
||
|
|
return '\n'.join(lines)
|
||
|
|
|
||
|
|
|
||
|
|
def duplicates_removed(it, already_seen=()):
|
||
|
|
"""
|
||
|
|
Returns a list with duplicates removed from the iterable `it`.
|
||
|
|
|
||
|
|
Order is preserved.
|
||
|
|
"""
|
||
|
|
lst = []
|
||
|
|
seen = set()
|
||
|
|
for i in it:
|
||
|
|
if i in seen or i in already_seen:
|
||
|
|
continue
|
||
|
|
lst.append(i)
|
||
|
|
seen.add(i)
|
||
|
|
return lst
|
||
|
|
|
||
|
|
|
||
|
|
class Future:
|
||
|
|
"""Generic class to defer some work.
|
||
|
|
|
||
|
|
Handled specially in RegexLexerMeta, to support regex string construction at
|
||
|
|
first use.
|
||
|
|
"""
|
||
|
|
def get(self):
|
||
|
|
raise NotImplementedError
|
||
|
|
|
||
|
|
|
||
|
|
def guess_decode(text):
|
||
|
|
"""Decode *text* with guessed encoding.
|
||
|
|
|
||
|
|
First try UTF-8; this should fail for non-UTF-8 encodings.
|
||
|
|
Then try the preferred locale encoding.
|
||
|
|
Fall back to latin-1, which always works.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
text = text.decode('utf-8')
|
||
|
|
return text, 'utf-8'
|
||
|
|
except UnicodeDecodeError:
|
||
|
|
try:
|
||
|
|
import locale
|
||
|
|
prefencoding = locale.getpreferredencoding()
|
||
|
|
text = text.decode()
|
||
|
|
return text, prefencoding
|
||
|
|
except (UnicodeDecodeError, LookupError):
|
||
|
|
text = text.decode('latin1')
|
||
|
|
return text, 'latin1'
|
||
|
|
|
||
|
|
|
||
|
|
def guess_decode_from_terminal(text, term):
|
||
|
|
"""Decode *text* coming from terminal *term*.
|
||
|
|
|
||
|
|
First try the terminal encoding, if given.
|
||
|
|
Then try UTF-8. Then try the preferred locale encoding.
|
||
|
|
Fall back to latin-1, which always works.
|
||
|
|
"""
|
||
|
|
if getattr(term, 'encoding', None):
|
||
|
|
try:
|
||
|
|
text = text.decode(term.encoding)
|
||
|
|
except UnicodeDecodeError:
|
||
|
|
pass
|
||
|
|
else:
|
||
|
|
return text, term.encoding
|
||
|
|
return guess_decode(text)
|
||
|
|
|
||
|
|
|
||
|
|
def terminal_encoding(term):
|
||
|
|
"""Return our best guess of encoding for the given *term*."""
|
||
|
|
if getattr(term, 'encoding', None):
|
||
|
|
return term.encoding
|
||
|
|
import locale
|
||
|
|
return locale.getpreferredencoding()
|
||
|
|
|
||
|
|
|
||
|
|
class UnclosingTextIOWrapper(TextIOWrapper):
|
||
|
|
# Don't close underlying buffer on destruction.
|
||
|
|
def close(self):
|
||
|
|
self.flush()
|