Re-land "Check in a simple pure-python based Markdown previewer."
This re-lands #352450 with a fix to make checklicenses.py happy.
R=thestig@chromium.org
TBR=jam@chromium.org
Review URL: https://codereview.chromium.org/1392733002
Cr-Original-Commit-Position: refs/heads/master@{#352731}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 27c171cd168807f85b95ae8aaa797bda02eff319
diff --git a/markdown/inlinepatterns.py b/markdown/inlinepatterns.py
new file mode 100644
index 0000000..95d358d
--- /dev/null
+++ b/markdown/inlinepatterns.py
@@ -0,0 +1,529 @@
+"""
+INLINE PATTERNS
+=============================================================================
+
+Inline patterns such as *emphasis* are handled by means of auxiliary
+objects, one per pattern. Pattern objects must be instances of classes
+that extend markdown.Pattern. Each pattern object uses a single regular
+expression and needs support the following methods:
+
+ pattern.getCompiledRegExp() # returns a regular expression
+
+ pattern.handleMatch(m) # takes a match object and returns
+ # an ElementTree element or just plain text
+
+All of python markdown's built-in patterns subclass from Pattern,
+but you can add additional patterns that don't.
+
+Also note that all the regular expressions used by inline must
+capture the whole block. For this reason, they all start with
+'^(.*)' and end with '(.*)!'. In case with built-in expression
+Pattern takes care of adding the "^(.*)" and "(.*)!".
+
+Finally, the order in which regular expressions are applied is very
+important - e.g. if we first replace http://.../ links with <a> tags
+and _then_ try to replace inline html, we would end up with a mess.
+So, we apply the expressions in the following order:
+
+* escape and backticks have to go before everything else, so
+ that we can preempt any markdown patterns by escaping them.
+
+* then we handle auto-links (must be done before inline html)
+
+* then we handle inline HTML. At this point we will simply
+ replace all inline HTML strings with a placeholder and add
+ the actual HTML to a hash.
+
+* then inline images (must be done before links)
+
+* then bracketed links, first regular then reference-style
+
+* finally we apply strong and emphasis
+"""
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from . import util
+from . import odict
+import re
+try: # pragma: no cover
+ from urllib.parse import urlparse, urlunparse
+except ImportError: # pragma: no cover
+ from urlparse import urlparse, urlunparse
+try: # pragma: no cover
+ from html import entities
+except ImportError: # pragma: no cover
+ import htmlentitydefs as entities
+
+
+def build_inlinepatterns(md_instance, **kwargs):
+ """ Build the default set of inline patterns for Markdown. """
+ inlinePatterns = odict.OrderedDict()
+ inlinePatterns["backtick"] = BacktickPattern(BACKTICK_RE)
+ inlinePatterns["escape"] = EscapePattern(ESCAPE_RE, md_instance)
+ inlinePatterns["reference"] = ReferencePattern(REFERENCE_RE, md_instance)
+ inlinePatterns["link"] = LinkPattern(LINK_RE, md_instance)
+ inlinePatterns["image_link"] = ImagePattern(IMAGE_LINK_RE, md_instance)
+ inlinePatterns["image_reference"] = ImageReferencePattern(
+ IMAGE_REFERENCE_RE, md_instance
+ )
+ inlinePatterns["short_reference"] = ReferencePattern(
+ SHORT_REF_RE, md_instance
+ )
+ inlinePatterns["autolink"] = AutolinkPattern(AUTOLINK_RE, md_instance)
+ inlinePatterns["automail"] = AutomailPattern(AUTOMAIL_RE, md_instance)
+ inlinePatterns["linebreak"] = SubstituteTagPattern(LINE_BREAK_RE, 'br')
+ if md_instance.safeMode != 'escape':
+ inlinePatterns["html"] = HtmlPattern(HTML_RE, md_instance)
+ inlinePatterns["entity"] = HtmlPattern(ENTITY_RE, md_instance)
+ inlinePatterns["not_strong"] = SimpleTextPattern(NOT_STRONG_RE)
+ inlinePatterns["em_strong"] = DoubleTagPattern(EM_STRONG_RE, 'strong,em')
+ inlinePatterns["strong_em"] = DoubleTagPattern(STRONG_EM_RE, 'em,strong')
+ inlinePatterns["strong"] = SimpleTagPattern(STRONG_RE, 'strong')
+ inlinePatterns["emphasis"] = SimpleTagPattern(EMPHASIS_RE, 'em')
+ if md_instance.smart_emphasis:
+ inlinePatterns["emphasis2"] = SimpleTagPattern(SMART_EMPHASIS_RE, 'em')
+ else:
+ inlinePatterns["emphasis2"] = SimpleTagPattern(EMPHASIS_2_RE, 'em')
+ return inlinePatterns
+
+"""
+The actual regular expressions for patterns
+-----------------------------------------------------------------------------
+"""
+
+NOBRACKET = r'[^\]\[]*'
+BRK = (
+ r'\[(' +
+ (NOBRACKET + r'(\[')*6 +
+ (NOBRACKET + r'\])*')*6 +
+ NOBRACKET + r')\]'
+)
+NOIMG = r'(?<!\!)'
+
+# `e=f()` or ``e=f("`")``
+BACKTICK_RE = r'(?<!\\)(`+)(.+?)(?<!`)\2(?!`)'
+
+# \<
+ESCAPE_RE = r'\\(.)'
+
+# *emphasis*
+EMPHASIS_RE = r'(\*)([^\*]+)\2'
+
+# **strong**
+STRONG_RE = r'(\*{2}|_{2})(.+?)\2'
+
+# ***strongem*** or ***em*strong**
+EM_STRONG_RE = r'(\*|_)\2{2}(.+?)\2(.*?)\2{2}'
+
+# ***strong**em*
+STRONG_EM_RE = r'(\*|_)\2{2}(.+?)\2{2}(.*?)\2'
+
+# _smart_emphasis_
+SMART_EMPHASIS_RE = r'(?<!\w)(_)(?!_)(.+?)(?<!_)\2(?!\w)'
+
+# _emphasis_
+EMPHASIS_2_RE = r'(_)(.+?)\2'
+
+# [text](url) or [text](<url>) or [text](url "title")
+LINK_RE = NOIMG + BRK + \
+ r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*?)\12\s*)?\)'''
+
+#  or 
+IMAGE_LINK_RE = r'\!' + BRK + r'\s*\((<.*?>|([^")]+"[^"]*"|[^\)]*))\)'
+
+# [Google][3]
+REFERENCE_RE = NOIMG + BRK + r'\s?\[([^\]]*)\]'
+
+# [Google]
+SHORT_REF_RE = NOIMG + r'\[([^\]]+)\]'
+
+# ![alt text][2]
+IMAGE_REFERENCE_RE = r'\!' + BRK + '\s?\[([^\]]*)\]'
+
+# stand-alone * or _
+NOT_STRONG_RE = r'((^| )(\*|_)( |$))'
+
+# <http://www.123.com>
+AUTOLINK_RE = r'<((?:[Ff]|[Hh][Tt])[Tt][Pp][Ss]?://[^>]*)>'
+
+# <me@example.com>
+AUTOMAIL_RE = r'<([^> \!]*@[^> ]*)>'
+
+# <...>
+HTML_RE = r'(\<([a-zA-Z/][^\>]*?|\!--.*?--)\>)'
+
+# &
+ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)'
+
+# two spaces at end of line
+LINE_BREAK_RE = r' \n'
+
+
+def dequote(string):
+ """Remove quotes from around a string."""
+ if ((string.startswith('"') and string.endswith('"')) or
+ (string.startswith("'") and string.endswith("'"))):
+ return string[1:-1]
+ else:
+ return string
+
+
+ATTR_RE = re.compile("\{@([^\}]*)=([^\}]*)}") # {@id=123}
+
+
+def handleAttributes(text, parent):
+ """Set values of an element based on attribute definitions ({@id=123})."""
+ def attributeCallback(match):
+ parent.set(match.group(1), match.group(2).replace('\n', ' '))
+ return ATTR_RE.sub(attributeCallback, text)
+
+
+"""
+The pattern classes
+-----------------------------------------------------------------------------
+"""
+
+
+class Pattern(object):
+ """Base class that inline patterns subclass. """
+
+ def __init__(self, pattern, markdown_instance=None):
+ """
+ Create an instant of an inline pattern.
+
+ Keyword arguments:
+
+ * pattern: A regular expression that matches a pattern
+
+ """
+ self.pattern = pattern
+ self.compiled_re = re.compile("^(.*?)%s(.*?)$" % pattern,
+ re.DOTALL | re.UNICODE)
+
+ # Api for Markdown to pass safe_mode into instance
+ self.safe_mode = False
+ if markdown_instance:
+ self.markdown = markdown_instance
+
+ def getCompiledRegExp(self):
+ """ Return a compiled regular expression. """
+ return self.compiled_re
+
+ def handleMatch(self, m):
+ """Return a ElementTree element from the given match.
+
+ Subclasses should override this method.
+
+ Keyword arguments:
+
+ * m: A re match object containing a match of the pattern.
+
+ """
+ pass # pragma: no cover
+
+ def type(self):
+ """ Return class name, to define pattern type """
+ return self.__class__.__name__
+
+ def unescape(self, text):
+ """ Return unescaped text given text with an inline placeholder. """
+ try:
+ stash = self.markdown.treeprocessors['inline'].stashed_nodes
+ except KeyError: # pragma: no cover
+ return text
+
+ def itertext(el): # pragma: no cover
+ ' Reimplement Element.itertext for older python versions '
+ tag = el.tag
+ if not isinstance(tag, util.string_type) and tag is not None:
+ return
+ if el.text:
+ yield el.text
+ for e in el:
+ for s in itertext(e):
+ yield s
+ if e.tail:
+ yield e.tail
+
+ def get_stash(m):
+ id = m.group(1)
+ if id in stash:
+ value = stash.get(id)
+ if isinstance(value, util.string_type):
+ return value
+ else:
+ # An etree Element - return text content only
+ return ''.join(itertext(value))
+ return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text)
+
+
+class SimpleTextPattern(Pattern):
+ """ Return a simple text of group(2) of a Pattern. """
+ def handleMatch(self, m):
+ return m.group(2)
+
+
+class EscapePattern(Pattern):
+ """ Return an escaped character. """
+
+ def handleMatch(self, m):
+ char = m.group(2)
+ if char in self.markdown.ESCAPED_CHARS:
+ return '%s%s%s' % (util.STX, ord(char), util.ETX)
+ else:
+ return None
+
+
+class SimpleTagPattern(Pattern):
+ """
+ Return element of type `tag` with a text attribute of group(3)
+ of a Pattern.
+
+ """
+ def __init__(self, pattern, tag):
+ Pattern.__init__(self, pattern)
+ self.tag = tag
+
+ def handleMatch(self, m):
+ el = util.etree.Element(self.tag)
+ el.text = m.group(3)
+ return el
+
+
+class SubstituteTagPattern(SimpleTagPattern):
+ """ Return an element of type `tag` with no children. """
+ def handleMatch(self, m):
+ return util.etree.Element(self.tag)
+
+
+class BacktickPattern(Pattern):
+ """ Return a `<code>` element containing the matching text. """
+ def __init__(self, pattern):
+ Pattern.__init__(self, pattern)
+ self.tag = "code"
+
+ def handleMatch(self, m):
+ el = util.etree.Element(self.tag)
+ el.text = util.AtomicString(m.group(3).strip())
+ return el
+
+
+class DoubleTagPattern(SimpleTagPattern):
+ """Return a ElementTree element nested in tag2 nested in tag1.
+
+ Useful for strong emphasis etc.
+
+ """
+ def handleMatch(self, m):
+ tag1, tag2 = self.tag.split(",")
+ el1 = util.etree.Element(tag1)
+ el2 = util.etree.SubElement(el1, tag2)
+ el2.text = m.group(3)
+ if len(m.groups()) == 5:
+ el2.tail = m.group(4)
+ return el1
+
+
+class HtmlPattern(Pattern):
+ """ Store raw inline html and return a placeholder. """
+ def handleMatch(self, m):
+ rawhtml = self.unescape(m.group(2))
+ place_holder = self.markdown.htmlStash.store(rawhtml)
+ return place_holder
+
+ def unescape(self, text):
+ """ Return unescaped text given text with an inline placeholder. """
+ try:
+ stash = self.markdown.treeprocessors['inline'].stashed_nodes
+ except KeyError: # pragma: no cover
+ return text
+
+ def get_stash(m):
+ id = m.group(1)
+ value = stash.get(id)
+ if value is not None:
+ try:
+ return self.markdown.serializer(value)
+ except:
+ return '\%s' % value
+
+ return util.INLINE_PLACEHOLDER_RE.sub(get_stash, text)
+
+
+class LinkPattern(Pattern):
+ """ Return a link element from the given match. """
+ def handleMatch(self, m):
+ el = util.etree.Element("a")
+ el.text = m.group(2)
+ title = m.group(13)
+ href = m.group(9)
+
+ if href:
+ if href[0] == "<":
+ href = href[1:-1]
+ el.set("href", self.sanitize_url(self.unescape(href.strip())))
+ else:
+ el.set("href", "")
+
+ if title:
+ title = dequote(self.unescape(title))
+ el.set("title", title)
+ return el
+
+ def sanitize_url(self, url):
+ """
+ Sanitize a url against xss attacks in "safe_mode".
+
+ Rather than specifically blacklisting `javascript:alert("XSS")` and all
+ its aliases (see <http://ha.ckers.org/xss.html>), we whitelist known
+ safe url formats. Most urls contain a network location, however some
+ are known not to (i.e.: mailto links). Script urls do not contain a
+ location. Additionally, for `javascript:...`, the scheme would be
+ "javascript" but some aliases will appear to `urlparse()` to have no
+ scheme. On top of that relative links (i.e.: "foo/bar.html") have no
+ scheme. Therefore we must check "path", "parameters", "query" and
+ "fragment" for any literal colons. We don't check "scheme" for colons
+ because it *should* never have any and "netloc" must allow the form:
+ `username:password@host:port`.
+
+ """
+ if not self.markdown.safeMode:
+ # Return immediately bipassing parsing.
+ return url
+
+ try:
+ scheme, netloc, path, params, query, fragment = url = urlparse(url)
+ except ValueError: # pragma: no cover
+ # Bad url - so bad it couldn't be parsed.
+ return ''
+
+ locless_schemes = ['', 'mailto', 'news']
+ allowed_schemes = locless_schemes + ['http', 'https', 'ftp', 'ftps']
+ if scheme not in allowed_schemes:
+ # Not a known (allowed) scheme. Not safe.
+ return ''
+
+ if netloc == '' and scheme not in locless_schemes: # pragma: no cover
+ # This should not happen. Treat as suspect.
+ return ''
+
+ for part in url[2:]:
+ if ":" in part:
+ # A colon in "path", "parameters", "query"
+ # or "fragment" is suspect.
+ return ''
+
+ # Url passes all tests. Return url as-is.
+ return urlunparse(url)
+
+
+class ImagePattern(LinkPattern):
+ """ Return a img element from the given match. """
+ def handleMatch(self, m):
+ el = util.etree.Element("img")
+ src_parts = m.group(9).split()
+ if src_parts:
+ src = src_parts[0]
+ if src[0] == "<" and src[-1] == ">":
+ src = src[1:-1]
+ el.set('src', self.sanitize_url(self.unescape(src)))
+ else:
+ el.set('src', "")
+ if len(src_parts) > 1:
+ el.set('title', dequote(self.unescape(" ".join(src_parts[1:]))))
+
+ if self.markdown.enable_attributes:
+ truealt = handleAttributes(m.group(2), el)
+ else:
+ truealt = m.group(2)
+
+ el.set('alt', self.unescape(truealt))
+ return el
+
+
+class ReferencePattern(LinkPattern):
+ """ Match to a stored reference and return link element. """
+
+ NEWLINE_CLEANUP_RE = re.compile(r'[ ]?\n', re.MULTILINE)
+
+ def handleMatch(self, m):
+ try:
+ id = m.group(9).lower()
+ except IndexError:
+ id = None
+ if not id:
+ # if we got something like "[Google][]" or "[Goggle]"
+ # we'll use "google" as the id
+ id = m.group(2).lower()
+
+ # Clean up linebreaks in id
+ id = self.NEWLINE_CLEANUP_RE.sub(' ', id)
+ if id not in self.markdown.references: # ignore undefined refs
+ return None
+ href, title = self.markdown.references[id]
+
+ text = m.group(2)
+ return self.makeTag(href, title, text)
+
+ def makeTag(self, href, title, text):
+ el = util.etree.Element('a')
+
+ el.set('href', self.sanitize_url(href))
+ if title:
+ el.set('title', title)
+
+ el.text = text
+ return el
+
+
+class ImageReferencePattern(ReferencePattern):
+ """ Match to a stored reference and return img element. """
+ def makeTag(self, href, title, text):
+ el = util.etree.Element("img")
+ el.set("src", self.sanitize_url(href))
+ if title:
+ el.set("title", title)
+
+ if self.markdown.enable_attributes:
+ text = handleAttributes(text, el)
+
+ el.set("alt", self.unescape(text))
+ return el
+
+
+class AutolinkPattern(Pattern):
+ """ Return a link Element given an autolink (`<http://example/com>`). """
+ def handleMatch(self, m):
+ el = util.etree.Element("a")
+ el.set('href', self.unescape(m.group(2)))
+ el.text = util.AtomicString(m.group(2))
+ return el
+
+
+class AutomailPattern(Pattern):
+ """
+ Return a mailto link Element given an automail link (`<foo@example.com>`).
+ """
+ def handleMatch(self, m):
+ el = util.etree.Element('a')
+ email = self.unescape(m.group(2))
+ if email.startswith("mailto:"):
+ email = email[len("mailto:"):]
+
+ def codepoint2name(code):
+ """Return entity definition by code, or the code if not defined."""
+ entity = entities.codepoint2name.get(code)
+ if entity:
+ return "%s%s;" % (util.AMP_SUBSTITUTE, entity)
+ else:
+ return "%s#%d;" % (util.AMP_SUBSTITUTE, code)
+
+ letters = [codepoint2name(ord(letter)) for letter in email]
+ el.text = util.AtomicString(''.join(letters))
+
+ mailto = "mailto:" + email
+ mailto = "".join([util.AMP_SUBSTITUTE + '#%d;' %
+ ord(letter) for letter in mailto])
+ el.set('href', mailto)
+ return el