Compare commits
No commits in common. 'c9' and 'i8c' have entirely different histories.
@ -1 +1 @@
|
||||
SOURCES/lxml-4.6.5.tar.gz
|
||||
SOURCES/lxml-4.2.3.tgz
|
||||
|
||||
@ -1 +1 @@
|
||||
04a3ed4d33a511b5796880461b0edb6f3b144547 SOURCES/lxml-4.6.5.tar.gz
|
||||
536c9ced5d03e8d871ebf21748ed36a2f7ddf668 SOURCES/lxml-4.2.3.tgz
|
||||
|
||||
@ -0,0 +1,121 @@
|
||||
diff --git a/src/lxml/html/clean.py b/src/lxml/html/clean.py
|
||||
index adc3f45..6f3f7de 100644
|
||||
--- a/src/lxml/html/clean.py
|
||||
+++ b/src/lxml/html/clean.py
|
||||
@@ -61,12 +61,15 @@ __all__ = ['clean_html', 'clean', 'Cleaner', 'autolink', 'autolink_html',
|
||||
|
||||
# This is an IE-specific construct you can have in a stylesheet to
|
||||
# run some Javascript:
|
||||
-_css_javascript_re = re.compile(
|
||||
- r'expression\s*\(.*?\)', re.S|re.I)
|
||||
+_replace_css_javascript = re.compile(
|
||||
+ r'expression\s*\(.*?\)', re.S|re.I).sub
|
||||
|
||||
# Do I have to worry about @\nimport?
|
||||
-_css_import_re = re.compile(
|
||||
- r'@\s*import', re.I)
|
||||
+_replace_css_import = re.compile(
|
||||
+ r'@\s*import', re.I).sub
|
||||
+
|
||||
+_looks_like_tag_content = re.compile(
|
||||
+ r'</?[a-zA-Z]+|\son[a-zA-Z]+\s*=', re.ASCII).search
|
||||
|
||||
# All kinds of schemes besides just javascript: that can cause
|
||||
# execution:
|
||||
@@ -292,8 +295,8 @@ class Cleaner(object):
|
||||
if not self.inline_style:
|
||||
for el in _find_styled_elements(doc):
|
||||
old = el.get('style')
|
||||
- new = _css_javascript_re.sub('', old)
|
||||
- new = _css_import_re.sub('', new)
|
||||
+ new = _replace_css_javascript('', old)
|
||||
+ new = _replace_css_import('', new)
|
||||
if self._has_sneaky_javascript(new):
|
||||
# Something tricky is going on...
|
||||
del el.attrib['style']
|
||||
@@ -305,9 +308,9 @@ class Cleaner(object):
|
||||
el.drop_tree()
|
||||
continue
|
||||
old = el.text or ''
|
||||
- new = _css_javascript_re.sub('', old)
|
||||
+ new = _replace_css_javascript('', old)
|
||||
# The imported CSS can do anything; we just can't allow:
|
||||
- new = _css_import_re.sub('', old)
|
||||
+ new = _replace_css_import('', new)
|
||||
if self._has_sneaky_javascript(new):
|
||||
# Something tricky is going on...
|
||||
el.text = '/* deleted */'
|
||||
@@ -509,6 +512,12 @@ class Cleaner(object):
|
||||
return True
|
||||
if 'expression(' in style:
|
||||
return True
|
||||
+ if '</noscript' in style:
|
||||
+ # e.g. '<noscript><style><a title="</noscript><img src=x onerror=alert(1)>">'
|
||||
+ return True
|
||||
+ if _looks_like_tag_content(style):
|
||||
+ # e.g. '<math><style><img src=x onerror=alert(1)></style></math>'
|
||||
+ return True
|
||||
return False
|
||||
|
||||
def clean_html(self, html):
|
||||
diff --git a/src/lxml/html/tests/test_clean.py b/src/lxml/html/tests/test_clean.py
|
||||
index 3bcaaf5..451eec2 100644
|
||||
--- a/src/lxml/html/tests/test_clean.py
|
||||
+++ b/src/lxml/html/tests/test_clean.py
|
||||
@@ -69,6 +69,26 @@ class CleanerTest(unittest.TestCase):
|
||||
s = lxml.html.fromstring('<invalid tag>child</another>')
|
||||
self.assertEqual('child', clean_html(s).text_content())
|
||||
|
||||
+ def test_sneaky_noscript_in_style(self):
|
||||
+ # This gets parsed as <noscript> -> <style>"...</noscript>..."</style>
|
||||
+ # thus passing the </noscript> through into the output.
|
||||
+ html = '<noscript><style><a title="</noscript><img src=x onerror=alert(1)>">'
|
||||
+ s = lxml.html.fragment_fromstring(html)
|
||||
+
|
||||
+ self.assertEqual(
|
||||
+ b'<noscript><style>/* deleted */</style></noscript>',
|
||||
+ lxml.html.tostring(clean_html(s)))
|
||||
+
|
||||
+ def test_sneaky_js_in_math_style(self):
|
||||
+ # This gets parsed as <math> -> <style>"..."</style>
|
||||
+ # thus passing any tag/script/whatever content through into the output.
|
||||
+ html = '<math><style><img src=x onerror=alert(1)></style></math>'
|
||||
+ s = lxml.html.fragment_fromstring(html)
|
||||
+
|
||||
+ self.assertEqual(
|
||||
+ b'<math><style>/* deleted */</style></math>',
|
||||
+ lxml.html.tostring(clean_html(s)))
|
||||
+
|
||||
|
||||
def test_suite():
|
||||
suite = unittest.TestSuite()
|
||||
diff --git a/src/lxml/html/tests/test_clean.txt b/src/lxml/html/tests/test_clean.txt
|
||||
index c78ab4f..c901871 100644
|
||||
--- a/src/lxml/html/tests/test_clean.txt
|
||||
+++ b/src/lxml/html/tests/test_clean.txt
|
||||
@@ -104,7 +104,11 @@
|
||||
>>> print(Cleaner(page_structure=False, safe_attrs_only=False).clean_html(doc))
|
||||
<html>
|
||||
<head>
|
||||
- <style>/* deleted */</style>
|
||||
+ <style>
|
||||
+ body {background-image: url()};
|
||||
+ div {background-image: url()};
|
||||
+ div {color: };
|
||||
+ </style>
|
||||
</head>
|
||||
<body>
|
||||
<a href="">a link</a>
|
||||
@@ -168,7 +172,11 @@
|
||||
<link rel="alternate" type="text/rss" src="evil-rss">
|
||||
<link rel="alternate" type="text/rss" href="http://example.com">
|
||||
<link rel="stylesheet" type="text/rss" href="http://example.com">
|
||||
- <style>/* deleted */</style>
|
||||
+ <style>
|
||||
+ body {background-image: url()};
|
||||
+ div {background-image: url()};
|
||||
+ div {color: };
|
||||
+ </style>
|
||||
</head>
|
||||
<body>
|
||||
<a href="">a link</a>
|
||||
@ -0,0 +1,39 @@
|
||||
diff --git a/src/lxml/html/defs.py b/src/lxml/html/defs.py
|
||||
index caf6b21..ea3c016 100644
|
||||
--- a/src/lxml/html/defs.py
|
||||
+++ b/src/lxml/html/defs.py
|
||||
@@ -21,6 +21,8 @@ link_attrs = frozenset([
|
||||
'usemap',
|
||||
# Not standard:
|
||||
'dynsrc', 'lowsrc',
|
||||
+ # HTML5 formaction
|
||||
+ 'formaction'
|
||||
])
|
||||
|
||||
# Not in the HTML 4 spec:
|
||||
diff --git a/src/lxml/html/tests/test_clean.py b/src/lxml/html/tests/test_clean.py
|
||||
index 451eec2..e40cdad 100644
|
||||
--- a/src/lxml/html/tests/test_clean.py
|
||||
+++ b/src/lxml/html/tests/test_clean.py
|
||||
@@ -89,6 +89,21 @@ class CleanerTest(unittest.TestCase):
|
||||
b'<math><style>/* deleted */</style></math>',
|
||||
lxml.html.tostring(clean_html(s)))
|
||||
|
||||
+ def test_formaction_attribute_in_button_input(self):
|
||||
+ # The formaction attribute overrides the form's action and should be
|
||||
+ # treated as a malicious link attribute
|
||||
+ html = ('<form id="test"><input type="submit" formaction="javascript:alert(1)"></form>'
|
||||
+ '<button form="test" formaction="javascript:alert(1)">X</button>')
|
||||
+ expected = ('<div><form id="test"><input type="submit" formaction=""></form>'
|
||||
+ '<button form="test" formaction="">X</button></div>')
|
||||
+ cleaner = Cleaner(
|
||||
+ forms=False,
|
||||
+ safe_attrs_only=False,
|
||||
+ )
|
||||
+ self.assertEqual(
|
||||
+ expected,
|
||||
+ cleaner.clean_html(html))
|
||||
+
|
||||
|
||||
def test_suite():
|
||||
suite = unittest.TestSuite()
|
||||
@ -0,0 +1,127 @@
|
||||
diff --git a/src/lxml/html/clean.py b/src/lxml/html/clean.py
|
||||
index 6f3f7de..da5af16 100644
|
||||
--- a/src/lxml/html/clean.py
|
||||
+++ b/src/lxml/html/clean.py
|
||||
@@ -73,18 +73,25 @@ _looks_like_tag_content = re.compile(
|
||||
|
||||
# All kinds of schemes besides just javascript: that can cause
|
||||
# execution:
|
||||
-_is_image_dataurl = re.compile(
|
||||
- r'^data:image/.+;base64', re.I).search
|
||||
+_find_image_dataurls = re.compile(
|
||||
+ r'^data:image/(.+);base64,', re.I).findall
|
||||
_is_possibly_malicious_scheme = re.compile(
|
||||
- r'(?:javascript|jscript|livescript|vbscript|data|about|mocha):',
|
||||
- re.I).search
|
||||
+ r'(javascript|jscript|livescript|vbscript|data|about|mocha):',
|
||||
+ re.I).findall
|
||||
+# SVG images can contain script content
|
||||
+_is_unsafe_image_type = re.compile(r"(xml|svg)", re.I).findall
|
||||
+
|
||||
def _is_javascript_scheme(s):
|
||||
- if _is_image_dataurl(s):
|
||||
- return None
|
||||
- return _is_possibly_malicious_scheme(s)
|
||||
+ is_image_url = False
|
||||
+ for image_type in _find_image_dataurls(s):
|
||||
+ is_image_url = True
|
||||
+ if _is_unsafe_image_type(image_type):
|
||||
+ return True
|
||||
+ if is_image_url:
|
||||
+ return False
|
||||
+ return bool(_is_possibly_malicious_scheme(s))
|
||||
|
||||
_substitute_whitespace = re.compile(r'[\s\x00-\x08\x0B\x0C\x0E-\x19]+').sub
|
||||
-# FIXME: should data: be blocked?
|
||||
|
||||
# FIXME: check against: http://msdn2.microsoft.com/en-us/library/ms537512.aspx
|
||||
_conditional_comment_re = re.compile(
|
||||
@@ -512,6 +519,8 @@ class Cleaner(object):
|
||||
return True
|
||||
if 'expression(' in style:
|
||||
return True
|
||||
+ if '@import' in style:
|
||||
+ return True
|
||||
if '</noscript' in style:
|
||||
# e.g. '<noscript><style><a title="</noscript><img src=x onerror=alert(1)>">'
|
||||
return True
|
||||
diff --git a/src/lxml/html/tests/test_clean.py b/src/lxml/html/tests/test_clean.py
|
||||
index e40cdad..ad9a598 100644
|
||||
--- a/src/lxml/html/tests/test_clean.py
|
||||
+++ b/src/lxml/html/tests/test_clean.py
|
||||
@@ -1,3 +1,5 @@
|
||||
+import base64
|
||||
+import gzip
|
||||
import unittest, sys
|
||||
from lxml.tests.common_imports import make_doctest
|
||||
from lxml.etree import LIBXML_VERSION
|
||||
@@ -89,6 +91,69 @@ class CleanerTest(unittest.TestCase):
|
||||
b'<math><style>/* deleted */</style></math>',
|
||||
lxml.html.tostring(clean_html(s)))
|
||||
|
||||
+ def test_sneaky_import_in_style(self):
|
||||
+ # Prevent "@@importimport" -> "@import" replacement.
|
||||
+ style_codes = [
|
||||
+ "@@importimport(extstyle.css)",
|
||||
+ "@ @ import import(extstyle.css)",
|
||||
+ "@ @ importimport(extstyle.css)",
|
||||
+ "@@ import import(extstyle.css)",
|
||||
+ "@ @import import(extstyle.css)",
|
||||
+ "@@importimport()",
|
||||
+ ]
|
||||
+ for style_code in style_codes:
|
||||
+ html = '<style>%s</style>' % style_code
|
||||
+ s = lxml.html.fragment_fromstring(html)
|
||||
+
|
||||
+ cleaned = lxml.html.tostring(clean_html(s))
|
||||
+ self.assertEqual(
|
||||
+ b'<style>/* deleted */</style>',
|
||||
+ cleaned,
|
||||
+ "%s -> %s" % (style_code, cleaned))
|
||||
+
|
||||
+ def test_svg_data_links(self):
|
||||
+ # Remove SVG images with potentially insecure content.
|
||||
+ svg = b'<svg onload="alert(123)" />'
|
||||
+ svgz = gzip.compress(svg)
|
||||
+ svg_b64 = base64.b64encode(svg).decode('ASCII')
|
||||
+ svgz_b64 = base64.b64encode(svgz).decode('ASCII')
|
||||
+ urls = [
|
||||
+ "data:image/svg+xml;base64," + svg_b64,
|
||||
+ "data:image/svg+xml-compressed;base64," + svgz_b64,
|
||||
+ ]
|
||||
+ for url in urls:
|
||||
+ html = '<img src="%s">' % url
|
||||
+ s = lxml.html.fragment_fromstring(html)
|
||||
+
|
||||
+ cleaned = lxml.html.tostring(clean_html(s))
|
||||
+ self.assertEqual(
|
||||
+ b'<img src="">',
|
||||
+ cleaned,
|
||||
+ "%s -> %s" % (url, cleaned))
|
||||
+
|
||||
+ def test_image_data_links(self):
|
||||
+ data = b'123'
|
||||
+ data_b64 = base64.b64encode(data).decode('ASCII')
|
||||
+ urls = [
|
||||
+ "data:image/jpeg;base64," + data_b64,
|
||||
+ "data:image/apng;base64," + data_b64,
|
||||
+ "data:image/png;base64," + data_b64,
|
||||
+ "data:image/gif;base64," + data_b64,
|
||||
+ "data:image/webp;base64," + data_b64,
|
||||
+ "data:image/bmp;base64," + data_b64,
|
||||
+ "data:image/tiff;base64," + data_b64,
|
||||
+ "data:image/x-icon;base64," + data_b64,
|
||||
+ ]
|
||||
+ for url in urls:
|
||||
+ html = '<img src="%s">' % url
|
||||
+ s = lxml.html.fragment_fromstring(html)
|
||||
+
|
||||
+ cleaned = lxml.html.tostring(clean_html(s))
|
||||
+ self.assertEqual(
|
||||
+ html.encode("UTF-8"),
|
||||
+ cleaned,
|
||||
+ "%s -> %s" % (url, cleaned))
|
||||
+
|
||||
def test_formaction_attribute_in_button_input(self):
|
||||
# The formaction attribute overrides the form's action and should be
|
||||
# treated as a malicious link attribute
|
||||
@ -1,104 +0,0 @@
|
||||
diff --git a/src/lxml/apihelpers.pxi b/src/lxml/apihelpers.pxi
|
||||
index 5eb3416..88a031d 100644
|
||||
--- a/src/lxml/apihelpers.pxi
|
||||
+++ b/src/lxml/apihelpers.pxi
|
||||
@@ -246,9 +246,10 @@ cdef dict _build_nsmap(xmlNode* c_node):
|
||||
while c_node is not NULL and c_node.type == tree.XML_ELEMENT_NODE:
|
||||
c_ns = c_node.nsDef
|
||||
while c_ns is not NULL:
|
||||
- prefix = funicodeOrNone(c_ns.prefix)
|
||||
- if prefix not in nsmap:
|
||||
- nsmap[prefix] = funicodeOrNone(c_ns.href)
|
||||
+ if c_ns.prefix or c_ns.href:
|
||||
+ prefix = funicodeOrNone(c_ns.prefix)
|
||||
+ if prefix not in nsmap:
|
||||
+ nsmap[prefix] = funicodeOrNone(c_ns.href)
|
||||
c_ns = c_ns.next
|
||||
c_node = c_node.parent
|
||||
return nsmap
|
||||
diff --git a/src/lxml/includes/xmlparser.pxd b/src/lxml/includes/xmlparser.pxd
|
||||
index a196e34..45acfc8 100644
|
||||
--- a/src/lxml/includes/xmlparser.pxd
|
||||
+++ b/src/lxml/includes/xmlparser.pxd
|
||||
@@ -144,6 +144,7 @@ cdef extern from "libxml/parser.h":
|
||||
void* userData
|
||||
int* spaceTab
|
||||
int spaceMax
|
||||
+ int nsNr
|
||||
bint html
|
||||
bint progressive
|
||||
int inSubset
|
||||
diff --git a/src/lxml/iterparse.pxi b/src/lxml/iterparse.pxi
|
||||
index 4c20506..3da7485 100644
|
||||
--- a/src/lxml/iterparse.pxi
|
||||
+++ b/src/lxml/iterparse.pxi
|
||||
@@ -419,7 +419,7 @@ cdef int _countNsDefs(xmlNode* c_node):
|
||||
count = 0
|
||||
c_ns = c_node.nsDef
|
||||
while c_ns is not NULL:
|
||||
- count += 1
|
||||
+ count += (c_ns.href is not NULL)
|
||||
c_ns = c_ns.next
|
||||
return count
|
||||
|
||||
@@ -430,9 +430,10 @@ cdef int _appendStartNsEvents(xmlNode* c_node, list event_list) except -1:
|
||||
count = 0
|
||||
c_ns = c_node.nsDef
|
||||
while c_ns is not NULL:
|
||||
- ns_tuple = (funicode(c_ns.prefix) if c_ns.prefix is not NULL else '',
|
||||
- funicode(c_ns.href))
|
||||
- event_list.append( (u"start-ns", ns_tuple) )
|
||||
- count += 1
|
||||
+ if c_ns.href:
|
||||
+ ns_tuple = (funicodeOrEmpty(c_ns.prefix),
|
||||
+ funicode(c_ns.href))
|
||||
+ event_list.append( (u"start-ns", ns_tuple) )
|
||||
+ count += 1
|
||||
c_ns = c_ns.next
|
||||
return count
|
||||
diff --git a/src/lxml/parser.pxi b/src/lxml/parser.pxi
|
||||
index 3ed223b..f5ff6b2 100644
|
||||
--- a/src/lxml/parser.pxi
|
||||
+++ b/src/lxml/parser.pxi
|
||||
@@ -569,6 +569,9 @@ cdef class _ParserContext(_ResolverContext):
|
||||
self._c_ctxt.disableSAX = 0 # work around bug in libxml2
|
||||
else:
|
||||
xmlparser.xmlClearParserCtxt(self._c_ctxt)
|
||||
+ # work around bug in libxml2 [2.9.10 .. 2.9.14]:
|
||||
+ # https://gitlab.gnome.org/GNOME/libxml2/-/issues/378
|
||||
+ self._c_ctxt.nsNr = 0
|
||||
|
||||
cdef int prepare(self, bint set_document_loader=True) except -1:
|
||||
cdef int result
|
||||
diff --git a/src/lxml/tests/test_etree.py b/src/lxml/tests/test_etree.py
|
||||
index 42613dc..db1f560 100644
|
||||
--- a/src/lxml/tests/test_etree.py
|
||||
+++ b/src/lxml/tests/test_etree.py
|
||||
@@ -1459,6 +1459,27 @@ class ETreeOnlyTestCase(HelperTestCase):
|
||||
[1,2,1,4],
|
||||
counts)
|
||||
|
||||
+ def test_walk_after_parse_failure(self):
|
||||
+ # This used to be an issue because libxml2 can leak empty namespaces
|
||||
+ # between failed parser runs. iterwalk() failed to handle such a tree.
|
||||
+ parser = etree.XMLParser()
|
||||
+
|
||||
+ try:
|
||||
+ etree.XML('''<anot xmlns="1">''', parser=parser)
|
||||
+ except etree.XMLSyntaxError:
|
||||
+ pass
|
||||
+ else:
|
||||
+ assert False, "invalid input did not fail to parse"
|
||||
+
|
||||
+ et = etree.XML('''<root> </root>''', parser=parser)
|
||||
+ try:
|
||||
+ ns = next(etree.iterwalk(et, events=('start-ns',)))
|
||||
+ except StopIteration:
|
||||
+ # This would be the expected result, because there was no namespace
|
||||
+ pass
|
||||
+ else:
|
||||
+ assert False, "Found unexpected namespace '%s'" % ns
|
||||
+
|
||||
def test_itertext_comment_pi(self):
|
||||
# https://bugs.launchpad.net/lxml/+bug/1844674
|
||||
XML = self.etree.XML
|
||||
@ -0,0 +1,26 @@
|
||||
diff --git a/src/lxml/tests/test_threading.py b/src/lxml/tests/test_threading.py
|
||||
index 8948c3e..5ede3f8 100644
|
||||
--- a/src/lxml/tests/test_threading.py
|
||||
+++ b/src/lxml/tests/test_threading.py
|
||||
@@ -130,7 +130,7 @@ class ThreadingTestCase(HelperTestCase):
|
||||
<xsl:template match="tag" />
|
||||
<!-- extend time for parsing + transform -->
|
||||
''' + '\n'.join('<xsl:template match="tag%x" />' % i for i in range(200)) + '''
|
||||
- <xsl:foo />
|
||||
+ <xsl:UnExpectedElement />
|
||||
</xsl:stylesheet>''')
|
||||
self.assertRaises(etree.XSLTParseError,
|
||||
etree.XSLT, style)
|
||||
@@ -153,9 +153,10 @@ class ThreadingTestCase(HelperTestCase):
|
||||
self.assertTrue(len(log))
|
||||
if last_log is not None:
|
||||
self.assertEqual(len(last_log), len(log))
|
||||
- self.assertEqual(4, len(log))
|
||||
+ self.assertTrue(len(log) >= 2, len(log))
|
||||
for error in log:
|
||||
- self.assertTrue(':ERROR:XSLT:' in str(error))
|
||||
+ self.assertTrue(':ERROR:XSLT:' in str(error), str(error))
|
||||
+ self.assertTrue(any('UnExpectedElement' in str(error) for error in log), log)
|
||||
last_log = log
|
||||
|
||||
def test_thread_xslt_apply_error_log(self):
|
||||
Loading…
Reference in new issue