source: TI12-security/trunk/NDGSecurity/python/ndg_security_test/ndg/security/test/unit/elementTreeC14n/testC14n.py @ 7077

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg-security/TI12-security/trunk/NDGSecurity/python/ndg_security_test/ndg/security/test/unit/elementTreeC14n/testC14n.py@7077
Revision 7077, 17.5 KB checked in by pjkersha, 11 years ago (diff)
  • Property svn:executable set to *
  • Property svn:keywords set to Id
Line 
1#!/usr/bin/env python
2"""NDG ElementTreeC14n class unit tests
3
4NERC Data Grid Project
5"""
6__author__ = "P J Kershaw"
7__date__ = "03/01/07"
8__copyright__ = "(C) 2009 Science and Technology Facilities Council"
9__license__ = "BSD - see LICENSE file in top-level directory"
10__contact__ = "Philip.Kershaw@stfc.ac.uk"
11__revision__ = '$Id$'
12
13import unittest
14import os
15import sys
16import getpass
17import traceback
18
19from difflib import unified_diff
20
21from StringIO import StringIO
22
23from elementtree import ElementC14N as ETC14N
24from elementtree import ElementTree as ET
25
26# Minidom based Canonicalization from ZSI for comparison
27from ZSI.wstools.c14n import Canonicalize
28
29from xml.xpath.Context import Context
30from xml import xpath
31from xml.dom.ext.reader import PyExpat
32
33xpdVars = os.path.expandvars
34jnPath = os.path.join
35
36class ElementTreeC14nTestCase(unittest.TestCase):
37   
38    def setUp(self):
39       
40        if 'NDGSEC_INT_DEBUG' in os.environ:
41            import pdb
42            pdb.set_trace()         
43
44    def assertEqual(self, a, b):
45        '''Override default to get something easy to read than super class
46        behaviour'''
47        if a != b:
48            diffGen = unified_diff(a.split('\n'), b.split('\n'))
49            raise AssertionError('\n'+'\n'.join(diffGen))
50       
51    def test01UTF8DocEncoding(self):
52       
53        # http://www.w3.org/TR/xml-c14n#Example-UTF8
54        xml = '<?xml version="1.0" encoding="ISO-8859-1"?><doc>&#169;</doc>'
55        elem = ET.fromstring(xml)
56        f = StringIO()
57        ETC14N.write(elem, f)
58        c14n = f.getvalue()
59        #self.assertEqual(c14n, '<doc>#xC2#xA9</doc>')
60        self.assertEqual(c14n, '<doc>\xC2\xA9</doc>')
61
62    def test01aPIsCommentsAndOutsideOfDocElem(self):
63        # http://www.w3.org/TR/xml-c14n#Example-OutsideDoc - PIs, Comments, and
64        # Outside of Document Element
65        xml = \
66'''<?xml version="1.0"?>
67
68<?xml-stylesheet   href="doc.xsl"
69   type="text/xsl"   ?>
70
71<!DOCTYPE doc SYSTEM "doc.dtd">
72
73<doc>Hello, world!<!-- Comment 1 --></doc>
74
75<?pi-without-data     ?>
76
77<!-- Comment 2 -->
78
79<!-- Comment 3 -->'''
80
81        exptdC14n = \
82'''<?xml-stylesheet href="doc.xsl"
83   type="text/xsl"   ?>
84<doc>Hello, world!</doc>
85<?pi-without-data?>'''
86
87        elem = ET.fromstring(xml)
88        f = StringIO()
89        ETC14N.write(elem, f)
90        c14n = f.getvalue()
91        self.assertEqual(c14n, exptdC14n)
92
93   
94    def test02NormalizeLineBreaks(self):
95        xml = '<?xml version="1.0" encoding="UTF-8"?>\r\n<a/>\r\n'
96        elem = ET.fromstring(xml)
97        f = StringIO()
98        ET.ElementTree(elem).write_c14n(f)
99        c14n = f.getvalue()
100        self.failIf('\r' in c14n, "Carriage return \r char found in c14n")
101
102   
103    def test03NormalizedAttrVals(self):
104        pass
105
106   
107    def test04CharAndParsedEntityRefsReplaced(self):
108        xml = '''<!DOCTYPE doc [
109<!ATTLIST doc attrExtEnt ENTITY #IMPLIED>
110<!ENTITY ent1 "Hello">
111<!ENTITY ent2 SYSTEM "world.txt">
112<!ENTITY entExt SYSTEM "earth.gif" NDATA gif>
113<!NOTATION gif SYSTEM "viewgif.exe">
114]>
115<doc attrExtEnt="entExt">
116   &ent1;, &ent2;!
117</doc>
118
119<!-- Let world.txt contain "world" (excluding the quotes) -->'''
120
121        exptdC14n = '''<doc attrExtEnt="entExt">
122   Hello, world!
123</doc>'''
124        elem = ET.fromstring(xml)
125        f = StringIO()
126        ET.ElementTree(elem).write_c14n(f)
127        c14n = f.getvalue()
128        self.assertEqual(c14n, exptdC14n)
129       
130   
131    def test05CDATASectionsReplaced(self):
132        xml = \
133"""<?xml version="1.0" encoding="UTF-8"?>
134<script>
135<![CDATA[
136function matchwo(a,b)
137{
138if (a < b && a > 0) then
139   {
140   print("Match");
141   return 1;
142   }
143else
144   {
145   print('Different');
146   return 0;
147   }
148}
149]]>
150</script>
151"""
152        elem = ET.fromstring(xml)
153        f = StringIO()
154        ET.ElementTree(elem).write_c14n(f)
155        c14n = f.getvalue()
156       
157        self.failIf('CDATA' in c14n, "CDATA not removed, c14n = %s" % c14n)
158        self.failUnless('&lt;' in c14n,
159                        "Less than not converted, c14n = %s" % c14n)
160        self.failUnless('&gt;' in c14n, 
161                        "Greater than not converted, c14n = %s" % c14n)
162        self.failUnless('&amp;' in c14n, 
163                        "Ampersand not converted, c14n = %s" % c14n)
164
165        # Test for double quotes / apostrophes?
166       
167   
168    def test06XMLDeclAndDTDRemoved(self):
169        xmlDecl = '<?xml version="1.0" encoding="UTF-8"?>'
170        dtd = \
171"""<!DOCTYPE note [
172  <!ELEMENT note (to,from,heading,body)>
173  <!ELEMENT to      (#PCDATA)>
174  <!ELEMENT from    (#PCDATA)>
175  <!ELEMENT heading (#PCDATA)>
176  <!ELEMENT body    (#PCDATA)>
177]>
178"""
179        xml = \
180"""%s
181%s<a/>""" % (xmlDecl, dtd)
182
183        elem = ET.fromstring(xml)
184        f = StringIO()
185        ET.ElementTree(elem).write_c14n(f)
186        c14n = f.getvalue()
187        self.failIf('<?xml version="1.0" encoding="UTF-8"?>' in c14n, 
188                    "XML Declaration not removed")
189        self.failIf(dtd in c14n, "DTD not removed")
190
191   
192    def test07EmptyElemsConvertedStartEndPairs(self):
193        elem = ET.fromstring('<?xml version="1.0" encoding="UTF-8"?><a/>')
194        f = StringIO()
195        ET.ElementTree(elem).write_c14n(f)
196        c14n = f.getvalue()
197        self.failUnless(c14n == '<a></a>', "C14N = %s" % c14n)
198
199         
200    def test08WhitespaceNormalized(self):
201        # ...outside the document element and within start and end tags
202        dat = \
203'''        1 2
204  3'''
205 
206        xml = \
207'''<?xml version="1.0" encoding="UTF-8"?>
208<doc xmlns="http://example.com/default">
209  <a
210     a2="2"   a1="1"
211  >%s</a>
212</doc>
213
214''' % dat
215
216        elem = ET.fromstring(xml)
217        f = StringIO()
218        ET.ElementTree(elem).write_c14n(f)
219        c14n = f.getvalue()
220       
221        self.failUnless('a1="1" a2="2"' in c14n, 
222                        "Expecting single space between attributes")
223        self.failUnless(dat in c14n, 
224                        "Expecting element content to be preserved")
225       
226        sub = c14n[c14n.find('<a'):c14n.find('>')]
227        self.failIf('\n' in sub, 
228                    "Expecting removal of line breaks for 'a' element")
229     
230     
231    def test09WhitespaceInCharContentRetained(self):
232        # http://www.w3.org/TR/xml-c14n#Example-WhitespaceInContent
233        # Nb. excludes chars removed during line break normalization
234        xml = \
235'''<doc>
236   <clean>   </clean>
237   <dirty>   A   B   </dirty>
238   <mixed>
239      A
240      <clean>   </clean>
241      B
242      <dirty>   A   B   </dirty>
243      C
244   </mixed>
245</doc>'''
246        elem = ET.fromstring(xml)
247        f = StringIO()
248        ETC14N.write(elem, f)
249        c14n = f.getvalue()
250       
251        # In this case the canonicalized form should be identical to the
252        # original
253        self.assertEqual(c14n, xml)
254
255       
256    def test10AttrValDelimitersSet2DblQuotes(self):
257        xml = \
258"""<?xml version="1.0" encoding="UTF-8"?>
259  <b y:a1='1' a3='"3"'
260     xmlns:y='http://example.com/y' y:a2='2'/>
261"""
262
263        elem = ET.fromstring(xml)
264        f = StringIO()
265        ET.ElementTree(elem).write_c14n(f)
266        c14n = f.getvalue()
267        self.failIf("'" in c14n, 
268                    "Expecting removal of apostrophes C14N = %s" % c14n)
269
270   
271    def test11SpecialCharsReplaced(self):
272        # i.e. within attribute values and character content
273        pass
274       
275       
276    def test12SuperflousNSdeclsRemoved(self):
277        extraNS = "http://example.com/default"
278        xml = \
279"""<?xml version="1.0" encoding="UTF-8"?>
280<doc xmlns:x="http://example.com/x" xmlns="%s">
281  <b y:a1='1' xmlns="%s" a3='"3"'
282     xmlns:y='http://example.com/y' y:a2='2'/>
283</doc>""" % (extraNS, extraNS)
284
285        elem = ET.fromstring(xml)
286        f = StringIO()
287        ET.ElementTree(elem).write_c14n(f)
288        c14n = f.getvalue()
289       
290        # Namespace should now only occur once...
291        self.failUnless(c14n.find(extraNS) == c14n.rfind(extraNS), 
292                    "Expecting removal of extra NS %s in output = %s" % \
293                    (extraNS, c14n))
294       
295       
296    def test13DefAttrsAdded2EachElem(self):
297        # Ref. http://www.w3.org/TR/xml-c14n#Example-SETags
298        xml = '''<!DOCTYPE doc [<!ATTLIST e9 attr CDATA "default">]>
299<doc>
300   <e1   />
301   <e2   ></e2>
302   <e3   name = "elem3"   id="elem3"   />
303   <e4   name="elem4"   id="elem4"   ></e4>
304   <e5 a:attr="out" b:attr="sorted" attr2="all" attr="I'm"
305      xmlns:b="http://www.ietf.org"
306      xmlns:a="http://www.w3.org"
307      xmlns="http://example.org"/>
308   <e6 xmlns="" xmlns:a="http://www.w3.org">
309      <e7 xmlns="http://www.ietf.org">
310         <e8 xmlns="" xmlns:a="http://www.w3.org">
311            <e9 xmlns="" xmlns:a="http://www.ietf.org"/>
312         </e8>
313      </e7>
314   </e6>
315</doc>'''
316
317        elem = ET.fromstring(xml)
318        f = StringIO()
319#        ET.ElementTree(elem).write_c14n(f)
320        ETC14N.write(elem, f)
321        c14n = f.getvalue()
322
323        exptdC14n = '''<doc>
324   <e1></e1>
325   <e2></e2>
326   <e3 id="elem3" name="elem3"></e3>
327   <e4 id="elem4" name="elem4"></e4>
328   <e5 xmlns="http://example.org" xmlns:a="http://www.w3.org" xmlns:b="http://www.ietf.org" attr="I'm" attr2="all" b:attr="sorted" a:attr="out"></e5>
329   <e6 xmlns:a="http://www.w3.org">
330      <e7 xmlns="http://www.ietf.org">
331         <e8 xmlns="">
332            <e9 xmlns:a="http://www.ietf.org" attr="default"></e9>
333         </e8>
334      </e7>
335   </e6>
336</doc>'''
337        self.assertEqual(c14n, exptdC14n)
338       
339    def test14DocumentSubsets(self):
340        # Ref. http://www.w3.org/TR/xml-c14n#Example-DocSubsets
341        xml = \
342"""<!DOCTYPE doc [
343<!ATTLIST e2 xml:space (default|preserve) 'preserve'>
344<!ATTLIST e3 id ID #IMPLIED>
345]>
346<doc xmlns="http://www.ietf.org" xmlns:w3c="http://www.w3.org">
347   <e1>
348      <e2 xmlns="">
349         <e3 id="E3"/>
350      </e2>
351   </e1>
352</doc>"""
353
354#'''<!-- Evaluate with declaration xmlns:ietf="http://www.ietf.org" -->
355        xpathExpr = \
356'''
357(//. | //@* | //namespace::*)
358[
359   self::ietf:e1 or (parent::ietf:e1 and not(self::text() or self::e2))
360   or
361   count(id("E3")|ancestor-or-self::node()) = count(ancestor-or-self::node())
362]'''
363
364        exptdC14n = \
365'<e1 xmlns="http://www.ietf.org" xmlns:w3c="http://www.w3.org"><e3 xmlns="" id="E3" xml:space="preserve"></e3></e1>'
366
367        elem = ET.fromstring(xml)
368        f = StringIO()
369        subElem = elem.find(xpathExpr)
370        ETC14N.write(elem, f, subset_element=subElem)
371        c14n = f.getvalue()
372
373    def test15CmpZSIc14n(self):
374        elem = ETC14N.parse('./windows-ac.xml')
375        ETC14N.write(elem, './et-c14n-ac.xml')
376       
377        from xml.dom.ext.reader import PyExpat
378        reader = PyExpat.Reader()
379        dom = reader.fromStream(open('./windows-ac.xml'))
380       
381        zsiC14n = Canonicalize(dom)
382        etC14n = open('./et-c14n-ac.xml').read()
383        open('./zsi-c14n-ac.xml', 'w').write(zsiC14n)
384       
385        etC14n = open('./et-c14n-ac.xml').read()
386
387        self.failUnless(etC14n == zsiC14n, "ZSI C14N output differs")
388       
389    def test16Cmplxmlc14n(self):
390        from StringIO import StringIO
391
392        elem = ETC14N.parse('./windows-ac.xml')
393        ETC14N.write(elem, './et-c14n-ac-2.xml')
394       
395       
396        from lxml import etree as lxmlET
397       
398        lxmlElem = lxmlET.parse('./windows-ac.xml')
399        lxmlETf = StringIO()
400        lxmlElem.write_c14n(lxmlETf)
401        open('./lxml-c14n-ac.xml', 'w').write(lxmlETf.getvalue())
402       
403        f1 = open('./et-c14n-ac-2.xml')
404        etC14n = f1.read()
405        f1.close()
406       
407        self.failUnless(etC14n == lxmlETf.getvalue(),
408                        "lxml C14N output differs")
409       
410       
411    def test17InclusiveC14nWithXPath(self):
412        # Inclusive Canonicalization of portions of a SOAP message extracted
413        # using XPath
414       
415        inputFile = './soapGetAttCertResponse.xml'
416       
417        reader = PyExpat.Reader()
418        dom = reader.fromStream(open(inputFile))
419        processorNss = \
420        {
421            'wsu': \
422"http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd",
423        }
424   
425        ctxt = Context(dom, processorNss=processorNss)
426        zsiRefNodes = xpath.Evaluate('//*[@wsu:Id]', 
427                                  contextNode=dom, 
428                                  context=ctxt)
429       
430        # ElementTree
431        elem = ETC14N.parse(inputFile)
432       
433        # Extract nodes for signing
434        etRefNodes = elem.findall('.//*[@wsu:Id]', namespaces=processorNss)
435       
436        for zsiRefNode, etRefNode in zip(zsiRefNodes, etRefNodes):
437            # Get ref node and all it's children
438            zsiRefC14n = Canonicalize(zsiRefNode)
439
440#            print "_"*80
441#            print "ZSI Inclusive C14N %s:\n" % zsiRefNode.nodeName
442#            print zsiRefC14n
443            open('soapGetAttCertResponse-%s-zsi-c14n.xml'%zsiRefNode.localName, 
444                 'w').write(zsiRefC14n)
445                 
446            f = StringIO()
447            ETC14N.write(elem, f, subset=etRefNode)
448            etRefC14n = f.getvalue()
449           
450#            print "_"*80
451#            print "ElementTree Inclusive C14N %s:\n" % etRefNode.tag
452#            print etRefC14n
453            open('soapGetAttCertResponse-%s-et-c14n.xml' % \
454                 etRefNode.tag.split('}')[-1], 
455                 'w').write(etRefC14n)
456            self.assertEqual(zsiRefC14n, etRefC14n)
457       
458    def test18ExclC14nWithXPath(self):
459        # Exclusive C14N applied to portions of a SOAP message by extracting
460        # using XPath
461       
462        inputFile = './soapGetAttCertResponse.xml'
463
464        reader = PyExpat.Reader()
465        dom = reader.fromStream(open(inputFile))
466        processorNss = \
467        {
468            'wsu': \
469"http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd",
470        }
471   
472        ctxt = Context(dom, processorNss=processorNss)
473        zsiRefNodes = xpath.Evaluate('//*[@wsu:Id]', 
474                                  contextNode=dom, 
475                                  context=ctxt)
476        # ElementTree
477        elem = ETC14N.parse(inputFile)
478       
479        # Extract nodes for signing
480        etRefNodes = elem.findall('.//*[@wsu:Id]', namespaces=processorNss)
481       
482        for zsiRefNode, etRefNode in zip(zsiRefNodes, etRefNodes):
483            # Get ref node and all it's children
484            refSubsetList = getChildNodes(zsiRefNode)
485            zsiRefC14n = Canonicalize(dom, None, subset=refSubsetList,
486                                   unsuppressedPrefixes=[])
487
488#            print "_"*80
489#            print "ZSI Exclusive C14N %s:\n" % zsiRefNode.nodeName
490#            print zsiRefC14n
491            open('soapGetAttCertResponse-%s-exclC14n.xml'%zsiRefNode.localName, 
492                 'w').write(zsiRefC14n)
493       
494            # ElementTree equivalent     
495            f = StringIO()
496            ETC14N.write(elem, f, etRefNode, exclusive=True)
497            etRefC14n = f.getvalue()
498           
499#            print "_"*80
500#            print "ElementTree Exclusive C14N %s:\n" % etRefNode.tag
501#            print etRefC14n
502            open('soapGetAttCertResponse-%s-et-exclC14n.xml' % \
503                 etRefNode.tag.split('}')[-1], 
504                 'w').write(etRefC14n)
505       
506            self.assertEqual(zsiRefC14n, etRefC14n)
507       
508    def test19ExclC14nWithXPathAndInclusiveNSPfx(self):
509        # Exclusive C14N applied to portions of a SOAP message by extracting
510        # using XPath
511        inputFile = './soapGetAttCertResponse.xml'
512       
513        from xml.xpath.Context import Context
514        from xml import xpath
515        from xml.dom.ext.reader import PyExpat
516        reader = PyExpat.Reader()
517        dom = reader.fromStream(open(inputFile))
518        processorNss = \
519        {
520            'wsu': \
521"http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd",
522        }
523   
524        ctxt = Context(dom, processorNss=processorNss)
525        zsiRefNodes = xpath.Evaluate('//*[@wsu:Id]', 
526                                  contextNode=dom, 
527                                  context=ctxt)
528
529        # ElementTree
530        elem = ETC14N.parse(inputFile)
531       
532        # Extract nodes for signing
533        etRefNodes = elem.findall('.//*[@wsu:Id]', namespaces=processorNss)
534       
535        nsPfx = ['SOAP-ENV', 'ds']
536        for zsiRefNode, etRefNode in zip(zsiRefNodes, etRefNodes):
537            # Get ref node and all it's children
538            refSubsetList = getChildNodes(zsiRefNode)
539            zsiRefC14n = Canonicalize(dom, None, subset=refSubsetList,
540                                   unsuppressedPrefixes=nsPfx)
541
542#            print "_"*80
543#            print "Exclusive C14N with Prefixes %s:\n" % zsiRefNode.nodeName
544#            print zsiRefC14n
545            open('soapGetAttCertResponse-%s-exclC14nWithInclPrefixes.xml' % \
546                 zsiRefNode.localName, 
547                 'w').write(zsiRefC14n)
548       
549            # ElementTree equivalent     
550            f = StringIO()
551            ETC14N.write(elem, f, subset=etRefNode, exclusive=True,
552                         inclusive_namespaces=nsPfx)
553            etRefC14n = f.getvalue()
554           
555#            print "_"*80
556#            print "ElementTree Exclusive C14N %s:\n" % etRefNode.tag
557#            print etRefC14n
558            open('soapGetAttCertResponse-%s-et-exclC14n.xml' % \
559                 etRefNode.tag.split('}')[-1], 
560                 'w').write(etRefC14n)
561
562            self.assertEqual(zsiRefC14n, etRefC14n)
563     
564
565def getChildNodes(node, nodeList=None):
566    if nodeList is None:
567        nodeList = [node] 
568    return _getChildNodes(node, nodeList=nodeList)
569           
570def _getChildNodes(node, nodeList=None):
571
572    if node.attributes is not None:
573        nodeList += node.attributes.values() 
574    nodeList += node.childNodes
575    for childNode in node.childNodes:
576        _getChildNodes(childNode, nodeList)
577    return nodeList
578
579if __name__ == "__main__":
580    unittest.main()
581
Note: See TracBrowser for help on using the repository browser.