source: TI12-security/trunk/NDG_XACML/ndg/xacml/utils/configfileparsers.py @ 7064

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg-security/TI12-security/trunk/NDG_XACML/ndg/xacml/utils/configfileparsers.py@7064
Revision 7064, 22.3 KB checked in by pjkersha, 11 years ago (diff)

Incomplete - task 2: XACML-Security Integration

  • added and and function and placeholders fro xpath-node-* functions
  • Property svn:keywords set to Id
Line 
1"""
2Generic parsers to use when reading in configuration data
3- methods available to deal with both XML and INI (flat text key/val) formats
4"""
5__author__ = "C Byrom - Tessella"
6__date__ = "20/05/08"
7__copyright__ = "(C) 2009 Science and Technology Facilities Council"
8__license__ = "BSD - see LICENSE file in top-level directory"
9__contact__ = "Philip.Kershaw@stfc.ac.uk"
10__revision__ = '$Id$'
11
12from ConfigParser import SafeConfigParser, InterpolationMissingOptionError, \
13    NoOptionError
14
15# For parsing of properties file
16try: # python 2.5
17    from xml.etree import cElementTree as ElementTree
18except ImportError:
19    # if you've installed it yourself it comes this way
20    import cElementTree as ElementTree
21
22import logging, os
23log = logging.getLogger(__name__)
24
25# lambda function to expand out any environment variables in properties read in
26expandEnvVars = lambda x: isinstance(x, basestring) and \
27                    os.path.expandvars(x).strip() or x
28
29
30class CaseSensitiveConfigParser(SafeConfigParser):
31    '''
32    Subclass the SafeConfigParser - to preserve the original string case of the
33    cfg section names - NB, the RawConfigParser default is to lowercase these
34    by default
35    '''
36   
37    def optionxform(self, optionstr):
38        return optionstr
39       
40class ConfigFileParseError(Exception):
41    """Raise for errors in configuration file formatting"""
42
43def readAndValidateProperties(propFilePath, validKeys={}, **iniPropertyFileKw):
44    """
45    Determine the type of properties file and load the contents appropriately.
46    If a dict of valid keys is also specified, check the loaded properties
47    against these.
48   
49    @param propFilePath: file path to properties file - either in xml or ini
50    format
51    @type propFilePath: string
52    @keywords validKeys: a dictionary of valid values to be read from the file
53    - if values are encountered that are not in this list, an exception will be
54    thrown
55    - if all info should be read, this keyword should be left to its default
56    value
57    - NB, this dict will also ensure list data is read in correctly
58    @type validKeys: dict
59    @raise ValueError: if a key is read in from the file that is not included
60    in the specified validKeys dict
61    """
62    log.debug("Reading properties from %s" % propFilePath)
63    properties = {}
64    if propFilePath.lower().endswith('.xml'):
65        log.debug("File has 'xml' suffix - treating as standard XML formatted "
66                  "properties file")
67        log.warning("Current version of code for properties handling with "
68                    "XML is untested - may be deprecated")
69        properties = readXMLPropertyFile(propFilePath, validKeys)
70        # if validKeys set, check that all loaded property values are featured
71        # in this list
72        if validKeys:
73            validateProperties(properties, validKeys)
74       
75            # lastly set any default values from the validKeys dict for vals
76            # not read in from the property file
77            _setDefaultValues(properties, validKeys)
78    else:
79        properties = readINIPropertyFile(propFilePath, validKeys,
80                                         **iniPropertyFileKw)
81       
82        # Ugly hack to allow for sections and option prefixes in the validation
83        # and setting of defaults
84        if validKeys:
85            sections = iniPropertyFileKw.get('sections')
86            prefix = iniPropertyFileKw.get('prefix')
87            if sections is not None:
88                for section in sections:
89                    if section == 'DEFAULT':
90                        propBranch = properties
91                    else:
92                        propBranch = properties[section]
93                       
94                    validateProperties(propBranch, validKeys)
95                    _setDefaultValues(propBranch, validKeys)
96                   
97            else:
98                validateProperties(properties, validKeys)
99                _setDefaultValues(properties, validKeys)
100
101   
102    # lastly, expand out any environment variables set in the properties file
103    properties = _expandEnvironmentVariables(properties)
104    log.info('Properties loaded')
105    return properties
106
107
108def readProperties(propFilePath, validKeys={}, **iniPropertyFileKw):
109    """
110    Determine the type of properties file and load the contents appropriately.
111    @param propFilePath: file path to properties file - either in xml or ini
112    format
113    @type propFilePath: string
114    """
115    log.debug("Reading properties from %s" %propFilePath)
116    properties = {}
117    if propFilePath.lower().endswith('.xml'):
118        log.debug("File has 'xml' suffix - treating as standard XML formatted "
119                  "properties file")
120        log.warning("Current version of code for properties handling with "
121                    "XML is untested - may be deprecated")
122        properties = readXMLPropertyFile(propFilePath, validKeys)
123    else:
124        properties = readINIPropertyFile(propFilePath, validKeys,
125                                         **iniPropertyFileKw)
126   
127    # lastly, expand out any environment variables set in the properties file
128    properties = _expandEnvironmentVariables(properties)
129    log.info('Properties loaded')
130    return properties
131       
132
133class INIPropertyFile(object):
134    '''INI Property file reading class
135   
136    __call__ method enables a standalone read function'''
137   
138    defaultOptionNames = ('here',)
139   
140    def read(self, 
141             propFilePath, 
142             validKeys, 
143             cfg=None, 
144             sections=None,
145             defaultItems={}, 
146             prefix=''):
147        """
148        Read 'ini' type property file - i.e. a flat text file with key/value
149        data separated into sections
150   
151        @param propFilePath: file path to properties file - either in xml or
152        ini format
153        @type propFilePath: string
154        @param validKeys: a dictionary of valid values to be read from the file
155        - if values are encountered that are not in this list, an exception
156        will be thrown
157        - if all info should be read, set this param to 'None'
158        @type validKeys: dict
159        @type sections: basestring
160        @param sections: sections to be read from - defaults to all sections in the
161        file
162        @type defaultItems: dict
163        @param defaultItems: add items via this input dictionary as well as by
164        retrieval from config file itself.  This only comes into effect if
165        cfg was not set and a new config object is created locally.
166        @rtype: dict
167        @return: dict with the loaded properties in
168        @raise ValueError: if a key is read in from the file that is not
169        included in the specified validKeys dict
170        """
171        log.debug("File is not marked as XML - treating as flat 'ini' format "
172                  "file")
173       
174        # Keep a record of property file path setting
175        self.propFilePath = propFilePath
176           
177        if cfg is None:
178            # Add default item for file location to enable convenient
179            # substitutions within the file
180            defaultItems['here'] = os.path.dirname(propFilePath)
181           
182            self.cfg = CaseSensitiveConfigParser(defaults=defaultItems)
183            self.cfg.read(propFilePath)
184            if not os.path.isfile(propFilePath):
185                raise IOError('Error parsing properties file "%s": No such '
186                              'file' % propFilePath)
187        else:
188            self.cfg = cfg
189               
190        properties = {}
191       
192        if sections is None:
193            # NB, add 'DEFAULT' section since this isn't returned by the
194            # 'sections()'
195            sections = self.cfg.sections()
196            sections.append('DEFAULT')
197       
198        # parse data from the specified sections of the config file
199        for section in sections:
200            if section == 'DEFAULT':
201                properties.update(_parseConfig(self.cfg, 
202                                               validKeys, 
203                                               section=section,
204                                               prefix=prefix))
205            else:                   
206                properties[section] = _parseConfig(self.cfg, 
207                                                   validKeys, 
208                                                   section=section,
209                                                   prefix=prefix)
210   
211               
212        # Get rid of 'here' default item to avoid interfering with later
213        # processing
214        for opt in INIPropertyFile.defaultOptionNames:
215            properties.pop(opt, None)
216       
217        log.debug("Finished reading from INI properties file")
218        return properties
219   
220    # Enables use of this class like a function see below ...
221    __call__ = read
222   
223   
224# Enable read INI of file as a one shot call
225readINIPropertyFile = INIPropertyFile()   
226
227
228class INIPropertyFileWithValidation(INIPropertyFile):
229    '''Extension of INI Property file reading class to make a callable that
230    validates as well as reads in the properties.  Also see
231    readAndValidateINIPropertyFile in this module'''
232   
233    def readAndValidate(self, propFilePath, validKeys, **kw):
234        prop = super(INIPropertyFileWithValidation,self).__call__(propFilePath,
235                                                                  validKeys, 
236                                                                  **kw)
237       
238        # Pass wsseSection but respect validateProperties default value
239        wsseSection = kw.get('wssSection')
240        if wsseSection is not None:
241            validatePropKw = {'wsseSection': wsseSection}
242        else:
243            validatePropKw = {}
244           
245        validateProperties(prop, validKeys, **validatePropKw)
246        return prop
247   
248    __call__ = readAndValidate
249   
250# Enable read and validation of INI file as a one shot call
251readAndValidateINIPropertyFile = INIPropertyFileWithValidation()
252
253
254def _parseConfig(cfg, validKeys, section='DEFAULT', prefix=''):
255    '''
256    Extract parameters from cfg config object
257    @param cfg: config object
258    @type cfg: CaseSensitiveConfigParser
259    @param validKeys: a dictionary of valid values to be read from the file -
260    used to check the type of the input parameter to ensure (lists) are handled
261    correctly
262    @type validKeys: dict
263    @keyword section: section of config file to parse from
264    @type section: string
265    @return: dict with the loaded properties in
266    '''
267    log.debug("Parsing section: %s" % section)
268
269    propRoot = {}
270    propThisBranch = propRoot
271   
272    if section == 'DEFAULT':
273        keys = cfg.defaults().keys()
274    else:
275        keys = cfg.options(section)
276        # NB, we need to be careful here - since this will return the section
277        # keywords AND the 'DEFAULT' section entries - so use the difference
278        # between the two
279        keys = filter(lambda x:x not in cfg.defaults().keys(), keys)
280
281    for key in keys:
282        try:
283            val = cfg.get(section, key)
284        except InterpolationMissingOptionError, e:
285            log.warning('Ignoring property "%s": %s' % (key, e))
286            continue
287       
288        # Allow for prefixes - 1st a prefix global to all parameters
289#        keyLevels = key.split('.')
290#        if prefix:
291#            if keyLevels[0] == prefix:
292#                keyLevels = keyLevels[1:]
293#                if keyLevels == []:
294#                    raise ConfigFileParseError('Expecting "%s.<option>"; got '
295#                                               '"%s"' % ((prefix,)*2))
296#            else:
297#                continue           
298        if prefix:
299            if key.startswith(prefix):
300                keyLevels = key.replace(prefix+'.', '', 1).split('.') 
301                if keyLevels == []:
302                    raise ConfigFileParseError('Expecting "%s.<option>"; got '
303                                               '"%s"' % ((prefix,)*2))
304            else:
305                continue
306        else:
307            keyLevels = key.split('.')
308                       
309        # 2nd - prefixes to denote sections
310        if len(keyLevels) > 1:
311               
312            # Nb. This allows only one level of nesting - subsequent levels if
313            # present are represented by a concatenation of the levels joined
314            # by underscores
315            subSectionKey = keyLevels[0]
316            subKey = '_'.join(keyLevels[1:])
317            if subSectionKey in validKeys and \
318               isinstance(validKeys[subSectionKey], dict):
319                val = _parseVal(cfg, section, key, validKeys[subSectionKey],
320                                subKey=subKey)
321                if subSectionKey in propThisBranch:
322                    propThisBranch[subSectionKey][subKey] = val
323                else:
324                    propThisBranch[subSectionKey] = {subKey: val}
325        else: 
326            # No sub-section present           
327            subKey = keyLevels[0]
328            val = _parseVal(cfg, section, key, validKeys, subKey=subKey)
329           
330            # check if key already exists; if so, append to list
331            if propThisBranch.has_key(subKey):
332                propThisBranch[subKey] = __listify(
333                                            propThisBranch[subKey]).extend(val)
334            else:
335                propThisBranch[subKey] = val
336
337    log.debug("Finished parsing section")
338    return propRoot
339
340def _parseVal(cfg, section, option, validKeys, subKey=None):
341    '''Convert option to correct type trying each parser config routine in
342    turn.  Convert to a list if validKeys dict item indicates so
343   
344    @type cfg: ndg.xacml.utils.configfileparsers.CaseSensitiveConfigParser
345    @param cfg: config file object
346    @type section: basestring
347    @param section: section in config file to read from
348    @type key: basestring
349    @param key: section option to read
350    @type validKeys: dict
351    @param validKeys: key look-up - if item is set to list type then the option
352    value in the config file will be split into a list.'''
353   
354    if subKey:
355        key = subKey
356    else:
357        key = option
358         
359    conversionFuncs = (cfg.getint, cfg.getfloat, cfg.getboolean, cfg.get)
360    for conversionFunc in conversionFuncs:
361        try:
362            val = conversionFunc(section, option)
363            if val == '':
364                # NB, the XML parser will return empty vals as None, so ensure
365                # consistency here
366                val = None
367               
368            elif isinstance(val, basestring):
369                # expand out any env vars
370                val = expandEnvVars(val)
371               
372                # ensure it is read in as the correct type
373                if key in validKeys and isinstance(validKeys[key], list):
374                    # Treat as a list of space separated string type elements
375                    # Nb. lists only cater for string type elements
376                    val = val.split()
377             
378            return val
379        except ValueError:
380            continue
381        except Exception, e:
382            log.error('Error parsing option "%s" in section "%s": %s' %
383                      (section, key, e))
384            raise
385
386    raise ValueError('Error parsing option "%s" in section "%s"'%(section,key))
387
388         
389def readXMLPropertyFile(propFilePath, validKeys, rootElem=None):
390    """
391    Read property file - assuming the standard XML schema
392
393    @param propFilePath: file path to properties file - either in xml or ini
394    format
395    @type propFilePath: string
396    @param validKeys: a dictionary of valid values to be read from the file -
397    used to check the type of the input parameter to ensure (lists) are handled
398    correctly
399    @keyword rootElem: a particular element of an ElementTree can be passed in
400    to use as the root element; NB, if this is set, it will take precedence
401    over any propFilePath specified
402    @type rootElem: ElementTree.Element
403    @return: dict with the loaded properties in
404    """
405    if rootElem is None:
406        try:
407            tree = ElementTree.parse(propFilePath)
408           
409        except IOError, ioErr:
410            raise ValueError("Error parsing properties file \"%s\": %s" % 
411                             (ioErr.filename, ioErr.strerror))
412   
413        rootElem = tree.getroot()
414        if rootElem is None:
415            raise ValueError('Parsing properties file "%s": root element is '
416                             'not defined' % propFilePath)
417
418    properties = {}
419    # Copy properties from file into a dictionary
420    try:
421        for elem in rootElem:
422            key = elem.tag
423            val = elem.text
424
425            # expand out any env vars
426            val = expandEnvVars(val)
427
428            # if the tag contains an integer, convert this appropriately
429            if val and val.isdigit():
430                val = int(val)
431           
432            # check for lists - don't recurse into these else the key names
433            # will end up being wrong
434            if key in validKeys and isinstance(validKeys[key], list):
435                # handle lists of elements
436                if len(elem) == 0:
437                    if elem.text is not None:
438                        # Treat as a list of space separated elements
439                        val = val.split()
440                else:
441                    # Parse from a list of sub-elements
442                    val = [expandEnvVars(subElem.text.strip()) \
443                           for subElem in elem]
444           
445            # otherwise check for subelements; if these exist, recurse and
446            # store properties in an inner dictionary
447            elif len(elem) > 0:
448                val = readXMLPropertyFile(propFilePath,validKeys,rootElem=elem)
449
450            # check if key already exists; if so, append to list
451            if properties.has_key(key):
452                properties[key] = __listify(properties[key]).extend(val)
453            else:
454                properties[key] = val
455           
456    except Exception, e:
457        raise ValueError('Error parsing tag "%s" in properties file "%s": %s' %
458                         (elem.tag, propFilePath, e))
459
460    log.debug("Finished reading from XML properties file")
461    return properties
462
463
464def __listify(val):
465    '''
466    Checks if val is a list; if so return as is, if not return as list
467   
468    @type val: list
469    @param val: object to turn into a list
470    @rtype: list
471    @return: val as a list (if it is not already)
472    '''
473    if isinstance(val, list):
474        return val
475    return [val]
476
477
478def validateProperties(properties, validKeys):
479    '''
480    Check the contents of the properties dict to ensure it doesn't contain
481    any keys not featured in the validKeys dict; if it does, throw an exception
482    @param properties: dictionary storing loaded properties
483    @type properties: dict
484    @param validKeys: a dictionary of valid values
485    @type validKeys: dict
486    @raise ValueError: if a key is read in from the file that is not included
487    in the specified validKeys dict
488    '''
489    log.debug("Checking for invalid properties")
490    invalidKeys = []
491    for key in validKeys:
492        # NB, this is a standard property used across most services - so check
493        # using the properties listed here
494        if validKeys[key] and isinstance(validKeys[key], dict):
495            validateProperties(properties.get(key, {}), validKeys[key])
496               
497        elif key not in properties and nonDefaultProperty(validKeys[key]):
498            invalidKeys += [key]
499
500    if invalidKeys != []:
501        errorMessage = "The following properties file " + \
502            "elements are missing and must be set: " + ', '.join(invalidKeys)
503        log.error(errorMessage)
504        raise ValueError(errorMessage)
505
506nonDefaultProperty = lambda prop:prop==NotImplemented or prop==[NotImplemented]
507
508def _expandEnvironmentVariables(properties):
509    '''
510    Iterate through the values in a dict and expand out environment variables
511    specified in any non password option entries
512    @param properties: dict of properties to expand
513    @type properties: dict
514    @return: dict with expanded values
515    '''
516    log.debug("Expanding out environment variables in properties dictionary")
517    for key, val in properties.items():
518        # only check strings or lists of strings
519        if isinstance(val, list):           
520            properties[key] = [_expandEnvironmentVariable(key, item) \
521                               for item in val]
522           
523        elif isinstance(val, str):
524            properties[key] = _expandEnvironmentVariable(key, val)
525           
526    log.debug("Finished expanding environment variables")
527    return properties
528
529
530def _expandEnvironmentVariable(key, val):
531    '''
532    Expand out a val, if it contains environment variables and
533    is not password related
534    @param key: key name for the value
535    @type key: str
536    @param val: value to expand env vars out in
537    @type val: str
538    @rtype: basestring
539    @return: val - with any environment variables expanded out
540    '''
541    if key.lower().find('pwd') == -1 and key.lower().find('password') == -1:
542        val = os.path.expandvars(val)
543    return val
544
545   
546def _setDefaultValues(properties, validKeys, sectionKey=''):
547    '''
548    Check the contents of the properties dict to ensure it contains all the
549    keys featured in the validKeys dict; if any of these are missing or have
550    no value set for them, set up default values for these in the properties
551    dict
552    @param properties: dictionary storing loaded properties
553    @type properties: dict
554    @param validKeys: a dictionary of valid values
555    @type validKeys: dict
556    @rtype: dict
557    @return properties: updated dict with default values for any missing values
558    '''
559   
560   
561    if sectionKey:
562        sectionKeyDot = sectionKey+'.'
563        log.debug("Checking for any unset keys for %s sub-section"%sectionKey)
564    else:
565        sectionKeyDot = ''
566        log.debug("Checking for any unset keys")
567       
568    for key in validKeys:
569        if key not in properties or not properties[key]:
570            if validKeys[key] == NotImplemented:
571                errorMessage = 'Missing property "%s" must be set.' % key
572                log.error(errorMessage)
573                raise ValueError(errorMessage)
574           
575            log.warning("Found missing/unset property - setting default "
576                        "values: %s%s=%s" % (sectionKeyDot,key,validKeys[key]))
577            properties[key] = validKeys[key]
578           
579        elif isinstance(properties[key], dict):
580            _setDefaultValues(properties[key], validKeys[key], sectionKey=key)
581       
582    log.debug("Finished checking for unset keys")
583
Note: See TracBrowser for help on using the repository browser.