Changeset 4134


Ignore:
Timestamp:
26/08/08 16:49:13 (12 years ago)
Author:
cbyrom
Message:

Extend config parser utilities - to cleanly separate the input and
validation - and setting of default values - of properties. Also,
allow separate property sections to be stored in child dictionaries +
properly traverse elementtree docs to retrieve all data + fix a few
issues along the way.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • TI12-security/trunk/python/ndg.security.common/ndg/security/common/utils/ConfigFileParsers.py

    r4131 r4134  
    1313 
    1414from ConfigParser import SafeConfigParser 
    15  
     15from ndg.security.common.wssecurity import WSSecurityConfig 
    1616# For parsing of properties file 
    1717try: # python 2.5 
    … …  
    2424log = logging.getLogger(__name__) 
    2525 
     26# lambda function to expand out any environment variables in properties read in 
     27expandEnvVars = lambda x: isinstance(x, basestring) and os.path.expandvars(x).strip() or x 
     28 
     29# since they are used across services, define the valid keys for the WS-Security section 
     30# here for easy access 
     31__wsSecurityValidKeys = { 
     32        'certFile':               None, 
     33        'keyFile':                None, 
     34        'keyPwd':                 None, 
     35        'clntCertFile':           None, 
     36        'caCertFileList':         [], 
     37        'wssRefInclNS':           [], 
     38        'wssSignedInfoInclNS':    [], 
     39        'reqBinSecTokValType':    'X509v3', 
     40        'applySignatureConfirmation': True, 
     41        'useSignatureHandler':    True 
     42        } 
    2643 
    2744class CaseSensitiveConfigParser(SafeConfigParser): 
    … …  
    3451        return optionstr 
    3552 
    36  
    37 def readProperties(propFilePath, validKeys={}, sections=[]): 
     53def readAndValidateProperties(propFilePath, validKeys={}): 
    3854    """ 
    3955    Determine the type of properties file and load the contents appropriately. 
     56    If a dict of valid keys is also specified, check the loaded properties against these. 
    4057    @param propFilePath: file path to properties file - either in xml or ini format 
    4158    @type propFilePath: string 
    … …  
    4360    - if values are encountered that are not in this list, an exception will be thrown 
    4461    - if all info should be read, this keyword should be left to its default value 
    45     @type validKeys: dict 
    46     @keyword sections: a list of sections to include data from - if reading from an 'ini' type file 
    47     @type sections: list  
     62    - NB, this dict will also ensure list data is read in correctely 
     63    @type validKeys: dict 
     64    @raise ValueError: if a key is read in from the file that is not included in the 
     65    specified validKeys dict 
     66    """ 
     67    log.debug("Reading properties from %s" %propFilePath) 
     68    properties = {} 
     69    if propFilePath.lower().endswith('xml'): 
     70        log.debug("File has 'xml' suffix - treating as standard XML formatted properties file") 
     71        properties = readXMLPropertyFile(propFilePath, validKeys) 
     72    else: 
     73        properties = readINIPropertyFile(propFilePath, validKeys) 
     74 
     75    # if validKeys set, check that all loaded property values are featured in  
     76    # this list 
     77    if validKeys: 
     78        validateProperties(properties, validKeys) 
     79     
     80        # lastly set any default values from the validKeys dict for vals not read in 
     81        # from the property file 
     82        _setDefaultValues(properties, validKeys) 
     83     
     84    # lastly, expand out any environment variables set in the properties file 
     85    properties = _expandEnvironmentVariables(properties) 
     86    log.info('Properties loaded') 
     87    return properties 
     88 
     89 
     90def readProperties(propFilePath, validKeys={}): 
     91    """ 
     92    Determine the type of properties file and load the contents appropriately. 
     93    @param propFilePath: file path to properties file - either in xml or ini format 
     94    @type propFilePath: string 
     95    """ 
     96    log.debug("Reading properties from %s" %propFilePath) 
     97    properties = {} 
     98    if propFilePath.lower().endswith('xml'): 
     99        log.debug("File has 'xml' suffix - treating as standard XML formatted properties file") 
     100        properties = readXMLPropertyFile(propFilePath, validKeys) 
     101    else: 
     102        properties = readINIPropertyFile(propFilePath, validKeys) 
     103     
     104    # lastly, expand out any environment variables set in the properties file 
     105    properties = _expandEnvironmentVariables(properties) 
     106    log.info('Properties loaded') 
     107    return properties 
     108         
     109 
     110def readINIPropertyFile(propFilePath, validKeys): 
     111    """ 
     112    Read 'ini' type property file - i.e. a flat text file with key/value 
     113    data separated into sections 
     114 
     115    @param propFilePath: file path to properties file - either in xml or ini format 
     116    @type propFilePath: string 
     117    @param validKeys: a dictionary of valid values to be read from the file 
     118    - if values are encountered that are not in this list, an exception will be thrown 
     119    - if all info should be read, set this param to 'None' 
     120    @type validKeys: dict 
    48121    @return: dict with the loaded properties in 
    49122    @raise ValueError: if a key is read in from the file that is not included in the 
    50123    specified validKeys dict 
    51124    """ 
    52     log.debug("Reading properties from %s" %propFilePath) 
    53     properties = {} 
    54     if propFilePath.lower().endswith('xml'): 
    55         properties = readXMLPropertyFile(propFilePath, validKeys) 
     125    log.debug("File is not marked as XML - treating as flat, 'ini' format file") 
     126     
     127    if not os.path.isfile(propFilePath): 
     128        raise ValueError("Error parsing properties file \"%s\": No such file" %propFilePath) 
     129 
     130    cfg = CaseSensitiveConfigParser() 
     131    cfg.read(propFilePath) 
     132    properties = {} 
     133     
     134    # NB, add 'DEFAULT' section since this isn't returned by the 'sections()' 
     135    sections = cfg.sections() 
     136    sections.append('DEFAULT') 
     137     
     138    # parse data from the specified sections of the config file 
     139    for section in sections: 
     140        if section == 'DEFAULT': 
     141            properties.update(_parseConfig(cfg, validKeys, section=section)) 
     142        else: 
     143            keys = validKeys 
     144            if section == 'WS-Security': 
     145                keys = __wsSecurityValidKeys #WSSecurityConfig.defParam 
     146            properties[section] = _parseConfig(cfg, keys, section=section) 
     147 
     148    log.debug("Finished reading from INI properties file") 
     149    return properties 
     150     
     151 
     152def _parseConfig(cfg, validKeys, section='DEFAULT'): 
     153    ''' 
     154    Extract parameters from cfg config object 
     155    @param cfg: config object 
     156    @type cfg: CaseSensitiveConfigParser 
     157    @param validKeys: a dictionary of valid values to be read from the file - used to check the 
     158    type of the input parameter to ensure (lists) are handled correctly 
     159    @type validKeys: dict 
     160    @keyword section: section of config file to parse from 
     161    @type section: string 
     162    @return: dict with the loaded properties in 
     163    ''' 
     164    log.debug("Parsing section: %s" %section) 
     165    keys = {} 
     166    properties = {} 
     167    if section == 'DEFAULT': 
     168        keys = cfg.defaults().keys() 
    56169    else: 
    57         properties = readINIPropertyFile(propFilePath, validKeys, sections) 
    58  
    59     # if validKeys set, check that all loaded property values are featured in  
    60     # this list 
    61     if validKeys: 
    62         _checkForInvalidElements(properties, validKeys) 
    63      
    64         # lastly set any default values from the validKeys dict for vals not read in 
    65         # from the property file 
    66         _setDefaultValues(properties, validKeys) 
    67      
    68     # lastly, expand out any environment variables set in the properties file 
    69     properties = _expandEnvironmentVariables(properties) 
    70     log.info('Properties loaded') 
    71     return properties 
     170        keys = cfg.options(section) 
     171        # NB, we need to be careful here - since this will return the section keywords 
     172        # AND the 'DEFAULT' section entries - so use the difference between the two 
     173        keys = filter(lambda x:x not in cfg.defaults().keys(), keys) 
     174 
     175    for key in keys: 
     176        val = cfg.get(section, key) 
     177        if val: 
     178             
     179            # expand out any env vars 
     180            val = expandEnvVars(val) 
     181             
     182            # if the tag contains an integer, convert this appropriately 
     183            if val.isdigit(): 
     184                val = int(val) 
     185             
     186            # if the value is in the validKeys dict, ensure it is read in as the correct type 
     187            if key in validKeys: 
     188                if isinstance(validKeys[key], list): 
     189                    # Treat as a list of space separated elements 
     190                    val = val.split() 
     191        else: 
     192            # NB, the XML parser will return empty vals as None, so ensure consistency here 
     193            val = None 
     194 
     195        # check if key already exists; if so, append to list 
     196        if properties.has_key(key): 
     197            properties[key] = __listify(properties[key]).extend(val) 
     198        else: 
     199            properties[key] = val 
     200 
     201    log.debug("Finished parsing section") 
     202    return properties 
     203 
     204 
     205def readXMLPropertyFile(propFilePath, validKeys, rootElem=None): 
     206    """ 
     207    Read property file - assuming the standard XML schema 
     208 
     209    @param propFilePath: file path to properties file - either in xml or ini format 
     210    @type propFilePath: string 
     211    @param validKeys: a dictionary of valid values to be read from the file - used to check the 
     212    type of the input parameter to ensure (lists) are handled correctly 
     213    @keyword rootElem: a particular element of an ElementTree can be passed in to 
     214    use as the root element; NB, if this is set, it will take precedence over any 
     215    propFilePath specified 
     216    @type rootElem: ElementTree.Element 
     217    @return: dict with the loaded properties in 
     218    """ 
     219    if rootElem is None: 
     220        try: 
     221            tree = ElementTree.parse(propFilePath) 
     222             
     223        except IOError, ioErr: 
     224            raise ValueError, "Error parsing properties file \"%s\": %s" % \ 
     225                                (ioErr.filename, ioErr.strerror) 
     226     
     227        rootElem = tree.getroot() 
     228        if rootElem is None: 
     229            raise ValueError, \ 
     230                "Parsing properties file \"%s\": root element is not defined" % \ 
     231                propFilePath 
     232 
     233    properties = {} 
     234    # Copy properties from file into a dictionary 
     235    try: 
     236        for elem in rootElem: 
     237            key = elem.tag 
     238            val = elem.text 
     239 
     240            # expand out any env vars 
     241            val = expandEnvVars(val) 
     242 
     243            # if the tag contains an integer, convert this appropriately 
     244            if val and val.isdigit(): 
     245                val = int(val) 
     246             
     247            # check for lists - don't recurse into these else the key names 
     248            # will end up being wrong 
     249            if key in validKeys and isinstance(validKeys[key], list): 
     250                # handle lists of elements 
     251                if len(elem) == 0: 
     252                    if elem.text is not None: 
     253                        # Treat as a list of space separated elements 
     254                        val = val.split() 
     255                else: 
     256                    # Parse from a list of sub-elements 
     257                    val = [expandEnvVars(subElem.text.strip()) for subElem in elem] 
     258             
     259            # otherwise check for subelements; if these exist, recurse and store properties 
     260            # in an inner dictionary 
     261            elif len(elem) > 0: 
     262                keys = validKeys 
     263                if key == 'WS-Security': 
     264                    keys = __wsSecurityValidKeys #WSSecurityConfig.defParam 
     265                val = readXMLPropertyFile(propFilePath, keys, rootElem=elem) 
     266 
     267            # check if key already exists; if so, append to list 
     268            if properties.has_key(key): 
     269                properties[key] = __listify(properties[key]).extend(val) 
     270            else: 
     271                properties[key] = val 
     272            #print "XML: Key: %s, Val: %s" %(key, properties[key]) 
     273             
     274    except Exception, e: 
     275        raise ValueError, \ 
     276            "Error parsing tag \"%s\" in properties file \"%s\": %s" % \ 
     277            (elem.tag, propFilePath, e) 
     278 
     279    log.debug("Finished reading from XML properties file") 
     280    return properties 
     281 
     282 
     283def __listify(val): 
     284    ''' 
     285    Checks if val is a list; if so return as is, if not return as list 
     286    @param val: object to turn into a list 
     287    @return: val as a list (if it is not already) 
     288    ''' 
     289    if isinstance(val, list): 
     290        return val 
     291    return [val] 
     292 
     293 
     294def validateProperties(properties, validKeys): 
     295    ''' 
     296    Check the contents of the properties dict to ensure it doesn't contain 
     297    any keys not featured in the validKeys dict; if it does, throw an exception 
     298    @param properties: dictionary storing loaded properties 
     299    @type properties: dict 
     300    @param validKeys: a dictionary of valid values 
     301    @type validKeys: dict 
     302    @raise ValueError: if a key is read in from the file that is not included in the 
     303    specified validKeys dict 
     304    ''' 
     305    log.debug("Checking for invalid properties") 
     306    invalidKeys = [] 
     307    for key in properties: 
     308        # NB, this is a standard property used across most services - so check 
     309        # using the properties listed here 
     310        if key == 'WS-Security': 
     311            validateProperties(properties[key], __wsSecurityValidKeys) 
     312        else: 
     313            if key not in validKeys: 
     314                invalidKeys.append(key) 
     315 
     316    if invalidKeys != []: 
     317        errorMessage = "The following properties file " + \ 
     318            "elements are invalid: " + ', '.join(invalidKeys) 
     319        print errorMessage 
     320        log.error(errorMessage) 
     321        raise ValueError, errorMessage 
    72322 
    73323 
    … …  
    110360        val = os.path.expandvars(val) 
    111361    return val 
    112          
    113  
    114 def readINIPropertyFile(propFilePath, validKeys, sections=[]): 
    115     """ 
    116     Read 'ini' type property file - i.e. a flat text file with key/value 
    117     data separated into sections 
    118  
    119     @param propFilePath: file path to properties file - either in xml or ini format 
    120     @type propFilePath: string 
    121     @param validKeys: a dictionary of valid values to be read from the file 
    122     - if values are encountered that are not in this list, an exception will be thrown 
    123     - if all info should be read, set this param to 'None' 
    124     @type validKeys: dict 
    125     @keyword sections: a list of sections to include data from 
    126     @type sections: list  
    127     @return: dict with the loaded properties in 
    128     @raise ValueError: if a key is read in from the file that is not included in the 
    129     specified validKeys dict 
    130     """ 
    131     log.debug("File is not marked as XML - treating as flat, 'ini' format file") 
    132      
    133     if not os.path.isfile(propFilePath): 
    134         raise ValueError("Error parsing properties file \"%s\": No such file" %propFilePath) 
    135  
    136     cfg = CaseSensitiveConfigParser() 
    137     cfg.read(propFilePath) 
    138     properties = {} 
    139      
    140     # if no sections are set, use all sections - including the default one 
    141     # - NB, this isn't returned by the 'sections()' method so needs to be added explicitly 
    142     if not sections: 
    143         sections = cfg.sections() 
    144         sections.append('DEFAULT') 
    145      
    146     # parse data from the specified sections of the config file 
    147     if len(sections): 
    148         for section in sections: 
    149             properties = _parseConfig(cfg, properties, validKeys, section=section) 
    150  
    151     log.debug("Finished reading from INI properties file") 
    152     return properties 
    153  
    154  
    155 def _checkForInvalidElements(properties, validKeys): 
    156     ''' 
    157     Check the contents of the properties dict to ensure it doesn't contain 
    158     any keys not featured in the validKeys dict; if it does, throw an exception 
    159     @param properties: dictionary storing loaded properties 
    160     @type properties: dict 
    161     @param validKeys: a dictionary of valid values 
    162     @type validKeys: dict 
    163     @raise ValueError: if a key is read in from the file that is not included in the 
    164     specified validKeys dict 
    165     ''' 
    166     log.debug("Checking for invalid properties") 
    167     invalidKeys = [] 
    168     for key in properties: 
    169         if key not in validKeys: 
    170             invalidKeys.append(key) 
    171  
    172     if invalidKeys != []: 
    173         errorMessage = "The following properties file " + \ 
    174             "elements are invalid: " + ', '.join(invalidKeys) 
    175         log.error(errorMessage) 
    176         raise ValueError, errorMessage 
    177362 
    178363     
    … …  
    195380            properties[key] = validKeys[key] 
    196381    log.debug("Finished checking for unset keys") 
    197      
    198  
    199 def _parseConfig(cfg, properties, validKeys, section='DEFAULT'): 
    200     ''' 
    201     Extract parameters from cfg config object 
    202     @param cfg: config object 
    203     @type cfg: CaseSensitiveConfigParser 
    204     @param properties: dictionary to store properties in 
    205     @type properties: dict 
    206     @param validKeys: a dictionary of valid values to be read from the file 
    207     - if values are encountered that are not in this list, an exception will be thrown 
    208     - if all info should be read, set this param to 'None' 
    209     @type validKeys: dict 
    210     @keyword section: section of config file to parse from 
    211     @type section: string 
    212     @return: dict with the loaded properties in 
    213     ''' 
    214     log.debug("Parsing section: %s" %section) 
    215     keys = {} 
    216     if section == 'DEFAULT': 
    217         keys = cfg.defaults().keys() 
    218     else: 
    219         keys = cfg.options(section) 
    220          
    221     for key in keys: 
    222         val = cfg.get(section, key) 
    223          
    224         # NB, the XML parser will return empty vals as None, so ensure consistency here 
    225         if not val: 
    226             properties[key] = None 
    227             continue 
    228          
    229         # if the value is in the validKeys dict, ensure it is read in as the correct type 
    230         if key in validKeys: 
    231             if isinstance(validKeys[key], list): 
    232                 # Treat as a list of space separated elements 
    233                 properties[key] = val.split() 
    234             else: 
    235                 properties[key] = val 
    236         else: 
    237             properties[key] = val 
    238                     
    239     log.debug("Finished parsing section") 
    240     return properties 
    241  
    242  
    243 def readXMLPropertyFile(propFilePath, validKeys): 
    244     """ 
    245     Read property file - assuming the standard XML schema 
    246  
    247     @param propFilePath: file path to properties file - either in xml or ini format 
    248     @type propFilePath: string 
    249     @param validKeys: a dictionary of valid values to be read from the file 
    250     - if values are encountered that are not in this list, an exception will be thrown 
    251     - if all info should be read, set this param to 'None' 
    252     @type validKeys: dict 
    253     @return: dict with the loaded properties in 
    254     """ 
    255     log.debug("File has 'xml' suffix - treating as standard XML formatted properties file") 
    256     try: 
    257         tree = ElementTree.parse(propFilePath) 
    258          
    259     except IOError, ioErr: 
    260         raise ValueError, "Error parsing properties file \"%s\": %s" % \ 
    261                             (ioErr.filename, ioErr.strerror) 
    262  
    263     rootElem = tree.getroot() 
    264     if rootElem is None: 
    265         raise ValueError, \ 
    266             "Parsing properties file \"%s\": root element is not defined" % \ 
    267             propFilePath 
    268  
    269     # Copy properties from file into a dictionary 
    270     properties = {} 
    271     try: 
    272         for elem in rootElem: 
    273             # if the value is in the validKeys dict, ensure it is read in as the correct type 
    274             if elem.tag in validKeys: 
    275                 if isinstance(validKeys[elem.tag], list): 
    276                     if len(elem) == 0: 
    277                         if elem.text is not None: 
    278                             # Treat as a list of space separated elements 
    279                             properties[elem.tag] = elem.text.split() 
    280                         else: 
    281                             properties[elem.tag] = [] 
    282                     else: 
    283                         # Parse from a list of sub-elements 
    284                         properties[elem.tag] = \ 
    285                             [subElem.text.strip() for subElem in elem] 
    286                 else: 
    287                     properties[elem.tag] = elem.text 
    288             else: 
    289                 properties[elem.tag] = elem.text 
    290              
    291     except Exception, e: 
    292         raise ValueError, \ 
    293             "Error parsing tag \"%s\" in properties file \"%s\": %s" % \ 
    294             (elem.tag, propFilePath, e) 
    295  
    296     log.debug("Finished reading from XML properties file") 
    297     return properties 
     382 
Note: See TracChangeset for help on using the changeset viewer.