Source code for pcvs.helpers.system

import os

import addict
import jsonschema
from ruamel.yaml import YAML, YAMLError

import pcvs
from pcvs import NAME_BUILDIR, PATH_INSTDIR
from pcvs.io import Verbosity
from pcvs.helpers import git, pm
from pcvs.helpers.exceptions import CommonException, ValidationException


####################################
####   YAML VALIDATION OBJECT   ####
####################q################
[docs]class ValidationScheme: """Object manipulating schemes (yaml) to enforce data formats. A validationScheme is instancied according to a 'model' (the format to validate). This instance can be used multiple times to check multiple streams belonging to the same model. """ avail_list = None
[docs] @classmethod def available_schemes(cls): """return list of currently supported formats to be validated. The list is extracted from INSTALL/schemes/<model>-scheme.yml """ if cls.avail_list is None: cls.avail_list = list() for f in os.listdir(os.path.join(PATH_INSTDIR, 'schemes/')): cls.avail_list.append(f.replace('-scheme.yml', '')) return cls.avail_list
def __init__(self, name): """Create a new ValidationScheme instancen based on a given model. During initialisatio, the file scheme is loaded from disk. :raises: ValidationException.SchemeError: file is not found OR unable to load the YAML scheme file. """ self._name = name try: with open(os.path.join( PATH_INSTDIR, 'schemes/{}-scheme.yml'.format(name)), 'r') as fh: self._scheme = YAML(typ='safe').load(fh) except (IOError, YAMLError): raise ValidationException.SchemeError( "Unable to load scheme {}".format(name))
[docs] def validate(self, content, filepath=None): """Validate a given datastructure (dict) agasint the loaded scheme. :param content: json to validate :type content: dict :param filepath: :raises ValidationException.FormatError: data are not valid :raises ValidationException.SchemeError: issue while applying scheme """ try: if filepath is None: filepath = "'data stream'" jsonschema.validate(instance=content, schema=self._scheme) except jsonschema.exceptions.ValidationError as e: raise ValidationException.FormatError( reason="Failed to validate input file: {}".format(e.message), file=filepath, scheme=self._scheme) except jsonschema.exceptions.SchemaError as e: raise ValidationException.SchemeError( name=self._name, content=self._scheme, error=e)
[docs]class MetaDict(addict.Dict): """Helps with managing large configuration sets, based on dictionaries. Once instanciated, an arbitraty subnode can be initialized like: o = MetaDict() o.field_a.subnode2 = 4 Currently, this class is just derived from addict.Dict. It is planned to remove this adherence. """
[docs] def to_dict(self): """Convert the object to a regular dict. :return: a regular Python dict :rtype: Dict """ return super().to_dict()
[docs]class Config(MetaDict): """a 'Config' is a dict extension (an MetaDict), used to manage all configuration fields. While it can contain arbitrary data, the whole PCVS configuration is composed of 5 distinct 'categories', each being a single Config. These are then gathered in a `MetaConfig` object (see below) """ def __init__(self, d={}, *args, **kwargs): """Init the object and propagate properly the init to MetaDict() object :param d: items of the configuration :type d: dict :param *arg: items of the configuration :type *arg: tuple :param **kwargs: items of the configuration :type **kwargs: dict""" super().__init__(*args, **kwargs) for n in d: self.__setitem__(n, d[n])
[docs] def validate(self, kw): """Check if the Config instance matches the expected format as declared in schemes/. As the 'category' is not carried by the object itself, it is provided by the function argument. :param kw: keyword describing the configuration to be validated (scheme) :type kw: str """ assert (kw in ValidationScheme.available_schemes()) ValidationScheme(kw).validate(self)
def __setitem__(self, param, value): """extend it to handle dict initialization (needs MetaDict conversion) :param param: name of value to add to configuration :type param: str :param value: value to add to configuration :type value: object""" if isinstance(value, dict): value = MetaDict(value) super().__setitem__(param, value)
[docs] def set_ifdef(self, k, v): """shortcut function: init self[k] only if v is not None :param k: name of value to add :type k: str :param v: value to add :type v: str""" if v is not None: self[k] = v
[docs] def set_nosquash(self, k, v): """shortcut function: init self[k] only if v is not already set :param k: name of value to add :type k: str :param v: value to add :type v: str""" if k not in self: self[k] = v
[docs] def to_dict(self): """Convert the Config() to regular dict.""" # this dirty hack is due to a type(self) used into addict.py # leading to misconvert derived classes from Dict() # --> to_dict() checks if a sub-value is instance of type(self) # In our case here, type(self) return Config(), addict not converting # sub-Dict() into dict(). This double call to to_dict() seems # to fix the issue. But an alternative to addict should be used # (customly handled ?) copy = MetaDict(super().to_dict()) return copy.to_dict()
[docs] def from_dict(self, d): """Fill the current Config from a given dict :param d: dictionary to add :type d: dict""" for k, v in d.items(): self[k] = v
[docs] def from_file(self, filename): """Fill the current config from a given file :raises CommonException.IOError: file does not exist OR badly formatted """ try: with open(filename, 'r') as fh: d = YAML(typ='safe').load(fh) self.from_dict(d) except (IOError, YAMLError) as e: raise CommonException.IOError( "{} invalid or badly formatted".format(filename))
[docs]class MetaConfig(MetaDict): """Root configuration object. It is composed of Config(), categorizing each configuration blocks. This MetaConfig() contains the whole profile along with any validation and current run information. This configuration is used as a dict extension. To avoid carrying a global instancied object over the whole code, a class-scoped attribute allows to browse the global configuration from anywhere through `Metaconfig.root`" """ root = None validation_default_file = pcvs.PATH_VALCFG def __init__(self, *args, **kwargs): """constructor method. :param args: list of positional arguments :type args: tuple :param kwargs: list of keyword-based arguments """ super().__init__(*args, **kwargs) # The 'internal' node is a special one. Put here anything not requiring # to be published (like conf.yml, etc...). mainly one-time Python # objects #self.__internal_config = Config() def __setitem__(self, param, value): """Extend the default MetaDict setter mthod to reach the base class one""" super().__setitem__(param, value)
[docs] def bootstrap_generic(self, subnode, node): """"Initialize a Config() object and store it under name 'node' :param subnode: node name :type subnode: str :param node: node to initialize and add :type node: dict :return: added subnode :rtype: dict""" if subnode not in self: self[subnode] = Config() for k, v in node.items(): self[subnode][k] = v self[subnode].validate(subnode) return self[subnode]
[docs] def bootstrap_from_profile(self, pf_as_dict): if not isinstance(pf_as_dict, MetaDict): pf_as_dict = MetaDict(pf_as_dict) self.bootstrap_compiler(pf_as_dict.compiler) self.bootstrap_runtime(pf_as_dict.runtime) self.bootstrap_machine(pf_as_dict.machine) self.bootstrap_criterion(pf_as_dict.criterion) self.bootstrap_group(pf_as_dict.group)
[docs] def bootstrap_compiler(self, node): """"Specific initialize for compiler config block :param node: compiler block to initialize :type node: dict :return: added node :rtype: dict""" subtree = self.bootstrap_generic('compiler', node) if 'package_manager' in subtree: self.set_internal('cc_pm', pm.identify(subtree.package_manager)) return subtree
[docs] def bootstrap_runtime(self, node): """"Specific initialize for runtime config block :param node: runtime block to initialize :type node: dict :return: added node :rtype: dict""" subtree = self.bootstrap_generic('runtime', node) if 'package_manager' in subtree: self.set_internal('rt_pm', pm.identify(subtree.package_manager)) return subtree
[docs] def bootstrap_group(self, node): """"Specific initialize for group config block. There is currently nothing to here but calling bootstrap_generic() :param node: runtime block to initialize :type node: dict :return: added node :rtype: dict """ return self.bootstrap_generic('group', node)
[docs] def bootstrap_validation_from_file(self, filepath): """Specific initialize for validation config block. This function loads a file containing the validation dict. :param filepath: path to file to be validated :type filepath: os.path, str :raises CommonException.IOError: file is not found or badly formatted """ node = MetaDict() if filepath is None: filepath = self.validation_default_file if os.path.isfile(filepath): try: with open(filepath, 'r') as fh: node = MetaDict(YAML(typ='safe').load(fh)) except (IOError, YAMLError) as e: raise CommonException.IOError( "Error(s) found while loading (}".format(filepath)) # some post-actions for field in ["output", "reused_build"]: if field in node: node[field] = os.path.abspath(node[field]) if node.dirs: node.dirs = {k: os.path.abspath(v) for k, v in node.dirs.items()} return self.bootstrap_validation(node)
[docs] def bootstrap_subnode(root_node, *default_subnode_dict): for k, v in default_subnode_dict: if k not in root_node: root_node[k] = v
[docs] def bootstrap_validation(self, node): """"Specific initialize for validation config block :param node: validation block to initialize :type node: dict :return: initialized node :rtype: dict""" subtree = self.bootstrap_generic('validation', node) # Initialize default values when not set by user or default files subtree.set_nosquash('verbose', str(Verbosity.COMPACT)) subtree.set_nosquash('print_level', 'none') subtree.set_nosquash('color', True) subtree.set_nosquash('default_profile', 'default') subtree.set_nosquash('output', os.path.join(os.getcwd(), NAME_BUILDIR)) subtree.set_nosquash('background', False) subtree.set_nosquash('override', False) subtree.set_nosquash('dirs', None) subtree.set_nosquash("spack_recipe", None) subtree.set_nosquash('simulated', False) subtree.set_nosquash('anonymize', False) subtree.set_nosquash('onlygen', False) subtree.set_nosquash('timeout', None) subtree.set_nosquash('target_bank', None) subtree.set_nosquash('reused_build', None) subtree.set_nosquash('webreport', None) subtree.set_nosquash("only_success", False) subtree.set_nosquash("enable_report", False) subtree.set_nosquash('job_timeout', 86400) subtree.set_nosquash('per_result_file_sz', 10 * 1024 * 1024) subtree.set_nosquash( 'buildcache', os.path.join(subtree.output, 'cache')) subtree.set_nosquash('result', {"format": ['json']}) subtree.set_nosquash('author', { "name": git.get_current_username(), "email": git.get_current_usermail()}) # Annoying here: # self.result should be allowed even without the 'set_nosquash' above # but because of inheritance, it does not result as a MetaDict() # As the 'set_nosquash' is required, it is solving the issue # but this corner-case should be remembered as it WILL happen again :( if 'format' not in subtree.result: subtree.result.format = ['json'] if 'log' not in subtree.result: subtree.result.log = 1 if 'logsz' not in subtree.result: subtree.result.logsz = 1024 return subtree
[docs] def bootstrap_machine(self, node): """"Specific initialize for machine config block :param node: machine block to initialize :type node: dict :return: initialized node :rtype: dict""" subtree = self.bootstrap_generic('machine', node) subtree.set_nosquash('name', 'default') subtree.set_nosquash('nodes', 1) subtree.set_nosquash('cores_per_node', 1) subtree.set_nosquash('concurrent_run', 1) if 'default_partition' not in subtree or 'partitions' not in subtree: return # override default values by selected partition for elt in subtree.partitions: if elt.get('name', subtree.default_partition) == subtree.default_partition: subtree.update(elt) break # redirect to direct programs if no wrapper is defined for kind in ['allocate', 'run', 'batch']: if not subtree.job_manager[kind].wrapper and subtree.job_manager[kind].program: subtree.job_manager[kind].wrapper = subtree.job_manager[kind].program return subtree
[docs] def bootstrap_criterion(self, node): """"Specific initialize for criterion config block :param node: criterion block to initialize :type node: dict :return: initialized node :rtype: dict""" return self.bootstrap_generic('criterion', node)
[docs] def set_internal(self, k, v): """manipulate the internal MetaConfig() node to store not-exportable data :param k: name of value to add :type k: str :param v: value to add :type v: str""" if not hasattr(self, "__internal_config"): self.__internal_config = {} self.__internal_config[k] = v
[docs] def get_internal(self, k): """manipulate the internal MetaConfig() node to load not-exportable data :param k: value to get :type k: str""" if not hasattr(self, "__internal_config"): return None if k in self.__internal_config: return self.__internal_config[k] else: return None
[docs] def dump_for_export(self): """Export the whole configuration as a dict. Prune any __internal node beforehand. """ if self.__internal_config: not_exported = self.__internal_config del self.__internal_config res = MetaDict(self.to_dict()) self.__internal_config = not_exported else: res = self return res.to_dict()