diff --git a/.appveyor.yml b/.appveyor.yml index ed692231..a403001d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -23,7 +23,7 @@ install: - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" # Download miktex portable (if not cached) - - ps: "If (!(Test-Path miktex-portable.exe)){wget http://mirrors.ctan.org/systems/win32/miktex/setup/miktex-portable-2.9.5719.exe -OutFile ./miktex-portable.exe}" + - ps: "If (!(Test-Path miktex-portable.exe)){wget http://mirrors.ctan.org/systems/win32/miktex/setup/miktex-portable-2.9.5857.exe -OutFile ./miktex-portable.exe}" # Unzip miktex portable - "7z x miktex-portable.exe * -aot -omiktex > nul" @@ -32,8 +32,8 @@ install: - cmd: set PATH=%PATH%;C:\projects\patacrep\miktex\miktex\bin # Update some packages to prevent ltluatex bug - - cmd: mpm.exe --update=miktex-bin-2.9 - - cmd: mpm.exe --update=ltxbase --update=luatexbase --update=luaotfload --update=miktex-luatex-base --update=fontspec + # - cmd: mpm.exe --update=miktex-bin-2.9 --verbose + # - cmd: mpm.exe --update=ltxbase --update=luatexbase --update=luaotfload --update=miktex-luatex-base --update=fontspec # Manually install required texlive packages - cmd: mpm.exe --install-some texlive_packages.txt diff --git a/examples/example-all.sb b/examples/example-all.sb index 8f5d80a6..8c0639d4 100644 --- a/examples/example-all.sb +++ b/examples/example-all.sb @@ -14,6 +14,6 @@ "sep" : ["and", "et"] }, "datadir" : ".", - "content": [["sorted"]] + "content": [["sort"]] } diff --git a/examples/example-all.yaml.sb b/examples/example-all.yaml.sb index 01a95015..814e8875 100644 --- a/examples/example-all.yaml.sb +++ b/examples/example-all.yaml.sb @@ -13,7 +13,7 @@ authors: separators: - "and" - "et" -content: [["sorted"]] +content: {"sort"} template: patacrep.tex: @@ -23,4 +23,4 @@ template: bgcolor: note: D1E4AE songnumber: AED1E4 - index: E4AED1 #not enough songs to see it \ No newline at end of file + index: E4AED1 #not enough songs to see it diff --git a/patacrep/Rx.py b/patacrep/Rx.py index 677a3461..a388f039 100644 --- a/patacrep/Rx.py +++ b/patacrep/Rx.py @@ -9,85 +9,242 @@ import re import types from numbers import Number - -core_types = [ ] +### Exception Classes -------------------------------------------------------- class SchemaError(Exception): pass class SchemaMismatch(Exception): - pass -class SchemaTypeMismatch(SchemaMismatch): - def __init__(self, name, desired_type): - SchemaMismatch.__init__(self, '{0} must be {1}'.format(name, desired_type)) + def __init__(self, message, schema, error=None): + Exception.__init__(self, message) + self.type = schema.subname() + self.error = error + +class TypeMismatch(SchemaMismatch): + + def __init__(self, schema, data): + message = 'must be of type {} (was {})'.format( + schema.subname(), + type(data).__name__ + ) -class SchemaValueMismatch(SchemaMismatch): - def __init__(self, name, value): - SchemaMismatch.__init__(self, '{0} must equal {1}'.format(name, value)) + SchemaMismatch.__init__(self, message, schema, 'type') + self.expected_type = schema.subname() + self.value = type(data).__name__ -class SchemaRangeMismatch(SchemaMismatch): - pass -def indent(text, level=1, whitespace=' '): - return '\n'.join(whitespace*level+line for line in text.split('\n')) +class ValueMismatch(SchemaMismatch): + + def __init__(self, schema, data): -class Util(object): - @staticmethod - def make_range_check(opt): - - if not {'min', 'max', 'min-ex', 'max-ex'}.issuperset(opt): - raise ValueError("illegal argument to make_range_check") - if {'min', 'min-ex'}.issubset(opt): - raise ValueError("Cannot define both exclusive and inclusive min") - if {'max', 'max-ex'}.issubset(opt): - raise ValueError("Cannot define both exclusive and inclusive max") - - r = opt.copy() - inf = float('inf') - - def check_range(value): - return( - r.get('min', -inf) <= value and \ - r.get('max', inf) >= value and \ - r.get('min-ex', -inf) < value and \ - r.get('max-ex', inf) > value + message = 'must equal {} (was {})'.format( + repr(schema.value), + repr(data) + ) + + SchemaMismatch.__init__(self, message, schema, 'value') + self.expected_value = schema.value + self.value = data + + + +class RangeMismatch(SchemaMismatch): + + def __init__(self, schema, data): + + message = 'must be in range {} (was {})'.format( + schema.range, + data + ) + + SchemaMismatch.__init__(self, message, schema, 'range') + self.range = schema.range + self.value = data + + +class LengthRangeMismatch(SchemaMismatch): + + def __init__(self, schema, data): + length_range = Range(schema.length) + + if not hasattr(length_range, 'min') and \ + not hasattr(length_range, 'min_ex'): + length_range.min = 0 + + message = 'length must be in range {} (was {})'.format( + length_range, + len(data) + ) + + SchemaMismatch.__init__(self, message, schema, 'range') + self.range = schema.length + self.value = len(data) + + +class MissingFieldMismatch(SchemaMismatch): + + def __init__(self, schema, fields): + + if len(fields) == 1: + message = 'missing required field: {}'.format( + repr(fields[0]) ) + else: + message = 'missing required fields: {}'.format( + ', '.join(fields) + ) + if len(message) >= 80: # if the line is too long + message = 'missing required fields:\n{}'.format( + _indent('\n'.join(fields)) + ) - return check_range + SchemaMismatch.__init__(self, message, schema, 'missing') + self.fields = fields - @staticmethod - def make_range_validator(opt): - check_range = Util.make_range_check(opt) - - r = opt.copy() - nan = float('nan') - - def validate_range(value, name='value'): - if not check_range(value): - if r.get('min', nan) == r.get('max', nan): - msg = '{0} must equal {1}'.format(name, r['min']) - raise SchemaRangeMismatch(msg) - - range_str = '' - if 'min' in r: - range_str = '[{0}, '.format(r['min']) - elif 'min-ex' in r: - range_str = '({0}, '.format(r['min-ex']) - else: - range_str = '(-inf, ' - - if 'max' in r: - range_str += '{0}]'.format(r['max']) - elif 'max-ex' in r: - range_str += '{0})'.format(r['max-ex']) - else: - range_str += 'inf)' - - raise SchemaRangeMismatch(name+' must be in range '+range_str) - - return validate_range +class UnknownFieldMismatch(SchemaMismatch): + + def __init__(self, schema, fields): + + if len(fields) == 1: + message = 'unknown field: {}'.format( + repr(fields[0]) + ) + else: + message = 'unknown fields: {}'.format( + ', '.join(fields) + ) + if len(message) >= 80: # if the line is too long + message = 'unknown fields:\n{}'.format( + _indent('\n'.join(fields)) + ) + + SchemaMismatch.__init__(self, message, schema, 'unexpected') + self.fields = fields + + +class SeqLengthMismatch(SchemaMismatch): + def __init__(self, schema, data): + + expected_length = len(schema.content_schema) + message = 'sequence must have {} element{} (had {})'.format( + expected_length, + 's'*(expected_length != 1), # plural + len(data) + ) + + SchemaMismatch.__init__(self, message, schema, 'size') + self.expected_length = expected_length + self.value = len(data) + + +class TreeMismatch(SchemaMismatch): + + def __init__(self, schema, errors=[], child_errors={}, message=None): + + ## Create error message + + error_messages = [] + + for err in errors: + error_messages.append(str(err)) + + for key, err in child_errors.items(): + + if isinstance(key, int): + index = '[item {}]'.format(key) + else: + index = '{}'.format(repr(key)) + + if isinstance(err, TreeMismatch) and \ + not err.errors and len(err.child_errors) == 1: + + template = '{} > {}' + + else: + template = '{} {}' + + msg = template.format(index, err) + error_messages.append(msg) + + if message is None: + message = 'does not match schema' + + if len(error_messages) == 1: + msg = error_messages[0] + + else: + msg = '{}:\n{}'.format( + message, + _indent('\n'.join(error_messages)) + ) + + SchemaMismatch.__init__(self, msg, schema, 'multiple') + self.errors = errors + self.child_errors = child_errors + +def _createTreeMismatch(schema, errors=[], child_errors={}, message=None): + if len(errors) == 1 and not child_errors: + return errors[0] + else: + return TreeMismatch(schema, errors, child_errors, message) + +### Utilities ---------------------------------------------------------------- + +class Range(object): + + def __init__(self, opt): + if isinstance(opt, Range): + for attr in ('min', 'max', 'min_ex', 'max_ex'): + if hasattr(opt, attr): + setattr(self, attr, getattr(opt, attr)) + else: + if not {'min', 'max', 'min-ex', 'max-ex'}.issuperset(opt): + raise ValueError("illegal argument to make_range_check") + if {'min', 'min-ex'}.issubset(opt): + raise ValueError("Cannot define both exclusive and inclusive min") + if {'max', 'max-ex'}.issubset(opt): + raise ValueError("Cannot define both exclusive and inclusive max") + + for boundary in ('min', 'max', 'min-ex', 'max-ex'): + if boundary in opt: + attr = boundary.replace('-', '_') + setattr(self, attr, opt[boundary]) + + def __call__(self, value): + INF = float('inf') + + get = lambda attr, default: getattr(self, attr, default) + + return( + get('min', -INF) <= value and \ + get('max', INF) >= value and \ + get('min_ex', -INF) < value and \ + get('max_ex', INF) > value + ) + + def __str__(self): + if hasattr(self, 'min'): + s = '[{}, '.format(self.min) + elif hasattr(self, 'min_ex'): + s = '({}, '.format(self.min_ex) + else: + s = '(-Inf, ' + + if hasattr(self, 'max'): + s += '{}]'.format(self.max) + elif hasattr(self, 'max_ex'): + s += '{})'.format(self.max_ex) + else: + s += 'Inf)' + + return s + +def _indent(text, level=1, whitespace=' '): + return '\n'.join(whitespace*level+line for line in text.split('\n')) + + ### Schema Factory Class ----------------------------------------------------- class Factory(object): def __init__(self, register_core_types=True): @@ -109,20 +266,20 @@ class Factory(object): m = re.match('^/([-._a-z0-9]*)/([-._a-z0-9]+)$', type_name) if not m: - raise ValueError("couldn't understand type name '{0}'".format(type_name)) + raise ValueError("couldn't understand type name '{}'".format(type_name)) prefix, suffix = m.groups() if prefix not in self.prefix_registry: raise KeyError( - "unknown prefix '{0}' in type name '{1}'".format(prefix, type_name) + "unknown prefix '{0}' in type name '{}'".format(prefix, type_name) ) return self.prefix_registry[ prefix ] + suffix def add_prefix(self, name, base): if self.prefix_registry.get(name): - raise SchemaError("the prefix '{0}' is already registered".format(name)) + raise SchemaError("the prefix '{}' is already registered".format(name)) self.prefix_registry[name] = base; @@ -130,13 +287,15 @@ class Factory(object): t_uri = t.uri() if t_uri in self.type_registry: - raise ValueError("type already registered for {0}".format(t_uri)) + raise ValueError("type already registered for {}".format(t_uri)) self.type_registry[t_uri] = t def learn_type(self, uri, schema): if self.type_registry.get(uri): - raise SchemaError("tried to learn type for already-registered uri {0}".format(uri)) + raise SchemaError( + "tried to learn type for already-registered uri {}".format(uri) + ) # make sure schema is valid # should this be in a try/except? @@ -153,17 +312,27 @@ class Factory(object): uri = self.expand_uri(schema['type']) - if not self.type_registry.get(uri): raise SchemaError("unknown type {0}".format(uri)) + if not self.type_registry.get(uri): + raise SchemaError("unknown type {}".format(uri)) type_class = self.type_registry[uri] if isinstance(type_class, dict): if not {'type'}.issuperset(schema): - raise SchemaError('composed type does not take check arguments'); + raise SchemaError('composed type does not take check arguments') return self.make_schema(type_class['schema']) else: return type_class(schema, self) +std_factory = None +def make_schema(schema): + global std_factory + if std_factory is None: + std_factory = Factory() + return std_factory.make_schema(schema) + +### Core Type Base Class ------------------------------------------------- + class _CoreType(object): @classmethod def uri(self): @@ -171,7 +340,7 @@ class _CoreType(object): def __init__(self, schema, rx): if not {'type'}.issuperset(schema): - raise SchemaError('unknown parameter for //{0}'.format(self.subname())) + raise SchemaError('unknown parameter for //{}'.format(self.subname())) def check(self, value): try: @@ -180,8 +349,10 @@ class _CoreType(object): return False return True - def validate(self, value, name='value'): - raise SchemaMismatch('Tried to validate abstract base schema class') + def validate(self, value): + raise SchemaMismatch('Tried to validate abstract base schema class', self) + +### Core Schema Types -------------------------------------------------------- class AllType(_CoreType): @staticmethod @@ -191,26 +362,22 @@ class AllType(_CoreType): if not {'type', 'of'}.issuperset(schema): raise SchemaError('unknown parameter for //all') - if not(schema.get('of') and len(schema.get('of'))): + if not schema.get('of'): raise SchemaError('no alternatives given in //all of') self.alts = [rx.make_schema(s) for s in schema['of']] - def validate(self, value, name='value'): - error_messages = [] + def validate(self, value): + errors = [] for schema in self.alts: try: - schema.validate(value, name) + schema.validate(value) except SchemaMismatch as e: - error_messages.append(str(e)) + errors.append(e) + + if errors: + raise _createTreeMismatch(self, errors) - if len(error_messages) > 1: - messages = indent('\n'.join(error_messages)) - message = '{0} failed to meet all schema requirements:\n{1}' - message = message.format(name, messages) - raise SchemaMismatch(message) - elif len(error_messages) == 1: - raise SchemaMismatch(error_messages[0]) class AnyType(_CoreType): @staticmethod @@ -223,25 +390,28 @@ class AnyType(_CoreType): raise SchemaError('unknown parameter for //any') if 'of' in schema: - if not schema['of']: raise SchemaError('no alternatives given in //any of') + if not schema['of']: + raise SchemaError('no alternatives given in //any of') + self.alts = [ rx.make_schema(alt) for alt in schema['of'] ] - def validate(self, value, name='value'): + def validate(self, value): if self.alts is None: return - error_messages = [] + + errors = [] + for schema in self.alts: try: - schema.validate(value, name) + schema.validate(value) break except SchemaMismatch as e: - error_messages.append(str(e)) + errors.append(e) + + if len(errors) == len(self.alts): + message = 'must satisfy at least one of the following' + raise _createTreeMismatch(self, errors, message=message) - if len(error_messages) == len(self.alts): - messages = indent('\n'.join(error_messages)) - message = '{0} failed to meet any schema requirements:\n{1}' - message = message.format(name, messages) - raise SchemaMismatch(message) class ArrType(_CoreType): @staticmethod @@ -259,47 +429,45 @@ class ArrType(_CoreType): self.content_schema = rx.make_schema(schema['contents']) if schema.get('length'): - self.length = Util.make_range_validator(schema['length']) + self.length = Range(schema['length']) - def validate(self, value, name='value'): + def validate(self, value): if not isinstance(value, (list, tuple)): - raise SchemaTypeMismatch(name, 'array') + raise TypeMismatch(self, value) - if self.length: - self.length(len(value), name+' length') + errors = [] + if self.length and not self.length(len(value)): + err = LengthRangeMismatch(self, value) + errors.append(err) - error_messages = [] + child_errors = {} - for i, item in enumerate(value): + for key, item in enumerate(value): try: - self.content_schema.validate(item, 'item '+str(i)) + self.content_schema.validate(item) except SchemaMismatch as e: - error_messages.append(str(e)) + child_errors[key] = e + if errors or child_errors: + raise _createTreeMismatch(self, errors, child_errors) - if len(error_messages) > 1: - messages = indent('\n'.join(error_messages)) - message = '{0} sequence contains invalid elements:\n{1}' - message = message.format(name, messages) - raise SchemaMismatch(message) - elif len(error_messages) == 1: - raise SchemaMismatch(name+': '+error_messages[0]) class BoolType(_CoreType): @staticmethod def subname(): return 'bool' - def validate(self, value, name='value'): + def validate(self, value,): if not isinstance(value, bool): - raise SchemaTypeMismatch(name, 'boolean') + raise TypeMismatch(self, value) + class DefType(_CoreType): @staticmethod def subname(): return 'def' - - def validate(self, value, name='value'): + def validate(self, value): if value is None: - raise SchemaMismatch(name+' must be non-null') + raise TypeMismatch(self, value) + class FailType(_CoreType): @staticmethod @@ -307,8 +475,13 @@ class FailType(_CoreType): def check(self, value): return False - def validate(self, value, name='value'): - raise SchemaMismatch(name+' is of fail type, automatically invalid.') + def validate(self, value): + raise SchemaMismatch( + 'is of fail type, automatically invalid.', + self, + 'fail' + ) + class IntType(_CoreType): @staticmethod @@ -326,17 +499,18 @@ class IntType(_CoreType): self.range = None if 'range' in schema: - self.range = Util.make_range_validator(schema['range']) + self.range = Range(schema['range']) - def validate(self, value, name='value'): + def validate(self, value): if not isinstance(value, Number) or isinstance(value, bool) or value%1: - raise SchemaTypeMismatch(name,'integer') + raise TypeMismatch(self, value) - if self.range: - self.range(value, name) + if self.range and not self.range(value): + raise RangeMismatch(self, value) if self.value is not None and value != self.value: - raise SchemaValueMismatch(name, self.value) + raise ValueMismatch(self, value) + class MapType(_CoreType): @staticmethod @@ -353,25 +527,21 @@ class MapType(_CoreType): self.value_schema = rx.make_schema(schema['values']) - def validate(self, value, name='value'): + def validate(self, value): if not isinstance(value, dict): - raise SchemaTypeMismatch(name, 'map') + raise TypeMismatch(self, value) - error_messages = [] + child_errors = {} for key, val in value.items(): try: - self.value_schema.validate(val, key) + self.value_schema.validate(val) except SchemaMismatch as e: - error_messages.append(str(e)) + child_errors[key] = e + + if child_errors: + raise _createTreeMismatch(self, child_errors=child_errors) - if len(error_messages) > 1: - messages = indent('\n'.join(error_messages)) - message = '{0} map contains invalid entries:\n{1}' - message = message.format(name, messages) - raise SchemaMismatch(message) - elif len(error_messages) == 1: - raise SchemaMismatch(name+': '+error_messages[0]) class NilType(_CoreType): @staticmethod @@ -379,9 +549,10 @@ class NilType(_CoreType): def check(self, value): return value is None - def validate(self, value, name='value'): + def validate(self, value): if value is not None: - raise SchemaTypeMismatch(name, 'null') + raise TypeMismatch(self, value) + class NumType(_CoreType): @staticmethod @@ -400,25 +571,27 @@ class NumType(_CoreType): self.range = None if schema.get('range'): - self.range = Util.make_range_validator(schema['range']) + self.range = Range(schema['range']) - def validate(self, value, name='value'): + def validate(self, value): if not isinstance(value, Number) or isinstance(value, bool): - raise SchemaTypeMismatch(name, 'number') + raise TypeMismatch(self, value) - if self.range: - self.range(value, name) + if self.range and not self.range(value): + raise RangeMismatch(self, value) if self.value is not None and value != self.value: - raise SchemaValueMismatch(name, self.value) + raise ValueMismatch(self, value) + class OneType(_CoreType): @staticmethod def subname(): return 'one' - def validate(self, value, name='value'): + def validate(self, value): if not isinstance(value, (Number, str)): - raise SchemaTypeMismatch(name, 'number or string') + raise TypeMismatch(self, value) + class RecType(_CoreType): @staticmethod @@ -436,7 +609,9 @@ class RecType(_CoreType): setattr(self, which, {}) for field in schema.get(which, {}).keys(): if field in self.known: - raise SchemaError('%s appears in both required and optional' % field) + raise SchemaError( + '%s appears in both required and optional' % field + ) self.known.add(field) @@ -444,47 +619,53 @@ class RecType(_CoreType): schema[which][field] ) - def validate(self, value, name='value'): + def validate(self, value): if not isinstance(value, dict): - raise SchemaTypeMismatch(name, 'record') + raise TypeMismatch(self, value) - unknown = [k for k in value.keys() if k not in self.known] + errors = [] + child_errors = {} - if unknown and not self.rest_schema: - fields = indent('\n'.join(unknown)) - raise SchemaMismatch(name+' contains unknown fields:\n'+fields) - - error_messages = [] + missing_fields = [] for field in self.required: - try: - if field not in value: - raise SchemaMismatch('missing required field: '+field) - self.required[field].validate(value[field], field) - except SchemaMismatch as e: - error_messages.append(str(e)) + + if field not in value: + missing_fields.append(field) + else: + try: + self.required[field].validate(value[field]) + except SchemaMismatch as e: + child_errors[field] = e + + if missing_fields: + err = MissingFieldMismatch(self, missing_fields) + errors.append(err) for field in self.optional: if field not in value: continue - try: - self.optional[field].validate(value[field], field) - except SchemaMismatch as e: - error_messages.append(str(e)) - if unknown: - rest = {key: value[key] for key in unknown} try: - self.rest_schema.validate(rest, name) + self.optional[field].validate(value[field]) except SchemaMismatch as e: - error_messages.append(str(e)) + child_errors[field] = e - if len(error_messages) > 1: - messages = indent('\n'.join(error_messages)) - message = '{0} record is invalid:\n{1}' - message = message.format(name, messages) - raise SchemaMismatch(message) - elif len(error_messages) == 1: - raise SchemaMismatch(name+': '+error_messages[0]) + unknown = [k for k in value.keys() if k not in self.known] + + if unknown: + if self.rest_schema: + rest = {key: value[key] for key in unknown} + try: + self.rest_schema.validate(rest) + except SchemaMismatch as e: + errors.append(e) + else: + fields = _indent('\n'.join(unknown)) + err = UnknownFieldMismatch(self, unknown) + errors.append(err) + + if errors or child_errors: + raise _createTreeMismatch(self, errors, child_errors) class SeqType(_CoreType): @@ -504,34 +685,33 @@ class SeqType(_CoreType): if (schema.get('tail')): self.tail_schema = rx.make_schema(schema['tail']) - def validate(self, value, name='value'): + def validate(self, value): if not isinstance(value, (list, tuple)): - raise SchemaTypeMismatch(name, 'sequence') + raise TypeMismatch(self, value) - if len(value) < len(self.content_schema): - raise SchemaMismatch(name+' is less than expected length') + errors = [] - if len(value) > len(self.content_schema) and not self.tail_schema: - raise SchemaMismatch(name+' exceeds expected length') + if len(value) != len(self.content_schema): + if len(value) > len(self.content_schema) and self.tail_schema: + try: + self.tail_schema.validate(value[len(self.content_schema):]) + except SchemaMismatch as e: + errors.append(e) + else: + err = SeqLengthMismatch(self, value) + errors.append(err) - error_messages = [] + child_errors = {} - for i, (schema, item) in enumerate(zip(self.content_schema, value)): + for index, (schema, item) in enumerate(zip(self.content_schema, value)): try: - schema.validate(item, 'item '+str(i)) + schema.validate(item) except SchemaMismatch as e: - error_messages.append(str(e)) + child_errors[index] = e - if len(error_messages) > 1: - messages = indent('\n'.join(error_messages)) - message = '{0} sequence is invalid:\n{1}' - message = message.format(name, messages) - raise SchemaMismatch(message) - elif len(error_messages) == 1: - raise SchemaMismatch(name+': '+error_messages[0]) + if errors or child_errors: + raise _createTreeMismatch(self, errors, child_errors) - if len(value) > len(self.content_schema): - self.tail_schema.validate(value[len(self.content_schema):], name) class StrType(_CoreType): @staticmethod @@ -549,18 +729,20 @@ class StrType(_CoreType): self.length = None if 'length' in schema: - self.length = Util.make_range_validator(schema['length']) + self.length = Range(schema['length']) - def validate(self, value, name='value'): + def validate(self, value): if not isinstance(value, str): - raise SchemaTypeMismatch(name, 'string') + raise TypeMismatch(self, value) + if self.value is not None and value != self.value: - raise SchemaValueMismatch(name, '"{0}"'.format(self.value)) - if self.length: - self.length(len(value), name+' length') + raise ValueMismatch(self, self) + + if self.length and not self.length(len(value)): + raise LengthRangeMismatch(self, value) core_types = [ AllType, AnyType, ArrType, BoolType, DefType, FailType, IntType, MapType, NilType, NumType, OneType, RecType, SeqType, StrType -] +] \ No newline at end of file diff --git a/patacrep/build.py b/patacrep/build.py index 96a09424..f123228d 100644 --- a/patacrep/build.py +++ b/patacrep/build.py @@ -44,7 +44,12 @@ class Songbook: def __init__(self, raw_songbook, basename): # Validate config schema = config_model('schema') - utils.validate_yaml_schema(raw_songbook, schema) + + try: + utils.validate_yaml_schema(raw_songbook, schema) + except errors.SchemaError as exception: + exception.message = "The songbook file '{}' is not valid\n".format(basename) + raise exception self._raw_config = raw_songbook self.basename = basename @@ -84,7 +89,13 @@ class Songbook: self._config['book']['lang'], self._config['book']['encoding'], ) - self._config['_template'] = renderer.get_all_variables(self._config.get('template', {})) + + try: + self._config['_template'] = renderer.get_all_variables(self._config.get('template', {})) + except errors.SchemaError as exception: + exception.message = "The songbook file '{}' is not valid\n{}".format( + self.basename, exception.message) + raise exception self._config['_compiled_authwords'] = authors.compile_authwords( copy.deepcopy(self._config['authors']) @@ -110,7 +121,7 @@ class Songbook: ) self._config['filename'] = output.name[:-4] - self._config['bookoptions'] = iter_bookoptions(self._config) + self._config['_bookoptions'] = iter_bookoptions(self._config) renderer.render_tex(output, self._config) self._errors.extend(renderer.errors) diff --git a/patacrep/content/__init__.py b/patacrep/content/__init__.py index a2268ee7..c265f5b7 100755 --- a/patacrep/content/__init__.py +++ b/patacrep/content/__init__.py @@ -17,13 +17,17 @@ dictionary where: When analysing the content field of the .sb file, when those keywords are met, the corresponding parser is called. +# Keyword examples + + - sort + - section* + - cwd + # Parsers A parser is a function which takes as arguments: - keyword: the keyword triggering this function; - argument: the argument of the keyword (see below); - - contentlist: the list of content, that is, the part of the list - following the keyword (see example below); - config: the configuration object of the current songbook. Plugins can change it. @@ -31,30 +35,20 @@ A parser returns a ContentList object (a list of instances of the ContentItem class), defined in this module (or of subclasses of this class). Example: When the following piece of content is met - - ["sorted(author, @title)", "a_song.sg", "another_song.sg"] - -the parser associated to keyword 'sorted' get the arguments: - - keyword = "sorted" - - argument = "author, @title" - - contentlist = ["a_song.sg", "another_song.sg"] + sort: + key: ["author", "title"] + content: + - "a_song.sg" + - "another_song.sg" + +the parser associated to keyword 'sort' get the arguments: + - keyword = "sort" + - argument = { + 'key': ["author", "title"], + 'content': ["a_song.sg", "another_song.sg"], + } - config = . -# Keyword - -A keyword is either an identifier (alphanumeric characters, and underscore), -or such an identifier, with some text surrounded by parenthesis (like a -function definition); this text is called the argument to the keyword. -Examples: - - sorted - - sorted(author, @title) - - cwd(some/path) - -If the keyword has an argument, it can be anything, given that it is -surrounded by parenthesis. It is up to the plugin to parse this argument. For -intance, keyword "foo()(( bar()" is a perfectly valid keyword, and the parser -associated to "foo" will get as argument the string ")(( bar(". - # ContentItem class The content classes are subclasses of class ContentItem defined in this module. @@ -72,8 +66,9 @@ import re import sys import jinja2 +import yaml -from patacrep import files +from patacrep import files, Rx from patacrep.errors import SharedError LOGGER = logging.getLogger(__name__) @@ -228,12 +223,37 @@ def render(context, content): return rendered +def validate_parser_argument(raw_schema): + """Check that the parser argument respects the schema + + Will raise `ContentError` if the schema is not respected. + """ + rx_checker = Rx.Factory({"register_core_types": True}) + schema = rx_checker.make_schema(yaml.load(raw_schema)) + + def wrap(parse): + """Wrap the parse function""" + def wrapped(keyword, argument, config): + """Check the argument schema before calling the plugin parser""" + try: + schema.validate(argument) + except Rx.SchemaMismatch as exception: + msg = 'Invalid syntax:\n---\n{}---\n{}'.format( + yaml.dump({keyword: argument}, default_flow_style=False), + str(exception) + ) + raise ContentError(keyword, msg) + return parse(keyword, argument=argument, config=config) + return wrapped + return wrap + + def process_content(content, config=None): """Process content, and return a list of ContentItem() objects. Arguments are: - - content: the content field of the .sb file, which should be a list, and - describe what is to be included in the songbook; + - content: the content field of the .sb file, which should be a nested list + and describe what is to be included in the songbook; - config: the configuration dictionary of the current songbook. Return: a list of ContentItem objects, corresponding to the content to be @@ -241,25 +261,25 @@ def process_content(content, config=None): """ contentlist = ContentList() plugins = config.get('_content_plugins', {}) - keyword_re = re.compile(r'^ *(?P[\w\*]*) *(\((?P.*)\))? *$') if not content: - content = [["song"]] + content = [{'song': None}] + elif isinstance(content, dict): + content = [content] for elem in content: if isinstance(elem, str): - elem = ["song", elem] - try: - match = keyword_re.match(elem[0]).groupdict() - except AttributeError: - contentlist.append_error(ContentError(elem[0], "Cannot parse content type.")) - continue - (keyword, argument) = (match['keyword'], match['argument']) - if keyword not in plugins: - contentlist.append_error(ContentError(keyword, "Unknown content type.")) - continue - contentlist.extend(plugins[keyword]( - keyword, - argument=argument, - contentlist=elem[1:], - config=config, - )) + elem = {'song': elem} + if isinstance(elem, dict): + for keyword, argument in elem.items(): + try: + if keyword not in plugins: + raise ContentError(keyword, "Unknown content keyword.") + contentlist.extend(plugins[keyword]( + keyword, + argument=argument, + config=config, + )) + except ContentError as error: + contentlist.append_error(error) + else: + contentlist.append_error(ContentError(str(elem), "Unknown content type.")) return contentlist diff --git a/patacrep/content/cwd.py b/patacrep/content/cwd.py index 07cc407e..efc37553 100755 --- a/patacrep/content/cwd.py +++ b/patacrep/content/cwd.py @@ -1,33 +1,46 @@ """Change base directory before importing songs.""" -from patacrep.content import process_content +import os + +from patacrep.content import process_content, validate_parser_argument from patacrep.songs import DataSubpath #pylint: disable=unused-argument -def parse(keyword, config, argument, contentlist): - """Return a list songs included in contentlist, whith a different base path. +@validate_parser_argument(""" +type: //rec +required: + path: //str +optional: + content: //any +""") +def parse(keyword, config, argument): + """Return a list songs, whith a different base path. Arguments: - keyword: unused; - config: the current songbook configuration dictionary; - - argument: a directory; - - contentlist: songbook content, that is parsed by - patacrep.content.process_content(). + - argument: a dict containing: + path: string specifying the path to use as root; + content: songbook content, that is parsed by + patacrep.content.process_content(). - This function adds 'argument' to the directories where songs are searched + This function adds 'path' to the directories where songs are searched for, and then processes the content. - The 'argument' is added: - - first as a relative path to the current directory; + The 'path' is added: + - first as a relative path to the *.sb file directory; - then as a relative path to every path already present in config['songdir']. """ + subpath = argument['path'] old_songdir = config['_songdir'] + sbdir = os.path.dirname(config['_filepath']) + config['_songdir'] = ( - [DataSubpath(".", argument)] + - [path.clone().join(argument) for path in config['_songdir']] + [DataSubpath(sbdir, subpath)] + + [path.clone().join(subpath) for path in config['_songdir']] ) - processed_content = process_content(contentlist, config) + processed_content = process_content(argument.get('content'), config) config['_songdir'] = old_songdir return processed_content diff --git a/patacrep/content/include.py b/patacrep/content/include.py index dbc10fce..0755c747 100644 --- a/patacrep/content/include.py +++ b/patacrep/content/include.py @@ -1,14 +1,15 @@ """Include an external list of songs This plugin provides keyword 'include', used to include an external list of -songs in JSON format. +songs in JSON or YAML format. """ -import json import os import logging -from patacrep.content import process_content, ContentError, ContentList +import yaml + +from patacrep.content import process_content, ContentError, ContentList, validate_parser_argument from patacrep import encoding, errors, files LOGGER = logging.getLogger(__name__) @@ -28,18 +29,29 @@ def load_from_datadirs(path, datadirs): ) #pylint: disable=unused-argument -def parse(keyword, config, argument, contentlist): +@validate_parser_argument(""" +type: //any +of: + - type: //str + - type: //arr + contents: //str +""") +def parse(keyword, config, argument): """Include an external file content. Arguments: - keyword: the string 'include'; - config: the current songbook configuration dictionary; - - argument: None; - - contentlist: a list of file paths to be included. + - argument: + a list of file paths to be included + or a string of the file to include + """ new_contentlist = ContentList() + if isinstance(argument, str): + argument = [argument] - for path in contentlist: + for path in argument: try: filepath = load_from_datadirs(path, config['_datadir']) except ContentError as error: @@ -51,7 +63,7 @@ def parse(keyword, config, argument, contentlist): filepath, encoding=config['book']['encoding'] ) as content_file: - new_content = json.load(content_file) + new_content = yaml.load(content_file) except Exception as error: # pylint: disable=broad-except new_contentlist.append_error(ContentError( keyword="include", diff --git a/patacrep/content/section.py b/patacrep/content/section.py index 6aed27c6..316b0fa3 100755 --- a/patacrep/content/section.py +++ b/patacrep/content/section.py @@ -1,6 +1,6 @@ """Allow LaTeX sections (starred or not) as content of a songbook.""" -from patacrep.content import ContentItem, ContentError, ContentList, EmptyContentList +from patacrep.content import ContentItem, ContentList, validate_parser_argument KEYWORDS = [ "part", @@ -23,36 +23,39 @@ class Section(ContentItem): self.short = short def render(self, __context): - if self.short is None: + if self.short is None or self.keyword not in KEYWORDS: return r'\{}{{{}}}'.format(self.keyword, self.name) else: return r'\{}[{}]{{{}}}'.format(self.keyword, self.short, self.name) #pylint: disable=unused-argument -def parse(keyword, argument, contentlist, config): - """Parse the contentlist. +@validate_parser_argument(""" +type: //any +of: + - type: //str + - type: //rec + required: + name: //str + optional: + short: //str +""") +def parse(keyword, argument, config): + """Parse the section. Arguments: - keyword (one of "part", "chapter", "section", ... , "subparagraph", and their starred versions "part*", "chapter*", ... , "subparagraph*"): the section to use; - - argument: unused; - - contentlist: a list of one or two strings, which are the names (long - and short) of the section; + - argument: + either a string describing the section name + or a dict + name: Name of the section + short: Shortname of the section (only for non starred sections) - config: configuration dictionary of the current songbook. """ - try: - if (keyword not in KEYWORDS) and (len(contentlist) != 1): - raise ContentError( - keyword, - "Starred section names must have exactly one argument." - ) - if (len(contentlist) not in [1, 2]): - raise ContentError(keyword, "Section can have one or two arguments.") - return ContentList([Section(keyword, *contentlist)]) - except ContentError as error: - return EmptyContentList(errors=[error]) - + if isinstance(argument, str): + argument = {'name': argument} + return ContentList([Section(keyword, **argument)]) CONTENT_PLUGINS = dict([ (word, parse) diff --git a/patacrep/content/song.py b/patacrep/content/song.py index 8e9519dc..56ed0205 100755 --- a/patacrep/content/song.py +++ b/patacrep/content/song.py @@ -7,7 +7,7 @@ import textwrap import jinja2 -from patacrep.content import process_content +from patacrep.content import process_content, validate_parser_argument from patacrep.content import ContentError, ContentItem, ContentList from patacrep import files, errors @@ -58,18 +58,28 @@ class SongRenderer(ContentItem): return self.song.fullpath < other.song.fullpath #pylint: disable=unused-argument -def parse(keyword, argument, contentlist, config): +@validate_parser_argument(""" +type: //any +of: + - type: //nil + - type: //str + - type: //arr + contents: //str +""") +def parse(keyword, argument, config): """Parse data associated with keyword 'song'. Arguments: - keyword: unused; - - argument: unused; - - contentlist: a list of strings, which are interpreted as regular + - argument: a list of strings, which are interpreted as regular expressions (interpreted using the glob module), referring to songs. - config: the current songbook configuration dictionary. Return a list of Song() instances. """ + contentlist = argument + if isinstance(contentlist, str): + contentlist = [contentlist] plugins = config['_song_plugins'] if '_langs' not in config: config['_langs'] = set() @@ -121,7 +131,6 @@ def parse(keyword, argument, contentlist, config): )) return sorted(songlist) - CONTENT_PLUGINS = {'song': parse} diff --git a/patacrep/content/songsection.py b/patacrep/content/songsection.py index c5fea4ab..abc9d609 100755 --- a/patacrep/content/songsection.py +++ b/patacrep/content/songsection.py @@ -1,6 +1,6 @@ """Allow 'songchapter' and 'songsection' as content of a songbook.""" -from patacrep.content import ContentItem, ContentError, ContentList, EmptyContentList +from patacrep.content import ContentItem, ContentList, validate_parser_argument KEYWORDS = [ "songchapter", @@ -20,25 +20,18 @@ class SongSection(ContentItem): return r'\{}{{{}}}'.format(self.keyword, self.name) #pylint: disable=unused-argument -def parse(keyword, argument, contentlist, config): - """Parse the contentlist. +@validate_parser_argument(""" +//str +""") +def parse(keyword, argument, config): + """Parse the songsection. Arguments: - keyword ("songsection" or "songchapter"): the section to use; - - argument: unused; - - contentlist: a list of one string, which is the name of the section; + - argument: name of the section; - config: configuration dictionary of the current songbook. """ - try: - if (keyword not in KEYWORDS) and (len(contentlist) != 1): - raise ContentError( - keyword, - "Starred section names must have exactly one argument.", - ) - return ContentList([SongSection(keyword, contentlist[0])]) - except ContentError as error: - return EmptyContentList(errors=[error]) - + return ContentList([SongSection(keyword, argument)]) CONTENT_PLUGINS = dict([ (keyword, parse) diff --git a/patacrep/content/sorted.py b/patacrep/content/sort.py similarity index 64% rename from patacrep/content/sorted.py rename to patacrep/content/sort.py index c14cc732..7ee5aff0 100755 --- a/patacrep/content/sorted.py +++ b/patacrep/content/sort.py @@ -1,6 +1,6 @@ """Sorted list of songs. -This plugin provides keyword 'sorted', used to include a sorted list of songs +This plugin provides keyword 'sort', used to include a sorted list of songs to a songbook. """ @@ -8,13 +8,13 @@ import logging import unidecode from patacrep import files -from patacrep.content import ContentError, EmptyContentList -from patacrep.content import process_content +from patacrep.content import ContentError +from patacrep.content import process_content, validate_parser_argument from patacrep.content.song import OnlySongsError LOGGER = logging.getLogger(__name__) -DEFAULT_SORT = ['by', 'album', '@title'] +DEFAULT_SORT = ['by', 'album', 'title'] def normalize_string(string): """Return a normalized string. @@ -45,9 +45,9 @@ def key_generator(sort): song = songrenderer.song songkey = [] for key in sort: - if key == "@title": + if key == "title": field = song.unprefixed_titles - elif key == "@path": + elif key == "path": field = song.fullpath elif key == "by": field = song.authors @@ -67,29 +67,44 @@ def key_generator(sort): return ordered_song_keys #pylint: disable=unused-argument -def parse(keyword, config, argument, contentlist): - """Return a sorted list of songs contained in 'contentlist'. +@validate_parser_argument(""" +type: //any +of: + - type: //nil + - type: //rec + optional: + key: + type: //any + of: + - //str + - type: //arr + contents: //str + content: //any +""") +def parse(keyword, config, argument): + """Return a sorted list of songs. Arguments: - - keyword: the string 'sorted'; + - keyword: the string 'sort'; - config: the current songbook configuration dictionary; - - argument: the list of the fields used to sort songs, as strings - separated by commas (e.g. "by, album, @title"); - - contentlist: the list of content to be sorted. If this content - contain something else than a song, an exception is raised. + - argument: a dict of: + key: the list of the fields used to sort songs (e.g. "by", "album", "title") + content: content to be sorted. If this content + contain something else than a song, an exception is raised. """ - if argument: - sort = [key.strip() for key in argument.split(",")] - else: - sort = DEFAULT_SORT + if argument is None: + argument = {} + sort = argument.get('key', DEFAULT_SORT) + if isinstance(sort, str): + sort = [sort] try: - songlist = process_content(contentlist, config) + songlist = process_content(argument.get('content'), config) except OnlySongsError as error: - return EmptyContentList(errors=[ContentError(keyword, ( + raise ContentError(keyword, ( "Content list of this keyword can be only songs (or content " "that result into songs), and the following are not:" + str(error.not_songs) - ))]) + )) return sorted(songlist, key=key_generator(sort)) -CONTENT_PLUGINS = {'sorted': parse} +CONTENT_PLUGINS = {'sort': parse} diff --git a/patacrep/content/tex.py b/patacrep/content/tex.py index dd506655..406084a8 100755 --- a/patacrep/content/tex.py +++ b/patacrep/content/tex.py @@ -5,7 +5,7 @@ import logging import os from patacrep import files, errors -from patacrep.content import ContentItem, ContentList, ContentError +from patacrep.content import ContentItem, ContentList, ContentError, validate_parser_argument LOGGER = logging.getLogger(__name__) @@ -22,26 +22,33 @@ class LaTeX(ContentItem): ))) #pylint: disable=unused-argument -def parse(keyword, argument, contentlist, config): - """Parse the contentlist. +@validate_parser_argument(""" +type: //any +of: + - type: //arr + contents: //str + - type: //str +""") +def parse(keyword, argument, config): + """Parse the tex files. Arguments: - keyword: unused; - - argument: unused; - - contentlist: a list of name of tex files; + - argument: + a list of tex files to include + or a string of the tex file to include; - config: configuration dictionary of the current songbook. """ - if not contentlist: - LOGGER.warning( - "Useless 'tex' content: list of files to include is empty." - ) + if isinstance(argument, str): + argument = [argument] + filelist = ContentList() basefolders = itertools.chain( (path.fullpath for path in config['_songdir']), files.iter_datadirs(config['_datadir']), files.iter_datadirs(config['_datadir'], 'latex'), ) - for filename in contentlist: + for filename in argument: checked_file = None for path in basefolders: if os.path.exists(os.path.join(path, filename)): @@ -62,5 +69,4 @@ def parse(keyword, argument, contentlist, config): return filelist - CONTENT_PLUGINS = {'tex': parse} diff --git a/patacrep/data/templates/patacrep.tex b/patacrep/data/templates/patacrep.tex index 48cb7bd4..fce3a9e1 100644 --- a/patacrep/data/templates/patacrep.tex +++ b/patacrep/data/templates/patacrep.tex @@ -63,7 +63,7 @@ default: (* block songbookpackages *) \usepackage[ - (* for option in bookoptions *)((option)), + (* for option in _bookoptions *)((option)), (* endfor *) ]{crepbook} (* endblock *) diff --git a/patacrep/data/templates/songbook_model.yml b/patacrep/data/templates/songbook_model.yml index 7567de47..fb8722bb 100644 --- a/patacrep/data/templates/songbook_model.yml +++ b/patacrep/data/templates/songbook_model.yml @@ -5,6 +5,7 @@ schema: template: //any required: _cache: //bool + _filepath: //str _datadir: type: //arr contents: //str @@ -82,7 +83,7 @@ default: lang: en encoding: utf-8 pictures: yes - template: default.tex + template: patacrep.tex onesongperpage: no chords: # Options relatives aux accords diff --git a/patacrep/data/templates/songs.tex b/patacrep/data/templates/songs.tex index 94846f0d..50e23ec4 100644 --- a/patacrep/data/templates/songs.tex +++ b/patacrep/data/templates/songs.tex @@ -22,7 +22,7 @@ (* block songbookpackages *) \usepackage[ - (* for option in bookoptions *)((option)), + (* for option in _bookoptions *)((option)), (* endfor *) ]{patacrep} (* endblock *) diff --git a/patacrep/errors.py b/patacrep/errors.py index 08eebc6a..5bb3896d 100644 --- a/patacrep/errors.py +++ b/patacrep/errors.py @@ -7,15 +7,19 @@ class SongbookError(Exception): """ pass -class SBFileError(SongbookError): - """Error during songbook file decoding""" +class SchemaError(SongbookError): + """Error on the songbook schema""" - def __init__(self, message=None): + def __init__(self, message='', rx_exception=None): super().__init__() self.message = message + self.rx_exception = rx_exception def __str__(self): - return self.message + if self.rx_exception: + return self.message + str(self.rx_exception) + else: + return self.message class TemplateError(SongbookError): """Error during template generation""" diff --git a/patacrep/songbook/__main__.py b/patacrep/songbook/__main__.py index adc581c5..3e9e592b 100644 --- a/patacrep/songbook/__main__.py +++ b/patacrep/songbook/__main__.py @@ -153,6 +153,9 @@ def main(): default_songbook.update(user_songbook) songbook = dict(default_songbook) + songbook['_filepath'] = os.path.abspath(songbook_path) + sbdir = os.path.dirname(songbook['_filepath']) + # Gathering datadirs datadirs = [] if options.datadir: @@ -162,16 +165,13 @@ def main(): if isinstance(songbook['book']['datadir'], str): songbook['book']['datadir'] = [songbook['book']['datadir']] datadirs += [ - os.path.join( - os.path.dirname(os.path.abspath(songbook_path)), - path - ) + os.path.join(sbdir, path) for path in songbook['book']['datadir'] ] del songbook['book']['datadir'] # Default value - datadirs.append(os.path.dirname(os.path.abspath(songbook_path))) + datadirs.append(sbdir) songbook['_datadir'] = datadirs songbook['_cache'] = options.cache[0] diff --git a/patacrep/templates.py b/patacrep/templates.py index f745a58e..b47dd683 100644 --- a/patacrep/templates.py +++ b/patacrep/templates.py @@ -8,7 +8,7 @@ import yaml from jinja2 import Environment, FileSystemLoader, ChoiceLoader, \ TemplateNotFound, nodes from jinja2.ext import Extension -from jinja2.meta import find_referenced_templates as find_templates +from jinja2.meta import find_referenced_templates from patacrep import errors, files, utils from patacrep.latex import lang2babel, UnknownLanguage @@ -162,82 +162,54 @@ class TexBookRenderer(Renderer): ) def get_all_variables(self, user_config): - ''' - Validate template variables (and set defaults when needed) + '''Validate template variables (and set defaults when needed) + + Will raise `SchemaError` if any data does not respect the schema ''' data = self.get_template_variables(self.template) variables = dict() - for name, param in data.items(): - template_config = user_config.get(name, {}) - variables[name] = self._get_variables(param, template_config) + for templatename, param in data.items(): + template_config = user_config.get(templatename, {}) + try: + variables[templatename] = self._get_variables(param, template_config) + except errors.SchemaError as exception: + exception.message += "'template' > '{}' > ".format(templatename) + raise exception return variables @staticmethod def _get_variables(parameter, user_config): '''Get the default value for the parameter, according to the language. - ''' - schema = parameter.get('schema', {}) + Will raise `SchemaError` if the data does not respect the schema + ''' data = utils.DictOfDict(parameter.get('default', {})) data.update(user_config) + schema = parameter.get('schema', {}) utils.validate_yaml_schema(data, schema) + return data - def get_template_variables(self, template, skip=None): + def get_template_variables(self, basetemplate): """Parse the template to extract the variables as a dictionary. If the template includes or extends other templates, load them as well. Arguments: - - template: the name of the template, as a string. - - skip: a list of templates (as strings) to skip: if they are included + - basetemplate: the name of the template, as a string. in 'template' (or one of its subtemplates), it is not parsed. """ - if not skip: - skip = [] variables = {} - (current, templates) = self.parse_template(template) - if current: - variables[template.name] = current - - for subtemplate in templates: - if subtemplate in skip: + for templatename, template in self._iter_template_content(basetemplate): + match = re.findall(_VARIABLE_REGEXP, template) + if not match: continue - subtemplate = self.jinjaenv.get_template(subtemplate) - variables.update( - self.get_template_variables( - subtemplate, - skip + templates - ) - ) - return variables - - def parse_template(self, template): - """Return (variables, templates). - - Argument: - - template: name of the template to parse. - - Return values: - - variables: a dictionary of variables contained in 'template', NOT - recursively (included templates are not parsed). - - templates: list of included temlates, NOT recursively. - """ - - subvariables = {} - templatename = self.jinjaenv.get_template(template).filename - with patacrep.encoding.open_read( - templatename, - encoding=self.encoding - ) as template_file: - content = template_file.read() - subtemplates = list(find_templates(self.jinjaenv.parse(content))) - match = re.findall(_VARIABLE_REGEXP, content) - if match: - for var in match: + if templatename not in variables: + variables[templatename] = {} + for variables_string in match: try: - subvariables.update(yaml.load(var)) + variables[templatename].update(yaml.load(variables_string)) except ValueError as exception: raise errors.TemplateError( exception, @@ -246,12 +218,29 @@ class TexBookRenderer(Renderer): "{filename}. The yaml string was:" "\n'''\n{yamlstring}\n'''" ).format( - filename=templatename, - yamlstring=var, + filename=template.filename, + yamlstring=variables_string, ) ) + return variables - return (subvariables, subtemplates) + def _iter_template_content(self, templatename, *, skip=None): + """Iterate over template (and subtemplate) content.""" + if skip is None: + skip = [] + template = self.jinjaenv.get_template(templatename) + with patacrep.encoding.open_read( + template.filename, + encoding=self.encoding + ) as contentfile: + content = contentfile.read() + for subtemplatename in find_referenced_templates(self.jinjaenv.parse(content)): + if subtemplatename not in skip: + yield from self._iter_template_content( + subtemplatename, + skip=skip + [templatename], + ) + yield template.name, content def render_tex(self, output, context): '''Render a template into a .tex file @@ -264,7 +253,7 @@ class TexBookRenderer(Renderer): output.write(self.template.render(context)) -def transform_options(config, equivalents): +def _transform_options(config, equivalents): """ Get the equivalent name of the checked options """ @@ -285,14 +274,14 @@ def iter_bookoptions(config): 'pictures': 'pictures', 'onesongperpage': 'onesongperpage', } - yield from transform_options(config['book'], book_equivalents) + yield from _transform_options(config['book'], book_equivalents) chords_equivalents = { 'lilypond': 'lilypond', 'tablatures': 'tabs', 'repeatchords': 'repeatchords', } - yield from transform_options(config['chords'], chords_equivalents) + yield from _transform_options(config['chords'], chords_equivalents) if config['chords']['show']: if config['chords']['diagramreminder'] == "important": diff --git a/patacrep/utils.py b/patacrep/utils.py index 7ddd0eca..5051a592 100644 --- a/patacrep/utils.py +++ b/patacrep/utils.py @@ -81,16 +81,14 @@ def yesno(string): def validate_yaml_schema(data, schema): """Check that the data respects the schema - Will raise `SBFileError` if the schema is not respected. + Will raise `SchemaError` if the schema is not respected. """ - rx_checker = Rx.Factory({"register_core_types": True}) - schema = rx_checker.make_schema(schema) + schema = Rx.make_schema(schema) - if not isinstance(data, dict): + if isinstance(data, DictOfDict): data = dict(data) try: schema.validate(data) except Rx.SchemaMismatch as exception: - msg = 'Could not parse songbook file:\n' + str(exception) - raise errors.SBFileError(msg) + raise errors.SchemaError(rx_exception=exception) diff --git a/test/test_content/cwd.control b/test/test_content/cwd.control index c0a741d2..e1ed6693 100644 --- a/test/test_content/cwd.control +++ b/test/test_content/cwd.control @@ -1 +1 @@ -["subdir/chordpro.csg"] \ No newline at end of file +- subdir/chordpro.csg \ No newline at end of file diff --git a/test/test_content/cwd.source b/test/test_content/cwd.source index 4dfc53e6..1d0663d7 100644 --- a/test/test_content/cwd.source +++ b/test/test_content/cwd.source @@ -1 +1,3 @@ -[["cwd(subdir)"]] \ No newline at end of file +- cwd: + path: subdir + content: \ No newline at end of file diff --git a/test/test_content/cwd_list.control b/test/test_content/cwd_list.control index 465f29df..3c9377a4 100644 --- a/test/test_content/cwd_list.control +++ b/test/test_content/cwd_list.control @@ -1 +1,2 @@ -["subdir/chordpro.csg", "exsong.sg"] \ No newline at end of file +- subdir/chordpro.csg +- exsong.sg \ No newline at end of file diff --git a/test/test_content/cwd_list.source b/test/test_content/cwd_list.source index c5700449..15f85307 100644 --- a/test/test_content/cwd_list.source +++ b/test/test_content/cwd_list.source @@ -1 +1,11 @@ -[["cwd(subdir)", "exsong.sg", "intersong.is", "jsonlist.json", "texfile.tex", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"], "exsong.sg"] \ No newline at end of file +- cwd: + path: subdir + content: + - "exsong.sg" + - "intersong.is" + - "jsonlist.json" + - "texfile.tex" + - "texsong.tsg" + - "chordpro.csg" + - "subdir/chordpro.csg" +- "exsong.sg" \ No newline at end of file diff --git a/test/test_content/datadir/songs/custom_list.yaml b/test/test_content/datadir/songs/custom_list.yaml new file mode 100644 index 00000000..440c12e4 --- /dev/null +++ b/test/test_content/datadir/songs/custom_list.yaml @@ -0,0 +1,6 @@ +- sort: + key: "title" + content: + - exsong.sg + - chordpro.csg + - subdir/chordpro.csg diff --git a/test/test_content/datadir_sort/path1_title1_author1.csg b/test/test_content/datadir_sort/path1_title1_author1.csg new file mode 100644 index 00000000..f5dd92fc --- /dev/null +++ b/test/test_content/datadir_sort/path1_title1_author1.csg @@ -0,0 +1,5 @@ +{title: Title1} +{artist: Author1} + +Foo bar + diff --git a/test/test_content/datadir_sort/path1_title1_author2.csg b/test/test_content/datadir_sort/path1_title1_author2.csg new file mode 100644 index 00000000..13d6ecac --- /dev/null +++ b/test/test_content/datadir_sort/path1_title1_author2.csg @@ -0,0 +1,5 @@ +{title: Title1} +{artist: Author2} + +Foo bar + diff --git a/test/test_content/datadir_sort/path1_title2_author1.csg b/test/test_content/datadir_sort/path1_title2_author1.csg new file mode 100644 index 00000000..35e93160 --- /dev/null +++ b/test/test_content/datadir_sort/path1_title2_author1.csg @@ -0,0 +1,5 @@ +{title: Title2} +{artist: Author1} + +Foo bar + diff --git a/test/test_content/datadir_sort/path1_title2_author2.csg b/test/test_content/datadir_sort/path1_title2_author2.csg new file mode 100644 index 00000000..f2db9337 --- /dev/null +++ b/test/test_content/datadir_sort/path1_title2_author2.csg @@ -0,0 +1,5 @@ +{title: Title2} +{artist: Author2} + +Foo bar + diff --git a/test/test_content/datadir_sort/path2_title1_author1.csg b/test/test_content/datadir_sort/path2_title1_author1.csg new file mode 100644 index 00000000..f5dd92fc --- /dev/null +++ b/test/test_content/datadir_sort/path2_title1_author1.csg @@ -0,0 +1,5 @@ +{title: Title1} +{artist: Author1} + +Foo bar + diff --git a/test/test_content/datadir_sort/path2_title1_author2.csg b/test/test_content/datadir_sort/path2_title1_author2.csg new file mode 100644 index 00000000..13d6ecac --- /dev/null +++ b/test/test_content/datadir_sort/path2_title1_author2.csg @@ -0,0 +1,5 @@ +{title: Title1} +{artist: Author2} + +Foo bar + diff --git a/test/test_content/datadir_sort/path2_title2_author1.csg b/test/test_content/datadir_sort/path2_title2_author1.csg new file mode 100644 index 00000000..35e93160 --- /dev/null +++ b/test/test_content/datadir_sort/path2_title2_author1.csg @@ -0,0 +1,5 @@ +{title: Title2} +{artist: Author1} + +Foo bar + diff --git a/test/test_content/datadir_sort/path2_title2_author2.csg b/test/test_content/datadir_sort/path2_title2_author2.csg new file mode 100644 index 00000000..f2db9337 --- /dev/null +++ b/test/test_content/datadir_sort/path2_title2_author2.csg @@ -0,0 +1,5 @@ +{title: Title2} +{artist: Author2} + +Foo bar + diff --git a/test/test_content/glob.control b/test/test_content/glob.control index a30583cc..177bc945 100644 --- a/test/test_content/glob.control +++ b/test/test_content/glob.control @@ -1 +1 @@ -["chordpro.csg"] \ No newline at end of file +- chordpro.csg \ No newline at end of file diff --git a/test/test_content/glob.source b/test/test_content/glob.source index 167ab6ab..175382a6 100644 --- a/test/test_content/glob.source +++ b/test/test_content/glob.source @@ -1 +1 @@ -["*.csg"] \ No newline at end of file +- "*.csg" \ No newline at end of file diff --git a/test/test_content/include.control b/test/test_content/include.control index 5ba82a4c..6aba97ae 100644 --- a/test/test_content/include.control +++ b/test/test_content/include.control @@ -1 +1,6 @@ -["exsong.sg", "chordpro.csg", "subdir/chordpro.csg"] \ No newline at end of file +- exsong.sg +- chordpro.csg +- subdir/chordpro.csg +- chordpro.csg +- subdir/chordpro.csg +- exsong.sg \ No newline at end of file diff --git a/test/test_content/include.source b/test/test_content/include.source index ccd69aa1..d7eb147c 100644 --- a/test/test_content/include.source +++ b/test/test_content/include.source @@ -1 +1,3 @@ -[["include" , "custom_list.json"]] \ No newline at end of file +- include: + - custom_list.json +- include: custom_list.yaml \ No newline at end of file diff --git a/test/test_content/invalid.control b/test/test_content/invalid.control new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/test/test_content/invalid.control @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/test/test_content/invalid.source b/test/test_content/invalid.source new file mode 100644 index 00000000..7952bf8f --- /dev/null +++ b/test/test_content/invalid.source @@ -0,0 +1,4 @@ +- ["directly", "a", "list"] +- invalid_keyword: Test +- section: + short: Missing name \ No newline at end of file diff --git a/test/test_content/sections.control b/test/test_content/sections.control index 3666bbce..5a87c67c 100644 --- a/test/test_content/sections.control +++ b/test/test_content/sections.control @@ -1 +1,11 @@ -["section:Traditional", "exsong.sg", "section:Example", "texsong.tsg", "chordpro.csg", "exsong.sg"] \ No newline at end of file +- section{First Section!} +- section{Named section} +- section[section_short_name]{Section with short name} +- section*{Section* with short name} +- part{part section test} +- chapter{chapter section test} +- section{section section test} +- subsection{subsection section test} +- subsubsection{subsubsection section test} +- paragraph{paragraph section test} +- subparagraph{subparagraph section test} \ No newline at end of file diff --git a/test/test_content/sections.source b/test/test_content/sections.source index 339815be..ff5bae23 100644 --- a/test/test_content/sections.source +++ b/test/test_content/sections.source @@ -1,6 +1,16 @@ -[["section", "Traditional"], - "exsong.sg", - ["section", "Example"], - "texsong.tsg", - "chordpro.csg", - "exsong.sg"] \ No newline at end of file +- section: First Section! +- section: + name: Named section +- section: + name: Section with short name + short: section_short_name +- section*: + name: Section* with short name + short: section_star_short_name +- part: part section test +- chapter: chapter section test +- section: section section test +- subsection: subsection section test +- subsubsection: subsubsection section test +- paragraph: paragraph section test +- subparagraph: subparagraph section test \ No newline at end of file diff --git a/test/test_content/sections_short.control b/test/test_content/sections_short.control deleted file mode 100644 index 706e86b8..00000000 --- a/test/test_content/sections_short.control +++ /dev/null @@ -1 +0,0 @@ -["section:(tradi)Traditional", "exsong.sg", "section*:Example", "texsong.tsg", "chordpro.csg", "exsong.sg"] \ No newline at end of file diff --git a/test/test_content/sections_short.source b/test/test_content/sections_short.source deleted file mode 100644 index fe70d510..00000000 --- a/test/test_content/sections_short.source +++ /dev/null @@ -1,6 +0,0 @@ -[["section", "Traditional", "tradi"], - "exsong.sg", - ["section*", "Example"], - "texsong.tsg", - "chordpro.csg", - "exsong.sg"] \ No newline at end of file diff --git a/test/test_content/songs.control b/test/test_content/songs.control index e85dfea7..6ad0622a 100644 --- a/test/test_content/songs.control +++ b/test/test_content/songs.control @@ -1 +1,4 @@ -["exsong.sg", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"] \ No newline at end of file +- exsong.sg +- texsong.tsg +- chordpro.csg +- subdir/chordpro.csg \ No newline at end of file diff --git a/test/test_content/songs.source b/test/test_content/songs.source index dcd19323..1c0b689d 100644 --- a/test/test_content/songs.source +++ b/test/test_content/songs.source @@ -1 +1,8 @@ -["exsong.sg", "intersong.is", "jsonlist.json", "texfile.tex", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"] \ No newline at end of file +- exsong.sg +- intersong.is +- jsonlist.json +- song: + - texfile.tex + - texsong.tsg +- song: chordpro.csg +- subdir/chordpro.csg \ No newline at end of file diff --git a/test/test_content/songsection.control b/test/test_content/songsection.control index 69dd034b..81197f08 100644 --- a/test/test_content/songsection.control +++ b/test/test_content/songsection.control @@ -1 +1,6 @@ -["songsection:Traditional", "exsong.sg", "songchapter:English", "texsong.tsg", "chordpro.csg", "exsong.sg"] \ No newline at end of file +- songsection{Traditional} +- exsong.sg +- songchapter{English} +- texsong.tsg +- chordpro.csg +- exsong.sg \ No newline at end of file diff --git a/test/test_content/songsection.source b/test/test_content/songsection.source index 089322c4..f4540159 100644 --- a/test/test_content/songsection.source +++ b/test/test_content/songsection.source @@ -1,6 +1,6 @@ -[["songsection", "Traditional"], - "exsong.sg", - ["songchapter", "English"], - "texsong.tsg", - "chordpro.csg", - "exsong.sg"] \ No newline at end of file +- songsection: Traditional +- "exsong.sg" +- songchapter: English +- "texsong.tsg" +- "chordpro.csg" +- "exsong.sg" \ No newline at end of file diff --git a/test/test_content/sort.control b/test/test_content/sort.control new file mode 100644 index 00000000..a7430273 --- /dev/null +++ b/test/test_content/sort.control @@ -0,0 +1,27 @@ +- section{Title} +- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg" +- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg" +- section{Author, Title} +- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg" +- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg" +- section{Path, Title} +- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg" +- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg" +- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg" diff --git a/test/test_content/sort.source b/test/test_content/sort.source new file mode 100644 index 00000000..0e6d71da --- /dev/null +++ b/test/test_content/sort.source @@ -0,0 +1,15 @@ +- cwd: + path: "datadir_sort" + content: + - section: + name: "Title" + - sort: + key: title + - section: + name: "Author, Title" + - sort: + key: [by, title] + - section: + name: "Path, Title" + - sort: + key: [path, title] diff --git a/test/test_content/sorted.control b/test/test_content/sorted.control deleted file mode 100644 index 5cfaa29d..00000000 --- a/test/test_content/sorted.control +++ /dev/null @@ -1 +0,0 @@ -["chordpro.csg", "exsong.sg", "subdir/chordpro.csg", "texsong.tsg"] \ No newline at end of file diff --git a/test/test_content/sorted.source b/test/test_content/sorted.source deleted file mode 100644 index 5b2bbf3a..00000000 --- a/test/test_content/sorted.source +++ /dev/null @@ -1 +0,0 @@ -[["sorted(fullpath)"]] \ No newline at end of file diff --git a/test/test_content/test_content.py b/test/test_content/test_content.py index 23d498bc..060467d9 100644 --- a/test/test_content/test_content.py +++ b/test/test_content/test_content.py @@ -5,7 +5,9 @@ import glob import os import unittest -import json +import yaml + +from pkg_resources import resource_filename from patacrep.songs import DataSubpath from patacrep import content, files @@ -18,7 +20,7 @@ from .. import dynamic # pylint: disable=unused-import class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): """Test of the content plugins. - For any given `foo.source`, it parses the content as a json "content" + For any given `foo.source`, it parses the content as a yaml "content" argument of a .sb file. It controls that the generated file list is equal to the one in `foo.control`. """ @@ -51,17 +53,22 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): """Test that `base.source` produces the correct file list""" sourcename = "{}.source".format(base) with open(sourcename, mode="r", encoding="utf8") as sourcefile: - sbcontent = json.load(sourcefile) + sbcontent = yaml.load(sourcefile) + config = cls.config.copy() + config['_filepath'] = base with logging_reduced('patacrep.content.song'): - expandedlist = content.process_content(sbcontent, cls.config.copy()) + expandedlist = content.process_content(sbcontent, config) sourcelist = [cls._clean_path(elem) for elem in expandedlist] controlname = "{}.control".format(base) if not os.path.exists(controlname): raise Exception("Missing control:" + str(sourcelist).replace("'", '"')) with open(controlname, mode="r", encoding="utf8") as controlfile: - controllist = json.load(controlfile) + controllist = [ + elem.replace("@TEST_FOLDER@", files.path2posix(resource_filename(__name__, ""))) + for elem in yaml.load(controlfile) + ] self.assertEqual(controllist, sourcelist) @@ -78,13 +85,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): return files.path2posix(files.relpath(elem.song.fullpath, songpath)) elif isinstance(elem, section.Section): - if elem.short is None: - return "{}:{}".format(elem.keyword, elem.name) - else: - return "{}:({}){}".format(elem.keyword, elem.short, elem.name) + return elem.render(None)[1:] elif isinstance(elem, songsection.SongSection): - return "{}:{}".format(elem.keyword, elem.name) + return elem.render(None)[1:] elif isinstance(elem, tex.LaTeX): return files.path2posix(elem.filename) diff --git a/test/test_content/tex.control b/test/test_content/tex.control index 85eeb47e..0c187461 100644 --- a/test/test_content/tex.control +++ b/test/test_content/tex.control @@ -1 +1,2 @@ -["test/test_content/datadir/songs/texfile.tex"] \ No newline at end of file +- test/test_content/datadir/songs/texfile.tex +- test/test_content/datadir/songs/texfile.tex \ No newline at end of file diff --git a/test/test_content/tex.source b/test/test_content/tex.source index 56852196..9cbed25b 100644 --- a/test/test_content/tex.source +++ b/test/test_content/tex.source @@ -1 +1,4 @@ -[["tex", "texfile.tex", "chordpro.csg"]] \ No newline at end of file +- tex: + - texfile.tex + - chordpro.csg +- tex: texfile.tex \ No newline at end of file diff --git a/test/test_songbook/content.sb b/test/test_songbook/content.sb index 7e5ee2c8..a49ff77f 100644 --- a/test/test_songbook/content.sb +++ b/test/test_songbook/content.sb @@ -2,17 +2,24 @@ book: pictures: yes datadir: content_datadir lang: en + template: default.tex chords: repeatchords: no diagramreminder: all -content: [ - ["section", "Test of section"], - ["sorted"], - ["songsection", "Test of song section"], - ["cwd(content_datadir/content)", - "song.csg", "song.tsg", - ["tex", "foo.tex"] - ], - ["include", "include.sbc"] - ] \ No newline at end of file +content: + - section: Test of section + - sort: + - songsection: Test of song section + - cwd: + # relative to sb file + path: content_datadir/content + content: + - "song.csg" + - "song.tsg" + - cwd: + # relative to datadir + path: ../content + content: + - tex: foo.tex + - include: include.sbc diff --git a/test/test_songbook/content_datadir/songs/include.sbc b/test/test_songbook/content_datadir/songs/include.sbc index fefa39b1..16351055 100644 --- a/test/test_songbook/content_datadir/songs/include.sbc +++ b/test/test_songbook/content_datadir/songs/include.sbc @@ -1 +1 @@ -[["section", "This is an included section"]] +[{"section": "This is an included section"}] diff --git a/test/test_songbook/datadir.sb b/test/test_songbook/datadir.sb index 0763bc7a..946d71e8 100644 --- a/test/test_songbook/datadir.sb +++ b/test/test_songbook/datadir.sb @@ -4,3 +4,4 @@ book: - datadir_datadir - datadir_datadir2 lang: en + template: default.tex diff --git a/test/test_songbook/languages.sb b/test/test_songbook/languages.sb index 43e7ab85..27343463 100644 --- a/test/test_songbook/languages.sb +++ b/test/test_songbook/languages.sb @@ -1,3 +1,4 @@ book: + template: default.tex datadir: - languages_datadir diff --git a/test/test_songbook/syntax.sb b/test/test_songbook/syntax.sb index aca40b87..9d6266ba 100644 --- a/test/test_songbook/syntax.sb +++ b/test/test_songbook/syntax.sb @@ -1,4 +1,5 @@ book: + template: default.tex datadir: - syntax_datadir lang: en diff --git a/test/test_songbook/unicode.sb b/test/test_songbook/unicode.sb index 9f5163c5..c6f8d75a 100644 --- a/test/test_songbook/unicode.sb +++ b/test/test_songbook/unicode.sb @@ -1,4 +1,5 @@ book: + template: default.tex datadir: - unicode_datadir lang: en diff --git a/texlive_packages.txt b/texlive_packages.txt index 7a3e79cb..f16d7e2a 100644 --- a/texlive_packages.txt +++ b/texlive_packages.txt @@ -1,11 +1,7 @@ -babel-english babel-esperanto -babel-french -babel-german babel-italian babel-latin babel-portuges -babel-spanish ctablestack etoolbox fancybox