Merge pull request #190 from patacrep/yaml_content

Content uses Yaml system

.appveyor.yml

@@ -23,7 +23,7 @@ install:
   - "python -c \"import struct; print(struct.calcsize('P') * 8)\""
 
   # Download miktex portable (if not cached)
-  - ps: "If (!(Test-Path miktex-portable.exe)){wget http://mirrors.ctan.org/systems/win32/miktex/setup/miktex-portable-2.9.5719.exe -OutFile ./miktex-portable.exe}"
+  - ps: "If (!(Test-Path miktex-portable.exe)){wget http://mirrors.ctan.org/systems/win32/miktex/setup/miktex-portable-2.9.5857.exe -OutFile ./miktex-portable.exe}"
 
   # Unzip miktex portable
   - "7z x miktex-portable.exe * -aot -omiktex > nul"
@@ -32,8 +32,8 @@ install:
   - cmd: set PATH=%PATH%;C:\projects\patacrep\miktex\miktex\bin
 
   # Update some packages to prevent ltluatex bug
-  - cmd: mpm.exe --update=miktex-bin-2.9
-  - cmd: mpm.exe --update=ltxbase --update=luatexbase --update=luaotfload --update=miktex-luatex-base --update=fontspec
+  # - cmd: mpm.exe --update=miktex-bin-2.9 --verbose
+  # - cmd: mpm.exe --update=ltxbase --update=luatexbase --update=luaotfload --update=miktex-luatex-base --update=fontspec
 
   # Manually install required texlive packages
   - cmd: mpm.exe --install-some texlive_packages.txt

examples/example-all.sb

@@ -14,6 +14,6 @@
   "sep" : ["and", "et"]
   },
  "datadir" : ".",
- "content": [["sorted"]]
+ "content": [["sort"]]
 
 }

examples/example-all.yaml.sb

@@ -13,7 +13,7 @@ authors:
   separators:
     - "and"
     - "et"
-content: [["sorted"]]
+content: {"sort"}
 
 template:
   patacrep.tex:

patacrep/Rx.py

@@ -9,33 +9,197 @@ import re
 import types
 from numbers import Number
 
-
-core_types = [ ]
+### Exception Classes --------------------------------------------------------
 
 class SchemaError(Exception):
   pass
 
 class SchemaMismatch(Exception):
-  pass
 
-class SchemaTypeMismatch(SchemaMismatch):
-  def __init__(self, name, desired_type):
-    SchemaMismatch.__init__(self, '{0} must be {1}'.format(name, desired_type))
+  def __init__(self, message, schema, error=None):
+    Exception.__init__(self, message)
+    self.type = schema.subname() 
+    self.error = error
 
-class SchemaValueMismatch(SchemaMismatch):
-  def __init__(self, name, value):
-    SchemaMismatch.__init__(self, '{0} must equal {1}'.format(name, value))
+class TypeMismatch(SchemaMismatch):
   
-class SchemaRangeMismatch(SchemaMismatch):
-  pass
+  def __init__(self, schema, data):
+    message = 'must be of type {} (was {})'.format(
+      schema.subname(),
+      type(data).__name__
+      )
 
-def indent(text, level=1, whitespace='  '):
-    return '\n'.join(whitespace*level+line for line in text.split('\n'))
+    SchemaMismatch.__init__(self, message, schema, 'type')
+    self.expected_type = schema.subname()
+    self.value = type(data).__name__
 
-class Util(object):
-  @staticmethod
-  def make_range_check(opt):
 
+class ValueMismatch(SchemaMismatch):
+  
+  def __init__(self, schema, data):
+
+    message = 'must equal {} (was {})'.format(
+      repr(schema.value),
+      repr(data)
+      )
+
+    SchemaMismatch.__init__(self, message, schema, 'value')
+    self.expected_value = schema.value
+    self.value = data
+
+ 
+
+class RangeMismatch(SchemaMismatch):
+  
+  def __init__(self, schema, data):
+    
+    message = 'must be in range {} (was {})'.format(
+      schema.range,
+      data
+      )
+
+    SchemaMismatch.__init__(self, message, schema, 'range')
+    self.range = schema.range
+    self.value = data
+
+
+class LengthRangeMismatch(SchemaMismatch):
+
+  def __init__(self, schema, data):
+    length_range = Range(schema.length)
+
+    if not hasattr(length_range, 'min') and \
+       not hasattr(length_range, 'min_ex'):
+      length_range.min = 0
+
+    message = 'length must be in range {} (was {})'.format(
+      length_range,
+      len(data)
+      )
+
+    SchemaMismatch.__init__(self, message, schema, 'range')
+    self.range = schema.length
+    self.value = len(data)
+
+
+class MissingFieldMismatch(SchemaMismatch):
+
+  def __init__(self, schema, fields):
+
+    if len(fields) == 1:
+      message = 'missing required field: {}'.format(
+        repr(fields[0])
+        )
+    else:
+      message = 'missing required fields: {}'.format(
+        ', '.join(fields)
+        )
+      if len(message) >= 80: # if the line is too long
+        message = 'missing required fields:\n{}'.format(
+          _indent('\n'.join(fields))
+          )
+
+    SchemaMismatch.__init__(self, message, schema, 'missing')
+    self.fields = fields
+
+
+class UnknownFieldMismatch(SchemaMismatch):
+
+  def __init__(self, schema, fields):
+
+    if len(fields) == 1:
+      message = 'unknown field: {}'.format(
+        repr(fields[0])
+        )
+    else:
+      message = 'unknown fields: {}'.format(
+        ', '.join(fields)
+        )
+      if len(message) >= 80: # if the line is too long
+        message = 'unknown fields:\n{}'.format(
+          _indent('\n'.join(fields))
+          )
+
+    SchemaMismatch.__init__(self, message, schema, 'unexpected')
+    self.fields = fields
+
+
+class SeqLengthMismatch(SchemaMismatch):
+  def __init__(self, schema, data):
+
+    expected_length = len(schema.content_schema)
+    message = 'sequence must have {} element{} (had {})'.format(
+      expected_length,
+      's'*(expected_length != 1), # plural
+      len(data)
+      )
+
+    SchemaMismatch.__init__(self, message, schema, 'size')
+    self.expected_length = expected_length
+    self.value = len(data)
+
+
+class TreeMismatch(SchemaMismatch):
+
+  def __init__(self, schema, errors=[], child_errors={}, message=None):
+    
+    ## Create error message
+
+    error_messages = []
+
+    for err in errors:
+      error_messages.append(str(err))
+
+    for key, err in child_errors.items():
+
+      if isinstance(key, int):
+        index = '[item {}]'.format(key)
+      else:
+        index = '{}'.format(repr(key))
+
+      if isinstance(err, TreeMismatch) and \
+          not err.errors and len(err.child_errors) == 1:
+
+        template = '{} > {}'
+
+      else:
+        template = '{} {}'
+
+      msg = template.format(index, err)
+      error_messages.append(msg)
+
+    if message is None:
+      message = 'does not match schema'
+
+    if len(error_messages) == 1:
+      msg = error_messages[0]
+
+    else:
+      msg = '{}:\n{}'.format(
+        message,
+        _indent('\n'.join(error_messages))
+        )
+
+    SchemaMismatch.__init__(self, msg, schema, 'multiple')
+    self.errors = errors
+    self.child_errors = child_errors
+
+def _createTreeMismatch(schema, errors=[], child_errors={}, message=None):
+  if len(errors) == 1 and not child_errors:
+    return errors[0]
+  else:
+    return TreeMismatch(schema, errors, child_errors, message)
+
+### Utilities ----------------------------------------------------------------
+
+class Range(object):
+
+  def __init__(self, opt):
+    if isinstance(opt, Range):
+      for attr in ('min', 'max', 'min_ex', 'max_ex'):
+        if hasattr(opt, attr):
+          setattr(self, attr, getattr(opt, attr))
+    else:
       if not {'min', 'max', 'min-ex', 'max-ex'}.issuperset(opt):
         raise ValueError("illegal argument to make_range_check")
       if {'min', 'min-ex'}.issubset(opt):
@@ -43,51 +207,44 @@ class Util(object):
       if {'max', 'max-ex'}.issubset(opt):
         raise ValueError("Cannot define both exclusive and inclusive max")
 
-    r = opt.copy()
-    inf = float('inf')
+      for boundary in ('min', 'max', 'min-ex', 'max-ex'):
+        if boundary in opt:
+          attr = boundary.replace('-', '_')
+          setattr(self, attr, opt[boundary])
+
+  def __call__(self, value):
+    INF = float('inf')
+
+    get = lambda attr, default: getattr(self, attr, default)
 
-    def check_range(value):
     return(
-        r.get('min',    -inf) <= value and \
-        r.get('max',     inf) >= value and \
-        r.get('min-ex', -inf) <  value and \
-        r.get('max-ex',  inf) >  value
+        get('min',    -INF) <= value and \
+        get('max',     INF) >= value and \
+        get('min_ex', -INF) <  value and \
+        get('max_ex',  INF) >  value
         )
 
-    return check_range
-
-  @staticmethod
-  def make_range_validator(opt):
-    check_range = Util.make_range_check(opt)
-
-    r = opt.copy()
-    nan = float('nan')
-
-    def validate_range(value, name='value'):
-      if not check_range(value):
-        if r.get('min', nan) == r.get('max', nan):
-          msg = '{0} must equal {1}'.format(name, r['min'])
-          raise SchemaRangeMismatch(msg)
-        
-        range_str = ''
-        if 'min' in r:
-          range_str = '[{0}, '.format(r['min'])
-        elif 'min-ex' in r:
-          range_str = '({0}, '.format(r['min-ex'])
+  def __str__(self):
+    if hasattr(self, 'min'):
+      s = '[{}, '.format(self.min)
+    elif hasattr(self, 'min_ex'):
+      s = '({}, '.format(self.min_ex)
     else:
-          range_str = '(-inf, '
+      s = '(-Inf, '
 
-        if 'max' in r:
-          range_str += '{0}]'.format(r['max'])
-        elif 'max-ex' in r:
-          range_str += '{0})'.format(r['max-ex'])
+    if hasattr(self, 'max'):
+      s += '{}]'.format(self.max)
+    elif hasattr(self, 'max_ex'):
+      s += '{})'.format(self.max_ex)
     else:
-          range_str += 'inf)'
+      s += 'Inf)'
     
-        raise SchemaRangeMismatch(name+' must be in range '+range_str)
+    return s
 
-    return validate_range
+def _indent(text, level=1, whitespace='  '):
+    return '\n'.join(whitespace*level+line for line in text.split('\n'))
 
+    ### Schema Factory Class -----------------------------------------------------
 
 class Factory(object):
   def __init__(self, register_core_types=True):
@@ -109,20 +266,20 @@ class Factory(object):
     m = re.match('^/([-._a-z0-9]*)/([-._a-z0-9]+)$', type_name)
 
     if not m:
-      raise ValueError("couldn't understand type name '{0}'".format(type_name))
+      raise ValueError("couldn't understand type name '{}'".format(type_name))
 
     prefix, suffix = m.groups()
 
     if prefix not in self.prefix_registry:
       raise KeyError(
-        "unknown prefix '{0}' in type name '{1}'".format(prefix, type_name)
+        "unknown prefix '{0}' in type name '{}'".format(prefix, type_name)
       )
 
     return self.prefix_registry[ prefix ] + suffix
 
   def add_prefix(self, name, base):
     if self.prefix_registry.get(name):
-      raise SchemaError("the prefix '{0}' is already registered".format(name))
+      raise SchemaError("the prefix '{}' is already registered".format(name))
 
     self.prefix_registry[name] = base;
 
@@ -130,13 +287,15 @@ class Factory(object):
     t_uri = t.uri()
 
     if t_uri in self.type_registry:
-      raise ValueError("type already registered for {0}".format(t_uri))
+      raise ValueError("type already registered for {}".format(t_uri))
 
     self.type_registry[t_uri] = t
 
   def learn_type(self, uri, schema):
     if self.type_registry.get(uri):
-      raise SchemaError("tried to learn type for already-registered uri {0}".format(uri))
+      raise SchemaError(
+        "tried to learn type for already-registered uri {}".format(uri)
+        )
 
     # make sure schema is valid
     # should this be in a try/except?
@@ -153,17 +312,27 @@ class Factory(object):
 
     uri = self.expand_uri(schema['type'])
 
-    if not self.type_registry.get(uri): raise SchemaError("unknown type {0}".format(uri))
+    if not self.type_registry.get(uri):
+      raise SchemaError("unknown type {}".format(uri))
 
     type_class = self.type_registry[uri]
 
     if isinstance(type_class, dict):
       if not {'type'}.issuperset(schema):
-        raise SchemaError('composed type does not take check arguments');
+        raise SchemaError('composed type does not take check arguments')
       return self.make_schema(type_class['schema'])
     else:
       return type_class(schema, self)
 
+std_factory = None
+def make_schema(schema):
+  global std_factory
+  if std_factory is None:
+    std_factory = Factory()
+  return std_factory.make_schema(schema)
+
+### Core Type Base Class -------------------------------------------------
+
 class _CoreType(object):
   @classmethod
   def uri(self):
@@ -171,7 +340,7 @@ class _CoreType(object):
 
   def __init__(self, schema, rx):
     if not {'type'}.issuperset(schema):
-      raise SchemaError('unknown parameter for //{0}'.format(self.subname()))
+      raise SchemaError('unknown parameter for //{}'.format(self.subname()))
 
   def check(self, value):
     try:
@@ -180,8 +349,10 @@ class _CoreType(object):
       return False
     return True
 
-  def validate(self, value, name='value'):
-    raise SchemaMismatch('Tried to validate abstract base schema class')
+  def validate(self, value):
+    raise SchemaMismatch('Tried to validate abstract base schema class', self)
+
+### Core Schema Types --------------------------------------------------------
 
 class AllType(_CoreType):
   @staticmethod
@@ -191,26 +362,22 @@ class AllType(_CoreType):
     if not {'type', 'of'}.issuperset(schema):
       raise SchemaError('unknown parameter for //all')
     
-    if not(schema.get('of') and len(schema.get('of'))):
+    if not schema.get('of'):
       raise SchemaError('no alternatives given in //all of')
 
     self.alts = [rx.make_schema(s) for s in schema['of']]
 
-  def validate(self, value, name='value'):
-    error_messages = []
+  def validate(self, value):
+    errors = []
     for schema in self.alts:
       try:
-        schema.validate(value, name)
+        schema.validate(value)
       except SchemaMismatch as e:
-        error_messages.append(str(e))
+        errors.append(e)
+
+    if errors:
+      raise _createTreeMismatch(self, errors)
 
-    if len(error_messages) > 1:
-      messages = indent('\n'.join(error_messages))
-      message = '{0} failed to meet all schema requirements:\n{1}'
-      message = message.format(name, messages)
-      raise SchemaMismatch(message)
-    elif len(error_messages) == 1:
-      raise SchemaMismatch(error_messages[0])
 
 class AnyType(_CoreType):
   @staticmethod
@@ -223,25 +390,28 @@ class AnyType(_CoreType):
       raise SchemaError('unknown parameter for //any')
     
     if 'of' in schema:
-      if not schema['of']: raise SchemaError('no alternatives given in //any of')
+      if not schema['of']: 
+        raise SchemaError('no alternatives given in //any of')
+
       self.alts = [ rx.make_schema(alt) for alt in schema['of'] ]
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if self.alts is None:
       return
-    error_messages = []
+    
+    errors = []
+
     for schema in self.alts:
       try:
-        schema.validate(value, name)
+        schema.validate(value)
         break
       except SchemaMismatch as e:
-        error_messages.append(str(e))
+        errors.append(e)
+
+    if len(errors) == len(self.alts):
+      message = 'must satisfy at least one of the following'
+      raise _createTreeMismatch(self, errors, message=message)
 
-    if len(error_messages) == len(self.alts):
-      messages = indent('\n'.join(error_messages))
-      message = '{0} failed to meet any schema requirements:\n{1}'
-      message = message.format(name, messages)
-      raise SchemaMismatch(message)
 
 class ArrType(_CoreType):
   @staticmethod
@@ -259,47 +429,45 @@ class ArrType(_CoreType):
     self.content_schema = rx.make_schema(schema['contents'])
 
     if schema.get('length'):
-      self.length = Util.make_range_validator(schema['length'])
+      self.length = Range(schema['length'])
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if not isinstance(value, (list, tuple)):
-      raise SchemaTypeMismatch(name, 'array')
+      raise TypeMismatch(self, value)
 
-    if self.length:
-      self.length(len(value), name+' length')
+    errors = []
+    if self.length and not self.length(len(value)):
+      err = LengthRangeMismatch(self, value)
+      errors.append(err)
 
-    error_messages = []
+    child_errors = {}
 
-    for i, item in enumerate(value):
+    for key, item in enumerate(value):
       try:
-        self.content_schema.validate(item, 'item '+str(i))
+        self.content_schema.validate(item)
       except SchemaMismatch as e:
-        error_messages.append(str(e))
+        child_errors[key] = e
+    if errors or child_errors:
+      raise _createTreeMismatch(self, errors, child_errors)
 
-    if len(error_messages) > 1:
-      messages = indent('\n'.join(error_messages))
-      message = '{0} sequence contains invalid elements:\n{1}'
-      message = message.format(name, messages)
-      raise SchemaMismatch(message)
-    elif len(error_messages) == 1:
-      raise SchemaMismatch(name+': '+error_messages[0])
 
 class BoolType(_CoreType):
   @staticmethod
   def subname(): return 'bool'
 
-  def validate(self, value, name='value'):
+  def validate(self, value,):
     if not isinstance(value, bool):
-      raise SchemaTypeMismatch(name, 'boolean')
+      raise TypeMismatch(self, value)
+
 
 class DefType(_CoreType):
   @staticmethod
   def subname(): return 'def'
 
-
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if value is None:
-      raise SchemaMismatch(name+' must be non-null')
+      raise TypeMismatch(self, value)
+
 
 class FailType(_CoreType):
   @staticmethod
@@ -307,8 +475,13 @@ class FailType(_CoreType):
 
   def check(self, value): return False
 
-  def validate(self, value, name='value'):
-    raise SchemaMismatch(name+' is of fail type, automatically invalid.')
+  def validate(self, value):
+    raise SchemaMismatch(
+      'is of fail type, automatically invalid.',
+      self,
+      'fail'
+      )
+
 
 class IntType(_CoreType):
   @staticmethod
@@ -326,17 +499,18 @@ class IntType(_CoreType):
 
     self.range = None
     if 'range' in schema:
-      self.range = Util.make_range_validator(schema['range'])
+      self.range = Range(schema['range'])
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if not isinstance(value, Number) or isinstance(value, bool) or value%1:
-      raise SchemaTypeMismatch(name,'integer')
+      raise TypeMismatch(self, value)
 
-    if self.range:
-      self.range(value, name)
+    if self.range and not self.range(value):
+      raise RangeMismatch(self, value)
 
     if self.value is not None and value != self.value:
-      raise SchemaValueMismatch(name, self.value)
+      raise ValueMismatch(self, value)
+
 
 class MapType(_CoreType):
   @staticmethod
@@ -353,25 +527,21 @@ class MapType(_CoreType):
 
     self.value_schema = rx.make_schema(schema['values'])
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if not isinstance(value, dict):
-      raise SchemaTypeMismatch(name, 'map')
+      raise TypeMismatch(self, value)
 
-    error_messages = []
+    child_errors = {}
 
     for key, val in value.items():
       try:
-        self.value_schema.validate(val, key)
+        self.value_schema.validate(val)
       except SchemaMismatch as e:
-        error_messages.append(str(e))
+        child_errors[key] = e
+
+    if child_errors:
+      raise _createTreeMismatch(self, child_errors=child_errors)
 
-    if len(error_messages) > 1:
-      messages = indent('\n'.join(error_messages))
-      message = '{0} map contains invalid entries:\n{1}'
-      message = message.format(name, messages)
-      raise SchemaMismatch(message)
-    elif len(error_messages) == 1:
-      raise SchemaMismatch(name+': '+error_messages[0])
 
 class NilType(_CoreType):
   @staticmethod
@@ -379,9 +549,10 @@ class NilType(_CoreType):
 
   def check(self, value): return value is None
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if value is not None:
-      raise SchemaTypeMismatch(name, 'null')
+      raise TypeMismatch(self, value)
+
 
 class NumType(_CoreType):
   @staticmethod
@@ -400,25 +571,27 @@ class NumType(_CoreType):
     self.range = None
 
     if schema.get('range'):
-      self.range = Util.make_range_validator(schema['range'])
+      self.range = Range(schema['range'])
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if not isinstance(value, Number) or isinstance(value, bool):
-      raise SchemaTypeMismatch(name, 'number')
+      raise TypeMismatch(self, value)
 
-    if self.range:
-      self.range(value, name)
+    if self.range and not self.range(value):
+      raise RangeMismatch(self, value)
 
     if self.value is not None and value != self.value:
-      raise SchemaValueMismatch(name, self.value)
+      raise ValueMismatch(self, value)
+
 
 class OneType(_CoreType):
   @staticmethod
   def subname(): return 'one'
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if not isinstance(value, (Number, str)):
-      raise SchemaTypeMismatch(name, 'number or string')
+      raise TypeMismatch(self, value)
+
 
 class RecType(_CoreType):
   @staticmethod
@@ -436,7 +609,9 @@ class RecType(_CoreType):
       setattr(self, which, {})
       for field in schema.get(which, {}).keys():
         if field in self.known:
-          raise SchemaError('%s appears in both required and optional' % field)
+          raise SchemaError(
+            '%s appears in both required and optional' % field
+            )
 
         self.known.add(field)
 
@@ -444,47 +619,53 @@ class RecType(_CoreType):
           schema[which][field]
         )
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if not isinstance(value, dict):
-      raise SchemaTypeMismatch(name, 'record')
+      raise TypeMismatch(self, value)
 
-    unknown = [k for k in value.keys() if k not in self.known]
-
-    if unknown and not self.rest_schema:
-      fields = indent('\n'.join(unknown))
-      raise SchemaMismatch(name+' contains unknown fields:\n'+fields)
+    errors = []
+    child_errors = {}
 
-    error_messages = []
+    missing_fields = []
 
     for field in self.required:
-      try:
+
       if field not in value:
-          raise SchemaMismatch('missing required field: '+field)
-        self.required[field].validate(value[field], field)
+        missing_fields.append(field)
+      else:
+        try:
+          self.required[field].validate(value[field])
         except SchemaMismatch as e:
-        error_messages.append(str(e))
+          child_errors[field] = e
+
+    if missing_fields:
+      err = MissingFieldMismatch(self, missing_fields)
+      errors.append(err)
 
     for field in self.optional:
       if field not in value: continue
+
       try:
-        self.optional[field].validate(value[field], field) 
+        self.optional[field].validate(value[field]) 
       except SchemaMismatch as e:
-        error_messages.append(str(e))
+        child_errors[field] = e
+
+    unknown = [k for k in value.keys() if k not in self.known]
 
     if unknown:
+      if self.rest_schema:
         rest = {key: value[key] for key in unknown}
         try:
-        self.rest_schema.validate(rest, name)
+          self.rest_schema.validate(rest)
         except SchemaMismatch as e:
-        error_messages.append(str(e))
+          errors.append(e)
+      else:
+        fields = _indent('\n'.join(unknown))
+        err = UnknownFieldMismatch(self, unknown)
+        errors.append(err)
     
-    if len(error_messages) > 1:
-      messages = indent('\n'.join(error_messages))
-      message = '{0} record is invalid:\n{1}'
-      message = message.format(name, messages)
-      raise SchemaMismatch(message)
-    elif len(error_messages) == 1:
-      raise SchemaMismatch(name+': '+error_messages[0])
+    if errors or child_errors:
+      raise _createTreeMismatch(self, errors, child_errors)
 
 
 class SeqType(_CoreType):
@@ -504,34 +685,33 @@ class SeqType(_CoreType):
     if (schema.get('tail')):
       self.tail_schema = rx.make_schema(schema['tail'])
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if not isinstance(value, (list, tuple)):
-      raise SchemaTypeMismatch(name, 'sequence')
+      raise TypeMismatch(self, value)
 
-    if len(value) < len(self.content_schema):
-      raise SchemaMismatch(name+' is less than expected length')
+    errors = []
 
-    if len(value) > len(self.content_schema) and not self.tail_schema:
-      raise SchemaMismatch(name+' exceeds expected length')
+    if len(value) != len(self.content_schema):
+      if len(value) > len(self.content_schema) and self.tail_schema:
+        try:
+          self.tail_schema.validate(value[len(self.content_schema):])
+        except SchemaMismatch as e:
+          errors.append(e)
+      else:
+        err = SeqLengthMismatch(self, value)
+        errors.append(err)
 
-    error_messages = []
+    child_errors = {}
 
-    for i, (schema, item) in enumerate(zip(self.content_schema, value)):
+    for index, (schema, item) in enumerate(zip(self.content_schema, value)):
       try:
-        schema.validate(item, 'item '+str(i))
+        schema.validate(item)
       except SchemaMismatch as e:
-        error_messages.append(str(e))   
+        child_errors[index] = e
 
-    if len(error_messages) > 1:
-      messages = indent('\n'.join(error_messages))
-      message = '{0} sequence is invalid:\n{1}'
-      message = message.format(name, messages)
-      raise SchemaMismatch(message)
-    elif len(error_messages) == 1:
-      raise SchemaMismatch(name+': '+error_messages[0])
+    if errors or child_errors:
+      raise _createTreeMismatch(self, errors, child_errors)
 
-    if len(value) > len(self.content_schema):
-      self.tail_schema.validate(value[len(self.content_schema):], name)
 
 class StrType(_CoreType):
   @staticmethod
@@ -549,15 +729,17 @@ class StrType(_CoreType):
 
     self.length = None
     if 'length' in schema:
-      self.length = Util.make_range_validator(schema['length'])
+      self.length = Range(schema['length'])
 
-  def validate(self, value, name='value'):
+  def validate(self, value):
     if not isinstance(value, str):
-      raise SchemaTypeMismatch(name, 'string')
+      raise TypeMismatch(self, value)
+
     if self.value is not None and value != self.value:
-      raise SchemaValueMismatch(name, '"{0}"'.format(self.value))
-    if self.length:
-      self.length(len(value), name+' length')
+      raise ValueMismatch(self, self)
+
+    if self.length and not self.length(len(value)):
+      raise LengthRangeMismatch(self, value)
 
 core_types = [
   AllType,  AnyType, ArrType, BoolType, DefType,

patacrep/build.py

@@ -44,7 +44,12 @@ class Songbook:
     def __init__(self, raw_songbook, basename):
         # Validate config
         schema = config_model('schema')
+
+        try:
             utils.validate_yaml_schema(raw_songbook, schema)
+        except errors.SchemaError as exception:
+            exception.message = "The songbook file '{}' is not valid\n".format(basename)
+            raise exception
 
         self._raw_config = raw_songbook
         self.basename = basename
@@ -84,7 +89,13 @@ class Songbook:
             self._config['book']['lang'],
             self._config['book']['encoding'],
             )
+
+        try:
             self._config['_template'] = renderer.get_all_variables(self._config.get('template', {}))
+        except errors.SchemaError as exception:
+            exception.message = "The songbook file '{}' is not valid\n{}".format(
+                self.basename, exception.message)
+            raise exception
 
         self._config['_compiled_authwords'] = authors.compile_authwords(
             copy.deepcopy(self._config['authors'])
@@ -110,7 +121,7 @@ class Songbook:
             )
         self._config['filename'] = output.name[:-4]
 
-        self._config['bookoptions'] = iter_bookoptions(self._config)
+        self._config['_bookoptions'] = iter_bookoptions(self._config)
 
         renderer.render_tex(output, self._config)
         self._errors.extend(renderer.errors)

patacrep/content/__init__.py

@@ -17,13 +17,17 @@ dictionary where:
 When analysing the content field of the .sb file, when those keywords are
 met, the corresponding parser is called.
 
+# Keyword examples
+
+    - sort
+    - section*
+    - cwd
+
 # Parsers
 
 A parser is a function which takes as arguments:
     - keyword: the keyword triggering this function;
     - argument: the argument of the keyword (see below);
-    - contentlist: the list of content, that is, the part of the list
-      following the keyword (see example below);
     - config: the configuration object of the current songbook. Plugins can
       change it.
 
@@ -31,30 +35,20 @@ A parser returns a ContentList object (a list of instances of the ContentItem
 class), defined in this module (or of subclasses of this class).
 
 Example: When the following piece of content is met
-
-    ["sorted(author, @title)", "a_song.sg", "another_song.sg"]
-
-the parser associated to keyword 'sorted' get the arguments:
-    - keyword = "sorted"
-    - argument = "author, @title"
-    - contentlist = ["a_song.sg", "another_song.sg"]
+    sort:
+      key: ["author", "title"]
+      content:
+        - "a_song.sg"
+        - "another_song.sg"
+
+the parser associated to keyword 'sort' get the arguments:
+    - keyword = "sort"
+    - argument = {
+        'key': ["author", "title"],
+        'content': ["a_song.sg", "another_song.sg"],
+    }
     - config = <the config file of the current songbook>.
 
-# Keyword
-
-A keyword is either an identifier (alphanumeric characters, and underscore),
-or such an identifier, with some text surrounded by parenthesis (like a
-function definition); this text is called the argument to the keyword.
-Examples:
-    - sorted
-    - sorted(author, @title)
-    - cwd(some/path)
-
-If the keyword has an argument, it can be anything, given that it is
-surrounded by parenthesis. It is up to the plugin to parse this argument. For
-intance, keyword "foo()(( bar()" is a perfectly valid keyword, and the parser
-associated to "foo" will get as argument the string ")(( bar(".
-
 # ContentItem class
 
 The content classes are subclasses of class ContentItem defined in this module.
@@ -72,8 +66,9 @@ import re
 import sys
 
 import jinja2
+import yaml
 
-from patacrep import files
+from patacrep import files, Rx
 from patacrep.errors import SharedError
 
 LOGGER = logging.getLogger(__name__)
@@ -228,12 +223,37 @@ def render(context, content):
 
     return rendered
 
+def validate_parser_argument(raw_schema):
+    """Check that the parser argument respects the schema
+
+    Will raise `ContentError` if the schema is not respected.
+    """
+    rx_checker = Rx.Factory({"register_core_types": True})
+    schema = rx_checker.make_schema(yaml.load(raw_schema))
+
+    def wrap(parse):
+        """Wrap the parse function"""
+        def wrapped(keyword, argument, config):
+            """Check the argument schema before calling the plugin parser"""
+            try:
+                schema.validate(argument)
+            except Rx.SchemaMismatch as exception:
+                msg = 'Invalid syntax:\n---\n{}---\n{}'.format(
+                    yaml.dump({keyword: argument}, default_flow_style=False),
+                    str(exception)
+                )
+                raise ContentError(keyword, msg)
+            return parse(keyword, argument=argument, config=config)
+        return wrapped
+    return wrap
+
+
 def process_content(content, config=None):
     """Process content, and return a list of ContentItem() objects.
 
     Arguments are:
-    - content: the content field of the .sb file, which should be a list, and
-      describe what is to be included in the songbook;
+    - content: the content field of the .sb file, which should be a nested list
+    and describe what is to be included in the songbook;
     - config: the configuration dictionary of the current songbook.
 
     Return: a list of ContentItem objects, corresponding to the content to be
@@ -241,25 +261,25 @@ def process_content(content, config=None):
     """
     contentlist = ContentList()
     plugins = config.get('_content_plugins', {})
-    keyword_re = re.compile(r'^ *(?P<keyword>[\w\*]*) *(\((?P<argument>.*)\))? *$')
     if not content:
-        content = [["song"]]
+        content = [{'song': None}]
+    elif isinstance(content, dict):
+        content = [content]
     for elem in content:
         if isinstance(elem, str):
-            elem = ["song", elem]
+            elem = {'song': elem}
+        if isinstance(elem, dict):
+            for keyword, argument in elem.items():
                 try:
-            match = keyword_re.match(elem[0]).groupdict()
-        except AttributeError:
-            contentlist.append_error(ContentError(elem[0], "Cannot parse content type."))
-            continue
-        (keyword, argument) = (match['keyword'], match['argument'])
                     if keyword not in plugins:
-            contentlist.append_error(ContentError(keyword, "Unknown content type."))
-            continue
+                        raise ContentError(keyword, "Unknown content keyword.")
                     contentlist.extend(plugins[keyword](
                         keyword,
                         argument=argument,
-            contentlist=elem[1:],
                         config=config,
                         ))
+                except ContentError as error:
+                    contentlist.append_error(error)
+        else:
+            contentlist.append_error(ContentError(str(elem), "Unknown content type."))
     return contentlist

patacrep/content/cwd.py

@@ -1,33 +1,46 @@
 """Change base directory before importing songs."""
 
-from patacrep.content import process_content
+import os
+
+from patacrep.content import process_content, validate_parser_argument
 from patacrep.songs import DataSubpath
 
 #pylint: disable=unused-argument
-def parse(keyword, config, argument, contentlist):
-    """Return a list songs included in contentlist, whith a different base path.
+@validate_parser_argument("""
+type: //rec
+required:
+  path: //str
+optional:
+  content: //any
+""")
+def parse(keyword, config, argument):
+    """Return a list songs, whith a different base path.
 
     Arguments:
     - keyword: unused;
     - config: the current songbook configuration dictionary;
-    - argument: a directory;
-    - contentlist: songbook content, that is parsed by
+    - argument: a dict containing:
+        path: string specifying the path to use as root;
+        content: songbook content, that is parsed by
             patacrep.content.process_content().
 
-    This function adds 'argument' to the directories where songs are searched
+    This function adds 'path' to the directories where songs are searched
     for, and then processes the content.
 
-    The 'argument' is added:
-    - first as a relative path to the current directory;
+    The 'path' is added:
+    - first as a relative path to the *.sb file directory;
     - then as a relative path to every path already present in
       config['songdir'].
     """
+    subpath = argument['path']
     old_songdir = config['_songdir']
+    sbdir = os.path.dirname(config['_filepath'])
+
     config['_songdir'] = (
-        [DataSubpath(".", argument)] +
-        [path.clone().join(argument) for path in config['_songdir']]
+        [DataSubpath(sbdir, subpath)] +
+        [path.clone().join(subpath) for path in config['_songdir']]
         )
-    processed_content = process_content(contentlist, config)
+    processed_content = process_content(argument.get('content'), config)
     config['_songdir'] = old_songdir
     return processed_content
 

patacrep/content/include.py

@@ -1,14 +1,15 @@
 """Include an external list of songs
 
 This plugin provides keyword 'include', used to include an external list of
-songs in JSON format.
+songs in JSON or YAML format.
 """
 
-import json
 import os
 import logging
 
-from patacrep.content import process_content, ContentError, ContentList
+import yaml
+
+from patacrep.content import process_content, ContentError, ContentList, validate_parser_argument
 from patacrep import encoding, errors, files
 
 LOGGER = logging.getLogger(__name__)
@@ -28,18 +29,29 @@ def load_from_datadirs(path, datadirs):
         )
 
 #pylint: disable=unused-argument
-def parse(keyword, config, argument, contentlist):
+@validate_parser_argument("""
+type: //any
+of:
+  - type: //str
+  - type: //arr
+    contents: //str
+""")
+def parse(keyword, config, argument):
     """Include an external file content.
 
     Arguments:
         - keyword: the string 'include';
         - config: the current songbook configuration dictionary;
-        - argument: None;
-        - contentlist: a list of file paths to be included.
+        - argument:
+            a list of file paths to be included
+            or a string of the file to include
+
     """
     new_contentlist = ContentList()
+    if isinstance(argument, str):
+        argument = [argument]
 
-    for path in contentlist:
+    for path in argument:
         try:
             filepath = load_from_datadirs(path, config['_datadir'])
         except ContentError as error:
@@ -51,7 +63,7 @@ def parse(keyword, config, argument, contentlist):
                 filepath,
                 encoding=config['book']['encoding']
                 ) as content_file:
-                new_content = json.load(content_file)
+                new_content = yaml.load(content_file)
         except Exception as error: # pylint: disable=broad-except
             new_contentlist.append_error(ContentError(
                 keyword="include",

patacrep/content/section.py

@@ -1,6 +1,6 @@
 """Allow LaTeX sections (starred or not) as content of a songbook."""
 
-from patacrep.content import ContentItem, ContentError, ContentList, EmptyContentList
+from patacrep.content import ContentItem, ContentList, validate_parser_argument
 
 KEYWORDS = [
     "part",
@@ -23,36 +23,39 @@ class Section(ContentItem):
         self.short = short
 
     def render(self, __context):
-        if self.short is None:
+        if self.short is None or self.keyword not in KEYWORDS:
             return r'\{}{{{}}}'.format(self.keyword, self.name)
         else:
             return r'\{}[{}]{{{}}}'.format(self.keyword, self.short, self.name)
 
 #pylint: disable=unused-argument
-def parse(keyword, argument, contentlist, config):
-    """Parse the contentlist.
+@validate_parser_argument("""
+type: //any
+of:
+  - type: //str
+  - type: //rec
+    required:
+      name: //str
+    optional:
+      short: //str
+""")
+def parse(keyword, argument, config):
+    """Parse the section.
 
     Arguments:
     - keyword (one of "part", "chapter", "section", ... , "subparagraph", and
       their starred versions "part*", "chapter*", ... , "subparagraph*"): the
       section to use;
-    - argument: unused;
-    - contentlist: a list of one or two strings, which are the names (long
-      and short) of the section;
+    - argument:
+        either a string describing the section name
+        or a dict
+            name: Name of the section
+            short: Shortname of the section (only for non starred sections)
     - config: configuration dictionary of the current songbook.
     """
-    try:
-        if (keyword not in KEYWORDS) and (len(contentlist) != 1):
-            raise ContentError(
-                keyword,
-                "Starred section names must have exactly one argument."
-                )
-        if (len(contentlist) not in [1, 2]):
-            raise ContentError(keyword, "Section can have one or two arguments.")
-        return ContentList([Section(keyword, *contentlist)])
-    except ContentError as error:
-        return EmptyContentList(errors=[error])
-
+    if isinstance(argument, str):
+        argument = {'name': argument}
+    return ContentList([Section(keyword, **argument)])
 
 CONTENT_PLUGINS = dict([
     (word, parse)

patacrep/content/song.py

@@ -7,7 +7,7 @@ import textwrap
 
 import jinja2
 
-from patacrep.content import process_content
+from patacrep.content import process_content, validate_parser_argument
 from patacrep.content import ContentError, ContentItem, ContentList
 from patacrep import files, errors
 
@@ -58,18 +58,28 @@ class SongRenderer(ContentItem):
         return self.song.fullpath < other.song.fullpath
 
 #pylint: disable=unused-argument
-def parse(keyword, argument, contentlist, config):
+@validate_parser_argument("""
+type: //any
+of:
+  - type: //nil
+  - type: //str
+  - type: //arr
+    contents: //str
+""")
+def parse(keyword, argument, config):
     """Parse data associated with keyword 'song'.
 
     Arguments:
     - keyword: unused;
-    - argument: unused;
-    - contentlist: a list of strings, which are interpreted as regular
+    - argument: a list of strings, which are interpreted as regular
       expressions (interpreted using the glob module), referring to songs.
     - config: the current songbook configuration dictionary.
 
     Return a list of Song() instances.
     """
+    contentlist = argument
+    if isinstance(contentlist, str):
+        contentlist = [contentlist]
     plugins = config['_song_plugins']
     if '_langs' not in config:
         config['_langs'] = set()
@@ -121,7 +131,6 @@ def parse(keyword, argument, contentlist, config):
                 ))
     return sorted(songlist)
 
-
 CONTENT_PLUGINS = {'song': parse}
 
 

patacrep/content/songsection.py

@@ -1,6 +1,6 @@
 """Allow 'songchapter' and 'songsection' as content of a songbook."""
 
-from patacrep.content import ContentItem, ContentError, ContentList, EmptyContentList
+from patacrep.content import ContentItem, ContentList, validate_parser_argument
 
 KEYWORDS = [
     "songchapter",
@@ -20,25 +20,18 @@ class SongSection(ContentItem):
         return r'\{}{{{}}}'.format(self.keyword, self.name)
 
 #pylint: disable=unused-argument
-def parse(keyword, argument, contentlist, config):
-    """Parse the contentlist.
+@validate_parser_argument("""
+//str
+""")
+def parse(keyword, argument, config):
+    """Parse the songsection.
 
     Arguments:
     - keyword ("songsection" or "songchapter"): the section to use;
-    - argument: unused;
-    - contentlist: a list of one string, which is the name of the section;
+    - argument: name of the section;
     - config: configuration dictionary of the current songbook.
     """
-    try:
-        if (keyword not in KEYWORDS) and (len(contentlist) != 1):
-            raise ContentError(
-                keyword,
-                "Starred section names must have exactly one argument.",
-                )
-        return ContentList([SongSection(keyword, contentlist[0])])
-    except ContentError as error:
-        return EmptyContentList(errors=[error])
-
+    return ContentList([SongSection(keyword, argument)])
 
 CONTENT_PLUGINS = dict([
     (keyword, parse)

patacrep/content/sorted.py → patacrep/content/sort.py

@@ -1,6 +1,6 @@
 """Sorted list of songs.
 
-This plugin provides keyword 'sorted', used to include a sorted list of songs
+This plugin provides keyword 'sort', used to include a sorted list of songs
 to a songbook.
 """
 
@@ -8,13 +8,13 @@ import logging
 import unidecode
 
 from patacrep import files
-from patacrep.content import ContentError, EmptyContentList
-from patacrep.content import process_content
+from patacrep.content import ContentError
+from patacrep.content import process_content, validate_parser_argument
 from patacrep.content.song import OnlySongsError
 
 LOGGER = logging.getLogger(__name__)
 
-DEFAULT_SORT = ['by', 'album', '@title']
+DEFAULT_SORT = ['by', 'album', 'title']
 
 def normalize_string(string):
     """Return a normalized string.
@@ -45,9 +45,9 @@ def key_generator(sort):
         song = songrenderer.song
         songkey = []
         for key in sort:
-            if key == "@title":
+            if key == "title":
                 field = song.unprefixed_titles
-            elif key == "@path":
+            elif key == "path":
                 field = song.fullpath
             elif key == "by":
                 field = song.authors
@@ -67,29 +67,44 @@ def key_generator(sort):
     return ordered_song_keys
 
 #pylint: disable=unused-argument
-def parse(keyword, config, argument, contentlist):
-    """Return a sorted list of songs contained in 'contentlist'.
+@validate_parser_argument("""
+type: //any
+of:
+  - type: //nil
+  - type: //rec
+    optional:
+      key:
+        type: //any
+        of:
+          - //str
+          - type: //arr
+            contents: //str
+      content: //any
+""")
+def parse(keyword, config, argument):
+    """Return a sorted list of songs.
 
     Arguments:
-        - keyword: the string 'sorted';
+        - keyword: the string 'sort';
         - config: the current songbook configuration dictionary;
-        - argument: the list of the fields used to sort songs, as strings
-          separated by commas (e.g. "by, album, @title");
-        - contentlist: the list of content to be sorted. If this content
+        - argument: a dict of:
+            key: the list of the fields used to sort songs (e.g. "by", "album", "title")
+            content: content to be sorted. If this content
                 contain something else than a song, an exception is raised.
     """
-    if argument:
-        sort = [key.strip() for key in argument.split(",")]
-    else:
-        sort = DEFAULT_SORT
+    if argument is None:
+        argument = {}
+    sort = argument.get('key', DEFAULT_SORT)
+    if isinstance(sort, str):
+        sort = [sort]
     try:
-        songlist = process_content(contentlist, config)
+        songlist = process_content(argument.get('content'), config)
     except OnlySongsError as error:
-        return EmptyContentList(errors=[ContentError(keyword, (
+        raise ContentError(keyword, (
             "Content list of this keyword can be only songs (or content "
             "that result into songs), and the following are not:" +
             str(error.not_songs)
-            ))])
+            ))
     return sorted(songlist, key=key_generator(sort))
 
-CONTENT_PLUGINS = {'sorted': parse}
+CONTENT_PLUGINS = {'sort': parse}

patacrep/content/tex.py

@@ -5,7 +5,7 @@ import logging
 import os
 
 from patacrep import files, errors
-from patacrep.content import ContentItem, ContentList, ContentError
+from patacrep.content import ContentItem, ContentList, ContentError, validate_parser_argument
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,26 +22,33 @@ class LaTeX(ContentItem):
             )))
 
 #pylint: disable=unused-argument
-def parse(keyword, argument, contentlist, config):
-    """Parse the contentlist.
+@validate_parser_argument("""
+type: //any
+of:
+  - type: //arr
+    contents: //str
+  - type: //str
+""")
+def parse(keyword, argument, config):
+    """Parse the tex files.
 
     Arguments:
     - keyword: unused;
-    - argument: unused;
-    - contentlist: a list of name of tex files;
+    - argument:
+        a list of tex files to include
+        or a string of the tex file to include;
     - config: configuration dictionary of the current songbook.
     """
-    if not contentlist:
-        LOGGER.warning(
-            "Useless 'tex' content: list of files to include is empty."
-            )
+    if isinstance(argument, str):
+        argument = [argument]
+
     filelist = ContentList()
     basefolders = itertools.chain(
         (path.fullpath for path in config['_songdir']),
         files.iter_datadirs(config['_datadir']),
         files.iter_datadirs(config['_datadir'], 'latex'),
         )
-    for filename in contentlist:
+    for filename in argument:
         checked_file = None
         for path in basefolders:
             if os.path.exists(os.path.join(path, filename)):
@@ -62,5 +69,4 @@ def parse(keyword, argument, contentlist, config):
 
     return filelist
 
-
 CONTENT_PLUGINS = {'tex': parse}

patacrep/data/templates/patacrep.tex

@@ -63,7 +63,7 @@ default:
 
 (* block songbookpackages *)
 \usepackage[
-    (* for option in bookoptions *)((option)),
+    (* for option in _bookoptions *)((option)),
     (* endfor *)
     ]{crepbook}
 (* endblock *)

patacrep/data/templates/songbook_model.yml

@@ -5,6 +5,7 @@ schema:
     template: //any
   required:
     _cache: //bool
+    _filepath: //str
     _datadir:
       type: //arr
       contents: //str
@@ -82,7 +83,7 @@ default:
     lang: en
     encoding: utf-8
     pictures: yes
-    template: default.tex
+    template: patacrep.tex
     onesongperpage: no
 
   chords: # Options relatives aux accords

patacrep/data/templates/songs.tex

@@ -22,7 +22,7 @@
 
 (* block songbookpackages *)
 \usepackage[
-    (* for option in bookoptions *)((option)),
+    (* for option in _bookoptions *)((option)),
     (* endfor *)
     ]{patacrep}
 (* endblock *)

patacrep/errors.py

@@ -7,14 +7,18 @@ class SongbookError(Exception):
     """
     pass
 
-class SBFileError(SongbookError):
-    """Error during songbook file decoding"""
+class SchemaError(SongbookError):
+    """Error on the songbook schema"""
 
-    def __init__(self, message=None):
+    def __init__(self, message='', rx_exception=None):
         super().__init__()
         self.message = message
+        self.rx_exception = rx_exception
 
     def __str__(self):
+        if self.rx_exception:
+            return self.message + str(self.rx_exception)
+        else:
             return self.message
 
 class TemplateError(SongbookError):

patacrep/songbook/__main__.py

@@ -153,6 +153,9 @@ def main():
     default_songbook.update(user_songbook)
     songbook = dict(default_songbook)
 
+    songbook['_filepath'] = os.path.abspath(songbook_path)
+    sbdir = os.path.dirname(songbook['_filepath'])
+
     # Gathering datadirs
     datadirs = []
     if options.datadir:
@@ -162,16 +165,13 @@ def main():
         if isinstance(songbook['book']['datadir'], str):
             songbook['book']['datadir'] = [songbook['book']['datadir']]
         datadirs += [
-            os.path.join(
-                os.path.dirname(os.path.abspath(songbook_path)),
-                path
-                )
+            os.path.join(sbdir, path)
             for path in songbook['book']['datadir']
             ]
         del songbook['book']['datadir']
 
     # Default value
-    datadirs.append(os.path.dirname(os.path.abspath(songbook_path)))
+    datadirs.append(sbdir)
     songbook['_datadir'] = datadirs
     songbook['_cache'] = options.cache[0]
 

patacrep/templates.py

@@ -8,7 +8,7 @@ import yaml
 from jinja2 import Environment, FileSystemLoader, ChoiceLoader, \
         TemplateNotFound, nodes
 from jinja2.ext import Extension
-from jinja2.meta import find_referenced_templates as find_templates
+from jinja2.meta import find_referenced_templates
 
 from patacrep import errors, files, utils
 from patacrep.latex import lang2babel, UnknownLanguage
@@ -162,82 +162,54 @@ class TexBookRenderer(Renderer):
                 )
 
     def get_all_variables(self, user_config):
-        '''
-        Validate template variables (and set defaults when needed)
+        '''Validate template variables (and set defaults when needed)
+
+        Will raise `SchemaError` if any data does not respect the schema
         '''
         data = self.get_template_variables(self.template)
         variables = dict()
-        for name, param in data.items():
-            template_config = user_config.get(name, {})
-            variables[name] = self._get_variables(param, template_config)
+        for templatename, param in data.items():
+            template_config = user_config.get(templatename, {})
+            try:
+                variables[templatename] = self._get_variables(param, template_config)
+            except errors.SchemaError as exception:
+                exception.message += "'template' > '{}' > ".format(templatename)
+                raise exception
         return variables
 
     @staticmethod
     def _get_variables(parameter, user_config):
         '''Get the default value for the parameter, according to the language.
-        '''
-        schema = parameter.get('schema', {})
 
+        Will raise `SchemaError` if the data does not respect the schema
+        '''
         data = utils.DictOfDict(parameter.get('default', {}))
         data.update(user_config)
 
+        schema = parameter.get('schema', {})
         utils.validate_yaml_schema(data, schema)
+
         return data
 
-    def get_template_variables(self, template, skip=None):
+    def get_template_variables(self, basetemplate):
         """Parse the template to extract the variables as a dictionary.
 
         If the template includes or extends other templates, load them as well.
 
         Arguments:
-        - template: the name of the template, as a string.
-        - skip: a list of templates (as strings) to skip: if they are included
+        - basetemplate: the name of the template, as a string.
           in 'template' (or one of its subtemplates), it is not parsed.
         """
-        if not skip:
-            skip = []
         variables = {}
-        (current, templates) = self.parse_template(template)
-        if current:
-            variables[template.name] = current
-
-        for subtemplate in templates:
-            if subtemplate in skip:
+        for templatename, template in self._iter_template_content(basetemplate):
+            match = re.findall(_VARIABLE_REGEXP, template)
+            if not match:
                 continue
-            subtemplate = self.jinjaenv.get_template(subtemplate)
-            variables.update(
-                self.get_template_variables(
-                    subtemplate,
-                    skip + templates
-                    )
-                )
-        return variables
-
-    def parse_template(self, template):
-        """Return (variables, templates).
-
-        Argument:
-        - template: name of the template to parse.
-
-        Return values:
-        - variables: a dictionary of variables contained in 'template', NOT
-          recursively (included templates are not parsed).
-        - templates: list of included temlates, NOT recursively.
-        """
-
-        subvariables = {}
-        templatename = self.jinjaenv.get_template(template).filename
-        with patacrep.encoding.open_read(
-            templatename,
-            encoding=self.encoding
-            ) as template_file:
-            content = template_file.read()
-        subtemplates = list(find_templates(self.jinjaenv.parse(content)))
-        match = re.findall(_VARIABLE_REGEXP, content)
-        if match:
-            for var in match:
+            if templatename not in variables:
+                variables[templatename] = {}
+            for variables_string in match:
                 try:
-                    subvariables.update(yaml.load(var))
+                    variables[templatename].update(yaml.load(variables_string))
                 except ValueError as exception:
                     raise errors.TemplateError(
                         exception,
@@ -246,12 +218,29 @@ class TexBookRenderer(Renderer):
                             "{filename}. The yaml string was:"
                             "\n'''\n{yamlstring}\n'''"
                         ).format(
-                            filename=templatename,
-                            yamlstring=var,
+                            filename=template.filename,
+                            yamlstring=variables_string,
                             )
                         )
+        return variables
 
-        return (subvariables, subtemplates)
+    def _iter_template_content(self, templatename, *, skip=None):
+        """Iterate over template (and subtemplate) content."""
+        if skip is None:
+            skip = []
+        template = self.jinjaenv.get_template(templatename)
+        with patacrep.encoding.open_read(
+            template.filename,
+            encoding=self.encoding
+            ) as contentfile:
+            content = contentfile.read()
+            for subtemplatename in find_referenced_templates(self.jinjaenv.parse(content)):
+                if subtemplatename not in skip:
+                    yield from self._iter_template_content(
+                        subtemplatename,
+                        skip=skip + [templatename],
+                        )
+            yield template.name, content
 
     def render_tex(self, output, context):
         '''Render a template into a .tex file
@@ -264,7 +253,7 @@ class TexBookRenderer(Renderer):
         output.write(self.template.render(context))
 
 
-def transform_options(config, equivalents):
+def _transform_options(config, equivalents):
     """
     Get the equivalent name of the checked options
     """
@@ -285,14 +274,14 @@ def iter_bookoptions(config):
         'pictures':         'pictures',
         'onesongperpage':   'onesongperpage',
     }
-    yield from transform_options(config['book'], book_equivalents)
+    yield from _transform_options(config['book'], book_equivalents)
 
     chords_equivalents = {
         'lilypond':     'lilypond',
         'tablatures':   'tabs',
         'repeatchords': 'repeatchords',
     }
-    yield from transform_options(config['chords'], chords_equivalents)
+    yield from _transform_options(config['chords'], chords_equivalents)
 
     if config['chords']['show']:
         if config['chords']['diagramreminder'] == "important":

patacrep/utils.py

@@ -81,16 +81,14 @@ def yesno(string):
 def validate_yaml_schema(data, schema):
     """Check that the data respects the schema
 
-    Will raise `SBFileError` if the schema is not respected.
+    Will raise `SchemaError` if the schema is not respected.
     """
-    rx_checker = Rx.Factory({"register_core_types": True})
-    schema = rx_checker.make_schema(schema)
+    schema = Rx.make_schema(schema)
 
-    if not isinstance(data, dict):
+    if isinstance(data, DictOfDict):
         data = dict(data)
 
     try:
         schema.validate(data)
     except Rx.SchemaMismatch as exception:
-        msg = 'Could not parse songbook file:\n' + str(exception)
-        raise errors.SBFileError(msg)
+        raise errors.SchemaError(rx_exception=exception)

test/test_content/cwd.control

@@ -1 +1 @@
-["subdir/chordpro.csg"]
+- subdir/chordpro.csg

test/test_content/cwd.source

@@ -1 +1,3 @@
-[["cwd(subdir)"]]
+- cwd:
+    path: subdir
+    content:

test/test_content/cwd_list.control

@@ -1 +1,2 @@
-["subdir/chordpro.csg", "exsong.sg"]
+- subdir/chordpro.csg
+- exsong.sg

test/test_content/cwd_list.source

@@ -1 +1,11 @@
-[["cwd(subdir)", "exsong.sg", "intersong.is", "jsonlist.json", "texfile.tex", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"], "exsong.sg"]
+- cwd:
+    path: subdir
+    content:
+        - "exsong.sg"
+        - "intersong.is"
+        - "jsonlist.json"
+        - "texfile.tex"
+        - "texsong.tsg"
+        - "chordpro.csg"
+        - "subdir/chordpro.csg"
+- "exsong.sg"

test/test_content/datadir/songs/custom_list.yaml

@@ -0,0 +1,6 @@
+- sort:
+    key: "title"
+    content:
+      - exsong.sg
+      - chordpro.csg
+      - subdir/chordpro.csg

test/test_content/datadir_sort/path1_title1_author1.csg

@@ -0,0 +1,5 @@
+{title: Title1}
+{artist: Author1}
+
+Foo bar
+

test/test_content/datadir_sort/path1_title1_author2.csg

@@ -0,0 +1,5 @@
+{title: Title1}
+{artist: Author2}
+
+Foo bar
+

test/test_content/datadir_sort/path1_title2_author1.csg

@@ -0,0 +1,5 @@
+{title: Title2}
+{artist: Author1}
+
+Foo bar
+

test/test_content/datadir_sort/path1_title2_author2.csg

@@ -0,0 +1,5 @@
+{title: Title2}
+{artist: Author2}
+
+Foo bar
+

test/test_content/datadir_sort/path2_title1_author1.csg

@@ -0,0 +1,5 @@
+{title: Title1}
+{artist: Author1}
+
+Foo bar
+

test/test_content/datadir_sort/path2_title1_author2.csg

@@ -0,0 +1,5 @@
+{title: Title1}
+{artist: Author2}
+
+Foo bar
+

test/test_content/datadir_sort/path2_title2_author1.csg

@@ -0,0 +1,5 @@
+{title: Title2}
+{artist: Author1}
+
+Foo bar
+

test/test_content/datadir_sort/path2_title2_author2.csg

@@ -0,0 +1,5 @@
+{title: Title2}
+{artist: Author2}
+
+Foo bar
+

test/test_content/glob.control

@@ -1 +1 @@
-["chordpro.csg"]
+- chordpro.csg

test/test_content/glob.source

@@ -1 +1 @@
-["*.csg"]
+- "*.csg"

test/test_content/include.control

@@ -1 +1,6 @@
-["exsong.sg", "chordpro.csg", "subdir/chordpro.csg"]
+- exsong.sg
+- chordpro.csg
+- subdir/chordpro.csg
+- chordpro.csg
+- subdir/chordpro.csg
+- exsong.sg

test/test_content/include.source

@@ -1 +1,3 @@
-[["include" , "custom_list.json"]]
+- include:
+    - custom_list.json
+- include: custom_list.yaml

test/test_content/invalid.control

@@ -0,0 +1 @@
+[]

test/test_content/invalid.source

@@ -0,0 +1,4 @@
+- ["directly", "a", "list"]
+- invalid_keyword: Test
+- section:
+    short: Missing name

test/test_content/sections.control

@@ -1 +1,11 @@
-["section:Traditional", "exsong.sg", "section:Example", "texsong.tsg", "chordpro.csg", "exsong.sg"]
+- section{First Section!}
+- section{Named section}
+- section[section_short_name]{Section with short name}
+- section*{Section* with short name}
+- part{part section test}
+- chapter{chapter section test}
+- section{section section test}
+- subsection{subsection section test}
+- subsubsection{subsubsection section test}
+- paragraph{paragraph section test}
+- subparagraph{subparagraph section test}

test/test_content/sections.source

@@ -1,6 +1,16 @@
-[["section", "Traditional"],
-                "exsong.sg",
-                ["section", "Example"],
-                "texsong.tsg",
-                "chordpro.csg",
-                "exsong.sg"]
+- section: First Section!
+- section:
+    name: Named section
+- section:
+    name: Section with short name
+    short: section_short_name
+- section*:
+    name: Section* with short name
+    short: section_star_short_name
+- part: part section test
+- chapter: chapter section test
+- section: section section test
+- subsection: subsection section test
+- subsubsection: subsubsection section test
+- paragraph: paragraph section test
+- subparagraph: subparagraph section test

test/test_content/sections_short.control

@@ -1 +0,0 @@
-["section:(tradi)Traditional", "exsong.sg", "section*:Example", "texsong.tsg", "chordpro.csg", "exsong.sg"]

test/test_content/sections_short.source

@@ -1,6 +0,0 @@
-[["section", "Traditional", "tradi"],
-                "exsong.sg",
-                ["section*", "Example"],
-                "texsong.tsg",
-                "chordpro.csg",
-                "exsong.sg"]

test/test_content/songs.control

@@ -1 +1,4 @@
-["exsong.sg", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"]
+- exsong.sg
+- texsong.tsg
+- chordpro.csg
+- subdir/chordpro.csg

test/test_content/songs.source

@@ -1 +1,8 @@
-["exsong.sg", "intersong.is", "jsonlist.json", "texfile.tex", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"]
+- exsong.sg
+- intersong.is
+- jsonlist.json
+- song:
+    - texfile.tex
+    - texsong.tsg
+- song: chordpro.csg
+- subdir/chordpro.csg

test/test_content/songsection.control

@@ -1 +1,6 @@
-["songsection:Traditional", "exsong.sg", "songchapter:English", "texsong.tsg", "chordpro.csg", "exsong.sg"]
+- songsection{Traditional}
+- exsong.sg
+- songchapter{English}
+- texsong.tsg
+- chordpro.csg
+- exsong.sg

test/test_content/songsection.source

@@ -1,6 +1,6 @@
-[["songsection", "Traditional"],
-                "exsong.sg",
-                ["songchapter", "English"],
-                "texsong.tsg",
-                "chordpro.csg",
-                "exsong.sg"]
+- songsection: Traditional
+- "exsong.sg"
+- songchapter: English
+- "texsong.tsg"
+- "chordpro.csg"
+- "exsong.sg"

test/test_content/sort.control

@@ -0,0 +1,27 @@
+- section{Title}
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg"
+- section{Author, Title}
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg"
+- section{Path, Title}
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg"

test/test_content/sort.source

@@ -0,0 +1,15 @@
+- cwd:
+    path: "datadir_sort"
+    content:
+      - section:
+          name: "Title"
+      - sort:
+          key: title
+      - section:
+          name: "Author, Title"
+      - sort:
+          key: [by, title]
+      - section:
+          name: "Path, Title"
+      - sort:
+          key: [path, title]

test/test_content/sorted.control

@@ -1 +0,0 @@
-["chordpro.csg", "exsong.sg", "subdir/chordpro.csg", "texsong.tsg"]

test/test_content/sorted.source

@@ -1 +0,0 @@
-[["sorted(fullpath)"]]

test/test_content/test_content.py

@@ -5,7 +5,9 @@
 import glob
 import os
 import unittest
-import json
+import yaml
+
+from pkg_resources import resource_filename
 
 from patacrep.songs import DataSubpath
 from patacrep import content, files
@@ -18,7 +20,7 @@ from .. import dynamic # pylint: disable=unused-import
 class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
     """Test of the content plugins.
 
-    For any given `foo.source`, it parses the content as a json "content"
+    For any given `foo.source`, it parses the content as a yaml "content"
     argument of a .sb file.
     It controls that the generated file list is equal to the one in `foo.control`.
     """
@@ -51,17 +53,22 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
             """Test that `base.source` produces the correct file list"""
             sourcename = "{}.source".format(base)
             with open(sourcename, mode="r", encoding="utf8") as sourcefile:
-                sbcontent = json.load(sourcefile)
+                sbcontent = yaml.load(sourcefile)
 
+            config = cls.config.copy()
+            config['_filepath'] = base
             with logging_reduced('patacrep.content.song'):
-                expandedlist = content.process_content(sbcontent, cls.config.copy())
+                expandedlist = content.process_content(sbcontent, config)
             sourcelist = [cls._clean_path(elem) for elem in expandedlist]
 
             controlname = "{}.control".format(base)
             if not os.path.exists(controlname):
                 raise Exception("Missing control:" + str(sourcelist).replace("'", '"'))
             with open(controlname, mode="r", encoding="utf8") as controlfile:
-                controllist = json.load(controlfile)
+                controllist = [
+                    elem.replace("@TEST_FOLDER@", files.path2posix(resource_filename(__name__, "")))
+                    for elem in yaml.load(controlfile)
+                    ]
 
             self.assertEqual(controllist, sourcelist)
 
@@ -78,13 +85,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
             return files.path2posix(files.relpath(elem.song.fullpath, songpath))
 
         elif isinstance(elem, section.Section):
-            if elem.short is None:
-                return "{}:{}".format(elem.keyword, elem.name)
-            else:
-                return "{}:({}){}".format(elem.keyword, elem.short, elem.name)
+            return elem.render(None)[1:]
 
         elif isinstance(elem, songsection.SongSection):
-            return "{}:{}".format(elem.keyword, elem.name)
+            return elem.render(None)[1:]
 
         elif isinstance(elem, tex.LaTeX):
             return files.path2posix(elem.filename)

test/test_content/tex.control

@@ -1 +1,2 @@
-["test/test_content/datadir/songs/texfile.tex"]
+- test/test_content/datadir/songs/texfile.tex
+- test/test_content/datadir/songs/texfile.tex

test/test_content/tex.source

@@ -1 +1,4 @@
-[["tex", "texfile.tex", "chordpro.csg"]]
+- tex:
+    - texfile.tex
+    - chordpro.csg
+- tex: texfile.tex

test/test_songbook/content.sb

@@ -2,17 +2,24 @@ book:
   pictures: yes
   datadir: content_datadir
   lang: en
+  template: default.tex
 chords:
   repeatchords: no
   diagramreminder: all
 
-content: [
-  ["section", "Test of section"],
-  ["sorted"],
-  ["songsection", "Test of song section"],
-  ["cwd(content_datadir/content)",
-    "song.csg", "song.tsg",
-    ["tex", "foo.tex"]
-  ],
-  ["include", "include.sbc"]
-  ]
+content:
+  - section: Test of section
+  - sort:
+  - songsection: Test of song section
+  - cwd:
+      # relative to sb file
+      path: content_datadir/content
+      content:
+        - "song.csg"
+        - "song.tsg"
+  - cwd:
+      # relative to datadir
+      path: ../content
+      content:
+        - tex: foo.tex
+  - include: include.sbc

test/test_songbook/content_datadir/songs/include.sbc

@@ -1 +1 @@
-[["section", "This is an included section"]]
+[{"section": "This is an included section"}]

test/test_songbook/datadir.sb

@@ -4,3 +4,4 @@ book:
     - datadir_datadir
     - datadir_datadir2
   lang: en
+  template: default.tex

test/test_songbook/languages.sb

@@ -1,3 +1,4 @@
 book:
+  template: default.tex
   datadir:
   - languages_datadir

test/test_songbook/syntax.sb

@@ -1,4 +1,5 @@
 book:
+  template: default.tex
   datadir:
   - syntax_datadir
   lang: en

test/test_songbook/unicode.sb

@@ -1,4 +1,5 @@
 book:
+  template: default.tex
   datadir:
   - unicode_datadir
   lang: en

texlive_packages.txt

@@ -1,11 +1,7 @@
-babel-english
 babel-esperanto
-babel-french
-babel-german
 babel-italian
 babel-latin
 babel-portuges
-babel-spanish
 ctablestack
 etoolbox
 fancybox