Merge pull request #193 from patacrep/yaml

Songbook file format is now yaml (was json before).

README.rst

@@ -55,7 +55,7 @@ Run
 
 ::
 
-    songbook <songbook_file.sb>
+    songbook <songbook_file.yaml>
     <pdfreader> <songbook_file.pdf>
 
 Look for existing songbook files in `patadata <http://github.com/patacrep/patadata>`_.

examples/README

@@ -1,3 +1,3 @@
 Run the following command to build an example song book.
 
-$ songbook example.sb
+$ songbook example.yaml

examples/example-all.sb

@@ -1,19 +0,0 @@
-{
-"bookoptions" : [
-    "diagram",
-    "repeatchords",
-    "lilypond",
-    "pictures"
-  ],
-"booktype" : "chorded",
-"datadir": ["datadir2"],
-"template" : "patacrep.tex",
-"lang" : "fr",
-"encoding": "utf8",
-"authwords" : {
-  "sep" : ["and", "et"]
-  },
- "datadir" : ".",
- "content": [["sorted"]]
-
-}

examples/example-all.yaml

@@ -0,0 +1,27 @@
+book:
+  lang: fr
+  encoding: utf8
+  template: patacrep.tex
+  datadir: "."
+  pictures: yes
+chords:
+  show: yes
+  diagramreminder: all
+  repeatchords: yes
+
+authors:
+  separators:
+    - "and"
+    - "et"
+content:
+    - sort:
+
+template:
+  patacrep.tex:
+    color:
+      songlink: FF0000
+      hyperlink: 0000FF
+    bgcolor:
+      note: D1E4AE
+      songnumber: AED1E4
+      index: E4AED1 #not enough songs to see it

examples/example-all.yaml.sb

@@ -1,17 +0,0 @@
-bookoptions: 
-  - "diagram"
-  - "repeatchords"
-  - "lilypond"
-  - "pictures"
-booktype: "chorded"
-datadir: "."
-template: "patacrep.tex"
-lang: "fr"
-encoding: "utf8"
-authwords: 
-  sep: 
-    - "and"
-    - "et"
-content: 
-  - 
-    - "sorted"

examples/example-crepbook.sb

@@ -1,22 +0,0 @@
-{
-"template" : "patacrep.tex",
-"bookoptions" : [
-    "importantdiagramonly",
-    "repeatchords",
-    "lilypond",
-    "pictures"
-  ],
-"booktype" : "chorded",
-"lang" : "fr",
-"authwords" : {
-  "sep" : ["and", "et"]
-  },
- "datadir" : ".",
- "content" : [["section", "Traditional"],
-                "chevaliers_de_la_table_ronde.sg",
-                "greensleeves.sg",
-                "vent_frais.sg",
-                ["section", "Example"],
-                "example-fr.sg",
-                "example-en.sg"]
-}

examples/example-crepbook.yaml

@@ -0,0 +1,24 @@
+book:
+  lang: fr
+  encoding: utf8
+  template: patacrep.tex
+  datadir: "."
+  pictures: yes
+chords:
+  show: yes
+  diagramreminder: important
+  repeatchords: yes
+  lilypond: yes
+
+authors:
+  separators:
+    - "and"
+    - "et"
+content:
+  - section: "Traditional"
+  - "chevaliers_de_la_table_ronde.tsg"
+  - "greensleeves.csg"
+  - "vent_frais.tsg"
+  - section: Example
+  - "example-fr.tsg"
+  - "example-en.tsg"

examples/example-layout.sb

@@ -1,22 +0,0 @@
-{
-"template" : "layout.tex",
-"bookoptions" : [
-    "importantdiagramonly",
-    "repeatchords",
-    "lilypond",
-    "pictures"
-  ],
-"booktype" : "chorded",
-"lang" : "fr",
-"authwords" : {
-  "sep" : ["and", "et"]
-  },
- "datadir" : ".",
- "content" : [["section", "Traditional"],
-                "chevaliers_de_la_table_ronde.sg",
-                "greensleeves.sg",
-                "vent_frais.sg",
-                ["section", "Example"],
-                "example-fr.sg",
-                "example-en.sg"]
-}

examples/example-songs.sb

@@ -1,22 +0,0 @@
-{
-"template" : "songs.tex",
-"bookoptions" : [
-    "importantdiagramonly",
-    "repeatchords",
-    "lilypond",
-    "pictures"
-  ],
-"booktype" : "chorded",
-"lang" : "fr",
-"authwords" : {
-  "sep" : ["and", "et"]
-  },
- "datadir" : ".",
- "content" : [["section", "Traditional"],
-                "chevaliers_de_la_table_ronde.sg",
-                "greensleeves.sg",
-                "vent_frais.sg",
-                ["section", "Example"],
-                "example-fr.sg",
-                "example-en.sg"]
-}

examples/example-songs.yaml

@@ -0,0 +1,24 @@
+book:
+  lang: fr
+  encoding: utf8
+  template: songs.tex
+  datadir: "."
+  pictures: yes
+chords:
+  show: yes
+  diagramreminder: important
+  repeatchords: yes
+  lilypond: yes
+
+authors:
+  separators:
+    - "and"
+    - "et"
+content:
+  - section: "Traditional"
+  - "chevaliers_de_la_table_ronde.tsg"
+  - "greensleeves.csg"
+  - "vent_frais.tsg"
+  - section: Example
+  - "example-fr.tsg"
+  - "example-en.tsg"

examples/example-test.sb

@@ -1,14 +0,0 @@
-{
-"bookoptions" : [
-    "diagram",
-    "repeatchords",
-    "lilypond",
-    "pictures"
-  ],
-"lang": "ERROR",
-"booktype" : "chorded",
-"template" : "patacrep.tex",
-"encoding": "utf8",
- "content": ["tests/*.csg", "tests/*.tsg"]
-
-}

examples/example-test.yaml

@@ -0,0 +1,14 @@
+book:
+  lang: ERROR
+  encoding: utf8
+  template: patacrep.tex
+  datadir: "."
+  pictures: yes
+chords:
+  show: yes
+  diagramreminder: all
+  repeatchords: yes
+
+content:
+  - "tests/*.csg"
+  - "tests/*.tsg"

examples/example.sb

@@ -1,21 +0,0 @@
-{
-"bookoptions" : [
-    "importantdiagramonly",
-    "repeatchords",
-    "lilypond",
-    "pictures"
-  ],
-"booktype" : "chorded",
-"lang" : "fr",
-"authwords" : {
-  "sep" : ["and", "et"]
-  },
- "datadir" : ".",
- "content" : [["section", "Traditional"],
-                "chevaliers_de_la_table_ronde.sg",
-                "greensleeves.sg",
-                "vent_frais.sg",
-                ["section", "Example"],
-                "example-fr.sg",
-                "example-en.sg"]
-}               

examples/example.yaml

@@ -0,0 +1,23 @@
+book:
+  lang: fr
+  encoding: utf8
+  datadir: "."
+  pictures: yes
+chords:
+  show: yes
+  diagramreminder: important
+  repeatchords: yes
+  lilypond: yes
+
+authors:
+  separators:
+    - "and"
+    - "et"
+content:
+  - section: "Traditional"
+  - "chevaliers_de_la_table_ronde.tsg"
+  - "greensleeves.csg"
+  - "vent_frais.tsg"
+  - section: Example
+  - "example-fr.tsg"
+  - "example-en.tsg"

examples/example_encoding.sb

@@ -1,24 +0,0 @@
-{
-"bookoptions" : [
-    "importantdiagramonly",
-    "repeatchords",
-    "lilypond",
-    "pictures"
-  ],
-"booktype" : "chorded",
-"lang" : "fr",
-"authwords" : {
-  "sep" : ["and", "et", "À"],
-  "ignore" : ["À"],
-  "after" : ["À"]
-  },
-  "titleprefixwords": ["À"],
- "datadir" : ".",
- "content" : [["section", "Traditional"],
-                "chevaliers_de_la_table_ronde.sg",
-                "greensleeves.sg",
-                "vent_frais.sg",
-                ["section", "Example"],
-                "example-fr.sg",
-                "example-en.sg"]
-}               

examples/example_encoding.yaml

@@ -0,0 +1,32 @@
+book:
+  lang: fr
+  encoding: utf8
+  datadir: "."
+  pictures: yes
+chords:
+  show: yes
+  diagramreminder: important
+  repeatchords: yes
+  lilypond: yes
+
+authors:
+  separators:
+    - "and"
+    - "et"
+    - "À"
+  ignore:
+    - "À"
+  after:
+    - "À"
+
+titles: # Comment sont analysés les titres
+  prefix:
+    - "À"
+content:
+  - section: "Traditionàl"
+  - "chevaliers_de_la_table_ronde.tsg"
+  - "greensleeves.csg"
+  - "vent_frais.tsg"
+  - section: Exâmple
+  - "example-fr.tsg"
+  - "example-en.tsg"

patacrep/Rx.py

@@ -0,0 +1,748 @@
+# Downloaded from https://github.com/rjbs/rx
+# The contents of the Rx repository are copyright (C) 2008, Ricardo SIGNES.
+# They may be distributed under the terms of the 
+# GNU  Public License (GPL) Version 2, June 1991
+
+#pylint: skip-file
+
+import re
+import types
+from numbers import Number
+
+### Exception Classes --------------------------------------------------------
+
+class SchemaError(Exception):
+  pass
+
+class SchemaMismatch(Exception):
+
+  def __init__(self, message, schema, error=None):
+    Exception.__init__(self, message)
+    self.type = schema.subname() 
+    self.error = error
+
+class TypeMismatch(SchemaMismatch):
+  
+  def __init__(self, schema, data):
+    message = 'must be of type {} (was {})'.format(
+      schema.subname(),
+      type(data).__name__
+      )
+
+    SchemaMismatch.__init__(self, message, schema, 'type')
+    self.expected_type = schema.subname()
+    self.value = type(data).__name__
+
+
+class ValueMismatch(SchemaMismatch):
+  
+  def __init__(self, schema, data):
+
+    message = 'must equal {} (was {})'.format(
+      repr(schema.value),
+      repr(data)
+      )
+
+    SchemaMismatch.__init__(self, message, schema, 'value')
+    self.expected_value = schema.value
+    self.value = data
+
+ 
+
+class RangeMismatch(SchemaMismatch):
+  
+  def __init__(self, schema, data):
+    
+    message = 'must be in range {} (was {})'.format(
+      schema.range,
+      data
+      )
+
+    SchemaMismatch.__init__(self, message, schema, 'range')
+    self.range = schema.range
+    self.value = data
+
+
+class LengthRangeMismatch(SchemaMismatch):
+
+  def __init__(self, schema, data):
+    length_range = Range(schema.length)
+
+    if not hasattr(length_range, 'min') and \
+       not hasattr(length_range, 'min_ex'):
+      length_range.min = 0
+
+    message = 'length must be in range {} (was {})'.format(
+      length_range,
+      len(data)
+      )
+
+    SchemaMismatch.__init__(self, message, schema, 'range')
+    self.range = schema.length
+    self.value = len(data)
+
+
+class MissingFieldMismatch(SchemaMismatch):
+
+  def __init__(self, schema, fields):
+
+    if len(fields) == 1:
+      message = 'missing required field: {}'.format(
+        repr(fields[0])
+        )
+    else:
+      message = 'missing required fields: {}'.format(
+        ', '.join(fields)
+        )
+      if len(message) >= 80: # if the line is too long
+        message = 'missing required fields:\n{}'.format(
+          _indent('\n'.join(fields))
+          )
+
+    SchemaMismatch.__init__(self, message, schema, 'missing')
+    self.fields = fields
+
+
+class UnknownFieldMismatch(SchemaMismatch):
+
+  def __init__(self, schema, fields):
+
+    if len(fields) == 1:
+      message = 'unknown field: {}'.format(
+        repr(fields[0])
+        )
+    else:
+      message = 'unknown fields: {}'.format(
+        ', '.join(fields)
+        )
+      if len(message) >= 80: # if the line is too long
+        message = 'unknown fields:\n{}'.format(
+          _indent('\n'.join(fields))
+          )
+
+    SchemaMismatch.__init__(self, message, schema, 'unexpected')
+    self.fields = fields
+
+
+class SeqLengthMismatch(SchemaMismatch):
+  def __init__(self, schema, data):
+
+    expected_length = len(schema.content_schema)
+    message = 'sequence must have {} element{} (had {})'.format(
+      expected_length,
+      's'*(expected_length != 1), # plural
+      len(data)
+      )
+
+    SchemaMismatch.__init__(self, message, schema, 'size')
+    self.expected_length = expected_length
+    self.value = len(data)
+
+
+class TreeMismatch(SchemaMismatch):
+
+  def __init__(self, schema, errors=[], child_errors={}, message=None):
+    
+    ## Create error message
+
+    error_messages = []
+
+    for err in errors:
+      error_messages.append(str(err))
+
+    for key, err in child_errors.items():
+
+      if isinstance(key, int):
+        index = '[item {}]'.format(key)
+      else:
+        index = '{}'.format(repr(key))
+
+      if isinstance(err, TreeMismatch) and \
+          not err.errors and len(err.child_errors) == 1:
+
+        template = '{} > {}'
+
+      else:
+        template = '{} {}'
+
+      msg = template.format(index, err)
+      error_messages.append(msg)
+
+    if message is None:
+      message = 'does not match schema'
+
+    if len(error_messages) == 1:
+      msg = error_messages[0]
+
+    else:
+      msg = '{}:\n{}'.format(
+        message,
+        _indent('\n'.join(error_messages))
+        )
+
+    SchemaMismatch.__init__(self, msg, schema, 'multiple')
+    self.errors = errors
+    self.child_errors = child_errors
+
+def _createTreeMismatch(schema, errors=[], child_errors={}, message=None):
+  if len(errors) == 1 and not child_errors:
+    return errors[0]
+  else:
+    return TreeMismatch(schema, errors, child_errors, message)
+
+### Utilities ----------------------------------------------------------------
+
+class Range(object):
+
+  def __init__(self, opt):
+    if isinstance(opt, Range):
+      for attr in ('min', 'max', 'min_ex', 'max_ex'):
+        if hasattr(opt, attr):
+          setattr(self, attr, getattr(opt, attr))
+    else:
+      if not {'min', 'max', 'min-ex', 'max-ex'}.issuperset(opt):
+        raise ValueError("illegal argument to make_range_check")
+      if {'min', 'min-ex'}.issubset(opt):
+        raise ValueError("Cannot define both exclusive and inclusive min")
+      if {'max', 'max-ex'}.issubset(opt):
+        raise ValueError("Cannot define both exclusive and inclusive max")
+
+      for boundary in ('min', 'max', 'min-ex', 'max-ex'):
+        if boundary in opt:
+          attr = boundary.replace('-', '_')
+          setattr(self, attr, opt[boundary])
+
+  def __call__(self, value):
+    INF = float('inf')
+
+    get = lambda attr, default: getattr(self, attr, default)
+
+    return(
+        get('min',    -INF) <= value and \
+        get('max',     INF) >= value and \
+        get('min_ex', -INF) <  value and \
+        get('max_ex',  INF) >  value
+        )
+
+  def __str__(self):
+    if hasattr(self, 'min'):
+      s = '[{}, '.format(self.min)
+    elif hasattr(self, 'min_ex'):
+      s = '({}, '.format(self.min_ex)
+    else:
+      s = '(-Inf, '
+
+    if hasattr(self, 'max'):
+      s += '{}]'.format(self.max)
+    elif hasattr(self, 'max_ex'):
+      s += '{})'.format(self.max_ex)
+    else:
+      s += 'Inf)'
+    
+    return s
+
+def _indent(text, level=1, whitespace='  '):
+    return '\n'.join(whitespace*level+line for line in text.split('\n'))
+
+    ### Schema Factory Class -----------------------------------------------------
+
+class Factory(object):
+  def __init__(self, register_core_types=True):
+    self.prefix_registry = {
+      '':      'tag:codesimply.com,2008:rx/core/',
+      '.meta': 'tag:codesimply.com,2008:rx/meta/',
+    }
+
+    self.type_registry = {}
+    if register_core_types:
+      for t in core_types: self.register_type(t)
+
+  @staticmethod
+  def _default_prefixes(): pass
+
+  def expand_uri(self, type_name):
+    if re.match('^\w+:', type_name): return type_name
+
+    m = re.match('^/([-._a-z0-9]*)/([-._a-z0-9]+)$', type_name)
+
+    if not m:
+      raise ValueError("couldn't understand type name '{}'".format(type_name))
+
+    prefix, suffix = m.groups()
+
+    if prefix not in self.prefix_registry:
+      raise KeyError(
+        "unknown prefix '{0}' in type name '{}'".format(prefix, type_name)
+      )
+
+    return self.prefix_registry[ prefix ] + suffix
+
+  def add_prefix(self, name, base):
+    if self.prefix_registry.get(name):
+      raise SchemaError("the prefix '{}' is already registered".format(name))
+
+    self.prefix_registry[name] = base;
+
+  def register_type(self, t):
+    t_uri = t.uri()
+
+    if t_uri in self.type_registry:
+      raise ValueError("type already registered for {}".format(t_uri))
+
+    self.type_registry[t_uri] = t
+
+  def learn_type(self, uri, schema):
+    if self.type_registry.get(uri):
+      raise SchemaError(
+        "tried to learn type for already-registered uri {}".format(uri)
+        )
+
+    # make sure schema is valid
+    # should this be in a try/except?
+    self.make_schema(schema)
+
+    self.type_registry[uri] = { 'schema': schema }
+
+  def make_schema(self, schema):
+    if isinstance(schema, str):
+      schema = { 'type': schema }
+
+    if not isinstance(schema, dict):
+      raise SchemaError('invalid schema argument to make_schema')
+
+    uri = self.expand_uri(schema['type'])
+
+    if not self.type_registry.get(uri):
+      raise SchemaError("unknown type {}".format(uri))
+
+    type_class = self.type_registry[uri]
+
+    if isinstance(type_class, dict):
+      if not {'type'}.issuperset(schema):
+        raise SchemaError('composed type does not take check arguments')
+      return self.make_schema(type_class['schema'])
+    else:
+      return type_class(schema, self)
+
+std_factory = None
+def make_schema(schema):
+  global std_factory
+  if std_factory is None:
+    std_factory = Factory()
+  return std_factory.make_schema(schema)
+
+### Core Type Base Class -------------------------------------------------
+
+class _CoreType(object):
+  @classmethod
+  def uri(self):
+    return 'tag:codesimply.com,2008:rx/core/' + self.subname()
+
+  def __init__(self, schema, rx):
+    if not {'type'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //{}'.format(self.subname()))
+
+  def check(self, value):
+    try:
+      self.validate(value)
+    except SchemaMismatch:
+      return False
+    return True
+
+  def validate(self, value):
+    raise SchemaMismatch('Tried to validate abstract base schema class', self)
+
+### Core Schema Types --------------------------------------------------------
+
+class AllType(_CoreType):
+  @staticmethod
+  def subname(): return 'all'
+
+  def __init__(self, schema, rx):
+    if not {'type', 'of'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //all')
+    
+    if not schema.get('of'):
+      raise SchemaError('no alternatives given in //all of')
+
+    self.alts = [rx.make_schema(s) for s in schema['of']]
+
+  def validate(self, value):
+    errors = []
+    for schema in self.alts:
+      try:
+        schema.validate(value)
+      except SchemaMismatch as e:
+        errors.append(e)
+
+    if errors:
+      raise _createTreeMismatch(self, errors)
+
+
+class AnyType(_CoreType):
+  @staticmethod
+  def subname(): return 'any'
+
+  def __init__(self, schema, rx):
+    self.alts = None
+
+    if not {'type', 'of'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //any')
+    
+    if 'of' in schema:
+      if not schema['of']: 
+        raise SchemaError('no alternatives given in //any of')
+
+      self.alts = [ rx.make_schema(alt) for alt in schema['of'] ]
+
+  def validate(self, value):
+    if self.alts is None:
+      return
+    
+    errors = []
+
+    for schema in self.alts:
+      try:
+        schema.validate(value)
+        break
+      except SchemaMismatch as e:
+        errors.append(e)
+
+    if len(errors) == len(self.alts):
+      message = 'must satisfy at least one of the following'
+      raise _createTreeMismatch(self, errors, message=message)
+
+
+class ArrType(_CoreType):
+  @staticmethod
+  def subname(): return 'arr'
+
+  def __init__(self, schema, rx):
+    self.length = None
+
+    if not {'type', 'contents', 'length'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //arr')
+
+    if not schema.get('contents'):
+      raise SchemaError('no contents provided for //arr')
+
+    self.content_schema = rx.make_schema(schema['contents'])
+
+    if schema.get('length'):
+      self.length = Range(schema['length'])
+
+  def validate(self, value):
+    if not isinstance(value, (list, tuple)):
+      raise TypeMismatch(self, value)
+
+    errors = []
+    if self.length and not self.length(len(value)):
+      err = LengthRangeMismatch(self, value)
+      errors.append(err)
+
+    child_errors = {}
+
+    for key, item in enumerate(value):
+      try:
+        self.content_schema.validate(item)
+      except SchemaMismatch as e:
+        child_errors[key] = e
+    if errors or child_errors:
+      raise _createTreeMismatch(self, errors, child_errors)
+
+
+class BoolType(_CoreType):
+  @staticmethod
+  def subname(): return 'bool'
+
+  def validate(self, value,):
+    if not isinstance(value, bool):
+      raise TypeMismatch(self, value)
+
+
+class DefType(_CoreType):
+  @staticmethod
+  def subname(): return 'def'
+
+  def validate(self, value):
+    if value is None:
+      raise TypeMismatch(self, value)
+
+
+class FailType(_CoreType):
+  @staticmethod
+  def subname(): return 'fail'
+
+  def check(self, value): return False
+
+  def validate(self, value):
+    raise SchemaMismatch(
+      'is of fail type, automatically invalid.',
+      self,
+      'fail'
+      )
+
+
+class IntType(_CoreType):
+  @staticmethod
+  def subname(): return 'int'
+
+  def __init__(self, schema, rx):
+    if not {'type', 'range', 'value'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //int')
+
+    self.value = None
+    if 'value' in schema:
+      if not isinstance(schema['value'], Number) or schema['value'] % 1 != 0:
+        raise SchemaError('invalid value parameter for //int')
+      self.value = schema['value']
+
+    self.range = None
+    if 'range' in schema:
+      self.range = Range(schema['range'])
+
+  def validate(self, value):
+    if not isinstance(value, Number) or isinstance(value, bool) or value%1:
+      raise TypeMismatch(self, value)
+
+    if self.range and not self.range(value):
+      raise RangeMismatch(self, value)
+
+    if self.value is not None and value != self.value:
+      raise ValueMismatch(self, value)
+
+
+class MapType(_CoreType):
+  @staticmethod
+  def subname(): return 'map'
+
+  def __init__(self, schema, rx):
+    self.allowed = set()
+
+    if not {'type', 'values'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //map')
+
+    if not schema.get('values'):
+      raise SchemaError('no values given for //map')
+
+    self.value_schema = rx.make_schema(schema['values'])
+
+  def validate(self, value):
+    if not isinstance(value, dict):
+      raise TypeMismatch(self, value)
+
+    child_errors = {}
+
+    for key, val in value.items():
+      try:
+        self.value_schema.validate(val)
+      except SchemaMismatch as e:
+        child_errors[key] = e
+
+    if child_errors:
+      raise _createTreeMismatch(self, child_errors=child_errors)
+
+
+class NilType(_CoreType):
+  @staticmethod
+  def subname(): return 'nil'
+
+  def check(self, value): return value is None
+
+  def validate(self, value):
+    if value is not None:
+      raise TypeMismatch(self, value)
+
+
+class NumType(_CoreType):
+  @staticmethod
+  def subname(): return 'num'
+
+  def __init__(self, schema, rx):
+    if not {'type', 'range', 'value'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //num')
+
+    self.value = None
+    if 'value' in schema:
+      if not isinstance(schema['value'], Number):
+        raise SchemaError('invalid value parameter for //num')
+      self.value = schema['value']
+
+    self.range = None
+
+    if schema.get('range'):
+      self.range = Range(schema['range'])
+
+  def validate(self, value):
+    if not isinstance(value, Number) or isinstance(value, bool):
+      raise TypeMismatch(self, value)
+
+    if self.range and not self.range(value):
+      raise RangeMismatch(self, value)
+
+    if self.value is not None and value != self.value:
+      raise ValueMismatch(self, value)
+
+
+class OneType(_CoreType):
+  @staticmethod
+  def subname(): return 'one'
+
+  def validate(self, value):
+    if not isinstance(value, (Number, str)):
+      raise TypeMismatch(self, value)
+
+
+class RecType(_CoreType):
+  @staticmethod
+  def subname(): return 'rec'
+
+  def __init__(self, schema, rx):
+    if not {'type', 'rest', 'required', 'optional'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //rec')
+
+    self.known = set()
+    self.rest_schema = None
+    if schema.get('rest'): self.rest_schema = rx.make_schema(schema['rest'])
+
+    for which in ('required', 'optional'):
+      setattr(self, which, {})
+      for field in schema.get(which, {}).keys():
+        if field in self.known:
+          raise SchemaError(
+            '%s appears in both required and optional' % field
+            )
+
+        self.known.add(field)
+
+        self.__getattribute__(which)[field] = rx.make_schema(
+          schema[which][field]
+        )
+
+  def validate(self, value):
+    if not isinstance(value, dict):
+      raise TypeMismatch(self, value)
+
+    errors = []
+    child_errors = {}
+
+    missing_fields = []
+
+    for field in self.required:
+
+      if field not in value:
+        missing_fields.append(field)
+      else:
+        try:
+          self.required[field].validate(value[field])
+        except SchemaMismatch as e:
+          child_errors[field] = e
+
+    if missing_fields:
+      err = MissingFieldMismatch(self, missing_fields)
+      errors.append(err)
+
+    for field in self.optional:
+      if field not in value: continue
+
+      try:
+        self.optional[field].validate(value[field]) 
+      except SchemaMismatch as e:
+        child_errors[field] = e
+
+    unknown = [k for k in value.keys() if k not in self.known]
+
+    if unknown:
+      if self.rest_schema:
+        rest = {key: value[key] for key in unknown}
+        try:
+          self.rest_schema.validate(rest)
+        except SchemaMismatch as e:
+          errors.append(e)
+      else:
+        fields = _indent('\n'.join(unknown))
+        err = UnknownFieldMismatch(self, unknown)
+        errors.append(err)
+    
+    if errors or child_errors:
+      raise _createTreeMismatch(self, errors, child_errors)
+
+
+class SeqType(_CoreType):
+  @staticmethod
+  def subname(): return 'seq'
+
+  def __init__(self, schema, rx):
+    if not {'type', 'contents', 'tail'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //seq')
+
+    if not schema.get('contents'):
+      raise SchemaError('no contents provided for //seq')
+
+    self.content_schema = [ rx.make_schema(s) for s in schema['contents'] ]
+
+    self.tail_schema = None
+    if (schema.get('tail')):
+      self.tail_schema = rx.make_schema(schema['tail'])
+
+  def validate(self, value):
+    if not isinstance(value, (list, tuple)):
+      raise TypeMismatch(self, value)
+
+    errors = []
+
+    if len(value) != len(self.content_schema):
+      if len(value) > len(self.content_schema) and self.tail_schema:
+        try:
+          self.tail_schema.validate(value[len(self.content_schema):])
+        except SchemaMismatch as e:
+          errors.append(e)
+      else:
+        err = SeqLengthMismatch(self, value)
+        errors.append(err)
+
+    child_errors = {}
+
+    for index, (schema, item) in enumerate(zip(self.content_schema, value)):
+      try:
+        schema.validate(item)
+      except SchemaMismatch as e:
+        child_errors[index] = e
+
+    if errors or child_errors:
+      raise _createTreeMismatch(self, errors, child_errors)
+
+
+class StrType(_CoreType):
+  @staticmethod
+  def subname(): return 'str'
+
+  def __init__(self, schema, rx):
+    if not {'type', 'value', 'length'}.issuperset(schema):
+      raise SchemaError('unknown parameter for //str')
+
+    self.value = None
+    if 'value' in schema:
+      if not isinstance(schema['value'], str):
+        raise SchemaError('invalid value parameter for //str')
+      self.value = schema['value']
+
+    self.length = None
+    if 'length' in schema:
+      self.length = Range(schema['length'])
+
+  def validate(self, value):
+    if not isinstance(value, str):
+      raise TypeMismatch(self, value)
+
+    if self.value is not None and value != self.value:
+      raise ValueMismatch(self, self)
+
+    if self.length and not self.length(len(value)):
+      raise LengthRangeMismatch(self, value)
+
+core_types = [
+  AllType,  AnyType, ArrType, BoolType, DefType,
+  FailType, IntType, MapType, NilType,  NumType,
+  OneType,  RecType, SeqType, StrType
+]

patacrep/authors.py

@@ -5,11 +5,6 @@ import re
 
 LOGGER = logging.getLogger(__name__)
 
-DEFAULT_AUTHWORDS = {
-    "after": ["by"],
-    "ignore": ["unknown"],
-    "sep": ["and"],
-    }
 RE_AFTER = r"^.*\b{}\b(.*)$"
 RE_SEPARATOR = r"^(.*)\b *{} *(\b.*)?$"
 
@@ -18,23 +13,17 @@ def compile_authwords(authwords):
 
     This regexp will later be used to match these words in authors strings.
     """
-    # Fill missing values
+    return {
-    for (key, value) in DEFAULT_AUTHWORDS.items():
+        'ignore': authwords.get('ignore', []),
-        if key not in authwords:
+        'after': [
-            authwords[key] = value
-
-    # Compilation
-    authwords['after'] = [
             re.compile(RE_AFTER.format(word), re.LOCALE)
             for word in authwords['after']
-        ]
+            ],
-    authwords['sep'] = [
+        'separators': [
             re.compile(RE_SEPARATOR.format(word), re.LOCALE)
-        for word in ([" %s" % word for word in authwords['sep']] + [',', ';'])
+            for word in ([" %s" % word for word in authwords['separators']] + [',', ';'])
-        ]
+            ],
-
+        }
-    return authwords
-
 
 def split_author_names(string):
     r"""Split author between first and last name.
@@ -60,12 +49,12 @@ def split_author_names(string):
     return (chunks[-1].strip(), " ".join(chunks[:-1]).strip())
 
 
-def split_sep_author(string, sep):
+def split_sep_author(string, separators):
     """Split authors string according to separators.
 
     Arguments:
     - string: string containing authors names ;
-    - sep: regexp matching a separator.
+    - separators: regexp matching a separator.
 
     >>> split_sep_author("Tintin and Milou", re.compile(RE_SEPARATOR.format("and")))
     ['Tintin', 'Milou']
@@ -73,12 +62,12 @@ def split_sep_author(string, sep):
     ['Tintin']
     """
     authors = []
-    match = sep.match(string)
+    match = separators.match(string)
     while match:
         if match.group(2) is not None:
             authors.append(match.group(2).strip())
         string = match.group(1)
-        match = sep.match(string)
+        match = separators.match(string)
     authors.insert(0, string.strip())
     return authors
 
@@ -105,7 +94,7 @@ def processauthors_removeparen(authors_string):
             dest += char
     return dest
 
-def processauthors_split_string(authors_string, sep):
+def processauthors_split_string(authors_string, separators):
     """Split strings
 
     See docstring of processauthors() for more information.
@@ -121,7 +110,7 @@ def processauthors_split_string(authors_string, sep):
     ['Tintin', 'Milou']
     """
     authors_list = [authors_string]
-    for sepword in sep:
+    for sepword in separators:
         dest = []
         for author in authors_list:
             dest.extend(split_sep_author(author, sepword))
@@ -171,7 +160,7 @@ def processauthors_clean_authors(authors_list):
         if author.lstrip()
         ]
 
-def processauthors(authors_string, after=None, ignore=None, sep=None):
+def processauthors(authors_string, after=None, ignore=None, separators=None):
     r"""Return an iterator of authors
 
     For example, in the following call:
@@ -186,7 +175,7 @@ def processauthors(authors_string, after=None, ignore=None, sep=None):
     ...   **compile_authwords({
     ...         'after': ["by"],
     ...         'ignore': ["anonymous"],
-    ...         'sep': ["and", ","],
+    ...         'separators': ["and", ","],
     ...         })
     ...   )) == {("Blake", "William"), ("Parry", "Hubert"), ("Royal~Choir~of~FooBar", "The")}
     True
@@ -198,7 +187,7 @@ def processauthors(authors_string, after=None, ignore=None, sep=None):
     # "Lyrics by William Blake, music by Hubert Parry,
                 and sung by The Royal~Choir~of~FooBar"
 
-    2) String is split, separators being comma and words from "sep".
+    2) String is split, separators being comma and words from "separators".
     # ["Lyrics by William Blake", "music by Hubert Parry",
                 "sung by The Royal~Choir~of~FooBar"]
 
@@ -216,8 +205,8 @@ def processauthors(authors_string, after=None, ignore=None, sep=None):
     # ]
     """
 
-    if not sep:
+    if not separators:
-        sep = []
+        separators = []
     if not after:
         after = []
     if not ignore:
@@ -230,17 +219,17 @@ def processauthors(authors_string, after=None, ignore=None, sep=None):
                         processauthors_removeparen(
                             authors_string
                             ),
-                        sep),
+                        separators),
                     after),
                 ignore)
         ):
         yield split_author_names(author)
 
-def process_listauthors(authors_list, after=None, ignore=None, sep=None):
+def process_listauthors(authors_list, after=None, ignore=None, separators=None):
     """Process a list of authors, and return the list of resulting authors."""
     authors = []
     for sublist in [
-            processauthors(string, after, ignore, sep)
+            processauthors(string, after, ignore, separators)
             for string in authors_list
         ]:
         authors.extend(sublist)

patacrep/build.py

@@ -1,4 +1,4 @@
-"""Build a songbook, according to parameters found in a .sb file."""
+"""Build a songbook, according to parameters found in a .yaml file."""
 
 import codecs
 import copy
@@ -8,10 +8,11 @@ import threading
 import os.path
 from subprocess import Popen, PIPE, call, check_call
 
-from patacrep import authors, content, errors, files
+import yaml
+
+from patacrep import authors, content, encoding, errors, files, pkg_datapath, utils
 from patacrep.index import process_sxd
-from patacrep.templates import TexBookRenderer
+from patacrep.templates import TexBookRenderer, iter_bookoptions
-from patacrep.songs import DataSubpath, DEFAULT_CONFIG
 
 LOGGER = logging.getLogger(__name__)
 EOL = "\n"
@@ -32,44 +33,27 @@ GENERATED_EXTENSIONS = [
 
 # pylint: disable=too-few-public-methods
 class Songbook:
-    """Represent a songbook (.sb) file.
+    """Represent a songbook (.yaml) file.
 
     - Low level: provide a Python representation of the values stored in the
-      '.sb' file.
+      '.yaml' file.
     - High level: provide some utility functions to manipulate these data.
     """
 
     def __init__(self, raw_songbook, basename):
+        # Validate config
+        schema = config_model('schema')
+
+        try:
+            utils.validate_yaml_schema(raw_songbook, schema)
+        except errors.SchemaError as exception:
+            exception.message = "The songbook file '{}' is not valid\n".format(basename)
+            raise exception
+
         self._raw_config = raw_songbook
-        self.config = raw_songbook
         self.basename = basename
         self._errors = list()
         self._config = dict()
-        # Some special keys have their value processed.
-        self._set_datadir()
-
-    def _set_datadir(self):
-        """Set the default values for datadir"""
-        try:
-            if isinstance(self._raw_config['datadir'], str):
-                self._raw_config['datadir'] = [self._raw_config['datadir']]
-        except KeyError:  # No datadir in the raw_songbook
-            self._raw_config['datadir'] = [os.path.abspath('.')]
-
-        abs_datadir = []
-        for path in self._raw_config['datadir']:
-            if os.path.exists(path) and os.path.isdir(path):
-                abs_datadir.append(os.path.abspath(path))
-            else:
-                LOGGER.warning(
-                    "Ignoring non-existent datadir '{}'.".format(path)
-                    )
-
-        self._raw_config['datadir'] = abs_datadir
-        self._raw_config['_songdir'] = [
-            DataSubpath(path, 'songs')
-            for path in self._raw_config['datadir']
-            ]
 
     def write_tex(self, output):
         """Build the '.tex' file corresponding to self.
@@ -78,29 +62,33 @@ class Songbook:
         - output: a file object, in which the file will be written.
         """
         # Updating configuration
-        self._config = DEFAULT_CONFIG.copy()
+        self._config = self._raw_config.copy()
-        self._config.update(self._raw_config)
         renderer = TexBookRenderer(
-            self._config['template'],
+            self._config['book']['template'],
-            self._config['datadir'],
+            self._config['_datadir'],
-            self._config['lang'],
+            self._config['book']['lang'],
-            self._config['encoding'],
+            self._config['book']['encoding'],
             )
-        self._config.update(renderer.get_variables())
+
-        self._config.update(self._raw_config)
+        try:
+            self._config['_template'] = renderer.get_all_variables(self._config.get('template', {}))
+        except errors.SchemaError as exception:
+            exception.message = "The songbook file '{}' is not valid\n{}".format(
+                self.basename, exception.message)
+            raise exception
 
         self._config['_compiled_authwords'] = authors.compile_authwords(
-            copy.deepcopy(self._config['authwords'])
+            copy.deepcopy(self._config['authors'])
             )
 
         # Loading custom plugins
         self._config['_content_plugins'] = files.load_plugins(
-            datadirs=self._config.get('datadir', []),
+            datadirs=self._config['_datadir'],
             root_modules=['content'],
             keyword='CONTENT_PLUGINS',
             )
         self._config['_song_plugins'] = files.load_plugins(
-            datadirs=self._config.get('datadir', []),
+            datadirs=self._config['_datadir'],
             root_modules=['songs'],
             keyword='SONG_RENDERERS',
             )['tsg']
@@ -113,11 +101,13 @@ class Songbook:
             )
         self._config['filename'] = output.name[:-4]
 
+        self._config['_bookoptions'] = iter_bookoptions(self._config)
+
         renderer.render_tex(output, self._config)
 
         # Get all errors, and maybe exit program
         self._errors.extend(renderer.errors)
-        if self.config['_error'] == "failonbook":
+        if self._config['_error'] == "failonbook":
             if self.has_errors():
                 raise errors.SongbookError("Some songs contain errors. Stopping as requested.")
 
@@ -154,7 +144,7 @@ class Songbook:
 
     def requires_lilypond(self):
         """Tell if lilypond is part of the bookoptions"""
-        return 'lilypond' in self.config.get('bookoptions', [])
+        return 'lilypond' in iter_bookoptions(self._raw_config)
 
 def _log_pipe(pipe):
     """Log content from `pipe`."""
@@ -179,8 +169,8 @@ class SongbookBuilder:
 
     def __init__(self, raw_songbook):
         # Basename of the songbook to be built.
-        self.basename = raw_songbook['_basename']
+        self.basename = raw_songbook['_outputname']
-        # Representation of the .sb songbook configuration file.
+        # Representation of the .yaml songbook configuration file.
         self.songbook = Songbook(raw_songbook, self.basename)
 
     def _run_once(self, function, *args, **kwargs):
@@ -310,6 +300,10 @@ class SongbookBuilder:
         standard_output.join()
         process.wait()
 
+        # Close the stdout and stderr to prevent ResourceWarning:
+        process.stdout.close()
+        process.stderr.close()
+
         if process.returncode:
             raise errors.LatexCompilationError(self.basename)
 
@@ -365,3 +359,18 @@ class SongbookBuilder:
                     os.unlink(self.basename + ext)
                 except Exception as exception:
                     raise errors.CleaningError(self.basename + ext, exception)
+
+
+def config_model(key):
+    """Get the model structure
+
+    key can be:
+        - schema
+        - default
+        - description
+    """
+    model_path = pkg_datapath('templates', 'songbook_model.yml')
+    with encoding.open_read(model_path) as model_file:
+        data = yaml.load(model_file)
+
+    return data.get(key, {})

patacrep/content/__init__.py

@@ -1,7 +1,7 @@
 """Content plugin management.
 
 Content that can be included in a songbook is controlled by plugins. From the
-user (or .sb file) point of view, each piece of content is introduced by a
+user (or .yaml file) point of view, each piece of content is introduced by a
 keyword. This keywold is associated with a plugin (a submodule of this very
 module), which parses the content, and return a ContentList object, which is
 little more than a list of instances of the ContentItem class.
@@ -14,16 +14,20 @@ dictionary where:
     - keys are keywords,
     - values are parsers (see below).
 
-When analysing the content field of the .sb file, when those keywords are
+When analysing the content field of the .yaml file, when those keywords are
 met, the corresponding parser is called.
 
+# Keyword examples
+
+    - sort
+    - section*
+    - cwd
+
 # Parsers
 
 A parser is a function which takes as arguments:
     - keyword: the keyword triggering this function;
     - argument: the argument of the keyword (see below);
-    - contentlist: the list of content, that is, the part of the list
-      following the keyword (see example below);
     - config: the configuration object of the current songbook. Plugins can
       change it.
 
@@ -31,30 +35,20 @@ A parser returns a ContentList object (a list of instances of the ContentItem
 class), defined in this module (or of subclasses of this class).
 
 Example: When the following piece of content is met
-
+    sort:
-    ["sorted(author, @title)", "a_song.sg", "another_song.sg"]
+      key: ["author", "title"]
-
+      content:
-the parser associated to keyword 'sorted' get the arguments:
+        - "a_song.sg"
-    - keyword = "sorted"
+        - "another_song.sg"
-    - argument = "author, @title"
+
-    - contentlist = ["a_song.sg", "another_song.sg"]
+the parser associated to keyword 'sort' get the arguments:
+    - keyword = "sort"
+    - argument = {
+        'key': ["author", "title"],
+        'content': ["a_song.sg", "another_song.sg"],
+    }
     - config = <the config file of the current songbook>.
 
-# Keyword
-
-A keyword is either an identifier (alphanumeric characters, and underscore),
-or such an identifier, with some text surrounded by parenthesis (like a
-function definition); this text is called the argument to the keyword.
-Examples:
-    - sorted
-    - sorted(author, @title)
-    - cwd(some/path)
-
-If the keyword has an argument, it can be anything, given that it is
-surrounded by parenthesis. It is up to the plugin to parse this argument. For
-intance, keyword "foo()(( bar()" is a perfectly valid keyword, and the parser
-associated to "foo" will get as argument the string ")(( bar(".
-
 # ContentItem class
 
 The content classes are subclasses of class ContentItem defined in this module.
@@ -72,8 +66,9 @@ import re
 import sys
 
 import jinja2
+import yaml
 
-from patacrep import files
+from patacrep import files, Rx
 from patacrep.errors import SharedError
 
 LOGGER = logging.getLogger(__name__)
@@ -228,12 +223,36 @@ def render(context, content):
 
     return rendered
 
+def validate_parser_argument(raw_schema):
+    """Check that the parser argument respects the schema
+
+    Will raise `ContentError` if the schema is not respected.
+    """
+    schema = Rx.make_schema(yaml.load(raw_schema))
+
+    def wrap(parse):
+        """Wrap the parse function"""
+        def wrapped(keyword, argument, config):
+            """Check the argument schema before calling the plugin parser"""
+            try:
+                schema.validate(argument)
+            except Rx.SchemaMismatch as exception:
+                msg = 'Invalid syntax:\n---\n{}---\n{}'.format(
+                    yaml.dump({keyword: argument}, default_flow_style=False),
+                    str(exception)
+                )
+                raise ContentError(keyword, msg)
+            return parse(keyword, argument=argument, config=config)
+        return wrapped
+    return wrap
+
+
 def process_content(content, config=None):
     """Process content, and return a list of ContentItem() objects.
 
     Arguments are:
-    - content: the content field of the .sb file, which should be a list, and
+    - content: the content field of the .yaml file, which should be a nested list
-      describe what is to be included in the songbook;
+    and describe what is to be included in the songbook;
     - config: the configuration dictionary of the current songbook.
 
     Return: a list of ContentItem objects, corresponding to the content to be
@@ -241,25 +260,25 @@ def process_content(content, config=None):
     """
     contentlist = ContentList()
     plugins = config.get('_content_plugins', {})
-    keyword_re = re.compile(r'^ *(?P<keyword>[\w\*]*) *(\((?P<argument>.*)\))? *$')
     if not content:
-        content = [["song"]]
+        content = [{'song': None}]
+    elif isinstance(content, dict):
+        content = [content]
     for elem in content:
         if isinstance(elem, str):
-            elem = ["song", elem]
+            elem = {'song': elem}
+        if isinstance(elem, dict):
+            for keyword, argument in elem.items():
                 try:
-            match = keyword_re.match(elem[0]).groupdict()
-        except AttributeError:
-            contentlist.append_error(ContentError(elem[0], "Cannot parse content type."))
-            continue
-        (keyword, argument) = (match['keyword'], match['argument'])
                     if keyword not in plugins:
-            contentlist.append_error(ContentError(keyword, "Unknown content type."))
+                        raise ContentError(keyword, "Unknown content keyword.")
-            continue
                     contentlist.extend(plugins[keyword](
                         keyword,
                         argument=argument,
-            contentlist=elem[1:],
                         config=config,
                         ))
+                except ContentError as error:
+                    contentlist.append_error(error)
+        else:
+            contentlist.append_error(ContentError(str(elem), "Unknown content type."))
     return contentlist

patacrep/content/cwd.py

@@ -1,33 +1,43 @@
 """Change base directory before importing songs."""
 
-from patacrep.content import process_content
+from patacrep.content import process_content, validate_parser_argument
 from patacrep.songs import DataSubpath
 
 #pylint: disable=unused-argument
-def parse(keyword, config, argument, contentlist):
+@validate_parser_argument("""
-    """Return a list songs included in contentlist, whith a different base path.
+type: //rec
+required:
+  path: //str
+optional:
+  content: //any
+""")
+def parse(keyword, config, argument):
+    """Return a list songs, whith a different base path.
 
     Arguments:
     - keyword: unused;
     - config: the current songbook configuration dictionary;
-    - argument: a directory;
+    - argument: a dict containing:
-    - contentlist: songbook content, that is parsed by
+        path: string specifying the path to use as root;
+        content: songbook content, that is parsed by
             patacrep.content.process_content().
 
-    This function adds 'argument' to the directories where songs are searched
+    This function adds 'path' to the directories where songs are searched
     for, and then processes the content.
 
-    The 'argument' is added:
+    The 'path' is added:
-    - first as a relative path to the current directory;
+    - first as a relative path to the *.yaml file directory;
     - then as a relative path to every path already present in
-      config['songdir'].
+      config['songdir'] (which are 'song' dir inside the datadirs).
     """
+    subpath = argument['path']
     old_songdir = config['_songdir']
-    config['_songdir'] = (
+
-        [DataSubpath(".", argument)] +
+    config['_songdir'] = [path.clone().join(subpath) for path in config['_songdir']]
-        [path.clone().join(argument) for path in config['_songdir']]
+    if '_songbookfile_dir' in config:
-        )
+        config['_songdir'].insert(0, DataSubpath(config['_songbookfile_dir'], subpath))
-    processed_content = process_content(contentlist, config)
+
+    processed_content = process_content(argument.get('content'), config)
     config['_songdir'] = old_songdir
     return processed_content
 

patacrep/content/include.py

@@ -1,14 +1,15 @@
 """Include an external list of songs
 
 This plugin provides keyword 'include', used to include an external list of
-songs in JSON format.
+songs in JSON or YAML format.
 """
 
-import json
 import os
 import logging
 
-from patacrep.content import process_content, ContentError, ContentList
+import yaml
+
+from patacrep.content import process_content, ContentError, ContentList, validate_parser_argument
 from patacrep import encoding, errors, files
 
 LOGGER = logging.getLogger(__name__)
@@ -28,20 +29,31 @@ def load_from_datadirs(path, datadirs):
         )
 
 #pylint: disable=unused-argument
-def parse(keyword, config, argument, contentlist):
+@validate_parser_argument("""
+type: //any
+of:
+  - type: //str
+  - type: //arr
+    contents: //str
+""")
+def parse(keyword, config, argument):
     """Include an external file content.
 
     Arguments:
         - keyword: the string 'include';
         - config: the current songbook configuration dictionary;
-        - argument: None;
+        - argument:
-        - contentlist: a list of file paths to be included.
+            a list of file paths to be included
+            or a string of the file to include
+
     """
     new_contentlist = ContentList()
+    if isinstance(argument, str):
+        argument = [argument]
 
-    for path in contentlist:
+    for path in argument:
         try:
-            filepath = load_from_datadirs(path, config.get('datadir', []))
+            filepath = load_from_datadirs(path, config['_datadir'])
         except ContentError as error:
             new_contentlist.append_error(error)
             continue
@@ -49,9 +61,9 @@ def parse(keyword, config, argument, contentlist):
         try:
             with encoding.open_read(
                 filepath,
-                encoding=config['encoding']
+                encoding=config['book']['encoding']
                 ) as content_file:
-                new_content = json.load(content_file)
+                new_content = yaml.load(content_file)
         except Exception as error: # pylint: disable=broad-except
             new_contentlist.append_error(ContentError(
                 keyword="include",
@@ -59,9 +71,9 @@ def parse(keyword, config, argument, contentlist):
                 ))
             continue
 
-        config["datadir"].append(os.path.abspath(os.path.dirname(filepath)))
+        config['_datadir'].append(os.path.abspath(os.path.dirname(filepath)))
         new_contentlist.extend(process_content(new_content, config))
-        config["datadir"].pop()
+        config['_datadir'].pop()
 
     return new_contentlist
 

patacrep/content/section.py

@@ -1,6 +1,6 @@
 """Allow LaTeX sections (starred or not) as content of a songbook."""
 
-from patacrep.content import ContentItem, ContentError, ContentList, EmptyContentList
+from patacrep.content import ContentItem, ContentList, validate_parser_argument
 
 KEYWORDS = [
     "part",
@@ -23,36 +23,39 @@ class Section(ContentItem):
         self.short = short
 
     def render(self, __context):
-        if self.short is None:
+        if self.short is None or self.keyword not in KEYWORDS:
             return r'\{}{{{}}}'.format(self.keyword, self.name)
         else:
             return r'\{}[{}]{{{}}}'.format(self.keyword, self.short, self.name)
 
 #pylint: disable=unused-argument
-def parse(keyword, argument, contentlist, config):
+@validate_parser_argument("""
-    """Parse the contentlist.
+type: //any
+of:
+  - type: //str
+  - type: //rec
+    required:
+      name: //str
+    optional:
+      short: //str
+""")
+def parse(keyword, argument, config):
+    """Parse the section.
 
     Arguments:
     - keyword (one of "part", "chapter", "section", ... , "subparagraph", and
       their starred versions "part*", "chapter*", ... , "subparagraph*"): the
       section to use;
-    - argument: unused;
+    - argument:
-    - contentlist: a list of one or two strings, which are the names (long
+        either a string describing the section name
-      and short) of the section;
+        or a dict
+            name: Name of the section
+            short: Shortname of the section (only for non starred sections)
     - config: configuration dictionary of the current songbook.
     """
-    try:
+    if isinstance(argument, str):
-        if (keyword not in KEYWORDS) and (len(contentlist) != 1):
+        argument = {'name': argument}
-            raise ContentError(
+    return ContentList([Section(keyword, **argument)])
-                keyword,
-                "Starred section names must have exactly one argument."
-                )
-        if (len(contentlist) not in [1, 2]):
-            raise ContentError(keyword, "Section can have one or two arguments.")
-        return ContentList([Section(keyword, *contentlist)])
-    except ContentError as error:
-        return EmptyContentList(errors=[error])
-
 
 CONTENT_PLUGINS = dict([
     (word, parse)

patacrep/content/song.py

@@ -7,7 +7,7 @@ import textwrap
 
 import jinja2
 
-from patacrep.content import process_content
+from patacrep.content import process_content, validate_parser_argument
 from patacrep.content import ContentError, ContentItem, ContentList
 from patacrep import files, errors
 
@@ -63,19 +63,30 @@ class SongRenderer(ContentItem):
         """Order by song path"""
         return self.song.fullpath < other.song.fullpath
 
-#pylint: disable=unused-argument, too-many-branches
+#pylint: disable=unused-argument
-def parse(keyword, argument, contentlist, config):
+#pylint: disable=too-many-branches
+@validate_parser_argument("""
+type: //any
+of:
+  - type: //nil
+  - type: //str
+  - type: //arr
+    contents: //str
+""")
+def parse(keyword, argument, config):
     """Parse data associated with keyword 'song'.
 
     Arguments:
     - keyword: unused;
-    - argument: unused;
+    - argument: a list of strings, which are interpreted as regular
-    - contentlist: a list of strings, which are interpreted as regular
       expressions (interpreted using the glob module), referring to songs.
     - config: the current songbook configuration dictionary.
 
     Return a list of Song() instances.
     """
+    contentlist = argument
+    if isinstance(contentlist, str):
+        contentlist = [contentlist]
     plugins = config['_song_plugins']
     if '_langs' not in config:
         config['_langs'] = set()
@@ -84,6 +95,8 @@ def parse(keyword, argument, contentlist, config):
         if contentlist:
             break
         contentlist = files.recursive_find(songdir.fullpath, plugins.keys())
+    if contentlist is None:
+        contentlist = [] # No content was set or found
     for elem in contentlist:
         before = len(songlist)
         for songdir in config['_songdir']:
@@ -137,7 +150,6 @@ def parse(keyword, argument, contentlist, config):
                 ))
     return sorted(songlist)
 
-
 CONTENT_PLUGINS = {'song': parse}
 
 

patacrep/content/songsection.py

@@ -1,6 +1,6 @@
 """Allow 'songchapter' and 'songsection' as content of a songbook."""
 
-from patacrep.content import ContentItem, ContentError, ContentList, EmptyContentList
+from patacrep.content import ContentItem, ContentList, validate_parser_argument
 
 KEYWORDS = [
     "songchapter",
@@ -20,25 +20,18 @@ class SongSection(ContentItem):
         return r'\{}{{{}}}'.format(self.keyword, self.name)
 
 #pylint: disable=unused-argument
-def parse(keyword, argument, contentlist, config):
+@validate_parser_argument("""
-    """Parse the contentlist.
+//str
+""")
+def parse(keyword, argument, config):
+    """Parse the songsection.
 
     Arguments:
     - keyword ("songsection" or "songchapter"): the section to use;
-    - argument: unused;
+    - argument: name of the section;
-    - contentlist: a list of one string, which is the name of the section;
     - config: configuration dictionary of the current songbook.
     """
-    try:
+    return ContentList([SongSection(keyword, argument)])
-        if (keyword not in KEYWORDS) and (len(contentlist) != 1):
-            raise ContentError(
-                keyword,
-                "Starred section names must have exactly one argument.",
-                )
-        return ContentList([SongSection(keyword, contentlist[0])])
-    except ContentError as error:
-        return EmptyContentList(errors=[error])
-
 
 CONTENT_PLUGINS = dict([
     (keyword, parse)

patacrep/content/sorted.py → patacrep/content/sort.py

@@ -1,6 +1,6 @@
 """Sorted list of songs.
 
-This plugin provides keyword 'sorted', used to include a sorted list of songs
+This plugin provides keyword 'sort', used to include a sorted list of songs
 to a songbook.
 """
 
@@ -8,13 +8,13 @@ import logging
 import unidecode
 
 from patacrep import files
-from patacrep.content import ContentError, EmptyContentList
+from patacrep.content import ContentError
-from patacrep.content import process_content
+from patacrep.content import process_content, validate_parser_argument
 from patacrep.content.song import OnlySongsError
 
 LOGGER = logging.getLogger(__name__)
 
-DEFAULT_SORT = ['by', 'album', '@title']
+DEFAULT_SORT = ['by', 'album', 'title']
 
 def normalize_string(string):
     """Return a normalized string.
@@ -45,9 +45,9 @@ def key_generator(sort):
         song = songrenderer.song
         songkey = []
         for key in sort:
-            if key == "@title":
+            if key == "title":
                 field = song.unprefixed_titles
-            elif key == "@path":
+            elif key == "path":
                 field = song.fullpath
             elif key == "by":
                 field = song.authors
@@ -67,29 +67,44 @@ def key_generator(sort):
     return ordered_song_keys
 
 #pylint: disable=unused-argument
-def parse(keyword, config, argument, contentlist):
+@validate_parser_argument("""
-    """Return a sorted list of songs contained in 'contentlist'.
+type: //any
+of:
+  - type: //nil
+  - type: //rec
+    optional:
+      key:
+        type: //any
+        of:
+          - //str
+          - type: //arr
+            contents: //str
+      content: //any
+""")
+def parse(keyword, config, argument):
+    """Return a sorted list of songs.
 
     Arguments:
-        - keyword: the string 'sorted';
+        - keyword: the string 'sort';
         - config: the current songbook configuration dictionary;
-        - argument: the list of the fields used to sort songs, as strings
+        - argument: a dict of:
-          separated by commas (e.g. "by, album, @title");
+            key: the list of the fields used to sort songs (e.g. "by", "album", "title")
-        - contentlist: the list of content to be sorted. If this content
+            content: content to be sorted. If this content
                 contain something else than a song, an exception is raised.
     """
-    if argument:
+    if argument is None:
-        sort = [key.strip() for key in argument.split(",")]
+        argument = {}
-    else:
+    sort = argument.get('key', DEFAULT_SORT)
-        sort = DEFAULT_SORT
+    if isinstance(sort, str):
+        sort = [sort]
     try:
-        songlist = process_content(contentlist, config)
+        songlist = process_content(argument.get('content'), config)
     except OnlySongsError as error:
-        return EmptyContentList(errors=[ContentError(keyword, (
+        raise ContentError(keyword, (
             "Content list of this keyword can be only songs (or content "
             "that result into songs), and the following are not:" +
             str(error.not_songs)
-            ))])
+            ))
     return sorted(songlist, key=key_generator(sort))
 
-CONTENT_PLUGINS = {'sorted': parse}
+CONTENT_PLUGINS = {'sort': parse}

patacrep/content/tex.py

@@ -5,7 +5,7 @@ import logging
 import os
 
 from patacrep import files, errors
-from patacrep.content import ContentItem, ContentList, ContentError
+from patacrep.content import ContentItem, ContentList, ContentError, validate_parser_argument
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,26 +22,33 @@ class LaTeX(ContentItem):
             )))
 
 #pylint: disable=unused-argument
-def parse(keyword, argument, contentlist, config):
+@validate_parser_argument("""
-    """Parse the contentlist.
+type: //any
+of:
+  - type: //arr
+    contents: //str
+  - type: //str
+""")
+def parse(keyword, argument, config):
+    """Parse the tex files.
 
     Arguments:
     - keyword: unused;
-    - argument: unused;
+    - argument:
-    - contentlist: a list of name of tex files;
+        a list of tex files to include
+        or a string of the tex file to include;
     - config: configuration dictionary of the current songbook.
     """
-    if not contentlist:
+    if isinstance(argument, str):
-        LOGGER.warning(
+        argument = [argument]
-            "Useless 'tex' content: list of files to include is empty."
+
-            )
     filelist = ContentList()
     basefolders = itertools.chain(
         (path.fullpath for path in config['_songdir']),
-        files.iter_datadirs(config['datadir']),
+        files.iter_datadirs(config['_datadir']),
-        files.iter_datadirs(config['datadir'], 'latex'),
+        files.iter_datadirs(config['_datadir'], 'latex'),
         )
-    for filename in contentlist:
+    for filename in argument:
         checked_file = None
         for path in basefolders:
             if os.path.exists(os.path.join(path, filename)):
@@ -62,5 +69,4 @@ def parse(keyword, argument, contentlist, config):
 
     return filelist
 
-
 CONTENT_PLUGINS = {'tex': parse}

patacrep/data/templates/default.tex

@@ -19,34 +19,42 @@
 %!- https://github.com/patacrep/
 
 (* variables *)
-{
+schema:
-"classoptions": {"description": {"en": "LaTeX class options", "fr": "Options de la classe LaTeX"},
+  type: //rec
-               "type": "flag",
+  required:
-               "join": ",",
+    title:
-               "mandatory": true,
+      type: //str
-               "default": {"default":[]}
+    author:
-            },
+      type: //str
-"title": {"description": {"en": "Title", "fr": "Titre"},
+  optional:
-            "default": {"en": "Guitar songbook", "fr": "Recueil de chansons pour guitare"},
+    classoptions:
-            "mandatory":true
+      type: //arr
-         },
+      contents: //str
-"author": {"description": {"en": "Author", "fr": "Auteur"},
+default:
-            "default": {"en": "The Patacrep Team", "fr": "L'équipe Patacrep"},
+  en:
-            "mandatory":true
+    title: "Guitar songbook"
-         },
+    author: "The Patacrep Team"
-"notenamesout": {"description": {"en": "Note names. Can be 'solfedge' (Do, Re, Mi...) or 'alphascale' (A, B, C...).",
+  fr:
-                              "fr": "Nom des notes : 'solfedge' (Do, Ré, Mi...) ou 'alphascale' (A, B, C...)."},
+    title: "Recueil de chansons pour guitare"
-                  "default": {"default": "alphascale", "fr": "solfedge"}
+    author: "L'équipe Patacrep"
-                  }
+description:
-}
+  en:
+    title: "Title"
+    author: "Author"
+    classoptions: "LaTeX class options"
+  fr:
+    title: "Titre"
+    author: "Auteur"
+    classoptions: "Options de la classe LaTeX"
 (* endvariables -*)
 
 (*- extends "songs.tex" -*)
 (*- set indexes = "titleidx,authidx" -*)
+(*- set template_var = _template["default.tex"] -*)
 
 (* block documentclass *)
 \documentclass[
-    (* for option in classoptions *)
+    (* for option in template_var.classoptions *)
     ((option)),
     (* endfor *)
     ]{article}
@@ -58,8 +66,8 @@
 
 \usepackage{chords}
 
-\title{((title))}
+\title{(( template_var.title ))}
-\author{((author))}
+\author{(( template_var.author ))}
 
 \newindex{titleidx}{((filename))_title}
 \newauthorindex{authidx}{((filename))_auth}
@@ -67,17 +75,17 @@
 (* for prefix in titleprefixwords -*)
    \titleprefixword{((prefix))}
 (* endfor*)
-(* for word in authwords.ignore -*)
+(* for word in authors.ignore -*)
    \authignoreword{((word))}
 (* endfor *)
-(* for word in authwords.after -*)
+(* for word in authors.after -*)
    \authbyword{((word))}
 (* endfor *)
-(* for word in authwords.sep -*)
+(* for word in authors.separators -*)
    \authsepword{((word))}
 (* endfor *)
 
-(* if notenamesout=="alphascale" -*)
+(* if chords.notation=="alphascale" -*)
   \notenamesout{A}{B}{C}{D}{E}{F}{G}
 (* else -*)
   \notenamesout{La}{Si}{Do}{R\'e}{Mi}{Fa}{Sol}

patacrep/data/templates/layout.tex

@@ -27,7 +27,7 @@
 
 \makeatletter
 \def\input@path{ %
-    (* for dir in datadir|iter_datadirs *)
+    (* for dir in _datadir|iter_datadirs *)
         {(( dir | path2posix ))/latex/} %
     (* endfor *)
 }

patacrep/data/templates/patacrep.tex

@@ -19,60 +19,85 @@
 %!- https://github.com/patacrep/
 
 (* variables *)
-{
+schema:
-"subtitle": {"description": {"en": "Subtitle", "fr": "Sous-titre"},
+  type: //rec
-            "default": {"default": ""}
+  optional:
-            },
+    version: //str
-"version":{ "description": {"en": "Version", "fr": "Version"},
+  required:
-            "default": {"default": "undefined"}
+    subtitle: //str
-         },
+    url: //str
-"web": {"description": {"en": "Website", "fr": "Site web"},
+    email: //str
-         "default": {"default": "http://www.patacrep.com"}
+    picture: //str
-      },
+    picturecopyright: //str
-"mail": {"description": {"en": "Email", "fr": "Adresse électronique"},
+    footer: //str
-         "default": {"default": "crep@team-on-fire.com"}
+    color:
-      },
+      type: //rec
-"picture": {"description": {"en": "Cover picture", "fr": "Image de couverture"},
+      required:
-            "type": "file",
+        songlink: //str
-            "default": {"default": "img/treble_a"}
+        hyperlink: //str
-         },
+    bgcolor:
-"picturecopyright": {"description": {"en": "Copyright for the cover picture", "fr": "Copyright pour l'image de couverture"},
+      type: //rec
-                     "default": {"default": "Dbolton \\url{http://commons.wikimedia.org/wiki/User:Dbolton}"}
+      required:
-                  },
+        songnumber: //str
-"footer": {"description": {"en": "Footer", "fr": "Pied de page"},
+        note: //str
-           "default": {"default": "Generated using Songbook (\\url{http://www.patacrep.com})"}
+        index: //str
-        },
+
-"songnumberbgcolor": {"description": {"en": "Number Shade", "fr": "Couleur des numéros"},
+default:
-                     "type": "color",
+  en:
-                     "default": {"default": "D1E4AE"}
+    subtitle: ""
-                  },
+    version: ""
-"notebgcolor": {"description": {"en": "Note Shade", "fr": "Couleur des notes"},
+    url: "http://www.patacrep.com"
-               "type": "color",
+    email: crep@team-on-fire.com
-               "default": {"default": "D1E4AE"}
+    picture: img/treble_a
-            },
+    picturecopyright: "Dbolton \\url{http://commons.wikimedia.org/wiki/User:Dbolton}"
-"indexbgcolor": {"description": {"en": "Index Shade", "fr": "Couleur d'index"},
+    footer: "Generated using Songbook (\\url{http://www.patacrep.com})"
-               "type": "color",
+    color:
-               "default": {"default": "D1E4AE"}
+      songlink: 4e9a06
-            },
+      hyperlink: 204a87
-"titleprefixwords": {"description": {"en": "Ignore some words in the beginning of song titles",
+    bgcolor:
-                                     "fr": "Ignore des mots dans le classement des chansons"},
+      songnumber: D1E4AE
-                     "default": {"default": ["The", "Le", "La", "L'", "A", "Au", "Ces", "De",
+      note: D1E4AE
-                     						 "Des", "El", "Les", "Ma", "Mon", "Un"]}
+      index: D1E4AE
-                     }
+  fr:
-}
+    footer: "Créé avec le programme Songbook (\\url{http://www.patacrep.com})"
+description:
+  en:
+    subtitle: "Subtitle"
+    version: "Version"
+    url: "Website"
+    email: "Email"
+    picture: "Cover picture"
+    picturecopyright: "Copyright for the cover picture"
+    footer: "Footer"
+    color:
+      songlink: "Songlink color"
+      hyperlink: "Hyperlink color"
+    bgcolor:
+      songnumber: "Number Shade"
+      note: "Note Shade"
+      index: "Index Shade"
+  fr:
+    subtitle: "Sous-titre"
+    version: "Version"
+    url: "Site web"
+    email: "Adresse électronique"
+    picture: "Image de couverture"
+    picturecopyright: "Copyright pour l'image de couverture"
+    footer: "Pied de page"
+    color:
+      songlink: "Couleur des liens vers les chants"
+      hyperlink: "Couleurs des liens hypertextes"
+    bgcolor:
+      songnumber: "Couleur des numéros"
+      note: "Couleur des notes"
+      index: "Couleur d'index"
 (* endvariables -*)
 
 (*- extends "default.tex" -*)
 
 (* block songbookpackages *)
-%! booktype, bookoptions and instruments are defined in "songs.tex"
 \usepackage[
-    ((booktype)),
+    (* for option in _bookoptions *)((option)),
-    (* for option in bookoptions *)
-    ((option)),
-    (* endfor *)
-    (* for instrument in instruments *)
-    ((instrument)),
     (* endfor *)
     ]{crepbook}
 (* endblock *)
@@ -91,16 +116,18 @@
 
 \pagestyle{empty}
 
-\definecolor{SongNumberBgColor}{HTML}{((songnumberbgcolor))}
+(*- set template_var = _template["patacrep.tex"] -*)
-\definecolor{NoteBgColor}{HTML}{((notebgcolor))}
+
-\definecolor{IndexBgColor}{HTML}{((indexbgcolor))}
+\definecolor{SongNumberBgColor}{HTML}{(( template_var.bgcolor.songnumber ))}
+\definecolor{NoteBgColor}{HTML}{(( template_var.bgcolor.note ))}
+\definecolor{IndexBgColor}{HTML}{(( template_var.bgcolor.index ))}
 
 \renewcommand{\snumbgcolor}{SongNumberBgColor}
 \renewcommand{\notebgcolor}{NoteBgColor}
 \renewcommand{\idxbgcolor}{IndexBgColor}
 
-\definecolor{tango-green-3}{HTML}{4e9a06}
+\definecolor{tango-green-3}{HTML}{(( template_var.color.songlink ))}
-\definecolor{tango-blue-3}{HTML}{204a87}
+\definecolor{tango-blue-3}{HTML}{(( template_var.color.hyperlink ))}
 \usepackage[
     bookmarks,
     bookmarksopen,
@@ -111,13 +138,13 @@
     ]{hyperref}
 
 
-\subtitle{((subtitle))}
+\subtitle{(( template_var.subtitle ))}
-(* if version!="undefined" -*)
+(* if template_var.version -*)
-    \version{((version))}
+    \version{(( template_var.version ))}
 (* endif *)
-\mail{((mail))}
+\mail{(( template_var.email ))}
-\web{((web))}
+\web{(( template_var.url ))}
-\picture{((picture))}
+\picture{(( template_var.picture ))}
-\picturecopyright{((picturecopyright))}
+\picturecopyright{(( template_var.picturecopyright ))}
-\footer{((footer))}
+\footer{(( template_var.footer ))}
 (* endblock *)

patacrep/data/templates/songbook_model.yml

@@ -0,0 +1,186 @@
+schema:
+  type: //rec
+  optional:
+    content: //any
+    template: //any
+    _songbookfile_dir: //str
+  required:
+    _cache: //bool
+    _outputdir: //str
+    _outputname: //str
+    _error: //str
+    _datadir:
+      type: //arr
+      contents: //str
+    _songdir:
+      type: //arr
+      contents: //any
+    book:
+      type: //rec
+      required:
+        encoding: //str
+        lang: //str
+        pictures: //bool
+        template: //str
+        onesongperpage: //bool
+    chords:
+      type: //rec
+      required:
+        show: //bool
+        diagrampage: //bool
+        repeatchords: //bool
+        lilypond: //bool
+        tablatures: //bool
+        diagramreminder:
+          type: //any
+          of:
+            - type: //str
+              value: "none"
+            - type: //str
+              value: "important"
+            - type: //str
+              value: "all"
+        instrument:
+          type: //any
+          of:
+            - type: //str
+              value: "guitar"
+            - type: //str
+              value: "ukulele"
+        notation:
+          type: //any
+          of:
+            - type: //str
+              value: "alphascale"
+            - type: //str
+              value: "solfedge"
+    authors:
+      type: //rec
+      required:
+        separators:
+          type: //any
+          of:
+            - type: //arr
+              contents: //str
+            - type: //nil
+        ignore:
+          type: //any
+          of:
+            - type: //arr
+              contents: //str
+            - type: //nil
+        after:
+          type: //any
+          of:
+            - type: //arr
+              contents: //str
+            - type: //nil
+    titles:
+      type: //rec
+      required:
+        prefix:
+          type: //any
+          of:
+            - type: //arr
+              contents: //str
+            - type: //nil
+default:
+  en:
+    _datadir: [] # For test reasons
+    book:
+      lang: en
+      encoding: utf-8
+      pictures: yes
+      template: patacrep.tex
+      onesongperpage: no
+
+    chords:
+      show: yes
+      diagramreminder: important
+      diagrampage: yes
+      repeatchords: yes
+      lilypond: no
+      tablatures: no
+      instrument: guitar
+      notation: alphascale
+
+    authors:
+      separators:
+      - and
+      ignore:
+      - unknown
+      after:
+      - by
+
+    titles:
+      prefix:
+      - The
+      - Le
+      - La
+      - "L'"
+      - A
+      - Au
+      - Ces
+      - De
+      - Des
+      - El
+      - Les
+      - Ma
+      - Mon
+      - Un
+
+  fr:
+    chords:
+      notation: solfedge
+description:
+  en:
+    book:
+      lang: "Main language of the songbook"
+      encoding: "Encoding of the files"
+      pictures: "Display the album pictures"
+      template: "Main template to use"
+      onesongperpage: "Start every song on a new page"
+
+    chords:
+      show: "Display chords"
+      diagramreminder: "Display some diagram reminders before the songs"
+      diagrampage: "Add a diagram page"
+      repeatchords: "Repeat the chords inside the songs"
+      lilypond: "Display lilypond scores"
+      tablatures: "Display tablatures"
+      instrument: "Instrument for the diagrams"
+      notation: "Chord notation"
+
+    authors:
+      separators: "Separator words between artists"
+      ignore: "Artist names to ignore"
+      after: "Word preceding artist names"
+
+    titles:
+      prefix: "Words to ignore at the beginning of song titles for the index"
+
+  fr:
+    book:
+      lang: "Langue principale"
+      encoding: "Encodage des fichiers"
+      pictures: "Afficher les illustrations d'albums"
+      template: "Template de base"
+      onesongperpage: "Commencer chaque chant sur une nouvelle page"
+
+    chords:
+      show: "Afficher les accords"
+      diagramreminder: "Afficher quelques diagrammes d'accords au début des chants"
+      diagrampage: "Inclure une page de rappel des accords"
+      repeatchords: "Répéter l'affichage des accords au sein des chants"
+      lilypond: "Inclure les partitions lilypond"
+      tablatures: "Inclure les tablatures"
+      instrument: "Instrument pour les diagrammes d'accords"
+      notation: "Notation des accords"
+
+    authors:
+      separators: "Mots de séparation entre les artistes"
+      ignore: "Noms d'artistes à ignorer"
+      after: "Mots précédents les noms d'artistes"
+
+    titles:
+      prefix: "Préfixes de chants à ignorer dans le classement"

patacrep/data/templates/songs.tex

@@ -18,61 +18,11 @@
 %!- The latest version of this program can be obtained from
 %!- https://github.com/patacrep/
 
-(* variables *)
-{
-"instruments": {"description": {"en": "Instruments", "fr": "Instruments"},
-               "type": "flag",
-               "values": {"guitar": {"en": "Guitare", "fr": "Guitare"},
-                          "ukulele": {"en": "Ukulele", "fr": "Ukulele"}
-                         },
-               "join": ",",
-               "mandatory": true,
-               "default": {"default":["guitar"]}
-            },
-"bookoptions": {"description": {"en": "Options", "fr": "Options"},
-               "type": "flag",
-               "values": {"diagram": {"en": "Chords diagrams", "fr": "Diagrammes d'accords"},
-                          "importantdiagramonly": {"en": "Only importants diagrames", "fr": "Diagrammes importants uniquement"},
-                          "lilypond": {"en": "Lilypond music sheets", "fr": "Partitions lilypond"},
-                          "pictures": {"en": "Cover pictures", "fr": "Couvertures d'albums"},
-                          "tabs": {"en": "Tablatures", "fr": "Tablatures"},
-                          "repeatchords": {"en": "Repeat chords", "fr": "Répéter les accords"},
-                          "onesongperpage": {"en": "One song per page", "fr": "Une chanson par page"}
-                       },
-               "join": ",",
-               "mandatory": true,
-               "default": {"default":["diagram","pictures"]}
-            },
-"booktype": {"description": {"en": "Type", "fr": "Type"},
-            "type": "enum",
-            "values": {"chorded": {"en": "With guitar chords", "fr": "Avec accords de guitare" },
-                                   "lyric": {"en": "Lyrics only", "fr": "Paroles uniquement"}
-                     },
-            "default": {"default":"chorded"},
-            "mandatory": true
-            },
-"lang": {"description": {"en": "Language", "fr": "Langue"},
-         "default": {"en": "en", "fr": "fr"}
-      },
-"titleprefixwords": {"description": {"en": "Ignore some words in the beginning of song titles",
-                                     "fr": "Ignore des mots dans le classement des chansons"},
-                     "default": {"default": []}
-                     },
-"authwords": {"description": {"en": "Set of options to process author string (LaTeX commands authsepword, authignoreword, authbyword)",
-                              "fr": "Options pour traiter les noms d'auteurs (commandes LaTeX authsepword, authignoreword, authbyword)"},
-               "default": {"default": {}}
-               }
-}
-(* endvariables -*)
-
 (*- extends "layout.tex" -*)
 
 (* block songbookpackages *)
 \usepackage[
-    ((booktype)),
+    (* for option in _bookoptions *)((option)),
-    (* for option in bookoptions *)((option)),
-    (* endfor *)
-    (* for instrument in instruments *)((instrument)),
     (* endfor *)
     ]{patacrep}
 (* endblock *)
@@ -83,12 +33,12 @@
 (* for lang in _langs|sort -*)
     \PassOptionsToPackage{(( lang | lang2babel ))}{babel}
 (* endfor *)
-\usepackage[(( lang | lang2babel ))]{babel}
+\usepackage[(( book.lang | lang2babel ))]{babel}
-\lang{(( lang | lang2babel ))}
+\lang{(( book.lang | lang2babel ))}
 
 \usepackage{graphicx}
 \graphicspath{ %
-    (* for dir in datadir|iter_datadirs*)
+    (* for dir in _datadir|iter_datadirs*)
         {(( dir | path2posix ))/} %
     (* endfor *)
 }

patacrep/errors.py

@@ -12,10 +12,18 @@ class SongbookError(Exception):
     def __str__(self):
         return self.message
 
+class SchemaError(SongbookError):
+    """Error on the songbook schema"""
 
-class YAMLError(SongbookError):
+    def __init__(self, message='', rx_exception=None):
-    """Error during songbook file decoding"""
+        super().__init__(message)
-    pass
+        self.rx_exception = rx_exception
+
+    def __str__(self):
+        if self.rx_exception:
+            return self.message + str(self.rx_exception)
+        else:
+            return self.message
 
 class TemplateError(SongbookError):
     """Error during template generation"""
@@ -43,13 +51,7 @@ class ExecutableNotFound(SongbookError):
 
 class StepError(SongbookError):
     """Error during execution of one compilation step."""
-
+    pass
-    def __init__(self, message):
-        super().__init__()
-        self.message = message
-
-    def __str__(self):
-        return self.message
 
 class LatexCompilationError(StepError):
     """Error during LaTeX compilation."""
@@ -96,13 +98,7 @@ class UnknownStep(StepError):
 
 class ParsingError(SongbookError):
     """Parsing error."""
-
+    pass
-    def __init__(self, message):
-        super().__init__(self)
-        self.message = message
-
-    def __str__(self):
-        return self.message
 
 class SharedError(SongbookError):
     """Error that is meant to be shared to third party tools using patacrep."""

patacrep/index.py

@@ -77,6 +77,9 @@ class Index:
 
     def add_keyword(self, key, word):
         """Add 'word' to self.keywords[key]."""
+        # Because LaTeX uses 'sep'
+        if key == 'sep':
+            key = 'separators'
         if key not in self.keywords:
             self.keywords[key] = []
         self.keywords[key].append(word)

patacrep/songbook/__init__.py

@@ -6,49 +6,126 @@ import sys
 import yaml
 
 from patacrep import encoding
+from patacrep.build import config_model
+from patacrep.utils import DictOfDict
+from patacrep.songs import DataSubpath
 import patacrep
 
 LOGGER = logging.getLogger()
 
 def open_songbook(filename):
-    """Open songbook, and return a raw songbook object.
+    """Open a songbook file, and prepare it to
+    return a raw songbook object.
 
     :param str filename: Filename of the yaml songbook.
     :rvalue: dict
     :return: Songbook, as a dictionary.
     """
-    if os.path.exists(filename + ".sb") and not os.path.exists(filename):
+    if os.path.exists(filename + ".yaml") and not os.path.exists(filename):
-        filename += ".sb"
+        filename += ".yaml"
 
     try:
         with patacrep.encoding.open_read(filename) as songbook_file:
-            songbook = yaml.load(songbook_file)
+            user_songbook = yaml.load(songbook_file)
-        if 'encoding' in songbook:
+        if 'encoding' in user_songbook.get('book', []):
             with encoding.open_read(
                 filename,
-                encoding=songbook['encoding']
+                encoding=user_songbook['book']['encoding']
                 ) as songbook_file:
-                songbook = yaml.load(songbook_file)
+                user_songbook = yaml.load(songbook_file)
     except Exception as error: # pylint: disable=broad-except
-        raise patacrep.errors.YAMLError(str(error))
+        raise patacrep.errors.SongbookError(str(error))
 
-    songbook['_basename'] = os.path.basename(filename)[:-3]
+    songbookfile_dir = os.path.dirname(os.path.abspath(filename))
+    # Output at the same place as the songbook file
+    outputdir = songbookfile_dir
+    outputname = os.path.splitext(os.path.basename(filename))[0]
+
+    return prepare_songbook(user_songbook, outputdir, outputname, songbookfile_dir)
+
+def prepare_songbook(songbook, outputdir, outputname, songbookfile_dir=None, datadir_prefix=None):
+    """Prepare a songbook by adding default values and datadirs
+    Returns a raw songbook object.
+
+    :param dict songbook: Initial yaml songbook.
+    :param str outputdir: Folder to put the output (tex, pdf...)
+    :param str outputname: Filename for the outputs (tex, pdf...)
+    :param str songbookfile_dir: Folder of the original songbook file (if there is one)
+    :param str datadir_prefix: Prefix for the datadirs
+    :rvalue: dict
+    :return: Songbook, as a dictionary.
+    """
+
+    songbook['_outputdir'] = outputdir
+    songbook['_outputname'] = outputname
+    if songbookfile_dir:
+        songbook['_songbookfile_dir'] = songbookfile_dir
+
+    songbook = _add_songbook_defaults(songbook)
 
     # Gathering datadirs
-    datadirs = []
+    songbook['_datadir'] = list(_iter_absolute_datadirs(songbook, datadir_prefix))
-    if 'datadir' in songbook:
+    if 'datadir' in songbook['book']:
-        if isinstance(songbook['datadir'], str):
+        del songbook['book']['datadir']
-            songbook['datadir'] = [songbook['datadir']]
-        datadirs += [
-            os.path.join(
-                os.path.dirname(os.path.abspath(filename)),
-                path
-                )
-            for path in songbook['datadir']
-            ]
-    # Default value
-    datadirs.append(os.path.dirname(os.path.abspath(filename)))
 
-    songbook['datadir'] = datadirs
+    songbook['_songdir'] = [
+        DataSubpath(path, 'songs')
+        for path in songbook['_datadir']
+    ]
 
     return songbook
+
+def _add_songbook_defaults(user_songbook):
+    """ Adds the defaults values to the songbook if missing from
+    the user songbook
+
+    Priority:
+        - User values
+        - Default values of the user lang (if set)
+        - Default english values
+    """
+
+    # Merge the default and user configs
+    locale_default = config_model('default')
+    # Initialize with default in english
+    default_songbook = locale_default.get('en', {})
+    default_songbook = DictOfDict(default_songbook)
+
+    if 'lang' in user_songbook.get('book', []):
+        # Update default with current lang
+        lang = user_songbook['book']['lang']
+        default_songbook.update(locale_default.get(lang, {}))
+    # Update default with user_songbook
+    default_songbook.update(user_songbook)
+
+    return dict(default_songbook)
+
+def _iter_absolute_datadirs(raw_songbook, datadir_prefix=None):
+    """Iterate on the absolute datadirs of the raw songbook
+
+    Appends the songfile dir at the end
+    """
+    songbookfile_dir = raw_songbook.get('_songbookfile_dir')
+
+    if datadir_prefix is None:
+        if songbookfile_dir is None:
+            raise patacrep.errors.SongbookError('Please specify where the datadir are located')
+        datadir_prefix = songbookfile_dir
+
+    datadir = raw_songbook.get('book', {}).get('datadir')
+
+    if datadir is None:
+        datadir = []
+    elif isinstance(datadir, str):
+        datadir = [datadir]
+
+    for path in datadir:
+        abspath = os.path.join(datadir_prefix, path)
+        if os.path.exists(abspath) and os.path.isdir(abspath):
+            yield abspath
+        else:
+            LOGGER.warning(
+                "Ignoring non-existent datadir '{}'.".format(path)
+                )
+    if songbookfile_dir:
+        yield songbookfile_dir

patacrep/songbook/__main__.py

@@ -6,11 +6,11 @@ import logging
 import sys
 import textwrap
 
+from patacrep.build import SongbookBuilder, DEFAULT_STEPS
+from patacrep.utils import yesno
 from patacrep import __version__
 from patacrep import errors
 from patacrep.songbook import open_songbook
-from patacrep.build import SongbookBuilder, DEFAULT_STEPS
-from patacrep.utils import yesno
 
 # Logging configuration
 logging.basicConfig(level=logging.INFO)
@@ -141,8 +141,11 @@ def main(args=None):
 
     options = argument_parser(args[1:])
 
+    songbook_path = options.book[-1]
+
+    # Load the user songbook config
     try:
-        songbook = open_songbook(options.book[-1])
+        songbook = open_songbook(songbook_path)
 
         # Command line options
         if options.datadir:

patacrep/songs/__init__.py

@@ -14,15 +14,6 @@ from patacrep.songs import errors as song_errors
 
 LOGGER = logging.getLogger(__name__)
 
-DEFAULT_CONFIG = {
-    'template': "default.tex",
-    'lang': 'en',
-    'content': [],
-    'titleprefixwords': [],
-    'encoding': None,
-    'datadir': [],
-    }
-
 def cached_name(datadir, filename):
     """Return the filename of the cache version of the file."""
     fullpath = os.path.abspath(os.path.join(datadir, '.cache', filename))
@@ -107,7 +98,7 @@ class Song:
 
     def __init__(self, subpath, config=None, *, datadir=None):
         if config is None:
-            config = DEFAULT_CONFIG.copy()
+            config = {}
 
         if datadir is None:
             self.datadir = ""
@@ -120,8 +111,8 @@ class Song:
         self.fullpath = os.path.join(self.datadir, subpath)
         self.subpath = subpath
         self._filehash = None
-        self.encoding = config["encoding"]
+        self.encoding = config['book']["encoding"]
-        self.lang = config["lang"]
+        self.lang = config['book']["lang"]
         self.config = config
         self.errors = []
 
@@ -138,7 +129,7 @@ class Song:
         self.unprefixed_titles = [
             unprefixed_title(
                 title,
-                config['titleprefixwords']
+                config['titles']['prefix']
                 )
             for title
             in self.titles
@@ -161,16 +152,16 @@ class Song:
     def filehash(self):
         """Compute (and cache) the md5 hash of the file"""
         if self._filehash is None:
-            self._filehash = hashlib.md5(
+            with open(self.fullpath, 'rb') as songfile:
-                open(self.fullpath, 'rb').read()
+                self._filehash = hashlib.md5(songfile.read()).hexdigest()
-                ).hexdigest()
         return self._filehash
 
     def _cache_retrieved(self):
         """If relevant, retrieve self from the cache."""
         if self.use_cache and os.path.exists(self.cached_name):
             try:
-                cached = pickle.load(open(self.cached_name, 'rb',))
+                with open(self.cached_name, 'rb',) as cachefile:
+                    cached = pickle.load(cachefile)
                 if (
                         cached['_filehash'] == self.filehash
                         and cached['_version'] == self.CACHE_VERSION
@@ -231,7 +222,7 @@ class Song:
     def iter_datadirs(self, *subpath):
         """Return an iterator of existing datadirs (with an optionnal subpath)
         """
-        yield from files.iter_datadirs(self.config['datadir'], *subpath)
+        yield from files.iter_datadirs(self.config['_datadir'], *subpath)
 
     def search_datadir_file(self, filename, extensions=None, directories=None):
         """Search for a file name.
@@ -259,7 +250,7 @@ class Song:
         if extensions is None:
             extensions = ['']
         if directories is None:
-            directories = self.config['datadir']
+            directories = self.config['_datadir']
 
         songdir = os.path.dirname(self.fullpath)
         for extension in extensions:

patacrep/templates.py

@@ -2,14 +2,15 @@
 
 import logging
 import re
-import json
+
+import yaml
 
 from jinja2 import Environment, FileSystemLoader, ChoiceLoader, \
         TemplateNotFound, nodes
 from jinja2.ext import Extension
-from jinja2.meta import find_referenced_templates as find_templates
+from jinja2.meta import find_referenced_templates
 
-from patacrep import errors, files
+from patacrep import errors, files, utils
 from patacrep.latex import lang2babel, UnknownLanguage
 import patacrep.encoding
 
@@ -160,103 +161,93 @@ class TexBookRenderer(Renderer):
                     ),
                 )
 
-    def get_variables(self):
+    def get_all_variables(self, user_config):
-        '''Get and return a dictionary with the default values
+        '''Validate template variables (and set defaults when needed)
-         for all the variables
+
+        Will raise `SchemaError` if any data does not respect the schema
         '''
         data = self.get_template_variables(self.template)
         variables = dict()
-        for name, param in data.items():
+        for templatename, param in data.items():
-            variables[name] = self._get_default(param)
+            template_config = user_config.get(templatename, {})
+            try:
+                variables[templatename] = self._get_variables(param, template_config, self.lang)
+            except errors.SchemaError as exception:
+                exception.message += "'template' > '{}' > ".format(templatename)
+                raise exception
         return variables
 
-    def _get_default(self, parameter):
+    @staticmethod
+    def _get_variables(parameter, user_config, lang):
         '''Get the default value for the parameter, according to the language.
+
+        Will raise `SchemaError` if the data does not respect the schema
         '''
-        default = None
+        locale_default = parameter.get('default', {})
-        try:
+        # Initialize with default in english
-            default = parameter['default']
+        data = locale_default.get('en', {})
-        except KeyError:
+        data = utils.DictOfDict(data)
-            return None
+
-
+        # Update default with current lang
-        if self.lang in default:
+        data.update(locale_default.get(lang, {}))
-            variable = default[self.lang]
+        # Update default with user_config
-        elif "default" in default:
+        data.update(user_config)
-            variable = default["default"]
+
-        elif "en" in default:
+        schema = parameter.get('schema', {})
-            variable = default["en"]
+        utils.validate_yaml_schema(data, schema)
-        elif len(default):
-            variable = default.popitem()[1]
-        else:
-            variable = None
 
-        return variable
+        return data
 
-    def get_template_variables(self, template, skip=None):
+    def get_template_variables(self, basetemplate):
         """Parse the template to extract the variables as a dictionary.
 
         If the template includes or extends other templates, load them as well.
 
         Arguments:
-        - template: the name of the template, as a string.
+        - basetemplate: the name of the template, as a string.
-        - skip: a list of templates (as strings) to skip: if they are included
           in 'template' (or one of its subtemplates), it is not parsed.
         """
-        if not skip:
-            skip = []
         variables = {}
-        (current, templates) = self.parse_template(template)
+        for templatename, template in self._iter_template_content(basetemplate):
-        for subtemplate in templates:
+            match = re.findall(_VARIABLE_REGEXP, template)
-            if subtemplate in skip:
+            if not match:
                 continue
-            variables.update(
+            if templatename not in variables:
-                self.get_template_variables(
+                variables[templatename] = {}
-                    subtemplate,
+            for variables_string in match:
-                    skip + templates
-                    )
-                )
-        variables.update(current)
-        return variables
-
-    def parse_template(self, template):
-        """Return (variables, templates).
-
-        Argument:
-        - template: name of the template to parse.
-
-        Return values:
-        - variables: a dictionary of variables contained in 'template', NOT
-          recursively (included templates are not parsed).
-        - templates: list of included temlates, NOT recursively.
-        """
-
-        subvariables = {}
-        templatename = self.jinjaenv.get_template(template).filename
-        with patacrep.encoding.open_read(
-            templatename,
-            encoding=self.encoding
-            ) as template_file:
-            content = template_file.read()
-        subtemplates = list(find_templates(self.jinjaenv.parse(content)))
-        match = re.findall(_VARIABLE_REGEXP, content)
-        if match:
-            for var in match:
                 try:
-                    subvariables.update(json.loads(var))
+                    variables[templatename].update(yaml.load(variables_string))
                 except ValueError as exception:
                     raise errors.TemplateError(
                         exception,
                         (
-                            "Error while parsing json in file "
+                            "Error while parsing yaml in file "
-                            "{filename}. The json string was:"
+                            "{filename}. The yaml string was:"
-                            "\n'''\n{jsonstring}\n'''"
+                            "\n'''\n{yamlstring}\n'''"
                         ).format(
-                            filename=templatename,
+                            filename=template.filename,
-                            jsonstring=var,
+                            yamlstring=variables_string,
                             )
                         )
+        return variables
 
-        return (subvariables, subtemplates)
+    def _iter_template_content(self, templatename, *, skip=None):
+        """Iterate over template (and subtemplate) content."""
+        if skip is None:
+            skip = []
+        template = self.jinjaenv.get_template(templatename)
+        with patacrep.encoding.open_read(
+            template.filename,
+            encoding=self.encoding
+            ) as contentfile:
+            content = contentfile.read()
+            for subtemplatename in find_referenced_templates(self.jinjaenv.parse(content)):
+                if subtemplatename not in skip:
+                    yield from self._iter_template_content(
+                        subtemplatename,
+                        skip=skip + [templatename],
+                        )
+            yield template.name, content
 
     def render_tex(self, output, context):
         '''Render a template into a .tex file
@@ -267,3 +258,42 @@ class TexBookRenderer(Renderer):
         '''
 
         output.write(self.template.render(context))
+
+
+def _transform_options(config, equivalents):
+    """
+    Get the equivalent name of the checked options
+    """
+    for option in config:
+        if config[option] and option in equivalents:
+            yield equivalents[option]
+
+def iter_bookoptions(config):
+    """
+    Extract the bookoptions from the config structure
+    """
+    if config['chords']['show']:
+        yield 'chorded'
+    else:
+        yield 'lyrics'
+
+    book_equivalents = {
+        'pictures':         'pictures',
+        'onesongperpage':   'onesongperpage',
+    }
+    yield from _transform_options(config['book'], book_equivalents)
+
+    chords_equivalents = {
+        'lilypond':     'lilypond',
+        'tablatures':   'tabs',
+        'repeatchords': 'repeatchords',
+    }
+    yield from _transform_options(config['chords'], chords_equivalents)
+
+    if config['chords']['show']:
+        if config['chords']['diagramreminder'] == "important":
+            yield 'importantdiagramonly'
+        elif config['chords']['diagramreminder'] == "all":
+            yield 'diagram'
+
+        yield config['chords']['instrument']

patacrep/tools/cache/__main__.py

@@ -54,7 +54,7 @@ def commandline_parser():
 
 def do_clean(namespace):
     """Execute the `patatools cache clean` command."""
-    for datadir in open_songbook(namespace.songbook)['datadir']:
+    for datadir in open_songbook(namespace.songbook)['_datadir']:
         cachedir = os.path.join(datadir, ".cache")
         LOGGER.info("Deleting cache directory '{}'...".format(cachedir))
         if os.path.isdir(cachedir):

patacrep/tools/convert/__main__.py

@@ -6,8 +6,8 @@ import sys
 
 from patacrep import files
 from patacrep.content import ContentError
-from patacrep.songs import DEFAULT_CONFIG
 from patacrep.utils import yesno
+from patacrep.build import config_model
 
 LOGGER = logging.getLogger("patatools.convert")
 SUBCOMMAND_DESCRIPTION = "Convert between song formats"
@@ -36,8 +36,9 @@ def main(args=None):
     dest = args[2]
     song_files = args[3:]
 
+    # todo : what is the datadir argument used for?
     renderers = files.load_plugins(
-        datadirs=DEFAULT_CONFIG.get('datadir', []),
+        datadirs=[],
         root_modules=['songs'],
         keyword='SONG_RENDERERS',
         )
@@ -59,7 +60,7 @@ def main(args=None):
 
     for file in song_files:
         try:
-            song = renderers[dest][source](file, DEFAULT_CONFIG)
+            song = renderers[dest][source](file, config_model('default')['en'])
             destname = "{}.{}".format(".".join(file.split(".")[:-1]), dest)
             if os.path.exists(destname):
                 if not confirm(destname):

patacrep/utils.py

@@ -2,6 +2,8 @@
 
 from collections import UserDict
 
+from patacrep import errors, Rx
+
 class DictOfDict(UserDict):
     """Dictionary, with a recursive :meth:`update` method.
 
@@ -75,3 +77,18 @@ def yesno(string):
         string,
         ", ".join(["'{}'".format(string) for string in yes_strings + no_strings]),
         ))
+
+def validate_yaml_schema(data, schema):
+    """Check that the data respects the schema
+
+    Will raise `SchemaError` if the schema is not respected.
+    """
+    schema = Rx.make_schema(schema)
+
+    if isinstance(data, DictOfDict):
+        data = dict(data)
+
+    try:
+        schema.validate(data)
+    except Rx.SchemaMismatch as exception:
+        raise errors.SchemaError(rx_exception=exception)

test/test_authors.py

@@ -49,7 +49,7 @@ PROCESS_AUTHORS_DATA = [
 AUTHWORDS = authors.compile_authwords({
     "after": ["by"],
     "ignore": ["anonymous", "Anonyme", "anonyme"],
-    "sep": ['and', 'et'],
+    "separators": ['and', 'et'],
     })
 
 class TestAutors(unittest.TestCase):

test/test_content/cwd.control

@@ -1 +1 @@
-["subdir/chordpro.csg"]
+- subdir/chordpro.csg

test/test_content/cwd.source

@@ -1 +1,3 @@
-[["cwd(subdir)"]]
+- cwd:
+    path: subdir
+    content:

test/test_content/cwd_list.control

@@ -1 +1,2 @@
-["subdir/chordpro.csg", "exsong.sg"]
+- subdir/chordpro.csg
+- exsong.sg

test/test_content/cwd_list.source

@@ -1 +1,11 @@
-[["cwd(subdir)", "exsong.sg", "intersong.is", "jsonlist.json", "texfile.tex", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"], "exsong.sg"]
+- cwd:
+    path: subdir
+    content:
+        - "exsong.sg"
+        - "intersong.is"
+        - "jsonlist.json"
+        - "texfile.tex"
+        - "texsong.tsg"
+        - "chordpro.csg"
+        - "subdir/chordpro.csg"
+- "exsong.sg"

test/test_content/datadir/songs/custom_list.yaml

@@ -0,0 +1,6 @@
+- sort:
+    key: "title"
+    content:
+      - exsong.sg
+      - chordpro.csg
+      - subdir/chordpro.csg

test/test_content/datadir_sort/path1_title1_author1.csg

@@ -0,0 +1,5 @@
+{title: Title1}
+{artist: Author1}
+
+Foo bar
+

test/test_content/datadir_sort/path1_title1_author2.csg

@@ -0,0 +1,5 @@
+{title: Title1}
+{artist: Author2}
+
+Foo bar
+

test/test_content/datadir_sort/path1_title2_author1.csg

@@ -0,0 +1,5 @@
+{title: Title2}
+{artist: Author1}
+
+Foo bar
+

test/test_content/datadir_sort/path1_title2_author2.csg

@@ -0,0 +1,5 @@
+{title: Title2}
+{artist: Author2}
+
+Foo bar
+

test/test_content/datadir_sort/path2_title1_author1.csg

@@ -0,0 +1,5 @@
+{title: Title1}
+{artist: Author1}
+
+Foo bar
+

test/test_content/datadir_sort/path2_title1_author2.csg

@@ -0,0 +1,5 @@
+{title: Title1}
+{artist: Author2}
+
+Foo bar
+

test/test_content/datadir_sort/path2_title2_author1.csg

@@ -0,0 +1,5 @@
+{title: Title2}
+{artist: Author1}
+
+Foo bar
+

test/test_content/datadir_sort/path2_title2_author2.csg

@@ -0,0 +1,5 @@
+{title: Title2}
+{artist: Author2}
+
+Foo bar
+

test/test_content/glob.control

@@ -1 +1 @@
-["chordpro.csg"]
+- chordpro.csg

test/test_content/glob.source

@@ -1 +1 @@
-["*.csg"]
+- "*.csg"

test/test_content/include.control

@@ -1 +1,6 @@
-["exsong.sg", "chordpro.csg", "subdir/chordpro.csg"]
+- exsong.sg
+- chordpro.csg
+- subdir/chordpro.csg
+- chordpro.csg
+- subdir/chordpro.csg
+- exsong.sg

test/test_content/include.source

@@ -1 +1,3 @@
-[["include" , "custom_list.json"]]
+- include:
+    - custom_list.json
+- include: custom_list.yaml

test/test_content/invalid.control

@@ -0,0 +1 @@
+[]

test/test_content/invalid.source

@@ -0,0 +1,4 @@
+- ["directly", "a", "list"]
+- invalid_keyword: Test
+- section:
+    short: Missing name

test/test_content/sections.control

@@ -1 +1,11 @@
-["section:Traditional", "exsong.sg", "section:Example", "texsong.tsg", "chordpro.csg", "exsong.sg"]
+- section{First Section!}
+- section{Named section}
+- section[section_short_name]{Section with short name}
+- section*{Section* with short name}
+- part{part section test}
+- chapter{chapter section test}
+- section{section section test}
+- subsection{subsection section test}
+- subsubsection{subsubsection section test}
+- paragraph{paragraph section test}
+- subparagraph{subparagraph section test}

test/test_content/sections.source

@@ -1,6 +1,16 @@
-[["section", "Traditional"],
+- section: First Section!
-                "exsong.sg",
+- section:
-                ["section", "Example"],
+    name: Named section
-                "texsong.tsg",
+- section:
-                "chordpro.csg",
+    name: Section with short name
-                "exsong.sg"]
+    short: section_short_name
+- section*:
+    name: Section* with short name
+    short: section_star_short_name
+- part: part section test
+- chapter: chapter section test
+- section: section section test
+- subsection: subsection section test
+- subsubsection: subsubsection section test
+- paragraph: paragraph section test
+- subparagraph: subparagraph section test

test/test_content/sections_short.control

@@ -1 +0,0 @@
-["section:(tradi)Traditional", "exsong.sg", "section*:Example", "texsong.tsg", "chordpro.csg", "exsong.sg"]

test/test_content/sections_short.source

@@ -1,6 +0,0 @@
-[["section", "Traditional", "tradi"],
-                "exsong.sg",
-                ["section*", "Example"],
-                "texsong.tsg",
-                "chordpro.csg",
-                "exsong.sg"]

test/test_content/songs.control

@@ -1 +1,4 @@
-["exsong.sg", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"]
+- exsong.sg
+- texsong.tsg
+- chordpro.csg
+- subdir/chordpro.csg

test/test_content/songs.source

@@ -1 +1,8 @@
-["exsong.sg", "intersong.is", "jsonlist.json", "texfile.tex", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"]
+- exsong.sg
+- intersong.is
+- jsonlist.json
+- song:
+    - texfile.tex
+    - texsong.tsg
+- song: chordpro.csg
+- subdir/chordpro.csg

test/test_content/songsection.control

@@ -1 +1,6 @@
-["songsection:Traditional", "exsong.sg", "songchapter:English", "texsong.tsg", "chordpro.csg", "exsong.sg"]
+- songsection{Traditional}
+- exsong.sg
+- songchapter{English}
+- texsong.tsg
+- chordpro.csg
+- exsong.sg

test/test_content/songsection.source

@@ -1,6 +1,6 @@
-[["songsection", "Traditional"],
+- songsection: Traditional
-                "exsong.sg",
+- "exsong.sg"
-                ["songchapter", "English"],
+- songchapter: English
-                "texsong.tsg",
+- "texsong.tsg"
-                "chordpro.csg",
+- "chordpro.csg"
-                "exsong.sg"]
+- "exsong.sg"

test/test_content/sort.control

@@ -0,0 +1,27 @@
+- section{Title}
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg"
+- section{Author, Title}
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg"
+- section{Path, Title}
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg"
+- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg"

test/test_content/sort.source

@@ -0,0 +1,15 @@
+- cwd:
+    path: "datadir_sort"
+    content:
+      - section:
+          name: "Title"
+      - sort:
+          key: title
+      - section:
+          name: "Author, Title"
+      - sort:
+          key: [by, title]
+      - section:
+          name: "Path, Title"
+      - sort:
+          key: [path, title]

test/test_content/sorted.control

@@ -1 +0,0 @@
-["chordpro.csg", "exsong.sg", "subdir/chordpro.csg", "texsong.tsg"]

test/test_content/sorted.source

@@ -1 +0,0 @@
-[["sorted(fullpath)"]]

test/test_content/test_content.py

@@ -5,11 +5,13 @@
 import glob
 import os
 import unittest
-import json
+import yaml
+
+from pkg_resources import resource_filename
 
-from patacrep.songs import DataSubpath, DEFAULT_CONFIG
 from patacrep import content, files
 from patacrep.content import song, section, songsection, tex
+from patacrep.songbook import prepare_songbook
 
 from .. import logging_reduced
 from .. import dynamic # pylint: disable=unused-import
@@ -17,18 +19,14 @@ from .. import dynamic # pylint: disable=unused-import
 class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
     """Test of the content plugins.
 
-    For any given `foo.source`, it parses the content as a json "content"
+    For any given `foo.source`, it parses the content as a yaml "content"
-    argument of a .sb file.
+    argument of a .yaml file.
     It controls that the generated file list is equal to the one in `foo.control`.
     """
 
     maxDiff = None
     config = None
 
-    @classmethod
-    def setUpClass(cls):
-        cls._generate_config()
-
     @classmethod
     def _iter_testmethods(cls):
         """Iterate over dynamically generated test methods"""
@@ -50,17 +48,23 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
             """Test that `base.source` produces the correct file list"""
             sourcename = "{}.source".format(base)
             with open(sourcename, mode="r", encoding="utf8") as sourcefile:
-                sbcontent = json.load(sourcefile)
+                sbcontent = yaml.load(sourcefile)
+
+            outputdir = os.path.dirname(base)
+            config = cls._generate_config(sbcontent, outputdir, base)
 
             with logging_reduced('patacrep.content.song'):
-                expandedlist = content.process_content(sbcontent, cls.config.copy())
+                expandedlist = content.process_content(sbcontent, config)
             sourcelist = [cls._clean_path(elem) for elem in expandedlist]
 
             controlname = "{}.control".format(base)
             if not os.path.exists(controlname):
                 raise Exception("Missing control:" + str(sourcelist).replace("'", '"'))
             with open(controlname, mode="r", encoding="utf8") as controlfile:
-                controllist = json.load(controlfile)
+                controllist = [
+                    elem.replace("@TEST_FOLDER@", files.path2posix(resource_filename(__name__, "")))
+                    for elem in yaml.load(controlfile)
+                    ]
 
             self.assertEqual(controllist, sourcelist)
 
@@ -77,13 +81,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
             return files.path2posix(files.relpath(elem.song.fullpath, songpath))
 
         elif isinstance(elem, section.Section):
-            if elem.short is None:
+            return elem.render(None)[1:]
-                return "{}:{}".format(elem.keyword, elem.name)
-            else:
-                return "{}:({}){}".format(elem.keyword, elem.short, elem.name)
 
         elif isinstance(elem, songsection.SongSection):
-            return "{}:{}".format(elem.keyword, elem.name)
+            return elem.render(None)[1:]
 
         elif isinstance(elem, tex.LaTeX):
             return files.path2posix(elem.filename)
@@ -92,27 +93,27 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
             raise Exception(elem)
 
     @classmethod
-    def _generate_config(cls):
+    def _generate_config(cls, sbcontent, outputdir, base):
         """Generate the config to process the content"""
 
-        config = DEFAULT_CONFIG.copy()
+        # Load the default songbook config
-
+        config = prepare_songbook(
-        datadirpaths = [os.path.join(os.path.dirname(__file__), 'datadir')]
+            {'book':{'datadir':'datadir'}, 'content': sbcontent},
-
+            outputdir,
-        config['datadir'] = datadirpaths
+            base,
+            outputdir
+            )
 
-        config['_songdir'] = [
+        # Load the plugins
-            DataSubpath(path, 'songs')
-            for path in datadirpaths
-            ]
         config['_content_plugins'] = files.load_plugins(
-            datadirs=datadirpaths,
+            datadirs=config['_datadir'],
             root_modules=['content'],
             keyword='CONTENT_PLUGINS',
             )
         config['_song_plugins'] = files.load_plugins(
-            datadirs=datadirpaths,
+            datadirs=config['_datadir'],
             root_modules=['songs'],
             keyword='SONG_RENDERERS',
             )['tsg']
-        cls.config = config
+
+        return config

test/test_content/tex.control

@@ -1 +1,2 @@
-["test/test_content/datadir/songs/texfile.tex"]
+- test/test_content/datadir/songs/texfile.tex
+- test/test_content/datadir/songs/texfile.tex

test/test_content/tex.source

@@ -1 +1,4 @@
-[["tex", "texfile.tex", "chordpro.csg"]]
+- tex:
+    - texfile.tex
+    - chordpro.csg
+- tex: texfile.tex

test/test_patatools/test_cache.py

@@ -11,6 +11,8 @@ from patacrep.tools.__main__ import main as tools_main
 from patacrep.tools.cache.__main__ import main as cache_main
 from patacrep.songbook.__main__ import main as songbook_main
 
+from .. import logging_reduced
+
 CACHEDIR = os.path.join(os.path.dirname(__file__), "test_cache_datadir", ".cache")
 
 class TestCache(unittest.TestCase):
@@ -41,15 +43,20 @@ class TestCache(unittest.TestCase):
     def test_clean_exists(self):
         """Test of the "patatools cache clean" subcommand"""
         for main, args in [
-                (tools_main, ["patatools", "cache", "clean", "test_cache.sb"]),
+                (tools_main, ["patatools", "cache", "clean", "test_cache.yaml"]),
-                (cache_main, ["patatools-cache", "clean", "test_cache.sb"]),
+                (cache_main, ["patatools-cache", "clean", "test_cache.yaml"]),
             ]:
             with self.subTest(main=main, args=args):
                 # First compilation. Ensure that cache exists afterwards
-                self._system(songbook_main, ["songbook", "--steps", "tex,clean", "test_cache.sb"])
+                with logging_reduced('patacrep.build'):
+                    self._system(
+                        songbook_main,
+                        ["songbook", "--steps", "tex,clean", "test_cache.yaml"]
+                    )
                 self.assertTrue(os.path.exists(CACHEDIR))
 
                 # Clean cache
+                with logging_reduced('patatools.cache'):
                     self._system(main, args)
 
                 # Ensure that cache does not exist
@@ -59,9 +66,10 @@ class TestCache(unittest.TestCase):
         """Test of the "patatools cache clean" subcommand"""
         # Clean non-existent cache
         for main, args in [
-                (tools_main, ["patatools", "cache", "clean", "test_cache.sb"]),
+                (tools_main, ["patatools", "cache", "clean", "test_cache.yaml"]),
-                (cache_main, ["patatools-cache", "clean", "test_cache.sb"]),
+                (cache_main, ["patatools-cache", "clean", "test_cache.yaml"]),
             ]:
             with self.subTest(main=main, args=args):
                 # Clean cache
+                with logging_reduced('patatools.cache'):
                     self._system(main, args)

test/test_patatools/test_cache.sb

@@ -1,3 +0,0 @@
-{
-"datadir": ["test_cache_datadir"],
-}

test/test_patatools/test_cache.yaml

@@ -0,0 +1,2 @@
+book:
+    datadir: test_cache_datadir

test/test_song/test_parser.py

@@ -9,8 +9,8 @@ import unittest
 from pkg_resources import resource_filename
 
 from patacrep import files
-from patacrep.songs import DEFAULT_CONFIG
 from patacrep.encoding import open_read
+from patacrep.build import config_model
 from patacrep.songs import errors
 
 from .. import logging_reduced
@@ -74,13 +74,15 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
     def _iter_testmethods(cls):
         """Iterate over song files to test."""
         # Setting datadir
-        cls.config = DEFAULT_CONFIG
+        # Load the default songbook config
-        if 'datadir' not in cls.config:
+        cls.config = config_model('default')['en']
-            cls.config['datadir'] = []
+
-        cls.config['datadir'].append('datadir')
+        if '_datadir' not in cls.config:
+            cls.config['_datadir'] = []
+        cls.config['_datadir'].append('datadir')
 
         cls.song_plugins = files.load_plugins(
-            datadirs=cls.config['datadir'],
+            datadirs=cls.config['_datadir'],
             root_modules=['songs'],
             keyword='SONG_RENDERERS',
             )

test/test_songbook/.gitignore

@@ -1,2 +1,2 @@
-/*tex
+**.tex
 .cache

test/test_songbook/content.sb

@@ -1,13 +0,0 @@
-{
-"datadir": ["content_datadir"],
-"content": [
-  ["section", "Test of section"],
-  ["sorted"],
-  ["songsection", "Test of song section"],
-  ["cwd(content_datadir/content)",
-    "song.csg", "song.tsg",
-    ["tex", "foo.tex"]
-  ],
-  ["include", "include.sbc"]
-  ]
-}

test/test_songbook/content.tex.control

@@ -23,9 +23,9 @@
     ]{article}
 
 \usepackage[
-    chorded,
+chorded,
-diagram,
 pictures,
+diagram,
 guitar,
     ]{patacrep}
 
@@ -63,6 +63,9 @@ guitar,
 \newindex{titleidx}{content_title}
 \newauthorindex{authidx}{content_auth}
 
+\authignoreword{unknown}
+\authbyword{by}
+\authsepword{and}
 
 \notenamesout{A}{B}{C}{D}{E}{F}{G}
 

test/test_songbook/content.yaml

@@ -0,0 +1,25 @@
+book:
+  pictures: yes
+  datadir: content_datadir
+  lang: en
+  template: default.tex
+chords:
+  repeatchords: no
+  diagramreminder: all
+
+content:
+  - section: Test of section
+  - sort:
+  - songsection: Test of song section
+  - cwd:
+      # relative to yaml songfile
+      path: content_datadir/content
+      content:
+        - "song.csg"
+        - "song.tsg"
+  - cwd:
+      # relative to datadir
+      path: ../content
+      content:
+        - tex: foo.tex
+  - include: include.sbc

test/test_songbook/content_datadir/songs/include.sbc

@@ -1 +1 @@
-[["section", "This is an included section"]]
+[{"section": "This is an included section"}]

test/test_songbook/datadir.sb

@@ -1,6 +0,0 @@
-bookoptions:
-- pictures
-datadir:
-- datadir_datadir
-- datadir_datadir2
-lang: en

test/test_songbook/datadir.tex.control

@@ -24,8 +24,10 @@
     ]{article}
 
 \usepackage[
-    chorded,
+chorded,
 pictures,
+repeatchords,
+importantdiagramonly,
 guitar,
     ]{patacrep}
 
@@ -64,6 +66,9 @@ guitar,
 \newindex{titleidx}{datadir_title}
 \newauthorindex{authidx}{datadir_auth}
 
+\authignoreword{unknown}
+\authbyword{by}
+\authsepword{and}
 
 \notenamesout{A}{B}{C}{D}{E}{F}{G}
 

test/test_songbook/datadir.yaml

@@ -0,0 +1,7 @@
+book:
+  pictures: yes
+  datadir:
+    - datadir_datadir
+    - datadir_datadir2
+  lang: en
+  template: default.tex

test/test_songbook/lang_default.tex.control

@@ -0,0 +1,131 @@
+
+
+
+
+
+
+%% Automatically generated document.
+%% You may edit this file but all changes will be overwritten.
+%% If you want to change this document, have a look at
+%% the templating system.
+%%
+%% Generated using Songbook <http://www.patacrep.com>
+
+\makeatletter
+\def\input@path{ %
+        {@TEST_FOLDER@/latex/} %
+        {@DATA_FOLDER@/latex/} %
+}
+\makeatother
+
+\documentclass[
+    ]{article}
+
+\usepackage[
+chorded,
+pictures,
+repeatchords,
+importantdiagramonly,
+guitar,
+    ]{crepbook}
+
+\usepackage[
+     a4paper % paper size
+     ,includeheadfoot % include header and footer into text size
+     ,hmarginratio=1:1 % ratio between inner and outer margin (default)
+     ,outer=1.8cm % outer margin (right)
+     ,vmarginratio=1:1 % ratio between top and bottom margin
+     ,bmargin=1.3cm % bottom margin
+     ]{geometry}
+
+\usepackage{lmodern}
+
+
+\usepackage[english]{babel}
+\lang{english}
+
+\usepackage{graphicx}
+\graphicspath{ %
+        {@TEST_FOLDER@/} %
+        {@DATA_FOLDER@/} %
+}
+
+
+\makeatletter
+\@ifpackageloaded{hyperref}{}{
+   \usepackage{url}
+   \newcommand{\phantomsection}{}
+   \newcommand{\hyperlink}[2]{#2}
+   \newcommand{\href}[2]{\expandafter\url\expandafter{#1}}
+}
+\makeatother
+
+
+\usepackage{chords}
+
+\title{Guitar songbook}
+\author{The Patacrep Team}
+
+\newindex{titleidx}{lang_default_title}
+\newauthorindex{authidx}{lang_default_auth}
+
+\authignoreword{unknown}
+\authbyword{by}
+\authsepword{and}
+
+\notenamesout{A}{B}{C}{D}{E}{F}{G}
+
+
+\pagestyle{empty}\definecolor{SongNumberBgColor}{HTML}{D1E4AE}
+\definecolor{NoteBgColor}{HTML}{D1E4AE}
+\definecolor{IndexBgColor}{HTML}{D1E4AE}
+
+\renewcommand{\snumbgcolor}{SongNumberBgColor}
+\renewcommand{\notebgcolor}{NoteBgColor}
+\renewcommand{\idxbgcolor}{IndexBgColor}
+
+\definecolor{tango-green-3}{HTML}{4e9a06}
+\definecolor{tango-blue-3}{HTML}{204a87}
+\usepackage[
+    bookmarks,
+    bookmarksopen,
+    hyperfigures=true,
+    colorlinks=true,
+    linkcolor=tango-green-3,
+    urlcolor=tango-blue-3
+    ]{hyperref}
+
+
+\subtitle{}
+\mail{crep@team-on-fire.com}
+\web{http://www.patacrep.com}
+\picture{img/treble_a}
+\picturecopyright{Dbolton \url{http://commons.wikimedia.org/wiki/User:Dbolton}}
+\footer{Generated using Songbook (\url{http://www.patacrep.com})}
+
+
+\begin{document}
+
+\maketitle
+
+
+\showindex{\songindexname}{titleidx}
+\showindex{\authorindexname}{authidx}
+
+% list of chords
+\ifchorded
+  \ifdiagram
+     \phantomsection
+     \addcontentsline{toc}{section}{\chordlistname}
+     \chords
+  \fi
+\fi
+
+\phantomsection
+\addcontentsline{toc}{section}{\songlistname}
+
+
+
+
+
+\end{document}

test/test_songbook/lang_default.yaml

@@ -0,0 +1,2 @@
+content:
+  - sort:

test/test_songbook/lang_en.tex.control

@@ -0,0 +1,131 @@
+
+
+
+
+
+
+%% Automatically generated document.
+%% You may edit this file but all changes will be overwritten.
+%% If you want to change this document, have a look at
+%% the templating system.
+%%
+%% Generated using Songbook <http://www.patacrep.com>
+
+\makeatletter
+\def\input@path{ %
+        {@TEST_FOLDER@/latex/} %
+        {@DATA_FOLDER@/latex/} %
+}
+\makeatother
+
+\documentclass[
+    ]{article}
+
+\usepackage[
+chorded,
+pictures,
+repeatchords,
+importantdiagramonly,
+guitar,
+    ]{crepbook}
+
+\usepackage[
+     a4paper % paper size
+     ,includeheadfoot % include header and footer into text size
+     ,hmarginratio=1:1 % ratio between inner and outer margin (default)
+     ,outer=1.8cm % outer margin (right)
+     ,vmarginratio=1:1 % ratio between top and bottom margin
+     ,bmargin=1.3cm % bottom margin
+     ]{geometry}
+
+\usepackage{lmodern}
+
+
+\usepackage[english]{babel}
+\lang{english}
+
+\usepackage{graphicx}
+\graphicspath{ %
+        {@TEST_FOLDER@/} %
+        {@DATA_FOLDER@/} %
+}
+
+
+\makeatletter
+\@ifpackageloaded{hyperref}{}{
+   \usepackage{url}
+   \newcommand{\phantomsection}{}
+   \newcommand{\hyperlink}[2]{#2}
+   \newcommand{\href}[2]{\expandafter\url\expandafter{#1}}
+}
+\makeatother
+
+
+\usepackage{chords}
+
+\title{Guitar songbook}
+\author{The Patacrep Team}
+
+\newindex{titleidx}{lang_en_title}
+\newauthorindex{authidx}{lang_en_auth}
+
+\authignoreword{unknown}
+\authbyword{by}
+\authsepword{and}
+
+\notenamesout{A}{B}{C}{D}{E}{F}{G}
+
+
+\pagestyle{empty}\definecolor{SongNumberBgColor}{HTML}{D1E4AE}
+\definecolor{NoteBgColor}{HTML}{D1E4AE}
+\definecolor{IndexBgColor}{HTML}{D1E4AE}
+
+\renewcommand{\snumbgcolor}{SongNumberBgColor}
+\renewcommand{\notebgcolor}{NoteBgColor}
+\renewcommand{\idxbgcolor}{IndexBgColor}
+
+\definecolor{tango-green-3}{HTML}{4e9a06}
+\definecolor{tango-blue-3}{HTML}{204a87}
+\usepackage[
+    bookmarks,
+    bookmarksopen,
+    hyperfigures=true,
+    colorlinks=true,
+    linkcolor=tango-green-3,
+    urlcolor=tango-blue-3
+    ]{hyperref}
+
+
+\subtitle{}
+\mail{crep@team-on-fire.com}
+\web{http://www.patacrep.com}
+\picture{img/treble_a}
+\picturecopyright{Dbolton \url{http://commons.wikimedia.org/wiki/User:Dbolton}}
+\footer{Generated using Songbook (\url{http://www.patacrep.com})}
+
+
+\begin{document}
+
+\maketitle
+
+
+\showindex{\songindexname}{titleidx}
+\showindex{\authorindexname}{authidx}
+
+% list of chords
+\ifchorded
+  \ifdiagram
+     \phantomsection
+     \addcontentsline{toc}{section}{\chordlistname}
+     \chords
+  \fi
+\fi
+
+\phantomsection
+\addcontentsline{toc}{section}{\songlistname}
+
+
+
+
+
+\end{document}

test/test_songbook/lang_en.yaml

@@ -0,0 +1,5 @@
+book:
+  lang: en
+
+content:
+  - sort:

test/test_songbook/lang_fr.tex.control

@@ -0,0 +1,131 @@
+
+
+
+
+
+
+%% Automatically generated document.
+%% You may edit this file but all changes will be overwritten.
+%% If you want to change this document, have a look at
+%% the templating system.
+%%
+%% Generated using Songbook <http://www.patacrep.com>
+
+\makeatletter
+\def\input@path{ %
+        {@TEST_FOLDER@/latex/} %
+        {@DATA_FOLDER@/latex/} %
+}
+\makeatother
+
+\documentclass[
+    ]{article}
+
+\usepackage[
+chorded,
+pictures,
+repeatchords,
+importantdiagramonly,
+guitar,
+    ]{crepbook}
+
+\usepackage[
+     a4paper % paper size
+     ,includeheadfoot % include header and footer into text size
+     ,hmarginratio=1:1 % ratio between inner and outer margin (default)
+     ,outer=1.8cm % outer margin (right)
+     ,vmarginratio=1:1 % ratio between top and bottom margin
+     ,bmargin=1.3cm % bottom margin
+     ]{geometry}
+
+\usepackage{lmodern}
+
+
+\usepackage[french]{babel}
+\lang{french}
+
+\usepackage{graphicx}
+\graphicspath{ %
+        {@TEST_FOLDER@/} %
+        {@DATA_FOLDER@/} %
+}
+
+
+\makeatletter
+\@ifpackageloaded{hyperref}{}{
+   \usepackage{url}
+   \newcommand{\phantomsection}{}
+   \newcommand{\hyperlink}[2]{#2}
+   \newcommand{\href}[2]{\expandafter\url\expandafter{#1}}
+}
+\makeatother
+
+
+\usepackage{chords}
+
+\title{Recueil de chansons pour guitare}
+\author{L'équipe Patacrep}
+
+\newindex{titleidx}{lang_fr_title}
+\newauthorindex{authidx}{lang_fr_auth}
+
+\authignoreword{unknown}
+\authbyword{by}
+\authsepword{and}
+
+\notenamesout{La}{Si}{Do}{R\'e}{Mi}{Fa}{Sol}
+
+
+\pagestyle{empty}\definecolor{SongNumberBgColor}{HTML}{D1E4AE}
+\definecolor{NoteBgColor}{HTML}{D1E4AE}
+\definecolor{IndexBgColor}{HTML}{D1E4AE}
+
+\renewcommand{\snumbgcolor}{SongNumberBgColor}
+\renewcommand{\notebgcolor}{NoteBgColor}
+\renewcommand{\idxbgcolor}{IndexBgColor}
+
+\definecolor{tango-green-3}{HTML}{4e9a06}
+\definecolor{tango-blue-3}{HTML}{204a87}
+\usepackage[
+    bookmarks,
+    bookmarksopen,
+    hyperfigures=true,
+    colorlinks=true,
+    linkcolor=tango-green-3,
+    urlcolor=tango-blue-3
+    ]{hyperref}
+
+
+\subtitle{}
+\mail{crep@team-on-fire.com}
+\web{http://www.patacrep.com}
+\picture{img/treble_a}
+\picturecopyright{Dbolton \url{http://commons.wikimedia.org/wiki/User:Dbolton}}
+\footer{Créé avec le programme Songbook (\url{http://www.patacrep.com})}
+
+
+\begin{document}
+
+\maketitle
+
+
+\showindex{\songindexname}{titleidx}
+\showindex{\authorindexname}{authidx}
+
+% list of chords
+\ifchorded
+  \ifdiagram
+     \phantomsection
+     \addcontentsline{toc}{section}{\chordlistname}
+     \chords
+  \fi
+\fi
+
+\phantomsection
+\addcontentsline{toc}{section}{\songlistname}
+
+
+
+
+
+\end{document}

test/test_songbook/lang_fr.yaml

@@ -0,0 +1,5 @@
+book:
+  lang: fr
+
+content:
+  - sort:

test/test_songbook/languages.sb

@@ -1,2 +0,0 @@
-datadir:
-- languages_datadir

test/test_songbook/languages.tex.control

@@ -23,9 +23,10 @@
     ]{article}
 
 \usepackage[
-    chorded,
+chorded,
-diagram,
 pictures,
+repeatchords,
+importantdiagramonly,
 guitar,
     ]{patacrep}
 
@@ -65,6 +66,9 @@ guitar,
 \newindex{titleidx}{languages_title}
 \newauthorindex{authidx}{languages_auth}
 
+\authignoreword{unknown}
+\authbyword{by}
+\authsepword{and}
 
 \notenamesout{A}{B}{C}{D}{E}{F}{G}
 

test/test_songbook/languages.yaml

@@ -0,0 +1,4 @@
+book:
+  template: default.tex
+  datadir:
+  - languages_datadir

test/test_songbook/onthefly/content.onthefly.tex.control

@@ -0,0 +1,133 @@
+
+
+
+
+
+
+%% Automatically generated document.
+%% You may edit this file but all changes will be overwritten.
+%% If you want to change this document, have a look at
+%% the templating system.
+%%
+%% Generated using Songbook <http://www.patacrep.com>
+
+\makeatletter
+\def\input@path{ %
+        {@TEST_FOLDER@/onthefly/../content_datadir/latex/} %
+        {@LOCAL_DATA_FOLDER@/latex/} %
+}
+\makeatother
+
+\documentclass[
+    ]{article}
+
+\usepackage[
+chorded,
+pictures,
+diagram,
+guitar,
+    ]{patacrep}
+
+\usepackage{lmodern}
+
+
+\PassOptionsToPackage{english}{babel}
+\PassOptionsToPackage{english}{babel}
+\usepackage[english]{babel}
+\lang{english}
+
+\usepackage{graphicx}
+\graphicspath{ %
+        {@TEST_FOLDER@/onthefly/../content_datadir/} %
+        {@LOCAL_DATA_FOLDER@/} %
+}
+
+
+\makeatletter
+\@ifpackageloaded{hyperref}{}{
+   \usepackage{url}
+   \newcommand{\phantomsection}{}
+   \newcommand{\hyperlink}[2]{#2}
+   \newcommand{\href}[2]{\expandafter\url\expandafter{#1}}
+}
+\makeatother
+
+
+\usepackage{chords}
+
+\title{Guitar songbook}
+\author{The Patacrep Team}
+
+\newindex{titleidx}{content.onthefly_title}
+\newauthorindex{authidx}{content.onthefly_auth}
+
+\authignoreword{unknown}
+\authbyword{by}
+\authsepword{and}
+
+\notenamesout{A}{B}{C}{D}{E}{F}{G}
+
+
+\begin{document}
+
+\maketitle
+
+
+\showindex{\songindexname}{titleidx}
+\showindex{\authorindexname}{authidx}
+
+% list of chords
+\ifchorded
+  \ifdiagram
+     \phantomsection
+     \addcontentsline{toc}{section}{\chordlistname}
+     \chords
+  \fi
+\fi
+
+\phantomsection
+\addcontentsline{toc}{section}{\songlistname}
+
+
+\section{Test of section}
+
+\begin{songs}{titleidx,authidx}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% songs/./song.csg
+
+\selectlanguage{english}
+
+\beginsong{This is a song}[
+  by={
+  },
+]
+
+
+
+
+\begin{verse}
+    Foo
+\end{verse}
+
+\endsong
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% songs/./song.tsg
+
+\import{@TEST_FOLDER@/content_datadir/songs/}{song.tsg}
+
+\end{songs}
+
+\songsection{Test of song section}
+
+
+\input{@TEST_FOLDER@/content_datadir/content/foo.tex}
+
+
+\section{This is an included section}
+
+
+
+
+
+\end{document}

test/test_songbook/onthefly/content.onthefly.yaml

@@ -0,0 +1,19 @@
+book:
+  pictures: yes
+  datadir: content_datadir
+  lang: en
+  template: default.tex
+chords:
+  repeatchords: no
+  diagramreminder: all
+
+content:
+  - section: Test of section
+  - sort:
+  - songsection: Test of song section
+  - cwd:
+      # relative to datadir 'song' dir
+      path: ../content
+      content:
+        - tex: foo.tex
+  - include: include.sbc

test/test_songbook/syntax.sb

@@ -1,3 +0,0 @@
-datadir:
-- syntax_datadir
-lang: en