Merge branch 'search_file'

examples/songs/subdir/datadir.sg

@@ -1,10 +1,10 @@
 \beginsong{Image included from datadir\\\LaTeX}
-  [cov={datadir}]
+  [cov={img/datadir}]
 
   \cover
 
-  \lilypond{datadir.ly}
+  \lilypond{scores/datadir.ly}
 
-  \image{datadir}
+  \image{img/datadir}
 
 \endsong

examples/songs/subdir/datadir2.sg

@@ -1,10 +1,10 @@
 \beginsong{Image included from a different datadir\\\LaTeX}
-  [cov={datadir2}]
+  [cov={img/datadir2}]
 
   \cover
 
-  \lilypond{datadir2.ly}
+  \lilypond{scores/datadir2.ly}
 
-  \image{datadir2}
+  \image{img/datadir2}
 
 \endsong

patacrep/build.py

@@ -110,8 +110,8 @@ class Songbook(object):
         config['_song_plugins'] = files.load_plugins(
             datadirs=config.get('datadir', []),
             root_modules=['songs'],
-            keyword='SONG_PARSERS',
+            keyword='SONG_RENDERERS',
-            )
+            )['latex']
 
         # Configuration set
         config['render'] = content.render

patacrep/content/song.py

@@ -43,7 +43,7 @@ class SongRenderer(Content):
                 """).format(
                     separator="%"*80,
                     path=self.song.subpath,
-                    song=self.song.render(output=context['filename'], output_format="latex"),
+                    song=self.song.render(output=context['filename']),
                 )
 
 #pylint: disable=unused-argument

patacrep/data/templates/patacrep.tex

@@ -34,7 +34,7 @@
       },
 "picture": {"description": {"english": "Cover picture", "french": "Image de couverture"},
             "type": "file",
-            "default": {"default": "treble_a"}
+            "default": {"default": "img/treble_a"}
          },
 "picturecopyright": {"description": {"english": "Copyright for the cover picture", "french": "Copyright pour l'image de couverture"},
                      "default": {"default": "Dbolton \\url{http://commons.wikimedia.org/wiki/User:Dbolton}"}

patacrep/data/templates/songs.tex

@@ -89,7 +89,7 @@
 \usepackage{graphicx}
 \graphicspath{ %
     (* for dir in datadir *)
-        {(( path2posix(dir) ))/img/} %
+        {(( path2posix(dir) ))/} %
     (* endfor *)
 }
 

patacrep/files.py

@@ -8,6 +8,8 @@ import posixpath
 import re
 import sys
 
+from patacrep import utils
+
 LOGGER = logging.getLogger(__name__)
 
 def recursive_find(root_directory, extensions):
@@ -104,7 +106,7 @@ def load_plugins(datadirs, root_modules, keyword):
     - keys are the keywords ;
     - values are functions triggered when this keyword is met.
     """
-    plugins = {}
+    plugins = utils.DictOfDict()
 
     datadir_path = [
         os.path.join(datadir, "python", *root_modules)
@@ -121,13 +123,5 @@ def load_plugins(datadirs, root_modules, keyword):
             prefix="patacrep.{}.".format(".".join(root_modules))
         ):
         if hasattr(module, keyword):
-            for (key, value) in getattr(module, keyword).items():
+            plugins.update(getattr(module, keyword))
-                if key in plugins:
-                    LOGGER.warning(
-                        "File %s: Keyword '%s' is already used. Ignored.",
-                        module.__file__,
-                        key,
-                        )
-                    continue
-                plugins[key] = value
     return plugins

patacrep/songs/__init__.py

@@ -168,16 +168,12 @@ class Song:
     def __repr__(self):
         return repr((self.titles, self.data, self.fullpath))
 
-    def render(self, output_format, output=None, *args, **kwargs):
+    def render(self, output=None, *args, **kwargs):
         """Return the code rendering this song.
 
         Arguments:
-        - output_format: Format of the output file (latex, chordpro...)
         - output: Name of the output file, or `None` if irrelevant.
         """
-        method = "render_{}".format(output_format)
-        if hasattr(self, method):
-            return getattr(self, method)(output, *args, **kwargs)
         raise NotImplementedError()
 
     def _parse(self, config): # pylint: disable=no-self-use
@@ -204,15 +200,24 @@ class Song:
             if os.path.isdir(fullpath):
                 yield fullpath
 
-    def search_file(self, filename, extensions=None, directories=None):
+    def search_datadir_file(self, filename, extensions=None, directories=None):
         """Search for a file name.
 
         :param str filename: The name, as provided in the chordpro file (with or without extension).
         :param list extensions: Possible extensions (with '.'). Default is no extension.
         :param iterator directories: Other directories where to search for the file
                                 The directory where the Song file is stored is added to the list.
-
+        :return: A tuple `(datadir, filename, extension)` if file has been
-        Returns None if nothing found.
+            found. It is guaranteed that `os.path.join(datadir,
+            filename+extension)` is a (relative or absolute) valid path to an
+            existing filename.
+            * `datadir` is the datadir in which the file has been found. Can be
+                the empty string.
+            * `filename` is the filename, relative to the datadir.
+            * `extension` is the extension that is to be appended to the
+                filename to get the real filename. Can be the empty string.
+
+        Raise `FileNotFoundError` if nothing found.
 
         This function can also be used as a preprocessor for a renderer: for
         instance, it can compile a file, place it in a temporary folder, and
@@ -224,27 +229,42 @@ class Song:
             directories = self.config['datadir']
 
         songdir = os.path.dirname(self.fullpath)
+        for extension in extensions:
+            if os.path.isfile(os.path.join(songdir, filename + extension)):
+                return "", os.path.join(songdir, filename), extension
 
-        for directory in [songdir] + list(directories):
+        for directory in directories:
             for extension in extensions:
-                fullpath = os.path.join(directory, filename + extension)
+                if os.path.isfile(os.path.join(directory, filename + extension)):
-                if os.path.isfile(fullpath):
+                    return directory, filename, extension
-                    return os.path.abspath(fullpath)
+
-        return None
+        raise FileNotFoundError(filename)
+
+    def search_file(self, filename, extensions=None, *, datadirs=None):
+        """Return the path to a file present in a datadir.
+
+        Implementation is specific to each renderer, as:
+        - some renderers can preprocess files;
+        - some renderers can return the absolute path, other can return something else;
+        - etc.
+        """
+        raise NotImplementedError()
 
-    def search_image(self, filename, none_if_not_found=False):
+    def search_image(self, filename):
         """Search for an image file"""
-        filepath = self.search_file(
+        return self.search_file(
             filename,
             ['', '.jpg', '.png'],
-            self.get_datadirs('img'),
+            datadirs=self.get_datadirs('img'),
             )
-        return filepath if none_if_not_found or filepath else filename
 
-    def search_partition(self, filename, none_if_not_found=False):
+    def search_partition(self, filename):
         """Search for a lilypond file"""
-        filepath = self.search_file(filename, ['', '.ly'])
+        return self.search_file(
-        return filepath if none_if_not_found or filepath else filename
+            filename,
+            ['', '.ly'],
+            datadirs=self.get_datadirs('scores'),
+            )
 
 def unprefixed_title(title, prefixes):
     """Remove the first prefix of the list in the beginning of title (if any).

patacrep/songs/chordpro/__init__.py

@@ -1,9 +1,10 @@
 """Chordpro parser"""
 
-from jinja2 import Environment, FileSystemLoader, contextfunction
+from jinja2 import Environment, FileSystemLoader, contextfunction, ChoiceLoader
 import jinja2
+import logging
 import os
-import pkg_resources
+from pkg_resources import resource_filename
 
 from patacrep import encoding, files
 from patacrep.songs import Song
@@ -11,24 +12,13 @@ from patacrep.songs.chordpro.syntax import parse_song
 from patacrep.templates import Renderer
 from patacrep.latex import lang2babel
 
+LOGGER = logging.getLogger(__name__)
+
 class ChordproSong(Song):
-    """Chordpros song parser."""
+    """Chordpro song parser"""
+    # pylint: disable=abstract-method
 
-    @staticmethod
+    output_language = None
-    def iter_template_paths(templatedirs, output_format):
-        """Iterate over paths in which templates are to be searched.
-
-        :param iterator templatedirs: Iterators of additional directories (the
-            default hard-coded template directory is returned last).
-        :param str output_format: Song output format, which is appended to
-            each directory.
-        """
-        for directory in templatedirs:
-            yield os.path.join(directory, output_format)
-        yield os.path.join(
-            os.path.abspath(pkg_resources.resource_filename(__name__, 'data')),
-            output_format,
-            )
 
     def _parse(self, config):
         """Parse content, and return the dictionary of song data."""
@@ -42,10 +32,7 @@ class ChordproSong(Song):
             'song': song,
             }
 
-    def render(self, output_format, output=None, template="song", templatedirs=None): # pylint: disable=arguments-differ
+    def render(self, output=None, template="song"): # pylint: disable=arguments-differ
-        if templatedirs is None:
-            templatedirs = []
-
         context = {
             'lang': self.lang,
             "titles": self.titles,
@@ -56,9 +43,14 @@ class ChordproSong(Song):
             "content": self.cached['song'].content,
             }
 
-        jinjaenv = Environment(loader=FileSystemLoader(
+        jinjaenv = Environment(loader=ChoiceLoader([
-            self.iter_template_paths(templatedirs, output_format)
+            FileSystemLoader(
-            ))
+                self.get_datadirs(os.path.join("templates", self.output_language))
+            ),
+            FileSystemLoader(
+                os.path.join(resource_filename(__name__, 'data'), self.output_language)
+            ),
+            ]))
         jinjaenv.filters['search_image'] = self.search_image
         jinjaenv.filters['search_partition'] = self.search_partition
         jinjaenv.filters['lang2babel'] = lang2babel
@@ -70,7 +62,7 @@ class ChordproSong(Song):
                 jinjaenv=jinjaenv,
                 ).template.render(context)
         except jinja2.exceptions.TemplateNotFound:
-            raise NotImplementedError("Cannot convert to format '{}'.".format(output_format))
+            raise NotImplementedError("Cannot convert to format '{}'.".format(self.output_language))
 
     @staticmethod
     @contextfunction
@@ -79,6 +71,72 @@ class ChordproSong(Song):
         context.vars['content'] = content
         return context.environment.get_template(content.template()).render(context)
 
-SONG_PARSERS = {
+class Chordpro2HtmlSong(ChordproSong):
-    'sgc': ChordproSong,
+    """Render chordpro song to html code"""
+
+    output_language = "html"
+
+    def search_file(self, filename, extensions=None, *, datadirs=None):
+        try:
+            datadir, filename, extension = self.search_datadir_file(filename, extensions, datadirs)
+            return os.path.join(datadir, filename + extension)
+        except FileNotFoundError:
+            LOGGER.warning(
+                "Song '%s' (datadir '%s'): File '%s' not found.",
+                self.subpath, self.datadir, filename,
+                )
+            return None
+
+class Chordpro2LatexSong(ChordproSong):
+    """Render chordpro song to latex code"""
+
+    output_language = "latex"
+
+    def search_file(self, filename, extensions=None, *, datadirs=None):
+        _datadir, filename, _extension = self.search_datadir_file(
+            filename,
+            extensions,
+            datadirs,
+            )
+        return filename
+
+    def search_partition(self, filename):
+        try:
+            return os.path.join("scores", super().search_partition(filename))
+        except FileNotFoundError:
+            LOGGER.warning(
+                "Song '%s' (datadir '%s'): Score '%s' not found.",
+                self.subpath, self.datadir, filename,
+                )
+            return None
+
+    def search_image(self, filename):
+        try:
+            return os.path.join("img", super().search_image(filename))
+        except FileNotFoundError:
+            LOGGER.warning(
+                "Song '%s' (datadir '%s'): Image '%s' not found.",
+                self.subpath, self.datadir, filename,
+                )
+            return None
+
+class Chordpro2ChordproSong(ChordproSong):
+    """Render chordpro song to chordpro code"""
+
+    output_language = "chordpro"
+
+    def search_file(self, filename, extensions=None, *, datadirs=None):
+        # pylint: disable=unused-variable
+        return filename
+
+SONG_RENDERERS = {
+    "latex": {
+        'sgc': Chordpro2LatexSong,
+        },
+    "html": {
+        'sgc': Chordpro2HtmlSong,
+        },
+    "chordpro": {
+        'sgc': Chordpro2ChordproSong,
+        },
     }

patacrep/songs/chordpro/data/html/content_image

@@ -1 +1,6 @@
+(* block image *)
+(* set image = content.argument|search_image *)
+(* if image *)
 <img src="(( content.argument|search_image ))">
+(* endif *)
+(* endblock *)

patacrep/songs/chordpro/data/html/content_metadata_cover

@@ -1,3 +1,8 @@
+(* block cov *)
 (* if 'cov' in metadata -*)
-  <img src="(( metadata['cov'].argument|search_image ))"><br>
+  (* set cov = metadata['cov'].argument|search_image *)
-(* endif *)
+  (* if cov *)
+    <img src="(( cov ))"><br>
+  (* endif *)
+(* endif *)
+(* endblock *)

patacrep/songs/chordpro/data/html/content_partition

@@ -1 +1,6 @@
-<a class="song-partition" href="(( content.argument|search_partition ))">((content.argument))</a>
+(* block partition *)
+(* set partition = content.argument|search_partition *)
+(* if partition *)
+<a class="song-partition" href="(( partition ))">((content.argument))</a>
+(* endif *)
+(* endblock *)

patacrep/songs/chordpro/data/latex/content_image

@@ -1 +1,6 @@
-\image{(( content.argument|search_image ))}
+(* block image *)
+(* set image = content.argument|search_image *)
+(* if image *)
+\image{(( image ))}
+(*- endif *)
+(*- endblock *)

patacrep/songs/chordpro/data/latex/content_partition

@@ -1 +1,6 @@
+(* block partition *)
+(* set partition = content.argument|search_partition *)
+(* if partition *)
 \lilypond{ ((- content.argument|search_partition -)) }
+(*- endif -*)
+(*- endblock -*)

patacrep/songs/chordpro/data/latex/song

@@ -28,7 +28,12 @@
   (* endif *)
   (* endfor *)
   (* if 'cov' in metadata *)
-  cov={(( metadata["cov"].argument|search_image ))},
+    (* block cov *)
+      (* set cov = metadata["cov"].argument|search_image *)
+      (* if cov *)
+    cov={(( cov ))},
+      (* endif *)
+    (* endblock *)
   (* endif *)
   (* for key in metadata.morekeys *)
   (( key.keyword ))={(( key.argument ))},

patacrep/songs/convert/__main__.py

@@ -36,22 +36,29 @@ if __name__ == "__main__":
     dest = sys.argv[2]
     song_files = sys.argv[3:]
 
-    song_parsers = files.load_plugins(
+    renderers = files.load_plugins(
         datadirs=DEFAULT_CONFIG.get('datadir', []),
         root_modules=['songs'],
-        keyword='SONG_PARSERS',
+        keyword='SONG_RENDERERS',
         )
 
-    if source not in song_parsers:
+    if dest not in renderers:
         LOGGER.error(
-            "Unknown file format '%s'. Available ones are %s.",
+            "Unknown destination file format '%s'. Available ones are %s.",
             source,
-            ", ".join(["'{}'".format(key) for key in song_parsers.keys()])
+            ", ".join(["'{}'".format(key) for key in renderers.keys()])
+            )
+        sys.exit(1)
+    if source not in renderers[dest]:
+        LOGGER.error(
+            "Unknown source file format '%s'. Available ones are %s.",
+            source,
+            ", ".join(["'{}'".format(key) for key in renderers[dest].keys()])
             )
         sys.exit(1)
 
     for file in song_files:
-        song = song_parsers[source](file, DEFAULT_CONFIG)
+        song = renderers[dest][source](file, DEFAULT_CONFIG)
         try:
             destname = "{}.{}".format(".".join(file.split(".")[:-1]), dest)
             if os.path.exists(destname):

patacrep/songs/latex/__init__.py

@@ -11,8 +11,9 @@ from patacrep import files, encoding
 from patacrep.latex import parse_song, BABEL_LANGUAGES
 from patacrep.songs import Song
 
-class LatexSong(Song):
+class Latex2LatexSong(Song):
-    """LaTeX song parser."""
+    """Song written in LaTeX, rendered in LaTeX"""
+    # pylint: disable=abstract-method
 
     def _parse(self, __config):
         """Parse content, and return the dictionary of song data."""
@@ -28,8 +29,9 @@ class LatexSong(Song):
         else:
             self.authors = []
 
-    def render_latex(self, output):
+    def render(self, output):
         """Return the code rendering the song."""
+        # pylint: disable=signature-differs
         if output is None:
             raise ValueError(output)
         path = files.path2posix(files.relpath(
@@ -50,7 +52,10 @@ class LatexSong(Song):
         BABEL_LANGUAGES[custom_lang] = language
         self.lang = custom_lang
 
-SONG_PARSERS = {
+SONG_RENDERERS = {
-    'is': LatexSong,
+    "latex": {
-    'sg': LatexSong,
+        'is': Latex2LatexSong,
-    }
+        'sg': Latex2LatexSong,
+    },
+}
+

patacrep/utils.py

@@ -0,0 +1,61 @@
+"""Some utility functions"""
+
+from collections import UserDict
+
+class DictOfDict(UserDict):
+    """Dictionary, with a recursive :meth:`update` method.
+
+    By "recursive", we mean: if `self.update(other)` is called, and for some
+    key both `self[key]` and `other[key]` are dictionary, then `self[key]` is
+    not replaced by `other[key]`, but instead is updated. This is done
+    recursively (that is, `self[foo][bar][baz]` is updated with
+    `other[foo][bar][baz]`, if the corresponding objects are dictionaries).
+
+    >>> ordinal = DictOfDict({
+    ... "francais": {
+    ...     1: "premier",
+    ...     2: "deuxieme",
+    ...     },
+    ... "english": {
+    ...     1: "first",
+    ...     },
+    ... })
+    >>> ordinal.update({
+    ... "francais": {
+    ...     2: "second",
+    ...     3: "troisieme",
+    ...     },
+    ... "espanol": {
+    ...     1: "primero",
+    ...     },
+    ... })
+    >>> ordinal == {
+    ...     "francais": {
+    ...         1: "premier",
+    ...         2: "second",
+    ...         3: "troisieme",
+    ...         },
+    ...     "english": {
+    ...         1: "first",
+    ...         },
+    ...     "espanol": {
+    ...         1: "primero",
+    ...         },
+    ...     }
+    True
+    """
+
+    def update(self, other):
+        # pylint: disable=arguments-differ
+        self._update(self, other)
+
+    @staticmethod
+    def _update(left, right):
+        """Equivalent to `left.update(right)`, with recursive update."""
+        for key in right:
+            if key not in left:
+                left[key] = right[key]
+            elif isinstance(left[key], dict) and isinstance(right[key], dict):
+                DictOfDict._update(left[key], right[key])
+            else:
+                left[key] = right[key]

test/test_chordpro/greensleeves.tex

@@ -7,14 +7,14 @@ Un sous titre}[
   by={
          Traditionnel  },
   album={Angleterre},
-  cov={traditionnel},
+    cov={img/traditionnel},
 ]
 
 \cover
 
 
 
-\lilypond{greensleeves.ly}
+\lilypond{scores/greensleeves.ly}
 
 
 

test/test_chordpro/metadata.sgc

@@ -10,13 +10,13 @@
 {artist:  Author2}
 {album: Album}
 {copyright: Copyright}
-{cov: Cover}
+{cov: metadata_cover}
 {key: foo: Foo}
 
 {comment: Comment}
 {guitar_comment: GuitarComment}
-{partition: Lilypond}
+{partition: metadata_lilypond}
-{image: Image}
+{image: metadata_image}
 
 
 Foo

test/test_chordpro/metadata.source

@@ -10,12 +10,12 @@
 {artist: Author2}
 {album: Album}
 {copyright: Copyright}
-{cover: Cover}
+{cover: metadata_cover}
 {capo: Capo}
 {key: foo: Foo}
 {comment: Comment}
 {guitar_comment: GuitarComment}
-{partition: Lilypond}
+{partition: metadata_lilypond}
-{image: Image}
+{image: metadata_image}
 
 Foo

test/test_chordpro/metadata.tex

@@ -11,7 +11,7 @@ Subtitle5}[
          Author2  },
   album={Album},
   copyright={Copyright},
-  cov={Cover},
+    cov={img/test/test_chordpro/metadata_cover},
   foo={Foo},
 ]
 
@@ -19,8 +19,8 @@ Subtitle5}[
 
 \textnote{Comment}
 \musicnote{GuitarComment}
-\lilypond{Lilypond}
+\lilypond{scores/test/test_chordpro/metadata_lilypond}
-\image{Image}
+\image{img/test/test_chordpro/metadata_image}
 
 
 

test/test_chordpro/test_parser.py

@@ -5,9 +5,10 @@
 import glob
 import os
 import unittest
+from pkg_resources import resource_filename
 
+from patacrep import files
 from patacrep.build import DEFAULT_CONFIG
-from patacrep.songs.chordpro import ChordproSong
 
 from .. import disable_logging
 
@@ -26,11 +27,22 @@ class FileTestMeta(type):
     def __init__(cls, name, bases, nmspc):
         super().__init__(name, bases, nmspc)
 
+        # Setting datadir
+        cls.config = DEFAULT_CONFIG
+        if 'datadir' not in cls.config:
+            cls.config['datadir'] = []
+        cls.config['datadir'].append(resource_filename(__name__, 'datadir'))
+
+        cls.song_plugins = files.load_plugins(
+            datadirs=cls.config['datadir'],
+            root_modules=['songs'],
+            keyword='SONG_RENDERERS',
+            )
         for source in sorted(glob.glob(os.path.join(
                 os.path.dirname(__file__),
                 '*.source',
             ))):
-            base = source[:-len(".source")]
+            base = os.path.relpath(source, os.getcwd())[:-len(".source")]
             for dest in LANGUAGES:
                 destname = "{}.{}".format(base, dest)
                 if not os.path.exists(destname):
@@ -55,9 +67,8 @@ class FileTestMeta(type):
                 chordproname = "{}.source".format(base)
                 with disable_logging():
                     self.assertMultiLineEqual(
-                        ChordproSong(chordproname, DEFAULT_CONFIG).render(
+                        self.song_plugins[LANGUAGES[dest]]['sgc'](chordproname, self.config).render(
                             output=chordproname,
-                            output_format=LANGUAGES[dest],
                             ).strip(),
                         expectfile.read().strip(),
                         )