Setting up file type plugins.

patacrep/build.py

@@ -9,7 +9,7 @@ import logging
 import os.path
 from subprocess import Popen, PIPE, call
 
-from patacrep import __DATADIR__, authors, content, errors
+from patacrep import __DATADIR__, authors, content, errors, files
 from patacrep.index import process_sxd
 from patacrep.templates import TexRenderer
 from patacrep.songs import DataSubpath
@@ -99,8 +99,25 @@ class Songbook(object):
                 copy.deepcopy(config['authwords'])
                 )
 
-        # Configuration set
+        # Loading custom plugins
+        config['_content_plugins'] = files.load_plugins(
+            datadirs=config.get('datadir', []),
+            subdir=['content'],
+            variable='CONTENT_PLUGINS',
+            error=(
+                "File {filename}: Keyword '{keyword}' is already used. Ignored."
+                ),
+            )
+        config['_file_plugins'] = files.load_plugins(
+            datadirs=config.get('datadir', []),
+            subdir=['songs'],
+            variable='FILE_PLUGINS',
+            error=(
+                "File {filename}: Keyword '{keyword}' is already used. Ignored."
+                ),
+            )
 
+        # Configuration set
         config['render_content'] = content.render_content
         config['content'] = content.process_content(
                 config.get('content', []),

patacrep/content/__init__.py

@@ -69,7 +69,6 @@ More documentation in the docstring of Content.
 """
 
 import glob
-import importlib
 import jinja2
 import logging
 import os
@@ -134,53 +133,6 @@ class ContentError(SongbookError):
     def __str__(self):
         return "Content: {}: {}".format(self.keyword, self.message)
 
-def load_plugins(config):
-    """Load all content plugins, and return a dictionary of those plugins.
-
-    Return value: a dictionary where:
-    - keys are the keywords ;
-    - values are functions triggered when this keyword is met.
-    """
-    plugins = {}
-    directory_list = (
-            [
-                os.path.join(datadir, "python", "content")
-                for datadir in config.get('datadir', [])
-            ]
-            + [os.path.dirname(__file__)]
-            )
-    for directory in directory_list:
-        if not os.path.exists(directory):
-            LOGGER.debug(
-                    "Ignoring non-existent directory '%s'.",
-                    directory
-                    )
-            continue
-        sys.path.append(directory)
-        for name in glob.glob(os.path.join(directory, '*.py')):
-            if name.endswith(".py") and os.path.basename(name) != "__init__.py":
-                if directory == os.path.dirname(__file__):
-                    plugin = importlib.import_module(
-                            'patacrep.content.{}'.format(
-                                os.path.basename(name[:-len('.py')])
-                                )
-                            )
-                else:
-                    plugin = importlib.import_module(
-                                os.path.basename(name[:-len('.py')])
-                                )
-                for (key, value) in plugin.CONTENT_PLUGINS.items():
-                    if key in plugins:
-                        LOGGER.warning(
-                            "File %s: Keyword '%s' is already used. Ignored.",
-                            files.relpath(name),
-                            key,
-                            )
-                        continue
-                    plugins[key] = value
-        del sys.path[-1]
-    return plugins
-
 @jinja2.contextfunction
 def render_content(context, content):
     """Render the content of the songbook as a LaTeX code.
@@ -224,7 +176,8 @@ def process_content(content, config=None):
     included in the .tex file.
     """
     contentlist = []
-    plugins = load_plugins(config)
+    plugins = config.get('_content_plugins', {})
+
     keyword_re = re.compile(r'^ *(?P<keyword>\w*) *(\((?P<argument>.*)\))? *$')
     if not content:
         content = [["song"]]

patacrep/content/song.py

@@ -4,42 +4,15 @@
 """Plugin to include songs to the songbook."""
 
 import glob
-import jinja2
 import logging
 import os
 
-from patacrep.content import Content, process_content, ContentError
+from patacrep.content import process_content, ContentError
 from patacrep import files, errors
 from patacrep.songs import Song
 
 LOGGER = logging.getLogger(__name__)
 
-class SongRenderer(Content, Song):
-    """Render a song in the .tex file."""
-
-    def begin_new_block(self, previous, __context):
-        """Return a boolean stating if a new block is to be created."""
-        return not isinstance(previous, SongRenderer)
-
-    def begin_block(self, context):
-        """Return the string to begin a block."""
-        indexes = context.resolve("indexes")
-        if isinstance(indexes, jinja2.runtime.Undefined):
-            indexes = ""
-        return r'\begin{songs}{%s}' % indexes
-
-    def end_block(self, __context):
-        """Return the string to end a block."""
-        return r'\end{songs}'
-
-    def render(self, context):
-        """Return the string that will render the song."""
-        return r'\input{{{}}}'.format(files.path2posix(
-                                    files.relpath(
-                                        self.fullpath,
-                                        os.path.dirname(context['filename'])
-                                    )))
-
 #pylint: disable=unused-argument
 def parse(keyword, argument, contentlist, config):
     """Parse data associated with keyword 'song'.
@@ -51,22 +24,17 @@ def parse(keyword, argument, contentlist, config):
       expressions (interpreted using the glob module), referring to songs.
     - config: the current songbook configuration dictionary.
 
-    Return a list of SongRenderer() instances.
+    Return a list of Song() instances.
     """
     if '_languages' not in config:
         config['_languages'] = set()
     songlist = []
+    plugins = config.get('_file_plugins', {})
     for songdir in config['_songdir']:
         if contentlist:
             break
-        contentlist = [
+        contentlist = files.recursive_find(songdir.fullpath, plugins.keys())
-                filename
+
-                for filename
-                in (
-                    files.recursive_find(songdir.fullpath, "*.sg")
-                    + files.recursive_find(songdir.fullpath, "*.is")
-                    )
-                ]
     for elem in contentlist:
         before = len(songlist)
         for songdir in config['_songdir']:
@@ -74,21 +42,16 @@ def parse(keyword, argument, contentlist, config):
                 continue
             with files.chdir(songdir.datadir):
                 for filename in glob.iglob(os.path.join(songdir.subpath, elem)):
-                    if not (
+                    LOGGER.debug('Parsing file "{}"…'.format(filename))
-                            filename.endswith('.sg') or
+                    try:
-                            filename.endswith('.is')
+                        renderer = plugins[filename.split('.')[-1]]
-                            ):
+                    except KeyError:
                         LOGGER.warning((
-                            'File "{}" is not a ".sg" or ".is" file. Ignored.'
+                            'I do not know how to parse file "{}". Ignored.'
                             ).format(os.path.join(songdir.datadir, filename))
                             )
                         continue
-                    LOGGER.debug('Parsing file "{}"…'.format(filename))
+                    song = renderer(songdir.datadir, filename, config)
-                    song = SongRenderer(
-                            songdir.datadir,
-                            filename,
-                            config,
-                            )
                     songlist.append(song)
                     config["_languages"].update(song.languages)
             if len(songlist) > before:
@@ -129,7 +92,7 @@ def process_songs(content, config=None):
             item
             for item
             in contentlist
-            if not isinstance(item, SongRenderer)
+            if not isinstance(item, Song)
             ]
     if not_songs:
         raise OnlySongsError(not_songs)

patacrep/files.py

@@ -2,22 +2,30 @@
 """File system utilities."""
 
 from contextlib import contextmanager
-import fnmatch
+import glob
+import importlib
+import logging
 import os
 import posixpath
+import re
+import sys
 
-def recursive_find(root_directory, pattern):
+LOGGER = logging.getLogger(__name__)
+
+def recursive_find(root_directory, extensions):
     """Recursively find files matching a pattern, from a root_directory.
 
-    Return a list of files matching the pattern.
+    Return a list of files matching the pattern. TODO
     """
     if not os.path.isdir(root_directory):
         return []
 
     matches = []
+    pattern = re.compile(r'.*\.({})$'.format('|'.join(extensions)))
     with chdir(root_directory):
         for root, __ignored, filenames in os.walk(os.curdir):
-            for filename in fnmatch.filter(filenames, pattern):
+            for filename in filenames:
+                if pattern.match(filename):
                     matches.append(os.path.join(root, filename))
     return matches
 
@@ -59,3 +67,63 @@ def chdir(path):
         os.chdir(olddir)
     else:
         yield
+
+def load_plugins(datadirs, subdir, variable, error):
+    """Load all content plugins, and return a dictionary of those plugins.
+
+    A plugin is a .py file, submodule of `subdir`, located in one of the
+    directories of `datadirs`. It contains a dictionary `variable`. The return
+    value is the union of the dictionaries of the loaded plugins.
+
+    Arguments:
+    - datadirs: list of directories (as strings) in which files has to be
+      searched.
+    - subdir: modules (as a list of strings) files has to be submodules of
+      (e.g. if `subdir` is `['first', 'second']`, search files are of the form
+      `first/second/*.py`.
+    - variable: Name of the variable holding the dictionary.
+    - error: Error message raised if a key appears several times.
+    """
+    plugins = {}
+    directory_list = (
+            [
+                os.path.join(datadir, "python", *subdir) #pylint: disable=star-args
+                for datadir in datadirs
+            ]
+            + [os.path.dirname(__file__)]
+            )
+    for directory in directory_list:
+        if not os.path.exists(directory):
+            LOGGER.debug(
+                    "Ignoring non-existent directory '%s'.",
+                    directory
+                    )
+            continue
+        sys.path.append(directory)
+        for name in glob.glob(os.path.join(directory, *(subdir + ['*.py']))):
+            if name.endswith(".py") and os.path.basename(name) != "__init__.py":
+                if directory == os.path.dirname(__file__):
+                    plugin = importlib.import_module(
+                            'patacrep.{}.{}'.format(
+                                ".".join(subdir),
+                                os.path.basename(name[:-len('.py')])
+                                )
+                            )
+                else:
+                    plugin = importlib.import_module(
+                                os.path.basename(name[:-len('.py')])
+                                )
+                for (key, value) in getattr(plugin, variable, {}).items():
+                    if key in plugins:
+                        LOGGER.warning(
+                                error.format(
+                                filename=relpath(name),
+                                key=key,
+                                )
+                                )
+                        continue
+                    plugins[key] = value
+        del sys.path[-1]
+    return plugins
+
+

patacrep/songs.py → patacrep/songs/__init__.py

@@ -4,13 +4,14 @@
 
 import errno
 import hashlib
+import jinja2
 import logging
 import os
 import pickle
 import re
 
 from patacrep.authors import processauthors
-from patacrep.latex import parsesong
+from patacrep.content import Content
 
 LOGGER = logging.getLogger(__name__)
 
@@ -62,18 +63,32 @@ class DataSubpath(object):
         self.subpath = os.path.join(self.subpath, path)
         return self
 
-# pylint: disable=too-few-public-methods, too-many-instance-attributes
+# pylint: disable=too-many-instance-attributes
-class Song(object):
+class Song(Content):
-    """Song management"""
+    """Song (or song metadata)
+
+    This class represents a song, bound to a file.
+
+    - It can parse the file given in arguments.
+    - It can render the song as some LaTeX code.
+    - Its content is cached, so that if the file has not been changed, the
+      file is not parsed again.
+
+    This class is inherited by classes implementing song management for
+    several file formats. Those subclasses must implement:
+    - `parse()` to parse the file;
+    - `render()` to render the song as LaTeX code.
+    """
 
     # Version format of cached song. Increment this number if we update
     # information stored in cache.
-    CACHE_VERSION = 0
+    CACHE_VERSION = 1
 
     # List of attributes to cache
     cached_attributes = [
             "titles",
             "unprefixed_titles",
+            "cached",
             "data",
             "datadir",
             "fullpath",
@@ -109,10 +124,14 @@ class Song(object):
                         self.fullpath
                         ))
 
-        # Data extraction from the latex song
+        # Default values
-        self.data = parsesong(self.fullpath)
+        self.data = {}
-        self.titles = self.data['@titles']
+        self.titles = []
-        self.languages = self.data['@languages']
+        self.languages = []
+        self.authors = []
+
+        # Parsing and data processing
+        self.parse()
         self.datadir = datadir
         self.unprefixed_titles = [
                 unprefixed_title(
@@ -123,13 +142,15 @@ class Song(object):
                 in self.titles
                 ]
         self.subpath = subpath
-        if "by" in self.data:
         self.authors = processauthors(
-                    self.data["by"],
+                self.authors,
                 **config["_compiled_authwords"]
                 )
-        else:
+
-            self.authors = []
+        # Cache management
+
+        #: Special attribute to allow plugins to store cached data
+        self.cached = None
 
         self._version = self.CACHE_VERSION
         self._write_cache()
@@ -149,6 +170,50 @@ class Song(object):
     def __repr__(self):
         return repr((self.titles, self.data, self.fullpath))
 
+    def begin_new_block(self, previous, __context):
+        """Return a boolean stating if a new block is to be created."""
+        return not isinstance(previous, Song)
+
+    def begin_block(self, context):
+        """Return the string to begin a block."""
+        indexes = context.resolve("indexes")
+        if isinstance(indexes, jinja2.runtime.Undefined):
+            indexes = ""
+        return r'\begin{songs}{%s}' % indexes
+
+    def end_block(self, __context):
+        """Return the string to end a block."""
+        return r'\end{songs}'
+
+    def render(self, __context):
+        """Returns the TeX code rendering the song.
+
+        This function is to be defined by subclasses.
+        """
+        return ''
+
+    def parse(self):
+        """Parse file `self.fullpath`.
+
+        This function is to be defined by subclasses.
+
+        It set the following attributes:
+
+        - titles: the list of (raw) titles. This list will be processed to
+          remove prefixes.
+        - languages: the list of languages used in the song, as languages
+          recognized by the LaTeX babel package.
+        - authors: the list of (raw) authors. This list will be processed to
+          'clean' it (see function :func:`patacrep.authors.processauthors`).
+        - data: song metadata. Used (among others) to sort the songs.
+        - cached: additional data that will be cached. Thus, data stored in
+          this attribute must be picklable.
+        """
+        self.data = {}
+        self.titles = []
+        self.languages = []
+        self.authors = []
+
 def unprefixed_title(title, prefixes):
     """Remove the first prefix of the list in the beginning of title (if any).
     """
@@ -158,4 +223,3 @@ def unprefixed_title(title, prefixes):
             return match.group(2)
     return title
 
-

patacrep/songs/tex.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+
+"""Very simple LaTeX parsing."""
+
+import os
+
+from patacrep import files
+from patacrep.latex import parsesong
+from patacrep.songs import Song
+
+class TexRenderer(Song):
+    """Renderer for song and intersong files."""
+
+    def parse(self):
+        """Parse song and set metadata."""
+        self.data = parsesong(self.fullpath)
+        self.titles = self.data['@titles']
+        self.languages = self.data['@languages']
+        self.authors = self.data['by']
+
+    def render(self, context):
+        """Return the string that will render the song."""
+        return r'\input{{{}}}'.format(files.path2posix(
+                                    files.relpath(
+                                        self.fullpath,
+                                        os.path.dirname(context['filename'])
+                                    )))
+
+FILE_PLUGINS = {
+        'sg': TexRenderer,
+        'is': TexRenderer,
+        }