From fe2d2da9584eb4e89f896b64ec0aa87f3f5e25eb Mon Sep 17 00:00:00 2001 From: Louis Date: Sat, 18 Oct 2014 18:39:55 +0200 Subject: [PATCH] Setting up file type plugins. --- patacrep/build.py | 21 ++++- patacrep/content/__init__.py | 51 +----------- patacrep/content/song.py | 61 +++------------ patacrep/files.py | 78 +++++++++++++++++-- patacrep/{songs.py => songs/__init__.py} | 98 ++++++++++++++++++++---- patacrep/songs/tex.py | 32 ++++++++ 6 files changed, 219 insertions(+), 122 deletions(-) rename patacrep/{songs.py => songs/__init__.py} (62%) create mode 100644 patacrep/songs/tex.py diff --git a/patacrep/build.py b/patacrep/build.py index 212305ca..be9b48f4 100644 --- a/patacrep/build.py +++ b/patacrep/build.py @@ -9,7 +9,7 @@ import logging import os.path from subprocess import Popen, PIPE, call -from patacrep import __DATADIR__, authors, content, errors +from patacrep import __DATADIR__, authors, content, errors, files from patacrep.index import process_sxd from patacrep.templates import TexRenderer from patacrep.songs import DataSubpath @@ -99,8 +99,25 @@ class Songbook(object): copy.deepcopy(config['authwords']) ) - # Configuration set + # Loading custom plugins + config['_content_plugins'] = files.load_plugins( + datadirs=config.get('datadir', []), + subdir=['content'], + variable='CONTENT_PLUGINS', + error=( + "File {filename}: Keyword '{keyword}' is already used. Ignored." + ), + ) + config['_file_plugins'] = files.load_plugins( + datadirs=config.get('datadir', []), + subdir=['songs'], + variable='FILE_PLUGINS', + error=( + "File {filename}: Keyword '{keyword}' is already used. Ignored." + ), + ) + # Configuration set config['render_content'] = content.render_content config['content'] = content.process_content( config.get('content', []), diff --git a/patacrep/content/__init__.py b/patacrep/content/__init__.py index a53d4d7e..5214a55b 100755 --- a/patacrep/content/__init__.py +++ b/patacrep/content/__init__.py @@ -69,7 +69,6 @@ More documentation in the docstring of Content. """ import glob -import importlib import jinja2 import logging import os @@ -134,53 +133,6 @@ class ContentError(SongbookError): def __str__(self): return "Content: {}: {}".format(self.keyword, self.message) -def load_plugins(config): - """Load all content plugins, and return a dictionary of those plugins. - - Return value: a dictionary where: - - keys are the keywords ; - - values are functions triggered when this keyword is met. - """ - plugins = {} - directory_list = ( - [ - os.path.join(datadir, "python", "content") - for datadir in config.get('datadir', []) - ] - + [os.path.dirname(__file__)] - ) - for directory in directory_list: - if not os.path.exists(directory): - LOGGER.debug( - "Ignoring non-existent directory '%s'.", - directory - ) - continue - sys.path.append(directory) - for name in glob.glob(os.path.join(directory, '*.py')): - if name.endswith(".py") and os.path.basename(name) != "__init__.py": - if directory == os.path.dirname(__file__): - plugin = importlib.import_module( - 'patacrep.content.{}'.format( - os.path.basename(name[:-len('.py')]) - ) - ) - else: - plugin = importlib.import_module( - os.path.basename(name[:-len('.py')]) - ) - for (key, value) in plugin.CONTENT_PLUGINS.items(): - if key in plugins: - LOGGER.warning( - "File %s: Keyword '%s' is already used. Ignored.", - files.relpath(name), - key, - ) - continue - plugins[key] = value - del sys.path[-1] - return plugins - @jinja2.contextfunction def render_content(context, content): """Render the content of the songbook as a LaTeX code. @@ -224,7 +176,8 @@ def process_content(content, config=None): included in the .tex file. """ contentlist = [] - plugins = load_plugins(config) + plugins = config.get('_content_plugins', {}) + keyword_re = re.compile(r'^ *(?P\w*) *(\((?P.*)\))? *$') if not content: content = [["song"]] diff --git a/patacrep/content/song.py b/patacrep/content/song.py index 02acf463..50cb7349 100755 --- a/patacrep/content/song.py +++ b/patacrep/content/song.py @@ -4,42 +4,15 @@ """Plugin to include songs to the songbook.""" import glob -import jinja2 import logging import os -from patacrep.content import Content, process_content, ContentError +from patacrep.content import process_content, ContentError from patacrep import files, errors from patacrep.songs import Song LOGGER = logging.getLogger(__name__) -class SongRenderer(Content, Song): - """Render a song in the .tex file.""" - - def begin_new_block(self, previous, __context): - """Return a boolean stating if a new block is to be created.""" - return not isinstance(previous, SongRenderer) - - def begin_block(self, context): - """Return the string to begin a block.""" - indexes = context.resolve("indexes") - if isinstance(indexes, jinja2.runtime.Undefined): - indexes = "" - return r'\begin{songs}{%s}' % indexes - - def end_block(self, __context): - """Return the string to end a block.""" - return r'\end{songs}' - - def render(self, context): - """Return the string that will render the song.""" - return r'\input{{{}}}'.format(files.path2posix( - files.relpath( - self.fullpath, - os.path.dirname(context['filename']) - ))) - #pylint: disable=unused-argument def parse(keyword, argument, contentlist, config): """Parse data associated with keyword 'song'. @@ -51,22 +24,17 @@ def parse(keyword, argument, contentlist, config): expressions (interpreted using the glob module), referring to songs. - config: the current songbook configuration dictionary. - Return a list of SongRenderer() instances. + Return a list of Song() instances. """ if '_languages' not in config: config['_languages'] = set() songlist = [] + plugins = config.get('_file_plugins', {}) for songdir in config['_songdir']: if contentlist: break - contentlist = [ - filename - for filename - in ( - files.recursive_find(songdir.fullpath, "*.sg") - + files.recursive_find(songdir.fullpath, "*.is") - ) - ] + contentlist = files.recursive_find(songdir.fullpath, plugins.keys()) + for elem in contentlist: before = len(songlist) for songdir in config['_songdir']: @@ -74,21 +42,16 @@ def parse(keyword, argument, contentlist, config): continue with files.chdir(songdir.datadir): for filename in glob.iglob(os.path.join(songdir.subpath, elem)): - if not ( - filename.endswith('.sg') or - filename.endswith('.is') - ): + LOGGER.debug('Parsing file "{}"…'.format(filename)) + try: + renderer = plugins[filename.split('.')[-1]] + except KeyError: LOGGER.warning(( - 'File "{}" is not a ".sg" or ".is" file. Ignored.' + 'I do not know how to parse file "{}". Ignored.' ).format(os.path.join(songdir.datadir, filename)) ) continue - LOGGER.debug('Parsing file "{}"…'.format(filename)) - song = SongRenderer( - songdir.datadir, - filename, - config, - ) + song = renderer(songdir.datadir, filename, config) songlist.append(song) config["_languages"].update(song.languages) if len(songlist) > before: @@ -129,7 +92,7 @@ def process_songs(content, config=None): item for item in contentlist - if not isinstance(item, SongRenderer) + if not isinstance(item, Song) ] if not_songs: raise OnlySongsError(not_songs) diff --git a/patacrep/files.py b/patacrep/files.py index 6e02d481..51737c23 100644 --- a/patacrep/files.py +++ b/patacrep/files.py @@ -2,23 +2,31 @@ """File system utilities.""" from contextlib import contextmanager -import fnmatch +import glob +import importlib +import logging import os import posixpath +import re +import sys -def recursive_find(root_directory, pattern): +LOGGER = logging.getLogger(__name__) + +def recursive_find(root_directory, extensions): """Recursively find files matching a pattern, from a root_directory. - Return a list of files matching the pattern. + Return a list of files matching the pattern. TODO """ if not os.path.isdir(root_directory): return [] matches = [] + pattern = re.compile(r'.*\.({})$'.format('|'.join(extensions))) with chdir(root_directory): for root, __ignored, filenames in os.walk(os.curdir): - for filename in fnmatch.filter(filenames, pattern): - matches.append(os.path.join(root, filename)) + for filename in filenames: + if pattern.match(filename): + matches.append(os.path.join(root, filename)) return matches def relpath(path, start=None): @@ -59,3 +67,63 @@ def chdir(path): os.chdir(olddir) else: yield + +def load_plugins(datadirs, subdir, variable, error): + """Load all content plugins, and return a dictionary of those plugins. + + A plugin is a .py file, submodule of `subdir`, located in one of the + directories of `datadirs`. It contains a dictionary `variable`. The return + value is the union of the dictionaries of the loaded plugins. + + Arguments: + - datadirs: list of directories (as strings) in which files has to be + searched. + - subdir: modules (as a list of strings) files has to be submodules of + (e.g. if `subdir` is `['first', 'second']`, search files are of the form + `first/second/*.py`. + - variable: Name of the variable holding the dictionary. + - error: Error message raised if a key appears several times. + """ + plugins = {} + directory_list = ( + [ + os.path.join(datadir, "python", *subdir) #pylint: disable=star-args + for datadir in datadirs + ] + + [os.path.dirname(__file__)] + ) + for directory in directory_list: + if not os.path.exists(directory): + LOGGER.debug( + "Ignoring non-existent directory '%s'.", + directory + ) + continue + sys.path.append(directory) + for name in glob.glob(os.path.join(directory, *(subdir + ['*.py']))): + if name.endswith(".py") and os.path.basename(name) != "__init__.py": + if directory == os.path.dirname(__file__): + plugin = importlib.import_module( + 'patacrep.{}.{}'.format( + ".".join(subdir), + os.path.basename(name[:-len('.py')]) + ) + ) + else: + plugin = importlib.import_module( + os.path.basename(name[:-len('.py')]) + ) + for (key, value) in getattr(plugin, variable, {}).items(): + if key in plugins: + LOGGER.warning( + error.format( + filename=relpath(name), + key=key, + ) + ) + continue + plugins[key] = value + del sys.path[-1] + return plugins + + diff --git a/patacrep/songs.py b/patacrep/songs/__init__.py similarity index 62% rename from patacrep/songs.py rename to patacrep/songs/__init__.py index 43e70fe7..a4a6b089 100644 --- a/patacrep/songs.py +++ b/patacrep/songs/__init__.py @@ -4,13 +4,14 @@ import errno import hashlib +import jinja2 import logging import os import pickle import re from patacrep.authors import processauthors -from patacrep.latex import parsesong +from patacrep.content import Content LOGGER = logging.getLogger(__name__) @@ -62,18 +63,32 @@ class DataSubpath(object): self.subpath = os.path.join(self.subpath, path) return self -# pylint: disable=too-few-public-methods, too-many-instance-attributes -class Song(object): - """Song management""" +# pylint: disable=too-many-instance-attributes +class Song(Content): + """Song (or song metadata) + + This class represents a song, bound to a file. + + - It can parse the file given in arguments. + - It can render the song as some LaTeX code. + - Its content is cached, so that if the file has not been changed, the + file is not parsed again. + + This class is inherited by classes implementing song management for + several file formats. Those subclasses must implement: + - `parse()` to parse the file; + - `render()` to render the song as LaTeX code. + """ # Version format of cached song. Increment this number if we update # information stored in cache. - CACHE_VERSION = 0 + CACHE_VERSION = 1 # List of attributes to cache cached_attributes = [ "titles", "unprefixed_titles", + "cached", "data", "datadir", "fullpath", @@ -109,10 +124,14 @@ class Song(object): self.fullpath )) - # Data extraction from the latex song - self.data = parsesong(self.fullpath) - self.titles = self.data['@titles'] - self.languages = self.data['@languages'] + # Default values + self.data = {} + self.titles = [] + self.languages = [] + self.authors = [] + + # Parsing and data processing + self.parse() self.datadir = datadir self.unprefixed_titles = [ unprefixed_title( @@ -123,13 +142,15 @@ class Song(object): in self.titles ] self.subpath = subpath - if "by" in self.data: - self.authors = processauthors( - self.data["by"], - **config["_compiled_authwords"] - ) - else: - self.authors = [] + self.authors = processauthors( + self.authors, + **config["_compiled_authwords"] + ) + + # Cache management + + #: Special attribute to allow plugins to store cached data + self.cached = None self._version = self.CACHE_VERSION self._write_cache() @@ -149,6 +170,50 @@ class Song(object): def __repr__(self): return repr((self.titles, self.data, self.fullpath)) + def begin_new_block(self, previous, __context): + """Return a boolean stating if a new block is to be created.""" + return not isinstance(previous, Song) + + def begin_block(self, context): + """Return the string to begin a block.""" + indexes = context.resolve("indexes") + if isinstance(indexes, jinja2.runtime.Undefined): + indexes = "" + return r'\begin{songs}{%s}' % indexes + + def end_block(self, __context): + """Return the string to end a block.""" + return r'\end{songs}' + + def render(self, __context): + """Returns the TeX code rendering the song. + + This function is to be defined by subclasses. + """ + return '' + + def parse(self): + """Parse file `self.fullpath`. + + This function is to be defined by subclasses. + + It set the following attributes: + + - titles: the list of (raw) titles. This list will be processed to + remove prefixes. + - languages: the list of languages used in the song, as languages + recognized by the LaTeX babel package. + - authors: the list of (raw) authors. This list will be processed to + 'clean' it (see function :func:`patacrep.authors.processauthors`). + - data: song metadata. Used (among others) to sort the songs. + - cached: additional data that will be cached. Thus, data stored in + this attribute must be picklable. + """ + self.data = {} + self.titles = [] + self.languages = [] + self.authors = [] + def unprefixed_title(title, prefixes): """Remove the first prefix of the list in the beginning of title (if any). """ @@ -158,4 +223,3 @@ def unprefixed_title(title, prefixes): return match.group(2) return title - diff --git a/patacrep/songs/tex.py b/patacrep/songs/tex.py new file mode 100644 index 00000000..b5710346 --- /dev/null +++ b/patacrep/songs/tex.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +"""Very simple LaTeX parsing.""" + +import os + +from patacrep import files +from patacrep.latex import parsesong +from patacrep.songs import Song + +class TexRenderer(Song): + """Renderer for song and intersong files.""" + + def parse(self): + """Parse song and set metadata.""" + self.data = parsesong(self.fullpath) + self.titles = self.data['@titles'] + self.languages = self.data['@languages'] + self.authors = self.data['by'] + + def render(self, context): + """Return the string that will render the song.""" + return r'\input{{{}}}'.format(files.path2posix( + files.relpath( + self.fullpath, + os.path.dirname(context['filename']) + ))) + +FILE_PLUGINS = { + 'sg': TexRenderer, + 'is': TexRenderer, + }