diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..fe4c1ab7 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE NEWS readme.md Requirements.txt +recursive-include patacrep/data * diff --git a/NEWS b/NEWS deleted file mode 100644 index 62413cc8..00000000 --- a/NEWS +++ /dev/null @@ -1,68 +0,0 @@ -songbook 3.7.2 - - (Louis) Undocumented bug corrections and improvements. - -songbook 3.4.7 to 3.7.1 - - Mainly new songs in the data (which was included in songbook at this - time), and a few undocumented bug corrections and improvements. - -songbook (v0.8) - - Undocumented. - -songbook (v0.7) - - (lohrun) New songbook format (not compatible with older version). - Changes have been made to the compilation toolchain that prevent - compilation of old format songbook. - (lohrun) Use LaTeX Songs package v2.10. - - -- Alexandre Dupas Sat, 17 Jul 2010 15:24:14 +0200 - -songbook (v0.6) - - (crep, lohrun) Corrections of mistakes and typos. - (lohrun) Use plain songs package v2.9 - (lohrun) Replace makeindex script with a new python version - (lohrun) Add script to produce the list of chords used in songs - (crep, lohrun) Correct chords and gtabs used in songs - (lohrun) Modification of the default geometry - (lohrun) Remove capos from the lyricbook - - -- Alexandre Dupas Fri, 11 Dec 2009 15:35:03 +0100 - -songbook (0.5) - - (crep, lohrun) Corrections of mistakes and typos. - (lohrun) Add a proper volume mechanism - (lohrun) Add volume-1 source containing about 165 songs - (crep) Add naheulbeuk special edition - (lohrun) Upgraded songs.sty with bits from songs package v2.9 - (lohrun) Add tabs option - (crep,lohrun) Add lilypond option - - -- Alexandre Dupas Tue, 18 Aug 2009 23:38:12 +0200 - -songbook (0.4) - - (crep, lohrun) Corrections of mistakes and typos. - (crep, lohrun) Add cover picture to each song - (lohrun) Update to the Songs Package v2.8 - (lohrun) Update makefile to be POSIX compilant - - -- Alexandre Dupas Sun, 31 May 2009 01:39:16 +0200 - -songbook (0.3) - - (crep) Corrections of a lot of mistakes. - (crep) Include image support. - (lohrun) Add make-html utility. - - -- Alexandre Dupas Sun, 15 Feb 2009 18:34:59 +0100 - -songbook (0.2) - - Initial version. - - -- Alexandre Dupas Sat, 11 Oct 2008 20:00:00 +0100 diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 00000000..6a7b76a7 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,113 @@ +# patacrep 4.0.0 + +* Project gestion + * Name change [#39](http://github.com/patacrep/patacrep/issues/39) + * Renew of the developement team + * Separation engine/data + * The engine is the [current poject](http://github.com/patacrep/patacrep) + * Data have their [own project](http://github.com/patacrep/patadata) + * And so does [various tools](http://github.com/patacrep/pataextra) + +* Internal changes + * Complete migration to Python + * No more Makefiles + * Creation of a `songbook` command + * patacrep uses Python3 [#65](http://github.com/patacrep/patacrep/issues/65) + * Massive code refactoring and simplification + * [PEP8](http://legacy.python.org/dev/peps/pep-0008/) conformity + * Better LaTeX Packages + * Better langages handling + * Better errors handling + * Better code documentation (in comments) + * Caching song AST, which gave an improvement of 45s for the compilation of all patadata [#51](http://github.com/patacrep/patacrep/issues/51) + * Lot of small improvements + +* Installation + * All from PyPi ! You can now use pip to install/update/remove patacrep + +* Fonctionnalities + * Change the template engine [#9](http://github.com/patacrep/patacrep/issues/9) + * Ability to add user variables [#18](http://github.com/patacrep/patacrep/issues/18) + * Change the song inclusion syntaxe [#47](http://github.com/patacrep/patacrep/issues/47) + * It is now possible to include other things than songs + * You can write a plugins to include your own type of content + * Personalisaitons of the songbook is easier with patadata templates (font, paper, colors, column, ...) [#41](http://github.com/patacrep/patacrep/issues/41) + * You can change the number of columns [#41](http://github.com/patacrep/patacrep/issues/41) + * Lilypond + * On the fly lylipond files compilation + * Adapt partition size to the paper size [#19](http://github.com/patacrep/patacrep/issues/19) + * You can choos how to sort the songs [#36](http://github.com/patacrep/patacrep/issues/36) + * Easier song repertories management [#43](http://github.com/patacrep/patacrep/issues/43) and [#45](http://github.com/patacrep/patacrep/issues/45) + * You can have more than one data folder + * Better index customisation + * Better gestion of files encoding [#62](http://github.com/patacrep/patacrep/issues/62). + + +# songbook 3.7.2 + + (Louis) Undocumented bug corrections and improvements. + +# songbook 3.4.7 to 3.7.1 + + Mainly new songs in the data (which was included in songbook at this + time), and a few undocumented bug corrections and improvements. + +# songbook (v0.8) + + Undocumented. + +# songbook (v0.7) + + (lohrun) New songbook format (not compatible with older version). + Changes have been made to the compilation toolchain that prevent + compilation of old format songbook. + (lohrun) Use LaTeX Songs package v2.10. + + -- Alexandre Dupas Sat, 17 Jul 2010 15:24:14 +0200 + +# songbook (v0.6) + + (crep, lohrun) Corrections of mistakes and typos. + (lohrun) Use plain songs package v2.9 + (lohrun) Replace makeindex script with a new python version + (lohrun) Add script to produce the list of chords used in songs + (crep, lohrun) Correct chords and gtabs used in songs + (lohrun) Modification of the default geometry + (lohrun) Remove capos from the lyricbook + + -- Alexandre Dupas Fri, 11 Dec 2009 15:35:03 +0100 + +# songbook (0.5) + + (crep, lohrun) Corrections of mistakes and typos. + (lohrun) Add a proper volume mechanism + (lohrun) Add volume-1 source containing about 165 songs + (crep) Add naheulbeuk special edition + (lohrun) Upgraded songs.sty with bits from songs package v2.9 + (lohrun) Add tabs option + (crep,lohrun) Add lilypond option + + -- Alexandre Dupas Tue, 18 Aug 2009 23:38:12 +0200 + +# songbook (0.4) + + (crep, lohrun) Corrections of mistakes and typos. + (crep, lohrun) Add cover picture to each song + (lohrun) Update to the Songs Package v2.8 + (lohrun) Update makefile to be POSIX compilant + + -- Alexandre Dupas Sun, 31 May 2009 01:39:16 +0200 + +# songbook (0.3) + + (crep) Corrections of a lot of mistakes. + (crep) Include image support. + (lohrun) Add make-html utility. + + -- Alexandre Dupas Sun, 15 Feb 2009 18:34:59 +0100 + +# songbook (0.2) + + Initial version. + + -- Alexandre Dupas Sat, 11 Oct 2008 20:00:00 +0100 diff --git a/patacrep/__init__.py b/patacrep/__init__.py index 104a538a..e63b5123 100644 --- a/patacrep/__init__.py +++ b/patacrep/__init__.py @@ -3,9 +3,14 @@ from pkg_resources import resource_filename import os -# Version +# Check Python version +import sys +if sys.version_info < (3, 3): + print("ERROR: Your Python version is too old. Please use a Python version > 3.3.") + sys.exit(1) -__TUPLE_VERSION__ = (4, 0, 0, "alpha") +# Patacrep version. +__TUPLE_VERSION__ = (4, 0, 0) __version__ = '.'.join([str(number) for number in __TUPLE_VERSION__]) # Directory containing shared data (default templates, custom LaTeX packages, diff --git a/patacrep/authors.py b/patacrep/authors.py index 8a25c92b..3bdb6b9a 100644 --- a/patacrep/authors.py +++ b/patacrep/authors.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Authors string management.""" import logging diff --git a/patacrep/build.py b/patacrep/build.py index f804f154..ac566e07 100644 --- a/patacrep/build.py +++ b/patacrep/build.py @@ -1,15 +1,14 @@ -# -*- coding: utf-8 -*- - """Build a songbook, according to parameters found in a .sb file.""" import codecs import copy import glob import logging +import threading import os.path from subprocess import Popen, PIPE, call -from patacrep import __DATADIR__, authors, content, errors +from patacrep import __DATADIR__, authors, content, errors, files from patacrep.index import process_sxd from patacrep.templates import TexRenderer from patacrep.songs import DataSubpath @@ -33,6 +32,7 @@ DEFAULT_CONFIG = { 'lang': 'english', 'content': [], 'titleprefixwords': [], + 'encoding': None, } @@ -91,6 +91,7 @@ class Songbook(object): config['template'], config['datadir'], config['lang'], + config['encoding'], ) config.update(renderer.get_variables()) config.update(self.config) @@ -99,8 +100,19 @@ class Songbook(object): copy.deepcopy(config['authwords']) ) - # Configuration set + # Loading custom plugins + config['_content_plugins'] = files.load_plugins( + datadirs=config.get('datadir', []), + root_modules=['content'], + keyword='CONTENT_PLUGINS', + ) + config['_song_plugins'] = files.load_plugins( + datadirs=config.get('datadir', []), + root_modules=['songs'], + keyword='SONG_PARSERS', + ) + # Configuration set config['render_content'] = content.render_content config['content'] = content.process_content( config.get('content', []), @@ -110,6 +122,13 @@ class Songbook(object): renderer.render_tex(output, config) +def _log_pipe(pipe): + """Log content from `pipe`.""" + while 1: + line = pipe.readline() + if not bool(line): + break + LOGGER.debug(line.strip()) class SongbookBuilder(object): """Provide methods to compile a songbook.""" @@ -200,22 +219,33 @@ class SongbookBuilder(object): stdin=PIPE, stdout=PIPE, stderr=PIPE, - env=os.environ, universal_newlines=True, - ) + env=os.environ) except Exception as error: LOGGER.debug(error) raise errors.LatexCompilationError(self.basename) if not self.interactive: process.stdin.close() - log = '' - line = process.stdout.readline() - while line: - log += str(line) - line = process.stdout.readline() - LOGGER.debug(log) + standard_output = threading.Thread( + target=_log_pipe, + kwargs={ + 'pipe' : process.stdout, + } + ) + standard_error = threading.Thread( + target=_log_pipe, + kwargs={ + 'pipe' : process.stderr, + } + ) + standard_output.daemon = True + standard_error.daemon = True + standard_error.start() + standard_output.start() + standard_error.join() + standard_output.join() process.wait() if process.returncode: diff --git a/patacrep/content/__init__.py b/patacrep/content/__init__.py index 03b2fb0d..d92dc99b 100755 --- a/patacrep/content/__init__.py +++ b/patacrep/content/__init__.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - """Content plugin management. Content that can be included in a songbook is controlled by plugins. From the @@ -69,7 +66,6 @@ More documentation in the docstring of Content. """ import glob -import importlib import jinja2 import logging import os @@ -177,7 +173,7 @@ def process_content(content, config=None): included in the .tex file. """ contentlist = [] - plugins = files.load_plugins(config, ["content"], "CONTENT_PLUGINS") + plugins = config.get('_content_plugins', {}) keyword_re = re.compile(r'^ *(?P\w*) *(\((?P.*)\))? *$') if not content: content = [["song"]] diff --git a/patacrep/content/cwd.py b/patacrep/content/cwd.py index 5e55d68d..67468761 100755 --- a/patacrep/content/cwd.py +++ b/patacrep/content/cwd.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - """Change base directory before importing songs.""" from patacrep.content import process_content diff --git a/patacrep/content/include.py b/patacrep/content/include.py index dcc969a9..43c79b95 100644 --- a/patacrep/content/include.py +++ b/patacrep/content/include.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Include an external list of songs This plugin provides keyword 'include', used to include an external list of @@ -48,15 +46,15 @@ def parse(keyword, config, argument, contentlist): filepath = load_from_datadirs(path, config) content_file = None try: - content_file = encoding.open_read(filepath, 'r') - new_content = json.load(content_file) + with encoding.open_read( + filepath, + encoding=config['encoding'] + ) as content_file: + new_content = json.load(content_file) except Exception as error: # pylint: disable=broad-except LOGGER.error(error) LOGGER.error("Error while loading file '{}'.".format(filepath)) sys.exit(1) - finally: - if content_file: - content_file.close() config["datadir"].append(os.path.abspath(os.path.dirname(filepath))) new_contentlist += process_content(new_content, config) diff --git a/patacrep/content/section.py b/patacrep/content/section.py index 96215f68..9bca685d 100755 --- a/patacrep/content/section.py +++ b/patacrep/content/section.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - """Allow LaTeX sections (starred or not) as content of a songbook.""" from patacrep.content import Content, ContentError diff --git a/patacrep/content/song.py b/patacrep/content/song.py index 1b75ba6b..8f5d0f45 100755 --- a/patacrep/content/song.py +++ b/patacrep/content/song.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - """Plugin to include songs to the songbook.""" import glob @@ -8,7 +5,7 @@ import jinja2 import logging import os -from patacrep.content import Content, process_content, ContentError +from patacrep.content import process_content, ContentError, Content from patacrep import files, errors LOGGER = logging.getLogger(__name__) @@ -50,20 +47,17 @@ def parse(keyword, argument, contentlist, config): expressions (interpreted using the glob module), referring to songs. - config: the current songbook configuration dictionary. - Return a list of SongRenderer() instances. + Return a list of Song() instances. """ - plugins = files.load_plugins(config, ["songs"], "SONG_PARSERS") + plugins = config['_song_plugins'] if '_languages' not in config: config['_languages'] = set() songlist = [] for songdir in config['_songdir']: if contentlist: break - contentlist = [ - filename - for filename - in files.recursive_find(songdir.fullpath, plugins.keys()) - ] + contentlist = files.recursive_find(songdir.fullpath, plugins.keys()) + for elem in contentlist: before = len(songlist) for songdir in config['_songdir']: @@ -71,22 +65,22 @@ def parse(keyword, argument, contentlist, config): continue with files.chdir(songdir.datadir): for filename in glob.iglob(os.path.join(songdir.subpath, elem)): + LOGGER.debug('Parsing file "{}"…'.format(filename)) extension = filename.split(".")[-1] - if extension not in plugins: + try: + renderer = SongRenderer(plugins[extension]( + songdir.datadir, + filename, + config, + )) + except KeyError: LOGGER.warning(( - 'File "{}" does not end with one of {}. Ignored.' + 'I do not know how to parse "{}". Ignored.' ).format( os.path.join(songdir.datadir, filename), - ", ".join(["'.{}'".format(key) for key in plugins.keys()]), ) ) continue - LOGGER.debug('Parsing file "{}"…'.format(filename)) - renderer = SongRenderer(plugins[extension]( - songdir.datadir, - filename, - config, - )) songlist.append(renderer) config["_languages"].update(renderer.song.languages) if len(songlist) > before: diff --git a/patacrep/content/songsection.py b/patacrep/content/songsection.py index 07153591..a51c503c 100755 --- a/patacrep/content/songsection.py +++ b/patacrep/content/songsection.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - """Allow 'songchapter' and 'songsection' as content of a songbook.""" from patacrep.content import Content, ContentError diff --git a/patacrep/content/sorted.py b/patacrep/content/sorted.py index 00844335..b3bbe40a 100755 --- a/patacrep/content/sorted.py +++ b/patacrep/content/sorted.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - """Sorted list of songs. This plugin provides keyword 'sorted', used to include a sorted list of songs diff --git a/patacrep/content/tex.py b/patacrep/content/tex.py index 38593f38..9a8227d3 100755 --- a/patacrep/content/tex.py +++ b/patacrep/content/tex.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - """Include LaTeX raw code in the songbook.""" import logging diff --git a/patacrep/encoding.py b/patacrep/encoding.py index ca917295..bd36b560 100644 --- a/patacrep/encoding.py +++ b/patacrep/encoding.py @@ -1,21 +1,29 @@ -# -*- coding: utf-8 -*- - """Dealing with encoding problems.""" import codecs import chardet import logging +import contextlib LOGGER = logging.getLogger(__name__) -def open_read(filename, mode='r'): + +@contextlib.contextmanager +def open_read(filename, mode='r', encoding=None): """Open a file for reading, guessing the right encoding. Return a fileobject, reading unicode strings. + If `encoding` is set, use it as the encoding (do not guess). """ - return codecs.open( + if encoding is None: + fileencoding = chardet.detect(open(filename, 'rb').read())['encoding'] + else: + fileencoding = encoding + + with codecs.open( filename, mode=mode, - encoding=chardet.detect(open(filename, 'rb').read())['encoding'], + encoding=fileencoding, errors='replace', - ) + ) as fileobject: + yield fileobject diff --git a/patacrep/errors.py b/patacrep/errors.py index a065c322..f6e4121a 100644 --- a/patacrep/errors.py +++ b/patacrep/errors.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Songbook exceptions and errors.""" class SongbookError(Exception): diff --git a/patacrep/files.py b/patacrep/files.py index a1a12d18..86208746 100644 --- a/patacrep/files.py +++ b/patacrep/files.py @@ -1,31 +1,32 @@ -# -*- coding: utf-8 -*- """File system utilities.""" from contextlib import contextmanager -import fnmatch import importlib import logging import os import posixpath +import re LOGGER = logging.getLogger(__name__) -def recursive_find(root_directory, patterns): - """Recursively find files matching one of the patterns, from a root_directory. +def recursive_find(root_directory, extensions): + """Recursively find files with some extension, from a root_directory. - Return a list of files matching one of the patterns. + Return a list of files matching those conditions. + + Arguments: + - `extensions`: list of accepted extensions. + - `root_directory`: root directory of the search. """ if not os.path.isdir(root_directory): return [] matches = [] + pattern = re.compile(r'.*\.({})$'.format('|'.join(extensions))) with chdir(root_directory): for root, __ignored, filenames in os.walk(os.curdir): - for pattern in patterns: - for filename in fnmatch.filter( - filenames, - "*.{}".format(pattern), - ): + for filename in filenames: + if pattern.match(filename): matches.append(os.path.join(root, filename)) return matches @@ -68,11 +69,15 @@ def chdir(path): else: yield -def load_plugins(config, root_modules, keyword): +def load_plugins(datadirs, root_modules, keyword): """Load all plugins, and return a dictionary of those plugins. + A plugin is a .py file, submodule of `subdir`, located in one of the + directories of `datadirs`. It contains a dictionary `keyword`. The return + value is the union of the dictionaries of the loaded plugins. + Arguments: - - config: the configuration dictionary of the songbook + - datadirs: List of directories in which plugins are to be searched. - root_modules: the submodule in which plugins are to be searched, as a list of modules (e.g. ["some", "deep", "module"] for "some.deep.module"). @@ -87,7 +92,7 @@ def load_plugins(config, root_modules, keyword): directory_list = ( [ os.path.join(datadir, "python", *root_modules) - for datadir in config.get('datadir', []) + for datadir in datadirs ] + [os.path.join( os.path.dirname(__file__), @@ -124,4 +129,3 @@ def load_plugins(config, root_modules, keyword): continue plugins[key] = value return plugins - diff --git a/patacrep/index.py b/patacrep/index.py index c715918a..6a4300a2 100644 --- a/patacrep/index.py +++ b/patacrep/index.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Manage indexes. Generate indexes files for the songbook compilation. This is a replacement for @@ -29,13 +27,9 @@ def process_sxd(filename): """ data = [] index_file = None - try: - index_file = encoding.open_read(filename, 'r') + with encoding.open_read(filename) as index_file: for line in index_file: data.append(line.strip()) - finally: - if index_file: - index_file.close() i = 1 idx = Index(data[0]) diff --git a/patacrep/latex/__init__.py b/patacrep/latex/__init__.py index 90c4e798..2c6db73a 100644 --- a/patacrep/latex/__init__.py +++ b/patacrep/latex/__init__.py @@ -1,3 +1,8 @@ -"""Dumb and very very incomplete LaTeX parser.""" +"""Dumb and very very incomplete LaTeX parser. + +This module uses an LALR parser to try to parse LaTeX code. LaTeX language +*cannot* be parsed by an LALR parser, so this is a very simple attemps, which +will work on simple cases, but not on complex ones. +""" from patacrep.latex.syntax import tex2plain, parse_song diff --git a/patacrep/latex/detex.py b/patacrep/latex/detex.py index d99c3467..ebfd721b 100644 --- a/patacrep/latex/detex.py +++ b/patacrep/latex/detex.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Render `very simple` TeX commands in a simple TeX code.""" import logging diff --git a/patacrep/latex/syntax.py b/patacrep/latex/syntax.py index 094ef124..aff3a885 100644 --- a/patacrep/latex/syntax.py +++ b/patacrep/latex/syntax.py @@ -41,11 +41,12 @@ class Parser: def p_error(self, token): """Manage parsing errors.""" - LOGGER.error("Erreur fichier {}, ligne {}, position {}.".format( - str(self.filename), - token.lineno, - self.__find_column(token), - ) + LOGGER.error( + "Error in file {}, line {} at position {}.".format( + str(self.filename), + token.lineno, + self.__find_column(token), + ) ) @staticmethod @@ -223,19 +224,26 @@ class Parser: else: symbols[0] = symbols[2].prepend(symbols[1]) +def silent_yacc(*args, **kwargs): + """Call yacc, suppressing (as far as possible) output and generated files. + """ + return yacc.yacc( + write_tables=0, + debug=0, + *args, + **kwargs + ) def tex2plain(string): """Parse string and return its plain text version.""" return detex( - yacc.yacc( - write_tables=0, - debug=0, - module=Parser(), - ).parse( - string, - lexer=SimpleLexer().lexer, - ) + silent_yacc( + module=Parser(), + ).parse( + string, + lexer=SimpleLexer().lexer, ) + ) def parse_song(content, filename=None): """Parse some LaTeX code, expected to be a song. @@ -246,12 +254,8 @@ def parse_song(content, filename=None): display error messages. """ return detex( - yacc.yacc( - module=Parser(filename), - write_tables=0, - debug=0, - ).parse( - content, - lexer=SongLexer().lexer, - ).metadata - ) + silent_yacc(module=Parser(filename)).parse( + content, + lexer=SongLexer().lexer, + ).metadata + ) diff --git a/patacrep/songbook.py b/patacrep/songbook.py new file mode 100755 index 00000000..45a32560 --- /dev/null +++ b/patacrep/songbook.py @@ -0,0 +1,154 @@ +"""Command line tool to compile songbooks using the songbook library.""" + +import argparse +import json +import locale +import logging +import os.path +import textwrap +import sys + +from patacrep.build import SongbookBuilder, DEFAULT_STEPS +from patacrep import __version__ +from patacrep import errors +import patacrep.encoding + +# Logging configuration +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger() + +# pylint: disable=too-few-public-methods +class ParseStepsAction(argparse.Action): + """Argparse action to split a string into a list.""" + def __call__(self, __parser, namespace, values, __option_string=None): + if not getattr(namespace, self.dest): + setattr(namespace, self.dest, []) + setattr( + namespace, + self.dest, + ( + getattr(namespace, self.dest) + + [value.strip() for value in values[0].split(',')] + ), + ) + +class VerboseAction(argparse.Action): + """Set verbosity level with option --verbose.""" + def __call__(self, *_args, **_kwargs): + LOGGER.setLevel(logging.DEBUG) + +def argument_parser(args): + """Parse arguments""" + parser = argparse.ArgumentParser(description="A song book compiler") + + parser.add_argument('--version', help='Show version', action='version', + version='%(prog)s ' + __version__) + + parser.add_argument('book', nargs=1, help=textwrap.dedent("""\ + Book to compile. + """)) + + parser.add_argument('--datadir', '-d', nargs='+', type=str, action='append', + help=textwrap.dedent("""\ + Data location. Expected (not necessarily required) + subdirectories are 'songs', 'img', 'latex', 'templates'. + """)) + + parser.add_argument('--verbose', '-v', nargs=0, action=VerboseAction, + help=textwrap.dedent("""\ + Show details about the compilation process. + """)) + + parser.add_argument('--steps', '-s', nargs=1, type=str, + action=ParseStepsAction, + help=textwrap.dedent("""\ + Steps to run. Default is "{steps}". + Available steps are: + "tex" produce .tex file from templates; + "pdf" compile .tex file; + "sbx" compile index files; + "clean" remove temporary files; + any string beginning with '%%' (in this case, it will be run + in a shell). Several steps (excepted the custom shell + command) can be combinend in one --steps argument, as a + comma separated string. + """.format(steps=','.join(DEFAULT_STEPS))), + default=None, + ) + + options = parser.parse_args(args) + + return options + + +def main(): + """Main function:""" + + # set script locale to match user's + try: + locale.setlocale(locale.LC_ALL, '') + except locale.Error as error: + # Locale is not installed on user's system, or wrongly configured. + LOGGER.error("Locale error: {}\n".format(str(error))) + + options = argument_parser(sys.argv[1:]) + + songbook_path = options.book[0] + + basename = os.path.basename(songbook_path)[:-3] + + try: + with patacrep.encoding.open_read(songbook_path) as songbook_file: + songbook = json.load(songbook_file) + if 'encoding' in songbook: + with patacrep.encoding.open_read( + songbook_path, + encoding=songbook['encoding'] + ) as songbook_file: + songbook = json.load(songbook_file) + except Exception as error: # pylint: disable=broad-except + LOGGER.error(error) + LOGGER.error("Error while loading file '{}'.".format(songbook_path)) + sys.exit(1) + + # Gathering datadirs + datadirs = [] + if options.datadir: + # Command line options + datadirs += [item[0] for item in options.datadir] + if 'datadir' in songbook: + # .sg file + if isinstance(songbook['datadir'], str): + songbook['datadir'] = [songbook['datadir']] + datadirs += [ + os.path.join( + os.path.dirname(os.path.abspath(songbook_path)), + path + ) + for path in songbook['datadir'] + ] + # Default value + datadirs.append(os.path.dirname(os.path.abspath(songbook_path))) + + songbook['datadir'] = datadirs + + try: + sb_builder = SongbookBuilder(songbook, basename) + sb_builder.unsafe = True + + sb_builder.build_steps(options.steps) + except errors.SongbookError as error: + LOGGER.error(error) + if LOGGER.level >= logging.INFO: + LOGGER.error( + "Running again with option '-v' may give more information." + ) + sys.exit(1) + except KeyboardInterrupt: + LOGGER.warning("Aborted by user.") + sys.exit(1) + + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/patacrep/songs/__init__.py b/patacrep/songs/__init__.py index 0744d8eb..20fb367e 100644 --- a/patacrep/songs/__init__.py +++ b/patacrep/songs/__init__.py @@ -1,9 +1,8 @@ -# -*- coding: utf-8 -*- - """Song management.""" import errno import hashlib +import jinja2 import logging import os import pickle @@ -11,6 +10,7 @@ import re from patacrep.authors import processauthors from patacrep import files, encoding +from patacrep.content import Content LOGGER = logging.getLogger(__name__) @@ -62,21 +62,33 @@ class DataSubpath(object): self.subpath = os.path.join(self.subpath, path) return self -# pylint: disable=too-few-public-methods, too-many-instance-attributes -class Song(object): - """Song management""" +# pylint: disable=too-many-instance-attributes +class Song(Content): + """Song (or song metadata) + + This class represents a song, bound to a file. + + - It can parse the file given in arguments. + - It can render the song as some LaTeX code. + - Its content is cached, so that if the file has not been changed, the + file is not parsed again. + + This class is inherited by classes implementing song management for + several file formats. Those subclasses must implement: + - `parse()` to parse the file; + - `render()` to render the song as LaTeX code. + """ # Version format of cached song. Increment this number if we update # information stored in cache. - CACHE_VERSION = 0 + CACHE_VERSION = 1 # List of attributes to cache cached_attributes = [ "titles", "unprefixed_titles", + "cached", "data", - "datadir", - "fullpath", "subpath", "languages", "authors", @@ -84,14 +96,11 @@ class Song(object): "_version", ] - # Default data - DEFAULT_DATA = { - '@titles': [], - '@languages': [], - } - def __init__(self, datadir, subpath, config): self.fullpath = os.path.join(datadir, subpath) + self.datadir = datadir + self.encoding = config["encoding"] + if datadir: # Only songs in datadirs are cached self._filehash = hashlib.md5( @@ -116,14 +125,13 @@ class Song(object): )) # Data extraction from the latex song - self.data = self.DEFAULT_DATA - self.data['@path'] = self.fullpath - self.data.update(self.parse( - encoding.open_read(self.fullpath).read() - )) - self.titles = self.data['@titles'] - self.languages = self.data['@languages'] + self.titles = [] + self.data = {} + self.parse() + + # Post processing of data self.datadir = datadir + self.subpath = subpath self.unprefixed_titles = [ unprefixed_title( title, @@ -132,14 +140,15 @@ class Song(object): for title in self.titles ] - self.subpath = subpath - if "by" in self.data: - self.authors = processauthors( - self.data["by"], - **config["_compiled_authwords"] - ) - else: - self.authors = [] + self.authors = processauthors( + self.authors, + **config["_compiled_authwords"] + ) + + # Cache management + + #: Special attribute to allow plugins to store cached data + self.cached = None self._version = self.CACHE_VERSION self._write_cache() @@ -165,11 +174,24 @@ class Song(object): Arguments: - output: Name of the output file. """ - return NotImplementedError() - - def parse(self, content): # pylint: disable=no-self-use, unused-argument - """Parse song, and return a dictionary of its data.""" - return NotImplementedError() + raise NotImplementedError() + + def parse(self): # pylint: disable=no-self-use + """Parse song. + + It set the following attributes: + + - titles: the list of (raw) titles. This list will be processed to + remove prefixes. + - languages: the list of languages used in the song, as languages + recognized by the LaTeX babel package. + - authors: the list of (raw) authors. This list will be processed to + 'clean' it (see function :func:`patacrep.authors.processauthors`). + - data: song metadata. Used (among others) to sort the songs. + - cached: additional data that will be cached. Thus, data stored in + this attribute must be picklable. + """ + raise NotImplementedError() def unprefixed_title(title, prefixes): """Remove the first prefix of the list in the beginning of title (if any). diff --git a/patacrep/songs/latex/__init__.py b/patacrep/songs/latex/__init__.py index 2de16338..13270bd4 100644 --- a/patacrep/songs/latex/__init__.py +++ b/patacrep/songs/latex/__init__.py @@ -7,16 +7,23 @@ will work on simple cases, but not on complex ones. import os -from patacrep import files +from patacrep import files, encoding from patacrep.latex import parse_song from patacrep.songs import Song class LatexSong(Song): """LaTeX song parser.""" - def parse(self, content): + def parse(self): """Parse content, and return the dictinory of song data.""" - return parse_song(content, self.fullpath) + with encoding.open_read(self.fullpath, encoding=self.encoding) as song: + self.data = parse_song(song.read(), self.fullpath) + self.titles = self.data['@titles'] + del self.data['@titles'] + self.languages = self.data['@languages'] + del self.data['@languages'] + self.authors = self.data['by'] + del self.data['by'] def tex(self, output): """Return the LaTeX code rendering the song.""" diff --git a/patacrep/templates.py b/patacrep/templates.py index d1e9a616..ed48529f 100644 --- a/patacrep/templates.py +++ b/patacrep/templates.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Template for .tex generation settings and utilities""" from jinja2 import Environment, FileSystemLoader, ChoiceLoader, \ @@ -9,7 +8,8 @@ import os import re import json -from patacrep import encoding, errors, files +from patacrep import errors, files +import patacrep.encoding _LATEX_SUBS = ( (re.compile(r'\\'), r'\\textbackslash'), @@ -67,7 +67,7 @@ def _escape_tex(value): class TexRenderer(object): """Render a template to a LaTeX file.""" - def __init__(self, template, datadirs, lang): + def __init__(self, template, datadirs, lang, encoding=None): '''Start a new jinja2 environment for .tex creation. Arguments: @@ -75,8 +75,10 @@ class TexRenderer(object): - datadirs: list of locations of the data directory (which may contain file /templates/