Merge branch 'master' into crlf_test

Requirements.txt

@@ -1,4 +1,3 @@
 ply
 Jinja2==2.7.3
-chardet==2.2.1
 unidecode>=0.04.16

patacrep/encoding.py

@@ -1,7 +1,6 @@
 """Dealing with encoding problems."""
 
 import codecs
-import chardet
 import logging
 import contextlib
 
@@ -16,15 +15,31 @@ def open_read(filename, mode='r', encoding=None):
     If `encoding` is set, use it as the encoding (do not guess).
     """
     if encoding is None:
-        with open(filename, 'rb') as file:
-            fileencoding = chardet.detect(file.read())['encoding']
-    else:
-        fileencoding = encoding
+        encoding = detect_encoding(filename)
 
     with codecs.open(
         filename,
         mode=mode,
-        encoding=fileencoding,
+        encoding=encoding,
         errors='replace',
         ) as fileobject:
         yield fileobject
+
+def detect_encoding(filename):
+    """Return the most likely encoding of the file
+    """
+    encodings = ['utf-8', 'windows-1250', 'windows-1252']
+    for encoding in encodings:
+        try:
+            filehandler = codecs.open(filename, 'r', encoding=encoding)
+            filehandler.readlines()
+            filehandler.seek(0)
+        except UnicodeDecodeError:
+            pass
+        else:
+            if encoding != 'utf-8':
+                LOGGER.info('Opening `{}` with `{}` encoding'.format(filename, encoding))
+            return encoding
+        finally:
+            filehandler.close()
+    raise UnicodeError('Not suitable encoding found for {}'.format(filename))

patacrep/files.py

@@ -12,20 +12,24 @@ from patacrep import utils
 
 LOGGER = logging.getLogger(__name__)
 
-def recursive_find(root_directory, extensions):
+def recursive_find(root_directory, extensions=None):
     """Recursively find files with the given extensions, from a root_directory.
 
     Return a list of files matching those conditions.
 
     Arguments:
-    - `extensions`: list of accepted extensions.
+    - `extensions`: list of accepted extensions (None means every file).
     - `root_directory`: root directory of the search.
     """
     if not os.path.isdir(root_directory):
         return []
 
     matches = []
-    pattern = re.compile(r'.*\.({})$'.format('|'.join(extensions)))
+    if extensions is None:
+        pattern = re.compile('.*')
+    else:
+        pattern = re.compile(r'.*\.({})$'.format('|'.join(extensions)))
+
     with chdir(root_directory):
         for root, __ignored, filenames in os.walk(os.curdir):
             for filename in filenames:

patacrep/songbook/__main__.py

@@ -9,6 +9,7 @@ import textwrap
 import sys
 
 from patacrep.build import SongbookBuilder, DEFAULT_STEPS
+from patacrep.utils import yesno
 from patacrep import __version__
 from patacrep import errors
 import patacrep.encoding
@@ -37,6 +38,16 @@ class VerboseAction(argparse.Action):
     def __call__(self, *_args, **_kwargs):
         LOGGER.setLevel(logging.DEBUG)
 
+def yesno_type(string):
+    """Interpret argument as a "yes" or a "no".
+
+    Raise `argparse.ArgumentTypeError` if string cannot be analysed.
+    """
+    try:
+        return yesno(string)
+    except ValueError as error:
+        raise argparse.ArgumentTypeError(str(error))
+
 def argument_parser(args):
     """Parse arguments"""
     parser = argparse.ArgumentParser(
@@ -68,6 +79,15 @@ def argument_parser(args):
         """)
         )
 
+    parser.add_argument(
+        '--cache', '-c', nargs=1,
+        help=textwrap.dedent("""\
+                Enable song cache.
+        """),
+        type=yesno_type,
+        default=[True],
+        )
+
     parser.add_argument(
         '--steps', '-s', nargs=1, type=str,
         action=ParseStepsAction,
@@ -107,7 +127,7 @@ def main():
 
     options = argument_parser(sys.argv[1:])
 
-    songbook_path = options.book[0]
+    songbook_path = options.book[-1]
     if os.path.exists(songbook_path + ".sb") and not os.path.exists(songbook_path):
         songbook_path += ".sb"
 
@@ -147,6 +167,7 @@ def main():
     datadirs.append(os.path.dirname(os.path.abspath(songbook_path)))
 
     songbook['datadir'] = datadirs
+    songbook['_cache'] = options.cache[0]
 
     try:
         sb_builder = SongbookBuilder(songbook, basename)

patacrep/songs/__init__.py

@@ -104,7 +104,7 @@ class Song:
         self.encoding = config["encoding"]
         self.config = config
 
-        if datadir:
+        if self.datadir and self.config['_cache']:
             # Only songs in datadirs are cached
             self._filehash = hashlib.md5(
                 open(self.fullpath, 'rb').read()
@@ -155,7 +155,7 @@ class Song:
 
     def _write_cache(self):
         """If relevant, write a dumbed down version of self to the cache."""
-        if self.datadir:
+        if self.datadir and self.config['_cache']:
             cached = {}
             for attribute in self.cached_attributes:
                 cached[attribute] = getattr(self, attribute)
@@ -192,11 +192,11 @@ class Song:
         """
         raise NotImplementedError()
 
-    def get_datadirs(self, subdir=None):
-        """Return an iterator of existing datadirs (with eventually a subdir)
+    def iter_datadirs(self, *subpath):
+        """Return an iterator of existing datadirs (with an optionnal subpath)
         """
         for directory in self.config['datadir']:
-            fullpath = os.path.join(directory, subdir)
+            fullpath = os.path.join(directory, *subpath)
             if os.path.isdir(fullpath):
                 yield fullpath
 
@@ -255,7 +255,7 @@ class Song:
         return self.search_file(
             filename,
             ['', '.jpg', '.png'],
-            datadirs=self.get_datadirs('img'),
+            datadirs=self.iter_datadirs('img'),
             )
 
     def search_partition(self, filename):
@@ -263,7 +263,7 @@ class Song:
         return self.search_file(
             filename,
             ['', '.ly'],
-            datadirs=self.get_datadirs('scores'),
+            datadirs=self.iter_datadirs('scores'),
             )
 
 def unprefixed_title(title, prefixes):

patacrep/songs/chordpro/__init__.py

@@ -48,14 +48,9 @@ class ChordproSong(Song):
             "content": self.cached['song'].content,
             }
 
-        jinjaenv = Environment(loader=ChoiceLoader([
-            FileSystemLoader(
-                self.get_datadirs(os.path.join("templates", self.output_language))
-            ),
-            FileSystemLoader(
-                pkg_datapath('ast_templates', 'chordpro', self.output_language)
-            ),
-            ]))
+        jinjaenv = Environment(loader=FileSystemLoader(
+            self.iter_datadirs("templates", "songs", "chordpro", self.output_language)
+            ))
         jinjaenv.filters['search_image'] = self.search_image
         jinjaenv.filters['search_partition'] = self.search_partition
         jinjaenv.filters['lang2babel'] = lang2babel

patacrep/songs/convert/__main__.py

@@ -7,24 +7,21 @@ import os
 import logging
 import sys
 
-from patacrep.build import DEFAULT_CONFIG
 from patacrep import files
+from patacrep.build import DEFAULT_CONFIG
+from patacrep.utils import yesno
 
 LOGGER = logging.getLogger(__name__)
 
 def __usage():
     return "python3 -m patacrep.songs.convert INPUTFORMAT OUTPUTFORMAT FILES"
 
-def yesno(prompt):
-    while True:
-        answer = input("{} [yn] ".format(prompt))
-        if answer.strip().lower() == "y":
-            return True
-        if answer.strip().lower() == "n":
-            return False
-
 def confirm(destname):
-    return yesno("File '{}' already exist. Overwrite?".format(destname))
+    while True
+        try:
+            return yesno(input("File '{}' already exist. Overwrite? [yn] ".format(destname)))
+        except ValueError:
+            continue
 
 if __name__ == "__main__":
     if len(sys.argv) < 4:

patacrep/utils.py

@@ -59,3 +59,19 @@ class DictOfDict(UserDict):
                 DictOfDict._update(left[key], right[key])
             else:
                 left[key] = right[key]
+
+def yesno(string):
+    """Interpret string argument as a boolean.
+
+    May raise `ValueError` if argument cannot be interpreted.
+    """
+    yes_strings = ["y", "yes", "true", "1"]
+    no_strings = ["n", "no", "false", "0"]
+    if string.lower() in yes_strings:
+        return True
+    if string.lower() in no_strings:
+        return False
+    raise ValueError("'{}' is supposed to be one of {}.".format(
+        string,
+        ", ".join(["'{}'".format(string) for string in yes_strings + no_strings]),
+        ))

setup.py

@@ -4,10 +4,14 @@
 
 $ python setup.py install
 """
-from patacrep import __version__
+from patacrep import __version__, __DATADIR__, files
 
 from setuptools import setup, find_packages
 
+# List the data files
+data_files = files.recursive_find(__DATADIR__)
+data_files = ["data/" + d for d in data_files]
+
 setup(
     name='patacrep',
     version=__version__,
@@ -18,15 +22,10 @@ setup(
     packages=find_packages(exclude=["test*"]),
     license="GPLv2 or any later version",
     install_requires=[
-        "unidecode", "jinja2", "chardet", "ply",
+        "unidecode", "jinja2", "ply",
         ],
     setup_requires=["hgtools"],
-    package_data={'patacrep': [
-        'data/ast_templates/*/*/*',
-        'data/img/*',
-        'data/latex/*',
-        'data/templates/*',
-        ]},
+    package_data={'patacrep': data_files},
     entry_points={
         'console_scripts': [
             "songbook = patacrep.songbook.__main__:main",

test/test_chordpro/test_parser.py

@@ -8,7 +8,7 @@ import os
 import unittest
 from pkg_resources import resource_filename
 
-from patacrep import files
+from patacrep import files, __DATADIR__
 from patacrep.build import DEFAULT_CONFIG
 from patacrep.encoding import open_read
 
@@ -74,6 +74,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
         if 'datadir' not in cls.config:
             cls.config['datadir'] = []
         cls.config['datadir'].append('datadir')
+        cls.config['datadir'].append(__DATADIR__)
 
         cls.song_plugins = files.load_plugins(
             datadirs=cls.config['datadir'],

test/test_compilation/syntax.tex.control

@@ -92,7 +92,7 @@ guitar,
 
 \selectlanguage{english}
 
-\beginsong{Song with Sharp in musicnote}[
+\beginsong{Song with Sharp in musicnote}[
   by={
   },
 ]

test/test_compilation/test_compilation.py

@@ -9,7 +9,6 @@ import sys
 import subprocess
 import unittest
 
-from patacrep.encoding import open_read
 from patacrep.files import path2posix
 
 from .. import dynamic # pylint: disable=unused-import
@@ -64,8 +63,8 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
                 raise unittest.SkipTest('No control file for {}'.format(songbook))
 
             tex = "{}.tex".format(base)
-            with open_read(control) as expectfile:
-                with open_read(tex) as latexfile:
+            with open(control, mode="r", encoding="utf8") as expectfile:
+                with open(tex, mode="r", encoding="utf8") as latexfile:
                     expected = expectfile.read().strip()
                     expected = expected.replace(
                         "@TEST_FOLDER@",
@@ -112,7 +111,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
     @staticmethod
     def compile_songbook(songbook, steps=None):
         """Compile songbook, and return the command return code."""
-        command = [sys.executable, '-m', 'patacrep.songbook', songbook]
+        command = [sys.executable, '-m', 'patacrep.songbook', '--cache=no', songbook]
         if steps:
             command.extend(['--steps', steps])