Songs are cached only if they are in a datadir

patacrep/build.py

@@ -13,6 +13,7 @@ from subprocess import Popen, PIPE, call
 from patacrep import __DATADIR__, authors, content, errors
 from patacrep.index import process_sxd
 from patacrep.templates import TexRenderer
+from patacrep.songs import DataSubpath
 
 LOGGER = logging.getLogger(__name__)
 EOL = "\n"
@@ -75,7 +76,7 @@ class Songbook(object):
 
         self.config['datadir'] = abs_datadir
         self.config['_songdir'] = [
-                os.path.join(path, 'songs')
+                DataSubpath(path, 'songs')
                 for path in self.config['datadir']
                 ]
 

patacrep/content/cwd.py

@@ -3,9 +3,8 @@
 
 """Change base directory before importing songs."""
 
-import os
-
 from patacrep.content import process_content
+from patacrep.songs import DataSubpath
 
 #pylint: disable=unused-argument
 def parse(keyword, config, argument, contentlist):
@@ -28,8 +27,8 @@ def parse(keyword, config, argument, contentlist):
     """
     old_songdir = config['_songdir']
     config['_songdir'] = (
-            [argument] +
+            [DataSubpath("", argument)] +
-            [os.path.join(path, argument) for path in config['_songdir']] +
+            [path.clone().join(argument) for path in config['_songdir']] +
             config['_songdir']
             )
     processed_content = process_content(contentlist, config)

patacrep/content/song.py

@@ -35,7 +35,7 @@ class SongRenderer(Content, Song):
     def render(self, context):
         """Return the string that will render the song."""
         return r'\input{{{}}}'.format(files.relpath(
-            self.path,
+            self.fullpath,
             os.path.dirname(context['filename'])
             ))
 
@@ -59,21 +59,28 @@ def parse(keyword, argument, contentlist, config):
         if contentlist:
             break
         contentlist = [
-                files.relpath(filename, songdir)
+                filename
                 for filename
                 in (
-                    files.recursive_find(songdir, "*.sg")
+                    files.recursive_find(songdir.fullpath, "*.sg")
-                    + files.recursive_find(songdir, "*.is")
+                    + files.recursive_find(songdir.fullpath, "*.is")
                     )
                 ]
     for elem in contentlist:
         before = len(songlist)
         for songdir in config['_songdir']:
-            for filename in glob.iglob(os.path.join(songdir, elem)):
+            if songdir.datadir and not os.path.isdir(songdir.datadir):
-                LOGGER.debug('Parsing file "{}"…'.format(filename))
+                continue
-                song = SongRenderer(filename, config)
+            with files.chdir(songdir.datadir):
-                songlist.append(song)
+                for filename in glob.iglob(os.path.join(songdir.subpath, elem)):
-                config["_languages"].update(song.languages)
+                    LOGGER.debug('Parsing file "{}"…'.format(filename))
+                    song = SongRenderer(
+                            songdir.datadir,
+                            filename,
+                            config,
+                            )
+                    songlist.append(song)
+                    config["_languages"].update(song.languages)
             if len(songlist) > before:
                 break
         if len(songlist) == before:

patacrep/content/sorted.py

@@ -49,7 +49,7 @@ def key_generator(sort):
             if key == "@title":
                 field = song.unprefixed_titles
             elif key == "@path":
-                field = song.path
+                field = song.fullpath
             elif key == "by":
                 field = song.authors
             else:
@@ -59,7 +59,7 @@ def key_generator(sort):
                     LOGGER.debug(
                             "Ignoring unknown key '{}' for song {}.".format(
                                 key,
-                                files.relpath(song.path),
+                                files.relpath(song.fullpath),
                                 )
                             )
                     field = ""

patacrep/content/tex.py

@@ -41,8 +41,11 @@ def parse(keyword, argument, contentlist, config):
     for filename in contentlist:
         checked_file = None
         for path in config['_songdir']:
-            if os.path.exists(os.path.join(path, filename)):
+            if os.path.exists(os.path.join(path.fullpath, filename)):
-                checked_file = os.path.relpath(os.path.join(path, filename))
+                checked_file = os.path.relpath(os.path.join(
+                    path.fullpath,
+                    filename,
+                    ))
                 break
         if not checked_file:
             LOGGER.warning(

patacrep/files.py

@@ -4,6 +4,7 @@
 
 """File system utilities."""
 
+from contextlib import contextmanager
 import fnmatch
 import os
 
@@ -12,10 +13,14 @@ def recursive_find(root_directory, pattern):
 
     Return a list of files matching the pattern.
     """
+    if not os.path.isdir(root_directory):
+        return []
+
     matches = []
-    for root, _, filenames in os.walk(root_directory):
+    with chdir(root_directory):
-        for filename in fnmatch.filter(filenames, pattern):
+        for root, _, filenames in os.walk(os.curdir):
-            matches.append(os.path.join(root, filename))
+            for filename in fnmatch.filter(filenames, pattern):
+                matches.append(os.path.join(root, filename))
     return matches
 
 def relpath(path, start=None):
@@ -26,3 +31,22 @@ def relpath(path, start=None):
         return os.path.relpath(path, start)
     else:
         return os.path.abspath(path)
+
+@contextmanager
+def chdir(path):
+    """Locally change dir
+
+    Can be used as:
+
+        with chdir("some/directory"):
+            do_stuff()
+
+    """
+    olddir = os.getcwd()
+    if path:
+        os.chdir(path)
+        yield
+        os.chdir(olddir)
+    else:
+        yield
+

patacrep/songs.py

@@ -4,22 +4,67 @@
 """Song management."""
 
 from unidecode import unidecode
+import errno
 import hashlib
 import os
 import re
 
 try:
-        import cPickle as pickle
+    import cPickle as pickle
 except ImportError:
-        import pickle
+    import pickle
 
 from patacrep.authors import processauthors
 from patacrep.plastex import parsetex
 
 
-def cached_name(filename):
+def cached_name(datadir, filename):
     """Return the filename of the cache version of the file."""
-    return filename + ".cache"
+    fullpath = os.path.join(datadir, '.cache', filename)
+    directory = os.path.dirname(fullpath)
+    try:
+        os.makedirs(directory)
+    except OSError as error:
+        if error.errno == errno.EEXIST and os.path.isdir(directory):
+            pass
+        else:
+            raise
+    return fullpath
+
+class DataSubpath(object):
+    """A path divided in two path: a datadir, and its subpath.
+
+    - This object can represent either a file or directory.
+    - If the datadir part is the empty string, it means that the represented
+      path does not belong to a datadir.
+    """
+
+    def __init__(self, datadir, subpath):
+        if os.path.isabs(subpath):
+            self.datadir = ""
+        else:
+            self.datadir = datadir
+        self.subpath = subpath
+
+    def __str__(self):
+        return os.path.join(self.datadir, self.subpath)
+
+    @property
+    def fullpath(self):
+        """Return the full path represented by self."""
+        return os.path.join(self.datadir, self.subpath)
+
+    def clone(self):
+        """Return a cloned object."""
+        return DataSubpath(self.datadir, self.subpath)
+
+    def join(self, path):
+        """Join "path" argument to self path.
+
+        Return self for commodity.
+        """
+        self.subpath = os.path.join(self.subpath, path)
+        return self
 
 # pylint: disable=too-few-public-methods, too-many-instance-attributes
 class Song(object):
@@ -34,28 +79,36 @@ class Song(object):
             "titles",
             "unprefixed_titles",
             "args",
-            "path",
+            "datadir",
+            "fullpath",
+            "subpath",
             "languages",
             "authors",
             "_filehash",
             "_version",
             ]
 
-    def __init__(self, filename, config):
+    def __init__(self, datadir, subpath, config):
-        self._filehash = hashlib.md5(open(filename, 'rb').read()).hexdigest()
+        self.fullpath = os.path.join(datadir, subpath)
-        if os.path.exists(cached_name(filename)):
+        if datadir:
-            cached = pickle.load(open(cached_name(filename), 'rb'))
+            # Only songs in datadirs are cached
-            if (
+            self._filehash = hashlib.md5(
-                    cached['_filehash'] == self._filehash
+                    open(self.fullpath, 'rb').read()
-                    and cached['_version'] == self.CACHE_VERSION
+                    ).hexdigest()
-                    ):
+            if os.path.exists(cached_name(datadir, subpath)):
-                for attribute in self.cached_attributes:
+                cached = pickle.load(open(cached_name(datadir, subpath), 'rb'))
-                    setattr(self, attribute, cached[attribute])
+                if (
-                return
+                        cached['_filehash'] == self._filehash
+                        and cached['_version'] == self.CACHE_VERSION
+                        ):
+                    for attribute in self.cached_attributes:
+                        setattr(self, attribute, cached[attribute])
+                    return
 
         # Data extraction from the song with plastex
-        data = parsetex(filename)
+        data = parsetex(self.fullpath)
         self.titles = data['titles']
+        self.datadir = datadir
         self.unprefixed_titles = [
                 unprefixed_title(
                     unidecode(unicode(title, "utf-8")),
@@ -65,7 +118,7 @@ class Song(object):
                 in self.titles
                 ]
         self.args = data['args']
-        self.path = filename
+        self.subpath = subpath
         self.languages = data['languages']
         if "by" in self.args.keys():
             self.authors = processauthors(
@@ -79,14 +132,18 @@ class Song(object):
         self._write_cache()
 
     def _write_cache(self):
-        """Write a dumbed down version of self to the cache."""
+        """If relevant, write a dumbed down version of self to the cache."""
-        cached = {}
+        if self.datadir:
-        for attribute in self.cached_attributes:
+            cached = {}
-            cached[attribute] = getattr(self, attribute)
+            for attribute in self.cached_attributes:
-        pickle.dump(cached, open(cached_name(self.path), 'wb'))
+                cached[attribute] = getattr(self, attribute)
+            pickle.dump(
+                    cached,
+                    open(cached_name(self.datadir, self.subpath), 'wb'),
+                    )
 
     def __repr__(self):
-        return repr((self.titles, self.args, self.path))
+        return repr((self.titles, self.args, self.fullpath))
 
 def unprefixed_title(title, prefixes):
     """Remove the first prefix of the list in the beginning of title (if any).