Merge pull request #203 from patacrep/yaml_ext

Support any songbook file extension and refactorization

patacrep/build.py

@@ -169,7 +169,7 @@ class SongbookBuilder:
 
     def __init__(self, raw_songbook):
         # Basename of the songbook to be built.
-        self.basename = raw_songbook['_basename']
+        self.basename = raw_songbook['_outputname']
         # Representation of the .yaml songbook configuration file.
         self.songbook = Songbook(raw_songbook, self.basename)
 
@@ -300,6 +300,10 @@ class SongbookBuilder:
         standard_output.join()
         process.wait()
 
+        # Close the stdout and stderr to prevent ResourceWarning:
+        process.stdout.close()
+        process.stderr.close()
+
         if process.returncode:
             raise errors.LatexCompilationError(self.basename)
 

patacrep/content/cwd.py

@@ -1,7 +1,5 @@
 """Change base directory before importing songs."""
 
-import os
-
 from patacrep.content import process_content, validate_parser_argument
 from patacrep.songs import DataSubpath
 
@@ -30,16 +28,15 @@ def parse(keyword, config, argument):
     The 'path' is added:
     - first as a relative path to the *.yaml file directory;
     - then as a relative path to every path already present in
-      config['songdir'].
+      config['songdir'] (which are 'song' dir inside the datadirs).
     """
     subpath = argument['path']
     old_songdir = config['_songdir']
-    sbdir = os.path.dirname(config['_filepath'])
 
-    config['_songdir'] = (
+    config['_songdir'] = [path.clone().join(subpath) for path in config['_songdir']]
-        [DataSubpath(sbdir, subpath)] +
+    if '_songbookfile_dir' in config:
-        [path.clone().join(subpath) for path in config['_songdir']]
+        config['_songdir'].insert(0, DataSubpath(config['_songbookfile_dir'], subpath))
-        )
+
     processed_content = process_content(argument.get('content'), config)
     config['_songdir'] = old_songdir
     return processed_content

patacrep/content/song.py

@@ -95,6 +95,8 @@ def parse(keyword, argument, config):
         if contentlist:
             break
         contentlist = files.recursive_find(songdir.fullpath, plugins.keys())
+    if contentlist is None:
+        contentlist = [] # No content was set or found
     for elem in contentlist:
         before = len(songlist)
         for songdir in config['_songdir']:

patacrep/data/templates/songbook_model.yml

@@ -3,10 +3,11 @@ schema:
   optional:
     content: //any
     template: //any
+    _songbookfile_dir: //str
   required:
     _cache: //bool
-    _filepath: //str
+    _outputdir: //str
-    _basename: //str
+    _outputname: //str
     _error: //str
     _datadir:
       type: //arr

patacrep/songbook/__init__.py

@@ -14,7 +14,8 @@ import patacrep
 LOGGER = logging.getLogger()
 
 def open_songbook(filename):
-    """Open songbook, and return a raw songbook object.
+    """Open a songbook file, and prepare it to
+    return a raw songbook object.
 
     :param str filename: Filename of the yaml songbook.
     :rvalue: dict
@@ -35,13 +36,35 @@ def open_songbook(filename):
     except Exception as error: # pylint: disable=broad-except
         raise patacrep.errors.SongbookError(str(error))
 
-    songbook = _add_songbook_defaults(user_songbook)
+    songbookfile_dir = os.path.dirname(os.path.abspath(filename))
+    # Output at the same place as the songbook file
+    outputdir = songbookfile_dir
+    outputname = os.path.splitext(os.path.basename(filename))[0]
 
-    songbook['_filepath'] = filename
+    return prepare_songbook(user_songbook, outputdir, outputname, songbookfile_dir)
-    songbook['_basename'] = os.path.basename(filename)[:-len(".yaml")]
+
+def prepare_songbook(songbook, outputdir, outputname, songbookfile_dir=None, datadir_prefix=None):
+    """Prepare a songbook by adding default values and datadirs
+    Returns a raw songbook object.
+
+    :param dict songbook: Initial yaml songbook.
+    :param str outputdir: Folder to put the output (tex, pdf...)
+    :param str outputname: Filename for the outputs (tex, pdf...)
+    :param str songbookfile_dir: Folder of the original songbook file (if there is one)
+    :param str datadir_prefix: Prefix for the datadirs
+    :rvalue: dict
+    :return: Songbook, as a dictionary.
+    """
+
+    songbook['_outputdir'] = outputdir
+    songbook['_outputname'] = outputname
+    if songbookfile_dir:
+        songbook['_songbookfile_dir'] = songbookfile_dir
+
+    songbook = _add_songbook_defaults(songbook)
 
     # Gathering datadirs
-    songbook['_datadir'] = list(_iter_absolute_datadirs(songbook))
+    songbook['_datadir'] = list(_iter_absolute_datadirs(songbook, datadir_prefix))
     if 'datadir' in songbook['book']:
         del songbook['book']['datadir']
 
@@ -50,7 +73,6 @@ def open_songbook(filename):
         for path in songbook['_datadir']
     ]
 
-
     return songbook
 
 def _add_songbook_defaults(user_songbook):
@@ -78,26 +100,32 @@ def _add_songbook_defaults(user_songbook):
 
     return dict(default_songbook)
 
-def _iter_absolute_datadirs(raw_songbook):
+def _iter_absolute_datadirs(raw_songbook, datadir_prefix=None):
     """Iterate on the absolute datadirs of the raw songbook
 
     Appends the songfile dir at the end
     """
+    songbookfile_dir = raw_songbook.get('_songbookfile_dir')
+
+    if datadir_prefix is None:
+        if songbookfile_dir is None:
+            raise patacrep.errors.SongbookError('Please specify where the datadir are located')
+        datadir_prefix = songbookfile_dir
+
     datadir = raw_songbook.get('book', {}).get('datadir')
-    filepath = raw_songbook['_filepath']
 
     if datadir is None:
         datadir = []
     elif isinstance(datadir, str):
         datadir = [datadir]
-    basedir = os.path.dirname(os.path.abspath(filepath))
 
     for path in datadir:
-        abspath = os.path.join(basedir, path)
+        abspath = os.path.join(datadir_prefix, path)
         if os.path.exists(abspath) and os.path.isdir(abspath):
             yield abspath
         else:
             LOGGER.warning(
                 "Ignoring non-existent datadir '{}'.".format(path)
                 )
-    yield basedir
+    if songbookfile_dir:
+        yield songbookfile_dir

patacrep/songs/__init__.py

@@ -152,16 +152,16 @@ class Song:
     def filehash(self):
         """Compute (and cache) the md5 hash of the file"""
         if self._filehash is None:
-            self._filehash = hashlib.md5(
+            with open(self.fullpath, 'rb') as songfile:
-                open(self.fullpath, 'rb').read()
+                self._filehash = hashlib.md5(songfile.read()).hexdigest()
-                ).hexdigest()
         return self._filehash
 
     def _cache_retrieved(self):
         """If relevant, retrieve self from the cache."""
         if self.use_cache and os.path.exists(self.cached_name):
             try:
-                cached = pickle.load(open(self.cached_name, 'rb',))
+                with open(self.cached_name, 'rb',) as cachefile:
+                    cached = pickle.load(cachefile)
                 if (
                         cached['_filehash'] == self.filehash
                         and cached['_version'] == self.CACHE_VERSION

test/test_content/test_content.py

@@ -9,10 +9,9 @@ import yaml
 
 from pkg_resources import resource_filename
 
-from patacrep.songs import DataSubpath
 from patacrep import content, files
 from patacrep.content import song, section, songsection, tex
-from patacrep.build import config_model
+from patacrep.songbook import prepare_songbook
 
 from .. import logging_reduced
 from .. import dynamic # pylint: disable=unused-import
@@ -28,10 +27,6 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
     maxDiff = None
     config = None
 
-    @classmethod
-    def setUpClass(cls):
-        cls._generate_config()
-
     @classmethod
     def _iter_testmethods(cls):
         """Iterate over dynamically generated test methods"""
@@ -55,8 +50,9 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
             with open(sourcename, mode="r", encoding="utf8") as sourcefile:
                 sbcontent = yaml.load(sourcefile)
 
-            config = cls.config.copy()
+            outputdir = os.path.dirname(base)
-            config['_filepath'] = base
+            config = cls._generate_config(sbcontent, outputdir, base)
+
             with logging_reduced('patacrep.content.song'):
                 expandedlist = content.process_content(sbcontent, config)
             sourcelist = [cls._clean_path(elem) for elem in expandedlist]
@@ -97,28 +93,27 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
             raise Exception(elem)
 
     @classmethod
-    def _generate_config(cls):
+    def _generate_config(cls, sbcontent, outputdir, base):
         """Generate the config to process the content"""
 
         # Load the default songbook config
-        config = config_model('default')['en']
+        config = prepare_songbook(
-
+            {'book':{'datadir':'datadir'}, 'content': sbcontent},
-        datadirpaths = [os.path.join(os.path.dirname(__file__), 'datadir')]
+            outputdir,
-
+            base,
-        config['_datadir'] = datadirpaths
+            outputdir
+            )
 
-        config['_songdir'] = [
+        # Load the plugins
-            DataSubpath(path, 'songs')
-            for path in datadirpaths
-            ]
         config['_content_plugins'] = files.load_plugins(
-            datadirs=datadirpaths,
+            datadirs=config['_datadir'],
             root_modules=['content'],
             keyword='CONTENT_PLUGINS',
             )
         config['_song_plugins'] = files.load_plugins(
-            datadirs=datadirpaths,
+            datadirs=config['_datadir'],
             root_modules=['songs'],
             keyword='SONG_RENDERERS',
             )['tsg']
-        cls.config = config
+
+        return config

test/test_patatools/test_cache.py

@@ -11,6 +11,8 @@ from patacrep.tools.__main__ import main as tools_main
 from patacrep.tools.cache.__main__ import main as cache_main
 from patacrep.songbook.__main__ import main as songbook_main
 
+from .. import logging_reduced
+
 CACHEDIR = os.path.join(os.path.dirname(__file__), "test_cache_datadir", ".cache")
 
 class TestCache(unittest.TestCase):
@@ -46,11 +48,16 @@ class TestCache(unittest.TestCase):
             ]:
             with self.subTest(main=main, args=args):
                 # First compilation. Ensure that cache exists afterwards
-                self._system(songbook_main, ["songbook", "--steps", "tex,clean", "test_cache.yaml"])
+                with logging_reduced('patacrep.build'):
+                    self._system(
+                        songbook_main,
+                        ["songbook", "--steps", "tex,clean", "test_cache.yaml"]
+                    )
                 self.assertTrue(os.path.exists(CACHEDIR))
 
                 # Clean cache
-                self._system(main, args)
+                with logging_reduced('patatools.cache'):
+                    self._system(main, args)
 
                 # Ensure that cache does not exist
                 self.assertFalse(os.path.exists(CACHEDIR))
@@ -64,4 +71,5 @@ class TestCache(unittest.TestCase):
             ]:
             with self.subTest(main=main, args=args):
                 # Clean cache
-                self._system(main, args)
+                with logging_reduced('patatools.cache'):
+                    self._system(main, args)

test/test_songbook/.gitignore

@@ -1,2 +1,2 @@
-/*tex
+**.tex
 .cache

test/test_songbook/onthefly/content.onthefly.tex.control

@@ -0,0 +1,133 @@
+
+
+
+
+
+
+%% Automatically generated document.
+%% You may edit this file but all changes will be overwritten.
+%% If you want to change this document, have a look at
+%% the templating system.
+%%
+%% Generated using Songbook <http://www.patacrep.com>
+
+\makeatletter
+\def\input@path{ %
+        {@TEST_FOLDER@/onthefly/../content_datadir/latex/} %
+        {@LOCAL_DATA_FOLDER@/latex/} %
+}
+\makeatother
+
+\documentclass[
+    ]{article}
+
+\usepackage[
+chorded,
+pictures,
+diagram,
+guitar,
+    ]{patacrep}
+
+\usepackage{lmodern}
+
+
+\PassOptionsToPackage{english}{babel}
+\PassOptionsToPackage{english}{babel}
+\usepackage[english]{babel}
+\lang{english}
+
+\usepackage{graphicx}
+\graphicspath{ %
+        {@TEST_FOLDER@/onthefly/../content_datadir/} %
+        {@LOCAL_DATA_FOLDER@/} %
+}
+
+
+\makeatletter
+\@ifpackageloaded{hyperref}{}{
+   \usepackage{url}
+   \newcommand{\phantomsection}{}
+   \newcommand{\hyperlink}[2]{#2}
+   \newcommand{\href}[2]{\expandafter\url\expandafter{#1}}
+}
+\makeatother
+
+
+\usepackage{chords}
+
+\title{Guitar songbook}
+\author{The Patacrep Team}
+
+\newindex{titleidx}{content.onthefly_title}
+\newauthorindex{authidx}{content.onthefly_auth}
+
+\authignoreword{unknown}
+\authbyword{by}
+\authsepword{and}
+
+\notenamesout{A}{B}{C}{D}{E}{F}{G}
+
+
+\begin{document}
+
+\maketitle
+
+
+\showindex{\songindexname}{titleidx}
+\showindex{\authorindexname}{authidx}
+
+% list of chords
+\ifchorded
+  \ifdiagram
+     \phantomsection
+     \addcontentsline{toc}{section}{\chordlistname}
+     \chords
+  \fi
+\fi
+
+\phantomsection
+\addcontentsline{toc}{section}{\songlistname}
+
+
+\section{Test of section}
+
+\begin{songs}{titleidx,authidx}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% songs/./song.csg
+
+\selectlanguage{english}
+
+\beginsong{This is a song}[
+  by={
+  },
+]
+
+
+
+
+\begin{verse}
+    Foo
+\end{verse}
+
+\endsong
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% songs/./song.tsg
+
+\import{@TEST_FOLDER@/content_datadir/songs/}{song.tsg}
+
+\end{songs}
+
+\songsection{Test of song section}
+
+
+\input{@TEST_FOLDER@/content_datadir/content/foo.tex}
+
+
+\section{This is an included section}
+
+
+
+
+
+\end{document}

test/test_songbook/onthefly/content.onthefly.yaml

@@ -0,0 +1,19 @@
+book:
+  pictures: yes
+  datadir: content_datadir
+  lang: en
+  template: default.tex
+chords:
+  repeatchords: no
+  diagramreminder: all
+
+content:
+  - section: Test of section
+  - sort:
+  - songsection: Test of song section
+  - cwd:
+      # relative to datadir 'song' dir
+      path: ../content
+      content:
+        - tex: foo.tex
+  - include: include.sbc

test/test_songbook/test_compilation.py

@@ -9,8 +9,14 @@ import sys
 import subprocess
 import unittest
 
-from patacrep.files import path2posix
+import yaml
 
+from patacrep.files import path2posix, chdir
+from patacrep.songbook import prepare_songbook
+from patacrep.build import SongbookBuilder
+from patacrep import __DATADIR__
+
+from .. import logging_reduced
 from .. import dynamic # pylint: disable=unused-import
 
 LOGGER = logging.getLogger(__name__)
@@ -36,7 +42,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
                 os.path.dirname(__file__),
                 '*.yaml',
             ))):
-            base = songbook[:-len(".yaml")]
+            base = os.path.splitext(songbook)[0]
             yield (
                 "test_latex_generation_{}".format(os.path.basename(base)),
                 cls._create_generation_test(base),
@@ -45,9 +51,23 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
                 "test_pdf_compilation_{}".format(os.path.basename(base)),
                 cls._create_compilation_test(base),
                 )
+        for songbook in sorted(glob.glob(os.path.join(
+                os.path.dirname(__file__),
+                'onthefly',
+                '*.yaml',
+            ))):
+            base = os.path.splitext(songbook)[0]
+            yield (
+                "test_latex_generation_onthefly_{}".format(os.path.basename(base)),
+                cls._create_generation_test(base, True),
+                )
+            yield (
+                "test_pdf_compilation_onthefly_{}".format(os.path.basename(base)),
+                cls._create_compilation_test(base, True),
+                )
 
     @classmethod
-    def _create_generation_test(cls, base):
+    def _create_generation_test(cls, base, onthefly=False):
         """Return a function testing that `base.tex` is correctly generated."""
 
         def test_generation(self):
@@ -55,7 +75,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
             songbook = "{}.yaml".format(base)
 
             # Check tex generation
-            self.assertEqual(0, self.compile_songbook(songbook, "tex"))
+            if onthefly:
+                self.compile_songbook_onthefly(base, ['tex'])
+            else:
+                self.assertEqual(0, self.compile_songbook(songbook, "tex"))
 
             # Check generated tex
             control = "{}.tex.control".format(base)
@@ -70,6 +93,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
                         "@TEST_FOLDER@",
                         path2posix(os.path.dirname(__file__)),
                         )
+                    expected = expected.replace(
+                        "@LOCAL_DATA_FOLDER@",
+                        path2posix(__DATADIR__),
+                        )
 
                     expected = expected.replace(
                         "@DATA_FOLDER@",
@@ -93,7 +120,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
         return test_generation
 
     @classmethod
-    def _create_compilation_test(cls, base):
+    def _create_compilation_test(cls, base, onthefly=False):
         """Return a function testing that `base.tex` is correctly compiled."""
         @unittest.skipIf('TRAVIS' in os.environ,
                          "Travis does not support lualatex compilation yet")
@@ -101,7 +128,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
             """Test that `base` is rendered to pdf."""
             # Check compilation
             songbook = "{}.yaml".format(base)
-            self.assertEqual(0, self.compile_songbook(songbook))
+            if onthefly:
+                self.compile_songbook_onthefly(base)
+            else:
+                self.assertEqual(0, self.compile_songbook(songbook))
 
         test_compilation.__doc__ = (
             "Test that '{base}' is correctly compiled."
@@ -130,3 +160,28 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
         except subprocess.CalledProcessError as error:
             LOGGER.warning(error.output)
             return error.returncode
+
+    @staticmethod
+    def compile_songbook_onthefly(base, steps=None):
+        """Compile songbook "on the fly": without a physical songbook file."""
+
+        with open(base + ".yaml", mode="r", encoding="utf8") as sbfile:
+            sbyaml = yaml.load(sbfile)
+
+        outputdir = os.path.dirname(base)
+        outputname = os.path.basename(base)
+        datadir_prefix = os.path.join(outputdir, '..')
+        songbook = prepare_songbook(sbyaml, outputdir, outputname, datadir_prefix=datadir_prefix)
+        songbook['_error'] = "fix"
+        songbook['_cache'] = True
+
+        sb_builder = SongbookBuilder(songbook)
+        sb_builder.unsafe = True
+
+        with chdir(outputdir):
+            # Continuous Integration will be verbose
+            if 'CI' in os.environ:
+                with logging_reduced(level=logging.DEBUG):
+                    sb_builder.build_steps(steps)
+            else:
+                sb_builder.build_steps(steps)