diff --git a/patacrep/build.py b/patacrep/build.py index c3ac9461..de7bb3ce 100644 --- a/patacrep/build.py +++ b/patacrep/build.py @@ -169,7 +169,7 @@ class SongbookBuilder: def __init__(self, raw_songbook): # Basename of the songbook to be built. - self.basename = raw_songbook['_basename'] + self.basename = raw_songbook['_outputname'] # Representation of the .yaml songbook configuration file. self.songbook = Songbook(raw_songbook, self.basename) @@ -300,6 +300,10 @@ class SongbookBuilder: standard_output.join() process.wait() + # Close the stdout and stderr to prevent ResourceWarning: + process.stdout.close() + process.stderr.close() + if process.returncode: raise errors.LatexCompilationError(self.basename) diff --git a/patacrep/content/cwd.py b/patacrep/content/cwd.py index 2c43af39..7efee676 100755 --- a/patacrep/content/cwd.py +++ b/patacrep/content/cwd.py @@ -1,7 +1,5 @@ """Change base directory before importing songs.""" -import os - from patacrep.content import process_content, validate_parser_argument from patacrep.songs import DataSubpath @@ -30,16 +28,15 @@ def parse(keyword, config, argument): The 'path' is added: - first as a relative path to the *.yaml file directory; - then as a relative path to every path already present in - config['songdir']. + config['songdir'] (which are 'song' dir inside the datadirs). """ subpath = argument['path'] old_songdir = config['_songdir'] - sbdir = os.path.dirname(config['_filepath']) - config['_songdir'] = ( - [DataSubpath(sbdir, subpath)] + - [path.clone().join(subpath) for path in config['_songdir']] - ) + config['_songdir'] = [path.clone().join(subpath) for path in config['_songdir']] + if '_songbookfile_dir' in config: + config['_songdir'].insert(0, DataSubpath(config['_songbookfile_dir'], subpath)) + processed_content = process_content(argument.get('content'), config) config['_songdir'] = old_songdir return processed_content diff --git a/patacrep/content/song.py b/patacrep/content/song.py index 192294a0..4f81cc12 100755 --- a/patacrep/content/song.py +++ b/patacrep/content/song.py @@ -95,6 +95,8 @@ def parse(keyword, argument, config): if contentlist: break contentlist = files.recursive_find(songdir.fullpath, plugins.keys()) + if contentlist is None: + contentlist = [] # No content was set or found for elem in contentlist: before = len(songlist) for songdir in config['_songdir']: diff --git a/patacrep/data/templates/songbook_model.yml b/patacrep/data/templates/songbook_model.yml index 19fb66fb..05c40f2b 100644 --- a/patacrep/data/templates/songbook_model.yml +++ b/patacrep/data/templates/songbook_model.yml @@ -3,10 +3,11 @@ schema: optional: content: //any template: //any + _songbookfile_dir: //str required: _cache: //bool - _filepath: //str - _basename: //str + _outputdir: //str + _outputname: //str _error: //str _datadir: type: //arr diff --git a/patacrep/songbook/__init__.py b/patacrep/songbook/__init__.py index 34c77eb5..0a7d394e 100644 --- a/patacrep/songbook/__init__.py +++ b/patacrep/songbook/__init__.py @@ -14,7 +14,8 @@ import patacrep LOGGER = logging.getLogger() def open_songbook(filename): - """Open songbook, and return a raw songbook object. + """Open a songbook file, and prepare it to + return a raw songbook object. :param str filename: Filename of the yaml songbook. :rvalue: dict @@ -35,13 +36,35 @@ def open_songbook(filename): except Exception as error: # pylint: disable=broad-except raise patacrep.errors.SongbookError(str(error)) - songbook = _add_songbook_defaults(user_songbook) + songbookfile_dir = os.path.dirname(os.path.abspath(filename)) + # Output at the same place as the songbook file + outputdir = songbookfile_dir + outputname = os.path.splitext(os.path.basename(filename))[0] - songbook['_filepath'] = filename - songbook['_basename'] = os.path.basename(filename)[:-len(".yaml")] + return prepare_songbook(user_songbook, outputdir, outputname, songbookfile_dir) + +def prepare_songbook(songbook, outputdir, outputname, songbookfile_dir=None, datadir_prefix=None): + """Prepare a songbook by adding default values and datadirs + Returns a raw songbook object. + + :param dict songbook: Initial yaml songbook. + :param str outputdir: Folder to put the output (tex, pdf...) + :param str outputname: Filename for the outputs (tex, pdf...) + :param str songbookfile_dir: Folder of the original songbook file (if there is one) + :param str datadir_prefix: Prefix for the datadirs + :rvalue: dict + :return: Songbook, as a dictionary. + """ + + songbook['_outputdir'] = outputdir + songbook['_outputname'] = outputname + if songbookfile_dir: + songbook['_songbookfile_dir'] = songbookfile_dir + + songbook = _add_songbook_defaults(songbook) # Gathering datadirs - songbook['_datadir'] = list(_iter_absolute_datadirs(songbook)) + songbook['_datadir'] = list(_iter_absolute_datadirs(songbook, datadir_prefix)) if 'datadir' in songbook['book']: del songbook['book']['datadir'] @@ -50,7 +73,6 @@ def open_songbook(filename): for path in songbook['_datadir'] ] - return songbook def _add_songbook_defaults(user_songbook): @@ -78,26 +100,32 @@ def _add_songbook_defaults(user_songbook): return dict(default_songbook) -def _iter_absolute_datadirs(raw_songbook): +def _iter_absolute_datadirs(raw_songbook, datadir_prefix=None): """Iterate on the absolute datadirs of the raw songbook Appends the songfile dir at the end """ + songbookfile_dir = raw_songbook.get('_songbookfile_dir') + + if datadir_prefix is None: + if songbookfile_dir is None: + raise patacrep.errors.SongbookError('Please specify where the datadir are located') + datadir_prefix = songbookfile_dir + datadir = raw_songbook.get('book', {}).get('datadir') - filepath = raw_songbook['_filepath'] if datadir is None: datadir = [] elif isinstance(datadir, str): datadir = [datadir] - basedir = os.path.dirname(os.path.abspath(filepath)) for path in datadir: - abspath = os.path.join(basedir, path) + abspath = os.path.join(datadir_prefix, path) if os.path.exists(abspath) and os.path.isdir(abspath): yield abspath else: LOGGER.warning( "Ignoring non-existent datadir '{}'.".format(path) ) - yield basedir + if songbookfile_dir: + yield songbookfile_dir diff --git a/patacrep/songs/__init__.py b/patacrep/songs/__init__.py index 1a24ff81..b1d80f5d 100644 --- a/patacrep/songs/__init__.py +++ b/patacrep/songs/__init__.py @@ -152,16 +152,16 @@ class Song: def filehash(self): """Compute (and cache) the md5 hash of the file""" if self._filehash is None: - self._filehash = hashlib.md5( - open(self.fullpath, 'rb').read() - ).hexdigest() + with open(self.fullpath, 'rb') as songfile: + self._filehash = hashlib.md5(songfile.read()).hexdigest() return self._filehash def _cache_retrieved(self): """If relevant, retrieve self from the cache.""" if self.use_cache and os.path.exists(self.cached_name): try: - cached = pickle.load(open(self.cached_name, 'rb',)) + with open(self.cached_name, 'rb',) as cachefile: + cached = pickle.load(cachefile) if ( cached['_filehash'] == self.filehash and cached['_version'] == self.CACHE_VERSION diff --git a/test/test_content/test_content.py b/test/test_content/test_content.py index d4fae378..221a3212 100644 --- a/test/test_content/test_content.py +++ b/test/test_content/test_content.py @@ -9,10 +9,9 @@ import yaml from pkg_resources import resource_filename -from patacrep.songs import DataSubpath from patacrep import content, files from patacrep.content import song, section, songsection, tex -from patacrep.build import config_model +from patacrep.songbook import prepare_songbook from .. import logging_reduced from .. import dynamic # pylint: disable=unused-import @@ -28,10 +27,6 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): maxDiff = None config = None - @classmethod - def setUpClass(cls): - cls._generate_config() - @classmethod def _iter_testmethods(cls): """Iterate over dynamically generated test methods""" @@ -55,8 +50,9 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): with open(sourcename, mode="r", encoding="utf8") as sourcefile: sbcontent = yaml.load(sourcefile) - config = cls.config.copy() - config['_filepath'] = base + outputdir = os.path.dirname(base) + config = cls._generate_config(sbcontent, outputdir, base) + with logging_reduced('patacrep.content.song'): expandedlist = content.process_content(sbcontent, config) sourcelist = [cls._clean_path(elem) for elem in expandedlist] @@ -97,28 +93,27 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): raise Exception(elem) @classmethod - def _generate_config(cls): + def _generate_config(cls, sbcontent, outputdir, base): """Generate the config to process the content""" # Load the default songbook config - config = config_model('default')['en'] - - datadirpaths = [os.path.join(os.path.dirname(__file__), 'datadir')] - - config['_datadir'] = datadirpaths + config = prepare_songbook( + {'book':{'datadir':'datadir'}, 'content': sbcontent}, + outputdir, + base, + outputdir + ) - config['_songdir'] = [ - DataSubpath(path, 'songs') - for path in datadirpaths - ] + # Load the plugins config['_content_plugins'] = files.load_plugins( - datadirs=datadirpaths, + datadirs=config['_datadir'], root_modules=['content'], keyword='CONTENT_PLUGINS', ) config['_song_plugins'] = files.load_plugins( - datadirs=datadirpaths, + datadirs=config['_datadir'], root_modules=['songs'], keyword='SONG_RENDERERS', )['tsg'] - cls.config = config + + return config diff --git a/test/test_patatools/test_cache.py b/test/test_patatools/test_cache.py index 447e0e73..13d7407a 100644 --- a/test/test_patatools/test_cache.py +++ b/test/test_patatools/test_cache.py @@ -11,6 +11,8 @@ from patacrep.tools.__main__ import main as tools_main from patacrep.tools.cache.__main__ import main as cache_main from patacrep.songbook.__main__ import main as songbook_main +from .. import logging_reduced + CACHEDIR = os.path.join(os.path.dirname(__file__), "test_cache_datadir", ".cache") class TestCache(unittest.TestCase): @@ -46,11 +48,16 @@ class TestCache(unittest.TestCase): ]: with self.subTest(main=main, args=args): # First compilation. Ensure that cache exists afterwards - self._system(songbook_main, ["songbook", "--steps", "tex,clean", "test_cache.yaml"]) + with logging_reduced('patacrep.build'): + self._system( + songbook_main, + ["songbook", "--steps", "tex,clean", "test_cache.yaml"] + ) self.assertTrue(os.path.exists(CACHEDIR)) # Clean cache - self._system(main, args) + with logging_reduced('patatools.cache'): + self._system(main, args) # Ensure that cache does not exist self.assertFalse(os.path.exists(CACHEDIR)) @@ -64,4 +71,5 @@ class TestCache(unittest.TestCase): ]: with self.subTest(main=main, args=args): # Clean cache - self._system(main, args) + with logging_reduced('patatools.cache'): + self._system(main, args) diff --git a/test/test_songbook/.gitignore b/test/test_songbook/.gitignore index 3f49c007..7a3ae1d8 100644 --- a/test/test_songbook/.gitignore +++ b/test/test_songbook/.gitignore @@ -1,2 +1,2 @@ -/*tex +**.tex .cache diff --git a/test/test_songbook/onthefly/content.onthefly.tex.control b/test/test_songbook/onthefly/content.onthefly.tex.control new file mode 100644 index 00000000..a41446bd --- /dev/null +++ b/test/test_songbook/onthefly/content.onthefly.tex.control @@ -0,0 +1,133 @@ + + + + + + +%% Automatically generated document. +%% You may edit this file but all changes will be overwritten. +%% If you want to change this document, have a look at +%% the templating system. +%% +%% Generated using Songbook + +\makeatletter +\def\input@path{ % + {@TEST_FOLDER@/onthefly/../content_datadir/latex/} % + {@LOCAL_DATA_FOLDER@/latex/} % +} +\makeatother + +\documentclass[ + ]{article} + +\usepackage[ +chorded, +pictures, +diagram, +guitar, + ]{patacrep} + +\usepackage{lmodern} + + +\PassOptionsToPackage{english}{babel} +\PassOptionsToPackage{english}{babel} +\usepackage[english]{babel} +\lang{english} + +\usepackage{graphicx} +\graphicspath{ % + {@TEST_FOLDER@/onthefly/../content_datadir/} % + {@LOCAL_DATA_FOLDER@/} % +} + + +\makeatletter +\@ifpackageloaded{hyperref}{}{ + \usepackage{url} + \newcommand{\phantomsection}{} + \newcommand{\hyperlink}[2]{#2} + \newcommand{\href}[2]{\expandafter\url\expandafter{#1}} +} +\makeatother + + +\usepackage{chords} + +\title{Guitar songbook} +\author{The Patacrep Team} + +\newindex{titleidx}{content.onthefly_title} +\newauthorindex{authidx}{content.onthefly_auth} + +\authignoreword{unknown} +\authbyword{by} +\authsepword{and} + +\notenamesout{A}{B}{C}{D}{E}{F}{G} + + +\begin{document} + +\maketitle + + +\showindex{\songindexname}{titleidx} +\showindex{\authorindexname}{authidx} + +% list of chords +\ifchorded + \ifdiagram + \phantomsection + \addcontentsline{toc}{section}{\chordlistname} + \chords + \fi +\fi + +\phantomsection +\addcontentsline{toc}{section}{\songlistname} + + +\section{Test of section} + +\begin{songs}{titleidx,authidx} +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% songs/./song.csg + +\selectlanguage{english} + +\beginsong{This is a song}[ + by={ + }, +] + + + + +\begin{verse} + Foo +\end{verse} + +\endsong + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% songs/./song.tsg + +\import{@TEST_FOLDER@/content_datadir/songs/}{song.tsg} + +\end{songs} + +\songsection{Test of song section} + + +\input{@TEST_FOLDER@/content_datadir/content/foo.tex} + + +\section{This is an included section} + + + + + +\end{document} diff --git a/test/test_songbook/onthefly/content.onthefly.yaml b/test/test_songbook/onthefly/content.onthefly.yaml new file mode 100644 index 00000000..3d5b585f --- /dev/null +++ b/test/test_songbook/onthefly/content.onthefly.yaml @@ -0,0 +1,19 @@ +book: + pictures: yes + datadir: content_datadir + lang: en + template: default.tex +chords: + repeatchords: no + diagramreminder: all + +content: + - section: Test of section + - sort: + - songsection: Test of song section + - cwd: + # relative to datadir 'song' dir + path: ../content + content: + - tex: foo.tex + - include: include.sbc diff --git a/test/test_songbook/test_compilation.py b/test/test_songbook/test_compilation.py index 09a3e382..0439becd 100644 --- a/test/test_songbook/test_compilation.py +++ b/test/test_songbook/test_compilation.py @@ -9,8 +9,14 @@ import sys import subprocess import unittest -from patacrep.files import path2posix +import yaml +from patacrep.files import path2posix, chdir +from patacrep.songbook import prepare_songbook +from patacrep.build import SongbookBuilder +from patacrep import __DATADIR__ + +from .. import logging_reduced from .. import dynamic # pylint: disable=unused-import LOGGER = logging.getLogger(__name__) @@ -36,7 +42,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): os.path.dirname(__file__), '*.yaml', ))): - base = songbook[:-len(".yaml")] + base = os.path.splitext(songbook)[0] yield ( "test_latex_generation_{}".format(os.path.basename(base)), cls._create_generation_test(base), @@ -45,9 +51,23 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): "test_pdf_compilation_{}".format(os.path.basename(base)), cls._create_compilation_test(base), ) + for songbook in sorted(glob.glob(os.path.join( + os.path.dirname(__file__), + 'onthefly', + '*.yaml', + ))): + base = os.path.splitext(songbook)[0] + yield ( + "test_latex_generation_onthefly_{}".format(os.path.basename(base)), + cls._create_generation_test(base, True), + ) + yield ( + "test_pdf_compilation_onthefly_{}".format(os.path.basename(base)), + cls._create_compilation_test(base, True), + ) @classmethod - def _create_generation_test(cls, base): + def _create_generation_test(cls, base, onthefly=False): """Return a function testing that `base.tex` is correctly generated.""" def test_generation(self): @@ -55,7 +75,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): songbook = "{}.yaml".format(base) # Check tex generation - self.assertEqual(0, self.compile_songbook(songbook, "tex")) + if onthefly: + self.compile_songbook_onthefly(base, ['tex']) + else: + self.assertEqual(0, self.compile_songbook(songbook, "tex")) # Check generated tex control = "{}.tex.control".format(base) @@ -70,6 +93,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): "@TEST_FOLDER@", path2posix(os.path.dirname(__file__)), ) + expected = expected.replace( + "@LOCAL_DATA_FOLDER@", + path2posix(__DATADIR__), + ) expected = expected.replace( "@DATA_FOLDER@", @@ -93,7 +120,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): return test_generation @classmethod - def _create_compilation_test(cls, base): + def _create_compilation_test(cls, base, onthefly=False): """Return a function testing that `base.tex` is correctly compiled.""" @unittest.skipIf('TRAVIS' in os.environ, "Travis does not support lualatex compilation yet") @@ -101,7 +128,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): """Test that `base` is rendered to pdf.""" # Check compilation songbook = "{}.yaml".format(base) - self.assertEqual(0, self.compile_songbook(songbook)) + if onthefly: + self.compile_songbook_onthefly(base) + else: + self.assertEqual(0, self.compile_songbook(songbook)) test_compilation.__doc__ = ( "Test that '{base}' is correctly compiled." @@ -130,3 +160,28 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): except subprocess.CalledProcessError as error: LOGGER.warning(error.output) return error.returncode + + @staticmethod + def compile_songbook_onthefly(base, steps=None): + """Compile songbook "on the fly": without a physical songbook file.""" + + with open(base + ".yaml", mode="r", encoding="utf8") as sbfile: + sbyaml = yaml.load(sbfile) + + outputdir = os.path.dirname(base) + outputname = os.path.basename(base) + datadir_prefix = os.path.join(outputdir, '..') + songbook = prepare_songbook(sbyaml, outputdir, outputname, datadir_prefix=datadir_prefix) + songbook['_error'] = "fix" + songbook['_cache'] = True + + sb_builder = SongbookBuilder(songbook) + sb_builder.unsafe = True + + with chdir(outputdir): + # Continuous Integration will be verbose + if 'CI' in os.environ: + with logging_reduced(level=logging.DEBUG): + sb_builder.build_steps(steps) + else: + sb_builder.build_steps(steps)