Browse Source

Merge pull request #203 from patacrep/yaml_ext

Support any songbook file extension and refactorization
pull/193/head
Louis 9 years ago
parent
commit
3452fd222a
  1. 6
      patacrep/build.py
  2. 13
      patacrep/content/cwd.py
  3. 2
      patacrep/content/song.py
  4. 5
      patacrep/data/templates/songbook_model.yml
  5. 50
      patacrep/songbook/__init__.py
  6. 8
      patacrep/songs/__init__.py
  7. 37
      test/test_content/test_content.py
  8. 14
      test/test_patatools/test_cache.py
  9. 2
      test/test_songbook/.gitignore
  10. 133
      test/test_songbook/onthefly/content.onthefly.tex.control
  11. 19
      test/test_songbook/onthefly/content.onthefly.yaml
  12. 67
      test/test_songbook/test_compilation.py

6
patacrep/build.py

@ -169,7 +169,7 @@ class SongbookBuilder:
def __init__(self, raw_songbook):
# Basename of the songbook to be built.
self.basename = raw_songbook['_basename']
self.basename = raw_songbook['_outputname']
# Representation of the .yaml songbook configuration file.
self.songbook = Songbook(raw_songbook, self.basename)
@ -300,6 +300,10 @@ class SongbookBuilder:
standard_output.join()
process.wait()
# Close the stdout and stderr to prevent ResourceWarning:
process.stdout.close()
process.stderr.close()
if process.returncode:
raise errors.LatexCompilationError(self.basename)

13
patacrep/content/cwd.py

@ -1,7 +1,5 @@
"""Change base directory before importing songs."""
import os
from patacrep.content import process_content, validate_parser_argument
from patacrep.songs import DataSubpath
@ -30,16 +28,15 @@ def parse(keyword, config, argument):
The 'path' is added:
- first as a relative path to the *.yaml file directory;
- then as a relative path to every path already present in
config['songdir'].
config['songdir'] (which are 'song' dir inside the datadirs).
"""
subpath = argument['path']
old_songdir = config['_songdir']
sbdir = os.path.dirname(config['_filepath'])
config['_songdir'] = (
[DataSubpath(sbdir, subpath)] +
[path.clone().join(subpath) for path in config['_songdir']]
)
config['_songdir'] = [path.clone().join(subpath) for path in config['_songdir']]
if '_songbookfile_dir' in config:
config['_songdir'].insert(0, DataSubpath(config['_songbookfile_dir'], subpath))
processed_content = process_content(argument.get('content'), config)
config['_songdir'] = old_songdir
return processed_content

2
patacrep/content/song.py

@ -95,6 +95,8 @@ def parse(keyword, argument, config):
if contentlist:
break
contentlist = files.recursive_find(songdir.fullpath, plugins.keys())
if contentlist is None:
contentlist = [] # No content was set or found
for elem in contentlist:
before = len(songlist)
for songdir in config['_songdir']:

5
patacrep/data/templates/songbook_model.yml

@ -3,10 +3,11 @@ schema:
optional:
content: //any
template: //any
_songbookfile_dir: //str
required:
_cache: //bool
_filepath: //str
_basename: //str
_outputdir: //str
_outputname: //str
_error: //str
_datadir:
type: //arr

50
patacrep/songbook/__init__.py

@ -14,7 +14,8 @@ import patacrep
LOGGER = logging.getLogger()
def open_songbook(filename):
"""Open songbook, and return a raw songbook object.
"""Open a songbook file, and prepare it to
return a raw songbook object.
:param str filename: Filename of the yaml songbook.
:rvalue: dict
@ -35,13 +36,35 @@ def open_songbook(filename):
except Exception as error: # pylint: disable=broad-except
raise patacrep.errors.SongbookError(str(error))
songbook = _add_songbook_defaults(user_songbook)
songbookfile_dir = os.path.dirname(os.path.abspath(filename))
# Output at the same place as the songbook file
outputdir = songbookfile_dir
outputname = os.path.splitext(os.path.basename(filename))[0]
songbook['_filepath'] = filename
songbook['_basename'] = os.path.basename(filename)[:-len(".yaml")]
return prepare_songbook(user_songbook, outputdir, outputname, songbookfile_dir)
def prepare_songbook(songbook, outputdir, outputname, songbookfile_dir=None, datadir_prefix=None):
"""Prepare a songbook by adding default values and datadirs
Returns a raw songbook object.
:param dict songbook: Initial yaml songbook.
:param str outputdir: Folder to put the output (tex, pdf...)
:param str outputname: Filename for the outputs (tex, pdf...)
:param str songbookfile_dir: Folder of the original songbook file (if there is one)
:param str datadir_prefix: Prefix for the datadirs
:rvalue: dict
:return: Songbook, as a dictionary.
"""
songbook['_outputdir'] = outputdir
songbook['_outputname'] = outputname
if songbookfile_dir:
songbook['_songbookfile_dir'] = songbookfile_dir
songbook = _add_songbook_defaults(songbook)
# Gathering datadirs
songbook['_datadir'] = list(_iter_absolute_datadirs(songbook))
songbook['_datadir'] = list(_iter_absolute_datadirs(songbook, datadir_prefix))
if 'datadir' in songbook['book']:
del songbook['book']['datadir']
@ -50,7 +73,6 @@ def open_songbook(filename):
for path in songbook['_datadir']
]
return songbook
def _add_songbook_defaults(user_songbook):
@ -78,26 +100,32 @@ def _add_songbook_defaults(user_songbook):
return dict(default_songbook)
def _iter_absolute_datadirs(raw_songbook):
def _iter_absolute_datadirs(raw_songbook, datadir_prefix=None):
"""Iterate on the absolute datadirs of the raw songbook
Appends the songfile dir at the end
"""
songbookfile_dir = raw_songbook.get('_songbookfile_dir')
if datadir_prefix is None:
if songbookfile_dir is None:
raise patacrep.errors.SongbookError('Please specify where the datadir are located')
datadir_prefix = songbookfile_dir
datadir = raw_songbook.get('book', {}).get('datadir')
filepath = raw_songbook['_filepath']
if datadir is None:
datadir = []
elif isinstance(datadir, str):
datadir = [datadir]
basedir = os.path.dirname(os.path.abspath(filepath))
for path in datadir:
abspath = os.path.join(basedir, path)
abspath = os.path.join(datadir_prefix, path)
if os.path.exists(abspath) and os.path.isdir(abspath):
yield abspath
else:
LOGGER.warning(
"Ignoring non-existent datadir '{}'.".format(path)
)
yield basedir
if songbookfile_dir:
yield songbookfile_dir

8
patacrep/songs/__init__.py

@ -152,16 +152,16 @@ class Song:
def filehash(self):
"""Compute (and cache) the md5 hash of the file"""
if self._filehash is None:
self._filehash = hashlib.md5(
open(self.fullpath, 'rb').read()
).hexdigest()
with open(self.fullpath, 'rb') as songfile:
self._filehash = hashlib.md5(songfile.read()).hexdigest()
return self._filehash
def _cache_retrieved(self):
"""If relevant, retrieve self from the cache."""
if self.use_cache and os.path.exists(self.cached_name):
try:
cached = pickle.load(open(self.cached_name, 'rb',))
with open(self.cached_name, 'rb',) as cachefile:
cached = pickle.load(cachefile)
if (
cached['_filehash'] == self.filehash
and cached['_version'] == self.CACHE_VERSION

37
test/test_content/test_content.py

@ -9,10 +9,9 @@ import yaml
from pkg_resources import resource_filename
from patacrep.songs import DataSubpath
from patacrep import content, files
from patacrep.content import song, section, songsection, tex
from patacrep.build import config_model
from patacrep.songbook import prepare_songbook
from .. import logging_reduced
from .. import dynamic # pylint: disable=unused-import
@ -28,10 +27,6 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
maxDiff = None
config = None
@classmethod
def setUpClass(cls):
cls._generate_config()
@classmethod
def _iter_testmethods(cls):
"""Iterate over dynamically generated test methods"""
@ -55,8 +50,9 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
with open(sourcename, mode="r", encoding="utf8") as sourcefile:
sbcontent = yaml.load(sourcefile)
config = cls.config.copy()
config['_filepath'] = base
outputdir = os.path.dirname(base)
config = cls._generate_config(sbcontent, outputdir, base)
with logging_reduced('patacrep.content.song'):
expandedlist = content.process_content(sbcontent, config)
sourcelist = [cls._clean_path(elem) for elem in expandedlist]
@ -97,28 +93,27 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
raise Exception(elem)
@classmethod
def _generate_config(cls):
def _generate_config(cls, sbcontent, outputdir, base):
"""Generate the config to process the content"""
# Load the default songbook config
config = config_model('default')['en']
datadirpaths = [os.path.join(os.path.dirname(__file__), 'datadir')]
config['_datadir'] = datadirpaths
config = prepare_songbook(
{'book':{'datadir':'datadir'}, 'content': sbcontent},
outputdir,
base,
outputdir
)
config['_songdir'] = [
DataSubpath(path, 'songs')
for path in datadirpaths
]
# Load the plugins
config['_content_plugins'] = files.load_plugins(
datadirs=datadirpaths,
datadirs=config['_datadir'],
root_modules=['content'],
keyword='CONTENT_PLUGINS',
)
config['_song_plugins'] = files.load_plugins(
datadirs=datadirpaths,
datadirs=config['_datadir'],
root_modules=['songs'],
keyword='SONG_RENDERERS',
)['tsg']
cls.config = config
return config

14
test/test_patatools/test_cache.py

@ -11,6 +11,8 @@ from patacrep.tools.__main__ import main as tools_main
from patacrep.tools.cache.__main__ import main as cache_main
from patacrep.songbook.__main__ import main as songbook_main
from .. import logging_reduced
CACHEDIR = os.path.join(os.path.dirname(__file__), "test_cache_datadir", ".cache")
class TestCache(unittest.TestCase):
@ -46,11 +48,16 @@ class TestCache(unittest.TestCase):
]:
with self.subTest(main=main, args=args):
# First compilation. Ensure that cache exists afterwards
self._system(songbook_main, ["songbook", "--steps", "tex,clean", "test_cache.yaml"])
with logging_reduced('patacrep.build'):
self._system(
songbook_main,
["songbook", "--steps", "tex,clean", "test_cache.yaml"]
)
self.assertTrue(os.path.exists(CACHEDIR))
# Clean cache
self._system(main, args)
with logging_reduced('patatools.cache'):
self._system(main, args)
# Ensure that cache does not exist
self.assertFalse(os.path.exists(CACHEDIR))
@ -64,4 +71,5 @@ class TestCache(unittest.TestCase):
]:
with self.subTest(main=main, args=args):
# Clean cache
self._system(main, args)
with logging_reduced('patatools.cache'):
self._system(main, args)

2
test/test_songbook/.gitignore

@ -1,2 +1,2 @@
/*tex
**.tex
.cache

133
test/test_songbook/onthefly/content.onthefly.tex.control

@ -0,0 +1,133 @@
%% Automatically generated document.
%% You may edit this file but all changes will be overwritten.
%% If you want to change this document, have a look at
%% the templating system.
%%
%% Generated using Songbook <http://www.patacrep.com>
\makeatletter
\def\input@path{ %
{@TEST_FOLDER@/onthefly/../content_datadir/latex/} %
{@LOCAL_DATA_FOLDER@/latex/} %
}
\makeatother
\documentclass[
]{article}
\usepackage[
chorded,
pictures,
diagram,
guitar,
]{patacrep}
\usepackage{lmodern}
\PassOptionsToPackage{english}{babel}
\PassOptionsToPackage{english}{babel}
\usepackage[english]{babel}
\lang{english}
\usepackage{graphicx}
\graphicspath{ %
{@TEST_FOLDER@/onthefly/../content_datadir/} %
{@LOCAL_DATA_FOLDER@/} %
}
\makeatletter
\@ifpackageloaded{hyperref}{}{
\usepackage{url}
\newcommand{\phantomsection}{}
\newcommand{\hyperlink}[2]{#2}
\newcommand{\href}[2]{\expandafter\url\expandafter{#1}}
}
\makeatother
\usepackage{chords}
\title{Guitar songbook}
\author{The Patacrep Team}
\newindex{titleidx}{content.onthefly_title}
\newauthorindex{authidx}{content.onthefly_auth}
\authignoreword{unknown}
\authbyword{by}
\authsepword{and}
\notenamesout{A}{B}{C}{D}{E}{F}{G}
\begin{document}
\maketitle
\showindex{\songindexname}{titleidx}
\showindex{\authorindexname}{authidx}
% list of chords
\ifchorded
\ifdiagram
\phantomsection
\addcontentsline{toc}{section}{\chordlistname}
\chords
\fi
\fi
\phantomsection
\addcontentsline{toc}{section}{\songlistname}
\section{Test of section}
\begin{songs}{titleidx,authidx}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% songs/./song.csg
\selectlanguage{english}
\beginsong{This is a song}[
by={
},
]
\begin{verse}
Foo
\end{verse}
\endsong
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% songs/./song.tsg
\import{@TEST_FOLDER@/content_datadir/songs/}{song.tsg}
\end{songs}
\songsection{Test of song section}
\input{@TEST_FOLDER@/content_datadir/content/foo.tex}
\section{This is an included section}
\end{document}

19
test/test_songbook/onthefly/content.onthefly.yaml

@ -0,0 +1,19 @@
book:
pictures: yes
datadir: content_datadir
lang: en
template: default.tex
chords:
repeatchords: no
diagramreminder: all
content:
- section: Test of section
- sort:
- songsection: Test of song section
- cwd:
# relative to datadir 'song' dir
path: ../content
content:
- tex: foo.tex
- include: include.sbc

67
test/test_songbook/test_compilation.py

@ -9,8 +9,14 @@ import sys
import subprocess
import unittest
from patacrep.files import path2posix
import yaml
from patacrep.files import path2posix, chdir
from patacrep.songbook import prepare_songbook
from patacrep.build import SongbookBuilder
from patacrep import __DATADIR__
from .. import logging_reduced
from .. import dynamic # pylint: disable=unused-import
LOGGER = logging.getLogger(__name__)
@ -36,7 +42,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
os.path.dirname(__file__),
'*.yaml',
))):
base = songbook[:-len(".yaml")]
base = os.path.splitext(songbook)[0]
yield (
"test_latex_generation_{}".format(os.path.basename(base)),
cls._create_generation_test(base),
@ -45,9 +51,23 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
"test_pdf_compilation_{}".format(os.path.basename(base)),
cls._create_compilation_test(base),
)
for songbook in sorted(glob.glob(os.path.join(
os.path.dirname(__file__),
'onthefly',
'*.yaml',
))):
base = os.path.splitext(songbook)[0]
yield (
"test_latex_generation_onthefly_{}".format(os.path.basename(base)),
cls._create_generation_test(base, True),
)
yield (
"test_pdf_compilation_onthefly_{}".format(os.path.basename(base)),
cls._create_compilation_test(base, True),
)
@classmethod
def _create_generation_test(cls, base):
def _create_generation_test(cls, base, onthefly=False):
"""Return a function testing that `base.tex` is correctly generated."""
def test_generation(self):
@ -55,7 +75,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
songbook = "{}.yaml".format(base)
# Check tex generation
self.assertEqual(0, self.compile_songbook(songbook, "tex"))
if onthefly:
self.compile_songbook_onthefly(base, ['tex'])
else:
self.assertEqual(0, self.compile_songbook(songbook, "tex"))
# Check generated tex
control = "{}.tex.control".format(base)
@ -70,6 +93,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
"@TEST_FOLDER@",
path2posix(os.path.dirname(__file__)),
)
expected = expected.replace(
"@LOCAL_DATA_FOLDER@",
path2posix(__DATADIR__),
)
expected = expected.replace(
"@DATA_FOLDER@",
@ -93,7 +120,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
return test_generation
@classmethod
def _create_compilation_test(cls, base):
def _create_compilation_test(cls, base, onthefly=False):
"""Return a function testing that `base.tex` is correctly compiled."""
@unittest.skipIf('TRAVIS' in os.environ,
"Travis does not support lualatex compilation yet")
@ -101,7 +128,10 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
"""Test that `base` is rendered to pdf."""
# Check compilation
songbook = "{}.yaml".format(base)
self.assertEqual(0, self.compile_songbook(songbook))
if onthefly:
self.compile_songbook_onthefly(base)
else:
self.assertEqual(0, self.compile_songbook(songbook))
test_compilation.__doc__ = (
"Test that '{base}' is correctly compiled."
@ -130,3 +160,28 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
except subprocess.CalledProcessError as error:
LOGGER.warning(error.output)
return error.returncode
@staticmethod
def compile_songbook_onthefly(base, steps=None):
"""Compile songbook "on the fly": without a physical songbook file."""
with open(base + ".yaml", mode="r", encoding="utf8") as sbfile:
sbyaml = yaml.load(sbfile)
outputdir = os.path.dirname(base)
outputname = os.path.basename(base)
datadir_prefix = os.path.join(outputdir, '..')
songbook = prepare_songbook(sbyaml, outputdir, outputname, datadir_prefix=datadir_prefix)
songbook['_error'] = "fix"
songbook['_cache'] = True
sb_builder = SongbookBuilder(songbook)
sb_builder.unsafe = True
with chdir(outputdir):
# Continuous Integration will be verbose
if 'CI' in os.environ:
with logging_reduced(level=logging.DEBUG):
sb_builder.build_steps(steps)
else:
sb_builder.build_steps(steps)

Loading…
Cancel
Save