diff --git a/NEWS.md b/NEWS.md index 1bb8b80a..1966a401 100644 --- a/NEWS.md +++ b/NEWS.md @@ -19,6 +19,9 @@ * LaTeX songs * The `meta` directive is now supported: `\metacrep{COMMANDNAME}{arg}` [#220](https://github.com/patacrep/patacrep/pull/220) * Faster index generation [#233](https://github.com/patacrep/patacrep/pull/233) + * Patatools + * New command to generate the list of the content items (songs, sections...): `patatools content items ` [#232](https://github.com/patacrep/patacrep/pull/232) + # patacrep 5.0.0 diff --git a/patacrep/build.py b/patacrep/build.py index 15a4ca78..87bb7a7f 100644 --- a/patacrep/build.py +++ b/patacrep/build.py @@ -53,6 +53,19 @@ class Songbook: self._errors = list() self._config = dict() + def get_content_items(self): + """Return: a list of ContentItem objects, corresponding to the content to be + included in the .tex file. + """ + content_config = self._raw_config.copy() + # Updates the '_langs' key + content_items = content.process_content( + content_config.get('content', []), + content_config, + ) + content_langs = content_config['_langs'] + return content_langs, content_items + def write_tex(self, output): """Build the '.tex' file corresponding to self. @@ -60,44 +73,41 @@ class Songbook: - output: a file object, in which the file will be written. """ # Updating configuration - self._config = self._raw_config.copy() + tex_config = self._raw_config.copy() renderer = TexBookRenderer( - self._config['book']['template'], - self._config['_datadir'], - self._config['book']['lang'], - self._config['book']['encoding'], + tex_config['book']['template'], + tex_config['_datadir'], + tex_config['book']['lang'], + tex_config['book']['encoding'], ) try: - self._config['_template'] = renderer.get_all_variables(self._config.get('template', {})) + tex_config['_template'] = renderer.get_all_variables(tex_config.get('template', {})) except errors.SchemaError as exception: exception.message = "The songbook file '{}' is not valid\n{}".format( self.basename, exception.message) raise exception - self._config['_compiled_authwords'] = authors.compile_authwords( - copy.deepcopy(self._config['authors']) + tex_config['_compiled_authwords'] = authors.compile_authwords( + copy.deepcopy(tex_config['authors']) ) # Configuration set - self._config['render'] = content.render - self._config['content'] = content.process_content( - self._config.get('content', []), - self._config, - ) - self._config['filename'] = output.name[:-4] + tex_config['render'] = content.render + tex_config['_langs'], tex_config['content'] = self.get_content_items() + tex_config['filename'] = output.name[:-4] # Processing special options - self._config['_bookoptions'] = iter_bookoptions(self._config) - self._config['chords']['_notenames'] = self._get_chord_names( - self._config['chords']['notation'] + tex_config['_bookoptions'] = iter_bookoptions(tex_config) + tex_config['chords']['_notenames'] = self._get_chord_names( + tex_config['chords']['notation'] ) - renderer.render_tex(output, self._config) + renderer.render_tex(output, tex_config) # Get all errors, and maybe exit program self._errors.extend(renderer.errors) - if self._config['_error'] == "failonbook": + if tex_config['_error'] == "failonbook": if self.has_errors(): raise errors.SongbookError("Some songs contain errors. Stopping as requested.") diff --git a/patacrep/content/__init__.py b/patacrep/content/__init__.py index 5acc2aae..12e9bed9 100755 --- a/patacrep/content/__init__.py +++ b/patacrep/content/__init__.py @@ -116,6 +116,10 @@ class ContentItem: """Return the string to end a block.""" return "" + def to_dict(self): + """Return the dict representation (as in the yaml file).""" + raise NotImplementedError() + class ContentError(SharedError): """Error in a content plugin.""" def __init__(self, keyword=None, message=None): diff --git a/patacrep/content/section.py b/patacrep/content/section.py index 316b0fa3..78776d5c 100755 --- a/patacrep/content/section.py +++ b/patacrep/content/section.py @@ -28,6 +28,12 @@ class Section(ContentItem): else: return r'\{}[{}]{{{}}}'.format(self.keyword, self.short, self.name) + def to_dict(self): + if self.short is None or self.keyword not in KEYWORDS: + return {self.keyword: self.name} + else: + return {self.keyword: {'name': self.name, 'short': self.short}} + #pylint: disable=unused-argument @validate_parser_argument(""" type: //any diff --git a/patacrep/content/setcounter.py b/patacrep/content/setcounter.py index 41729b52..400ce26b 100755 --- a/patacrep/content/setcounter.py +++ b/patacrep/content/setcounter.py @@ -14,6 +14,9 @@ class CounterSetter(ContentItem): """Set the value of the counter.""" return r'\setcounter{{{}}}{{{}}}'.format(self.name, self.value) + def to_dict(self): + return {'setcounter': {'name': self.name, 'value': self.value}} + #pylint: disable=unused-argument @validate_parser_argument(""" type: //any diff --git a/patacrep/content/song.py b/patacrep/content/song.py index 3063a700..dc35b5ae 100755 --- a/patacrep/content/song.py +++ b/patacrep/content/song.py @@ -63,6 +63,9 @@ class SongRenderer(ContentItem): """Order by song path""" return self.song.fullpath < other.song.fullpath + def to_dict(self): + return {'song': self.song.fullpath} + #pylint: disable=unused-argument #pylint: disable=too-many-branches @validate_parser_argument(""" diff --git a/patacrep/content/songsection.py b/patacrep/content/songsection.py index abc9d609..132c0666 100755 --- a/patacrep/content/songsection.py +++ b/patacrep/content/songsection.py @@ -19,6 +19,9 @@ class SongSection(ContentItem): """Render this section or chapter.""" return r'\{}{{{}}}'.format(self.keyword, self.name) + def to_dict(self): + return {self.keyword: self.name} + #pylint: disable=unused-argument @validate_parser_argument(""" //str diff --git a/patacrep/content/tex.py b/patacrep/content/tex.py index b00f3539..ef588d04 100755 --- a/patacrep/content/tex.py +++ b/patacrep/content/tex.py @@ -20,6 +20,9 @@ class LaTeX(ContentItem): os.path.dirname(context['filename']), ))) + def to_dict(self): + return {'tex': self.filename} + #pylint: disable=unused-argument @validate_parser_argument(""" type: //any diff --git a/patacrep/songs/__init__.py b/patacrep/songs/__init__.py index c58aaae9..0afcf9cf 100644 --- a/patacrep/songs/__init__.py +++ b/patacrep/songs/__init__.py @@ -43,7 +43,7 @@ class DataSubpath: self.subpath = subpath def __str__(self): - return os.path.join(self.datadir, self.subpath) + return self.fullpath @property def fullpath(self): diff --git a/patacrep/tools/__init__.py b/patacrep/tools/__init__.py index e69de29b..90ae902b 100644 --- a/patacrep/tools/__init__.py +++ b/patacrep/tools/__init__.py @@ -0,0 +1,13 @@ +"""Common functions for patatools""" + +import argparse +import os + +def existing_file(name): + """Check that argument is an existing, readable file name. + + Return the argument for convenience. + """ + if os.path.isfile(name) and os.access(name, os.R_OK): + return name + raise argparse.ArgumentTypeError("Cannot read file '{}'.".format(name)) diff --git a/patacrep/tools/cache/__main__.py b/patacrep/tools/cache/__main__.py index f48de224..6faf7989 100644 --- a/patacrep/tools/cache/__main__.py +++ b/patacrep/tools/cache/__main__.py @@ -9,18 +9,10 @@ import textwrap from patacrep import errors from patacrep.songbook import open_songbook +from .. import existing_file LOGGER = logging.getLogger("patatools.cache") -def filename(name): - """Check that argument is an existing, readable file name. - - Return the argument for convenience. - """ - if os.path.isfile(name) and os.access(name, os.R_OK): - return name - raise argparse.ArgumentTypeError("Cannot read file '{}'.".format(name)) - def commandline_parser(): """Return a command line parser.""" @@ -30,10 +22,7 @@ def commandline_parser(): formatter_class=argparse.RawTextHelpFormatter, ) - subparsers = parser.add_subparsers( - description="", - dest="command", - ) + subparsers = parser.add_subparsers() subparsers.required = True clean = subparsers.add_parser( @@ -45,7 +34,7 @@ def commandline_parser(): 'songbook', metavar="SONGBOOK", help=textwrap.dedent("""Songbook file to be used to look for cache path."""), - type=filename, + type=existing_file, ) clean.set_defaults(command=do_clean) diff --git a/patacrep/tools/content/__init__.py b/patacrep/tools/content/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/patacrep/tools/content/__main__.py b/patacrep/tools/content/__main__.py new file mode 100644 index 00000000..0d64c480 --- /dev/null +++ b/patacrep/tools/content/__main__.py @@ -0,0 +1,70 @@ +"""Perform operations on songbook content.""" + +import argparse +import logging +import os +import sys +import textwrap +import yaml + +from patacrep.songbook import open_songbook +from patacrep.build import Songbook +from .. import existing_file + +LOGGER = logging.getLogger("patatools.content") + +def commandline_parser(): + """Return a command line parser.""" + + parser = argparse.ArgumentParser( + prog="patatools content", + description="Operations related to the content of a songbook.", + formatter_class=argparse.RawTextHelpFormatter, + ) + + subparsers = parser.add_subparsers() + subparsers.required = True + + content_items = subparsers.add_parser( + "items", + description="Display the content items of a songbook.", + help="Return the content items.", + ) + content_items.add_argument( + 'songbook', + metavar="SONGBOOK", + help=textwrap.dedent("""Songbook file to be used to look for content items."""), + type=existing_file, + ) + content_items.set_defaults(command=do_content_items) + + return parser + +def do_content_items(namespace): + """Execute the `patatools content items` command.""" + config = open_songbook(namespace.songbook) + config['_cache'] = True + config['_error'] = "fix" + songbook = Songbook(config, config['_outputname']) + _, content_items = songbook.get_content_items() + yaml_dir = os.path.dirname(os.path.abspath(namespace.songbook)) + ref_dir = os.path.join(yaml_dir, 'songs') + content_items = [ + normalize_song_path(item.to_dict(), ref_dir) + for item in content_items + ] + sys.stdout.write(yaml.safe_dump(content_items, allow_unicode=True, default_flow_style=False)) + +def normalize_song_path(file_entry, ref_dir): + """Normalize the 'song' value, relative to ref_dir""" + if 'song' in file_entry: + file_entry['song'] = os.path.relpath(file_entry['song'], ref_dir) + return file_entry + +def main(args): + """Main function: run from command line.""" + options = commandline_parser().parse_args(args[1:]) + options.command(options) + +if __name__ == "__main__": + main(sys.argv) diff --git a/test/test_content/custom.control b/test/test_content/custom.control index b549c6a3..52e29862 100644 --- a/test/test_content/custom.control +++ b/test/test_content/custom.control @@ -1 +1 @@ -- "{'customname:': ''}" \ No newline at end of file +- customname: '' \ No newline at end of file diff --git a/test/test_content/custom.source b/test/test_content/custom.source index ded39db8..52e29862 100644 --- a/test/test_content/custom.source +++ b/test/test_content/custom.source @@ -1 +1 @@ -- customname: \ No newline at end of file +- customname: '' \ No newline at end of file diff --git a/test/test_content/customzipped.control b/test/test_content/customzipped.control index 6d3fd533..8a6bbc0d 100644 --- a/test/test_content/customzipped.control +++ b/test/test_content/customzipped.control @@ -1 +1 @@ -- "{'customzippedname:': ''}" \ No newline at end of file +- customzippedname: '' diff --git a/test/test_content/customzipped.source b/test/test_content/customzipped.source index 7c67530d..8a6bbc0d 100644 --- a/test/test_content/customzipped.source +++ b/test/test_content/customzipped.source @@ -1 +1 @@ -- customzippedname: +- customzippedname: '' diff --git a/test/test_content/cwd.control b/test/test_content/cwd.control index e1ed6693..9ea8e105 100644 --- a/test/test_content/cwd.control +++ b/test/test_content/cwd.control @@ -1 +1 @@ -- subdir/chordpro.csg \ No newline at end of file +- song: datadir/songs/subdir/chordpro.csg \ No newline at end of file diff --git a/test/test_content/cwd_list.control b/test/test_content/cwd_list.control index 3c9377a4..c52db118 100644 --- a/test/test_content/cwd_list.control +++ b/test/test_content/cwd_list.control @@ -1,2 +1,2 @@ -- subdir/chordpro.csg -- exsong.sg \ No newline at end of file +- song: datadir/songs/subdir/chordpro.csg +- song: datadir/songs/exsong.sg \ No newline at end of file diff --git a/test/test_content/datadir/python/content/customplugin.py b/test/test_content/datadir/python/content/customplugin.py index e7333b30..b835459c 100755 --- a/test/test_content/datadir/python/content/customplugin.py +++ b/test/test_content/datadir/python/content/customplugin.py @@ -5,8 +5,8 @@ from patacrep.content import ContentItem, ContentList, validate_parser_argument class FakeContent(ContentItem): """Fake content.""" - def file_entry(self): - return {'customname:':''} + def to_dict(self): + return {'customname':''} def parse(keyword, argument, config): return ContentList([FakeContent()]) diff --git a/test/test_content/datadir_zippedcontent/python/content b/test/test_content/datadir_zippedcontent/python/content index a333d318..2c5e1ee5 100644 Binary files a/test/test_content/datadir_zippedcontent/python/content and b/test/test_content/datadir_zippedcontent/python/content differ diff --git a/test/test_content/glob.control b/test/test_content/glob.control index 177bc945..2cb022f2 100644 --- a/test/test_content/glob.control +++ b/test/test_content/glob.control @@ -1 +1 @@ -- chordpro.csg \ No newline at end of file +- song: datadir/songs/chordpro.csg \ No newline at end of file diff --git a/test/test_content/include.control b/test/test_content/include.control index 6aba97ae..4b39aced 100644 --- a/test/test_content/include.control +++ b/test/test_content/include.control @@ -1,6 +1,6 @@ -- exsong.sg -- chordpro.csg -- subdir/chordpro.csg -- chordpro.csg -- subdir/chordpro.csg -- exsong.sg \ No newline at end of file +- song: datadir/songs/exsong.sg +- song: datadir/songs/chordpro.csg +- song: datadir/songs/subdir/chordpro.csg +- song: datadir/songs/chordpro.csg +- song: datadir/songs/subdir/chordpro.csg +- song: datadir/songs/exsong.sg \ No newline at end of file diff --git a/test/test_content/sections.control b/test/test_content/sections.control index 5a87c67c..f96345f2 100644 --- a/test/test_content/sections.control +++ b/test/test_content/sections.control @@ -1,11 +1,13 @@ -- section{First Section!} -- section{Named section} -- section[section_short_name]{Section with short name} -- section*{Section* with short name} -- part{part section test} -- chapter{chapter section test} -- section{section section test} -- subsection{subsection section test} -- subsubsection{subsubsection section test} -- paragraph{paragraph section test} -- subparagraph{subparagraph section test} \ No newline at end of file +- section: First Section! +- section: Named section +- section: + name: Section with short name + short: section_short_name +- section*: Section* with short name +- part: part section test +- chapter: chapter section test +- section: section section test +- subsection: subsection section test +- subsubsection: subsubsection section test +- paragraph: paragraph section test +- subparagraph: subparagraph section test \ No newline at end of file diff --git a/test/test_content/setcounter.control b/test/test_content/setcounter.control index dff14be1..453c67e4 100644 --- a/test/test_content/setcounter.control +++ b/test/test_content/setcounter.control @@ -1,4 +1,12 @@ -- setcounter{songnum}{101} -- setcounter{songnum}{1} -- setcounter{songnum}{5} -- setcounter{counter_name}{-1} \ No newline at end of file +- setcounter: + name: songnum + value: 101 +- setcounter: + name: songnum + value: 1 +- setcounter: + name: songnum + value: 5 +- setcounter: + name: counter_name + value: -1 \ No newline at end of file diff --git a/test/test_content/songs.control b/test/test_content/songs.control index 6ad0622a..61723113 100644 --- a/test/test_content/songs.control +++ b/test/test_content/songs.control @@ -1,4 +1,4 @@ -- exsong.sg -- texsong.tsg -- chordpro.csg -- subdir/chordpro.csg \ No newline at end of file +- song: datadir/songs/exsong.sg +- song: datadir/songs/texsong.tsg +- song: datadir/songs/chordpro.csg +- song: datadir/songs/subdir/chordpro.csg \ No newline at end of file diff --git a/test/test_content/songsection.control b/test/test_content/songsection.control index 81197f08..4bbd48f8 100644 --- a/test/test_content/songsection.control +++ b/test/test_content/songsection.control @@ -1,6 +1,6 @@ -- songsection{Traditional} -- exsong.sg -- songchapter{English} -- texsong.tsg -- chordpro.csg -- exsong.sg \ No newline at end of file +- songsection: Traditional +- song: datadir/songs/exsong.sg +- songchapter: English +- song: datadir/songs/texsong.tsg +- song: datadir/songs/chordpro.csg +- song: datadir/songs/exsong.sg \ No newline at end of file diff --git a/test/test_content/sort.control b/test/test_content/sort.control index a7430273..4778a6e5 100644 --- a/test/test_content/sort.control +++ b/test/test_content/sort.control @@ -1,27 +1,27 @@ -- section{Title} -- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg" -- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg" -- section{Author, Title} -- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg" -- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg" -- section{Path, Title} -- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg" -- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg" -- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg" +- section: Title +- song: datadir_sort/path1_title1_author1.csg +- song: datadir_sort/path1_title1_author2.csg +- song: datadir_sort/path2_title1_author1.csg +- song: datadir_sort/path2_title1_author2.csg +- song: datadir_sort/path1_title2_author1.csg +- song: datadir_sort/path1_title2_author2.csg +- song: datadir_sort/path2_title2_author1.csg +- song: datadir_sort/path2_title2_author2.csg +- section: Author, Title +- song: datadir_sort/path1_title1_author1.csg +- song: datadir_sort/path2_title1_author1.csg +- song: datadir_sort/path1_title2_author1.csg +- song: datadir_sort/path2_title2_author1.csg +- song: datadir_sort/path1_title1_author2.csg +- song: datadir_sort/path2_title1_author2.csg +- song: datadir_sort/path1_title2_author2.csg +- song: datadir_sort/path2_title2_author2.csg +- section: Path, Title +- song: datadir_sort/path1_title1_author1.csg +- song: datadir_sort/path1_title1_author2.csg +- song: datadir_sort/path1_title2_author1.csg +- song: datadir_sort/path1_title2_author2.csg +- song: datadir_sort/path2_title1_author1.csg +- song: datadir_sort/path2_title1_author2.csg +- song: datadir_sort/path2_title2_author1.csg +- song: datadir_sort/path2_title2_author2.csg diff --git a/test/test_content/test_content.py b/test/test_content/test_content.py index 1ca8c67d..eb405c90 100644 --- a/test/test_content/test_content.py +++ b/test/test_content/test_content.py @@ -10,7 +10,6 @@ import yaml from pkg_resources import resource_filename from patacrep import content, files -from patacrep.content import song, section, setcounter, songsection, tex from patacrep.songbook import prepare_songbook from .. import logging_reduced @@ -55,16 +54,13 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): with logging_reduced('patacrep.content.song'): expandedlist = content.process_content(sbcontent, config) - sourcelist = [cls._clean_path(elem) for elem in expandedlist] + sourcelist = [cls._clean_path(elem.to_dict()) for elem in expandedlist] controlname = "{}.control".format(base) if not os.path.exists(controlname): - raise Exception("Missing control:" + str(sourcelist).replace("'", '"')) + raise Exception("Missing control:" + str(controlname).replace("'", '"')) with open(controlname, mode="r", encoding="utf8") as controlfile: - controllist = [ - elem.replace("@TEST_FOLDER@", files.path2posix(resource_filename(__name__, ""))) - for elem in yaml.load(controlfile) - ] + controllist = yaml.load(controlfile) self.assertEqual(controllist, sourcelist) @@ -75,25 +71,17 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): @classmethod def _clean_path(cls, elem): - """Shorten the path relative to the `songs` directory""" - - latex_command_classes = ( - section.Section, - songsection.SongSection, - setcounter.CounterSetter, - ) - if isinstance(elem, latex_command_classes): - return elem.render(None)[1:] - - elif isinstance(elem, song.SongRenderer): - songpath = os.path.join(os.path.dirname(__file__), 'datadir', 'songs') - return files.path2posix(files.relpath(elem.song.fullpath, songpath)) - - elif isinstance(elem, tex.LaTeX): - return files.path2posix(elem.filename) - - else: - return str(elem.file_entry()) + """Shorten the path relative to the test directory""" + if not isinstance(elem, dict): + return elem + + test_path = files.path2posix(resource_filename(__name__, ""))+"/" + for key in ['song', 'tex']: + if key in elem: + elem[key] = files.path2posix( + os.path.normpath(elem[key]) + ).replace(test_path, "") + return elem @classmethod def _generate_config(cls, sbcontent, outputdir, base): diff --git a/test/test_content/tex.control b/test/test_content/tex.control index 47d8fe2c..6a9a4749 100644 --- a/test/test_content/tex.control +++ b/test/test_content/tex.control @@ -1,3 +1,3 @@ -- test/test_content/datadir/songs/texfile.tex -- test/test_content/datadir/songs/texfile.tex -- test/test_content/datadir/songs/texfile.tex \ No newline at end of file +- tex: test/test_content/datadir/songs/texfile.tex +- tex: test/test_content/datadir/songs/texfile.tex +- tex: test/test_content/datadir/songs/texfile.tex \ No newline at end of file