Merge pull request #232 from patacrep/sb_get_content

Retrieve processed songbook content

NEWS.md

@@ -19,6 +19,9 @@
   * LaTeX songs
     * The `meta` directive is now supported: `\metacrep{COMMANDNAME}{arg}` [#220](https://github.com/patacrep/patacrep/pull/220)
   * Faster index generation [#233](https://github.com/patacrep/patacrep/pull/233)
+  * Patatools
+    * New command to generate the list of the content items (songs, sections...): `patatools content items <songbook>` [#232](https://github.com/patacrep/patacrep/pull/232)
+
 
 # patacrep 5.0.0
 

patacrep/build.py

@@ -53,6 +53,19 @@ class Songbook:
         self._errors = list()
         self._config = dict()
 
+    def get_content_items(self):
+        """Return: a list of ContentItem objects, corresponding to the content to be
+        included in the .tex file.
+        """
+        content_config = self._raw_config.copy()
+        # Updates the '_langs' key
+        content_items = content.process_content(
+            content_config.get('content', []),
+            content_config,
+            )
+        content_langs = content_config['_langs']
+        return content_langs, content_items
+
     def write_tex(self, output):
         """Build the '.tex' file corresponding to self.
 
@@ -60,44 +73,41 @@ class Songbook:
         - output: a file object, in which the file will be written.
         """
         # Updating configuration
-        self._config = self._raw_config.copy()
+        tex_config = self._raw_config.copy()
         renderer = TexBookRenderer(
-            self._config['book']['template'],
-            self._config['_datadir'],
-            self._config['book']['lang'],
-            self._config['book']['encoding'],
+            tex_config['book']['template'],
+            tex_config['_datadir'],
+            tex_config['book']['lang'],
+            tex_config['book']['encoding'],
             )
 
         try:
-            self._config['_template'] = renderer.get_all_variables(self._config.get('template', {}))
+            tex_config['_template'] = renderer.get_all_variables(tex_config.get('template', {}))
         except errors.SchemaError as exception:
             exception.message = "The songbook file '{}' is not valid\n{}".format(
                 self.basename, exception.message)
             raise exception
 
-        self._config['_compiled_authwords'] = authors.compile_authwords(
-            copy.deepcopy(self._config['authors'])
+        tex_config['_compiled_authwords'] = authors.compile_authwords(
+            copy.deepcopy(tex_config['authors'])
             )
 
         # Configuration set
-        self._config['render'] = content.render
-        self._config['content'] = content.process_content(
-            self._config.get('content', []),
-            self._config,
-            )
-        self._config['filename'] = output.name[:-4]
+        tex_config['render'] = content.render
+        tex_config['_langs'], tex_config['content'] = self.get_content_items()
+        tex_config['filename'] = output.name[:-4]
 
         # Processing special options
-        self._config['_bookoptions'] = iter_bookoptions(self._config)
-        self._config['chords']['_notenames'] = self._get_chord_names(
-            self._config['chords']['notation']
+        tex_config['_bookoptions'] = iter_bookoptions(tex_config)
+        tex_config['chords']['_notenames'] = self._get_chord_names(
+            tex_config['chords']['notation']
             )
 
-        renderer.render_tex(output, self._config)
+        renderer.render_tex(output, tex_config)
 
         # Get all errors, and maybe exit program
         self._errors.extend(renderer.errors)
-        if self._config['_error'] == "failonbook":
+        if tex_config['_error'] == "failonbook":
             if self.has_errors():
                 raise errors.SongbookError("Some songs contain errors. Stopping as requested.")
 

patacrep/content/__init__.py

@@ -116,6 +116,10 @@ class ContentItem:
         """Return the string to end a block."""
         return ""
 
+    def to_dict(self):
+        """Return the dict representation (as in the yaml file)."""
+        raise NotImplementedError()
+
 class ContentError(SharedError):
     """Error in a content plugin."""
     def __init__(self, keyword=None, message=None):

patacrep/content/section.py

@@ -28,6 +28,12 @@ class Section(ContentItem):
         else:
             return r'\{}[{}]{{{}}}'.format(self.keyword, self.short, self.name)
 
+    def to_dict(self):
+        if self.short is None or self.keyword not in KEYWORDS:
+            return {self.keyword: self.name}
+        else:
+            return {self.keyword: {'name': self.name, 'short': self.short}}
+
 #pylint: disable=unused-argument
 @validate_parser_argument("""
 type: //any

patacrep/content/setcounter.py

@@ -14,6 +14,9 @@ class CounterSetter(ContentItem):
         """Set the value of the counter."""
         return r'\setcounter{{{}}}{{{}}}'.format(self.name, self.value)
 
+    def to_dict(self):
+        return {'setcounter': {'name': self.name, 'value': self.value}}
+
 #pylint: disable=unused-argument
 @validate_parser_argument("""
 type: //any

patacrep/content/song.py

@@ -63,6 +63,9 @@ class SongRenderer(ContentItem):
         """Order by song path"""
         return self.song.fullpath < other.song.fullpath
 
+    def to_dict(self):
+        return {'song': self.song.fullpath}
+
 #pylint: disable=unused-argument
 #pylint: disable=too-many-branches
 @validate_parser_argument("""

patacrep/content/songsection.py

@@ -19,6 +19,9 @@ class SongSection(ContentItem):
         """Render this section or chapter."""
         return r'\{}{{{}}}'.format(self.keyword, self.name)
 
+    def to_dict(self):
+        return {self.keyword: self.name}
+
 #pylint: disable=unused-argument
 @validate_parser_argument("""
 //str

patacrep/content/tex.py

@@ -20,6 +20,9 @@ class LaTeX(ContentItem):
             os.path.dirname(context['filename']),
             )))
 
+    def to_dict(self):
+        return {'tex': self.filename}
+
 #pylint: disable=unused-argument
 @validate_parser_argument("""
 type: //any

patacrep/songs/__init__.py

@@ -43,7 +43,7 @@ class DataSubpath:
         self.subpath = subpath
 
     def __str__(self):
-        return os.path.join(self.datadir, self.subpath)
+        return self.fullpath
 
     @property
     def fullpath(self):

patacrep/tools/__init__.py

@@ -0,0 +1,13 @@
+"""Common functions for patatools"""
+
+import argparse
+import os
+
+def existing_file(name):
+    """Check that argument is an existing, readable file name.
+
+    Return the argument for convenience.
+    """
+    if os.path.isfile(name) and os.access(name, os.R_OK):
+        return name
+    raise argparse.ArgumentTypeError("Cannot read file '{}'.".format(name))

patacrep/tools/cache/__main__.py

@@ -9,18 +9,10 @@ import textwrap
 
 from patacrep import errors
 from patacrep.songbook import open_songbook
+from .. import existing_file
 
 LOGGER = logging.getLogger("patatools.cache")
 
-def filename(name):
-    """Check that argument is an existing, readable file name.
-
-    Return the argument for convenience.
-    """
-    if os.path.isfile(name) and os.access(name, os.R_OK):
-        return name
-    raise argparse.ArgumentTypeError("Cannot read file '{}'.".format(name))
-
 def commandline_parser():
     """Return a command line parser."""
 
@@ -30,10 +22,7 @@ def commandline_parser():
         formatter_class=argparse.RawTextHelpFormatter,
         )
 
-    subparsers = parser.add_subparsers(
-        description="",
-        dest="command",
-        )
+    subparsers = parser.add_subparsers()
     subparsers.required = True
 
     clean = subparsers.add_parser(
@@ -45,7 +34,7 @@ def commandline_parser():
         'songbook',
         metavar="SONGBOOK",
         help=textwrap.dedent("""Songbook file to be used to look for cache path."""),
-        type=filename,
+        type=existing_file,
         )
     clean.set_defaults(command=do_clean)
 

patacrep/tools/content/__main__.py

@@ -0,0 +1,70 @@
+"""Perform operations on songbook content."""
+
+import argparse
+import logging
+import os
+import sys
+import textwrap
+import yaml
+
+from patacrep.songbook import open_songbook
+from patacrep.build import Songbook
+from .. import existing_file
+
+LOGGER = logging.getLogger("patatools.content")
+
+def commandline_parser():
+    """Return a command line parser."""
+
+    parser = argparse.ArgumentParser(
+        prog="patatools content",
+        description="Operations related to the content of a songbook.",
+        formatter_class=argparse.RawTextHelpFormatter,
+        )
+
+    subparsers = parser.add_subparsers()
+    subparsers.required = True
+
+    content_items = subparsers.add_parser(
+        "items",
+        description="Display the content items of a songbook.",
+        help="Return the content items.",
+        )
+    content_items.add_argument(
+        'songbook',
+        metavar="SONGBOOK",
+        help=textwrap.dedent("""Songbook file to be used to look for content items."""),
+        type=existing_file,
+        )
+    content_items.set_defaults(command=do_content_items)
+
+    return parser
+
+def do_content_items(namespace):
+    """Execute the `patatools content items` command."""
+    config = open_songbook(namespace.songbook)
+    config['_cache'] = True
+    config['_error'] = "fix"
+    songbook = Songbook(config, config['_outputname'])
+    _, content_items = songbook.get_content_items()
+    yaml_dir = os.path.dirname(os.path.abspath(namespace.songbook))
+    ref_dir = os.path.join(yaml_dir, 'songs')
+    content_items = [
+        normalize_song_path(item.to_dict(), ref_dir)
+        for item in content_items
+    ]
+    sys.stdout.write(yaml.safe_dump(content_items, allow_unicode=True, default_flow_style=False))
+
+def normalize_song_path(file_entry, ref_dir):
+    """Normalize the 'song' value, relative to ref_dir"""
+    if 'song' in file_entry:
+        file_entry['song'] = os.path.relpath(file_entry['song'], ref_dir)
+    return file_entry
+
+def main(args):
+    """Main function: run from command line."""
+    options = commandline_parser().parse_args(args[1:])
+    options.command(options)
+
+if __name__ == "__main__":
+    main(sys.argv)

test/test_content/custom.control

@@ -1 +1 @@
-- "{'customname:': ''}"
+- customname: ''

test/test_content/custom.source

@@ -1 +1 @@
-- customname:
+- customname: ''

test/test_content/customzipped.control

@@ -1 +1 @@
-- "{'customzippedname:': ''}"
+- customzippedname: ''

test/test_content/customzipped.source

@@ -1 +1 @@
-- customzippedname:
+- customzippedname: ''

test/test_content/cwd.control

@@ -1 +1 @@
-- subdir/chordpro.csg
+- song: datadir/songs/subdir/chordpro.csg

test/test_content/cwd_list.control

@@ -1,2 +1,2 @@
-- subdir/chordpro.csg
-- exsong.sg
+- song: datadir/songs/subdir/chordpro.csg
+- song: datadir/songs/exsong.sg

test/test_content/datadir/python/content/customplugin.py

@@ -5,8 +5,8 @@ from patacrep.content import ContentItem, ContentList, validate_parser_argument
 class FakeContent(ContentItem):
     """Fake content."""
 
-    def file_entry(self):
-        return {'customname:':''}
+    def to_dict(self):
+        return {'customname':''}
 
 def parse(keyword, argument, config):
     return ContentList([FakeContent()])

test/test_content/glob.control

@@ -1 +1 @@
-- chordpro.csg
+- song: datadir/songs/chordpro.csg

test/test_content/include.control

@@ -1,6 +1,6 @@
-- exsong.sg
-- chordpro.csg
-- subdir/chordpro.csg
-- chordpro.csg
-- subdir/chordpro.csg
-- exsong.sg
+- song: datadir/songs/exsong.sg
+- song: datadir/songs/chordpro.csg
+- song: datadir/songs/subdir/chordpro.csg
+- song: datadir/songs/chordpro.csg
+- song: datadir/songs/subdir/chordpro.csg
+- song: datadir/songs/exsong.sg

test/test_content/sections.control

@@ -1,11 +1,13 @@
-- section{First Section!}
-- section{Named section}
-- section[section_short_name]{Section with short name}
-- section*{Section* with short name}
-- part{part section test}
-- chapter{chapter section test}
-- section{section section test}
-- subsection{subsection section test}
-- subsubsection{subsubsection section test}
-- paragraph{paragraph section test}
-- subparagraph{subparagraph section test}
+- section: First Section!
+- section: Named section
+- section:
+    name: Section with short name
+    short: section_short_name
+- section*: Section* with short name
+- part: part section test
+- chapter: chapter section test
+- section: section section test
+- subsection: subsection section test
+- subsubsection: subsubsection section test
+- paragraph: paragraph section test
+- subparagraph: subparagraph section test

test/test_content/setcounter.control

@@ -1,4 +1,12 @@
-- setcounter{songnum}{101}
-- setcounter{songnum}{1}
-- setcounter{songnum}{5}
-- setcounter{counter_name}{-1}
+- setcounter:
+    name: songnum
+    value: 101
+- setcounter:
+    name: songnum
+    value: 1
+- setcounter:
+    name: songnum
+    value: 5
+- setcounter:
+    name: counter_name
+    value: -1

test/test_content/songs.control

@@ -1,4 +1,4 @@
-- exsong.sg
-- texsong.tsg
-- chordpro.csg
-- subdir/chordpro.csg
+- song: datadir/songs/exsong.sg
+- song: datadir/songs/texsong.tsg
+- song: datadir/songs/chordpro.csg
+- song: datadir/songs/subdir/chordpro.csg

test/test_content/songsection.control

@@ -1,6 +1,6 @@
-- songsection{Traditional}
-- exsong.sg
-- songchapter{English}
-- texsong.tsg
-- chordpro.csg
-- exsong.sg
+- songsection: Traditional
+- song: datadir/songs/exsong.sg
+- songchapter: English
+- song: datadir/songs/texsong.tsg
+- song: datadir/songs/chordpro.csg
+- song: datadir/songs/exsong.sg

test/test_content/sort.control

@@ -1,27 +1,27 @@
-- section{Title}
-- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg"
-- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg"
-- section{Author, Title}
-- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg"
-- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg"
-- section{Path, Title}
-- "@TEST_FOLDER@/datadir_sort/path1_title1_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path1_title1_author2.csg"
-- "@TEST_FOLDER@/datadir_sort/path1_title2_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path1_title2_author2.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title1_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title1_author2.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title2_author1.csg"
-- "@TEST_FOLDER@/datadir_sort/path2_title2_author2.csg"
+- section: Title
+- song: datadir_sort/path1_title1_author1.csg
+- song: datadir_sort/path1_title1_author2.csg
+- song: datadir_sort/path2_title1_author1.csg
+- song: datadir_sort/path2_title1_author2.csg
+- song: datadir_sort/path1_title2_author1.csg
+- song: datadir_sort/path1_title2_author2.csg
+- song: datadir_sort/path2_title2_author1.csg
+- song: datadir_sort/path2_title2_author2.csg
+- section: Author, Title
+- song: datadir_sort/path1_title1_author1.csg
+- song: datadir_sort/path2_title1_author1.csg
+- song: datadir_sort/path1_title2_author1.csg
+- song: datadir_sort/path2_title2_author1.csg
+- song: datadir_sort/path1_title1_author2.csg
+- song: datadir_sort/path2_title1_author2.csg
+- song: datadir_sort/path1_title2_author2.csg
+- song: datadir_sort/path2_title2_author2.csg
+- section: Path, Title
+- song: datadir_sort/path1_title1_author1.csg
+- song: datadir_sort/path1_title1_author2.csg
+- song: datadir_sort/path1_title2_author1.csg
+- song: datadir_sort/path1_title2_author2.csg
+- song: datadir_sort/path2_title1_author1.csg
+- song: datadir_sort/path2_title1_author2.csg
+- song: datadir_sort/path2_title2_author1.csg
+- song: datadir_sort/path2_title2_author2.csg

test/test_content/test_content.py

@@ -10,7 +10,6 @@ import yaml
 from pkg_resources import resource_filename
 
 from patacrep import content, files
-from patacrep.content import song, section, setcounter, songsection, tex
 from patacrep.songbook import prepare_songbook
 
 from .. import logging_reduced
@@ -55,16 +54,13 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
 
             with logging_reduced('patacrep.content.song'):
                 expandedlist = content.process_content(sbcontent, config)
-            sourcelist = [cls._clean_path(elem) for elem in expandedlist]
+            sourcelist = [cls._clean_path(elem.to_dict()) for elem in expandedlist]
 
             controlname = "{}.control".format(base)
             if not os.path.exists(controlname):
-                raise Exception("Missing control:" + str(sourcelist).replace("'", '"'))
+                raise Exception("Missing control:" + str(controlname).replace("'", '"'))
             with open(controlname, mode="r", encoding="utf8") as controlfile:
-                controllist = [
-                    elem.replace("@TEST_FOLDER@", files.path2posix(resource_filename(__name__, "")))
-                    for elem in yaml.load(controlfile)
-                    ]
+                controllist = yaml.load(controlfile)
 
             self.assertEqual(controllist, sourcelist)
 
@@ -75,25 +71,17 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
 
     @classmethod
     def _clean_path(cls, elem):
-        """Shorten the path relative to the `songs` directory"""
-
-        latex_command_classes = (
-            section.Section,
-            songsection.SongSection,
-            setcounter.CounterSetter,
-            )
-        if isinstance(elem, latex_command_classes):
-            return elem.render(None)[1:]
-
-        elif isinstance(elem, song.SongRenderer):
-            songpath = os.path.join(os.path.dirname(__file__), 'datadir', 'songs')
-            return files.path2posix(files.relpath(elem.song.fullpath, songpath))
-
-        elif isinstance(elem, tex.LaTeX):
-            return files.path2posix(elem.filename)
-
-        else:
-            return str(elem.file_entry())
+        """Shorten the path relative to the test directory"""
+        if not isinstance(elem, dict):
+            return elem
+
+        test_path = files.path2posix(resource_filename(__name__, ""))+"/"
+        for key in ['song', 'tex']:
+            if key in elem:
+                elem[key] = files.path2posix(
+                    os.path.normpath(elem[key])
+                ).replace(test_path, "")
+        return elem
 
     @classmethod
     def _generate_config(cls, sbcontent, outputdir, base):

test/test_content/tex.control

@@ -1,3 +1,3 @@
-- test/test_content/datadir/songs/texfile.tex
-- test/test_content/datadir/songs/texfile.tex
-- test/test_content/datadir/songs/texfile.tex
+- tex: test/test_content/datadir/songs/texfile.tex
+- tex: test/test_content/datadir/songs/texfile.tex
+- tex: test/test_content/datadir/songs/texfile.tex