Merge pull request #179 from patacrep/content_improve

Content tests and improvements

patacrep/content/__init__.py

@@ -174,14 +174,12 @@ def process_content(content, config=None):
     """
     contentlist = []
     plugins = config.get('_content_plugins', {})
-    keyword_re = re.compile(r'^ *(?P<keyword>\w*) *(\((?P<argument>.*)\))? *$')
+    keyword_re = re.compile(r'^ *(?P<keyword>[\w\*]*) *(\((?P<argument>.*)\))? *$')
     if not content:
         content = [["song"]]
     for elem in content:
         if isinstance(elem, str):
             elem = ["song", elem]
-        if len(content) == 0:
-            content = ["song"]
         try:
             match = keyword_re.match(elem[0]).groupdict()
         except AttributeError:

patacrep/content/cwd.py

@@ -25,8 +25,7 @@ def parse(keyword, config, argument, contentlist):
     old_songdir = config['_songdir']
     config['_songdir'] = (
         [DataSubpath("", argument)] +
-        [path.clone().join(argument) for path in config['_songdir']] +
-        config['_songdir']
+        [path.clone().join(argument) for path in config['_songdir']]
         )
     processed_content = process_content(contentlist, config)
     config['_songdir'] = old_songdir

patacrep/content/section.py

@@ -37,8 +37,8 @@ def parse(keyword, argument, contentlist, config):
       their starred versions "part*", "chapter*", ... , "subparagraph*"): the
       section to use;
     - argument: unused;
-    - contentlist: a list of one or two strings, which are the names (short
-      and long) of the section;
+    - contentlist: a list of one or two strings, which are the names (long
+      and short) of the section;
     - config: configuration dictionary of the current songbook.
     """
     if (keyword not in KEYWORDS) and (len(contentlist) != 1):

patacrep/content/song.py

@@ -78,6 +78,8 @@ def parse(keyword, argument, contentlist, config):
             if not os.path.isdir(songdir.datadir):
                 continue
             with files.chdir(songdir.datadir):
+                # Starting with Python 3.5 glob can be recursive: **/*.csg for instance
+                # for filename in glob.iglob(os.path.join(songdir.subpath, elem), recursive=True):
                 for filename in glob.iglob(os.path.join(songdir.subpath, elem)):
                     LOGGER.debug('Parsing file "{}"…'.format(filename))
                     extension = filename.split(".")[-1]

patacrep/content/sorted.py

@@ -9,8 +9,8 @@ import logging
 import unidecode
 
 from patacrep import files
-from patacrep.content import ContentError
-from patacrep.content.song import OnlySongsError, process_songs
+from patacrep.content import ContentError, process_content
+from patacrep.content.song import OnlySongsError
 
 LOGGER = logging.getLogger(__name__)
 
@@ -83,7 +83,7 @@ def parse(keyword, config, argument, contentlist):
     else:
         sort = DEFAULT_SORT
     try:
-        songlist = process_songs(contentlist, config)
+        songlist = process_content(contentlist, config)
     except OnlySongsError as error:
         raise ContentError(keyword, (
             "Content list of this keyword can be only songs (or content "

test/__init__.py

@@ -9,8 +9,17 @@ import unittest
 import patacrep
 
 @contextlib.contextmanager
-def disable_logging():
-    """Context locally disabling logging."""
+def logging_reduced(module_name=None, tmp_level=logging.CRITICAL):
+    """Temporarly reduce the logging level of a specific module
+    or globally if None
+    """
+    if module_name:
+        logger = logging.getLogger(module_name)
+        old_level = logger.getEffectiveLevel()
+        logger.setLevel(tmp_level)
+        yield
+        logger.setLevel(old_level)
+    else:
         logging.disable(logging.CRITICAL)
         yield
         logging.disable(logging.NOTSET)

test/test_content/cwd.control

@@ -0,0 +1 @@
+["subdir/chordpro.csg"]

test/test_content/cwd.source

@@ -0,0 +1 @@
+[["cwd(subdir)"]]

test/test_content/cwd_list.control

@@ -0,0 +1 @@
+["subdir/chordpro.csg", "exsong.sg"]

test/test_content/cwd_list.source

@@ -0,0 +1 @@
+[["cwd(subdir)", "exsong.sg", "intersong.is", "jsonlist.json", "texfile.tex", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"], "exsong.sg"]

test/test_content/datadir/custom_list.json

@@ -0,0 +1 @@
+["exsong.sg", "chordpro.csg", "subdir/chordpro.csg"]

test/test_content/datadir/songs/chordpro.csg

@@ -0,0 +1,51 @@
+{lang : en}
+{columns :  2}
+{ title : Greensleeves}
+{subtitle: Test of the chordpro format}
+{artist: Traditionnel}
+{artist: Prénom Nom}
+{cover : traditionnel.jpg }
+{album :Angleterre}
+
+{partition : greensleeves.ly}
+
+
+A[Am]las, my love, ye [G]do me wrong
+To [Am]cast me oft dis[E]curteously
+And [Am]I have loved [G]you so long
+De[Am]lighting [E]in your [Am]companie
+
+{start_of_chorus}
+   [C]Greensleeves was [G]all my joy
+   [Am]Greensleeves was [E]my delight
+   [C]Greensleeves was my [G]heart of gold
+   And [Am]who but [E]Ladie [Am]Greensleeves
+{end_of_chorus}
+
+I [Am]have been ready [G]at your hand
+To [Am]grant what ever [E]you would crave
+I [Am]have both waged [G]life and land
+Your [Am]love and [E]good will [Am]for to have
+
+I [Am]bought thee kerchers [G]to thy head
+That [Am]were wrought fine and [E]gallantly
+I [Am]kept thee both at [G]boord and bed
+Which [Am]cost my [E]purse well [Am]favouredly
+
+I [Am]bought thee peticotes [G]of the best
+The [Am]cloth so fine as [E]fine might be
+I [Am]gave thee jewels [G]for thy chest
+And [Am]all this [E]cost I [Am]spent on thee
+
+{c:test of comment}
+
+{gc: test of guitar comment}
+
+{image: traditionnel.jpg}
+
+Thy [Am]smock of silke, both [G]faire and white
+With [Am]gold embrodered [E]gorgeously
+Thy [Am]peticote of [G]sendall right
+And [Am]this I [E]bought thee [Am]gladly
+
+

test/test_content/datadir/songs/exsong.sg

@@ -0,0 +1,10 @@
+\beginsong{Image included from a different datadir\\\LaTeX}
+  [cover={img/datadir2}]
+
+  \cover
+
+  \lilypond{scores/datadir2.ly}
+
+  \image{img/datadir2}
+
+\endsong

test/test_content/datadir/songs/intersong.is

@@ -0,0 +1,6 @@
+\selectlanguage{french}
+\sortassong{}[by={QQ}]
+\begin{intersong}
+
+Lorem ipsum
+\end{intersong}

test/test_content/datadir/songs/subdir/chordpro.csg

@@ -0,0 +1,51 @@
+{lang : en}
+{columns :  2}
+{ title : Greensleeves}
+{subtitle: Test of the chordpro format}
+{artist: Traditionnel}
+{artist: Prénom Nom}
+{cover : traditionnel.jpg }
+{album :Angleterre}
+
+{partition : greensleeves.ly}
+
+
+A[Am]las, my love, ye [G]do me wrong
+To [Am]cast me oft dis[E]curteously
+And [Am]I have loved [G]you so long
+De[Am]lighting [E]in your [Am]companie
+
+{start_of_chorus}
+   [C]Greensleeves was [G]all my joy
+   [Am]Greensleeves was [E]my delight
+   [C]Greensleeves was my [G]heart of gold
+   And [Am]who but [E]Ladie [Am]Greensleeves
+{end_of_chorus}
+
+I [Am]have been ready [G]at your hand
+To [Am]grant what ever [E]you would crave
+I [Am]have both waged [G]life and land
+Your [Am]love and [E]good will [Am]for to have
+
+I [Am]bought thee kerchers [G]to thy head
+That [Am]were wrought fine and [E]gallantly
+I [Am]kept thee both at [G]boord and bed
+Which [Am]cost my [E]purse well [Am]favouredly
+
+I [Am]bought thee peticotes [G]of the best
+The [Am]cloth so fine as [E]fine might be
+I [Am]gave thee jewels [G]for thy chest
+And [Am]all this [E]cost I [Am]spent on thee
+
+{c:test of comment}
+
+{gc: test of guitar comment}
+
+{image: traditionnel.jpg}
+
+Thy [Am]smock of silke, both [G]faire and white
+With [Am]gold embrodered [E]gorgeously
+Thy [Am]peticote of [G]sendall right
+And [Am]this I [E]bought thee [Am]gladly
+
+

test/test_content/datadir/songs/texfile.tex

@@ -0,0 +1,70 @@
+\selectlanguage{french}
+\songcolumns{2}
+\beginsong{Chevaliers de la table ronde}
+  [by={Traditionnel},cover={traditionnel},album={France}]
+
+  \cover
+  \gtab{C}{X32010}
+  \gtab{G7}{320001}
+  \gtab{F}{1:022100}
+
+  \begin{verse}
+    Cheva\[C]liers de la Table Ronde
+    Goûtons \[G7]voir si le vin est \[C]bon
+    \rep{2}
+  \end{verse}
+
+  \begin{chorus}
+    Goûtons \[F]voir, \echo{oui, oui, oui}
+    Goûtons \[C]voir, \echo{non, non, non}
+    Goûtons \[G7]voir si le vin est bon
+    \rep{2}
+  \end{chorus}
+
+  \begin{verse}
+    S'il est bon, s'il est agréable
+    J'en boirai jusqu'à mon plaisir
+  \end{verse}
+
+  \begin{verse}
+    J'en boirai cinq à six bouteilles
+    Et encore, ce n'est pas beaucoup
+  \end{verse}
+
+  \begin{verse}
+    Si je meurs, je veux qu'on m'enterre
+    Dans une cave où il y a du bon vin
+  \end{verse}
+
+  \begin{verse}
+    Les deux pieds contre la muraille
+    Et la tête sous le robinet
+  \end{verse}
+
+  \begin{verse}
+    Et les quatre plus grands ivrognes
+    Porteront les quatre coins du drap
+  \end{verse}
+
+  \begin{verse}
+    Pour donner le discours d'usage
+    On prendra le bistrot du coin
+  \end{verse}
+
+  \begin{verse}
+    Et si le tonneau se débouche
+    J'en boirai jusqu'à mon plaisir
+  \end{verse}
+
+  \begin{verse}
+    Et s'il en reste quelques gouttes
+    Ce sera pour nous rafraîchir
+  \end{verse}
+
+  \begin{verse}
+    Sur ma tombe, je veux qu'on inscrive
+    \emph{Ici gît le roi des buveurs}
+  \end{verse}
+
+\endsong
+

test/test_content/datadir/songs/texsong.tsg

@@ -0,0 +1,70 @@
+\selectlanguage{french}
+\songcolumns{2}
+\beginsong{Chevaliers de la table ronde}
+  [by={Traditionnel},cover={traditionnel},album={France}]
+
+  \cover
+  \gtab{C}{X32010}
+  \gtab{G7}{320001}
+  \gtab{F}{1:022100}
+
+  \begin{verse}
+    Cheva\[C]liers de la Table Ronde
+    Goûtons \[G7]voir si le vin est \[C]bon
+    \rep{2}
+  \end{verse}
+
+  \begin{chorus}
+    Goûtons \[F]voir, \echo{oui, oui, oui}
+    Goûtons \[C]voir, \echo{non, non, non}
+    Goûtons \[G7]voir si le vin est bon
+    \rep{2}
+  \end{chorus}
+
+  \begin{verse}
+    S'il est bon, s'il est agréable
+    J'en boirai jusqu'à mon plaisir
+  \end{verse}
+
+  \begin{verse}
+    J'en boirai cinq à six bouteilles
+    Et encore, ce n'est pas beaucoup
+  \end{verse}
+
+  \begin{verse}
+    Si je meurs, je veux qu'on m'enterre
+    Dans une cave où il y a du bon vin
+  \end{verse}
+
+  \begin{verse}
+    Les deux pieds contre la muraille
+    Et la tête sous le robinet
+  \end{verse}
+
+  \begin{verse}
+    Et les quatre plus grands ivrognes
+    Porteront les quatre coins du drap
+  \end{verse}
+
+  \begin{verse}
+    Pour donner le discours d'usage
+    On prendra le bistrot du coin
+  \end{verse}
+
+  \begin{verse}
+    Et si le tonneau se débouche
+    J'en boirai jusqu'à mon plaisir
+  \end{verse}
+
+  \begin{verse}
+    Et s'il en reste quelques gouttes
+    Ce sera pour nous rafraîchir
+  \end{verse}
+
+  \begin{verse}
+    Sur ma tombe, je veux qu'on inscrive
+    \emph{Ici gît le roi des buveurs}
+  \end{verse}
+
+\endsong
+

test/test_content/glob.control

@@ -0,0 +1 @@
+["chordpro.csg"]

test/test_content/glob.source

@@ -0,0 +1 @@
+["*.csg"]

test/test_content/include.control

@@ -0,0 +1 @@
+["exsong.sg", "chordpro.csg", "subdir/chordpro.csg"]

test/test_content/include.source

@@ -0,0 +1 @@
+[["include" , "custom_list.json"]]

test/test_content/sections.control

@@ -0,0 +1 @@
+["section:Traditional", "exsong.sg", "section:Example", "texsong.tsg", "chordpro.csg", "exsong.sg"]

test/test_content/sections.source

@@ -0,0 +1,6 @@
+[["section", "Traditional"],
+                "exsong.sg",
+                ["section", "Example"],
+                "texsong.tsg",
+                "chordpro.csg",
+                "exsong.sg"]

test/test_content/sections_short.control

@@ -0,0 +1 @@
+["section:(tradi)Traditional", "exsong.sg", "section*:Example", "texsong.tsg", "chordpro.csg", "exsong.sg"]

test/test_content/sections_short.source

@@ -0,0 +1,6 @@
+[["section", "Traditional", "tradi"],
+                "exsong.sg",
+                ["section*", "Example"],
+                "texsong.tsg",
+                "chordpro.csg",
+                "exsong.sg"]

test/test_content/songs.control

@@ -0,0 +1 @@
+["exsong.sg", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"]

test/test_content/songs.source

@@ -0,0 +1 @@
+["exsong.sg", "intersong.is", "jsonlist.json", "texfile.tex", "texsong.tsg", "chordpro.csg", "subdir/chordpro.csg"]

test/test_content/songsection.control

@@ -0,0 +1 @@
+["songsection:Traditional", "exsong.sg", "songchapter:English", "texsong.tsg", "chordpro.csg", "exsong.sg"]

test/test_content/songsection.source

@@ -0,0 +1,6 @@
+[["songsection", "Traditional"],
+                "exsong.sg",
+                ["songchapter", "English"],
+                "texsong.tsg",
+                "chordpro.csg",
+                "exsong.sg"]

test/test_content/sorted.control

@@ -0,0 +1 @@
+["chordpro.csg", "exsong.sg", "subdir/chordpro.csg", "texsong.tsg"]

test/test_content/sorted.source

@@ -0,0 +1 @@
+[["sorted(fullpath)"]]

test/test_content/test_content.py

@@ -0,0 +1,118 @@
+"""Tests for the content plugins."""
+
+# pylint: disable=too-few-public-methods
+
+import glob
+import os
+import unittest
+import json
+
+from patacrep.songs import DataSubpath, DEFAULT_CONFIG
+from patacrep import content, files
+from patacrep.content import song, section, songsection, tex
+
+from .. import logging_reduced
+from .. import dynamic # pylint: disable=unused-import
+
+class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
+    """Test of the content plugins.
+
+    For any given `foo.source`, it parses the content as a json "content"
+    argument of a .sb file.
+    It controls that the generated file list is equal to the one in `foo.control`.
+    """
+
+    maxDiff = None
+    config = None
+
+    @classmethod
+    def setUpClass(cls):
+        cls._generate_config()
+
+    @classmethod
+    def _iter_testmethods(cls):
+        """Iterate over dynamically generated test methods"""
+        for source in sorted(glob.glob(os.path.join(
+                os.path.dirname(__file__),
+                '*.source',
+            ))):
+            base = source[:-len(".source")]
+            yield (
+                "test_content_{}".format(os.path.basename(base)),
+                cls._create_content_test(base),
+                )
+
+    @classmethod
+    def _create_content_test(cls, base):
+        """Return a function that `base.source` produces the correct file list"""
+
+        def test_content(self):
+            """Test that `base.source` produces the correct file list"""
+            sourcename = "{}.source".format(base)
+            with open(sourcename, mode="r", encoding="utf8") as sourcefile:
+                sbcontent = json.load(sourcefile)
+
+            with logging_reduced('patacrep.content.song'):
+                expandedlist = content.process_content(sbcontent, cls.config.copy())
+            sourcelist = [cls._clean_path(elem) for elem in expandedlist]
+
+            controlname = "{}.control".format(base)
+            if not os.path.exists(controlname):
+                raise Exception("Missing control:" + str(sourcelist).replace("'", '"'))
+            with open(controlname, mode="r", encoding="utf8") as controlfile:
+                controllist = json.load(controlfile)
+
+            self.assertEqual(controllist, sourcelist)
+
+        test_content.__doc__ = (
+            "Test that '{base}.source' produces the correct file list"""
+            ).format(base=os.path.basename(base))
+        return test_content
+
+    @classmethod
+    def _clean_path(cls, elem):
+        """Shorten the path relative to the `songs` directory"""
+        if isinstance(elem, song.SongRenderer):
+            songpath = os.path.join(os.path.dirname(__file__), 'datadir', 'songs')
+            return files.path2posix(files.relpath(elem.song.fullpath, songpath))
+
+        elif isinstance(elem, section.Section):
+            if elem.short is None:
+                return "{}:{}".format(elem.keyword, elem.name)
+            else:
+                return "{}:({}){}".format(elem.keyword, elem.short, elem.name)
+
+        elif isinstance(elem, songsection.SongSection):
+            return "{}:{}".format(elem.keyword, elem.name)
+
+        elif isinstance(elem, tex.LaTeX):
+            return files.path2posix(elem.filename)
+
+        else:
+            raise Exception(elem)
+
+    @classmethod
+    def _generate_config(cls):
+        """Generate the config to process the content"""
+
+        config = DEFAULT_CONFIG.copy()
+
+        datadirpaths = [os.path.join(os.path.dirname(__file__), 'datadir')]
+
+        config['datadir'] = datadirpaths
+
+        config['_songdir'] = [
+            DataSubpath(path, 'songs')
+            for path in datadirpaths
+            ]
+        config['_content_plugins'] = files.load_plugins(
+            datadirs=datadirpaths,
+            root_modules=['content'],
+            keyword='CONTENT_PLUGINS',
+            )
+        config['_song_plugins'] = files.load_plugins(
+            datadirs=datadirpaths,
+            root_modules=['songs'],
+            keyword='SONG_RENDERERS',
+            )['tsg']
+        cls.config = config

test/test_content/tex.control

@@ -0,0 +1 @@
+["test/test_content/datadir/songs/texfile.tex"]

test/test_content/tex.source

@@ -0,0 +1 @@
+[["tex", "texfile.tex", "chordpro.csg"]]

test/test_song/test_parser.py

@@ -12,7 +12,7 @@ from patacrep import files
 from patacrep.songs import DEFAULT_CONFIG
 from patacrep.encoding import open_read
 
-from .. import disable_logging
+from .. import logging_reduced
 from .. import dynamic # pylint: disable=unused-import
 
 OUTPUTS = {
@@ -59,7 +59,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest):
         destname = "{}.{}".format(base, out_format)
         with self.chdir():
             with open_read(destname) as expectfile:
-                with disable_logging():
+                with logging_reduced():
                     song = self.song_plugins[out_format][in_format](sourcename, self.config)
                     expected = expectfile.read().strip().replace(
                         "@TEST_FOLDER@",

test/test_songbook/test_compilation.py

@@ -1,4 +1,4 @@
-"""Tests for the chordpro parser."""
+"""Tests for the songbook compilation."""
 
 # pylint: disable=too-few-public-methods