From fc328658ce1f506bf3fe9db14e4d445bfddc2857 Mon Sep 17 00:00:00 2001 From: Louis Date: Sat, 3 Oct 2015 21:33:49 +0200 Subject: [PATCH 1/5] [WIP] Does not work --- patacrep/latex/lexer.py | 6 +- patacrep/latex/syntax.py | 4 +- patacrep/songs/chordpro/ast.py | 108 ++++++++---------- .../chordpro/data/chordpro/content_endofline | 1 + .../chordpro/data/chordpro/content_newline | 2 +- .../songs/chordpro/data/chordpro/song_header | 2 +- .../chordpro/data/html/content_endofline | 1 + .../songs/chordpro/data/html/content_newline | 2 +- patacrep/songs/chordpro/data/html/song_header | 2 +- .../chordpro/data/latex/content_endofline | 2 + .../songs/chordpro/data/latex/content_newline | 3 +- patacrep/songs/chordpro/data/latex/song | 2 +- patacrep/songs/chordpro/lexer.py | 6 +- patacrep/songs/chordpro/syntax.py | 41 ++++--- test/test_chordpro/metadata.sgc | 4 +- test/test_chordpro/metadata.source | 4 +- test/test_chordpro/metadata.tex | 5 +- test/test_chordpro/newline.sgc | 41 +++++++ test/test_chordpro/newline.source | 32 ++++++ test/test_chordpro/newline.tex | 61 ++++++++++ 20 files changed, 235 insertions(+), 94 deletions(-) create mode 100644 patacrep/songs/chordpro/data/chordpro/content_endofline create mode 100644 patacrep/songs/chordpro/data/html/content_endofline create mode 100644 patacrep/songs/chordpro/data/latex/content_endofline create mode 100644 test/test_chordpro/newline.sgc create mode 100644 test/test_chordpro/newline.source create mode 100644 test/test_chordpro/newline.tex diff --git a/patacrep/latex/lexer.py b/patacrep/latex/lexer.py index b8a762c0..f76500f8 100644 --- a/patacrep/latex/lexer.py +++ b/patacrep/latex/lexer.py @@ -12,7 +12,7 @@ tokens = ( 'LBRACE', 'RBRACE', 'COMMAND', - 'NEWLINE', + 'ENDOFLINE', 'COMMA', 'EQUAL', 'CHARACTER', @@ -34,7 +34,7 @@ class SimpleLexer: t_LBRACE = r'{' t_RBRACE = r'}' t_COMMAND = r'\\([@a-zA-Z]+|[^\\])' - t_NEWLINE = r'\\\\' + t_ENDOFLINE = r'\\\\' SPECIAL_CHARACTERS = ( t_LBRACKET + t_RBRACKET + @@ -59,7 +59,7 @@ class SimpleLexer: # Define a rule so we can track line numbers @staticmethod - def t_newline(token): + def t_endofline(token): r'\n+' token.lexer.lineno += len(token.value) diff --git a/patacrep/latex/syntax.py b/patacrep/latex/syntax.py index 46c42ea1..8915816b 100644 --- a/patacrep/latex/syntax.py +++ b/patacrep/latex/syntax.py @@ -27,7 +27,7 @@ class LatexParser(Parser): """expression : brackets expression | braces expression | command expression - | NEWLINE expression + | ENDOFLINE expression | beginsong expression | word expression | SPACE expression @@ -172,7 +172,7 @@ class LatexParser(Parser): @staticmethod def p_titles_next(symbols): - """titles_next : NEWLINE title titles_next + """titles_next : ENDOFLINE title titles_next | empty """ if len(symbols) == 2: diff --git a/patacrep/songs/chordpro/ast.py b/patacrep/songs/chordpro/ast.py index 0227d457..7f563dea 100644 --- a/patacrep/songs/chordpro/ast.py +++ b/patacrep/songs/chordpro/ast.py @@ -2,47 +2,11 @@ # pylint: disable=too-few-public-methods +from collections import OrderedDict import logging LOGGER = logging.getLogger() -class OrderedLifoDict: - """Ordered (LIFO) dictionary. - - Mimics the :class:`dict` dictionary, with: - - dictionary is ordered: the order the keys are kept (as with - :class:`collections.OrderedDict`), excepted that: - - LIFO: the last item is reterned first when iterating. - """ - - def __init__(self, default=None): - if default is None: - self._keys = [] - self._values = {} - else: - self._keys = list(default.keys()) - self._values = default.copy() - - def values(self): - """Same as :meth:`dict.values`.""" - for key in self: - yield self._values[key] - - def __iter__(self): - yield from self._keys - - def __setitem__(self, key, value): - if key not in self._keys: - self._keys.insert(0, key) - self._values[key] = value - - def __getitem__(self, key): - return self._values[key] - - def get(self, key, default=None): - """Same as :meth:`dict.get`.""" - return self._values.get(key, default) - def _indent(string): """Return and indented version of argument.""" return "\n".join([" {}".format(line) for line in string.split('\n')]) @@ -100,8 +64,12 @@ class Line(AST): self.line = [] def prepend(self, data): - """Add an object at the beginning of line.""" - self.line.insert(0, data) + """Add an object at the beginning of line. + + Does nothing if argument is `None`. + """ + if data is not None: + self.line.insert(0, data) return self def strip(self): @@ -109,14 +77,17 @@ class Line(AST): while True: if not self.line: return self - if isinstance(self.line[0], Space): + if isinstance(self.line[0], Space) or isinstance(self.line[0], Error): del self.line[0] continue - if isinstance(self.line[-1], Space): + if isinstance(self.line[-1], Space) or isinstance(self.line[-1], Error): del self.line[-1] continue return self + def is_empty(self): + return len(self.strip().line) == 0 + class LineElement(AST): """Something present on a line.""" # pylint: disable=abstract-method @@ -206,26 +177,29 @@ class Song(AST): def __init__(self, filename): super().__init__() self.content = [] - self.meta = OrderedLifoDict() + self.meta = OrderedDict() self._authors = [] self._titles = [] self._subtitles = [] - self._keys = [] self.filename = filename def add(self, data): """Add an element to the song""" if isinstance(data, Error): - return self + pass elif data is None: # New line - if not (self.content and isinstance(self.content[0], Newline)): - self.content.insert(0, Newline()) - elif isinstance(data, Line): + if not (self.content and isinstance(self.content[0], EndOfLine)): + self.content.insert(0, EndOfLine()) + elif isinstance(data, Line) or isinstance(data, NewLine): # Add a new line, maybe in the current verse. - if not (self.content and isinstance(self.content[0], Verse)): - self.content.insert(0, Verse()) - self.content[0].prepend(data.strip()) + if not data.is_empty(): + if not (self.content and isinstance(self.content[0], Verse)): + self.content.insert(0, Verse()) + self.content[0].prepend(data.strip()) + elif isinstance(data, Directive) and data.inline: + # Add a directive in the content of the song. + self.content.append(data) elif data.inline: # Add an object in the content of the song. self.content.insert(0, data) @@ -242,13 +216,13 @@ class Song(AST): def add_title(self, data): """Add a title""" - self._titles.insert(0, data.argument) + self._titles.append(data.argument) def add_cumulative(self, data): """Add a cumulative argument into metadata""" if data.keyword not in self.meta: self.meta[data.keyword] = [] - self.meta[data.keyword].insert(0, data) + self.meta[data.keyword].append(data) def get_data_argument(self, keyword, default): """Return `self.meta[keyword].argument`. @@ -267,7 +241,7 @@ class Song(AST): def add_subtitle(self, data): """Add a subtitle""" - self._subtitles.insert(0, data.argument) + self._subtitles.append(data.argument) @property def titles(self): @@ -276,7 +250,7 @@ class Song(AST): def add_author(self, data): """Add an auhor.""" - self._authors.insert(0, data.argument) + self._authors.append(data.argument) @property def authors(self): @@ -286,19 +260,20 @@ class Song(AST): def add_key(self, data): """Add a new {key: foo: bar} directive.""" key, *argument = data.argument.split(":") - if 'keys' not in self.meta: - self.meta['keys'] = [] - self.meta['keys'].insert(0, Directive( + if 'morekeys' not in self.meta: + self.meta['morekeys'] = [] + self.meta['morekeys'].append(Directive( key.strip(), ":".join(argument).strip(), )) -class Newline(AST): +class EndOfLine(AST): """New line""" - _template = "newline" + _template = "endofline" class Directive(AST): """A directive""" + meta = True def __init__(self, keyword, argument=None): super().__init__() @@ -322,6 +297,21 @@ class Directive(AST): def __str__(self): return self.argument + @classmethod + def create(cls, keyword, argument=None): + if keyword == "newline": + return NewLine(keyword, argument) + else: + return cls(keyword, argument) + +class NewLine(Directive): + keyword = "newline" + _template = "newline" + meta = False + + def strip(self): + return self + class Define(Directive): """A chord definition. diff --git a/patacrep/songs/chordpro/data/chordpro/content_endofline b/patacrep/songs/chordpro/data/chordpro/content_endofline new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/patacrep/songs/chordpro/data/chordpro/content_endofline @@ -0,0 +1 @@ + diff --git a/patacrep/songs/chordpro/data/chordpro/content_newline b/patacrep/songs/chordpro/data/chordpro/content_newline index 8b137891..01d04291 100644 --- a/patacrep/songs/chordpro/data/chordpro/content_newline +++ b/patacrep/songs/chordpro/data/chordpro/content_newline @@ -1 +1 @@ - +{newline} diff --git a/patacrep/songs/chordpro/data/chordpro/song_header b/patacrep/songs/chordpro/data/chordpro/song_header index d3e65f0b..5289d96f 100644 --- a/patacrep/songs/chordpro/data/chordpro/song_header +++ b/patacrep/songs/chordpro/data/chordpro/song_header @@ -25,7 +25,7 @@ {(( 'cov' )): (( metadata['cov'].argument|search_image ))} (* endif *) -(*- for key in metadata.keys -*) +(*- for key in metadata.morekeys -*) {key: (( key.keyword )): (( key.argument ))} (* endfor *) diff --git a/patacrep/songs/chordpro/data/html/content_endofline b/patacrep/songs/chordpro/data/html/content_endofline new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/patacrep/songs/chordpro/data/html/content_endofline @@ -0,0 +1 @@ + diff --git a/patacrep/songs/chordpro/data/html/content_newline b/patacrep/songs/chordpro/data/html/content_newline index 8b137891..1915ec14 100644 --- a/patacrep/songs/chordpro/data/html/content_newline +++ b/patacrep/songs/chordpro/data/html/content_newline @@ -1 +1 @@ - +
diff --git a/patacrep/songs/chordpro/data/html/song_header b/patacrep/songs/chordpro/data/html/song_header index cc23ad17..dbc40433 100644 --- a/patacrep/songs/chordpro/data/html/song_header +++ b/patacrep/songs/chordpro/data/html/song_header @@ -23,7 +23,7 @@ (* include 'content_metadata_cover' *) -(*- for key in metadata.keys -*) +(*- for key in metadata.morekeys -*) {key: (( key.keyword )): (( key.argument ))} (* endfor *) diff --git a/patacrep/songs/chordpro/data/latex/content_endofline b/patacrep/songs/chordpro/data/latex/content_endofline new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/patacrep/songs/chordpro/data/latex/content_endofline @@ -0,0 +1,2 @@ + + diff --git a/patacrep/songs/chordpro/data/latex/content_newline b/patacrep/songs/chordpro/data/latex/content_newline index 139597f9..c0846fba 100644 --- a/patacrep/songs/chordpro/data/latex/content_newline +++ b/patacrep/songs/chordpro/data/latex/content_newline @@ -1,2 +1 @@ - - +~\\ diff --git a/patacrep/songs/chordpro/data/latex/song b/patacrep/songs/chordpro/data/latex/song index 021dd04f..044ee87b 100644 --- a/patacrep/songs/chordpro/data/latex/song +++ b/patacrep/songs/chordpro/data/latex/song @@ -30,7 +30,7 @@ (* if 'cov' in metadata *) cov={(( metadata["cov"].argument|search_image ))}, (* endif *) - (* for key in metadata.keys *) + (* for key in metadata.morekeys *) (( key.keyword ))={(( key.argument ))}, (* endfor *) ] diff --git a/patacrep/songs/chordpro/lexer.py b/patacrep/songs/chordpro/lexer.py index aba37a1c..4ee20c4b 100644 --- a/patacrep/songs/chordpro/lexer.py +++ b/patacrep/songs/chordpro/lexer.py @@ -9,7 +9,7 @@ LOGGER = logging.getLogger() tokens = ( 'LBRACE', 'RBRACE', - 'NEWLINE', + 'ENDOFLINE', 'COLON', 'WORD', 'SPACE', @@ -81,14 +81,14 @@ class ChordProLexer: return token t_tablature_TEXT = r'[^\n]+' - t_tablature_NEWLINE = r'\n' + t_tablature_ENDOFLINE = r'\n' def __init__(self): self.__class__.lexer = lex.lex(module=self) # Define a rule so we can track line numbers @staticmethod - def t_NEWLINE(token): + def t_ENDOFLINE(token): r'[\n\r]' token.lexer.lineno += 1 return token diff --git a/patacrep/songs/chordpro/syntax.py b/patacrep/songs/chordpro/syntax.py index a5d1b102..b06ec13c 100644 --- a/patacrep/songs/chordpro/syntax.py +++ b/patacrep/songs/chordpro/syntax.py @@ -16,26 +16,25 @@ class ChordproParser(Parser): def __init__(self, filename=None): super().__init__() self.tokens = tokens - self.filename = filename + self.song = ast.Song(filename) def p_song(self, symbols): """song : block song | empty """ if len(symbols) == 2: - symbols[0] = ast.Song(self.filename) + symbols[0] = self.song else: symbols[0] = symbols[2].add(symbols[1]) @staticmethod def p_block(symbols): """block : SPACE block - | directive NEWLINE - | line NEWLINE - | chorus NEWLINE - | tab NEWLINE - | bridge NEWLINE - | NEWLINE + | line ENDOFLINE + | chorus ENDOFLINE + | tab ENDOFLINE + | bridge ENDOFLINE + | ENDOFLINE """ if len(symbols) == 3 and isinstance(symbols[1], str): symbols[0] = symbols[2] @@ -133,16 +132,22 @@ class ChordproParser(Parser): symbols[0] = ast.Error() return - symbols[0] = self._parse_define(match.groupdict()) - if symbols[0] is None: + define = self._parse_define(match.groupdict()) + if define is None: self.error( line=symbols.lexer.lineno, message="Invalid chord definition '{}'.".format(argument), ) symbols[0] = ast.Error() + return + self.song.add(define) else: - symbols[0] = ast.Directive(keyword, argument) + directive = ast.Directive.create(keyword, argument) + if directive.meta: + self.song.add(directive) + else: + symbols[0] = directive @staticmethod def p_directive_next(symbols): @@ -164,6 +169,7 @@ class ChordproParser(Parser): def p_line(symbols): """line : word line_next | chord line_next + | directive line_next """ symbols[0] = symbols[2].prepend(symbols[1]) @@ -172,6 +178,7 @@ class ChordproParser(Parser): """line_next : word line_next | space line_next | chord line_next + | directive line_next | empty """ if len(symbols) == 2: @@ -196,13 +203,13 @@ class ChordproParser(Parser): @staticmethod def p_chorus(symbols): - """chorus : SOC maybespace NEWLINE chorus_content EOC maybespace + """chorus : SOC maybespace ENDOFLINE chorus_content EOC maybespace """ symbols[0] = symbols[4] @staticmethod def p_chorus_content(symbols): - """chorus_content : line NEWLINE chorus_content + """chorus_content : line ENDOFLINE chorus_content | SPACE chorus_content | empty """ @@ -215,13 +222,13 @@ class ChordproParser(Parser): @staticmethod def p_bridge(symbols): - """bridge : SOB maybespace NEWLINE bridge_content EOB maybespace + """bridge : SOB maybespace ENDOFLINE bridge_content EOB maybespace """ symbols[0] = symbols[4] @staticmethod def p_bridge_content(symbols): - """bridge_content : line NEWLINE bridge_content + """bridge_content : line ENDOFLINE bridge_content | SPACE bridge_content | empty """ @@ -235,13 +242,13 @@ class ChordproParser(Parser): @staticmethod def p_tab(symbols): - """tab : SOT maybespace NEWLINE tab_content EOT maybespace + """tab : SOT maybespace ENDOFLINE tab_content EOT maybespace """ symbols[0] = symbols[4] @staticmethod def p_tab_content(symbols): - """tab_content : NEWLINE tab_content + """tab_content : ENDOFLINE tab_content | TEXT tab_content | SPACE tab_content | empty diff --git a/test/test_chordpro/metadata.sgc b/test/test_chordpro/metadata.sgc index e7b35506..b0428d3a 100644 --- a/test/test_chordpro/metadata.sgc +++ b/test/test_chordpro/metadata.sgc @@ -15,5 +15,7 @@ {comment: Comment} {guitar_comment: GuitarComment} -{image: Image} {partition: Lilypond} +{image: Image} + +Foo diff --git a/test/test_chordpro/metadata.source b/test/test_chordpro/metadata.source index 2e106444..65a2359a 100644 --- a/test/test_chordpro/metadata.source +++ b/test/test_chordpro/metadata.source @@ -15,5 +15,7 @@ {key: foo: Foo} {comment: Comment} {guitar_comment: GuitarComment} -{image: Image} {partition: Lilypond} +{image: Image} + +Foo diff --git a/test/test_chordpro/metadata.tex b/test/test_chordpro/metadata.tex index 555f59d4..5481121d 100644 --- a/test/test_chordpro/metadata.tex +++ b/test/test_chordpro/metadata.tex @@ -19,7 +19,10 @@ Subtitle5}[ \textnote{Comment} \musicnote{GuitarComment} -\image{Image} \lilypond{Lilypond} +\image{Image} +\begin{verse} + Foo +\end{verse} \endsong diff --git a/test/test_chordpro/newline.sgc b/test/test_chordpro/newline.sgc new file mode 100644 index 00000000..426c45f5 --- /dev/null +++ b/test/test_chordpro/newline.sgc @@ -0,0 +1,41 @@ +{language: english} + +This is a verse +With a new line +{newline} +The second part of the verse +Is this line + + +Here is a new line at the end +{newline} + + +Foo bar + + +{newline} +And a new line +At the beginning + + +{start_of_chorus} + New lines can also + {newline} + Be in chorus +{end_of_chorus} + + +{start_of_bridge} + New lines can also + {newline} + Be in bridges +{end_of_bridge} + + +New lines can also +{newline} +Be surrounded by spaces + + +New lines can {newline} appear in the middle of a line diff --git a/test/test_chordpro/newline.source b/test/test_chordpro/newline.source new file mode 100644 index 00000000..717639ee --- /dev/null +++ b/test/test_chordpro/newline.source @@ -0,0 +1,32 @@ +This is a verse +With a new line +{newline} +The second part of the verse +Is this line + +Here is a new line at the end +{newline} + +Foo bar + +{newline} +And a new line +At the beginning + +{soc} +New lines can also +{newline} +Be in chorus +{eoc} + +{sob} +New lines can also +{newline} +Be in bridges +{eob} + +New lines can also + {newline} +Be surrounded by spaces + +New lines can {newline} appear in the middle of a line diff --git a/test/test_chordpro/newline.tex b/test/test_chordpro/newline.tex new file mode 100644 index 00000000..f8ab7d02 --- /dev/null +++ b/test/test_chordpro/newline.tex @@ -0,0 +1,61 @@ +\selectlanguage{english} + +\beginsong{}[ + by={ + }, +] + + +\begin{verse} + This is a verse + With a new line + ~\\ + The second part of the verse + Is this line +\end{verse} + + +\begin{verse} + Here is a new line at the end + ~\\ +\end{verse} + + +\begin{verse} + Foo bar +\end{verse} + + +\begin{verse} + ~\\ + And a new line + At the beginning +\end{verse} + + +\begin{chorus} + New lines can also + ~\\ + Be in chorus +\end{chorus} + + +\begin{bridge} + New lines can also + ~\\ + Be in bridges +\end{bridge} + + +\begin{verse} + New lines can also + ~\\ + Be surrounded by spaces +\end{verse} + + +\begin{verse} + New lines can ~\\ appear in the middle of a line +\end{verse} + +\endsong From 39b61959f8848971bab17390161c30f7ab2669ff Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 12 Oct 2015 00:32:18 +0200 Subject: [PATCH 2/5] Directive {newline} is correctly parsed and rendered * Moreover, error messages also mention file name * Directives can now appear inside text (and not only on their own line) --- patacrep/data/examples/songs/errors.sgc | 2 + .../data/examples/songs/tests/newline.sgc | 35 +++++++++++++++ patacrep/songs/chordpro/ast.py | 43 +++++++++++-------- .../songs/chordpro/data/latex/content_verse | 6 +++ patacrep/songs/chordpro/lexer.py | 19 ++++---- patacrep/songs/chordpro/syntax.py | 10 +++-- patacrep/songs/syntax.py | 8 ++-- test/test_chordpro/greensleeves.sgc | 1 + test/test_chordpro/greensleeves.tex | 1 + test/test_chordpro/metadata.sgc | 1 + test/test_chordpro/metadata.tex | 3 ++ 11 files changed, 95 insertions(+), 34 deletions(-) create mode 100644 patacrep/data/examples/songs/tests/newline.sgc diff --git a/patacrep/data/examples/songs/errors.sgc b/patacrep/data/examples/songs/errors.sgc index 0b006cab..56cb61b7 100644 --- a/patacrep/data/examples/songs/errors.sgc +++ b/patacrep/data/examples/songs/errors.sgc @@ -10,4 +10,6 @@ Bla []bla Bla [H]bla +{fo#: bar} + diff --git a/patacrep/data/examples/songs/tests/newline.sgc b/patacrep/data/examples/songs/tests/newline.sgc new file mode 100644 index 00000000..4510d5ff --- /dev/null +++ b/patacrep/data/examples/songs/tests/newline.sgc @@ -0,0 +1,35 @@ +{title: Newline} +{subtitle: Test of "newline" directive} + +This is a verse +With a new line +{newline} +The second part of the verse +Is this line + +Here is a new line at the end +{newline} + +Foo bar + +{newline} +And a new line +At the beginning + +{soc} +New lines can also +{newline} +Be in chorus +{eoc} + +{sob} +New lines can also +{newline} +Be in bridges +{eob} + +New lines can also + {newline} +Be surrounded by spaces + +New lines can {newline} appear in the middle of a line diff --git a/patacrep/songs/chordpro/ast.py b/patacrep/songs/chordpro/ast.py index 7f563dea..9ee1a137 100644 --- a/patacrep/songs/chordpro/ast.py +++ b/patacrep/songs/chordpro/ast.py @@ -13,11 +13,12 @@ def _indent(string): #: List of properties that are to be displayed in the flow of the song (not as #: metadata at the beginning or end of song. -INLINE_PROPERTIES = { +INLINE_DIRECTIVES = { "partition", "comment", "guitar_comment", "image", + "newline", } #: Some directive have alternative names. For instance `{title: Foo}` and `{t: @@ -63,6 +64,9 @@ class Line(AST): super().__init__() self.line = [] + def __iter__(self): + yield from self.line + def prepend(self, data): """Add an object at the beginning of line. @@ -86,6 +90,7 @@ class Line(AST): return self def is_empty(self): + """Return `True` iff line is empty.""" return len(self.strip().line) == 0 class LineElement(AST): @@ -144,6 +149,14 @@ class Verse(AST): self.lines.insert(0, data) return self + def directive(self): + """Return `True` iff the verse is composed only of directives.""" + for line in self.lines: + for element in line: + if not isinstance(element, Directive): + return False + return True + class Chorus(Verse): """Chorus""" type = 'chorus' @@ -191,7 +204,7 @@ class Song(AST): # New line if not (self.content and isinstance(self.content[0], EndOfLine)): self.content.insert(0, EndOfLine()) - elif isinstance(data, Line) or isinstance(data, NewLine): + elif isinstance(data, Line): # Add a new line, maybe in the current verse. if not data.is_empty(): if not (self.content and isinstance(self.content[0], Verse)): @@ -273,7 +286,7 @@ class EndOfLine(AST): class Directive(AST): """A directive""" - meta = True + inline = False def __init__(self, keyword, argument=None): super().__init__() @@ -288,29 +301,23 @@ class Directive(AST): """ return self.keyword - @property - def inline(self): - """True iff this directive is to be rendered in the flow on the song. - """ - return self.keyword in INLINE_PROPERTIES - def __str__(self): return self.argument @classmethod def create(cls, keyword, argument=None): - if keyword == "newline": - return NewLine(keyword, argument) + """Create a :class:`Directive` or :class:`InlineDirective`. + + Depending on the keyword. + """ + if keyword in INLINE_DIRECTIVES: + return InlineDirective(keyword, argument) else: return cls(keyword, argument) -class NewLine(Directive): - keyword = "newline" - _template = "newline" - meta = False - - def strip(self): - return self +class InlineDirective(Directive): + """Directive displayed in the flow of the song""" + inline = True class Define(Directive): """A chord definition. diff --git a/patacrep/songs/chordpro/data/latex/content_verse b/patacrep/songs/chordpro/data/latex/content_verse index b6065a7f..8417534f 100644 --- a/patacrep/songs/chordpro/data/latex/content_verse +++ b/patacrep/songs/chordpro/data/latex/content_verse @@ -1,5 +1,11 @@ +(* if content.directive() *) + (* for line in content.lines -*) + ((- render(line) )) + (* endfor -*) +(*- else -*) \begin{(( content.type ))} (* for line in content.lines *) (( render(line) )) (* endfor *) \end{(( content.type ))} +(*- endif *) diff --git a/patacrep/songs/chordpro/lexer.py b/patacrep/songs/chordpro/lexer.py index 4ee20c4b..a0b57826 100644 --- a/patacrep/songs/chordpro/lexer.py +++ b/patacrep/songs/chordpro/lexer.py @@ -83,8 +83,9 @@ class ChordProLexer: t_tablature_TEXT = r'[^\n]+' t_tablature_ENDOFLINE = r'\n' - def __init__(self): + def __init__(self, *, filename=None): self.__class__.lexer = lex.lex(module=self) + self.filename = filename # Define a rule so we can track line numbers @staticmethod @@ -132,16 +133,16 @@ class ChordProLexer: self.lexer.push_state('directiveargument') return token - @staticmethod - def error(token, more=""): + def error(self, token, more=""): """Display error message, and skip illegal token.""" - LOGGER.error( - "Line {line}: Illegal character '{char}'{more}.".format( - line=token.lexer.lineno, - char=token.value[0], - more=more, - ) + message = "Line {line}: Illegal character '{char}'{more}.".format( + line=token.lexer.lineno, + char=token.value[0], + more=more, ) + if self.filename is not None: + message = "File {}: {}".format(self.filename, message) + LOGGER.error(message) token.lexer.skip(1) def t_error(self, token): diff --git a/patacrep/songs/chordpro/syntax.py b/patacrep/songs/chordpro/syntax.py index b06ec13c..8837f501 100644 --- a/patacrep/songs/chordpro/syntax.py +++ b/patacrep/songs/chordpro/syntax.py @@ -17,6 +17,7 @@ class ChordproParser(Parser): super().__init__() self.tokens = tokens self.song = ast.Song(filename) + self.filename = filename def p_song(self, symbols): """song : block song @@ -144,10 +145,11 @@ class ChordproParser(Parser): else: directive = ast.Directive.create(keyword, argument) - if directive.meta: - self.song.add(directive) - else: + if directive.inline: symbols[0] = directive + else: + self.song.add(directive) + @staticmethod def p_directive_next(symbols): @@ -273,5 +275,5 @@ def parse_song(content, filename=None): write_tables=0, ).parse( content, - lexer=ChordProLexer().lexer, + lexer=ChordProLexer(filename=filename).lexer, ) diff --git a/patacrep/songs/syntax.py b/patacrep/songs/syntax.py index 7f019374..ee7d95da 100644 --- a/patacrep/songs/syntax.py +++ b/patacrep/songs/syntax.py @@ -20,8 +20,7 @@ class Parser: column = (token.lexpos - last_cr) + 1 return column - @staticmethod - def error(*, line=None, column=None, message=""): + def error(self, *, line=None, column=None, message=""): """Display an error message""" coordinates = [] if line is not None: @@ -35,7 +34,10 @@ class Parser: text += message else: text += "." - LOGGER.error(text) + if self.filename is None: + LOGGER.error(text) + else: + LOGGER.error("File {}: {}".format(self.filename, text)) def p_error(self, token): """Manage parsing errors.""" diff --git a/test/test_chordpro/greensleeves.sgc b/test/test_chordpro/greensleeves.sgc index 566669b0..eafed1a2 100644 --- a/test/test_chordpro/greensleeves.sgc +++ b/test/test_chordpro/greensleeves.sgc @@ -10,6 +10,7 @@ {partition: greensleeves.ly} + A[Am]las, my love, ye [G]do me wrong To [Am]cast me oft dis[E]curteously And [Am]I have loved [G]you so long diff --git a/test/test_chordpro/greensleeves.tex b/test/test_chordpro/greensleeves.tex index 905e1664..f760253c 100644 --- a/test/test_chordpro/greensleeves.tex +++ b/test/test_chordpro/greensleeves.tex @@ -17,6 +17,7 @@ Un sous titre}[ \lilypond{greensleeves.ly} + \begin{verse} A\[Am]las, my love, ye \[G]do me wrong To \[Am]cast me oft dis\[E]curteously diff --git a/test/test_chordpro/metadata.sgc b/test/test_chordpro/metadata.sgc index b0428d3a..499bb833 100644 --- a/test/test_chordpro/metadata.sgc +++ b/test/test_chordpro/metadata.sgc @@ -18,4 +18,5 @@ {partition: Lilypond} {image: Image} + Foo diff --git a/test/test_chordpro/metadata.tex b/test/test_chordpro/metadata.tex index 5481121d..bb5393a1 100644 --- a/test/test_chordpro/metadata.tex +++ b/test/test_chordpro/metadata.tex @@ -22,7 +22,10 @@ Subtitle5}[ \lilypond{Lilypond} \image{Image} + + \begin{verse} Foo \end{verse} + \endsong From 124e11b20c74b2a165071c56315ccc585f3b3033 Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 12 Oct 2015 17:10:29 +0200 Subject: [PATCH 3/5] Code simplification --- patacrep/songs/chordpro/ast.py | 19 ++++--------------- patacrep/songs/chordpro/syntax.py | 2 +- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/patacrep/songs/chordpro/ast.py b/patacrep/songs/chordpro/ast.py index 9ee1a137..7b331ecd 100644 --- a/patacrep/songs/chordpro/ast.py +++ b/patacrep/songs/chordpro/ast.py @@ -286,7 +286,6 @@ class EndOfLine(AST): class Directive(AST): """A directive""" - inline = False def __init__(self, keyword, argument=None): super().__init__() @@ -304,20 +303,10 @@ class Directive(AST): def __str__(self): return self.argument - @classmethod - def create(cls, keyword, argument=None): - """Create a :class:`Directive` or :class:`InlineDirective`. - - Depending on the keyword. - """ - if keyword in INLINE_DIRECTIVES: - return InlineDirective(keyword, argument) - else: - return cls(keyword, argument) - -class InlineDirective(Directive): - """Directive displayed in the flow of the song""" - inline = True + @property + def inline(self): + """Return `True` iff `self` is an inline directive.""" + return self.keyword in INLINE_DIRECTIVES class Define(Directive): """A chord definition. diff --git a/patacrep/songs/chordpro/syntax.py b/patacrep/songs/chordpro/syntax.py index 8837f501..c86e7ef4 100644 --- a/patacrep/songs/chordpro/syntax.py +++ b/patacrep/songs/chordpro/syntax.py @@ -144,7 +144,7 @@ class ChordproParser(Parser): self.song.add(define) else: - directive = ast.Directive.create(keyword, argument) + directive = ast.Directive(keyword, argument) if directive.inline: symbols[0] = directive else: From 20310279c573adf358cdb99511ec8d0f8a846449 Mon Sep 17 00:00:00 2001 From: Oliverpool Date: Mon, 12 Oct 2015 18:53:43 +0200 Subject: [PATCH 4/5] [WIP] add an html test (failing) --- .../songs/chordpro/data/html/content_newline | 1 - .../songs/chordpro/data/html/content_verse | 8 +-- test/test_chordpro/newline.html | 50 +++++++++++++++++++ test/test_chordpro/test_parser.py | 1 + 4 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 test/test_chordpro/newline.html diff --git a/patacrep/songs/chordpro/data/html/content_newline b/patacrep/songs/chordpro/data/html/content_newline index 1915ec14..e69de29b 100644 --- a/patacrep/songs/chordpro/data/html/content_newline +++ b/patacrep/songs/chordpro/data/html/content_newline @@ -1 +0,0 @@ -
diff --git a/patacrep/songs/chordpro/data/html/content_verse b/patacrep/songs/chordpro/data/html/content_verse index eab4d361..6fe21f6b 100644 --- a/patacrep/songs/chordpro/data/html/content_verse +++ b/patacrep/songs/chordpro/data/html/content_verse @@ -1,6 +1,8 @@

- (*- for line in content.lines -*) - (* if not loop.first *)
(* endif -*) + (* for line in content.lines *) (( render(line) )) - (* endfor -*) + (*- if not loop.last *)
+ (* endif *) + (* endfor *) +

diff --git a/test/test_chordpro/newline.html b/test/test_chordpro/newline.html new file mode 100644 index 00000000..16cba958 --- /dev/null +++ b/test/test_chordpro/newline.html @@ -0,0 +1,50 @@ +Language: english
+ + + +
+

+ This is a verse
+ With a new line
+
+ The second part of the verse
+ Is this line +

+ +

+ Here is a new line at the end
+ +

+ +

+ Foo bar +

+ +

+
+ And a new line
+ At the beginning +

+ +

+ New lines can also
+
+ Be in chorus +

+ +

+ New lines can also
+
+ Be in bridges +

+ +

+ New lines can also
+
+ Be surrounded by spaces +

+ +

+ New lines can't appear in the middle of a line +

+
diff --git a/test/test_chordpro/test_parser.py b/test/test_chordpro/test_parser.py index 8f857156..805d1583 100644 --- a/test/test_chordpro/test_parser.py +++ b/test/test_chordpro/test_parser.py @@ -14,6 +14,7 @@ from .. import disable_logging LANGUAGES = { 'tex': 'latex', 'sgc': 'chordpro', + 'html': 'html', } class FileTestMeta(type): From 33328e789fcf5a5097db8f39bd64bf05b83bffbf Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 12 Oct 2015 23:45:01 +0200 Subject: [PATCH 5/5] Directives are allowed only on their own line (with nothing else on this line) Better error handling of the parser. --- .../data/examples/songs/tests/newline.sgc | 6 +- patacrep/songs/chordpro/ast.py | 6 +- patacrep/songs/chordpro/syntax.py | 61 +++++++++++++++---- test/test_chordpro/newline.html | 2 +- test/test_chordpro/newline.sgc | 2 +- test/test_chordpro/newline.source | 2 +- test/test_chordpro/newline.tex | 2 +- 7 files changed, 62 insertions(+), 19 deletions(-) diff --git a/patacrep/data/examples/songs/tests/newline.sgc b/patacrep/data/examples/songs/tests/newline.sgc index 4510d5ff..c14acb20 100644 --- a/patacrep/data/examples/songs/tests/newline.sgc +++ b/patacrep/data/examples/songs/tests/newline.sgc @@ -32,4 +32,8 @@ New lines can also {newline} Be surrounded by spaces -New lines can {newline} appear in the middle of a line +New lines cannot have text before them {newline} + +{newline} New lines cannot have text after them + +New lines cannot be {newline} surrounded by text. diff --git a/patacrep/songs/chordpro/ast.py b/patacrep/songs/chordpro/ast.py index 80949e10..18805e9b 100644 --- a/patacrep/songs/chordpro/ast.py +++ b/patacrep/songs/chordpro/ast.py @@ -60,9 +60,9 @@ class Line(AST): _template = "line" - def __init__(self): + def __init__(self, *items): super().__init__() - self.line = [] + self.line = list(items) def __iter__(self): yield from self.line @@ -310,7 +310,7 @@ class Directive(AST): return self.keyword def __str__(self): - return self.argument + return str(self.argument) @property def inline(self): diff --git a/patacrep/songs/chordpro/syntax.py b/patacrep/songs/chordpro/syntax.py index c86e7ef4..e88d7360 100644 --- a/patacrep/songs/chordpro/syntax.py +++ b/patacrep/songs/chordpro/syntax.py @@ -1,5 +1,6 @@ """ChordPro parser""" +import logging import ply.yacc as yacc import re @@ -7,6 +8,8 @@ from patacrep.songs.syntax import Parser from patacrep.songs.chordpro import ast from patacrep.songs.chordpro.lexer import tokens, ChordProLexer +LOGGER = logging.getLogger() + class ChordproParser(Parser): """ChordPro parser class""" # pylint: disable=too-many-public-methods @@ -18,6 +21,19 @@ class ChordproParser(Parser): self.tokens = tokens self.song = ast.Song(filename) self.filename = filename + self.parser = yacc.yacc( + module=self, + debug=0, + write_tables=0, + ) + + def parse(self, *args, **kwargs): + """Parse file + + This is a shortcut to `yacc.yacc(...).parse()`. The arguments are + transmitted to this method. + """ + return self.parser.parse(*args, **kwargs) def p_song(self, symbols): """song : block song @@ -32,6 +48,7 @@ class ChordproParser(Parser): def p_block(symbols): """block : SPACE block | line ENDOFLINE + | line_error ENDOFLINE | chorus ENDOFLINE | tab ENDOFLINE | bridge ENDOFLINE @@ -167,20 +184,35 @@ class ChordproParser(Parser): else: symbols[0] = None + @staticmethod + def p_line_error(symbols): + """line_error : error directive""" + LOGGER.error("Directive can only be preceded or followed by spaces") + symbols[0] = ast.Line() + @staticmethod def p_line(symbols): """line : word line_next | chord line_next - | directive line_next + | directive maybespace """ - symbols[0] = symbols[2].prepend(symbols[1]) + if isinstance(symbols[2], ast.Line): + # Line with words, etc. + symbols[0] = symbols[2].prepend(symbols[1]) + else: + # Directive + if symbols[1] is None: + # Meta directive. Nothing to do + symbols[0] = ast.Line() + else: + # Inline directive + symbols[0] = ast.Line(symbols[1]) @staticmethod def p_line_next(symbols): """line_next : word line_next | space line_next | chord line_next - | directive line_next | empty """ if len(symbols) == 2: @@ -212,6 +244,7 @@ class ChordproParser(Parser): @staticmethod def p_chorus_content(symbols): """chorus_content : line ENDOFLINE chorus_content + | line_error ENDOFLINE chorus_content | SPACE chorus_content | empty """ @@ -231,6 +264,7 @@ class ChordproParser(Parser): @staticmethod def p_bridge_content(symbols): """bridge_content : line ENDOFLINE bridge_content + | line_error ENDOFLINE bridge_content | SPACE bridge_content | empty """ @@ -267,13 +301,18 @@ class ChordproParser(Parser): """empty :""" symbols[0] = None + def p_error(self, token): + super().p_error(token) + while True: + token = self.parser.token() + if not token or token.type == "ENDOFLINE": + break + self.parser.errok() + return token + def parse_song(content, filename=None): """Parse song and return its metadata.""" - return yacc.yacc( - module=ChordproParser(filename), - debug=0, - write_tables=0, - ).parse( - content, - lexer=ChordProLexer(filename=filename).lexer, - ) + return ChordproParser(filename).parse( + content, + lexer=ChordProLexer(filename=filename).lexer, + ) diff --git a/test/test_chordpro/newline.html b/test/test_chordpro/newline.html index 16cba958..1fbb02a3 100644 --- a/test/test_chordpro/newline.html +++ b/test/test_chordpro/newline.html @@ -45,6 +45,6 @@

- New lines can't appear in the middle of a line + New lines cannot

diff --git a/test/test_chordpro/newline.sgc b/test/test_chordpro/newline.sgc index 426c45f5..72bdc1a2 100644 --- a/test/test_chordpro/newline.sgc +++ b/test/test_chordpro/newline.sgc @@ -38,4 +38,4 @@ New lines can also Be surrounded by spaces -New lines can {newline} appear in the middle of a line +New lines cannot diff --git a/test/test_chordpro/newline.source b/test/test_chordpro/newline.source index 717639ee..03095b88 100644 --- a/test/test_chordpro/newline.source +++ b/test/test_chordpro/newline.source @@ -29,4 +29,4 @@ New lines can also {newline} Be surrounded by spaces -New lines can {newline} appear in the middle of a line +New lines cannot {newline} appear in the middle of a line diff --git a/test/test_chordpro/newline.tex b/test/test_chordpro/newline.tex index f8ab7d02..d557de5f 100644 --- a/test/test_chordpro/newline.tex +++ b/test/test_chordpro/newline.tex @@ -55,7 +55,7 @@ \begin{verse} - New lines can ~\\ appear in the middle of a line + New lines cannot \end{verse} \endsong