Merge pull request #213 from patacrep/latex_special

Handle LaTeX special characters

patacrep/data/templates/songbook/default.tex

@@ -66,8 +66,8 @@ description:
 
 \usepackage{chords}
 
-\title{(( template_var.title ))}
+\title{(( template_var.title|escape_specials() ))}
-\author{(( template_var.author ))}
+\author{(( template_var.author|escape_specials() ))}
 
 \newindex{titleidx}{((filename))_title}
 \newauthorindex{authidx}{((filename))_auth}

patacrep/data/templates/songbook/patacrep.tex

@@ -138,12 +138,12 @@ description:
     ]{hyperref}
 
 
-\subtitle{(( template_var.subtitle ))}
+\subtitle{(( template_var.subtitle|escape_specials ))}
 (* if template_var.version -*)
     \version{(( template_var.version ))}
 (* endif *)
-\mail{(( template_var.email ))}
+\mail{(( template_var.email|escape_url ))}
-\web{(( template_var.url ))}
+\web{(( template_var.url|escape_url ))}
 \picture{(( template_var.picture ))}
 \picturecopyright{(( template_var.picturecopyright ))}
 \footer{(( template_var.footer ))}

patacrep/data/templates/songs/chordpro/chordpro/content_word

@@ -1 +1 @@
-(( content.value ))
+(( content.value|escape_specials('{}\\#') ))

patacrep/data/templates/songs/chordpro/chordpro/song_header

@@ -9,16 +9,16 @@
 (* endif *)
 
 (*- for title in titles -*)
-  {title: (( title ))}
+  {title: (( title|escape_specials('{}\\') ))}
 (* endfor -*)
 
 (*- for author in authors -*)
-  {artist: (( author[1] )) (( author[0] ))}
+  {artist: (( author[1]|escape_specials('{}\\') )) (( author[0]|escape_specials('{}\\') ))}
 (* endfor -*)
 
 (*- for key in ['album', 'copyright'] *)
   (* if key in metadata -*)
-    {(( key )): (( metadata[key] ))}
+    {(( key )): (( metadata[key]|escape_specials('{}\\') ))}
   (* endif *)
 (* endfor *)
 (* if 'cover' in metadata -*)
@@ -30,9 +30,13 @@
 (* endfor -*)
 
 (*- for key in metadata.morekeys -*)
-  {key: (( key.keyword )): (( key.argument ))}
+  {key: (( key.keyword )): (( key.argument|escape_specials('{}\\') ))}
 (* endfor *)
 
+(*- if 'url' in metadata -*)
+  {url: (( metadata.url|escape_url ))}
+(* endif -*)
+
 (*- for chord in metadata['define'] *)
   ((- render(chord) ))
 (* endfor *)

patacrep/data/templates/songs/chordpro/latex/content_word

@@ -1 +1 @@
-(( content.value ))
+(( content.value|escape_specials('{}&#_^%~$\\') ))

patacrep/data/templates/songs/chordpro/latex/song

@@ -8,7 +8,7 @@
 
 \beginsong{
     (*- for title in titles -*)
-        (( title ))
+        (( title|escape_specials('{}&#_^%~$\\') ))
         (*- if not loop.last -*)
             \\
         (* endif *)
@@ -16,7 +16,7 @@
 }[
   by={
       (* for author in authors *)
-        (( author[1] )) (( author[0] ))
+        (( author[1]|escape_specials('{}&#_^%~$\\') )) (( author[0]|escape_specials('{}&#_^%~$\\') ))
         (*- if not loop.last -*)
         ,
         (* endif *)
@@ -24,9 +24,12 @@
   },
   (* for key in ['album', 'copyright'] *)
   (* if key in metadata *)
-  (( key ))={(( metadata[key] ))},
+  (( key ))={(( metadata[key]|escape_specials('{}&#_^%~$\\') ))},
   (* endif *)
   (* endfor *)
+  (* if 'url' in metadata *)
+  url={(( metadata.url|escape_url ))},
+  (* endif *)
   (* if 'cover' in metadata *)
     (* block cover *)
       (* set cover = metadata["cover"].argument|search_image|path2posix *)
@@ -36,7 +39,7 @@
     (* endblock *)
   (* endif *)
   (* for key in metadata.morekeys *)
-  (( key.keyword ))={(( key.argument ))},
+  (( key.keyword ))={(( key.argument|escape_specials('{}&#_^%~$\\') ))},
   (* endfor *)
 ]
 

patacrep/data/templates/styles/crepbook.sty

@@ -79,8 +79,8 @@
 % Title page
 \long\def\subtitle#1{\long\def\@subtitle{#1}}
 \def\version#1{\def\@version{#1}}
-\def\web#1{\def\@web{#1}}
+\def\web#1{\def\@web{\url{#1}}}
-\def\mail#1{\def\@mail{#1}}
+\def\mail#1{\def\@mail{\href{mailto:#1}{\nolinkurl{#1}}}}
 \def\email#1{\def\@email{#1}}
 \def\picture#1{\def\@picture{#1}}
 \def\picturecopyright#1{\def\@picturecopyright{#1}}

patacrep/songs/chordpro/__init__.py

@@ -3,6 +3,7 @@
 import logging
 import operator
 import os
+import urllib
 
 from jinja2 import Environment, FileSystemLoader, ChoiceLoader
 from jinja2 import contextfunction
@@ -30,6 +31,13 @@ class ChordproSong(Song):
     # pylint: disable=abstract-method
 
     output_language = None
+    _translation_map = {}
+    _translation_map_url = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self._translation_map_url is None:
+            self._translation_map_url = self._translation_map
 
     def _parse(self):
         """Parse content, and return the dictionary of song data."""
@@ -50,6 +58,8 @@ class ChordproSong(Song):
         filters.update({
             'search_image': self.search_image,
             'search_partition': self.search_partition,
+            'escape_specials': self._escape_specials,
+            'escape_url': self._escape_url,
         })
         return filters
 
@@ -84,6 +94,20 @@ class ChordproSong(Song):
         context.vars['content'] = content
         return context.environment.get_template(content.template()).render(context)
 
+    def _escape_specials(self, content, chars=None, *, translation_map=None):
+        if translation_map is None:
+            translation_map = self._translation_map
+        if chars is None:
+            chars = translation_map.keys()
+        return str(content).translate(str.maketrans({
+            key: value
+            for key, value in translation_map.items()
+            if key in chars
+            }))
+
+    def _escape_url(self, content):
+        return self._escape_specials(content, translation_map=self._translation_map_url)
+
 class Chordpro2HtmlSong(ChordproSong):
     """Render chordpro song to html code"""
 
@@ -104,6 +128,25 @@ class Chordpro2LatexSong(ChordproSong):
     """Render chordpro song to latex code"""
 
     output_language = "latex"
+    _translation_map = {
+        '{': r'\{',
+        '}': r'\}',
+        '\\': r'\textbackslash{}',
+        '^': r'\textasciicircum{}',
+        '~': r'\textasciitilde{}',
+        '#': r'\#',
+        '&': r'\&',
+        '$': r'\$',
+        '%': r'\%',
+        '_': r'\_',
+    }
+    _translation_map_url = {
+        " ": urllib.parse.quote(" "),
+        "{": urllib.parse.quote("{"),
+        "}": urllib.parse.quote("}"),
+        '%': r'\%',
+        '#': r'\#',
+        }
 
     def search_file(self, filename, extensions=None, *, datadirs=None):
         _datadir, filename, _extension = self.search_datadir_file(
@@ -164,6 +207,17 @@ class Chordpro2ChordproSong(ChordproSong):
     """Render chordpro song to chordpro code"""
 
     output_language = "chordpro"
+    _translation_map = {
+        '{': r'\{',
+        '}': r'\}',
+        '\\': '\\\\',
+        '#': r'\#',
+    }
+    _translation_map_url = {
+        '{': r'\{',
+        '}': r'\}',
+        '\\': '\\\\',
+    }
 
     def search_file(self, filename, extensions=None, *, datadirs=None):
         # pylint: disable=unused-variable

patacrep/songs/chordpro/lexer.py

@@ -49,7 +49,7 @@ class ChordProLexer:
 
     t_directive_SPACE = r'[ \t]+'
     t_directive_KEYWORD = r'[a-zA-Z_]+'
-    t_directiveargument_TEXT = r'[^}]+'
+    t_directiveargument_TEXT = r'[^\\}]+'
 
     @staticmethod
     def t_SOC(token):
@@ -118,7 +118,7 @@ class ChordProLexer:
 
     @staticmethod
     def t_WORD(token):
-        r'[^{}\r\n\]\[\t ]+'
+        r'[^{}\\\r\n\]\[\t ]+'
         return token
 
     def t_LBRACKET(self, __token):
@@ -150,6 +150,20 @@ class ChordProLexer:
         self.lexer.push_state('directiveargument')
         return token
 
+    @staticmethod
+    def t_ESCAPED(token):
+        r'\\[{} #\\]'
+        token.value = token.value[1]
+        token.type = "WORD"
+        return token
+
+    @staticmethod
+    def t_directiveargument_ESCAPED(token):
+        r'\\[{} #\\]'
+        token.value = token.value[1]
+        token.type = "TEXT"
+        return token
+
     def error(self, token, more=""):
         """Display error message, and skip illegal token."""
         message = "Illegal character '{char}'{more}.".format(

patacrep/songs/chordpro/syntax.py

@@ -182,9 +182,8 @@ class ChordproParser(Parser):
 
     @staticmethod
     def p_directive_next(symbols):
-        """directive_next : SPACE COLON TEXT
+        """directive_next : SPACE COLON directive_argument
-                          | COLON TEXT
+                          | COLON directive_argument
-                          | COLON
                           | empty
         """
         if len(symbols) == 3:
@@ -196,6 +195,16 @@ class ChordproParser(Parser):
         else:
             symbols[0] = None
 
+    @staticmethod
+    def p_directive_argument(symbols):
+        """directive_argument : TEXT directive_argument
+                              | empty
+        """
+        if len(symbols) == 3:
+            symbols[0] = symbols[1] + symbols[2]
+        else:
+            symbols[0] = ""
+
     def p_line_error(self, symbols):
         """line_error : error directive"""
         self.error(

patacrep/templates.py

@@ -2,6 +2,7 @@
 
 import logging
 import re
+import urllib
 
 import yaml
 
@@ -16,15 +17,6 @@ import patacrep.encoding
 
 LOGGER = logging.getLogger(__name__)
 
-_LATEX_SUBS = (
-    (re.compile(r'\\'), r'\\textbackslash'),
-    (re.compile(r'([{}_#%&$])'), r'\\\1'),
-    (re.compile(r'~'), r'\~{}'),
-    (re.compile(r'\^'), r'\^{}'),
-    (re.compile(r'"'), r"''"),
-    (re.compile(r'\.\.\.+'), r'\\ldots'),
-)
-
 _VARIABLE_REGEXP = re.compile(
     r"""
     \(\*-?\ *variables\ *\*\)    # Match (* variables *) or (*- variables *)
@@ -46,15 +38,47 @@ _VARIABLE_REGEXP = re.compile(
     """,
     re.VERBOSE|re.DOTALL)
 
-def _escape_tex(value):
+TRANSLATION_MAP = {
+    '{': r'\{',
+    '}': r'\}',
+    '\\': r'\textbackslash{}',
+    '^': r'\textasciicircum{}',
+    '~': r'\textasciitilde{}',
+    '#': r'\#',
+    '&': r'\&',
+    '$': r'\$',
+    '%': r'\%',
+    '_': r'\_',
+}
+TRANSLATION_MAP_URL = {
+    ' ': '\\' + urllib.parse.quote(" "),
+    '{': '\\' + urllib.parse.quote("{"),
+    '}': '\\' + urllib.parse.quote("}"),
+    '%': '\\%',
+    '\\': '\\\\',
+    '#': '\\#',
+    '&': '\\&',
+    }
+
+def _escape_specials(text, *, chars=None, translation_map=None):
     '''Escape TeX special characters'''
-    newval = value
+    if translation_map is None:
-    for pattern, replacement in _LATEX_SUBS:
+        translation_map = TRANSLATION_MAP
-        newval = pattern.sub(replacement, newval)
+    if chars is None:
-    return newval
+        chars = translation_map.keys()
+    return str(text).translate(str.maketrans({
+        key: value
+        for key, value in translation_map.items()
+        if key in chars
+        }))
+
+def _escape_url(text):
+    """Escape TeX special characters, in url."""
+    return _escape_specials(text, translation_map=TRANSLATION_MAP_URL)
 
 DEFAULT_FILTERS = {
-    "escape_tex": _escape_tex,
+    "escape_specials": _escape_specials,
+    "escape_url": _escape_url,
     "iter_datadirs": files.iter_datadirs,
     "path2posix": files.path2posix,
     }

test/test_book/special.tex.control

@@ -0,0 +1,162 @@
+
+
+
+
+
+
+%% Automatically generated document.
+%% You may edit this file but all changes will be overwritten.
+%% If you want to change this document, have a look at
+%% the templating system.
+%%
+%% Generated using Songbook <http://www.patacrep.com>
+
+\makeatletter
+\def\input@path{ %
+        {@TEST_FOLDER@/special_datadir/templates/styles/} %
+        {@TEST_FOLDER@/templates/styles/} %
+        {@DATA_FOLDER@/templates/styles/} %
+}
+\makeatother
+
+\documentclass[
+    ]{article}
+
+\usepackage[
+chorded,
+pictures,
+repeatchords,
+importantdiagramonly,
+diagrampage,
+guitar,
+    ]{crepbook}
+
+\usepackage[
+     a4paper % paper size
+     ,includeheadfoot % include header and footer into text size
+     ,hmarginratio=1:1 % ratio between inner and outer margin (default)
+     ,outer=1.8cm % outer margin (right)
+     ,vmarginratio=1:1 % ratio between top and bottom margin
+     ,bmargin=1.3cm % bottom margin
+     ]{geometry}
+
+\usepackage{lmodern}
+
+
+\PassOptionsToPackage{english}{babel}
+\usepackage[english]{babel}
+\lang{english}
+
+\usepackage{graphicx}
+\graphicspath{ %
+        {@TEST_FOLDER@/special_datadir/} %
+        {@TEST_FOLDER@/} %
+        {@DATA_FOLDER@/} %
+}
+
+
+\makeatletter
+\@ifpackageloaded{hyperref}{}{
+   \usepackage{url}
+   \newcommand{\phantomsection}{}
+   \newcommand{\hyperlink}[2]{#2}
+   \newcommand{\href}[2]{\expandafter\url\expandafter{#1}}
+}
+\makeatother
+
+
+\usepackage{chords}
+
+\title{\& \% \$ \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}}
+\author{\& \% \$ \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}}
+
+\newindex{titleidx}{special_title}
+\newauthorindex{authidx}{special_auth}
+
+\authignoreword{unknown}
+\authbyword{by}
+\authsepword{and}
+
+\notenamesout{A}{B}{C}{D}{E}{F}{G}
+
+
+\pagestyle{empty}\definecolor{SongNumberBgColor}{HTML}{D1E4AE}
+\definecolor{NoteBgColor}{HTML}{D1E4AE}
+\definecolor{IndexBgColor}{HTML}{D1E4AE}
+
+\renewcommand{\snumbgcolor}{SongNumberBgColor}
+\renewcommand{\notebgcolor}{NoteBgColor}
+\renewcommand{\idxbgcolor}{IndexBgColor}
+
+\definecolor{tango-green-3}{HTML}{4e9a06}
+\definecolor{tango-blue-3}{HTML}{204a87}
+\usepackage[
+    bookmarks,
+    bookmarksopen,
+    hyperfigures=true,
+    colorlinks=true,
+    linkcolor=tango-green-3,
+    urlcolor=tango-blue-3
+    ]{hyperref}
+
+
+\subtitle{\& \% \$ \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}}
+\mail{foo@\\\%\&$\#_~^\%20\%7B\%7D}
+\web{http://\\\%\&$\#_~^\%20\%7B\%7D}
+\picture{img/treble_a}
+\picturecopyright{Dbolton \url{http://commons.wikimedia.org/wiki/User:Dbolton}}
+\footer{Generated using Songbook (\url{http://www.patacrep.com})}
+
+
+\begin{document}
+
+\maketitle
+
+
+\showindex{\songindexname}{titleidx}
+\showindex{\authorindexname}{authidx}
+
+% list of chords
+\ifdiagrampage
+   \phantomsection
+   \addcontentsline{toc}{section}{\chordlistname}
+   \chords
+\fi
+\setcounter{songnum}{1}%
+
+\phantomsection
+\addcontentsline{toc}{section}{\songlistname}
+
+\begin{songs}{titleidx,authidx}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% songs/./special.csg
+
+\selectlanguage{english}
+
+\beginsong{\& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}}[
+  by={
+        \& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}  },
+  album={\& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}},
+  url={http://&$\%\#_~^},
+]
+
+
+
+
+\begin{verse}
+    \& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}
+\end{verse}
+
+
+\begin{chorus}
+    \& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}
+\end{chorus}
+
+\endsong
+
+\end{songs}
+
+
+
+
+\end{document}

test/test_book/special.yaml

@@ -0,0 +1,14 @@
+book:
+  datadir:
+  - special_datadir
+
+template:
+  default.tex:
+    title: "& % $ # _ } { ~ ^ \\"
+    author: "& % $ # _ } { ~ ^ \\"
+  patacrep.tex:
+    subtitle: "& % $ # _ } { ~ ^ \\"
+    url: "http://\\%&$#_~^ {}"
+    email: "foo@\\%&$#_~^ {}"
+    picture: "img/treble_a"
+

test/test_book/special_datadir/songs/special.csg

@@ -0,0 +1,10 @@
+{title: & $ % # _ \} \{ ~ ^ \\}
+{artist:  & $ % # _ \} \{ ~ ^ \\}
+{album: & $ % # _ \} \{ ~ ^ \\}
+{url: http://&$%#_~^}
+
+& $ % \# _ \} \{ ~ ^ \\
+
+{start_of_chorus}
+& $ % \# _ \} \{ ~ ^ \\
+{end_of_chorus}

test/test_song/special.csg

@@ -0,0 +1,13 @@
+{lang: en}
+{title: & $ % # _ \} \{ ~ ^ \\}
+{artist: & $ % # _ \} \{ ~ ^ \\}
+{album: & $ % # _ \} \{ ~ ^ \\}
+{url: http://&$%#_~^}
+
+
+& $ % \# _ \} \{ ~ ^ \\
+
+
+{start_of_chorus}
+    & $ % \# _ \} \{ ~ ^ \\
+{end_of_chorus}

test/test_song/special.csg.source

@@ -0,0 +1,10 @@
+{title: & $ % # _ \} \{ ~ ^ \\}
+{artist:  & $ % # _ \} \{ ~ ^ \\}
+{album: & $ % # _ \} \{ ~ ^ \\}
+{url: http://&$%#_~^}
+
+& $ % \# _ \} \{ ~ ^ \\
+
+{start_of_chorus}
+& $ % \# _ \} \{ ~ ^ \\
+{end_of_chorus}

test/test_song/special.tsg

@@ -0,0 +1,22 @@
+\selectlanguage{english}
+
+\beginsong{\& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}}[
+  by={
+        \& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}  },
+  album={\& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}},
+  url={http://&$\%\#_~^},
+]
+
+
+
+
+\begin{verse}
+    \& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}
+\end{verse}
+
+
+\begin{chorus}
+    \& \$ \% \# \_ \} \{ \textasciitilde{} \textasciicircum{} \textbackslash{}
+\end{chorus}
+
+\endsong