diff --git a/.gitignore b/.gitignore index 69298645..a8e9e8cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ deb_dist build dist -.gitignore +#.gitignore *~ *.aux *.sbd diff --git a/NEWS.md b/NEWS.md index 4e374f6c..06eb7aab 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,7 @@ * Fix the import when the content folder is zipped [#235](https://github.com/patacrep/patacrep/pull/235) * PDF generation * The bookmarks correctly use the unicode encoding [#225](https://github.com/patacrep/patacrep/pull/225) + * The first letters of the index are "unidecoded" [#231](https://github.com/patacrep/patacrep/pull/231) * Enhancements * Error management * The 'error' option is considered for contentlist errors (incorrect syntax for instance) [#226](https://github.com/patacrep/patacrep/pull/226) diff --git a/patacrep/build.py b/patacrep/build.py index a66619d7..15a4ca78 100644 --- a/patacrep/build.py +++ b/patacrep/build.py @@ -307,8 +307,8 @@ class SongbookBuilder: raise errors.LatexCompilationError(self.basename) def build_sbx(self): - """Make index""" - LOGGER.info("Building indexes…") + """Make .sbx indexes from .sxd files""" + LOGGER.info("Building .sbx indexes…") sxd_files = glob.glob("%s_*.sxd" % self.basename) for sxd_file in sxd_files: LOGGER.debug("Processing " + sxd_file) diff --git a/patacrep/content/sort.py b/patacrep/content/sort.py index cb10d157..e4bde299 100755 --- a/patacrep/content/sort.py +++ b/patacrep/content/sort.py @@ -5,27 +5,17 @@ to a songbook. """ import logging -import unidecode from patacrep import files from patacrep.content import ContentError from patacrep.content import process_content, validate_parser_argument from patacrep.content.song import OnlySongsError +from patacrep.utils import normalize_string LOGGER = logging.getLogger(__name__) DEFAULT_SORT = ['by', 'album', 'title'] -def normalize_string(string): - """Return a normalized string. - - Normalized means: - - no surrounding spaces; - - lower case; - - passed through unidecode.unidecode(). - """ - return unidecode.unidecode(string.lower().strip()) - def normalize_field(field): """Return a normalized field, it being a string or a list of strings.""" if isinstance(field, str): diff --git a/patacrep/index.py b/patacrep/index.py index e520e713..75d55107 100644 --- a/patacrep/index.py +++ b/patacrep/index.py @@ -7,11 +7,11 @@ from a file generated by the latex compilation of the songbook (.sxd). import locale import re -import unidecode from patacrep import authors from patacrep import encoding from patacrep.latex import tex2plain +from patacrep.utils import normalize_string EOL = "\n" @@ -67,7 +67,7 @@ class Index: def get_first_letter(key): """Return the uppercase first letter of key.""" try: - letter = FIRST_LETTER_PATTERN.match(key).group(1) + letter = FIRST_LETTER_PATTERN.match(normalize_string(key)).group(1) except AttributeError: # classify as number all the non letter characters letter = "0" @@ -108,7 +108,7 @@ class Index: if key not in self.data[first]: self.data[first][key] = { 'sortingkey': [ - unidecode.unidecode(tex2plain(item)).lower() + normalize_string(tex2plain(item)) for item in key ], 'entries': [], diff --git a/patacrep/utils.py b/patacrep/utils.py index 5051a592..1d139f6a 100644 --- a/patacrep/utils.py +++ b/patacrep/utils.py @@ -1,6 +1,7 @@ """Some utility functions""" from collections import UserDict +import unidecode from patacrep import errors, Rx @@ -92,3 +93,13 @@ def validate_yaml_schema(data, schema): schema.validate(data) except Rx.SchemaMismatch as exception: raise errors.SchemaError(rx_exception=exception) + +def normalize_string(string): + """Return a normalized string. + + Normalized means: + - no surrounding spaces; + - lower case; + - passed through unidecode.unidecode(). + """ + return unidecode.unidecode(string.lower().strip()) diff --git a/setup.py b/setup.py index aacfe910..aedd2009 100755 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ $ python setup.py install from patacrep import __version__ from setuptools import setup, find_packages +import os import sys setup_kwargs = { @@ -14,10 +15,28 @@ setup_kwargs = { } if sys.platform[0:3] == 'win': - from patacrep import __DATADIR__, files + from patacrep import __DATADIR__ + + def recursive_find(root_directory): + """Recursively find files from a root_directory. + + Return a list of files matching those conditions. + + Arguments: + - `root_directory`: root directory of the search. + """ + if not os.path.isdir(root_directory): + return + + olddir = os.getcwd() + os.chdir(root_directory) + for root, __ignored, filenames in os.walk(os.curdir): + for filename in filenames: + yield os.path.join(root, filename) + os.chdir(olddir) # List the data files - data_files = files.recursive_find(__DATADIR__) + data_files = recursive_find(__DATADIR__) data_files = ["data/" + d for d in data_files] setup_kwargs['package_data'] = {'patacrep': data_files} else: diff --git a/test/test_content/test_content.py b/test/test_content/test_content.py index 5edb9f96..1ca8c67d 100644 --- a/test/test_content/test_content.py +++ b/test/test_content/test_content.py @@ -42,7 +42,7 @@ class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): @classmethod def _create_content_test(cls, base): - """Return a function that `base.source` produces the correct file list""" + """Return a function that tests that `base.source` produces the correct file list""" def test_content(self): """Test that `base.source` produces the correct file list""" diff --git a/test/test_index/.gitignore b/test/test_index/.gitignore new file mode 100644 index 00000000..371a68d1 --- /dev/null +++ b/test/test_index/.gitignore @@ -0,0 +1,2 @@ +!*.sxd +!*.sbx diff --git a/test/test_index/__init__.py b/test/test_index/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_index/idx_auth.sbx b/test/test_index/idx_auth.sbx new file mode 100644 index 00000000..1063af69 --- /dev/null +++ b/test/test_index/idx_auth.sbx @@ -0,0 +1,26 @@ +\begin{idxblock}{A} + \idxentry{ + \indexauthor{}{Aba} + }{ + \hyperlink{song1-4.2}{4} + } \idxentry{ + \indexauthor{}{Äbc} + }{ + \hyperlink{song1-2.2}{2} + } \idxentry{ + \indexauthor{}{Abd} + }{ + \hyperlink{song1-3.2}{3} + } +\end{idxblock} +\begin{idxblock}{J} + \idxentry{ + \indexauthor{}{Jack} + }{ + \hyperlink{song1-1.2}{1} + } \idxentry{ + \indexauthor{}{Johns} + }{ + \hyperlink{song1-1.2}{1} + } +\end{idxblock} diff --git a/test/test_index/idx_auth.sxd b/test/test_index/idx_auth.sxd new file mode 100644 index 00000000..5d2f091b --- /dev/null +++ b/test/test_index/idx_auth.sxd @@ -0,0 +1,19 @@ +AUTHOR INDEX DATA FILE +%ignore unknown +%after by +%sep and +Jack and Johns +1 +song1-1.2 +Äbc +2 +song1-2.2 +Abd +3 +song1-3.2 +Aba +4 +song1-4.2 +unknown +5 +song1-5.2 diff --git a/test/test_index/idx_title.sbx b/test/test_index/idx_title.sbx new file mode 100644 index 00000000..5cc83540 --- /dev/null +++ b/test/test_index/idx_title.sbx @@ -0,0 +1,22 @@ +\begin{idxblock}{C} + \idxentry{ + \indextitle{}{Caa} + }{ + \hyperlink{song1-3.2}{3} + } \idxentry{ + \indextitle{}{Çab} + }{ + \hyperlink{song1-2.2}{2} + } \idxentry{ + \indextitle{}{Cac} + }{ + \hyperlink{song1-4.2}{4} + } +\end{idxblock} +\begin{idxblock}{M} + \idxentry{ + \indextitle{}{My song} + }{ + \hyperlink{song1-1.2}{1} + } +\end{idxblock} diff --git a/test/test_index/idx_title.sxd b/test/test_index/idx_title.sxd new file mode 100644 index 00000000..f6ba287f --- /dev/null +++ b/test/test_index/idx_title.sxd @@ -0,0 +1,13 @@ +TITLE INDEX DATA FILE +My song +1 +song1-1.2 +Çab +2 +song1-2.2 +Caa +3 +song1-3.2 +Cac +4 +song1-4.2 diff --git a/test/test_index/test_index.py b/test/test_index/test_index.py new file mode 100644 index 00000000..94748fc4 --- /dev/null +++ b/test/test_index/test_index.py @@ -0,0 +1,46 @@ +"""Tests for the index generation.""" + +import codecs +import glob +import os +import unittest + +from patacrep.index import process_sxd + +from .. import dynamic # pylint: disable=unused-import + + +class FileTest(unittest.TestCase, metaclass=dynamic.DynamicTest): + """Test of the index generation. + + For any given `foo.sxd`, it generates the index. + It controls that the generated file is equal to the one in `foo.sbx`. + """ + + @classmethod + def _iter_testmethods(cls): + """Iterate over dynamically generated test methods""" + for source in sorted(glob.glob(os.path.join( + os.path.dirname(__file__), + '*.sxd', + ))): + base = source[:-len(".sxd")] + yield ( + "test_index_{}".format(os.path.basename(base)), + cls._create_index_test(base), + ) + + @classmethod + def _create_index_test(cls, base): + """Return a function that tests that `foo.sxd` produces the sbx file""" + + def test_index(self): + """Test that `foo.sxd` produces the correct sbx file""" + generated_index = process_sxd(base + ".sxd").entries_to_str() + with codecs.open(base + ".sbx", "r", "utf-8") as control_index: + self.assertEqual(control_index.read(), generated_index, ) + + test_index.__doc__ = ( + "Test that '{base}.sxd' produces the correct sbx file""" + ).format(base=os.path.basename(base)) + return test_index