From 607e064bfb45b768c5fce74d6509cce1ec371f94 Mon Sep 17 00:00:00 2001 From: Louis Date: Sat, 22 Nov 2014 11:02:13 +0100 Subject: [PATCH 1/2] Transformation de encoding.open_read en un 'context manager' --- patacrep/content/include.py | 7 ++----- patacrep/encoding.py | 8 ++++++-- patacrep/index.py | 6 +----- patacrep/latex/__init__.py | 3 ++- patacrep/songbook.py | 8 ++------ patacrep/templates.py | 41 ++++++++++++++++--------------------- 6 files changed, 31 insertions(+), 42 deletions(-) diff --git a/patacrep/content/include.py b/patacrep/content/include.py index dcc969a9..9fc73621 100644 --- a/patacrep/content/include.py +++ b/patacrep/content/include.py @@ -48,15 +48,12 @@ def parse(keyword, config, argument, contentlist): filepath = load_from_datadirs(path, config) content_file = None try: - content_file = encoding.open_read(filepath, 'r') - new_content = json.load(content_file) + with encoding.open_read(filepath, 'r') as content_file: + new_content = json.load(content_file) except Exception as error: # pylint: disable=broad-except LOGGER.error(error) LOGGER.error("Error while loading file '{}'.".format(filepath)) sys.exit(1) - finally: - if content_file: - content_file.close() config["datadir"].append(os.path.abspath(os.path.dirname(filepath))) new_contentlist += process_content(new_content, config) diff --git a/patacrep/encoding.py b/patacrep/encoding.py index ca917295..a1084aa8 100644 --- a/patacrep/encoding.py +++ b/patacrep/encoding.py @@ -5,17 +5,21 @@ import codecs import chardet import logging +import contextlib LOGGER = logging.getLogger(__name__) + +@contextlib.contextmanager def open_read(filename, mode='r'): """Open a file for reading, guessing the right encoding. Return a fileobject, reading unicode strings. """ - return codecs.open( + with codecs.open( filename, mode=mode, encoding=chardet.detect(open(filename, 'rb').read())['encoding'], errors='replace', - ) + ) as fileobject: + yield fileobject diff --git a/patacrep/index.py b/patacrep/index.py index c715918a..f111f059 100644 --- a/patacrep/index.py +++ b/patacrep/index.py @@ -29,13 +29,9 @@ def process_sxd(filename): """ data = [] index_file = None - try: - index_file = encoding.open_read(filename, 'r') + with encoding.open_read(filename, 'r') as index_file: for line in index_file: data.append(line.strip()) - finally: - if index_file: - index_file.close() i = 1 idx = Index(data[0]) diff --git a/patacrep/latex/__init__.py b/patacrep/latex/__init__.py index fce5470f..f923b822 100644 --- a/patacrep/latex/__init__.py +++ b/patacrep/latex/__init__.py @@ -15,6 +15,7 @@ def parsesong(path): """Return a dictonary of data read from the latex file `path`. """ - data = syntax_parsesong(encoding.open_read(path).read(), path) + with encoding.open_read(path) as songfile: + data = syntax_parsesong(songfile.read(), path) data['@path'] = path return data diff --git a/patacrep/songbook.py b/patacrep/songbook.py index e3bcf555..dfa21651 100755 --- a/patacrep/songbook.py +++ b/patacrep/songbook.py @@ -100,17 +100,13 @@ def main(): basename = os.path.basename(songbook_path)[:-3] - songbook_file = None try: - songbook_file = encoding.open_read(songbook_path) - songbook = json.load(songbook_file) + with encoding.open_read(songbook_path) as songbook_file: + songbook = json.load(songbook_file) except Exception as error: # pylint: disable=broad-except LOGGER.error(error) LOGGER.error("Error while loading file '{}'.".format(songbook_path)) sys.exit(1) - finally: - if songbook_file: - songbook_file.close() # Gathering datadirs datadirs = [] diff --git a/patacrep/templates.py b/patacrep/templates.py index d1e9a616..55a354fb 100644 --- a/patacrep/templates.py +++ b/patacrep/templates.py @@ -186,32 +186,27 @@ class TexRenderer(object): """ subvariables = {} - template_file = None templatename = self.texenv.get_template(template).filename - try: - template_file = encoding.open_read(templatename, 'r') + with encoding.open_read(templatename, 'r') as template_file: content = template_file.read() - subtemplates = list(find_templates(self.texenv.parse(content))) - match = re.findall(_VARIABLE_REGEXP, content) - if match: - for var in match: - try: - subvariables.update(json.loads(var)) - except ValueError as exception: - raise errors.TemplateError( - exception, - ( - "Error while parsing json in file " - "{filename}. The json string was:" - "\n'''\n{jsonstring}\n'''" - ).format( - filename=templatename, - jsonstring=var, - ) + subtemplates = list(find_templates(self.texenv.parse(content))) + match = re.findall(_VARIABLE_REGEXP, content) + if match: + for var in match: + try: + subvariables.update(json.loads(var)) + except ValueError as exception: + raise errors.TemplateError( + exception, + ( + "Error while parsing json in file " + "{filename}. The json string was:" + "\n'''\n{jsonstring}\n'''" + ).format( + filename=templatename, + jsonstring=var, ) - finally: - if template_file: - template_file.close() + ) return (subvariables, subtemplates) From 88ba800fb36c07ae2e735ae19c8d8b2a5477329b Mon Sep 17 00:00:00 2001 From: Louis Date: Sat, 22 Nov 2014 11:27:54 +0100 Subject: [PATCH 2/2] Added option 'encoding' to songbook fiels --- patacrep/build.py | 2 ++ patacrep/content/include.py | 5 ++++- patacrep/encoding.py | 10 ++++++++-- patacrep/index.py | 2 +- patacrep/latex/__init__.py | 4 ++-- patacrep/songbook.py | 10 ++++++++-- patacrep/songs/__init__.py | 2 ++ patacrep/songs/tex.py | 2 +- patacrep/templates.py | 12 +++++++++--- 9 files changed, 37 insertions(+), 12 deletions(-) diff --git a/patacrep/build.py b/patacrep/build.py index be9b48f4..1bc683a4 100644 --- a/patacrep/build.py +++ b/patacrep/build.py @@ -33,6 +33,7 @@ DEFAULT_CONFIG = { 'lang': 'english', 'content': [], 'titleprefixwords': [], + 'encoding': None, } @@ -91,6 +92,7 @@ class Songbook(object): config['template'], config['datadir'], config['lang'], + config['encoding'], ) config.update(renderer.get_variables()) config.update(self.config) diff --git a/patacrep/content/include.py b/patacrep/content/include.py index 9fc73621..03492544 100644 --- a/patacrep/content/include.py +++ b/patacrep/content/include.py @@ -48,7 +48,10 @@ def parse(keyword, config, argument, contentlist): filepath = load_from_datadirs(path, config) content_file = None try: - with encoding.open_read(filepath, 'r') as content_file: + with encoding.open_read( + filepath, + encoding=config['encoding'] + ) as content_file: new_content = json.load(content_file) except Exception as error: # pylint: disable=broad-except LOGGER.error(error) diff --git a/patacrep/encoding.py b/patacrep/encoding.py index a1084aa8..a0501e4d 100644 --- a/patacrep/encoding.py +++ b/patacrep/encoding.py @@ -11,15 +11,21 @@ LOGGER = logging.getLogger(__name__) @contextlib.contextmanager -def open_read(filename, mode='r'): +def open_read(filename, mode='r', encoding=None): """Open a file for reading, guessing the right encoding. Return a fileobject, reading unicode strings. + If `encoding` is set, use it as the encoding (do not guess). """ + if encoding is None: + fileencoding = chardet.detect(open(filename, 'rb').read())['encoding'] + else: + fileencoding = encoding + with codecs.open( filename, mode=mode, - encoding=chardet.detect(open(filename, 'rb').read())['encoding'], + encoding=fileencoding, errors='replace', ) as fileobject: yield fileobject diff --git a/patacrep/index.py b/patacrep/index.py index f111f059..351e155e 100644 --- a/patacrep/index.py +++ b/patacrep/index.py @@ -29,7 +29,7 @@ def process_sxd(filename): """ data = [] index_file = None - with encoding.open_read(filename, 'r') as index_file: + with encoding.open_read(filename) as index_file: for line in index_file: data.append(line.strip()) diff --git a/patacrep/latex/__init__.py b/patacrep/latex/__init__.py index f923b822..8d0efed9 100644 --- a/patacrep/latex/__init__.py +++ b/patacrep/latex/__init__.py @@ -11,11 +11,11 @@ from patacrep.latex.syntax import tex2plain from patacrep.latex.syntax import parsesong as syntax_parsesong from patacrep import encoding -def parsesong(path): +def parsesong(path, fileencoding=None): """Return a dictonary of data read from the latex file `path`. """ - with encoding.open_read(path) as songfile: + with encoding.open_read(path, encoding=fileencoding) as songfile: data = syntax_parsesong(songfile.read(), path) data['@path'] = path return data diff --git a/patacrep/songbook.py b/patacrep/songbook.py index dfa21651..8417be91 100755 --- a/patacrep/songbook.py +++ b/patacrep/songbook.py @@ -14,7 +14,7 @@ import sys from patacrep.build import SongbookBuilder, DEFAULT_STEPS from patacrep import __version__ from patacrep import errors -from patacrep import encoding +import patacrep.encoding # Logging configuration logging.basicConfig(level=logging.INFO) @@ -101,8 +101,14 @@ def main(): basename = os.path.basename(songbook_path)[:-3] try: - with encoding.open_read(songbook_path) as songbook_file: + with patacrep.encoding.open_read(songbook_path) as songbook_file: songbook = json.load(songbook_file) + if 'encoding' in songbook: + with patacrep.encoding.open_read( + songbook_path, + encoding=songbook['encoding'] + ) as songbook_file: + songbook = json.load(songbook_file) except Exception as error: # pylint: disable=broad-except LOGGER.error(error) LOGGER.error("Error while loading file '{}'.".format(songbook_path)) diff --git a/patacrep/songs/__init__.py b/patacrep/songs/__init__.py index 512a2ced..0a1072f9 100644 --- a/patacrep/songs/__init__.py +++ b/patacrep/songs/__init__.py @@ -100,6 +100,8 @@ class Song(Content): def __init__(self, datadir, subpath, config): self.fullpath = os.path.join(datadir, subpath) self.datadir = datadir + self.encoding = config["encoding"] + if datadir: # Only songs in datadirs are cached self._filehash = hashlib.md5( diff --git a/patacrep/songs/tex.py b/patacrep/songs/tex.py index b5710346..c2a51d62 100644 --- a/patacrep/songs/tex.py +++ b/patacrep/songs/tex.py @@ -13,7 +13,7 @@ class TexRenderer(Song): def parse(self): """Parse song and set metadata.""" - self.data = parsesong(self.fullpath) + self.data = parsesong(self.fullpath, self.encoding) self.titles = self.data['@titles'] self.languages = self.data['@languages'] self.authors = self.data['by'] diff --git a/patacrep/templates.py b/patacrep/templates.py index 55a354fb..be507a27 100644 --- a/patacrep/templates.py +++ b/patacrep/templates.py @@ -9,7 +9,8 @@ import os import re import json -from patacrep import encoding, errors, files +from patacrep import errors, files +import patacrep.encoding _LATEX_SUBS = ( (re.compile(r'\\'), r'\\textbackslash'), @@ -67,7 +68,7 @@ def _escape_tex(value): class TexRenderer(object): """Render a template to a LaTeX file.""" - def __init__(self, template, datadirs, lang): + def __init__(self, template, datadirs, lang, encoding=None): '''Start a new jinja2 environment for .tex creation. Arguments: @@ -75,8 +76,10 @@ class TexRenderer(object): - datadirs: list of locations of the data directory (which may contain file /templates/