Skip to content

Commit

Permalink
cli, docutils: Unify docutils pipelines and propagate error code on exit
Browse files Browse the repository at this point in the history 
Closes GH-57.

Suggested-by: @gares
  • Loading branch information
@cpitclaudel
cpitclaudel committed Aug 17, 2021
1 parent a310ec5 commit dffde22
Show file tree
Hide file tree
Showing 8 changed files with 87 additions and 88 deletions.
66 changes: 33 additions & 33 deletions alectryon/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ def register_docutils(v, ctx):
'stylesheet_path': None,
'input_encoding': 'utf-8',
'output_encoding': 'utf-8',
'exit_status_level': 3,
'alectryon_banner': ctx["include_banner"],
'alectryon_vernums': ctx["include_vernums"],
'alectryon_webpage_style': ctx["webpage_style"],
Expand All @@ -102,17 +103,26 @@ def register_docutils(v, ctx):
def _gen_docutils(source, fpath,
Parser, Reader, Writer,
settings_overrides):
from docutils.core import publish_string
from docutils.core import publish_programmatically
from docutils.io import StringInput, StringOutput

parser = Parser()
return publish_string(
source=source.encode("utf-8"),
output, pub = publish_programmatically(
source_class=StringInput, destination_class=StringOutput,
source=source.encode("utf-8"), destination=None,
source_path=fpath, destination_path=None,

reader=Reader(parser), reader_name=None,
parser=parser, parser_name=None,
writer=Writer(), writer_name=None,
settings_overrides=settings_overrides,
enable_exit_status=True).decode("utf-8")

settings=None, settings_spec=None,
settings_overrides=settings_overrides, config_section=None,
enable_exit_status=False)

max_level = pub.document.reporter.max_level
exit_code = max_level + 10 if max_level >= pub.settings.exit_status_level else 0
return output.decode("utf-8"), exit_code

def _resolve_dialect(backend, html_dialect, latex_dialect):
return {"webpage": html_dialect, "latex": latex_dialect}.get(backend, None)
Expand All @@ -122,15 +132,17 @@ def _record_assets(assets, path, names):
assets.append((path, name))

def gen_docutils(src, frontend, backend, fpath, dialect,
docutils_settings_overrides, assets):
docutils_settings_overrides, assets, exit_code):
from .docutils import get_pipeline

pipeline = get_pipeline(frontend, backend, dialect)
_record_assets(assets, pipeline.translator.ASSETS_PATH, pipeline.translator.ASSETS)

return _gen_docutils(src, fpath,
pipeline.parser, pipeline.reader, pipeline.writer,
docutils_settings_overrides)
output, exit_code.val = \
_gen_docutils(src, fpath,
pipeline.parser, pipeline.reader, pipeline.writer,
docutils_settings_overrides)
return output

def _docutils_cmdline(description, frontend, backend):
import locale
Expand All @@ -146,24 +158,7 @@ def _docutils_cmdline(description, frontend, backend):
publish_cmdline(
parser=pipeline.parser(), writer=pipeline.writer(),
settings_overrides={'stylesheet_path': None},
description="{} {}".format(description, default_description)
)

def lint_docutils(source, fpath, frontend, docutils_settings_overrides):
from docutils.core import publish_doctree
from .docutils import get_parser, LintingReader

parser = get_parser(frontend)()
reader = LintingReader(parser)

publish_doctree(
source=source.encode("utf-8"), source_path=fpath,
reader=reader, reader_name=None,
parser=parser, parser_name=None,
settings_overrides=docutils_settings_overrides,
enable_exit_status=True)

return reader.error_stream.getvalue() # FIXME exit code
description="{} {}".format(description, default_description))

def _scrub_fname(fname):
import re
Expand Down Expand Up @@ -380,7 +375,7 @@ def write_file(ext):
(read_plain, parse_coq_plain, annotate_chunks, apply_transforms,
gen_latex_snippets, dump_latex_snippets, write_file(".snippets.tex")),
'lint':
(read_plain, register_docutils, lint_docutils,
(read_plain, register_docutils, gen_docutils,
write_file(".lint.json")),
'rst':
(read_plain, coq_to_rst, write_file(".v.rst")),
Expand All @@ -396,7 +391,7 @@ def write_file(ext):
(read_plain, register_docutils, gen_docutils, copy_assets,
write_file(".tex")),
'lint':
(read_plain, register_docutils, lint_docutils,
(read_plain, register_docutils, gen_docutils,
write_file(".lint.json")),
'rst':
(read_plain, coq_to_rst, write_file(".v.rst"))
Expand All @@ -415,7 +410,7 @@ def write_file(ext):
(read_plain, register_docutils, gen_docutils, copy_assets,
write_file(".tex")),
'lint':
(read_plain, register_docutils, lint_docutils,
(read_plain, register_docutils, gen_docutils,
write_file(".lint.json")),
'coq':
(read_plain, rst_to_coq, write_file(".v")),
Expand All @@ -430,7 +425,7 @@ def write_file(ext):
(read_plain, register_docutils, gen_docutils, copy_assets,
write_file(".tex")),
'lint':
(read_plain, register_docutils, lint_docutils,
(read_plain, register_docutils, gen_docutils,
write_file(".lint.json"))
}
}
Expand Down Expand Up @@ -694,6 +689,10 @@ def parse_arguments():
# Entry point
# ===========

class ExitCode:
def __init__(self, n):
self.val = n

def call_pipeline_step(step, state, ctx):
params = list(inspect.signature(step).parameters.keys())[1:]
return step(state, **{p: ctx[p] for p in params})
Expand All @@ -708,7 +707,7 @@ def build_context(fpath, args, frontend, backend):
ctx = {**vars(args),
"fpath": fpath, "fname": fname,
"frontend": frontend, "backend": backend, "dialect": dialect,
"assets": [], "html_classes": []}
"assets": [], "html_classes": [], "exit_code": ExitCode(0)}
ctx["ctx"] = ctx

if args.output_directory is None:
Expand Down Expand Up @@ -742,11 +741,12 @@ def process_pipelines(args):
state, ctx = None, build_context(fpath, args, frontend, backend)
for step in pipeline:
state = call_pipeline_step(step, state, ctx)
yield ctx["exit_code"].val

def main():
try:
args = parse_arguments()
process_pipelines(args)
sys.exit(max(process_pipelines(args), default=0))
except (ValueError, FileNotFoundError, ImportError, argparse.ArgumentTypeError) as e:
if core.TRACEBACK:
raise e
Expand Down
74 changes: 45 additions & 29 deletions alectryon/docutils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import docutils.frontend
import docutils.transforms
import docutils.utils
import docutils.writers
from docutils import nodes

from docutils.parsers.rst import directives, roles, Directive # type: ignore
Expand Down Expand Up @@ -904,7 +905,7 @@ def marker_quote_role(role, rawtext, text, lineno, inliner,
# Error printer
# -------------

class JsErrorPrinter:
class JsErrorObserver:
@staticmethod
def json_of_message(msg):
message = msg.children[0].astext() if msg.children else "Unknown error"
Expand All @@ -919,12 +920,14 @@ def json_of_message(msg):
return js

def __init__(self, stream, settings):
self.errors = []
self.stream = stream
self.report_level = settings.report_level

def __call__(self, msg):
import json
if msg['level'] >= self.report_level:
self.errors.append(msg)
if self.stream and msg['level'] >= self.report_level:
js = self.json_of_message(msg)
json.dump(js, self.stream)
self.stream.write('\n')
Expand Down Expand Up @@ -1119,29 +1122,7 @@ def __init__(self, *args, **kwargs):

class DummyTranslator:
ASSETS: List[str] = []

Pipeline = namedtuple("Pipeline", "parser reader translator writer")

PARSERS = {
"coq+rst": (__name__, "RSTCoqParser"),
"rst": ("docutils.parsers.rst", "Parser"),
"md": ("alectryon.myst", "Parser"),
}

BACKENDS = {
'webpage': {
'html4': (HtmlTranslator, HtmlWriter),
'html5': (Html5Translator, Html5Writer),
},
'latex': {
'pdflatex': (LatexTranslator, LatexWriter),
'xelatex': (XeLatexTranslator, XeLatexWriter),
'lualatex': (LuaLatexTranslator, LuaLatexWriter),
},
'pseudoxml': {
None: (DummyTranslator, ("docutils.writers.pseudoxml", "Writer")),
}
}
ASSETS_PATH = ""

# Linter
# ======
Expand All @@ -1158,7 +1139,7 @@ class LintingReader(StandaloneReader):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
from io import StringIO
self.error_stream = StringIO()
self.error_stream = kwargs.get("error_stream", StringIO())

def get_transforms(self):
return super().get_transforms() + [LoadConfigTransform]
Expand All @@ -1167,20 +1148,54 @@ def new_document(self):
doc = super().new_document()
doc.transformer = EarlyTransformer(doc)

observer = JsErrorPrinter(self.error_stream, self.settings)
js_observer = JsErrorObserver(self.error_stream, self.settings)
doc.reporter.report_level = 0 # Report all messages
doc.reporter.halt_level = docutils.utils.Reporter.SEVERE_LEVEL + 1 # Do not exit early
doc.reporter.stream = False # Disable textual reporting
doc.reporter.attach_observer(observer)
doc.reporter.attach_observer(js_observer)
doc["js_observer"] = js_observer

return doc

class LintingWriter(docutils.writers.UnfilteredWriter):
def translate(self):
self.output = self.document["js_observer"].stream.getvalue()

# API
# ===

Pipeline = namedtuple("Pipeline", "reader parser translator writer")

PARSERS = {
"coq+rst": (__name__, "RSTCoqParser"),
"rst": ("docutils.parsers.rst", "Parser"),
"md": ("alectryon.myst", "Parser"),
}

BACKENDS = {
'webpage': {
'html4': (HtmlTranslator, HtmlWriter),
'html5': (Html5Translator, Html5Writer),
},
'latex': {
'pdflatex': (LatexTranslator, LatexWriter),
'xelatex': (XeLatexTranslator, XeLatexWriter),
'lualatex': (LuaLatexTranslator, LuaLatexWriter),
},
'lint': {
None: (DummyTranslator, LintingWriter),
},
'pseudoxml': {
None: (DummyTranslator, ("docutils.writers.pseudoxml", "Writer")),
}
}

def _maybe_import(tp):
return getattr(import_module(tp[0]), tp[1]) if isinstance(tp, tuple) else tp

def get_reader(_frontend, backend):
return LintingReader if backend == 'lint' else StandaloneReader

def get_parser(frontend):
if frontend not in PARSERS:
raise ValueError("Unsupported docutils frontend: {}".format(frontend))
Expand All @@ -1195,9 +1210,10 @@ def get_writer(backend, dialect):
return _maybe_import(translator), _maybe_import(writer)

def get_pipeline(frontend, backend, dialect):
reader = get_reader(frontend, backend)
parser = get_parser(frontend)
translator, writer = get_writer(backend, dialect)
return Pipeline(parser, StandaloneReader, translator, writer)
return Pipeline(reader, parser, translator, writer)

# Entry points
# ============
Expand Down
1 change: 1 addition & 0 deletions recipes/_output/tests/errors.lint.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,4 @@
{"level": "error", "message": "Error in \"coq\" directive:\nUnknown flag `unknown`.", "source": "tests/errors.rst", "line": 77, "column": null, "end_line": null, "end_column": null}
{"level": "error", "message": "Error in \"coq\" directive:\nMissing search pattern for key ``.s`` in expression ``.s.g``. (maybe an invalid pattern?)", "source": "tests/errors.rst", "line": 82, "column": null, "end_line": null, "end_column": null}
{"level": "error", "message": "In `:alectryon/pygments/xyz:`: Unknown token kind: xyz", "source": "tests/errors.rst", "line": 5, "column": null, "end_line": null, "end_column": null}
exit: 13
1 change: 1 addition & 0 deletions recipes/_output/tests/errors.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,4 @@ tests/errors.rst:93: (ERROR/3) In `.. coq::`:
tests/errors.rst:100: (ERROR/3) In `.. coq:: unfold out`: Cannot show output of 'Check nat.' without .in or .unfold.
tests/errors.rst:70: (ERROR/3) In :mref:`.io#nope.s(123)`: Reference to unknown Alectryon block.
tests/errors.rst:71: (ERROR/3) In :mref:`.s(Goal).g#25`: No goal matches '25'.
exit: 13
2 changes: 0 additions & 2 deletions recipes/_output/tests/linter.lint.json
View file

This file was deleted.

12 changes: 4 additions & 8 deletions recipes/tests.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ _output/tests/doctests.out: tests/doctests.py | _output/tests/

# reST → JSON errors
_output/tests/errors.lint.json: tests/errors.rst
$(alectryon) $< --backend lint
$(alectryon) $< --backend lint; echo "exit: $$?" >> $@
# reST → HTML + errors
_output/tests/errors.txt: tests/errors.rst
$(alectryon) $< --copy-assets none --backend webpage -o /dev/null 2> $@
$(alectryon) $< --copy-assets none --backend webpage -o /dev/null 2> $@; echo "exit: $$?" >> $@

# Coq → HTML
_output/tests/goal-name.html: tests/goal-name.v
Expand All @@ -54,10 +54,6 @@ _output/tests/goal-name.xe.tex: tests/goal-name.v
_output/tests/latex_formatting.tex: tests/latex_formatting.v
$(alectryon) $< --backend latex

# Coq+reST → JSON
_output/tests/linter.lint.json: tests/linter.v
$(alectryon) $< --backend lint

# reST → Coq
_output/tests/literate.v: tests/literate.rst
$(alectryon) $< --backend coq
Expand All @@ -74,6 +70,6 @@ _output/tests/screenshot.html: tests/screenshot.v
_output/tests/syntax_highlighting.html: tests/syntax_highlighting.v
$(alectryon) $<

_output/tests/dialects.4.html _output/tests/dialects.5.html _output/tests/dialects.tex _output/tests/dialects.xe.tex _output/tests/dialects.lua.tex _output/tests/directive-options.html _output/tests/directive-options.xe.tex _output/tests/display-flags.html _output/tests/doctests.out _output/tests/errors.lint.json _output/tests/errors.txt _output/tests/goal-name.html _output/tests/goal-name.xe.tex _output/tests/latex_formatting.tex _output/tests/linter.lint.json _output/tests/literate.v _output/tests/literate.v.rst _output/tests/screenshot.html _output/tests/syntax_highlighting.html: out_dir := _output/tests
_output/tests/dialects.4.html _output/tests/dialects.5.html _output/tests/dialects.tex _output/tests/dialects.xe.tex _output/tests/dialects.lua.tex _output/tests/directive-options.html _output/tests/directive-options.xe.tex _output/tests/display-flags.html _output/tests/doctests.out _output/tests/errors.lint.json _output/tests/errors.txt _output/tests/goal-name.html _output/tests/goal-name.xe.tex _output/tests/latex_formatting.tex _output/tests/literate.v _output/tests/literate.v.rst _output/tests/screenshot.html _output/tests/syntax_highlighting.html: out_dir := _output/tests

targets += _output/tests/dialects.4.html _output/tests/dialects.5.html _output/tests/dialects.tex _output/tests/dialects.xe.tex _output/tests/dialects.lua.tex _output/tests/directive-options.html _output/tests/directive-options.xe.tex _output/tests/display-flags.html _output/tests/doctests.out _output/tests/errors.lint.json _output/tests/errors.txt _output/tests/goal-name.html _output/tests/goal-name.xe.tex _output/tests/latex_formatting.tex _output/tests/linter.lint.json _output/tests/literate.v _output/tests/literate.v.rst _output/tests/screenshot.html _output/tests/syntax_highlighting.html
targets += _output/tests/dialects.4.html _output/tests/dialects.5.html _output/tests/dialects.tex _output/tests/dialects.xe.tex _output/tests/dialects.lua.tex _output/tests/directive-options.html _output/tests/directive-options.xe.tex _output/tests/display-flags.html _output/tests/doctests.out _output/tests/errors.lint.json _output/tests/errors.txt _output/tests/goal-name.html _output/tests/goal-name.xe.tex _output/tests/latex_formatting.tex _output/tests/literate.v _output/tests/literate.v.rst _output/tests/screenshot.html _output/tests/syntax_highlighting.html
6 changes: 3 additions & 3 deletions recipes/tests/errors.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@

:alectryon/pygments/xyz: test

To compile::
The ``lint`` backend in Alectryon runs the compiler and reports errors on ``stderr``::

alectryon errors.rst --backend lint
alectryon errors.rst --backend lint; echo "exit: $?" >> errors.lint.json
# reST → JSON errors; produces ‘errors.lint.json’
alectryon errors.rst --copy-assets none --backend webpage -o /dev/null 2> errors.txt
alectryon errors.rst --copy-assets none --backend webpage -o /dev/null 2> errors.txt; echo "exit: $?" >> errors.txt
# reST → HTML + errors; produces ‘errors.txt’

.. coq::
Expand Down
13 changes: 0 additions & 13 deletions recipes/tests/linter.v
View file

This file was deleted.

0 comments on commit dffde22

Please sign in to comment.