add CHANGELOG to API docs, point to license on GitHub, improve API do…

…c formatting (#4472)

* add CHANGELOG to API docs

* point to license directly on GitHub

* tweak renderer

* fix failing test

* improvements

* update CHANGELOG

Co-authored-by: Dirk Groeneveld <dirkg@allenai.org>

CHANGELOG.md

@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed a bug that prevented `cached_path` from downloading assets from GitHub releases.
 - Fixed a bug that erronously increased last label's false positive count in calculating fbeta metrics.
 - `Tqdm` output now looks much better when the output is being piped or redirected.
+- Small improvements to how the API documentation is rendered.
 
 ### Added
 

Makefile

@@ -8,7 +8,7 @@ MD_DOCS_CMD = python scripts/py2md.py
 MD_DOCS_CONF = mkdocs.yml
 MD_DOCS_CONF_SRC = mkdocs-skeleton.yml
 MD_DOCS_TGT = site/
-MD_DOCS_EXTRAS = $(addprefix $(MD_DOCS_ROOT),README.md LICENSE.md CONTRIBUTING.md)
+MD_DOCS_EXTRAS = $(addprefix $(MD_DOCS_ROOT),README.md CHANGELOG.md CONTRIBUTING.md)
 
 DOCKER_TAG = latest
 DOCKER_IMAGE_NAME = allennlp/allennlp:$(DOCKER_TAG)
@@ -120,9 +120,6 @@ $(MD_DOCS_ROOT)README.md : README.md
 	# Alter the relative path of the README image for the docs.
 	$(SED) -i '1s/docs/./' $@
 
-$(MD_DOCS_ROOT)LICENSE.md : LICENSE
-	cp $< $@
-
 $(MD_DOCS_ROOT)%.md : %.md
 	cp $< $@
 

mkdocs-skeleton.yml

@@ -31,7 +31,8 @@ nav:
   - Master: /master/
 - API: 'This section is autogenerated, do not edit.'
 - Contributing: CONTRIBUTING.md
-- License: LICENSE.md
+- CHANGELOG: CHANGELOG.md
+- License: https://raw.githubusercontent.com/allenai/allennlp/master/LICENSE
 
 markdown_extensions:
 - toc:

scripts/py2md.py

@@ -24,7 +24,7 @@
 from pydoc_markdown.contrib.loaders.python import PythonLoader
 from pydoc_markdown.contrib.renderers.markdown import MarkdownRenderer
 from pydoc_markdown.interfaces import Processor, Renderer
-from pydoc_markdown.reflection import Argument, Module, Function, Class
+from pydoc_markdown.reflection import Argument, Module, Function, Class, Data
 
 
 logging.basicConfig(level=logging.INFO)
@@ -173,6 +173,7 @@ class AllenNlpDocstringProcessor(Struct):
     """
 
     CROSS_REF_RE = re.compile("(:(class|func|mod):`~?([a-zA-Z0-9_.]+)`)")
+    UNDERSCORE_HEADER_RE = re.compile(r"(.*)\n-{3,}\n")
 
     @override
     def process(self, graph, resolver):
@@ -185,7 +186,12 @@ def process_node(self, node):
         lines: List[str] = []
         state: ProcessorState = ProcessorState(parameters=OrderedDict())
 
-        for line in node.docstring.split("\n"):
+        docstring = node.docstring
+
+        # Standardize header syntax to use '#' instead of underscores.
+        docstring = self.UNDERSCORE_HEADER_RE.sub(r"# \g<1>", docstring)
+
+        for line in docstring.split("\n"):
             # Check if we're starting or ending a codeblock.
             if line.startswith("```"):
                 state.codeblock_opened = not state.codeblock_opened
@@ -281,7 +287,11 @@ def _check(node):
 @implements(Renderer)
 class AllenNlpRenderer(MarkdownRenderer):
     def _format_function_signature(
-        self, func: Function, override_name: str = None, add_method_bar: bool = True
+        self,
+        func: Function,
+        override_name: str = None,
+        add_method_bar: bool = True,
+        include_parent_class: bool = True,
     ) -> str:
         parts = []
         for dec in func.decorators:
@@ -317,20 +327,54 @@ def _format_function_signature(
         result = "".join(parts)
         if add_method_bar and func.is_method():
             result = "\n".join(" | " + line for line in result.split("\n"))
+            if include_parent_class:
+                bases = ", ".join(map(str, func.parent.bases))
+                if func.parent.metaclass:
+                    bases += ", metaclass=" + str(func.parent.metaclass)
+                if bases:
+                    class_signature = f"class {func.parent.name}({bases})"
+                else:
+                    class_signature = f"class {func.parent.name}"
+                result = f"{class_signature}:\n | ...\n{result}"
         return result
 
+    def _format_data_signature(self, data: Data) -> str:
+        expr = str(data.expr)
+        if len(expr) > self.data_expression_maxlength:
+            expr = expr[: self.data_expression_maxlength] + " ..."
+
+        if data.annotation:
+            signature = f"{data.name}: {data.annotation} = {expr}"
+        else:
+            signature = f"{data.name} = {expr}"
+
+        if data.parent and data.parent.is_class():
+            bases = ", ".join(map(str, data.parent.bases))
+            if data.parent.metaclass:
+                bases += ", metaclass=" + str(data.parent.metaclass)
+            if bases:
+                class_signature = f"class {data.parent.name}({bases})"
+            else:
+                class_signature = f"class {data.parent.name}"
+            return f"{class_signature}:\n | ...\n | {signature}"
+        else:
+            return signature
+
     def _format_classdef_signature(self, cls: Class) -> str:
         bases = ", ".join(map(str, cls.bases))
         if cls.metaclass:
             bases += ", metaclass=" + str(cls.metaclass)
-        code = "class {}({})".format(cls.name, bases)
+        if bases:
+            code = "class {}({})".format(cls.name, bases)
+        else:
+            code = "class {}".format(cls.name)
         if self.signature_python_help_style:
             code = cls.path() + " = " + code
         if self.classdef_render_init_signature_if_needed and (
             "__init__" in cls.members and not cls.members["__init__"].visible
         ):
             code += ":\n" + self._format_function_signature(
-                cls.members["__init__"], add_method_bar=True
+                cls.members["__init__"], add_method_bar=True, include_parent_class=False,
             )
         return code
 
@@ -381,6 +425,7 @@ def py2md(module: str, out: Optional[str] = None) -> bool:
             signature_with_def=True,
             use_fixed_header_levels=False,
             render_module_header=False,
+            descriptive_class_title=False,
         ),
     )
     if out:

scripts/tests/py2md/basic_example.py

@@ -28,6 +28,11 @@ def func_with_args(a: int, b: int, c: int = 3) -> int:
     c : `int`, optional (default = `3`)
         Yet another number.
 
+    Notes
+    -----
+
+    These are some notes.
+
     # Returns
 
     `int`
@@ -40,7 +45,7 @@ class SomeClass:
     """
     I'm a class!
 
-    # Paramaters
+    # Parameters
 
     x : `float`
         This attribute is called `x`.

scripts/tests/py2md/basic_example_expected_output.md

@@ -37,33 +37,39 @@ This function has some args.
 - __c__ : `int`, optional (default = `3`) <br>
     Yet another number.
 
+<strong>Notes</strong>
+
+These are some notes.
+
 <strong>Returns</strong>
 
 
 - `int` <br>
     The result of `a + b * c`.
 
 <a name=".scripts.tests.py2md.basic_example.SomeClass"></a>
-## SomeClass Objects
+## SomeClass
 
 ```python
-class SomeClass():
+class SomeClass:
  | def __init__(self) -> None
 ```
 
 I'm a class!
 
-<strong>Paramaters</strong>
+<strong>Parameters</strong>
 
 
-x : `float`
+- __x__ : `float` <br>
     This attribute is called `x`.
 
 <a name=".scripts.tests.py2md.basic_example.SomeClass.some_class_level_variable"></a>
 ### some\_class\_level\_variable
 
 ```python
-some_class_level_variable = 1
+class SomeClass:
+ | ...
+ | some_class_level_variable = 1
 ```
 
 This is how you document a class-level variable.
@@ -72,13 +78,17 @@ This is how you document a class-level variable.
 ### some\_class\_level\_var\_with\_type
 
 ```python
-some_class_level_var_with_type = 1
+class SomeClass:
+ | ...
+ | some_class_level_var_with_type: int = 1
 ```
 
 <a name=".scripts.tests.py2md.basic_example.SomeClass.some_method"></a>
 ### some\_method
 
 ```python
+class SomeClass:
+ | ...
  | def some_method(self) -> None
 ```
 
@@ -95,6 +105,8 @@ But I don't do anything.
 ### method\_with\_alternative\_return\_section
 
 ```python
+class SomeClass:
+ | ...
  | def method_with_alternative_return_section(self) -> int
 ```
 
@@ -109,6 +121,8 @@ Another method.
 ### method\_with\_alternative\_return\_section3
 
 ```python
+class SomeClass:
+ | ...
  | def method_with_alternative_return_section3(self) -> int
 ```
 
@@ -121,10 +135,10 @@ Another method.
     A completely arbitrary number.
 
 <a name=".scripts.tests.py2md.basic_example.AnotherClassWithReallyLongConstructor"></a>
-## AnotherClassWithReallyLongConstructor Objects
+## AnotherClassWithReallyLongConstructor
 
 ```python
-class AnotherClassWithReallyLongConstructor():
+class AnotherClassWithReallyLongConstructor:
  | def __init__(
  |     self,
  |     a_really_long_argument_name: int = 0,