[AIRFLOW-3811] automatic generation of API Reference in docs (#4788)

Co-authored-by: Jarek Potiuk <jarek.potiuk@polidea.com>

.gitignore

@@ -83,6 +83,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/_api/
 
 # PyBuilder
 target/

airflow/contrib/hooks/gcp_transfer_hook.py

@@ -267,7 +267,7 @@ def list_transfer_operations(self, filter):
 
             * project_id is optional if you have a project id defined
               in the connection
-              See: :doc:`howto/connection/gcp`
+              See: :ref:`howto/connection:gcp`
 
         :type filter: dict
         :return: transfer operation

airflow/contrib/operators/kubernetes_pod_operator.py

@@ -21,9 +21,6 @@
 from airflow.contrib.kubernetes import kube_client, pod_generator, pod_launcher
 from airflow.contrib.kubernetes.pod import Resources
 from airflow.utils.state import State
-from airflow.contrib.kubernetes.volume_mount import VolumeMount  # noqa
-from airflow.contrib.kubernetes.volume import Volume  # noqa
-from airflow.contrib.kubernetes.secret import Secret  # noqa
 
 
 class KubernetesPodOperator(BaseOperator):

airflow/hooks/hdfs_hook.py

@@ -16,18 +16,15 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-from six import PY2
-
 from airflow import configuration
 from airflow.exceptions import AirflowException
 from airflow.hooks.base_hook import BaseHook
 
-
-snakebite_imported = False
-if PY2:
+try:
     from snakebite.client import Client, HAClient, Namenode, AutoConfigClient
-    snakebite_imported = True
+    snakebite_loaded = True
+except ImportError:
+    snakebite_loaded = False
 
 
 class HDFSHookException(AirflowException):
@@ -47,7 +44,7 @@ class HDFSHook(BaseHook):
     """
     def __init__(self, hdfs_conn_id='hdfs_default', proxy_user=None,
                  autoconfig=False):
-        if not snakebite_imported:
+        if not snakebite_loaded:
             raise ImportError(
                 'This HDFSHook implementation requires snakebite, but '
                 'snakebite is not compatible with Python 3 '

airflow/operators/bash_operator.py

@@ -37,7 +37,7 @@ class BashOperator(BaseOperator):
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
-        :doc:`howto/operator/bash`
+        :ref:`howto/operator:BashOperator`
 
     If BaseOperator.do_xcom_push is True, the last line written to stdout
     will also be pushed to an XCom when the bash command completes

airflow/operators/python_operator.py

@@ -43,7 +43,7 @@ class PythonOperator(BaseOperator):
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
-        :doc:`howto/operator/python`
+        :ref:`howto/operator:PythonOperator`
 
     :param python_callable: A reference to an object that is callable
     :type python_callable: python callable

docs/autoapi_templates/index.rst

@@ -0,0 +1,129 @@
+..  Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+..    http://www.apache.org/licenses/LICENSE-2.0
+
+..  Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+API Reference
+=============
+
+Operators
+---------
+Operators allow for generation of certain types of tasks that become nodes in
+the DAG when instantiated. All operators derive from :class:`~airflow.models.BaseOperator` and
+inherit many attributes and methods that way.
+
+There are 3 main types of operators:
+
+- Operators that performs an **action**, or tell another system to
+  perform an action
+- **Transfer** operators move data from one system to another
+- **Sensors** are a certain type of operator that will keep running until a
+  certain criterion is met. Examples include a specific file landing in HDFS or
+  S3, a partition appearing in Hive, or a specific time of the day. Sensors
+  are derived from :class:`~airflow.sensors.base_sensor_operator.BaseSensorOperator` and run a poke
+  method at a specified :attr:`~airflow.sensors.base_sensor_operator.BaseSensorOperator.poke_interval` until it returns ``True``.
+
+BaseOperator
+''''''''''''
+All operators are derived from :class:`~airflow.models.BaseOperator` and acquire much
+functionality through inheritance. Since this is the core of the engine,
+it's worth taking the time to understand the parameters of :class:`~airflow.models.BaseOperator`
+to understand the primitive features that can be leveraged in your
+DAGs.
+
+BaseSensorOperator
+''''''''''''''''''
+All sensors are derived from :class:`~airflow.sensors.base_sensor_operator.BaseSensorOperator`. All sensors inherit
+the :attr:`~airflow.sensors.base_sensor_operator.BaseSensorOperator.timeout` and :attr:`~airflow.sensors.base_sensor_operator.BaseSensorOperator.poke_interval` on top of the :class:`~airflow.models.BaseOperator`
+attributes.
+
+Operators packages
+''''''''''''''''''
+All operators are in the following packages:
+
+.. toctree::
+  :includehidden:
+  :glob:
+  :maxdepth: 1
+
+  airflow/operators/index
+
+  airflow/sensors/index
+
+  airflow/contrib/operators/index
+
+  airflow/contrib/sensors/index
+
+
+Hooks
+-----
+Hooks are interfaces to external platforms and databases, implementing a common
+interface when possible and acting as building blocks for operators. All hooks
+are derived from :class:`~airflow.hooks.base_hook.BaseHook`.
+
+Hooks packages
+''''''''''''''
+All hooks are in the following packages:
+
+.. toctree::
+  :includehidden:
+  :glob:
+  :maxdepth: 1
+
+  airflow/hooks/index
+
+  airflow/contrib/hooks/index
+
+
+Executors
+---------
+Executors are the mechanism by which task instances get run. All executors are
+derived from :class:`~airflow.executors.base_executor.BaseExecutor`.
+
+Executors packages
+''''''''''''''''''
+All executors are in the following packages:
+
+.. toctree::
+  :includehidden:
+  :glob:
+  :maxdepth: 1
+
+  airflow/executors/index
+
+  airflow/contrib/executors/index
+
+
+Models
+------
+Models are built on top of the SQLAlchemy ORM Base class, and instances are
+persisted in the database.
+
+.. toctree::
+  :includehidden:
+  :glob:
+  :maxdepth: 1
+
+  airflow/models/index
+
+
+Core and community package
+--------------------------
+Formerly the core code was maintained by the original creators - Airbnb. The code
+that was in the contrib package was supported by the community. The project
+was passed to the Apache community and currently the entire code is
+maintained by the community, so now the division has no justification,
+and it is only due to historical reasons. Currently, all new classes are
+added only to the contrib package.

docs/build.sh

@@ -23,7 +23,8 @@ set -e
 FWDIR="$(cd "`dirname "$0"`"; pwd)"
 cd "$FWDIR"
 
-[ -d _build ] && rm -r _build
+[[ -d "_build" ]] && rm -r _build
+[[ -d "_api" ]] && rm -r _api
 
 SUCCEED_LINE=$(make html |\
     tee /dev/tty |\