Run Python OP tests in a single Python process to improve test time.

Currently, our tests run with 2 GPUs, the init time is absurdly long:
about 4s for each process.  Currently, we run each OP test on
different processes. This PR:

1. create cmake function py_test_modules which will generate the
Makefile that runs a list of Python unittest module in a single Python
process.

1. move all "python unittest compatible" (e.g., used the unittest
package, not just a regular python file). from fluid/tests to
fluid/tests/unittests.

1. cmake now will run all OP tests in fluid/tests/unittests in a
single process, except the time-consuming tests, they are separated
into different processes to utilize parallelism. Please make sure to
use the unittest package if you put the python test file in
fluid/tests/unittests

1. remove all exit(0) from fluid/tests/unittests/*.py, exit(0) is used
to disable unittest, we can not do it when running all tests in a
single process since it will terminate the process without running the
other tests. Instead, the test is disabled in
fluid/tests/unittests/CMakeLists.txt. FIXME is added for each disabled
item. Please disable the unittest from
fluid/tests/unittests/CMakeLists.txt, instead of adding exit(0) to the
Python file, for all Python file in fluid/tests/unittests/.

cmake/generic.cmake

@@ -479,6 +479,19 @@ function(py_test TARGET_NAME)
   endif()
 endfunction()
 
+function(py_test_modules TARGET_NAME)
+  if(WITH_TESTING)
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs MODULES DEPS ARGS ENVS)
+    cmake_parse_arguments(py_test_modules "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+    add_test(NAME ${TARGET_NAME}
+             COMMAND env PYTHONPATH=${PADDLE_PYTHON_BUILD_DIR}/lib-python ${py_test_modules_ENVS}
+             ${PYTHON_EXECUTABLE} -u -m unittest --verbose ${py_test_modules_MODULES} ${py_test_modules_ARGS}
+             WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+endfunction()
+
 # grpc_library generate grpc code using grpc_cpp_plugin and protoc
 # then build the generated protobuf code and grpc code with your
 # implementation source codes together. Use SRCS argument for your

python/paddle/v2/fluid/tests/CMakeLists.txt

@@ -1,16 +1,11 @@
 file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
 string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
 
-if(NOT WITH_DISTRIBUTE)
-    list(REMOVE_ITEM TEST_OPS test_recv_op)
-endif(NOT WITH_DISTRIBUTE)
-
-list(REMOVE_ITEM TEST_OPS test_warpctc_op)
 foreach(src ${TEST_OPS})
-    py_test(${src} SRCS ${src}.py)
+  py_test(${src} SRCS ${src}.py)
 endforeach()
-py_test(test_warpctc_op SRCS test_warpctc_op.py ENVS FLAGS_warpctc_dir=${WARPCTC_LIB_DIR})
 
+add_subdirectory(unittests)
 add_subdirectory(book)
 add_subdirectory(book_distribute)
 add_subdirectory(book_memory_optimization)

python/paddle/v2/fluid/tests/unittests/CMakeLists.txt

@@ -0,0 +1,29 @@
+file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py")
+string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}")
+
+if(NOT WITH_DISTRIBUTE)
+    list(REMOVE_ITEM TEST_OPS test_recv_op)
+endif(NOT WITH_DISTRIBUTE)
+
+list(REMOVE_ITEM TEST_OPS test_seq_concat_op) # FIXME(helin): /~https://github.com/PaddlePaddle/Paddle/issues/8290
+list(REMOVE_ITEM TEST_OPS test_modified_huber_loss_op) # FIXME(qijun) /~https://github.com/PaddlePaddle/Paddle/issues/5184 
+list(REMOVE_ITEM TEST_OPS test_lstm_unit_op) # # FIXME(qijun) /~https://github.com/PaddlePaddle/Paddle/issues/5185
+list(REMOVE_ITEM TEST_OPS test_nce) # IXME(qijun) /~https://github.com/PaddlePaddle/Paddle/issues/7778
+list(REMOVE_ITEM TEST_OPS test_recurrent_op) # FIXME(qijun) /~https://github.com/PaddlePaddle/Paddle/issues/6152
+list(REMOVE_ITEM TEST_OPS test_cond_op) # FIXME(qijun): /~https://github.com/PaddlePaddle/Paddle/issues/5101#issuecomment-339814957
+list(REMOVE_ITEM TEST_OPS test_detection_output_op) # FIXME: detection_output_op will be rewritten. This unittest should be
+
+list(REMOVE_ITEM TEST_OPS op_test) # op_test is a helper python file, not a test
+list(REMOVE_ITEM TEST_OPS decorators) # decorators is a helper python file, not a test
+
+# test time consuming OPs in a separate process for expliot parallism
+list(REMOVE_ITEM TEST_OPS test_warpctc_op)
+py_test_modules(test_warpctc_op MODULES test_warpctc_op ENVS FLAGS_warpctc_dir=${WARPCTC_LIB_DIR})
+
+list(REMOVE_ITEM TEST_OPS test_dyn_rnn)
+py_test_modules(test_train_dyn_rnn MODULES test_dyn_rnn)
+
+list(REMOVE_ITEM TEST_OPS test_mul_op)
+py_test_modules(test_mul_op MODULES test_mul_op)
+
+py_test_modules("test_all_ops" MODULES ${TEST_OPS})

python/paddle/v2/fluid/tests/unittests/__init__.py

@@ -0,0 +1,13 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

python/paddle/v2/fluid/tests/unittests/nvprof_config_file

@@ -0,0 +1,7 @@
+gpustarttimestamp
+gpuendtimestamp
+gridsize3d
+threadblocksize
+streamid
+enableonstart 0
+conckerneltrace

python/paddle/v2/fluid/tests/test_cond_op.py → python/paddle/v2/fluid/tests/unittests/test_cond_op.py

@@ -125,7 +125,4 @@ def test_forward(self):
 
 
 if __name__ == "__main__":
-    exit(
-        0
-    )  # FIXME(qijun): /~https://github.com/PaddlePaddle/Paddle/issues/5101#issuecomment-339814957
     unittest.main()

python/paddle/v2/fluid/tests/test_detection_output_op.py → python/paddle/v2/fluid/tests/unittests/test_detection_output_op.py

@@ -68,6 +68,4 @@ def init_test_case(self):
 
 
 if __name__ == '__main__':
-    # FIXME: detection_output_op will be rewritten. This unittest should be
-    # enabled after rewriting.
-    exit(0)  # temporary disable this unittest
+    unittest.main()

python/paddle/v2/fluid/tests/test_lstm_unit_op.py → python/paddle/v2/fluid/tests/unittests/test_lstm_unit_op.py

@@ -49,6 +49,4 @@ def test_check_grad(self):
 
 
 if __name__ == "__main__":
-    # FIXME(qijun) /~https://github.com/PaddlePaddle/Paddle/issues/5185
-    exit(0)
     unittest.main()

python/paddle/v2/fluid/tests/test_modified_huber_loss_op.py → python/paddle/v2/fluid/tests/unittests/test_modified_huber_loss_op.py

@@ -59,6 +59,4 @@ def test_check_grad(self):
 
 
 if __name__ == '__main__':
-    exit(0)
-    # FIXME(qijun) /~https://github.com/PaddlePaddle/Paddle/issues/5184
     unittest.main()

python/paddle/v2/fluid/tests/test_nce.py → python/paddle/v2/fluid/tests/unittests/test_nce.py

@@ -109,6 +109,4 @@ def set_data(self):
 
 
 if __name__ == '__main__':
-    # FIXME(qijun) /~https://github.com/PaddlePaddle/Paddle/issues/7778
-    exit(0)
     unittest.main()

python/paddle/v2/fluid/tests/test_recurrent_op.py → python/paddle/v2/fluid/tests/unittests/test_recurrent_op.py

@@ -468,6 +468,4 @@ def create_rnn_op(self):
 
 
 if __name__ == '__main__':
-    # FIXME(qijun) /~https://github.com/PaddlePaddle/Paddle/issues/6152
-    exit(0)
     unittest.main()

python/paddle/v2/fluid/tests/test_seq_concat_op.py → python/paddle/v2/fluid/tests/unittests/test_seq_concat_op.py

@@ -16,7 +16,6 @@
 import numpy as np
 import sys
 from op_test import OpTest
-exit(0)
 
 
 def to_abs_lod(lod):