Add xavier initializer

.clang-format

@@ -0,0 +1,27 @@
+# This file is used by clang-format to autoformat paddle source code
+#
+# The clang-format is part of llvm toolchain.
+# It need to install llvm and clang to format source code style.
+#
+# The basic usage is,
+#   clang-format -i -style=file PATH/TO/SOURCE/CODE
+#
+# The -style=file implicit use ".clang-format" file located in one of 
+# parent directory. 
+# The -i means inplace change.
+#
+# The document of clang-format is 
+#   http://clang.llvm.org/docs/ClangFormat.html
+#   http://clang.llvm.org/docs/ClangFormatStyleOptions.html
+---
+Language:        Cpp
+BasedOnStyle:  Google
+IndentWidth:     2
+TabWidth:        2
+ContinuationIndentWidth: 4
+AccessModifierOffset: -1  # The private/protected/public has no indent in class
+Standard:  Cpp11 
+AllowAllParametersOfDeclarationOnNextLine: true
+BinPackParameters: false
+BinPackArguments: false
+...

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0)
 
-include_directories(${CMAKE_CURRENT_SRC_DIR}/paddle)
+include_directories(${CMAKE_SOURCE_DIR})
 add_subdirectory(paddle)
 
 add_subdirectory(src)

src/CMakeLists.txt

@@ -29,7 +29,6 @@ if(OPENMP_FOUND)
     set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
 endif()
 
-include_directories(${CMAKE_CURRENT_SRC_DIR}) # for header files in src/
 include_directories(${CMAKE_SOURCE_DIR}/paddle) # for header files in paddle/
 include_directories(${CMAKE_BINARY_DIR}/paddle) # for *.pb.h
 include_directories(${CMAKE_BINARY_DIR}/third_party/install/glog/include)

src/function.h

@@ -14,17 +14,24 @@
 
 #pragma once
 
+#include <cmath>
 #include <string>
+#include <vector>
 
-#include "paddle/contrib/tape/tape.h"
-#include "paddle/contrib/tape/variable.h"
 #include "paddle/fluid/framework/type_defs.h"
+#include "src/tape.h"
+#include "src/variable.h"
 
 namespace paddle {
 namespace tape {
 
 class Function {};
 
+class RandomSeed {
+ public:
+  static int GetRandomSeed() { return 0; }
+};
+
 class Fill {
  public:
   Fill(const std::string &initializer, const framework::AttributeMap &attrs)
@@ -56,17 +63,21 @@ class Linear {
         act_(act) {
     Tape init_tape;
 
-    std::string initializer = "fill_constant";
+    // Use Xavier to initialize Weight
+    float limit = sqrt(6.0 / static_cast<float>(in_dim + out_dim));
     framework::AttributeMap attrs;
-    attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32;
     attrs["shape"] = std::vector<int>{in_dim, out_dim};
-    attrs["value"] = 1.0f;
-    init_tape.AddOp(initializer, {}, {{"Out", {w_}}}, attrs);
+    attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32;
+    attrs["min"] = -limit;
+    attrs["max"] = limit;
+    attrs["seed"] = RandomSeed::GetRandomSeed();
+    init_tape.AddOp("uniform_random", {}, {{"Out", {w_}}}, attrs);
 
+    // Use fill zero to initialize Bias
     attrs["dtype"] = paddle::framework::proto::VarType::Type::VarType_Type_FP32;
     attrs["shape"] = std::vector<int>{out_dim};
-    attrs["value"] = 1.0f;
-    init_tape.AddOp(initializer, {}, {{"Out", {b_}}}, attrs);
+    attrs["value"] = 0.0f;
+    init_tape.AddOp("fill_constant", {}, {{"Out", {b_}}}, attrs);
 
     init_tape.Forward();
   }
@@ -98,7 +109,7 @@ class Linear {
 
 class SGD {
  public:
-  SGD(float learning_rate) : learning_rate_(new Variable("sgd")) {
+  explicit SGD(float learning_rate) : learning_rate_(new Variable("sgd")) {
     Tape init_tape;
 
     std::string initializer = "fill_constant";
@@ -111,7 +122,7 @@ class SGD {
     init_tape.Forward();
   }
 
-  void operator()(VariableHandle input) {
+  void Update(VariableHandle input) {
     PADDLE_ENFORCE(get_global_tape().HasBeenBackwarded(),
                    "optimization must happen after the backward");
     Tape temp_tape;
@@ -127,5 +138,5 @@ class SGD {
  private:
   VariableHandle learning_rate_;
 };
-}
-}
+}  // namespace tape
+}  // namespace paddle

src/tape.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/contrib/tape/tape.h"
+#include "src/tape.h"
 
 #include <list>
 #include <map>
@@ -261,5 +261,5 @@ Tape &get_global_tape() {
 }
 
 void reset_global_tape() { get_global_tape() = Tape(); }
-}
-}
+}  // namespace tape
+}  // namespace paddle

src/tape.h

@@ -18,7 +18,7 @@
 #include <string>
 #include <vector>
 
-#include "paddle/contrib/tape/variable.h"
+#include "src/variable.h"
 
 namespace paddle {
 namespace tape {
@@ -60,5 +60,5 @@ class Tape {
 Tape &get_global_tape();
 
 void reset_global_tape();
-}
-}
+}  // namespace tape
+}  // namespace paddle

src/test_tape.cc

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "paddle/contrib/tape/function.h"
 #include "gtest/gtest.h"
+#include "src/function.h"
 
 using paddle::tape::VariableHandle;
 using paddle::tape::Variable;
@@ -50,10 +50,10 @@ TEST(Tape, TestMLP) {
     get_global_tape().Backward(loss);
 
     for (auto w : linear1.Params()) {
-      sgd(w);
+      sgd.Update(w);
     }
     for (auto w : linear2.Params()) {
-      sgd(w);
+      sgd.Update(w);
     }
   }
 }

src/variable.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "variable.h"
+#include "src/variable.h"
 
 namespace paddle {
 namespace tape {
@@ -29,5 +29,6 @@ void Variable::InitializeVariable() {
                  var_type);
   }
 }
-}
-}
+
+}  // namespace tape
+}  // namespace paddle

src/variable.h

@@ -14,6 +14,7 @@
 #pragma once
 
 #include <memory>
+#include <string>
 
 #include "paddle/fluid/framework/operator.h"  // framework::kGradVarSuffix
 #include "paddle/fluid/framework/program_desc.h"
@@ -32,7 +33,7 @@ using VariableHandle = std::shared_ptr<Variable>;
  */
 class Variable {
  public:
-  Variable(const std::string pre_fix)
+  explicit Variable(const std::string pre_fix)
       : desc_(pre_fix + std::to_string(count())) {}
 
   Variable(const std::string pre_fix, bool is_grad)
@@ -62,13 +63,13 @@ class Variable {
 
   // void value() {};
 
-  const framework::VarDesc& Desc() const { return desc_; }
-  framework::VarDesc* MutableDesc() { return &desc_; }
+  const framework::VarDesc &Desc() const { return desc_; }
+  framework::VarDesc *MutableDesc() { return &desc_; }
 
   // TODO(tonyyang-svail): No need to expose name
   std::string Name() const { return desc_.Name(); }
 
-  framework::Variable* Var() { return &var_; }
+  framework::Variable *Var() { return &var_; }
 
  private:
   int count() {
@@ -79,7 +80,8 @@ class Variable {
   framework::VarDesc desc_;
   framework::Variable var_;
 
+  // Not own
   std::weak_ptr<Variable> grad_;
 };
-}
-}
+}  // namespace tape
+}  // namespace paddle