Add StringTensor

.gitignore

@@ -9,10 +9,12 @@ paddle/phi/api/backward/backward_api.h
 paddle/phi/api/backward/sparse_bw_api.h
 paddle/phi/api/include/api.h
 paddle/phi/api/include/sparse_api.h
+paddle/phi/api/include/strings_api.h
 paddle/phi/api/lib/api.cc
 paddle/phi/api/lib/dygraph_api.*
 paddle/phi/api/lib/backward_api.cc
 paddle/phi/api/lib/sparse_api.cc
+paddle/phi/api/lib/strings_api.cc
 paddle/phi/api/lib/sparse_bw_api.cc
 paddle/phi/extension.h
 paddle/phi/include/*

cmake/phi.cmake

@@ -173,6 +173,7 @@ function(kernel_library TARGET)
             string(REGEX MATCHALL "#include \"paddle\/phi\/kernels\/[a-z0-9_]+_kernel.h\"" include_kernels ${target_content})
         else()
             string(REGEX MATCHALL "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/[a-z0-9_]+_kernel.h\"" include_kernels ${target_content})
+            string(REGEX MATCHALL "#include \"paddle\/phi\/kernels\/[a-z0-9_]+_kernel.h\"" include_dense_kernels ${target_content})
         endif()
         foreach(include_kernel ${include_kernels})
             if ("${kernel_library_SUB_DIR}" STREQUAL "")
@@ -183,6 +184,16 @@ function(kernel_library TARGET)
             string(REGEX REPLACE ".h\"" "" kernel_name ${kernel_name})
             list(APPEND kernel_deps ${kernel_name})
         endforeach()
+
+        if (NOT "${kernel_library_SUB_DIR}" STREQUAL "")
+            foreach(include_dense_kernel ${include_dense_kernels})
+                string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/" "" kernel_name ${include_dense_kernel})
+                string(REGEX REPLACE ".h\"" "" kernel_name ${kernel_name})
+                list(APPEND kernel_deps ${kernel_name})
+            endforeach()
+        endif()
+
+
     endforeach()
     list(REMOVE_DUPLICATES kernel_deps)
     list(REMOVE_ITEM kernel_deps ${TARGET})
@@ -331,7 +342,6 @@ function(register_kernels)
     file(GLOB KERNELS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*_kernel.h")
     string(REPLACE ".h" "" KERNELS "${KERNELS}")
     list(LENGTH register_kernels_DEPS register_kernels_DEPS_len)
-
     foreach(target ${KERNELS})
         list(FIND register_kernels_EXCLUDES ${target} _index)
         if (${_index} EQUAL -1)

paddle/fluid/framework/convert_utils.cc

@@ -81,6 +81,8 @@ paddle::framework::proto::VarType::Type TransToProtoVarType(
       return paddle::framework::proto::VarType::BF16;
     case DataType::BOOL:
       return paddle::framework::proto::VarType::BOOL;
+    case DataType::PSTRING:
+      return paddle::framework::proto::VarType::RAW;
     default:
       PADDLE_THROW(paddle::platform::errors::Unimplemented(
           "Unsupported data type `%s` when casting it into "

paddle/phi/CMakeLists.txt

@@ -23,7 +23,7 @@ add_subdirectory(tools)
 add_subdirectory(tests)
 
 # make an unity target for compile deps
-set(PHI_DEPS convert_utils dense_tensor phi_context kernel_factory kernel_context arg_map_context infermeta lod_utils op_compat_infos sparse_csr_tensor sparse_coo_tensor)
+set(PHI_DEPS convert_utils dense_tensor phi_context kernel_factory kernel_context arg_map_context infermeta lod_utils op_compat_infos sparse_csr_tensor sparse_coo_tensor string_tensor)
 get_property(phi_kernels GLOBAL PROPERTY PHI_KERNELS)
 set(PHI_DEPS ${PHI_DEPS} ${phi_kernels})
 

paddle/phi/api/CMakeLists.txt

@@ -1,2 +1,2 @@
 add_subdirectory(lib)
-cc_library(phi_api SRCS all.cc DEPS phi_function_api phi_bw_function_api sparse_api sparse_bw_api)
+cc_library(phi_api SRCS all.cc DEPS phi_function_api phi_bw_function_api sparse_api sparse_bw_api strings_api)

paddle/phi/api/lib/CMakeLists.txt

@@ -56,6 +56,14 @@ set(sparse_bw_api_source_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/lib/sparse_bw_a
 set(sparse_bw_api_header_file_tmp ${sparse_bw_api_header_file}.tmp)
 set(sparse_bw_api_source_file_tmp ${sparse_bw_api_source_file}.tmp)
 
+# strings api file
+set(strings_api_gen_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/strings_api_gen.py)
+set(strings_api_yaml_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/strings_api.yaml)
+set(strings_api_header_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/include/strings_api.h)
+set(strings_api_source_file ${CMAKE_SOURCE_DIR}/paddle/phi/api/lib/strings_api.cc)
+set(strings_api_header_file_tmp ${strings_api_header_file}.tmp)
+set(strings_api_source_file_tmp ${strings_api_source_file}.tmp)
+
 # wrapped infermeta file
 set(wrapped_infermeta_gen_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/wrapped_infermeta_gen.py)
 set(api_yaml_file ${CMAKE_SOURCE_DIR}/python/paddle/utils/code_gen/api.yaml)
@@ -123,6 +131,19 @@ add_custom_command(
   DEPENDS ${sparse_bw_api_yaml_file} ${sparse_bw_api_gen_file} ${api_gen_base} ${api_gen_file} ${sparse_api_gen_file} ${bw_api_gen_file}
   VERBATIM)
 
+# generate strings api
+add_custom_command(
+  OUTPUT ${strings_api_header_file} ${strings_api_source_file}
+  COMMAND ${PYTHON_EXECUTABLE} ${strings_api_gen_file}
+                 --api_yaml_path ${strings_api_yaml_file}
+                 --api_header_path ${strings_api_header_file_tmp}
+                 --api_source_path ${strings_api_source_file_tmp}
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${strings_api_header_file_tmp} ${strings_api_header_file}
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different ${strings_api_source_file_tmp} ${strings_api_source_file}
+  COMMENT "copy_if_different ${strings_api_header_file} ${strings_strings_api_source_file}"
+  DEPENDS ${strings_api_yaml_file} ${strings_api_gen_file} ${api_gen_base} ${api_gen_file}
+  VERBATIM)
+
 # generate wrapped infermeta
 add_custom_command(
   OUTPUT ${wrapped_infermeta_header_file} ${wrapped_infermeta_source_file}
@@ -147,5 +168,5 @@ cc_library(phi_dygraph_api SRCS ${dygraph_api_source_file} DEPS phi_tensor_raw p
 cc_library(phi_bw_function_api SRCS ${bw_api_source_file} DEPS phi_tensor_raw phi kernel_dispatch api_gen_utils backward_infermeta phi_data_transform phi_function_api api_custom_impl)
 cc_library(sparse_api SRCS ${sparse_api_source_file} DEPS phi_tensor_raw phi kernel_dispatch api_gen_utils sparse_api_custom_impl)
 cc_library(sparse_bw_api SRCS ${sparse_bw_api_source_file} DEPS phi_tensor_raw phi kernel_dispatch api_gen_utils sparse_api sparse_api_custom_impl)
-
+cc_library(strings_api SRCS ${strings_api_source_file} DEPS phi_tensor_raw phi kernel_dispatch api_gen_utils phi_data_transform)
 cc_library(phi_tensor SRCS tensor_method.cc DEPS phi_tensor_raw phi_function_api api_gen_utils kernel_dispatch infermeta)

paddle/phi/api/lib/api_declare.h

@@ -19,3 +19,4 @@ limitations under the License. */
 
 PD_DECLARE_API(Math);
 PD_DECLARE_API(SparseApi);
+PD_DECLARE_API(StringsApi);

paddle/phi/api/lib/api_gen_utils.cc

@@ -56,6 +56,10 @@ std::shared_ptr<phi::SelectedRows> TensorToSelectedRows(
   return nullptr;
 }
 
+std::shared_ptr<phi::StringTensor> TensorToStringTensor(const Tensor& tensor) {
+  return std::dynamic_pointer_cast<phi::StringTensor>(tensor.impl());
+}
+
 /* ----------------- for infer_meta --------------------- */
 
 phi::MetaTensor MakeMetaTensor(const phi::DenseTensor& tensor) {
@@ -92,6 +96,10 @@ paddle::optional<phi::MetaTensor> MakeMetaTensor(
   return {paddle::none};
 }
 
+phi::MetaTensor MakeMetaTensor(const phi::StringTensor& tensor) {
+  return phi::MetaTensor(tensor);
+}
+
 /* ------------------ for output ----------------------- */
 
 phi::DenseTensor* SetKernelOutput(Backend backend, Tensor* out) {
@@ -148,5 +156,20 @@ phi::TensorBase* SetSparseKernelOutput(Tensor* out, TensorType type) {
   return out->impl().get();
 }
 
+phi::TensorBase* SetStringsKernelOutput(Backend backend,
+                                        Tensor* out,
+                                        TensorType type) {
+  if (!out->initialized()) {
+    auto place = phi::TransToPhiPlace(backend);
+    if (type == TensorType::STRING_TENSOR) {
+      auto strings_tensor = std::make_shared<phi::StringTensor>(
+          paddle::memory::AllocShared(place, 0), phi::StringTensorMeta());
+      out->set_impl(strings_tensor);
+      return strings_tensor.get();
+    }
+  }
+  return out->impl().get();
+}
+
 }  // namespace experimental
 }  // namespace paddle

paddle/phi/api/lib/api_gen_utils.h

@@ -22,11 +22,12 @@ limitations under the License. */
 #include "paddle/phi/core/selected_rows.h"
 #include "paddle/phi/core/sparse_coo_tensor.h"
 #include "paddle/phi/core/sparse_csr_tensor.h"
+#include "paddle/phi/core/string_tensor.h"
 
 namespace paddle {
 namespace experimental {
 
-enum class TensorType { DENSE_TENSOR, SPARSE_CSR, SPARSE_COO };
+enum class TensorType { DENSE_TENSOR, SPARSE_CSR, SPARSE_COO, STRING_TENSOR };
 
 /* ------------------ for input ----------------------- */
 
@@ -43,6 +44,8 @@ std::shared_ptr<phi::SelectedRows> TensorToSelectedRows(const Tensor& tensor);
 std::shared_ptr<phi::SelectedRows> TensorToSelectedRows(
     const paddle::optional<Tensor>& tensor);
 
+std::shared_ptr<phi::StringTensor> TensorToStringTensor(const Tensor& tensor);
+
 /* ----------------- for infer_meta --------------------- */
 
 phi::MetaTensor MakeMetaTensor(const phi::DenseTensor& tensor);
@@ -58,6 +61,8 @@ phi::MetaTensor MakeMetaTensor(const phi::SelectedRows& tensor);
 paddle::optional<phi::MetaTensor> MakeMetaTensor(
     const paddle::optional<const phi::SelectedRows&>& tensor);
 
+phi::MetaTensor MakeMetaTensor(const phi::StringTensor& tensor);
+
 /* ------------------ for output ----------------------- */
 
 phi::DenseTensor* SetKernelOutput(Backend backend, Tensor* out);
@@ -70,5 +75,9 @@ phi::SelectedRows* SetSelectedRowsKernelOutput(Backend backend, Tensor* out);
 
 phi::TensorBase* SetSparseKernelOutput(Tensor* out, TensorType type);
 
+phi::TensorBase* SetStringsKernelOutput(Backend backend,
+                                        Tensor* out,
+                                        TensorType type);
+
 }  // namespace experimental
 }  // namespace paddle

paddle/phi/api/lib/utils/CMakeLists.txt

@@ -1,2 +1,2 @@
 cc_library(phi_api_utils SRCS storage.cc tensor_utils.cc DEPS
-tensor_base convert_utils dense_tensor lod_tensor selected_rows_utils place var_type_traits scalar)
+tensor_base convert_utils dense_tensor lod_tensor selected_rows_utils place var_type_traits scalar string_tensor)