add device_matrix_data kernels

common/unified/components/device_matrix_data_kernels.cpp

@@ -0,0 +1,81 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include "core/components/device_matrix_data_kernels.hpp"
+
+
+#include "common/unified/base/kernel_launch.hpp"
+#include "ginkgo/core/base/types.hpp"
+
+
+namespace gko {
+namespace kernels {
+namespace GKO_DEVICE_NAMESPACE {
+namespace components {
+
+
+template <typename ValueType, typename IndexType>
+void build_row_pointers(std::shared_ptr<const DefaultExecutor> exec,
+                        device_matrix_data<ValueType, IndexType>& data,
+                        int64* row_ptrs)
+{
+    if (data.nonzeros.get_num_elems() == 0) {
+        run_kernel(
+            exec, [] GKO_KERNEL(auto i, auto row_ptrs) { row_ptrs[i] = 0; },
+            data.size[0] + 1, row_ptrs);
+    } else {
+        run_kernel(
+            exec,
+            [] GKO_KERNEL(auto i, auto num_nonzeros, auto num_rows,
+                          auto nonzeros, auto row_ptrs) {
+                auto begin_row = i == 0 ? size_type{} : nonzeros[i - 1].row;
+                auto end_row = i == num_nonzeros ? num_rows : nonzeros[i].row;
+                for (auto row = begin_row; row < end_row; row++) {
+                    row_ptrs[row + 1] = i;
+                }
+                if (i == 0) {
+                    row_ptrs[0] = 0;
+                }
+            },
+            data.nonzeros.get_num_elems() + 1, data.nonzeros.get_num_elems(),
+            data.size[0], data.nonzeros, row_ptrs);
+    }
+}
+
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DEVICE_MATRIX_DATA_BUILD_ROW_PTRS_KERNEL);
+
+
+}  // namespace components
+}  // namespace GKO_DEVICE_NAMESPACE
+}  // namespace kernels
+}  // namespace gko

core/CMakeLists.txt

@@ -6,6 +6,7 @@ target_sources(ginkgo
     base/array.cpp
     base/combination.cpp
     base/composition.cpp
+    base/device_matrix_data.cpp
     base/executor.cpp
     base/mtx_io.cpp
     base/perturbation.cpp

core/base/device_matrix_data.cpp

@@ -0,0 +1,116 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/device_matrix_data.hpp>
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+#include "core/components/device_matrix_data_kernels.hpp"
+
+
+namespace gko {
+namespace components {
+namespace {
+
+
+GKO_REGISTER_OPERATION(remove_zeros, components::remove_zeros);
+GKO_REGISTER_OPERATION(sort_row_major, components::sort_row_major);
+
+
+}  // anonymous namespace
+}  // namespace components
+
+
+template <typename ValueType, typename IndexType>
+device_matrix_data<ValueType, IndexType>::device_matrix_data(
+    std::shared_ptr<const Executor> exec, dim<2> size, size_type nnz)
+    : size{size}, nonzeros{exec, nnz}
+{}
+
+
+template <typename ValueType, typename IndexType>
+device_matrix_data<ValueType, IndexType>::device_matrix_data(
+    dim<2> size, Array<nonzero_type> data)
+    : size{size}, nonzeros{std::move(data)}
+{}
+
+
+template <typename ValueType, typename IndexType>
+matrix_data<ValueType, IndexType>
+device_matrix_data<ValueType, IndexType>::copy_to_host() const
+{
+    const auto nnz = nonzeros.get_num_elems();
+    matrix_data<ValueType, IndexType> result{size};
+    result.nonzeros.resize(nnz);
+    nonzeros.get_executor()->get_master()->copy_from(
+        nonzeros.get_executor().get(), nnz, nonzeros.get_const_data(),
+        result.nonzeros.data());
+    return result;
+}
+
+
+template <typename ValueType, typename IndexType>
+device_matrix_data<ValueType, IndexType>
+device_matrix_data<ValueType, IndexType>::create_from_host(
+    std::shared_ptr<const Executor> exec, host_type& data)
+{
+    auto host_view = Array<nonzero_type>::view(
+        exec->get_master(), data.nonzeros.size(), data.nonzeros.data());
+    auto device_view = Array<nonzero_type>{exec, std::move(host_view)};
+    return device_matrix_data{data.size, std::move(device_view)};
+}
+
+
+template <typename ValueType, typename IndexType>
+void device_matrix_data<ValueType, IndexType>::remove_zeros()
+{
+    this->nonzeros.get_executor()->run(
+        components::make_remove_zeros(this->nonzeros));
+}
+
+
+template <typename ValueType, typename IndexType>
+void device_matrix_data<ValueType, IndexType>::sort_row_major()
+{
+    this->nonzeros.get_executor()->run(
+        components::make_sort_row_major(this->nonzeros));
+}
+
+
+#define GKO_DECLARE_DEVICE_MATRIX_DATA(ValueType, IndexType) \
+    struct device_matrix_data<ValueType, IndexType>
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DEVICE_MATRIX_DATA);
+
+
+}  // namespace gko

core/components/device_matrix_data_kernels.hpp

@@ -0,0 +1,129 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#ifndef GKO_CORE_COMPONENTS_DEVICE_MATRIX_DATA_KERNELS_HPP_
+#define GKO_CORE_COMPONENTS_DEVICE_MATRIX_DATA_KERNELS_HPP_
+
+
+#include <ginkgo/core/base/device_matrix_data.hpp>
+
+
+#include <memory>
+
+
+#include <ginkgo/core/base/executor.hpp>
+
+
+namespace gko {
+namespace kernels {
+
+
+#define GKO_DECLARE_DEVICE_MATRIX_DATA_REMOVE_ZEROS_KERNEL(ValueType, \
+                                                           IndexType) \
+    void remove_zeros(std::shared_ptr<const DefaultExecutor> exec,    \
+                      Array<matrix_data_entry<ValueType, IndexType>>& data)
+
+#define GKO_DECLARE_DEVICE_MATRIX_DATA_SORT_ROW_MAJOR_KERNEL(ValueType, \
+                                                             IndexType) \
+    void sort_row_major(std::shared_ptr<const DefaultExecutor> exec,    \
+                        Array<matrix_data_entry<ValueType, IndexType>>& data)
+
+#define GKO_DECLARE_DEVICE_MATRIX_DATA_BUILD_ROW_PTRS_KERNEL(ValueType,     \
+                                                             IndexType)     \
+    void build_row_pointers(std::shared_ptr<const DefaultExecutor> exec,    \
+                            device_matrix_data<ValueType, IndexType>& data, \
+                            int64* row_ptrs)
+
+
+#define GKO_DECLARE_ALL_AS_TEMPLATES                                          \
+    template <typename ValueType, typename IndexType>                         \
+    GKO_DECLARE_DEVICE_MATRIX_DATA_REMOVE_ZEROS_KERNEL(ValueType, IndexType); \
+    template <typename ValueType, typename IndexType>                         \
+    GKO_DECLARE_DEVICE_MATRIX_DATA_SORT_ROW_MAJOR_KERNEL(ValueType,           \
+                                                         IndexType);          \
+    template <typename ValueType, typename IndexType>                         \
+    GKO_DECLARE_DEVICE_MATRIX_DATA_BUILD_ROW_PTRS_KERNEL(ValueType, IndexType)
+
+
+namespace omp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace omp
+
+
+namespace cuda {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace cuda
+
+
+namespace reference {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace reference
+
+
+namespace hip {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace hip
+
+
+namespace dpcpp {
+namespace components {
+
+GKO_DECLARE_ALL_AS_TEMPLATES;
+
+}  // namespace components
+}  // namespace dpcpp
+
+
+#undef GKO_DECLARE_ALL_AS_TEMPLATES
+
+
+}  // namespace kernels
+}  // namespace gko
+
+
+#endif  // GKO_CORE_COMPONENTS_DEVICE_MATRIX_DATA_KERNELS_HPP_

core/device_hooks/common_kernels.inc.cpp

@@ -35,6 +35,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "core/base/mixed_precision_types.hpp"
 #include "core/components/absolute_array.hpp"
+#include "core/components/device_matrix_data_kernels.hpp"
 #include "core/components/fill_array.hpp"
 #include "core/components/precision_conversion.hpp"
 #include "core/components/prefix_sum.hpp"
@@ -118,6 +119,24 @@ GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL(ValueType)
 GKO_NOT_COMPILED(GKO_HOOK_MODULE);
 GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_ARRAY_KERNEL);
 
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_DEVICE_MATRIX_DATA_REMOVE_ZEROS_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DEVICE_MATRIX_DATA_REMOVE_ZEROS_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_DEVICE_MATRIX_DATA_SORT_ROW_MAJOR_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DEVICE_MATRIX_DATA_SORT_ROW_MAJOR_KERNEL);
+
+template <typename ValueType, typename IndexType>
+GKO_DECLARE_DEVICE_MATRIX_DATA_BUILD_ROW_PTRS_KERNEL(ValueType, IndexType)
+GKO_NOT_COMPILED(GKO_HOOK_MODULE);
+GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
+    GKO_DECLARE_DEVICE_MATRIX_DATA_BUILD_ROW_PTRS_KERNEL);
+
 
 }  // namespace components
 

cuda/CMakeLists.txt

@@ -66,6 +66,7 @@ find_library(CURAND curand
 
 add_library(ginkgo_cuda $<TARGET_OBJECTS:ginkgo_cuda_device> "")
 set(GKO_CUDA_COMMON_SOURCES
+    ../common/unified/components/device_matrix_data_kernels.cpp
     ../common/unified/components/precision_conversion.cpp
     ../common/unified/matrix/coo_kernels.cpp
     ../common/unified/matrix/csr_kernels.cpp
@@ -85,6 +86,7 @@ target_sources(ginkgo_cuda
     base/executor.cpp
     base/version.cpp
     components/absolute_array.cu
+    components/device_matrix_data_kernels.cu
     components/fill_array.cu
     components/prefix_sum.cu
     factorization/factorization_kernels.cu