Skip to content

Commit

Permalink
wip matrix read
Browse files Browse the repository at this point in the history
  • Loading branch information
@upsj
upsj committed Nov 4, 2021
1 parent e7e3af5 commit 167c2be
Show file tree
Hide file tree
Showing 31 changed files with 421 additions and 114 deletions.
23 changes: 23 additions & 0 deletions common/unified/matrix/coo_kernels.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,29 @@ namespace GKO_DEVICE_NAMESPACE {
namespace coo {


template <typename ValueType, typename IndexType>
void from_matrix_data(
std::shared_ptr<const DefaultExecutor> exec,
const Array<matrix_data_entry<ValueType, IndexType>>& nonzeros,
matrix::Coo<ValueType, IndexType>* output)
{
run_kernel(
exec,
[] GKO_KERNEL(auto i, auto nonzeros, auto rows, auto cols,
auto values) {
auto nonzero = nonzeros[i];
rows[i] = nonzero.row;
cols[i] = nonzero.column;
values[i] = nonzero.value;
},
nonzeros.get_num_elems(), nonzeros, output->get_row_idxs(),
output->get_col_idxs(), output->get_values());
}

GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_COO_FROM_MATRIX_DATA_KERNEL);


template <typename ValueType, typename IndexType>
void extract_diagonal(std::shared_ptr<const DefaultExecutor> exec,
const matrix::Coo<ValueType, IndexType>* orig,
Expand Down
38 changes: 38 additions & 0 deletions common/unified/matrix/csr_kernels.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,44 @@ namespace GKO_DEVICE_NAMESPACE {
namespace csr {


template <typename ValueType, typename IndexType>
void from_matrix_data(
std::shared_ptr<const DefaultExecutor> exec,
const Array<matrix_data_entry<ValueType, IndexType>>& nonzeros,
matrix::Csr<ValueType, IndexType>* output)
{
if (nonzeros.get_num_elems() == 0) {
run_kernel(
exec, [] GKO_KERNEL(auto i, auto row_ptrs) { row_ptrs[i] = 0; },
output->get_size()[0] + 1, output->get_row_ptrs());
} else {
run_kernel(
exec,
[] GKO_KERNEL(auto i, auto num_nonzeros, auto num_rows,
auto nonzeros, auto row_ptrs, auto cols,
auto values) {
auto begin_row = i == 0 ? size_type{} : nonzeros[i - 1].row;
auto end_row = i == num_nonzeros ? num_rows : nonzeros[i].row;
for (auto row = begin_row; row < end_row; row++) {
row_ptrs[row + 1] = i;
}
if (i < num_nonzeros) {
cols[i] = nonzeros[i].column;
values[i] = nonzeros[i].value;
} else {
row_ptrs[0] = 0;
}
},
nonzeros.get_num_elems() + 1, nonzeros.get_num_elems(),
output->get_size()[0], nonzeros, output->get_row_ptrs(),
output->get_col_idxs(), output->get_values());
}
}

GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_CSR_FROM_MATRIX_DATA_KERNEL);


template <typename IndexType>
void invert_permutation(std::shared_ptr<const DefaultExecutor> exec,
size_type size, const IndexType* permutation_indices,
Expand Down
19 changes: 19 additions & 0 deletions common/unified/matrix/dense_kernels.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,25 @@ void fill(std::shared_ptr<const DefaultExecutor> exec,
GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_FILL_KERNEL);


template <typename ValueType, typename IndexType>
void fill_in_matrix_data(
std::shared_ptr<const DefaultExecutor> exec,
const Array<matrix_data_entry<ValueType, IndexType>>& nonzeros,
matrix::Dense<ValueType>* output)
{
run_kernel(
exec,
[] GKO_KERNEL(auto i, auto data, auto output) {
const auto entry = data[i];
output(entry.row, entry.column) = entry.value;
},
nonzeros.get_num_elems(), nonzeros, output);
}

GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_DENSE_FILL_IN_MATRIX_DATA_KERNEL);


template <typename ValueType, typename ScalarType>
void scale(std::shared_ptr<const DefaultExecutor> exec,
const matrix::Dense<ScalarType>* alpha, matrix::Dense<ValueType>* x)
Expand Down
91 changes: 91 additions & 0 deletions common/unified/matrix/ell_kernels.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*******************************<GINKGO LICENSE>******************************
Copyright (c) 2017-2021, the Ginkgo authors
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
******************************<GINKGO LICENSE>*******************************/

#include "core/matrix/ell_kernels.hpp"


#include <ginkgo/core/base/math.hpp>


#include "common/unified/base/kernel_launch.hpp"


namespace gko {
namespace kernels {
namespace GKO_DEVICE_NAMESPACE {
/**
* @brief The Ell matrix format namespace.
*
* @ingroup ell
*/
namespace ell {


template <typename ValueType, typename IndexType>
void from_matrix_data(
std::shared_ptr<const DefaultExecutor> exec,
const Array<matrix_data_entry<ValueType, IndexType>>& nonzeros,
const int64* row_ptrs, matrix::Ell<ValueType, IndexType>* output)
{
constexpr auto missing_column = output->missing_column;
run_kernel(
exec,
[] GKO_KERNEL(size_type row, size_type stride, size_type num_cols,
auto row_ptrs, auto nonzeros, auto cols, auto values) {
const auto begin = row_ptrs[row];
const auto end = row_ptrs[row + 1];
const auto nnz = end - begin;
auto out_idx = row;
for (auto i = begin; i < end; i++) {
const auto nonzero = nonzeros[i];
cols[out_idx] = nonzero.column;
values[out_idx] = nonzero.value;
out_idx += stride;
}
for (auto i = nnz; i < num_cols; i++) {
cols[out_idx] = 0;
values[out_idx] = zero(values[out_idx]);
}
},
output->get_size()[0], output->get_stride(),
output->get_num_stored_elements_per_row(), row_ptrs, nonzeros,
output->get_col_idxs(), output->get_values());
}

GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE(
GKO_DECLARE_ELL_FROM_MATRIX_DATA_KERNEL);


} // namespace ell
} // namespace GKO_DEVICE_NAMESPACE
} // namespace kernels
} // namespace gko
3 changes: 3 additions & 0 deletions core/device_hooks/common_kernels.inc.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_DOT_KERNEL);
GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_CONJ_DOT_KERNEL);
GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_NORM2_KERNEL);
GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_NORM1_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_FILL_IN_MATRIX_DATA_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL);
Expand Down Expand Up @@ -409,6 +410,7 @@ GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_ADVANCED_SPMV_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEMM_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_ADVANCED_SPGEMM_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_SPGEAM_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_FROM_MATRIX_DATA_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_CONVERT_TO_DENSE_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_CONVERT_TO_COO_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_CSR_CONVERT_TO_ELL_KERNEL);
Expand Down Expand Up @@ -471,6 +473,7 @@ GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_SPMV_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_ADVANCED_SPMV_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_SPMV2_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_ADVANCED_SPMV2_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_FROM_MATRIX_DATA_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_CONVERT_TO_CSR_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL);
GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_COO_EXTRACT_DIAGONAL_KERNEL);
Expand Down
42 changes: 18 additions & 24 deletions core/matrix/coo.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ GKO_REGISTER_OPERATION(spmv, coo::spmv);
GKO_REGISTER_OPERATION(advanced_spmv, coo::advanced_spmv);
GKO_REGISTER_OPERATION(spmv2, coo::spmv2);
GKO_REGISTER_OPERATION(advanced_spmv2, coo::advanced_spmv2);
GKO_REGISTER_OPERATION(from_matrix_data, coo::from_matrix_data);
GKO_REGISTER_OPERATION(convert_to_csr, coo::convert_to_csr);
GKO_REGISTER_OPERATION(convert_to_dense, coo::convert_to_dense);
GKO_REGISTER_OPERATION(extract_diagonal, coo::extract_diagonal);
Expand Down Expand Up @@ -193,36 +194,29 @@ void Coo<ValueType, IndexType>::move_to(Dense<ValueType>* result)
template <typename ValueType, typename IndexType>
void Coo<ValueType, IndexType>::read(const mat_data& data)
{
size_type nnz = 0;
for (const auto& elem : data.nonzeros) {
nnz += (elem.value != zero<ValueType>());
}
auto tmp = Coo::create(this->get_executor()->get_master(), data.size, nnz);
size_type elt = 0;
for (const auto& elem : data.nonzeros) {
auto val = elem.value;
if (val != zero<ValueType>()) {
tmp->get_row_idxs()[elt] = elem.row;
tmp->get_col_idxs()[elt] = elem.column;
tmp->get_values()[elt] = elem.value;
elt++;
}
}
this->copy_from(std::move(tmp));
this->read(device_mat_data::create_from_host(this->get_executor(),
const_cast<mat_data&>(data)));
}


template <typename ValueType, typename IndexType>
void Coo<ValueType, IndexType>::read(const device_mat_data& data)
{
const auto nnz = data.nonzeros.get_num_elems();
auto exec = this->get_executor();
this->set_size(data.size);
this->row_idxs_.resize_and_reset(nnz);
this->col_idxs_.resize_and_reset(nnz);
this->values_.resize_and_reset(nnz);
auto nonzeros = make_temporary_clone(exec, &data.nonzeros);
exec->run(coo::make_from_matrix_data(*nonzeros, this));
}


template <typename ValueType, typename IndexType>
void Coo<ValueType, IndexType>::write(mat_data& data) const
{
std::unique_ptr<const LinOp> op{};
const Coo* tmp{};
if (this->get_executor()->get_master() != this->get_executor()) {
op = this->clone(this->get_executor()->get_master());
tmp = static_cast<const Coo*>(op.get());
} else {
tmp = this;
}
auto tmp = make_temporary_clone(this->get_executor()->get_master(), this);

data = {this->get_size(), {}};

Expand Down
8 changes: 8 additions & 0 deletions core/matrix/coo_kernels.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ namespace kernels {
const matrix::Dense<ValueType>* b, \
matrix::Dense<ValueType>* c)

#define GKO_DECLARE_COO_FROM_MATRIX_DATA_KERNEL(ValueType, IndexType) \
void from_matrix_data( \
std::shared_ptr<const DefaultExecutor> exec, \
const Array<matrix_data_entry<ValueType, IndexType>>& data, \
matrix::Coo<ValueType, IndexType>* output)

#define GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType) \
void convert_to_dense(std::shared_ptr<const DefaultExecutor> exec, \
const matrix::Coo<ValueType, IndexType>* source, \
Expand All @@ -100,6 +106,8 @@ namespace kernels {
template <typename ValueType, typename IndexType> \
GKO_DECLARE_COO_ADVANCED_SPMV2_KERNEL(ValueType, IndexType); \
template <typename ValueType, typename IndexType> \
GKO_DECLARE_COO_FROM_MATRIX_DATA_KERNEL(ValueType, IndexType); \
template <typename ValueType, typename IndexType> \
GKO_DECLARE_COO_CONVERT_TO_CSR_KERNEL(ValueType, IndexType); \
template <typename ValueType, typename IndexType> \
GKO_DECLARE_COO_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType); \
Expand Down
52 changes: 19 additions & 33 deletions core/matrix/csr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ GKO_REGISTER_OPERATION(advanced_spmv, csr::advanced_spmv);
GKO_REGISTER_OPERATION(spgemm, csr::spgemm);
GKO_REGISTER_OPERATION(advanced_spgemm, csr::advanced_spgemm);
GKO_REGISTER_OPERATION(spgeam, csr::spgeam);
GKO_REGISTER_OPERATION(from_matrix_data, csr::from_matrix_data);
GKO_REGISTER_OPERATION(convert_to_coo, csr::convert_to_coo);
GKO_REGISTER_OPERATION(convert_to_dense, csr::convert_to_dense);
GKO_REGISTER_OPERATION(convert_to_sellp, csr::convert_to_sellp);
Expand Down Expand Up @@ -316,45 +317,30 @@ void Csr<ValueType, IndexType>::move_to(Ell<ValueType, IndexType>* result)
template <typename ValueType, typename IndexType>
void Csr<ValueType, IndexType>::read(const mat_data& data)
{
size_type nnz = 0;
for (const auto& elem : data.nonzeros) {
nnz += (elem.value != zero<ValueType>());
}
auto tmp = Csr::create(this->get_executor()->get_master(), data.size, nnz,
this->get_strategy());
size_type ind = 0;
size_type cur_ptr = 0;
tmp->get_row_ptrs()[0] = cur_ptr;
for (size_type row = 0; row < data.size[0]; ++row) {
for (; ind < data.nonzeros.size(); ++ind) {
if (data.nonzeros[ind].row > row) {
break;
}
auto val = data.nonzeros[ind].value;
if (val != zero<ValueType>()) {
tmp->get_values()[cur_ptr] = val;
tmp->get_col_idxs()[cur_ptr] = data.nonzeros[ind].column;
++cur_ptr;
}
}
tmp->get_row_ptrs()[row + 1] = cur_ptr;
}
tmp->make_srow();
tmp->move_to(this);
this->read(device_mat_data::create_from_host(this->get_executor(),
const_cast<mat_data&>(data)));
}


template <typename ValueType, typename IndexType>
void Csr<ValueType, IndexType>::read(const device_mat_data& data)
{
const auto nnz = data.nonzeros.get_num_elems();
auto exec = this->get_executor();
this->set_size(data.size);
this->row_ptrs_.resize_and_reset(data.size[0] + 1);
this->col_idxs_.resize_and_reset(nnz);
this->values_.resize_and_reset(nnz);
exec->run(csr::make_from_matrix_data(
*make_temporary_clone(exec, &data.nonzeros), this));
this->make_srow();
}


template <typename ValueType, typename IndexType>
void Csr<ValueType, IndexType>::write(mat_data& data) const
{
std::unique_ptr<const LinOp> op{};
const Csr* tmp{};
if (this->get_executor()->get_master() != this->get_executor()) {
op = this->clone(this->get_executor()->get_master());
tmp = static_cast<const Csr*>(op.get());
} else {
tmp = this;
}
auto tmp = make_temporary_clone(this->get_executor()->get_master(), this);

data = {tmp->get_size(), {}};

Expand Down
8 changes: 8 additions & 0 deletions core/matrix/csr_kernels.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,12 @@ namespace kernels {
const matrix::Csr<ValueType, IndexType>* b, \
matrix::Csr<ValueType, IndexType>* c)

#define GKO_DECLARE_CSR_FROM_MATRIX_DATA_KERNEL(ValueType, IndexType) \
void from_matrix_data( \
std::shared_ptr<const DefaultExecutor> exec, \
const Array<matrix_data_entry<ValueType, IndexType>>& data, \
matrix::Csr<ValueType, IndexType>* output)

#define GKO_DECLARE_CSR_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType) \
void convert_to_dense(std::shared_ptr<const DefaultExecutor> exec, \
const matrix::Csr<ValueType, IndexType>* source, \
Expand Down Expand Up @@ -210,6 +216,8 @@ namespace kernels {
template <typename ValueType, typename IndexType> \
GKO_DECLARE_CSR_SPGEAM_KERNEL(ValueType, IndexType); \
template <typename ValueType, typename IndexType> \
GKO_DECLARE_CSR_FROM_MATRIX_DATA_KERNEL(ValueType, IndexType); \
template <typename ValueType, typename IndexType> \
GKO_DECLARE_CSR_CONVERT_TO_DENSE_KERNEL(ValueType, IndexType); \
template <typename ValueType, typename IndexType> \
GKO_DECLARE_CSR_CONVERT_TO_COO_KERNEL(ValueType, IndexType); \
Expand Down
Loading

0 comments on commit 167c2be

Please sign in to comment.