Skip to content

Commit

Permalink
[QualcommQnn] add ops (#9538)
Browse files Browse the repository at this point in the history 
support fusion_elementwise_mul_activation, fusion_elementwise_sub_activation, fusion_elementwise_div_activation, fusion_elementwise_min_activation, fusion_elementwise_max_activation, fusion_elementwise_pow_activation, instance_norm, prelu, arg_max, arg_min, flatten, flatten2, norm
  • Loading branch information
@zhupengyang
zhupengyang authored Oct 17, 2022
1 parent ca7e580 commit f8656fd
Show file tree
Hide file tree
Showing 13 changed files with 211 additions and 72 deletions.
54 changes: 27 additions & 27 deletions lite/backends/nnadapter/nnadapter/include/nnadapter/operation/lp_normalization.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,33 +17,33 @@
namespace nnadapter {
namespace operation {

#define LP_NORMALIZATION_OPERATION_EXTRACT_INPUTS_OUTPUTS \
auto& input_operands = operation->input_operands; \
auto& output_operands = operation->output_operands; \
auto input_count = input_operands.size(); \
auto output_count = output_operands.size(); \
NNADAPTER_CHECK_EQ(input_count, 4); \
NNADAPTER_CHECK_EQ(output_count, 1); \
/* Input */ \
auto input_operand = input_operands[0]; \
NNADAPTER_VLOG(5) << "input: " << OperandToString(input_operand); \
/* Output */ \
auto output_operand = output_operands[0]; \
NNADAPTER_VLOG(5) << "output: " << OperandToString(output_operand); \
/* Axis */ \
auto axis_operand = input_operands[1]; \
auto axis_count = axis_operand->length / sizeof(int32_t); \
auto axis_data = reinterpret_cast<int32_t*>(axis_operand->buffer); \
for (uint32_t i = 0; i < axis_count; i++) { \
NNADAPTER_VLOG(5) << "axis[" << i << "]=" << axis_data[i]; \
} \
/* P */ \
auto p = *reinterpret_cast<int32_t*>(input_operands[2]->buffer); \
NNADAPTER_VLOG(5) << "p: " << p; \
NNADAPTER_CHECK(p == 1 || p == 2) \
<< "lp normalization only support p = 1 or p = 2."; \
/* Epsilon */ \
auto epsilon = *reinterpret_cast<float*>(input_operands[3]->buffer); \
#define LP_NORMALIZATION_OPERATION_EXTRACT_INPUTS_OUTPUTS \
auto& input_operands = operation->input_operands; \
auto& output_operands = operation->output_operands; \
auto input_count = input_operands.size(); \
auto output_count = output_operands.size(); \
NNADAPTER_CHECK_EQ(input_count, 4); \
NNADAPTER_CHECK_EQ(output_count, 1); \
/* Input */ \
auto input_operand = input_operands[0]; \
NNADAPTER_VLOG(5) << "input: " << OperandToString(input_operand); \
/* Output */ \
auto output_operand = output_operands[0]; \
NNADAPTER_VLOG(5) << "output: " << OperandToString(output_operand); \
/* Axis */ \
auto axis_operand = input_operands[1]; \
auto axis_count = axis_operand->length / sizeof(int32_t); \
auto axis_data = reinterpret_cast<int32_t*>(axis_operand->buffer); \
for (uint32_t i = 0; i < axis_count; i++) { \
NNADAPTER_VLOG(5) << "axis[" << i << "]=" << axis_data[i]; \
} \
/* P */ \
auto p = *reinterpret_cast<int32_t*>(input_operands[2]->buffer); \
NNADAPTER_VLOG(5) << "p: " << p; \
NNADAPTER_CHECK(p == 1 || p == 2) \
<< "lp normalization only support p = 1 or p = 2. Received p is " << p; \
/* Epsilon */ \
auto epsilon = *reinterpret_cast<float*>(input_operands[3]->buffer); \
NNADAPTER_VLOG(5) << "epsilon: " << epsilon;

} // namespace operation
Expand Down
11 changes: 7 additions & 4 deletions ...ackends/nnadapter/nnadapter/include/nnadapter/optimizer/convert_datalayout_nchw_to_nhwc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,29 +36,33 @@ class NCHW2NHWCDataLayoutConverter {

private:
// Operation converters
void ConvertActivation(core::Operation* operation);
void ConvertAdaptivePool2D(core::Operation* operation);
void ConvertArgMinMax(core::Operation* operation);
void ConvertBatchNormalization(core::Operation* operation);
void ConvertCast(core::Operation* operation);
void ConvertChannelShuffle(core::Operation* operation);
void ConvertClip(core::Operation* operation);
void ConvertComparisons(core::Operation* operation);
void ConvertConcat(core::Operation* operation);
void ConvertCumSum(core::Operation* operation);
void ConvertDequantize(core::Operation* operation);
void ConvertElementwise(core::Operation* operation);
void ConvertPool2D(core::Operation* operation);
void ConvertConcat(core::Operation* operation);
void ConvertFill(core::Operation* operation);
void ConvertFillLike(core::Operation* operation);
void ConvertFlatten(core::Operation* operation);
void ConvertFullyConnected(core::Operation* operation);
void ConvertGather(core::Operation* operation);
void ConvertGelu(core::Operation* operation);
void ConvertInstanceNormalization(core::Operation* operation);
void ConvertLayerNormalization(core::Operation* operation);
void ConvertLeakyRelu(core::Operation* operation);
void ConvertLpNormalization(core::Operation* operation);
void ConvertActivation(core::Operation* operation);
void ConvertMatMul(core::Operation* operation);
void ConvertPad(core::Operation* operation);
void ConvertPow(core::Operation* operation);
void ConvertPool2D(core::Operation* operation);
void ConvertPrelu(core::Operation* operation);
void ConvertQuantize(core::Operation* operation);
void ConvertReduce(core::Operation* operation);
void ConvertReshape(core::Operation* operation);
Expand All @@ -73,7 +77,6 @@ class NCHW2NHWCDataLayoutConverter {
void ConvertStack(core::Operation* operation);
void ConvertTile(core::Operation* operation);
void ConvertTranspose(core::Operation* operation);
void ConvertMatMul(core::Operation* operation);
void ConvertUnsqueeze(core::Operation* operation);
void ConvertUnstack(core::Operation* operation);

Expand Down
108 changes: 108 additions & 0 deletions lite/backends/nnadapter/nnadapter/src/optimizer/convert_datalayout_nchw_to_nhwc.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,46 @@ void NCHW2NHWCDataLayoutConverter::ConvertAdaptivePool2D(
SetOperationLayout(operation);
}

void NCHW2NHWCDataLayoutConverter::ConvertArgMinMax(
core::Operation* operation) {
auto& input_operands = operation->input_operands;
auto& output_operands = operation->output_operands;
auto input_count = input_operands.size();
auto output_count = output_operands.size();
NNADAPTER_CHECK_EQ(input_count, 4);
NNADAPTER_CHECK_EQ(output_count, 1);
auto input_operand = input_operands[0];
auto input_dimensions_count = input_operand->type.dimensions.count;
auto axis = reinterpret_cast<int32_t*>(input_operands[1]->buffer);
if (*axis < 0) {
*axis += input_dimensions_count;
}
auto keepdim = *reinterpret_cast<bool*>(input_operands[2]->buffer);
auto output_operand = output_operands[0];
auto input_permutation = GetPermutation(input_operand);
if (keepdim) {
// Recalculate the axis according to the dimorder vector of the input
// operand
*axis = TransposeAxis(*axis, input_permutation);
TransposeOperand(output_operand, input_permutation);
SetPermutation(output_operand, input_permutation);
} else {
// Force to restore the dimorder vector of the input operand
auto transpose_input_permutation = InversePermutation(input_permutation);
if (!IsIdentityPermutation(transpose_input_permutation)) {
auto transpose_input_operand = AppendTransposeOperation(
model_, input_operand, transpose_input_permutation);
UpdateOperationInputOperands(
{operation}, input_operand, transpose_input_operand);
SetPermutation(transpose_input_operand,
IdentityPermutation(input_dimensions_count));
}
int output_dimensions_count = output_operand->type.dimensions.count;
SetPermutation(output_operand,
IdentityPermutation(output_dimensions_count));
}
}

void NCHW2NHWCDataLayoutConverter::ConvertBatchNormalization(
core::Operation* operation) {
auto& input_operands = operation->input_operands;
Expand Down Expand Up @@ -704,6 +744,38 @@ void NCHW2NHWCDataLayoutConverter::ConvertPow(core::Operation* operation) {
SetPermutation(output_operand, input_permutation);
}

void NCHW2NHWCDataLayoutConverter::ConvertPrelu(core::Operation* operation) {
auto& input_operands = operation->input_operands;
auto& output_operands = operation->output_operands;
auto input_count = input_operands.size();
auto output_count = output_operands.size();
NNADAPTER_CHECK_EQ(input_count, 2);
NNADAPTER_CHECK_EQ(output_count, 1);
auto input_operand = input_operands[0];
auto output_operand = output_operands[0];
auto input_permutation = GetPermutation(input_operand);
auto slope_size = input_operands[1]->type.dimensions.data[0];
if (slope_size == 1) {
// The input and output operands share the same dimorder vector
TransposeOperand(output_operand, input_permutation);
SetPermutation(output_operand, input_permutation);
} else {
// Force to apply the dimorder vector of NCHW2NHWC conversion
auto transpose_input_permutation =
MultiplyPermutation(InversePermutation(input_permutation), kNCHW2NHWC);
if (!IsIdentityPermutation(transpose_input_permutation)) {
auto transpose_input_operand = AppendTransposeOperation(
model_, input_operand, transpose_input_permutation);
UpdateOperationInputOperands(
{operation}, input_operand, transpose_input_operand);
SetPermutation(transpose_input_operand, kNCHW2NHWC);
}
TransposeOperand(output_operand, kNCHW2NHWC);
SetPermutation(output_operand, kNCHW2NHWC);
SetOperationLayout(operation);
}
}

void NCHW2NHWCDataLayoutConverter::ConvertQuantize(core::Operation* operation) {
auto& input_operands = operation->input_operands;
auto& output_operands = operation->output_operands;
Expand Down Expand Up @@ -921,6 +993,32 @@ void NCHW2NHWCDataLayoutConverter::ConvertFlatten(core::Operation* operation) {
SetPermutation(output_operand, IdentityPermutation(output_dimensions_count));
}

void NCHW2NHWCDataLayoutConverter::ConvertInstanceNormalization(
core::Operation* operation) {
auto& input_operands = operation->input_operands;
auto& output_operands = operation->output_operands;
auto input_count = input_operands.size();
auto output_count = output_operands.size();
NNADAPTER_CHECK_EQ(input_count, 4);
NNADAPTER_CHECK_EQ(output_count, 1);
auto input_operand = input_operands[0];
auto output_operand = output_operands[0];
// Force to apply the dimorder vector of NCHW2NHWC conversion
auto input_permutation = GetPermutation(input_operand);
auto transpose_input_permutation =
MultiplyPermutation(InversePermutation(input_permutation), kNCHW2NHWC);
if (!IsIdentityPermutation(transpose_input_permutation)) {
auto transpose_input_operand = AppendTransposeOperation(
model_, input_operand, transpose_input_permutation);
UpdateOperationInputOperands(
{operation}, input_operand, transpose_input_operand);
SetPermutation(transpose_input_operand, kNCHW2NHWC);
}
TransposeOperand(output_operand, kNCHW2NHWC);
SetPermutation(output_operand, kNCHW2NHWC);
SetOperationLayout(operation);
}

void NCHW2NHWCDataLayoutConverter::ConvertLayerNormalization(
core::Operation* operation) {
auto& input_operands = operation->input_operands;
Expand Down Expand Up @@ -1297,6 +1395,10 @@ void NCHW2NHWCDataLayoutConverter::Apply(core::Model* model) {
case NNADAPTER_ADAPTIVE_AVERAGE_POOL_2D:
ConvertAdaptivePool2D(operation);
break;
case NNADAPTER_ARG_MAX:
case NNADAPTER_ARG_MIN:
ConvertArgMinMax(operation);
break;
case NNADAPTER_AVERAGE_POOL_2D:
case NNADAPTER_MAX_POOL_2D:
ConvertPool2D(operation);
Expand Down Expand Up @@ -1354,6 +1456,9 @@ void NCHW2NHWCDataLayoutConverter::Apply(core::Model* model) {
case NNADAPTER_GELU:
ConvertGelu(operation);
break;
case NNADAPTER_INSTANCE_NORMALIZATION:
ConvertInstanceNormalization(operation);
break;
case NNADAPTER_LAYER_NORMALIZATION:
ConvertLayerNormalization(operation);
break;
Expand All @@ -1372,6 +1477,9 @@ void NCHW2NHWCDataLayoutConverter::Apply(core::Model* model) {
case NNADAPTER_POW:
ConvertPow(operation);
break;
case NNADAPTER_PRELU:
ConvertPrelu(operation);
break;
case NNADAPTER_QUANTIZE:
ConvertQuantize(operation);
break;
Expand Down
5 changes: 5 additions & 0 deletions lite/backends/nnadapter/nnadapter/src/optimizer/convert_quantization_symm_to_asymm.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ NNADAPTER_EXPORT void ConvertQuantizationSymmToAsymm(core::Model* model) {
ConvertOperandSymmToAsymm(output_operands[0], 128);
} break;
case NNADAPTER_ABS:
case NNADAPTER_ARG_MAX:
case NNADAPTER_ARG_MIN:
case NNADAPTER_AVERAGE_POOL_2D:
case NNADAPTER_BATCH_NORMALIZATION:
case NNADAPTER_CAST:
Expand All @@ -114,11 +116,14 @@ NNADAPTER_EXPORT void ConvertQuantizationSymmToAsymm(core::Model* model) {
case NNADAPTER_GELU:
case NNADAPTER_HARD_SIGMOID:
case NNADAPTER_HARD_SWISH:
case NNADAPTER_INSTANCE_NORMALIZATION:
case NNADAPTER_LAYER_NORMALIZATION:
case NNADAPTER_LEAKY_RELU:
case NNADAPTER_LOG:
case NNADAPTER_LP_NORMALIZATION:
case NNADAPTER_MAX_POOL_2D:
case NNADAPTER_PAD:
case NNADAPTER_PRELU:
case NNADAPTER_REDUCE_MAX:
case NNADAPTER_REDUCE_MEAN:
case NNADAPTER_REDUCE_SUM:
Expand Down
Loading

0 comments on commit f8656fd

Please sign in to comment.