diff --git a/src/operator/mshadow_op.h b/src/operator/mshadow_op.h
index d96782a27ebf..12b92ca8658d 100644
--- a/src/operator/mshadow_op.h
+++ b/src/operator/mshadow_op.h
@@ -780,7 +780,7 @@ namespace isnan_typed {
 
 MXNET_UNARY_MATH_OP_NC(relu, isnan_typed::IsNan(a) || (a > DType(0)) ? a : DType(0));
 
-MXNET_UNARY_MATH_OP_NC(relu_grad, a > DType(0) ? DType(1) : DType(0));
+MXNET_UNARY_MATH_OP_NC(relu_grad, isnan_typed::IsNan(a) || a > DType(0) ? DType(1) : DType(0));
 
 /*! \brief used for computing binary operator maximum */
 struct maximum : public mxnet_op::tunable {