[Cherry-Pick][Performance]Remove CudaStreamSychornize in ClipGradByGl…

…obalNorm and fix shape op (#42170) * [Performance]Set ShapeKernel with ALL_BACKEND and ALL_LAYOUT (#42138) * [Performance]Set ShapeKernel with ALL_BACKEND and ALL_LAYOUT * [Performance]Set ShapeKernel with ALL_BACKEND and ALL_LAYOUT * [Performance]Remove CudaStreamSychornize in ClipGradByGlobalNorm (#42132)
PaddlePaddle · Apr 25, 2022 · 58d0d15 · 58d0d15
1 parent 4feca75
commit 58d0d15
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 6 deletions.
diff --git a/paddle/phi/kernels/shape_kernel.cc b/paddle/phi/kernels/shape_kernel.cc
@@ -63,5 +63,7 @@ PD_REGISTER_KERNEL(shape,
                    double,
                    phi::dtype::complex<float>,
                    phi::dtype::complex<double>,
-                   phi::dtype::float16) {}
+                   phi::dtype::float16) {
+  kernel->InputAt(0).SetBackend(phi::Backend::ALL_BACKEND);
+}
 #endif
diff --git a/python/paddle/fluid/clip.py b/python/paddle/fluid/clip.py
@@ -468,10 +468,15 @@ class ClipGradByGlobalNorm(ClipGradBase):
             sdg.step()
     """
 
-    def __init__(self, clip_norm, group_name="default_group"):
+    def __init__(self,
+                 clip_norm,
+                 group_name="default_group",
+                 auto_skip_clip=False):
         super(ClipGradByGlobalNorm, self).__init__()
         self.clip_norm = float(clip_norm)
         self.group_name = group_name
+        assert isinstance(auto_skip_clip, bool)
+        self.auto_skip_clip = auto_skip_clip
 
     def __str__(self):
         return "Gradient Clip By GlobalNorm, global_norm=%f" % (self.clip_norm)
@@ -524,14 +529,19 @@ def _dygraph_clip(self, params_grads):
         max_global_norm = layers.fill_constant(
             shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm)
 
-        # only when global_norm_var > max_global_norm, grad need clip
         need_clip = False
-        if global_norm_var > max_global_norm:
+        if not self.auto_skip_clip:  # always apply clip
+            need_clip = True
+            clip_var = layers.elementwise_div(
+                x=max_global_norm,
+                y=layers.elementwise_max(
+                    x=global_norm_var, y=max_global_norm))
+        elif global_norm_var > max_global_norm:
+            # only when global_norm_var > max_global_norm, grad need clip
             need_clip = True
-
-        if need_clip:
             clip_var = layers.elementwise_div(
                 x=max_global_norm, y=global_norm_var)
+
         for p, g in params_grads:
             if g is None:
                 continue