HDR: Apply OETF in HDR external sampler GLSL shader.

This should fix how color matrix transforms look when applied on HDR colors

PiperOrigin-RevId: 491323228

libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl

@@ -14,12 +14,13 @@
 // limitations under the License.
 
 // ES 3 fragment shader that:
-// 1. samples optical linear BT.2020 RGB from a (non-external) texture with
-//    uTexSampler, and applies a 4x4 RGB color matrix to change the pixel
-//    colors,
-// 2. applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB,
-//    and
-// 3. copies this converted texture color to the current output.
+// 1. Samples optical linear BT.2020 RGB from a (non-external) texture with
+//    uTexSampler.
+// 2. Applies a 4x4 RGB color matrix to change the pixel colors.
+// 3. Applies the HLG or PQ OETF to yield electrical (HLG or PQ) BT.2020 RGB,
+//    based on uOetfColorTransfer.
+// 4. Copies this converted texture color to the current output.
+// The output will be red if an error has occurred.
 
 precision mediump float;
 uniform sampler2D uTexSampler;
@@ -75,8 +76,8 @@ highp vec3 pqOetf(highp vec3 linearColor) {
 }
 
 // Applies the appropriate OETF to convert linear optical signals to nonlinear
-// electrical signals. Input and output are both normalzied to [0, 1].
-highp vec3 getElectricalColor(highp vec3 linearColor) {
+// electrical signals. Input and output are both normalized to [0, 1].
+highp vec3 applyOetf(highp vec3 linearColor) {
   // LINT.IfChange(color_transfer)
   const int COLOR_TRANSFER_ST2084 = 6;
   const int COLOR_TRANSFER_HLG = 7;
@@ -94,5 +95,5 @@ void main() {
   vec4 inputColor = texture(uTexSampler, vTexSamplingCoord);
   // transformedColors is an optical color.
   vec4 transformedColors = uRgbMatrix * vec4(inputColor.rgb, 1);
-  outColor = vec4(getElectricalColor(transformedColors.rgb), inputColor.a);
+  outColor = vec4(applyOetf(transformedColors.rgb), inputColor.a);
 }

libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl

@@ -20,11 +20,13 @@
 //    https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt,
 // 2. Applies a YUV to RGB conversion using the specified color transform
 //    uYuvToRgbColorTransform, yielding electrical (HLG or PQ) BT.2020 RGB,
-// 3. If uEotfColorTransfer is COLOR_TRANSFER_NO_VALUE, outputs electrical
-//    (HLG or PQ) BT.2020 RGB. Otherwise, outputs optical linear BT.2020 RGB for
-//    intermediate shaders by applying the HLG or PQ EOTF.
-// 4. Copies this converted texture color to the current output, with alpha = 1,
-//    while applying a 4x4 RGB color matrix to change the pixel colors.
+// 3. Applies an EOTF based on uEotfColorTransfer, yielding optical linear
+//    BT.2020 RGB.
+// 4. Applies a 4x4 RGB color matrix to change the pixel colors.
+// 5. If uOetfColorTransfer is COLOR_TRANSFER_LINEAR, outputs linear colors as
+//    is to intermediate shaders. Otherwise, applies the HLG or PQ OETF, based
+// on uOetfColorTransfer, to provide the corresponding output electrical color.
+// The output will be red if an error has occurred.
 
 #extension GL_OES_EGL_image_external : require
 #extension GL_EXT_YUV_target : require
@@ -33,7 +35,12 @@ uniform __samplerExternal2DY2YEXT uTexSampler;
 uniform mat3 uYuvToRgbColorTransform;
 uniform mat4 uRgbMatrix;
 // C.java#ColorTransfer value.
+// Only COLOR_TRANSFER_ST2084 and COLOR_TRANSFER_HLG are allowed.
 uniform int uEotfColorTransfer;
+// C.java#ColorTransfer value.
+// Only COLOR_TRANSFER_LINEAR, COLOR_TRANSFER_ST2084, and COLOR_TRANSFER_HLG are
+// allowed.
+uniform int uOetfColorTransfer;
 in vec2 vTexSamplingCoord;
 out vec4 outColor;
 
@@ -81,19 +88,74 @@ highp vec3 pqEotf(highp vec3 pqColor) {
 
 // Applies the appropriate EOTF to convert nonlinear electrical values to linear
 // optical values. Input and output are both normalized to [0, 1].
-highp vec3 getOpticalColor(highp vec3 electricalColor) {
+highp vec3 applyEotf(highp vec3 electricalColor) {
   // LINT.IfChange(color_transfer)
   const int COLOR_TRANSFER_ST2084 = 6;
   const int COLOR_TRANSFER_HLG = 7;
 
-  const int FORMAT_NO_VALUE = -1;
-
   if (uEotfColorTransfer == COLOR_TRANSFER_ST2084) {
     return pqEotf(electricalColor);
   } else if (uEotfColorTransfer == COLOR_TRANSFER_HLG) {
     return hlgEotf(electricalColor);
-  } else if (uEotfColorTransfer == FORMAT_NO_VALUE) {
-    return electricalColor;
+  } else {
+    // Output red as an obviously visible error.
+    return vec3(1.0, 0.0, 0.0);
+  }
+}
+
+// HLG OETF for one channel.
+highp float hlgOetfSingleChannel(highp float linearChannel) {
+  // Specification:
+  // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
+  // Reference implementation:
+  // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
+  const highp float a = 0.17883277;
+  const highp float b = 0.28466892;
+  const highp float c = 0.55991073;
+
+  return linearChannel <= 1.0 / 12.0 ? sqrt(3.0 * linearChannel) :
+      a * log(12.0 * linearChannel - b) + c;
+}
+
+// BT.2100 / BT.2020 HLG OETF.
+highp vec3 hlgOetf(highp vec3 linearColor) {
+  return vec3(
+      hlgOetfSingleChannel(linearColor.r),
+      hlgOetfSingleChannel(linearColor.g),
+      hlgOetfSingleChannel(linearColor.b)
+  );
+}
+
+// BT.2100 / BT.2020, PQ / ST2084 OETF.
+highp vec3 pqOetf(highp vec3 linearColor) {
+  // Specification:
+  // https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ
+  // Reference implementation:
+  // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=514-527;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
+  const highp float m1 = (2610.0 / 16384.0);
+  const highp float m2 = (2523.0 / 4096.0) * 128.0;
+  const highp float c1 = (3424.0 / 4096.0);
+  const highp float c2 = (2413.0 / 4096.0) * 32.0;
+  const highp float c3 = (2392.0 / 4096.0) * 32.0;
+
+  highp vec3 temp = pow(linearColor, vec3(m1));
+  temp = (c1 + c2 * temp) / (1.0 + c3 * temp);
+  return pow(temp, vec3(m2));
+}
+
+// Applies the appropriate OETF to convert linear optical signals to nonlinear
+// electrical signals. Input and output are both normalized to [0, 1].
+highp vec3 applyOetf(highp vec3 linearColor) {
+  // LINT.IfChange(color_transfer_oetf)
+  const int COLOR_TRANSFER_LINEAR = 1;
+  const int COLOR_TRANSFER_ST2084 = 6;
+  const int COLOR_TRANSFER_HLG = 7;
+  if(uOetfColorTransfer == COLOR_TRANSFER_ST2084) {
+    return pqOetf(linearColor);
+  } else if(uOetfColorTransfer == COLOR_TRANSFER_HLG) {
+    return hlgOetf(linearColor);
+  } else if (uOetfColorTransfer == COLOR_TRANSFER_LINEAR) {
+    return linearColor;
   } else {
     // Output red as an obviously visible error.
     return vec3(1.0, 0.0, 0.0);
@@ -107,7 +169,7 @@ vec3 yuvToRgb(vec3 yuv) {
 
 void main() {
   vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
-  vec3 rgb = yuvToRgb(srcYuv);
-  outColor = uRgbMatrix * vec4(getOpticalColor(rgb), 1.0);
-  // TODO(b/241902517): Transform optical to electrical colors.
+  vec4 opticalColor = vec4(applyEotf(yuvToRgb(srcYuv)), 1.0);
+  vec4 transformedColors = uRgbMatrix * opticalColor;
+  outColor = vec4(applyOetf(transformedColors.rgb), 1.0);
 }

libraries/effect/src/main/java/androidx/media3/effect/MatrixTextureProcessor.java

@@ -24,7 +24,6 @@
 import android.util.Pair;
 import androidx.media3.common.C;
 import androidx.media3.common.ColorInfo;
-import androidx.media3.common.Format;
 import androidx.media3.common.FrameProcessingException;
 import androidx.media3.common.util.GlProgram;
 import androidx.media3.common.util.GlUtil;
@@ -154,7 +153,6 @@ public static MatrixTextureProcessor create(
             context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH);
 
     // No transfer functions needed, because input and output are both optical colors.
-    // TODO(b/241902517): Add transfer functions since existing color filters may change the colors.
     return new MatrixTextureProcessor(
         glProgram,
         ImmutableList.copyOf(matrixTransformations),
@@ -216,6 +214,8 @@ public static MatrixTextureProcessor createWithExternalSamplerApplyingEotf(
       checkArgument(
           colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
       glProgram.setIntUniform("uEotfColorTransfer", colorTransfer);
+      // No OETF needed, because the intended output here is optical colors.
+      glProgram.setIntUniform("uOetfColorTransfer", C.COLOR_TRANSFER_LINEAR);
     } else {
       glProgram.setIntUniform("uApplyOetf", 0);
     }
@@ -319,8 +319,11 @@ public static MatrixTextureProcessor createWithExternalSamplerApplyingEotfThenOe
               ? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
               : BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
 
-      // No transfer functions needed, because the EOTF and OETF cancel out.
-      glProgram.setIntUniform("uEotfColorTransfer", Format.NO_VALUE);
+      @C.ColorTransfer int colorTransfer = electricalColorInfo.colorTransfer;
+      checkArgument(
+          colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084);
+      glProgram.setIntUniform("uEotfColorTransfer", colorTransfer);
+      glProgram.setIntUniform("uOetfColorTransfer", colorTransfer);
     } else {
       glProgram.setIntUniform("uApplyOetf", 1);
     }