From 92690aee954a0fef85b3779b694ba25676648f3a Mon Sep 17 00:00:00 2001
From: David Rosca <nowrep@gmail.com>
Date: Fri, 18 Aug 2023 12:33:04 +0200
Subject: [PATCH] frontends/va: Set csc matrix in postproc

Set correct matrix according to format, color standard and range.

Change default value for color range when not explicitly specified.
Use limited range for YUV and full range for RGB.

This also adds support for converting from full range YUV to RGB.

Reviewed-by: Leo Liu <leo.liu@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24788>
---
 src/gallium/frontends/va/postproc.c | 52 +++++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 6 deletions(-)

diff --git a/src/gallium/frontends/va/postproc.c b/src/gallium/frontends/va/postproc.c
index 9e96de0b715..dce3d3618c0 100644
--- a/src/gallium/frontends/va/postproc.c
+++ b/src/gallium/frontends/va/postproc.c
@@ -111,6 +111,46 @@ static void vlVaGetBox(struct pipe_video_buffer *buf, unsigned idx,
    box->height = height;
 }
 
+static bool vlVaGetFullRange(vlVaSurface *surface, uint8_t va_range)
+{
+   if (va_range != VA_SOURCE_RANGE_UNKNOWN)
+      return va_range == VA_SOURCE_RANGE_FULL;
+
+   /* Assume limited for YUV, full for RGB */
+   return !util_format_is_yuv(surface->buffer->buffer_format);
+}
+
+static void vlVaSetCscMatrix(vlVaDriver *drv,
+                             vlVaSurface *src,
+                             vlVaSurface *dst,
+                             VAProcPipelineParameterBuffer *param)
+{
+   enum VL_CSC_COLOR_STANDARD color_standard;
+   bool src_yuv = util_format_is_yuv(src->buffer->buffer_format);
+   bool dst_yuv = util_format_is_yuv(dst->buffer->buffer_format);
+
+   if (src_yuv == dst_yuv) {
+      color_standard = VL_CSC_COLOR_STANDARD_IDENTITY;
+   } else if (src_yuv) {
+      switch (param->surface_color_standard) {
+      case VAProcColorStandardBT601:
+         color_standard = VL_CSC_COLOR_STANDARD_BT_601;
+         break;
+      case VAProcColorStandardBT709:
+      default:
+         color_standard = src->full_range ?
+            VL_CSC_COLOR_STANDARD_BT_709_FULL :
+            VL_CSC_COLOR_STANDARD_BT_709;
+         break;
+      }
+   } else {
+      color_standard = VL_CSC_COLOR_STANDARD_BT_709_REV;
+   }
+
+   vl_csc_get_matrix(color_standard, NULL, dst->full_range, &drv->csc);
+   vl_compositor_set_csc_matrix(&drv->cstate, &drv->csc, 1.0f, 0.0f);
+}
+
 static VAStatus vlVaVidEngineBlit(vlVaDriver *drv, vlVaContext *context,
                                  const VARectangle *src_region,
                                  const VARectangle *dst_region,
@@ -422,12 +462,10 @@ vlVaHandleVAProcPipelineParameterBufferType(vlVaDriver *drv, vlVaContext *contex
    if (!src_surface->buffer || !dst_surface->buffer)
       return VA_STATUS_ERROR_INVALID_SURFACE;
 
-   /* Assume full range input when not set */
-   src_surface->full_range =
-      param->input_color_properties.color_range != VA_SOURCE_RANGE_REDUCED;
-   /* Assume limited range output when not set */
-   dst_surface->full_range =
-      param->output_color_properties.color_range == VA_SOURCE_RANGE_FULL;
+   src_surface->full_range = vlVaGetFullRange(src_surface,
+      param->input_color_properties.color_range);
+   dst_surface->full_range = vlVaGetFullRange(dst_surface,
+      param->output_color_properties.color_range);
 
    pscreen = drv->vscreen->pscreen;
 
@@ -541,6 +579,8 @@ vlVaHandleVAProcPipelineParameterBufferType(vlVaDriver *drv, vlVaContext *contex
          return VA_STATUS_SUCCESS;
    }
 
+   vlVaSetCscMatrix(drv, src_surface, dst_surface, param);
+
    /* Try other post proc implementations */
    if (context->target->buffer_format != PIPE_FORMAT_NV12 &&
        context->target->buffer_format != PIPE_FORMAT_P010 &&