mesa: various fixes for ClearTexImage/ClearTexSubImage

Fixes some upcoming CTS tests for texture clears.

* some drivers will attempt to issue clears with zero range
  and hit asserts/crashes (spec clarification for negative
  values)

* fix error thrown with negative values to match spec

* fix cases for clearing generic compressed formats

* fix negative case of using color format while having
  depth/stencil internalformat and vice versa

Cc: mesa-stable
Signed-off-by: Tapani Pälli <tapani.palli@intel.com>
Reviewed-by: Erik Faye-Lund <erik.faye-lund@collabora.com>
Reviewed-by: Marek Olšák <marek.olsak@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/34428>

src/mesa/main/formatquery.c

@@ -885,29 +885,6 @@ _get_min_dimensions(GLenum pname)
    }
 }
 
-static bool
-_is_generic_compressed_format(const struct gl_context *ctx,
-                              GLenum intFormat)
-{
-   switch (intFormat) {
-   case GL_COMPRESSED_SRGB:
-   case GL_COMPRESSED_SRGB_ALPHA:
-   case GL_COMPRESSED_SLUMINANCE:
-   case GL_COMPRESSED_SLUMINANCE_ALPHA:
-      return _mesa_has_EXT_texture_sRGB(ctx);
-   case GL_COMPRESSED_RG:
-   case GL_COMPRESSED_RED:
-      return _mesa_is_gles(ctx) ?
-             _mesa_has_EXT_texture_rg(ctx) :
-             _mesa_has_ARB_texture_rg(ctx);
-   case GL_COMPRESSED_RGB:
-   case GL_COMPRESSED_RGBA:
-      return true;
-   default:
-      return false;
-   }
-}
-
 /*
  * Similar to teximage.c:check_multisample_target, but independent of the
  * dimensions.
@@ -1634,7 +1611,7 @@ _mesa_GetInternalformativ(GLenum target, GLenum internalformat, GLenum pname,
          goto end;
 
       if (_mesa_is_compressed_format(ctx, internalformat) ||
-          _is_generic_compressed_format(ctx, internalformat))
+          _mesa_is_generic_compressed_format(ctx, internalformat))
          goto end;
 
       st_QueryInternalFormat(ctx, target, internalformat, pname,

src/mesa/main/glformats.c

@@ -1298,6 +1298,29 @@ _mesa_has_depth_float_channel(GLenum internalFormat)
           internalFormat == GL_DEPTH_COMPONENT32F;
 }
 
+GLboolean
+_mesa_is_generic_compressed_format(const struct gl_context *ctx,
+                                   GLenum format)
+{
+   switch (format) {
+   case GL_COMPRESSED_SRGB:
+   case GL_COMPRESSED_SRGB_ALPHA:
+   case GL_COMPRESSED_SLUMINANCE:
+   case GL_COMPRESSED_SLUMINANCE_ALPHA:
+      return _mesa_has_EXT_texture_sRGB(ctx);
+   case GL_COMPRESSED_RG:
+   case GL_COMPRESSED_RED:
+      return _mesa_is_gles(ctx) ?
+             _mesa_has_EXT_texture_rg(ctx) :
+             _mesa_has_ARB_texture_rg(ctx);
+   case GL_COMPRESSED_RGB:
+   case GL_COMPRESSED_RGBA:
+      return true;
+   default:
+      return false;
+   }
+}
+
 /**
  * Test if an image format is a supported compressed format.
  * \param format the internal format token provided by the user.

src/mesa/main/glformats.h

@@ -105,6 +105,10 @@ _mesa_is_depth_or_stencil_format(GLenum format);
 extern GLboolean
 _mesa_has_depth_float_channel(GLenum internalFormat);
 
+extern GLboolean
+_mesa_is_generic_compressed_format(const struct gl_context *ctx,
+                                   GLenum format);
+
 extern GLboolean
 _mesa_is_compressed_format(const struct gl_context *ctx, GLenum format);
 

src/mesa/main/teximage.c

@@ -1876,6 +1876,12 @@ texture_formats_agree(GLenum internalFormat,
    if (_mesa_is_ycbcr_format(internalFormat) != _mesa_is_ycbcr_format(format))
       return false;
 
+   if ((_mesa_is_depth_format(internalFormat) ||
+        _mesa_is_stencil_format(internalFormat)) &&
+       _mesa_is_color_format(format)) {
+      return false;
+   }
+
    return true;
 }
 
@@ -5284,12 +5290,26 @@ check_clear_tex_image(struct gl_context *ctx,
       return false;
    }
 
-   if (_mesa_is_compressed_format(ctx, internalFormat)) {
+   if (_mesa_is_compressed_format(ctx, internalFormat) ||
+       _mesa_is_generic_compressed_format(ctx, internalFormat)) {
       _mesa_error(ctx, GL_INVALID_OPERATION,
                   "%s(compressed texture)", function);
       return false;
    }
 
+   /* This is a special case where we might throw GL_INVALID_ENUM
+    * below but should do GL_INVALID_OPERATION with glClearTexImage.
+    */
+   if (_mesa_is_color_format(internalFormat) &&
+       _mesa_is_depthstencil_format(format)) {
+      _mesa_error(ctx, GL_INVALID_OPERATION,
+                  "%s(incompatible internalFormat = %s, format = %s)",
+                  function,
+                  _mesa_enum_to_string(internalFormat),
+                  _mesa_enum_to_string(format));
+      return false;
+   }
+
    err = _mesa_error_check_format_and_type(ctx, format, type);
    if (err != GL_NO_ERROR) {
       _mesa_error(ctx, err,
@@ -5437,12 +5457,19 @@ _mesa_ClearTexSubImage(GLuint texture, GLint level,
       maxDepth = numImages;
    }
 
+   /* Nothing to clear, skip. */
+   if (width == 0 || height == 0 || depth == 0)
+      goto out;
+
+   if (width < 0 || height < 0 || depth < 0) {
+      _mesa_error(ctx, GL_INVALID_VALUE,
+                  "glClearSubTexImage(invalid dimensions)");
+      goto out;
+   }
+
    if (xoffset < -(GLint) texImages[0]->Border ||
        yoffset < -(GLint) texImages[0]->Border ||
        zoffset < minDepth ||
-       width < 0 ||
-       height < 0 ||
-       depth < 0 ||
        xoffset + width > texImages[0]->Width ||
        yoffset + height > texImages[0]->Height ||
        zoffset + depth > maxDepth) {