st/xorg: Fix comp alpha code and deal with luminance masks

There are two fixes in here one is a one liner that fixes
component alpha logic. The other deals better with luminance
formats used for masks, sources not yet implemented.

Fixes component alpha text and icons in gnome. There are a one
or two cases that this code misses. Like if src_luminance is set
but no mask image is given.

src/gallium/state_trackers/xorg/xorg_composite.c

@@ -235,8 +235,11 @@ picture_format_fixups(struct exa_pixmap_priv *pSrc, PicturePtr pSrcPicture, bool
    boolean swizzle = FALSE;
    unsigned ret = 0;
 
-   if (pSrc->picture_format == pSrcPicture->format)
+   if (pSrc->picture_format == pSrcPicture->format) {
+      if (pSrc->picture_format == PICT_a8)
+         return mask ? FS_MASK_LUMINANCE : FS_MASK_LUMINANCE;
       return 0;
+   }
 
    if (pSrc->picture_format != PICT_a8r8g8b8) {
       assert(!"can not handle formats");

src/gallium/state_trackers/xorg/xorg_exa.c

@@ -551,6 +551,9 @@ ExaPrepareComposite(int op, PicturePtr pSrcPicture,
          XORG_FALLBACK("pSrc pic_format: %s != %s",
                        render_format_name(priv->picture_format),
                        render_format_name(pSrcPicture->format));
+
+      if (priv->picture_format == PICT_a8)
+         XORG_FALLBACK("pSrc pic_format == PICT_a8");
    }
 
    if (pMask) {

src/gallium/state_trackers/xorg/xorg_exa_tgsi.c

@@ -43,6 +43,38 @@
  * OUT[0] = color
  */
 
+static void
+print_fs_traits(int fs_traits)
+{
+   const char *strings[] = {
+      "FS_COMPOSITE",       // = 1 << 0,
+      "FS_MASK",            // = 1 << 1,
+      "FS_SOLID_FILL",      // = 1 << 2,
+      "FS_LINGRAD_FILL",    // = 1 << 3,
+      "FS_RADGRAD_FILL",    // = 1 << 4,
+      "FS_CA_FULL",         // = 1 << 5, /* src.rgba * mask.rgba */
+      "FS_CA_SRCALPHA",     // = 1 << 6, /* src.aaaa * mask.rgba */
+      "FS_YUV",             // = 1 << 7,
+      "FS_SRC_REPEAT_NONE", // = 1 << 8,
+      "FS_MASK_REPEAT_NONE",// = 1 << 9,
+      "FS_SRC_SWIZZLE_RGB", // = 1 << 10,
+      "FS_MASK_SWIZZLE_RGB",// = 1 << 11,
+      "FS_SRC_SET_ALPHA",   // = 1 << 12,
+      "FS_MASK_SET_ALPHA",  // = 1 << 13,
+      "FS_SRC_LUMINANCE",   // = 1 << 14,
+      "FS_MASK_LUMINANCE",  // = 1 << 15,
+   };
+   int i, k;
+   debug_printf("%s: ", __func__);
+
+   for (i = 0, k = 1; k < (1 << 16); i++, k <<= 1) {
+      if (fs_traits & k)
+         debug_printf("%s, ", strings[i]);
+   }
+
+   debug_printf("\n");
+}
+
 struct xorg_shaders {
    struct xorg_renderer *r;
 
@@ -55,7 +87,8 @@ src_in_mask(struct ureg_program *ureg,
             struct ureg_dst dst,
             struct ureg_src src,
             struct ureg_src mask,
-            int component_alpha)
+            unsigned component_alpha,
+            unsigned mask_luminance)
 {
    if (component_alpha == FS_CA_FULL) {
       ureg_MUL(ureg, dst, src, mask);
@@ -64,8 +97,12 @@ src_in_mask(struct ureg_program *ureg,
                ureg_scalar(src, TGSI_SWIZZLE_W), mask);
    }
    else {
-      ureg_MUL(ureg, dst, src,
-               ureg_scalar(mask, TGSI_SWIZZLE_X));
+      if (mask_luminance)
+         ureg_MUL(ureg, dst, src,
+                  ureg_scalar(mask, TGSI_SWIZZLE_X));
+      else
+         ureg_MUL(ureg, dst, src,
+                  ureg_scalar(mask, TGSI_SWIZZLE_W));
    }
 }
 
@@ -435,7 +472,7 @@ create_fs(struct pipe_context *pipe,
    unsigned is_solid   = (fs_traits & FS_SOLID_FILL) != 0;
    unsigned is_lingrad = (fs_traits & FS_LINGRAD_FILL) != 0;
    unsigned is_radgrad = (fs_traits & FS_RADGRAD_FILL) != 0;
-   unsigned comp_alpha = (fs_traits & FS_COMPONENT_ALPHA) != 0;
+   unsigned comp_alpha_mask = fs_traits & FS_COMPONENT_ALPHA;
    unsigned is_yuv = (fs_traits & FS_YUV) != 0;
    unsigned src_repeat_none = (fs_traits & FS_SRC_REPEAT_NONE) != 0;
    unsigned mask_repeat_none = (fs_traits & FS_MASK_REPEAT_NONE) != 0;
@@ -443,6 +480,16 @@ create_fs(struct pipe_context *pipe,
    unsigned mask_swizzle = (fs_traits & FS_MASK_SWIZZLE_RGB) != 0;
    unsigned src_set_alpha = (fs_traits & FS_SRC_SET_ALPHA) != 0;
    unsigned mask_set_alpha = (fs_traits & FS_MASK_SET_ALPHA) != 0;
+   unsigned src_luminance = (fs_traits & FS_SRC_LUMINANCE) != 0;
+   unsigned mask_luminance = (fs_traits & FS_MASK_LUMINANCE) != 0;
+
+   if (src_luminance)
+      assert(!"src_luminance not supported");
+#if 0
+   print_fs_traits(fs_traits);
+#else
+   (void)print_fs_traits;
+#endif
 
    ureg = ureg_create(TGSI_PROCESSOR_FRAGMENT);
    if (ureg == NULL)
@@ -541,7 +588,8 @@ create_fs(struct pipe_context *pipe,
       xrender_tex(ureg, mask, mask_pos, mask_sampler,
                   mask_repeat_none, mask_swizzle, mask_set_alpha);
       /* src IN mask */
-      src_in_mask(ureg, out, ureg_src(src), ureg_src(mask), comp_alpha);
+      src_in_mask(ureg, out, ureg_src(src), ureg_src(mask),
+                  comp_alpha_mask, mask_luminance);
       ureg_release_temporary(ureg, mask);
    }
 

src/gallium/state_trackers/xorg/xorg_exa_tgsi.h

@@ -32,6 +32,8 @@ enum xorg_fs_traits {
    FS_MASK_SWIZZLE_RGB = 1 << 11,
    FS_SRC_SET_ALPHA    = 1 << 12,
    FS_MASK_SET_ALPHA   = 1 << 13,
+   FS_SRC_LUMINANCE    = 1 << 14,
+   FS_MASK_LUMINANCE   = 1 << 15,
 
    FS_FILL             = (FS_SOLID_FILL |
                           FS_LINGRAD_FILL |