mesa: strengthen the condition that triggers generating VS with edge flags

Check the cull face state to see if polygon mode has any effect.
It could happen that polygon mode is GL_LINE, but that face is always
culled, in which case polygon mode has no effect, thus edge flags
have no effect either.

Reviewed-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/35314>

src/mesa/main/enable.c

@@ -501,6 +501,7 @@ _mesa_set_enable(struct gl_context *ctx, GLenum cap, GLboolean state)
                         GL_POLYGON_BIT | GL_ENABLE_BIT);
          ctx->NewDriverState |= ST_NEW_RASTERIZER;
          ctx->Polygon.CullFlag = state;
+         _mesa_update_edgeflag_state_vao(ctx);
          break;
       case GL_DEPTH_TEST:
          if (ctx->Depth.Test == state)

src/mesa/main/polygon.c

@@ -70,6 +70,7 @@ cull_face(struct gl_context *ctx, GLenum mode, bool no_error)
                   GL_POLYGON_BIT);
    ctx->NewDriverState |= ST_NEW_RASTERIZER;
    ctx->Polygon.CullFaceMode = mode;
+   _mesa_update_edgeflag_state_vao(ctx);
 }
 
 

src/mesa/main/varray.c

@@ -2096,11 +2096,16 @@ _mesa_update_edgeflag_state_explicit(struct gl_context *ctx,
    if (ctx->API != API_OPENGL_COMPAT)
       return;
 
-   /* Edge flags take effect only if the polygon mode is not FILL, and they
-    * determine whether a line or point is drawn with that polygon mode.
+   /* Edge flags take effect only if the polygon mode is not FILL on the side
+    * of the face that isn't culled.
+    *
+    * Edge flags determine whether a line or a point is drawn by polygon mode.
     */
-   bool edgeflags_have_effect = ctx->Polygon.FrontMode != GL_FILL ||
-                                ctx->Polygon.BackMode != GL_FILL;
+   bool edgeflags_have_effect =
+      (ctx->Polygon.FrontMode != GL_FILL &&
+       (!ctx->Polygon.CullFlag || ctx->Polygon.CullFaceMode == GL_BACK)) ||
+      (ctx->Polygon.BackMode != GL_FILL &&
+       (!ctx->Polygon.CullFlag || ctx->Polygon.CullFaceMode == GL_FRONT));
    per_vertex_enable &= edgeflags_have_effect;
 
    if (per_vertex_enable != ctx->Array._PerVertexEdgeFlagsEnabled) {