diff --git a/src/panfrost/bifrost/bi_lower_swizzle.c b/src/panfrost/bifrost/bi_lower_swizzle.c
index bec9b93aded..bae7739660d 100644
--- a/src/panfrost/bifrost/bi_lower_swizzle.c
+++ b/src/panfrost/bifrost/bi_lower_swizzle.c
@@ -74,6 +74,11 @@ bi_lower_swizzle_16(bi_context *ctx, bi_instr *ins, unsigned src)
         if (ins->src[src].swizzle == BI_SWIZZLE_H01)
                 return;
 
+        /* If the instruction is scalar we can ignore the other component */
+        if (ins->dest[0].swizzle == BI_SWIZZLE_H00 &&
+                        ins->src[src].swizzle == BI_SWIZZLE_H00)
+                return;
+
         /* Lower it away */
         bi_builder b = bi_init_builder(ctx, bi_before_instr(ins));
         ins->src[src] = bi_replace_index(ins->src[src],