diff --git a/src/asahi/compiler/agx_compile.c b/src/asahi/compiler/agx_compile.c
index 680406f4139..9e701a502ed 100644
--- a/src/asahi/compiler/agx_compile.c
+++ b/src/asahi/compiler/agx_compile.c
@@ -3909,12 +3909,14 @@ agx_compile_shader_nir(nir_shader *nir, struct agx_shader_key *key,
    };
    NIR_PASS(_, nir, nir_lower_mem_access_bit_sizes, &lower_mem_access_options);
 
-   /* Optimize scratch access */
-   NIR_PASS(_, nir, nir_lower_scratch_to_var);
-   NIR_PASS(_, nir, nir_lower_vars_to_scratch, nir_var_function_temp, 256,
-            glsl_get_natural_size_align_bytes,
-            glsl_get_natural_size_align_bytes);
-   NIR_PASS(_, nir, nir_lower_indirect_derefs, nir_var_function_temp, ~0);
+   /* Optimize scratch access for silly internal CL shaders */
+   if (nir->info.internal) {
+      NIR_PASS(_, nir, nir_lower_scratch_to_var);
+      NIR_PASS(_, nir, nir_lower_vars_to_scratch, nir_var_function_temp, 256,
+               glsl_get_natural_size_align_bytes,
+               glsl_get_natural_size_align_bytes);
+      NIR_PASS(_, nir, nir_lower_indirect_derefs, nir_var_function_temp, ~0);
+   }
 
    /* Cleanup 8-bit math before lowering */
    bool progress;