diff --git a/src/amd/common/nir/ac_nir.c b/src/amd/common/nir/ac_nir.c
index 9ca5ce1b8c1..bb2a0675eda 100644
--- a/src/amd/common/nir/ac_nir.c
+++ b/src/amd/common/nir/ac_nir.c
@@ -470,6 +470,7 @@ ac_nir_mem_vectorize_callback(unsigned align_mul, unsigned align_offset, unsigne
                     low->intrinsic == nir_intrinsic_store_shared ||
                     low->intrinsic == nir_intrinsic_load_deref ||
                     low->intrinsic == nir_intrinsic_store_deref;
+   unsigned swizzle_element_size = config->gfx_level <= GFX8 ? 4 : 16;
 
    assert(!is_store || hole_size <= 0);
 
@@ -523,7 +524,7 @@ ac_nir_mem_vectorize_callback(unsigned align_mul, unsigned align_offset, unsigne
          return false;
 
       /* GFX6-8 only support 32-bit scratch loads/stores. */
-      if (config->gfx_level <= GFX8 && swizzled && aligned_new_size > 32)
+      if (swizzled && aligned_new_size > (swizzle_element_size * 8))
          return false;
    }
 
@@ -588,6 +589,15 @@ ac_nir_mem_vectorize_callback(unsigned align_mul, unsigned align_offset, unsigne
    else
       align = align_mul;
 
+   /* Don't cross swizzle elements. stack/scratch intrinsics use scratch_* instructions, which
+    * seem to work fine.
+    */
+   if ((low->intrinsic == nir_intrinsic_load_buffer_amd ||
+        low->intrinsic == nir_intrinsic_store_buffer_amd) && swizzled &&
+       (align_offset % swizzle_element_size + unaligned_new_size / 8u) > MIN2(align_mul, swizzle_element_size)) {
+      return false;
+   }
+
    /* Validate the alignment and number of components. */
    if (!is_shared) {
       unsigned max_components;