diff --git a/src/compiler/nir/nir_divergence_analysis.c b/src/compiler/nir/nir_divergence_analysis.c index 7efbca39f46..b9a6a2d2f56 100644 --- a/src/compiler/nir/nir_divergence_analysis.c +++ b/src/compiler/nir/nir_divergence_analysis.c @@ -506,6 +506,20 @@ visit_intrinsic(nir_shader *shader, nir_intrinsic_instr *instr) case nir_intrinsic_shared_atomic_fmin: case nir_intrinsic_shared_atomic_fmax: case nir_intrinsic_shared_atomic_fcomp_swap: + case nir_intrinsic_task_payload_atomic_add: + case nir_intrinsic_task_payload_atomic_imin: + case nir_intrinsic_task_payload_atomic_umin: + case nir_intrinsic_task_payload_atomic_imax: + case nir_intrinsic_task_payload_atomic_umax: + case nir_intrinsic_task_payload_atomic_and: + case nir_intrinsic_task_payload_atomic_or: + case nir_intrinsic_task_payload_atomic_xor: + case nir_intrinsic_task_payload_atomic_exchange: + case nir_intrinsic_task_payload_atomic_comp_swap: + case nir_intrinsic_task_payload_atomic_fadd: + case nir_intrinsic_task_payload_atomic_fmin: + case nir_intrinsic_task_payload_atomic_fmax: + case nir_intrinsic_task_payload_atomic_fcomp_swap: case nir_intrinsic_global_atomic_add: case nir_intrinsic_global_atomic_imin: case nir_intrinsic_global_atomic_umin: @@ -644,6 +658,7 @@ nir_variable_mode_is_uniform(nir_variable_mode mode) { case nir_var_mem_ubo: case nir_var_mem_ssbo: case nir_var_mem_shared: + case nir_var_mem_task_payload: case nir_var_mem_global: case nir_var_image: return true; diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index b4247efd44b..5487a46548a 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -692,7 +692,8 @@ intrinsic("load_vulkan_descriptor", src_comp=[-1], dest_comp=0, # in ssbo_atomic_add, etc). # 3: For CompSwap only: the second data parameter. # -# All shared variable operations take 2 sources except CompSwap that takes 3. +# All shared (and task payload) variable operations take 2 sources +# except CompSwap that takes 3. # These sources represent: # # 0: The offset into the shared variable storage region that the atomic @@ -716,6 +717,7 @@ def memory_atomic_data1(name): intrinsic("deref_atomic_" + name, src_comp=[-1, 1], dest_comp=1, indices=[ACCESS]) intrinsic("ssbo_atomic_" + name, src_comp=[-1, 1, 1], dest_comp=1, indices=[ACCESS]) intrinsic("shared_atomic_" + name, src_comp=[1, 1], dest_comp=1, indices=[BASE]) + intrinsic("task_payload_atomic_" + name, src_comp=[1, 1], dest_comp=1, indices=[BASE]) intrinsic("global_atomic_" + name, src_comp=[1, 1], dest_comp=1, indices=[]) intrinsic("global_atomic_" + name + "_amd", src_comp=[1, 1, 1], dest_comp=1, indices=[BASE]) if not name.startswith('f'): @@ -725,6 +727,7 @@ def memory_atomic_data2(name): intrinsic("deref_atomic_" + name, src_comp=[-1, 1, 1], dest_comp=1, indices=[ACCESS]) intrinsic("ssbo_atomic_" + name, src_comp=[-1, 1, 1, 1], dest_comp=1, indices=[ACCESS]) intrinsic("shared_atomic_" + name, src_comp=[1, 1, 1], dest_comp=1, indices=[BASE]) + intrinsic("task_payload_atomic_" + name, src_comp=[1, 1, 1], dest_comp=1, indices=[BASE]) intrinsic("global_atomic_" + name, src_comp=[1, 1, 1], dest_comp=1, indices=[]) intrinsic("global_atomic_" + name + "_amd", src_comp=[1, 1, 1, 1], dest_comp=1, indices=[BASE]) if not name.startswith('f'): diff --git a/src/compiler/nir/nir_lower_io.c b/src/compiler/nir/nir_lower_io.c index 5e3b866ff20..a5e33bdd6a2 100644 --- a/src/compiler/nir/nir_lower_io.c +++ b/src/compiler/nir/nir_lower_io.c @@ -121,6 +121,31 @@ shared_atomic_for_deref(nir_intrinsic_op deref_op) } } +static nir_intrinsic_op +task_payload_atomic_for_deref(nir_intrinsic_op deref_op) +{ + switch (deref_op) { +#define OP(O) case nir_intrinsic_deref_##O: return nir_intrinsic_task_payload_##O; + OP(atomic_exchange) + OP(atomic_comp_swap) + OP(atomic_add) + OP(atomic_imin) + OP(atomic_umin) + OP(atomic_imax) + OP(atomic_umax) + OP(atomic_and) + OP(atomic_or) + OP(atomic_xor) + OP(atomic_fadd) + OP(atomic_fmin) + OP(atomic_fmax) + OP(atomic_fcomp_swap) +#undef OP + default: + unreachable("Invalid task payload atomic"); + } +} + void nir_assign_var_locations(nir_shader *shader, nir_variable_mode mode, unsigned *size, @@ -1734,6 +1759,10 @@ build_explicit_io_atomic(nir_builder *b, nir_intrinsic_instr *intrin, assert(addr_format_is_offset(addr_format, mode)); op = shared_atomic_for_deref(intrin->intrinsic); break; + case nir_var_mem_task_payload: + assert(addr_format_is_offset(addr_format, mode)); + op = task_payload_atomic_for_deref(intrin->intrinsic); + break; default: unreachable("Unsupported explicit IO variable mode"); } @@ -2560,6 +2589,20 @@ nir_get_io_offset_src(nir_intrinsic_instr *instr) case nir_intrinsic_shared_atomic_umax: case nir_intrinsic_shared_atomic_umin: case nir_intrinsic_shared_atomic_xor: + case nir_intrinsic_task_payload_atomic_add: + case nir_intrinsic_task_payload_atomic_imin: + case nir_intrinsic_task_payload_atomic_umin: + case nir_intrinsic_task_payload_atomic_imax: + case nir_intrinsic_task_payload_atomic_umax: + case nir_intrinsic_task_payload_atomic_and: + case nir_intrinsic_task_payload_atomic_or: + case nir_intrinsic_task_payload_atomic_xor: + case nir_intrinsic_task_payload_atomic_exchange: + case nir_intrinsic_task_payload_atomic_comp_swap: + case nir_intrinsic_task_payload_atomic_fadd: + case nir_intrinsic_task_payload_atomic_fmin: + case nir_intrinsic_task_payload_atomic_fmax: + case nir_intrinsic_task_payload_atomic_fcomp_swap: case nir_intrinsic_global_atomic_add: case nir_intrinsic_global_atomic_and: case nir_intrinsic_global_atomic_comp_swap: diff --git a/src/compiler/nir/nir_lower_multiview.c b/src/compiler/nir/nir_lower_multiview.c index 59e76ce5c3b..8242b9a46c5 100644 --- a/src/compiler/nir/nir_lower_multiview.c +++ b/src/compiler/nir/nir_lower_multiview.c @@ -88,6 +88,20 @@ shader_writes_to_memory(nir_shader *shader) case nir_intrinsic_shared_atomic_xor: case nir_intrinsic_shared_atomic_exchange: case nir_intrinsic_shared_atomic_comp_swap: + case nir_intrinsic_task_payload_atomic_add: + case nir_intrinsic_task_payload_atomic_imin: + case nir_intrinsic_task_payload_atomic_umin: + case nir_intrinsic_task_payload_atomic_imax: + case nir_intrinsic_task_payload_atomic_umax: + case nir_intrinsic_task_payload_atomic_and: + case nir_intrinsic_task_payload_atomic_or: + case nir_intrinsic_task_payload_atomic_xor: + case nir_intrinsic_task_payload_atomic_exchange: + case nir_intrinsic_task_payload_atomic_comp_swap: + case nir_intrinsic_task_payload_atomic_fadd: + case nir_intrinsic_task_payload_atomic_fmin: + case nir_intrinsic_task_payload_atomic_fmax: + case nir_intrinsic_task_payload_atomic_fcomp_swap: case nir_intrinsic_image_deref_store: case nir_intrinsic_image_deref_atomic_add: case nir_intrinsic_image_deref_atomic_fadd: