From 7280b0911d8ce22092251a8db6d37b11bdc5a59e Mon Sep 17 00:00:00 2001 From: Jason Ekstrand Date: Wed, 21 Oct 2020 14:46:50 -0500 Subject: [PATCH] intel/compiler: Add support for bindless shaders The Intel bindless thread dispatch model is very simple. When a compute shader is to be used for bindless dispatch, it can request a set of stack IDs. These are allocated per-dual-subslice by the hardware and recycled automatically when the stack ID is returned. Passed to the bindless dispatch are a global argument address, a stack ID, and an address of the BINDLESS_SHADER_RECORD to invoke. When the bindless shader is dispatched, it is passed its stack ID as well as the global and local argument pointers. The local argument pointer is the address of the BINDLESS_SHADER_RECORD plus some offset which is specified as part of the BINDLESS_SHADER_RECORD. Reviewed-by: Caio Marcelo de Oliveira Filho Part-of: --- src/compiler/nir/nir_intrinsics.py | 9 + src/intel/compiler/brw_compiler.c | 43 ++-- src/intel/compiler/brw_compiler.h | 35 +++ src/intel/compiler/brw_eu.h | 39 ++++ src/intel/compiler/brw_eu_defines.h | 8 + src/intel/compiler/brw_fs.cpp | 212 +++++++++++++++++- src/intel/compiler/brw_fs.h | 3 + src/intel/compiler/brw_fs_generator.cpp | 29 +++ src/intel/compiler/brw_fs_nir.cpp | 71 ++++++ src/intel/compiler/brw_fs_visitor.cpp | 3 - src/intel/compiler/brw_ir_performance.cpp | 6 + .../compiler/brw_schedule_instructions.cpp | 9 + src/intel/compiler/brw_shader.cpp | 8 + 13 files changed, 457 insertions(+), 18 deletions(-) diff --git a/src/compiler/nir/nir_intrinsics.py b/src/compiler/nir/nir_intrinsics.py index aee01cd6a8f..6d1e0c42b06 100644 --- a/src/compiler/nir/nir_intrinsics.py +++ b/src/compiler/nir/nir_intrinsics.py @@ -1081,3 +1081,12 @@ store("ssbo_block_intel", [-1, 1], [WRMASK, ACCESS, ALIGN_MUL, ALIGN_OFFSET]) # src[] = { value, offset }. store("shared_block_intel", [1], [BASE, WRMASK, ALIGN_MUL, ALIGN_OFFSET]) +# Intrinsics for Intel bindless thread dispatch +system_value("btd_dss_id_intel", 1) +system_value("btd_stack_id_intel", 1) +system_value("btd_global_arg_addr_intel", 1, bit_sizes=[64]) +system_value("btd_local_arg_addr_intel", 1, bit_sizes=[64]) +# src[] = { global_arg_addr, btd_record } +intrinsic("btd_spawn_intel", src_comp=[1, 1]) +# src[] = { } +intrinsic("btd_retire_intel") diff --git a/src/intel/compiler/brw_compiler.c b/src/intel/compiler/brw_compiler.c index 933b00440dd..943b621daa9 100644 --- a/src/intel/compiler/brw_compiler.c +++ b/src/intel/compiler/brw_compiler.c @@ -118,6 +118,9 @@ brw_compiler_create(void *mem_ctx, const struct gen_device_info *devinfo) i == MESA_SHADER_FRAGMENT || i == MESA_SHADER_COMPUTE; } + for (int i = MESA_SHADER_TASK; i < MESA_VULKAN_SHADER_STAGES; i++) + compiler->scalar_stage[i] = true; + nir_lower_int64_options int64_options = nir_lower_imul64 | nir_lower_isign64 | @@ -230,13 +233,19 @@ unsigned brw_prog_data_size(gl_shader_stage stage) { static const size_t stage_sizes[] = { - [MESA_SHADER_VERTEX] = sizeof(struct brw_vs_prog_data), - [MESA_SHADER_TESS_CTRL] = sizeof(struct brw_tcs_prog_data), - [MESA_SHADER_TESS_EVAL] = sizeof(struct brw_tes_prog_data), - [MESA_SHADER_GEOMETRY] = sizeof(struct brw_gs_prog_data), - [MESA_SHADER_FRAGMENT] = sizeof(struct brw_wm_prog_data), - [MESA_SHADER_COMPUTE] = sizeof(struct brw_cs_prog_data), - [MESA_SHADER_KERNEL] = sizeof(struct brw_cs_prog_data), + [MESA_SHADER_VERTEX] = sizeof(struct brw_vs_prog_data), + [MESA_SHADER_TESS_CTRL] = sizeof(struct brw_tcs_prog_data), + [MESA_SHADER_TESS_EVAL] = sizeof(struct brw_tes_prog_data), + [MESA_SHADER_GEOMETRY] = sizeof(struct brw_gs_prog_data), + [MESA_SHADER_FRAGMENT] = sizeof(struct brw_wm_prog_data), + [MESA_SHADER_COMPUTE] = sizeof(struct brw_cs_prog_data), + [MESA_SHADER_RAYGEN] = sizeof(struct brw_bs_prog_data), + [MESA_SHADER_ANY_HIT] = sizeof(struct brw_bs_prog_data), + [MESA_SHADER_CLOSEST_HIT] = sizeof(struct brw_bs_prog_data), + [MESA_SHADER_MISS] = sizeof(struct brw_bs_prog_data), + [MESA_SHADER_INTERSECTION] = sizeof(struct brw_bs_prog_data), + [MESA_SHADER_CALLABLE] = sizeof(struct brw_bs_prog_data), + [MESA_SHADER_KERNEL] = sizeof(struct brw_cs_prog_data), }; assert((int)stage >= 0 && stage < ARRAY_SIZE(stage_sizes)); return stage_sizes[stage]; @@ -246,13 +255,19 @@ unsigned brw_prog_key_size(gl_shader_stage stage) { static const size_t stage_sizes[] = { - [MESA_SHADER_VERTEX] = sizeof(struct brw_vs_prog_key), - [MESA_SHADER_TESS_CTRL] = sizeof(struct brw_tcs_prog_key), - [MESA_SHADER_TESS_EVAL] = sizeof(struct brw_tes_prog_key), - [MESA_SHADER_GEOMETRY] = sizeof(struct brw_gs_prog_key), - [MESA_SHADER_FRAGMENT] = sizeof(struct brw_wm_prog_key), - [MESA_SHADER_COMPUTE] = sizeof(struct brw_cs_prog_key), - [MESA_SHADER_KERNEL] = sizeof(struct brw_cs_prog_key), + [MESA_SHADER_VERTEX] = sizeof(struct brw_vs_prog_key), + [MESA_SHADER_TESS_CTRL] = sizeof(struct brw_tcs_prog_key), + [MESA_SHADER_TESS_EVAL] = sizeof(struct brw_tes_prog_key), + [MESA_SHADER_GEOMETRY] = sizeof(struct brw_gs_prog_key), + [MESA_SHADER_FRAGMENT] = sizeof(struct brw_wm_prog_key), + [MESA_SHADER_COMPUTE] = sizeof(struct brw_cs_prog_key), + [MESA_SHADER_RAYGEN] = sizeof(struct brw_bs_prog_key), + [MESA_SHADER_ANY_HIT] = sizeof(struct brw_bs_prog_key), + [MESA_SHADER_CLOSEST_HIT] = sizeof(struct brw_bs_prog_key), + [MESA_SHADER_MISS] = sizeof(struct brw_bs_prog_key), + [MESA_SHADER_INTERSECTION] = sizeof(struct brw_bs_prog_key), + [MESA_SHADER_CALLABLE] = sizeof(struct brw_bs_prog_key), + [MESA_SHADER_KERNEL] = sizeof(struct brw_cs_prog_key), }; assert((int)stage >= 0 && stage < ARRAY_SIZE(stage_sizes)); return stage_sizes[stage]; diff --git a/src/intel/compiler/brw_compiler.h b/src/intel/compiler/brw_compiler.h index e02409436b3..bfec4e8f0db 100644 --- a/src/intel/compiler/brw_compiler.h +++ b/src/intel/compiler/brw_compiler.h @@ -152,6 +152,13 @@ struct brw_compiler { */ #define BRW_SUBGROUP_SIZE 32 +static inline bool +brw_shader_stage_is_bindless(gl_shader_stage stage) +{ + return stage >= MESA_SHADER_RAYGEN && + stage <= MESA_SHADER_CALLABLE; +} + /** * Program key structures. * @@ -481,6 +488,10 @@ struct brw_cs_prog_key { struct brw_base_prog_key base; }; +struct brw_bs_prog_key { + struct brw_base_prog_key base; +}; + /* brw_any_prog_key is any of the keys that map to an API stage */ union brw_any_prog_key { struct brw_base_prog_key base; @@ -490,6 +501,7 @@ union brw_any_prog_key { struct brw_gs_prog_key gs; struct brw_wm_prog_key wm; struct brw_cs_prog_key cs; + struct brw_bs_prog_key bs; }; /* @@ -986,6 +998,7 @@ struct brw_cs_prog_data { bool uses_barrier; bool uses_num_work_groups; + bool uses_btd_stack_ids; struct { struct brw_push_const_block cross_thread; @@ -1013,6 +1026,12 @@ brw_cs_prog_data_prog_offset(const struct brw_cs_prog_data *prog_data, return prog_data->prog_offset[index]; } +struct brw_bs_prog_data { + struct brw_stage_prog_data base; + uint8_t simd_size; + uint32_t stack_size; +}; + /** * Enum representing the i965-specific vertex results that don't correspond * exactly to any element of gl_varying_slot. The values of this enum are @@ -1340,6 +1359,7 @@ union brw_any_prog_data { struct brw_gs_prog_data gs; struct brw_wm_prog_data wm; struct brw_cs_prog_data cs; + struct brw_bs_prog_data bs; }; #define DEFINE_PROG_DATA_DOWNCAST(STAGE, CHECK) \ @@ -1364,6 +1384,7 @@ DEFINE_PROG_DATA_DOWNCAST(tes, prog_data->stage == MESA_SHADER_TESS_EVAL) DEFINE_PROG_DATA_DOWNCAST(gs, prog_data->stage == MESA_SHADER_GEOMETRY) DEFINE_PROG_DATA_DOWNCAST(wm, prog_data->stage == MESA_SHADER_FRAGMENT) DEFINE_PROG_DATA_DOWNCAST(cs, prog_data->stage == MESA_SHADER_COMPUTE) +DEFINE_PROG_DATA_DOWNCAST(bs, brw_shader_stage_is_bindless(prog_data->stage)) DEFINE_PROG_DATA_DOWNCAST(vue, prog_data->stage == MESA_SHADER_VERTEX || prog_data->stage == MESA_SHADER_TESS_CTRL || @@ -1541,6 +1562,20 @@ brw_compile_cs(const struct brw_compiler *compiler, void *log_data, struct brw_compile_stats *stats, char **error_str); +/** + * Compile a Ray Tracing shader. + * + * Returns the final assembly and the program's size. + */ +const unsigned * +brw_compile_bs(const struct brw_compiler *compiler, void *log_data, + void *mem_ctx, + const struct brw_bs_prog_key *key, + struct brw_bs_prog_data *prog_data, + struct nir_shader *shader, + struct brw_compile_stats *stats, + char **error_str); + void brw_debug_key_recompile(const struct brw_compiler *c, void *log, gl_shader_stage stage, const struct brw_base_prog_key *old_key, diff --git a/src/intel/compiler/brw_eu.h b/src/intel/compiler/brw_eu.h index 8547be16659..a70e35fc1ea 100644 --- a/src/intel/compiler/brw_eu.h +++ b/src/intel/compiler/brw_eu.h @@ -1011,6 +1011,45 @@ brw_dp_typed_surface_rw_desc(const struct gen_device_info *devinfo, return brw_dp_surface_desc(devinfo, msg_type, msg_control); } +static inline uint32_t +brw_mdc_sm2(unsigned exec_size) +{ + assert(exec_size == 8 || exec_size == 16); + return exec_size > 8; +} + +static inline uint32_t +brw_mdc_sm2_exec_size(uint32_t sm2) +{ + assert(sm2 <= 1); + return 8 << sm2; +} + +static inline uint32_t +brw_btd_spawn_desc(const struct gen_device_info *devinfo, + unsigned exec_size, unsigned msg_type) +{ + assert(devinfo->has_ray_tracing); + + return SET_BITS(0, 19, 19) | /* No header */ + SET_BITS(msg_type, 17, 14) | + SET_BITS(brw_mdc_sm2(exec_size), 8, 8); +} + +static inline uint32_t +brw_btd_spawn_msg_type(const struct gen_device_info *devinfo, + uint32_t desc) +{ + return GET_BITS(desc, 17, 14); +} + +static inline uint32_t +brw_btd_spawn_exec_size(const struct gen_device_info *devinfo, + uint32_t desc) +{ + return brw_mdc_sm2_exec_size(GET_BITS(desc, 8, 8)); +} + /** * Construct a message descriptor immediate with the specified pixel * interpolator function controls. diff --git a/src/intel/compiler/brw_eu_defines.h b/src/intel/compiler/brw_eu_defines.h index c1114a78968..8bba0dd5380 100644 --- a/src/intel/compiler/brw_eu_defines.h +++ b/src/intel/compiler/brw_eu_defines.h @@ -793,6 +793,10 @@ enum opcode { TES_OPCODE_GET_PRIMITIVE_ID, TES_OPCODE_CREATE_INPUT_READ_HEADER, TES_OPCODE_ADD_INDIRECT_URB_OFFSET, + + SHADER_OPCODE_GET_DSS_ID, + SHADER_OPCODE_BTD_SPAWN_LOGICAL, + SHADER_OPCODE_BTD_RETIRE_LOGICAL, }; enum brw_urb_write_flags { @@ -1230,6 +1234,8 @@ enum brw_message_target { GEN7_SFID_PIXEL_INTERPOLATOR = 11, HSW_SFID_DATAPORT_DATA_CACHE_1 = 12, HSW_SFID_CRE = 13, + + GEN_RT_SFID_BINDLESS_THREAD_DISPATCH = 7, }; #define GEN7_MESSAGE_TARGET_DP_DATA_CACHE 10 @@ -1622,4 +1628,6 @@ enum PACKED brw_rnd_mode { #define GEN7_BYTE_SCATTERED_DATA_ELEMENT_WORD 1 #define GEN7_BYTE_SCATTERED_DATA_ELEMENT_DWORD 2 +#define GEN_RT_BTD_MESSAGE_SPAWN 1 + #endif /* BRW_EU_DEFINES_H */ diff --git a/src/intel/compiler/brw_fs.cpp b/src/intel/compiler/brw_fs.cpp index 1caaa93dd17..34d767d5c90 100644 --- a/src/intel/compiler/brw_fs.cpp +++ b/src/intel/compiler/brw_fs.cpp @@ -6033,6 +6033,76 @@ lower_math_logical_send(const fs_builder &bld, fs_inst *inst) } } +static void +lower_btd_logical_send(const fs_builder &bld, fs_inst *inst) +{ + const gen_device_info *devinfo = bld.shader->devinfo; + fs_reg global_addr = inst->src[0]; + const fs_reg &btd_record = inst->src[1]; + + const unsigned mlen = 2; + const fs_builder ubld = bld.exec_all().group(8, 0); + fs_reg header = ubld.vgrf(BRW_REGISTER_TYPE_UD, 2); + + ubld.MOV(header, brw_imm_ud(0)); + switch (inst->opcode) { + case SHADER_OPCODE_BTD_SPAWN_LOGICAL: + assert(type_sz(global_addr.type) == 8 && global_addr.stride == 0); + global_addr.type = BRW_REGISTER_TYPE_UD; + global_addr.stride = 1; + ubld.group(2, 0).MOV(header, global_addr); + break; + + case SHADER_OPCODE_BTD_RETIRE_LOGICAL: + /* The bottom bit is the Stack ID release bit */ + ubld.group(1, 0).MOV(header, brw_imm_ud(1)); + break; + + default: + unreachable("Invalid BTD message"); + } + + /* Stack IDs are always in R1 regardless of whether we're coming from a + * bindless shader or a regular compute shader. + */ + fs_reg stack_ids = + retype(byte_offset(header, REG_SIZE), BRW_REGISTER_TYPE_UW); + bld.MOV(stack_ids, retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UW)); + + unsigned ex_mlen = 0; + fs_reg payload; + if (inst->opcode == SHADER_OPCODE_BTD_SPAWN_LOGICAL) { + ex_mlen = 2 * (inst->exec_size / 8); + payload = bld.move_to_vgrf(btd_record, 1); + } else { + assert(inst->opcode == SHADER_OPCODE_BTD_RETIRE_LOGICAL); + /* All these messages take a BTD and things complain if we don't provide + * one for RETIRE. However, it shouldn't ever actually get used so fill + * it with zero. + */ + ex_mlen = 2 * (inst->exec_size / 8); + payload = bld.move_to_vgrf(brw_imm_uq(0), 1); + } + + /* Update the original instruction. */ + inst->opcode = SHADER_OPCODE_SEND; + inst->mlen = mlen; + inst->ex_mlen = ex_mlen; + inst->header_size = 0; /* HW docs require has_header = false */ + inst->send_has_side_effects = true; + inst->send_is_volatile = false; + + /* Set up SFID and descriptors */ + inst->sfid = GEN_RT_SFID_BINDLESS_THREAD_DISPATCH; + inst->desc = brw_btd_spawn_desc(devinfo, inst->exec_size, + GEN_RT_BTD_MESSAGE_SPAWN); + inst->resize_sources(4); + inst->src[0] = brw_imm_ud(0); /* desc */ + inst->src[1] = brw_imm_ud(0); /* ex_desc */ + inst->src[2] = header; + inst->src[3] = payload; +} + bool fs_visitor::lower_logical_sends() { @@ -6177,6 +6247,11 @@ fs_visitor::lower_logical_sends() continue; } + case SHADER_OPCODE_BTD_SPAWN_LOGICAL: + case SHADER_OPCODE_BTD_RETIRE_LOGICAL: + lower_btd_logical_send(ibld, inst); + break; + default: continue; } @@ -7546,7 +7621,8 @@ void fs_visitor::setup_cs_payload() { assert(devinfo->gen >= 7); - payload.num_regs = 1; + /* TODO: Fill out uses_btd_stack_ids automatically */ + payload.num_regs = 1 + brw_cs_prog_data(prog_data)->uses_btd_stack_ids; } brw::register_pressure::register_pressure(const fs_visitor *v) @@ -8474,6 +8550,43 @@ fs_visitor::run_cs(bool allow_spilling) return !failed; } +bool +fs_visitor::run_bs(bool allow_spilling) +{ + assert(stage >= MESA_SHADER_RAYGEN && stage <= MESA_SHADER_CALLABLE); + + /* R0: thread header, R1: stack IDs, R2: argument addresses */ + payload.num_regs = 3; + + if (shader_time_index >= 0) + emit_shader_time_begin(); + + emit_nir_code(); + + if (failed) + return false; + + /* TODO(RT): Perhaps rename this? */ + emit_cs_terminate(); + + if (shader_time_index >= 0) + emit_shader_time_end(); + + calculate_cfg(); + + optimize(); + + assign_curb_setup(); + + fixup_3src_null_dest(); + allocate_registers(allow_spilling); + + if (failed) + return false; + + return !failed; +} + static bool is_used_in_not_interp_frag_coord(nir_ssa_def *def) { @@ -9423,6 +9536,103 @@ brw_cs_simd_size_for_group_size(const struct gen_device_info *devinfo, return 32; } +const unsigned * +brw_compile_bs(const struct brw_compiler *compiler, void *log_data, + void *mem_ctx, + const struct brw_bs_prog_key *key, + struct brw_bs_prog_data *prog_data, + nir_shader *shader, + struct brw_compile_stats *stats, + char **error_str) +{ + prog_data->base.stage = shader->info.stage; + prog_data->stack_size = shader->scratch_size; + + const unsigned max_dispatch_width = 16; + brw_nir_apply_key(shader, compiler, &key->base, max_dispatch_width, true); + brw_postprocess_nir(shader, compiler, true); + + fs_visitor *v = NULL, *v8 = NULL, *v16 = NULL; + bool has_spilled = false; + + if (likely(!(INTEL_DEBUG & DEBUG_NO8))) { + v8 = new fs_visitor(compiler, log_data, mem_ctx, &key->base, + &prog_data->base, shader, + 8, -1 /* shader time */); + const bool allow_spilling = true; + if (!v8->run_bs(allow_spilling)) { + if (error_str) + *error_str = ralloc_strdup(mem_ctx, v8->fail_msg); + delete v8; + return NULL; + } else { + v = v8; + prog_data->simd_size = 8; + if (v8->spilled_any_registers) + has_spilled = true; + } + } + + if (!has_spilled && likely(!(INTEL_DEBUG & DEBUG_NO16))) { + v16 = new fs_visitor(compiler, log_data, mem_ctx, &key->base, + &prog_data->base, shader, + 16, -1 /* shader time */); + const bool allow_spilling = (v == NULL); + if (!v16->run_bs(allow_spilling)) { + compiler->shader_perf_log(log_data, + "SIMD16 shader failed to compile: %s", + v16->fail_msg); + if (v == NULL) { + assert(v8 == NULL); + if (error_str) { + *error_str = ralloc_asprintf( + mem_ctx, "SIMD8 disabled and couldn't generate SIMD16: %s", + v16->fail_msg); + } + delete v16; + return NULL; + } + } else { + v = v16; + prog_data->simd_size = 16; + if (v16->spilled_any_registers) + has_spilled = true; + } + } + + if (unlikely(v == NULL)) { + assert(INTEL_DEBUG & (DEBUG_NO8 | DEBUG_NO16)); + if (error_str) { + *error_str = ralloc_strdup(mem_ctx, + "Cannot satisfy INTEL_DEBUG flags SIMD restrictions"); + } + return NULL; + } + + assert(v); + + fs_generator g(compiler, log_data, mem_ctx, &prog_data->base, + v->runtime_check_aads_emit, shader->info.stage); + if (INTEL_DEBUG & DEBUG_RT) { + char *name = ralloc_asprintf(mem_ctx, "%s %s shader %s", + shader->info.label ? + shader->info.label : "unnamed", + gl_shader_stage_name(shader->info.stage), + shader->info.name); + g.enable_debug(name); + } + + g.generate_code(v->cfg, prog_data->simd_size, v->shader_stats, + v->performance_analysis.require(), stats); + + delete v8; + delete v16; + + g.add_const_data(shader->constant_data, shader->constant_data_size); + + return g.get_assembly(); +} + /** * Test the dispatch mask packing assumptions of * brw_stage_has_packed_dispatch(). Call this from e.g. the top of diff --git a/src/intel/compiler/brw_fs.h b/src/intel/compiler/brw_fs.h index 8f543e1ec88..b3b280192dc 100644 --- a/src/intel/compiler/brw_fs.h +++ b/src/intel/compiler/brw_fs.h @@ -127,6 +127,7 @@ public: bool run_tes(); bool run_gs(); bool run_cs(bool allow_spilling); + bool run_bs(bool allow_spilling); void optimize(); void allocate_registers(bool allow_spilling); void setup_fs_payload_gen4(); @@ -250,6 +251,8 @@ public: nir_intrinsic_instr *instr); void nir_emit_cs_intrinsic(const brw::fs_builder &bld, nir_intrinsic_instr *instr); + void nir_emit_bs_intrinsic(const brw::fs_builder &bld, + nir_intrinsic_instr *instr); fs_reg get_nir_image_intrinsic_image(const brw::fs_builder &bld, nir_intrinsic_instr *instr); fs_reg get_nir_ssbo_intrinsic_index(const brw::fs_builder &bld, diff --git a/src/intel/compiler/brw_fs_generator.cpp b/src/intel/compiler/brw_fs_generator.cpp index 1ad82aaf282..da5f69c45c7 100644 --- a/src/intel/compiler/brw_fs_generator.cpp +++ b/src/intel/compiler/brw_fs_generator.cpp @@ -2593,6 +2593,35 @@ fs_generator::generate_code(const cfg_t *cfg, int dispatch_width, brw_float_controls_mode(p, src[0].d, src[1].d); break; + case SHADER_OPCODE_GET_DSS_ID: + /* The Slice, Dual-SubSlice, SubSlice, EU, and Thread IDs are all + * stored in sr0.0. Normally, for reading from HW regs, we'd just do + * this in the IR and let the back-end generate some code but these + * live in the state register which tends to have special rules. + * + * For convenience, we combine Slice ID and Dual-SubSlice ID into a + * single ID. + */ + if (devinfo->gen == 12) { + /* There is a SWSB restriction that requires that any time sr0 is + * accessed both the instruction doing the access and the next one + * have SWSB set to RegDist(1). + */ + if (brw_get_default_swsb(p).mode != TGL_SBID_NULL) + brw_SYNC(p, TGL_SYNC_NOP); + brw_set_default_swsb(p, tgl_swsb_regdist(1)); + brw_SHR(p, dst, brw_sr0_reg(0), brw_imm_ud(9)); + brw_set_default_swsb(p, tgl_swsb_regdist(1)); + brw_AND(p, dst, dst, brw_imm_ud(0x1f)); + } else { + /* These move around basically every hardware generation, so don't + * do any >= checks and fail if the platform hasn't explicitly + * been enabled here. + */ + unreachable("Unsupported platform"); + } + break; + default: unreachable("Unsupported opcode"); diff --git a/src/intel/compiler/brw_fs_nir.cpp b/src/intel/compiler/brw_fs_nir.cpp index 018be4588e8..2201389dd78 100644 --- a/src/intel/compiler/brw_fs_nir.cpp +++ b/src/intel/compiler/brw_fs_nir.cpp @@ -497,6 +497,14 @@ fs_visitor::nir_emit_instr(nir_instr *instr) case MESA_SHADER_KERNEL: nir_emit_cs_intrinsic(abld, nir_instr_as_intrinsic(instr)); break; + case MESA_SHADER_RAYGEN: + case MESA_SHADER_ANY_HIT: + case MESA_SHADER_CLOSEST_HIT: + case MESA_SHADER_MISS: + case MESA_SHADER_INTERSECTION: + case MESA_SHADER_CALLABLE: + nir_emit_bs_intrinsic(abld, nir_instr_as_intrinsic(instr)); + break; default: unreachable("unsupported shader stage"); } @@ -3871,6 +3879,31 @@ fs_visitor::nir_emit_cs_intrinsic(const fs_builder &bld, } } +void +fs_visitor::nir_emit_bs_intrinsic(const fs_builder &bld, + nir_intrinsic_instr *instr) +{ + assert(brw_shader_stage_is_bindless(stage)); + + fs_reg dest; + if (nir_intrinsic_infos[instr->intrinsic].has_dest) + dest = get_nir_dest(instr->dest); + + switch (instr->intrinsic) { + case nir_intrinsic_load_btd_global_arg_addr_intel: + bld.MOV(dest, retype(brw_vec1_grf(2, 0), dest.type)); + break; + + case nir_intrinsic_load_btd_local_arg_addr_intel: + bld.MOV(dest, retype(brw_vec1_grf(2, 2), dest.type)); + break; + + default: + nir_emit_intrinsic(bld, instr); + break; + } +} + static fs_reg brw_nir_reduction_op_identity(const fs_builder &bld, nir_op op, brw_reg_type type) @@ -5436,6 +5469,44 @@ fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr break; } + case nir_intrinsic_load_btd_dss_id_intel: + bld.emit(SHADER_OPCODE_GET_DSS_ID, + retype(dest, BRW_REGISTER_TYPE_UD)); + break; + + case nir_intrinsic_load_btd_stack_id_intel: + if (stage == MESA_SHADER_COMPUTE) { + assert(brw_cs_prog_data(prog_data)->uses_btd_stack_ids); + } else { + assert(brw_shader_stage_is_bindless(stage)); + } + /* Stack IDs are always in R1 regardless of whether we're coming from a + * bindless shader or a regular compute shader. + */ + bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), + retype(brw_vec8_grf(1, 0), BRW_REGISTER_TYPE_UW)); + break; + + case nir_intrinsic_btd_spawn_intel: + if (stage == MESA_SHADER_COMPUTE) { + assert(brw_cs_prog_data(prog_data)->uses_btd_stack_ids); + } else { + assert(brw_shader_stage_is_bindless(stage)); + } + bld.emit(SHADER_OPCODE_BTD_SPAWN_LOGICAL, bld.null_reg_ud(), + bld.emit_uniformize(get_nir_src(instr->src[0])), + get_nir_src(instr->src[1])); + break; + + case nir_intrinsic_btd_retire_intel: + if (stage == MESA_SHADER_COMPUTE) { + assert(brw_cs_prog_data(prog_data)->uses_btd_stack_ids); + } else { + assert(brw_shader_stage_is_bindless(stage)); + } + bld.emit(SHADER_OPCODE_BTD_RETIRE_LOGICAL); + break; + default: unreachable("unknown intrinsic"); } diff --git a/src/intel/compiler/brw_fs_visitor.cpp b/src/intel/compiler/brw_fs_visitor.cpp index 233e1c80bd9..7c46086a1b2 100644 --- a/src/intel/compiler/brw_fs_visitor.cpp +++ b/src/intel/compiler/brw_fs_visitor.cpp @@ -844,9 +844,6 @@ fs_visitor::emit_cs_terminate() { assert(devinfo->gen >= 7); - /* We are getting the thread ID from the compute shader header */ - assert(stage == MESA_SHADER_COMPUTE || stage == MESA_SHADER_KERNEL); - /* We can't directly send from g0, since sends with EOT have to use * g112-127. So, copy it to a virtual register, The register allocator will * make sure it uses the appropriate register range. diff --git a/src/intel/compiler/brw_ir_performance.cpp b/src/intel/compiler/brw_ir_performance.cpp index c0fae6187f3..98049449c12 100644 --- a/src/intel/compiler/brw_ir_performance.cpp +++ b/src/intel/compiler/brw_ir_performance.cpp @@ -355,6 +355,7 @@ namespace { case TCS_OPCODE_SRC0_010_IS_ZERO: case TCS_OPCODE_GET_PRIMITIVE_ID: case TES_OPCODE_GET_PRIMITIVE_ID: + case SHADER_OPCODE_GET_DSS_ID: if (devinfo->gen >= 11) { return calculate_desc(info, unit_fpu, 0, 2, 0, 0, 2, 0, 10, 6 /* XXX */, 14, 0, 0); @@ -1086,6 +1087,11 @@ namespace { } else { abort(); } + + case GEN_RT_SFID_BINDLESS_THREAD_DISPATCH: + return calculate_desc(info, unit_spawner, 2, 0, 0, 0 /* XXX */, 0, + 10 /* XXX */, 0, 0, 0, 0, 0); + default: abort(); } diff --git a/src/intel/compiler/brw_schedule_instructions.cpp b/src/intel/compiler/brw_schedule_instructions.cpp index d9f6f9d852f..38f74a7988f 100644 --- a/src/intel/compiler/brw_schedule_instructions.cpp +++ b/src/intel/compiler/brw_schedule_instructions.cpp @@ -524,6 +524,15 @@ schedule_node::set_latency_gen7(bool is_haswell) } break; + case GEN_RT_SFID_BINDLESS_THREAD_DISPATCH: + /* TODO. + * + * We'll assume for the moment that this is pretty quick as it + * doesn't actually return any data. + */ + latency = 200; + break; + default: unreachable("Unknown SFID"); } diff --git a/src/intel/compiler/brw_shader.cpp b/src/intel/compiler/brw_shader.cpp index 19acf8fa87e..11da2825500 100644 --- a/src/intel/compiler/brw_shader.cpp +++ b/src/intel/compiler/brw_shader.cpp @@ -547,6 +547,12 @@ brw_instruction_name(const struct gen_device_info *devinfo, enum opcode op) return "rnd_mode"; case SHADER_OPCODE_FLOAT_CONTROL_MODE: return "float_control_mode"; + case SHADER_OPCODE_GET_DSS_ID: + return "get_dss_id"; + case SHADER_OPCODE_BTD_SPAWN_LOGICAL: + return "btd_spawn_logical"; + case SHADER_OPCODE_BTD_RETIRE_LOGICAL: + return "btd_retire_logical"; } unreachable("not reached"); @@ -1104,6 +1110,8 @@ backend_instruction::has_side_effects() const case FS_OPCODE_SCHEDULING_FENCE: case SHADER_OPCODE_OWORD_BLOCK_WRITE_LOGICAL: case SHADER_OPCODE_A64_OWORD_BLOCK_WRITE_LOGICAL: + case SHADER_OPCODE_BTD_SPAWN_LOGICAL: + case SHADER_OPCODE_BTD_RETIRE_LOGICAL: return true; default: return eot;