diff --git a/src/freedreno/ir3/ir3_a4xx.c b/src/freedreno/ir3/ir3_a4xx.c index 4939ba98373..0334588f13e 100644 --- a/src/freedreno/ir3/ir3_a4xx.c +++ b/src/freedreno/ir3/ir3_a4xx.c @@ -30,10 +30,41 @@ #include "ir3_context.h" #include "ir3_image.h" +/* SSBO data is available at this CB address, addressed like regular consts + * containing the following data in each vec4: + * + * [ base address, pitch, array_pitch, cpp ] + * + * These mirror the values uploaded to A4XX_SSBO_0 state. For A5XX, these are + * uploaded manually by the driver. + */ +#define A4XX_SSBO_CB_BASE(i) (0x700 + ((i) << 2)) + /* * Handlers for instructions changed/added in a4xx: */ +/* Convert byte offset to address of appropriate width for GPU */ +static struct ir3_instruction * +byte_offset_to_address(struct ir3_context *ctx, + nir_src *ssbo, + struct ir3_instruction *byte_offset) +{ + struct ir3_block *b = ctx->block; + + if (ctx->compiler->gen == 4) { + uint32_t index = nir_src_as_uint(*ssbo); + unsigned cb = A4XX_SSBO_CB_BASE(index); + byte_offset = ir3_ADD_U(b, create_uniform(b, cb), 0, byte_offset, 0); + } + + if (fd_dev_64b(ctx->compiler->dev_id)) { + return ir3_collect(b, byte_offset, create_immed(b, 0)); + } else { + return byte_offset; + } +} + /* src[] = { buffer_index, offset }. No const_index */ static void emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr, @@ -48,7 +79,7 @@ emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr, offset = ir3_get_src(ctx, &intr->src[2])[0]; /* src0 is uvec2(offset*4, 0), src1 is offset.. nir already *= 4: */ - src0 = ir3_collect(b, byte_offset, create_immed(b, 0)); + src0 = byte_offset_to_address(ctx, &intr->src[0], byte_offset); src1 = offset; ldgb = ir3_LDGB(b, ssbo, 0, src0, 0, src1, 0); @@ -83,7 +114,7 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) */ src0 = ir3_create_collect(b, ir3_get_src(ctx, &intr->src[0]), ncomp); src1 = offset; - src2 = ir3_collect(b, byte_offset, create_immed(b, 0)); + src2 = byte_offset_to_address(ctx, &intr->src[1], byte_offset); stgb = ir3_STGB(b, ssbo, 0, src0, 0, src1, 0, src2, 0); stgb->cat6.iim_val = ncomp; @@ -129,7 +160,7 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr) struct ir3_instruction *data = ir3_get_src(ctx, &intr->src[2])[0]; /* 64b byte offset */ struct ir3_instruction *byte_offset = - ir3_collect(b, ir3_get_src(ctx, &intr->src[1])[0], create_immed(b, 0)); + byte_offset_to_address(ctx, &intr->src[0], ir3_get_src(ctx, &intr->src[1])[0]); /* dword offset for everything but comp_swap */ struct ir3_instruction *src3 = ir3_get_src(ctx, &intr->src[3])[0]; @@ -198,14 +229,23 @@ get_image_offset(struct ir3_context *ctx, const nir_intrinsic_instr *instr, /* to calculate the byte offset (yes, uggg) we need (up to) three * const values to know the bytes per pixel, and y and z stride: */ - const struct ir3_const_state *const_state = ir3_const_state(ctx->so); - unsigned cb = regid(const_state->offsets.image_dims, 0) + - const_state->image_dims.off[index]; + unsigned cb; + if (ctx->compiler->gen > 4) { + const struct ir3_const_state *const_state = ir3_const_state(ctx->so); + debug_assert(const_state->image_dims.mask & (1 << index)); - debug_assert(const_state->image_dims.mask & (1 << index)); + cb = regid(const_state->offsets.image_dims, 0) + + const_state->image_dims.off[index]; + } else { + index += ctx->s->info.num_ssbos; + cb = A4XX_SSBO_CB_BASE(index); + } /* offset = coords.x * bytes_per_pixel: */ - offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 0), 0); + if (ctx->compiler->gen == 4) + offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 3), 0); + else + offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 0), 0); if (ncoords > 1) { /* offset += coords.y * y_pitch: */ offset = @@ -217,6 +257,10 @@ get_image_offset(struct ir3_context *ctx, const nir_intrinsic_instr *instr, ir3_MAD_S24(b, create_uniform(b, cb + 2), 0, coords[2], 0, offset, 0); } + /* a4xx: must add in the base address: */ + if (ctx->compiler->gen == 4) + offset = ir3_ADD_U(b, offset, 0, create_uniform(b, cb + 0), 0); + if (!byteoff) { /* Some cases, like atomics, seem to use dword offset instead * of byte offsets.. blob just puts an extra shr.b in there @@ -225,7 +269,10 @@ get_image_offset(struct ir3_context *ctx, const nir_intrinsic_instr *instr, offset = ir3_SHR_B(b, offset, 0, create_immed(b, 2), 0); } - return ir3_collect(b, offset, create_immed(b, 0)); + if (fd_dev_64b(ctx->compiler->dev_id)) + return ir3_collect(b, offset, create_immed(b, 0)); + else + return offset; } /* src[] = { deref, coord, sample_index }. const_index[] = {} */ @@ -241,8 +288,30 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr, unsigned ncomp = ir3_get_num_components_for_image_format(nir_intrinsic_format(intr)); - struct ir3_instruction *ldib = ir3_LDIB( - b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0); + struct ir3_instruction *ldib; + /* At least A420 does not have LDIB. Use LDGB and perform conversion + * ourselves. + * + * TODO: Actually do the conversion. ES 3.1 only requires this for + * single-component 32-bit types anyways. + */ + if (ctx->compiler->gen > 4) { + ldib = ir3_LDIB( + b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0); + } else { + ldib = ir3_LDGB( + b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0); + switch (nir_intrinsic_format(intr)) { + case PIPE_FORMAT_R32_UINT: + case PIPE_FORMAT_R32_SINT: + case PIPE_FORMAT_R32_FLOAT: + break; + default: + /* For some reason even more 32-bit components don't work. */ + debug_assert(0); + break; + } + } ldib->dsts[0]->wrmask = MASK(intr->num_components); ldib->cat6.iim_val = ncomp; ldib->cat6.d = ncoords; @@ -307,7 +376,7 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) */ src0 = ir3_get_src(ctx, &intr->src[3])[0]; src1 = ir3_create_collect(b, coords, ncoords); - src2 = get_image_offset(ctx, intr, coords, false); + src2 = get_image_offset(ctx, intr, coords, ctx->compiler->gen == 4); switch (intr->intrinsic) { case nir_intrinsic_image_atomic_add: @@ -345,7 +414,7 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr) atomic->cat6.iim_val = 1; atomic->cat6.d = ncoords; atomic->cat6.type = ir3_get_type_for_image_intrinsic(intr); - atomic->cat6.typed = true; + atomic->cat6.typed = ctx->compiler->gen == 5; atomic->barrier_class = IR3_BARRIER_IMAGE_W; atomic->barrier_conflict = IR3_BARRIER_IMAGE_R | IR3_BARRIER_IMAGE_W; diff --git a/src/freedreno/ir3/ir3_compiler_nir.c b/src/freedreno/ir3/ir3_compiler_nir.c index ad1a275685b..8337fb855d3 100644 --- a/src/freedreno/ir3/ir3_compiler_nir.c +++ b/src/freedreno/ir3/ir3_compiler_nir.c @@ -1396,7 +1396,7 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr, /* If the image can be written, must use LDIB to retrieve data, rather than * through ISAM (which uses the texture cache and won't get previous writes). */ - if (!(nir_intrinsic_access(intr) & ACCESS_CAN_REORDER) && ctx->compiler->gen >= 5) { + if (!(nir_intrinsic_access(intr) & ACCESS_CAN_REORDER)) { ctx->funcs->emit_intrinsic_load_image(ctx, intr, dst); return; } diff --git a/src/freedreno/ir3/ir3_nir.c b/src/freedreno/ir3/ir3_nir.c index 539dc2a5502..c59cecac0dc 100644 --- a/src/freedreno/ir3/ir3_nir.c +++ b/src/freedreno/ir3/ir3_nir.c @@ -199,15 +199,9 @@ ir3_nir_lower_ssbo_size_instr(nir_builder *b, nir_instr *instr, void *data) return nir_ishl(b, &intr->dest.ssa, nir_imm_int(b, ssbo_size_to_bytes_shift)); } -/** - * The resinfo opcode we have for getting the SSBO size on a6xx returns a byte - * length divided by IBO_0_FMT, while the NIR intrinsic coming in is a number of - * bytes. Switch things so the NIR intrinsic in our backend means dwords. - */ static bool -ir3_nir_lower_ssbo_size(nir_shader *s, bool storage_16bit) +ir3_nir_lower_ssbo_size(nir_shader *s, uint8_t ssbo_size_to_bytes_shift) { - uint8_t ssbo_size_to_bytes_shift = storage_16bit ? 1 : 2; return nir_shader_lower_instructions(s, ir3_nir_lower_ssbo_size_filter, ir3_nir_lower_ssbo_size_instr, &ssbo_size_to_bytes_shift); @@ -490,8 +484,18 @@ ir3_nir_post_finalize(struct ir3_shader *shader) }; NIR_PASS_V(s, nir_lower_idiv, &lower_idiv_options); /* idiv generated by cube lowering */ + + /* The resinfo opcode returns the size in dwords on a4xx */ + if (compiler->gen == 4) + OPT_V(s, ir3_nir_lower_ssbo_size, 2); + + /* The resinfo opcode we have for getting the SSBO size on a6xx returns a + * byte length divided by IBO_0_FMT, while the NIR intrinsic coming in is a + * number of bytes. Switch things so the NIR intrinsic in our backend means + * dwords. + */ if (compiler->gen >= 6) - OPT_V(s, ir3_nir_lower_ssbo_size, compiler->storage_16bit); + OPT_V(s, ir3_nir_lower_ssbo_size, compiler->storage_16bit ? 1 : 2); ir3_optimize_loop(compiler, s); } @@ -722,7 +726,8 @@ ir3_nir_scan_driver_consts(struct ir3_compiler *compiler, nir_shader *shader, st case nir_intrinsic_image_load: case nir_intrinsic_image_store: case nir_intrinsic_image_size: - if (compiler->gen < 6 && + /* a4xx gets these supplied by the hw directly (maybe CP?) */ + if (compiler->gen == 5 && !(intr->intrinsic == nir_intrinsic_image_load && !(nir_intrinsic_access(intr) & ACCESS_COHERENT))) { idx = nir_src_as_uint(intr->src[0]);