AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

freedreno/ir3: support a4xx in load/store buffer/image emission

Browse Source 
Signed-off-by: Ilia Mirkin <imirkin@alum.mit.edu>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/14794>
This commit is contained in:
Ilia Mirkin
2021-11-14 13:04:26 -05:00
parent e9cd4fba6f
commit 6fb5e64ead
3 changed files with 97 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/freedreno/ir3/ir3_a4xx.c
+82 -13
View File
@@ -30,10 +30,41 @@
#include "ir3_context.h"
#include "ir3_image.h"
/* SSBO data is available at this CB address, addressed like regular consts
* containing the following data in each vec4:
*
* [ base address, pitch, array_pitch, cpp ]
*
* These mirror the values uploaded to A4XX_SSBO_0 state. For A5XX, these are
* uploaded manually by the driver.
*/
#define A4XX_SSBO_CB_BASE(i) (0x700 + ((i) << 2))
/*
* Handlers for instructions changed/added in a4xx:
*/
/* Convert byte offset to address of appropriate width for GPU */
static struct ir3_instruction *
byte_offset_to_address(struct ir3_context *ctx,
nir_src *ssbo,
struct ir3_instruction *byte_offset)
{
struct ir3_block *b = ctx->block;
if (ctx->compiler->gen == 4) {
uint32_t index = nir_src_as_uint(*ssbo);
unsigned cb = A4XX_SSBO_CB_BASE(index);
byte_offset = ir3_ADD_U(b, create_uniform(b, cb), 0, byte_offset, 0);
}
if (fd_dev_64b(ctx->compiler->dev_id)) {
return ir3_collect(b, byte_offset, create_immed(b, 0));
} else {
return byte_offset;
}
}
/* src[] = { buffer_index, offset }. No const_index */
static void
emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr,
@@ -48,7 +79,7 @@ emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr,
offset = ir3_get_src(ctx, &intr->src[2])[0];
/* src0 is uvec2(offset*4, 0), src1 is offset.. nir already *= 4: */
src0 = ir3_collect(b, byte_offset, create_immed(b, 0));
src0 = byte_offset_to_address(ctx, &intr->src[0], byte_offset);
src1 = offset;
ldgb = ir3_LDGB(b, ssbo, 0, src0, 0, src1, 0);
@@ -83,7 +114,7 @@ emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
*/
src0 = ir3_create_collect(b, ir3_get_src(ctx, &intr->src[0]), ncomp);
src1 = offset;
src2 = ir3_collect(b, byte_offset, create_immed(b, 0));
src2 = byte_offset_to_address(ctx, &intr->src[1], byte_offset);
stgb = ir3_STGB(b, ssbo, 0, src0, 0, src1, 0, src2, 0);
stgb->cat6.iim_val = ncomp;
@@ -129,7 +160,7 @@ emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
struct ir3_instruction *data = ir3_get_src(ctx, &intr->src[2])[0];
/* 64b byte offset */
struct ir3_instruction *byte_offset =
ir3_collect(b, ir3_get_src(ctx, &intr->src[1])[0], create_immed(b, 0));
byte_offset_to_address(ctx, &intr->src[0], ir3_get_src(ctx, &intr->src[1])[0]);
/* dword offset for everything but comp_swap */
struct ir3_instruction *src3 = ir3_get_src(ctx, &intr->src[3])[0];
@@ -198,14 +229,23 @@ get_image_offset(struct ir3_context *ctx, const nir_intrinsic_instr *instr,
/* to calculate the byte offset (yes, uggg) we need (up to) three
* const values to know the bytes per pixel, and y and z stride:
*/
const struct ir3_const_state *const_state = ir3_const_state(ctx->so);
unsigned cb = regid(const_state->offsets.image_dims, 0) +
const_state->image_dims.off[index];
unsigned cb;
if (ctx->compiler->gen > 4) {
const struct ir3_const_state *const_state = ir3_const_state(ctx->so);
debug_assert(const_state->image_dims.mask & (1 << index));
debug_assert(const_state->image_dims.mask & (1 << index));
cb = regid(const_state->offsets.image_dims, 0) +
const_state->image_dims.off[index];
} else {
index += ctx->s->info.num_ssbos;
cb = A4XX_SSBO_CB_BASE(index);
}
/* offset = coords.x * bytes_per_pixel: */
offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 0), 0);
if (ctx->compiler->gen == 4)
offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 3), 0);
else
offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 0), 0);
if (ncoords > 1) {
/* offset += coords.y * y_pitch: */
offset =
@@ -217,6 +257,10 @@ get_image_offset(struct ir3_context *ctx, const nir_intrinsic_instr *instr,
ir3_MAD_S24(b, create_uniform(b, cb + 2), 0, coords[2], 0, offset, 0);
}
/* a4xx: must add in the base address: */
if (ctx->compiler->gen == 4)
offset = ir3_ADD_U(b, offset, 0, create_uniform(b, cb + 0), 0);
if (!byteoff) {
/* Some cases, like atomics, seem to use dword offset instead
* of byte offsets.. blob just puts an extra shr.b in there
@@ -225,7 +269,10 @@ get_image_offset(struct ir3_context *ctx, const nir_intrinsic_instr *instr,
offset = ir3_SHR_B(b, offset, 0, create_immed(b, 2), 0);
}
return ir3_collect(b, offset, create_immed(b, 0));
if (fd_dev_64b(ctx->compiler->dev_id))
return ir3_collect(b, offset, create_immed(b, 0));
else
return offset;
}
/* src[] = { deref, coord, sample_index }. const_index[] = {} */
@@ -241,8 +288,30 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr,
unsigned ncomp =
ir3_get_num_components_for_image_format(nir_intrinsic_format(intr));
struct ir3_instruction *ldib = ir3_LDIB(
b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0);
struct ir3_instruction *ldib;
/* At least A420 does not have LDIB. Use LDGB and perform conversion
* ourselves.
*
* TODO: Actually do the conversion. ES 3.1 only requires this for
* single-component 32-bit types anyways.
*/
if (ctx->compiler->gen > 4) {
ldib = ir3_LDIB(
b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0);
} else {
ldib = ir3_LDGB(
b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0);
switch (nir_intrinsic_format(intr)) {
case PIPE_FORMAT_R32_UINT:
case PIPE_FORMAT_R32_SINT:
case PIPE_FORMAT_R32_FLOAT:
break;
default:
/* For some reason even more 32-bit components don't work. */
debug_assert(0);
break;
}
}
ldib->dsts[0]->wrmask = MASK(intr->num_components);
ldib->cat6.iim_val = ncomp;
ldib->cat6.d = ncoords;
@@ -307,7 +376,7 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
*/
src0 = ir3_get_src(ctx, &intr->src[3])[0];
src1 = ir3_create_collect(b, coords, ncoords);
src2 = get_image_offset(ctx, intr, coords, false);
src2 = get_image_offset(ctx, intr, coords, ctx->compiler->gen == 4);
switch (intr->intrinsic) {
case nir_intrinsic_image_atomic_add:
@@ -345,7 +414,7 @@ emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
atomic->cat6.iim_val = 1;
atomic->cat6.d = ncoords;
atomic->cat6.type = ir3_get_type_for_image_intrinsic(intr);
atomic->cat6.typed = true;
atomic->cat6.typed = ctx->compiler->gen == 5;
atomic->barrier_class = IR3_BARRIER_IMAGE_W;
atomic->barrier_conflict = IR3_BARRIER_IMAGE_R | IR3_BARRIER_IMAGE_W;
src/freedreno/ir3/ir3_compiler_nir.c
+1 -1
View File
@@ -1396,7 +1396,7 @@ emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr,
/* If the image can be written, must use LDIB to retrieve data, rather than
* through ISAM (which uses the texture cache and won't get previous writes).
*/
if (!(nir_intrinsic_access(intr) & ACCESS_CAN_REORDER) && ctx->compiler->gen >= 5) {
if (!(nir_intrinsic_access(intr) & ACCESS_CAN_REORDER)) {
ctx->funcs->emit_intrinsic_load_image(ctx, intr, dst);
return;
}
src/freedreno/ir3/ir3_nir.c
+14 -9
View File
@@ -199,15 +199,9 @@ ir3_nir_lower_ssbo_size_instr(nir_builder *b, nir_instr *instr, void *data)
return nir_ishl(b, &intr->dest.ssa, nir_imm_int(b, ssbo_size_to_bytes_shift));
}
/**
* The resinfo opcode we have for getting the SSBO size on a6xx returns a byte
* length divided by IBO_0_FMT, while the NIR intrinsic coming in is a number of
* bytes. Switch things so the NIR intrinsic in our backend means dwords.
*/
static bool
ir3_nir_lower_ssbo_size(nir_shader *s, bool storage_16bit)
ir3_nir_lower_ssbo_size(nir_shader *s, uint8_t ssbo_size_to_bytes_shift)
{
uint8_t ssbo_size_to_bytes_shift = storage_16bit ? 1 : 2;
return nir_shader_lower_instructions(s, ir3_nir_lower_ssbo_size_filter,
ir3_nir_lower_ssbo_size_instr,
&ssbo_size_to_bytes_shift);
@@ -490,8 +484,18 @@ ir3_nir_post_finalize(struct ir3_shader *shader)
};
NIR_PASS_V(s, nir_lower_idiv, &lower_idiv_options); /* idiv generated by cube lowering */
/* The resinfo opcode returns the size in dwords on a4xx */
if (compiler->gen == 4)
OPT_V(s, ir3_nir_lower_ssbo_size, 2);
/* The resinfo opcode we have for getting the SSBO size on a6xx returns a
* byte length divided by IBO_0_FMT, while the NIR intrinsic coming in is a
* number of bytes. Switch things so the NIR intrinsic in our backend means
* dwords.
*/
if (compiler->gen >= 6)
OPT_V(s, ir3_nir_lower_ssbo_size, compiler->storage_16bit);
OPT_V(s, ir3_nir_lower_ssbo_size, compiler->storage_16bit ? 1 : 2);
ir3_optimize_loop(compiler, s);
}
@@ -722,7 +726,8 @@ ir3_nir_scan_driver_consts(struct ir3_compiler *compiler, nir_shader *shader, st
case nir_intrinsic_image_load:
case nir_intrinsic_image_store:
case nir_intrinsic_image_size:
if (compiler->gen < 6 &&
/* a4xx gets these supplied by the hw directly (maybe CP?) */
if (compiler->gen == 5 &&
!(intr->intrinsic == nir_intrinsic_image_load &&
!(nir_intrinsic_access(intr) & ACCESS_COHERENT))) {
idx = nir_src_as_uint(intr->src[0]);