AlexIndustrial/mesa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ac/llvm: add new cache flags for gfx12

Browse Source 
Reviewed-by: Timur Kristóf <timur.kristof@gmail.com>
Acked-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/29007>
This commit is contained in:
Marek Olšák
2023-04-27 03:56:03 -04:00
parent a6c46509cc
commit 0356209543
2 changed files with 128 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/amd/common/ac_shader_util.c
+28 -3
View File
@@ -1240,7 +1240,28 @@ union ac_hw_cache_flags ac_get_hw_cache_flags(const struct radeon_info *info,
bool scope_is_device = access & (ACCESS_COHERENT | ACCESS_VOLATILE);
if (info->gfx_level >= GFX11) {
if (info->gfx_level >= GFX12) {
if (access & ACCESS_CP_GE_COHERENT_AMD) {
result.gfx12.scope = info->cp_sdma_ge_use_system_memory_scope ?
gfx12_scope_memory : gfx12_scope_device;
} else if (scope_is_device) {
result.gfx12.scope = gfx12_scope_device;
} else {
result.gfx12.scope = gfx12_scope_cu;
}
if (access & ACCESS_NON_TEMPORAL) {
if (access & ACCESS_TYPE_LOAD) {
/* Don't use non_temporal for SMEM because it can't set regular_temporal for MALL. */
if (!(access & ACCESS_TYPE_SMEM))
result.gfx12.temporal_hint = gfx12_load_near_non_temporal_far_regular_temporal;
} else if (access & ACCESS_TYPE_STORE) {
result.gfx12.temporal_hint = gfx12_store_near_non_temporal_far_regular_temporal;
} else {
result.gfx12.temporal_hint = gfx12_atomic_non_temporal;
}
}
} else if (info->gfx_level >= GFX11) {
/* GFX11 simplified it and exposes what is actually useful.
*
* GLC means device scope for loads only. (stores and atomics are always device scope)
@@ -1322,8 +1343,12 @@ union ac_hw_cache_flags ac_get_hw_cache_flags(const struct radeon_info *info,
result.value |= ac_glc;
}
if (access & ACCESS_IS_SWIZZLED_AMD)
result.value |= ac_swizzled;
if (access & ACCESS_IS_SWIZZLED_AMD) {
if (info->gfx_level >= GFX12)
result.gfx12.swizzled = true;
else
result.value |= ac_swizzled;
}
return result;
}
src/amd/common/ac_shader_util.h
+100 -3
View File
@@ -47,7 +47,7 @@ enum {
ACCESS_MAY_STORE_SUBDWORD = BITFIELD_BIT(31),
};
/* The meaning of these enums is different between chips. They match LLVM definitions,
/* GFX6-11. The meaning of these enums is different between chips. They match LLVM definitions,
* but they can also be used by ACO. Use ac_get_hw_cache_flags to get these.
*/
enum ac_cache_flags
@@ -58,10 +58,107 @@ enum ac_cache_flags
ac_swizzled = BITFIELD_BIT(3),
};
/* Cache-agnostic scope flags. */
enum gfx12_scope
{
/* Memory access is coherent within a workgroup in CU mode.
* There is no coherency between VMEM and SMEM.
*/
gfx12_scope_cu,
/* Memory access is coherent within an SE.
* If there is no SE cache, this resolves to the device scope in the gfx domain.
*/
gfx12_scope_se,
/* Memory access is globally coherent within the device for all gfx blocks except CP and GE
* depending on the chip (see below). This is referred to as the device scope. It's not coherent
* with non-gfx blocks like DCN and VCN.
*
* If there a single global GL2 cache:
* - The device scope in the gfx domain resolves to GL2 scope in hw.
* - Memory access is cached in GL2.
* - radeon_info::cp_sdma_ge_use_system_memory_scope says whether CP, SDMA, and GE are
* not coherent. If true, some features need special handling. The list of the features
* and the suggested programming is:
* * tess factor ring for GE: use ACCESS_CP_GE_COHERENT_AMD (it selects the correct scope
* automatically)
* * query results accessed by shaders: Range-invalidate GL2 before the shaders
* * vertex indices for GE: flush GL2 after buffer stores, but don't invalidate
* * draw indirect for CP: flush GL2 after buffer stores, but don't invalidate
* * shader uploads via SDMA: invalidate GL2 at the beginning of IBs
* * PRIME buffer read by SDMA: the kernel flushes GL2 at the end of IBs
* * CP DMA clears/copies: use compute shaders or range-flush/invalidate GL2 around it
* * CP DMA prefetch: no change
* * COPY_DATA - FILLED_SIZE state for streamout, range-flush/invalidate GL2
* * WRITE_DATA - bindless descriptors: range-invalidate GL2
*
* If there is a separate GL2 cache per SE:
* - The device scope resolves to memory scope in hw.
* - Memory access is cached in MALL if MALL (infinity cache) is present.
* - radeon_info::cp_sdma_ge_use_system_memory_scope is always false in this case.
*/
gfx12_scope_device,
/* Memory scope. It's cached if MALL is present. This is called "system scope" in the ISA
* documentation.
*/
gfx12_scope_memory,
};
enum gfx12_load_temporal_hint
{
/* VMEM and SMEM */
gfx12_load_regular_temporal,
gfx12_load_non_temporal,
gfx12_load_high_temporal,
/* VMEM$ treats SCOPE=3 and TH=3 as MALL bypass on GFX12. Don't use this combination in shaders. */
gfx12_load_last_use_discard,
/* VMEM only, far means the last level cache, near means other caches. */
gfx12_load_near_non_temporal_far_regular_temporal,
gfx12_load_near_regular_temporal_far_non_temporal,
gfx12_load_near_non_temporal_far_high_temporal,
gfx12_load_reserved,
};
enum gfx12_store_temporal_hint
{
gfx12_store_regular_temporal,
gfx12_store_non_temporal,
gfx12_store_high_temporal,
gfx12_store_high_temporal_stay_dirty,
gfx12_store_near_non_temporal_far_regular_temporal,
gfx12_store_near_regular_temporal_far_non_temporal,
gfx12_store_near_non_temporal_far_high_temporal,
gfx12_store_near_non_temporal_far_writeback,
};
enum gfx12_atomic_temporal_hint
{
gfx12_atomic_return = BITFIELD_BIT(0),
gfx12_atomic_non_temporal = BITFIELD_BIT(1),
gfx12_atomic_accum_deferred_scope = BITFIELD_BIT(2), /* requires no return */
};
enum gfx12_speculative_data_read
{
gfx12_spec_read_auto,
gfx12_spec_read_force_on,
gfx12_spec_read_force_off,
};
union ac_hw_cache_flags
{
/* NOTE: This will contain more fields in the future. */
enum ac_cache_flags value;
struct {
/* This matches LLVM, but it can also be used by ACO for translation of ac_memop_flags. */
uint8_t temporal_hint:3; /* gfx12_{load,store,atomic}_temporal_hint */
uint8_t scope:2; /* gfx12_scope */
uint8_t _reserved:1;
uint8_t swizzled:1; /* for swizzled buffer access (attribute ring) */
uint8_t _pad:1;
} gfx12;
uint8_t value; /* ac_cache_flags (GFX6-11) or the gfx12 structure */
};
enum ac_image_dim