From d96cac61c51d58a30460e8a8b3606399bf823024 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Wed, 22 Mar 2023 10:10:13 +1000 Subject: [PATCH] nvk: add compute support for ampere ampere has support for QMDv3, it's not mandatory but lets use it for now. the more important change is the PCAS2_B call. Part-of: --- src/nouveau/nvidia-headers/meson.build | 1 + src/nouveau/nvidia-headers/nv_push.h | 1 + src/nouveau/vulkan/nvk_cmd_dispatch.c | 66 +++++++++++++++++++---- src/nouveau/vulkan/nvk_compute_pipeline.c | 29 +++++++++- 4 files changed, 87 insertions(+), 10 deletions(-) diff --git a/src/nouveau/nvidia-headers/meson.build b/src/nouveau/nvidia-headers/meson.build index 410b29e02d5..3fd421ad6f3 100644 --- a/src/nouveau/nvidia-headers/meson.build +++ b/src/nouveau/nvidia-headers/meson.build @@ -18,6 +18,7 @@ nvk_classes = [ 'clc3c0', 'clc597', 'clc5c0', + 'clc6c0', ] nvk_cl_header_depend_files = [ diff --git a/src/nouveau/nvidia-headers/nv_push.h b/src/nouveau/nvidia-headers/nv_push.h index c190a831f1a..b537ff4334a 100644 --- a/src/nouveau/nvidia-headers/nv_push.h +++ b/src/nouveau/nvidia-headers/nv_push.h @@ -57,6 +57,7 @@ void vk_push_print(FILE *fp, const struct nv_push *push, #define SUBC_NVB0C0 1 #define SUBC_NVC0C0 1 #define SUBC_NVC3C0 1 +#define SUBC_NVC6C0 1 #define SUBC_NV9039 2 diff --git a/src/nouveau/vulkan/nvk_cmd_dispatch.c b/src/nouveau/vulkan/nvk_cmd_dispatch.c index 8a667b96e4d..882ae3ac3c5 100644 --- a/src/nouveau/vulkan/nvk_cmd_dispatch.c +++ b/src/nouveau/vulkan/nvk_cmd_dispatch.c @@ -14,6 +14,8 @@ #include "nvk_cla0c0.h" #include "cla1c0.h" #include "clc0c0.h" +#include "clc5c0.h" +#include "nvk_clc6c0.h" #include "nvk_clc3c0.h" #include "nvk_clc597.h" @@ -21,6 +23,7 @@ #include "cla0c0qmd.h" #include "clc0c0qmd.h" #include "clc3c0qmd.h" +#include "clc6c0qmd.h" #define NVA0C0_QMDV00_06_VAL_SET(p,a...) NVVAL_MW_SET((p), NVA0C0, QMDV00_06, ##a) #define NVA0C0_QMDV00_06_DEF_SET(p,a...) NVDEF_MW_SET((p), NVA0C0, QMDV00_06, ##a) @@ -28,6 +31,8 @@ #define NVC0C0_QMDV02_01_DEF_SET(p,a...) NVDEF_MW_SET((p), NVC0C0, QMDV02_01, ##a) #define NVC3C0_QMDV02_02_VAL_SET(p,a...) NVVAL_MW_SET((p), NVC3C0, QMDV02_02, ##a) #define NVC3C0_QMDV02_02_DEF_SET(p,a...) NVDEF_MW_SET((p), NVC3C0, QMDV02_02, ##a) +#define NVC6C0_QMDV03_00_VAL_SET(p,a...) NVVAL_MW_SET((p), NVC6C0, QMDV03_00, ##a) +#define NVC6C0_QMDV03_00_DEF_SET(p,a...) NVDEF_MW_SET((p), NVC6C0, QMDV03_00, ##a) void nvk_cmd_buffer_begin_compute(struct nvk_cmd_buffer *cmd, @@ -53,6 +58,16 @@ nvc0c0_qmd_set_dispatch_size(UNUSED struct nvk_device *dev, uint32_t *qmd, NVC0C0_QMDV02_01_VAL_SET(qmd, CTA_RASTER_DEPTH, z); } +static void +nvc6c0_qmd_set_dispatch_size(UNUSED struct nvk_device *dev, uint32_t *qmd, + uint32_t x, uint32_t y, uint32_t z) +{ + NVC6C0_QMDV03_00_VAL_SET(qmd, CTA_RASTER_WIDTH, x); + NVC6C0_QMDV03_00_VAL_SET(qmd, CTA_RASTER_HEIGHT, y); + /* this field is different from older QMD versions */ + NVC6C0_QMDV03_00_VAL_SET(qmd, CTA_RASTER_DEPTH, z); +} + static uint32_t qmd_dispatch_size_offset(const struct nv_device_info *devinfo) { @@ -85,6 +100,18 @@ nvc0c0_cp_launch_desc_set_cb(uint32_t *qmd, unsigned index, NVC0C0_QMDV02_01_DEF_SET(qmd, CONSTANT_BUFFER_VALID, index, TRUE); } +static inline void +nvc6c0_cp_launch_desc_set_cb(uint32_t *qmd, unsigned index, + uint32_t size, uint64_t address) +{ + NVC6C0_QMDV03_00_VAL_SET(qmd, CONSTANT_BUFFER_ADDR_LOWER, index, address); + NVC6C0_QMDV03_00_VAL_SET(qmd, CONSTANT_BUFFER_ADDR_UPPER, index, address >> 32); + NVC6C0_QMDV03_00_VAL_SET(qmd, CONSTANT_BUFFER_SIZE_SHIFTED4, index, + DIV_ROUND_UP(size, 16)); + NVC6C0_QMDV03_00_DEF_SET(qmd, CONSTANT_BUFFER_VALID, index, TRUE); +} + + void nvk_cmd_bind_compute_pipeline(struct nvk_cmd_buffer *cmd, struct nvk_compute_pipeline *pipeline) @@ -133,7 +160,15 @@ nvk_flush_compute_state(struct nvk_cmd_buffer *cmd, memset(qmd, 0, sizeof(qmd)); memcpy(qmd, pipeline->qmd_template, sizeof(pipeline->qmd_template)); - if (dev->ctx->compute.cls >= PASCAL_COMPUTE_A) { + if (dev->ctx->compute.cls >= AMPERE_COMPUTE_A) { + nvc6c0_qmd_set_dispatch_size(nvk_cmd_buffer_device(cmd), qmd, + desc->root.cs.group_count[0], + desc->root.cs.group_count[1], + desc->root.cs.group_count[2]); + + nvc6c0_cp_launch_desc_set_cb(qmd, 0, sizeof(desc->root), root_desc_addr); + nvc6c0_cp_launch_desc_set_cb(qmd, 1, sizeof(desc->root), root_desc_addr); + } else if (dev->ctx->compute.cls >= PASCAL_COMPUTE_A) { nvc0c0_qmd_set_dispatch_size(nvk_cmd_buffer_device(cmd), qmd, desc->root.cs.group_count[0], desc->root.cs.group_count[1], @@ -205,6 +240,7 @@ nvk_CmdDispatchBase(VkCommandBuffer commandBuffer, uint32_t groupCountZ) { VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer); + const struct nvk_device *dev = nvk_cmd_buffer_device(cmd); struct nvk_descriptor_state *desc = &cmd->state.cs.descriptors; desc->root.cs.base_group[0] = baseGroupX; @@ -236,10 +272,16 @@ nvk_CmdDispatchBase(VkCommandBuffer commandBuffer, P_MTHD(p, NVA0C0, SEND_PCAS_A); P_NVA0C0_SEND_PCAS_A(p, qmd_addr >> 8); - P_IMMD(p, NVA0C0, SEND_SIGNALING_PCAS_B, { - .invalidate = INVALIDATE_TRUE, - .schedule = SCHEDULE_TRUE - }); + + if (dev->pdev->info.cls_compute <= TURING_COMPUTE_A) { + P_IMMD(p, NVA0C0, SEND_SIGNALING_PCAS_B, { + .invalidate = INVALIDATE_TRUE, + .schedule = SCHEDULE_TRUE + }); + } else { + P_IMMD(p, NVC6C0, SEND_SIGNALING_PCAS2_B, + PCAS_ACTION_INVALIDATE_COPY_SCHEDULE); + } } static void @@ -313,6 +355,7 @@ nvk_CmdDispatchIndirect(VkCommandBuffer commandBuffer, { VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer); VK_FROM_HANDLE(nvk_buffer, buffer, _buffer); + const struct nvk_device *dev = nvk_cmd_buffer_device(cmd); struct nvk_descriptor_state *desc = &cmd->state.cs.descriptors; /* TODO: Indirect dispatch pre-Turing */ @@ -348,8 +391,13 @@ nvk_CmdDispatchIndirect(VkCommandBuffer commandBuffer, P_MTHD(p, NVA0C0, SEND_PCAS_A); P_NVA0C0_SEND_PCAS_A(p, qmd_addr >> 8); - P_IMMD(p, NVA0C0, SEND_SIGNALING_PCAS_B, { - .invalidate = INVALIDATE_TRUE, - .schedule = SCHEDULE_TRUE - }); + if (dev->pdev->info.cls_compute <= TURING_COMPUTE_A) { + P_IMMD(p, NVA0C0, SEND_SIGNALING_PCAS_B, { + .invalidate = INVALIDATE_TRUE, + .schedule = SCHEDULE_TRUE + }); + } else { + P_IMMD(p, NVC6C0, SEND_SIGNALING_PCAS2_B, + PCAS_ACTION_INVALIDATE_COPY_SCHEDULE); + } } diff --git a/src/nouveau/vulkan/nvk_compute_pipeline.c b/src/nouveau/vulkan/nvk_compute_pipeline.c index 1f63776eb09..8ec0cfc6e9c 100644 --- a/src/nouveau/vulkan/nvk_compute_pipeline.c +++ b/src/nouveau/vulkan/nvk_compute_pipeline.c @@ -18,12 +18,16 @@ #include "clc0c0qmd.h" #include "clc3c0.h" #include "clc3c0qmd.h" +#include "clc6c0.h" +#include "clc6c0qmd.h" #define NVA0C0_QMDV00_06_VAL_SET(p,a...) NVVAL_MW_SET((p), NVA0C0, QMDV00_06, ##a) #define NVA0C0_QMDV00_06_DEF_SET(p,a...) NVDEF_MW_SET((p), NVA0C0, QMDV00_06, ##a) #define NVC0C0_QMDV02_01_VAL_SET(p,a...) NVVAL_MW_SET((p), NVC0C0, QMDV02_01, ##a) #define NVC0C0_QMDV02_01_DEF_SET(p,a...) NVDEF_MW_SET((p), NVC0C0, QMDV02_01, ##a) #define NVC3C0_QMDV02_02_VAL_SET(p,a...) NVVAL_MW_SET((p), NVC3C0, QMDV02_02, ##a) #define NVC3C0_QMDV02_02_DEF_SET(p,a...) NVDEF_MW_SET((p), NVC3C0, QMDV02_02, ##a) +#define NVC6C0_QMDV03_00_VAL_SET(p,a...) NVVAL_MW_SET((p), NVC6C0, QMDV03_00, ##a) +#define NVC6C0_QMDV03_00_DEF_SET(p,a...) NVDEF_MW_SET((p), NVC6C0, QMDV03_00, ##a) #define QMD_DEF_SET(qmd, class_id, version_major, version_minor, a...) \ NVDEF_MW_SET((qmd), NV##class_id, QMDV##version_major##_##version_minor, ##a) @@ -123,7 +127,28 @@ nvc3c0_compute_setup_launch_desc_template(uint32_t *qmd, uint64_t addr = nvk_shader_address(shader); NVC3C0_QMDV02_02_VAL_SET(qmd, PROGRAM_ADDRESS_LOWER, addr & 0xffffffff); NVC3C0_QMDV02_02_VAL_SET(qmd, PROGRAM_ADDRESS_UPPER, addr >> 32); +} +static void +nvc6c0_compute_setup_launch_desc_template(uint32_t *qmd, + struct nvk_shader *shader) +{ + base_compute_setup_launch_desc_template(qmd, shader, C6C0, 03, 00); + + NVC6C0_QMDV03_00_VAL_SET(qmd, SM_GLOBAL_CACHING_ENABLE, 1); + /* those are all QMD 2.2+ */ + NVC6C0_QMDV03_00_VAL_SET(qmd, MIN_SM_CONFIG_SHARED_MEM_SIZE, + gv100_sm_config_smem_size(8 * 1024)); + NVC6C0_QMDV03_00_VAL_SET(qmd, MAX_SM_CONFIG_SHARED_MEM_SIZE, + gv100_sm_config_smem_size(96 * 1024)); + NVC6C0_QMDV03_00_VAL_SET(qmd, TARGET_SM_CONFIG_SHARED_MEM_SIZE, + gv100_sm_config_smem_size(shader->cp.smem_size)); + + NVC6C0_QMDV03_00_VAL_SET(qmd, REGISTER_COUNT_V, shader->num_gprs); + + uint64_t addr = nvk_shader_address(shader); + NVC6C0_QMDV03_00_VAL_SET(qmd, PROGRAM_ADDRESS_LOWER, addr & 0xffffffff); + NVC6C0_QMDV03_00_VAL_SET(qmd, PROGRAM_ADDRESS_UPPER, addr >> 32); } VkResult @@ -177,7 +202,9 @@ nvk_compute_pipeline_create(struct nvk_device *device, goto fail; struct nvk_shader *shader = &pipeline->base.shaders[MESA_SHADER_COMPUTE]; - if (device->ctx->compute.cls >= VOLTA_COMPUTE_A) + if (device->ctx->compute.cls >= AMPERE_COMPUTE_A) + nvc6c0_compute_setup_launch_desc_template(pipeline->qmd_template, shader); + else if (device->ctx->compute.cls >= VOLTA_COMPUTE_A) nvc3c0_compute_setup_launch_desc_template(pipeline->qmd_template, shader); else if (device->ctx->compute.cls >= PASCAL_COMPUTE_A) nvc0c0_compute_setup_launch_desc_template(pipeline->qmd_template, shader);