From f3d765ed5d12b47ad92494dd71a3a2882504e76e Mon Sep 17 00:00:00 2001 From: Tomeu Vizoso Date: Wed, 23 Oct 2024 07:59:44 +0200 Subject: [PATCH] etnaviv/ml: Split V7 coefficient encoding to a new file In preparation for V8 support, which uses a completely different encoding. Reviewed-by: Philipp Zabel Part-of: --- src/gallium/drivers/etnaviv/etnaviv_ml_nn.c | 481 +----------------- src/gallium/drivers/etnaviv/etnaviv_ml_nn.h | 13 +- .../drivers/etnaviv/etnaviv_ml_nn_v7.c | 475 +++++++++++++++++ src/gallium/drivers/etnaviv/meson.build | 1 + 4 files changed, 499 insertions(+), 471 deletions(-) create mode 100644 src/gallium/drivers/etnaviv/etnaviv_ml_nn_v7.c diff --git a/src/gallium/drivers/etnaviv/etnaviv_ml_nn.c b/src/gallium/drivers/etnaviv/etnaviv_ml_nn.c index 09554e8a069..0276a93048e 100644 --- a/src/gallium/drivers/etnaviv/etnaviv_ml_nn.c +++ b/src/gallium/drivers/etnaviv/etnaviv_ml_nn.c @@ -555,9 +555,9 @@ calc_interleave_mode(unsigned tile_width, unsigned weight_height) return MIN2(mode, 2); } -static void -calc_addition_sizes(unsigned *input_width, unsigned *input_height, unsigned *input_channels, - unsigned *output_width, unsigned *output_height, unsigned *output_channels) +void +etna_ml_calc_addition_sizes(unsigned *input_width, unsigned *input_height, unsigned *input_channels, + unsigned *output_width, unsigned *output_height, unsigned *output_channels) { ML_DBG("addition input width %d channels %d\n", *input_width, *input_channels); @@ -587,8 +587,8 @@ calc_addition_sizes(unsigned *input_width, unsigned *input_height, unsigned *inp *output_channels = 1; } -static unsigned -calculate_tiling(struct etna_context *ctx, const struct etna_operation *operation, unsigned *tile_width_out, unsigned *tile_height_out) +unsigned +etna_ml_calculate_tiling(struct etna_context *ctx, const struct etna_operation *operation, unsigned *tile_width_out, unsigned *tile_height_out) { unsigned nn_input_buffer_depth = etna_ml_get_core_info(ctx)->nn_input_buffer_depth; unsigned nn_accum_buffer_depth = etna_ml_get_core_info(ctx)->nn_accum_buffer_depth; @@ -604,8 +604,8 @@ calculate_tiling(struct etna_context *ctx, const struct etna_operation *operatio unsigned interleave_mode; if (operation->addition) - calc_addition_sizes(&input_width, &input_height, &input_channels, - &output_width, &output_height, &output_channels); + etna_ml_calc_addition_sizes(&input_width, &input_height, &input_channels, + &output_width, &output_height, &output_channels); if (operation->pooling_first_pixel) { output_width *= 2; @@ -658,8 +658,8 @@ create_nn_config(struct etna_ml_subgraph *subgraph, const struct etna_operation weight_width = weight_height = 2; if (operation->addition) - calc_addition_sizes(&input_width, &input_height, &input_channels, - &output_width, &output_height, &output_channels); + etna_ml_calc_addition_sizes(&input_width, &input_height, &input_channels, + &output_width, &output_height, &output_channels); etna_bo_cpu_prep(bo, DRM_ETNA_PREP_WRITE); @@ -766,7 +766,7 @@ create_nn_config(struct etna_ml_subgraph *subgraph, const struct etna_operation } unsigned tile_x, tile_y; - unsigned superblocks = calculate_tiling(ctx, operation, &tile_x, &tile_y); + unsigned superblocks = etna_ml_calculate_tiling(ctx, operation, &tile_x, &tile_y); map->out_image_tile_x_size = tile_x; map->out_image_tile_y_size = tile_y; @@ -888,465 +888,6 @@ create_nn_config(struct etna_ml_subgraph *subgraph, const struct etna_operation return bo; } -static uint32_t calculate_bias_correction(uint8_t *weights, const struct etna_operation *operation) -{ - int32_t correction = 0; - - for (unsigned i = 0; i < operation->weight_width * operation->weight_height * operation->input_channels; i++) { - correction += (weights[i] - operation->weight_zero_point) * operation->input_zero_point; - } - - return correction; -} - - -static void -append_bits(uint32_t value, size_t size, unsigned *bits_in_buffer, uint64_t *buffer, uint32_t **dest, bool do_write) -{ - *buffer |= (uint64_t)value << *bits_in_buffer; - *bits_in_buffer += size; - if (*bits_in_buffer >= 32) { - if (do_write) - **dest = *buffer & 0xffffffff; - *dest += 1; - *buffer >>= 32; - *bits_in_buffer -= 32; - } -} - -struct wb_stream { - unsigned zero_point; - unsigned zrl_bits; - unsigned *bits_in_buffer; - uint64_t *buffer; - uint32_t **map; - bool do_write; - - unsigned accum_zeroes; -}; - -static void -wb_stream_flush_zeroes(struct wb_stream *wb_stream) -{ - if (wb_stream->accum_zeroes == 0) - return; - - append_bits(wb_stream->accum_zeroes - 1, wb_stream->zrl_bits, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); - wb_stream->accum_zeroes = 0; - append_bits(wb_stream->zero_point, 8, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); -} - -static void -wb_stream_write(struct wb_stream *wb_stream, unsigned value) -{ - unsigned max_zeroes = (1 << wb_stream->zrl_bits) - 1; - - if (wb_stream->zrl_bits == 0) { - append_bits(value, 8, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); - return; - } - - if (wb_stream->accum_zeroes == max_zeroes) { - append_bits(max_zeroes, wb_stream->zrl_bits, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); - wb_stream->accum_zeroes = 0; - append_bits(value, 8, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); - return; - } - - if (value == wb_stream->zero_point) { - wb_stream->accum_zeroes++; - return; - } - - append_bits(wb_stream->accum_zeroes, wb_stream->zrl_bits, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); - wb_stream->accum_zeroes = 0; - append_bits(value, 8, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); -} - -static unsigned -write_core_6(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned core, const struct etna_operation *operation, unsigned zrl_bits) -{ - struct pipe_context *pctx = subgraph->base.context; - unsigned nn_core_count = etna_ml_get_core_info(etna_context(pctx))->nn_core_count; - unsigned input_channels = operation->addition ? 1 : operation->input_channels; - unsigned output_channels = operation->addition ? 1 : operation->output_channels; - unsigned cores_used = MIN2(output_channels, nn_core_count); - unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); - uint8_t *input = map_resource(operation->weight_tensor); - uint32_t *biases = map_resource(operation->bias_tensor); - unsigned out_values_per_channel = operation->output_width * operation->output_height; - unsigned stride = MIN2(input_channels, 6); - unsigned superblocks = calculate_tiling(etna_context(pctx), operation, NULL, NULL); - uint8_t *weights_maps[DIV_ROUND_UP(kernels_per_core, superblocks)]; - uint32_t *initial_ptr = map; - bool do_write = initial_ptr != NULL; - uint64_t buffer = 0; - unsigned bits_in_buffer = 0; - struct wb_stream wb_stream = { - .zero_point = operation->weight_zero_point, - .zrl_bits = zrl_bits, - .bits_in_buffer = &bits_in_buffer, - .buffer = &buffer, - .map = &map, - .do_write = do_write, - }; - - ML_DBG("%s core %d zrl_bits %d\n", __func__, core, zrl_bits); - - append_bits(zrl_bits, 8, &bits_in_buffer, &buffer, &map, do_write); - append_bits(kernels_per_core, 16, &bits_in_buffer, &buffer, &map, do_write); - - for (unsigned superblock = 0; superblock < superblocks; superblock++) { - - unsigned kernels_in_superblock = DIV_ROUND_UP(kernels_per_core, superblocks); - if (superblock == superblocks - 1) - kernels_in_superblock = kernels_per_core - kernels_in_superblock * (superblocks - 1); - - for (unsigned kernel = 0; kernel < kernels_in_superblock; kernel++) { - unsigned out_channel = core * kernels_in_superblock + kernel + superblock * DIV_ROUND_UP(kernels_per_core, superblocks) * cores_used; - weights_maps[kernel] = input + out_channel * operation->weight_width * operation->weight_height * input_channels; - } - - for (unsigned block = 0; block < DIV_ROUND_UP(input_channels, stride); block++) { - for (unsigned kernel = 0; kernel < kernels_in_superblock; kernel++) { - unsigned out_channel = core * kernels_in_superblock + kernel + superblock * DIV_ROUND_UP(kernels_per_core, superblocks) * cores_used; - - if (block == 0) { - wb_stream_write(&wb_stream, weights_maps[kernel][0]); - - uint32_t corr = calculate_bias_correction(weights_maps[kernel], operation); - wb_stream_flush_zeroes(&wb_stream); - append_bits(biases[out_channel] - corr, 32, &bits_in_buffer, &buffer, &map, do_write); - - for (int i = 1; i < stride; i++) { - wb_stream_write(&wb_stream, weights_maps[kernel][i]); - } - } else { - for (int i = 0; i < stride; i++) { - if (i + block * stride < input_channels) - wb_stream_write(&wb_stream, weights_maps[kernel][i + block * stride]); - } - } - if (block == DIV_ROUND_UP(input_channels, stride) - 1) { - wb_stream_flush_zeroes(&wb_stream); - append_bits(out_values_per_channel * out_channel, 32, &bits_in_buffer, &buffer, &map, do_write); - } - } - } - } - - wb_stream_flush_zeroes(&wb_stream); - - if (bits_in_buffer > 0) - append_bits(0, 32 - bits_in_buffer, &bits_in_buffer, &buffer, &map, do_write); - - return (uint8_t *)map - (uint8_t *)initial_ptr - 1; -} - -static unsigned -write_core_interleaved(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned core, const struct etna_operation *operation, unsigned zrl_bits) -{ - struct pipe_context *pctx = subgraph->base.context; - unsigned nn_core_count = etna_ml_get_core_info(etna_context(pctx))->nn_core_count; - unsigned input_channels = operation->addition ? 1 : operation->input_channels; - unsigned output_channels = operation->addition ? 1 : operation->output_channels; - unsigned cores_used = MIN2(output_channels, nn_core_count); - unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); - uint8_t *input = map_resource(operation->weight_tensor); - uint32_t *biases = map_resource(operation->bias_tensor); - unsigned out_values_per_channel = operation->output_width * operation->output_height; - unsigned superblocks = calculate_tiling(etna_context(pctx), operation, NULL, NULL); - uint8_t (*weights_map)[input_channels][operation->weight_width][operation->weight_height] = (void *)input; - uint32_t *initial_ptr = map; - bool do_write = initial_ptr != NULL; - uint64_t buffer = 0; - unsigned bits_in_buffer = 0; - struct wb_stream wb_stream = { - .zero_point = operation->weight_zero_point, - .zrl_bits = zrl_bits, - .bits_in_buffer = &bits_in_buffer, - .buffer = &buffer, - .map = &map, - .do_write = do_write, - }; - - ML_DBG("%s core %d zrl_bits %d map %p\n", __func__, core, zrl_bits, map); - - append_bits(zrl_bits, 8, &bits_in_buffer, &buffer, &map, do_write); - append_bits(kernels_per_core, 16, &bits_in_buffer, &buffer, &map, do_write); - - for (unsigned superblock = 0; superblock < superblocks; superblock++) { - - unsigned kernels_in_superblock = DIV_ROUND_UP(kernels_per_core, superblocks); - if (superblock == superblocks - 1) - kernels_in_superblock = kernels_per_core - kernels_in_superblock * (superblocks - 1); - - for (unsigned z = 0; z < input_channels; z++) { - for (unsigned kernel = 0; kernel < kernels_in_superblock; kernel++) { - unsigned out_channel = core * kernels_in_superblock + kernel + superblock * DIV_ROUND_UP(kernels_per_core, superblocks) * cores_used; - - for (unsigned block = 0; block < DIV_ROUND_UP(operation->weight_width, 2); block++) { - unsigned stride = operation->weight_height; - if (operation->weight_height > 3) - stride = 3; - for (unsigned x = block * 2; x < (block + 1) * 2; x++ ) { - if (x >= operation->weight_width) - break; - for (unsigned y = 0; y < stride; y++) { - wb_stream_write(&wb_stream, weights_map[out_channel][z][x][y]); - if (x == 0 && y == 0 && z == 0) { - uint32_t corr = calculate_bias_correction((uint8_t *)weights_map[out_channel], operation); - wb_stream_flush_zeroes(&wb_stream); - append_bits(biases[out_channel] - corr, 32, &bits_in_buffer, &buffer, &map, do_write); - } - } - } - if (operation->weight_height > 3) { - for (unsigned x = block * 2; x < (block + 1) * 2; x++ ) { - if (x >= operation->weight_width) - break; - for (unsigned y = stride; y < operation->weight_width; y++) { - wb_stream_write(&wb_stream, weights_map[out_channel][z][x][y]); - } - } - } - } - - if (z == input_channels - 1) { - wb_stream_flush_zeroes(&wb_stream); - append_bits(out_values_per_channel * out_channel, 32, &bits_in_buffer, &buffer, &map, do_write); - } - } - if (superblock == superblocks - 1) - wb_stream_flush_zeroes(&wb_stream); - } - } - - wb_stream_flush_zeroes(&wb_stream); - - if (bits_in_buffer > 0) - append_bits(0, 32 - bits_in_buffer, &bits_in_buffer, &buffer, &map, do_write); - - return (uint8_t *)map - (uint8_t *)initial_ptr; -} - -static unsigned -write_core_sequential(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned core, const struct etna_operation *operation, unsigned zrl_bits) -{ - struct pipe_context *pctx = subgraph->base.context; - unsigned nn_core_count = etna_ml_get_core_info(etna_context(pctx))->nn_core_count; - unsigned output_channels = operation->addition ? 1 : operation->output_channels; - unsigned cores_used = MIN2(output_channels, nn_core_count); - unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); - uint8_t *input = map_resource(operation->weight_tensor); - uint32_t *biases = map_resource(operation->bias_tensor); - unsigned out_values_per_channel = operation->output_width * operation->output_height; - unsigned superblocks = calculate_tiling(etna_context(pctx), operation, NULL, NULL); - uint32_t *initial_ptr = map; - bool do_write = initial_ptr != NULL; - uint64_t buffer = 0; - unsigned bits_in_buffer = 0; - struct wb_stream wb_stream = { - .zero_point = operation->weight_zero_point, - .zrl_bits = zrl_bits, - .bits_in_buffer = &bits_in_buffer, - .buffer = &buffer, - .map = &map, - .do_write = do_write, - }; - - ML_DBG("%s core %d zrl_bits %d superblocks %d\n", __func__, core, zrl_bits, superblocks); - - append_bits(zrl_bits, 8, &bits_in_buffer, &buffer, &map, do_write); - append_bits(kernels_per_core, 16, &bits_in_buffer, &buffer, &map, do_write); - - for (unsigned superblock = 0; superblock < superblocks; superblock++) { - - unsigned kernels_in_superblock = DIV_ROUND_UP(kernels_per_core, superblocks); - if (superblock == superblocks - 1) - kernels_in_superblock = kernels_per_core - kernels_in_superblock * (superblocks - 1); - - for (unsigned kernel = 0; kernel < kernels_in_superblock; kernel++) { - unsigned out_channel = core * kernels_in_superblock + kernel + superblock * DIV_ROUND_UP(kernels_per_core, superblocks) * cores_used; - - uint8_t (*weights_map)[operation->weight_height] = (void*) input + out_channel * operation->weight_width * operation->weight_height; - - for (unsigned block = 0; block < DIV_ROUND_UP(operation->weight_width, 2); block++) { - unsigned stride = operation->weight_height; - if ((operation->depthwise || operation->input_width > 64) && \ - operation->weight_height > 3) - stride = 3; - for (unsigned x = block * 2; x < (block + 1) * 2; x++ ) { - if (x >= operation->weight_width) - break; - for (unsigned y = 0; y < stride; y++) { - - wb_stream_write(&wb_stream, weights_map[x][y]); - if (x == 0 && y == 0) { - uint32_t corr = calculate_bias_correction((uint8_t *)weights_map, operation); - wb_stream_flush_zeroes(&wb_stream); - append_bits(biases[out_channel] - corr, 32, &bits_in_buffer, &buffer, &map, do_write); - } - } - } - if ((operation->depthwise || operation->input_width > 64) && \ - operation->weight_height > 3) { - for (unsigned x = block * 2; x < (block + 1) * 2; x++ ) { - if (x >= operation->weight_width) - break; - for (unsigned y = stride; y < operation->weight_width; y++) { - wb_stream_write(&wb_stream, weights_map[x][y]); - } - } - } - } - wb_stream_flush_zeroes(&wb_stream); - if (operation->addition) - append_bits(operation->addition_offset, 32, &bits_in_buffer, &buffer, &map, do_write); - else - append_bits(out_values_per_channel * out_channel, 32, &bits_in_buffer, &buffer, &map, do_write); - } - } - - wb_stream_flush_zeroes(&wb_stream); - - if (bits_in_buffer > 0) - append_bits(0, 32 - bits_in_buffer, &bits_in_buffer, &buffer, &map, do_write); - - return (uint8_t *)map - (uint8_t *)initial_ptr - 1; -} - -static unsigned -calculate_weight_bo_size(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation) -{ - struct pipe_context *context = subgraph->base.context; - struct etna_context *ctx = etna_context(context); - unsigned nn_core_count = etna_ml_get_core_info(ctx)->nn_core_count; - unsigned header_size = ALIGN(nn_core_count * 4, 64); - unsigned input_channels = operation->addition ? 1 : operation->input_channels; - unsigned output_channels = operation->addition ? 1 : operation->output_channels; - unsigned cores_used = MIN2(output_channels, nn_core_count); - unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); - unsigned weights_size; - unsigned core_size; - unsigned core_size_aligned; - unsigned compressed_size_aligned; - - weights_size = operation->weight_width * operation->weight_height * input_channels; - core_size = 1 + 2 + (weights_size + 4 + 4) * kernels_per_core; - core_size_aligned = ALIGN(core_size, 64); - compressed_size_aligned = header_size + core_size_aligned * cores_used; - - return compressed_size_aligned; -} - -static unsigned -calculate_zrl_bits(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation) -{ - struct pipe_context *context = subgraph->base.context; - struct etna_context *ctx = etna_context(context); - unsigned nn_core_count = etna_ml_get_core_info(ctx)->nn_core_count; - unsigned max_zrl_bits = etna_ml_get_core_info(ctx)->nn_zrl_bits; - unsigned header_size = ALIGN(nn_core_count * 4, 64); - unsigned input_channels = operation->addition ? 1 : operation->input_channels; - unsigned output_channels = operation->addition ? 1 : operation->output_channels; - unsigned cores_used = MIN2(output_channels, nn_core_count); - unsigned best_compressed_size; - unsigned best_zrl_bits; - - /* These are very unlikely to have enough zeroes for compression to be useful. */ - if (operation->addition || - operation->pointwise) { - - return 0; - } - - /* This calculation can be really slow. Start from max_zrl_bits as big - * buffers will benefit the most from high zero compression. - */ - best_compressed_size = UINT_MAX; - best_zrl_bits = 0; - for (int zrl_bits = max_zrl_bits; zrl_bits >= 0; zrl_bits--) { - - unsigned compressed_size = header_size; - for (unsigned core = 0; core < cores_used; core++) { - - unsigned actual_size; - if (operation->pointwise && output_channels > 8) - actual_size = write_core_6(subgraph, NULL, core, operation, zrl_bits); - else if (input_channels > 1) - actual_size = write_core_interleaved(subgraph, NULL, core, operation, zrl_bits); - else - actual_size = write_core_sequential(subgraph, NULL, core, operation, zrl_bits); - - compressed_size += actual_size; - } - - /* If more bits don't compress further, then stop */ - if (compressed_size <= best_compressed_size) { - best_compressed_size = compressed_size; - best_zrl_bits = zrl_bits; - } else - break; - } - - return best_zrl_bits; -} - -static struct etna_bo * -create_coefficients_bo(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation, unsigned *cache_size) -{ - struct pipe_context *context = subgraph->base.context; - struct etna_context *ctx = etna_context(context); - unsigned nn_core_count = etna_ml_get_core_info(ctx)->nn_core_count; - unsigned header_size = ALIGN(nn_core_count * 4, 64); - unsigned input_channels = operation->addition ? 1 : operation->input_channels; - unsigned output_channels = operation->addition ? 1 : operation->output_channels; - unsigned cores_used = MIN2(output_channels, nn_core_count); - unsigned zrl_bits; - unsigned max_core_size = 0; - unsigned bo_size; - - bo_size = calculate_weight_bo_size(subgraph, operation); - zrl_bits = calculate_zrl_bits(subgraph, operation); - - struct etna_bo *compressed = etna_bo_new(ctx->screen->dev, - bo_size, - DRM_ETNA_GEM_CACHE_WC); - - etna_bo_cpu_prep(compressed, DRM_ETNA_PREP_WRITE); - - uint32_t *map = etna_bo_map(compressed); - memset(map, 0, bo_size); - - uint32_t *header = map; - map += header_size / 4; - - for (unsigned core = 0; core < cores_used; core++) { - - unsigned actual_size; - if (operation->pointwise && output_channels > 8) - actual_size = write_core_6(subgraph, map, core, operation, zrl_bits); - else if (input_channels > 1) - actual_size = write_core_interleaved(subgraph, map, core, operation, zrl_bits); - else - actual_size = write_core_sequential(subgraph, map, core, operation, zrl_bits); - - actual_size = ALIGN(actual_size, 64); - max_core_size = MAX2(actual_size, max_core_size); - - header[core] = actual_size; - - map += actual_size / 4; - } - - etna_bo_cpu_fini(compressed); - - *cache_size = max_core_size * cores_used; - - return compressed; -} - void etna_ml_compile_operation_nn(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation, struct etna_vip_instruction *instruction) @@ -1354,7 +895,7 @@ etna_ml_compile_operation_nn(struct etna_ml_subgraph *subgraph, const struct etn unsigned coef_cache_size; instruction->type = ETNA_JOB_TYPE_NN; - instruction->coefficients = create_coefficients_bo(subgraph, operation, &coef_cache_size); + instruction->coefficients = etna_ml_create_coeffs_v7(subgraph, operation, &coef_cache_size); struct pipe_resource *input = etna_ml_get_tensor(subgraph, operation->input_tensor); assert(input); diff --git a/src/gallium/drivers/etnaviv/etnaviv_ml_nn.h b/src/gallium/drivers/etnaviv/etnaviv_ml_nn.h index 468db300f6a..202bcc81332 100644 --- a/src/gallium/drivers/etnaviv/etnaviv_ml_nn.h +++ b/src/gallium/drivers/etnaviv/etnaviv_ml_nn.h @@ -4,6 +4,17 @@ */ #include "etnaviv_ml.h" +#include "etnaviv_context.h" + +void +etna_ml_calc_addition_sizes(unsigned *input_width, unsigned *input_height, unsigned *input_channels, + unsigned *output_width, unsigned *output_height, unsigned *output_channels); + +struct etna_bo * +etna_ml_create_coeffs_v7(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation, unsigned *cache_size); + +unsigned +etna_ml_calculate_tiling(struct etna_context *ctx, const struct etna_operation *operation, unsigned *tile_width_out, unsigned *tile_height_out); void etna_ml_lower_convolution(struct etna_ml_subgraph *subgraph, @@ -23,4 +34,4 @@ etna_ml_compile_operation_nn(struct etna_ml_subgraph *subgraph, void etna_ml_emit_operation_nn(struct etna_ml_subgraph *subgraph, struct etna_vip_instruction *operation, - unsigned idx); \ No newline at end of file + unsigned idx); diff --git a/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v7.c b/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v7.c new file mode 100644 index 00000000000..beada6a59c8 --- /dev/null +++ b/src/gallium/drivers/etnaviv/etnaviv_ml_nn_v7.c @@ -0,0 +1,475 @@ +/* + * Copyright (c) 2023-2024 Tomeu Vizoso + * SPDX-License-Identifier: MIT + */ + +#include "etnaviv_context.h" +#include "etnaviv_debug.h" +#include "etnaviv_ml_nn.h" +#include "etnaviv_screen.h" + +static void * +map_resource(struct pipe_resource *resource) +{ + return etna_bo_map(etna_resource(resource)->bo); +} + +static uint32_t +calculate_bias_correction(uint8_t *weights, const struct etna_operation *operation) +{ + int32_t correction = 0; + + for (unsigned i = 0; i < operation->weight_width * operation->weight_height * operation->input_channels; i++) { + correction += (weights[i] - operation->weight_zero_point) * operation->input_zero_point; + } + + return correction; +} + + +static void +append_bits(uint32_t value, size_t size, unsigned *bits_in_buffer, uint64_t *buffer, uint32_t **dest, bool do_write) +{ + *buffer |= (uint64_t)value << *bits_in_buffer; + *bits_in_buffer += size; + if (*bits_in_buffer >= 32) { + if (do_write) + **dest = *buffer & 0xffffffff; + *dest += 1; + *buffer >>= 32; + *bits_in_buffer -= 32; + } +} + +struct wb_stream { + unsigned zero_point; + unsigned zrl_bits; + unsigned *bits_in_buffer; + uint64_t *buffer; + uint32_t **map; + bool do_write; + + unsigned accum_zeroes; +}; + +static void +wb_stream_flush_zeroes(struct wb_stream *wb_stream) +{ + if (wb_stream->accum_zeroes == 0) + return; + + append_bits(wb_stream->accum_zeroes - 1, wb_stream->zrl_bits, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); + wb_stream->accum_zeroes = 0; + append_bits(wb_stream->zero_point, 8, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); +} + +static void +wb_stream_write(struct wb_stream *wb_stream, unsigned value) +{ + unsigned max_zeroes = (1 << wb_stream->zrl_bits) - 1; + + if (wb_stream->zrl_bits == 0) { + append_bits(value, 8, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); + return; + } + + if (wb_stream->accum_zeroes == max_zeroes) { + append_bits(max_zeroes, wb_stream->zrl_bits, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); + wb_stream->accum_zeroes = 0; + append_bits(value, 8, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); + return; + } + + if (value == wb_stream->zero_point) { + wb_stream->accum_zeroes++; + return; + } + + append_bits(wb_stream->accum_zeroes, wb_stream->zrl_bits, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); + wb_stream->accum_zeroes = 0; + append_bits(value, 8, wb_stream->bits_in_buffer, wb_stream->buffer, wb_stream->map, wb_stream->do_write); +} + +static unsigned +write_core_6(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned core, const struct etna_operation *operation, unsigned zrl_bits) +{ + struct pipe_context *pctx = subgraph->base.context; + unsigned nn_core_count = etna_ml_get_core_info(etna_context(pctx))->nn_core_count; + unsigned input_channels = operation->addition ? 1 : operation->input_channels; + unsigned output_channels = operation->addition ? 1 : operation->output_channels; + unsigned cores_used = MIN2(output_channels, nn_core_count); + unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); + uint8_t *input = map_resource(operation->weight_tensor); + uint32_t *biases = map_resource(operation->bias_tensor); + unsigned out_values_per_channel = operation->output_width * operation->output_height; + unsigned stride = MIN2(input_channels, 6); + unsigned superblocks = etna_ml_calculate_tiling(etna_context(pctx), operation, NULL, NULL); + uint8_t *weights_maps[DIV_ROUND_UP(kernels_per_core, superblocks)]; + uint32_t *initial_ptr = map; + bool do_write = initial_ptr != NULL; + uint64_t buffer = 0; + unsigned bits_in_buffer = 0; + struct wb_stream wb_stream = { + .zero_point = operation->weight_zero_point, + .zrl_bits = zrl_bits, + .bits_in_buffer = &bits_in_buffer, + .buffer = &buffer, + .map = &map, + .do_write = do_write, + }; + + ML_DBG("%s core %d zrl_bits %d\n", __func__, core, zrl_bits); + + append_bits(zrl_bits, 8, &bits_in_buffer, &buffer, &map, do_write); + append_bits(kernels_per_core, 16, &bits_in_buffer, &buffer, &map, do_write); + + for (unsigned superblock = 0; superblock < superblocks; superblock++) { + + unsigned kernels_in_superblock = DIV_ROUND_UP(kernels_per_core, superblocks); + if (superblock == superblocks - 1) + kernels_in_superblock = kernels_per_core - kernels_in_superblock * (superblocks - 1); + + for (unsigned kernel = 0; kernel < kernels_in_superblock; kernel++) { + unsigned out_channel = core * kernels_in_superblock + kernel + superblock * DIV_ROUND_UP(kernels_per_core, superblocks) * cores_used; + weights_maps[kernel] = input + out_channel * operation->weight_width * operation->weight_height * input_channels; + } + + for (unsigned block = 0; block < DIV_ROUND_UP(input_channels, stride); block++) { + for (unsigned kernel = 0; kernel < kernels_in_superblock; kernel++) { + unsigned out_channel = core * kernels_in_superblock + kernel + superblock * DIV_ROUND_UP(kernels_per_core, superblocks) * cores_used; + + if (block == 0) { + wb_stream_write(&wb_stream, weights_maps[kernel][0]); + + uint32_t corr = calculate_bias_correction(weights_maps[kernel], operation); + wb_stream_flush_zeroes(&wb_stream); + append_bits(biases[out_channel] - corr, 32, &bits_in_buffer, &buffer, &map, do_write); + + for (int i = 1; i < stride; i++) { + wb_stream_write(&wb_stream, weights_maps[kernel][i]); + } + } else { + for (int i = 0; i < stride; i++) { + if (i + block * stride < input_channels) + wb_stream_write(&wb_stream, weights_maps[kernel][i + block * stride]); + } + } + if (block == DIV_ROUND_UP(input_channels, stride) - 1) { + wb_stream_flush_zeroes(&wb_stream); + append_bits(out_values_per_channel * out_channel, 32, &bits_in_buffer, &buffer, &map, do_write); + } + } + } + } + + wb_stream_flush_zeroes(&wb_stream); + + if (bits_in_buffer > 0) + append_bits(0, 32 - bits_in_buffer, &bits_in_buffer, &buffer, &map, do_write); + + return (uint8_t *)map - (uint8_t *)initial_ptr - 1; +} + +static unsigned +write_core_interleaved(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned core, const struct etna_operation *operation, unsigned zrl_bits) +{ + struct pipe_context *pctx = subgraph->base.context; + unsigned nn_core_count = etna_ml_get_core_info(etna_context(pctx))->nn_core_count; + unsigned input_channels = operation->addition ? 1 : operation->input_channels; + unsigned output_channels = operation->addition ? 1 : operation->output_channels; + unsigned cores_used = MIN2(output_channels, nn_core_count); + unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); + uint8_t *input = map_resource(operation->weight_tensor); + uint32_t *biases = map_resource(operation->bias_tensor); + unsigned out_values_per_channel = operation->output_width * operation->output_height; + unsigned superblocks = etna_ml_calculate_tiling(etna_context(pctx), operation, NULL, NULL); + uint8_t (*weights_map)[input_channels][operation->weight_width][operation->weight_height] = (void *)input; + uint32_t *initial_ptr = map; + bool do_write = initial_ptr != NULL; + uint64_t buffer = 0; + unsigned bits_in_buffer = 0; + struct wb_stream wb_stream = { + .zero_point = operation->weight_zero_point, + .zrl_bits = zrl_bits, + .bits_in_buffer = &bits_in_buffer, + .buffer = &buffer, + .map = &map, + .do_write = do_write, + }; + + ML_DBG("%s core %d zrl_bits %d map %p\n", __func__, core, zrl_bits, map); + + append_bits(zrl_bits, 8, &bits_in_buffer, &buffer, &map, do_write); + append_bits(kernels_per_core, 16, &bits_in_buffer, &buffer, &map, do_write); + + for (unsigned superblock = 0; superblock < superblocks; superblock++) { + + unsigned kernels_in_superblock = DIV_ROUND_UP(kernels_per_core, superblocks); + if (superblock == superblocks - 1) + kernels_in_superblock = kernels_per_core - kernels_in_superblock * (superblocks - 1); + + for (unsigned z = 0; z < input_channels; z++) { + for (unsigned kernel = 0; kernel < kernels_in_superblock; kernel++) { + unsigned out_channel = core * kernels_in_superblock + kernel + superblock * DIV_ROUND_UP(kernels_per_core, superblocks) * cores_used; + + for (unsigned block = 0; block < DIV_ROUND_UP(operation->weight_width, 2); block++) { + unsigned stride = operation->weight_height; + if (operation->weight_height > 3) + stride = 3; + for (unsigned x = block * 2; x < (block + 1) * 2; x++ ) { + if (x >= operation->weight_width) + break; + for (unsigned y = 0; y < stride; y++) { + wb_stream_write(&wb_stream, weights_map[out_channel][z][x][y]); + if (x == 0 && y == 0 && z == 0) { + uint32_t corr = calculate_bias_correction((uint8_t *)weights_map[out_channel], operation); + wb_stream_flush_zeroes(&wb_stream); + append_bits(biases[out_channel] - corr, 32, &bits_in_buffer, &buffer, &map, do_write); + } + } + } + if (operation->weight_height > 3) { + for (unsigned x = block * 2; x < (block + 1) * 2; x++ ) { + if (x >= operation->weight_width) + break; + for (unsigned y = stride; y < operation->weight_width; y++) { + wb_stream_write(&wb_stream, weights_map[out_channel][z][x][y]); + } + } + } + } + + if (z == input_channels - 1) { + wb_stream_flush_zeroes(&wb_stream); + append_bits(out_values_per_channel * out_channel, 32, &bits_in_buffer, &buffer, &map, do_write); + } + } + if (superblock == superblocks - 1) + wb_stream_flush_zeroes(&wb_stream); + } + } + + wb_stream_flush_zeroes(&wb_stream); + + if (bits_in_buffer > 0) + append_bits(0, 32 - bits_in_buffer, &bits_in_buffer, &buffer, &map, do_write); + + return (uint8_t *)map - (uint8_t *)initial_ptr; +} + +static unsigned +write_core_sequential(struct etna_ml_subgraph *subgraph, uint32_t *map, unsigned core, const struct etna_operation *operation, unsigned zrl_bits) +{ + struct pipe_context *pctx = subgraph->base.context; + unsigned nn_core_count = etna_ml_get_core_info(etna_context(pctx))->nn_core_count; + unsigned output_channels = operation->addition ? 1 : operation->output_channels; + unsigned cores_used = MIN2(output_channels, nn_core_count); + unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); + uint8_t *input = map_resource(operation->weight_tensor); + uint32_t *biases = map_resource(operation->bias_tensor); + unsigned out_values_per_channel = operation->output_width * operation->output_height; + unsigned superblocks = etna_ml_calculate_tiling(etna_context(pctx), operation, NULL, NULL); + uint32_t *initial_ptr = map; + bool do_write = initial_ptr != NULL; + uint64_t buffer = 0; + unsigned bits_in_buffer = 0; + struct wb_stream wb_stream = { + .zero_point = operation->weight_zero_point, + .zrl_bits = zrl_bits, + .bits_in_buffer = &bits_in_buffer, + .buffer = &buffer, + .map = &map, + .do_write = do_write, + }; + + ML_DBG("%s core %d zrl_bits %d superblocks %d\n", __func__, core, zrl_bits, superblocks); + + append_bits(zrl_bits, 8, &bits_in_buffer, &buffer, &map, do_write); + append_bits(kernels_per_core, 16, &bits_in_buffer, &buffer, &map, do_write); + + for (unsigned superblock = 0; superblock < superblocks; superblock++) { + + unsigned kernels_in_superblock = DIV_ROUND_UP(kernels_per_core, superblocks); + if (superblock == superblocks - 1) + kernels_in_superblock = kernels_per_core - kernels_in_superblock * (superblocks - 1); + + for (unsigned kernel = 0; kernel < kernels_in_superblock; kernel++) { + unsigned out_channel = core * kernels_in_superblock + kernel + superblock * DIV_ROUND_UP(kernels_per_core, superblocks) * cores_used; + + uint8_t (*weights_map)[operation->weight_height] = (void*) input + out_channel * operation->weight_width * operation->weight_height; + + for (unsigned block = 0; block < DIV_ROUND_UP(operation->weight_width, 2); block++) { + unsigned stride = operation->weight_height; + if ((operation->depthwise || operation->input_width > 64) && \ + operation->weight_height > 3) + stride = 3; + for (unsigned x = block * 2; x < (block + 1) * 2; x++ ) { + if (x >= operation->weight_width) + break; + for (unsigned y = 0; y < stride; y++) { + + wb_stream_write(&wb_stream, weights_map[x][y]); + if (x == 0 && y == 0) { + uint32_t corr = calculate_bias_correction((uint8_t *)weights_map, operation); + wb_stream_flush_zeroes(&wb_stream); + append_bits(biases[out_channel] - corr, 32, &bits_in_buffer, &buffer, &map, do_write); + } + } + } + if ((operation->depthwise || operation->input_width > 64) && \ + operation->weight_height > 3) { + for (unsigned x = block * 2; x < (block + 1) * 2; x++ ) { + if (x >= operation->weight_width) + break; + for (unsigned y = stride; y < operation->weight_width; y++) { + wb_stream_write(&wb_stream, weights_map[x][y]); + } + } + } + } + wb_stream_flush_zeroes(&wb_stream); + if (operation->addition) + append_bits(operation->addition_offset, 32, &bits_in_buffer, &buffer, &map, do_write); + else + append_bits(out_values_per_channel * out_channel, 32, &bits_in_buffer, &buffer, &map, do_write); + } + } + + wb_stream_flush_zeroes(&wb_stream); + + if (bits_in_buffer > 0) + append_bits(0, 32 - bits_in_buffer, &bits_in_buffer, &buffer, &map, do_write); + + return (uint8_t *)map - (uint8_t *)initial_ptr - 1; +} + +static unsigned +calculate_weight_bo_size(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation) +{ + struct pipe_context *context = subgraph->base.context; + struct etna_context *ctx = etna_context(context); + unsigned nn_core_count = etna_ml_get_core_info(ctx)->nn_core_count; + unsigned header_size = ALIGN(nn_core_count * 4, 64); + unsigned input_channels = operation->addition ? 1 : operation->input_channels; + unsigned output_channels = operation->addition ? 1 : operation->output_channels; + unsigned cores_used = MIN2(output_channels, nn_core_count); + unsigned kernels_per_core = DIV_ROUND_UP(output_channels, cores_used); + unsigned weights_size; + unsigned core_size; + unsigned core_size_aligned; + unsigned compressed_size_aligned; + + weights_size = operation->weight_width * operation->weight_height * input_channels; + core_size = 1 + 2 + (weights_size + 4 + 4) * kernels_per_core; + core_size_aligned = ALIGN(core_size, 64); + compressed_size_aligned = header_size + core_size_aligned * cores_used; + + return compressed_size_aligned; +} + +static unsigned +calculate_zrl_bits(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation) +{ + struct pipe_context *context = subgraph->base.context; + struct etna_context *ctx = etna_context(context); + unsigned nn_core_count = etna_ml_get_core_info(ctx)->nn_core_count; + unsigned max_zrl_bits = etna_ml_get_core_info(ctx)->nn_zrl_bits; + unsigned header_size = ALIGN(nn_core_count * 4, 64); + unsigned input_channels = operation->addition ? 1 : operation->input_channels; + unsigned output_channels = operation->addition ? 1 : operation->output_channels; + unsigned cores_used = MIN2(output_channels, nn_core_count); + unsigned best_compressed_size; + unsigned best_zrl_bits; + + /* These are very unlikely to have enough zeroes for compression to be useful. */ + if (operation->addition || + operation->pointwise) { + + return 0; + } + + /* This calculation can be really slow. Start from max_zrl_bits as big + * buffers will benefit the most from high zero compression. + */ + best_compressed_size = UINT_MAX; + best_zrl_bits = 0; + for (int zrl_bits = max_zrl_bits; zrl_bits >= 0; zrl_bits--) { + + unsigned compressed_size = header_size; + for (unsigned core = 0; core < cores_used; core++) { + + unsigned actual_size; + if (operation->pointwise && output_channels > 8) + actual_size = write_core_6(subgraph, NULL, core, operation, zrl_bits); + else if (input_channels > 1) + actual_size = write_core_interleaved(subgraph, NULL, core, operation, zrl_bits); + else + actual_size = write_core_sequential(subgraph, NULL, core, operation, zrl_bits); + + compressed_size += actual_size; + } + + /* If more bits don't compress further, then stop */ + if (compressed_size <= best_compressed_size) { + best_compressed_size = compressed_size; + best_zrl_bits = zrl_bits; + } else + break; + } + + return best_zrl_bits; +} + +struct etna_bo * +etna_ml_create_coeffs_v7(struct etna_ml_subgraph *subgraph, const struct etna_operation *operation, unsigned *cache_size) +{ + struct pipe_context *context = subgraph->base.context; + struct etna_context *ctx = etna_context(context); + unsigned nn_core_count = etna_ml_get_core_info(ctx)->nn_core_count; + unsigned header_size = ALIGN(nn_core_count * 4, 64); + unsigned input_channels = operation->addition ? 1 : operation->input_channels; + unsigned output_channels = operation->addition ? 1 : operation->output_channels; + unsigned cores_used = MIN2(output_channels, nn_core_count); + unsigned zrl_bits; + unsigned max_core_size = 0; + unsigned bo_size; + + bo_size = calculate_weight_bo_size(subgraph, operation); + zrl_bits = calculate_zrl_bits(subgraph, operation); + + struct etna_bo *compressed = etna_bo_new(ctx->screen->dev, + bo_size, + DRM_ETNA_GEM_CACHE_WC); + + etna_bo_cpu_prep(compressed, DRM_ETNA_PREP_WRITE); + + uint32_t *map = etna_bo_map(compressed); + memset(map, 0, bo_size); + + uint32_t *header = map; + map += header_size / 4; + + for (unsigned core = 0; core < cores_used; core++) { + + unsigned actual_size; + if (operation->pointwise && output_channels > 8) + actual_size = write_core_6(subgraph, map, core, operation, zrl_bits); + else if (input_channels > 1) + actual_size = write_core_interleaved(subgraph, map, core, operation, zrl_bits); + else + actual_size = write_core_sequential(subgraph, map, core, operation, zrl_bits); + + actual_size = ALIGN(actual_size, 64); + max_core_size = MAX2(actual_size, max_core_size); + + header[core] = actual_size; + + map += actual_size / 4; + } + + etna_bo_cpu_fini(compressed); + + *cache_size = max_core_size * cores_used; + + return compressed; +} diff --git a/src/gallium/drivers/etnaviv/meson.build b/src/gallium/drivers/etnaviv/meson.build index 6adec123dec..1ba1ce859a3 100644 --- a/src/gallium/drivers/etnaviv/meson.build +++ b/src/gallium/drivers/etnaviv/meson.build @@ -34,6 +34,7 @@ files_etnaviv = files( 'etnaviv_internal.h', 'etnaviv_ml.c', 'etnaviv_ml.h', + 'etnaviv_ml_nn_v7.c', 'etnaviv_ml_nn.c', 'etnaviv_ml_nn.h', 'etnaviv_ml_tp.c',