diff --git a/src/amd/common/ac_vcn_enc.h b/src/amd/common/ac_vcn_enc.h
index eaa96e284be..a7f5bbf558f 100644
--- a/src/amd/common/ac_vcn_enc.h
+++ b/src/amd/common/ac_vcn_enc.h
@@ -534,13 +534,6 @@ typedef struct rvcn_enc_reconstructed_picture_s {
    uint32_t encode_metadata_offset; /* vcn5 only */
 } rvcn_enc_reconstructed_picture_t;
 
-typedef struct rvcn_enc_picture_info_s
-{
-   bool in_use;
-   bool is_ltr;
-   uint32_t pic_num;
-} rvcn_enc_picture_info_t;
-
 typedef struct rvcn_enc_pre_encode_input_picture_s {
    union {
       struct {
diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_enc.c b/src/gallium/drivers/radeonsi/radeon_vcn_enc.c
index 732284f039f..0fc402f93ac 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_enc.c
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_enc.c
@@ -483,6 +483,12 @@ static void radeon_vcn_enc_h264_get_param(struct radeon_encoder *enc,
    enc->enc_pic.ref_idx_l0_is_ltr = pic->l0_is_long_term[0];
    enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1_list[0];
    enc->enc_pic.ref_idx_l1_is_ltr = pic->l1_is_long_term[0];
+   enc->enc_pic.enc_params.reference_picture_index =
+      pic->ref_list0[0] == PIPE_H2645_LIST_REF_INVALID_ENTRY ? 0xffffffff : pic->ref_list0[0];
+   enc->enc_pic.h264_enc_params.l1_reference_picture0_index =
+      pic->ref_list1[0] == PIPE_H2645_LIST_REF_INVALID_ENTRY ? 0xffffffff : pic->ref_list1[0];
+   enc->enc_pic.enc_params.reconstructed_picture_index = pic->dpb_curr_pic;
+   enc->enc_pic.h264_enc_params.is_reference = !pic->not_referenced;
    enc->enc_pic.not_referenced = pic->not_referenced;
    enc->enc_pic.is_idr = (pic->picture_type == PIPE_H2645_ENC_PICTURE_TYPE_IDR);
    enc->enc_pic.is_ltr = pic->is_ltr;
@@ -729,6 +735,9 @@ static void radeon_vcn_enc_hevc_get_param(struct radeon_encoder *enc,
    enc->enc_pic.ref_idx_l0 = pic->ref_idx_l0_list[0];
    enc->enc_pic.ref_idx_l1 = pic->ref_idx_l1_list[0];
    enc->enc_pic.not_referenced = pic->not_referenced;
+   enc->enc_pic.enc_params.reference_picture_index =
+      pic->ref_list0[0] == PIPE_H2645_LIST_REF_INVALID_ENTRY ? 0xffffffff : pic->ref_list0[0];
+   enc->enc_pic.enc_params.reconstructed_picture_index = pic->dpb_curr_pic;
    enc->enc_pic.is_idr = (pic->picture_type == PIPE_H2645_ENC_PICTURE_TYPE_IDR);
    radeon_vcn_enc_hevc_get_cropping_param(enc, pic);
    enc->enc_pic.general_tier_flag = pic->seq.general_tier_flag;
@@ -1190,7 +1199,7 @@ static void pre_encode_size(struct radeon_encoder *enc,
       *offset += align((pre_size * 52 + full_size) * sizeof(uint32_t), enc->alignment);
 }
 
-static int setup_dpb(struct radeon_encoder *enc)
+static int setup_dpb(struct radeon_encoder *enc, uint32_t num_reconstructed_pictures)
 {
    struct si_screen *sscreen = (struct si_screen *)enc->screen;
    bool is_h264 = u_reduce_video_profile(enc->base.profile)
@@ -1201,7 +1210,6 @@ static int setup_dpb(struct radeon_encoder *enc)
    uint32_t aligned_width = align(enc->base.width, rec_alignment);
    uint32_t aligned_height = align(enc->base.height, rec_alignment);
    uint32_t pitch = align(aligned_width, enc->alignment);
-   uint32_t num_reconstructed_pictures = enc->base.max_references + 1;
    uint32_t luma_size, chroma_size, offset;
    struct radeon_enc_pic *enc_pic = &enc->enc_pic;
    int i;
@@ -1229,11 +1237,26 @@ static int setup_dpb(struct radeon_encoder *enc)
    if (sscreen->info.vcn_ip_version < VCN_5_0_0) {
       enc_pic->ctx_buf.rec_chroma_pitch = pitch;
       enc_pic->ctx_buf.pre_encode_picture_chroma_pitch = pitch;
+      if (has_b) {
+         enc_pic->ctx_buf.colloc_buffer_offset = offset;
+         offset += total_coloc_bytes;
+      } else
+         enc_pic->ctx_buf.colloc_buffer_offset = 0;
+
       if (enc_pic->quality_modes.pre_encode_mode)
          pre_encode_size(enc, &offset);
       else
          enc_pic->ctx_buf.two_pass_search_center_map_offset = 0;
 
+      if (enc_pic->quality_modes.pre_encode_mode) {
+         enc_pic->ctx_buf.pre_encode_input_picture.rgb.red_offset = offset;
+         offset += luma_size;
+         enc_pic->ctx_buf.pre_encode_input_picture.rgb.green_offset = offset;
+         offset += luma_size;
+         enc_pic->ctx_buf.pre_encode_input_picture.rgb.blue_offset = offset;
+         offset += luma_size;
+      }
+
       if (is_av1) {
          enc_pic->ctx_buf.av1.av1_sdb_intermediate_context_offset = offset;
          offset += RENCODE_AV1_SDB_FRAME_CONTEXT_SIZE;
@@ -1256,21 +1279,6 @@ static int setup_dpb(struct radeon_encoder *enc)
                                   NULL, 0, 0, false);
       }
 
-      if (enc_pic->quality_modes.pre_encode_mode) {
-         enc_pic->ctx_buf.pre_encode_input_picture.rgb.red_offset = offset;
-         offset += luma_size;
-         enc_pic->ctx_buf.pre_encode_input_picture.rgb.green_offset = offset;
-         offset += luma_size;
-         enc_pic->ctx_buf.pre_encode_input_picture.rgb.blue_offset = offset;
-         offset += luma_size;
-      }
-
-      if (has_b) {
-         enc_pic->ctx_buf.colloc_buffer_offset = offset;
-         offset += total_coloc_bytes;
-      } else
-         enc_pic->ctx_buf.colloc_buffer_offset = 0;
-
       enc->dpb_size = offset;
    } else { /* vcn 5.0 */
       enc_pic->ctx_buf.rec_chroma_pitch = pitch / 2;
@@ -1282,6 +1290,15 @@ static int setup_dpb(struct radeon_encoder *enc)
       } else
          enc_pic->ctx_buf.av1.av1_sdb_intermediate_context_offset = 0;
 
+      if (enc_pic->quality_modes.pre_encode_mode) {
+         enc_pic->ctx_buf.pre_encode_input_picture.rgb.red_offset = offset;
+         offset += luma_size;
+         enc_pic->ctx_buf.pre_encode_input_picture.rgb.green_offset = offset;
+         offset += luma_size;
+         enc_pic->ctx_buf.pre_encode_input_picture.rgb.blue_offset = offset;
+         offset += luma_size;
+      }
+
       for (i = 0; i < num_reconstructed_pictures; i++) {
          radeon_enc_rec_offset(&enc_pic->ctx_buf.reconstructed_pictures[i],
                                &offset, luma_size, chroma_size, false);
@@ -1299,15 +1316,6 @@ static int setup_dpb(struct radeon_encoder *enc)
                                   NULL, 0, 0, false);
       }
 
-      if (enc_pic->quality_modes.pre_encode_mode) {
-         enc_pic->ctx_buf.pre_encode_input_picture.rgb.red_offset = offset;
-         offset += luma_size;
-         enc_pic->ctx_buf.pre_encode_input_picture.rgb.green_offset = offset;
-         offset += luma_size;
-         enc_pic->ctx_buf.pre_encode_input_picture.rgb.blue_offset = offset;
-         offset += luma_size;
-      }
-
       enc->dpb_size = offset;
 
       /* meta buffer*/
@@ -1334,6 +1342,8 @@ static int setup_dpb(struct radeon_encoder *enc)
       enc->metadata_size = offset;
    }
 
+   enc->dpb_slots = num_reconstructed_pictures;
+
    return offset;
 }
 
@@ -1418,11 +1428,14 @@ static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
    struct radeon_encoder *enc = (struct radeon_encoder *)encoder;
    struct si_screen *sscreen = (struct si_screen *)enc->screen;
    struct vl_video_buffer *vid_buf = (struct vl_video_buffer *)source;
+   unsigned dpb_slots = 0;
+
    enc->need_rate_control = false;
    enc->need_rc_per_pic = false;
 
    if (u_reduce_video_profile(enc->base.profile) == PIPE_VIDEO_FORMAT_MPEG4_AVC) {
       struct pipe_h264_enc_picture_desc *pic = (struct pipe_h264_enc_picture_desc *)picture;
+      dpb_slots = MAX2(pic->seq.max_num_ref_frames + 1, pic->dpb_size);
       enc->need_rate_control =
          (enc->enc_pic.rc_layer_init[0].target_bit_rate != pic->rate_ctrl[0].target_bitrate) ||
          (enc->enc_pic.rc_layer_init[0].frame_rate_num != pic->rate_ctrl[0].frame_rate_num) ||
@@ -1441,6 +1454,7 @@ static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
          (enc->enc_pic.rc_per_pic.qvbr_quality_level != pic->rate_ctrl[0].vbr_quality_factor);
    } else if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_HEVC) {
       struct pipe_h265_enc_picture_desc *pic = (struct pipe_h265_enc_picture_desc *)picture;
+      dpb_slots = MAX2(pic->pic.num_ref_idx_l0_default_active_minus1 + 2, pic->dpb_size);
       enc->need_rate_control =
          (enc->enc_pic.rc_layer_init[0].target_bit_rate != pic->rc[0].target_bitrate) ||
          (enc->enc_pic.rc_layer_init[0].frame_rate_num != pic->rc[0].frame_rate_num) ||
@@ -1482,9 +1496,11 @@ static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
    }
 
    radeon_vcn_enc_get_param(enc, picture);
+   if (u_reduce_video_profile(picture->profile) == PIPE_VIDEO_FORMAT_AV1)
+      dpb_slots = enc->base.max_references + 1;
    if (!enc->dpb) {
       enc->dpb = CALLOC_STRUCT(rvid_buffer);
-      setup_dpb(enc);
+      setup_dpb(enc, dpb_slots);
       if (!enc->dpb ||
           !si_vid_create_buffer(enc->screen, enc->dpb, enc->dpb_size, PIPE_USAGE_DEFAULT)) {
          RVID_ERR("Can't create DPB buffer.\n");
@@ -1501,6 +1517,19 @@ static void radeon_enc_begin_frame(struct pipe_video_codec *encoder,
       }
    }
 
+   if (dpb_slots > enc->dpb_slots) {
+      setup_dpb(enc, dpb_slots);
+      if (!si_vid_resize_buffer(enc->base.context, &enc->cs, enc->dpb, enc->dpb_size, NULL)) {
+         RVID_ERR("Can't resize DPB buffer.\n");
+         goto error;
+      }
+      if (sscreen->info.vcn_ip_version >= VCN_5_0_0 && enc->metadata_size &&
+          !si_vid_resize_buffer(enc->base.context, &enc->cs, enc->meta, enc->metadata_size, NULL)) {
+         RVID_ERR("Can't resize meta buffer.\n");
+         goto error;
+      }
+   }
+
    /* qp map buffer could be created here, and release at the end */
    if (enc->enc_pic.enc_qp_map.qp_map_type != RENCODE_QP_MAP_TYPE_NONE) {
       if (!enc->roi) {
diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_enc.h b/src/gallium/drivers/radeonsi/radeon_vcn_enc.h
index be60baaa9d7..796b7388f51 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_enc.h
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_enc.h
@@ -296,9 +296,9 @@ struct radeon_encoder {
    bool need_rate_control;
    bool need_rc_per_pic;
    unsigned dpb_size;
+   unsigned dpb_slots;
    unsigned roi_size;
    unsigned metadata_size;
-   rvcn_enc_picture_info_t dpb_info[RENCODE_MAX_NUM_RECONSTRUCTED_PICTURES];
    unsigned max_ltr_idx;
 
    struct pipe_context *ectx;
diff --git a/src/gallium/drivers/radeonsi/radeon_vcn_enc_1_2.c b/src/gallium/drivers/radeonsi/radeon_vcn_enc_1_2.c
index 8cce71ac21e..2e08171e2d2 100644
--- a/src/gallium/drivers/radeonsi/radeon_vcn_enc_1_2.c
+++ b/src/gallium/drivers/radeonsi/radeon_vcn_enc_1_2.c
@@ -1546,125 +1546,9 @@ static void destroy(struct radeon_encoder *enc)
    *enc->p_task_size = (enc->total_task_size);
 }
 
-static int find_ref_idx(struct radeon_encoder *enc, int pic_num, bool is_ltr)
-{
-   for (int i = 0; i < enc->base.max_references + 1; i++) {
-      if (enc->dpb_info[i].pic_num == pic_num &&
-	  enc->dpb_info[i].in_use &&
-	  enc->dpb_info[i].is_ltr == is_ltr)
-         return i;
-   }
-
-   return -1;
-}
-
-static int get_picture_storage(struct radeon_encoder *enc)
-{
-   if (enc->enc_pic.is_ltr) {
-      if (enc->enc_pic.is_idr) {
-         enc->enc_pic.ltr_idx = 0;
-         enc->max_ltr_idx = 0;
-      }
-      /*
-         find ltr with the same ltr_idx to replace
-         if this is a new ltr_idx, increase max_ltr_idx and use the normal logic to find slot
-      */
-     if (enc->enc_pic.ltr_idx <= enc->max_ltr_idx) {
-        for (int i = 0; i < enc->base.max_references + 1; i++) {
-            if (enc->dpb_info[i].in_use &&
-		enc->dpb_info[i].is_ltr &&
-		enc->enc_pic.ltr_idx == enc->dpb_info[i].pic_num) {
-               enc->dpb_info[i].in_use = false;
-               return i;
-            }
-         }
-     } else
-        enc->max_ltr_idx = enc->enc_pic.ltr_idx;
-   }
-
-   for (int i = 0; i < enc->base.max_references + 1; i++) {
-      if (!enc->dpb_info[i].in_use) {
-         memset(&(enc->dpb_info[i]), 0, sizeof(rvcn_enc_picture_info_t));
-         return i;
-      }
-   }
-
-   /* look for the oldest short term ref pic */
-   unsigned int oldest_frame_num = 0xFFFFFFFF;
-   int oldest_idx = -1;
-   for (int i = 0; i < enc->base.max_references + 1; i++)
-      if (!enc->dpb_info[i].is_ltr && enc->dpb_info[i].pic_num < oldest_frame_num) {
-         oldest_frame_num = enc->dpb_info[i].pic_num;
-         oldest_idx = i;
-      }
-
-   if (oldest_idx >= 0)
-      enc->dpb_info[oldest_idx].in_use = false;
-
-   return oldest_idx;
-}
-
-static void manage_dpb_before_encode(struct radeon_encoder *enc)
-{
-   int current_pic_idx = 0;
-
-   if (enc->enc_pic.picture_type == PIPE_H2645_ENC_PICTURE_TYPE_IDR) {
-      /* clear reference frames */
-      for (int i = 0; i < enc->base.max_references + 1; i++)
-         memset(&(enc->dpb_info[i]), 0, sizeof(rvcn_enc_picture_info_t));
-   }
-
-   current_pic_idx = get_picture_storage(enc);
-   assert(current_pic_idx >= 0);
-
-   int ref0_idx = find_ref_idx(enc, enc->enc_pic.ref_idx_l0, enc->enc_pic.ref_idx_l0_is_ltr);
-   /* B-frames only supported on VCN >= 3.0 */
-   int ref1_idx = find_ref_idx(enc, enc->enc_pic.ref_idx_l1, enc->enc_pic.ref_idx_l1_is_ltr);
-
-   assert(enc->enc_pic.picture_type != PIPE_H2645_ENC_PICTURE_TYPE_P ||
-          ref0_idx != -1);
-   assert(enc->enc_pic.picture_type != PIPE_H2645_ENC_PICTURE_TYPE_B ||
-          (ref0_idx != -1 && ref1_idx != -1));
-
-   /* In case we didn't find the reference in dpb, we have to pick
-    * some valid index to prevent GPU hang. */
-   if ((enc->enc_pic.picture_type == PIPE_H2645_ENC_PICTURE_TYPE_P ||
-        enc->enc_pic.picture_type == PIPE_H2645_ENC_PICTURE_TYPE_B) &&
-       ref0_idx == -1) {
-      RVID_ERR("Failed to find ref0 (%u).\n", enc->enc_pic.ref_idx_l0);
-      ref0_idx = (current_pic_idx + 1) % (enc->base.max_references + 1);
-   }
-
-   if (enc->enc_pic.picture_type == PIPE_H2645_ENC_PICTURE_TYPE_B && ref1_idx == -1) {
-      RVID_ERR("Failed to find ref1 (%u).\n", enc->enc_pic.ref_idx_l1);
-      ref1_idx = (current_pic_idx + 2) % (enc->base.max_references + 1);
-   }
-
-   if (!enc->enc_pic.not_referenced)
-      enc->dpb_info[current_pic_idx].in_use = true;
-
-   if (enc->enc_pic.is_ltr) {
-      enc->dpb_info[current_pic_idx].pic_num = enc->enc_pic.ltr_idx;
-      enc->dpb_info[current_pic_idx].is_ltr = true;
-   } else {
-      enc->dpb_info[current_pic_idx].pic_num = enc->enc_pic.frame_num;
-      enc->dpb_info[current_pic_idx].is_ltr = false;
-   }
-
-   if (enc->enc_pic.picture_type == PIPE_H2645_ENC_PICTURE_TYPE_IDR) {
-      enc->enc_pic.enc_params.reference_picture_index = 0xFFFFFFFF;
-      enc->enc_pic.h264_enc_params.l1_reference_picture0_index = 0xFFFFFFFF;
-   } else {
-      enc->enc_pic.enc_params.reference_picture_index = ref0_idx;
-      enc->enc_pic.h264_enc_params.l1_reference_picture0_index = ref1_idx;
-   }
-   enc->enc_pic.enc_params.reconstructed_picture_index = current_pic_idx;
-   enc->enc_pic.h264_enc_params.is_reference = !enc->enc_pic.not_referenced;
-}
-
 void radeon_enc_1_2_init(struct radeon_encoder *enc)
 {
-   enc->before_encode = manage_dpb_before_encode;
+   enc->before_encode = radeon_enc_dummy;
    enc->begin = begin;
    enc->encode = encode;
    enc->destroy = destroy;