diff --git a/src/gallium/frontends/va/context.c b/src/gallium/frontends/va/context.c
index b95aa7ddbd1..c142587955f 100644
--- a/src/gallium/frontends/va/context.c
+++ b/src/gallium/frontends/va/context.c
@@ -378,6 +378,7 @@ vlVaCreateContext(VADriverContextP ctx, VAConfigID config_id, int picture_width,
    if (config->entrypoint == PIPE_VIDEO_ENTRYPOINT_ENCODE) {
       switch (u_reduce_video_profile(context->templat.profile)) {
       case PIPE_VIDEO_FORMAT_MPEG4_AVC:
+         context->templat.max_references = PIPE_H264_MAX_REFERENCES;
          for (unsigned i = 0; i < ARRAY_SIZE(context->desc.h264enc.rate_ctrl); i++) {
             context->desc.h264enc.rate_ctrl[i].rate_ctrl_method = config->rc;
             context->desc.h264enc.rate_ctrl[i].vbv_buffer_size = 20000000;
@@ -392,6 +393,7 @@ vlVaCreateContext(VADriverContextP ctx, VAConfigID config_id, int picture_width,
          util_dynarray_init(&context->desc.h264enc.raw_headers, NULL);
          break;
       case PIPE_VIDEO_FORMAT_HEVC:
+         context->templat.max_references = PIPE_H265_MAX_REFERENCES;
          for (unsigned i = 0; i < ARRAY_SIZE(context->desc.h265enc.rc); i++) {
             context->desc.h265enc.rc[i].rate_ctrl_method = config->rc;
             context->desc.h265enc.rc[i].vbv_buffer_size = 20000000;
@@ -406,6 +408,7 @@ vlVaCreateContext(VADriverContextP ctx, VAConfigID config_id, int picture_width,
          util_dynarray_init(&context->desc.h265enc.raw_headers, NULL);
          break;
       case PIPE_VIDEO_FORMAT_AV1:
+         context->templat.max_references = PIPE_AV1_MAX_REFERENCES;
          for (unsigned i = 0; i < ARRAY_SIZE(context->desc.av1enc.rc); i++) {
             context->desc.av1enc.rc[i].rate_ctrl_method = config->rc;
             context->desc.av1enc.rc[i].vbv_buffer_size = 20000000;
@@ -422,6 +425,12 @@ vlVaCreateContext(VADriverContextP ctx, VAConfigID config_id, int picture_width,
          break;
       }
       context->desc.base.packed_headers = config->packed_headers;
+
+      mtx_lock(&drv->mutex);
+      context->decoder = drv->pipe->create_video_codec(drv->pipe, &context->templat);
+      mtx_unlock(&drv->mutex);
+      if (!context->decoder)
+         return VA_STATUS_ERROR_ALLOCATION_FAILED;
    }
 
    context->surfaces = _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal);
diff --git a/src/gallium/frontends/va/picture_av1_enc.c b/src/gallium/frontends/va/picture_av1_enc.c
index 5c3b746df01..5bbc36fa547 100644
--- a/src/gallium/frontends/va/picture_av1_enc.c
+++ b/src/gallium/frontends/va/picture_av1_enc.c
@@ -107,15 +107,6 @@ VAStatus vlVaHandleVAEncSequenceParameterBufferTypeAV1(vlVaDriver *drv, vlVaCont
 {
    VAEncSequenceParameterBufferAV1 *av1 = buf->data;
 
-   if (!context->decoder) {
-      context->templat.max_references = PIPE_AV1_MAX_REFERENCES;
-      context->templat.level = av1->seq_level_idx;
-      context->decoder = drv->pipe->create_video_codec(drv->pipe, &context->templat);
-
-      if (!context->decoder)
-         return VA_STATUS_ERROR_ALLOCATION_FAILED;
-   }
-
    context->desc.av1enc.seq.tier = av1->seq_tier;
    context->desc.av1enc.seq.level = av1->seq_level_idx;
    context->desc.av1enc.seq.intra_period = av1->intra_period;
diff --git a/src/gallium/frontends/va/picture_h264_enc.c b/src/gallium/frontends/va/picture_h264_enc.c
index 65c89ffcfd2..14d747430d2 100644
--- a/src/gallium/frontends/va/picture_h264_enc.c
+++ b/src/gallium/frontends/va/picture_h264_enc.c
@@ -233,27 +233,9 @@ vlVaHandleVAEncSliceParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *contex
 VAStatus
 vlVaHandleVAEncSequenceParameterBufferTypeH264(vlVaDriver *drv, vlVaContext *context, vlVaBuffer *buf)
 {
+   VAEncSequenceParameterBufferH264 *h264 = buf->data;
    uint32_t num_units_in_tick = 0, time_scale  = 0;
 
-   VAEncSequenceParameterBufferH264 *h264 = (VAEncSequenceParameterBufferH264 *)buf->data;
-   if (!context->decoder) {
-      context->templat.max_references = h264->max_num_ref_frames;
-      context->templat.level = h264->level_idc;
-      context->decoder = drv->pipe->create_video_codec(drv->pipe, &context->templat);
-      if (!context->decoder)
-         return VA_STATUS_ERROR_ALLOCATION_FAILED;
-
-      struct pipe_h264_enc_dpb_entry *dpb =
-         &context->desc.h264enc.dpb[context->desc.h264enc.dpb_curr_pic];
-      if (dpb->id && !dpb->buffer && context->decoder->create_dpb_buffer) {
-         vlVaSurface *surf = handle_table_get(drv->htab, dpb->id);
-         if (!surf)
-            return VA_STATUS_ERROR_INVALID_PARAMETER;
-         surf->buffer = context->decoder->create_dpb_buffer(context->decoder, &context->desc.base, &surf->templat);
-         dpb->buffer = surf->buffer;
-      }
-   }
-
    context->desc.h264enc.ip_period = h264->ip_period;
    context->desc.h264enc.intra_idr_period =
       h264->intra_idr_period != 0 ? h264->intra_idr_period : PIPE_DEFAULT_INTRA_IDR_PERIOD;
diff --git a/src/gallium/frontends/va/picture_hevc_enc.c b/src/gallium/frontends/va/picture_hevc_enc.c
index 29b1697f6d0..fdf53e81ba1 100644
--- a/src/gallium/frontends/va/picture_hevc_enc.c
+++ b/src/gallium/frontends/va/picture_hevc_enc.c
@@ -232,28 +232,9 @@ vlVaHandleVAEncSliceParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *contex
 VAStatus
 vlVaHandleVAEncSequenceParameterBufferTypeHEVC(vlVaDriver *drv, vlVaContext *context, vlVaBuffer *buf)
 {
-   VAEncSequenceParameterBufferHEVC *h265 = (VAEncSequenceParameterBufferHEVC *)buf->data;
+   VAEncSequenceParameterBufferHEVC *h265 = buf->data;
    uint32_t num_units_in_tick = 0, time_scale = 0;
 
-   if (!context->decoder) {
-      context->templat.max_references = PIPE_H265_MAX_NUM_LIST_REF;
-      context->templat.level = h265->general_level_idc;
-      context->decoder = drv->pipe->create_video_codec(drv->pipe, &context->templat);
-
-      if (!context->decoder)
-         return VA_STATUS_ERROR_ALLOCATION_FAILED;
-
-      struct pipe_h265_enc_dpb_entry *dpb =
-         &context->desc.h265enc.dpb[context->desc.h265enc.dpb_curr_pic];
-      if (dpb->id && !dpb->buffer && context->decoder->create_dpb_buffer) {
-         vlVaSurface *surf = handle_table_get(drv->htab, dpb->id);
-         if (!surf)
-            return VA_STATUS_ERROR_INVALID_PARAMETER;
-         surf->buffer = context->decoder->create_dpb_buffer(context->decoder, &context->desc.base, &surf->templat);
-         dpb->buffer = surf->buffer;
-      }
-   }
-
    context->desc.h265enc.seq.general_profile_idc = h265->general_profile_idc;
    context->desc.h265enc.seq.general_level_idc = h265->general_level_idc;
    context->desc.h265enc.seq.general_tier_flag = h265->general_tier_flag;