Tvheadend

Hi,
I worked through the vaapi code and found a solution for the bad picture quality in h264 and hevc encoding when using vaapi via av-lib.
The following changes needed to be done in /src/transcoding/codec/codecs/libs/vaapi.c :

For H.264:

static int
tvh_codec_profile_vaapi_h264_open(tvh_codec_profile_vaapi_t *self,
                                  AVDictionary **opts)
{
    // bit_rate or qp
    if (self->bit_rate) {
        AV_DICT_SET_BIT_RATE(opts, self->bit_rate);
        AV_DICT_SET_INT(opts, "maxrate", (self->bit_rate) * 1000, AV_DICT_DONT_OVERWRITE);
        AV_DICT_SET_INT(opts, "bufsize", ((self->bit_rate) * 1000) * 2, AV_DICT_DONT_OVERWRITE);
        AV_DICT_SET(opts, "force_key_frames", "expr:gte(t,n_forced*3)", AV_DICT_DONT_OVERWRITE);
    }
    else {
        AV_DICT_SET_QP(opts, self->qp, 20);
    }
    AV_DICT_SET_INT(opts, "quality", self->quality, 0);
    return 0;
}

For HEVC:

static int
tvh_codec_profile_vaapi_hevc_open(tvh_codec_profile_vaapi_t *self,
                                  AVDictionary **opts)
{
    // bit_rate or qp
    if (self->bit_rate) {
        AV_DICT_SET_BIT_RATE(opts, self->bit_rate);
        AV_DICT_SET_INT(opts, "maxrate", (self->bit_rate) * 1000, AV_DICT_DONT_OVERWRITE);
        AV_DICT_SET_INT(opts, "bufsize", ((self->bit_rate) * 1000) * 2, AV_DICT_DONT_OVERWRITE);
        AV_DICT_SET(opts, "force_key_frames", "expr:gte(t,n_forced*3)", AV_DICT_DONT_OVERWRITE);
    }
    else {
        AV_DICT_SET_QP(opts, self->qp, 25);
    }
    // max_b_frames
    // XXX: remove when b-frames handling in vaapi_encode is fixed
    AV_DICT_SET_INT(opts, "bf", 0, 0);

    return 0;
}

In both functions the following lines needed to be added:

        AV_DICT_SET_INT(opts, "maxrate", (self->bit_rate) * 1000, AV_DICT_DONT_OVERWRITE);
        AV_DICT_SET_INT(opts, "bufsize", ((self->bit_rate) * 1000) * 2, AV_DICT_DONT_OVERWRITE);
        AV_DICT_SET(opts, "force_key_frames", "expr:gte(t,n_forced*3)", AV_DICT_DONT_OVERWRITE);

After this change picture quality is very good from the first second on.
Because I an not having a git account I cannot commit those changes. Maybe Jaroslav or someone else can do this.

Best regards
Stefan