Tvheadend

Not all transcoding combinations are working (VAAPI).
IN + OUT
1. Decoder HW + Encoder HW --> works (with VAAPI)
2. Decoder SW + Encoder HW --> works (with VAAPI)
3. Decoder SW + Encoder SW --> works
4. Decoder HW + Encoder SW --> not working (with VAAPI)

Error:
2023-01-09 12:02:30.266 transcode: 0015: 01:MPEG2VIDEO: > Using profile webtv-h264
2023-01-09 12:02:30.267 transcode: 0015: 02:AC3: > Using profile webtv-aac
2023-01-09 12:02:31.195 libav: AVCodecContext: using SAR=40/33
2023-01-09 12:02:31.196 libav: AVCodecContext: using cpu capabilities: MMX2 SSE2Fast SSSE3 SSE4.2 AVX FMA3 BMI2 AVX2 AVX512
2023-01-09 12:02:31.197 libav: AVCodecContext: profile High, level 3.0, 4:2:0, 8-bit
2023-01-09 12:02:31.198 libav: TVHGraph: Impossible to convert between the formats supported by the filter 'Parsed_scale_vaapi_1' and the filter 'auto_scaler_0'
2023-01-09 12:02:31.198 transcode: 0015: 01:H264: [mpeg2video => libx264]: filters: failed to config filter graph

the fix is:
replace function _video_filters_get_filters() from src/transcoding/transcode/video.c with function below.

static int
_video_filters_get_filters(TVHContext *self, AVDictionary **opts, char **filters)
{
    char download[48];
    char deint[8];
    char hw_deint[64];
    char scale[24];
    char hw_scale[64];
    char upload[48];
    int ihw = _video_filters_hw_pix_fmt(self->iavctx->pix_fmt);
    int ohw = _video_filters_hw_pix_fmt(self->oavctx->pix_fmt);
    int filter_scale = (self->iavctx->height != self->oavctx->height);
    int filter_deint = 0, filter_download = 0, filter_upload = 0;

    if (tvh_context_get_int_opt(opts, "tvh_filter_deint", &filter_deint)) {
        return -1;
    }
    filter_download = (ihw && (!ohw || filter_scale || filter_deint)) ? 1 : 0;
    filter_upload = ((filter_download || !ihw) && ohw) ? 1 : 0;

    memset(deint, 0, sizeof(deint));
    memset(hw_deint, 0, sizeof(hw_deint));
#if ENABLE_HWACCELS
    if (filter_deint) {
        // when hwaccel is enabled we have two options:
        if (ihw) {
            // hw deint
            hwaccels_get_deint_filter(self->iavctx, hw_deint, sizeof(hw_deint));
        }
        else {
            // sw deint
            if (str_snprintf(deint, sizeof(deint), "yadif")) {
                return -1;
            }
        }
    }
#else
    if (filter_deint) {
        if (str_snprintf(deint, sizeof(deint), "yadif")) {
            return -1;
        }
    }
#endif

    memset(scale, 0, sizeof(scale));
    memset(hw_scale, 0, sizeof(hw_scale));
#if ENABLE_HWACCELS
    if (filter_scale) {
        // when hwaccel is enabled we have two options:
        if (ihw) {
            // hw scale
            hwaccels_get_scale_filter(self->iavctx, self->oavctx, hw_scale, sizeof(hw_scale));
        }
        else {
            // sw scale
            if (str_snprintf(scale, sizeof(scale), "scale=w=-2:h=%d",
                         self->oavctx->height)) {
                return -1;
            }
        }
    }
#else
    if (filter_scale) {
        if (str_snprintf(scale, sizeof(scale), "scale=w=-2:h=%d",
                         self->oavctx->height)) {
            return -1;
        }
    }
#endif

#if ENABLE_HWACCELS
    if (hw_deint[0] == '\0' && deint[0] == '\0' && hw_scale[0] == '\0' && scale[0] == '\0') {
        filter_download = filter_upload = 0;
    }
#else
    if (deint[0] == '\0' && scale[0] == '\0') {
        filter_download = filter_upload = 0;
    }
#endif

    memset(download, 0, sizeof(download));
    if (filter_download &&
        str_snprintf(download, sizeof(download), "hwdownload,format=pix_fmts=%s",
                     av_get_pix_fmt_name(self->iavctx->sw_pix_fmt))) {
        return -1;
    }

    memset(upload, 0, sizeof(upload));
    if (filter_upload &&
        str_snprintf(upload, sizeof(upload), "format=pix_fmts=%s|%s,hwupload",
                     av_get_pix_fmt_name(self->oavctx->sw_pix_fmt),
                     av_get_pix_fmt_name(self->oavctx->pix_fmt))) {
        return -1;
    }

    if (!(*filters = str_join(",", hw_deint, hw_scale, download, deint, scale, upload, NULL))) {
        return -1;
    }

    return 0;
}

I will file PR later.