Bug 110719

Kernel 5.0.0
Mesa 19.2.0-devel (git-629806b55b)
OpenGL renderer string: AMD Radeon (TM) RX 460 Graphics (POLARIS11, DRM 3.27.0, 5.0.0, LLVM 8.0.0)

I can reproduce the crash with this command:

`ffmpeg -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -i input.mp4 -vf 'format=rgb0,hwupload' -vcodec h264_vaapi -bf 0 output.mp4`

The input video does not have to be rgb0 format, this command just tells it that it is to demonstrate the problem. av_hwframe_transfer_get_formats() reports rgb0 is supported but clearly there's a problem. This is the backtrace for h264_vaapi:

Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
create (enc=0x555555ec73a0) at ../src/gallium/drivers/radeon/radeon_vce_52.c:188
188			RVCE_CS(enc->chroma->u.legacy.level[0].nblk_x * enc->chroma->bpe); // encRefPicChromaPitch
(gdb) bt full
#0  0x00007fffd740a407 in create (enc=0x555555ec73a0) at ../src/gallium/drivers/radeon/radeon_vce_52.c:188
        begin = 0x7fffc060102c
        sscreen = 0x555555812020
#1  0x00007fffd73bfe13 in rvce_begin_frame (encoder=0x555555ec73a0, source=0x5555561c2ea0, picture=<optimized out>) at ../src/gallium/drivers/radeon/radeon_vce.c:291
        fb = {usage = 4, res = 0x555555ed02f0}
        enc = 0x555555ec73a0
        vid_buf = 0x5555561c2ea0
        pic = <optimized out>
        need_rate_control = true
#2  0x00007fffd72d32ed in vlVaEndPicture (ctx=<optimized out>, context_id=<optimized out>) at ../src/gallium/state_trackers/va/picture.c:655
        drv = 0x55555580c6c0
        context = 0x555555ec3180
        coded_buf = 0x5555561e6360
        surf = 0x5555561f0320
        feedback = 0x555555a6eec0
        screen = <optimized out>
        supported = <optimized out>
        realloc = <optimized out>
        format = <optimized out>
#3  0x00007ffff085d820 in vaEndPicture () at /usr/lib/x86_64-linux-gnu/libva.so.2
#4  0x00007ffff6212e53 in  () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#5  0x00007ffff6213149 in  () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#6  0x00007ffff62137be in  () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#7  0x00007ffff5dbc6c7 in avcodec_encode_video2 () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#8  0x00007ffff5dbcaad in  () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#9  0x00007ffff5dbcc5a in avcodec_send_frame () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57

and for hevc_vaapi:

Thread 1 "ffmpeg" received signal SIGSEGV, Segmentation fault.
0x00007fffd7406f4f in radeon_uvd_enc_encode_params_hevc (enc=0x555555ec7040) at ../src/gallium/drivers/radeon/radeon_uvd_enc_1_1.c:972
972	   enc->enc_pic.enc_params.allowed_max_bitstream_size = enc->bs_size;
(gdb) bt full
#0  0x00007fffd7406f4f in radeon_uvd_enc_encode_params_hevc (enc=0x555555ec7040) at ../src/gallium/drivers/radeon/radeon_uvd_enc_1_1.c:972
        sscreen = 0x555555812020
#1  0x00007fffd7406f4f in encode (enc=0x555555ec7040) at ../src/gallium/drivers/radeon/radeon_uvd_enc_1_1.c:1104
#2  0x00007fffd72d3304 in vlVaEndPicture (ctx=<optimized out>, context_id=<optimized out>) at ../src/gallium/state_trackers/va/picture.c:656
        drv = 0x55555580c6c0
        context = 0x555555ec4420
        coded_buf = 0x555555ec6180
        surf = 0x5555561f0570
        feedback = 0x5555557bd0c0
        screen = <optimized out>
        supported = <optimized out>
        realloc = <optimized out>
        format = <optimized out>
#3  0x00007ffff085d820 in vaEndPicture () at /usr/lib/x86_64-linux-gnu/libva.so.2
#4  0x00007ffff6212e53 in  () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#5  0x00007ffff6213149 in  () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#6  0x00007ffff62137be in  () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#7  0x00007ffff5dbc6c7 in avcodec_encode_video2 () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#8  0x00007ffff5dbcaad in  () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57
#9  0x00007ffff5dbcc5a in avcodec_send_frame () at /usr/lib/x86_64-linux-gnu/libavcodec.so.57

It seems the driver should support this format or at least just report only the formats it actually supports.

Please note that this bug was discovered by a screen recorder implementing vaapi. Passing rgb data to the intel vaapi driver works as expected.