基于ffmpeg接口sws_scale抽取视频thumbnail 实例

最近在做Android上thumbnail的抽取功能，没有使用原生的Android系统自带功能而基于ffmpeg提供的API, 主要涉及一下几方面：

1. 解码视频数据 得到 YUV数据，保存在AVFrame中。

2. 使用sws_scale 接口库进行YUV数据格式转换及缩放，由于Android默认保存thumbnail使用的是RGB565格式，因此需要把YUV420p转换为RGB565，并缩放到指定的thumbnail 图像大小。

3. 对解码出来的YUV数据进行过滤，去除黑图， 采用像素Y值得平均值检测方式。

相关的示例代码程序已经放到github上了，有兴趣的朋友可以参考：

https://github.com/coloriy/thumbnail_demo

下面是具体的示例代码，可以供大家参考;

0. 初始化ffmpeg

int initFFmpegContext()
{
avcodec_register_all();
av_register_all();

return 0;
}

1. 打开文件并获取视频相关信息

int setDataSource(const char* url)
{
int ret = -1;
if (m_pFormatContext)
{
delete m_pFormatContext;
m_pFormatContext = NULL;
}

m_pFormatContext = avformat_alloc_context();
if (!m_pFormatContext)
{
return -1;
}
ret = avformat_open_input(&m_pFormatContext, url, NULL, NULL);
if (ret != 0)
{
delete m_pFormatContext;
return ret;
}

ret = avformat_find_stream_info(m_pFormatContext, NULL);
if (ret != 0)
{
delete m_pFormatContext;
return ret;
}

m_nStreamIndex[AVMEDIA_TYPE_VIDEO] = av_find_best_stream(m_pFormatContext, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0);
m_nStreamIndex[AVMEDIA_TYPE_AUDIO] = av_find_best_stream(m_pFormatContext, AVMEDIA_TYPE_AUDIO, -1, -1, NULL, 0);

return ret;
}

2. 打开解码器

int openDecoder()
{
int ret = -1;
m_pCodecContext = m_pFormatContext->streams[m_nStreamIndex[AVMEDIA_TYPE_VIDEO]]->codec;
if (m_pCodecContext)
{
m_pVideoCodec = avcodec_find_decoder(m_pCodecContext->codec_id);
ret = avcodec_open2(m_pCodecContext, m_pVideoCodec, NULL);
if (ret != 0)
{
return ret;
}
avcodec_flush_buffers(m_pCodecContext);
}
return ret;
}

3. 解码符合条件的YUV图像数据（该过程去除黑图）

int decodeOneFrame(AVFrame* pFrame)
{
    int ret = 0;

    bool frame_found = false;
    int  decoded_frame_count = 0;
    AVPacket pkt;
    do
    {
        int got_frame = 0;
        ret = av_read_frame(m_pFormatContext, &pkt);
        if (ret < 0)
        {
            break;
        }
        if (pkt.stream_index == m_nStreamIndex[AVMEDIA_TYPE_VIDEO])
        {
            ret = avcodec_decode_video2(m_pCodecContext, pFrame, &got_frame, &pkt);
            if (got_frame)
            {
                decoded_frame_count++;
                //skip black and white pitures
                uint32_t y_value = 0;
                uint32_t y_half = 0;
                uint32_t y_count = 0;
                int pixel_count = pFrame->width * pFrame->height;
                bool bHalf = false;
                for (int i = 0; i < pixel_count; i+=3)
                {
                    uint8_t y_temp = (uint8_t)(*(uint8_t*)((uint8_t*)(pFrame->data[0]) + i));
                    y_value += y_temp;
                    y_count++;

                    if (!bHalf && i > pixel_count / 6)
                    {
                        y_half = y_value / y_count;
                        bHalf = true;
                    }                    
                }

                y_value /= y_count;
                if (y_half == y_value)
                {
                    printf("decoded frame count = %d y_half=%d == y_value=%d, skip this frame!\n", decoded_frame_count, y_half, y_value);
                    continue;
                }
                if (y_value < BRIGHTNESS_VALUE && y_value > DARKNESS_VALUE)
                {
                    frame_found = true;
                    printf("frame_found = true -----------------------decoded frame count = %d\n", decoded_frame_count);
                }
            }
#ifdef SAVE_YUV_FRAME    
            char szName[128];
            sprintf(szName, "D:\\test_%d.yuv", frame_count);

            // save the yuv
            FILE *pFile = fopen(szName, "ab");
            if (pFile)
            {
                fwrite(pFrame->data[0], 1, pFrame->width * pFrame->height, pFile);
                fwrite(pFrame->data[1], 1, pFrame->width * pFrame->height * 1 / 4, pFile);
                fwrite(pFrame->data[2], 1, pFrame->width * pFrame->height * 1 / 4, pFile);
                fclose(pFile);
            }
#endif            
        }
        av_free_packet(&pkt);

    } while ((!frame_found) && (ret >= 0));

    av_free_packet(&pkt);

    return ret;
}

void closeDecoder()
{
    if (m_pVideoCodec)
    {
        avcodec_close(m_pCodecContext);
        m_pCodecContext = NULL;
    }
}

4. YUV图像数据转换为RGB565，并缩放

int getThumbnail(AVFrame* pInputFrame, AVFrame* pOutputFrame, int desW, int desH)
{
if (pInputFrame == NULL || pOutputFrame == NULL)
{
return -1;
}
SwsContext* pSwsContext = NULL;
pSwsContext = sws_getCachedContext(pSwsContext, pInputFrame->width, pInputFrame->height, (AVPixelFormat)pInputFrame->format,
desW, desH, AV_PIX_FMT_RGB565, SWS_BICUBIC, NULL, NULL, NULL);

if (pSwsContext == NULL)
{
return -1;
}

m_pThumbFrame->width = desW;
m_pThumbFrame->height = desH;
m_pThumbFrame->format = AV_PIX_FMT_RGB565;

av_frame_get_buffer(m_pThumbFrame, 16);

sws_scale(pSwsContext, pInputFrame->data, pInputFrame->linesize, 0, pInputFrame->height, m_pThumbFrame->data, m_pThumbFrame->linesize);

sws_freeContext(pSwsContext);

return 0;
}

5. 综合调用上述函数接口，并保存RGB565数据到文件。

int getFrameAt(int64_t timeUs, int width, int height)
{
int ret = -1;
AVFrame* pFrame = NULL;

ret = avformat_seek_file(m_pFormatContext, -1, INT16_MIN, timeUs, INT16_MAX, 0);

pFrame = av_frame_alloc();
m_pThumbFrame = av_frame_alloc();
ret = openDecoder();
if (ret != 0)
{
av_frame_free(&pFrame);
av_frame_free(&m_pThumbFrame);
}
ret = decodeOneFrame(pFrame);
if (ret < 0)
{
av_frame_free(&pFrame);
av_frame_free(&m_pThumbFrame);
}
ret = getThumbnail(pFrame, m_pThumbFrame, width, height);
if (ret < 0)
{
av_frame_free(&pFrame);
av_frame_free(&m_pThumbFrame);
}

// save the rgb565
FILE *pFile = fopen(strThumbFileName, "ab");
if (pFile)
{
fwrite(m_pThumbFrame->data[0], 1, m_pThumbFrame->width * m_pThumbFrame->height * 2, pFile);
fclose(pFile);
}

av_frame_free(&pFrame);
av_frame_free(&m_pThumbFrame);

return ret;
}

6. 测试程序main函数及全局变量定义

#include "stdafx.h"
extern "C"
{
#include "libavformat\avformat.h"
#include "libswscale\swscale.h"
}

AVFormatContext* m_pFormatContext = NULL;
AVCodecContext*  m_pCodecContext = NULL;
int              m_nStreamIndex[AVMEDIA_TYPE_NB] = {-1};
AVCodec*         m_pVideoCodec = NULL;
AVFrame*         m_pAVFrame = NULL;
AVFrame*         m_pThumbFrame = NULL;

const char* strInputFileName[] = {
    "D:\\testcontent\\MP4_H.264_Base_L4.0_1080P_7Mbps_25fps_AAC_LC.mp4",
    "D:\\testcontent\\MP4_H.264_BP_L1.3_480P_1.1Mbps_29.970fps_AMR_NB.mp4",
    "D:\\testcontent\\MP4_H.264_BP_L2.1_480x270_500Kbps_23.959fps_AAC_LC.mp4",
    "D:\\testcontent\\MP4_H.264_BP_L2.1_512x288_1.2Mbps_15fps_AAC_LC.mp4",
    "D:\\testcontent\\MP4_H.264_BP_L4.1_CIF_2Mbps_25fps_AAC_LC.mp4",
    "D:\\testcontent\\MP4_H.264_MP_L4.0_1080P_762Kbps_30fps_AAC_LC.mp4",
    "D:\\testcontent\\MP4_H.264_MP_L3.1_720P_2Mbps_30fps_AAC_LC.mp4",
    "D:\\testcontent\\MP4_H.264_MP_L2.2_640x320_463Kbps_25fps_AAC_LC.mp4",
    "D:\\testcontent\\MP4_H.264_MP_L3.1_720P_1Mbps_29.585fps_AAC_LC.mp4"
};
const char strThumbFileName[] = "D:\\test_thumb.rgb";
#define THUMB_WIDTH   640
#define THUMB_HEIGHT  480
#define BRIGHTNESS_VALUE 0xF0
#define DARKNESS_VALUE   0x10

main 函数

int _tmain(int argc, _TCHAR* argv[])
{
    int ret = -1;
    initFFmpegContext();

    int file_count = sizeof(strInputFileName) / sizeof(strInputFileName[0]);
    for (int i = 0; i < file_count; i++)
    {
        const char* pFileName = strInputFileName[i];
        ret = setDataSource(pFileName);

        getFrameAt(-1, THUMB_WIDTH, THUMB_HEIGHT);
    }

    //pause
    printf("finished, pause ....\n");
    getchar();

    return 0;
}

保存的RGB565图像数据使用ffmpeg命令转换为YUV420,然后使用YUVplayer播放显示， 来验证是否正确。

ffmpeg转命令：

D:\work\opensource\ffmpeg\bin>ffmpeg.exe -s 640x480 -pix_fmt rgb565le -i D:\test_thumb.rgb -s 640x480 -pix_fmt yuv420p D:\yuv_420p.yuv

使用VS2013 编译遇到问题：

1. fopen 编译错误问题：

在配置中添加：

C\C++->preprocessor 中添加宏定义： _CRT_SECURE_NO_WARNINGS;

2. release 编译出现错误

error LNK2026: module unsafe for SAFESEH image

去掉属性设置：



最终结果：

Thumbnail图片显示如下: