logo标签列表

从RTP流中解码h264帧

c++ffmpegh.264rtp
4

我正在使用live555和ffmpeg库从服务器获取和解码RTP H264流；视频流是由ffmpeg编码的，使用基本配置文件和

x264_param_default_preset(m_params, "veryfast", "zerolatency")

我阅读了这篇文章，并将从网络接收到的每一帧数据中添加SPS和PPS数据；

void ClientSink::NewFrameHandler(unsigned frameSize, unsigned numTruncatedBytes,
    timeval presentationTime, unsigned durationInMicroseconds)
{
     ...
EncodedFrame tmp;
    tmp.m_frame = std::vector<unsigned char>(m_tempBuffer.data(), m_tempBuffer.data() + frameSize);
    tmp.m_duration = durationInMicroseconds;
    tmp.m_pts = presentationTime;

    //Add SPS and PPS data into the frame; TODO: some devices may send SPS and PPs data already into frame;
    tmp.m_frame.insert(tmp.m_frame.begin(), m_spsPpsData.cbegin(), m_spsPpsData.cend());

    emit newEncodedFrame( SharedEncodedFrame(tmp) );
    m_frameCounter++;

    this->continuePlaying();
}

这些帧是我在解码器中接收到的。

bool H264Decoder::decodeFrame(SharedEncodedFrame orig_frame)
{
...
while(m_packet.size > 0)
    {
        int got_picture;
        int len = avcodec_decode_video2(m_decoderContext, m_picture, &got_picture, &m_packet);
        if (len < 0)
        {
            emit criticalError(QString("Decoding error"));
            return false;
        }
        if (got_picture)
        {
            std::vector<unsigned char> result;
            this->storePicture(result);

            if ( m_picture->format == AVPixelFormat::AV_PIX_FMT_YUV420P )
            {
                //QImage img = QImage(result.data(), m_picture->width, m_picture->height, QImage::Format_RGB888);
                Frame_t result_rgb;
                if (!convert_yuv420p_to_rgb32(result, m_picture->width, m_picture->height, result_rgb))
                {
                    emit criticalError( QString("Failed to convert YUV420p image into rgb32; can't create QImage!"));
                    return false;
                }
                unsigned char* copy_img = new unsigned char[result_rgb.size()];
//this needed because QImage shared buffer, which used, and it will crash, if i use this qimage after result_rgb deleting
                std::copy(result_rgb.cbegin(), result_rgb.cend(), copy_img);
                QImage img = QImage(copy_img, m_picture->width, m_picture->height, QImage::Format_RGB32,
                [](void* array)
                {
                    delete[] array;
                }, copy_img);
                img.save(QString("123.bmp"));
                emit newDecodedFrame(img);
            }

avcodec_decode_video2可以无错误消息地解码帧，但是将其转换（从yuv420p转换为rgb32）后的解码帧是无效的。可在此链接上查看示例图像。

您有什么想法吗？我错在哪里了吗？

2个回答
6
我怀疑错误在于convert_yuv420p_to_rgb32()代码中。请尝试以下操作：
static SwsContext *m_swsCtx = NULL;
QImage frame =  QImage ( m_picture->width, m_picture->height,
                         QImage::Format_RGB32 );
m_swsCtx = sws_getCachedContext ( m_swsCtx, m_picture->width,
                                  m_picture->height, PIX_FMT_YUV420P,
                                  m_picture->width, m_picture->height,
                                  AV_PIX_FMT_RGB32, SWS_BICUBIC,
                                  NULL, NULL, NULL );
uint8_t *dstSlice[] = { frame.bits() };
int dstStride = frame.width() * 4;
sws_scale ( m_swsCtx, &m_picture.data, &m_picture.linesize,
            0, m_picture->height, dstSlice, &dstStride );

如果您还没有包含/链接swscale，您需要这样做。

注意：您不需要每帧都有SPS / PPS（只在关键帧上即可），但这也不会有害。

你的代码比我之前的版本更好。但现在帧看起来像这样 http://img-fotki.yandex.ru/get/9351/157039962.6/0_a9d9a_5a5fdaee_orig 。可能是我的错误（解码器没有接收到RTSPClient发送给它的所有编码帧）。 - Dmitry Bakhtiyarov
不确定，但我猜测这与您如何插入SPS / PPS有关。编码器每帧是否产生多个切片？如果是这样，在AU的中间插入SPS / PPS，解码器会认为帧已经结束了。 - szatmary
我不知道编码器每帧产生多少个切片（我该如何检查？）。但所有输入帧都没有被分割 - 我使用“int fragment_type = Data [0]＆0x1F;”和fragment_type不等于28。如果我在帧的开头添加SPS和PPS数据 - 解码器只解码一半的帧:(如果我不添加SPS和PPS，解码器将不解码任何帧; - Dmitry Bakhtiyarov
好的。我解决了问题。我在解码器初始化方法中添加了这一行代码：m_decoderContext->flags2 |= CODEC_FLAG2_CHUNKS; - Dmitry Bakhtiyarov
4

好的。这是我简单的示例，解码从liveMedia库收到的H264流。它可以解码被截断的帧，而不仅仅是在帧边界处。

   class H264Decoder : public AbstractDecoder
    {
        Q_OBJECT
    public:
        H264Decoder( QObject* parent = nullptr );
        virtual ~H264Decoder();

    public slots:

        virtual bool decodeFrame(SharedEncodedFrame orig_frame) override;

    signals:
        //sended when we have new decoded frame
        void newDecodedFrame( QImage img );

    protected:
        void storePicture(std::vector<unsigned char>& res);

        AVCodec* m_decoder;
        AVCodecContext* m_decoderContext;
        int m_got_picture;
        AVFrame* m_picture;
        AVPacket m_packet;
    };

这是实现：

#include "H264Decoder.hpp"
#include "ImgConverting.hpp"
#include <QPixmap>
extern "C"
{
#include <libswscale/swscale.h>
}

using namespace std;

H264Decoder::H264Decoder( QObject *parent)
    : AbstractDecoder(parent), m_decoder(nullptr), m_decoderContext(nullptr),
      m_got_picture(0), m_picture(nullptr)
{
    avcodec_register_all();
    av_init_packet(&m_packet);
    m_decoder = avcodec_find_decoder(CODEC_ID_H264);
    if (!m_decoder)
    {
        QString str = QString("Can't find H264 decoder!");
        emit criticalError(str);
    }
    m_decoderContext = avcodec_alloc_context3(m_decoder);

    if (m_decoder->capabilities & CODEC_CAP_TRUNCATED)
        m_decoderContext->flags |= CODEC_FLAG_TRUNCATED;


    //we can receive truncated frames
    m_decoderContext->flags2 |= CODEC_FLAG2_CHUNKS;
    m_decoderContext->thread_count = 4;//TODO: random value. May be changing can make decoding faster

    AVDictionary* dictionary = nullptr;
    if (avcodec_open2(m_decoderContext, m_decoder, &dictionary) < 0)
    {
        QString str = QString("Failed to open decoder!");
        emit criticalError(str);
    }
    qDebug() << "H264 Decoder successfully opened";
    m_picture = avcodec_alloc_frame();
}

H264Decoder::~H264Decoder()
{
    qDebug() << "ACHTUNG!!! H264Decoder deleted!!!\r\n\r\n";
    if (m_decoderContext)
    {
        avcodec_close(m_decoderContext);
        delete m_decoderContext;
    }
}


bool H264Decoder::decodeFrame(SharedEncodedFrame orig_frame)
{
    Frame_t enc_frame;
    orig_frame >> enc_frame;
    m_packet.size = enc_frame.size();
    m_packet.data = enc_frame.data();

    qDebug() << "H264Decoder: received encoded frame with framesize " << enc_frame.size();

    while(m_packet.size > 0)
    {
        int got_picture;
        int len = avcodec_decode_video2(m_decoderContext, m_picture, &got_picture, &m_packet);
        if (len < 0)
        {
            QString err("Decoding error");
            qDebug() << err;
            return false;
        }
        if (got_picture)
        {
            qDebug() << "H264Decoder: frame decoded!";
            std::vector<unsigned char> result;
            this->storePicture(result);

            if ( m_picture->format == PIX_FMT_YUV420P )
            {
                static SwsContext *m_swsCtx = NULL;
                QImage frame_img = QImage(m_picture->width, m_picture->height, QImage::Format_RGB888);
                m_swsCtx = sws_getCachedContext ( m_swsCtx, m_picture->width,
                    m_picture->height, PIX_FMT_YUV420P,
                    m_picture->width, m_picture->height,
                    PIX_FMT_RGB24, SWS_GAUSS,
                    NULL, NULL, NULL );
                uint8_t *dstSlice[] = { frame_img.bits() };
                int dstStride = frame_img.width() * 3;
                if (sws_scale ( m_swsCtx, m_picture->data, m_picture->linesize,
                    0, m_picture->height, dstSlice, &dstStride ) != m_picture->height )
                {
                    qDebug() << "PIZDETS!!!";
                    exit(-5);
                }
                qDebug() << "New decoded image!";
                emit newDecodedFrame(frame_img);
            }
            else if (m_picture->format == PIX_FMT_RGB32)
            {
                QImage img = QImage(result.data(), m_picture->width, m_picture->height, QImage::Format_RGB32);
                qDebug() << "New decoded image!";
                emit newDecodedFrame(img);
            }
            else if (m_picture->format == PIX_FMT_RGB24)
            {
                QImage img = QImage(result.data(), m_picture->width, m_picture->height, QImage::Format_RGB888);
                qDebug() << "New decoded image!";
                emit newDecodedFrame(img);
            }
            else
            {
                QString err = QString( "Unsupported pixel format! Can't create QImage!");
                qDebug() << err;
                emit criticalError( err );
                return false;
            }
        }
        m_packet.size -= len;
        m_packet.data += len;
    }

    return true;
}

void H264Decoder::storePicture(std::vector<unsigned char>& res)
{
    for (size_t i = 0; i < AV_NUM_DATA_POINTERS; i++)
    {
        std::copy(m_picture->data[i], m_picture->data[i] + 
            m_picture->linesize[i]*m_picture->height, std::back_inserter(res));
    }
}

我将newDecodedFrame发送到GUI线程，在某个小部件中绘制该QImage。
注：这条评论过长，无法作为评论的评论发布。
网页内容由stack overflow 提供, 点击上面的
可以查看英文原文，
原文链接