SRS(simple-rtmp-server)流媒体服务器源码分析--HLS切片

一、前言

        SRS流媒体服务器支持rtmp协议，但是rtmp协议仅仅支持PC直播。移动端直播需要HLS协议，HLS协议是苹果公司开发出来的，用于移动端视频直播，Android也对HLS做了友好支持。所以，SRS流媒体服务器支持rtmp协议和hls协议，满足了PC和移动端直播要求。
     HLS协议有两个关键文件：.m3u8文件和.ts文件：

.m3u8文件：播放控制文件，存放了地址和播放参数。
.ts文件：真正存储视频文件。

相关HLS协议详细说明参见：HLS协议详解。

       SRS流媒体接受到通过rtmp传输协议传输的编码格式为H264/AAC（注意：HLS协议只支持Video编码：H264；Audio编码：AAC/mp3）音视频数据，进行切片成.m3u8文件和.ts文件，存储在磁盘或者内存当中（注意：一般为了提高cpu使用率，将.m3u8和.ts文件存储在内存中）。再通过nginx分发到端（注意：nginx工作目录要和存储.m3u8路径一致）。

       HLS切片处理，其实就是ts编码的处理，通过将H264/AAC编码数据按照TS协议来分成一个一个的TS包。TS协议规定，TS首包内容为PAT（Program
Association Table 节目关联表）表，其PID为0x0；接下来为PMT（Program Map Table 节目映射表）表，其PID为0x1001。其次，视频帧PID为：0x100，音频PID为：0x101。后面TS包为实际音视频数据。

二、代码分析

SRS源码相关其他总结：
       SRS(simple-rtmp-server)流媒体服务器源码分析--系统启动

       SRS(simple-rtmp-server)流媒体服务器源码分析--RTMP消息play

       SRS(simple-rtmp-server)流媒体服务器源码分析--RTMP信息Publish

       SRS(simple-rtmp-server)流媒体服务器源码分析--HLS切片



SRS源码HLS处理框架如下：



接着SRS(simple-rtmp-server)流媒体服务器源码分析--RTMP消息play分析，在接受到rtmp信息之后，进行HLS处理。

int SrsSource::on_video_imp(SrsSharedPtrMessage* msg)

在函数中，除了转发给client和forward之外，还有HLS处理。注意：在HLS这一讲中，只分析跟video相关的代码，audio类同。

#ifdef SRS_AUTO_HLS
if ((ret = hls->on_video(msg, is_sequence_header)) != ERROR_SUCCESS) {
// apply the error strategy for hls.
// @see https://github.com/ossrs/srs/issues/264 std::string hls_error_strategy = _srs_config->get_hls_on_error(_req->vhost);
if (srs_config_hls_is_on_error_ignore(hls_error_strategy)) {
srs_warn("hls process video message failed, ignore and disable hls. ret=%d", ret);
// unpublish, ignore ret.
hls->on_unpublish();
// ignore.
ret = ERROR_SUCCESS;
} else if (srs_config_hls_is_on_error_continue(hls_error_strategy)) {
if (srs_hls_can_continue(ret, cache_sh_video, msg)) {
ret = ERROR_SUCCESS;
} else {
srs_warn("hls continue video failed. ret=%d", ret);
return ret;
}
} else {
srs_warn("hls disconnect publisher for video error. ret=%d", ret);
return ret;
}
}
#endif

进入on_video()，

int SrsHls::on_video(SrsSharedPtrMessage* shared_video, bool is_sps_pps)
{
int ret = ERROR_SUCCESS;

if (!hls_enabled) {
return ret;
}

// update the hls time, for hls_dispose.
last_update_time = srs_get_system_time_ms();

SrsSharedPtrMessage* video = shared_video->copy();
SrsAutoFree(SrsSharedPtrMessage, video);

// user can disable the sps parse to workaround when parse sps failed.
// @see https://github.com/ossrs/srs/issues/474 /*
SPS:
H.264中定义的sequence parameter sets中包括了一个图像序列的所有信息.它也是H.264的基础之一
PPS:
H.264中定义的picture parameter sets中包括了一个图像的所有切片信息.它也是H.264的基础之一
*/
if (is_sps_pps) {
codec->avc_parse_sps = _srs_config->get_parse_sps(_req->vhost);
}

sample->clear();
//检查H264编码，序列头
if ((ret = codec->video_avc_demux(video->payload, video->size, sample)) != ERROR_SUCCESS) {
srs_error("hls codec demux video failed. ret=%d", ret);
return ret;
}
srs_info("video decoded, type=%d, codec=%d, avc=%d, cts=%d, size=%d, time=%"PRId64,
sample->frame_type, codec->video_codec_id, sample->avc_packet_type, sample->cts, video->size, video->timestamp);

// ignore info frame,
// @see https://github.com/ossrs/srs/issues/288#issuecomment-69863909 if (sample->frame_type == SrsCodecVideoAVCFrameVideoInfoFrame) {
return ret;
}

if (codec->video_codec_id != SrsCodecVideoAVC) {
return ret;
}

// ignore sequence header
if (sample->frame_type == SrsCodecVideoAVCFrameKeyFrame
&& sample->avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) {
return hls_cache->on_sequence_header(muxer);
}
// rtmp抖动矫正
// TODO: FIXME: config the jitter of HLS.
if ((ret = jitter->correct(video, SrsRtmpJitterAlgorithmOFF)) != ERROR_SUCCESS) {
srs_error("rtmp jitter correct video failed. ret=%d", ret);
return ret;
}
// PAR(pixel aspect ratio):像素横纵比 方形为1，长方形小于1
// DAR(display aspect ratio):显示比例 16:9   4:3
/*
DTS:Decode Time Stamp.DTS主要是标识读入内存的字节流在什么时候开始送人解码器中进行解码
DTS主要是用于视频解码，在解码阶段使用
PTS主要用于视频同步和输出，在显示阶段使用，在没有B frame的情况下，DTS和PTS的输出顺序是一样的
*/
int64_t dts = video->timestamp * 90;
stream_dts = dts;
if ((ret = hls_cache->write_video(codec, muxer, dts, sample)) != ERROR_SUCCESS) {
srs_error("hls cache write video failed. ret=%d", ret);
return ret;
}

// pithy print message.
hls_show_mux_log();

return ret;
}

在该函数中做了几个工作：

1、获取H264编码信息SPS和PPS，注意：SPS和PPS不是每个流中都会有的，它只包含在头帧当中。

2、检测视频压缩编码格式：必须为H264，否则直接退出。

3、RTMP抖动矫正（不清楚做了些什么事，不管了）

4、HLS切片处理。

以上4点，前三点不在这里详细说明。只看HLS切片部分：

int SrsHlsCache::write_video(SrsAvcAacCodec* codec, SrsHlsMuxer* muxer, int64_t dts, SrsCodecSample* sample)
{
int ret = ERROR_SUCCESS;

// write video to cache.
if ((ret = cache->cache_video(codec, dts, sample)) != ERROR_SUCCESS) {
return ret;
}

// when segment overflow, reap if possible.
if (muxer->is_segment_overflow()) {
// do reap ts if any of:
//      a. wait keyframe and got keyframe.
//      b. always reap when not wait keyframe.
if (!muxer->wait_keyframe() || sample->frame_type == SrsCodecVideoAVCFrameKeyFrame) {
// reap the segment, which will also flush the video.
if ((ret = reap_segment("video", muxer, cache->video->dts)) != ERROR_SUCCESS) {
return ret;
}
}
}

// flush video when got one
if ((ret = muxer->flush_video(cache)) != ERROR_SUCCESS) {
srs_error("m3u8 muxer flush video failed. ret=%d", ret);
return ret;
}

return ret;
}

该函数做一下两点工作：

1、首次或者.ts文件时间溢出时，进入reap_segment()函数，该函数主要负责.m3u8和.ts文件管理（创建，打开，关闭）。注意：.m3u8文件是在ts文件写入完毕之后，在关闭操作中一次性将播放参数，ts地址等等参数写入。

2、其他时间，直接进入后面flush_video()函数，该函数负责ts流编码和.ts文件写入。

其中ts流编码设计到很多知识，大家自行查看相关知识：http://blog.csdn.net/u013354805/article/details/51578457

int SrsHlsMuxer::flush_video(SrsTsCache* cache)
{
int ret = ERROR_SUCCESS;

// if current is NULL, segment is not open, ignore the flush event.
if (!current) {
srs_warn("flush video ignored, for segment is not open.");
return ret;
}

if (!cache->video || cache->video->payload->length() <= 0) {
return ret;
}

srs_assert(current);
//更新该ts文件持续时间
// update the duration of segment.
current->update_duration(cache->video->dts);
//贴片处理
if ((ret = current->muxer->write_video(cache->video)) != ERROR_SUCCESS) {
return ret;
}

// write success, clear and free the msg
srs_freep(cache->video);

return ret;
}

进入编码函数

int SrsTsContext::encode(SrsFileWriter* writer, SrsTsMessage* msg, SrsCodecVideo vc, SrsCodecAudio ac)
{
int ret = ERROR_SUCCESS;

SrsTsStream vs, as;
int16_t video_pid = 0, audio_pid = 0;
switch (vc) {
case SrsCodecVideoAVC:
vs = SrsTsStreamVideoH264;
video_pid = TS_VIDEO_AVC_PID;
break;
case SrsCodecVideoDisabled:
vs = SrsTsStreamReserved;
break;
case SrsCodecVideoReserved:
case SrsCodecVideoReserved1:
case SrsCodecVideoReserved2:
case SrsCodecVideoSorensonH263:
case SrsCodecVideoScreenVideo:
case SrsCodecVideoOn2VP6:
case SrsCodecVideoOn2VP6WithAlphaChannel:
case SrsCodecVideoScreenVideoVersion2:
vs = SrsTsStreamReserved;
break;
}
switch (ac) {
case SrsCodecAudioAAC:
as = SrsTsStreamAudioAAC;
audio_pid = TS_AUDIO_AAC_PID;
break;
case SrsCodecAudioMP3:
as = SrsTsStreamAudioMp3;
audio_pid = TS_AUDIO_MP3_PID;
break;
case SrsCodecAudioDisabled:
as = SrsTsStreamReserved;
break;
case SrsCodecAudioReserved1:
case SrsCodecAudioLinearPCMPlatformEndian:
case SrsCodecAudioADPCM:
case SrsCodecAudioLinearPCMLittleEndian:
case SrsCodecAudioNellymoser16kHzMono:
case SrsCodecAudioNellymoser8kHzMono:
case SrsCodecAudioNellymoser:
case SrsCodecAudioReservedG711AlawLogarithmicPCM:
case SrsCodecAudioReservedG711MuLawLogarithmicPCM:
case SrsCodecAudioReserved:
case SrsCodecAudioSpeex:
case SrsCodecAudioReservedMP3_8kHz:
case SrsCodecAudioReservedDeviceSpecificSound:
as = SrsTsStreamReserved;
break;
}

if (as == SrsTsStreamReserved && vs == SrsTsStreamReserved) {
ret = ERROR_HLS_NO_STREAM;
srs_error("hls: no video or audio stream, vcodec=%d, acodec=%d. ret=%d", vc, ac, ret);
return ret;
}

// when any codec changed, write PAT/PMT table.
if (vcodec != vc || acodec != ac) {
vcodec = vc;
acodec = ac;
if ((ret = encode_pat_pmt(writer, video_pid, vs, audio_pid, as)) != ERROR_SUCCESS) {
return ret;
}
}

// encode the media frame to PES packets over TS.
if (msg->is_audio()) {
return encode_pes(writer, msg, audio_pid, as, vs == SrsTsStreamReserved);
} else {
return encode_pes(writer, msg, video_pid, vs, vs == SrsTsStreamReserved);
}
}

该函数内容有点多：
1、根据音视频类型，获取不同的PID（在TS协议当中，TS包包含了音视频数据，它们是通过TS头协议PID来区分的）

2、TS编码，PAT帧，PMT帧(首帧TS流)

3、TS编码，音视频数据编码（TS流数据部分）

可以先进入encode_pat_pmt（）函数函数中看看：

int SrsTsContext::encode_pat_pmt(SrsFileWriter* writer, int16_t vpid, SrsTsStream vs, int16_t apid, SrsTsStream as)
{
int ret = ERROR_SUCCESS;

if (vs != SrsTsStreamVideoH264 && as != SrsTsStreamAudioAAC && as != SrsTsStreamAudioMp3) {
ret = ERROR_HLS_NO_STREAM;
srs_error("hls: no pmt pcr pid, vs=%d, as=%d. ret=%d", vs, as, ret);
return ret;
}

int16_t pmt_number = TS_PMT_NUMBER;
int16_t pmt_pid = TS_PMT_PID;
if (true) {
// 创建一个PAT(节目关联表)
SrsTsPacket* pkt = SrsTsPacket::create_pat(this, pmt_number, pmt_pid);
SrsAutoFree(SrsTsPacket, pkt);
// TS包188字节，其中头字节(4字节，184个字节)。加上16字节CRC，总共204字节
char* buf = new char[SRS_TS_PACKET_SIZE];
SrsAutoFreeA(char, buf);

// set the left bytes with 0xFF.
int nb_buf = pkt->size();
srs_assert(nb_buf < SRS_TS_PACKET_SIZE);
memset(buf + nb_buf, 0xFF, SRS_TS_PACKET_SIZE - nb_buf);

SrsStream stream;
if ((ret = stream.initialize(buf, nb_buf)) != ERROR_SUCCESS) {
return ret;
}
// 编码
if ((ret = pkt->encode(&stream)) != ERROR_SUCCESS) {
srs_error("ts encode ts packet failed. ret=%d", ret);
return ret;
}
if ((ret = writer->write(buf, SRS_TS_PACKET_SIZE, NULL)) != ERROR_SUCCESS) {
srs_error("ts write ts packet failed. ret=%d", ret);
return ret;
}
}
if (true) {
//创建一个PMT
SrsTsPacket* pkt = SrsTsPacket::create_pmt(this, pmt_number, pmt_pid, vpid, vs, apid, as);
SrsAutoFree(SrsTsPacket, pkt);

char* buf = new char[SRS_TS_PACKET_SIZE];
SrsAutoFreeA(char, buf);

// set the left bytes with 0xFF.
int nb_buf = pkt->size();
srs_assert(nb_buf < SRS_TS_PACKET_SIZE);
memset(buf + nb_buf, 0xFF, SRS_TS_PACKET_SIZE - nb_buf);

SrsStream stream;
if ((ret = stream.initialize(buf, nb_buf)) != ERROR_SUCCESS) {
return ret;
}
// 编码
if ((ret = pkt->encode(&stream)) != ERROR_SUCCESS) {
srs_error("ts encode ts packet failed. ret=%d", ret);
return ret;
}

if ((ret = writer->write(buf, SRS_TS_PACKET_SIZE, NULL)) != ERROR_SUCCESS) {
srs_error("ts write ts packet failed. ret=%d", ret);
return ret;
}
}

// When PAT and PMT are writen, the context is ready now.
ready = true;

return ret;
}

上面代码创建PAT和PMT两个TS包。将TS文件前两个包解析，如下：



再来看encode_pes()函数

int SrsTsContext::encode_pes(SrsFileWriter* writer, SrsTsMessage* msg, int16_t pid, SrsTsStream sid, bool pure_audio)
{
int ret = ERROR_SUCCESS;

// Sometimes, the context is not ready(PAT/PMT write failed), error in this situation.
if (!ready) {
ret = ERROR_TS_CONTEXT_NOT_READY;
srs_error("TS: context not ready, ret=%d", ret);
return ret;
}

if (msg->payload->length() == 0) {
return ret;
}

if (sid != SrsTsStreamVideoH264 && sid != SrsTsStreamAudioMp3 && sid != SrsTsStreamAudioAAC) {
srs_info("ts: ignore the unknown stream, sid=%d", sid);
return ret;
}

SrsTsChannel* channel = get(pid);
srs_assert(channel);

char* start = msg->payload->bytes();
char* end = start + msg->payload->length();
char* p = start;

while (p < end) {
SrsTsPacket* pkt = NULL;
if (p == start) {
// write pcr according to message.
bool write_pcr = msg->write_pcr;

// for pure audio, always write pcr.
// TODO: FIXME: maybe only need to write at begin and end of ts.
if (pure_audio && msg->is_audio()) {
write_pcr = true;
}

// it's ok to set pcr equals to dts,
// @see https://github.com/ossrs/srs/issues/311 // Fig. 3.18. Program Clock Reference of Digital-Video-and-Audio-Broadcasting-Technology, page 65
// In MPEG-2, these are the "Program Clock Refer- ence" (PCR) values which are
// nothing else than an up-to-date copy of the STC counter fed into the transport
// stream at a certain time. The data stream thus carries an accurate internal
// "clock time". All coding and de- coding processes are controlled by this clock
// time. To do this, the receiver, i.e. the MPEG decoder, must read out the
// "clock time", namely the PCR values, and compare them with its own internal
// system clock, that is to say its own 42 bit counter.
int64_t pcr = write_pcr? msg->dts : -1;

// TODO: FIXME: finger it why use discontinuity of msg.
pkt = SrsTsPacket::create_pes_first(this,
pid, msg->sid, channel->continuity_counter++, msg->is_discontinuity,
pcr, msg->dts, msg->pts, msg->payload->length()
);
} else {
pkt = SrsTsPacket::create_pes_continue(this,
pid, msg->sid, channel->continuity_counter++
);
}
SrsAutoFree(SrsTsPacket, pkt);

char* buf = new char[SRS_TS_PACKET_SIZE];
SrsAutoFreeA(char, buf);

// set the left bytes with 0xFF.
int nb_buf = pkt->size();
srs_assert(nb_buf < SRS_TS_PACKET_SIZE);

int left = (int)srs_min(end - p, SRS_TS_PACKET_SIZE - nb_buf);
int nb_stuffings = SRS_TS_PACKET_SIZE - nb_buf - left;
if (nb_stuffings > 0) {
// set all bytes to stuffings.
memset(buf, 0xFF, SRS_TS_PACKET_SIZE);

// padding with stuffings.
pkt->padding(nb_stuffings);

// size changed, recalc it.
nb_buf = pkt->size();
srs_assert(nb_buf < SRS_TS_PACKET_SIZE);

left = (int)srs_min(end - p, SRS_TS_PACKET_SIZE - nb_buf);
nb_stuffings = SRS_TS_PACKET_SIZE - nb_buf - left;
srs_assert(nb_stuffings == 0);
}
memcpy(buf + nb_buf, p, left);
p += left;

SrsStream stream;
if ((ret = stream.initialize(buf, nb_buf)) != ERROR_SUCCESS) {
return ret;
}
if ((ret = pkt->encode(&stream)) != ERROR_SUCCESS) {
srs_error("ts encode ts packet failed. ret=%d", ret);
return ret;
}
if ((ret = writer->write(buf, SRS_TS_PACKET_SIZE, NULL)) != ERROR_SUCCESS) {
srs_error("ts write ts packet failed. ret=%d", ret);
return ret;
}
}

return ret;
}

编码成TS包，写入TS文件。

三、总结

      1、TS编码，实际是按照TS协议来重新整理数据，TS协议是传输协议。TS协议传输的音视频数据压缩格式还是H264/AAC。

      2、TS切片，实际是按照TS协议要求的字段长度对音视频数据进行分包处理。

      3、TS文件，TS包存储的位置。又涉及到HLS协议和TS协议关系。