Update VVC packet handler
Save the frame info to the media object and fetch it from there when a packet is received, update the code to better handle out-of-order packets and add support for RCE_H26X_PREPEND_SC
This commit is contained in:
Aaro Altonen
parent
5b3a70adad
commit
4bacbaf36d
|
|
@ -8,6 +8,16 @@ namespace uvgrtp {
|
|||
|
||||
namespace formats {
|
||||
|
||||
enum H266_NAL_TYPES {
|
||||
H266_PKT_FRAG = 29
|
||||
};
|
||||
|
||||
struct h266_aggregation_packet {
|
||||
uint8_t nal_header[uvgrtp::frame::HEADER_SIZE_H266_NAL];
|
||||
uvgrtp::buf_vec nalus; /* discrete NAL units */
|
||||
uvgrtp::buf_vec aggr_pkt; /* crafted aggregation packet */
|
||||
};
|
||||
|
||||
struct h266_headers {
|
||||
uint8_t nal_header[uvgrtp::frame::HEADER_SIZE_H266_NAL];
|
||||
|
||||
|
|
@ -18,6 +28,37 @@ namespace uvgrtp {
|
|||
uint8_t fu_headers[3 * uvgrtp::frame::HEADER_SIZE_H266_FU];
|
||||
};
|
||||
|
||||
typedef struct h266_info {
|
||||
/* clock reading when the first fragment is received */
|
||||
uvgrtp::clock::hrc::hrc_t sframe_time;
|
||||
|
||||
/* sequence number of the frame with s-bit */
|
||||
uint32_t s_seq;
|
||||
|
||||
/* sequence number of the frame with e-bit */
|
||||
uint32_t e_seq;
|
||||
|
||||
/* how many fragments have been received */
|
||||
size_t pkts_received;
|
||||
|
||||
/* total size of all fragments */
|
||||
size_t total_size;
|
||||
|
||||
/* map of frame's fragments,
|
||||
* allows out-of-order insertion and loop-through in order */
|
||||
std::map<uint32_t, uvgrtp::frame::rtp_frame *> fragments;
|
||||
|
||||
/* storage for fragments that require relocation */
|
||||
std::vector<uvgrtp::frame::rtp_frame *> temporary;
|
||||
} h266_info_t;
|
||||
|
||||
typedef struct {
|
||||
std::deque<uvgrtp::frame::rtp_frame *> queued;
|
||||
std::unordered_map<uint32_t, h266_info_t> frames;
|
||||
std::unordered_set<uint32_t> dropped;
|
||||
uvgrtp::rtp *rtp_ctx;
|
||||
} h266_frame_info_t;
|
||||
|
||||
class h266 : public h26x {
|
||||
public:
|
||||
h266(uvgrtp::socket *socket, uvgrtp::rtp *rtp, int flags);
|
||||
|
|
@ -42,8 +83,14 @@ namespace uvgrtp {
|
|||
* Return RTP_GENERIC_ERROR if the packet was corrupted in some way */
|
||||
static rtp_error_t packet_handler(void *arg, int flags, frame::rtp_frame **frame);
|
||||
|
||||
/* Return pointer to the internal frame info structure which is relayed to packet handler */
|
||||
h266_frame_info_t *get_h266_frame_info();
|
||||
|
||||
protected:
|
||||
rtp_error_t push_nal_unit(uint8_t *data, size_t data_len, bool more);
|
||||
|
||||
private:
|
||||
h266_frame_info_t finfo_;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -96,10 +96,16 @@ rtp_error_t uvgrtp::formats::h266::push_nal_unit(uint8_t *data, size_t data_len,
|
|||
}
|
||||
|
||||
uvgrtp::formats::h266::h266(uvgrtp::socket *socket, uvgrtp::rtp *rtp, int flags):
|
||||
h26x(socket, rtp, flags)
|
||||
h26x(socket, rtp, flags), finfo_{}
|
||||
{
|
||||
finfo_.rtp_ctx = rtp;
|
||||
}
|
||||
|
||||
uvgrtp::formats::h266::~h266()
|
||||
{
|
||||
}
|
||||
|
||||
uvgrtp::formats::h266_frame_info_t *uvgrtp::formats::h266::get_h266_frame_info()
|
||||
{
|
||||
return &finfo_;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@
|
|||
|
||||
#include "formats/h266.hh"
|
||||
|
||||
#define RTP_FRAME_MAX_DELAY 100
|
||||
#define INVALID_SEQ 0x13371338
|
||||
#define INVALID_TS 0xffffffff
|
||||
|
||||
|
|
@ -18,11 +17,10 @@
|
|||
|
||||
enum FRAG_TYPES {
|
||||
FT_INVALID = -2, /* invalid combination of S and E bits */
|
||||
FT_NOT_FRAG = -1, /* frame doesn't contain VVC fragment */
|
||||
FT_NOT_FRAG = -1, /* frame doesn't contain h266 fragment */
|
||||
FT_START = 1, /* frame contains a fragment with S bit set */
|
||||
FT_MIDDLE = 2, /* frame is fragment but not S or E fragment */
|
||||
FT_END = 3, /* frame contains a fragment with E bit set */
|
||||
FT_AP = 4 /* aggregation packet */
|
||||
FT_END = 3 /* frame contains a fragment with E bit set */
|
||||
};
|
||||
|
||||
enum NAL_TYPES {
|
||||
|
|
@ -31,38 +29,12 @@ enum NAL_TYPES {
|
|||
NT_OTHER = 0xff
|
||||
};
|
||||
|
||||
typedef std::unordered_map<uint32_t, struct vvc_info> frame_info_t;
|
||||
|
||||
struct vvc_info {
|
||||
/* clock reading when the first fragment is received */
|
||||
uvgrtp::clock::hrc::hrc_t sframe_time;
|
||||
|
||||
/* sequence number of the frame with s-bit */
|
||||
uint32_t s_seq;
|
||||
|
||||
/* sequence number of the frame with e-bit */
|
||||
uint32_t e_seq;
|
||||
|
||||
/* how many fragments have been received */
|
||||
size_t pkts_received;
|
||||
|
||||
/* total size of all fragments */
|
||||
size_t total_size;
|
||||
|
||||
/* map of frame's fragments,
|
||||
* allows out-of-order insertion and loop-through in order */
|
||||
std::map<uint16_t, uvgrtp::frame::rtp_frame *> fragments;
|
||||
};
|
||||
|
||||
static int __get_frag(uvgrtp::frame::rtp_frame *frame)
|
||||
{
|
||||
bool first_frag = frame->payload[2] & 0x80;
|
||||
bool last_frag = frame->payload[2] & 0x40;
|
||||
|
||||
if (((frame->payload[1] >> 3) & 0x1f) == 28)
|
||||
return FT_NOT_FRAG;
|
||||
|
||||
if (((frame->payload[1] >> 3) & 0x1f) != 29)
|
||||
if ((frame->payload[1] >> 3) != uvgrtp::formats::H266_PKT_FRAG)
|
||||
return FT_NOT_FRAG;
|
||||
|
||||
if (first_frag && last_frag)
|
||||
|
|
@ -88,39 +60,29 @@ static inline uint8_t __get_nal(uvgrtp::frame::rtp_frame *frame)
|
|||
return NT_OTHER;
|
||||
}
|
||||
|
||||
static inline bool __frame_late(vvc_info& hinfo)
|
||||
static inline bool __frame_late(uvgrtp::formats::h266_info_t& hinfo, size_t max_delay)
|
||||
{
|
||||
return (uvgrtp::clock::hrc::diff_now(hinfo.sframe_time) >= RTP_FRAME_MAX_DELAY);
|
||||
return (uvgrtp::clock::hrc::diff_now(hinfo.sframe_time) >= max_delay);
|
||||
}
|
||||
|
||||
/* TODO: pkt dispatcher support needed */
|
||||
static rtp_error_t __handle_ap(uvgrtp::frame::rtp_frame **out)
|
||||
static void __drop_frame(uvgrtp::formats::h266_frame_info_t *finfo, uint32_t ts)
|
||||
{
|
||||
return RTP_PKT_READY;
|
||||
}
|
||||
|
||||
static void __drop_frame(frame_info_t& finfo, uint32_t ts)
|
||||
{
|
||||
uint16_t s_seq = finfo.at(ts).s_seq;
|
||||
uint16_t e_seq = finfo.at(ts).e_seq;
|
||||
uint16_t s_seq = finfo->frames.at(ts).s_seq;
|
||||
uint16_t e_seq = finfo->frames.at(ts).e_seq;
|
||||
|
||||
LOG_INFO("Dropping frame %u, %u - %u", ts, s_seq, e_seq);
|
||||
|
||||
for (auto& fragment : finfo.at(ts).fragments)
|
||||
for (auto& fragment : finfo->frames.at(ts).fragments)
|
||||
(void)uvgrtp::frame::dealloc_frame(fragment.second);
|
||||
|
||||
finfo.erase(ts);
|
||||
finfo->frames.erase(ts);
|
||||
}
|
||||
|
||||
rtp_error_t uvgrtp::formats::h266::packet_handler(void *arg, int flags, uvgrtp::frame::rtp_frame **out)
|
||||
{
|
||||
(void)arg;
|
||||
|
||||
static frame_info_t finfo;
|
||||
static std::unordered_set<uint32_t> dropped;
|
||||
|
||||
uvgrtp::frame::rtp_frame *frame;
|
||||
bool enable_idelay = !(flags & RCE_NO_H26X_INTRA_DELAY);
|
||||
auto finfo = (uvgrtp::formats::h266_frame_info_t *)arg;
|
||||
|
||||
/* Use "intra" to keep track of intra frames
|
||||
*
|
||||
|
|
@ -136,7 +98,7 @@ rtp_error_t uvgrtp::formats::h266::packet_handler(void *arg, int flags, uvgrtp::
|
|||
* pointed to by "intra" and new intra frame shall take the place of active intra frame */
|
||||
uint32_t intra = INVALID_TS;
|
||||
|
||||
const size_t VVC_HDR_SIZE =
|
||||
const size_t H266_HDR_SIZE =
|
||||
uvgrtp::frame::HEADER_SIZE_H266_NAL +
|
||||
uvgrtp::frame::HEADER_SIZE_H266_FU;
|
||||
|
||||
|
|
@ -147,11 +109,32 @@ rtp_error_t uvgrtp::formats::h266::packet_handler(void *arg, int flags, uvgrtp::
|
|||
int frag_type = __get_frag(frame);
|
||||
uint8_t nal_type = __get_nal(frame);
|
||||
|
||||
if (frag_type == FT_AP)
|
||||
return __handle_ap(out);
|
||||
/* if (frag_type == FT_AGGR) */
|
||||
/* return __handle_ap(finfo, out); */
|
||||
|
||||
if (frag_type == FT_NOT_FRAG) {
|
||||
if (flags & RCE_H26X_PREPEND_SC) {
|
||||
uint8_t *pl = new uint8_t[(*out)->payload_len + 4];
|
||||
|
||||
if (!pl) {
|
||||
LOG_ERROR("Failed to allocate space for a start code");
|
||||
return RTP_GENERIC_ERROR;
|
||||
}
|
||||
|
||||
pl[0] = 0;
|
||||
pl[1] = 0;
|
||||
pl[2] = 0;
|
||||
pl[3] = 1;
|
||||
|
||||
std::memcpy(pl + 4, (*out)->payload, (*out)->payload_len);
|
||||
delete[] (*out)->payload;
|
||||
|
||||
(*out)->payload = pl;
|
||||
(*out)->payload_len += 4;
|
||||
}
|
||||
|
||||
if (frag_type == FT_NOT_FRAG)
|
||||
return RTP_PKT_READY;
|
||||
}
|
||||
|
||||
if (frag_type == FT_INVALID) {
|
||||
LOG_WARN("invalid frame received!");
|
||||
|
|
@ -161,10 +144,10 @@ rtp_error_t uvgrtp::formats::h266::packet_handler(void *arg, int flags, uvgrtp::
|
|||
}
|
||||
|
||||
/* initialize new frame */
|
||||
if (finfo.find(c_ts) == finfo.end()) {
|
||||
if (finfo->frames.find(c_ts) == finfo->frames.end()) {
|
||||
|
||||
/* make sure we haven't discarded the frame "c_ts" before */
|
||||
if (dropped.find(c_ts) != dropped.end()) {
|
||||
if (finfo->dropped.find(c_ts) != finfo->dropped.end()) {
|
||||
LOG_WARN("packet belonging to a dropped frame was received!");
|
||||
return RTP_GENERIC_ERROR;
|
||||
}
|
||||
|
|
@ -173,40 +156,72 @@ rtp_error_t uvgrtp::formats::h266::packet_handler(void *arg, int flags, uvgrtp::
|
|||
if (nal_type == NT_INTRA) {
|
||||
if (intra != INVALID_TS && enable_idelay) {
|
||||
__drop_frame(finfo, intra);
|
||||
dropped.insert(intra);
|
||||
finfo->dropped.insert(intra);
|
||||
}
|
||||
intra = c_ts;
|
||||
}
|
||||
|
||||
finfo[c_ts].s_seq = INVALID_SEQ;
|
||||
finfo[c_ts].e_seq = INVALID_SEQ;
|
||||
finfo->frames[c_ts].s_seq = INVALID_SEQ;
|
||||
finfo->frames[c_ts].e_seq = INVALID_SEQ;
|
||||
|
||||
if (frag_type == FT_START) finfo[c_ts].s_seq = c_seq;
|
||||
if (frag_type == FT_END) finfo[c_ts].e_seq = c_seq;
|
||||
if (frag_type == FT_START) finfo->frames[c_ts].s_seq = c_seq;
|
||||
if (frag_type == FT_END) finfo->frames[c_ts].e_seq = c_seq;
|
||||
|
||||
finfo[c_ts].sframe_time = uvgrtp::clock::hrc::now();
|
||||
finfo[c_ts].total_size = frame->payload_len - VVC_HDR_SIZE;
|
||||
finfo[c_ts].pkts_received = 1;
|
||||
finfo->frames[c_ts].sframe_time = uvgrtp::clock::hrc::now();
|
||||
finfo->frames[c_ts].total_size = frame->payload_len - H266_HDR_SIZE;
|
||||
finfo->frames[c_ts].pkts_received = 1;
|
||||
|
||||
finfo[c_ts].fragments[c_seq] = frame;
|
||||
finfo->frames[c_ts].fragments[c_seq] = frame;
|
||||
return RTP_OK;
|
||||
}
|
||||
finfo[c_ts].fragments[c_seq] = frame;
|
||||
|
||||
finfo[c_ts].pkts_received += 1;
|
||||
finfo[c_ts].total_size += (frame->payload_len - VVC_HDR_SIZE);
|
||||
finfo->frames[c_ts].pkts_received += 1;
|
||||
finfo->frames[c_ts].total_size += (frame->payload_len - H266_HDR_SIZE);
|
||||
|
||||
if (frag_type == FT_START)
|
||||
finfo[c_ts].s_seq = c_seq;
|
||||
if (frag_type == FT_START) {
|
||||
finfo->frames[c_ts].s_seq = c_seq;
|
||||
finfo->frames[c_ts].fragments[c_seq] = frame;
|
||||
|
||||
for (auto& fragment : finfo->frames[c_ts].temporary) {
|
||||
uint16_t fseq = fragment->header.seq;
|
||||
uint32_t seq = (c_seq > fseq) ? 0x10000 + fseq : fseq;
|
||||
|
||||
finfo->frames[c_ts].fragments[seq] = fragment;
|
||||
}
|
||||
finfo->frames[c_ts].temporary.clear();
|
||||
}
|
||||
|
||||
if (frag_type == FT_END)
|
||||
finfo[c_ts].e_seq = c_seq;
|
||||
finfo->frames[c_ts].e_seq = c_seq;
|
||||
|
||||
if (finfo[c_ts].s_seq != INVALID_SEQ && finfo[c_ts].e_seq != INVALID_SEQ) {
|
||||
/* Out-of-order nature poses an interesting problem when reconstructing the frame:
|
||||
* how to store the fragments such that we mustn't shuffle them around when frame reconstruction takes place?
|
||||
*
|
||||
* std::map is an option but the overflow of 16-bit sequence number counter makes that a little harder because
|
||||
* if the first few fragments of a frame are near 65535, the rest of the fragments are going to have sequence
|
||||
* numbers less than that and thus our frame reconstruction breaks.
|
||||
*
|
||||
* This can be solved by checking if current fragment's sequence is less than start fragment's sequence number
|
||||
* (overflow has occurred) and correcting the current sequence by adding 0x10000 to its value so it appears
|
||||
* in order with other fragments */
|
||||
if (frag_type != FT_START) {
|
||||
if (finfo->frames[c_ts].s_seq != INVALID_SEQ) {
|
||||
/* overflow has occurred, adjust the sequence number of current
|
||||
* fragment so it appears in order with other fragments of the frame
|
||||
*
|
||||
* Note: if the frame is huge (~94 MB), this will not work but it's not a realistic scenario */
|
||||
finfo->frames[c_ts].fragments[((finfo->frames[c_ts].s_seq > c_seq) ? 0x10000 + c_seq : c_seq)] = frame;
|
||||
} else {
|
||||
/* position for the fragment cannot be calculated so move the fragment to a temporary storage */
|
||||
finfo->frames[c_ts].temporary.push_back(frame);
|
||||
}
|
||||
}
|
||||
|
||||
if (finfo->frames[c_ts].s_seq != INVALID_SEQ && finfo->frames[c_ts].e_seq != INVALID_SEQ) {
|
||||
size_t received = 0;
|
||||
size_t fptr = 0;
|
||||
size_t s_seq = finfo[c_ts].s_seq;
|
||||
size_t e_seq = finfo[c_ts].e_seq;
|
||||
size_t s_seq = finfo->frames[c_ts].s_seq;
|
||||
size_t e_seq = finfo->frames[c_ts].e_seq;
|
||||
|
||||
if (s_seq > e_seq)
|
||||
received = 0xffff - s_seq + e_seq + 2;
|
||||
|
|
@ -214,38 +229,52 @@ rtp_error_t uvgrtp::formats::h266::packet_handler(void *arg, int flags, uvgrtp::
|
|||
received = e_seq - s_seq + 1;
|
||||
|
||||
/* we've received every fragment and the frame can be reconstructed */
|
||||
if (received == finfo[c_ts].pkts_received) {
|
||||
if (received == finfo->frames[c_ts].pkts_received) {
|
||||
|
||||
/* intra is still in progress, do not return the inter */
|
||||
if (nal_type == NT_INTER && intra != INVALID_TS && enable_idelay) {
|
||||
__drop_frame(finfo, c_ts);
|
||||
dropped.insert(c_ts);
|
||||
finfo->dropped.insert(c_ts);
|
||||
return RTP_OK;
|
||||
}
|
||||
|
||||
/* TODO: */
|
||||
uint8_t nal_header[2] = {
|
||||
(uint8_t)((frame->payload[0] & 0x81) | ((frame->payload[2] & 0x3f) << 1)),
|
||||
(uint8_t)frame->payload[1]
|
||||
};
|
||||
/* uvgrtp::frame::rtp_frame *out = uvgrtp::frame::alloc_rtp_frame(); */
|
||||
|
||||
uvgrtp::frame::rtp_frame *complete = uvgrtp::frame::alloc_rtp_frame();
|
||||
|
||||
complete->payload_len = finfo[c_ts].total_size + uvgrtp::frame::HEADER_SIZE_H266_NAL;
|
||||
complete->payload = new uint8_t[complete->payload_len];
|
||||
complete->payload_len =
|
||||
finfo->frames[c_ts].total_size
|
||||
+ uvgrtp::frame::HEADER_SIZE_H266_NAL +
|
||||
+ ((flags & RCE_H26X_PREPEND_SC) ? 4 : 0);
|
||||
|
||||
std::memcpy(&complete->header, &(*out)->header, RTP_HDR_SIZE);
|
||||
std::memcpy(complete->payload, nal_header, NAL_HDR_SIZE);
|
||||
if (!(complete->payload = new uint8_t[complete->payload_len])) {
|
||||
LOG_ERROR("Failed to allocate memory for RTP frame");
|
||||
return RTP_GENERIC_ERROR;
|
||||
}
|
||||
|
||||
if (flags & RCE_H26X_PREPEND_SC) {
|
||||
complete->payload[0] = 0;
|
||||
complete->payload[1] = 0;
|
||||
complete->payload[2] = 0;
|
||||
complete->payload[3] = 1;
|
||||
fptr += 4;
|
||||
}
|
||||
|
||||
std::memcpy(&complete->header, &(*out)->header, RTP_HDR_SIZE);
|
||||
std::memcpy(&complete->payload[fptr], nal_header, NAL_HDR_SIZE);
|
||||
|
||||
fptr += uvgrtp::frame::HEADER_SIZE_H266_NAL;
|
||||
|
||||
for (auto& fragment : finfo.at(c_ts).fragments) {
|
||||
for (auto& fragment : finfo->frames.at(c_ts).fragments) {
|
||||
std::memcpy(
|
||||
&complete->payload[fptr],
|
||||
&fragment.second->payload[VVC_HDR_SIZE],
|
||||
fragment.second->payload_len - VVC_HDR_SIZE
|
||||
&fragment.second->payload[H266_HDR_SIZE],
|
||||
fragment.second->payload_len - H266_HDR_SIZE
|
||||
);
|
||||
fptr += fragment.second->payload_len - VVC_HDR_SIZE;
|
||||
fptr += fragment.second->payload_len - H266_HDR_SIZE;
|
||||
(void)uvgrtp::frame::dealloc_frame(fragment.second);
|
||||
}
|
||||
|
||||
|
|
@ -253,15 +282,15 @@ rtp_error_t uvgrtp::formats::h266::packet_handler(void *arg, int flags, uvgrtp::
|
|||
intra = INVALID_TS;
|
||||
|
||||
*out = complete;
|
||||
finfo.erase(c_ts);
|
||||
finfo->frames.erase(c_ts);
|
||||
return RTP_PKT_READY;
|
||||
}
|
||||
}
|
||||
|
||||
if (__frame_late(finfo.at(c_ts))) {
|
||||
if (__frame_late(finfo->frames.at(c_ts), finfo->rtp_ctx->get_pkt_max_delay())) {
|
||||
if (nal_type != NT_INTRA || (nal_type == NT_INTRA && !enable_idelay)) {
|
||||
__drop_frame(finfo, c_ts);
|
||||
dropped.insert(c_ts);
|
||||
finfo->dropped.insert(c_ts);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ rtp_error_t uvgrtp::media_stream::init()
|
|||
media_ = new uvgrtp::formats::h266(socket_, rtp_, ctx_config_.flags);
|
||||
pkt_dispatcher_->install_aux_handler(
|
||||
rtp_handler_key_,
|
||||
nullptr,
|
||||
dynamic_cast<uvgrtp::formats::h266 *>(media_)->get_h266_frame_info(),
|
||||
dynamic_cast<uvgrtp::formats::h266 *>(media_)->packet_handler,
|
||||
nullptr
|
||||
);
|
||||
|
|
|
|||
Loading…
Reference in New Issue