qiurui
/
uvgrtp-base
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

formats: Fix dropped frames based on size bug 

The allowed payload sizes were calculated slightly incorrectly and
this caused frames with sizes near the allowed frame size to be not
sent or sent incorrectly. Now all these errors should be fixed.
This commit is contained in:
Joni Räsänen 2022-03-31 10:05:41 +03:00
parent c3c1f60140
commit 4ecfedcdc8
11 changed files with 106 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/uvgrtp/frame.hh
View File

@ -23,10 +23,13 @@ namespace uvgrtp {
HEADER_SIZE_RTP = 12,
HEADER_SIZE_OPUS = 1,
HEADER_SIZE_H264_INDICATOR = 1,
HEADER_SIZE_H264_NAL = 1,
HEADER_SIZE_H264_FU = 1,
HEADER_SIZE_H265_PAYLOAD = 2,
HEADER_SIZE_H265_NAL = 2,
HEADER_SIZE_H265_FU = 1,
HEADER_SIZE_H266_PAYLOAD = 2,
HEADER_SIZE_H266_NAL = 2,
HEADER_SIZE_H266_FU = 1,
};

14
src/formats/h264.cc
View File

@ -30,6 +30,11 @@ uint8_t uvgrtp::formats::h264::get_payload_header_size() const
return uvgrtp::frame::HEADER_SIZE_H264_INDICATOR;
}
uint8_t uvgrtp::formats::h264::get_nal_header_size() const
{
return uvgrtp::frame::HEADER_SIZE_H264_NAL;
}
uint8_t uvgrtp::formats::h264::get_fu_header_size() const
{
return uvgrtp::frame::HEADER_SIZE_H264_FU;
@ -49,7 +54,7 @@ int uvgrtp::formats::h264::get_fragment_type(uvgrtp::frame::rtp_frame* frame) co
return uvgrtp::formats::FT_AGGR;
if ((frame->payload[0] & 0x1f) != uvgrtp::formats::H264_PKT_FRAG)
return uvgrtp::formats::FT_NOT_FRAG;
return uvgrtp::formats::FT_NOT_FRAG; // Single NAL unit
if (first_frag && last_frag)
return uvgrtp::formats::FT_INVALID;
@ -147,13 +152,10 @@ rtp_error_t uvgrtp::formats::h264::construct_format_header_divide_fus(uint8_t* d
buffers.push_back(std::make_pair(sizeof(uint8_t), &headers->fu_headers[0]));
buffers.push_back(std::make_pair(payload_size, nullptr));
size_t data_pos = uvgrtp::frame::HEADER_SIZE_H264_INDICATOR;
data_len -= uvgrtp::frame::HEADER_SIZE_H264_INDICATOR;
return divide_frame_to_fus(data, data_len, data_pos, payload_size, buffers, headers->fu_headers);
return divide_frame_to_fus(data, data_len, payload_size, buffers, headers->fu_headers);
}
void uvgrtp::formats::h264::copy_payload_header(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload)
void uvgrtp::formats::h264::get_nal_header_from_fu_headers(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload)
{
complete_payload[fptr] = (frame_payload[0] & 0xe0) | (frame_payload[1] & 0x1f);
}

3
src/formats/h264.hh
View File

@ -59,13 +59,14 @@ namespace uvgrtp {
virtual uint8_t get_nal_type(uint8_t* data) const;
virtual uint8_t get_payload_header_size() const;
virtual uint8_t get_nal_header_size() const;
virtual uint8_t get_fu_header_size() const;
virtual uint8_t get_start_code_range() const;
virtual int get_fragment_type(uvgrtp::frame::rtp_frame* frame) const;
virtual uvgrtp::formats::NAL_TYPES get_nal_type(uvgrtp::frame::rtp_frame* frame) const;
virtual void copy_payload_header(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload);
virtual void get_nal_header_from_fu_headers(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload);
private:
h264_aggregation_packet aggr_pkt_info_;

16
src/formats/h265.cc
View File

@ -33,6 +33,11 @@ uint8_t uvgrtp::formats::h265::get_payload_header_size() const
return uvgrtp::frame::HEADER_SIZE_H265_PAYLOAD;
}
uint8_t uvgrtp::formats::h265::get_nal_header_size() const
{
return uvgrtp::frame::HEADER_SIZE_H265_NAL;
}
uint8_t uvgrtp::formats::h265::get_fu_header_size() const
{
return uvgrtp::frame::HEADER_SIZE_H265_FU;
@ -50,14 +55,14 @@ uint8_t uvgrtp::formats::h265::get_nal_type(uint8_t* data) const
int uvgrtp::formats::h265::get_fragment_type(uvgrtp::frame::rtp_frame* frame) const
{
bool first_frag = frame->payload[2] & 0x80;
bool last_frag = frame->payload[2] & 0x40;
bool first_frag = frame->payload[2] & 0x80; // S bit
bool last_frag = frame->payload[2] & 0x40; // E bit
if ((frame->payload[0] >> 1) == uvgrtp::formats::H265_PKT_AGGR)
return uvgrtp::formats::FT_AGGR;
if ((frame->payload[0] >> 1) != uvgrtp::formats::H265_PKT_FRAG)
return uvgrtp::formats::FT_NOT_FRAG;
return uvgrtp::formats::FT_NOT_FRAG; // Single NAL unit
if (first_frag && last_frag)
return uvgrtp::formats::FT_INVALID;
@ -163,8 +168,5 @@ rtp_error_t uvgrtp::formats::h265::construct_format_header_divide_fus(uint8_t* d
buffers.push_back(std::make_pair(sizeof(uint8_t), &headers->fu_headers[0])); // first fragment
buffers.push_back(std::make_pair(payload_size, nullptr));
size_t data_pos = uvgrtp::frame::HEADER_SIZE_H265_PAYLOAD;
data_len -= uvgrtp::frame::HEADER_SIZE_H265_PAYLOAD;
return divide_frame_to_fus(data, data_len, data_pos, payload_size, buffers, headers->fu_headers);
return divide_frame_to_fus(data, data_len, payload_size, buffers, headers->fu_headers);
}

1
src/formats/h265.hh
View File

@ -61,6 +61,7 @@ namespace uvgrtp {
virtual uint8_t get_nal_type(uint8_t* data) const;
virtual uint8_t get_payload_header_size() const;
virtual uint8_t get_nal_header_size() const;
virtual uint8_t get_fu_header_size() const;
virtual uint8_t get_start_code_range() const;
virtual int get_fragment_type(uvgrtp::frame::rtp_frame* frame) const;

12
src/formats/h266.cc
View File

@ -34,6 +34,11 @@ uint8_t uvgrtp::formats::h266::get_payload_header_size() const
return uvgrtp::frame::HEADER_SIZE_H266_PAYLOAD;
}
uint8_t uvgrtp::formats::h266::get_nal_header_size() const
{
return uvgrtp::frame::HEADER_SIZE_H266_NAL;
}
uint8_t uvgrtp::formats::h266::get_fu_header_size() const
{
return uvgrtp::frame::HEADER_SIZE_H266_FU;
@ -55,7 +60,7 @@ int uvgrtp::formats::h266::get_fragment_type(uvgrtp::frame::rtp_frame* frame) co
bool last_frag = frame->payload[2] & 0x40;
if ((frame->payload[1] >> 3) != uvgrtp::formats::H266_PKT_FRAG)
return uvgrtp::formats::FT_NOT_FRAG;
return uvgrtp::formats::FT_NOT_FRAG; // Single NAL unit
if (first_frag && last_frag)
return uvgrtp::formats::FT_INVALID;
@ -94,8 +99,5 @@ rtp_error_t uvgrtp::formats::h266::construct_format_header_divide_fus(uint8_t* d
buffers.push_back(std::make_pair(sizeof(uint8_t), &headers->fu_headers[0]));
buffers.push_back(std::make_pair(payload_size, nullptr));
size_t data_pos = uvgrtp::frame::HEADER_SIZE_H266_PAYLOAD;
data_len -= uvgrtp::frame::HEADER_SIZE_H266_PAYLOAD;
return divide_frame_to_fus(data, data_len, data_pos, payload_size, buffers, headers->fu_headers);
return divide_frame_to_fus(data, data_len, payload_size, buffers, headers->fu_headers);
}

1
src/formats/h266.hh
View File

@ -50,6 +50,7 @@ namespace uvgrtp {
virtual uint8_t get_nal_type(uint8_t* data) const;
virtual uint8_t get_payload_header_size() const;
virtual uint8_t get_nal_header_size() const;
virtual uint8_t get_fu_header_size() const;
virtual uint8_t get_start_code_range() const;
virtual int get_fragment_type(uvgrtp::frame::rtp_frame* frame) const;

100
src/formats/h26x.cc
View File

@ -283,12 +283,7 @@ rtp_error_t uvgrtp::formats::h26x::push_media_frame(uint8_t* data, size_t data_l
return ret;
}
size_t payload_size = rtp_ctx_->get_payload_size() - get_payload_header_size() - get_fu_header_size();
if (payload_size <= 0)
{
return RTP_INVALID_VALUE;
}
size_t payload_size = rtp_ctx_->get_payload_size();
// find all the locations of NAL units using Start Code Lookup (SCL)
std::vector<nal_info> nals;
@ -321,28 +316,25 @@ rtp_error_t uvgrtp::formats::h26x::push_media_frame(uint8_t* data, size_t data_l
(void)finalize_aggregation_pkt();
}
if (nals.size() == 1 && nals[0].size <= payload_size && !should_aggregate) // Single NAL unit, non-aggretable
for (auto& nal : nals) // non-aggregatable NAL units
{
if ((ret = single_nal_unit(data + nals.at(0).prefix_len, data_len - nals.at(0).prefix_len)) != RTP_OK)
if (!nal.aggregate || !should_aggregate)
{
fqueue_->deinit_transaction();
return ret;
}
}
else // send all NAL units requiring FU division (single NAL unit is an alternative to this)
{
// push NAL units
for (auto& nal : nals)
{
if (!nal.aggregate || !should_aggregate)
// single NAL unit uses the NAL unit header as the payload header meaning that it does not
// add anything extra to the packet and we can just compare the NAL size with the payload size allowed
if (nal.size <= payload_size) // send as a single NAL unit packet
{
ret = single_nal_unit(data + nal.offset, nal.size);
}
else // send divided based on payload_size
{
ret = fu_division(&data[nal.offset], nal.size, payload_size);
}
if (ret != RTP_OK)
{
fqueue_->deinit_transaction();
return ret;
}
if (ret != RTP_OK)
{
fqueue_->deinit_transaction();
return ret;
}
}
}
@ -402,9 +394,8 @@ void uvgrtp::formats::h26x::clear_aggregation_info()
rtp_error_t uvgrtp::formats::h26x::single_nal_unit(uint8_t* data, size_t data_len)
{
// TODO: I have a hunch that the payload is missing payload header with single NAL unit
// single NAL unit packets use NAL header directly as payload header so the packet is
// correct as is
rtp_error_t ret = RTP_OK;
if ((ret = fqueue_->enqueue_message(data, data_len)) != RTP_OK) {
LOG_ERROR("Failed to enqueue single h26x NAL Unit packet!");
@ -414,7 +405,7 @@ rtp_error_t uvgrtp::formats::h26x::single_nal_unit(uint8_t* data, size_t data_le
}
rtp_error_t uvgrtp::formats::h26x::divide_frame_to_fus(uint8_t* data, size_t& data_left,
size_t& data_pos, size_t payload_size, uvgrtp::buf_vec& buffers, uint8_t fu_headers[])
size_t payload_size, uvgrtp::buf_vec& buffers, uint8_t fu_headers[])
{
if (data_left <= payload_size)
{
@ -424,26 +415,31 @@ rtp_error_t uvgrtp::formats::h26x::divide_frame_to_fus(uint8_t* data, size_t& da
rtp_error_t ret = RTP_OK;
// This seems to work by always using the headers in first and second index of buffer
// (and modifying those) and replacing the payload in third, then sending all
// the FU structure has both payload header and an fu header
size_t fu_payload_size = payload_size - get_payload_header_size() - get_fu_header_size();
// the headers for first fragment are already in buffers.at(0)
// skip NAL header of data since it is incorporated in payload and fu headers (which are repeated
// for each packet, but NAL header is only at the beginning of NAL unit)
size_t data_pos = get_nal_header_size();
data_left -= get_nal_header_size();
while (data_left > payload_size) {
while (data_left > fu_payload_size) {
/* This seems to work by always using the payload headers in first and fu headers in the second index
* of buffer (and modifying those) and replacing the payload in third, then sending all.
* The headers for first fragment are already in buffers.at(1) */
// set the payload for this fragment
buffers.at(2).first = payload_size;
buffers.at(2).first = fu_payload_size;
buffers.at(2).second = &data[data_pos];
if ((ret = fqueue_->enqueue_message(buffers)) != RTP_OK) {
LOG_ERROR("Queueing the message failed!");
clear_aggregation_info();
fqueue_->deinit_transaction();
LOG_ERROR("Queueing the FU packet failed!");
return ret;
}
data_pos += payload_size;
data_left -= payload_size;
data_pos += fu_payload_size;
data_left -= fu_payload_size;
buffers.at(1).second = &fu_headers[1]; // middle fragment header
}
@ -619,11 +615,9 @@ rtp_error_t uvgrtp::formats::h26x::packet_handler(int flags, uvgrtp::frame::rtp_
* pointed to by "intra" and new intra frame shall take the place of active intra frame */
uint32_t intra = INVALID_TS;
const size_t format_header_size = get_payload_header_size() + get_fu_header_size();
frame = *out;
int frag_type = get_fragment_type(frame);
if (frag_type == FT_AGGR) {
// handle aggregate packets (packets with multiple NAL units in them)
@ -670,8 +664,10 @@ rtp_error_t uvgrtp::formats::h26x::packet_handler(int flags, uvgrtp::frame::rtp_
initialize_new_fragmented_frame(c_ts);
}
const size_t sizeof_fu_headers = get_payload_header_size() + get_fu_header_size();
frames_[c_ts].pkts_received += 1;
frames_[c_ts].total_size += (frame->payload_len - format_header_size);
frames_[c_ts].total_size += (frame->payload_len - sizeof_fu_headers);
if (frag_type == FT_START) {
frames_[c_ts].s_seq = c_seq;
@ -729,18 +725,19 @@ rtp_error_t uvgrtp::formats::h26x::packet_handler(int flags, uvgrtp::frame::rtp_
size_t fptr = 0;
uvgrtp::frame::rtp_frame* complete = allocate_rtp_frame_with_startcode((flags & RCE_H26X_PREPEND_SC),
(*out)->header, frames_[c_ts].total_size + get_payload_header_size(), fptr);
(*out)->header, get_nal_header_size() + frames_[c_ts].total_size, fptr);
copy_payload_header(fptr, frame->payload, complete->payload); // NAL header
fptr += get_payload_header_size();
get_nal_header_from_fu_headers(fptr, frame->payload, complete->payload); // NAL header
fptr += get_nal_header_size();
for (auto& fragment : frames_.at(c_ts).fragments) {
// copy everything expect fu headers (which repeat for every fu)
std::memcpy(
&complete->payload[fptr],
&fragment.second->payload[format_header_size],
fragment.second->payload_len - format_header_size
&fragment.second->payload[sizeof_fu_headers],
fragment.second->payload_len - sizeof_fu_headers
);
fptr += fragment.second->payload_len - format_header_size;
fptr += fragment.second->payload_len - sizeof_fu_headers;
(void)uvgrtp::frame::dealloc_frame(fragment.second);
}
@ -762,11 +759,10 @@ rtp_error_t uvgrtp::formats::h26x::packet_handler(int flags, uvgrtp::frame::rtp_
}
garbage_collect_lost_frames();
return RTP_OK;
}
void uvgrtp::formats::h26x::copy_payload_header(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload)
void uvgrtp::formats::h26x::get_nal_header_from_fu_headers(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload)
{
uint8_t payload_header[2] = {
(uint8_t)((frame_payload[0] & 0x81) | ((frame_payload[2] & 0x3f) << 1)),
@ -838,6 +834,7 @@ void uvgrtp::formats::h26x::scl(uint8_t* data, size_t data_len, size_t packet_si
uint8_t start_len = 0;
ssize_t offset = find_h26x_start_code(data, data_len, 0, start_len);
packet_size -= get_payload_header_size(); // aggregate packet has a payload header
while (offset != -1) {
nal_info nal;
@ -869,9 +866,12 @@ void uvgrtp::formats::h26x::scl(uint8_t* data, size_t data_len, size_t packet_si
nals.at(i).size = data_len - nals[i].offset;
}
if (aggregate_size + nals.at(i).size <= packet_size)
// each NAL unit added to aggregate packet needs the size added which has to be taken into account
// when calculating the aggregate packet
// (NOTE: This is not enough for MTAP in h264, but I doubt uvgRTP will support it)
if (aggregate_size + nals.at(i).size + sizeof(uint16_t) <= packet_size)
{
aggregate_size += nals.at(i).size;
aggregate_size += nals.at(i).size + sizeof(uint16_t);
nals.at(i).aggregate = true;
++aggregatable_packets;
}

5
src/formats/h26x.hh
View File

@ -130,7 +130,7 @@ namespace uvgrtp {
size_t payload_size, uvgrtp::buf_vec& buffers) = 0;
// a helper function that handles the fu division.
rtp_error_t divide_frame_to_fus(uint8_t* data, size_t& data_left, size_t& data_pos, size_t payload_size,
rtp_error_t divide_frame_to_fus(uint8_t* data, size_t& data_left, size_t payload_size,
uvgrtp::buf_vec& buffers, uint8_t fu_headers[]);
void initialize_fu_headers(uint8_t nal_type, uint8_t fu_headers[]);
@ -141,12 +141,13 @@ namespace uvgrtp {
virtual uint8_t get_nal_type(uint8_t* data) const = 0;
virtual uint8_t get_payload_header_size() const = 0;
virtual uint8_t get_nal_header_size() const = 0;
virtual uint8_t get_fu_header_size() const = 0;
virtual uint8_t get_start_code_range() const = 0;
virtual int get_fragment_type(uvgrtp::frame::rtp_frame* frame) const = 0;
virtual uvgrtp::formats::NAL_TYPES get_nal_type(uvgrtp::frame::rtp_frame* frame) const = 0;
virtual void copy_payload_header(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload);
virtual void get_nal_header_from_fu_headers(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload);
private:

21
test/test_4_formats.cpp
View File

@ -28,11 +28,16 @@ TEST(FormatTests, h264)
test_packet_size(1443, sess, sender, receiver);
test_packet_size(1444, sess, sender, receiver);
test_packet_size(1445, sess, sender, receiver);
test_packet_size(1446, sess, sender, receiver);
test_packet_size(1446, sess, sender, receiver); // packet limit
test_packet_size(1447, sess, sender, receiver);
test_packet_size(1448, sess, sender, receiver);
test_packet_size(1449, sess, sender, receiver);
test_packet_size(1450, sess, sender, receiver);
test_packet_size(1451, sess, sender, receiver);
test_packet_size(1452, sess, sender, receiver);
test_packet_size(1453, sess, sender, receiver);
test_packet_size(1454, sess, sender, receiver);
test_packet_size(1455, sess, sender, receiver);
test_packet_size(1501, sess, sender, receiver);
test_packet_size(15000, sess, sender, receiver);
test_packet_size(150000, sess, sender, receiver);
@ -61,11 +66,16 @@ TEST(FormatTests, h265)
test_packet_size(1443, sess, sender, receiver);
test_packet_size(1444, sess, sender, receiver);
test_packet_size(1445, sess, sender, receiver);
test_packet_size(1446, sess, sender, receiver);
test_packet_size(1446, sess, sender, receiver); // packet limit
test_packet_size(1447, sess, sender, receiver);
test_packet_size(1448, sess, sender, receiver);
test_packet_size(1449, sess, sender, receiver);
test_packet_size(1450, sess, sender, receiver);
test_packet_size(1451, sess, sender, receiver);
test_packet_size(1452, sess, sender, receiver);
test_packet_size(1453, sess, sender, receiver);
test_packet_size(1454, sess, sender, receiver);
test_packet_size(1455, sess, sender, receiver);
test_packet_size(1501, sess, sender, receiver);
test_packet_size(15000, sess, sender, receiver);
test_packet_size(150000, sess, sender, receiver);
@ -94,11 +104,16 @@ TEST(FormatTests, h266)
test_packet_size(1443, sess, sender, receiver);
test_packet_size(1444, sess, sender, receiver);
test_packet_size(1445, sess, sender, receiver);
test_packet_size(1446, sess, sender, receiver);
test_packet_size(1446, sess, sender, receiver); // packet limit
test_packet_size(1447, sess, sender, receiver);
test_packet_size(1448, sess, sender, receiver);
test_packet_size(1449, sess, sender, receiver);
test_packet_size(1450, sess, sender, receiver);
test_packet_size(1451, sess, sender, receiver);
test_packet_size(1452, sess, sender, receiver);
test_packet_size(1453, sess, sender, receiver);
test_packet_size(1454, sess, sender, receiver);
test_packet_size(1455, sess, sender, receiver);
test_packet_size(1501, sess, sender, receiver);
test_packet_size(15000, sess, sender, receiver);
test_packet_size(150000, sess, sender, receiver);

6
test/test_common.hh
View File

@ -69,9 +69,11 @@ inline void send_packets(uvgrtp::session* sess, uvgrtp::media_stream* sender,
memset(dummy_frame.get(), 'a', size);
}
if (sender->push_frame(std::move(dummy_frame), size, RTP_NO_FLAGS) != RTP_OK)
rtp_error_t ret = RTP_OK;
if ((ret = sender->push_frame(std::move(dummy_frame), size, RTP_NO_FLAGS)) != RTP_OK)
{
std::cout << "Failed to send test packet!" << std::endl;
std::cout << "Failed to send test packet! Return value: " << ret << std::endl;
return;
}
if (i % (packets / 10) == packets / 10 - 1 && print_progress)