diff --git a/include/uvgrtp/frame.hh b/include/uvgrtp/frame.hh index 405407f..424059c 100644 --- a/include/uvgrtp/frame.hh +++ b/include/uvgrtp/frame.hh @@ -23,10 +23,13 @@ namespace uvgrtp { HEADER_SIZE_RTP = 12, HEADER_SIZE_OPUS = 1, HEADER_SIZE_H264_INDICATOR = 1, + HEADER_SIZE_H264_NAL = 1, HEADER_SIZE_H264_FU = 1, HEADER_SIZE_H265_PAYLOAD = 2, + HEADER_SIZE_H265_NAL = 2, HEADER_SIZE_H265_FU = 1, HEADER_SIZE_H266_PAYLOAD = 2, + HEADER_SIZE_H266_NAL = 2, HEADER_SIZE_H266_FU = 1, }; diff --git a/src/formats/h264.cc b/src/formats/h264.cc index 425d2f7..782b5ea 100644 --- a/src/formats/h264.cc +++ b/src/formats/h264.cc @@ -30,6 +30,11 @@ uint8_t uvgrtp::formats::h264::get_payload_header_size() const return uvgrtp::frame::HEADER_SIZE_H264_INDICATOR; } +uint8_t uvgrtp::formats::h264::get_nal_header_size() const +{ + return uvgrtp::frame::HEADER_SIZE_H264_NAL; +} + uint8_t uvgrtp::formats::h264::get_fu_header_size() const { return uvgrtp::frame::HEADER_SIZE_H264_FU; @@ -49,7 +54,7 @@ int uvgrtp::formats::h264::get_fragment_type(uvgrtp::frame::rtp_frame* frame) co return uvgrtp::formats::FT_AGGR; if ((frame->payload[0] & 0x1f) != uvgrtp::formats::H264_PKT_FRAG) - return uvgrtp::formats::FT_NOT_FRAG; + return uvgrtp::formats::FT_NOT_FRAG; // Single NAL unit if (first_frag && last_frag) return uvgrtp::formats::FT_INVALID; @@ -147,13 +152,10 @@ rtp_error_t uvgrtp::formats::h264::construct_format_header_divide_fus(uint8_t* d buffers.push_back(std::make_pair(sizeof(uint8_t), &headers->fu_headers[0])); buffers.push_back(std::make_pair(payload_size, nullptr)); - size_t data_pos = uvgrtp::frame::HEADER_SIZE_H264_INDICATOR; - data_len -= uvgrtp::frame::HEADER_SIZE_H264_INDICATOR; - - return divide_frame_to_fus(data, data_len, data_pos, payload_size, buffers, headers->fu_headers); + return divide_frame_to_fus(data, data_len, payload_size, buffers, headers->fu_headers); } -void uvgrtp::formats::h264::copy_payload_header(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload) +void uvgrtp::formats::h264::get_nal_header_from_fu_headers(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload) { complete_payload[fptr] = (frame_payload[0] & 0xe0) | (frame_payload[1] & 0x1f); } \ No newline at end of file diff --git a/src/formats/h264.hh b/src/formats/h264.hh index 7acf2dd..96949ba 100644 --- a/src/formats/h264.hh +++ b/src/formats/h264.hh @@ -59,13 +59,14 @@ namespace uvgrtp { virtual uint8_t get_nal_type(uint8_t* data) const; virtual uint8_t get_payload_header_size() const; + virtual uint8_t get_nal_header_size() const; virtual uint8_t get_fu_header_size() const; virtual uint8_t get_start_code_range() const; virtual int get_fragment_type(uvgrtp::frame::rtp_frame* frame) const; virtual uvgrtp::formats::NAL_TYPES get_nal_type(uvgrtp::frame::rtp_frame* frame) const; - virtual void copy_payload_header(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload); + virtual void get_nal_header_from_fu_headers(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload); private: h264_aggregation_packet aggr_pkt_info_; diff --git a/src/formats/h265.cc b/src/formats/h265.cc index 435243f..5cef3bc 100644 --- a/src/formats/h265.cc +++ b/src/formats/h265.cc @@ -33,6 +33,11 @@ uint8_t uvgrtp::formats::h265::get_payload_header_size() const return uvgrtp::frame::HEADER_SIZE_H265_PAYLOAD; } +uint8_t uvgrtp::formats::h265::get_nal_header_size() const +{ + return uvgrtp::frame::HEADER_SIZE_H265_NAL; +} + uint8_t uvgrtp::formats::h265::get_fu_header_size() const { return uvgrtp::frame::HEADER_SIZE_H265_FU; @@ -50,14 +55,14 @@ uint8_t uvgrtp::formats::h265::get_nal_type(uint8_t* data) const int uvgrtp::formats::h265::get_fragment_type(uvgrtp::frame::rtp_frame* frame) const { - bool first_frag = frame->payload[2] & 0x80; - bool last_frag = frame->payload[2] & 0x40; + bool first_frag = frame->payload[2] & 0x80; // S bit + bool last_frag = frame->payload[2] & 0x40; // E bit if ((frame->payload[0] >> 1) == uvgrtp::formats::H265_PKT_AGGR) return uvgrtp::formats::FT_AGGR; if ((frame->payload[0] >> 1) != uvgrtp::formats::H265_PKT_FRAG) - return uvgrtp::formats::FT_NOT_FRAG; + return uvgrtp::formats::FT_NOT_FRAG; // Single NAL unit if (first_frag && last_frag) return uvgrtp::formats::FT_INVALID; @@ -163,8 +168,5 @@ rtp_error_t uvgrtp::formats::h265::construct_format_header_divide_fus(uint8_t* d buffers.push_back(std::make_pair(sizeof(uint8_t), &headers->fu_headers[0])); // first fragment buffers.push_back(std::make_pair(payload_size, nullptr)); - size_t data_pos = uvgrtp::frame::HEADER_SIZE_H265_PAYLOAD; - data_len -= uvgrtp::frame::HEADER_SIZE_H265_PAYLOAD; - - return divide_frame_to_fus(data, data_len, data_pos, payload_size, buffers, headers->fu_headers); + return divide_frame_to_fus(data, data_len, payload_size, buffers, headers->fu_headers); } \ No newline at end of file diff --git a/src/formats/h265.hh b/src/formats/h265.hh index 04c97b8..8ca56db 100644 --- a/src/formats/h265.hh +++ b/src/formats/h265.hh @@ -61,6 +61,7 @@ namespace uvgrtp { virtual uint8_t get_nal_type(uint8_t* data) const; virtual uint8_t get_payload_header_size() const; + virtual uint8_t get_nal_header_size() const; virtual uint8_t get_fu_header_size() const; virtual uint8_t get_start_code_range() const; virtual int get_fragment_type(uvgrtp::frame::rtp_frame* frame) const; diff --git a/src/formats/h266.cc b/src/formats/h266.cc index 9e0b8fc..54c85aa 100644 --- a/src/formats/h266.cc +++ b/src/formats/h266.cc @@ -34,6 +34,11 @@ uint8_t uvgrtp::formats::h266::get_payload_header_size() const return uvgrtp::frame::HEADER_SIZE_H266_PAYLOAD; } +uint8_t uvgrtp::formats::h266::get_nal_header_size() const +{ + return uvgrtp::frame::HEADER_SIZE_H266_NAL; +} + uint8_t uvgrtp::formats::h266::get_fu_header_size() const { return uvgrtp::frame::HEADER_SIZE_H266_FU; @@ -55,7 +60,7 @@ int uvgrtp::formats::h266::get_fragment_type(uvgrtp::frame::rtp_frame* frame) co bool last_frag = frame->payload[2] & 0x40; if ((frame->payload[1] >> 3) != uvgrtp::formats::H266_PKT_FRAG) - return uvgrtp::formats::FT_NOT_FRAG; + return uvgrtp::formats::FT_NOT_FRAG; // Single NAL unit if (first_frag && last_frag) return uvgrtp::formats::FT_INVALID; @@ -94,8 +99,5 @@ rtp_error_t uvgrtp::formats::h266::construct_format_header_divide_fus(uint8_t* d buffers.push_back(std::make_pair(sizeof(uint8_t), &headers->fu_headers[0])); buffers.push_back(std::make_pair(payload_size, nullptr)); - size_t data_pos = uvgrtp::frame::HEADER_SIZE_H266_PAYLOAD; - data_len -= uvgrtp::frame::HEADER_SIZE_H266_PAYLOAD; - - return divide_frame_to_fus(data, data_len, data_pos, payload_size, buffers, headers->fu_headers); + return divide_frame_to_fus(data, data_len, payload_size, buffers, headers->fu_headers); } \ No newline at end of file diff --git a/src/formats/h266.hh b/src/formats/h266.hh index e640da3..6fc7c62 100644 --- a/src/formats/h266.hh +++ b/src/formats/h266.hh @@ -50,6 +50,7 @@ namespace uvgrtp { virtual uint8_t get_nal_type(uint8_t* data) const; virtual uint8_t get_payload_header_size() const; + virtual uint8_t get_nal_header_size() const; virtual uint8_t get_fu_header_size() const; virtual uint8_t get_start_code_range() const; virtual int get_fragment_type(uvgrtp::frame::rtp_frame* frame) const; diff --git a/src/formats/h26x.cc b/src/formats/h26x.cc index 905e459..a21fb68 100644 --- a/src/formats/h26x.cc +++ b/src/formats/h26x.cc @@ -283,12 +283,7 @@ rtp_error_t uvgrtp::formats::h26x::push_media_frame(uint8_t* data, size_t data_l return ret; } - size_t payload_size = rtp_ctx_->get_payload_size() - get_payload_header_size() - get_fu_header_size(); - - if (payload_size <= 0) - { - return RTP_INVALID_VALUE; - } + size_t payload_size = rtp_ctx_->get_payload_size(); // find all the locations of NAL units using Start Code Lookup (SCL) std::vector nals; @@ -321,28 +316,25 @@ rtp_error_t uvgrtp::formats::h26x::push_media_frame(uint8_t* data, size_t data_l (void)finalize_aggregation_pkt(); } - if (nals.size() == 1 && nals[0].size <= payload_size && !should_aggregate) // Single NAL unit, non-aggretable + for (auto& nal : nals) // non-aggregatable NAL units { - if ((ret = single_nal_unit(data + nals.at(0).prefix_len, data_len - nals.at(0).prefix_len)) != RTP_OK) + if (!nal.aggregate || !should_aggregate) { - fqueue_->deinit_transaction(); - return ret; - } - } - else // send all NAL units requiring FU division (single NAL unit is an alternative to this) - { - // push NAL units - for (auto& nal : nals) - { - if (!nal.aggregate || !should_aggregate) + // single NAL unit uses the NAL unit header as the payload header meaning that it does not + // add anything extra to the packet and we can just compare the NAL size with the payload size allowed + if (nal.size <= payload_size) // send as a single NAL unit packet + { + ret = single_nal_unit(data + nal.offset, nal.size); + } + else // send divided based on payload_size { ret = fu_division(&data[nal.offset], nal.size, payload_size); + } - if (ret != RTP_OK) - { - fqueue_->deinit_transaction(); - return ret; - } + if (ret != RTP_OK) + { + fqueue_->deinit_transaction(); + return ret; } } } @@ -402,9 +394,8 @@ void uvgrtp::formats::h26x::clear_aggregation_info() rtp_error_t uvgrtp::formats::h26x::single_nal_unit(uint8_t* data, size_t data_len) { - // TODO: I have a hunch that the payload is missing payload header with single NAL unit - - + // single NAL unit packets use NAL header directly as payload header so the packet is + // correct as is rtp_error_t ret = RTP_OK; if ((ret = fqueue_->enqueue_message(data, data_len)) != RTP_OK) { LOG_ERROR("Failed to enqueue single h26x NAL Unit packet!"); @@ -414,7 +405,7 @@ rtp_error_t uvgrtp::formats::h26x::single_nal_unit(uint8_t* data, size_t data_le } rtp_error_t uvgrtp::formats::h26x::divide_frame_to_fus(uint8_t* data, size_t& data_left, - size_t& data_pos, size_t payload_size, uvgrtp::buf_vec& buffers, uint8_t fu_headers[]) + size_t payload_size, uvgrtp::buf_vec& buffers, uint8_t fu_headers[]) { if (data_left <= payload_size) { @@ -424,26 +415,31 @@ rtp_error_t uvgrtp::formats::h26x::divide_frame_to_fus(uint8_t* data, size_t& da rtp_error_t ret = RTP_OK; - // This seems to work by always using the headers in first and second index of buffer - // (and modifying those) and replacing the payload in third, then sending all + // the FU structure has both payload header and an fu header + size_t fu_payload_size = payload_size - get_payload_header_size() - get_fu_header_size(); - // the headers for first fragment are already in buffers.at(0) + // skip NAL header of data since it is incorporated in payload and fu headers (which are repeated + // for each packet, but NAL header is only at the beginning of NAL unit) + size_t data_pos = get_nal_header_size(); + data_left -= get_nal_header_size(); - while (data_left > payload_size) { + while (data_left > fu_payload_size) { + + /* This seems to work by always using the payload headers in first and fu headers in the second index + * of buffer (and modifying those) and replacing the payload in third, then sending all. + * The headers for first fragment are already in buffers.at(1) */ // set the payload for this fragment - buffers.at(2).first = payload_size; + buffers.at(2).first = fu_payload_size; buffers.at(2).second = &data[data_pos]; if ((ret = fqueue_->enqueue_message(buffers)) != RTP_OK) { - LOG_ERROR("Queueing the message failed!"); - clear_aggregation_info(); - fqueue_->deinit_transaction(); + LOG_ERROR("Queueing the FU packet failed!"); return ret; } - data_pos += payload_size; - data_left -= payload_size; + data_pos += fu_payload_size; + data_left -= fu_payload_size; buffers.at(1).second = &fu_headers[1]; // middle fragment header } @@ -619,11 +615,9 @@ rtp_error_t uvgrtp::formats::h26x::packet_handler(int flags, uvgrtp::frame::rtp_ * pointed to by "intra" and new intra frame shall take the place of active intra frame */ uint32_t intra = INVALID_TS; - const size_t format_header_size = get_payload_header_size() + get_fu_header_size(); frame = *out; int frag_type = get_fragment_type(frame); - if (frag_type == FT_AGGR) { // handle aggregate packets (packets with multiple NAL units in them) @@ -670,8 +664,10 @@ rtp_error_t uvgrtp::formats::h26x::packet_handler(int flags, uvgrtp::frame::rtp_ initialize_new_fragmented_frame(c_ts); } + const size_t sizeof_fu_headers = get_payload_header_size() + get_fu_header_size(); + frames_[c_ts].pkts_received += 1; - frames_[c_ts].total_size += (frame->payload_len - format_header_size); + frames_[c_ts].total_size += (frame->payload_len - sizeof_fu_headers); if (frag_type == FT_START) { frames_[c_ts].s_seq = c_seq; @@ -729,18 +725,19 @@ rtp_error_t uvgrtp::formats::h26x::packet_handler(int flags, uvgrtp::frame::rtp_ size_t fptr = 0; uvgrtp::frame::rtp_frame* complete = allocate_rtp_frame_with_startcode((flags & RCE_H26X_PREPEND_SC), - (*out)->header, frames_[c_ts].total_size + get_payload_header_size(), fptr); + (*out)->header, get_nal_header_size() + frames_[c_ts].total_size, fptr); - copy_payload_header(fptr, frame->payload, complete->payload); // NAL header - fptr += get_payload_header_size(); + get_nal_header_from_fu_headers(fptr, frame->payload, complete->payload); // NAL header + fptr += get_nal_header_size(); for (auto& fragment : frames_.at(c_ts).fragments) { + // copy everything expect fu headers (which repeat for every fu) std::memcpy( &complete->payload[fptr], - &fragment.second->payload[format_header_size], - fragment.second->payload_len - format_header_size + &fragment.second->payload[sizeof_fu_headers], + fragment.second->payload_len - sizeof_fu_headers ); - fptr += fragment.second->payload_len - format_header_size; + fptr += fragment.second->payload_len - sizeof_fu_headers; (void)uvgrtp::frame::dealloc_frame(fragment.second); } @@ -762,11 +759,10 @@ rtp_error_t uvgrtp::formats::h26x::packet_handler(int flags, uvgrtp::frame::rtp_ } garbage_collect_lost_frames(); - return RTP_OK; } -void uvgrtp::formats::h26x::copy_payload_header(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload) +void uvgrtp::formats::h26x::get_nal_header_from_fu_headers(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload) { uint8_t payload_header[2] = { (uint8_t)((frame_payload[0] & 0x81) | ((frame_payload[2] & 0x3f) << 1)), @@ -838,6 +834,7 @@ void uvgrtp::formats::h26x::scl(uint8_t* data, size_t data_len, size_t packet_si uint8_t start_len = 0; ssize_t offset = find_h26x_start_code(data, data_len, 0, start_len); + packet_size -= get_payload_header_size(); // aggregate packet has a payload header while (offset != -1) { nal_info nal; @@ -869,9 +866,12 @@ void uvgrtp::formats::h26x::scl(uint8_t* data, size_t data_len, size_t packet_si nals.at(i).size = data_len - nals[i].offset; } - if (aggregate_size + nals.at(i).size <= packet_size) + // each NAL unit added to aggregate packet needs the size added which has to be taken into account + // when calculating the aggregate packet + // (NOTE: This is not enough for MTAP in h264, but I doubt uvgRTP will support it) + if (aggregate_size + nals.at(i).size + sizeof(uint16_t) <= packet_size) { - aggregate_size += nals.at(i).size; + aggregate_size += nals.at(i).size + sizeof(uint16_t); nals.at(i).aggregate = true; ++aggregatable_packets; } diff --git a/src/formats/h26x.hh b/src/formats/h26x.hh index 0fad0c2..7da510d 100644 --- a/src/formats/h26x.hh +++ b/src/formats/h26x.hh @@ -130,7 +130,7 @@ namespace uvgrtp { size_t payload_size, uvgrtp::buf_vec& buffers) = 0; // a helper function that handles the fu division. - rtp_error_t divide_frame_to_fus(uint8_t* data, size_t& data_left, size_t& data_pos, size_t payload_size, + rtp_error_t divide_frame_to_fus(uint8_t* data, size_t& data_left, size_t payload_size, uvgrtp::buf_vec& buffers, uint8_t fu_headers[]); void initialize_fu_headers(uint8_t nal_type, uint8_t fu_headers[]); @@ -141,12 +141,13 @@ namespace uvgrtp { virtual uint8_t get_nal_type(uint8_t* data) const = 0; virtual uint8_t get_payload_header_size() const = 0; + virtual uint8_t get_nal_header_size() const = 0; virtual uint8_t get_fu_header_size() const = 0; virtual uint8_t get_start_code_range() const = 0; virtual int get_fragment_type(uvgrtp::frame::rtp_frame* frame) const = 0; virtual uvgrtp::formats::NAL_TYPES get_nal_type(uvgrtp::frame::rtp_frame* frame) const = 0; - virtual void copy_payload_header(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload); + virtual void get_nal_header_from_fu_headers(size_t fptr, uint8_t* frame_payload, uint8_t* complete_payload); private: diff --git a/test/test_4_formats.cpp b/test/test_4_formats.cpp index 818d46e..d2b5ff3 100644 --- a/test/test_4_formats.cpp +++ b/test/test_4_formats.cpp @@ -28,11 +28,16 @@ TEST(FormatTests, h264) test_packet_size(1443, sess, sender, receiver); test_packet_size(1444, sess, sender, receiver); test_packet_size(1445, sess, sender, receiver); - test_packet_size(1446, sess, sender, receiver); + test_packet_size(1446, sess, sender, receiver); // packet limit test_packet_size(1447, sess, sender, receiver); test_packet_size(1448, sess, sender, receiver); test_packet_size(1449, sess, sender, receiver); test_packet_size(1450, sess, sender, receiver); + test_packet_size(1451, sess, sender, receiver); + test_packet_size(1452, sess, sender, receiver); + test_packet_size(1453, sess, sender, receiver); + test_packet_size(1454, sess, sender, receiver); + test_packet_size(1455, sess, sender, receiver); test_packet_size(1501, sess, sender, receiver); test_packet_size(15000, sess, sender, receiver); test_packet_size(150000, sess, sender, receiver); @@ -61,11 +66,16 @@ TEST(FormatTests, h265) test_packet_size(1443, sess, sender, receiver); test_packet_size(1444, sess, sender, receiver); test_packet_size(1445, sess, sender, receiver); - test_packet_size(1446, sess, sender, receiver); + test_packet_size(1446, sess, sender, receiver); // packet limit test_packet_size(1447, sess, sender, receiver); test_packet_size(1448, sess, sender, receiver); test_packet_size(1449, sess, sender, receiver); test_packet_size(1450, sess, sender, receiver); + test_packet_size(1451, sess, sender, receiver); + test_packet_size(1452, sess, sender, receiver); + test_packet_size(1453, sess, sender, receiver); + test_packet_size(1454, sess, sender, receiver); + test_packet_size(1455, sess, sender, receiver); test_packet_size(1501, sess, sender, receiver); test_packet_size(15000, sess, sender, receiver); test_packet_size(150000, sess, sender, receiver); @@ -94,11 +104,16 @@ TEST(FormatTests, h266) test_packet_size(1443, sess, sender, receiver); test_packet_size(1444, sess, sender, receiver); test_packet_size(1445, sess, sender, receiver); - test_packet_size(1446, sess, sender, receiver); + test_packet_size(1446, sess, sender, receiver); // packet limit test_packet_size(1447, sess, sender, receiver); test_packet_size(1448, sess, sender, receiver); test_packet_size(1449, sess, sender, receiver); test_packet_size(1450, sess, sender, receiver); + test_packet_size(1451, sess, sender, receiver); + test_packet_size(1452, sess, sender, receiver); + test_packet_size(1453, sess, sender, receiver); + test_packet_size(1454, sess, sender, receiver); + test_packet_size(1455, sess, sender, receiver); test_packet_size(1501, sess, sender, receiver); test_packet_size(15000, sess, sender, receiver); test_packet_size(150000, sess, sender, receiver); diff --git a/test/test_common.hh b/test/test_common.hh index 6886418..49448f0 100644 --- a/test/test_common.hh +++ b/test/test_common.hh @@ -69,9 +69,11 @@ inline void send_packets(uvgrtp::session* sess, uvgrtp::media_stream* sender, memset(dummy_frame.get(), 'a', size); } - if (sender->push_frame(std::move(dummy_frame), size, RTP_NO_FLAGS) != RTP_OK) + rtp_error_t ret = RTP_OK; + if ((ret = sender->push_frame(std::move(dummy_frame), size, RTP_NO_FLAGS)) != RTP_OK) { - std::cout << "Failed to send test packet!" << std::endl; + std::cout << "Failed to send test packet! Return value: " << ret << std::endl; + return; } if (i % (packets / 10) == packets / 10 - 1 && print_progress)