From 56b5b129c2fbea3828faaf0cdf882534c2433eff Mon Sep 17 00:00:00 2001 From: dknn Date: Wed, 29 Aug 2012 13:14:36 +0200 Subject: Better error resilience diff --git a/Swift/Controllers/ScreenSharing/ScreenSharingController.cpp b/Swift/Controllers/ScreenSharing/ScreenSharingController.cpp index a42e9bb..d96dd2f 100644 --- a/Swift/Controllers/ScreenSharing/ScreenSharingController.cpp +++ b/Swift/Controllers/ScreenSharing/ScreenSharingController.cpp @@ -23,7 +23,7 @@ namespace Swift { ScreenSharingController::ScreenSharingController(ScreenSharingManager* screenSharingManager, TimerFactory* timerFactory, DesktopScreenGrabber* desktopScreenGrabber, const JID& to) - : screenGrabber(desktopScreenGrabber), grabTimer(timerFactory->createTimer(1000/0.5)), remoteScreenWindowFactory(0), + : screenGrabber(desktopScreenGrabber), grabTimer(timerFactory->createTimer(1000/1)), remoteScreenWindowFactory(0), remoteScreenWindow(0), otherParty(to), incoming(false), chatWindow(0) { OutgoingScreenSharing::ref oss = screenSharingManager->createOutgoingScreenSharing(to); diff --git a/Swiften/ScreenSharing/RTPSessionImpl.cpp b/Swiften/ScreenSharing/RTPSessionImpl.cpp index 33b9c2e..2f90944 100644 --- a/Swiften/ScreenSharing/RTPSessionImpl.cpp +++ b/Swiften/ScreenSharing/RTPSessionImpl.cpp @@ -106,8 +106,8 @@ void RTPSessionImpl::sendPacket(const SafeByteArray& data, int timestampinc, boo void RTPSessionImpl::injectData(const SafeByteArray& data) { packetInjecter->InjectRTPorRTCP((void*)(data.data()), data.size(), *jRTPRemotePeer); - checkIncomingPackets(); poll(); + checkIncomingPackets(); } void RTPSessionImpl::stop(int maxWaitMs) @@ -122,26 +122,22 @@ void RTPSessionImpl::sendSLIFeedback(int pictureID) int first = 0; // 13 bits int number = 0; // 13 bits // TODO : Find the total number of macroblocks per frame - uint32_t data = 0; - data |= first << 19; - data |= (number & 2047) << 6; - data |= (pictureID & 63); + uint8_t data[4] = {0, 0, 0, (pictureID & 63)}; - SendUnknownPacket(false, PSFB, PSFB_SLI, (void*)&data, sizeof(uint32_t)); + SendUnknownPacket(false, PSFB, PSFB_SLI, (void*)data, sizeof(data)); } void RTPSessionImpl::sendRPSIFeedback(int pictureID) { + SWIFT_LOG(debug) << "sendRPSIFeedback, pictureId = " << pictureID << std::endl; + // Send an RPSI as positive feedback. With VP8, it only contains the picture ID (7 bits) - int pb = 9; // trailing padding bits - int pt = payloadType.getID(); // payload type (7 bits) + uint8_t pb = 9; // trailing padding bits + uint8_t pt = payloadType.getID(); // payload type (7 bits) - uint32_t data = 0; - data |= pb << 24; - data |= (pt & 127) << 16; - data |= (pictureID & 127) << 9; + uint8_t data[4] = {pb, pt & 127, (pictureID & 127) << 1, 0}; - SendUnknownPacket(false, PSFB, PSFB_RPSI, (void*)&data, sizeof(uint32_t)); + SendUnknownPacket(false, PSFB, PSFB_RPSI, (void*)data, sizeof(data)); } size_t RTPSessionImpl::getMaxRTPPayloadSize() const @@ -162,13 +158,14 @@ void RTPSessionImpl::OnUnknownPacketType(jrtplib::RTCPPacket* rtcpPack, const jr if (type != PSFB) return; + size_t hdrSize = 8; int subtype = rtcpHdr->count; switch (subtype) { case PSFB_SLI: - parseSLIFeedBack(data + 8, len - 8); + parseSLIFeedBack(data + hdrSize, len - hdrSize); break; case PSFB_RPSI: - parseRPSIFeedBack(data + 8, len - 8); + parseRPSIFeedBack(data + hdrSize, len - hdrSize); break; default: break; @@ -188,24 +185,21 @@ void RTPSessionImpl::handleDataRead(boost::shared_ptr data) void RTPSessionImpl::parseSLIFeedBack(uint8_t* data, size_t len) { - SWIFT_LOG(debug) << "Got SLI feedback (negative)" << std::endl; - if (len < 4) return; - int pictureID = data[len - 1] & 63; // pictureID correspond to the 6 last bits + + int pictureID = (data[3] & 63); // pictureID correspond to the 6 lsb + SWIFT_LOG(debug) << "Got SLI feedback (negative), pictureID = " << pictureID << std::endl; onSLIFeedback(pictureID); } void RTPSessionImpl::parseRPSIFeedBack(uint8_t* data, size_t len) { - SWIFT_LOG(debug) << "Got RPSI feedback (postive)" << std::endl; - if (len < 4) return; - int pb = data[0]; // First byte : trailing padding bits - uint32_t intData = *((uint32_t*)(data)); - intData >>= pb; // remove padding bits - int pictureID = (intData & 127); // pictureID correspond to the 7 last bits + + int pictureID = (data[2] >> 1); + SWIFT_LOG(debug) << "Got RPSI feedback (postive), pictureID = " << pictureID << std::endl; onRPSIFeedback(pictureID); } diff --git a/Swiften/ScreenSharing/ReferencePictureSelection.cpp b/Swiften/ScreenSharing/ReferencePictureSelection.cpp new file mode 100644 index 0000000..dc2a81e --- /dev/null +++ b/Swiften/ScreenSharing/ReferencePictureSelection.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "vpx/vpx_encoder.h" +#include "vpx/vp8cx.h" + +namespace Swift { + +ReferencePictureSelection::ReferencePictureSelection() + : update_golden_next_(true), + established_golden_(false), + received_ack_(false), + last_sent_ref_picture_id_(0), + established_ref_picture_id_(0), + send_refresh(false) { +} + +void ReferencePictureSelection::ReceivedRPSI(int rpsi_picture_id) { + // Assume RPSI is signaled with 7 bits. + SWIFT_LOG(debug) << "ReceivedRPSI: " << rpsi_picture_id << ", " << last_sent_ref_picture_id_ << std::endl; + if ((rpsi_picture_id & 127) == (last_sent_ref_picture_id_ & 127)) { + // Remote peer has received our last reference frame, switch frame type. + received_ack_ = true; + established_golden_ = update_golden_next_; + update_golden_next_ = !update_golden_next_; + established_ref_picture_id_ = last_sent_ref_picture_id_; + SWIFT_LOG(debug) << "Ref established: " << established_ref_picture_id_ << std::endl; + } +} + +void ReferencePictureSelection::ReceivedSLI() { + send_refresh = true; +} + +int ReferencePictureSelection::EncodeFlags(int picture_id) { + int flags = 0; + // We can't refresh the decoder until we have established the key frame. + if (send_refresh/* && received_ack_*/) { + flags = VPX_EFLAG_FORCE_KF; +// flags |= VPX_EFLAG_FORCE_KF; + /* + flags |= VP8_EFLAG_NO_REF_LAST; // Don't reference the last frame + + if (established_golden_) + flags |= VP8_EFLAG_NO_REF_ARF; // Don't reference the alt-ref frame. + else + flags |= VP8_EFLAG_NO_REF_GF; // Don't reference the golden frame + */ + + + + /*SWIFT_LOG(debug) << "Send refresh" << std::endl; + if (established_golden_) { + flags |= VP8_EFLAG_FORCE_ARF; + flags |= VP8_EFLAG_NO_UPD_GF; + flags |= VP8_EFLAG_NO_REF_ARF; + } else { + flags |= VP8_EFLAG_FORCE_GF; + flags |= VP8_EFLAG_NO_UPD_ARF; + flags |= VP8_EFLAG_NO_REF_GF; + }*/ + } + send_refresh = false; + + // Don't send reference frame updates until we have an established reference. + if (received_ack_ && picture_id % 8 == 0 && false) { + flags |= VP8_EFLAG_NO_REF_LAST; // Don't reference the last frame. + SWIFT_LOG(debug) << "Send reference : " << std::boolalpha << update_golden_next_ << std::endl; + if (update_golden_next_) { + flags |= VP8_EFLAG_FORCE_GF; // Update the golden reference. + flags |= VP8_EFLAG_NO_UPD_ARF; // Don't update alt-ref. + flags |= VP8_EFLAG_NO_REF_GF; // Don't reference the golden frame. + } else { + flags |= VP8_EFLAG_FORCE_ARF; // Update the alt-ref reference. + flags |= VP8_EFLAG_NO_UPD_GF; // Don't update the golden frame. + flags |= VP8_EFLAG_NO_REF_ARF; // Don't reference the alt-ref frame. + } + } else { + // No update of golden or alt-ref. We can therefore freely reference the + // established reference frame and the last frame. + if (established_golden_) + flags |= VP8_EFLAG_NO_REF_ARF; // Don't reference the alt-ref frame. + else + flags |= VP8_EFLAG_NO_REF_GF; // Don't reference the golden frame. + flags |= VP8_EFLAG_NO_UPD_GF; // Don't update the golden frame. + flags |= VP8_EFLAG_NO_UPD_ARF; // Don't update the alt-ref frame. + } + return flags; +} + +void ReferencePictureSelection::refFrameSent(int picture_id) { + last_sent_ref_picture_id_ = picture_id; + received_ack_ = false; +} + +} diff --git a/Swiften/ScreenSharing/ReferencePictureSelection.h b/Swiften/ScreenSharing/ReferencePictureSelection.h new file mode 100644 index 0000000..ff2f4c0 --- /dev/null +++ b/Swiften/ScreenSharing/ReferencePictureSelection.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +/* + * This file defines classes for doing reference picture selection, primarily + * with VP8. + */ + +/* Modified by Yoann Blein, 2012 (remove time handling for simplicity) + */ + +#pragma once + +#include + +namespace Swift { + class ReferencePictureSelection { + public: + ReferencePictureSelection(); + + // Report a received reference picture selection indication. This will + // introduce a new established reference if the received RPSI isn't too late. + void ReceivedRPSI(int rpsi_picture_id); + + // Report a received slice loss indication. Returns true if a refresh frame + // must be sent to the receiver, which is accomplished by only predicting + // from the established reference. + // |now_ts| is the RTP timestamp corresponding to the current time. Typically + // the capture timestamp of the frame currently being processed. + // Returns true if it's time to encode a decoder refresh, otherwise false. + void ReceivedSLI(); + + // Returns the recommended VP8 encode flags needed. May refresh the decoder + // and/or update the reference buffers. + // |picture_id| picture id of the frame to be encoded. + // |send_refresh| should be set to true if a decoder refresh should be + // encoded, otherwise false. + // |now_ts| is the RTP timestamp corresponding to the current time. Typically + // the capture timestamp of the frame currently being processed. + // Returns the flags to be given to the libvpx encoder when encoding the next + // frame. + int EncodeFlags(int picture_id); + + // Notify the RPS that the frame with picture id |picture_id| was encoded as + // a key frame, effectively updating all reference buffers. + void refFrameSent(int picture_id); + + private: + bool update_golden_next_; + bool established_golden_; + bool received_ack_; + int last_sent_ref_picture_id_; + int established_ref_picture_id_; + bool send_refresh; + }; +} diff --git a/Swiften/ScreenSharing/SConscript b/Swiften/ScreenSharing/SConscript index aefe45b..1797828 100644 --- a/Swiften/ScreenSharing/SConscript +++ b/Swiften/ScreenSharing/SConscript @@ -10,6 +10,7 @@ sources = [ "ScreenSharingManagerImpl.cpp", "VP8RTPParser.cpp", "InputEventResponder.cpp", + "ReferencePictureSelection.cpp", ] objects = swiften_env.SwiftenObject(sources) diff --git a/Swiften/ScreenSharing/UnitTest/DummyScreenSharingManager.h b/Swiften/ScreenSharing/UnitTest/DummyScreenSharingManager.h new file mode 100644 index 0000000..f8c963c --- /dev/null +++ b/Swiften/ScreenSharing/UnitTest/DummyScreenSharingManager.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2012 Yoann Blein + * Licensed under the simplified BSD license. + * See Documentation/Licenses/BSD-simplified.txt for more information. + */ + +#pragma once + +#include + +namespace Swift { + class DummyScreenSharingManager : public ScreenSharingManager { + public: + DummyScreenSharingManager() : ScreenSharingManager() {} + + virtual boost::shared_ptr createOutgoingScreenSharing(const JID& to) { + return boost::shared_ptr(); + } + + virtual void handleInputEvent(const JID& from, boost::shared_ptr payload) {} + }; +} diff --git a/Swiften/ScreenSharing/VP8Decoder.cpp b/Swiften/ScreenSharing/VP8Decoder.cpp index 90f5101..2c86b68 100644 --- a/Swiften/ScreenSharing/VP8Decoder.cpp +++ b/Swiften/ScreenSharing/VP8Decoder.cpp @@ -47,14 +47,24 @@ void VP8Decoder::updateCodecConfig() void VP8Decoder::decodeFrame(const std::vector& frame, int pictureID) { + SWIFT_LOG(debug) << "decodeFrame, pictureID = " << pictureID << std::endl; // Decode the frame - vpx_codec_err_t err = vpx_codec_decode(&codecContext, &frame[0], frame.size(), NULL, 0); + vpx_codec_err_t err = vpx_codec_decode(&codecContext, frame.data(), frame.size(), 0, 0); if (err) { SWIFT_LOG(debug) << "VP8 Decoder: Failed to decode frame, " << vpx_codec_err_to_string(err) << std::endl; - onNewFrameDecoded(pictureID, false); - return; } else { - onNewFrameDecoded(pictureID, true); + } + + int reference_updates = 0; + vpx_codec_control(&codecContext, VP8D_GET_LAST_REF_UPDATES, &reference_updates); + int corrupted = 0; + vpx_codec_control(&codecContext, VP8D_GET_FRAME_CORRUPTED, &corrupted); + + if (!corrupted) { + if ((reference_updates & VP8_GOLD_FRAME) || (reference_updates & VP8_ALTR_FRAME)) + onNewRef(pictureID); + } else { + onCorrupted(pictureID); } vpx_codec_iter_t iter = NULL; @@ -62,6 +72,7 @@ void VP8Decoder::decodeFrame(const std::vector& frame, int pictureID) while ((decodedImg = vpx_codec_get_frame(&codecContext, &iter))) { Image rgbImg = convertYV12toRGB(decodedImg); onNewImageAvailable(rgbImg); + /* // Restore ref from last save if corrupted, else save the ref int corrupted = 0; vpx_codec_control(&codecContext, VP8D_GET_FRAME_CORRUPTED, &corrupted); @@ -86,6 +97,7 @@ void VP8Decoder::decodeFrame(const std::vector& frame, int pictureID) } vpx_codec_control(&codecContext, VP8_COPY_REFERENCE, refFrame); } + */ } } diff --git a/Swiften/ScreenSharing/VP8Encoder.cpp b/Swiften/ScreenSharing/VP8Encoder.cpp index e48d6d9..a7f5a69 100644 --- a/Swiften/ScreenSharing/VP8Encoder.cpp +++ b/Swiften/ScreenSharing/VP8Encoder.cpp @@ -14,7 +14,7 @@ namespace Swift { VP8Encoder::VP8Encoder(VP8RTPPacketizer* packetizer, unsigned int width, unsigned int height) : VideoEncoder(), - packetizer(packetizer), codecInterface(vpx_codec_vp8_cx()), imageBuffer(0), codecFlags(0), frameFlags(0), frameNumber(0), pictureID(0), rpsiReceived(true) + packetizer(packetizer), codecInterface(vpx_codec_vp8_cx()), imageBuffer(0), codecFlags(0), frameFlags(0), frameNumber(0), pictureID(0) { SWIFT_LOG(debug) << "VP8 Encoder:" << vpx_codec_iface_name(codecInterface) << std::endl; @@ -63,7 +63,6 @@ void VP8Encoder::updateCodecConfig() frameNumber = 0; pictureID = rand() % 128; - rpsiReceived = true; } void VP8Encoder::encodeImage(const Image& frame) @@ -74,13 +73,16 @@ void VP8Encoder::encodeImage(const Image& frame) return; } + /* vpx_enc_frame_flags_t localFrameFlags = frameFlags; if (!rpsiReceived && frameNumber > 1) { SWIFT_LOG(error) << "VP8 Encoder: RPSI not received, do not ref last frame" << std::endl; localFrameFlags |= VP8_EFLAG_NO_REF_LAST; // VP8_EFLAG_FORCE_GF; // TODO: VP8_EFLAG_NO_REF_LAST; } + */ - vpx_codec_err_t err = vpx_codec_encode(&codecContext, imageBuffer, frameNumber, 1, localFrameFlags, VPX_DL_REALTIME); + vpx_enc_frame_flags_t flags = rps.encodeFlags(pictureID); + vpx_codec_err_t err = vpx_codec_encode(&codecContext, imageBuffer, frameNumber, 1, flags, VPX_DL_REALTIME); if (err) { SWIFT_LOG(error) << "VP8 Encoder: Failed to encode frame, " << vpx_codec_err_to_string(err) << std::endl; // TODO: exception ? @@ -89,9 +91,15 @@ void VP8Encoder::encodeImage(const Image& frame) const vpx_codec_cx_pkt_t* pkt; vpx_codec_iter_t iter = NULL; + int vp8Flags; while ((pkt = vpx_codec_get_cx_data(&codecContext, &iter))) { switch (pkt->kind) { case VPX_CODEC_CX_FRAME_PKT: + vp8Flags = pkt->data.frame.flags >> 16; + SWIFT_LOG(debug) << "vp8Flags: " << vp8Flags << ", len: " << pkt->data.frame.sz << std::endl; + if ((vp8Flags & VP8_ALTR_FRAME) || (vp8Flags & VP8_GOLD_FRAME)) { + rps.refFrameSent(pictureID); + } packetizer->packetizeFrame(pkt, pictureID); break; default: @@ -101,12 +109,16 @@ void VP8Encoder::encodeImage(const Image& frame) ++frameNumber; pictureID = (pictureID + 1) % 128; - rpsiReceived = false; } -void VP8Encoder::handleRPSIFeedback(int /*pictureID*/) +void VP8Encoder::handleRPSIFeedback(int pictureID) { - rpsiReceived = true; + rps.receivedRPSI(pictureID); +} + +void VP8Encoder::handleSLIFeedback(int /*pictureID*/) +{ + rps.receivedSLI(); } bool VP8Encoder::convertRGB24toYV12inBuffer(const Image& frame) diff --git a/Swiften/ScreenSharing/VP8Encoder.h b/Swiften/ScreenSharing/VP8Encoder.h index 62a6ea8..8722f1b 100644 --- a/Swiften/ScreenSharing/VP8Encoder.h +++ b/Swiften/ScreenSharing/VP8Encoder.h @@ -7,6 +7,7 @@ #pragma once #include +#include #define VPX_CODEC_DISABLE_COMPAT 1 // Recomended #include "vpx/vpx_encoder.h" @@ -25,7 +26,8 @@ namespace Swift { virtual void encodeImage(const Image& frame); - virtual void handleRPSIFeedback(int /*pictureID*/); + virtual void handleRPSIFeedback(int pictureID); + virtual void handleSLIFeedback(int /*pictureID*/); private: bool convertRGB24toYV12inBuffer(const Image& frame); @@ -40,7 +42,7 @@ namespace Swift { vpx_enc_frame_flags_t frameFlags; int frameNumber; int pictureID; - bool rpsiReceived; + ReferencePictureSelection rps; }; } diff --git a/Swiften/ScreenSharing/VP8RTPParser.cpp b/Swiften/ScreenSharing/VP8RTPParser.cpp index d44140b..3701ed2 100644 --- a/Swiften/ScreenSharing/VP8RTPParser.cpp +++ b/Swiften/ScreenSharing/VP8RTPParser.cpp @@ -27,7 +27,7 @@ void VP8RTPParser::newPayloadReceived(const uint8_t* data, size_t len, bool hasM uint8_t optX = data[headerSize++]; if (optX & IBit) { // PictureID byte is present uint8_t optI = data[headerSize++]; - pictureID = optI >> 1; + pictureID = (optI & 127); } } diff --git a/Swiften/ScreenSharing/VideoDecoder.h b/Swiften/ScreenSharing/VideoDecoder.h index fdec6b3..d7ad55e 100644 --- a/Swiften/ScreenSharing/VideoDecoder.h +++ b/Swiften/ScreenSharing/VideoDecoder.h @@ -21,7 +21,8 @@ namespace Swift { virtual void decodeFrame(const std::vector& frame, int pictureID) = 0; public: - boost::signal onNewFrameDecoded; + boost::signal onNewRef; + boost::signal onCorrupted; boost::signal onNewImageAvailable; }; } diff --git a/Swiften/ScreenSharing/VideoEncoder.h b/Swiften/ScreenSharing/VideoEncoder.h index c28b6d1..c07f4a3 100644 --- a/Swiften/ScreenSharing/VideoEncoder.h +++ b/Swiften/ScreenSharing/VideoEncoder.h @@ -20,6 +20,7 @@ namespace Swift { virtual void encodeImage(const Image& frame) = 0; - virtual void handleRPSIFeedback(int /*pictureID*/) = 0; + virtual void handleRPSIFeedback(int pictureID) = 0; + virtual void handleSLIFeedback(int pictureID) = 0; }; } -- cgit v0.10.2-6-g49f6