From 56b5b129c2fbea3828faaf0cdf882534c2433eff Mon Sep 17 00:00:00 2001
From: dknn <yoann.blein@free.fr>
Date: Wed, 29 Aug 2012 13:14:36 +0200
Subject: Better error resilience
diff --git a/Swift/Controllers/ScreenSharing/ScreenSharingController.cpp b/Swift/Controllers/ScreenSharing/ScreenSharingController.cpp
index a42e9bb..d96dd2f 100644
--- a/Swift/Controllers/ScreenSharing/ScreenSharingController.cpp
+++ b/Swift/Controllers/ScreenSharing/ScreenSharingController.cpp
@@ -23,7 +23,7 @@
namespace Swift {
ScreenSharingController::ScreenSharingController(ScreenSharingManager* screenSharingManager, TimerFactory* timerFactory, DesktopScreenGrabber* desktopScreenGrabber, const JID& to)
- : screenGrabber(desktopScreenGrabber), grabTimer(timerFactory->createTimer(1000/0.5)), remoteScreenWindowFactory(0),
+ : screenGrabber(desktopScreenGrabber), grabTimer(timerFactory->createTimer(1000/1)), remoteScreenWindowFactory(0),
remoteScreenWindow(0), otherParty(to), incoming(false), chatWindow(0)
{
OutgoingScreenSharing::ref oss = screenSharingManager->createOutgoingScreenSharing(to);
diff --git a/Swiften/ScreenSharing/RTPSessionImpl.cpp b/Swiften/ScreenSharing/RTPSessionImpl.cpp
index 33b9c2e..2f90944 100644
--- a/Swiften/ScreenSharing/RTPSessionImpl.cpp
+++ b/Swiften/ScreenSharing/RTPSessionImpl.cpp
@@ -106,8 +106,8 @@ void RTPSessionImpl::sendPacket(const SafeByteArray& data, int timestampinc, boo
void RTPSessionImpl::injectData(const SafeByteArray& data)
{
packetInjecter->InjectRTPorRTCP((void*)(data.data()), data.size(), *jRTPRemotePeer);
- checkIncomingPackets();
poll();
+ checkIncomingPackets();
}
void RTPSessionImpl::stop(int maxWaitMs)
@@ -122,26 +122,22 @@ void RTPSessionImpl::sendSLIFeedback(int pictureID)
int first = 0; // 13 bits
int number = 0; // 13 bits // TODO : Find the total number of macroblocks per frame
- uint32_t data = 0;
- data |= first << 19;
- data |= (number & 2047) << 6;
- data |= (pictureID & 63);
+ uint8_t data[4] = {0, 0, 0, (pictureID & 63)};
- SendUnknownPacket(false, PSFB, PSFB_SLI, (void*)&data, sizeof(uint32_t));
+ SendUnknownPacket(false, PSFB, PSFB_SLI, (void*)data, sizeof(data));
}
void RTPSessionImpl::sendRPSIFeedback(int pictureID)
{
+ SWIFT_LOG(debug) << "sendRPSIFeedback, pictureId = " << pictureID << std::endl;
+
// Send an RPSI as positive feedback. With VP8, it only contains the picture ID (7 bits)
- int pb = 9; // trailing padding bits
- int pt = payloadType.getID(); // payload type (7 bits)
+ uint8_t pb = 9; // trailing padding bits
+ uint8_t pt = payloadType.getID(); // payload type (7 bits)
- uint32_t data = 0;
- data |= pb << 24;
- data |= (pt & 127) << 16;
- data |= (pictureID & 127) << 9;
+ uint8_t data[4] = {pb, pt & 127, (pictureID & 127) << 1, 0};
- SendUnknownPacket(false, PSFB, PSFB_RPSI, (void*)&data, sizeof(uint32_t));
+ SendUnknownPacket(false, PSFB, PSFB_RPSI, (void*)data, sizeof(data));
}
size_t RTPSessionImpl::getMaxRTPPayloadSize() const
@@ -162,13 +158,14 @@ void RTPSessionImpl::OnUnknownPacketType(jrtplib::RTCPPacket* rtcpPack, const jr
if (type != PSFB)
return;
+ size_t hdrSize = 8;
int subtype = rtcpHdr->count;
switch (subtype) {
case PSFB_SLI:
- parseSLIFeedBack(data + 8, len - 8);
+ parseSLIFeedBack(data + hdrSize, len - hdrSize);
break;
case PSFB_RPSI:
- parseRPSIFeedBack(data + 8, len - 8);
+ parseRPSIFeedBack(data + hdrSize, len - hdrSize);
break;
default:
break;
@@ -188,24 +185,21 @@ void RTPSessionImpl::handleDataRead(boost::shared_ptr<SafeByteArray> data)
void RTPSessionImpl::parseSLIFeedBack(uint8_t* data, size_t len)
{
- SWIFT_LOG(debug) << "Got SLI feedback (negative)" << std::endl;
-
if (len < 4)
return;
- int pictureID = data[len - 1] & 63; // pictureID correspond to the 6 last bits
+
+ int pictureID = (data[3] & 63); // pictureID correspond to the 6 lsb
+ SWIFT_LOG(debug) << "Got SLI feedback (negative), pictureID = " << pictureID << std::endl;
onSLIFeedback(pictureID);
}
void RTPSessionImpl::parseRPSIFeedBack(uint8_t* data, size_t len)
{
- SWIFT_LOG(debug) << "Got RPSI feedback (postive)" << std::endl;
-
if (len < 4)
return;
- int pb = data[0]; // First byte : trailing padding bits
- uint32_t intData = *((uint32_t*)(data));
- intData >>= pb; // remove padding bits
- int pictureID = (intData & 127); // pictureID correspond to the 7 last bits
+
+ int pictureID = (data[2] >> 1);
+ SWIFT_LOG(debug) << "Got RPSI feedback (postive), pictureID = " << pictureID << std::endl;
onRPSIFeedback(pictureID);
}
diff --git a/Swiften/ScreenSharing/ReferencePictureSelection.cpp b/Swiften/ScreenSharing/ReferencePictureSelection.cpp
new file mode 100644
index 0000000..dc2a81e
--- /dev/null
+++ b/Swiften/ScreenSharing/ReferencePictureSelection.cpp
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <Swiften/ScreenSharing/ReferencePictureSelection.h>
+#include <Swiften/Base/Log.h>
+
+#include "vpx/vpx_encoder.h"
+#include "vpx/vp8cx.h"
+
+namespace Swift {
+
+ReferencePictureSelection::ReferencePictureSelection()
+ : update_golden_next_(true),
+ established_golden_(false),
+ received_ack_(false),
+ last_sent_ref_picture_id_(0),
+ established_ref_picture_id_(0),
+ send_refresh(false) {
+}
+
+void ReferencePictureSelection::ReceivedRPSI(int rpsi_picture_id) {
+ // Assume RPSI is signaled with 7 bits.
+ SWIFT_LOG(debug) << "ReceivedRPSI: " << rpsi_picture_id << ", " << last_sent_ref_picture_id_ << std::endl;
+ if ((rpsi_picture_id & 127) == (last_sent_ref_picture_id_ & 127)) {
+ // Remote peer has received our last reference frame, switch frame type.
+ received_ack_ = true;
+ established_golden_ = update_golden_next_;
+ update_golden_next_ = !update_golden_next_;
+ established_ref_picture_id_ = last_sent_ref_picture_id_;
+ SWIFT_LOG(debug) << "Ref established: " << established_ref_picture_id_ << std::endl;
+ }
+}
+
+void ReferencePictureSelection::ReceivedSLI() {
+ send_refresh = true;
+}
+
+int ReferencePictureSelection::EncodeFlags(int picture_id) {
+ int flags = 0;
+ // We can't refresh the decoder until we have established the key frame.
+ if (send_refresh/* && received_ack_*/) {
+ flags = VPX_EFLAG_FORCE_KF;
+// flags |= VPX_EFLAG_FORCE_KF;
+ /*
+ flags |= VP8_EFLAG_NO_REF_LAST; // Don't reference the last frame
+
+ if (established_golden_)
+ flags |= VP8_EFLAG_NO_REF_ARF; // Don't reference the alt-ref frame.
+ else
+ flags |= VP8_EFLAG_NO_REF_GF; // Don't reference the golden frame
+ */
+
+
+
+ /*SWIFT_LOG(debug) << "Send refresh" << std::endl;
+ if (established_golden_) {
+ flags |= VP8_EFLAG_FORCE_ARF;
+ flags |= VP8_EFLAG_NO_UPD_GF;
+ flags |= VP8_EFLAG_NO_REF_ARF;
+ } else {
+ flags |= VP8_EFLAG_FORCE_GF;
+ flags |= VP8_EFLAG_NO_UPD_ARF;
+ flags |= VP8_EFLAG_NO_REF_GF;
+ }*/
+ }
+ send_refresh = false;
+
+ // Don't send reference frame updates until we have an established reference.
+ if (received_ack_ && picture_id % 8 == 0 && false) {
+ flags |= VP8_EFLAG_NO_REF_LAST; // Don't reference the last frame.
+ SWIFT_LOG(debug) << "Send reference : " << std::boolalpha << update_golden_next_ << std::endl;
+ if (update_golden_next_) {
+ flags |= VP8_EFLAG_FORCE_GF; // Update the golden reference.
+ flags |= VP8_EFLAG_NO_UPD_ARF; // Don't update alt-ref.
+ flags |= VP8_EFLAG_NO_REF_GF; // Don't reference the golden frame.
+ } else {
+ flags |= VP8_EFLAG_FORCE_ARF; // Update the alt-ref reference.
+ flags |= VP8_EFLAG_NO_UPD_GF; // Don't update the golden frame.
+ flags |= VP8_EFLAG_NO_REF_ARF; // Don't reference the alt-ref frame.
+ }
+ } else {
+ // No update of golden or alt-ref. We can therefore freely reference the
+ // established reference frame and the last frame.
+ if (established_golden_)
+ flags |= VP8_EFLAG_NO_REF_ARF; // Don't reference the alt-ref frame.
+ else
+ flags |= VP8_EFLAG_NO_REF_GF; // Don't reference the golden frame.
+ flags |= VP8_EFLAG_NO_UPD_GF; // Don't update the golden frame.
+ flags |= VP8_EFLAG_NO_UPD_ARF; // Don't update the alt-ref frame.
+ }
+ return flags;
+}
+
+void ReferencePictureSelection::refFrameSent(int picture_id) {
+ last_sent_ref_picture_id_ = picture_id;
+ received_ack_ = false;
+}
+
+}
diff --git a/Swiften/ScreenSharing/ReferencePictureSelection.h b/Swiften/ScreenSharing/ReferencePictureSelection.h
new file mode 100644
index 0000000..ff2f4c0
--- /dev/null
+++ b/Swiften/ScreenSharing/ReferencePictureSelection.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+/*
+ * This file defines classes for doing reference picture selection, primarily
+ * with VP8.
+ */
+
+/* Modified by Yoann Blein, 2012 (remove time handling for simplicity)
+ */
+
+#pragma once
+
+#include <stdint.h>
+
+namespace Swift {
+ class ReferencePictureSelection {
+ public:
+ ReferencePictureSelection();
+
+ // Report a received reference picture selection indication. This will
+ // introduce a new established reference if the received RPSI isn't too late.
+ void ReceivedRPSI(int rpsi_picture_id);
+
+ // Report a received slice loss indication. Returns true if a refresh frame
+ // must be sent to the receiver, which is accomplished by only predicting
+ // from the established reference.
+ // |now_ts| is the RTP timestamp corresponding to the current time. Typically
+ // the capture timestamp of the frame currently being processed.
+ // Returns true if it's time to encode a decoder refresh, otherwise false.
+ void ReceivedSLI();
+
+ // Returns the recommended VP8 encode flags needed. May refresh the decoder
+ // and/or update the reference buffers.
+ // |picture_id| picture id of the frame to be encoded.
+ // |send_refresh| should be set to true if a decoder refresh should be
+ // encoded, otherwise false.
+ // |now_ts| is the RTP timestamp corresponding to the current time. Typically
+ // the capture timestamp of the frame currently being processed.
+ // Returns the flags to be given to the libvpx encoder when encoding the next
+ // frame.
+ int EncodeFlags(int picture_id);
+
+ // Notify the RPS that the frame with picture id |picture_id| was encoded as
+ // a key frame, effectively updating all reference buffers.
+ void refFrameSent(int picture_id);
+
+ private:
+ bool update_golden_next_;
+ bool established_golden_;
+ bool received_ack_;
+ int last_sent_ref_picture_id_;
+ int established_ref_picture_id_;
+ bool send_refresh;
+ };
+}
diff --git a/Swiften/ScreenSharing/SConscript b/Swiften/ScreenSharing/SConscript
index aefe45b..1797828 100644
--- a/Swiften/ScreenSharing/SConscript
+++ b/Swiften/ScreenSharing/SConscript
@@ -10,6 +10,7 @@ sources = [
"ScreenSharingManagerImpl.cpp",
"VP8RTPParser.cpp",
"InputEventResponder.cpp",
+ "ReferencePictureSelection.cpp",
]
objects = swiften_env.SwiftenObject(sources)
diff --git a/Swiften/ScreenSharing/UnitTest/DummyScreenSharingManager.h b/Swiften/ScreenSharing/UnitTest/DummyScreenSharingManager.h
new file mode 100644
index 0000000..f8c963c
--- /dev/null
+++ b/Swiften/ScreenSharing/UnitTest/DummyScreenSharingManager.h
@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2012 Yoann Blein
+ * Licensed under the simplified BSD license.
+ * See Documentation/Licenses/BSD-simplified.txt for more information.
+ */
+
+#pragma once
+
+#include <Swiften/ScreenSharing/ScreenSharingManager.h>
+
+namespace Swift {
+ class DummyScreenSharingManager : public ScreenSharingManager {
+ public:
+ DummyScreenSharingManager() : ScreenSharingManager() {}
+
+ virtual boost::shared_ptr<OutgoingScreenSharing> createOutgoingScreenSharing(const JID& to) {
+ return boost::shared_ptr<OutgoingScreenSharing>();
+ }
+
+ virtual void handleInputEvent(const JID& from, boost::shared_ptr<InputEventPayload> payload) {}
+ };
+}
diff --git a/Swiften/ScreenSharing/VP8Decoder.cpp b/Swiften/ScreenSharing/VP8Decoder.cpp
index 90f5101..2c86b68 100644
--- a/Swiften/ScreenSharing/VP8Decoder.cpp
+++ b/Swiften/ScreenSharing/VP8Decoder.cpp
@@ -47,14 +47,24 @@ void VP8Decoder::updateCodecConfig()
void VP8Decoder::decodeFrame(const std::vector<uint8_t>& frame, int pictureID)
{
+ SWIFT_LOG(debug) << "decodeFrame, pictureID = " << pictureID << std::endl;
// Decode the frame
- vpx_codec_err_t err = vpx_codec_decode(&codecContext, &frame[0], frame.size(), NULL, 0);
+ vpx_codec_err_t err = vpx_codec_decode(&codecContext, frame.data(), frame.size(), 0, 0);
if (err) {
SWIFT_LOG(debug) << "VP8 Decoder: Failed to decode frame, " << vpx_codec_err_to_string(err) << std::endl;
- onNewFrameDecoded(pictureID, false);
- return;
} else {
- onNewFrameDecoded(pictureID, true);
+ }
+
+ int reference_updates = 0;
+ vpx_codec_control(&codecContext, VP8D_GET_LAST_REF_UPDATES, &reference_updates);
+ int corrupted = 0;
+ vpx_codec_control(&codecContext, VP8D_GET_FRAME_CORRUPTED, &corrupted);
+
+ if (!corrupted) {
+ if ((reference_updates & VP8_GOLD_FRAME) || (reference_updates & VP8_ALTR_FRAME))
+ onNewRef(pictureID);
+ } else {
+ onCorrupted(pictureID);
}
vpx_codec_iter_t iter = NULL;
@@ -62,6 +72,7 @@ void VP8Decoder::decodeFrame(const std::vector<uint8_t>& frame, int pictureID)
while ((decodedImg = vpx_codec_get_frame(&codecContext, &iter))) {
Image rgbImg = convertYV12toRGB(decodedImg);
onNewImageAvailable(rgbImg);
+ /*
// Restore ref from last save if corrupted, else save the ref
int corrupted = 0;
vpx_codec_control(&codecContext, VP8D_GET_FRAME_CORRUPTED, &corrupted);
@@ -86,6 +97,7 @@ void VP8Decoder::decodeFrame(const std::vector<uint8_t>& frame, int pictureID)
}
vpx_codec_control(&codecContext, VP8_COPY_REFERENCE, refFrame);
}
+ */
}
}
diff --git a/Swiften/ScreenSharing/VP8Encoder.cpp b/Swiften/ScreenSharing/VP8Encoder.cpp
index e48d6d9..a7f5a69 100644
--- a/Swiften/ScreenSharing/VP8Encoder.cpp
+++ b/Swiften/ScreenSharing/VP8Encoder.cpp
@@ -14,7 +14,7 @@ namespace Swift {
VP8Encoder::VP8Encoder(VP8RTPPacketizer* packetizer, unsigned int width, unsigned int height)
: VideoEncoder(),
- packetizer(packetizer), codecInterface(vpx_codec_vp8_cx()), imageBuffer(0), codecFlags(0), frameFlags(0), frameNumber(0), pictureID(0), rpsiReceived(true)
+ packetizer(packetizer), codecInterface(vpx_codec_vp8_cx()), imageBuffer(0), codecFlags(0), frameFlags(0), frameNumber(0), pictureID(0)
{
SWIFT_LOG(debug) << "VP8 Encoder:" << vpx_codec_iface_name(codecInterface) << std::endl;
@@ -63,7 +63,6 @@ void VP8Encoder::updateCodecConfig()
frameNumber = 0;
pictureID = rand() % 128;
- rpsiReceived = true;
}
void VP8Encoder::encodeImage(const Image& frame)
@@ -74,13 +73,16 @@ void VP8Encoder::encodeImage(const Image& frame)
return;
}
+ /*
vpx_enc_frame_flags_t localFrameFlags = frameFlags;
if (!rpsiReceived && frameNumber > 1) {
SWIFT_LOG(error) << "VP8 Encoder: RPSI not received, do not ref last frame" << std::endl;
localFrameFlags |= VP8_EFLAG_NO_REF_LAST; // VP8_EFLAG_FORCE_GF; // TODO: VP8_EFLAG_NO_REF_LAST;
}
+ */
- vpx_codec_err_t err = vpx_codec_encode(&codecContext, imageBuffer, frameNumber, 1, localFrameFlags, VPX_DL_REALTIME);
+ vpx_enc_frame_flags_t flags = rps.encodeFlags(pictureID);
+ vpx_codec_err_t err = vpx_codec_encode(&codecContext, imageBuffer, frameNumber, 1, flags, VPX_DL_REALTIME);
if (err) {
SWIFT_LOG(error) << "VP8 Encoder: Failed to encode frame, " << vpx_codec_err_to_string(err) << std::endl;
// TODO: exception ?
@@ -89,9 +91,15 @@ void VP8Encoder::encodeImage(const Image& frame)
const vpx_codec_cx_pkt_t* pkt;
vpx_codec_iter_t iter = NULL;
+ int vp8Flags;
while ((pkt = vpx_codec_get_cx_data(&codecContext, &iter))) {
switch (pkt->kind) {
case VPX_CODEC_CX_FRAME_PKT:
+ vp8Flags = pkt->data.frame.flags >> 16;
+ SWIFT_LOG(debug) << "vp8Flags: " << vp8Flags << ", len: " << pkt->data.frame.sz << std::endl;
+ if ((vp8Flags & VP8_ALTR_FRAME) || (vp8Flags & VP8_GOLD_FRAME)) {
+ rps.refFrameSent(pictureID);
+ }
packetizer->packetizeFrame(pkt, pictureID);
break;
default:
@@ -101,12 +109,16 @@ void VP8Encoder::encodeImage(const Image& frame)
++frameNumber;
pictureID = (pictureID + 1) % 128;
- rpsiReceived = false;
}
-void VP8Encoder::handleRPSIFeedback(int /*pictureID*/)
+void VP8Encoder::handleRPSIFeedback(int pictureID)
{
- rpsiReceived = true;
+ rps.receivedRPSI(pictureID);
+}
+
+void VP8Encoder::handleSLIFeedback(int /*pictureID*/)
+{
+ rps.receivedSLI();
}
bool VP8Encoder::convertRGB24toYV12inBuffer(const Image& frame)
diff --git a/Swiften/ScreenSharing/VP8Encoder.h b/Swiften/ScreenSharing/VP8Encoder.h
index 62a6ea8..8722f1b 100644
--- a/Swiften/ScreenSharing/VP8Encoder.h
+++ b/Swiften/ScreenSharing/VP8Encoder.h
@@ -7,6 +7,7 @@
#pragma once
#include <Swiften/ScreenSharing/VideoEncoder.h>
+#include <Swiften/ScreenSharing/ReferencePictureSelection.h>
#define VPX_CODEC_DISABLE_COMPAT 1 // Recomended
#include "vpx/vpx_encoder.h"
@@ -25,7 +26,8 @@ namespace Swift {
virtual void encodeImage(const Image& frame);
- virtual void handleRPSIFeedback(int /*pictureID*/);
+ virtual void handleRPSIFeedback(int pictureID);
+ virtual void handleSLIFeedback(int /*pictureID*/);
private:
bool convertRGB24toYV12inBuffer(const Image& frame);
@@ -40,7 +42,7 @@ namespace Swift {
vpx_enc_frame_flags_t frameFlags;
int frameNumber;
int pictureID;
- bool rpsiReceived;
+ ReferencePictureSelection rps;
};
}
diff --git a/Swiften/ScreenSharing/VP8RTPParser.cpp b/Swiften/ScreenSharing/VP8RTPParser.cpp
index d44140b..3701ed2 100644
--- a/Swiften/ScreenSharing/VP8RTPParser.cpp
+++ b/Swiften/ScreenSharing/VP8RTPParser.cpp
@@ -27,7 +27,7 @@ void VP8RTPParser::newPayloadReceived(const uint8_t* data, size_t len, bool hasM
uint8_t optX = data[headerSize++];
if (optX & IBit) { // PictureID byte is present
uint8_t optI = data[headerSize++];
- pictureID = optI >> 1;
+ pictureID = (optI & 127);
}
}
diff --git a/Swiften/ScreenSharing/VideoDecoder.h b/Swiften/ScreenSharing/VideoDecoder.h
index fdec6b3..d7ad55e 100644
--- a/Swiften/ScreenSharing/VideoDecoder.h
+++ b/Swiften/ScreenSharing/VideoDecoder.h
@@ -21,7 +21,8 @@ namespace Swift {
virtual void decodeFrame(const std::vector<uint8_t>& frame, int pictureID) = 0;
public:
- boost::signal<void (int pictureID, bool success)> onNewFrameDecoded;
+ boost::signal<void (int pictureID)> onNewRef;
+ boost::signal<void (int pictureID)> onCorrupted;
boost::signal<void (const Image&)> onNewImageAvailable;
};
}
diff --git a/Swiften/ScreenSharing/VideoEncoder.h b/Swiften/ScreenSharing/VideoEncoder.h
index c28b6d1..c07f4a3 100644
--- a/Swiften/ScreenSharing/VideoEncoder.h
+++ b/Swiften/ScreenSharing/VideoEncoder.h
@@ -20,6 +20,7 @@ namespace Swift {
virtual void encodeImage(const Image& frame) = 0;
- virtual void handleRPSIFeedback(int /*pictureID*/) = 0;
+ virtual void handleRPSIFeedback(int pictureID) = 0;
+ virtual void handleSLIFeedback(int pictureID) = 0;
};
}
--
cgit v0.10.2-6-g49f6