Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial support for VP9 and AV1 simulcast (and fix for broken AV1/SVC) #3218

Merged
merged 6 commits into from
May 22, 2023
+316 −135
Merged

Initial support for VP9 and AV1 simulcast (and fix for broken AV1/SVC) #3218

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
73c3de2
Initial support for VP9 and AV1 simulcast
lminiero May 10, 2023
c64793b
Read VP9 temporal layers from payload descriptor (SVC code)
lminiero May 11, 2023
cf6eb00
Use Dependency Descriptor for temporal layers in AV1 simulcasting, if…
lminiero May 11, 2023
f282022
Show temporal layer buttons in EchoTest demos also for VP9 and AV1
lminiero May 11, 2023
8de2c78
Made AV1/SVC finally work
lminiero May 12, 2023
a001ded
Streamlined simulcast usage when involving DD, and added support to o…
lminiero May 16, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion html/echotest.js
View file
Original file line number Diff line number Diff line change
@@ -203,7 +203,7 @@ $(document).ready(function() {
if((substream !== null && substream !== undefined) || (temporal !== null && temporal !== undefined)) {
if(!simulcastStarted) {
simulcastStarted = true;
addSimulcastSvcButtons(msg["videocodec"] === "vp8");
addSimulcastSvcButtons(msg["videocodec"] === "vp8" || msg["videocodec"] === "vp9" || msg["videocodec"] === "av1");
}
// We just received notice that there's been a switch, update the buttons
updateSimulcastSvcButtons(substream, temporal);
6 changes: 3 additions & 3 deletions html/janus.js
View file
Original file line number Diff line number Diff line change
@@ -2497,9 +2497,9 @@ function Janus(gatewayCallbacks) {
direction: 'sendrecv',
streams: [config.myStream],
sendEncodings: track.sendEncodings || [
{ rid: 'h', active: true, maxBitrate: maxBitrates.high },
{ rid: 'm', active: true, maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 },
{ rid: 'l', active: true, maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 }
{ rid: 'h', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.high },
{ rid: 'm', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.medium, scaleResolutionDownBy: 2 },
{ rid: 'l', active: true, scalabilityMode: 'L1T2', maxBitrate: maxBitrates.low, scaleResolutionDownBy: 4 }
]
});
} else {
2 changes: 2 additions & 0 deletions src/plugins/duktape/janus-sdp.js
View file
Original file line number Diff line number Diff line change
@@ -399,6 +399,8 @@ JANUSSDP.generateAnswer = function(offer, options) {
answer.push({ type: "a", name: a.name, value: value });
} else if(options.enableAudioLevel !== false && a.value.indexOf("urn:ietf:params:rtp-hdrext:ssrc-audio-level") !== -1) {
answer.push({ type: "a", name: a.name, value: value });
} else if(options.enableAudioLevel !== false && a.value.indexOf("dependency-descriptor-rtp-header-extension") !== -1) {
answer.push({ type: "a", name: a.name, value: value });
}
}
} else {
13 changes: 3 additions & 10 deletions src/plugins/janus_duktape.c
View file
Original file line number Diff line number Diff line change
@@ -619,10 +619,6 @@ static duk_ret_t janus_duktape_method_pushevent(duk_context *ctx) {
janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
if(vcodec)
session->vcodec = janus_videocodec_from_name(vcodec);
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
}
}
janus_sdp_destroy(parsed_sdp);
/* Send asynchronously */
@@ -2240,10 +2236,6 @@ struct janus_plugin_result *janus_duktape_handle_message(janus_plugin_session *h
janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
if(vcodec)
session->vcodec = janus_videocodec_from_name(vcodec);
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
}
janus_sdp_destroy(parsed_sdp);
}
if(json_is_true(json_object_get(jsep, "e2ee")))
@@ -2459,7 +2451,7 @@ void janus_duktape_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *
} else {
/* We're simulcasting, save the best video quality */
gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx,
buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
if(save) {
uint32_t seq_number = ntohs(rtp->seq_number);
uint32_t timestamp = ntohl(rtp->timestamp);
@@ -2781,7 +2773,8 @@ static void janus_duktape_relay_rtp_packet(gpointer data, gpointer user_data) {
return;
/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
(char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
(char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len,
packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
if(session->sim_context.need_pli && sender->handle) {
/* Send a PLI */
JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n");
11 changes: 4 additions & 7 deletions src/plugins/janus_echotest.c
View file
Original file line number Diff line number Diff line change
@@ -624,11 +624,12 @@ void janus_echotest_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp
if(simulcast) {
/* Process this simulcast packet: don't relay if it's not the SSRC/layer we wanted to handle */
relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
buf, len, session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex);
buf, len, packet->extensions.dd_content, packet->extensions.dd_len,
session->ssrc, session->rid, session->vcodec, &session->context, &session->rid_mutex);
} else {
/* Process this SVC packet: don't relay if it's not the layer we wanted to handle */
relay = janus_rtp_svc_context_process_rtp(&session->svc_context,
buf, len, session->vcodec, NULL, &session->context);
buf, len, packet->extensions.dd_content, packet->extensions.dd_len, session->vcodec, NULL, &session->context);
}
if(session->sim_context.need_pli || session->svc_context.need_pli) {
/* Send a PLI */
@@ -1088,7 +1089,7 @@ static void *janus_echotest_handler(void *data) {
session->sim_context.templayer_target = json_integer_value(temporal);
JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n",
session->sim_context.templayer_target, session->sim_context.templayer);
if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) {
if(session->sim_context.templayer_target == session->sim_context.templayer) {
/* No need to do anything, we're already getting the right temporal, so notify the user */
json_t *event = json_object();
json_object_set_new(event, "echotest", json_string("event"));
@@ -1280,10 +1281,6 @@ static void *janus_echotest_handler(void *data) {
session->vcodec = janus_videocodec_from_name(vcodec);
session->has_audio = session->acodec != JANUS_AUDIOCODEC_NONE;
session->has_video = session->vcodec != JANUS_VIDEOCODEC_NONE;
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
}
g_free(session->vfmtp);
session->vfmtp = NULL;
if(session->has_video) {
13 changes: 3 additions & 10 deletions src/plugins/janus_lua.c
View file
Original file line number Diff line number Diff line change
@@ -547,10 +547,6 @@ static int janus_lua_method_pushevent(lua_State *s) {
janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
if(vcodec)
session->vcodec = janus_videocodec_from_name(vcodec);
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
}
}
janus_sdp_destroy(parsed_sdp);
/* Send asynchronously */
@@ -1936,10 +1932,6 @@ struct janus_plugin_result *janus_lua_handle_message(janus_plugin_session *handl
janus_sdp_find_first_codec(parsed_sdp, JANUS_SDP_VIDEO, -1, &vcodec);
if(vcodec)
session->vcodec = janus_videocodec_from_name(vcodec);
if(session->vcodec != JANUS_VIDEOCODEC_VP8 && session->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(&session->rid_extmap_id, session->ssrc, session->rid, &session->rid_mutex);
}
janus_sdp_destroy(parsed_sdp);
}
if(json_is_true(json_object_get(jsep, "e2ee")))
@@ -2127,7 +2119,7 @@ void janus_lua_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp *rtp_
} else {
/* We're simulcasting, save the best video quality */
gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->rec_simctx,
buf, len, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
buf, len, NULL, 0, session->ssrc, session->rid, session->vcodec, &session->rec_ctx, &session->rid_mutex);
if(save) {
uint32_t seq_number = ntohs(rtp->seq_number);
uint32_t timestamp = ntohl(rtp->timestamp);
@@ -2421,7 +2413,8 @@ static void janus_lua_relay_rtp_packet(gpointer data, gpointer user_data) {
return;
/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
gboolean relay = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
(char *)packet->data, packet->length, packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
(char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len,
packet->ssrc, NULL, sender->vcodec, &session->vrtpctx, NULL);
if(session->sim_context.need_pli && sender->handle) {
/* Send a PLI */
JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n");
7 changes: 2 additions & 5 deletions src/plugins/janus_recordplay.c
View file
Original file line number Diff line number Diff line change
@@ -1304,7 +1304,8 @@ void janus_recordplay_incoming_rtp(janus_plugin_session *handle, janus_plugin_rt
uint32_t ssrc = ntohl(header->ssrc);
/* Process this packet: don't save if it's not the SSRC/layer we wanted to handle */
gboolean save = janus_rtp_simulcasting_context_process_rtp(&session->sim_context,
buf, len, session->ssrc, session->rid, session->recording->vcodec, &session->context, &session->rid_mutex);
buf, len, NULL, 0, session->ssrc, session->rid, session->recording->vcodec,
&session->context, &session->rid_mutex);
if(session->sim_context.need_pli) {
/* Send a PLI */
JANUS_LOG(LOG_VERB, "We need a PLI for the simulcast context\n");
@@ -1879,10 +1880,6 @@ static void *janus_recordplay_handler(void *data) {
janus_mutex_unlock(&session->rid_mutex);
session->sim_context.substream_target = 2; /* Let's aim for the highest quality */
session->sim_context.templayer_target = 2; /* Let's aim for all temporal layers */
if(rec->vcodec != JANUS_VIDEOCODEC_VP8 && rec->vcodec != JANUS_VIDEOCODEC_H264) {
/* VP8 r H.264 were not negotiated, if simulcasting was enabled then disable it here */
janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
}
/* FIXME We're stopping at the first item, there may be more */
break;
}
3 changes: 2 additions & 1 deletion src/plugins/janus_streaming.c
View file
Original file line number Diff line number Diff line change
@@ -10030,7 +10030,8 @@ static void janus_streaming_relay_rtp_packet(gpointer data, gpointer user_data)
return;
/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
gboolean relay = janus_rtp_simulcasting_context_process_rtp(&s->sim_context,
(char *)packet->data, packet->length, packet->ssrc, NULL, packet->codec, &s->context, NULL);
(char *)packet->data, packet->length, NULL, 0,
packet->ssrc, NULL, packet->codec, &s->context, NULL);
if(!relay) {
/* Did a lot of time pass before we could relay a packet? */
gint64 now = janus_get_monotonic_time();
9 changes: 5 additions & 4 deletions src/plugins/janus_videocall.c
View file
Original file line number Diff line number Diff line change
@@ -778,7 +778,8 @@ void janus_videocall_incoming_rtp(janus_plugin_session *handle, janus_plugin_rtp
/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle
* The caveat is that the targets in OUR simulcast context are the PEER's targets */
gboolean relay = janus_rtp_simulcasting_context_process_rtp(&peer->sim_context,
buf, len, session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex);
buf, len, packet->extensions.dd_content, packet->extensions.dd_len,
session->ssrc, session->rid, session->vcodec, &peer->context, &session->rid_mutex);
/* Do we need to drop this? */
if(!relay)
return;
@@ -1374,7 +1375,7 @@ static void *janus_videocall_handler(void *data) {
session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL);
/* Check if this user will simulcast */
json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast");
if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) {
if(msg_simulcast) {
JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username);
} else {
janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
@@ -1533,7 +1534,7 @@ static void *janus_videocall_handler(void *data) {
session->sim_context.templayer_target = json_integer_value(temporal);
JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n",
session->sim_context.templayer_target, session->sim_context.templayer);
if(session->vcodec == JANUS_VIDEOCODEC_VP8 && session->sim_context.templayer_target == session->sim_context.templayer) {
if(session->sim_context.templayer_target == session->sim_context.templayer) {
/* No need to do anything, we're already getting the right temporal, so notify the user */
json_t *event = json_object();
json_object_set_new(event, "videocall", json_string("event"));
@@ -1569,7 +1570,7 @@ static void *janus_videocall_handler(void *data) {
session->has_data = (strstr(msg_sdp, "DTLS/SCTP") != NULL);
/* Check if this user will simulcast */
json_t *msg_simulcast = json_object_get(msg->jsep, "simulcast");
if(msg_simulcast && janus_get_codec_pt(msg_sdp, "vp8") > 0) {
if(msg_simulcast) {
JANUS_LOG(LOG_VERB, "VideoCall callee (%s) cannot do simulcast.\n", session->username);
} else {
janus_rtp_simulcasting_cleanup(NULL, session->ssrc, session->rid, &session->rid_mutex);
67 changes: 37 additions & 30 deletions src/plugins/janus_videoroom.c
View file
Original file line number Diff line number Diff line change
@@ -530,8 +530,8 @@ room-<unique room ID>: {
"codec" : "<codec used for published stream #1>",
"description" : "<text description of published stream #1, if any>",
"moderated" : <true if this stream audio has been moderated for this participant>,
"simulcast" : "<true if published stream #1 uses simulcast (VP8 and H.264 only)>",
"svc" : "<true if published stream #1 uses SVC (VP9 only)>",
"simulcast" : "<true if published stream #1 uses simulcast>",
"svc" : "<true if published stream #1 uses SVC (VP9 and AV1 only)>",
"talking" : <true|false, whether the publisher stream has audio activity or not (only if audio levels are used)>,
},
// Other streams, if any
@@ -675,8 +675,8 @@ room-<unique room ID>: {
"codec" : "<codec used for published stream #1>",
"description" : "<text description of published stream #1, if any>",
"moderated" : <true if this stream audio has been moderated for this participant>,
"simulcast" : "<true if published stream #1 uses simulcast (VP8 and H.264 only)>",
"svc" : "<true if published stream #1 uses SVC (VP9 only)>",
"simulcast" : "<true if published stream #1 uses simulcast>",
"svc" : "<true if published stream #1 uses SVC (VP9 and AV1 only)>",
"talking" : <true|false, whether the publisher stream has audio activity or not (only if audio levels are used)>,
},
// Other streams, if any
@@ -1365,8 +1365,8 @@ room-<unique room ID>: {
"substream" : <substream to receive (0-2), in case simulcasting is enabled; optional>,
"temporal" : <temporal layers to receive (0-2), in case simulcasting is enabled; optional>,
"fallback" : <How much time (in us, default 250000) without receiving packets will make us drop to the substream below; optional>,
"spatial_layer" : <spatial layer to receive (0-2), in case VP9-SVC is enabled; optional>,
"temporal_layer" : <temporal layers to receive (0-2), in case VP9-SVC is enabled; optional>,
"spatial_layer" : <spatial layer to receive (0-2), in case SVC is enabled; optional>,
"temporal_layer" : <temporal layers to receive (0-2), in case SVC is enabled; optional>,
"audio_level_average" : "<if provided, overrides the room audio_level_average for this user; optional>",
"audio_active_packets" : "<if provided, overrides the room audio_active_packets for this user; optional>",
"min_delay" : <minimum delay to enforce via the playout-delay RTP extension, in blocks of 10ms; optional>,
@@ -1388,7 +1388,7 @@ room-<unique room ID>: {
* that for them to work you'll have to specify the \c mid as well, as the same
* subscription may be receiving simulcast stream from multiple publishers.
* The \c spatial_layer and \c temporal_layer have exactly the same meaning,
* but within the context of VP9-SVC publishers, and will have no effect
* but within the context of SVC publishers, and will have no effect
* on subscriptions associated to regular publishers.
*
* As anticipated, \c configure is also the request you use when you want
@@ -1802,11 +1802,11 @@ static struct janus_json_parameter configure_stream_parameters[] = {
/* For talk detection */
{"audio_level_averge", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"audio_active_packets", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
/* For VP8 (or H.264) simulcast */
/* For simulcast */
{"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"fallback", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
/* For VP9 SVC */
/* For SVC */
{"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
/* For the playout-delay RTP extension, if negotiated */
@@ -1835,22 +1835,22 @@ static struct janus_json_parameter subscriber_parameters[] = {
{"offer_audio", JANUS_JSON_BOOL, 0},
{"offer_video", JANUS_JSON_BOOL, 0},
{"offer_data", JANUS_JSON_BOOL, 0},
/* For VP8 (or H.264) simulcast */
/* For simulcast */
{"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"fallback", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
/* For VP9 SVC */
/* For SVC */
{"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
};
static struct janus_json_parameter subscriber_stream_parameters[] = {
{"mid", JANUS_JSON_STRING, 0},
{"crossrefid", JANUS_JSON_STRING, 0},
{"send", JANUS_JSON_BOOL, 0},
/* For VP8 (or H.264) simulcast */
/* For simulcast */
{"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
/* For VP9 SVC */
/* For SVC */
{"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
/* For the playout-delay RTP extension, if negotiated */
@@ -1876,10 +1876,10 @@ static struct janus_json_parameter switch_update_parameters[] = {
//~ {"feed", JANUS_JSON_INTEGER, JANUS_JSON_PARAM_REQUIRED | JANUS_JSON_PARAM_POSITIVE},
{"mid", JANUS_JSON_STRING, JANUS_JSON_PARAM_REQUIRED},
{"sub_mid", JANUS_JSON_STRING, JANUS_JSON_PARAM_REQUIRED},
/* For VP8 (or H.264) simulcast */
/* For simulcast */
{"substream", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"temporal", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
/* For VP9 SVC */
/* For SVC */
{"spatial_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE},
{"temporal_layer", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}
};
@@ -2129,8 +2129,8 @@ typedef struct janus_videoroom_publisher_stream {
gboolean opusfec; /* Whether this stream is sending inband Opus FEC */
gboolean opusdtx; /* Whether this publisher is using Opus DTX (Discontinuous Transmission) */
gboolean opusstereo; /* Whether this publisher is doing stereo Opus */
gboolean simulcast, svc; /* Whether this stream uses simulcast or VP9 SVC */
uint32_t vssrc[3]; /* Only needed in case VP8 (or H.264) simulcasting is involved */
gboolean simulcast, svc; /* Whether this stream uses simulcast or SVC */
uint32_t vssrc[3]; /* Only needed in case simulcasting is involved */
char *rid[3]; /* Only needed if simulcasting is rid-based */
int rid_extmap_id; /* rid extmap ID */
janus_mutex rid_mutex; /* Mutex to protect access to the rid array and the extmap ID */
@@ -2252,7 +2252,7 @@ typedef struct janus_videoroom_rtp_relay_packet {
janus_plugin_rtp_extensions extensions;
/* Whether simulcast is involved */
gboolean simulcast;
/* The following are only relevant if we're doing VP9 SVC*/
/* The following are only relevant if we're doing SVC*/
gboolean svc;
janus_vp9_svc_info svc_info;
/* The following is only relevant for datachannels */
@@ -4077,7 +4077,7 @@ json_t *janus_videoroom_query_session(janus_plugin_session *handle) {
if(ps->simulcast)
json_object_set_new(m, "simulcast", json_true());
if(ps->svc)
json_object_set_new(m, "vp9-svc", json_true());
json_object_set_new(m, "svc", json_true());
if(ps->rc && ps->rc->filename)
json_object_set_new(m, "recording", json_string(ps->rc->filename));
if(ps->audio_level_extmap_id > 0) {
@@ -7998,7 +7998,8 @@ static void janus_videoroom_incoming_rtp_internal(janus_videoroom_session *sessi
} else {
/* We're simulcasting, save the best video quality */
gboolean save = janus_rtp_simulcasting_context_process_rtp(&ps->rec_simctx,
buf, len, ps->vssrc, ps->rid, ps->vcodec, &ps->rec_ctx, &ps->rid_mutex);
buf, len, pkt->extensions.dd_content, pkt->extensions.dd_len,
ps->vssrc, ps->rid, ps->vcodec, &ps->rec_ctx, &ps->rid_mutex);
if(save) {
uint32_t seq_number = ntohs(rtp->seq_number);
uint32_t timestamp = ntohl(rtp->timestamp);
@@ -8030,10 +8031,14 @@ static void janus_videoroom_incoming_rtp_internal(janus_videoroom_session *sessi
janus_videoroom_publisher_dereference_nodebug(participant);
return;
}
gboolean found = FALSE;
memset(&packet.svc_info, 0, sizeof(packet.svc_info));
if(janus_vp9_parse_svc(payload, plen, &found, &packet.svc_info) == 0) {
packet.svc = found;
if(ps->vcodec == JANUS_VIDEOCODEC_VP9) {
gboolean found = FALSE;
memset(&packet.svc_info, 0, sizeof(packet.svc_info));
if(janus_vp9_parse_svc(payload, plen, &found, &packet.svc_info) == 0) {
packet.svc = found;
}
} else if(ps->vcodec == JANUS_VIDEOCODEC_AV1) {
packet.svc = (pkt->extensions.dd_len > 0);
}
}
if(video && ps->simulcast)
@@ -10884,7 +10889,7 @@ static void *janus_videoroom_handler(void *data) {
janus_videoroom_reqpli(ps, "Simulcasting substream change");
}
}
if(ps->vcodec == JANUS_VIDEOCODEC_VP8 && ps->simulcast && sc_temporal) {
if(ps->simulcast && sc_temporal) {
stream->sim_context.templayer_target = json_integer_value(sc_temporal);
JANUS_LOG(LOG_VERB, "Setting video temporal layer to let through (simulcast): %d (was %d)\n",
stream->sim_context.templayer_target, stream->sim_context.templayer);
@@ -11795,8 +11800,7 @@ static void *janus_videoroom_handler(void *data) {
}
}
/* Check if simulcast or SVC is in place */
if(msg_simulcast != NULL && json_array_size(msg_simulcast) > 0 &&
(ps->vcodec == JANUS_VIDEOCODEC_VP8 || ps->vcodec == JANUS_VIDEOCODEC_H264)) {
if(msg_simulcast != NULL && json_array_size(msg_simulcast) > 0) {
size_t i = 0;
for(i=0; i<json_array_size(msg_simulcast); i++) {
json_t *s = json_array_get(msg_simulcast, i);
@@ -11814,7 +11818,7 @@ static void *janus_videoroom_handler(void *data) {
janus_mutex_unlock(&ps->rid_mutex);
}
} else if(msg_svc != NULL && json_array_size(msg_svc) > 0 &&
ps->vcodec == JANUS_VIDEOCODEC_VP9) {
(ps->vcodec == JANUS_VIDEOCODEC_VP9 || ps->vcodec == JANUS_VIDEOCODEC_AV1)) {
size_t i = 0;
for(i=0; i<json_array_size(msg_svc); i++) {
json_t *s = json_array_get(msg_svc, i);
@@ -11877,6 +11881,7 @@ static void *janus_videoroom_handler(void *data) {
JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_MID,
JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_RID,
JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_REPAIRED_RID,
JANUS_SDP_OA_ACCEPT_EXTMAP, JANUS_RTP_EXTMAP_DEPENDENCY_DESC,
JANUS_SDP_OA_ACCEPT_EXTMAP, videoroom->videoorient_ext ? JANUS_RTP_EXTMAP_VIDEO_ORIENTATION : NULL,
JANUS_SDP_OA_ACCEPT_EXTMAP, videoroom->playoutdelay_ext ? JANUS_RTP_EXTMAP_PLAYOUT_DELAY : NULL,
JANUS_SDP_OA_ACCEPT_EXTMAP, videoroom->transport_wide_cc_ext ? JANUS_RTP_EXTMAP_TRANSPORT_WIDE_CC : NULL,
@@ -12149,7 +12154,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data)
/* Process this packet: don't relay if it's not the layer we wanted to handle */
janus_rtp_header rtp = *(packet->data);
gboolean relay = janus_rtp_svc_context_process_rtp(&stream->svc_context,
(char *)packet->data, packet->length, ps->vcodec, &packet->svc_info, &stream->context);
(char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len,
ps->vcodec, &packet->svc_info, &stream->context);
if(stream->svc_context.need_pli) {
/* Send a PLI */
JANUS_LOG(LOG_VERB, "We need a PLI for the SVC context\n");
@@ -12201,7 +12207,8 @@ static void janus_videoroom_relay_rtp_packet(gpointer data, gpointer user_data)
return;
/* Process this packet: don't relay if it's not the SSRC/layer we wanted to handle */
gboolean relay = janus_rtp_simulcasting_context_process_rtp(&stream->sim_context,
(char *)packet->data, packet->length, packet->ssrc, NULL, ps->vcodec, &stream->context, &ps->rid_mutex);
(char *)packet->data, packet->length, packet->extensions.dd_content, packet->extensions.dd_len,
packet->ssrc, NULL, ps->vcodec, &stream->context, &ps->rid_mutex);
if(!relay) {
/* Did a lot of time pass before we could relay a packet? */
gint64 now = janus_get_monotonic_time();
2 changes: 2 additions & 0 deletions src/plugins/lua/janus-sdp.lua
View file
Original file line number Diff line number Diff line change
@@ -446,6 +446,8 @@ function JANUSSDP.generateAnswer(offer, options)
answer[#answer+1] = { type = "a", name = a.name, value = value }
elseif options.enableAudioLevel ~= false and a.value:find("urn:ietf:params:rtp-hdrext:ssrc-audio-level", 1, true) then
answer[#answer+1] = a
elseif options.enableAudioLevel ~= false and a.value:find("dependency-descriptor-rtp-header-extension", 1, true) then
answer[#answer+1] = a
end
end
else
205 changes: 192 additions & 13 deletions src/rtp.c
View file
Original file line number Diff line number Diff line change
@@ -1023,6 +1023,9 @@ void janus_rtp_simulcasting_context_reset(janus_rtp_simulcasting_context *contex
if(context == NULL)
return;
/* Reset the context values */
janus_av1_svc_context_reset(&context->av1_context[0]);
janus_av1_svc_context_reset(&context->av1_context[1]);
janus_av1_svc_context_reset(&context->av1_context[2]);
memset(context, 0, sizeof(*context));
context->rid_ext_id = -1;
context->substream = -1;
@@ -1081,7 +1084,7 @@ void janus_rtp_simulcasting_cleanup(int *rid_ext_id, uint32_t *ssrcs, char **rid
}

gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_context *context,
char *buf, int len, uint32_t *ssrcs, char **rids,
char *buf, int len, uint8_t *dd_content, int dd_len, uint32_t *ssrcs, char **rids,
janus_videocodec vcodec, janus_rtp_switching_context *sc, janus_mutex *rid_mutex) {
if(!context || !buf || len < 1)
return FALSE;
@@ -1146,7 +1149,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte
if(context->substream == -1) {
if((vcodec == JANUS_VIDEOCODEC_VP8 && janus_vp8_is_keyframe(payload, plen)) ||
(vcodec == JANUS_VIDEOCODEC_VP9 && janus_vp9_is_keyframe(payload, plen)) ||
(vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen))) {
(vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen)) ||
(vcodec == JANUS_VIDEOCODEC_AV1 && janus_av1_is_keyframe(payload, plen)) ||
(vcodec == JANUS_VIDEOCODEC_H265 && janus_h265_is_keyframe(payload, plen))) {
context->substream = substream;
/* Notify the caller that the substream changed */
context->changed_substream = TRUE;
@@ -1161,7 +1166,9 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte
(context->substream > target && substream < context->substream)) &&
((vcodec == JANUS_VIDEOCODEC_VP8 && janus_vp8_is_keyframe(payload, plen)) ||
(vcodec == JANUS_VIDEOCODEC_VP9 && janus_vp9_is_keyframe(payload, plen)) ||
(vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen)))) {
(vcodec == JANUS_VIDEOCODEC_H264 && janus_h264_is_keyframe(payload, plen)) ||
(vcodec == JANUS_VIDEOCODEC_AV1 && janus_av1_is_keyframe(payload, plen)) ||
(vcodec == JANUS_VIDEOCODEC_H265 && janus_h265_is_keyframe(payload, plen)))) {
JANUS_LOG(LOG_VERB, "Received keyframe on #%d (SSRC %"SCNu32"), switching (was #%d/%"SCNu32")\n",
substream, ssrc, context->substream, *(ssrcs + context->substream));
context->substream = substream;
@@ -1206,7 +1213,7 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte
return FALSE;
}
context->last_relayed = janus_get_monotonic_time();
/* Temporal layers are only available for VP8, so don't do anything else for other codecs */
/* Temporal layers are only easily available for some codecs */
if(vcodec == JANUS_VIDEOCODEC_VP8) {
/* Check if there's any temporal scalability to take into account */
gboolean m = FALSE;
@@ -1232,6 +1239,73 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte
return FALSE;
}
}
} else if(vcodec == JANUS_VIDEOCODEC_VP9) {
/* We use the VP9 SVC parser to extract info on temporal layers */
gboolean found = FALSE;
janus_vp9_svc_info svc_info = { 0 };
if(janus_vp9_parse_svc(payload, plen, &found, &svc_info) == 0 && found) {
int temporal_layer = context->templayer;
if(context->templayer_target > context->templayer) {
/* We need to upscale */
if(svc_info.ubit && svc_info.bbit &&
svc_info.temporal_layer > context->templayer &&
svc_info.temporal_layer <= context->templayer_target) {
context->templayer = svc_info.temporal_layer;
temporal_layer = context->templayer;
context->changed_temporal = TRUE;
}
} else if(context->templayer_target < context->templayer) {
/* We need to downscale */
if(svc_info.ebit && svc_info.temporal_layer == context->templayer_target) {
context->templayer = context->templayer_target;
context->changed_temporal = TRUE;
}
}
if(temporal_layer < svc_info.temporal_layer) {
JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n",
svc_info.temporal_layer, context->templayer);
/* We increase the base sequence number, or there will be gaps when delivering later */
if(sc)
sc->base_seq++;
return FALSE;
}
}
} else if(vcodec == JANUS_VIDEOCODEC_AV1 && dd_content != NULL && dd_len > 0) {
/* Use the Dependency Descriptor to check temporal layers */
janus_av1_svc_context *av1ctx = NULL;
if(context->substream >= 0 && context->substream <= 2)
av1ctx = &context->av1_context[context->substream];
if(av1ctx != NULL) {
uint8_t template = 0;
if(janus_av1_svc_context_process_dd(av1ctx, dd_content, dd_len, &template, NULL)) {
janus_av1_svc_template *t = g_hash_table_lookup(av1ctx->templates, GUINT_TO_POINTER(template));
if(t) {
int temporal_layer = context->templayer;
if(context->templayer_target > context->templayer) {
/* We need to upscale */
if(t->temporal > context->templayer && t->temporal <= context->templayer_target) {
context->templayer = t->temporal;
temporal_layer = context->templayer;
context->changed_temporal = TRUE;
}
} else if(context->templayer_target < context->templayer) {
/* We need to downscale */
if(t->temporal == context->templayer_target) {
context->templayer = context->templayer_target;
context->changed_temporal = TRUE;
}
}
if(temporal_layer < t->temporal) {
JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n",
t->temporal, context->templayer);
/* We increase the base sequence number, or there will be gaps when delivering later */
if(sc)
sc->base_seq++;
return FALSE;
}
}
}
}
}
/* If we got here, the packet can be relayed */
return TRUE;
@@ -1242,14 +1316,16 @@ void janus_rtp_svc_context_reset(janus_rtp_svc_context *context) {
if(context == NULL)
return;
/* Reset the context values */
janus_av1_svc_context_reset(&context->dd_context);
memset(context, 0, sizeof(*context));
context->spatial = -1;
context->temporal = -1;
}

gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char *buf, int len,
gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context,
char *buf, int len, uint8_t *dd_content, int dd_len,
janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc) {
if(!context || !buf || len < 1 || vcodec != JANUS_VIDEOCODEC_VP9)
if(!context || !buf || len < 1 || (vcodec != JANUS_VIDEOCODEC_VP9 && vcodec != JANUS_VIDEOCODEC_AV1))
return FALSE;
janus_rtp_header *header = (janus_rtp_header *)buf;
/* Reset the flags */
@@ -1262,7 +1338,108 @@ gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char
char *payload = janus_rtp_payload(buf, len, &plen);
if(payload == NULL)
return FALSE;
/* If we don't have any info parsed from the VP9 payload header, get it now */
/* Check if we should use the Dependency Descriptor */
if(vcodec == JANUS_VIDEOCODEC_AV1) {
/* We do, make sure the data is there */
if(dd_content == NULL || dd_len < 1) {
/* No Dependency Descriptor, relay as it is */
return TRUE;
}
uint8_t template = 0, ebit = 0;
if(!janus_av1_svc_context_process_dd(&context->dd_context, dd_content, dd_len, &template, &ebit)) {
/* We couldn't parse the Dependency Descriptor, relay as it is */
return TRUE;
}
janus_av1_svc_template *t = g_hash_table_lookup(context->dd_context.templates, GUINT_TO_POINTER(template));
if(t == NULL) {
/* We couldn't find the template, relay as it is */
return TRUE;
}
/* Now let's check if we should let the packet through or not */
gboolean keyframe = janus_av1_is_keyframe((const char *)payload, plen);
gboolean override_mark_bit = FALSE, has_marker_bit = header->markerbit;
int spatial_layer = context->spatial;
if(t->spatial >= 0 && t->spatial <= 2)
context->last_spatial_layer[t->spatial] = now;
if(context->spatial_target > context->spatial) {
JANUS_LOG(LOG_HUGE, "We need to upscale spatially: (%d < %d)\n",
context->spatial, context->spatial_target);
/* We need to upscale: wait for a keyframe */
if(keyframe) {
int new_spatial_layer = context->spatial_target;
while(new_spatial_layer > context->spatial && new_spatial_layer > 0) {
if(now - context->last_spatial_layer[new_spatial_layer] >= (context->drop_trigger ? context->drop_trigger : 250000)) {
/* We haven't received packets from this layer for a while, try a lower layer */
JANUS_LOG(LOG_HUGE, "Haven't received packets from layer %d for a while, trying %d instead...\n",
new_spatial_layer, new_spatial_layer-1);
new_spatial_layer--;
} else {
break;
}
}
if(new_spatial_layer > context->spatial) {
JANUS_LOG(LOG_HUGE, " -- Upscaling spatial layer: %d --> %d (need %d)\n",
context->spatial, new_spatial_layer, context->spatial_target);
context->spatial = new_spatial_layer;
spatial_layer = context->spatial;
context->changed_spatial = TRUE;
}
}
} else if(context->spatial_target < context->spatial) {
/* We need to scale: wait for a keyframe */
JANUS_LOG(LOG_HUGE, "We need to downscale spatially: (%d > %d)\n",
context->spatial, context->spatial_target);
/* Check the E bit to see if this is an end-of-frame */
if(ebit) {
JANUS_LOG(LOG_HUGE, " -- Downscaling spatial layer: %d --> %d\n",
context->spatial, context->spatial_target);
context->spatial = context->spatial_target;
context->changed_spatial = TRUE;
}
}
if(spatial_layer < t->spatial) {
/* Drop the packet: update the context to make sure sequence number is increased normally later */
JANUS_LOG(LOG_HUGE, "Dropping packet (spatial layer %d < %d)\n", spatial_layer, t->spatial);
if(sc)
sc->base_seq++;
return FALSE;
} else if(ebit && spatial_layer == t->spatial) {
/* If we stop at layer 0, we need a marker bit now, as the one from layer 1 will not be received */
override_mark_bit = TRUE;
}
int temporal = context->temporal;
if(context->temporal_target > context->temporal) {
/* We need to upscale */
if(t->temporal > context->temporal && t->temporal <= context->temporal_target) {
context->temporal = t->temporal;
temporal = context->temporal;
context->changed_temporal = TRUE;
}
} else if(context->temporal_target < context->temporal) {
/* We need to downscale */
if(t->temporal == context->temporal_target) {
context->temporal = context->temporal_target;
context->changed_temporal = TRUE;
}
}
if(temporal < t->temporal) {
JANUS_LOG(LOG_HUGE, "Dropping packet (it's temporal layer %d, but we're capping at %d)\n",
t->temporal, context->temporal);
/* We increase the base sequence number, or there will be gaps when delivering later */
if(sc)
sc->base_seq++;
return FALSE;
}
/* If we got here, we can send the frame: this doesn't necessarily mean it's
* one of the layers the user wants, as there may be dependencies involved */
JANUS_LOG(LOG_HUGE, "Sending packet (spatial=%d, temporal=%d)\n",
t->spatial, t->temporal);
if(override_mark_bit && !has_marker_bit)
header->markerbit = 1;
return TRUE;
}
/* If we got here, it's VP9, for which we parse the payload manually:
* if we don't have any info parsed from the VP9 payload header, get it now */
janus_vp9_svc_info svc_info = { 0 };
if(!info) {
gboolean found = FALSE;
@@ -1390,7 +1567,7 @@ void janus_av1_svc_context_reset(janus_av1_svc_context *context) {
}

gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context,
uint8_t *dd, int dd_len, uint8_t *template_id) {
uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit) {
if(!context || !dd || dd_len < 3)
return FALSE;

@@ -1400,9 +1577,11 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context,
/* mandatory_descriptor_fields() */
uint8_t start = janus_bitstream_getbit(dd, offset++);
uint8_t end = janus_bitstream_getbit(dd, offset++);
if(ebit)
*ebit = end;
uint8_t template = janus_bitstream_getbits(dd, 6, &offset);
uint16_t frame = janus_bitstream_getbits(dd, 16, &offset);
JANUS_LOG(LOG_WARN, " -- s=%u, e=%u, t=%u, f=%u\n",
JANUS_LOG(LOG_HUGE, " -- s=%u, e=%u, t=%u, f=%u\n",
start, end, template, frame);
if(blen > 24) {
/* extended_descriptor_fields() */
@@ -1434,7 +1613,7 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context,
}
t->spatial = spatial_layers;
t->temporal = temporal_layers;
JANUS_LOG(LOG_WARN, " -- -- -- [%u] spatial=%u, temporal=%u\n",
JANUS_LOG(LOG_HUGE, " -- -- -- [%u] spatial=%u, temporal=%u\n",
tcnt, t->spatial, t->temporal);
if(nlidc == 1) {
temporal_layers++;
@@ -1458,14 +1637,14 @@ gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context,
}
/* frame_dependency_definition() */
uint8_t tindex = (template + 64 - context->tioff) % 64;
janus_av1_svc_template *t = g_hash_table_lookup(context->templates,
GUINT_TO_POINTER(tindex));
janus_av1_svc_template *t = context->templates ? g_hash_table_lookup(context->templates,
GUINT_TO_POINTER(tindex)) : NULL;
if(t == NULL) {
JANUS_LOG(LOG_WARN, "Invalid template ID '%u' (count is %u), ignoring packet...\n",
tindex, context->tcnt);
return FALSE;
}
JANUS_LOG(LOG_WARN, " -- spatial=%u, temporal=%u (tindex %u)\n",
JANUS_LOG(LOG_HUGE, " -- spatial=%u, temporal=%u (tindex %u)\n",
t->spatial, t->temporal, t->id);
/* FIXME We currently don't care about the other fields */

109 changes: 59 additions & 50 deletions src/rtp.h
View file
Original file line number Diff line number Diff line change
@@ -295,13 +295,63 @@ int janus_rtp_skew_compensate_audio(janus_rtp_header *header, janus_rtp_switchin
int janus_rtp_skew_compensate_video(janus_rtp_header *header, janus_rtp_switching_context *context, gint64 now);


/** @name Janus AV1-SVC processing methods
*/
///@{
/*! \brief Helper struct for processing and tracking AV1-SVC streams */
typedef struct janus_av1_svc_context {
/*! \brief Number of templates advertised via Dependency Descriptor */
uint8_t tcnt;
/*! \brief Template ID offset, as advertised via Dependency Descriptor */
uint8_t tioff;
/*! \brief Map of templates advertised via Dependency Descriptor, indexed by ID */
GHashTable *templates;
/*! \brief How many spatial and temporal layers are available */
int spatial_layers, temporal_layers;
/*! \brief Whether this context changed since the last update */
gboolean updated;
} janus_av1_svc_context;

/*! \brief Helper struct to track SVC templates
* \note This is very incomplete, since we only track the spatial and
* temporal layer associated with a specific template ID for now */
typedef struct janus_av1_svc_template {
/*! \brief Template ID */
uint8_t id;
/*! \brief Spatial layer associated to this template */
int spatial;
/*! \brief Temporal layer associated to this template */
int temporal;
} janus_av1_svc_template;

/*! \brief Set (or reset) the context fields to their default values
* @param[in] context The context to (re)set */
void janus_av1_svc_context_reset(janus_av1_svc_context *context);

/*! \brief Process a Dependency Descriptor payload, updating the SVC context accordingly
* \note At the moment, this code is quite naive, as it mostly looks at the target
* spatial/temporal layers, and the one written in the Dependency Descriptor data.
* In the future, this should become more sophisticated, and use additional
* information like dependency chains and stuff like that
* @param[in] context The av1svc context to use
* @param[in] dd Pointer to the Dependency Descriptor data
* @param[in] dd_len The length of the Dependendy Descriptor data
* @param[out] template_id Pointer to the ID of the template referenced in this packet
* @param[out] ebit Whether this packet is an end of frame or not
* @returns TRUE if the packet is valid, FALSE if it should be dropped instead */
gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context,
uint8_t *dd, int dd_len, uint8_t *template_id, uint8_t *ebit);
///@}

/** @name Janus simulcast processing methods
*/
///@{
/*! \brief Helper struct for processing and tracking simulcast streams */
typedef struct janus_rtp_simulcasting_context {
/*! \brief RTP Stream extension ID, if any */
gint rid_ext_id;
/*! \brief Dependency Descriptors contexts, if any */
janus_av1_svc_context av1_context[3];
/*! \brief Which simulcast substream we should forward back */
int substream;
/*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */
@@ -348,14 +398,16 @@ void janus_rtp_simulcasting_cleanup(int *rid_ext_id, uint32_t *ssrcs, char **rid
* @param[in] context The simulcasting context to use
* @param[in] buf The RTP packet to process
* @param[in] len The length of the RTP packet (header, extension and payload)
* @param[in] dd_content The Dependency Descriptor RTP extension data, if available
* @param[in] dd_len Length of the Dependency Descriptor data, if available
* @param[in] ssrcs The simulcast SSRCs to refer to (may be updated if rids are involved)
* @param[in] rids The simulcast rids to refer to, if any
* @param[in] vcodec Video codec of the RTP payload
* @param[in] sc RTP switching context to refer to, if any (only needed for VP8 and dropping temporal layers)
* @param[in] rid_mutex A mutex that must be acquired before reading the rids array, if any
* @returns TRUE if the packet should be relayed, FALSE if it should be dropped instead */
gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_context *context,
char *buf, int len, uint32_t *ssrcs, char **rids,
char *buf, int len, uint8_t *dd_content, int dd_len, uint32_t *ssrcs, char **rids,
janus_videocodec vcodec, janus_rtp_switching_context *sc, janus_mutex *rid_mutex);
///@}

@@ -364,6 +416,8 @@ gboolean janus_rtp_simulcasting_context_process_rtp(janus_rtp_simulcasting_conte
///@{
/*! \brief Helper struct for processing and tracking VP9-SVC streams */
typedef struct janus_rtp_svc_context {
/*! \brief Dependency Descriptor context, in case it's needed */
struct janus_av1_svc_context dd_context;
/*! \brief Which SVC spatial layer we should forward back */
int spatial;
/*! \brief As above, but to handle transitions (e.g., wait for keyframe, or get this if available) */
@@ -394,60 +448,15 @@ void janus_rtp_svc_context_reset(janus_rtp_svc_context *context);
* @param[in] context The VP9 SVC context to use
* @param[in] buf The RTP packet to process
* @param[in] len The length of the RTP packet (header, extension and payload)
* @param[in] dd_content The Dependency Descriptor RTP extension data, if available
* @param[in] dd_len Length of the Dependency Descriptor data, if available
* @param[in] vcodec Video codec of the RTP payload
* @param[in] info Parsed info on VP9-SVC, if any
* @param[in] sc RTP switching context to refer to, if any
* @returns TRUE if the packet should be relayed, FALSE if it should be dropped instead */
gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context, char *buf, int len,
gboolean janus_rtp_svc_context_process_rtp(janus_rtp_svc_context *context,
char *buf, int len, uint8_t *dd_content, int dd_len,
janus_videocodec vcodec, janus_vp9_svc_info *info, janus_rtp_switching_context *sc);
///@}

/** @name Janus AV1-SVC processing methods (still WIP)
*/
///@{
/*! \brief Helper struct for processing and tracking AV1-SVC streams */
typedef struct janus_av1_svc_context {
/*! \brief Number of templates advertised via Dependency Descriptor */
uint8_t tcnt;
/*! \brief Template ID offset, as advertised via Dependency Descriptor */
uint8_t tioff;
/*! \brief Map of templates advertised via Dependency Descriptor, indexed by ID */
GHashTable *templates;
/*! \brief How many spatial and temporal layers are available */
int spatial_layers, temporal_layers;
/*! \brief Whether this context changed since the last update */
gboolean updated;
} janus_av1_svc_context;

/*! \brief Helper struct to track SVC templates
* \note This is very incomplete, since we only track the spatial and
* temporal layer associated with a specific template ID for now */
typedef struct janus_av1_svc_template {
/*! \brief Template ID */
uint8_t id;
/*! \brief Spatial layer associated to this template */
int spatial;
/*! \brief Temporal layer associated to this template */
int temporal;
} janus_av1_svc_template;

/*! \brief Set (or reset) the context fields to their default values
* @param[in] context The context to (re)set */
void janus_av1_svc_context_reset(janus_av1_svc_context *context);

/*! \brief Process a Dependency Descriptor payload, updating the SVC context accordingly
* \note At the moment, this code is quite naive, as it mostly looks at the target
* spatial/temporal layers, and the one written in the Dependency Descriptor data.
* In the future, this should become more sophisticated, and use additional
* information like dependency chains and stuff like that
* @param[in] context The av1svc context to use
* @param[in] dd Pointer to the Dependency Descriptor data
* @param[in] dd_len The length of the Dependendy Descriptor data
* @param[out] template_id Pointer to the ID of the template referenced in this packet
* @returns TRUE if the packet is valid, FALSE if it should be dropped instead */
gboolean janus_av1_svc_context_process_dd(janus_av1_svc_context *context,
uint8_t *dd, int dd_len, uint8_t *template_id);
///@}


#endif
2 changes: 1 addition & 1 deletion src/rtpfwd.c
View file
Original file line number Diff line number Diff line change
@@ -354,7 +354,7 @@ void janus_rtp_forwarder_send_rtp_full(janus_rtp_forwarder *rf, char *buffer, in
if(rf->is_video && rf->simulcast) {
/* This is video and we're simulcasting, check if we need to forward this frame */
if(!janus_rtp_simulcasting_context_process_rtp(&rf->sim_context,
buffer, len, ssrcs, rids, vcodec, &rf->rtp_context, rid_mutex)) {
buffer, len, NULL, 0, ssrcs, rids, vcodec, &rf->rtp_context, rid_mutex)) {
/* There was an error processing simulcasting for this packet */
return;
}