FS-7501: add auto sync of jb and fps detection

src/include/switch_core_media.h

@@ -192,6 +192,7 @@ SWITCH_DECLARE(int) switch_core_session_check_incoming_crypto(switch_core_sessio
 															  switch_media_type_t type, const char *crypto, int crypto_tag, switch_sdp_type_t sdp_type);
 
 
+SWITCH_DECLARE(uint32_t) switch_core_media_get_video_fps(switch_core_session_t *session);
 SWITCH_DECLARE(void) switch_core_media_set_rtp_session(switch_core_session_t *session, switch_media_type_t type, switch_rtp_t *rtp_session);
 
 SWITCH_DECLARE(const char *)switch_core_media_get_codec_string(switch_core_session_t *session);

src/include/switch_rtp.h

@@ -327,6 +327,8 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_activate_rtcp(switch_rtp_t *rtp_sessi
 
 SWITCH_DECLARE(switch_timer_t *) switch_rtp_get_media_timer(switch_rtp_t *rtp_session);
 
+SWITCH_DECLARE(switch_status_t) switch_rtp_set_video_buffer_size(switch_rtp_t *rtp_session, uint32_t frames);
+
 /*! 
   \brief Acvite a jitter buffer on an RTP session
   \param rtp_session the rtp session

src/include/switch_vidderbuffer.h

@@ -40,6 +40,7 @@ typedef enum {
 
 SWITCH_BEGIN_EXTERN_C
 SWITCH_DECLARE(switch_status_t) switch_vb_create(switch_vb_t **vbp, uint32_t min_frame_len, uint32_t max_frame_len, switch_memory_pool_t *pool);
+SWITCH_DECLARE(switch_status_t) switch_vb_set_frames(switch_vb_t *vb, uint32_t min_frame_len, uint32_t max_frame_len);
 SWITCH_DECLARE(switch_status_t) switch_vb_destroy(switch_vb_t **vbp);
 SWITCH_DECLARE(void) switch_vb_reset(switch_vb_t *vb);
 SWITCH_DECLARE(void) switch_vb_debug_level(switch_vb_t *vb, uint8_t level);

src/mod/endpoints/mod_verto/mod_verto.c

@@ -2263,7 +2263,7 @@ static void verto_set_media_options(verto_pvt_t *tech_pvt, verto_profile_t *prof
 	tech_pvt->mparams->inbound_codec_string = switch_core_session_strdup(tech_pvt->session, profile->inbound_codec_string);
 	tech_pvt->mparams->outbound_codec_string = switch_core_session_strdup(tech_pvt->session, profile->outbound_codec_string);
 	
-	tech_pvt->mparams->jb_msec = "-1";
+	//tech_pvt->mparams->jb_msec = "-1";
 	switch_media_handle_set_media_flag(tech_pvt->smh, SCMF_SUPPRESS_CNG);
 
 	switch_media_handle_set_media_flag(tech_pvt->smh, SCMF_RENEG_ON_REINVITE);

src/mod/formats/mod_avformat/mod_avformat.c

@@ -38,6 +38,7 @@
 #include <libavutil/channel_layout.h>
 // #include <libavutil/timestamp.h>
 #include <libavresample/avresample.h>
+#define DFT_RECORD_OFFSET 350
 
 SWITCH_MODULE_LOAD_FUNCTION(mod_avformat_load);
 SWITCH_MODULE_DEFINITION(mod_avformat, mod_avformat_load, NULL, NULL);
@@ -665,7 +666,7 @@ SWITCH_STANDARD_APP(record_av_function)
 		char buf[SWITCH_RECOMMENDED_BUFFER_SIZE] = {0};
 		switch_size_t datalen = codec.implementation->decoded_bytes_per_packet;
 		switch_size_t samples = datalen / 2 / codec.implementation->number_of_channels;
-		int offset = 1200;
+		int offset = DFT_RECORD_OFFSET;
 		int fps = codec.implementation->actual_samples_per_second / samples;
 		int lead_frames = (offset * fps) / 1000;
 		
@@ -1139,7 +1140,7 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
 
 	memset(context, 0, sizeof(av_file_context_t));
 
-	context->offset = 1200;
+	context->offset = DFT_RECORD_OFFSET;
 	if (handle->params && (tmp = switch_event_get_header(handle->params, "av_video_offset"))) {
 		context->offset = atoi(tmp);
 	}

src/switch_core_media.c

@@ -208,6 +208,9 @@ struct switch_media_handle_s {
 	switch_vid_params_t vid_params; 
 	switch_file_handle_t *video_read_fh;
 	switch_file_handle_t *video_write_fh;
+
+	uint64_t vid_frames;
+	time_t vid_started;
 };
 
 
@@ -309,6 +312,46 @@ static void _switch_core_media_pass_zrtp_hash2(switch_core_session_t *aleg_sessi
 	}
 }
 
+static uint32_t round_to_step(uint32_t num, uint32_t step)
+{
+	uint32_t r;
+	uint32_t x;
+
+	if (!num) return 0;
+	
+	r = (num % step);
+	x = num - r;
+	
+	if (r > step / 2) {
+		x += step;
+	}
+	
+	return x;
+}
+SWITCH_DECLARE(uint32_t) switch_core_media_get_video_fps(switch_core_session_t *session)
+{
+	switch_media_handle_t *smh;
+	time_t now;
+
+	switch_assert(session);
+	
+	if (!(smh = session->media_handle)) {
+		return SWITCH_STATUS_FALSE;
+	}               
+
+	if (!switch_channel_test_flag(session->channel, CF_VIDEO)) {
+		return 0;
+	}
+
+	now = switch_epoch_time_now(NULL);
+
+	if (!(smh->vid_started && smh->vid_frames && smh->vid_started < now)) {
+		return 0;
+	}
+	
+	return round_to_step(smh->vid_frames / (now - smh->vid_started), 5);
+}
+
 SWITCH_DECLARE(void) switch_core_media_pass_zrtp_hash2(switch_core_session_t *aleg_session, switch_core_session_t *bleg_session)
 {
 	_switch_core_media_pass_zrtp_hash2(aleg_session, bleg_session, SWITCH_MEDIA_TYPE_AUDIO);
@@ -1766,7 +1809,8 @@ SWITCH_DECLARE(void) switch_core_media_prepare_codecs(switch_core_session_t *ses
 }
 
 
-static void check_jb(switch_core_session_t *session, const char *input)
+
+static void check_jb(switch_core_session_t *session, const char *input, int32_t jb_msec, int32_t maxlen)
 {
 	const char *val;
 	switch_media_handle_t *smh;
@@ -1820,18 +1864,20 @@ static void check_jb(switch_core_session_t *session, const char *input)
 	}
 	
 
-	if ((val = switch_channel_get_variable(session->channel, "jitterbuffer_msec")) || (val = smh->mparams->jb_msec)) {
-		int jb_msec = atoi(val);
-		int maxlen = 0, max_drift = 0;
+	if (jb_msec || (val = switch_channel_get_variable(session->channel, "jitterbuffer_msec")) || (val = smh->mparams->jb_msec)) {
+		int max_drift = 0;
 		char *p, *q;
 
+		if (!jb_msec) {
+			jb_msec = atoi(val);
 					
-		if ((p = strchr(val, ':'))) {
-			p++;
-			maxlen = atoi(p);
-			if ((q = strchr(p, ':'))) {
-				q++;
-				max_drift = abs(atoi(q));
+			if ((p = strchr(val, ':'))) {
+				p++;
+				maxlen = atoi(p);
+				if ((q = strchr(p, ':'))) {
+					q++;
+					max_drift = abs(atoi(q));
+				}
 			}
 		}
 
@@ -1878,6 +1924,61 @@ static void check_jb(switch_core_session_t *session, const char *input)
 
 }
 
+static void check_jb_sync(switch_core_session_t *session)
+{
+	int32_t jb_sync_msec = 200;
+	uint32_t fps, frames = 0;
+	switch_media_handle_t *smh;
+	switch_rtp_engine_t *v_engine = NULL;
+	const char *var;
+
+	switch_assert(session);
+
+	if (!switch_channel_test_flag(session->channel, CF_VIDEO)) {
+		return;
+	}
+
+	if (!(smh = session->media_handle)) {
+		return;
+	}
+
+	v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO];
+	
+	if ((var = switch_channel_get_variable(session->channel, "jb_sync_msec"))) {
+		int tmp;
+
+		if (!strcasecmp(var, "disabled")) {
+			return;
+		}
+
+		tmp = atol(var);
+
+		if (tmp > -50 && tmp < 10000) {
+			jb_sync_msec = tmp;
+		}
+	}
+
+	fps = switch_core_media_get_video_fps(session);
+		
+	if (!fps) return;
+
+	if (jb_sync_msec < 0) {
+		frames = abs(jb_sync_msec);
+		jb_sync_msec = 1000 / (fps / frames);
+	} else {
+		frames = fps / (1000 / jb_sync_msec);
+	}
+
+	switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), 
+					  SWITCH_LOG_DEBUG, "%s Sync Audio and Video Jitterbuffer to %dms %u Video Frames FPS %u\n", 
+					  switch_channel_get_name(session->channel),
+					  jb_sync_msec, frames, fps);
+	
+	switch_rtp_set_video_buffer_size(v_engine->rtp_session, frames);
+	check_jb(session, NULL, jb_sync_msec, jb_sync_msec);
+}
+
+
 //?
 SWITCH_DECLARE(switch_status_t) switch_core_media_read_lock_unlock(switch_core_session_t *session, switch_media_type_t type, switch_bool_t lock)
 {
@@ -1991,6 +2092,18 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_read_frame(switch_core_session
 			goto end;
 		}
 
+		if (type == SWITCH_MEDIA_TYPE_VIDEO && engine->read_frame.m) {
+			if (!smh->vid_started) {
+				smh->vid_started = switch_epoch_time_now(NULL);
+			}
+			smh->vid_frames++;
+
+			if (smh->vid_frames == 45) {
+				check_jb_sync(session);
+			}
+		}
+
+
 		/* re-set codec if necessary */
 		if (engine->reset_codec > 0) {
 			const char *val;
@@ -2039,7 +2152,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_read_frame(switch_core_session
 				}
 			}
 
-			check_jb(session, NULL);
+			check_jb(session, NULL, 0, 0);
 
 			engine->check_frames = 0;
 			engine->last_ts = 0;
@@ -5919,7 +6032,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi
 
 		}
 
-		check_jb(session, NULL);
+		check_jb(session, NULL, 0, 0);
 
 		if ((val = switch_channel_get_variable(session->channel, "rtp_timeout_sec"))) {
 			int v = atoi(val);
@@ -8467,7 +8580,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_receive_message(switch_core_se
 	case SWITCH_MESSAGE_INDICATE_JITTER_BUFFER:
 		{
 			if (switch_rtp_ready(a_engine->rtp_session)) {
-				check_jb(session, msg->string_arg);
+				check_jb(session, msg->string_arg, 0, 0);
 			}
 		}
 		break;

src/switch_rtp.c

@@ -3549,9 +3549,12 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session
 				switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Starting video timer.\n");
 			}
 
-			switch_vb_create(&rtp_session->vb, 5, 30, rtp_session->pool);
+			//switch_vb_create(&rtp_session->vb, 5, 30, rtp_session->pool);
 			//switch_vb_debug_level(rtp_session->vb, 10);
-			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Starting video buffer.\n");
+			
+			
+			
+
 
 		} else {
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Not using a timer\n");
@@ -3834,6 +3837,23 @@ static void jb_logger(const char *file, const char *func, int line, int level, c
 	va_end(ap);
 }
 
+SWITCH_DECLARE(switch_status_t) switch_rtp_set_video_buffer_size(switch_rtp_t *rtp_session, uint32_t frames)
+{
+	if (!switch_rtp_ready(rtp_session)) {
+		return SWITCH_STATUS_FALSE;
+	}
+
+	if (!rtp_session->vb) {  
+		switch_vb_create(&rtp_session->vb, frames, frames * 3, rtp_session->pool);
+	} else {
+		switch_vb_set_frames(rtp_session->vb, frames, frames * 3);
+	}
+
+	switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Setting video buffer %u Frames.\n", frames);
+	
+	return SWITCH_STATUS_SUCCESS;
+}
+
 SWITCH_DECLARE(switch_status_t) switch_rtp_debug_jitter_buffer(switch_rtp_t *rtp_session, const char *name)
 {
 

src/switch_vidderbuffer.c

@@ -415,6 +415,16 @@ SWITCH_DECLARE(void) switch_vb_reset(switch_vb_t *vb)
 	switch_mutex_unlock(vb->mutex);
 }
 
+SWITCH_DECLARE(switch_status_t) switch_vb_set_frames(switch_vb_t *vb, uint32_t min_frame_len, uint32_t max_frame_len)
+{
+	switch_mutex_lock(vb->mutex);
+	vb->min_frame_len = vb->frame_len = min_frame_len;
+	vb->max_frame_len = max_frame_len;
+	switch_mutex_unlock(vb->mutex);
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
 SWITCH_DECLARE(switch_status_t) switch_vb_create(switch_vb_t **vbp, uint32_t min_frame_len, uint32_t max_frame_len, switch_memory_pool_t *pool)
 {
 	switch_vb_t *vb;