FS-7500: Work in progress. Added codec config params that can be set from…

FS-7500: Work in progress.  Added codec config params that can be set from session and made vpx codec re-init on size change.  Also add periodic key frame timer

src/include/switch_core_media.h

@@ -157,6 +157,9 @@ typedef struct switch_core_media_params_s {
 
 	switch_bool_t external_video_source;
 
+	uint32_t video_key_freq;
+	uint32_t video_key_first;
+
 } switch_core_media_params_t;
 
 static inline const char *switch_media_type2str(switch_media_type_t type)
@@ -305,6 +308,11 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
 																switch_codec_control_type_t *rtype,
 																void **ret_data);
 
+
+#define switch_core_media_gen_key_frame(_session) switch_core_media_codec_control(_session, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_IO_WRITE, \
+																				  SCC_VIDEO_REFRESH, SCCT_NONE, NULL, NULL, NULL) \
+
+
 SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session_t *session, switch_media_type_t mtype);
 
 SWITCH_END_EXTERN_C

src/include/switch_module_interfaces.h

@@ -591,12 +591,21 @@ struct switch_directory_handle {
 	void *private_info;
 };
 
-/* nobody has more setting than speex so we will let them set the standard */
-/*! \brief Various codec settings (currently only relevant to speex) */
-struct switch_codec_settings {
+struct switch_audio_codec_settings {
 	int unused;
 };
 
+struct switch_video_codec_settings {
+	uint32_t bandwidth;
+	int32_t width;
+	int32_t height;
+};
+
+union switch_codec_settings {
+	struct switch_audio_codec_settings audio;
+	struct switch_video_codec_settings video;
+};
+
 /*! an abstract handle of a fmtp parsed by codec */
 struct switch_codec_fmtp {
 	/*! actual samples transferred per second for those who are not moron g722 RFC writers */

src/include/switch_rtp.h

@@ -517,6 +517,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_del_dtls(switch_rtp_t *rtp_session, d
 
 SWITCH_DECLARE(int) switch_rtp_has_dtls(void);
 SWITCH_DECLARE(void) switch_rtp_video_refresh(switch_rtp_t *rtp_session);
+SWITCH_DECLARE(void) switch_rtp_video_loss(switch_rtp_t *rtp_session);
 
 /*!
   \}

src/include/switch_types.h

@@ -1496,7 +1496,8 @@ typedef enum {
 	SFF_RTCP = (1 << 10),
 	SFF_MARKER = (1 << 11),
 	SFF_WAIT_KEY_FRAME = (1 << 12),
-	SFF_RAW_RTP_PARSE_FRAME = (1 << 13)
+	SFF_RAW_RTP_PARSE_FRAME = (1 << 13),
+	SFF_PICTURE_RESET = (1 << 14)
 } switch_frame_flag_enum_t;
 typedef uint32_t switch_frame_flag_t;
 
@@ -2101,7 +2102,7 @@ typedef struct switch_codec switch_codec_t;
 typedef struct switch_core_thread_session switch_core_thread_session_t;
 typedef struct switch_codec_implementation switch_codec_implementation_t;
 typedef struct switch_buffer switch_buffer_t;
-typedef struct switch_codec_settings switch_codec_settings_t;
+typedef union  switch_codec_settings switch_codec_settings_t;
 typedef struct switch_codec_fmtp switch_codec_fmtp_t;
 typedef struct switch_odbc_handle switch_odbc_handle_t;
 typedef struct switch_pgsql_handle switch_pgsql_handle_t;

src/mod/formats/mod_vlc/mod_vlc.c

@@ -102,6 +102,7 @@ struct vlc_video_context {
 	switch_mutex_t *video_mutex;
 
 	switch_core_session_t *session;
+	switch_channel_t *channel;
 	switch_frame_t *aud_frame;
 	switch_frame_t *vid_frame;
 	uint8_t video_packet[1500 + 12];
@@ -230,20 +231,11 @@ static void *vlc_video_lock_callback(void *data, void **p_pixels)
 	return NULL; /* picture identifier, not needed here */
 }
 
-/* dummy callback so it should be good when no video on channel */
-static void vlc_video_unlock_dummy_callback(void *data, void *id, void *const *p_pixels)
-{
-	vlc_video_context_t *context = (vlc_video_context_t *)data;
-	assert(id == NULL); /* picture identifier, not needed here */
-	switch_mutex_unlock(context->video_mutex);
-}
-
 static void vlc_video_unlock_callback(void *data, void *id, void *const *p_pixels)
 {
 	vlc_video_context_t *context = (vlc_video_context_t *) data;
-	switch_frame_t *frame = context->vid_frame;
 
-	switch_assert(id == NULL); /* picture identifier, not needed here */
+	if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return;
 
 	if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0);
 
@@ -251,14 +243,11 @@ static void vlc_video_unlock_callback(void *data, void *id, void *const *p_pixel
 
 	yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height);
 
-	switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, NULL);
-
 	switch_mutex_unlock(context->video_mutex);
 }
 
-static void do_buffer_frame(vlc_video_context_t *context)
+static void do_buffer_frame(vlc_video_context_t *context, switch_frame_t *frame)
 {
-	switch_frame_t *frame = context->vid_frame;
 	uint32_t size = sizeof(*frame) + frame->packetlen;
 
 	switch_mutex_lock(context->video_mutex);
@@ -277,32 +266,35 @@ static void do_buffer_frame(vlc_video_context_t *context)
 static void vlc_video_channel_unlock_callback(void *data, void *id, void *const *p_pixels)
 {
 	vlc_video_context_t *context = (vlc_video_context_t *)data;
-	uint32_t flag = 0;
-	switch_frame_t *frame = context->vid_frame;
+
 
 	switch_assert(id == NULL); /* picture identifier, not needed here */
 
+	if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return;
+
 	if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0);
 	switch_assert(context->img);
 
 	yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height);
 
-
-	if (context->video_refresh_req > 0) {
-		flag |= SFF_WAIT_KEY_FRAME;
-		context->video_refresh_req--;
-	}
-
-	switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, &flag);
-
 	switch_mutex_unlock(context->video_mutex);
 }
 
 static void vlc_video_display_callback(void *data, void *id)
 {
+	vlc_video_context_t *context = (vlc_video_context_t *) data;
+	int32_t flag = 0;
+
 	/* VLC wants to display the video */
-	(void) data;
-	assert(id == NULL);
+
+	if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return;
+
+	if (context->video_refresh_req > 0) {
+		flag |= SFF_WAIT_KEY_FRAME;
+		context->video_refresh_req--;
+	}
+
+	switch_core_session_write_video_image(context->session, context->vid_frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, NULL);
 }
 
 unsigned video_format_setup_callback(void **opaque, char *chroma, unsigned *width, unsigned *height, unsigned *pitches, unsigned *lines)
@@ -723,7 +715,9 @@ SWITCH_STANDARD_APP(play_video_function)
 
 	audio_datalen = read_impl.decoded_bytes_per_packet; //codec.implementation->actual_samples_per_second / 1000 * (read_impl.microseconds_per_packet / 1000);
 
+
 	context->session = session;
+	context->channel = channel;
 	context->pool = pool;
 	context->aud_frame = &audio_frame;
 	context->vid_frame = &video_frame;
@@ -770,14 +764,8 @@ SWITCH_STANDARD_APP(play_video_function)
 	libvlc_audio_set_format(context->mp, "S16N", read_impl.actual_samples_per_second, read_impl.number_of_channels);
 	libvlc_audio_set_callbacks(context->mp, vlc_play_audio_callback, NULL,NULL,NULL,NULL, (void *) context);
 
-	if (switch_channel_test_flag(channel, CF_VIDEO)) {
-		// libvlc_video_set_format(context->mp, "YUYV", VIDEOWIDTH, VIDEOHEIGHT, VIDEOWIDTH * 2);
-		libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
-		libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_callback, vlc_video_display_callback, context);
-	} else {
-		libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
-		libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_dummy_callback, vlc_video_display_callback, context);
-	}
+	libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
+	libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_callback, vlc_video_display_callback, context);
 
 	// start play
 	if (-1 == libvlc_media_player_play(context->mp)) {
@@ -928,6 +916,15 @@ switch_io_routines_t vlc_io_routines = {
 	/*state_run*/ NULL
 };
 
+static switch_status_t vlc_channel_img_callback(switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data)
+{
+	vlc_video_context_t *context = (vlc_video_context_t *) user_data;
+
+	do_buffer_frame(context, frame);
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
 static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char *path)
 {
 	switch_channel_t *channel = switch_core_session_get_channel(session);
@@ -952,11 +949,12 @@ static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char
 	memset(context, 0, sizeof(vlc_file_context_t));
 	tech_pvt->context = context;
 
+
 	switch_buffer_create_dynamic(&(context->audio_buffer), VLC_BUFFER_SIZE, VLC_BUFFER_SIZE * 8, 0);
 
-	if (switch_channel_test_flag(channel, CF_VIDEO)) {
-		switch_buffer_create_dynamic(&(context->video_buffer), VLC_BUFFER_SIZE * 2, VLC_BUFFER_SIZE * 16, 0);
-	}
+	switch_buffer_create_dynamic(&(context->video_buffer), VLC_BUFFER_SIZE * 2, VLC_BUFFER_SIZE * 16, 0);
+
+	switch_core_session_set_image_write_callback(session, vlc_channel_img_callback, tech_pvt->context);
 
 	if (switch_core_timer_init(&tech_pvt->timer, "soft", 20,
 							   8000 / (1000 / 20), pool) != SWITCH_STATUS_SUCCESS) {
@@ -969,6 +967,9 @@ static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char
 	context->pool = pool;
 	context->aud_frame = &tech_pvt->read_frame;
 	context->vid_frame = &tech_pvt->read_video_frame;
+	context->vid_frame->packet = context->video_packet;
+	context->vid_frame->data = context->video_packet + 12;
+
 	context->playing = 0;
 	// context->err = 0;
 
@@ -1010,14 +1011,10 @@ static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char
 	libvlc_audio_set_format(context->mp, "S16N", 8000, 1);
 	libvlc_audio_set_callbacks(context->mp, vlc_play_audio_callback, NULL,NULL,NULL,NULL, (void *) context);
 
-	if (switch_channel_test_flag(channel, CF_VIDEO)) {
-		// libvlc_video_set_format(context->mp, "YUYV", VIDEOWIDTH, VIDEOHEIGHT, VIDEOWIDTH * 2);
-		libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
-		libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_channel_unlock_callback, vlc_video_display_callback, context);
-	} else {
-		libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
-		libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_dummy_callback, vlc_video_display_callback, context);
-	}
+	
+	libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback);
+	libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_channel_unlock_callback, vlc_video_display_callback, context);
+
 
 	return SWITCH_STATUS_SUCCESS;
 
@@ -1026,25 +1023,13 @@ fail:
 	return status;
 }
 
-static switch_status_t vlc_channel_img_callback(switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data)
-{
-	vlc_video_context_t *context = (vlc_video_context_t *) user_data;
-
-	do_buffer_frame(context);
-
-	return SWITCH_STATUS_SUCCESS;
-}
-
-
 static switch_status_t channel_on_init(switch_core_session_t *session)
 {
 	switch_channel_t *channel = switch_core_session_get_channel(session);
-	vlc_private_t *tech_pvt = switch_core_session_get_private(session);
+	//vlc_private_t *tech_pvt = switch_core_session_get_private(session);
 
 	switch_channel_set_state(channel, CS_CONSUME_MEDIA);
 
-	switch_core_session_set_image_write_callback(session, vlc_channel_img_callback, tech_pvt->context);
-
 	return SWITCH_STATUS_SUCCESS;
 }
 
@@ -1096,6 +1081,10 @@ static switch_status_t channel_on_destroy(switch_core_session_t *session)
 		switch_buffer_destroy(&tech_pvt->context->audio_buffer);
 	}
 
+	if (tech_pvt->context->video_buffer) {
+		switch_buffer_destroy(&tech_pvt->context->video_buffer);
+	}
+
 	if (tech_pvt->timer.interval) {
 		switch_core_timer_destroy(&tech_pvt->timer);
 	}

src/switch_core_codec.c

@@ -646,6 +646,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_init_with_bitrate(switch_codec
 
 	memset(codec, 0, sizeof(*codec));
 
+	if (pool) {
+		codec->session = switch_core_memory_pool_get_data(pool, "__session");
+	}
+
 	if ((codec_interface = switch_loadable_module_get_codec_interface(codec_name)) == 0) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid codec %s!\n", codec_name);
 		return SWITCH_STATUS_GENERR;

src/switch_core_io.c

@@ -36,174 +36,6 @@
 #include <switch.h>
 #include "private/switch_core_pvt.h"
 
-SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_session_t *session, switch_image_write_callback_t callback, void *user_data)
-{
-	session->image_write_callback = callback;
-	session->image_write_callback_user_data = user_data;
-}
-
-SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame, 
-																	  switch_image_t *img, switch_size_t size, uint32_t *flag)
-{
-	uint32_t encoded_data_len = size, lflag = 0, *flagp = flag;
-	switch_codec_t *codec = switch_core_session_get_video_write_codec(session);
-	switch_timer_t *timer;
-
-	switch_assert(session);
-
-	if (!flag) {
-		flagp = &lflag;
-	}
-
-	timer = switch_core_media_get_timer(session, SWITCH_MEDIA_TYPE_VIDEO);
-	switch_assert(timer);
-
-	switch_core_codec_encode_video(codec, img, frame->data, &encoded_data_len, flagp);
-
-	while(encoded_data_len) {
-
-		frame->datalen = encoded_data_len;
-		frame->packetlen = frame->datalen + 12;
-		frame->m = (*flagp & SFF_MARKER) ? 1 : 0;
-		frame->timestamp = timer->samplecount;
-
-		switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME);
-		
-		switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0);
-
-		if (session->image_write_callback) {
-			session->image_write_callback(session, frame, img, session->image_write_callback_user_data);
-		}
-
-		encoded_data_len = size;
-		switch_core_codec_encode_video(codec, NULL, frame->data, &encoded_data_len, flagp);	
-	}
-
-	return SWITCH_STATUS_SUCCESS;
-}
-
-
-SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags,
-																	  int stream_id)
-{
-	switch_io_event_hook_video_write_frame_t *ptr;
-	switch_status_t status = SWITCH_STATUS_FALSE;
-
-	if (switch_channel_down(session->channel)) {
-		return SWITCH_STATUS_FALSE;
-	}
-
-	if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) {
-		return SWITCH_STATUS_SUCCESS;
-	}
-
-	if (session->endpoint_interface->io_routines->write_video_frame) {
-		if ((status = session->endpoint_interface->io_routines->write_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) {
-			for (ptr = session->event_hooks.video_write_frame; ptr; ptr = ptr->next) {
-				if ((status = ptr->video_write_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) {
-					break;
-				}
-			}
-		}
-	}
-	return status;
-}
-
-SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags,
-																	 int stream_id)
-{
-	switch_status_t status = SWITCH_STATUS_FALSE;
-	switch_io_event_hook_video_read_frame_t *ptr;
-
-	switch_assert(session != NULL);
-
-	if (switch_channel_down(session->channel)) {
-		return SWITCH_STATUS_FALSE;
-	}
-
-	if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) {
-		*frame = &runtime.dummy_cng_frame;
-		switch_yield(20000);
-		return SWITCH_STATUS_SUCCESS;
-	}
-
-	if (session->endpoint_interface->io_routines->read_video_frame) {
-		if ((status = session->endpoint_interface->io_routines->read_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) {
-			for (ptr = session->event_hooks.video_read_frame; ptr; ptr = ptr->next) {
-				if ((status = ptr->video_read_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) {
-					break;
-				}
-			}
-		}
-	}
-
-	if (status == SWITCH_STATUS_INUSE) {
-		*frame = &runtime.dummy_cng_frame;
-		switch_yield(20000);
-		return SWITCH_STATUS_SUCCESS;
-	}
-
-	if (status != SWITCH_STATUS_SUCCESS) {
-		goto done;
-	}
-
-	if (!(*frame)) {
-		goto done;
-	}
-
-	if (switch_test_flag(*frame, SFF_CNG)) {
-		status = SWITCH_STATUS_SUCCESS;
-		goto done;
-	}
-
-	if (session->bugs) {
-		switch_media_bug_t *bp;
-		switch_bool_t ok = SWITCH_TRUE;
-		int prune = 0;
-		switch_thread_rwlock_rdlock(session->bug_rwlock);
-		for (bp = session->bugs; bp; bp = bp->next) {
-			if (switch_channel_test_flag(session->channel, CF_PAUSE_BUGS) && !switch_core_media_bug_test_flag(bp, SMBF_NO_PAUSE)) {
-				continue;
-			}
-
-			if (!switch_channel_test_flag(session->channel, CF_ANSWERED) && switch_core_media_bug_test_flag(bp, SMBF_ANSWER_REQ)) {
-				continue;
-			}
-
-			if (switch_test_flag(bp, SMBF_PRUNE)) {
-				prune++;
-				continue;
-			}
-
-			if (bp->ready && switch_test_flag(bp, SMBF_READ_PING)) {
-				switch_mutex_lock(bp->read_mutex);
-				bp->ping_frame = *frame;
-				if (bp->callback) {
-					if (bp->callback(bp, bp->user_data, SWITCH_ABC_TYPE_READ_VIDEO_PING) == SWITCH_FALSE
-						|| (bp->stop_time && bp->stop_time <= switch_epoch_time_now(NULL))) {
-						ok = SWITCH_FALSE;
-					}
-				}
-				bp->ping_frame = NULL;;
-				switch_mutex_unlock(bp->read_mutex);
-			}
-
-			if (ok == SWITCH_FALSE) {
-				switch_set_flag(bp, SMBF_PRUNE);
-				prune++;
-			}
-		}
-		switch_thread_rwlock_unlock(session->bug_rwlock);
-		if (prune) {
-			switch_core_media_bug_prune(session);
-		}
-	}
-
-  done:
-
-	return status;
-}
-
 SWITCH_DECLARE(void) switch_core_gen_encoded_silence(unsigned char *data, const switch_codec_implementation_t *read_impl, switch_size_t len)
 {
 	unsigned char g729_filler[] = {