FS-10249: [mod_av] Audio gradually falls behind video in recordings #comment backport to 1.6

src/include/private/switch_core_pvt.h

@@ -228,6 +228,7 @@ struct switch_media_bug {
 	switch_image_t *spy_img[2];
 	switch_vid_spy_fmt_t spy_fmt;
 	switch_thread_t *video_bug_thread;
+	switch_mm_t mm;
 	struct switch_media_bug *next;
 };
 

src/include/switch_core.h

@@ -351,6 +351,9 @@ SWITCH_DECLARE(void) switch_core_media_bug_set_read_demux_frame(_In_ switch_medi
 */
 SWITCH_DECLARE(switch_core_session_t *) switch_core_media_bug_get_session(_In_ switch_media_bug_t *bug);
 
+SWITCH_DECLARE(void) switch_core_media_bug_set_media_params(switch_media_bug_t *bug, switch_mm_t *mm);
+SWITCH_DECLARE(void) switch_core_media_bug_get_media_params(switch_media_bug_t *bug, switch_mm_t *mm);
+
 /*!
   \brief Test for the existance of a flag on an media bug
   \param bug the object to test

src/include/switch_module_interfaces.h

@@ -301,41 +301,6 @@ struct switch_file_interface {
 	struct switch_file_interface *next;
 };
 
-typedef enum {
-	SWITCH_VIDEO_ENCODE_SPEED_DEFAULT = 0,
-	SWITCH_VIDEO_ENCODE_SPEED_FAST = 0,
-	SWITCH_VIDEO_ENCODE_SPEED_MEDIUM,
-	SWITCH_VIDEO_ENCODE_SPEED_SLOW
-} switch_video_encode_speed_t;
-
-typedef enum {
-	SWITCH_VIDEO_PROFILE_BASELINE,
-	SWITCH_VIDEO_PROFILE_MAIN,
-	SWITCH_VIDEO_PROFILE_HIGH
-} switch_video_profile_t;
-
-typedef struct switch_mm_s {
-	int samplerate;
-	int channels;
-	int keyint;
-	int ab;
-	int vb;
-	int vw;
-	int vh;
-	int cbr;
-	float fps;
-	float source_fps;
-	int vbuf;
-	switch_video_profile_t vprofile;
-	switch_video_encode_speed_t vencspd;
-	uint8_t try_hardware_encoder;
-	int scale_w;
-	int scale_h;
-	switch_img_fmt_t fmt;
-	char *auth_username;
-	char *auth_password;
-} switch_mm_t;
-
 /*! an abstract representation of a file handle (some parameters based on compat with libsndfile) */
 struct switch_file_handle {
 	/*! the interface of the module that implemented the current file type */

src/include/switch_types.h

@@ -2624,6 +2624,42 @@ typedef enum {
 	SCFC_PAUSE_READ
 } switch_file_command_t;
 
+typedef enum {
+	SWITCH_VIDEO_ENCODE_SPEED_DEFAULT = 0,
+	SWITCH_VIDEO_ENCODE_SPEED_FAST = 0,
+	SWITCH_VIDEO_ENCODE_SPEED_MEDIUM,
+	SWITCH_VIDEO_ENCODE_SPEED_SLOW
+} switch_video_encode_speed_t;
+
+typedef enum {
+	SWITCH_VIDEO_PROFILE_BASELINE,
+	SWITCH_VIDEO_PROFILE_MAIN,
+	SWITCH_VIDEO_PROFILE_HIGH
+} switch_video_profile_t;
+
+typedef struct switch_mm_s {
+	int samplerate;
+	int channels;
+	int keyint;
+	int ab;
+	int vb;
+	int vw;
+	int vh;
+	int cbr;
+	float fps;
+	float source_fps;
+	int vbuf;
+	switch_video_profile_t vprofile;
+	switch_video_encode_speed_t vencspd;
+	uint8_t try_hardware_encoder;
+	int scale_w;
+	int scale_h;
+	switch_img_fmt_t fmt;
+	char *auth_username;
+	char *auth_password;
+} switch_mm_t;
+
+
 SWITCH_END_EXTERN_C
 #endif
 /* For Emacs:

src/mod/applications/mod_av/avformat.c

@@ -49,6 +49,88 @@
 
 #define AV_TS_MAX_STRING_SIZE 32
 
+
+/* App interface */
+
+// a wrapper around a single output AVStream
+typedef struct MediaStream {
+	AVStream *st;
+	AVFrame *frame;
+	AVFrame *tmp_frame;
+
+	// audio
+	int channels;
+	int sample_rate;
+	struct AVAudioResampleContext *resample_ctx;
+
+	//video
+	int width;
+	int height;
+	struct SwsContext *sws_ctx;
+	int64_t next_pts;
+
+} MediaStream;
+
+typedef struct record_helper_s {
+	switch_mutex_t *mutex;
+	AVFormatContext *fc;
+	MediaStream *video_st;
+	switch_timer_t *timer;
+	int in_callback;
+	switch_queue_t *video_queue;
+	switch_thread_t *video_thread;
+	switch_mm_t *mm;
+	int finalize;
+} record_helper_t;
+
+
+
+/* file interface */
+
+struct av_file_context {
+	switch_memory_pool_t *pool;
+	switch_mutex_t *mutex;
+	switch_thread_cond_t *cond;
+	switch_buffer_t *buf;
+	switch_buffer_t *audio_buffer;
+	switch_timer_t video_timer;
+	int offset;
+	int audio_start;
+	int aud_ready;
+	int vid_ready;
+	int audio_ready;
+	int closed;
+
+	MediaStream video_st;
+	MediaStream audio_st;
+	AVFormatContext *fc;
+	AVCodec *audio_codec;
+	AVCodec *video_codec;
+
+	int has_audio;
+	int has_video;
+
+	record_helper_t eh;
+	switch_thread_t *file_read_thread;
+	int file_read_thread_running;
+	int file_read_thread_started;
+	switch_time_t video_start_time;
+	switch_image_t *last_img;
+	int read_fps;
+	switch_time_t last_vid_push;
+	int64_t seek_ts;
+	switch_bool_t read_paused;
+	int errs;
+	switch_file_handle_t *handle;
+};
+
+typedef struct av_file_context av_file_context_t;
+
+
+
+//FUCK
+
+
 /**
  * Fill the provided buffer with a string containing a timestamp
  * representation.
@@ -126,20 +208,61 @@ static void av_unused fill_avframe(AVFrame *pict, switch_image_t *img)
 
 static void av_unused avframe2img(AVFrame *pict, switch_image_t *img)
 {
-	int i;
-	uint8_t *y = pict->data[0];
-	uint8_t *u = pict->data[1];
-	uint8_t *v = pict->data[2];
+	int i, j;
 
-	/* Y */
-	for (i = 0; i < img->d_h; i++) {
-		memcpy(&img->planes[0][i * img->stride[0]], y + i * pict->linesize[0], img->d_w);
+	if (img->fmt == SWITCH_IMG_FMT_I420) {
+		if (pict->format == AV_PIX_FMT_YUV420P) {
+			switch_I420_copy2(pict->data, pict->linesize, img->planes, img->stride, img->d_w, img->d_h);
+		} else if (pict->format == AV_PIX_FMT_YUVA420P) {
+			int linesize[3];
+			linesize[0] = pict->linesize[0];
+			linesize[1] = pict->linesize[1];
+			linesize[2] = pict->linesize[2] + pict->linesize[0];
+
+			switch_I420_copy2(pict->data, linesize, img->planes, img->stride, img->d_w, img->d_h);
 		}
 
-	/* U/V */
-	for(i = 0; i < pict->height / 2; i++) {
-		memcpy(&img->planes[1][i * img->stride[1]], u + i * pict->linesize[1], img->d_w / 2);
-		memcpy(&img->planes[2][i * img->stride[2]], v + i * pict->linesize[2], img->d_w / 2);
+	} else if (img->fmt == SWITCH_IMG_FMT_ARGB) {
+		if (pict->format == AV_PIX_FMT_YUV420P) {
+			switch_rgb_color_t *color = (switch_rgb_color_t *)img->planes[SWITCH_PLANE_PACKED];
+			uint8_t *alpha = pict->data[3];
+
+			/*!\brief I420 to ARGB Convertion*/
+			switch_I420ToARGB(pict->data[0], pict->linesize[0],
+				pict->data[1], pict->linesize[1],
+				pict->data[2], pict->linesize[2],
+				img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED],
+				img->d_w, img->d_h);
+
+
+			for (j = 0; j < img->d_h; j++) {
+				for (i = 0; i < img->d_w; i++) {
+					color->a = *alpha++;
+					color++;
+				}
+				color = (switch_rgb_color_t *)(img->planes[SWITCH_PLANE_PACKED] + img->stride[SWITCH_PLANE_PACKED] * j);
+			}
+		} else if (pict->format == AV_PIX_FMT_RGBA) {
+#if SWITCH_BYTE_ORDER == __BIG_ENDIAN
+			switch_RGBAToARGB(pict->data[0], pict->linesize[0],
+				mg->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED],
+				img->d_w, img->d_h);
+#else
+			switch_ABGRToARGB(pict->data[0], pict->linesize[0],
+				img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED],
+				img->d_w, img->d_h);
+#endif
+		} else if (pict->format == AV_PIX_FMT_BGRA) {
+#if SWITCH_BYTE_ORDER == __BIG_ENDIAN
+			switch_BGRAToARGB(pict->data[0], pict->linesize[0],
+			, img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED],
+			, img->d_w, img->d_h);
+#else
+			switch_ARGBToARGB(pict->data[0], pict->linesize[0],
+			img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED],
+			img->d_w, img->d_h);
+#endif
+		}
 	}
 }
 
@@ -165,39 +288,6 @@ static void av_unused avframe2fd(AVFrame *pict, int fd)
 	}
 }
 
-/* App interface */
-
-// a wrapper around a single output AVStream
-typedef struct MediaStream {
-	AVStream *st;
-	AVFrame *frame;
-	AVFrame *tmp_frame;
-
-	// audio
-	int channels;
-	int sample_rate;
-	struct AVAudioResampleContext *resample_ctx;
-
-	//video
-	int width;
-	int height;
-	struct SwsContext *sws_ctx;
-	int64_t next_pts;
-
-} MediaStream;
-
-typedef struct record_helper_s {
-	switch_mutex_t *mutex;
-	AVFormatContext *fc;
-	MediaStream *video_st;
-	switch_timer_t *timer;
-	int in_callback;
-	switch_queue_t *video_queue;
-	switch_thread_t *video_thread;
-	switch_mm_t *mm;
-	int finalize;
-} record_helper_t;
-
 static void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt)
 {
 	AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
@@ -610,15 +700,15 @@ static int flush_video_queue(switch_queue_t *q, int min)
 
 static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *obj)
 {
-	record_helper_t *eh = (record_helper_t *) obj;
+	av_file_context_t *context = (av_file_context_t *) obj;
 	void *pop = NULL;
 	switch_image_t *img = NULL, *tmp_img = NULL;
-	int d_w = eh->video_st->width, d_h = eh->video_st->height;
+	int d_w = context->eh.video_st->width, d_h = context->eh.video_st->height;
 	int size = 0, skip = 0, skip_freq = 0, skip_count = 0, skip_total = 0, skip_total_count = 0;
 	uint64_t delta_avg = 0, delta_sum = 0, delta_i = 0, delta = 0, last_ts = 0;
 
 	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "video thread start\n");
-
+		switch_assert(context->eh.video_queue);
 	for(;;) {
 		AVPacket pkt = { 0 };
 		int got_packet;
@@ -626,12 +716,15 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
 
 	top:
 
-		if (switch_queue_pop(eh->video_queue, &pop) == SWITCH_STATUS_SUCCESS) {
+		switch_assert(context->eh.video_queue);
+
+		if (switch_queue_pop(context->eh.video_queue, &pop) == SWITCH_STATUS_SUCCESS) {
             switch_img_free(&img);
 
 			if (!pop) {
 				goto endfor;
 			}
+
 			img = (switch_image_t *) pop;
 			
 			if (!d_w) d_w = img->d_w;
@@ -639,11 +732,14 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
 
 			if (d_w && d_h && (d_w != img->d_w || d_h != img->d_h)) {
 				/* scale to match established stream */
+
 				switch_img_scale(img, &tmp_img, d_w, d_h);
+				if (tmp_img) {
 					switch_img_free(&img);
 					img = tmp_img;
 					tmp_img = NULL;
 				}
+			}
 		} else {
 			continue;
 		}
@@ -658,9 +754,9 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
 			}
 		} else {
 
-			size = switch_queue_size(eh->video_queue);
+			size = switch_queue_size(context->eh.video_queue);
 
-			if (size > 5 && !eh->finalize) {
+			if (size > 5 && !context->eh.finalize) {
 				skip = size;
 
 				if (size > 10) {
@@ -673,65 +769,71 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
 			}
 		}
 
-		//switch_mutex_lock(eh->mutex);
+		//switch_mutex_lock(context->eh.mutex);
 
-		eh->in_callback = 1;
+		context->eh.in_callback = 1;
 
 		av_init_packet(&pkt);
 
-		if (eh->video_st->frame) {
-			ret = av_frame_make_writable(eh->video_st->frame);
+		if (context->eh.video_st->frame) {
+			ret = av_frame_make_writable(context->eh.video_st->frame);
 		}
 
 		if (ret < 0) {
 			continue;
 		}
 
-		fill_avframe(eh->video_st->frame, img);
+		fill_avframe(context->eh.video_st->frame, img);
 
-		if (eh->finalize) {
+		if (context->eh.finalize) {
 			if (delta_i && !delta_avg) {
 				delta_avg = (int)(double)(delta_sum / delta_i);
-				delta_i = 0;
+				delta_i = 1;
 				delta_sum = delta_avg;
 			}
 
 			if (delta_avg) {
 				delta = delta_avg;
-			} else if (eh->mm->fps) {
-				delta = 1000 / eh->mm->fps;
+			} else if (context->eh.mm->fps) {
+				delta = 1000 / context->eh.mm->fps;
 			} else {
 				delta = 33;
 			}
 
-			eh->video_st->frame->pts += delta;
+			context->eh.video_st->frame->pts += delta;
 		} else {
 			uint64_t delta_tmp;
 
-			switch_core_timer_sync(eh->timer);
-			delta_tmp = eh->timer->samplecount - last_ts;
+			switch_core_timer_sync(context->eh.timer);
+			delta_tmp = context->eh.timer->samplecount - last_ts;
 
-			if (delta_tmp != last_ts) {
+			if (delta_tmp != 0) {
 				delta_sum += delta_tmp;
 				delta_i++;
 				
-				if (delta_i >= 60) {
-					delta_avg = (int)(double)(delta_sum / delta_i);
-					delta_i = 0;
+				if (delta_i == UINT64_MAX) {
+					delta_i = 1;
 					delta_sum = delta_avg;
 				}
 				
-				eh->video_st->frame->pts = eh->timer->samplecount;
+				
+				if ((delta_i % 10) == 0) {
+					delta_avg = (int)(double)(delta_sum / delta_i);
+				}
+
+				context->eh.video_st->frame->pts = context->eh.timer->samplecount;
+			} else {
+				context->eh.video_st->frame->pts = context->eh.timer->samplecount + 1;
 			}
 		}
 
-		last_ts = eh->video_st->frame->pts;
+		last_ts = context->eh.video_st->frame->pts;
 
-		//eh->video_st->frame->pts = switch_time_now() / 1000 - eh->video_st->next_pts;
-		//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pts: %ld\n", eh->video_st->frame->pts);
+		//context->eh.video_st->frame->pts = switch_time_now() / 1000 - context->eh.video_st->next_pts;
+		//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pts: %ld\n", context->eh.video_st->frame->pts);
 
 		/* encode the image */
-		ret = avcodec_encode_video2(eh->video_st->st->codec, &pkt, eh->video_st->frame, &got_packet);
+		ret = avcodec_encode_video2(context->eh.video_st->st->codec, &pkt, context->eh.video_st->frame, &got_packet);
 
 		if (ret < 0) {
 			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoding Error %d\n", ret);
@@ -739,14 +841,14 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
 		}
 
 		if (got_packet) {
-			switch_mutex_lock(eh->mutex);
-			ret = write_frame(eh->fc, &eh->video_st->st->codec->time_base, eh->video_st->st, &pkt);
-			switch_mutex_unlock(eh->mutex);
+			switch_mutex_lock(context->eh.mutex);
+			ret = write_frame(context->eh.fc, &context->eh.video_st->st->codec->time_base, context->eh.video_st->st, &pkt);
+			switch_mutex_unlock(context->eh.mutex);
 			av_packet_unref(&pkt);
 		}
 
-		eh->in_callback = 0;
-		//switch_mutex_unlock(eh->mutex);
+		context->eh.in_callback = 0;
+		//switch_mutex_unlock(context->eh.mutex);
 	}
 
  endfor:
@@ -758,14 +860,14 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
 
 		av_init_packet(&pkt);
 
-		ret = avcodec_encode_video2(eh->video_st->st->codec, &pkt, NULL, &got_packet);
+		ret = avcodec_encode_video2(context->eh.video_st->st->codec, &pkt, NULL, &got_packet);
 
 		if (ret < 0) {
 			break;
 		} else if (got_packet) {
-			switch_mutex_lock(eh->mutex);
-			ret = write_frame(eh->fc, &eh->video_st->st->codec->time_base, eh->video_st->st, &pkt);
-			switch_mutex_unlock(eh->mutex);
+			switch_mutex_lock(context->eh.mutex);
+			ret = write_frame(context->eh.fc, &context->eh.video_st->st->codec->time_base, context->eh.video_st->st, &pkt);
+			switch_mutex_unlock(context->eh.mutex);
 			av_packet_unref(&pkt);
 			if (ret < 0) break;
 		} else {
@@ -773,7 +875,7 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
 		}
 	}
 
-	while(switch_queue_trypop(eh->video_queue, &pop) == SWITCH_STATUS_SUCCESS) {
+	while(switch_queue_trypop(context->eh.video_queue, &pop) == SWITCH_STATUS_SUCCESS) {
 		if (!pop) break;
 		img = (switch_image_t *) pop;
 		switch_img_free(&img);
@@ -813,7 +915,6 @@ SWITCH_STANDARD_APP(record_av_function)
 	switch_status_t status;
 	switch_frame_t *read_frame;
 	switch_channel_t *channel = switch_core_session_get_channel(session);
-	record_helper_t eh = { 0 };
 	switch_timer_t timer = { 0 };
 	switch_mutex_t *mutex = NULL;
 	switch_codec_t codec;//, *vid_codec;
@@ -831,6 +932,7 @@ SWITCH_STANDARD_APP(record_av_function)
 	AVCodec *audio_codec, *video_codec;
 	int has_audio = 0, has_video = 0;
 	int ret;
+	av_file_context_t context = { 0 };
 
 	switch_channel_answer(channel);
 	switch_core_session_get_read_impl(session, &read_impl);
@@ -949,23 +1051,22 @@ SWITCH_STANDARD_APP(record_av_function)
 
 	if (has_video) {
 		switch_threadattr_t *thd_attr = NULL;
-
 		switch_mutex_init(&mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
-		eh.mutex = mutex;
-		eh.video_st = &video_st;
-		eh.fc = fc;
+		context.eh.mutex = mutex;
+		context.eh.video_st = &video_st;
+		context.eh.fc = fc;
 		if (switch_core_timer_init(&timer, "soft", 1, 1, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
 			switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Timer Activation Fail\n");
 			goto end;
 		}
-		eh.timer = &timer;
-		switch_queue_create(&eh.video_queue, SWITCH_CORE_QUEUE_LEN, switch_core_session_get_pool(session));
-		switch_core_session_set_video_read_callback(session, video_read_callback, (void *)&eh);
+		context.eh.timer = &timer;
+		switch_queue_create(&context.eh.video_queue, SWITCH_CORE_QUEUE_LEN, switch_core_session_get_pool(session));
+		switch_core_session_set_video_read_callback(session, video_read_callback, (void *)&context.eh);
 
 		switch_threadattr_create(&thd_attr, switch_core_session_get_pool(session));
 		//switch_threadattr_priority_set(thd_attr, SWITCH_PRI_REALTIME);
 		switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
-		switch_thread_create(&eh.video_thread, thd_attr, video_thread_run, &eh, switch_core_session_get_pool(session));
+		switch_thread_create(&context.eh.video_thread, thd_attr, video_thread_run, &context, switch_core_session_get_pool(session));
 	}
 
 	switch_core_session_set_read_codec(session, &codec);
@@ -1096,8 +1197,8 @@ SWITCH_STANDARD_APP(record_av_function)
 		int ret = 0;
 		switch_status_t st;
 
-		switch_queue_push(eh.video_queue, NULL);
-		switch_thread_join(&st, eh.video_thread);
+		switch_queue_push(context.eh.video_queue, NULL);
+		switch_thread_join(&st, context.eh.video_thread);
 
 	again:
 		av_init_packet(&pkt);
@@ -1252,45 +1353,6 @@ SWITCH_STANDARD_API(av_format_api_function)
 	return SWITCH_STATUS_SUCCESS;
 }
 
-/* file interface */
-
-struct av_file_context {
-	switch_memory_pool_t *pool;
-	switch_mutex_t *mutex;
-	switch_thread_cond_t *cond;
-	switch_buffer_t *buf;
-	switch_buffer_t *audio_buffer;
-	switch_timer_t video_timer;
-	int offset;
-	int audio_start;
-	int aud_ready;
-	int vid_ready;
-	int audio_ready;
-	int closed;
-
-	MediaStream video_st;
-	MediaStream audio_st;
-	AVFormatContext *fc;
-	AVCodec *audio_codec;
-	AVCodec *video_codec;
-
-	int has_audio;
-	int has_video;
-
-	record_helper_t eh;
-	switch_thread_t *file_read_thread;
-	int file_read_thread_running;
-	int file_read_thread_started;
-	switch_time_t video_start_time;
-	switch_image_t *last_img;
-	int read_fps;
-	switch_time_t last_vid_push;
-	int64_t seek_ts;
-	switch_bool_t read_paused;
-	int errs;
-};
-
-typedef struct av_file_context av_file_context_t;
 
 static void mod_avformat_destroy_output_context(av_file_context_t *context)
 {
@@ -1309,13 +1371,15 @@ static switch_status_t open_input_file(av_file_context_t *context, switch_file_h
 {
 	AVCodec *audio_codec = NULL;
 	AVCodec *video_codec = NULL;
+	AVDictionary *opts = NULL;
 	int error;
 	int i;
 	switch_status_t status = SWITCH_STATUS_SUCCESS;
 
+	// av_dict_set(&opts, "c:v", "libvpx", 0);
+
 	/** Open the input file to read from it. */
-	if ((error = avformat_open_input(&context->fc, filename, NULL,
-									 NULL)) < 0) {
+	if ((error = avformat_open_input(&context->fc, filename, NULL, NULL)) < 0) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open input file '%s' (error '%s')\n", filename, get_error_text(error));
 		switch_goto_status(SWITCH_STATUS_FALSE, err);
 	}
@@ -1324,11 +1388,14 @@ static switch_status_t open_input_file(av_file_context_t *context, switch_file_h
 	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "file %s is %sseekable\n", filename, handle->seekable ? "" : "not ");
 
 	/** Get information on the input file (number of streams etc.). */
-	if ((error = avformat_find_stream_info(context->fc, NULL)) < 0) {
+	if ((error = avformat_find_stream_info(context->fc, opts ? &opts : NULL)) < 0) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open find stream info (error '%s')\n", get_error_text(error));
+		if (opts) av_dict_free(&opts);
 		switch_goto_status(SWITCH_STATUS_FALSE, err);
 	}
 
+	if (opts) av_dict_free(&opts);
+
 	av_dump_format(context->fc, 0, filename, 0);
 
 	for (i = 0; i< context->fc->nb_streams; i++) {
@@ -1341,7 +1408,12 @@ static switch_status_t open_input_file(av_file_context_t *context, switch_file_h
 				context->has_video = 1;
 				handle->duration = av_rescale_q(context->video_st.st->duration, context->video_st.st->time_base, AV_TIME_BASE_Q);
 			}
+			if (context->video_st.st->avg_frame_rate.num) {
 				handle->mm.source_fps = ceil(av_q2d(context->video_st.st->avg_frame_rate));
+			} else {
+				handle->mm.source_fps = 25;
+			}
+
 			context->read_fps = (int)handle->mm.source_fps;
 		}
 	}
@@ -1416,6 +1488,21 @@ static switch_status_t open_input_file(av_file_context_t *context, switch_file_h
 
 	if (!context->has_video) {
 		switch_clear_flag(handle, SWITCH_FILE_FLAG_VIDEO);
+	} else {
+		switch (context->video_st.st->codec->pix_fmt) {
+		case AV_PIX_FMT_YUVA420P:
+		case AV_PIX_FMT_RGBA:
+		case AV_PIX_FMT_ARGB:
+		case AV_PIX_FMT_BGRA:
+			context->handle->mm.fmt = SWITCH_IMG_FMT_ARGB;
+			break;
+		default:
+			context->handle->mm.fmt = SWITCH_IMG_FMT_I420;
+			break;
+		}
+		
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, 
+						  "Opening file in mode: %s\n", context->handle->mm.fmt == SWITCH_IMG_FMT_ARGB ? "ARGB" : "I420");
 	}
 
 	return status;
@@ -1458,9 +1545,7 @@ static void *SWITCH_THREAD_FUNC file_read_thread_run(switch_thread_t *thread, vo
 			switch_buffer_zero(context->audio_buffer);
 			switch_mutex_unlock(context->mutex);
 
-			if (context->eh.video_queue) {
-				flush_video_queue(context->eh.video_queue, 0);
-			}
+
 			
 			// if (context->has_audio) stream_id = context->audio_st.st->index;
 			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "seeking to %" SWITCH_INT64_T_FMT "\n", context->seek_ts);
@@ -1484,6 +1569,8 @@ static void *SWITCH_THREAD_FUNC file_read_thread_run(switch_thread_t *thread, vo
 			continue;
 		}
 
+
+
 		av_init_packet(&pkt);
 		pkt.data = NULL;
 		pkt.size = 0;
@@ -1531,11 +1618,17 @@ again:
 			//	av_frame_free(&vframe);
 			//	continue;
 			//}
+			// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got_data=%d, error=%d\n", got_data, error);
 
 			if (got_data && error >= 0) {
-				// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got picture %dx%d fmt: %d pktpts:%lld pktdts:%lld\n", vframe->width, vframe->height, vframe->format, vframe->pkt_pts, vframe->pkt_dts);
-
-				if (vframe->format != AV_PIX_FMT_YUV420P) {
+				switch_img_fmt_t fmt = SWITCH_IMG_FMT_I420;
+				if ((
+						vframe->format == AV_PIX_FMT_YUVA420P ||
+						vframe->format == AV_PIX_FMT_RGBA ||
+						vframe->format == AV_PIX_FMT_ARGB ||
+						vframe->format == AV_PIX_FMT_BGRA )) {
+					fmt = SWITCH_IMG_FMT_ARGB;
+				} else if (vframe->format != AV_PIX_FMT_YUV420P) {
 					AVFrame *frm = vframe;
 					int ret;
 
@@ -1577,7 +1670,10 @@ again:
 					}
 				}
 
-				img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, vframe->width, vframe->height, 1);
+				context->handle->mm.fmt = fmt;
+
+				img = switch_img_alloc(NULL, fmt, vframe->width, vframe->height, 1);
+
 				if (img) {
 					int64_t *pts = malloc(sizeof(int64_t));
 
@@ -1603,9 +1699,13 @@ again:
 						context->vid_ready = 1;
 						switch_queue_push(context->eh.video_queue, img);
 						context->last_vid_push = switch_time_now();
+						
+						
+
 					}
 				}
 			}
+
 			av_frame_free(&vframe);
 
 			if (eof) {
@@ -1721,9 +1821,13 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
 	context->pool = handle->memory_pool;
 	context->seek_ts = -1;
 	context->offset = DFT_RECORD_OFFSET;
-	if (handle->params && (tmp = switch_event_get_header(handle->params, "av_video_offset"))) {
+	context->handle = handle;
+
+	if (handle->params) {
+		if ((tmp = switch_event_get_header(handle->params, "av_video_offset"))) {
 			context->offset = atoi(tmp);
 		}
+	}
 
 	switch_mutex_init(&context->mutex, SWITCH_MUTEX_NESTED, handle->memory_pool);
 	switch_thread_cond_create(&context->cond, handle->memory_pool);
@@ -1745,7 +1849,7 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
 		}
 
 		if (context->has_video) {
-			switch_queue_create(&context->eh.video_queue, SWITCH_CORE_QUEUE_LEN, handle->memory_pool);
+			switch_queue_create(&context->eh.video_queue, context->read_fps, handle->memory_pool);
 			switch_mutex_init(&context->eh.mutex, SWITCH_MUTEX_NESTED, handle->memory_pool);
 			switch_core_timer_init(&context->video_timer, "soft", 66, 1, context->pool);
 		}
@@ -2058,8 +2162,14 @@ static switch_status_t av_file_close(switch_file_handle_t *handle)
 	context->eh.finalize = 1;
 
 	if (context->eh.video_queue) {
+		if (!switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) {
+			flush_video_queue(context->eh.video_queue, 0);
+			switch_queue_push(context->eh.video_queue, NULL);
+			switch_queue_term(context->eh.video_queue);
+		} else {
 			switch_queue_push(context->eh.video_queue, NULL);
 		}
+	}
 
 	if (context->eh.video_thread) {
 		switch_thread_join(&status, context->eh.video_thread);
@@ -2135,7 +2245,7 @@ static switch_status_t av_file_read(switch_file_handle_t *handle, void *data, si
 	int size;
 	size_t need = *len * 2 * context->audio_st.channels;
 
-	if (!context->has_audio && context->has_video && switch_queue_size(context->eh.video_queue) > 0) {
+	if (!context->has_audio && context->has_video && context->file_read_thread_running) {
 		memset(data, 0, *len * handle->channels * 2);
 		return SWITCH_STATUS_SUCCESS;
 	}
@@ -2187,7 +2297,6 @@ static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_f
 
 
 	if (!context->has_video || context->closed) return SWITCH_STATUS_FALSE;
-
 	if ((flags & SVR_CHECK)) {
 		return SWITCH_STATUS_BREAK;
 	}
@@ -2210,6 +2319,16 @@ static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_f
 		context->vid_ready = 1;
 
 		frame->img = (switch_image_t *) pop;
+
+		if (frame->img) {
+			if (frame->img && context->handle->mm.scale_w && context->handle->mm.scale_h) {
+				if (frame->img->d_w != context->handle->mm.scale_w || frame->img->d_h != context->handle->mm.scale_h) {
+					switch_img_fit(&frame->img, context->handle->mm.scale_w, context->handle->mm.scale_h, SWITCH_FIT_SIZE);
+				}
+			}
+			context->vid_ready = 1;
+		}		
+
 		return SWITCH_STATUS_SUCCESS;
 	}
 
@@ -2413,6 +2532,11 @@ static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_f
 	}
 
 	if (frame->img) {
+		if (frame->img && context->handle->mm.scale_w && context->handle->mm.scale_h) {
+			if (frame->img->d_w != context->handle->mm.scale_w || frame->img->d_h != context->handle->mm.scale_h) {
+				switch_img_fit(&frame->img, context->handle->mm.scale_w, context->handle->mm.scale_h, SWITCH_FIT_SCALE);
+			}
+		}
 		context->vid_ready = 1;
 	}
 	if (!frame->img) context->closed = 1;
@@ -2457,7 +2581,10 @@ static switch_status_t av_file_write_video(switch_file_handle_t *handle, switch_
 		}
 	}
 
-	if (!context->vid_ready) {
+	if (context->has_video) {
+		switch_image_t *img = NULL;
+
+		if (!context->eh.video_thread) {
 			switch_threadattr_t *thd_attr = NULL;
 			
 			switch_mutex_init(&context->mutex, SWITCH_MUTEX_NESTED, handle->memory_pool);
@@ -2465,27 +2592,28 @@ static switch_status_t av_file_write_video(switch_file_handle_t *handle, switch_
 			context->eh.video_st = &context->video_st;
 			context->eh.fc = context->fc;
 			context->eh.mm = &handle->mm;
-		context->eh.timer = &context->video_timer;
 			switch_queue_create(&context->eh.video_queue, SWITCH_CORE_QUEUE_LEN, handle->memory_pool);
-
 			switch_threadattr_create(&thd_attr, handle->memory_pool);
 			//switch_threadattr_priority_set(thd_attr, SWITCH_PRI_REALTIME);
 			switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
-		switch_thread_create(&context->eh.video_thread, thd_attr, video_thread_run, &context->eh, handle->memory_pool);
 			switch_core_timer_init(&context->video_timer, "soft", 1, 1, context->pool);
-		switch_buffer_zero(context->audio_buffer);
+			context->eh.timer = &context->video_timer;
 			context->audio_st.frame->pts = 0;
 			context->audio_st.next_pts = 0;
+			switch_thread_create(&context->eh.video_thread, thd_attr, video_thread_run, context, handle->memory_pool);
 		}
 
-	if (context->has_video) {
-		switch_image_t *img = NULL;
-
 		switch_img_copy(frame->img, &img);
 		switch_queue_push(context->eh.video_queue, img);
-	}
 		
+		if (!context->vid_ready) {
+			switch_mutex_lock(context->mutex);
+			switch_buffer_zero(context->audio_buffer);
+			switch_mutex_unlock(context->mutex);
 			context->vid_ready = 1;
+		}
+
+	}
 
 end:
 	return status;

src/switch_core_media_bug.c

@@ -83,6 +83,16 @@ SWITCH_DECLARE(uint32_t) switch_core_media_bug_clear_flag(switch_media_bug_t *bu
 	return switch_clear_flag(bug, flag);
 }
 
+SWITCH_DECLARE(void) switch_core_media_bug_set_media_params(switch_media_bug_t *bug, switch_mm_t *mm)
+{
+	bug->mm = *mm;
+}
+
+SWITCH_DECLARE(void) switch_core_media_bug_get_media_params(switch_media_bug_t *bug, switch_mm_t *mm)
+{
+	*mm = bug->mm;
+}
+
 SWITCH_DECLARE(switch_core_session_t *) switch_core_media_bug_get_session(switch_media_bug_t *bug)
 {
 	return bug->session;
@@ -522,15 +532,36 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_bug_patch_spy_frame(switch_med
 	return SWITCH_STATUS_FALSE;
 }
 
+
+static int flush_video_queue(switch_queue_t *q, int min)
+{
+	void *pop;
+
+	if (switch_queue_size(q) > min) {
+		while (switch_queue_trypop(q, &pop) == SWITCH_STATUS_SUCCESS) {
+			switch_image_t *img = (switch_image_t *) pop;
+			switch_img_free(&img);
+			if (min && switch_queue_size(q) <= min) {
+				break;
+			}
+		}
+	}
+
+	return switch_queue_size(q);
+}
+
 static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *obj)
 {
 	switch_media_bug_t *bug = (switch_media_bug_t *) obj;
 	switch_queue_t *main_q = NULL, *other_q = NULL;
 	switch_image_t *IMG = NULL, *img = NULL, *other_img = NULL;
-	void *pop;
+	void *pop, *other_pop;
 	uint8_t *buf;
 	switch_size_t buflen = SWITCH_RTP_MAX_BUF_LEN;
 	switch_frame_t frame = { 0 };
+	switch_timer_t timer = { 0 };
+	switch_mm_t mm = { 0 };
+	int fps = 15;
 
 	buf = switch_core_session_alloc(bug->session, buflen);
 	frame.packet = buf;
@@ -551,12 +582,29 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
 		return NULL;
 	}
 
+	switch_core_media_bug_get_media_params(bug, &mm);
+
+	if (mm.fps) {
+		fps = (int) mm.fps;
+	}
+
+	switch_core_timer_init(&timer, "soft", 1000 / fps, (90000 / (1000 / fps)), NULL);
+
 	while (bug->ready) {
 		switch_status_t status;
-		int w = 0, h = 0, ok = 1;
+		int w = 0, h = 0, ok = 1, new_main = 0, new_other = 0, new_canvas = 0;
+		
+		switch_core_timer_next(&timer);
+
+		if (!switch_channel_test_flag(bug->session->channel, CF_ANSWERED) && switch_core_media_bug_test_flag(bug, SMBF_ANSWER_REQ)) {
+			flush_video_queue(main_q, 0);
+			if (other_q) flush_video_queue(other_q, 0);
+			continue;
+		}
 
+		flush_video_queue(main_q, 1);
 
-		if ((status = switch_queue_pop(main_q, &pop)) == SWITCH_STATUS_SUCCESS) {
+		if ((status = switch_queue_trypop(main_q, &pop)) == SWITCH_STATUS_SUCCESS) {
 			switch_img_free(&img);
 
 			if (!pop) {
@@ -564,21 +612,25 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
 			}
 
 			img = (switch_image_t *) pop;
+			new_main = 1;
 
 			w = img->d_w;
 			h = img->d_h;
+		}
 		
 		if (other_q) {
-				while(switch_queue_size(other_q) > 0) {
-					if ((status = switch_queue_trypop(other_q, &pop)) == SWITCH_STATUS_SUCCESS) {
+			flush_video_queue(other_q, 1);
+			
+			if ((status = switch_queue_trypop(other_q, &other_pop)) == SWITCH_STATUS_SUCCESS) {
 				switch_img_free(&other_img);
-						if (!(other_img = (switch_image_t *) pop)) {
-							goto end;
-						}
-					}
+				other_img = (switch_image_t *) other_pop;
+				new_other = 1;
 			}
 
 			if (other_img) {
+				if (!w) w = other_img->d_w;
+				if (!h) h = other_img->d_h;
+				
 				if (other_img->d_w != w || other_img->d_h != h) {
 					switch_image_t *tmp_img = NULL;
 
@@ -588,29 +640,55 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
 				}
 			}
 
+			if (!(w&&h)) continue;
+
+			if (img) {
+				if (img->d_w != w || img->d_h != h) {
+					switch_image_t *tmp_img = NULL;
+					
+					switch_img_scale(img, &tmp_img, w, h);
+					switch_img_free(&img);
+					img = tmp_img;
+				}
+			}
+			
 			w *= 2;
 
 			if (!IMG || IMG->d_h != h || IMG->d_w != w) {
+				switch_rgb_color_t color = { 0 };
+
 				switch_img_free(&IMG);
 				IMG = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, w, h, 1);
+				new_canvas = 1;
+				switch_color_set_rgb(&color, "#000000");
+				switch_img_fill(IMG, 0, 0, IMG->d_w, IMG->d_h, &color);
+			}
 		}
 
+		
+		if (IMG) {
+			if (img && (new_canvas || new_main)) {
 				switch_img_patch(IMG, img, 0, 0);
+			}
 		
-				if (other_img) {
+			if (other_img && (new_canvas || new_other)) {
 				switch_img_patch(IMG, other_img, w / 2, 0);
 			}
 		}
 		
+		if (IMG || img) {
 			switch_thread_rwlock_rdlock(bug->session->bug_rwlock);
 			frame.img = other_q ? IMG : img;
+
 			bug->video_ping_frame = &frame;
+			
 			if (bug->callback) {
 				if (bug->callback(bug, bug->user_data, SWITCH_ABC_TYPE_STREAM_VIDEO_PING) == SWITCH_FALSE
 					|| (bug->stop_time && bug->stop_time <= switch_epoch_time_now(NULL))) {
 					ok = SWITCH_FALSE;
 				}
 			}
+
 			bug->video_ping_frame = NULL;
 			switch_thread_rwlock_unlock(bug->session->bug_rwlock);
 
@@ -623,6 +701,8 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
 
  end:
 
+	switch_core_timer_destroy(&timer);
+	
 	switch_img_free(&IMG);
 	switch_img_free(&img);
 	switch_img_free(&other_img);

src/switch_ivr_async.c

@@ -1507,7 +1507,7 @@ static switch_bool_t record_callback(switch_media_bug_t *bug, void *user_data, s
 		break;
 	case SWITCH_ABC_TYPE_READ_VIDEO_PING:
 	case SWITCH_ABC_TYPE_STREAM_VIDEO_PING:
-		if (rh->fh) {
+		if (rh->fh && switch_test_flag(rh->fh, SWITCH_FILE_OPEN)) {
 			if (!bug->video_ping_frame) break;
 			
 			if ((len || bug->video_ping_frame->img) && switch_core_file_write_video(rh->fh, bug->video_ping_frame) != SWITCH_STATUS_SUCCESS && 
@@ -2559,11 +2559,13 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_record_session(switch_core_session_t
 			//switch_core_media_set_video_file(session, fh, SWITCH_RW_READ);
 			//switch_channel_set_flag_recursive(session->channel, CF_VIDEO_DECODED_READ);
 			
-			if ((vval = switch_channel_get_variable(channel, "record_concat_video")) && switch_true(vval)) { 
+			if (switch_channel_var_true(channel, "record_concat_video")) {
 				flags |= SMBF_READ_VIDEO_STREAM;
 				flags |= SMBF_WRITE_VIDEO_STREAM;
+			} else if (switch_channel_var_true(channel, "record_bleg_video")) {
+				flags |= SMBF_WRITE_VIDEO_STREAM;
 			} else {
-				flags |= SMBF_READ_VIDEO_PING;
+				flags |= SMBF_READ_VIDEO_STREAM;
 			}
 		} else {
 			flags &= ~SMBF_READ_VIDEO_PING;