FS-10050: [core] chromakey

src/include/switch_core_video.h

@@ -44,6 +44,16 @@
 
 SWITCH_BEGIN_EXTERN_C
 
+#define CHROMAKEY_MAX_MASK 25
+
+typedef enum {
+	SWITCH_SHADE_NONE = 0,
+	SWITCH_SHADE_RED,
+	SWITCH_SHADE_GREEN,
+	SWITCH_SHADE_BLUE,
+	SWITCH_SHADE_AUTO
+} switch_shade_t;
+
 typedef enum {
 	POS_LEFT_TOP = 0,
 	POS_LEFT_MID,
@@ -420,8 +430,16 @@ SWITCH_DECLARE(switch_status_t) switch_I420_copy2(uint8_t *src_planes[], int src
 /*!\brief chromakey an img, img must be RGBA and return modified img */
 
 SWITCH_DECLARE(void) switch_img_chromakey(switch_image_t *img, switch_rgb_color_t *mask, int threshold);
-SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_image_t *cache_img, switch_rgb_color_t *mask, int *thresholds, int count);
-
+SWITCH_DECLARE(switch_status_t) switch_chromakey_clear_colors(switch_chromakey_t *ck);
+								
+SWITCH_DECLARE(switch_status_t) switch_chromakey_autocolor(switch_chromakey_t *ck, switch_shade_t autocolor, uint32_t threshold);
+SWITCH_DECLARE(switch_status_t) switch_chromakey_add_color(switch_chromakey_t *ck, switch_rgb_color_t *color, uint32_t threshold);
+SWITCH_DECLARE(switch_status_t) switch_chromakey_destroy(switch_chromakey_t **ckP);
+SWITCH_DECLARE(switch_status_t) switch_chromakey_create(switch_chromakey_t **ckP);
+SWITCH_DECLARE(void) switch_chromakey_set_default_threshold(switch_chromakey_t *ck, uint32_t threshold);
+SWITCH_DECLARE(void) switch_chromakey_process(switch_chromakey_t *ck, switch_image_t *img);
+SWITCH_DECLARE(switch_image_t *) switch_chromakey_cache_image(switch_chromakey_t *ck);
+SWITCH_DECLARE(switch_shade_t) switch_chromakey_str2shade(switch_chromakey_t *ck, const char *shade_name);
 
 SWITCH_END_EXTERN_C
 #endif

src/include/switch_types.h

@@ -2666,6 +2666,9 @@ struct switch_rtp_text_factory_s;
 typedef struct switch_rtp_text_factory_s  switch_rtp_text_factory_t;
 typedef struct switch_agc_s switch_agc_t;
 
+struct switch_chromakey_s;
+typedef struct switch_chromakey_s switch_chromakey_t;
+
 SWITCH_END_EXTERN_C
 #endif
 /* For Emacs:

src/mod/applications/mod_video_filter/mod_video_filter.c

@@ -38,13 +38,11 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_video_filter_load);
 SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_video_filter_shutdown);
 SWITCH_MODULE_DEFINITION(mod_video_filter, mod_video_filter_load, mod_video_filter_shutdown, NULL);
 
-#define MAX_MASK 25
 
 typedef struct chromakey_context_s {
 	int threshold;
 	switch_image_t *bgimg;
 	switch_image_t *bgimg_scaled;
-	switch_image_t *last_img;
 	switch_image_t *imgfg;
 	switch_image_t *imgbg;
 	void *data;
@@ -53,13 +51,11 @@ typedef struct chromakey_context_s {
 	switch_size_t patch_datalen;
 	switch_file_handle_t vfh;
 	switch_rgb_color_t bgcolor;
-	switch_rgb_color_t mask[MAX_MASK];
-	int thresholds[MAX_MASK];
-	int mask_len;
 	switch_core_session_t *session;
 	switch_mutex_t *command_mutex;
 	int patch;
 	int mod;
+	switch_chromakey_t *ck;
 } chromakey_context_t;
 
 static void init_context(chromakey_context_t *context)
@@ -67,6 +63,7 @@ static void init_context(chromakey_context_t *context)
 	switch_color_set_rgb(&context->bgcolor, "#000000");
 	context->threshold = 300;
 	switch_mutex_init(&context->command_mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(context->session));
+	switch_chromakey_create(&context->ck);
 }
 
 static void uninit_context(chromakey_context_t *context)
@@ -80,9 +77,9 @@ static void uninit_context(chromakey_context_t *context)
 		memset(&context->vfh, 0, sizeof(context->vfh));
 	}
 
-	switch_img_free(&context->last_img);
 	switch_safe_free(context->data);
 	switch_safe_free(context->patch_data);
+	switch_chromakey_destroy(&context->ck);
 }
 
 static void parse_params(chromakey_context_t *context, int start, int argc, char **argv, const char **function, switch_media_bug_flag_t *flags)
@@ -96,29 +93,30 @@ static void parse_params(chromakey_context_t *context, int start, int argc, char
 
 	if (n > 0 && argv[i]) { // color
 		int j = 0;
-		char *list[MAX_MASK];
+		char *list[CHROMAKEY_MAX_MASK];
 		int list_argc;
 
 		list_argc = switch_split(argv[i], ':', list);
 
-		context->mask_len = 0;
-		memset(context->thresholds, 0, sizeof(context->thresholds[0]) * MAX_MASK);
+		switch_chromakey_clear_colors(context->ck);
 
 		for (j = 0; j < list_argc; j++) {
 			char *p;
 			int thresh = 0;
-
+			switch_rgb_color_t color = { 0 };
+			
 			if ((p = strchr(list[j], '+'))) {
 				*p++ = '\0';
 				thresh = atoi(p);
 				if (thresh < 0) thresh = 0;
 			}
-			
-			switch_color_set_rgb(&context->mask[j], list[j]);
-			if (thresh) {
-				context->thresholds[j] = thresh*thresh;
+	
+			if (*list[j] == '#') {
+				switch_color_set_rgb(&color, list[j]);
+				switch_chromakey_add_color(context->ck, &color, thresh);
+			} else {
+				switch_chromakey_autocolor(context->ck, switch_chromakey_str2shade(context->ck, list[j]), thresh);
 			}
-			context->mask_len++;
 		}
 	}
 
@@ -126,14 +124,9 @@ static void parse_params(chromakey_context_t *context, int start, int argc, char
 
 	if (n > 1 && argv[i]) { // thresh
 		int thresh = atoi(argv[i]);
-		int j;
 
 		if (thresh > 0) {
-			context->threshold = thresh * thresh;
-
-			for (j = 0; j < context->mask_len; j++) {
-				if (!context->thresholds[j]) context->thresholds[j] = context->threshold;
-			}
+			switch_chromakey_set_default_threshold(context->ck, thresh);
 		}
 	}
 
@@ -237,7 +230,12 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi
 	context->mod = 0;
 
 	if (switch_mutex_trylock(context->command_mutex) != SWITCH_STATUS_SUCCESS) {
-		switch_img_patch(frame->img, context->last_img, 0, 0);
+		switch_image_t *last_img = switch_chromakey_cache_image(context->ck);
+
+		if (last_img) {
+			switch_img_patch(frame->img, last_img, 0, 0);
+		}
+
 		return SWITCH_STATUS_SUCCESS;
 	}
 
@@ -256,10 +254,7 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi
 	img = switch_img_wrap(NULL, SWITCH_IMG_FMT_ARGB, frame->img->d_w, frame->img->d_h, 1, context->data);
 
 	switch_assert(img);
-	switch_img_chromakey_multi(img, context->last_img, context->mask, context->thresholds, context->mask_len);
-
-	switch_img_free(&context->last_img);
-	switch_img_copy(img, &context->last_img);
+	switch_chromakey_process(context->ck, img);
 
 	if (context->bgimg) {
 		switch_image_t *tmp = NULL;

src/switch_core_video.c

@@ -87,7 +87,7 @@ static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color
 * \param[in]    count     number of colors in list
 * \param[in]    threshold hint of target threshold to stop processing list
 */
-static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, int *thresholds);
+static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, uint32_t *thresholds);
 
 /*!\brief Draw a pixel on an image
 *
@@ -334,7 +334,7 @@ SWITCH_DECLARE(void) switch_img_patch_rgb(switch_image_t *IMG, switch_image_t *i
 						tmp_a = ((RGB->a * (255 - alpha)) >> 8) + ((rgb->a * alpha) >> 8);
 						RGB->a = RGB->a > tmp_a ? RGB->a : tmp_a;
 					}
-
+					
 					RGB->r = ((RGB->r * (255 - alpha)) >> 8) + ((rgb->r * alpha) >> 8);
 					RGB->g = ((RGB->g * (255 - alpha)) >> 8) + ((rgb->g * alpha) >> 8);
 					RGB->b = ((RGB->b * (255 - alpha)) >> 8) + ((rgb->b * alpha) >> 8);
@@ -827,7 +827,7 @@ static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color
 
 }
 
-static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, int *thresholds)
+static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, uint32_t *thresholds)
 {
 	int x = 0, hits = 0;
 
@@ -844,38 +844,212 @@ static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb
 }
 
 
+struct switch_chromakey_s {
+	switch_image_t *cache_img;
+	switch_rgb_color_t mask[CHROMAKEY_MAX_MASK];
+	uint32_t thresholds[CHROMAKEY_MAX_MASK];
+	int mask_len;
+	switch_shade_t autocolor;
+	uint32_t dft_thresh;
+	uint32_t dft_thresh_squared;
+
+	uint32_t rr;
+	uint32_t gg;
+	uint32_t bb;
+	uint32_t color_count;
+	
+	switch_rgb_color_t auto_color;
+	int no_cache;
+};
+
+SWITCH_DECLARE(switch_shade_t) switch_chromakey_str2shade(switch_chromakey_t *ck, const char *shade_name)
+{
+	switch_shade_t shade = SWITCH_SHADE_NONE;
+
+	if (!strcasecmp(shade_name, "red")) {
+		shade = SWITCH_SHADE_RED;
+	} else if (!strcasecmp(shade_name, "green")) {
+		shade = SWITCH_SHADE_GREEN;
+	} else if (!strcasecmp(shade_name, "blue")) {
+		shade = SWITCH_SHADE_BLUE;
+	} else if (!strcasecmp(shade_name, "auto")) {
+		shade = SWITCH_SHADE_AUTO;
+	}
+
+	return shade;
+}
+
+SWITCH_DECLARE(void) switch_chromakey_set_default_threshold(switch_chromakey_t *ck, uint32_t threshold)
+{
+	int i;
+
+	ck->dft_thresh = threshold;
+	ck->dft_thresh_squared = threshold * threshold;
+
+	for (i = 0; i < ck->mask_len; i++) {
+		if (!ck->thresholds[i]) ck->thresholds[i] = ck->dft_thresh_squared;
+	}
+}
+
+SWITCH_DECLARE(switch_status_t) switch_chromakey_clear_colors(switch_chromakey_t *ck)
+{
+	switch_assert(ck);
+
+	ck->autocolor = SWITCH_SHADE_NONE;
+	ck->mask_len = 0;
+	memset(ck->mask, 0, sizeof(ck->mask[0]) * CHROMAKEY_MAX_MASK);
+	memset(ck->thresholds, 0, sizeof(ck->thresholds[0]) * CHROMAKEY_MAX_MASK);
+	ck->no_cache = 1;
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+SWITCH_DECLARE(switch_status_t) switch_chromakey_autocolor(switch_chromakey_t *ck, switch_shade_t autocolor, uint32_t threshold)
+{
+	switch_assert(ck);
+
+	switch_chromakey_clear_colors(ck);
+	ck->autocolor = autocolor;
+	ck->dft_thresh = threshold;
+	ck->dft_thresh_squared = threshold * threshold;
+	switch_img_free(&ck->cache_img);
+	ck->no_cache = 90;
+	memset(&ck->auto_color, 0, sizeof(ck->auto_color));
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+SWITCH_DECLARE(switch_status_t) switch_chromakey_add_color(switch_chromakey_t *ck, switch_rgb_color_t *color, uint32_t threshold) 
+{
+	switch_assert(ck);
+
+	if (ck->mask_len == CHROMAKEY_MAX_MASK) {
+		return SWITCH_STATUS_FALSE;
+	}
+
+	ck->mask[ck->mask_len] = *color;
+	ck->thresholds[ck->mask_len] = threshold * threshold;
+	ck->mask_len++;
+
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Adding color %d:%d:%d #%.2x%.2x%.2x\n",
+					  ck->auto_color.r, ck->auto_color.g, ck->auto_color.b, ck->auto_color.r, ck->auto_color.g, ck->auto_color.b);
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+SWITCH_DECLARE(switch_status_t) switch_chromakey_destroy(switch_chromakey_t **ckP)
+{
+	switch_chromakey_t *ck;
+
+	switch_assert(ckP);
+
+	ck = *ckP;
+	*ckP = NULL;
+
+	if (ck) {
+		switch_img_free(&ck->cache_img);
+		free(ck);
+	}
+
+	return SWITCH_STATUS_SUCCESS;
+}
+
+SWITCH_DECLARE(switch_status_t) switch_chromakey_create(switch_chromakey_t **ckP)
+{
+	switch_chromakey_t *ck;
+
+	switch_assert(ckP);
+
+	switch_zmalloc(ck, sizeof(*ck));
+
+	*ckP = ck;
+
+	return SWITCH_STATUS_SUCCESS;
+
+}
+
+SWITCH_DECLARE(switch_image_t *) switch_chromakey_cache_image(switch_chromakey_t *ck)
+{
+	switch_assert(ck);
+
+	return ck->cache_img;
+}
+
+static inline int get_max(switch_rgb_color_t *c1)
+{
+	if (c1->r > c1->g && c1->r > c1->b) {
+		return 1;
+	}
+
+	if (c1->g > c1->r && c1->g > c1->b) {
+		return 2;
+	}
+
+	if (c1->b > c1->r && c1->b > c1->g) {
+		return 3;
+	}
+
+	return 0;
+}
+
+static inline int switch_color_dom_cmp(switch_rgb_color_t *c1, switch_rgb_color_t *c2)
+{
+	
+	int c1_max = get_max(c1);
+	int c2_max = get_max(c2);
+	
+	if (c1_max && c1_max == c2_max) return 1;
+
+	return 0;
+
+}
 
+static inline int switch_color_distance_literal(switch_rgb_color_t *c1, switch_rgb_color_t *c2, int distance)
+{
+	int r = abs(c1->r - c2->r);
+	int g = abs(c1->g - c2->g);
+	int b = abs(c1->b - c2->b);
 
+	if (r < distance && g < distance && b < distance) return 1;
+
+	return 0;
+}
 
 static inline int switch_color_distance_cheap(switch_rgb_color_t *c1, switch_rgb_color_t *c2)
 {
-	int r = c1->r - c2->r;
-	int g = c1->g - c2->g;
-	int b = c1->b - c2->b;
+	int r = abs(c1->r - c2->r);
+	int g = abs(c1->g - c2->g);
+	int b = abs(c1->b - c2->b);
 
-	if (!r && !g && !b) return 0;
+	if (r < 5 && g < 5 && b < 5) return 0;
 
-	return (3*abs(r)) + (4*abs(g)) + (3*abs(b));
+	return (3*r) + (4*g) + (3*b);
 }
 
-SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_image_t *cache_img, switch_rgb_color_t *mask, int *thresholds, int count)
+SWITCH_DECLARE(void) switch_chromakey_process(switch_chromakey_t *ck, switch_image_t *img)
 {
 	uint8_t *pixel, *last_pixel = NULL, *cache_pixel = NULL, *end_pixel = NULL;
 	int last_hits = 0;
+	switch_image_t *cache_img;
+	int same = 0;
+	int same_same = 0;
 
 #ifdef DEBUG_CHROMA
 	int other_img_cached = 0, color_cached = 0, checked = 0, hit_total = 0, total_pixel = 0, delta_hits = 0;
 #endif
 
-
+	switch_assert(ck);
 	switch_assert(img);
 
 	if (img->fmt != SWITCH_IMG_FMT_ARGB) return;
 
 	pixel = img->planes[SWITCH_PLANE_PACKED];
 
+	cache_img = ck->cache_img;
+	ck->cache_img = NULL;
+
 	if (cache_img && (cache_img->d_w != img->d_w || cache_img->d_h != img->d_h)) {
-		cache_img = NULL;
+		switch_img_free(&cache_img);
 	}
 
 	if (cache_img) {
@@ -884,16 +1058,23 @@ SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_imag
 
 	end_pixel = (img->planes[SWITCH_PLANE_PACKED] + img->d_w * img->d_h * 4);
 
+	if (ck->autocolor) {
+		ck->color_count = 0;
+		ck->rr = ck->gg = ck->bb = 0;
+	}
+
 	for (; pixel < end_pixel; pixel += 4) {
 		switch_rgb_color_t *color = (switch_rgb_color_t *)pixel;
 		switch_rgb_color_t *last_color = (switch_rgb_color_t *)last_pixel;
 		int hits = 0;
 
+
+
 #ifdef DEBUG_CHROMA
 		total_pixel++;
 #endif
-
-		if (cache_img && cache_pixel) {
+		
+		if (!ck->no_cache && cache_img && cache_pixel) {
 			switch_rgb_color_t *cache_color = (switch_rgb_color_t *)cache_pixel;
 				
 			if (switch_color_distance_cheap(color, cache_color) < 5) {
@@ -902,28 +1083,92 @@ SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_imag
 #endif
 				*pixel = *cache_pixel;
 				goto end;
-			}				
+			}
 		}
 
 
-		if (last_color && switch_color_distance_cheap(color, last_color) < 5) {
+		if (last_color) {
+			if (switch_color_distance_cheap(color, last_color) < 5) {
 
-			hits = last_hits;
+				hits = last_hits;
 #ifdef DEBUG_CHROMA
-			color_cached++;
+				color_cached++;
 #endif
+
+				same++;
+			} else {
+				same = 0;
+			}
 		} 
 
 		if (!hits) {
-			hits = switch_color_distance_multi(color, mask, count, thresholds);
+
+			if (ck->autocolor) {
+				int dom, a, b;
+
+				switch(ck->autocolor) {
+				case SWITCH_SHADE_RED:
+					dom = color->r;
+					a = color->g;
+					b = color->b;
+					break;
+				case SWITCH_SHADE_GREEN:
+					dom = color->g;
+					a = color->r;
+					b = color->b;
+					break;
+				case SWITCH_SHADE_BLUE:
+					dom = color->b;
+					a = color->r;
+					b = color->g;
+					break;
+				default:
+					dom = 0;
+					a = 0;
+					b = 0;
+					break;
+				}
+
+				if (ck->autocolor != SWITCH_SHADE_AUTO) {
+					//printf("WTF %d\n", ck->dft_thresh);
+
+					int tol = ck->dft_thresh;
+					int a_tol = tol/6;
+					int b_tol = tol/6;
+
+					if (dom > a && dom > b && dom > tol) {
+						if (dom - a > a_tol && dom - b > b_tol) {
+							hits = 1;
+						} 
+					}
+				}
+			}
+
+			if (!hits && ck->mask_len) {
+				hits = switch_color_distance_multi(color, ck->mask, ck->mask_len, ck->thresholds);
+			}
+
 
 #ifdef DEBUG_CHROMA
 			checked++;
 #endif
 		}
-
+		
 	end:
 
+		if (same > 100 && last_color && switch_color_dom_cmp(color, last_color)) {
+			same_same++;
+		} else {
+			same_same = 0;
+		}
+
+		if (!hits && ck->autocolor == SWITCH_SHADE_AUTO && (same > 300 || same_same > 50) && ck->no_cache) {
+			ck->color_count++;
+			ck->rr += color->r;
+			ck->gg += color->g;
+			ck->bb += color->b;
+		}
+
 		if (cache_pixel) {
 			cache_pixel += 4;
 		}
@@ -938,10 +1183,48 @@ SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_imag
 		last_pixel = pixel;
 		last_hits = hits;
 	}
+
+	if (ck->color_count > 1000) {
+		switch_rgb_color_t *last_color = NULL;
+		int skip = 0;
+
+		ck->auto_color.r = ck->rr / ck->color_count;
+		ck->auto_color.g = ck->gg / ck->color_count;
+		ck->auto_color.b = ck->bb / ck->color_count;
+
+		if (ck->mask_len) {
+			int i = 0;
+
+			for (i = 0; i < ck->mask_len; i++) {
+				last_color = &ck->mask[i];
+				if (switch_color_distance_literal(&ck->auto_color, last_color, 10) || !switch_color_dom_cmp(&ck->auto_color, last_color)) {
+					skip = 1;
+					break;
+				}
+			}
+		}
+
+		if (!ck->mask_len || !skip) {
+			switch_chromakey_add_color(ck, &ck->auto_color, ck->dft_thresh);
+		}
+
+	}
+
+	if (ck->no_cache > 0 && ck->mask_len) {
+		ck->no_cache--;
+	}
+
+
 #ifdef DEBUG_CHROMA
 	printf("total %d: other img cache %d color cache %d Checked %d Hit Total %d Delta hits: %d\n", total_pixel, other_img_cached, color_cached, checked, hit_total, delta_hits);
 #endif
 
+	if (!ck->no_cache) {
+		switch_img_copy(img, &ck->cache_img);
+	}
+
+	switch_img_free(&cache_img);
+
 	return;
 }