FS-8688 #improve vp9 processing to avoid chrome hang

src/switch_vpx.c

@@ -40,12 +40,15 @@
 #include <vpx/vp8dx.h>
 #include <vpx/vp8.h>
 
+// #define DEBUG_VP9
+
 #define SLICE_SIZE SWITCH_DEFAULT_VIDEO_SIZE
 #define KEY_FRAME_MIN_FREQ 250000
 
 /*	http://tools.ietf.org/html/draft-ietf-payload-vp8-10
 
 	The first octets after the RTP header are the VP8 payload descriptor, with the following structure.
+#endif
 
 	     0 1 2 3 4 5 6 7
 	    +-+-+-+-+-+-+-+-+
@@ -112,7 +115,7 @@ typedef struct {
 	unsigned n_s:3;
 	unsigned y:1;
 	unsigned g:1;
-	unsigned zero:0;
+	unsigned zero:3;
 } vp9_ss_t;
 
 typedef struct {
@@ -122,6 +125,13 @@ typedef struct {
 	unsigned zero:2;
 } vp9_n_g_t;
 
+typedef struct {
+	unsigned temporal_id:3;
+	unsigned temporal_up_switch:1;
+	unsigned spatial_id:3;
+	unsigned inter_layer_predicted:1;
+} vp9_p_layer_t;
+
 #else /* ELSE LITTLE */
 
 typedef struct {
@@ -145,7 +155,7 @@ typedef struct {
 } vp9_payload_descriptor_t;
 
 typedef struct {
-	unsigned zero:4;
+	unsigned zero:3;
 	unsigned g:1;
 	unsigned y:1;
 	unsigned n_s:3;
@@ -159,10 +169,11 @@ typedef struct {
 } vp9_n_g_t;
 
 typedef struct {
-	unsigned d:1;
-	unsigned s:3;
-	unsigned gof_idx:4;
-} vp9_layer_t;
+	unsigned inter_layer_predicted:1;
+	unsigned spatial_id:3;
+	unsigned temporal_up_switch:1;
+	unsigned temporal_id:3;
+} vp9_p_layer_t;
 
 #endif
 
@@ -239,8 +250,8 @@ static inline int IS_VP8_KEY_FRAME(uint8_t *data)
 	}
 }
 
-#define IS_VP9_KEY_FRAME(byte) ((((byte) & 0x10) == 0) && ((byte) & 0x02))
-#define IS_VP9_START_PKT(byte) ((byte) & 0x02)
+#define IS_VP9_KEY_FRAME(byte) ((((byte) & 0x40) == 0) && ((byte) & 0x0A))
+#define IS_VP9_START_PKT(byte) ((byte) & 0x08)
 
 SWITCH_MODULE_LOAD_FUNCTION(mod_vpx_load);
 SWITCH_MODULE_DEFINITION(CORE_VPX_MODULE, mod_vpx_load, NULL, NULL);
@@ -663,28 +674,38 @@ static switch_status_t consume_partition(vpx_context_t *context, switch_frame_t
 
 		if (1) {
 			// payload_descriptor->vp9.have_p_layer = key; // key?
-			payload_descriptor->vp9.have_pid = 0;
+			payload_descriptor->vp9.have_pid = 1;
+
+			if (payload_descriptor->vp9.have_pid) {
+
+				if (context->vp9.picture_id < 0) context->vp9.picture_id = 0;
+
+				if (context->vp9.picture_id > 0x7f) {
+					*body++ = (context->vp9.picture_id >> 8) | 0x80;
+					*body++ = context->vp9.picture_id & 0xff;
+					payload_size--;
+					frame->datalen++;
+				} else {
+					*body++ = context->vp9.picture_id;
+				}
 
-			if (start) {
-				// context->vp9.picture_id++;
-			}
 
-			// if (context->vp9.picture_id > 0x7f) { // todo rewind to 0
-			// 	*body++ = context->vp9.picture_id >> 8;
-			// 	*body++ = context->vp9.picture_id & 0xff;
-			// 	payload_size--;
-			// 	frame->datalen++;
-			// } else {
-			// 	*body++ = context->vp9.picture_id;
-			// }
+				payload_size--;
+				frame->datalen++;
+			}
 
-			// payload_size--;
-			// frame->datalen++;
+			if (start) {
+				context->vp9.picture_id++;
+#ifdef DEBUG_VP9
+				// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "sending pid: %d\n", context->vp9.picture_id);
+#endif
+			}
 
 			if (key) {
 				vp9_ss_t *ss = (vp9_ss_t *)body;
 
 				payload_descriptor->vp9.have_ss = 1;
+				payload_descriptor->vp9.have_p_layer = 0;
 				ss->n_s = 0;
 				ss->g = 0;
 				ss->y = 0;
@@ -710,6 +731,8 @@ static switch_status_t consume_partition(vpx_context_t *context, switch_frame_t
 					payload_size-= (ss->n_s + 1) * 4;
 					frame->datalen+= (ss->n_s + 1) * 4;
 				}
+			} else {
+				payload_descriptor->vp9.have_p_layer = 1;
 			}
 		}
 
@@ -939,58 +962,99 @@ static switch_status_t buffer_vp9_packets(vpx_context_t *context, switch_frame_t
 	vp9_payload_descriptor_t *desc = (vp9_payload_descriptor_t *)vp9;
 	int len = 0;
 
-	// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%02x %02x %02x %02x m=%d start=%d end=%d len=%d\n", *data, *(data+1), *(data+2), *(data+3), frame->m, desc->start, desc->end, frame->datalen);
+#ifdef DEBUG_VP9
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%02x %02x %02x %02x m=%d len=%4d "
+					"have_pid=%d "
+					"have_p_layer=%d "
+					"have_layer_ind=%d "
+					"is_flexible=%d "
+					"start=%d "
+					"end=%d "
+					"have_ss=%d "
+					"zero=%d\n",
+					*data, *(data+1), *(data+2), *(data+3), frame->m, frame->datalen,
+					desc->have_pid,
+					desc->have_p_layer,
+					desc->have_layer_ind,
+					desc->is_flexible,
+					desc->start,
+					desc->end,
+					desc->have_ss,
+					desc->zero);
+#endif
 
 	vp9++;
 
-	if (desc->is_flexible) {
-		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "VP9 Flexiable mode is not supported yet\n");
-		switch_buffer_zero(context->vpx_packet_buffer);
-		goto end;
-	}
-
 	if (desc->have_pid) {
 		uint16_t pid = 0;
 
-		pid = *vp9 & 0x7f;
+		pid = *vp9 & 0x7f;	//0 bit is M , 1-7 bit is pid.
 
-		if (*vp9 & 0x80) {
+		if (*vp9 & 0x80) {	//if (M==1)
 			vp9++;
 			pid = (pid << 8) + *vp9;
 		}
 
 		vp9++;
-		// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "have pid: %d start=%d end=%d\n", pid, desc->start, desc->end);
+
+#ifdef DEBUG_VP9
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "have pid: %d start=%d end=%d\n", pid, desc->start, desc->end);
+#endif
+
 	}
 
 	if (desc->have_layer_ind) {
-		vp9_layer_t *layer = (vp9_layer_t *)vp9;
+#ifdef DEBUG_VP9
+		vp9_p_layer_t *layer = (vp9_p_layer_t *)vp9;
+
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "temporal_id=%d temporal_up_switch=%d spatial_id=%d inter_layer_predicted=%d\n",
+			layer->temporal_id, layer->temporal_up_switch, layer->spatial_id, layer->inter_layer_predicted);
+#endif
+
+		vp9++;
+		if (!desc->is_flexible) {
+			vp9++; // TL0PICIDX
+		}
+	}
 
-		vp9 += 2;
-		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "have layer idx: %d\n", layer->s);
+	//When P and F are both set to one, indicating a non-key frame in flexible mode
+	if (desc->have_p_layer && desc->is_flexible) { // P & F set, P_DIFF
+		if (*vp9 & 1) { // N
+			vp9++;
+			if (*vp9 & 1) { // N
+				vp9++;
+				if (*vp9 & 1) {
+					switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid VP9 packet!");
+					switch_buffer_zero(context->vpx_packet_buffer);
+					goto end;
+				}
+			}
+		}
+		vp9++;
 	}
 
 	if (desc->have_ss) {
 		vp9_ss_t *ss = (vp9_ss_t *)(vp9++);
 
-		context->got_key_frame = 1;
-		context->got_start_frame = 1;
-		// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "have ss: %02x n_s: %d y:%d g:%d\n", *(uint8_t *)ss, ss->n_s, ss->y, ss->g);
-
+#ifdef DEBUG_VP9
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "have ss: %02x n_s: %d y:%d g:%d\n", *(uint8_t *)ss, ss->n_s, ss->y, ss->g);
+#endif
 		if (ss->y) {
 			int i;
 
 			for (i=0; i<=ss->n_s; i++) {
+#ifdef DEBUG_VP9
 				int width = ntohs(*(uint16_t *)vp9);
 				int height = ntohs(*(uint16_t *)(vp9 + 2));
-				vp9 += 4;
 				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "SS: %d %dx%d\n", i, width, height);
+#endif
+				vp9 += 4;
 			}
 		}
 
 		if (ss->g) {
 			int i;
-			uint8_t ng = *vp9++;
+			uint8_t ng = *vp9++;	//N_G indicates the number of frames in a GOF
 
 			for (i = 0; ng > 0 && i < ng; i++) {
 				vp9_n_g_t *n_g = (vp9_n_g_t *)(vp9++);
@@ -1000,14 +1064,14 @@ static switch_status_t buffer_vp9_packets(vpx_context_t *context, switch_frame_t
 	}
 
 	if (vp9 - data >= frame->datalen) {
-		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid VP9 Packet\n");
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Invalid VP9 Packet %" SWITCH_SSIZE_T_FMT " > %d\n", vp9 - data, frame->datalen);
 		switch_buffer_zero(context->vpx_packet_buffer);
 		goto end;
 	}
 
 	if (switch_buffer_inuse(context->vpx_packet_buffer)) { // middle packet
 		if (desc->start) {
-			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got invalid vp9 packet, packet loss? resetting buffer\n");
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "got invalid vp9 packet, packet loss? resetting buffer\n");
 			switch_buffer_zero(context->vpx_packet_buffer);
 		}
 	} else { // start packet
@@ -1021,7 +1085,10 @@ static switch_status_t buffer_vp9_packets(vpx_context_t *context, switch_frame_t
 	switch_buffer_write(context->vpx_packet_buffer, vp9, len);
 
 end:
-	// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "buffered %d bytes, buffer size: %" SWITCH_SIZE_T_FMT "\n", len, switch_buffer_inuse(context->vpx_packet_buffer));
+
+#ifdef DEBUG_VP9
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "buffered %d bytes, buffer size: %" SWITCH_SIZE_T_FMT "\n", len, switch_buffer_inuse(context->vpx_packet_buffer));
+#endif
 
 	return SWITCH_STATUS_SUCCESS;
 }
@@ -1034,24 +1101,42 @@ static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t *
 	switch_status_t status = SWITCH_STATUS_SUCCESS;
 	int is_start = 0, is_keyframe = 0, get_refresh = 0;
 
+#if 0
+	vp9_payload_descriptor_t *desc = (vp9_payload_descriptor_t *)frame->data;
+	uint8_t *data = frame->data;
+	switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%02x %02x %02x %02x m=%d start=%d end=%d m=%d len=%d\n",
+		*data, *(data+1), *(data+2), *(data+3), frame->m, desc->start, desc->end, frame->m, frame->datalen);
+#endif
+
 	if (context->is_vp9) {
-		is_start = is_keyframe = IS_VP9_KEY_FRAME(*(unsigned char *)frame->data);
+		is_keyframe = IS_VP9_KEY_FRAME(*(unsigned char *)frame->data);
+		is_start = IS_VP9_START_PKT(*(unsigned char *)frame->data);
+
+#ifdef DEBUG_VP9
+		if (is_keyframe) {
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "================Got a key frame!!!!========================\n");
+		}
+#endif
 	} else { // vp8
 		is_start = (*(unsigned char *)frame->data & 0x10);
 		is_keyframe = IS_VP8_KEY_FRAME((uint8_t *)frame->data);
+
+#ifdef DEBUG_VP9
+		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "VP8\n");
+#endif
 	}
-	
-	if (context->got_key_frame <= 0) {
+
+    if (!is_keyframe && context->got_key_frame <= 0) {
 		context->no_key_frame++;
 		//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "no keyframe, %d\n", context->no_key_frame);
 		if (context->no_key_frame > 50) {
 			if ((is_keyframe = is_start)) {
-				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "no keyframe, treating start as key.\n");
+				switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "no keyframe, treating start as key. frames=%d\n", context->no_key_frame);
 			}
 		}
-	}
+    }
 
-	// if (is_keyframe) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got key %d\n", is_keyframe);
+	// if (is_keyframe) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got key %d\n", context->got_key_frame);
 
 	if (context->need_decoder_reset != 0) {
 		vpx_codec_destroy(&context->decoder);
@@ -1093,7 +1178,7 @@ static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t *
 		}
 
 		get_refresh = 1;
-		
+
 		if (!context->got_start_frame) {
 			switch_goto_status(SWITCH_STATUS_MORE_DATA, end);
 		}
@@ -1115,7 +1200,6 @@ static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t *
 		//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "WTF????? %d %ld\n", status, len);
 	//}
 
-
 	if (status == SWITCH_STATUS_SUCCESS && frame->m && len) {
 		uint8_t *data;
 		int corrupted = 0;
@@ -1139,9 +1223,15 @@ static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t *
 		
 		if (corrupted) {
 			frame->img = NULL;
+#ifdef DEBUG_VP9
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "corrupted!!\n");
+#endif
 		} else {
 			frame->img = (switch_image_t *) vpx_codec_get_frame(decoder, &context->dec_iter);
-			// switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%dx%d\n", frame->img->d_w, frame->img->d_h);
+
+#ifdef DEBUG_VP9
+			switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "decoded: %dx%d\n", frame->img->d_w, frame->img->d_h);
+#endif
 		}
 		
 		switch_buffer_zero(context->vpx_packet_buffer);