openstack/deb-spice-html5
Code Issues Proposed changes

Add support for the vp8 codec type.

Browse Source
This commit is contained in:
Jeremy White 2016-06-28 08:00:51 -06:00
parent 5b2e795bf6
commit 3778053138
5 changed files with 337 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

230
display.js
View File

@ -542,8 +542,40 @@ SpiceDisplayConn.prototype.process_channel_message = function(msg)
console.log("Stream " + m.id + " already exists");
else
this.streams[m.id] = m;
if (m.codec_type != SPICE_VIDEO_CODEC_TYPE_MJPEG)
console.log("Unhandled stream codec: " + m.codec_type);
if (m.codec_type == SPICE_VIDEO_CODEC_TYPE_VP8)
{
var media = new MediaSource();
var v = document.createElement("video");
v.src = window.URL.createObjectURL(media);
v.setAttribute('autoplay', true);
v.setAttribute('width', m.stream_width);
v.setAttribute('height', m.stream_height);
var left = m.dest.left;
var top = m.dest.top;
if (this.surfaces[m.surface_id] !== undefined)
{
left += this.surfaces[m.surface_id].canvas.offsetLeft;
top += this.surfaces[m.surface_id].canvas.offsetTop;
}
document.getElementById(this.parent.screen_id).appendChild(v);
v.setAttribute('style', "position: absolute; top:" + top + "px; left:" + left + "px;");
media.addEventListener('sourceopen', handle_video_source_open, false);
media.addEventListener('sourceended', handle_video_source_ended, false);
media.addEventListener('sourceclosed', handle_video_source_closed, false);
this.streams[m.id].video = v;
this.streams[m.id].media = media;
media.stream = this.streams[m.id];
media.spiceconn = this;
v.spice_stream = this.streams[m.id];
}
else if (m.codec_type != SPICE_VIDEO_CODEC_TYPE_MJPEG)
console.log("Unhandled stream codec: "+m.codec_type);
return true;
}
@ -568,6 +600,9 @@ SpiceDisplayConn.prototype.process_channel_message = function(msg)
if (this.streams[m.base.id].codec_type === SPICE_VIDEO_CODEC_TYPE_MJPEG)
process_mjpeg_stream_data(this, m, latency);
if (this.streams[m.base.id].codec_type === SPICE_VIDEO_CODEC_TYPE_VP8)
process_video_stream_data(this.streams[m.base.id], m);
if ("report" in this.streams[m.base.id])
process_stream_data_report(this, m, mmtime, latency);
@ -601,6 +636,14 @@ SpiceDisplayConn.prototype.process_channel_message = function(msg)
{
var m = new SpiceMsgDisplayStreamDestroy(msg.data);
DEBUG > 1 && console.log(this.type + ": MsgStreamDestroy id" + m.id);
if (this.streams[m.id].codec_type == SPICE_VIDEO_CODEC_TYPE_VP8)
{
document.getElementById(this.parent.screen_id).removeChild(this.streams[m.id].video);
this.streams[m.id].source_buffer = null;
this.streams[m.id].media = null;
this.streams[m.id].video = null;
}
this.streams[m.id] = undefined;
return true;
}
@ -971,3 +1014,186 @@ function process_stream_data_report(sc, m, mmtime, latency)
sc.streams[m.base.id].report.num_drops = 0;
}
}
function handle_video_source_open(e)
{
var stream = this.stream;
var p = this.spiceconn;
if (stream.source_buffer)
return;
var s = this.addSourceBuffer(SPICE_VP8_CODEC);
if (! s)
{
p.log_err('Codec ' + SPICE_VP8_CODEC + ' not available.');
return;
}
stream.source_buffer = s;
s.spiceconn = p;
s.stream = stream;
stream.queue = new Array();
stream.start_time = 0;
stream.cluster_time = 0;
listen_for_video_events(stream);
var h = new webm_Header();
var te = new webm_VideoTrackEntry(this.stream.stream_width, this.stream.stream_height);
var t = new webm_Tracks(te);
var mb = new ArrayBuffer(h.buffer_size() + t.buffer_size())
var b = h.to_buffer(mb);
t.to_buffer(mb, b);
s.addEventListener('error', handle_video_buffer_error, false);
s.addEventListener('updateend', handle_append_video_buffer_done, false);
append_video_buffer(s, mb);
}
function handle_video_source_ended(e)
{
var p = this.spiceconn;
p.log_err('Video source unexpectedly ended.');
}
function handle_video_source_closed(e)
{
var p = this.spiceconn;
p.log_err('Video source unexpectedly closed.');
}
function append_video_buffer(sb, mb)
{
try
{
sb.stream.append_okay = false;
sb.appendBuffer(mb);
}
catch (e)
{
var p = sb.spiceconn;
p.log_err("Error invoking appendBuffer: " + e.message);
}
}
function handle_append_video_buffer_done(e)
{
var stream = this.stream;
if (stream.queue.length > 0)
{
var mb = stream.queue.shift();
append_video_buffer(stream.source_buffer, mb);
}
else
{
stream.append_okay = true;
}
}
function handle_video_buffer_error(e)
{
var p = this.spiceconn;
p.log_err('source_buffer error ' + e.message);
}
function video_simple_block(stream, msg, keyframe)
{
var simple = new webm_SimpleBlock(msg.base.multi_media_time - stream.cluster_time, msg.data, keyframe);
var mb = new ArrayBuffer(simple.buffer_size());
simple.to_buffer(mb);
if (stream.append_okay)
append_video_buffer(stream.source_buffer, mb);
else
stream.queue.push(mb);
}
function new_video_cluster(stream, msg)
{
stream.cluster_time = msg.base.multi_media_time;
var c = new webm_Cluster(stream.cluster_time - stream.start_time, msg.data);
var mb = new ArrayBuffer(c.buffer_size());
c.to_buffer(mb);
if (stream.append_okay)
append_video_buffer(stream.source_buffer, mb);
else
stream.queue.push(mb);
video_simple_block(stream, msg, true);
}
function process_video_stream_data(stream, msg)
{
if (! stream.source_buffer)
return true;
if (stream.start_time == 0)
{
stream.start_time = msg.base.multi_media_time;
stream.video.play();
new_video_cluster(stream, msg);
}
else if (msg.base.multi_media_time - stream.cluster_time >= MAX_CLUSTER_TIME)
new_video_cluster(stream, msg);
else
video_simple_block(stream, msg, false);
}
function video_handle_event_debug(e)
{
var s = this.spice_stream;
if (s.video)
{
if (STREAM_DEBUG > 0 || s.video.buffered.len > 1)
console.log(s.video.currentTime + ":id " + s.id + " event " + e.type +
dump_media_element(s.video));
}
if (STREAM_DEBUG > 1 && s.media)
console.log(" media_source " + dump_media_source(s.media));
if (STREAM_DEBUG > 1 && s.source_buffer)
console.log(" source_buffer " + dump_source_buffer(s.source_buffer));
if (STREAM_DEBUG > 0 || s.queue.length > 1)
console.log(' queue len ' + s.queue.length + '; append_okay: ' + s.append_okay);
}
function video_debug_listen_for_one_event(name)
{
this.addEventListener(name, video_handle_event_debug);
}
function listen_for_video_events(stream)
{
var video_0_events = [
"abort", "error"
];
var video_1_events = [
"loadstart", "suspend", "emptied", "stalled", "loadedmetadata", "loadeddata", "canplay",
"canplaythrough", "playing", "waiting", "seeking", "seeked", "ended", "durationchange",
"timeupdate", "play", "pause", "ratechange"
];
var video_2_events = [
"progress",
"resize",
"volumechange"
];
video_0_events.forEach(video_debug_listen_for_one_event, stream.video);
if (STREAM_DEBUG > 0)
video_1_events.forEach(video_debug_listen_for_one_event, stream.video);
if (STREAM_DEBUG > 1)
video_2_events.forEach(video_debug_listen_for_one_event, stream.video);
}

6
enums.js
View File

@ -182,6 +182,11 @@ var SPICE_DISPLAY_CAP_COMPOSITE = 2;
var SPICE_DISPLAY_CAP_A8_SURFACE = 3;
var SPICE_DISPLAY_CAP_STREAM_REPORT = 4;
var SPICE_DISPLAY_CAP_LZ4_COMPRESSION = 5;
var SPICE_DISPLAY_CAP_PREF_COMPRESSION = 6;
var SPICE_DISPLAY_CAP_GL_SCANOUT = 7;
var SPICE_DISPLAY_CAP_MULTI_CODEC = 8;
var SPICE_DISPLAY_CAP_CODEC_MJPEG = 9;
var SPICE_DISPLAY_CAP_CODEC_VP8 = 10;
var SPICE_AUDIO_DATA_MODE_INVALID = 0;
var SPICE_AUDIO_DATA_MODE_RAW = 1;
@ -324,6 +329,7 @@ var SPICE_CURSOR_TYPE_ALPHA = 0,
SPICE_CURSOR_TYPE_COLOR32 = 6;
var SPICE_VIDEO_CODEC_TYPE_MJPEG = 1;
var SPICE_VIDEO_CODEC_TYPE_VP8 = 2;
var VD_AGENT_PROTOCOL = 1;
var VD_AGENT_MAX_DATA_SIZE = 2048;

5
spiceconn.js
View File

@ -136,7 +136,10 @@ SpiceConn.prototype =
else if (msg.channel_type == SPICE_CHANNEL_DISPLAY)
msg.channel_caps.push(
(1 << SPICE_DISPLAY_CAP_SIZED_STREAM) |
(1 << SPICE_DISPLAY_CAP_STREAM_REPORT)
(1 << SPICE_DISPLAY_CAP_STREAM_REPORT) |
(1 << SPICE_DISPLAY_CAP_MULTI_CODEC) |
(1 << SPICE_DISPLAY_CAP_CODEC_MJPEG) |
(1 << SPICE_DISPLAY_CAP_CODEC_VP8)
);
hdr.size = msg.buffer_size();

1
utils.js
View File

@ -23,6 +23,7 @@
**--------------------------------------------------------------------------*/
var DEBUG = 0;
var PLAYBACK_DEBUG = 0;
var STREAM_DEBUG = 0;
var DUMP_DRAWS = false;
var DUMP_CANVASES = false;

98
webm.js
View File

@ -61,6 +61,10 @@ var WEBM_CODEC_DELAY = [ 0x56, 0xAA ];
var WEBM_CODEC_PRIVATE = [ 0x63, 0xA2 ];
var WEBM_CODEC_ID = [ 0x86 ];
var WEBM_VIDEO = [ 0xE0 ] ;
var WEBM_PIXEL_WIDTH = [ 0xB0 ] ;
var WEBM_PIXEL_HEIGHT = [ 0xBA ] ;
var WEBM_AUDIO = [ 0xE1 ] ;
var WEBM_SAMPLING_FREQUENCY = [ 0xB5 ] ;
var WEBM_CHANNELS = [ 0x9F ] ;
@ -82,6 +86,8 @@ var MAX_CLUSTER_TIME = 1000;
var GAP_DETECTION_THRESHOLD = 50;
var SPICE_VP8_CODEC = 'video/webm; codecs="vp8"';
/*----------------------------------------------------------------------------
** EBML utility functions
** These classes can create the binary representation of a webm file
@ -291,6 +297,34 @@ webm_Audio.prototype =
},
}
function webm_Video(width, height)
{
this.id = WEBM_VIDEO;
this.width = width;
this.height = height;
}
webm_Video.prototype =
{
to_buffer: function(a, at)
{
at = at || 0;
var dv = new DataView(a);
at = EBML_write_array(this.id, dv, at);
at = EBML_write_u64_data_len(this.buffer_size() - 8 - this.id.length, dv, at);
at = EBML_write_u16_value(WEBM_PIXEL_WIDTH, this.width, dv, at)
at = EBML_write_u16_value(WEBM_PIXEL_HEIGHT, this.height, dv, at)
return at;
},
buffer_size: function()
{
return this.id.length + 8 +
WEBM_PIXEL_WIDTH.length + 1 + 2 +
WEBM_PIXEL_HEIGHT.length + 1 + 2;
},
}
/* ---------------------------
SeekHead not currently used. Hopefully not needed.
@ -431,6 +465,70 @@ webm_AudioTrackEntry.prototype =
this.audio.buffer_size();
},
}
function webm_VideoTrackEntry(width, height)
{
this.id = WEBM_TRACK_ENTRY;
this.number = 1;
this.uid = 1;
this.type = 1; // Video
this.flag_enabled = 1;
this.flag_default = 1;
this.flag_forced = 1;
this.flag_lacing = 0;
this.min_cache = 0; // fixme - check
this.max_block_addition_id = 0;
this.codec_decode_all = 0; // fixme - check
this.seek_pre_roll = 0; // 80000000; // fixme - check
this.codec_delay = 80000000; // Must match codec_private.preskip
this.codec_id = "V_VP8";
this.video = new webm_Video(width, height);
}
webm_VideoTrackEntry.prototype =
{
to_buffer: function(a, at)
{
at = at || 0;
var dv = new DataView(a);
at = EBML_write_array(this.id, dv, at);
at = EBML_write_u64_data_len(this.buffer_size() - 8 - this.id.length, dv, at);
at = EBML_write_u8_value(WEBM_TRACK_NUMBER, this.number, dv, at);
at = EBML_write_u8_value(WEBM_TRACK_UID, this.uid, dv, at);
at = EBML_write_u8_value(WEBM_FLAG_ENABLED, this.flag_enabled, dv, at);
at = EBML_write_u8_value(WEBM_FLAG_DEFAULT, this.flag_default, dv, at);
at = EBML_write_u8_value(WEBM_FLAG_FORCED, this.flag_forced, dv, at);
at = EBML_write_u8_value(WEBM_FLAG_LACING, this.flag_lacing, dv, at);
at = EBML_write_data(WEBM_CODEC_ID, this.codec_id, dv, at);
at = EBML_write_u8_value(WEBM_MIN_CACHE, this.min_cache, dv, at);
at = EBML_write_u8_value(WEBM_MAX_BLOCK_ADDITION_ID, this.max_block_addition_id, dv, at);
at = EBML_write_u8_value(WEBM_CODEC_DECODE_ALL, this.codec_decode_all, dv, at);
at = EBML_write_u32_value(WEBM_CODEC_DELAY, this.codec_delay, dv, at);
at = EBML_write_u32_value(WEBM_SEEK_PRE_ROLL, this.seek_pre_roll, dv, at);
at = EBML_write_u8_value(WEBM_TRACK_TYPE, this.type, dv, at);
at = this.video.to_buffer(a, at);
return at;
},
buffer_size: function()
{
return this.id.length + 8 +
WEBM_TRACK_NUMBER.length + 1 + 1 +
WEBM_TRACK_UID.length + 1 + 1 +
WEBM_FLAG_ENABLED.length + 1 + 1 +
WEBM_FLAG_DEFAULT.length + 1 + 1 +
WEBM_FLAG_FORCED.length + 1 + 1 +
WEBM_FLAG_LACING.length + 1 + 1 +
WEBM_CODEC_ID.length + this.codec_id.length + 1 +
WEBM_MIN_CACHE.length + 1 + 1 +
WEBM_MAX_BLOCK_ADDITION_ID.length + 1 + 1 +
WEBM_CODEC_DECODE_ALL.length + 1 + 1 +
WEBM_CODEC_DELAY.length + 1 + 4 +
WEBM_SEEK_PRE_ROLL.length + 1 + 4 +
WEBM_TRACK_TYPE.length + 1 + 1 +
this.video.buffer_size();
},
}
function webm_Tracks(entry)
{
this.id = WEBM_TRACKS;