来自WebGL视频帧的YouTube纹理
来自WebGL视频帧的YouTube纹理
提问于 2022-01-07 21:11:57
我正在使用描述为这里 (代码，演示)的技术来使用视频帧作为WebGL纹理，以及来自这里的简单场景(仅以2D显示图像，而不是三维旋转立方体)。
目标是一个用于YouTube的(带有YouTube着色器，即视频效果)。
由于gl.clearColor(0.5,0.5,0.5,1)的缘故，帆布是灰色的。但是接下来的代码行，应该从视频中画出帧，没有明显的效果。哪一部分可能是错的？没有错误。
我试着在发布之前缩短代码，但显然即使是简单的WebGL场景也需要大量的样板代码。
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey.net/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube.com/*
// @icon https://www.google.com/s2/favicons?domain=youtube.com
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
var playing = false;
var timeupdate = false;
// Waiting for these 2 events ensures
// there is data in the video
video.addEventListener('playing', function() {
playing = true;
checkReady();
}, true);
video.addEventListener('timeupdate', function() {
timeupdate = true;
checkReady();
}, true);
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
attribute vec2 a_texCoord;
uniform vec2 u_resolution;
varying vec2 v_texCoord;
void main() {
// convert the rectangle from pixels to 0.0 to 1.0
vec2 zeroToOne = a_position / u_resolution;
// convert from 0->1 to 0->2
vec2 zeroToTwo = zeroToOne * 2.0;
// convert from 0->2 to -1->+1 (clipspace)
vec2 clipSpace = zeroToTwo - 1.0;
gl_Position = vec4(clipSpace * vec2(1, -1), 0, 1);
// pass the texCoord to the fragment shader
// The GPU will interpolate this value between points.
v_texCoord = a_texCoord;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
// our texture
uniform sampler2D u_image;
// the texCoords passed in from the vertex shader.
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord).bgra;
}
`;
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const vertexShader = loadShader(gl, gl.VERTEX_SHADER, vsSource);
const fragmentShader = loadShader(gl, gl.FRAGMENT_SHADER, fsSource);
// Create the shader program
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, vertexShader);
gl.attachShader(shaderProgram, fragmentShader);
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and compiles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
// Send the source to the shader object
gl.shaderSource(shader, source);
// Compile the shader program
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize a shader program; this is where all the lighting
// for the vertices and so forth is established.
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var texcoordLocation = gl.getAttribLocation(shaderProgram, "a_texCoord");
// Create a buffer to put three 2d clip space points in
var positionBuffer = gl.createBuffer();
// Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Set a rectangle the same size as the image.
setRectangle(gl, 0, 0, video.width, video.height);
// provide texture coordinates for the rectangle.
var texcoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]), gl.STATIC_DRAW);
// Create a texture.
var texture = initTexture(gl);
function drawScene() {
// lookup uniforms
var resolutionLocation = gl.getUniformLocation(shaderProgram, "u_resolution");
//webglUtils.resizeCanvasToDisplaySize(gl.canvas);
// Tell WebGL how to convert from clip space to pixels
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
// Clear the canvas
gl.clearColor(0.5,0.5,0.5,1);
gl.clear(gl.COLOR_BUFFER_BIT);
// Tell it to use our program (pair of shaders)
gl.useProgram(shaderProgram);
// Turn on the position attribute
gl.enableVertexAttribArray(positionLocation);
// Bind the position buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
// Tell the position attribute how to get data out of positionBuffer (ARRAY_BUFFER)
var size = 2; // 2 components per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
positionLocation, size, type, normalize, stride, offset);
// Turn on the texcoord attribute
gl.enableVertexAttribArray(texcoordLocation);
// bind the texcoord buffer.
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
// Tell the texcoord attribute how to get data out of texcoordBuffer (ARRAY_BUFFER)
var size = 2; // 2 components per iteration
var type = gl.FLOAT; // the data is 32bit floats
var normalize = false; // don't normalize the data
var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
var offset = 0; // start at the beginning of the buffer
gl.vertexAttribPointer(
texcoordLocation, size, type, normalize, stride, offset);
// set the resolution
gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
// Draw the rectangle.
var primitiveType = gl.TRIANGLES;
var offset = 0;
var count = 6;
gl.drawArrays(primitiveType, offset, count);
}
function setRectangle(gl, x, y, width, height) {
var x1 = x;
var x2 = x + width;
var y1 = y;
var y2 = y + height;
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
x1, y1,
x2, y1,
x1, y2,
x1, y2,
x2, y1,
x2, y2,
]), gl.STATIC_DRAW);
}
var then = 0;
// Draw the scene repeatedly
function render(now) {
now *= 0.001; // convert to seconds
const deltaTime = now - then;
then = now;
if (copyVideo) {
updateTexture(gl, texture, video);
}
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
function initTexture(gl) {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
// Because video has to be download over the internet
// they might take a moment until it's ready so
// put a single pixel in the texture so we can
// use it immediately.
const level = 0;
const internalFormat = gl.RGBA;
const width = 1;
const height = 1;
const border = 0;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
width, height, border, srcFormat, srcType,
pixel);
// Turn off mips and set wrapping to clamp to edge so it
// will work regardless of the dimensions of the video.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
return texture;
}
function updateTexture(gl, texture, video) {
const level = 0;
const internalFormat = gl.RGBA;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, level, internalFormat,
srcFormat, srcType, video);
}
})();回答 2
Stack Overflow用户
回答已采纳
发布于 2022-01-08 13:41:52
第一个问题被Atekihcan正确地指出了，你把事情和NDC坐标计算搞混了，实际上直接发送它们要容易得多。此外，您可以很容易地从这些纹理坐标中派生出纹理坐标，从而保存第二个缓冲区的设置。
第二个问题是，您的事件没有在您期望的链中触发(至少对我来说不是重新加载播放视频并运行脚本)。我认为这应该是足够听timeupdate事件，因为时间不会更新，如果视频不能播放。工作守则：
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey.net/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube.com/*
// @icon https://www.google.com/s2/favicons?domain=youtube.com
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
video.addEventListener('timeupdate', function() {
copyVideo=true;
}, true);
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
varying vec2 v_texCoord;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
v_texCoord = a_position*.5+.5;
v_texCoord.y = 1.-v_texCoord.y;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
uniform sampler2D u_image;
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord);
}
`;
const positionData = new Float32Array([
-1.0,-1.0,
1.0,-1.0,
-1.0, 1.0,
1.0,-1.0,
1.0, 1.0,
-1.0, 1.0
]);
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, loadShader(gl, gl.VERTEX_SHADER, vsSource));
gl.attachShader(shaderProgram, loadShader(gl, gl.FRAGMENT_SHADER, fsSource));
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and compiles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize shader program
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var textureLoc = gl.getUniformLocation(shaderProgram, "u_image");
// Create a vertex buffer
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, positionData, gl.STATIC_DRAW);
// Create texture
var texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, 1, 1, 0, gl.RGB, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255]));
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
// Initialize rendering
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clearColor(1.0,0.0,0.0,1.0);
function drawScene() {
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(shaderProgram);
// Turn on the vertex attribute
gl.enableVertexAttribArray(positionLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
// Draw the rectangle
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
// Draw the scene repeatedly
function render() {
if (copyVideo)
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGB, gl.RGB, gl.UNSIGNED_BYTE, video);
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
})();注意:我还将纹理格式更改为RGB (alpha通道隐式为1)，但这与此无关。
Stack Overflow用户
发布于 2022-01-08 10:48:18
编辑:正如已经指出的，这个答案的前两部分是完全错误的。
TLDR:如果没有后端服务器首先获取视频数据，这可能是不可行的。
如果检查所遵循的MDN教程，则传递给texImage2D的视频对象实际上是一个MP4视频。但是，在脚本中，您可以访问的视频对象(document.getElementsByTagName("video")[0])只是一个DOM对象。你没有实际的视频数据。对于YouTube来说，要访问它并不容易。YouTube播放机不会一次抓取视频数据，而是YouTube流服务器确保流视频块。我不确定这一点，但我认为这将是非常困难的工作，如果你的目标是有一个实时视频效果。我发现一些关于这个(link1，link2)的讨论可能会有所帮助。
尽管如此，从WebGL的角度来看，您的代码中存在一些问题。理想情况下，您所拥有的代码应该显示一个蓝色矩形，因为这是您正在创建的纹理数据，而不是初始的glClearColor颜色。在视频开始播放后，它应该切换到视频纹理(由于我前面解释的问题，它将显示为黑色)。
我认为这是由于您已经设置了您的位置数据和剪辑空间计算的方式在着色器。它可以跳过以直接发送规范化设备坐标位置数据。下面是更新的代码，并对代码进行了一些清理，使其更短，其行为与预期一致：
// ==UserScript==
// @name tmp
// @namespace http://tampermonkey.net/
// @version 0.1
// @description try to take over the world!
// @author You
// @match https://www.youtube.com/*
// @icon https://www.google.com/s2/favicons?domain=youtube.com
// @grant none
// ==/UserScript==
(function() {
// will set to true when video can be copied to texture
var copyVideo = false;
const video = document.getElementsByTagName("video")[0];
// immediately after finding the video, create canvas and set its dimensions
let canvas = document.createElement('canvas');
canvas.setAttribute('id', 'glcanvas');
canvas.setAttribute('width', '300');
canvas.setAttribute('height', '200');
canvas.setAttribute('style', 'position: absolute;');
video.parentElement.appendChild(canvas);
var playing = false;
var timeupdate = false;
// Waiting for these 2 events ensures
// there is data in the video
video.addEventListener('playing', function() {
playing = true;
checkReady();
}, true);
video.addEventListener('timeupdate', function() {
timeupdate = true;
checkReady();
}, true);
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
// Initialize the GL context
const gl = canvas.getContext("webgl");
// Only continue if WebGL is available and working
if (gl === null) {
alert("Unable to initialize WebGL. Your browser or machine may not support it.");
return;
}
// Vertex shader program
const vsSource = `
attribute vec2 a_position;
attribute vec2 a_texCoord;
varying vec2 v_texCoord;
void main() {
gl_Position = vec4(a_position, 0.0, 1.0);
v_texCoord = a_texCoord;
}
`;
// Fragment shader program
const fsSource = `
precision mediump float;
uniform sampler2D u_image;
varying vec2 v_texCoord;
void main() {
gl_FragColor = texture2D(u_image, v_texCoord);
}
`;
const positionData = new Float32Array([
-1.0,-1.0,
1.0,-1.0,
-1.0, 1.0,
1.0,-1.0,
1.0, 1.0,
-1.0, 1.0
]);
const texcoordData = new Float32Array([
0.0, 0.0,
1.0, 0.0,
0.0, 1.0,
0.0, 1.0,
1.0, 0.0,
1.0, 1.0,
]);
// Initialize a shader program, so WebGL knows how to draw our data
function initShaderProgram(gl, vsSource, fsSource) {
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, loadShader(gl, gl.VERTEX_SHADER, vsSource));
gl.attachShader(shaderProgram, loadShader(gl, gl.FRAGMENT_SHADER, fsSource));
gl.linkProgram(shaderProgram);
// If creating the shader program failed, alert
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert('Unable to initialize the shader program: ' + gl.getProgramInfoLog(shaderProgram));
return null;
}
return shaderProgram;
}
// creates a shader of the given type, uploads the source and compiles it.
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
// See if it compiled successfully
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert('An error occurred compiling the shaders: ' + gl.getShaderInfoLog(shader));
gl.deleteShader(shader);
return null;
}
return shader;
}
// Initialize shader program
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
// look up where the vertex data needs to go.
var positionLocation = gl.getAttribLocation(shaderProgram, "a_position");
var texcoordLocation = gl.getAttribLocation(shaderProgram, "a_texCoord");
var textureLoc = gl.getUniformLocation(shaderProgram, "u_image");
// Create a vertex buffer
var positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.bufferData(gl.ARRAY_BUFFER, positionData, gl.STATIC_DRAW);
// Create texture coordinate buffer
var texcoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.bufferData(gl.ARRAY_BUFFER, texcoordData, gl.STATIC_DRAW);
// Create texture
var texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array([0, 0, 255, 255]));
// Initialize rendering
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.clearColor(1.0,0.0,0.0,1.0);
function drawScene() {
gl.clear(gl.COLOR_BUFFER_BIT);
gl.useProgram(shaderProgram);
// Turn on the vertex attribute
gl.enableVertexAttribArray(positionLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
gl.vertexAttribPointer(positionLocation, 2, gl.FLOAT, false, 0, 0);
// Turn on the texcoord attribute
gl.enableVertexAttribArray(texcoordLocation);
gl.bindBuffer(gl.ARRAY_BUFFER, texcoordBuffer);
gl.vertexAttribPointer(texcoordLocation, 2, gl.FLOAT, false, 0, 0);
// Draw the rectangle
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
// Draw the scene repeatedly
function render() {
if (copyVideo) {
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
}
drawScene();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
})();页面原文内容由Stack Overflow提供。腾讯云小微IT领域专用引擎提供翻译支持
原文链接：
https://stackoverflow.com/questions/70627240
复制相关文章
相似问题
腾讯云开发者
