From a3d257c9a4d07bd64a9852d86934fd460f0af213 Mon Sep 17 00:00:00 2001 From: donny Date: Fri, 24 Nov 2023 02:14:36 +0000 Subject: [PATCH] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=96=87=E4=BB=B6=E8=87=B3?= =?UTF-8?q?=20libavfilter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libavfilter/vf_gltransition.c | 579 ++++++++++++++++++++++++++++++++++ 1 file changed, 579 insertions(+) create mode 100644 libavfilter/vf_gltransition.c diff --git a/libavfilter/vf_gltransition.c b/libavfilter/vf_gltransition.c new file mode 100644 index 0000000..48712c4 --- /dev/null +++ b/libavfilter/vf_gltransition.c @@ -0,0 +1,579 @@ +/** + * FFmpeg filter for applying GLSL transitions between video streams. + * + * @see https://gl-transitions.com/ + */ + +#include "libavutil/opt.h" +#include "internal.h" +#include "framesync.h" + +#ifndef __APPLE__ +//# define GL_TRANSITION_USING_EGL //remove this line if you don't want to use EGL +#endif + +#ifdef __APPLE__ +# define __gl_h_ +# define GL_DO_NOT_WARN_IF_MULTI_GL_VERSION_HEADERS_INCLUDED +# include +#else +# include +#endif + +#ifdef GL_TRANSITION_USING_EGL +# include +# include +#else +# include +#endif + +#include +#include +#include + +#define FROM (0) +#define TO (1) + +#define PIXEL_FORMAT (GL_RGB) + +#ifdef GL_TRANSITION_USING_EGL +static const EGLint configAttribs[] = { + EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, + EGL_BLUE_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_RED_SIZE, 8, + EGL_DEPTH_SIZE, 8, + EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, + EGL_NONE}; +#endif +static const float position[12] = { + -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f +}; + +static const GLchar *v_shader_source = + "attribute vec2 position;\n" + "varying vec2 _uv;\n" + "void main(void) {\n" + " gl_Position = vec4(position, 0, 1);\n" + " vec2 uv = position * 0.5 + 0.5;\n" + " _uv = vec2(uv.x, 1.0 - uv.y);\n" + "}\n"; + +static const GLchar *f_shader_template = + "varying vec2 _uv;\n" + "uniform sampler2D from;\n" + "uniform sampler2D to;\n" + "uniform float progress;\n" + "uniform float ratio;\n" + "uniform float _fromR;\n" + "uniform float _toR;\n" + "\n" + "vec4 getFromColor(vec2 uv) {\n" + " return texture2D(from, vec2(uv.x, 1.0 - uv.y));\n" + "}\n" + "\n" + "vec4 getToColor(vec2 uv) {\n" + " return texture2D(to, vec2(uv.x, 1.0 - uv.y));\n" + "}\n" + "\n" + "\n%s\n" + "void main() {\n" + " gl_FragColor = transition(_uv);\n" + "}\n"; + +// default to a basic fade effect +static const GLchar *f_default_transition_source = + "vec4 transition (vec2 uv) {\n" + " return mix(\n" + " getFromColor(uv),\n" + " getToColor(uv),\n" + " progress\n" + " );\n" + "}\n"; + +typedef struct { + const AVClass *class; + FFFrameSync fs; + + // input options + double duration; + double offset; + char *source; + + // timestamp of the first frame in the output, in the timebase units + int64_t first_pts; + + // uniforms + GLuint from; + GLuint to; + GLint progress; + GLint ratio; + GLint _fromR; + GLint _toR; + + // internal state + GLuint posBuf; + GLuint program; +#ifdef GL_TRANSITION_USING_EGL + EGLDisplay eglDpy; + EGLConfig eglCfg; + EGLSurface eglSurf; + EGLContext eglCtx; +#else + GLFWwindow *window; +#endif + + GLchar *f_shader_source; +} GLTransitionContext; + +#define OFFSET(x) offsetof(GLTransitionContext, x) +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM + +static const AVOption gltransition_options[] = { + { "duration", "transition duration in seconds", OFFSET(duration), AV_OPT_TYPE_DOUBLE, {.dbl=1.0}, 0, DBL_MAX, FLAGS }, + { "offset", "delay before startingtransition in seconds", OFFSET(offset), AV_OPT_TYPE_DOUBLE, {.dbl=0.0}, 0, DBL_MAX, FLAGS }, + { "source", "path to the gl-transition source file (defaults to basic fade)", OFFSET(source), AV_OPT_TYPE_STRING, {.str = NULL}, CHAR_MIN, CHAR_MAX, FLAGS }, + {NULL} +}; + +FRAMESYNC_DEFINE_CLASS(gltransition, GLTransitionContext, fs); + +static GLuint build_shader(AVFilterContext *ctx, const GLchar *shader_source, GLenum type) +{ + GLuint shader = glCreateShader(type); + if (!shader || !glIsShader(shader)) { + return 0; + } + + glShaderSource(shader, 1, &shader_source, 0); + glCompileShader(shader); + + GLint status; + glGetShaderiv(shader, GL_COMPILE_STATUS, &status); + + return (status == GL_TRUE ? shader : 0); +} + +static int build_program(AVFilterContext *ctx) +{ + GLuint v_shader, f_shader; + GLTransitionContext *c = ctx->priv; + + if (!(v_shader = build_shader(ctx, v_shader_source, GL_VERTEX_SHADER))) { + av_log(ctx, AV_LOG_ERROR, "invalid vertex shader\n"); + return -1; + } + + char *source = NULL; + + if (c->source) { + FILE *f = fopen(c->source, "rb"); + + if (!f) { + av_log(ctx, AV_LOG_ERROR, "invalid transition source file \"%s\"\n", c->source); + return -1; + } + + fseek(f, 0, SEEK_END); + unsigned long fsize = ftell(f); + fseek(f, 0, SEEK_SET); + + source = malloc(fsize + 1); + fread(source, fsize, 1, f); + fclose(f); + + source[fsize] = 0; + } + + const char *transition_source = source ? source : f_default_transition_source; + + int len = strlen(f_shader_template) + strlen(transition_source); + c->f_shader_source = av_calloc(len, sizeof(*c->f_shader_source)); + if (!c->f_shader_source) { + return AVERROR(ENOMEM); + } + + snprintf(c->f_shader_source, len * sizeof(*c->f_shader_source), f_shader_template, transition_source); + av_log(ctx, AV_LOG_DEBUG, "\n%s\n", c->f_shader_source); + + if (source) { + free(source); + source = NULL; + } + + if (!(f_shader = build_shader(ctx, c->f_shader_source, GL_FRAGMENT_SHADER))) { + av_log(ctx, AV_LOG_ERROR, "invalid fragment shader\n"); + return -1; + } + + c->program = glCreateProgram(); + glAttachShader(c->program, v_shader); + glAttachShader(c->program, f_shader); + glLinkProgram(c->program); + + GLint status; + glGetProgramiv(c->program, GL_LINK_STATUS, &status); + return status == GL_TRUE ? 0 : -1; +} + +static void setup_vbo(GLTransitionContext *c) +{ + glGenBuffers(1, &c->posBuf); + glBindBuffer(GL_ARRAY_BUFFER, c->posBuf); + glBufferData(GL_ARRAY_BUFFER, sizeof(position), position, GL_STATIC_DRAW); + + GLint loc = glGetAttribLocation(c->program, "position"); + glEnableVertexAttribArray(loc); + glVertexAttribPointer(loc, 2, GL_FLOAT, GL_FALSE, 0, 0); +} + +static void setup_tex(AVFilterLink *fromLink) +{ + AVFilterContext *ctx = fromLink->dst; + GLTransitionContext *c = ctx->priv; + + { // from + glGenTextures(1, &c->from); + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, c->from); + + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, fromLink->w, fromLink->h, 0, PIXEL_FORMAT, GL_UNSIGNED_BYTE, NULL); + + glUniform1i(glGetUniformLocation(c->program, "from"), 0); + } + + { // to + glGenTextures(1, &c->to); + glActiveTexture(GL_TEXTURE0 + 1); + glBindTexture(GL_TEXTURE_2D, c->to); + + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, fromLink->w, fromLink->h, 0, PIXEL_FORMAT, GL_UNSIGNED_BYTE, NULL); + + glUniform1i(glGetUniformLocation(c->program, "to"), 1); + } +} + +static void setup_uniforms(AVFilterLink *fromLink) +{ + AVFilterContext *ctx = fromLink->dst; + GLTransitionContext *c = ctx->priv; + + c->progress = glGetUniformLocation(c->program, "progress"); + glUniform1f(c->progress, 0.0f); + + // TODO: this should be output ratio + c->ratio = glGetUniformLocation(c->program, "ratio"); + glUniform1f(c->ratio, fromLink->w / (float)fromLink->h); + + c->_fromR = glGetUniformLocation(c->program, "_fromR"); + glUniform1f(c->_fromR, fromLink->w / (float)fromLink->h); + + // TODO: initialize this in config_props for "to" input + c->_toR = glGetUniformLocation(c->program, "_toR"); + glUniform1f(c->_toR, fromLink->w / (float)fromLink->h); +} + +static int setup_gl(AVFilterLink *inLink) +{ + AVFilterContext *ctx = inLink->dst; + GLTransitionContext *c = ctx->priv; + + +#ifdef GL_TRANSITION_USING_EGL + //init EGL + // 1. Initialize EGL + // c->eglDpy = eglGetDisplay(EGL_DEFAULT_DISPLAY); + + #define MAX_DEVICES 4 + EGLDeviceEXT eglDevs[MAX_DEVICES]; + EGLint numDevices; + + PFNEGLQUERYDEVICESEXTPROC eglQueryDevicesEXT =(PFNEGLQUERYDEVICESEXTPROC) + eglGetProcAddress("eglQueryDevicesEXT"); + + eglQueryDevicesEXT(MAX_DEVICES, eglDevs, &numDevices); + + PFNEGLGETPLATFORMDISPLAYEXTPROC eglGetPlatformDisplayEXT = (PFNEGLGETPLATFORMDISPLAYEXTPROC) + eglGetProcAddress("eglGetPlatformDisplayEXT"); + + c->eglDpy = eglGetPlatformDisplayEXT(EGL_PLATFORM_DEVICE_EXT, eglDevs[0], 0); + + EGLint major, minor; + eglInitialize(c->eglDpy, &major, &minor); + av_log(ctx, AV_LOG_DEBUG, "%d%d", major, minor); + // 2. Select an appropriate configuration + EGLint numConfigs; + EGLint pbufferAttribs[] = { + EGL_WIDTH, + inLink->w, + EGL_HEIGHT, + inLink->h, + EGL_NONE, + }; + eglChooseConfig(c->eglDpy, configAttribs, &c->eglCfg, 1, &numConfigs); + // 3. Create a surface + c->eglSurf = eglCreatePbufferSurface(c->eglDpy, c->eglCfg, + pbufferAttribs); + // 4. Bind the API + eglBindAPI(EGL_OPENGL_API); + // 5. Create a context and make it current + c->eglCtx = eglCreateContext(c->eglDpy, c->eglCfg, EGL_NO_CONTEXT, NULL); + eglMakeCurrent(c->eglDpy, c->eglSurf, c->eglSurf, c->eglCtx); +#else + //glfw + + glfwWindowHint(GLFW_VISIBLE, 0); + c->window = glfwCreateWindow(inLink->w, inLink->h, "", NULL, NULL); + if (!c->window) { + av_log(ctx, AV_LOG_ERROR, "setup_gl ERROR"); + return -1; + } + glfwMakeContextCurrent(c->window); + +#endif + +#ifndef __APPLE__ + glewExperimental = GL_TRUE; + glewInit(); +#endif + + glViewport(0, 0, inLink->w, inLink->h); + + int ret; + if((ret = build_program(ctx)) < 0) { + return ret; + } + + glUseProgram(c->program); + setup_vbo(c); + setup_uniforms(inLink); + setup_tex(inLink); + + return 0; +} + +static AVFrame *apply_transition(FFFrameSync *fs, + AVFilterContext *ctx, + AVFrame *fromFrame, + const AVFrame *toFrame) +{ + GLTransitionContext *c = ctx->priv; + AVFilterLink *fromLink = ctx->inputs[FROM]; + AVFilterLink *toLink = ctx->inputs[TO]; + AVFilterLink *outLink = ctx->outputs[0]; + AVFrame *outFrame; + + outFrame = ff_get_video_buffer(outLink, outLink->w, outLink->h); + if (!outFrame) { + return NULL; + } + + av_frame_copy_props(outFrame, fromFrame); + +#ifdef GL_TRANSITION_USING_EGL + eglMakeCurrent(c->eglDpy, c->eglSurf, c->eglSurf, c->eglCtx); +#else + glfwMakeContextCurrent(c->window); +#endif + + glUseProgram(c->program); + + const float ts = ((fs->pts - c->first_pts) / (float)fs->time_base.den) - c->offset; + const float progress = FFMAX(0.0f, FFMIN(1.0f, ts / c->duration)); + // av_log(ctx, AV_LOG_ERROR, "transition '%s' %llu %f %f\n", c->source, fs->pts - c->first_pts, ts, progress); + glUniform1f(c->progress, progress); + + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, c->from); + glPixelStorei(GL_UNPACK_ROW_LENGTH, fromFrame->linesize[0] / 3); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, fromLink->w, fromLink->h, 0, PIXEL_FORMAT, GL_UNSIGNED_BYTE, fromFrame->data[0]); + + glActiveTexture(GL_TEXTURE0 + 1); + glBindTexture(GL_TEXTURE_2D, c->to); + glPixelStorei(GL_UNPACK_ROW_LENGTH, toFrame->linesize[0] / 3); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, toLink->w, toLink->h, 0, PIXEL_FORMAT, GL_UNSIGNED_BYTE, toFrame->data[0]); + + glDrawArrays(GL_TRIANGLES, 0, 6); + glPixelStorei(GL_PACK_ROW_LENGTH, outFrame->linesize[0] / 3); + glReadPixels(0, 0, outLink->w, outLink->h, PIXEL_FORMAT, GL_UNSIGNED_BYTE, (GLvoid *)outFrame->data[0]); + + glPixelStorei(GL_PACK_ROW_LENGTH, 0); + glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + + av_frame_free(&fromFrame); + + return outFrame; +} + +static int blend_frame(FFFrameSync *fs) +{ + AVFilterContext *ctx = fs->parent; + GLTransitionContext *c = ctx->priv; + + AVFrame *fromFrame, *toFrame, *outFrame; + int ret; + + ret = ff_framesync_dualinput_get(fs, &fromFrame, &toFrame); + if (ret < 0) { + return ret; + } + + if (c->first_pts == AV_NOPTS_VALUE && fromFrame && fromFrame->pts != AV_NOPTS_VALUE) { + c->first_pts = fromFrame->pts; + } + + if (!toFrame) { + return ff_filter_frame(ctx->outputs[0], fromFrame); + } + + outFrame = apply_transition(fs, ctx, fromFrame, toFrame); + if (!outFrame) { + return AVERROR(ENOMEM); + } + + return ff_filter_frame(ctx->outputs[0], outFrame); +} + +static av_cold int init(AVFilterContext *ctx) +{ + GLTransitionContext *c = ctx->priv; + c->fs.on_event = blend_frame; + c->first_pts = AV_NOPTS_VALUE; + + +#ifndef GL_TRANSITION_USING_EGL + if (!glfwInit()) + { + return -1; + } +#endif + + return 0; +} + +static av_cold void uninit(AVFilterContext *ctx) { + GLTransitionContext *c = ctx->priv; + ff_framesync_uninit(&c->fs); + +#ifdef GL_TRANSITION_USING_EGL + if (c->eglDpy) { + glDeleteTextures(1, &c->from); + glDeleteTextures(1, &c->to); + glDeleteBuffers(1, &c->posBuf); + glDeleteProgram(c->program); + eglTerminate(c->eglDpy); + } +#else + if (c->window) { + glDeleteTextures(1, &c->from); + glDeleteTextures(1, &c->to); + glDeleteBuffers(1, &c->posBuf); + glDeleteProgram(c->program); + glfwDestroyWindow(c->window); + } +#endif + + if (c->f_shader_source) { + av_freep(&c->f_shader_source); + } +} + +static int query_formats(AVFilterContext *ctx) +{ + static const enum AVPixelFormat formats[] = { + AV_PIX_FMT_RGB24, + AV_PIX_FMT_NONE + }; + + return ff_set_common_formats(ctx, ff_make_format_list(formats)); +} + +static int activate(AVFilterContext *ctx) +{ + GLTransitionContext *c = ctx->priv; + return ff_framesync_activate(&c->fs); +} + +static int config_output(AVFilterLink *outLink) +{ + AVFilterContext *ctx = outLink->src; + GLTransitionContext *c = ctx->priv; + AVFilterLink *fromLink = ctx->inputs[FROM]; + AVFilterLink *toLink = ctx->inputs[TO]; + int ret; + + if (fromLink->format != toLink->format) { + av_log(ctx, AV_LOG_ERROR, "inputs must be of same pixel format\n"); + return AVERROR(EINVAL); + } + + if (fromLink->w != toLink->w || fromLink->h != toLink->h) { + av_log(ctx, AV_LOG_ERROR, "First input link %s parameters " + "(size %dx%d) do not match the corresponding " + "second input link %s parameters (size %dx%d)\n", + ctx->input_pads[FROM].name, fromLink->w, fromLink->h, + ctx->input_pads[TO].name, toLink->w, toLink->h); + return AVERROR(EINVAL); + } + + outLink->w = fromLink->w; + outLink->h = fromLink->h; + // outLink->time_base = fromLink->time_base; + outLink->frame_rate = fromLink->frame_rate; + + if ((ret = ff_framesync_init_dualinput(&c->fs, ctx)) < 0) { + return ret; + } + + return ff_framesync_configure(&c->fs); +} + +static const AVFilterPad gltransition_inputs[] = { + { + .name = "from", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = setup_gl, + }, + { + .name = "to", + .type = AVMEDIA_TYPE_VIDEO, + }, + {NULL} +}; + +static const AVFilterPad gltransition_outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = config_output, + }, + {NULL} +}; + +AVFilter ff_vf_gltransition = { + .name = "gltransition", + .description = NULL_IF_CONFIG_SMALL("OpenGL blend transitions"), + .priv_size = sizeof(GLTransitionContext), + .preinit = gltransition_framesync_preinit, + .init = init, + .uninit = uninit, + .query_formats = query_formats, + .activate = activate, + .inputs = gltransition_inputs, + .outputs = gltransition_outputs, + .priv_class = &gltransition_class, + .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC +};