AuroraMiddleware/gtk2
1
0
Fork 0
forked from AuroraMiddleware/gtk
Code Issues Pull Requests Projects Releases Wiki Activity

media: Add support for OpenGL to GtkGstMediaFile

Browse Source
This commit is contained in:
Benjamin Otte 2020-12-29 14:11:51 -05:00 committed by Matthias Clasen
parent 8f585f7a53
commit 7901ab857b
6 changed files with 381 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
modules/media/gtkgstmediafile.c
View File

@ -297,6 +297,24 @@ gtk_gst_media_file_update_audio (GtkMediaStream *stream,
gst_player_set_volume (self->player, volume * volume * volume);
}
static void
gtk_gst_media_file_realize (GtkMediaStream *stream,
GdkSurface *surface)
{
GtkGstMediaFile *self = GTK_GST_MEDIA_FILE (stream);
gtk_gst_paintable_realize (GTK_GST_PAINTABLE (self->paintable), surface);
}
static void
gtk_gst_media_file_unrealize (GtkMediaStream *stream,
GdkSurface *surface)
{
GtkGstMediaFile *self = GTK_GST_MEDIA_FILE (stream);
gtk_gst_paintable_unrealize (GTK_GST_PAINTABLE (self->paintable), surface);
}
static void
gtk_gst_media_file_dispose (GObject *object)
{
@ -327,6 +345,8 @@ gtk_gst_media_file_class_init (GtkGstMediaFileClass *klass)
stream_class->pause = gtk_gst_media_file_pause;
stream_class->seek = gtk_gst_media_file_seek;
stream_class->update_audio = gtk_gst_media_file_update_audio;
stream_class->realize = gtk_gst_media_file_realize;
stream_class->unrealize = gtk_gst_media_file_unrealize;
gobject_class->dispose = gtk_gst_media_file_dispose;
}

44
modules/media/gtkgstpaintable.c
View File

@ -33,6 +33,8 @@ struct _GtkGstPaintable
GdkPaintable *image;
double pixel_aspect_ratio;
GdkGLContext *context;
};
struct _GtkGstPaintableClass
@ -116,6 +118,7 @@ gtk_gst_paintable_video_renderer_create_video_sink (GstPlayerVideoRenderer *rend
return g_object_new (GTK_TYPE_GST_SINK,
"paintable", self,
"gl-context", self->context,
NULL);
}
@ -160,6 +163,47 @@ gtk_gst_paintable_new (void)
return g_object_new (GTK_TYPE_GST_PAINTABLE, NULL);
}
void
gtk_gst_paintable_realize (GtkGstPaintable *self,
GdkSurface *surface)
{
GError *error = NULL;
if (self->context)
return;
self->context = gdk_surface_create_gl_context (surface, &error);
if (self->context == NULL)
{
GST_INFO ("failed to create GDK GL context: %s", error->message);
g_error_free (error);
return;
}
if (!gdk_gl_context_realize (self->context, &error))
{
GST_INFO ("failed to realize GDK GL context: %s", error->message);
g_clear_object (&self->context);
g_error_free (error);
return;
}
}
void
gtk_gst_paintable_unrealize (GtkGstPaintable *self,
GdkSurface *surface)
{
/* XXX: We could be smarter here and:
* - track how often we were realized with that surface
* - track alternate surfaces
*/
if (self->context == NULL)
return;
if (gdk_gl_context_get_surface (self->context) == surface)
g_clear_object (&self->context);
}
static void
gtk_gst_paintable_set_paintable (GtkGstPaintable *self,
GdkPaintable *paintable,

4
modules/media/gtkgstpaintableprivate.h
View File

@ -30,6 +30,10 @@ G_DECLARE_FINAL_TYPE (GtkGstPaintable, gtk_gst_paintable, GTK, GST_PAINTABLE, GO
GdkPaintable * gtk_gst_paintable_new (void);
void gtk_gst_paintable_realize (GtkGstPaintable *self,
GdkSurface *surface);
void gtk_gst_paintable_unrealize (GtkGstPaintable *self,
GdkSurface *surface);
void gtk_gst_paintable_queue_set_texture (GtkGstPaintable *self,
GdkTexture *texture,
double pixel_aspect_ratio);

334
modules/media/gtkgstsink.c
View File

@ -25,9 +25,20 @@
#include "gtkgstpaintableprivate.h"
#include "gtkintl.h"
#if GST_GL_HAVE_WINDOW_X11 && GST_GL_HAVE_PLATFORM_GLX && defined (GDK_WINDOWING_X11)
#include <gdk/x11/gdkx.h>
#include <gst/gl/x11/gstgldisplay_x11.h>
#endif
#if GST_GL_HAVE_WINDOW_WAYLAND && GST_GL_HAVE_PLATFORM_EGL && defined (GDK_WINDOWING_WAYLAND)
#include <gdk/wayland/gdkwayland.h>
#include <gst/gl/wayland/gstgldisplay_wayland.h>
#endif
enum {
PROP_0,
PROP_PAINTABLE,
PROP_GL_CONTEXT,
N_PROPS,
};
@ -37,11 +48,14 @@ GST_DEBUG_CATEGORY (gtk_debug_gst_sink);
#define FORMATS "{ BGRA, ARGB, RGBA, ABGR, RGB, BGR }"
#define NOGL_CAPS GST_VIDEO_CAPS_MAKE (FORMATS)
#define GL_CAPS GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA")
static GstStaticPadTemplate gtk_gst_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS))
GST_STATIC_CAPS (GL_CAPS "; " NOGL_CAPS)
);
G_DEFINE_TYPE_WITH_CODE (GtkGstSink, gtk_gst_sink,
@ -53,34 +67,75 @@ static GParamSpec *properties[N_PROPS] = { NULL, };
static void
gtk_gst_sink_get_times (GstBaseSink * bsink, GstBuffer * buf,
GstClockTime * start, GstClockTime * end)
gtk_gst_sink_get_times (GstBaseSink *bsink,
GstBuffer *buf,
GstClockTime *start,
GstClockTime *end)
{
GtkGstSink *gtk_sink;
gtk_sink = GTK_GST_SINK (bsink);
if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) {
*start = GST_BUFFER_TIMESTAMP (buf);
if (GST_BUFFER_DURATION_IS_VALID (buf))
*end = *start + GST_BUFFER_DURATION (buf);
else {
if (GST_VIDEO_INFO_FPS_N (&gtk_sink->v_info) > 0) {
*end = *start +
gst_util_uint64_scale_int (GST_SECOND,
GST_VIDEO_INFO_FPS_D (&gtk_sink->v_info),
GST_VIDEO_INFO_FPS_N (&gtk_sink->v_info));
}
if (GST_BUFFER_TIMESTAMP_IS_VALID (buf))
{
*start = GST_BUFFER_TIMESTAMP (buf);
if (GST_BUFFER_DURATION_IS_VALID (buf))
*end = *start + GST_BUFFER_DURATION (buf);
else
{
if (GST_VIDEO_INFO_FPS_N (&gtk_sink->v_info) > 0)
{
*end = *start +
gst_util_uint64_scale_int (GST_SECOND,
GST_VIDEO_INFO_FPS_D (&gtk_sink->v_info),
GST_VIDEO_INFO_FPS_N (&gtk_sink->v_info));
}
}
}
}
}
static GstCaps *
gtk_gst_sink_get_caps (GstBaseSink *bsink,
GstCaps *filter)
{
GtkGstSink *self = GTK_GST_SINK (bsink);
GstCaps *tmp;
GstCaps *result;
if (self->gst_context)
{
tmp = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (bsink));
}
else
{
tmp = gst_caps_from_string (NOGL_CAPS);
}
GST_DEBUG_OBJECT (self, "advertising own caps %" GST_PTR_FORMAT, tmp);
if (filter)
{
GST_DEBUG_OBJECT (self, "intersecting with filter caps %" GST_PTR_FORMAT, filter);
result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (tmp);
}
else
{
result = tmp;
}
GST_DEBUG_OBJECT (self, "returning caps: %" GST_PTR_FORMAT, result);
return result;
}
static gboolean
gtk_gst_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
gtk_gst_sink_set_caps (GstBaseSink *bsink,
GstCaps *caps)
{
GtkGstSink *self = GTK_GST_SINK (bsink);
GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps);
GST_DEBUG_OBJECT (self, "set caps with %" GST_PTR_FORMAT, caps);
if (!gst_video_info_from_caps (&self->v_info, caps))
return FALSE;
@ -88,6 +143,82 @@ gtk_gst_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
return TRUE;
}
static gboolean
gtk_gst_sink_query (GstBaseSink *bsink,
GstQuery *query)
{
GtkGstSink *self = GTK_GST_SINK (bsink);
if (GST_QUERY_TYPE (query) == GST_QUERY_CONTEXT &&
self->gst_display != NULL &&
gst_gl_handle_context_query (GST_ELEMENT (self), query, self->gst_display, self->gst_context, self->gst_app_context))
return TRUE;
return GST_BASE_SINK_CLASS (gtk_gst_sink_parent_class)->query (bsink, query);
}
static gboolean
gtk_gst_sink_propose_allocation (GstBaseSink *bsink,
GstQuery *query)
{
GtkGstSink *self = GTK_GST_SINK (bsink);
GstBufferPool *pool = NULL;
GstStructure *config;
GstCaps *caps;
guint size;
gboolean need_pool;
if (!self->gst_context)
return FALSE;
gst_query_parse_allocation (query, &caps, &need_pool);
if (caps == NULL)
{
GST_DEBUG_OBJECT (bsink, "no caps specified");
return FALSE;
}
if (!gst_caps_features_contains (gst_caps_get_features (caps, 0), GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
return FALSE;
if (need_pool)
{
GstVideoInfo info;
if (!gst_video_info_from_caps (&info, caps))
{
GST_DEBUG_OBJECT (self, "invalid caps specified");
return FALSE;
}
GST_DEBUG_OBJECT (self, "create new pool");
pool = gst_gl_buffer_pool_new (self->gst_context);
/* the normal size of a frame */
size = info.size;
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
if (!gst_buffer_pool_set_config (pool, config))
{
GST_DEBUG_OBJECT (bsink, "failed setting config");
gst_object_unref (pool);
return FALSE;
}
/* we need at least 2 buffer because we hold on to the last one */
gst_query_add_allocation_pool (query, pool, size, 2, 0);
gst_object_unref (pool);
}
/* we also support various metadata */
gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0);
return TRUE;
}
static GdkMemoryFormat
gtk_gst_memory_format_from_video (GstVideoFormat format)
{
@ -125,29 +256,50 @@ gtk_gst_sink_texture_from_buffer (GtkGstSink *self,
{
GstVideoFrame frame;
GdkTexture *texture;
GBytes *bytes;
if (!gst_video_frame_map (&frame, &self->v_info, buffer, GST_MAP_READ))
return NULL;
bytes = g_bytes_new_with_free_func (frame.data[0],
frame.info.width * frame.info.stride[0],
(GDestroyNotify) video_frame_free,
g_memdup (&frame, sizeof (frame)));
texture = gdk_memory_texture_new (frame.info.width,
if (self->gdk_context &&
gst_video_frame_map (&frame, &self->v_info, buffer, GST_MAP_READ | GST_MAP_GL))
{
texture = gdk_gl_texture_new (self->gdk_context,
*(guint *) frame.data[0],
frame.info.width,
frame.info.height,
gtk_gst_memory_format_from_video (GST_VIDEO_FRAME_FORMAT (&frame)),
bytes,
frame.info.stride[0]);
g_bytes_unref (bytes);
(GDestroyNotify) gst_buffer_unref,
gst_buffer_ref (buffer));
*pixel_aspect_ratio = ((double) frame.info.par_n) / ((double) frame.info.par_d);
*pixel_aspect_ratio = ((double) frame.info.par_n) / ((double) frame.info.par_d);
gst_video_frame_unmap (&frame);
}
else if (gst_video_frame_map (&frame, &self->v_info, buffer, GST_MAP_READ))
{
GBytes *bytes;
bytes = g_bytes_new_with_free_func (frame.data[0],
frame.info.height * frame.info.stride[0],
(GDestroyNotify) video_frame_free,
g_memdup (&frame, sizeof (frame)));
texture = gdk_memory_texture_new (frame.info.width,
frame.info.height,
gtk_gst_memory_format_from_video (GST_VIDEO_FRAME_FORMAT (&frame)),
bytes,
frame.info.stride[0]);
g_bytes_unref (bytes);
*pixel_aspect_ratio = ((double) frame.info.par_n) / ((double) frame.info.par_d);
}
else
{
GST_ERROR_OBJECT (self, "Could not convert buffer to texture.");
texture = NULL;
}
return texture;
}
static GstFlowReturn
gtk_gst_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
gtk_gst_sink_show_frame (GstVideoSink *vsink,
GstBuffer *buf)
{
GtkGstSink *self;
GdkTexture *texture;
@ -171,6 +323,100 @@ gtk_gst_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
return GST_FLOW_OK;
}
static void
gtk_gst_sink_initialize_gl (GtkGstSink *self)
{
GdkDisplay *display;
GError *error = NULL;
display = gdk_gl_context_get_display (self->gdk_context);
gdk_gl_context_make_current (self->gdk_context);
#if GST_GL_HAVE_WINDOW_X11 && GST_GL_HAVE_PLATFORM_GLX && defined (GDK_WINDOWING_X11)
if (GDK_IS_X11_DISPLAY (display))
{
GstGLPlatform platform = GST_GL_PLATFORM_GLX;
GstGLAPI gl_api;
guintptr gl_handle;
GST_DEBUG_OBJECT (self, "got GLX on X11!");
gl_api = gst_gl_context_get_current_gl_api (platform, NULL, NULL);
gl_handle = gst_gl_context_get_current_gl_context (platform);
if (gl_handle)
{
self->gst_display = GST_GL_DISPLAY (gst_gl_display_x11_new_with_display (gdk_x11_display_get_xdisplay (display)));
self->gst_app_context = gst_gl_context_new_wrapped (self->gst_display, gl_handle, platform, gl_api);
}
else
{
GST_ERROR_OBJECT (self, "Failed to get handle from GdkGLContext, not using GLX");
return;
}
}
else
#endif
#if GST_GL_HAVE_WINDOW_WAYLAND && GST_GL_HAVE_PLATFORM_EGL && defined (GDK_WINDOWING_WAYLAND)
if (GDK_IS_WAYLAND_DISPLAY (display))
{
GstGLPlatform platform = GST_GL_PLATFORM_GLX;
GstGLAPI gl_api;
guintptr gl_handle;
GST_DEBUG_OBJECT (self, "got EGL on Wayland!");
platform = GST_GL_PLATFORM_EGL;
gl_api = gst_gl_context_get_current_gl_api (platform, NULL, NULL);
gl_handle = gst_gl_context_get_current_gl_context (platform);
if (gl_handle)
{
struct wl_display *wayland_display;
wayland_display = gdk_wayland_display_get_wl_display (display);
self->gst_display = GST_GL_DISPLAY (gst_gl_display_wayland_new_with_display (wayland_display));
self->gst_app_context = gst_gl_context_new_wrapped (self->gst_display, gl_handle, platform, gl_api);
}
else
{
GST_ERROR_OBJECT (self, "Failed to get handle from GdkGLContext, not using Wayland EGL");
return;
}
}
else
#endif
{
GST_INFO_OBJECT (self, "Unsupported GDK display %s for GL", G_OBJECT_TYPE_NAME (display));
return;
}
g_assert (self->gst_app_context != NULL);
gst_gl_context_activate (self->gst_app_context, TRUE);
if (!gst_gl_context_fill_info (self->gst_app_context, &error))
{
GST_ERROR_OBJECT (self, "failed to retrieve GDK context info: %s", error->message);
g_clear_error (&error);
g_clear_object (&self->gst_app_context);
g_clear_object (&self->gst_display);
return;
}
else
{
gst_gl_context_activate (self->gst_app_context, FALSE);
}
if (!gst_gl_display_create_context (self->gst_display, self->gst_app_context, &self->gst_context, &error))
{
GST_ERROR_OBJECT (self, "Couldn't create GL context: %s", error->message);
g_error_free (error);
g_clear_object (&self->gst_app_context);
g_clear_object (&self->gst_display);
return;
}
}
static void
gtk_gst_sink_set_property (GObject *object,
guint prop_id,
@ -188,6 +434,12 @@ gtk_gst_sink_set_property (GObject *object,
self->paintable = GTK_GST_PAINTABLE (gtk_gst_paintable_new ());
break;
case PROP_GL_CONTEXT:
self->gdk_context = g_value_dup_object (value);
if (self->gdk_context != NULL)
gtk_gst_sink_initialize_gl (self);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@ -220,6 +472,9 @@ gtk_gst_sink_dispose (GObject *object)
GtkGstSink *self = GTK_GST_SINK (object);
g_clear_object (&self->paintable);
g_clear_object (&self->gst_app_context);
g_clear_object (&self->gst_display);
g_clear_object (&self->gdk_context);
G_OBJECT_CLASS (gtk_gst_sink_parent_class)->dispose (object);
}
@ -238,6 +493,9 @@ gtk_gst_sink_class_init (GtkGstSinkClass * klass)
gstbasesink_class->set_caps = gtk_gst_sink_set_caps;
gstbasesink_class->get_times = gtk_gst_sink_get_times;
gstbasesink_class->query = gtk_gst_sink_query;
gstbasesink_class->propose_allocation = gtk_gst_sink_propose_allocation;
gstbasesink_class->get_caps = gtk_gst_sink_get_caps;
gstvideosink_class->show_frame = gtk_gst_sink_show_frame;
@ -253,6 +511,18 @@ gtk_gst_sink_class_init (GtkGstSinkClass * klass)
GTK_TYPE_GST_PAINTABLE,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS);
/**
* GtkGstSink:gl-context:
*
* The #GdkGLContext to use for GL rendering.
*/
properties[PROP_GL_CONTEXT] =
g_param_spec_object ("gl-context",
P_("gl-context"),
P_("GL context to use for rendering"),
GDK_TYPE_GL_CONTEXT,
G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS);
g_object_class_install_properties (gobject_class, N_PROPS, properties);
gst_element_class_set_metadata (gstelement_class,

6
modules/media/gtkgstsinkprivate.h
View File

@ -24,6 +24,8 @@
#include "gtkgstpaintableprivate.h"
#include <gst/gst.h>
#define GST_USE_UNSTABLE_API
#include <gst/gl/gl.h>
#include <gst/video/gstvideosink.h>
#include <gst/video/video.h>
@ -47,6 +49,10 @@ struct _GtkGstSink
GstVideoInfo v_info;
GtkGstPaintable * paintable;
GdkGLContext * gdk_context;
GstGLDisplay * gst_display;
GstGLContext * gst_app_context;
GstGLContext * gst_context;
};
struct _GtkGstSinkClass

7
modules/media/meson.build
View File

@ -42,7 +42,10 @@ endif
gstplayer_dep = dependency('gstreamer-player-1.0', version: '>= 1.12.3',
required: get_option('media-gstreamer'))
if gstplayer_dep.found()
gstgl_dep = dependency('gstreamer-gl-1.0', version: '>= 1.12.3',
required: get_option('media-gstreamer'))
if gstplayer_dep.found() and gstgl_dep.found()
media_backends += 'gstreamer'
cdata.set('HAVE_GSTREAMER', 1)
shared_module('media-gstreamer',
@ -52,7 +55,7 @@ if gstplayer_dep.found()
'gtkgstsink.c',
],
c_args: extra_c_args,
dependencies: [ libm, libgtk_dep, gstplayer_dep ],
dependencies: [ libm, libgtk_dep, gstplayer_dep, gstgl_dep ],
install_dir: media_install_dir,
install: true,
)