/*
* Copyright © 2018 Benjamin Otte
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library. If not, see .
*
* Authors: Benjamin Otte
*/
#include "config.h"
#include "gtkffmediafileprivate.h"
#include "gtkintl.h"
#include "gdk/gdkmemorytextureprivate.h"
#include
#include
#include
#include
typedef struct _GtkVideoFrameFFMpeg GtkVideoFrameFFMpeg;
struct _GtkVideoFrameFFMpeg
{
GdkTexture *texture;
gint64 timestamp;
};
struct _GtkFfMediaFile
{
GtkMediaFile parent_instance;
GFile *file;
GInputStream *input_stream;
AVFormatContext *format_ctx;
AVCodecContext *codec_ctx;
int stream_id;
struct SwsContext *sws_ctx;
enum AVPixelFormat sws_pix_fmt;
GdkMemoryFormat memory_format;
GtkVideoFrameFFMpeg current_frame;
GtkVideoFrameFFMpeg next_frame;
gint64 start_time; /* monotonic time when we displayed the last frame */
guint next_frame_cb; /* Source ID of next frame callback */
};
struct _GtkFfMediaFileClass
{
GtkMediaFileClass parent_class;
};
static void
gtk_video_frame_ffmpeg_init (GtkVideoFrameFFMpeg *frame,
GdkTexture *texture,
gint64 timestamp)
{
frame->texture = texture;
frame->timestamp = timestamp;
}
static void
gtk_video_frame_ffmpeg_clear (GtkVideoFrameFFMpeg *frame)
{
g_clear_object (&frame->texture);
frame->timestamp = 0;
}
static gboolean
gtk_video_frame_ffmpeg_is_empty (GtkVideoFrameFFMpeg *frame)
{
return frame->texture == NULL;
}
static void
gtk_video_frame_ffmpeg_move (GtkVideoFrameFFMpeg *dest,
GtkVideoFrameFFMpeg *src)
{
*dest = *src;
src->texture = NULL;
src->timestamp = 0;
}
static void
gtk_ff_media_file_paintable_snapshot (GdkPaintable *paintable,
GdkSnapshot *snapshot,
double width,
double height)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (paintable);
if (!gtk_video_frame_ffmpeg_is_empty (&video->current_frame))
{
gdk_paintable_snapshot (GDK_PAINTABLE (video->current_frame.texture), snapshot, width, height);
}
}
static GdkPaintable *
gtk_ff_media_file_paintable_get_current_image (GdkPaintable *paintable)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (paintable);
if (gtk_video_frame_ffmpeg_is_empty (&video->current_frame))
{
if (video->codec_ctx)
return gdk_paintable_new_empty (video->codec_ctx->width, video->codec_ctx->height);
else
return gdk_paintable_new_empty (0, 0);
}
return GDK_PAINTABLE (g_object_ref (video->current_frame.texture));
}
static int
gtk_ff_media_file_paintable_get_intrinsic_width (GdkPaintable *paintable)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (paintable);
if (video->codec_ctx)
return video->codec_ctx->width;
return 0;
}
static int
gtk_ff_media_file_paintable_get_intrinsic_height (GdkPaintable *paintable)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (paintable);
if (video->codec_ctx)
return video->codec_ctx->height;
return 0;
}
static double gtk_ff_media_file_paintable_get_intrinsic_aspect_ratio (GdkPaintable *paintable)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (paintable);
if (video->codec_ctx)
return (double) video->codec_ctx->width / video->codec_ctx->height;
return 0.0;
};
static void
gtk_ff_media_file_paintable_init (GdkPaintableInterface *iface)
{
iface->snapshot = gtk_ff_media_file_paintable_snapshot;
iface->get_current_image = gtk_ff_media_file_paintable_get_current_image;
iface->get_intrinsic_width = gtk_ff_media_file_paintable_get_intrinsic_width;
iface->get_intrinsic_height = gtk_ff_media_file_paintable_get_intrinsic_height;
iface->get_intrinsic_aspect_ratio = gtk_ff_media_file_paintable_get_intrinsic_aspect_ratio;
}
G_DEFINE_TYPE_EXTENDED (GtkFfMediaFile, gtk_ff_media_file, GTK_TYPE_MEDIA_FILE, 0,
G_IMPLEMENT_INTERFACE (GDK_TYPE_PAINTABLE,
gtk_ff_media_file_paintable_init))
void
g_io_module_load (GIOModule *module)
{
g_type_module_use (G_TYPE_MODULE (module));
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT (58, 9, 100)
av_register_all ();
#endif
g_io_extension_point_implement (GTK_MEDIA_FILE_EXTENSION_POINT_NAME,
GTK_TYPE_FF_MEDIA_FILE,
"ffmpeg",
0);
}
void
g_io_module_unload (GIOModule *module)
{
g_assert_not_reached ();
}
char **
g_io_module_query (void)
{
char *eps[] = {
GTK_MEDIA_FILE_EXTENSION_POINT_NAME,
NULL
};
return g_strdupv (eps);
}
static void
gtk_ff_media_file_set_ffmpeg_error (GtkFfMediaFile *video,
int av_errnum)
{
char s[AV_ERROR_MAX_STRING_SIZE];
if (gtk_media_stream_get_error (GTK_MEDIA_STREAM (video)))
return;
if (av_strerror (av_errnum, s, sizeof (s) != 0))
snprintf (s, sizeof (s), _("Unspecified error decoding video"));
gtk_media_stream_error (GTK_MEDIA_STREAM (video),
G_IO_ERROR,
G_IO_ERROR_FAILED,
"%s",
s);
}
static int
gtk_ff_media_file_read_packet_cb (void *data,
uint8_t *buf,
int buf_size)
{
GtkFfMediaFile *video = data;
GError *error = NULL;
gssize n_read;
n_read = g_input_stream_read (video->input_stream,
buf,
buf_size,
NULL,
&error);
if (n_read < 0)
{
gtk_media_stream_gerror (GTK_MEDIA_STREAM (video), error);
}
else if (n_read == 0)
{
n_read = AVERROR_EOF;
}
return n_read;
}
static GdkMemoryFormat
memory_format_from_pix_fmt (enum AVPixelFormat pix_fmt)
{
switch ((int) pix_fmt)
{
case AV_PIX_FMT_RGBA:
return GDK_MEMORY_R8G8B8A8;
case AV_PIX_FMT_RGB24:
return GDK_MEMORY_R8G8B8;
default:
g_assert_not_reached ();
return GDK_MEMORY_R8G8B8A8;
}
}
static gboolean
gtk_ff_media_file_decode_frame (GtkFfMediaFile *video,
GtkVideoFrameFFMpeg *result)
{
GdkTexture *texture;
AVPacket packet;
AVFrame *frame;
int errnum;
GBytes *bytes;
guchar *data;
frame = av_frame_alloc ();
for (errnum = av_read_frame (video->format_ctx, &packet);
errnum >= 0;
errnum = av_read_frame (video->format_ctx, &packet))
{
if (packet.stream_index == video->stream_id)
{
errnum = avcodec_send_packet (video->codec_ctx, &packet);
if (errnum < 0)
G_BREAKPOINT();
if (errnum >= 0)
{
errnum = avcodec_receive_frame (video->codec_ctx, frame);
if (errnum < 0)
G_BREAKPOINT();
if (errnum >= 0)
{
av_packet_unref (&packet);
break;
}
}
}
av_packet_unref (&packet);
}
if (errnum < 0)
{
if (errnum != AVERROR_EOF)
gtk_ff_media_file_set_ffmpeg_error (video, errnum);
av_frame_free (&frame);
return FALSE;
}
data = g_try_malloc0 (video->codec_ctx->width * video->codec_ctx->height * 4);
if (data == NULL)
{
gtk_media_stream_error (GTK_MEDIA_STREAM (video),
G_IO_ERROR,
G_IO_ERROR_FAILED,
_("Not enough memory"));
av_frame_free (&frame);
return FALSE;
}
if (video->sws_ctx == NULL ||
video->sws_pix_fmt != frame->format)
{
const AVPixFmtDescriptor *desc;
enum AVPixelFormat gdk_pix_fmt;
g_clear_pointer (&video->sws_ctx, sws_freeContext);
video->sws_pix_fmt = frame->format;
desc = av_pix_fmt_desc_get (video->sws_pix_fmt);
/* Use gdk-pixbuf formats because ffmpeg can't premultiply */
if (desc != NULL && (desc->flags & AV_PIX_FMT_FLAG_ALPHA))
gdk_pix_fmt = AV_PIX_FMT_RGBA;
else
gdk_pix_fmt = AV_PIX_FMT_RGB24;
video->sws_ctx = sws_getContext (video->codec_ctx->width,
video->codec_ctx->height,
frame->format,
video->codec_ctx->width,
video->codec_ctx->height,
gdk_pix_fmt,
0,
NULL,
NULL,
NULL);
video->memory_format = memory_format_from_pix_fmt (gdk_pix_fmt);
}
sws_scale(video->sws_ctx,
(const uint8_t * const *) frame->data, frame->linesize,
0, video->codec_ctx->height,
(uint8_t *[1]) { data }, (int[1]) { video->codec_ctx->width * 4 });
bytes = g_bytes_new_take (data, video->codec_ctx->width * video->codec_ctx->height * 4);
texture = gdk_memory_texture_new (video->codec_ctx->width,
video->codec_ctx->height,
video->memory_format,
bytes,
video->codec_ctx->width * 4);
g_bytes_unref (bytes);
gtk_video_frame_ffmpeg_init (result,
texture,
av_rescale_q (frame->best_effort_timestamp,
video->format_ctx->streams[video->stream_id]->time_base,
(AVRational) { 1, G_USEC_PER_SEC }));
av_frame_free (&frame);
return TRUE;
}
static int64_t
gtk_ff_media_file_seek_cb (void *data,
int64_t offset,
int whence)
{
GtkFfMediaFile *video = data;
GSeekType seek_type;
gboolean result;
switch (whence)
{
case SEEK_SET:
seek_type = G_SEEK_SET;
break;
case SEEK_CUR:
seek_type = G_SEEK_CUR;
break;
case SEEK_END:
seek_type = G_SEEK_END;
break;
case AVSEEK_SIZE:
/* FIXME: Handle size querying */
return -1;
default:
g_assert_not_reached ();
return -1;
}
result = g_seekable_seek (G_SEEKABLE (video->input_stream),
offset,
seek_type,
NULL,
NULL);
if (!result)
return -1;
return g_seekable_tell (G_SEEKABLE (video->input_stream));
}
static gboolean
gtk_ff_media_file_create_input_stream (GtkFfMediaFile *video)
{
GError *error = NULL;
GFile *file;
file = gtk_media_file_get_file (GTK_MEDIA_FILE (video));
if (file)
{
video->input_stream = G_INPUT_STREAM (g_file_read (file, NULL, &error));
if (video->input_stream == NULL)
{
gtk_media_stream_gerror (GTK_MEDIA_STREAM (video), error);
g_error_free (error);
return FALSE;
}
}
else
{
video->input_stream = g_object_ref (gtk_media_file_get_input_stream (GTK_MEDIA_FILE (video)));
}
return TRUE;
}
static AVIOContext *
gtk_ff_media_file_create_io_context (GtkFfMediaFile *video)
{
AVIOContext *result;
int buffer_size = 4096; /* it's what everybody else uses... */
unsigned char *buffer;
if (!gtk_ff_media_file_create_input_stream (video))
return NULL;
buffer = av_malloc (buffer_size);
if (buffer == NULL)
return NULL;
result = avio_alloc_context (buffer,
buffer_size,
AVIO_FLAG_READ,
video,
gtk_ff_media_file_read_packet_cb,
NULL,
G_IS_SEEKABLE (video->input_stream)
? gtk_ff_media_file_seek_cb
: NULL);
result->buf_ptr = result->buf_end;
result->write_flag = 0;
return result;
}
static gboolean gtk_ff_media_file_play (GtkMediaStream *stream);
static void
gtk_ff_media_file_open (GtkMediaFile *file)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (file);
AVStream *stream;
AVCodec *codec;
int errnum;
video->format_ctx = avformat_alloc_context ();
video->format_ctx->pb = gtk_ff_media_file_create_io_context (video);
if (video->format_ctx->pb == NULL)
{
gtk_media_stream_error (GTK_MEDIA_STREAM (video),
G_IO_ERROR,
G_IO_ERROR_FAILED,
_("Not enough memory"));
return;
}
errnum = avformat_open_input (&video->format_ctx, NULL, NULL, NULL);
if (errnum != 0)
{
gtk_ff_media_file_set_ffmpeg_error (video, errnum);
return;
}
errnum = avformat_find_stream_info (video->format_ctx, NULL);
if (errnum < 0)
{
gtk_ff_media_file_set_ffmpeg_error (video, errnum);
return;
}
video->stream_id = av_find_best_stream (video->format_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, NULL, 0);
if (video->stream_id < 0)
{
gtk_media_stream_error (GTK_MEDIA_STREAM (video),
G_IO_ERROR,
G_IO_ERROR_INVALID_DATA,
_("Not a video file"));
return;
}
stream = video->format_ctx->streams[video->stream_id];
/* alpha transparency requires the libvpx codecs, not the ffmpeg builtin ones */
if (stream->codecpar->codec_id == AV_CODEC_ID_VP8)
codec = avcodec_find_decoder_by_name ("libvpx");
else if (stream->codecpar->codec_id == AV_CODEC_ID_VP9)
codec = avcodec_find_decoder_by_name ("libvpx-vp9");
else
codec = NULL;
if (codec == NULL)
codec = avcodec_find_decoder (stream->codecpar->codec_id);
if (codec == NULL)
{
gtk_media_stream_error (GTK_MEDIA_STREAM (video),
G_IO_ERROR,
G_IO_ERROR_NOT_SUPPORTED,
_("Unsupported video codec"));
return;
}
video->codec_ctx = avcodec_alloc_context3 (codec);
errnum = avcodec_parameters_to_context (video->codec_ctx, stream->codecpar);
if (errnum < 0)
{
gtk_ff_media_file_set_ffmpeg_error (video, errnum);
return;
}
errnum = avcodec_open2 (video->codec_ctx, codec, &stream->metadata);
if (errnum < 0)
{
gtk_ff_media_file_set_ffmpeg_error (video, errnum);
return;
}
gtk_media_stream_prepared (GTK_MEDIA_STREAM (video),
FALSE,
video->codec_ctx != NULL,
TRUE,
video->format_ctx->duration != AV_NOPTS_VALUE
? av_rescale (video->format_ctx->duration, G_USEC_PER_SEC, AV_TIME_BASE)
: 0);
gdk_paintable_invalidate_size (GDK_PAINTABLE (video));
if (gtk_ff_media_file_decode_frame (video, &video->current_frame))
gdk_paintable_invalidate_contents (GDK_PAINTABLE (video));
if (gtk_media_stream_get_playing (GTK_MEDIA_STREAM (video)))
gtk_ff_media_file_play (GTK_MEDIA_STREAM (video));
}
static void
gtk_ff_media_file_close (GtkMediaFile *file)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (file);
g_clear_object (&video->input_stream);
g_clear_pointer (&video->sws_ctx, sws_freeContext);
g_clear_pointer (&video->codec_ctx, avcodec_close);
avformat_close_input (&video->format_ctx);
video->stream_id = -1;
gtk_video_frame_ffmpeg_clear (&video->next_frame);
gtk_video_frame_ffmpeg_clear (&video->current_frame);
gdk_paintable_invalidate_size (GDK_PAINTABLE (video));
gdk_paintable_invalidate_contents (GDK_PAINTABLE (video));
}
static gboolean
gtk_ff_media_file_next_frame_cb (gpointer data);
static void
gtk_ff_media_file_queue_frame (GtkFfMediaFile *video)
{
gint64 time, frame_time;
guint delay;
time = g_get_monotonic_time ();
frame_time = video->start_time + video->next_frame.timestamp;
delay = time > frame_time ? 0 : (frame_time - time) / 1000;
video->next_frame_cb = g_timeout_add (delay, gtk_ff_media_file_next_frame_cb, video);
}
static gboolean
gtk_ff_media_file_restart (GtkFfMediaFile *video)
{
if (av_seek_frame (video->format_ctx,
video->stream_id,
av_rescale_q (0,
(AVRational) { 1, G_USEC_PER_SEC },
video->format_ctx->streams[video->stream_id]->time_base),
AVSEEK_FLAG_BACKWARD) < 0)
return FALSE;
if (!gtk_ff_media_file_decode_frame (video, &video->next_frame))
return FALSE;
return TRUE;
}
static gboolean
gtk_ff_media_file_next_frame_cb (gpointer data)
{
GtkFfMediaFile *video = data;
video->next_frame_cb = 0;
if (gtk_video_frame_ffmpeg_is_empty (&video->next_frame))
{
if (!gtk_media_stream_get_loop (GTK_MEDIA_STREAM (video)) ||
!gtk_ff_media_file_restart (video))
{
gtk_media_stream_ended (GTK_MEDIA_STREAM (video));
return G_SOURCE_REMOVE;
}
video->start_time += video->current_frame.timestamp - video->next_frame.timestamp;
}
gtk_video_frame_ffmpeg_clear (&video->current_frame);
gtk_video_frame_ffmpeg_move (&video->current_frame,
&video->next_frame);
gtk_media_stream_update (GTK_MEDIA_STREAM (video),
video->current_frame.timestamp);
gdk_paintable_invalidate_contents (GDK_PAINTABLE (video));
/* ignore failure here, we'll handle the empty frame case above
* the next time we're called. */
gtk_ff_media_file_decode_frame (video, &video->next_frame);
gtk_ff_media_file_queue_frame (video);
return G_SOURCE_REMOVE;
}
static gboolean
gtk_ff_media_file_play (GtkMediaStream *stream)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (stream);
if (video->format_ctx == NULL)
return FALSE;
if (!gtk_media_stream_is_prepared (stream))
return TRUE;
if (gtk_video_frame_ffmpeg_is_empty (&video->next_frame) &&
!gtk_ff_media_file_decode_frame (video, &video->next_frame))
{
if (gtk_ff_media_file_restart (video))
{
video->start_time = g_get_monotonic_time () - video->next_frame.timestamp;
}
else
{
return FALSE;
}
}
else
{
video->start_time = g_get_monotonic_time () - video->current_frame.timestamp;
}
gtk_ff_media_file_queue_frame (video);
return TRUE;
}
static void
gtk_ff_media_file_pause (GtkMediaStream *stream)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (stream);
if (video->next_frame_cb)
{
g_source_remove (video->next_frame_cb);
video->next_frame_cb = 0;
}
video->start_time = 0;
}
static void
gtk_ff_media_file_seek (GtkMediaStream *stream,
gint64 timestamp)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (stream);
int errnum;
errnum = av_seek_frame (video->format_ctx,
video->stream_id,
av_rescale_q (timestamp,
(AVRational) { 1, G_USEC_PER_SEC },
video->format_ctx->streams[video->stream_id]->time_base),
AVSEEK_FLAG_BACKWARD);
if (errnum < 0)
{
gtk_media_stream_seek_failed (stream);
return;
}
gtk_media_stream_seek_success (stream);
gtk_video_frame_ffmpeg_clear (&video->next_frame);
gtk_video_frame_ffmpeg_clear (&video->current_frame);
if (gtk_ff_media_file_decode_frame (video, &video->current_frame))
gtk_media_stream_update (stream, video->current_frame.timestamp);
gdk_paintable_invalidate_contents (GDK_PAINTABLE (video));
if (gtk_media_stream_get_playing (stream))
{
gtk_ff_media_file_pause (stream);
if (!gtk_ff_media_file_play (stream))
gtk_media_stream_ended (stream);
}
}
static void
gtk_ff_media_file_dispose (GObject *object)
{
GtkFfMediaFile *video = GTK_FF_MEDIA_FILE (object);
gtk_ff_media_file_pause (GTK_MEDIA_STREAM (video));
gtk_ff_media_file_close (GTK_MEDIA_FILE (video));
G_OBJECT_CLASS (gtk_ff_media_file_parent_class)->dispose (object);
}
static void
gtk_ff_media_file_class_init (GtkFfMediaFileClass *klass)
{
GtkMediaFileClass *file_class = GTK_MEDIA_FILE_CLASS (klass);
GtkMediaStreamClass *stream_class = GTK_MEDIA_STREAM_CLASS (klass);
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
file_class->open = gtk_ff_media_file_open;
file_class->close = gtk_ff_media_file_close;
stream_class->play = gtk_ff_media_file_play;
stream_class->pause = gtk_ff_media_file_pause;
stream_class->seek = gtk_ff_media_file_seek;
gobject_class->dispose = gtk_ff_media_file_dispose;
}
static void
gtk_ff_media_file_init (GtkFfMediaFile *video)
{
video->stream_id = -1;
}