use std::{error::Error, fmt, str::FromStr, sync::Arc};
use gettextrs::gettext;
use gtk::{gdk, gio, graphene, gsk, prelude::*};
use matrix_sdk::{
attachment::{BaseImageInfo, Thumbnail},
media::{MediaFormat, MediaRequestParameters, MediaThumbnailSettings},
Client,
};
use ruma::{
api::client::media::get_content_thumbnail::v3::Method,
events::{
room::{
avatar::ImageInfo as AvatarImageInfo, ImageInfo, MediaSource as CommonMediaSource,
ThumbnailInfo,
},
sticker::StickerMediaSource,
},
OwnedMxcUri,
};
use tracing::{error, warn};
mod queue;
pub(crate) use queue::{ImageRequestPriority, IMAGE_QUEUE};
use super::{FrameDimensions, MediaFileError};
use crate::{components::AnimatedImagePaintable, spawn_tokio, utils::File, DISABLE_GLYCIN_SANDBOX};
pub const THUMBNAIL_MAX_DIMENSIONS: FrameDimensions = FrameDimensions {
width: 600,
height: 400,
};
const SVG_CONTENT_TYPE: &str = "image/svg+xml";
const WEBP_CONTENT_TYPE: &str = "image/webp";
const WEBP_DEFAULT_QUALITY: f32 = 60.0;
const THUMBNAIL_MAX_FILESIZE_THRESHOLD: u32 = 1024 * 1024;
const THUMBNAIL_DIMENSIONS_THRESHOLD: u32 = 200;
const SUPPORTED_ANIMATED_IMAGE_MIME_TYPES: &[&str] = &["image/gif", "image/png", "image/webp"];
async fn image_loader(file: gio::File) -> Result<glycin::Image<'static>, glycin::ErrorCtx> {
let mut loader = glycin::Loader::new(file);
if DISABLE_GLYCIN_SANDBOX {
loader.sandbox_selector(glycin::SandboxSelector::NotSandboxed);
}
spawn_tokio!(async move { loader.load().await })
.await
.unwrap()
}
async fn load_image(
file: File,
request_dimensions: Option<FrameDimensions>,
) -> Result<Image, glycin::ErrorCtx> {
let image_loader = image_loader(file.as_gfile()).await?;
let frame_request = request_dimensions.map(|request| {
let image_info = image_loader.info();
let original_dimensions = FrameDimensions {
width: image_info.width,
height: image_info.height,
};
original_dimensions.to_image_loader_request(request)
});
spawn_tokio!(async move {
let first_frame = if let Some(frame_request) = frame_request {
image_loader.specific_frame(frame_request).await?
} else {
image_loader.next_frame().await?
};
Ok(Image {
file,
loader: image_loader.into(),
first_frame: first_frame.into(),
})
})
.await
.expect("task was not aborted")
}
#[derive(Clone)]
pub struct Image {
file: File,
loader: Arc<glycin::Image<'static>>,
first_frame: Arc<glycin::Frame>,
}
impl fmt::Debug for Image {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Image").finish_non_exhaustive()
}
}
impl From<Image> for gdk::Paintable {
fn from(value: Image) -> Self {
if value.first_frame.delay().is_some() {
AnimatedImagePaintable::new(value.file, value.loader, value.first_frame).upcast()
} else {
value.first_frame.texture().upcast()
}
}
}
pub enum ImageInfoLoader {
File(gio::File),
Texture(gdk::Texture),
}
impl ImageInfoLoader {
async fn into_first_frame(self) -> Option<Frame> {
match self {
Self::File(file) => {
let image_loader = image_loader(file).await.ok()?;
let handle = spawn_tokio!(async move { image_loader.next_frame().await });
Some(Frame::Glycin(handle.await.unwrap().ok()?))
}
Self::Texture(texture) => Some(Frame::Texture(texture)),
}
}
pub async fn load_info(self) -> BaseImageInfo {
self.into_first_frame()
.await
.map(|f| f.info())
.unwrap_or_default()
}
pub async fn load_info_and_thumbnail(
self,
filesize: Option<u32>,
widget: &impl IsA<gtk::Widget>,
) -> (BaseImageInfo, Option<Thumbnail>) {
let Some(frame) = self.into_first_frame().await else {
return (BaseImageInfo::default(), None);
};
let info = frame.info();
let scale_factor = widget.scale_factor();
let max_thumbnail_dimensions =
FrameDimensions::thumbnail_max_dimensions(widget.scale_factor());
if !filesize_is_too_big(filesize)
&& !frame
.dimensions()
.is_some_and(|d| d.needs_thumbnail(max_thumbnail_dimensions))
{
return (info, None);
}
let Some(renderer) = widget
.root()
.and_downcast::<gtk::Window>()
.and_then(|w| w.renderer())
else {
error!("Could not get GdkRenderer");
return (info, None);
};
let thumbnail = frame.generate_thumbnail(scale_factor, &renderer);
(info, thumbnail)
}
}
impl From<gio::File> for ImageInfoLoader {
fn from(value: gio::File) -> Self {
Self::File(value)
}
}
impl From<gdk::Texture> for ImageInfoLoader {
fn from(value: gdk::Texture) -> Self {
Self::Texture(value)
}
}
#[derive(Debug, Clone)]
enum Frame {
Glycin(glycin::Frame),
Texture(gdk::Texture),
}
impl Frame {
fn dimensions(&self) -> Option<FrameDimensions> {
match self {
Self::Glycin(frame) => Some(FrameDimensions {
width: frame.width(),
height: frame.height(),
}),
Self::Texture(texture) => FrameDimensions::with_texture(texture),
}
}
fn is_animated(&self) -> bool {
match self {
Self::Glycin(frame) => frame.delay().is_some(),
Self::Texture(_) => false,
}
}
fn info(&self) -> BaseImageInfo {
let dimensions = self.dimensions();
BaseImageInfo {
width: dimensions.map(|d| d.width.into()),
height: dimensions.map(|d| d.height.into()),
is_animated: Some(self.is_animated()),
..Default::default()
}
}
fn generate_thumbnail(self, scale_factor: i32, renderer: &gsk::Renderer) -> Option<Thumbnail> {
let texture = match self {
Self::Glycin(frame) => frame.texture(),
Self::Texture(texture) => texture,
};
let thumbnail = TextureThumbnailer(texture).generate_thumbnail(scale_factor, renderer);
if thumbnail.is_none() {
warn!("Could not generate thumbnail from GdkTexture");
}
thumbnail
}
}
impl FrameDimensions {
pub fn thumbnail_max_dimensions(scale_factor: i32) -> Self {
let scale_factor = scale_factor.try_into().unwrap_or(1);
THUMBNAIL_MAX_DIMENSIONS.scale(scale_factor)
}
fn with_texture(texture: &gdk::Texture) -> Option<Self> {
Some(Self {
width: texture.width().try_into().ok()?,
height: texture.height().try_into().ok()?,
})
}
pub(super) fn needs_thumbnail(self, thumbnail_dimensions: FrameDimensions) -> bool {
self.ge(thumbnail_dimensions.increase_by(THUMBNAIL_DIMENSIONS_THRESHOLD))
}
pub(super) fn downscale_for(self, max_dimensions: FrameDimensions) -> Option<Self> {
if !self.ge(max_dimensions) {
return None;
}
Some(self.scale_to_fit(max_dimensions, gtk::ContentFit::ScaleDown))
}
fn to_image_loader_request(self, requested: Self) -> glycin::FrameRequest {
let scaled = self.scale_to_fit(requested, gtk::ContentFit::Cover);
glycin::FrameRequest::new().scale(scaled.width, scaled.height)
}
}
#[derive(Debug, Clone)]
pub(super) struct TextureThumbnailer(pub(super) gdk::Texture);
impl TextureThumbnailer {
fn downscale_texture_if_needed(
self,
max_dimensions: FrameDimensions,
renderer: &gsk::Renderer,
) -> Option<gdk::Texture> {
let dimensions = FrameDimensions::with_texture(&self.0)?;
let texture = if let Some(target_dimensions) = dimensions.downscale_for(max_dimensions) {
let snapshot = gtk::Snapshot::new();
let bounds = graphene::Rect::new(
0.0,
0.0,
target_dimensions.width as f32,
target_dimensions.height as f32,
);
snapshot.append_texture(&self.0, &bounds);
let node = snapshot.to_node()?;
renderer.render_texture(node, None)
} else {
self.0
};
Some(texture)
}
fn texture_format_to_thumbnail_format(
format: gdk::MemoryFormat,
) -> Option<(gdk::MemoryFormat, webp::PixelLayout)> {
match format {
gdk::MemoryFormat::B8g8r8a8Premultiplied
| gdk::MemoryFormat::A8r8g8b8Premultiplied
| gdk::MemoryFormat::R8g8b8a8Premultiplied
| gdk::MemoryFormat::B8g8r8a8
| gdk::MemoryFormat::A8r8g8b8
| gdk::MemoryFormat::R8g8b8a8
| gdk::MemoryFormat::R16g16b16a16Premultiplied
| gdk::MemoryFormat::R16g16b16a16
| gdk::MemoryFormat::R16g16b16a16FloatPremultiplied
| gdk::MemoryFormat::R16g16b16a16Float
| gdk::MemoryFormat::R32g32b32a32FloatPremultiplied
| gdk::MemoryFormat::R32g32b32a32Float
| gdk::MemoryFormat::G8a8Premultiplied
| gdk::MemoryFormat::G8a8
| gdk::MemoryFormat::G16a16Premultiplied
| gdk::MemoryFormat::G16a16
| gdk::MemoryFormat::A8
| gdk::MemoryFormat::A16
| gdk::MemoryFormat::A16Float
| gdk::MemoryFormat::A32Float
| gdk::MemoryFormat::A8b8g8r8Premultiplied
| gdk::MemoryFormat::A8b8g8r8 => {
Some((gdk::MemoryFormat::R8g8b8a8, webp::PixelLayout::Rgba))
}
gdk::MemoryFormat::R8g8b8
| gdk::MemoryFormat::B8g8r8
| gdk::MemoryFormat::R16g16b16
| gdk::MemoryFormat::R16g16b16Float
| gdk::MemoryFormat::R32g32b32Float
| gdk::MemoryFormat::G8
| gdk::MemoryFormat::G16
| gdk::MemoryFormat::B8g8r8x8
| gdk::MemoryFormat::X8r8g8b8
| gdk::MemoryFormat::R8g8b8x8
| gdk::MemoryFormat::X8b8g8r8 => {
Some((gdk::MemoryFormat::R8g8b8, webp::PixelLayout::Rgb))
}
_ => None,
}
}
pub(super) fn generate_thumbnail(
self,
scale_factor: i32,
renderer: &gsk::Renderer,
) -> Option<Thumbnail> {
let max_thumbnail_dimensions = FrameDimensions::thumbnail_max_dimensions(scale_factor);
let thumbnail = self.downscale_texture_if_needed(max_thumbnail_dimensions, renderer)?;
let dimensions = FrameDimensions::with_texture(&thumbnail)?;
let (downloader_format, webp_layout) =
Self::texture_format_to_thumbnail_format(thumbnail.format())?;
let mut downloader = gdk::TextureDownloader::new(&thumbnail);
downloader.set_format(downloader_format);
let (data, _) = downloader.download_bytes();
let encoder = webp::Encoder::new(&data, webp_layout, dimensions.width, dimensions.height);
let data = encoder.encode(WEBP_DEFAULT_QUALITY).to_vec();
let size = data.len().try_into().ok()?;
let content_type =
mime::Mime::from_str(WEBP_CONTENT_TYPE).expect("content type should be valid");
Some(Thumbnail {
data,
content_type,
width: dimensions.width.into(),
height: dimensions.height.into(),
size,
})
}
}
#[derive(Debug, Clone, Copy)]
pub struct ThumbnailDownloader<'a> {
pub main: ImageSource<'a>,
pub alt: Option<ImageSource<'a>>,
}
impl ThumbnailDownloader<'_> {
pub async fn download(
self,
client: Client,
settings: ThumbnailSettings,
priority: ImageRequestPriority,
) -> Result<Image, ImageError> {
let dimensions = settings.dimensions;
let source = if let Some(alt) = self.alt {
if !self.main.can_be_thumbnailed()
&& (filesize_is_too_big(self.main.filesize())
|| alt.dimensions().is_some_and(|s| s.ge(settings.dimensions)))
{
alt
} else {
self.main
}
} else {
self.main
};
if source.should_thumbnail(settings.prefer_thumbnail, settings.dimensions) {
let request = MediaRequestParameters {
source: source.source.to_common_media_source(),
format: MediaFormat::Thumbnail(settings.into()),
};
let handle = IMAGE_QUEUE
.add_download_request(client.clone(), request, Some(dimensions), priority)
.await;
if let Ok(image) = handle.await {
return Ok(image);
}
}
let request = MediaRequestParameters {
source: source.source.to_common_media_source(),
format: MediaFormat::File,
};
let handle = IMAGE_QUEUE
.add_download_request(client, request, Some(dimensions), priority)
.await;
handle.await
}
}
#[derive(Debug, Clone, Copy)]
pub struct ImageSource<'a> {
pub source: MediaSource<'a>,
pub info: Option<ImageSourceInfo<'a>>,
}
impl ImageSource<'_> {
fn should_thumbnail(
&self,
prefer_thumbnail: bool,
thumbnail_dimensions: FrameDimensions,
) -> bool {
if !self.can_be_thumbnailed() {
return false;
}
if self.is_animated() {
return false;
}
let dimensions = self.dimensions();
if prefer_thumbnail && dimensions.is_none() {
return true;
}
dimensions.is_some_and(|d| d.needs_thumbnail(thumbnail_dimensions))
|| filesize_is_too_big(self.filesize())
}
fn can_be_thumbnailed(&self) -> bool {
!self.source.is_encrypted()
&& self
.info
.and_then(|i| i.mimetype)
.is_none_or(|m| m != SVG_CONTENT_TYPE)
}
fn filesize(&self) -> Option<u32> {
self.info.and_then(|i| i.filesize)
}
fn dimensions(&self) -> Option<FrameDimensions> {
self.info.and_then(|i| i.dimensions)
}
fn is_animated(&self) -> bool {
if self
.info
.and_then(|i| i.is_animated)
.is_none_or(|is_animated| !is_animated)
{
return false;
}
self.info
.and_then(|i| i.mimetype)
.is_some_and(|mimetype| SUPPORTED_ANIMATED_IMAGE_MIME_TYPES.contains(&mimetype))
}
}
fn filesize_is_too_big(filesize: Option<u32>) -> bool {
filesize.is_some_and(|s| s > THUMBNAIL_MAX_FILESIZE_THRESHOLD)
}
#[derive(Debug, Clone, Copy)]
pub enum MediaSource<'a> {
Common(&'a CommonMediaSource),
Sticker(&'a StickerMediaSource),
Uri(&'a OwnedMxcUri),
}
impl MediaSource<'_> {
fn is_encrypted(&self) -> bool {
match self {
Self::Common(source) => matches!(source, CommonMediaSource::Encrypted(_)),
Self::Sticker(source) => matches!(source, StickerMediaSource::Encrypted(_)),
Self::Uri(_) => false,
}
}
fn to_common_media_source(self) -> CommonMediaSource {
match self {
Self::Common(source) => source.clone(),
Self::Sticker(source) => source.clone().into(),
Self::Uri(uri) => CommonMediaSource::Plain(uri.clone()),
}
}
}
impl<'a> From<&'a CommonMediaSource> for MediaSource<'a> {
fn from(value: &'a CommonMediaSource) -> Self {
Self::Common(value)
}
}
impl<'a> From<&'a StickerMediaSource> for MediaSource<'a> {
fn from(value: &'a StickerMediaSource) -> Self {
Self::Sticker(value)
}
}
impl<'a> From<&'a OwnedMxcUri> for MediaSource<'a> {
fn from(value: &'a OwnedMxcUri) -> Self {
Self::Uri(value)
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct ImageSourceInfo<'a> {
dimensions: Option<FrameDimensions>,
mimetype: Option<&'a str>,
filesize: Option<u32>,
is_animated: Option<bool>,
}
impl<'a> From<&'a ImageInfo> for ImageSourceInfo<'a> {
fn from(value: &'a ImageInfo) -> Self {
Self {
dimensions: FrameDimensions::from_options(value.width, value.height),
mimetype: value.mimetype.as_deref(),
filesize: value.size.and_then(|u| u.try_into().ok()),
is_animated: value.is_animated,
}
}
}
impl<'a> From<&'a ThumbnailInfo> for ImageSourceInfo<'a> {
fn from(value: &'a ThumbnailInfo) -> Self {
Self {
dimensions: FrameDimensions::from_options(value.width, value.height),
mimetype: value.mimetype.as_deref(),
filesize: value.size.and_then(|u| u.try_into().ok()),
is_animated: None,
}
}
}
impl<'a> From<&'a AvatarImageInfo> for ImageSourceInfo<'a> {
fn from(value: &'a AvatarImageInfo) -> Self {
Self {
dimensions: FrameDimensions::from_options(value.width, value.height),
mimetype: value.mimetype.as_deref(),
filesize: value.size.and_then(|u| u.try_into().ok()),
is_animated: None,
}
}
}
#[derive(Debug, Clone)]
pub struct ThumbnailSettings {
pub dimensions: FrameDimensions,
pub method: Method,
pub animated: bool,
pub prefer_thumbnail: bool,
}
impl From<ThumbnailSettings> for MediaThumbnailSettings {
fn from(value: ThumbnailSettings) -> Self {
let ThumbnailSettings {
dimensions,
method,
animated,
..
} = value;
MediaThumbnailSettings {
method,
width: dimensions.width.into(),
height: dimensions.height.into(),
animated,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ImageError {
Download,
File,
UnsupportedFormat,
Io,
Unknown,
Aborted,
}
impl Error for ImageError {}
impl fmt::Display for ImageError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let s = match self {
Self::Download => gettext("Could not retrieve media"),
Self::UnsupportedFormat => gettext("Image format not supported"),
Self::File | Self::Io | Self::Unknown | Self::Aborted => {
gettext("An unexpected error occurred")
}
};
f.write_str(&s)
}
}
impl From<MediaFileError> for ImageError {
fn from(value: MediaFileError) -> Self {
match value {
MediaFileError::Sdk(_) => Self::Download,
MediaFileError::File(_) => Self::File,
}
}
}
impl From<glycin::ErrorCtx> for ImageError {
fn from(value: glycin::ErrorCtx) -> Self {
if value.unsupported_format().is_some() {
Self::UnsupportedFormat
} else if matches!(value.error(), glycin::Error::StdIoError { .. }) {
Self::Io
} else {
Self::Unknown
}
}
}