// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System.Runtime.InteropServices;
using SkiaSharp;
namespace Microsoft.Maui.Platform.Linux.Services;
///
/// Supported hardware video acceleration APIs.
///
public enum VideoAccelerationApi
{
///
/// Automatically select the best available API.
///
Auto,
///
/// VA-API (Video Acceleration API) - Intel, AMD, and some NVIDIA.
///
VaApi,
///
/// VDPAU (Video Decode and Presentation API for Unix) - NVIDIA.
///
Vdpau,
///
/// Software decoding fallback.
///
Software
}
///
/// Video codec profiles supported by hardware acceleration.
///
public enum VideoProfile
{
H264Baseline,
H264Main,
H264High,
H265Main,
H265Main10,
Vp8,
Vp9Profile0,
Vp9Profile2,
Av1Main
}
///
/// Information about a decoded video frame.
///
public class VideoFrame : IDisposable
{
public int Width { get; init; }
public int Height { get; init; }
public IntPtr DataY { get; init; }
public IntPtr DataU { get; init; }
public IntPtr DataV { get; init; }
public int StrideY { get; init; }
public int StrideU { get; init; }
public int StrideV { get; init; }
public long Timestamp { get; init; }
public bool IsKeyFrame { get; init; }
private bool _disposed;
private Action? _releaseCallback;
internal void SetReleaseCallback(Action callback) => _releaseCallback = callback;
public void Dispose()
{
if (!_disposed)
{
_releaseCallback?.Invoke();
_disposed = true;
}
}
}
///
/// Hardware-accelerated video decoding service using VA-API or VDPAU.
/// Provides efficient video decode for media playback on Linux.
///
public class HardwareVideoService : IDisposable
{
#region VA-API Native Interop
private const string LibVa = "libva.so.2";
private const string LibVaDrm = "libva-drm.so.2";
private const string LibVaX11 = "libva-x11.so.2";
// VA-API error codes
private const int VA_STATUS_SUCCESS = 0;
// VA-API profile constants
private const int VAProfileH264Baseline = 5;
private const int VAProfileH264Main = 6;
private const int VAProfileH264High = 7;
private const int VAProfileHEVCMain = 12;
private const int VAProfileHEVCMain10 = 13;
private const int VAProfileVP8Version0_3 = 14;
private const int VAProfileVP9Profile0 = 15;
private const int VAProfileVP9Profile2 = 17;
private const int VAProfileAV1Profile0 = 20;
// VA-API entrypoint
private const int VAEntrypointVLD = 1; // Video Decode
// Surface formats
private const uint VA_RT_FORMAT_YUV420 = 0x00000001;
private const uint VA_RT_FORMAT_YUV420_10 = 0x00000100;
[DllImport(LibVa)]
private static extern IntPtr vaGetDisplayDRM(int fd);
[DllImport(LibVaX11)]
private static extern IntPtr vaGetDisplay(IntPtr x11Display);
[DllImport(LibVa)]
private static extern int vaInitialize(IntPtr display, out int majorVersion, out int minorVersion);
[DllImport(LibVa)]
private static extern int vaTerminate(IntPtr display);
[DllImport(LibVa)]
private static extern IntPtr vaErrorStr(int errorCode);
[DllImport(LibVa)]
private static extern int vaQueryConfigProfiles(IntPtr display, [Out] int[] profileList, out int numProfiles);
[DllImport(LibVa)]
private static extern int vaQueryConfigEntrypoints(IntPtr display, int profile, [Out] int[] entrypoints, out int numEntrypoints);
[DllImport(LibVa)]
private static extern int vaCreateConfig(IntPtr display, int profile, int entrypoint, IntPtr attribList, int numAttribs, out uint configId);
[DllImport(LibVa)]
private static extern int vaDestroyConfig(IntPtr display, uint configId);
[DllImport(LibVa)]
private static extern int vaCreateContext(IntPtr display, uint configId, int pictureWidth, int pictureHeight, int flag, IntPtr renderTargets, int numRenderTargets, out uint contextId);
[DllImport(LibVa)]
private static extern int vaDestroyContext(IntPtr display, uint contextId);
[DllImport(LibVa)]
private static extern int vaCreateSurfaces(IntPtr display, uint format, uint width, uint height, [Out] uint[] surfaces, uint numSurfaces, IntPtr attribList, uint numAttribs);
[DllImport(LibVa)]
private static extern int vaDestroySurfaces(IntPtr display, [In] uint[] surfaces, int numSurfaces);
[DllImport(LibVa)]
private static extern int vaSyncSurface(IntPtr display, uint surfaceId);
[DllImport(LibVa)]
private static extern int vaMapBuffer(IntPtr display, uint bufferId, out IntPtr data);
[DllImport(LibVa)]
private static extern int vaUnmapBuffer(IntPtr display, uint bufferId);
[DllImport(LibVa)]
private static extern int vaDeriveImage(IntPtr display, uint surfaceId, out VaImage image);
[DllImport(LibVa)]
private static extern int vaDestroyImage(IntPtr display, uint imageId);
[StructLayout(LayoutKind.Sequential)]
private struct VaImage
{
public uint ImageId;
public uint Format; // VAImageFormat (simplified)
public uint FormatFourCC;
public int Width;
public int Height;
public uint DataSize;
public uint NumPlanes;
public uint PitchesPlane0;
public uint PitchesPlane1;
public uint PitchesPlane2;
public uint PitchesPlane3;
public uint OffsetsPlane0;
public uint OffsetsPlane1;
public uint OffsetsPlane2;
public uint OffsetsPlane3;
public uint BufferId;
}
#endregion
#region VDPAU Native Interop
private const string LibVdpau = "libvdpau.so.1";
[DllImport(LibVdpau)]
private static extern int vdp_device_create_x11(IntPtr display, int screen, out IntPtr device, out IntPtr getProcAddress);
#endregion
#region DRM Interop
[DllImport("libc", EntryPoint = "open")]
private static extern int open([MarshalAs(UnmanagedType.LPStr)] string path, int flags);
[DllImport("libc", EntryPoint = "close")]
private static extern int close(int fd);
private const int O_RDWR = 2;
#endregion
#region Fields
private IntPtr _vaDisplay;
private uint _vaConfigId;
private uint _vaContextId;
private uint[] _vaSurfaces = Array.Empty();
private int _drmFd = -1;
private bool _initialized;
private bool _disposed;
private VideoAccelerationApi _currentApi = VideoAccelerationApi.Software;
private int _width;
private int _height;
private VideoProfile _profile;
private readonly HashSet _supportedProfiles = new();
private readonly object _lock = new();
#endregion
#region Properties
///
/// Gets the currently active video acceleration API.
///
public VideoAccelerationApi CurrentApi => _currentApi;
///
/// Gets whether hardware acceleration is available and initialized.
///
public bool IsHardwareAccelerated => _currentApi != VideoAccelerationApi.Software && _initialized;
///
/// Gets the supported video profiles.
///
public IReadOnlySet SupportedProfiles => _supportedProfiles;
#endregion
#region Initialization
///
/// Creates a new hardware video service.
///
public HardwareVideoService()
{
}
///
/// Initializes the hardware video acceleration.
///
/// The preferred API to use.
/// Optional X11 display for VA-API X11 backend.
/// True if initialization succeeded.
public bool Initialize(VideoAccelerationApi api = VideoAccelerationApi.Auto, IntPtr x11Display = default)
{
if (_initialized)
return true;
lock (_lock)
{
if (_initialized)
return true;
// Try VA-API first (works with Intel, AMD, and some NVIDIA)
if (api == VideoAccelerationApi.Auto || api == VideoAccelerationApi.VaApi)
{
if (TryInitializeVaApi(x11Display))
{
_currentApi = VideoAccelerationApi.VaApi;
_initialized = true;
Console.WriteLine($"[HardwareVideo] Initialized VA-API with {_supportedProfiles.Count} supported profiles");
return true;
}
}
// Try VDPAU (NVIDIA proprietary)
if (api == VideoAccelerationApi.Auto || api == VideoAccelerationApi.Vdpau)
{
if (TryInitializeVdpau(x11Display))
{
_currentApi = VideoAccelerationApi.Vdpau;
_initialized = true;
Console.WriteLine("[HardwareVideo] Initialized VDPAU");
return true;
}
}
Console.WriteLine("[HardwareVideo] No hardware acceleration available, using software");
_currentApi = VideoAccelerationApi.Software;
return false;
}
}
private bool TryInitializeVaApi(IntPtr x11Display)
{
try
{
// Try DRM backend first (works in Wayland and headless)
string[] drmDevices = { "/dev/dri/renderD128", "/dev/dri/renderD129", "/dev/dri/card0" };
foreach (var device in drmDevices)
{
_drmFd = open(device, O_RDWR);
if (_drmFd >= 0)
{
_vaDisplay = vaGetDisplayDRM(_drmFd);
if (_vaDisplay != IntPtr.Zero)
{
if (InitializeVaDisplay())
return true;
}
close(_drmFd);
_drmFd = -1;
}
}
// Fall back to X11 backend if display provided
if (x11Display != IntPtr.Zero)
{
_vaDisplay = vaGetDisplay(x11Display);
if (_vaDisplay != IntPtr.Zero && InitializeVaDisplay())
return true;
}
return false;
}
catch (DllNotFoundException)
{
Console.WriteLine("[HardwareVideo] VA-API libraries not found");
return false;
}
catch (Exception ex)
{
Console.WriteLine($"[HardwareVideo] VA-API initialization failed: {ex.Message}");
return false;
}
}
private bool InitializeVaDisplay()
{
int status = vaInitialize(_vaDisplay, out int major, out int minor);
if (status != VA_STATUS_SUCCESS)
{
Console.WriteLine($"[HardwareVideo] vaInitialize failed: {GetVaError(status)}");
return false;
}
Console.WriteLine($"[HardwareVideo] VA-API {major}.{minor} initialized");
// Query supported profiles
int[] profiles = new int[32];
status = vaQueryConfigProfiles(_vaDisplay, profiles, out int numProfiles);
if (status == VA_STATUS_SUCCESS)
{
for (int i = 0; i < numProfiles; i++)
{
if (TryMapVaProfile(profiles[i], out var videoProfile))
{
// Check if VLD (decode) entrypoint is supported
int[] entrypoints = new int[8];
if (vaQueryConfigEntrypoints(_vaDisplay, profiles[i], entrypoints, out int numEntrypoints) == VA_STATUS_SUCCESS)
{
for (int j = 0; j < numEntrypoints; j++)
{
if (entrypoints[j] == VAEntrypointVLD)
{
_supportedProfiles.Add(videoProfile);
break;
}
}
}
}
}
}
return true;
}
private bool TryInitializeVdpau(IntPtr x11Display)
{
if (x11Display == IntPtr.Zero)
return false;
try
{
int result = vdp_device_create_x11(x11Display, 0, out IntPtr device, out IntPtr getProcAddress);
if (result == 0 && device != IntPtr.Zero)
{
// VDPAU initialized - would need additional setup for actual use
// For now, just mark as available
_supportedProfiles.Add(VideoProfile.H264Baseline);
_supportedProfiles.Add(VideoProfile.H264Main);
_supportedProfiles.Add(VideoProfile.H264High);
return true;
}
}
catch (DllNotFoundException)
{
Console.WriteLine("[HardwareVideo] VDPAU libraries not found");
}
catch (Exception ex)
{
Console.WriteLine($"[HardwareVideo] VDPAU initialization failed: {ex.Message}");
}
return false;
}
#endregion
#region Decoder Creation
///
/// Creates a decoder context for the specified profile and dimensions.
///
public bool CreateDecoder(VideoProfile profile, int width, int height)
{
if (!_initialized || _currentApi == VideoAccelerationApi.Software)
return false;
if (!_supportedProfiles.Contains(profile))
{
Console.WriteLine($"[HardwareVideo] Profile {profile} not supported");
return false;
}
lock (_lock)
{
// Destroy existing context
DestroyDecoder();
_width = width;
_height = height;
_profile = profile;
if (_currentApi == VideoAccelerationApi.VaApi)
return CreateVaApiDecoder(profile, width, height);
return false;
}
}
private bool CreateVaApiDecoder(VideoProfile profile, int width, int height)
{
int vaProfile = MapToVaProfile(profile);
// Create config
int status = vaCreateConfig(_vaDisplay, vaProfile, VAEntrypointVLD, IntPtr.Zero, 0, out _vaConfigId);
if (status != VA_STATUS_SUCCESS)
{
Console.WriteLine($"[HardwareVideo] vaCreateConfig failed: {GetVaError(status)}");
return false;
}
// Create surfaces for decoded frames (use a pool of 8)
uint format = profile == VideoProfile.H265Main10 || profile == VideoProfile.Vp9Profile2
? VA_RT_FORMAT_YUV420_10
: VA_RT_FORMAT_YUV420;
_vaSurfaces = new uint[8];
status = vaCreateSurfaces(_vaDisplay, format, (uint)width, (uint)height, _vaSurfaces, 8, IntPtr.Zero, 0);
if (status != VA_STATUS_SUCCESS)
{
Console.WriteLine($"[HardwareVideo] vaCreateSurfaces failed: {GetVaError(status)}");
vaDestroyConfig(_vaDisplay, _vaConfigId);
return false;
}
// Create context
status = vaCreateContext(_vaDisplay, _vaConfigId, width, height, 0, IntPtr.Zero, 0, out _vaContextId);
if (status != VA_STATUS_SUCCESS)
{
Console.WriteLine($"[HardwareVideo] vaCreateContext failed: {GetVaError(status)}");
vaDestroySurfaces(_vaDisplay, _vaSurfaces, _vaSurfaces.Length);
vaDestroyConfig(_vaDisplay, _vaConfigId);
return false;
}
Console.WriteLine($"[HardwareVideo] Created decoder: {profile} {width}x{height}");
return true;
}
///
/// Destroys the current decoder context.
///
public void DestroyDecoder()
{
lock (_lock)
{
if (_currentApi == VideoAccelerationApi.VaApi && _vaDisplay != IntPtr.Zero)
{
if (_vaContextId != 0)
{
vaDestroyContext(_vaDisplay, _vaContextId);
_vaContextId = 0;
}
if (_vaSurfaces.Length > 0)
{
vaDestroySurfaces(_vaDisplay, _vaSurfaces, _vaSurfaces.Length);
_vaSurfaces = Array.Empty();
}
if (_vaConfigId != 0)
{
vaDestroyConfig(_vaDisplay, _vaConfigId);
_vaConfigId = 0;
}
}
}
}
#endregion
#region Frame Retrieval
///
/// Retrieves a decoded frame from the specified surface.
///
public VideoFrame? GetDecodedFrame(int surfaceIndex, long timestamp, bool isKeyFrame)
{
if (!_initialized || _currentApi != VideoAccelerationApi.VaApi)
return null;
if (surfaceIndex < 0 || surfaceIndex >= _vaSurfaces.Length)
return null;
uint surfaceId = _vaSurfaces[surfaceIndex];
// Wait for decode to complete
int status = vaSyncSurface(_vaDisplay, surfaceId);
if (status != VA_STATUS_SUCCESS)
return null;
// Derive image from surface
status = vaDeriveImage(_vaDisplay, surfaceId, out VaImage image);
if (status != VA_STATUS_SUCCESS)
return null;
// Map the buffer
status = vaMapBuffer(_vaDisplay, image.BufferId, out IntPtr data);
if (status != VA_STATUS_SUCCESS)
{
vaDestroyImage(_vaDisplay, image.ImageId);
return null;
}
var frame = new VideoFrame
{
Width = image.Width,
Height = image.Height,
DataY = data + (int)image.OffsetsPlane0,
DataU = data + (int)image.OffsetsPlane1,
DataV = data + (int)image.OffsetsPlane2,
StrideY = (int)image.PitchesPlane0,
StrideU = (int)image.PitchesPlane1,
StrideV = (int)image.PitchesPlane2,
Timestamp = timestamp,
IsKeyFrame = isKeyFrame
};
// Set cleanup callback
frame.SetReleaseCallback(() =>
{
vaUnmapBuffer(_vaDisplay, image.BufferId);
vaDestroyImage(_vaDisplay, image.ImageId);
});
return frame;
}
///
/// Converts a decoded frame to an SKBitmap for display.
///
public SKBitmap? ConvertFrameToSkia(VideoFrame frame)
{
if (frame == null)
return null;
// Create BGRA bitmap
var bitmap = new SKBitmap(frame.Width, frame.Height, SKColorType.Bgra8888, SKAlphaType.Opaque);
// Convert YUV to BGRA
unsafe
{
byte* yPtr = (byte*)frame.DataY;
byte* uPtr = (byte*)frame.DataU;
byte* vPtr = (byte*)frame.DataV;
byte* dst = (byte*)bitmap.GetPixels();
for (int y = 0; y < frame.Height; y++)
{
for (int x = 0; x < frame.Width; x++)
{
int yIndex = y * frame.StrideY + x;
int uvIndex = (y / 2) * frame.StrideU + (x / 2);
int yVal = yPtr[yIndex];
int uVal = uPtr[uvIndex] - 128;
int vVal = vPtr[uvIndex] - 128;
// YUV to RGB conversion
int r = (int)(yVal + 1.402 * vVal);
int g = (int)(yVal - 0.344 * uVal - 0.714 * vVal);
int b = (int)(yVal + 1.772 * uVal);
r = Math.Clamp(r, 0, 255);
g = Math.Clamp(g, 0, 255);
b = Math.Clamp(b, 0, 255);
int dstIndex = (y * frame.Width + x) * 4;
dst[dstIndex] = (byte)b;
dst[dstIndex + 1] = (byte)g;
dst[dstIndex + 2] = (byte)r;
dst[dstIndex + 3] = 255;
}
}
}
return bitmap;
}
#endregion
#region Helpers
private static bool TryMapVaProfile(int vaProfile, out VideoProfile profile)
{
profile = vaProfile switch
{
VAProfileH264Baseline => VideoProfile.H264Baseline,
VAProfileH264Main => VideoProfile.H264Main,
VAProfileH264High => VideoProfile.H264High,
VAProfileHEVCMain => VideoProfile.H265Main,
VAProfileHEVCMain10 => VideoProfile.H265Main10,
VAProfileVP8Version0_3 => VideoProfile.Vp8,
VAProfileVP9Profile0 => VideoProfile.Vp9Profile0,
VAProfileVP9Profile2 => VideoProfile.Vp9Profile2,
VAProfileAV1Profile0 => VideoProfile.Av1Main,
_ => VideoProfile.H264Main
};
return vaProfile >= VAProfileH264Baseline && vaProfile <= VAProfileAV1Profile0;
}
private static int MapToVaProfile(VideoProfile profile)
{
return profile switch
{
VideoProfile.H264Baseline => VAProfileH264Baseline,
VideoProfile.H264Main => VAProfileH264Main,
VideoProfile.H264High => VAProfileH264High,
VideoProfile.H265Main => VAProfileHEVCMain,
VideoProfile.H265Main10 => VAProfileHEVCMain10,
VideoProfile.Vp8 => VAProfileVP8Version0_3,
VideoProfile.Vp9Profile0 => VAProfileVP9Profile0,
VideoProfile.Vp9Profile2 => VAProfileVP9Profile2,
VideoProfile.Av1Main => VAProfileAV1Profile0,
_ => VAProfileH264Main
};
}
private static string GetVaError(int status)
{
try
{
IntPtr errPtr = vaErrorStr(status);
return Marshal.PtrToStringAnsi(errPtr) ?? $"Unknown error {status}";
}
catch
{
return $"Error code {status}";
}
}
#endregion
#region IDisposable
public void Dispose()
{
if (_disposed)
return;
_disposed = true;
DestroyDecoder();
if (_currentApi == VideoAccelerationApi.VaApi && _vaDisplay != IntPtr.Zero)
{
vaTerminate(_vaDisplay);
_vaDisplay = IntPtr.Zero;
}
if (_drmFd >= 0)
{
close(_drmFd);
_drmFd = -1;
}
GC.SuppressFinalize(this);
}
~HardwareVideoService()
{
Dispose();
}
#endregion
}