From 17e4746687abc53f0a042620a7af6dd6cea44b80 Mon Sep 17 00:00:00 2001
From: Timo Rothenpieler <timo@rothenpieler.org>
Date: Thu, 5 Dec 2024 20:57:46 +0100
Subject: [PATCH] avcodec/libx265: add alpha layer encoding support

---
 Changelog            |  1 +
 libavcodec/libx265.c | 50 +++++++++++++++++++++++++++++++-------------
 libavcodec/version.h |  2 +-
 3 files changed, 37 insertions(+), 16 deletions(-)

diff --git a/Changelog b/Changelog
index a744faeb29..b78e61a617 100644
--- a/Changelog
+++ b/Changelog
@@ -6,6 +6,7 @@ version <next>:
 - VVC VAAPI decoder
 - RealVideo 6.0 decoder
 - OpenMAX encoders deprecated
+- libx265 alpha layer encoding
 
 version 7.1:
 - Raw Captions with Time (RCWT) closed caption demuxer
diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c
index 63cc497f83..aa9b9e4c46 100644
--- a/libavcodec/libx265.c
+++ b/libavcodec/libx265.c
@@ -42,6 +42,14 @@
 #include "atsc_a53.h"
 #include "sei.h"
 
+#if defined(X265_ENABLE_ALPHA) && MAX_LAYERS > 2
+#define FF_X265_MAX_LAYERS MAX_LAYERS
+#elif X265_BUILD >= 210
+#define FF_X265_MAX_LAYERS 2
+#else
+#define FF_X265_MAX_LAYERS 1
+#endif
+
 typedef struct ReorderedData {
     int64_t duration;
 
@@ -539,6 +547,15 @@ FF_ENABLE_DEPRECATION_WARNINGS
                                 ctx->dovi.cfg.dv_bl_signal_compatibility_id;
 #endif
 
+#if X265_BUILD >= 210 && FF_X265_MAX_LAYERS > 1
+    if (desc->flags & AV_PIX_FMT_FLAG_ALPHA) {
+        if (ctx->api->param_parse(ctx->params, "alpha", "1") < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Loaded libx265 does not support alpha layer encoding.\n");
+            return AVERROR(ENOTSUP);
+        }
+    }
+#endif
+
     ctx->encoder = ctx->api->encoder_open(ctx->params);
     if (!ctx->encoder) {
         av_log(avctx, AV_LOG_ERROR, "Cannot open libx265 encoder.\n");
@@ -659,15 +676,13 @@ static void free_picture(libx265Context *ctx, x265_picture *pic)
 static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                                 const AVFrame *pic, int *got_packet)
 {
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->pix_fmt);
     libx265Context *ctx = avctx->priv_data;
     x265_picture x265pic;
+    x265_picture x265pic_out[FF_X265_MAX_LAYERS] = { 0 };
 #if (X265_BUILD >= 210) && (X265_BUILD < 213)
-    x265_picture x265pic_layers_out[MAX_SCALABLE_LAYERS];
-    x265_picture* x265pic_lyrptr_out[MAX_SCALABLE_LAYERS];
-#else
-    x265_picture x265pic_solo_out = { 0 };
+    x265_picture *x265pic_lyrptr_out[FF_X265_MAX_LAYERS];
 #endif
-    x265_picture* x265pic_out;
     x265_nal *nal;
     x265_sei *sei;
     uint8_t *dst;
@@ -687,7 +702,7 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
         ReorderedData *rd;
         int rd_idx;
 
-        for (i = 0; i < 3; i++) {
+        for (i = 0; i < desc->nb_components; i++) {
            x265pic.planes[i] = pic->data[i];
            x265pic.stride[i] = pic->linesize[i];
         }
@@ -806,14 +821,14 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     }
 
 #if (X265_BUILD >= 210) && (X265_BUILD < 213)
-    for (i = 0; i < MAX_SCALABLE_LAYERS; i++)
-        x265pic_lyrptr_out[i] = &x265pic_layers_out[i];
+    for (i = 0; i < FF_ARRAY_ELEMS(x265pic_out); i++)
+        x265pic_lyrptr_out[i] = &x265pic_out[i];
 
     ret = ctx->api->encoder_encode(ctx->encoder, &nal, &nnal,
                                    pic ? &x265pic : NULL, x265pic_lyrptr_out);
 #else
     ret = ctx->api->encoder_encode(ctx->encoder, &nal, &nnal,
-                                   pic ? &x265pic : NULL, &x265pic_solo_out);
+                                   pic ? &x265pic : NULL, x265pic_out);
 #endif
 
     for (i = 0; i < sei->numPayloads; i++)
@@ -844,12 +859,6 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
             pkt->flags |= AV_PKT_FLAG_KEY;
     }
 
-#if (X265_BUILD >= 210) && (X265_BUILD < 213)
-    x265pic_out = x265pic_lyrptr_out[0];
-#else
-    x265pic_out = &x265pic_solo_out;
-#endif
-
     pkt->pts = x265pic_out->pts;
     pkt->dts = x265pic_out->dts;
 
@@ -907,6 +916,9 @@ static const enum AVPixelFormat x265_csp_eight[] = {
     AV_PIX_FMT_YUVJ444P,
     AV_PIX_FMT_GBRP,
     AV_PIX_FMT_GRAY8,
+#if X265_BUILD >= 210 && FF_X265_MAX_LAYERS > 1
+    AV_PIX_FMT_YUVA420P,
+#endif
     AV_PIX_FMT_NONE
 };
 
@@ -924,6 +936,10 @@ static const enum AVPixelFormat x265_csp_ten[] = {
     AV_PIX_FMT_GBRP10,
     AV_PIX_FMT_GRAY8,
     AV_PIX_FMT_GRAY10,
+#if X265_BUILD >= 210 && FF_X265_MAX_LAYERS > 1
+    AV_PIX_FMT_YUVA420P,
+    AV_PIX_FMT_YUVA420P10,
+#endif
     AV_PIX_FMT_NONE
 };
 
@@ -946,6 +962,10 @@ static const enum AVPixelFormat x265_csp_twelve[] = {
     AV_PIX_FMT_GRAY8,
     AV_PIX_FMT_GRAY10,
     AV_PIX_FMT_GRAY12,
+#if X265_BUILD >= 210 && FF_X265_MAX_LAYERS > 1
+    AV_PIX_FMT_YUVA420P,
+    AV_PIX_FMT_YUVA420P10,
+#endif
     AV_PIX_FMT_NONE
 };
 
diff --git a/libavcodec/version.h b/libavcodec/version.h
index 735c8b813c..eee1508084 100644
--- a/libavcodec/version.h
+++ b/libavcodec/version.h
@@ -30,7 +30,7 @@
 #include "version_major.h"
 
 #define LIBAVCODEC_VERSION_MINOR  27
-#define LIBAVCODEC_VERSION_MICRO 100
+#define LIBAVCODEC_VERSION_MICRO 101
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \