...

ffmpeg/libavcodec/aarch64/vvc/Makefile

@@ -0,0 +1,10 @@
+clean::
+	$(RM) $(CLEANSUFFIXES:%=libavcodec/aarch64/vvc/%)
+
+OBJS-$(CONFIG_VVC_DECODER)              += aarch64/vvc/dsp_init.o
+NEON-OBJS-$(CONFIG_VVC_DECODER)         += aarch64/vvc/alf.o \
+                                           aarch64/vvc/inter.o \
+                                           aarch64/vvc/sad.o \
+                                           aarch64/h26x/epel_neon.o \
+                                           aarch64/h26x/qpel_neon.o \
+                                           aarch64/h26x/sao_neon.o

ffmpeg/libavcodec/aarch64/vvc/alf.S

@@ -0,0 +1,293 @@
+/*
+ * Copyright (c) 2024 Zhao Zhili <quinkblack@foxmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/aarch64/asm.S"
+
+.macro alf_luma_filter_pixel index, pix_size, addr1, addr2, offset1, offset2
+    .if \pix_size == 1
+        ldur            d3, [\addr1, #\offset1]
+        ldur            d4, [\addr2, #\offset2]
+        uxtl            v6.8h, v3.8b
+        uxtl            v7.8h, v4.8b
+    .else
+        ldur            q6, [\addr1, #(2*\offset1)]
+        ldur            q7, [\addr2, #(2*\offset2)]
+    .endif
+    .if \index < 8
+        dup             v17.4h, v0.h[\index]        // clip
+        dup             v18.4h, v16.h[\index]       // -clip
+        dup             v19.4h, v1.h[\index]        // filter
+
+        dup             v26.4h, v22.h[\index]       // clip
+        dup             v27.4h, v23.h[\index]       // -clip
+        dup             v28.4h, v24.h[\index]       // filter
+    .else
+        dup             v17.4h, v0.h[\index - 8]    // clip
+        dup             v18.4h, v16.h[\index - 8]   // -clip
+        dup             v19.4h, v1.h[\index - 8]    // filter
+
+        dup             v26.4h, v22.h[\index - 8]   // clip
+        dup             v27.4h, v23.h[\index - 8]   // -clip
+        dup             v28.4h, v24.h[\index - 8]   // filter
+    .endif
+        ins             v17.d[1], v26.d[0]
+        ins             v18.d[1], v27.d[0]
+        ins             v19.d[1], v28.d[0]
+
+        sub             v6.8h, v6.8h, v5.8h
+        sub             v7.8h, v7.8h, v5.8h
+        smin            v6.8h, v6.8h, v17.8h
+        smin            v7.8h, v7.8h, v17.8h
+        smax            v6.8h, v6.8h, v18.8h
+        smax            v7.8h, v7.8h, v18.8h
+        add             v6.8h, v6.8h, v7.8h
+        smlal           v20.4s, v19.4h, v6.4h       // v20: sum
+        smlal2          v21.4s, v19.8h, v6.8h       // v21: sum
+.endm
+
+/* x0: dst
+ * x1: pp
+ * x2: filter
+ * x3: clip
+ * w4: is_near_vb
+ * w5: pix_max
+ */
+.macro alf_filter_luma_kernel, pix_size
+        dst             .req x0
+        pp              .req x1
+        filter          .req x2
+        clip            .req x3
+        is_near_vb      .req w4
+        pix_max         .req w5
+    .if \pix_size > 1
+        dup             v25.8h, pix_max             // pix_max
+    .endif
+        ldr             q0, [clip]                  // clip
+        ldr             q1, [filter]                // filter
+        ldur            q22, [clip, #24]            // clip
+        ldur            q24, [filter, #24]          // filter
+
+        ldr             x5, [pp]                    // x5: p0
+        ldr             x6, [pp, #(5*8)]            // x6: p5
+        ldr             x7, [pp, #(6*8)]            // x7: p6
+        neg             v16.8h, v0.8h               // -clip
+        neg             v23.8h, v22.8h              // -clip
+
+    .if \pix_size == 1
+        ldr             d2, [x5]                    // curr
+    .else
+        ldr             q5, [x5]                    // curr
+    .endif
+        movi            v20.4s, #64
+        cbz             is_near_vb, 1f
+        shl             v20.4s, v20.4s, #3
+1:
+    .if \pix_size == 1
+        uxtl            v5.8h, v2.8b
+    .endif
+        mov             v21.16b, v20.16b
+        ldr             x8, [pp, #(3*8)]            // p3
+        ldr             x9, [pp, #(4*8)]            // p4
+        alf_luma_filter_pixel 0, \pix_size, x6, x7, 0, 0
+
+        ldr             x6, [pp, #(1*8)]            // p1
+        ldr             x7, [pp, #(2*8)]            // p2
+        alf_luma_filter_pixel 1, \pix_size, x8, x9, 1, -1
+        alf_luma_filter_pixel 2, \pix_size, x8, x9, 0, 0
+        alf_luma_filter_pixel 3, \pix_size, x8, x9, -1, 1
+
+        alf_luma_filter_pixel 4, \pix_size, x6, x7, 2, -2
+        alf_luma_filter_pixel 5, \pix_size, x6, x7, 1, -1
+        alf_luma_filter_pixel 6, \pix_size, x6, x7, 0, 0
+        alf_luma_filter_pixel 7, \pix_size, x6, x7, -1, 1
+
+        ldr             d0, [clip, #16]             // clip
+        ldr             d1, [filter, #16]           // filter
+        neg             v16.4h, v0.4h               // -clip
+
+        ldr             d22, [clip, #40]            // clip
+        ldr             d24, [filter, #40]          // filter
+        neg             v23.4h, v22.4h              // -clip
+        alf_luma_filter_pixel 8, \pix_size, x6, x7, -2, 2
+        alf_luma_filter_pixel 9, \pix_size, x5, x5, 3, -3
+        alf_luma_filter_pixel 10, \pix_size, x5, x5, 2, -2
+        alf_luma_filter_pixel 11, \pix_size, x5, x5, 1, -1
+
+        cbz             is_near_vb, 2f
+        sshr            v20.4s, v20.4s, #10
+        sshr            v21.4s, v21.4s, #10
+        b               3f
+2:
+        sshr            v20.4s, v20.4s, #7
+        sshr            v21.4s, v21.4s, #7
+3:
+        uxtl            v22.4s, v5.4h
+        uxtl2           v23.4s, v5.8h
+        add             v20.4s, v20.4s, v22.4s
+        add             v21.4s, v21.4s, v23.4s
+        sqxtun          v20.4h, v20.4s
+        sqxtun2         v20.8h, v21.4s
+    .if \pix_size == 1
+        sqxtun          v20.8b, v20.8h
+        str             d20, [dst]
+    .else
+        umin            v20.8h, v20.8h, v25.8h
+        str             q20, [dst]
+    .endif
+        ret
+
+        .unreq          dst
+        .unreq          pp
+        .unreq          filter
+        .unreq          clip
+        .unreq          is_near_vb
+        .unreq          pix_max
+.endm
+
+.macro alf_chroma_filter_pixel index, pix_size, addr1, addr2, offset1, offset2
+    .if \pix_size == 1
+        ldur            s3, [\addr1, #\offset1]
+        ldur            s4, [\addr2, #\offset2]
+        uxtl            v6.8h, v3.8b
+        uxtl            v7.8h, v4.8b
+    .else
+        ldur            d6, [\addr1, #(2*\offset1)]
+        ldur            d7, [\addr2, #(2*\offset2)]
+    .endif
+    .if \index < 8
+        dup             v17.4h, v0.h[\index]        // v17: clip[0]
+        dup             v18.4h, v16.h[\index]       // v18: -clip[0]
+        dup             v19.4h, v1.h[\index]        // v19: filter[0]
+    .else
+        dup             v17.4h, v0.h[\index - 8]    // v17: clip[0]
+        dup             v18.4h, v16.h[\index - 8]   // v18: -clip[0]
+        dup             v19.4h, v1.h[\index - 8]    // v19: filter[0]
+    .endif
+
+        sub             v6.4h, v6.4h, v5.4h
+        sub             v7.4h, v7.4h, v5.4h
+        smin            v6.4h, v6.4h, v17.4h
+        smin            v7.4h, v7.4h, v17.4h
+        smax            v6.4h, v6.4h, v18.4h
+        smax            v7.4h, v7.4h, v18.4h
+        add             v6.4h, v6.4h, v7.4h
+        smlal           v20.4s, v19.4h, v6.4h       // v20: sum
+.endm
+
+/* x0: dst
+ * x1: pp
+ * x2: filter
+ * x3: clip
+ * w4: is_near_vb
+ * w5: pix_max
+ */
+.macro alf_filter_chroma_kernel, pix_size
+        dst             .req x0
+        pp              .req x1
+        filter          .req x2
+        clip            .req x3
+        is_near_vb      .req w4
+        pix_max         .req w5
+    .if \pix_size > 1
+        dup             v25.4h, pix_max             // pix_max
+    .endif
+        ldr             q0, [clip]                  // clip
+        ldr             q1, [filter]                // filter
+        ldr             x5, [pp]                    // p0
+        ldr             x6, [pp, #(3*8)]            // p3
+        ldr             x7, [pp, #(4*8)]            // p4
+        neg             v16.8h, v0.8h               // -clip
+
+    .if \pix_size == 1
+        ldr             s2, [x5]                    // curr
+    .else
+        ldr             d5, [x5]                    // curr
+    .endif
+        movi            v20.4s, #64
+        cbz             is_near_vb, 1f
+        shl             v20.4s, v20.4s, #3
+1:
+    .if \pix_size == 1
+        uxtl            v5.8h, v2.8b
+    .endif
+        ldr             x8, [pp, #(1*8)]            // p1
+        ldr             x9, [pp, #(2*8)]            // p2
+        alf_chroma_filter_pixel 0, \pix_size, x6, x7, 0, 0
+        alf_chroma_filter_pixel 1, \pix_size, x8, x9, 1, -1
+        alf_chroma_filter_pixel 2, \pix_size, x8, x9, 0, 0
+        alf_chroma_filter_pixel 3, \pix_size, x8, x9, -1, 1
+        alf_chroma_filter_pixel 4, \pix_size, x5, x5, 2, -2
+        alf_chroma_filter_pixel 5, \pix_size, x5, x5, 1, -1
+
+        uxtl            v22.4s, v5.4h
+        cbz             is_near_vb, 2f
+        sshr            v20.4s, v20.4s, #10
+        b               3f
+2:
+        sshr            v20.4s, v20.4s, #7
+3:
+        add             v20.4s, v20.4s, v22.4s
+        sqxtun          v20.4h, v20.4s
+    .if \pix_size == 1
+        sqxtun          v20.8b, v20.8h
+        str             s20, [dst]
+    .else
+        umin            v20.4h, v20.4h, v25.4h
+        str             d20, [dst]
+    .endif
+        ret
+
+        .unreq          dst
+        .unreq          pp
+        .unreq          filter
+        .unreq          clip
+        .unreq          is_near_vb
+        .unreq          pix_max
+.endm
+
+function ff_alf_filter_luma_kernel_8_neon, export=1
+        alf_filter_luma_kernel 1
+endfunc
+
+function ff_alf_filter_luma_kernel_12_neon, export=1
+        mov             w5, #4095
+        b               1f
+endfunc
+
+function ff_alf_filter_luma_kernel_10_neon, export=1
+        mov             w5, #1023
+1:
+        alf_filter_luma_kernel 2
+endfunc
+
+function ff_alf_filter_chroma_kernel_8_neon, export=1
+        alf_filter_chroma_kernel 1
+endfunc
+
+function ff_alf_filter_chroma_kernel_12_neon, export=1
+        mov             w5, #4095
+        b               1f
+endfunc
+
+function ff_alf_filter_chroma_kernel_10_neon, export=1
+        mov             w5, #1023
+1:
+        alf_filter_chroma_kernel 2
+endfunc

ffmpeg/libavcodec/aarch64/vvc/alf_template.c

@@ -0,0 +1,157 @@
+/*
+ * VVC filters DSP
+ *
+ * Copyright (C) 2024 Zhao Zhili
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavcodec/bit_depth_template.c"
+
+void FUNC2(ff_alf_filter_luma_kernel, BIT_DEPTH, _neon)(pixel *dst,
+                                            const pixel **p,
+                                            const int16_t *filter,
+                                            const int16_t *clip,
+                                            int is_near_vb);
+
+void FUNC2(ff_alf_filter_chroma_kernel, BIT_DEPTH, _neon)(pixel *dst,
+                                            const pixel **p,
+                                            const int16_t *filter,
+                                            const int16_t *clip,
+                                            int is_near_vb);
+
+static void FUNC2(alf_filter_luma, BIT_DEPTH, _neon)(uint8_t *_dst,
+                                            ptrdiff_t dst_stride,
+                                            const uint8_t *_src,
+                                            ptrdiff_t src_stride,
+                                            const int width, const int height,
+                                            const int16_t *filter,
+                                            const int16_t *clip,
+                                            const int vb_pos)
+{
+    const pixel *src    = (pixel *)_src;
+
+    dst_stride /= sizeof(pixel);
+    src_stride /= sizeof(pixel);
+
+    for (int y = 0; y < height; y += ALF_BLOCK_SIZE) {
+        int far = (y + 3 < vb_pos - 3) || (y > vb_pos + 2);
+
+        for (int x = 0; x < width; x += 2 * ALF_BLOCK_SIZE) {
+            const pixel *s0 = src + y * src_stride + x;
+            const pixel *s1 = s0 + src_stride;
+            const pixel *s2 = s0 - src_stride;
+            const pixel *s3 = s1 + src_stride;
+            const pixel *s4 = s2 - src_stride;
+            const pixel *s5 = s3 + src_stride;
+            const pixel *s6 = s4 - src_stride;
+
+            for (int i = 0; i < ALF_BLOCK_SIZE; i++) {
+                pixel *dst = (pixel *) _dst + (y + i) * dst_stride + x;
+
+                const pixel *p0 = s0 + i * src_stride;
+                const pixel *p1 = s1 + i * src_stride;
+                const pixel *p2 = s2 + i * src_stride;
+                const pixel *p3 = s3 + i * src_stride;
+                const pixel *p4 = s4 + i * src_stride;
+                const pixel *p5 = s5 + i * src_stride;
+                const pixel *p6 = s6 + i * src_stride;
+                int is_near_vb = 0;
+
+                if (!far) {
+                    is_near_vb = (y + i == vb_pos - 1) || (y + i == vb_pos);
+                    if (is_near_vb) {
+                        p1 = p0;
+                        p2 = p0;
+                    }
+                    if (y + i >= vb_pos - 2 && y + i <= vb_pos + 1) {
+                        p3 = p1;
+                        p4 = p2;
+                    }
+                    if (y + i >= vb_pos - 3 && y + i <= vb_pos + 2) {
+                        p5 = p3;
+                        p6 = p4;
+                    }
+                }
+                FUNC2(ff_alf_filter_luma_kernel, BIT_DEPTH, _neon)(dst,
+                           (const pixel *[]) { p0, p1, p2, p3, p4, p5, p6},
+                           filter,
+                           clip,
+                           is_near_vb);
+            }
+            filter += 2 * ALF_NUM_COEFF_LUMA;
+            clip += 2 * ALF_NUM_COEFF_LUMA;
+        }
+    }
+}
+
+static void FUNC2(alf_filter_chroma, BIT_DEPTH, _neon)(uint8_t *_dst,
+                                                         ptrdiff_t dst_stride,
+                                                         const uint8_t *_src,
+                                                         ptrdiff_t src_stride,
+                                                         const int width,
+                                                         const int height,
+                                                         const int16_t *filter,
+                                                         const int16_t *clip,
+                                                         const int vb_pos)
+{
+    const pixel *src = (pixel *)_src;
+
+    dst_stride /= sizeof(pixel);
+    src_stride /= sizeof(pixel);
+
+    for (int y = 0; y < height; y += ALF_BLOCK_SIZE) {
+        int far = (y + 3 < vb_pos - 2) || (y > vb_pos + 1);
+
+        for (int x = 0; x < width; x += ALF_BLOCK_SIZE) {
+            const pixel *s0 = src + y * src_stride + x;
+            const pixel *s1 = s0 + src_stride;
+            const pixel *s2 = s0 - src_stride;
+            const pixel *s3 = s1 + src_stride;
+            const pixel *s4 = s2 - src_stride;
+
+            for (int i = 0; i < ALF_BLOCK_SIZE; i++) {
+                pixel *dst = (pixel *)_dst + (y + i) * dst_stride + x;
+
+                const pixel *p0 = s0 + i * src_stride;
+                const pixel *p1 = s1 + i * src_stride;
+                const pixel *p2 = s2 + i * src_stride;
+                const pixel *p3 = s3 + i * src_stride;
+                const pixel *p4 = s4 + i * src_stride;
+                int is_near_vb = 0;
+
+                if (!far) {
+                    is_near_vb = (y + i == vb_pos - 1) || (y + i == vb_pos);
+                    if (is_near_vb) {
+                        p1 = p0;
+                        p2 = p0;
+                    }
+
+                    if (y + i >= vb_pos - 2 && y + i <= vb_pos + 1) {
+                        p3 = p1;
+                        p4 = p2;
+                    }
+                }
+
+                FUNC2(ff_alf_filter_chroma_kernel, BIT_DEPTH, _neon)(dst,
+                             (const pixel *[]){p0, p1, p2, p3, p4},
+                             filter, clip,
+                             is_near_vb);
+            }
+        }
+    }
+}

ffmpeg/libavcodec/aarch64/vvc/dsp_init.c

@@ -0,0 +1,258 @@
+/*
+ * VVC filters DSP
+ *
+ * Copyright (C) 2024 Zhao Zhili
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/cpu.h"
+#include "libavutil/aarch64/cpu.h"
+#include "libavcodec/aarch64/h26x/dsp.h"
+#include "libavcodec/vvc/dsp.h"
+#include "libavcodec/vvc/dec.h"
+#include "libavcodec/vvc/ctu.h"
+
+#define BDOF_BLOCK_SIZE         16
+#define BDOF_MIN_BLOCK_SIZE     4
+
+void ff_vvc_prof_grad_filter_8x_neon(int16_t *gradient_h,
+                                     int16_t *gradient_v,
+                                     ptrdiff_t gradient_stride,
+                                     const int16_t *_src,
+                                     ptrdiff_t src_stride,
+                                     int width, int height);
+
+void ff_vvc_derive_bdof_vx_vy_neon(const int16_t *_src0, const int16_t *_src1,
+                                   int pad_mask,
+                                   const int16_t **gradient_h,
+                                   const int16_t **gradient_v,
+                                   int16_t *vx, int16_t *vy);
+#define BIT_DEPTH 8
+#include "alf_template.c"
+#include "of_template.c"
+#undef BIT_DEPTH
+
+#define BIT_DEPTH 10
+#include "alf_template.c"
+#include "of_template.c"
+#undef BIT_DEPTH
+
+#define BIT_DEPTH 12
+#include "alf_template.c"
+#include "of_template.c"
+#undef BIT_DEPTH
+
+int ff_vvc_sad_neon(const int16_t *src0, const int16_t *src1, int dx, int dy,
+                    const int block_w, const int block_h);
+
+void ff_vvc_avg_8_neon(uint8_t *dst, ptrdiff_t dst_stride,
+                       const int16_t *src0, const int16_t *src1, int width,
+                       int height);
+void ff_vvc_avg_10_neon(uint8_t *dst, ptrdiff_t dst_stride,
+                       const int16_t *src0, const int16_t *src1, int width,
+                       int height);
+void ff_vvc_avg_12_neon(uint8_t *dst, ptrdiff_t dst_stride,
+                        const int16_t *src0, const int16_t *src1, int width,
+                        int height);
+
+void ff_vvc_w_avg_8_neon(uint8_t *_dst, ptrdiff_t _dst_stride,
+                         const int16_t *src0, const int16_t *src1,
+                         int width, int height,
+                         uintptr_t w0_w1, uintptr_t offset_shift);
+void ff_vvc_w_avg_10_neon(uint8_t *_dst, ptrdiff_t _dst_stride,
+                         const int16_t *src0, const int16_t *src1,
+                         int width, int height,
+                         uintptr_t w0_w1, uintptr_t offset_shift);
+void ff_vvc_w_avg_12_neon(uint8_t *_dst, ptrdiff_t _dst_stride,
+                          const int16_t *src0, const int16_t *src1,
+                          int width, int height,
+                          uintptr_t w0_w1, uintptr_t offset_shift);
+/* When passing arguments to functions, Apple platforms diverge from the ARM64
+ * standard ABI for functions that require passing arguments on the stack. To
+ * simplify portability in the assembly function interface, use a different
+ * function signature that doesn't require passing arguments on the stack.
+ */
+#define W_AVG_FUN(bit_depth) \
+static void vvc_w_avg_ ## bit_depth(uint8_t *dst, ptrdiff_t dst_stride, \
+    const int16_t *src0, const int16_t *src1, int width, int height, \
+    int denom, int w0, int w1, int o0, int o1) \
+{ \
+    int shift = denom + FFMAX(3, 15 - bit_depth); \
+    int offset = ((o0 + o1) * (1 << (bit_depth - 8)) + 1) * (1 << (shift - 1)); \
+    uintptr_t w0_w1 = ((uintptr_t)w0 << 32) | (uint32_t)w1; \
+    uintptr_t offset_shift = ((uintptr_t)offset << 32) | (uint32_t)shift; \
+    ff_vvc_w_avg_ ## bit_depth ## _neon(dst, dst_stride, src0, src1, width, height, w0_w1, offset_shift); \
+}
+
+W_AVG_FUN(8)
+W_AVG_FUN(10)
+W_AVG_FUN(12)
+
+#define DMVR_FUN(fn, bd) \
+    void ff_vvc_dmvr_ ## fn ## bd ## _neon(int16_t *dst, \
+        const uint8_t *_src, ptrdiff_t _src_stride, int height, \
+        intptr_t mx, intptr_t my, int width);
+
+DMVR_FUN(, 8)
+DMVR_FUN(, 12)
+DMVR_FUN(hv_, 8)
+DMVR_FUN(hv_, 10)
+DMVR_FUN(hv_, 12)
+
+void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd)
+{
+    int cpu_flags = av_get_cpu_flags();
+    if (!have_neon(cpu_flags))
+        return;
+
+    if (bd == 8) {
+        c->inter.put[0][1][0][0] = ff_vvc_put_pel_pixels4_8_neon;
+        c->inter.put[0][2][0][0] = ff_vvc_put_pel_pixels8_8_neon;
+        c->inter.put[0][3][0][0] = ff_vvc_put_pel_pixels16_8_neon;
+        c->inter.put[0][4][0][0] = ff_vvc_put_pel_pixels32_8_neon;
+        c->inter.put[0][5][0][0] = ff_vvc_put_pel_pixels64_8_neon;
+        c->inter.put[0][6][0][0] = ff_vvc_put_pel_pixels128_8_neon;
+
+        c->inter.put[0][1][0][1] = ff_vvc_put_qpel_h4_8_neon;
+        c->inter.put[0][2][0][1] = ff_vvc_put_qpel_h8_8_neon;
+        c->inter.put[0][3][0][1] = ff_vvc_put_qpel_h16_8_neon;
+        c->inter.put[0][4][0][1] =
+        c->inter.put[0][5][0][1] =
+        c->inter.put[0][6][0][1] = ff_vvc_put_qpel_h32_8_neon;
+
+        c->inter.put[0][1][1][0] = ff_vvc_put_qpel_v4_8_neon;
+        c->inter.put[0][2][1][0] =
+        c->inter.put[0][3][1][0] =
+        c->inter.put[0][4][1][0] =
+        c->inter.put[0][5][1][0] =
+        c->inter.put[0][6][1][0] = ff_vvc_put_qpel_v8_8_neon;
+
+        c->inter.put[0][1][1][1] = ff_vvc_put_qpel_hv4_8_neon;
+        c->inter.put[0][2][1][1] = ff_vvc_put_qpel_hv8_8_neon;
+        c->inter.put[0][3][1][1] = ff_vvc_put_qpel_hv16_8_neon;
+        c->inter.put[0][4][1][1] = ff_vvc_put_qpel_hv32_8_neon;
+        c->inter.put[0][5][1][1] = ff_vvc_put_qpel_hv64_8_neon;
+        c->inter.put[0][6][1][1] = ff_vvc_put_qpel_hv128_8_neon;
+
+        c->inter.put[1][1][0][0] = ff_vvc_put_pel_pixels4_8_neon;
+        c->inter.put[1][2][0][0] = ff_vvc_put_pel_pixels8_8_neon;
+        c->inter.put[1][3][0][0] = ff_vvc_put_pel_pixels16_8_neon;
+        c->inter.put[1][4][0][0] = ff_vvc_put_pel_pixels32_8_neon;
+        c->inter.put[1][5][0][0] = ff_vvc_put_pel_pixels64_8_neon;
+        c->inter.put[1][6][0][0] = ff_vvc_put_pel_pixels128_8_neon;
+
+        c->inter.put[1][1][0][1] = ff_vvc_put_epel_h4_8_neon;
+        c->inter.put[1][2][0][1] = ff_vvc_put_epel_h8_8_neon;
+        c->inter.put[1][3][0][1] = ff_vvc_put_epel_h16_8_neon;
+        c->inter.put[1][4][0][1] =
+        c->inter.put[1][5][0][1] =
+        c->inter.put[1][6][0][1] = ff_vvc_put_epel_h32_8_neon;
+
+        c->inter.put[1][1][1][1] = ff_vvc_put_epel_hv4_8_neon;
+        c->inter.put[1][2][1][1] = ff_vvc_put_epel_hv8_8_neon;
+        c->inter.put[1][3][1][1] = ff_vvc_put_epel_hv16_8_neon;
+        c->inter.put[1][4][1][1] = ff_vvc_put_epel_hv32_8_neon;
+        c->inter.put[1][5][1][1] = ff_vvc_put_epel_hv64_8_neon;
+        c->inter.put[1][6][1][1] = ff_vvc_put_epel_hv128_8_neon;
+
+        c->inter.put_uni[0][1][0][0] = ff_vvc_put_pel_uni_pixels4_8_neon;
+        c->inter.put_uni[0][2][0][0] = ff_vvc_put_pel_uni_pixels8_8_neon;
+        c->inter.put_uni[0][3][0][0] = ff_vvc_put_pel_uni_pixels16_8_neon;
+        c->inter.put_uni[0][4][0][0] = ff_vvc_put_pel_uni_pixels32_8_neon;
+        c->inter.put_uni[0][5][0][0] = ff_vvc_put_pel_uni_pixels64_8_neon;
+        c->inter.put_uni[0][6][0][0] = ff_vvc_put_pel_uni_pixels128_8_neon;
+
+        c->inter.put_uni[0][1][0][1] = ff_vvc_put_qpel_uni_h4_8_neon;
+        c->inter.put_uni[0][2][0][1] = ff_vvc_put_qpel_uni_h8_8_neon;
+        c->inter.put_uni[0][3][0][1] = ff_vvc_put_qpel_uni_h16_8_neon;
+        c->inter.put_uni[0][4][0][1] =
+        c->inter.put_uni[0][5][0][1] =
+        c->inter.put_uni[0][6][0][1] = ff_vvc_put_qpel_uni_h32_8_neon;
+
+        c->inter.put_uni_w[0][1][0][0] = ff_vvc_put_pel_uni_w_pixels4_8_neon;
+        c->inter.put_uni_w[0][2][0][0] = ff_vvc_put_pel_uni_w_pixels8_8_neon;
+        c->inter.put_uni_w[0][3][0][0] = ff_vvc_put_pel_uni_w_pixels16_8_neon;
+        c->inter.put_uni_w[0][4][0][0] = ff_vvc_put_pel_uni_w_pixels32_8_neon;
+        c->inter.put_uni_w[0][5][0][0] = ff_vvc_put_pel_uni_w_pixels64_8_neon;
+        c->inter.put_uni_w[0][6][0][0] = ff_vvc_put_pel_uni_w_pixels128_8_neon;
+
+        c->inter.avg = ff_vvc_avg_8_neon;
+        c->inter.w_avg = vvc_w_avg_8;
+        c->inter.dmvr[0][0] = ff_vvc_dmvr_8_neon;
+        c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_8_neon;
+        c->inter.apply_bdof = apply_bdof_8;
+
+        c->sao.band_filter[0] = ff_h26x_sao_band_filter_8x8_8_neon;
+        for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.band_filter); i++)
+            c->sao.band_filter[i] = ff_h26x_sao_band_filter_16x16_8_neon;
+        c->sao.edge_filter[0] = ff_vvc_sao_edge_filter_8x8_8_neon;
+        for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.edge_filter); i++)
+            c->sao.edge_filter[i] = ff_vvc_sao_edge_filter_16x16_8_neon;
+        c->alf.filter[LUMA] = alf_filter_luma_8_neon;
+        c->alf.filter[CHROMA] = alf_filter_chroma_8_neon;
+
+        if (have_i8mm(cpu_flags)) {
+            c->inter.put[0][1][0][1] = ff_vvc_put_qpel_h4_8_neon_i8mm;
+            c->inter.put[0][2][0][1] = ff_vvc_put_qpel_h8_8_neon_i8mm;
+            c->inter.put[0][3][0][1] = ff_vvc_put_qpel_h16_8_neon_i8mm;
+            c->inter.put[0][4][0][1] = ff_vvc_put_qpel_h32_8_neon_i8mm;
+            c->inter.put[0][5][0][1] = ff_vvc_put_qpel_h64_8_neon_i8mm;
+            c->inter.put[0][6][0][1] = ff_vvc_put_qpel_h128_8_neon_i8mm;
+
+            c->inter.put[0][1][1][1] = ff_vvc_put_qpel_hv4_8_neon_i8mm;
+            c->inter.put[0][2][1][1] = ff_vvc_put_qpel_hv8_8_neon_i8mm;
+            c->inter.put[0][3][1][1] = ff_vvc_put_qpel_hv16_8_neon_i8mm;
+            c->inter.put[0][4][1][1] = ff_vvc_put_qpel_hv32_8_neon_i8mm;
+            c->inter.put[0][5][1][1] = ff_vvc_put_qpel_hv64_8_neon_i8mm;
+            c->inter.put[0][6][1][1] = ff_vvc_put_qpel_hv128_8_neon_i8mm;
+
+            c->inter.put[1][1][0][1] = ff_vvc_put_epel_h4_8_neon_i8mm;
+            c->inter.put[1][2][0][1] = ff_vvc_put_epel_h8_8_neon_i8mm;
+            c->inter.put[1][3][0][1] = ff_vvc_put_epel_h16_8_neon_i8mm;
+            c->inter.put[1][4][0][1] = ff_vvc_put_epel_h32_8_neon_i8mm;
+            c->inter.put[1][5][0][1] = ff_vvc_put_epel_h64_8_neon_i8mm;
+            c->inter.put[1][6][0][1] = ff_vvc_put_epel_h128_8_neon_i8mm;
+
+            c->inter.put[1][1][1][1] = ff_vvc_put_epel_hv4_8_neon_i8mm;
+            c->inter.put[1][2][1][1] = ff_vvc_put_epel_hv8_8_neon_i8mm;
+            c->inter.put[1][3][1][1] = ff_vvc_put_epel_hv16_8_neon_i8mm;
+            c->inter.put[1][4][1][1] = ff_vvc_put_epel_hv32_8_neon_i8mm;
+            c->inter.put[1][5][1][1] = ff_vvc_put_epel_hv64_8_neon_i8mm;
+            c->inter.put[1][6][1][1] = ff_vvc_put_epel_hv128_8_neon_i8mm;
+        }
+    } else if (bd == 10) {
+        c->inter.avg = ff_vvc_avg_10_neon;
+        c->inter.w_avg = vvc_w_avg_10;
+        c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_10_neon;
+        c->inter.apply_bdof = apply_bdof_10;
+
+        c->alf.filter[LUMA] = alf_filter_luma_10_neon;
+        c->alf.filter[CHROMA] = alf_filter_chroma_10_neon;
+    } else if (bd == 12) {
+        c->inter.avg = ff_vvc_avg_12_neon;
+        c->inter.w_avg = vvc_w_avg_12;
+        c->inter.dmvr[0][0] = ff_vvc_dmvr_12_neon;
+        c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_12_neon;
+        c->inter.apply_bdof = apply_bdof_12;
+
+        c->alf.filter[LUMA] = alf_filter_luma_12_neon;
+        c->alf.filter[CHROMA] = alf_filter_chroma_12_neon;
+    }
+
+    c->inter.sad = ff_vvc_sad_neon;
+}

ffmpeg/libavcodec/aarch64/vvc/of_template.c

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2024 Zhao Zhili <quinkblack@foxmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavcodec/bit_depth_template.c"
+
+void FUNC2(ff_vvc_apply_bdof_block, BIT_DEPTH, _neon)(pixel* dst,
+        ptrdiff_t dst_stride, const int16_t *src0, const int16_t *src1,
+        const int16_t **gh, const int16_t **gv, int16_t *vx, int16_t *vy);
+
+static void FUNC(apply_bdof)(uint8_t *_dst, ptrdiff_t _dst_stride,
+                             const int16_t *_src0, const int16_t *_src1,
+                             int block_w, int block_h) {
+    // +2 for pad left and right
+    int16_t gradient_buf_h[2][BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2];
+    int16_t gradient_buf_v[2][BDOF_BLOCK_SIZE * BDOF_BLOCK_SIZE + 2];
+    int16_t *gradient_h[2] = {&gradient_buf_h[0][1], &gradient_buf_h[1][1]};
+    int16_t *gradient_v[2] = {&gradient_buf_v[0][1], &gradient_buf_v[1][1]};
+    ptrdiff_t dst_stride = _dst_stride / sizeof(pixel);
+    pixel *dst = (pixel *) _dst;
+
+    ff_vvc_prof_grad_filter_8x_neon(gradient_h[0], gradient_v[0],
+                                    BDOF_BLOCK_SIZE,
+                                    _src0, MAX_PB_SIZE, block_w, block_h);
+    ff_vvc_prof_grad_filter_8x_neon(gradient_h[1], gradient_v[1],
+                                    BDOF_BLOCK_SIZE,
+                                    _src1, MAX_PB_SIZE, block_w, block_h);
+
+    for (int y = 0; y < block_h; y += BDOF_MIN_BLOCK_SIZE) {
+        for (int x = 0; x < block_w; x += BDOF_MIN_BLOCK_SIZE * 2) {
+            const int16_t *src0 = _src0 + y * MAX_PB_SIZE + x;
+            const int16_t *src1 = _src1 + y * MAX_PB_SIZE + x;
+            pixel *d = dst + x;
+            int idx = BDOF_BLOCK_SIZE * y + x;
+            const int16_t *gh[] = {gradient_h[0] + idx, gradient_h[1] + idx};
+            const int16_t *gv[] = {gradient_v[0] + idx, gradient_v[1] + idx};
+            int16_t vx[2], vy[2];
+            int pad_mask = !x | ((!y) << 1) |
+                           ((x + 2 * BDOF_MIN_BLOCK_SIZE == block_w) << 2) |
+                           ((y + BDOF_MIN_BLOCK_SIZE == block_h) << 3);
+            ff_vvc_derive_bdof_vx_vy_neon(src0, src1, pad_mask, gh, gv, vx, vy);
+            FUNC2(ff_vvc_apply_bdof_block, BIT_DEPTH, _neon)(d, dst_stride,
+                                                             src0, src1, gh, gv,
+                                                             vx, vy);
+        }
+        dst += BDOF_MIN_BLOCK_SIZE * dst_stride;
+    }
+}

ffmpeg/libavcodec/aarch64/vvc/sad.S

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2024 Zhao Zhili <quinkblack@foxmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/aarch64/asm.S"
+
+#define VVC_MAX_PB_SIZE 128
+
+function ff_vvc_sad_neon, export=1
+        src0            .req x0
+        src1            .req x1
+        dx              .req w2
+        dy              .req w3
+        block_w         .req w4
+        block_h         .req w5
+
+        sub             w7, dx, #4
+        sub             w8, dy, #4
+        add             w6, dx, dy, lsl #7
+        add             w7, w7, w8, lsl #7
+        sxtw            x6, w6
+        sxtw            x7, w7
+        add             src0, src0, x6, lsl #1
+        sub             src1, src1, x7, lsl #1
+
+        cmp             block_w, #16
+        movi            v16.4s, #0
+        b.ge            2f
+1:
+        // block_w == 8
+        ldr             q0, [src0]
+        ldr             q2, [src1]
+        subs            block_h, block_h, #2
+        sabal           v16.4s, v0.4h, v2.4h
+        sabal2          v16.4s, v0.8h, v2.8h
+
+        add             src0, src0, #(2 * VVC_MAX_PB_SIZE * 2)
+        add             src1, src1, #(2 * VVC_MAX_PB_SIZE * 2)
+        b.ne            1b
+        b               4f
+2:
+        // block_w == 16, no block_w > 16 according the spec
+        movi            v17.4s, #0
+3:
+        ldp             q0, q1, [src0], #(2 * VVC_MAX_PB_SIZE * 2)
+        ldp             q2, q3, [src1], #(2 * VVC_MAX_PB_SIZE * 2)
+        subs            block_h, block_h, #2
+        sabal           v16.4s, v0.4h, v2.4h
+        sabal2          v16.4s, v0.8h, v2.8h
+        sabal           v17.4s, v1.4h, v3.4h
+        sabal2          v17.4s, v1.8h, v3.8h
+
+        b.ne            3b
+        add             v16.4s, v16.4s, v17.4s
+4:
+        addv            s16, v16.4s
+        mov             w0, v16.s[0]
+        ret
+endfunc