lun_admin/android_multiviewwer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
lun_admin
2025-08-27 15:09:05 +09:00
parent 57961d84de
commit a13f66d917
9896 changed files with 2193048 additions and 730 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ffmpeg/libavcodec/x86/vvc/Makefile
+13
View File
@@ -0,0 +1,13 @@
clean::
$(RM) $(CLEANSUFFIXES:%=libavcodec/x86/vvc/%) $(CLEANSUFFIXES:%=libavcodec/x86/h26x/%)
OBJS-$(CONFIG_VVC_DECODER) += x86/vvc/dsp_init.o \
x86/h26x/h2656dsp.o
X86ASM-OBJS-$(CONFIG_VVC_DECODER) += x86/vvc/alf.o \
x86/vvc/dmvr.o \
x86/vvc/mc.o \
x86/vvc/of.o \
x86/vvc/sad.o \
x86/vvc/sao.o \
x86/vvc/sao_10bit.o \
x86/h26x/h2656_inter.o
ffmpeg/libavcodec/x86/vvc/alf.asm
+876
View File
@@ -0,0 +1,876 @@
;******************************************************************************
;* VVC Adaptive Loop Filter SIMD optimizations
;*
;* Copyright (c) 2023-2024 Nuo Mi <nuomi2021@gmail.com>
;* Copyright (c) 2023-2024 Wu Jianhua <toqsxw@outlook.com>
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%include "libavutil/x86/x86util.asm"
SECTION_RODATA
%macro PARAM_SHUFFE 1
%assign i (%1 * 2)
%assign j ((i + 1) << 8) + (i)
param_shuffe_ %+ %1:
%rep 2
times 4 dw j
times 4 dw (j + 0x0808)
%endrep
%endmacro
PARAM_SHUFFE 0
PARAM_SHUFFE 1
PARAM_SHUFFE 2
PARAM_SHUFFE 3
CLASSIFY_SHUFFE: times 2 db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13
TRANSPOSE_PERMUTE: dd 0, 1, 4, 5, 2, 3, 6, 7
ARG_VAR_SHUFFE: times 2 db 0, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4
cextern pd_64
dd448: times 8 dd 512 - 64
dd2: times 8 dd 2
dw3: times 8 dd 3
dw5: times 8 dd 5
dd15: times 8 dd 15
SECTION .text
%define ALF_NUM_COEFF_LUMA 12
%define ALF_NUM_COEFF_CHROMA 6
%define ALF_NUM_COEFF_CC 7
;%1-%3 out
;%4 clip or filter
%macro LOAD_LUMA_PARAMS_W16 4
lea offsetq, [3 * xq] ;xq * ALF_NUM_COEFF_LUMA / ALF_BLOCK_SIZE
movu m%1, [%4q + 2 * offsetq + 0 * 32] ; 2 * for sizeof(int16_t)
movu m%2, [%4q + 2 * offsetq + 1 * 32]
movu m%3, [%4q + 2 * offsetq + 2 * 32]
%endmacro
%macro LOAD_LUMA_PARAMS_W16 6
LOAD_LUMA_PARAMS_W16 %1, %2, %3, %4
;m%1 = 03 02 01 00
;m%2 = 07 06 05 04
;m%3 = 11 10 09 08
shufpd m%5, m%1, m%2, 0011b ;06 02 05 01
shufpd m%6, m%3, m%5, 1001b ;06 10 01 09
shufpd m%1, m%1, m%6, 1100b ;06 03 09 00
shufpd m%2, m%2, m%6, 0110b ;10 07 01 04
shufpd m%3, m%3, m%5, 0110b ;02 11 05 08
vpermpd m%1, m%1, 01111000b ;09 06 03 00
shufpd m%2, m%2, m%2, 1001b ;10 07 04 01
vpermpd m%3, m%3, 10000111b ;11 08 05 02
%endmacro
; %1-%3 out
; %4 clip or filter
; %5-%6 tmp
%macro LOAD_LUMA_PARAMS 6
LOAD_LUMA_PARAMS_W16 %1, %2, %3, %4, %5, %6
%endmacro
%macro LOAD_CHROMA_PARAMS 4
; LOAD_CHROMA_PARAMS_W %+ WIDTH %1, %2, %3, %4
movq xm%1, [%3q]
movd xm%2, [%3q + 8]
vpbroadcastq m%1, xm%1
vpbroadcastq m%2, xm%2
%endmacro
%macro LOAD_PARAMS 0
%if LUMA
LOAD_LUMA_PARAMS 3, 4, 5, filter, 6, 7
LOAD_LUMA_PARAMS 6, 7, 8, clip, 9, 10
%else
LOAD_CHROMA_PARAMS 3, 4, filter, 5
LOAD_CHROMA_PARAMS 6, 7, clip, 8
%endif
%endmacro
; FILTER(param_idx)
; input: m2, m9, m10
; output: m0, m1
; tmp: m11-m13
%macro FILTER 1
%assign i (%1 % 4)
%assign j (%1 / 4 + 3)
%assign k (%1 / 4 + 6)
%define filters m %+ j
%define clips m %+ k
pshufb m12, clips, [param_shuffe_ %+ i] ;clip
pxor m11, m11
psubw m11, m12 ;-clip
psubw m9, m2
CLIPW m9, m11, m12
psubw m10, m2
CLIPW m10, m11, m12
punpckhwd m13, m9, m10
punpcklwd m9, m9, m10
pshufb m12, filters, [param_shuffe_ %+ i] ;filter
punpcklwd m10, m12, m12
punpckhwd m12, m12, m12
pmaddwd m9, m10
pmaddwd m12, m13
paddd m0, m9
paddd m1, m12
%endmacro
; FILTER(param_idx, bottom, top, byte_offset)
; input: param_idx, bottom, top, byte_offset
; output: m0, m1
; temp: m9, m10
%macro FILTER 4
LOAD_PIXELS m10, [%2 + %4]
LOAD_PIXELS m9, [%3 - %4]
FILTER %1
%endmacro
; GET_SRCS(line)
; brief: get source lines
; input: src, src_stride, vb_pos
; output: s1...s6
%macro GET_SRCS 1
lea s1q, [srcq + src_strideq]
lea s3q, [s1q + src_strideq]
%if LUMA
lea s5q, [s3q + src_strideq]
%endif
neg src_strideq
lea s2q, [srcq + src_strideq]
lea s4q, [s2q + src_strideq]
%if LUMA
lea s6q, [s4q + src_strideq]
%endif
neg src_strideq
%if LUMA
cmp vb_posq, 0
je %%vb_bottom
cmp vb_posq, 4
jne %%vb_end
%else
cmp vb_posq, 2
jne %%vb_end
cmp %1, 2
jge %%vb_bottom
%endif
%%vb_above:
; above
; p1 = (y + i == vb_pos - 1) ? p0 : p1;
; p2 = (y + i == vb_pos - 1) ? p0 : p2;
; p3 = (y + i >= vb_pos - 2) ? p1 : p3;
; p4 = (y + i >= vb_pos - 2) ? p2 : p4;
; p5 = (y + i >= vb_pos - 3) ? p3 : p5;
; p6 = (y + i >= vb_pos - 3) ? p4 : p6;
dec vb_posq
cmp vb_posq, %1
cmove s1q, srcq
cmove s2q, srcq
dec vb_posq
cmp vb_posq, %1
cmovbe s3q, s1q
cmovbe s4q, s2q
dec vb_posq
%if LUMA
cmp vb_posq, %1
cmovbe s5q, s3q
cmovbe s6q, s4q
%endif
add vb_posq, 3
jmp %%vb_end
%%vb_bottom:
; bottom
; p1 = (y + i == vb_pos ) ? p0 : p1;
; p2 = (y + i == vb_pos ) ? p0 : p2;
; p3 = (y + i <= vb_pos + 1) ? p1 : p3;
; p4 = (y + i <= vb_pos + 1) ? p2 : p4;
; p5 = (y + i <= vb_pos + 2) ? p3 : p5;
; p6 = (y + i <= vb_pos + 2) ? p4 : p6;
cmp vb_posq, %1
cmove s1q, srcq
cmove s2q, srcq
inc vb_posq
cmp vb_posq, %1
cmovae s3q, s1q
cmovae s4q, s2q
inc vb_posq
%if LUMA
cmp vb_posq, %1
cmovae s5q, s3q
cmovae s6q, s4q
%endif
sub vb_posq, 2
%%vb_end:
%endmacro
; SHIFT_VB(line)
; brief: shift filter result
; input: m0, m1, vb_pos
; output: m0
; temp: m9
%macro SHIFT_VB 1
%define SHIFT 7
%if LUMA
cmp %1, 3
je %%near_above
cmp %1, 0
je %%near_below
jmp %%no_vb
%%near_above:
cmp vb_posq, 4
je %%near_vb
jmp %%no_vb
%%near_below:
cmp vb_posq, 0
je %%near_vb
%else
cmp %1, 0
je %%no_vb
cmp %1, 3
je %%no_vb
cmp vb_posq, 2
je %%near_vb
%endif
%%no_vb:
psrad m0, SHIFT
psrad m1, SHIFT
jmp %%shift_end
%%near_vb:
vpbroadcastd m9, [dd448]
paddd m0, m9
paddd m1, m9
psrad m0, SHIFT + 3
psrad m1, SHIFT + 3
%%shift_end:
packssdw m0, m0, m1
%endmacro
; FILTER_VB(line)
; brief: filter pixels for luma and chroma
; input: line
; output: m0, m1
; temp: s0q...s1q
%macro FILTER_VB 1
vpbroadcastd m0, [pd_64]
vpbroadcastd m1, [pd_64]
GET_SRCS %1
%if LUMA
FILTER 0, s5q, s6q, 0 * ps
FILTER 1, s3q, s4q, 1 * ps
FILTER 2, s3q, s4q, 0 * ps
FILTER 3, s3q, s4q, -1 * ps
FILTER 4, s1q, s2q, 2 * ps
FILTER 5, s1q, s2q, 1 * ps
FILTER 6, s1q, s2q, 0 * ps
FILTER 7, s1q, s2q, -1 * ps
FILTER 8, s1q, s2q, -2 * ps
FILTER 9, srcq, srcq, 3 * ps
FILTER 10, srcq, srcq, 2 * ps
FILTER 11, srcq, srcq, 1 * ps
%else
FILTER 0, s3q, s4q, 0 * ps
FILTER 1, s1q, s2q, 1 * ps
FILTER 2, s1q, s2q, 0 * ps
FILTER 3, s1q, s2q, -1 * ps
FILTER 4, srcq, srcq, 2 * ps
FILTER 5, srcq, srcq, 1 * ps
%endif
SHIFT_VB %1
%endmacro
; LOAD_PIXELS(dest, src)
%macro LOAD_PIXELS 2
%if ps == 2
movu %1, %2
%else
pmovzxbw %1, %2
%endif
%endmacro
; STORE_PIXELS_W16(dst, src)
%macro STORE_PIXELS_W16 2
%if ps == 2
movu [%1], m%2
%else
movu [%1], xm%2
%endif
%endmacro
%macro STORE_PIXELS_W8 2
%if ps == 2
movu [%1], xm%2
%else
movq [%1], xm%2
%endif
%endmacro
; STORE_PIXELS_W4(dst, src, offset)
%macro STORE_PIXELS_W4 3
%if ps == 2
movq [%1 + %3 * ps], xm%2
%else
movd [%1 + %3], xm%2
%endif
%endmacro
%macro STORE_PIXELS_W8LE 3
cmp %3, 8
jl .w4
STORE_PIXELS_W8 %1, %2
cmp %3, 12
%if ps == 2
vpermq m%2, m%2, q0302
%else
vpermq m%2, m%2, q0101
%endif
jl .end
STORE_PIXELS_W4 %1, %2, 8
jmp .end
.w4:
STORE_PIXELS_W4 %1, %2, 0
.end:
%endmacro
; STORE_PIXELS(dst, src, width)
%macro STORE_PIXELS 3
%if ps == 1
packuswb m%2, m%2
vpermq m%2, m%2, 0x8
%endif
%ifidn %3, 16
STORE_PIXELS_W16 %1, %2
%else
%if LUMA
STORE_PIXELS_W8 %1, %2
%else
STORE_PIXELS_W8LE %1, %2, %3
%endif
%endif
%endmacro
%macro FILTER_16x4 1
%if LUMA
push clipq
push strideq
%define s1q clipq
%define s2q strideq
%else
%define s1q s5q
%define s2q s6q
%endif
%define s3q pixel_maxq
%define s4q offsetq
push xq
xor xq, xq
%%filter_16x4_loop:
LOAD_PIXELS m2, [srcq] ;p0
FILTER_VB xq
; sum += curr
paddsw m0, m2
; clip to pixel
CLIPW m0, m14, m15
STORE_PIXELS dstq, 0, %1
lea srcq, [srcq + src_strideq]
lea dstq, [dstq + dst_strideq]
inc xq
cmp xq, 4
jl %%filter_16x4_loop
mov xq, src_strideq
neg xq
lea srcq, [srcq + xq * 4]
mov xq, dst_strideq
neg xq
lea dstq, [dstq + xq * 4]
pop xq
%if LUMA
pop strideq
pop clipq
%endif
%endmacro
; FILTER(bpc, luma/chroma)
%macro ALF_FILTER 2
%xdefine BPC %1
%ifidn %2, luma
%xdefine LUMA 1
%else
%xdefine LUMA 0
%endif
; ******************************
; void vvc_alf_filter_%2_%1bpc_avx2(uint8_t *dst, ptrdiff_t dst_stride,
; const uint8_t *src, ptrdiff_t src_stride, const ptrdiff_t width, cosnt ptr_diff_t height,
; const int16_t *filter, const int16_t *clip, ptrdiff_t stride, ptrdiff_t vb_pos, ptrdiff_t pixel_max);
; ******************************
cglobal vvc_alf_filter_%2_%1bpc, 11, 15, 16, 0-0x30, dst, dst_stride, src, src_stride, width, height, filter, clip, stride, vb_pos, pixel_max, \
offset, x, s5, s6
%define ps (%1 / 8) ; pixel size
movd xm15, pixel_maxd
vpbroadcastw m15, xm15
pxor m14, m14
.loop:
push srcq
push dstq
push widthq
xor xq, xq
.loop_w:
cmp widthq, 16
jl .loop_w_end
LOAD_PARAMS
FILTER_16x4 16
add srcq, 16 * ps
add dstq, 16 * ps
add xq, 16
sub widthq, 16
jmp .loop_w
.loop_w_end:
cmp widthq, 0
je .w_end
LOAD_PARAMS
FILTER_16x4 widthq
.w_end:
pop widthq
pop dstq
pop srcq
lea srcq, [srcq + 4 * src_strideq]
lea dstq, [dstq + 4 * dst_strideq]
lea filterq, [filterq + 2 * strideq]
lea clipq, [clipq + 2 * strideq]
sub vb_posq, 4
sub heightq, 4
jg .loop
RET
%endmacro
; FILTER(bpc)
%macro ALF_FILTER 1
ALF_FILTER %1, luma
ALF_FILTER %1, chroma
%endmacro
%define ALF_GRADIENT_BORDER 2
%define ALF_BORDER_LUMA 3
; ******************************
; void ff_vvc_alf_classify_grad(int *gradient_sum, const uint8_t *src,
; ptrdiff_t src_stride, intptr_t width, intptr_t height, intptr_t vb_pos);
; ******************************
%macro ALF_CLASSIFY_GRAD 1
cglobal vvc_alf_classify_grad_%1bpc, 6, 14, 16, gradient_sum, src, src_stride, width, height, vb_pos, \
x, y, s0, s1, s2, s3, vb_pos_below, src_stride3
lea src_stride3q, [src_strideq * 2 + src_strideq]
lea vb_pos_belowd, [vb_posd + ALF_GRADIENT_BORDER]
; src = src - ALF_BORDER_LUMA * src_stride - ALF_BORDER_LUMA
sub srcq, src_stride3q
sub srcq, ALF_BORDER_LUMA * ps
add widthd, ALF_GRADIENT_BORDER * 2
add heightd, ALF_GRADIENT_BORDER * 2
xor yd, yd
.loop_h:
xor xd, xd
pxor m15, m15 ; prev
.loop_w:
lea s0q, [srcq + xq * ps]
lea s1q, [s0q + src_strideq]
lea s2q, [s0q + 2 * src_strideq]
lea s3q, [s0q + src_stride3q]
cmp yd, vb_pos_belowd
cmove s0q, s1q
cmp yd, vb_posd
cmove s3q, s2q
LOAD_PIXELS m0, [s0q]
LOAD_PIXELS m1, [s1q]
LOAD_PIXELS m2, [s2q]
LOAD_PIXELS m3, [s3q]
LOAD_PIXELS m4, [s0q + 2 * ps]
LOAD_PIXELS m5, [s1q + 2 * ps]
LOAD_PIXELS m6, [s2q + 2 * ps]
LOAD_PIXELS m7, [s3q + 2 * ps]
pblendw m8, m0, m1, 0xaa ; nw
pblendw m9, m0, m5, 0x55 ; n
pblendw m10, m4, m5, 0xaa ; ne
pblendw m11, m1, m2, 0xaa ; w
pblendw m12, m5, m6, 0xaa ; e
pblendw m13, m2, m3, 0xaa ; sw
pblendw m14, m2, m7, 0x55 ; s
pblendw m0, m1, m6, 0x55
paddw m0, m0 ; c
movu m1, [CLASSIFY_SHUFFE]
pshufb m1, m0, m1 ; d
paddw m9, m14 ; n + s
psubw m9, m0 ; (n + s) - c
pabsw m9, m9 ; ver
paddw m11, m12 ; w + e
psubw m11, m1 ; (w + e) - d
pabsw m11, m11 ; hor
pblendw m14, m6, m7, 0xaa ; se
paddw m8, m14 ; nw + se
psubw m8, m1 ; (nw + se) - d
pabsw m8, m8 ; di0
paddw m10, m13 ; ne + sw
psubw m10, m1 ; (nw + se) - d
pabsw m10, m10 ; di1
phaddw m9, m11 ; vh, each word represent 2x2 pixels
phaddw m8, m10 ; di, each word represent 2x2 pixels
phaddw m0, m9, m8 ; all = each word represent 4x2 pixels, order is v_h_d0_d1 x 4
vinserti128 m15, m15, xm0, 1
pblendw m1, m0, m15, 0xaa ; t
phaddw m1, m0 ; each word represent 8x2 pixels, adjacent word share 4x2 pixels
vextracti128 xm15, m0, 1 ; prev
movu [gradient_sumq], m1
add gradient_sumq, 32
add xd, 16
cmp xd, widthd
jl .loop_w
lea srcq, [srcq + 2 * src_strideq]
add yd, 2
cmp yd, heightd
jl .loop_h
RET
%endmacro
; SAVE_CLASSIFY_PARAM_W16(dest, src)
%macro SAVE_CLASSIFY_PARAM_W16 2
lea tempq, [%1q + xq]
movu [tempq], xm%2
vperm2i128 m%2, m%2, m%2, 1
movu [tempq + widthq], xm%2
%endmacro
; SAVE_CLASSIFY_PARAM_W8
%macro SAVE_CLASSIFY_PARAM_W8 2
movq [%1], xm%2
vperm2i128 m%2, m%2, m%2, 1
movq [%1 + widthq], xm%2
%endmacro
; SAVE_CLASSIFY_PARAM_W4
%macro SAVE_CLASSIFY_PARAM_W4 2
movd [%1], xm%2
vperm2i128 m%2, m%2, m%2, 1
movd [%1 + widthq], xm%2
%endmacro
; SAVE_CLASSIFY_PARAM_W(dest, src)
%macro SAVE_CLASSIFY_PARAM_W 2
lea tempq, [%1q + xq]
cmp wd, 8
jl %%w4
SAVE_CLASSIFY_PARAM_W8 tempq, %2
vpermq m%2, m%2, 00010011b
add tempq, 8
cmp wd, 8
je %%end
%%w4:
SAVE_CLASSIFY_PARAM_W4 tempq, %2
%%end:
%endmacro
%macro ALF_CLASSIFY_H8 0
; first line, sum of 16x4 pixels (includes borders)
lea gradq, [gradient_sumq + 2 * xq]
movu m0, [gradq]
movu m1, [gradq + sum_strideq]
movu m2, [gradq + 2 * sum_strideq]
pcmpeqb m11, m11
movd xm13, yd
vpbroadcastd m13, xm13
movd xm12, vb_posd
vpbroadcastd m12, xm12
pcmpeqd m13, m12 ; y == vb_pos
pandn m13, m11 ; y != vb_pos
vpbroadcastd m14, [dw3]
pblendvb m14, m14, [dd2], m13 ; ac
pblendvb m3, m15, [gradq + sum_stride3q], m13
; extent to dword to avoid overflow
punpcklwd m4, m0, m15
punpckhwd m5, m0, m15
punpcklwd m6, m1, m15
punpckhwd m7, m1, m15
punpcklwd m8, m2, m15
punpckhwd m9, m2, m15
punpcklwd m10, m3, m15
punpckhwd m11, m3, m15
paddd m0, m4, m6
paddd m1, m5, m7
paddd m2, m8, m10
paddd m3, m9, m11
; sum of the first row
paddd m0, m2 ; low
paddd m1, m3 ; high
lea gradq, [gradq + 2 * sum_strideq]
pblendvb m10, m15, [gradq], m13
movu m11, [gradq + sum_strideq]
movu m12, [gradq + 2 * sum_strideq]
movu m13, [gradq + sum_stride3q]
punpcklwd m4, m10, m15
punpckhwd m5, m10, m15
punpcklwd m6, m11, m15
punpckhwd m7, m11, m15
punpcklwd m8, m12, m15
punpckhwd m9, m12, m15
punpcklwd m10, m13, m15
punpckhwd m11, m13, m15
paddd m2, m4, m6
paddd m3, m5, m7
paddd m4, m8, m10
paddd m5, m9, m11
; sum of the second row
paddd m2, m4 ; low
paddd m3, m5 ; high
punpckldq m4, m0, m2
punpckhdq m5, m0, m2
punpckldq m6, m1, m3
punpckhdq m7, m1, m3
; each dword represent 4x2 alf blocks
; the order is 01452367
punpckldq m0, m4, m6 ; sum_v
punpckhdq m1, m4, m6 ; sum_h
punpckldq m2, m5, m7 ; sum_d0
punpckhdq m3, m5, m7 ; sum_d1
pcmpgtd m4, m0, m1 ; dir_hv - 1
pmaxsd m5, m0, m1 ; hv1
pminsd m6, m0, m1 ; hv0
paddd m0, m1; ; sum_hv
pcmpgtd m7, m2, m3 ; dir_d - 1
pmaxsd m8, m2, m3 ; d1
pminsd m9, m2, m3 ; d0
; *transpose_idx = dir_d * 2 + dir_hv;
vpbroadcastd m10, [dw3]
paddd m11, m7, m7
paddd m11, m4
paddd m10, m11
vpermq m10, m10, 11011000b
SAVE_CLASSIFY_PARAM transpose_idx, 10
psrlq m10, m8, 32
psrlq m11, m6, 32
pmuldq m12, m10, m11 ; d1 * hv0 high
psrlq m1, m9, 32
psrlq m2, m5, 32
pmuldq m3, m1, m2 ; d0 * hv1 high
pcmpgtq m10, m12, m3 ; dir1 - 1 high
pmuldq m1, m8, m6 ; d1 * hv0 low
pmuldq m2, m9, m5 ; d0 * hv1 low
pcmpgtq m1, m2 ; dir1 - 1 low
vpblendd m1, m1, m10, 0xaa ; dir1 - 1
pblendvb m2, m5, m8, m1 ; hvd1
pblendvb m3, m6, m9, m1 ; hvd0
movd xm5, bit_depthd
vpbroadcastd m5, xm5
;*class_idx = arg_var[av_clip_uintp2(sum_hv * ac >> (BIT_DEPTH - 1), 4)];
pmulld m0, m14 ; sum_hv * ac
vpsrlvd m0, m0, m5
pminsd m0, [dd15]
movu m6, [ARG_VAR_SHUFFE]
pshufb m6, m0 ; class_idx
vpbroadcastd m10, [dw5]
; if (hvd1 * 2 > 9 * hvd0)
; *class_idx += ((dir1 << 1) + 2) * 5;
; else if (hvd1 > 2 * hvd0)
; *class_idx += ((dir1 << 1) + 1) * 5;
paddd m7, m3, m3
pcmpgtd m7, m2, m7 ; hvd1 > 2 * hvd0
pand m7, m10
paddd m6, m7 ; class_idx
paddd m8, m2, m2
pslld m9, m3, 3
paddd m9, m3
pcmpgtd m8, m9 ; hvd1 * 2 > 9 * hvd0
pand m8, m10
paddd m6, m8 ; class_idx
pandn m1, m7
paddd m1, m1 ; dir1 << 1
paddd m6, m1 ; class_idx
vpermq m6, m6, 11011000b
SAVE_CLASSIFY_PARAM class_idx, 6
%endmacro
%macro ALF_CLASSIFY_16x8 0
%define SAVE_CLASSIFY_PARAM SAVE_CLASSIFY_PARAM_W16
ALF_CLASSIFY_H8
%undef SAVE_CLASSIFY_PARAM
%endmacro
%macro ALF_CLASSIFY_Wx8 0
%define SAVE_CLASSIFY_PARAM SAVE_CLASSIFY_PARAM_W
ALF_CLASSIFY_H8
%undef SAVE_CLASSIFY_PARAM
%endmacro
; ******************************
;void ff_vvc_alf_classify(int *class_idx, int *transpose_idx, const int *gradient_sum,
; intptr_t width, intptr_t height, intptr_t vb_pos, int *gradient_tmp, intptr_t bit_depth);
; ******************************
%macro ALF_CLASSIFY 1
%define ps (%1 / 8)
ALF_CLASSIFY_GRAD %1
cglobal vvc_alf_classify_%1bpc, 7, 15, 16, class_idx, transpose_idx, gradient_sum, width, height, vb_pos, bit_depth, \
x, y, grad, sum_stride, sum_stride3, temp, w
sub bit_depthq, 1
; now we can use gradient to get class idx and transpose idx
lea sum_strideq, [widthd + ALF_GRADIENT_BORDER * 2]
add sum_strideq, 15
and sum_strideq, ~15 ; align to 16
add sum_strideq, sum_strideq ; two rows a time
add gradient_sumq, 8 ; first 4 words are garbage
lea sum_stride3q, [3 * sum_strideq]
xor yd, yd
and vb_posd, ~7 ; floor align to 8
pxor m15, m15
.loop_sum_h:
xor xd, xd
.loop_sum_w16:
lea wd, [widthd]
sub wd, xd
cmp wd, 16
jl .loop_sum_w16_end
ALF_CLASSIFY_16x8
add xd, 16
jmp .loop_sum_w16
.loop_sum_w16_end:
cmp wd, 0
je .loop_sum_w_end
ALF_CLASSIFY_Wx8
.loop_sum_w_end:
lea gradient_sumq, [gradient_sumq + 4 * sum_strideq]
lea transpose_idxq, [transpose_idxq + 2 * widthq]
lea class_idxq, [class_idxq + 2 * widthq]
add yd, 8
cmp yd, heightd
jl .loop_sum_h
RET
%endmacro
%if ARCH_X86_64
%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2
ALF_FILTER 16
ALF_FILTER 8
ALF_CLASSIFY 16
ALF_CLASSIFY 8
%endif
%endif
ffmpeg/libavcodec/x86/vvc/dmvr.asm
+373
View File
@@ -0,0 +1,373 @@
; /*
; * Provide AVX2 luma dmvr functions for VVC decoding
; * Copyright (c) 2024 Nuo Mi
; *
; * This file is part of FFmpeg.
; *
; * FFmpeg is free software; you can redistribute it and/or
; * modify it under the terms of the GNU Lesser General Public
; * License as published by the Free Software Foundation; either
; * version 2.1 of the License, or (at your option) any later version.
; *
; * FFmpeg is distributed in the hope that it will be useful,
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
; * Lesser General Public License for more details.
; *
; * You should have received a copy of the GNU Lesser General Public
; * License along with FFmpeg; if not, write to the Free Software
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
; */
%include "libavutil/x86/x86util.asm"
%define MAX_PB_SIZE 128
SECTION_RODATA 32
shift_12 times 2 dw 1 << (15 - (12 - 10))
shift3_8 times 2 dw 1 << (15 - (8 - 6))
shift3_10 times 2 dw 1 << (15 - (10 - 6))
shift3_12 times 2 dw 1 << (15 - (12 - 6))
pw_16 times 2 dw 16
%if ARCH_X86_64
%if HAVE_AVX2_EXTERNAL
SECTION .text
%define pstride (bd / 10 + 1)
; LOAD(dst, src)
%macro LOAD_W16 2
%if bd == 8
pmovzxbw %1, %2
%else
movu %1, %2
%endif
%endmacro
%macro SHIFT_W16 2
%if bd == 8
psllw %1, (10 - bd)
%elif bd == 10
; nothing
%else
pmulhrsw %1, %2
%endif
%endmacro
%macro SAVE_W16 2
movu %1, %2
%endmacro
; NEXT_4_LINES(is_h)
%macro NEXT_4_LINES 1
lea dstq, [dstq + dsq*4]
lea srcq, [srcq + ssq*4]
%if %1
lea src1q, [srcq + pstride]
%endif
%endmacro
; DMVR_4xW16(dst, dst_stride, dst_stride3, src, src_stride, src_stride3)
%macro DMVR_4xW16 6
LOAD_W16 m0, [%4]
LOAD_W16 m1, [%4 + %5]
LOAD_W16 m2, [%4 + 2 * %5]
LOAD_W16 m3, [%4 + %6]
SHIFT_W16 m0, m4
SHIFT_W16 m1, m4
SHIFT_W16 m2, m4
SHIFT_W16 m3, m4
SAVE_W16 [%1] , m0
SAVE_W16 [%1 + %2] , m1
SAVE_W16 [%1 + 2 * %2], m2
SAVE_W16 [%1 + %3] , m3
%endmacro
; buf += -stride * h + off
; OFFSET_TO_W4(buf, stride, off)
%macro OFFSET_TO_W4 3
mov id, hd
imul iq, %2
sub %1, iq
lea %1, [%1 + %3]
%endmacro
%macro OFFSET_TO_W4 0
OFFSET_TO_W4 srcq, ssq, 16 * (bd / 10 + 1)
OFFSET_TO_W4 dstq, dsq, 16 * 2
%endmacro
; void ff_vvc_dmvr_%1_avx2(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride,
; int height, intptr_t mx, intptr_t my, int width);
%macro DMVR_AVX2 1
cglobal vvc_dmvr_%1, 4, 9, 5, dst, src, ss, h, ds, ds3, w, ss3, i
%define bd %1
LOAD_STRIDES
%if %1 > 10
vpbroadcastd m4, [shift_%1]
%endif
mov wd, wm
mov id, hd
.w16:
sub id, 4
jl .w16_end
DMVR_4xW16 dstq, dsq, ds3q, srcq, ssq, ss3q
NEXT_4_LINES 0
jmp .w16
.w16_end:
sub wd, 16
jl .w4_end
OFFSET_TO_W4
.w4:
sub hd, 4
jl .w4_end
DMVR_4xW16 dstq, dsq, ds3q, srcq, ssq, ss3q
NEXT_4_LINES 0
jmp .w4
.w4_end:
RET
%endmacro
; LOAD_COEFFS(coeffs0, coeffs1, src)
%macro LOAD_COEFFS 3
movd xm%2, %3
vpbroadcastw m%2, xm%2
vpbroadcastd m%1, [pw_16]
psubw m%1, m%2
%endmacro
; LOAD_SHIFT(shift, src)
%macro LOAD_SHIFT 2
vpbroadcastd %1, [%2]
%if bd == 12
psllw %1, 1 ; avoid signed mul for pmulhrsw
%endif
%endmacro
; LOAD_STRIDES(shift, src)
%macro LOAD_STRIDES 0
mov dsq, MAX_PB_SIZE * 2
lea ss3q, [ssq*3]
lea ds3q, [dsq*3]
%endmacro
; BILINEAR(dst/src0, src1, coeff0, coeff1, round, tmp)
%macro BILINEAR 6
pmullw %1, %3
pmullw %6, %2, %4
paddw %1, %6
%if bd == 12
psrlw %1, 1 ; avoid signed mul for pmulhrsw
%endif
pmulhrsw %1, %5
%endmacro
; DMVR_H_1xW16(dst, src0, src1, offset, tmp)
%macro DMVR_H_1xW16 5
LOAD_W16 %1, [%2 + %4]
LOAD_W16 %5, [%3 + %4]
BILINEAR %1, %5, m10, m11, m12, %5
%endmacro
; DMVR_H_4xW16(dst, dst_stride, dst_stride3, src, src_stride, src_stride3, src1)
%macro DMVR_H_4xW16 7
DMVR_H_1xW16 m0, %4, %7, 0, m4
DMVR_H_1xW16 m1, %4, %7, %5, m5
DMVR_H_1xW16 m2, %4, %7, 2 * %5, m6
DMVR_H_1xW16 m3, %4, %7, %6, m7
SAVE_W16 [%1] , m0
SAVE_W16 [%1 + %2] , m1
SAVE_W16 [%1 + 2 * %2], m2
SAVE_W16 [%1 + %3] , m3
%endmacro
; void ff_vvc_dmvr_h_%1_avx2(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride,
; int height, intptr_t mx, intptr_t my, int width);
%macro DMVR_H_AVX2 1
cglobal vvc_dmvr_h_%1, 4, 10, 13, dst, src, ss, h, ds, ds3, w, ss3, src1, i
%define bd %1
LOAD_COEFFS 10, 11, dsm
LOAD_SHIFT m12, shift3_%1
LOAD_STRIDES
lea src1q, [srcq + pstride]
mov wd, wm
mov id, hd
.w16:
sub id, 4
jl .w16_end
DMVR_H_4xW16 dstq, dsq, ds3q, srcq, ssq, ss3q, src1q
NEXT_4_LINES 1
jmp .w16
.w16_end:
sub wd, 16
jl .w4_end
OFFSET_TO_W4
lea src1q, [srcq + pstride]
.w4:
sub hd, 4
jl .w4_end
DMVR_H_4xW16 dstq, dsq, ds3q, srcq, ssq, ss3q, src1q
NEXT_4_LINES 1
jmp .w4
.w4_end:
RET
%endmacro
; DMVR_V_4xW16(dst, dst_stride, dst_stride3, src, src_stride, src_stride3)
%macro DMVR_V_4xW16 6
LOAD_W16 m1, [%4 + %5]
LOAD_W16 m2, [%4 + 2 * %5]
LOAD_W16 m3, [%4 + %6]
LOAD_W16 m4, [%4 + 4 * %5]
BILINEAR m0, m1, m8, m9, m10, m11
BILINEAR m1, m2, m8, m9, m10, m12
BILINEAR m2, m3, m8, m9, m10, m13
BILINEAR m3, m4, m8, m9, m10, m14
SAVE_W16 [%1] , m0
SAVE_W16 [%1 + %2] , m1
SAVE_W16 [%1 + 2 * %2], m2
SAVE_W16 [%1 + %3] , m3
; why can't we use SWAP m0, m4 here?
movaps m0, m4
%endmacro
; void ff_vvc_dmvr_v_%1_avx2(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride,
; int height, intptr_t mx, intptr_t my, int width);
%macro DMVR_V_AVX2 1
cglobal vvc_dmvr_v_%1, 4, 9, 15, dst, src, ss, h, ds, ds3, w, ss3, i
%define bd %1
LOAD_COEFFS 8, 9, ds3m
LOAD_SHIFT m10, shift3_%1
LOAD_STRIDES
mov wd, wm
mov id, hd
LOAD_W16 m0, [srcq]
.w16:
sub id, 4
jl .w16_end
DMVR_V_4xW16 dstq, dsq, ds3q, srcq, ssq, ss3q
NEXT_4_LINES 0
jmp .w16
.w16_end:
sub wd, 16
jl .w4_end
OFFSET_TO_W4
LOAD_W16 m0, [srcq]
.w4:
sub hd, 4
jl .w4_end
DMVR_V_4xW16 dstq, dsq, ds3q, srcq, ssq, ss3q
NEXT_4_LINES 0
jmp .w4
.w4_end:
RET
%endmacro
; DMVR_HV_4xW16(dst, dst_stride, dst_stride3, src, src_stride, src_stride3, src1)
%macro DMVR_HV_4xW16 7
DMVR_H_1xW16 m1, %4, %7, %5, m6
DMVR_H_1xW16 m2, %4, %7, 2 * %5, m7
DMVR_H_1xW16 m3, %4, %7, %6, m8
DMVR_H_1xW16 m4, %4, %7, 4 * %5, m9
BILINEAR m0, m1, m13, m14, m15, m6
BILINEAR m1, m2, m13, m14, m15, m7
BILINEAR m2, m3, m13, m14, m15, m8
BILINEAR m3, m4, m13, m14, m15, m9
SAVE_W16 [%1] , m0
SAVE_W16 [%1 + %2] , m1
SAVE_W16 [%1 + 2 * %2], m2
SAVE_W16 [%1 + %3] , m3
; why can't we use SWAP m0, m4 here?
movaps m0, m4
%endmacro
; void ff_vvc_dmvr_hv_%1_avx2(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride,
; int height, intptr_t mx, intptr_t my, int width);
%macro DMVR_HV_AVX2 1
cglobal vvc_dmvr_hv_%1, 7, 10, 16, dst, src, ss, h, ds, ds3, w, ss3, src1, i
%define bd %1
LOAD_COEFFS 10, 11, dsm
LOAD_SHIFT m12, shift3_%1
LOAD_COEFFS 13, 14, ds3m
LOAD_SHIFT m15, shift3_10
LOAD_STRIDES
lea src1q, [srcq + pstride]
mov id, hd
DMVR_H_1xW16 m0, srcq, src1q, 0, m5
.w16:
sub id, 4
jl .w16_end
DMVR_HV_4xW16 dstq, dsq, ds3q, srcq, ssq, ss3q, src1q
NEXT_4_LINES 1
jmp .w16
.w16_end:
sub wd, 16
jl .w4_end
OFFSET_TO_W4
lea src1q, [srcq + pstride]
DMVR_H_1xW16 m0, srcq, src1q, 0, m5
.w4:
sub hd, 4
jl .w4_end
DMVR_HV_4xW16 dstq, dsq, ds3q, srcq, ssq, ss3q, src1q
NEXT_4_LINES 1
jmp .w4
.w4_end:
RET
%endmacro
%macro VVC_DMVR_AVX2 1
DMVR_AVX2 %1
DMVR_H_AVX2 %1
DMVR_V_AVX2 %1
DMVR_HV_AVX2 %1
%endmacro
INIT_YMM avx2
VVC_DMVR_AVX2 8
VVC_DMVR_AVX2 10
VVC_DMVR_AVX2 12
%endif ; HAVE_AVX2_EXTERNAL
%endif ; ARCH_X86_64
ffmpeg/libavcodec/x86/vvc/dsp_init.c
+422
View File
@@ -0,0 +1,422 @@
/*
* VVC DSP init for x86
*
* Copyright (C) 2022-2024 Nuo Mi
* Copyright (c) 2023-2024 Wu Jianhua
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
#include "config.h"
#include "libavutil/cpu.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/vvc/dec.h"
#include "libavcodec/vvc/ctu.h"
#include "libavcodec/vvc/dsp.h"
#include "libavcodec/x86/h26x/h2656dsp.h"
#if ARCH_X86_64
#define bf(fn, bd, opt) fn##_##bd##_##opt
#define BF(fn, bpc, opt) fn##_##bpc##bpc_##opt
#define AVG_BPC_PROTOTYPES(bpc, opt) \
void BF(ff_vvc_avg, bpc, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
const int16_t *src0, const int16_t *src1, intptr_t width, intptr_t height, intptr_t pixel_max); \
void BF(ff_vvc_w_avg, bpc, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
const int16_t *src0, const int16_t *src1, intptr_t width, intptr_t height, \
intptr_t denom, intptr_t w0, intptr_t w1, intptr_t o0, intptr_t o1, intptr_t pixel_max);
AVG_BPC_PROTOTYPES( 8, avx2)
AVG_BPC_PROTOTYPES(16, avx2)
#define DMVR_PROTOTYPES(bd, opt) \
void ff_vvc_dmvr_##bd##_##opt(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride, \
int height, intptr_t mx, intptr_t my, int width); \
void ff_vvc_dmvr_h_##bd##_##opt(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride, \
int height, intptr_t mx, intptr_t my, int width); \
void ff_vvc_dmvr_v_##bd##_##opt(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride, \
int height, intptr_t mx, intptr_t my, int width); \
void ff_vvc_dmvr_hv_##bd##_##opt(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride, \
int height, intptr_t mx, intptr_t my, int width); \
DMVR_PROTOTYPES( 8, avx2)
DMVR_PROTOTYPES(10, avx2)
DMVR_PROTOTYPES(12, avx2)
#if ARCH_X86_64 && HAVE_AVX2_EXTERNAL
void ff_vvc_apply_bdof_avx2(uint8_t *dst, ptrdiff_t dst_stride,
const int16_t *src0, const int16_t *src1,
int w, int h, int pixel_max);
#define OF_FUNC(bd, opt) \
static void vvc_apply_bdof_##bd##_##opt(uint8_t *dst, ptrdiff_t dst_stride, \
const int16_t *src0, const int16_t *src1, int w, int h) \
{ \
ff_vvc_apply_bdof##_##opt(dst, dst_stride, src0, src1, w, h, (1 << bd) - 1); \
} \
OF_FUNC( 8, avx2)
OF_FUNC(10, avx2)
OF_FUNC(12, avx2)
#define OF_INIT(bd) c->inter.apply_bdof = vvc_apply_bdof_##bd##_avx2
#endif
#define ALF_BPC_PROTOTYPES(bpc, opt) \
void BF(ff_vvc_alf_filter_luma, bpc, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
const uint8_t *src, ptrdiff_t src_stride, ptrdiff_t width, ptrdiff_t height, \
const int16_t *filter, const int16_t *clip, ptrdiff_t stride, ptrdiff_t vb_pos, ptrdiff_t pixel_max); \
void BF(ff_vvc_alf_filter_chroma, bpc, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
const uint8_t *src, ptrdiff_t src_stride, ptrdiff_t width, ptrdiff_t height, \
const int16_t *filter, const int16_t *clip, ptrdiff_t stride, ptrdiff_t vb_pos, ptrdiff_t pixel_max); \
void BF(ff_vvc_alf_classify_grad, bpc, opt)(int *gradient_sum, \
const uint8_t *src, ptrdiff_t src_stride, intptr_t width, intptr_t height, intptr_t vb_pos); \
void BF(ff_vvc_alf_classify, bpc, opt)(int *class_idx, int *transpose_idx, const int *gradient_sum, \
intptr_t width, intptr_t height, intptr_t vb_pos, intptr_t bit_depth); \
ALF_BPC_PROTOTYPES(8, avx2)
ALF_BPC_PROTOTYPES(16, avx2)
#if ARCH_X86_64
#define FW_PUT(name, depth, opt) \
static void vvc_put_ ## name ## _ ## depth ## _##opt(int16_t *dst, const uint8_t *src, ptrdiff_t srcstride, \
int height, const int8_t *hf, const int8_t *vf, int width) \
{ \
ff_h2656_put_## name ## _ ## depth ## _##opt(dst, 2 * MAX_PB_SIZE, src, srcstride, height, hf, vf, width); \
}
#if HAVE_SSE4_EXTERNAL
#define FW_PUT_TAP(fname, bitd, opt ) \
FW_PUT(fname##4, bitd, opt ) \
FW_PUT(fname##8, bitd, opt ) \
FW_PUT(fname##16, bitd, opt ) \
FW_PUT(fname##32, bitd, opt ) \
FW_PUT(fname##64, bitd, opt ) \
FW_PUT(fname##128, bitd, opt ) \
#define FW_PUT_4TAP(fname, bitd, opt) \
FW_PUT(fname ## 2, bitd, opt) \
FW_PUT_TAP(fname, bitd, opt)
#define FW_PUT_4TAP_SSE4(bitd) \
FW_PUT_4TAP(pixels, bitd, sse4) \
FW_PUT_4TAP(4tap_h, bitd, sse4) \
FW_PUT_4TAP(4tap_v, bitd, sse4) \
FW_PUT_4TAP(4tap_hv, bitd, sse4)
#define FW_PUT_8TAP_SSE4(bitd) \
FW_PUT_TAP(8tap_h, bitd, sse4) \
FW_PUT_TAP(8tap_v, bitd, sse4) \
FW_PUT_TAP(8tap_hv, bitd, sse4)
#define FW_PUT_SSE4(bitd) \
FW_PUT_4TAP_SSE4(bitd) \
FW_PUT_8TAP_SSE4(bitd)
FW_PUT_SSE4( 8)
FW_PUT_SSE4(10)
FW_PUT_SSE4(12)
#endif
#if HAVE_AVX2_EXTERNAL
#define FW_PUT_TAP_AVX2(n, bitd) \
FW_PUT(n ## tap_h32, bitd, avx2) \
FW_PUT(n ## tap_h64, bitd, avx2) \
FW_PUT(n ## tap_h128, bitd, avx2) \
FW_PUT(n ## tap_v32, bitd, avx2) \
FW_PUT(n ## tap_v64, bitd, avx2) \
FW_PUT(n ## tap_v128, bitd, avx2)
#define FW_PUT_AVX2(bitd) \
FW_PUT(pixels32, bitd, avx2) \
FW_PUT(pixels64, bitd, avx2) \
FW_PUT(pixels128, bitd, avx2) \
FW_PUT_TAP_AVX2(4, bitd) \
FW_PUT_TAP_AVX2(8, bitd) \
FW_PUT_AVX2( 8)
FW_PUT_AVX2(10)
FW_PUT_AVX2(12)
#define FW_PUT_TAP_16BPC_AVX2(n, bitd) \
FW_PUT(n ## tap_h16, bitd, avx2) \
FW_PUT(n ## tap_v16, bitd, avx2) \
FW_PUT(n ## tap_hv16, bitd, avx2) \
FW_PUT(n ## tap_hv32, bitd, avx2) \
FW_PUT(n ## tap_hv64, bitd, avx2) \
FW_PUT(n ## tap_hv128, bitd, avx2)
#define FW_PUT_16BPC_AVX2(bitd) \
FW_PUT(pixels16, bitd, avx2) \
FW_PUT_TAP_16BPC_AVX2(4, bitd) \
FW_PUT_TAP_16BPC_AVX2(8, bitd)
FW_PUT_16BPC_AVX2(10)
FW_PUT_16BPC_AVX2(12)
#define AVG_FUNCS(bpc, bd, opt) \
static void bf(vvc_avg, bd, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
const int16_t *src0, const int16_t *src1, int width, int height) \
{ \
BF(ff_vvc_avg, bpc, opt)(dst, dst_stride, src0, src1, width, height, (1 << bd) - 1); \
} \
static void bf(vvc_w_avg, bd, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
const int16_t *src0, const int16_t *src1, int width, int height, \
int denom, int w0, int w1, int o0, int o1) \
{ \
BF(ff_vvc_w_avg, bpc, opt)(dst, dst_stride, src0, src1, width, height, \
denom, w0, w1, o0, o1, (1 << bd) - 1); \
}
AVG_FUNCS(8, 8, avx2)
AVG_FUNCS(16, 10, avx2)
AVG_FUNCS(16, 12, avx2)
#define ALF_FUNCS(bpc, bd, opt) \
static void bf(vvc_alf_filter_luma, bd, opt)(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, ptrdiff_t src_stride, \
int width, int height, const int16_t *filter, const int16_t *clip, const int vb_pos) \
{ \
const int param_stride = (width >> 2) * ALF_NUM_COEFF_LUMA; \
BF(ff_vvc_alf_filter_luma, bpc, opt)(dst, dst_stride, src, src_stride, width, height, \
filter, clip, param_stride, vb_pos, (1 << bd) - 1); \
} \
static void bf(vvc_alf_filter_chroma, bd, opt)(uint8_t *dst, ptrdiff_t dst_stride, const uint8_t *src, ptrdiff_t src_stride, \
int width, int height, const int16_t *filter, const int16_t *clip, const int vb_pos) \
{ \
BF(ff_vvc_alf_filter_chroma, bpc, opt)(dst, dst_stride, src, src_stride, width, height, \
filter, clip, 0, vb_pos,(1 << bd) - 1); \
} \
static void bf(vvc_alf_classify, bd, opt)(int *class_idx, int *transpose_idx, \
const uint8_t *src, ptrdiff_t src_stride, int width, int height, int vb_pos, int *gradient_tmp) \
{ \
BF(ff_vvc_alf_classify_grad, bpc, opt)(gradient_tmp, src, src_stride, width, height, vb_pos); \
BF(ff_vvc_alf_classify, bpc, opt)(class_idx, transpose_idx, gradient_tmp, width, height, vb_pos, bd); \
} \
ALF_FUNCS(8, 8, avx2)
ALF_FUNCS(16, 10, avx2)
ALF_FUNCS(16, 12, avx2)
#endif
#define SAO_FILTER_FUNC(wd, bitd, opt) \
void ff_vvc_sao_band_filter_##wd##_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
void ff_vvc_sao_edge_filter_##wd##_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
const int16_t *sao_offset_val, int eo, int width, int height); \
#define SAO_FILTER_FUNCS(bitd, opt) \
SAO_FILTER_FUNC(8, bitd, opt) \
SAO_FILTER_FUNC(16, bitd, opt) \
SAO_FILTER_FUNC(32, bitd, opt) \
SAO_FILTER_FUNC(48, bitd, opt) \
SAO_FILTER_FUNC(64, bitd, opt) \
SAO_FILTER_FUNC(80, bitd, opt) \
SAO_FILTER_FUNC(96, bitd, opt) \
SAO_FILTER_FUNC(112, bitd, opt) \
SAO_FILTER_FUNC(128, bitd, opt) \
SAO_FILTER_FUNCS(8, avx2)
SAO_FILTER_FUNCS(10, avx2)
SAO_FILTER_FUNCS(12, avx2)
#define SAO_FILTER_INIT(type, bitd, opt) do { \
c->sao.type##_filter[0] = ff_vvc_sao_##type##_filter_8_##bitd##_##opt; \
c->sao.type##_filter[1] = ff_vvc_sao_##type##_filter_16_##bitd##_##opt; \
c->sao.type##_filter[2] = ff_vvc_sao_##type##_filter_32_##bitd##_##opt; \
c->sao.type##_filter[3] = ff_vvc_sao_##type##_filter_48_##bitd##_##opt; \
c->sao.type##_filter[4] = ff_vvc_sao_##type##_filter_64_##bitd##_##opt; \
c->sao.type##_filter[5] = ff_vvc_sao_##type##_filter_80_##bitd##_##opt; \
c->sao.type##_filter[6] = ff_vvc_sao_##type##_filter_96_##bitd##_##opt; \
c->sao.type##_filter[7] = ff_vvc_sao_##type##_filter_112_##bitd##_##opt; \
c->sao.type##_filter[8] = ff_vvc_sao_##type##_filter_128_##bitd##_##opt; \
} while (0)
#define SAO_INIT(bitd, opt) do { \
SAO_FILTER_INIT(band, bitd, opt); \
SAO_FILTER_INIT(edge, bitd, opt); \
} while (0)
#define AVG_INIT(bd, opt) do { \
c->inter.avg = bf(vvc_avg, bd, opt); \
c->inter.w_avg = bf(vvc_w_avg, bd, opt); \
} while (0)
#define DMVR_INIT(bd) do { \
c->inter.dmvr[0][0] = ff_vvc_dmvr_##bd##_avx2; \
c->inter.dmvr[0][1] = ff_vvc_dmvr_h_##bd##_avx2; \
c->inter.dmvr[1][0] = ff_vvc_dmvr_v_##bd##_avx2; \
c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_##bd##_avx2; \
} while (0)
#define PEL_LINK(dst, C, W, idx1, idx2, name, D, opt) \
dst[C][W][idx1][idx2] = vvc_put_## name ## _ ## D ## _##opt; \
dst ## _uni[C][W][idx1][idx2] = ff_h2656_put_uni_ ## name ## _ ## D ## _##opt; \
#define MC_TAP_LINKS(pointer, C, my, mx, fname, bitd, opt ) \
PEL_LINK(pointer, C, 1, my , mx , fname##4 , bitd, opt ); \
PEL_LINK(pointer, C, 2, my , mx , fname##8 , bitd, opt ); \
PEL_LINK(pointer, C, 3, my , mx , fname##16, bitd, opt ); \
PEL_LINK(pointer, C, 4, my , mx , fname##32, bitd, opt ); \
PEL_LINK(pointer, C, 5, my , mx , fname##64, bitd, opt ); \
PEL_LINK(pointer, C, 6, my , mx , fname##128, bitd, opt );
#define MC_8TAP_LINKS(pointer, my, mx, fname, bitd, opt) \
MC_TAP_LINKS(pointer, LUMA, my, mx, fname, bitd, opt)
#define MC_8TAP_LINKS_SSE4(bd) \
MC_8TAP_LINKS(c->inter.put, 0, 0, pixels, bd, sse4); \
MC_8TAP_LINKS(c->inter.put, 0, 1, 8tap_h, bd, sse4); \
MC_8TAP_LINKS(c->inter.put, 1, 0, 8tap_v, bd, sse4); \
MC_8TAP_LINKS(c->inter.put, 1, 1, 8tap_hv, bd, sse4)
#define MC_4TAP_LINKS(pointer, my, mx, fname, bitd, opt) \
PEL_LINK(pointer, CHROMA, 0, my , mx , fname##2 , bitd, opt ); \
MC_TAP_LINKS(pointer, CHROMA, my, mx, fname, bitd, opt) \
#define MC_4TAP_LINKS_SSE4(bd) \
MC_4TAP_LINKS(c->inter.put, 0, 0, pixels, bd, sse4); \
MC_4TAP_LINKS(c->inter.put, 0, 1, 4tap_h, bd, sse4); \
MC_4TAP_LINKS(c->inter.put, 1, 0, 4tap_v, bd, sse4); \
MC_4TAP_LINKS(c->inter.put, 1, 1, 4tap_hv, bd, sse4)
#define MC_LINK_SSE4(bd) \
MC_4TAP_LINKS_SSE4(bd) \
MC_8TAP_LINKS_SSE4(bd)
#define MC_TAP_LINKS_AVX2(C,tap,bd) do { \
PEL_LINK(c->inter.put, C, 4, 0, 0, pixels32, bd, avx2) \
PEL_LINK(c->inter.put, C, 5, 0, 0, pixels64, bd, avx2) \
PEL_LINK(c->inter.put, C, 6, 0, 0, pixels128, bd, avx2) \
PEL_LINK(c->inter.put, C, 4, 0, 1, tap##tap_h32, bd, avx2) \
PEL_LINK(c->inter.put, C, 5, 0, 1, tap##tap_h64, bd, avx2) \
PEL_LINK(c->inter.put, C, 6, 0, 1, tap##tap_h128, bd, avx2) \
PEL_LINK(c->inter.put, C, 4, 1, 0, tap##tap_v32, bd, avx2) \
PEL_LINK(c->inter.put, C, 5, 1, 0, tap##tap_v64, bd, avx2) \
PEL_LINK(c->inter.put, C, 6, 1, 0, tap##tap_v128, bd, avx2) \
} while (0)
#define MC_LINKS_AVX2(bd) \
MC_TAP_LINKS_AVX2(LUMA, 8, bd); \
MC_TAP_LINKS_AVX2(CHROMA, 4, bd);
#define MC_TAP_LINKS_16BPC_AVX2(C, tap, bd) do { \
PEL_LINK(c->inter.put, C, 3, 0, 0, pixels16, bd, avx2) \
PEL_LINK(c->inter.put, C, 3, 0, 1, tap##tap_h16, bd, avx2) \
PEL_LINK(c->inter.put, C, 3, 1, 0, tap##tap_v16, bd, avx2) \
PEL_LINK(c->inter.put, C, 3, 1, 1, tap##tap_hv16, bd, avx2) \
PEL_LINK(c->inter.put, C, 4, 1, 1, tap##tap_hv32, bd, avx2) \
PEL_LINK(c->inter.put, C, 5, 1, 1, tap##tap_hv64, bd, avx2) \
PEL_LINK(c->inter.put, C, 6, 1, 1, tap##tap_hv128, bd, avx2) \
} while (0)
#define MC_LINKS_16BPC_AVX2(bd) \
MC_TAP_LINKS_16BPC_AVX2(LUMA, 8, bd); \
MC_TAP_LINKS_16BPC_AVX2(CHROMA, 4, bd);
int ff_vvc_sad_avx2(const int16_t *src0, const int16_t *src1, int dx, int dy, int block_w, int block_h);
#define SAD_INIT() c->inter.sad = ff_vvc_sad_avx2
#define ALF_INIT(bd) do { \
c->alf.filter[LUMA] = vvc_alf_filter_luma_##bd##_avx2; \
c->alf.filter[CHROMA] = vvc_alf_filter_chroma_##bd##_avx2; \
c->alf.classify = vvc_alf_classify_##bd##_avx2; \
} while (0)
#endif
#endif // ARCH_X86_64
void ff_vvc_dsp_init_x86(VVCDSPContext *const c, const int bd)
{
#if ARCH_X86_64
const int cpu_flags = av_get_cpu_flags();
switch (bd) {
case 8:
#if HAVE_SSE4_EXTERNAL
if (EXTERNAL_SSE4(cpu_flags)) {
MC_LINK_SSE4(8);
}
#endif
#if HAVE_AVX2_EXTERNAL
if (EXTERNAL_AVX2_FAST(cpu_flags)) {
// inter
AVG_INIT(8, avx2);
DMVR_INIT(8);
MC_LINKS_AVX2(8);
OF_INIT(8);
SAD_INIT();
// filter
ALF_INIT(8);
SAO_INIT(8, avx2);
}
#endif
break;
case 10:
#if HAVE_SSE4_EXTERNAL
if (EXTERNAL_SSE4(cpu_flags)) {
MC_LINK_SSE4(10);
}
#endif
#if HAVE_AVX2_EXTERNAL
if (EXTERNAL_AVX2_FAST(cpu_flags)) {
// inter
AVG_INIT(10, avx2);
DMVR_INIT(10);
MC_LINKS_AVX2(10);
MC_LINKS_16BPC_AVX2(10);
OF_INIT(10);
SAD_INIT();
// filter
ALF_INIT(10);
SAO_INIT(10, avx2);
}
#endif
break;
case 12:
#if HAVE_SSE4_EXTERNAL
if (EXTERNAL_SSE4(cpu_flags)) {
MC_LINK_SSE4(12);
}
#endif
#if HAVE_AVX2_EXTERNAL
if (EXTERNAL_AVX2_FAST(cpu_flags)) {
// inter
AVG_INIT(12, avx2);
DMVR_INIT(12);
MC_LINKS_AVX2(12);
MC_LINKS_16BPC_AVX2(12);
OF_INIT(12);
SAD_INIT();
// filter
ALF_INIT(12);
SAO_INIT(12, avx2);
}
#endif
break;
default:
break;
}
#endif
}
ffmpeg/libavcodec/x86/vvc/mc.asm
+302
View File
@@ -0,0 +1,302 @@
; /*
; * Provide SIMD MC functions for VVC decoding
; *
; * Copyright © 2021, VideoLAN and dav1d authors
; * Copyright © 2021, Two Orioles, LLC
; * All rights reserved.
; *
; * Copyright (c) 2023-2024 Nuo Mi
; * Copyright (c) 2023-2024 Wu Jianhua
; *
; * This file is part of FFmpeg.
; *
; * FFmpeg is free software; you can redistribute it and/or
; * modify it under the terms of the GNU Lesser General Public
; * License as published by the Free Software Foundation; either
; * version 2.1 of the License, or (at your option) any later version.
; *
; * FFmpeg is distributed in the hope that it will be useful,
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
; * Lesser General Public License for more details.
; *
; * You should have received a copy of the GNU Lesser General Public
; * License along with FFmpeg; if not, write to the Free Software
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
; */
%include "libavutil/x86/x86util.asm"
%define MAX_PB_SIZE 128
SECTION_RODATA 32
%if ARCH_X86_64
%if HAVE_AVX2_EXTERNAL
pw_0 times 2 dw 0
pw_1 times 2 dw 1
pw_4 times 2 dw 4
pw_12 times 2 dw 12
pw_256 times 2 dw 256
%macro AVG_JMP_TABLE 3-*
%xdefine %1_%2_%3_table (%%table - 2*%4)
%xdefine %%base %1_%2_%3_table
%xdefine %%prefix mangle(private_prefix %+ _vvc_%1_%2bpc_%3)
%%table:
%rep %0 - 3
dd %%prefix %+ .w%4 - %%base
%rotate 1
%endrep
%endmacro
AVG_JMP_TABLE avg, 8, avx2, 2, 4, 8, 16, 32, 64, 128
AVG_JMP_TABLE avg, 16, avx2, 2, 4, 8, 16, 32, 64, 128
AVG_JMP_TABLE w_avg, 8, avx2, 2, 4, 8, 16, 32, 64, 128
AVG_JMP_TABLE w_avg, 16, avx2, 2, 4, 8, 16, 32, 64, 128
SECTION .text
%macro AVG_W16_FN 3 ; bpc, op, count
%assign %%i 0
%rep %3
%define off %%i
AVG_LOAD_W16 0, off
%2
AVG_SAVE_W16 %1, 0, off
AVG_LOAD_W16 1, off
%2
AVG_SAVE_W16 %1, 1, off
%assign %%i %%i+1
%endrep
%endmacro
%macro AVG_FN 2 ; bpc, op
jmp wq
.w2:
movd xm0, [src0q]
pinsrd xm0, [src0q + AVG_SRC_STRIDE], 1
movd xm1, [src1q]
pinsrd xm1, [src1q + AVG_SRC_STRIDE], 1
%2
AVG_SAVE_W2 %1
AVG_LOOP_END .w2
.w4:
movq xm0, [src0q]
pinsrq xm0, [src0q + AVG_SRC_STRIDE], 1
movq xm1, [src1q]
pinsrq xm1, [src1q + AVG_SRC_STRIDE], 1
%2
AVG_SAVE_W4 %1
AVG_LOOP_END .w4
.w8:
vinserti128 m0, m0, [src0q], 0
vinserti128 m0, m0, [src0q + AVG_SRC_STRIDE], 1
vinserti128 m1, m1, [src1q], 0
vinserti128 m1, m1, [src1q + AVG_SRC_STRIDE], 1
%2
AVG_SAVE_W8 %1
AVG_LOOP_END .w8
.w16:
AVG_W16_FN %1, %2, 1
AVG_LOOP_END .w16
.w32:
AVG_W16_FN %1, %2, 2
AVG_LOOP_END .w32
.w64:
AVG_W16_FN %1, %2, 4
AVG_LOOP_END .w64
.w128:
AVG_W16_FN %1, %2, 8
AVG_LOOP_END .w128
.ret:
RET
%endmacro
%macro AVG 0
paddsw m0, m1
pmulhrsw m0, m2
CLIPW m0, m3, m4
%endmacro
%macro W_AVG 0
punpckhwd m5, m0, m1
pmaddwd m5, m3
paddd m5, m4
psrad m5, xm2
punpcklwd m0, m0, m1
pmaddwd m0, m3
paddd m0, m4
psrad m0, xm2
packssdw m0, m5
CLIPW m0, m6, m7
%endmacro
%macro AVG_LOAD_W16 2 ; line, offset
movu m0, [src0q + %1 * AVG_SRC_STRIDE + %2 * 32]
movu m1, [src1q + %1 * AVG_SRC_STRIDE + %2 * 32]
%endmacro
%macro AVG_SAVE_W2 1 ;bpc
%if %1 == 16
pextrd [dstq], xm0, 0
pextrd [dstq + strideq], xm0, 1
%else
packuswb m0, m0
pextrw [dstq], xm0, 0
pextrw [dstq + strideq], xm0, 1
%endif
%endmacro
%macro AVG_SAVE_W4 1 ;bpc
%if %1 == 16
pextrq [dstq], xm0, 0
pextrq [dstq + strideq], xm0, 1
%else
packuswb m0, m0
pextrd [dstq], xm0, 0
pextrd [dstq + strideq], xm0, 1
%endif
%endmacro
%macro AVG_SAVE_W8 1 ;bpc
%if %1 == 16
vextracti128 [dstq], m0, 0
vextracti128 [dstq + strideq], m0, 1
%else
packuswb m0, m0
vpermq m0, m0, 1000b
pextrq [dstq], xm0, 0
pextrq [dstq + strideq], xm0, 1
%endif
%endmacro
%macro AVG_SAVE_W16 3 ; bpc, line, offset
%if %1 == 16
movu [dstq + %2 * strideq + %3 * 32], m0
%else
packuswb m0, m0
vpermq m0, m0, 1000b
vextracti128 [dstq + %2 * strideq + %3 * 16], m0, 0
%endif
%endmacro
%macro AVG_LOOP_END 1
sub hd, 2
je .ret
lea src0q, [src0q + 2 * AVG_SRC_STRIDE]
lea src1q, [src1q + 2 * AVG_SRC_STRIDE]
lea dstq, [dstq + 2 * strideq]
jmp %1
%endmacro
%define AVG_SRC_STRIDE MAX_PB_SIZE*2
;void ff_vvc_avg_%1bpc_avx2(uint8_t *dst, ptrdiff_t dst_stride,
; const int16_t *src0, const int16_t *src1, intptr_t width, intptr_t height, intptr_t pixel_max);
%macro VVC_AVG_AVX2 1
cglobal vvc_avg_%1bpc, 4, 7, 5, dst, stride, src0, src1, w, h, bd
movifnidn hd, hm
pxor m3, m3 ; pixel min
vpbroadcastw m4, bdm ; pixel max
movifnidn bdd, bdm
inc bdd
tzcnt bdd, bdd ; bit depth
sub bdd, 8
movd xm0, bdd
vpbroadcastd m1, [pw_4]
pminuw m0, m1
vpbroadcastd m2, [pw_256]
psllw m2, xm0 ; shift
lea r6, [avg_%1 %+ SUFFIX %+ _table]
tzcnt wd, wm
movsxd wq, dword [r6+wq*4]
add wq, r6
AVG_FN %1, AVG
%endmacro
;void ff_vvc_w_avg_%1bpc_avx(uint8_t *dst, ptrdiff_t dst_stride,
; const int16_t *src0, const int16_t *src1, intptr_t width, intptr_t height,
; intptr_t denom, intptr_t w0, intptr_t w1, intptr_t o0, intptr_t o1, intptr_t pixel_max);
%macro VVC_W_AVG_AVX2 1
cglobal vvc_w_avg_%1bpc, 4, 8, 8, dst, stride, src0, src1, w, h, t0, t1
movifnidn hd, hm
movifnidn t0d, r8m ; w1
shl t0d, 16
mov t0w, r7m ; w0
movd xm3, t0d
vpbroadcastd m3, xm3 ; w0, w1
pxor m6, m6 ;pixel min
vpbroadcastw m7, r11m ;pixel max
mov t1q, rcx ; save ecx
mov ecx, r11m
inc ecx ; bd
tzcnt ecx, ecx
sub ecx, 8
mov t0d, r9m ; o0
add t0d, r10m ; o1
shl t0d, cl
inc t0d ;((o0 + o1) << (BIT_DEPTH - 8)) + 1
neg ecx
add ecx, 4 ; bd - 12
cmovl ecx, [pw_0]
add ecx, 3
add ecx, r6m
movd xm2, ecx ; shift
dec ecx
shl t0d, cl
movd xm4, t0d
vpbroadcastd m4, xm4 ; offset
mov rcx, t1q ; restore ecx
lea r6, [w_avg_%1 %+ SUFFIX %+ _table]
tzcnt wd, wm
movsxd wq, dword [r6+wq*4]
add wq, r6
AVG_FN %1, W_AVG
%endmacro
INIT_YMM avx2
VVC_AVG_AVX2 16
VVC_AVG_AVX2 8
VVC_W_AVG_AVX2 16
VVC_W_AVG_AVX2 8
%endif
%endif
ffmpeg/libavcodec/x86/vvc/of.asm
+385
View File
@@ -0,0 +1,385 @@
; /*
; * Provide AVX2 luma optical flow functions for VVC decoding
; * Copyright (c) 2024 Nuo Mi
; *
; * This file is part of FFmpeg.
; *
; * FFmpeg is free software; you can redistribute it and/or
; * modify it under the terms of the GNU Lesser General Public
; * License as published by the Free Software Foundation; either
; * version 2.1 of the License, or (at your option) any later version.
; *
; * FFmpeg is distributed in the hope that it will be useful,
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
; * Lesser General Public License for more details.
; *
; * You should have received a copy of the GNU Lesser General Public
; * License along with FFmpeg; if not, write to the Free Software
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
; */
%include "libavutil/x86/x86util.asm"
%define MAX_PB_SIZE 128
%define SRC_STRIDE (MAX_PB_SIZE * 2)
%define SRC_PS 2 ; source pixel size, sizeof(int16_t)
%define BDOF_STACK_SIZE 10 ; (4 + 1) * 2, 4 lines + the first line, *2 for h and v
%define bdof_stack_offset(line) ((line) * 2 % BDOF_STACK_SIZE * mmsize)
%define SHIFT 6
%define SHIFT2 4
SECTION_RODATA 32
pd_15 times 8 dd 15
pd_m15 times 8 dd -15
pb_shuffle_w8 times 2 db 0, 1, 0xff, 0xff, 8, 9, 0xff, 0xff, 6, 7, 0xff, 0xff, 14, 15, 0xff, 0xff
pb_shuffle_w16 times 2 db 0, 1, 0xff, 0xff, 6, 7, 0xff, 0xff, 8, 9, 0xff, 0xff, 14, 15, 0xff, 0xff
pd_perm_w16 dd 0, 2, 1, 4, 3, 6, 5, 7
%if ARCH_X86_64
%if HAVE_AVX2_EXTERNAL
SECTION .text
INIT_YMM avx2
; dst = (src0 >> shift) - (src1 >> shift)
%macro DIFF 5 ; dst, src0, src1, shift, tmp
psraw %1, %2, %4
psraw %5, %3, %4
psubw %1, %5
%endmacro
%macro LOAD_GRAD_H 4 ; dst, src, off, tmp
movu %1, [%2 + %3 + 2 * SRC_PS]
movu %4, [%2 + %3]
DIFF %1, %1, %4, SHIFT, %4
%endmacro
%macro SUM_GRAD 2 ;(dst/grad0, grad1)
paddw %1, %2
psraw %1, 1 ; shift3
%endmacro
%macro APPLY_BDOF_MIN_BLOCK_LINE 5 ; dst, vx, vy, tmp, line_num
%define off bdof_stack_offset(%5)
pmullw %1, %2, [rsp + off + 0 * mmsize] ; vx * (gradient_h[0] - gradient_h[1])
pmullw %4, %3, [rsp + off + 1 * mmsize] ; vy * (gradient_v[0] - gradient_v[1])
paddw %1, [src0q + (%5 + 1) * SRC_STRIDE + SRC_PS]
paddw %4, [src1q + (%5 + 1) * SRC_STRIDE + SRC_PS]
paddsw %1, %4 ; src0[x] + src1[x] + bdof_offset
pmulhrsw %1, m11
CLIPW %1, m9, m10
%endmacro
%macro SAVE_8BPC 2 ; dst, src
packuswb m%2, m%2
vpermq m%2, m%2, q0020
cmp wd, 16
je %%w16
movq %1, xm%2
jmp %%wend
%%w16:
movu %1, xm%2
%%wend:
%endmacro
%macro SAVE_16BPC 2 ; dst, src
cmp wd, 16
je %%w16
movu %1, xm%2
jmp %%wend
%%w16:
movu %1, m%2
%%wend:
%endmacro
%macro SAVE 2 ; dst, src
cmp pixel_maxd, (1 << 8) - 1
jne %%save_16bpc
SAVE_8BPC %1, %2
jmp %%end
%%save_16bpc:
SAVE_16BPC %1, %2
%%end:
%endmacro
; [rsp + even * mmsize] are gradient_h[0] - gradient_h[1]
; [rsp + odd * mmsize] are gradient_v[0] - gradient_v[1]
%macro APPLY_BDOF_MIN_BLOCK 4 ; block_num, vx, vy, bd
pxor m9, m9
movd xm10, pixel_maxd
vpbroadcastw m10, xm10
lea tmp0d, [pixel_maxd + 1]
movd xm11, tmp0d
VPBROADCASTW m11, xm11 ;shift_4 for pmulhrsw
APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 0
SAVE [dstq + 0 * dsq], 6
APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 1
SAVE [dstq + 1 * dsq], 6
APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 2
SAVE [dstq + 2 * dsq], 6
APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 3
SAVE [dstq + ds3q], 6
%endmacro
%macro SUM_MIN_BLOCK_W16 4 ; src/dst, shuffle, perm, tmp
pshufb %4, %1, %2
vpermd %4, %3, %4
paddw %1, %4
%endmacro
%macro SUM_MIN_BLOCK_W8 3 ; src/dst, shuffle, tmp
pshufb %3, %1, %2
paddw %1, %3
%endmacro
%macro BDOF_PROF_GRAD 2 ; line_no, last_line
%assign i0 (%1 + 0) % 3
%assign j0 (%1 + 1) % 3
%assign k0 (%1 + 2) % 3
%assign i1 3 + (%1 + 0) % 3
%assign j1 3 + (%1 + 1) % 3
%assign k1 3 + (%1 + 2) % 3
; we cached src0 in m0 to m2
%define t0 m %+ i0
%define c0 m %+ j0
%define b0 m %+ k0
; we cached src1 in m3 to m5
%define t1 m %+ i1
%define c1 m %+ j1
%define b1 m %+ k1
%define ndiff t1
%define off bdof_stack_offset(%1)
movu b0, [src0q + (%1 + 2) * SRC_STRIDE + SRC_PS]
movu b1, [src1q + (%1 + 2) * SRC_STRIDE + SRC_PS]
; gradient_v[0], gradient_v[1]
DIFF m6, b0, t0, SHIFT, t0
DIFF m7, b1, t1, SHIFT, t1
; save gradient_v[0] - gradient_v[1]
psubw m10, m6, m7
mova [rsp + off + mmsize], m10
; gradient_h[0], gradient_h[1]
LOAD_GRAD_H m8, src0q, (%1 + 1) * SRC_STRIDE, t0
LOAD_GRAD_H m9, src1q, (%1 + 1) * SRC_STRIDE, t1
; save gradient_h[0] - gradient_h[1]
psubw m11, m8, m9
mova [rsp + off], m11
SUM_GRAD m8, m9 ; temph
SUM_GRAD m6, m7 ; tempv
DIFF ndiff, c1, c0, SHIFT2, t0 ; -diff
psignw m7, ndiff, m8 ; sgxdi
psignw m9, ndiff, m6 ; sgydi
psignw m10, m8, m6 ; sgxgy
pabsw m6, m6 ; sgy2
pabsw m8, m8 ; sgx2
; use t0, t1 as temporary buffers
cmp wd, 16
je %%w16
mova t0, [pb_shuffle_w8]
SUM_MIN_BLOCK_W8 m6, t0, m11
SUM_MIN_BLOCK_W8 m7, t0, m11
SUM_MIN_BLOCK_W8 m8, t0, m11
SUM_MIN_BLOCK_W8 m9, t0, m11
SUM_MIN_BLOCK_W8 m10, t0, m11
jmp %%wend
%%w16:
mova t0, [pb_shuffle_w16]
mova t1, [pd_perm_w16]
SUM_MIN_BLOCK_W16 m6, t0, t1, m11
SUM_MIN_BLOCK_W16 m7, t0, t1, m11
SUM_MIN_BLOCK_W16 m8, t0, t1, m11
SUM_MIN_BLOCK_W16 m9, t0, t1, m11
SUM_MIN_BLOCK_W16 m10, t0, t1, m11
%%wend:
vpblendd m11, m8, m7, 10101010b
vpblendd m7, m8, m7, 01010101b
pshufd m7, m7, q2301
paddw m8, m7, m11 ;4 x (2sgx2, 2sgxdi)
vpblendd m11, m6, m9, 10101010b
vpblendd m9, m6, m9, 01010101b
pshufd m9, m9, q2301
paddw m6, m9, m11 ;4 x (2sgy2, 2sgydi)
vpblendw m11, m8, m6, 10101010b
vpblendw m6, m8, m6, 01010101b
pshuflw m6, m6, q2301
pshufhw m6, m6, q2301
paddw m8, m6, m11 ; 4 x (4sgx2, 4sgy2, 4sgxdi, 4sgydi)
%if (%1) == 0 || (%2)
; pad for top and bottom
paddw m8, m8
paddw m10, m10
%endif
paddw m12, m8
paddw m13, m10
%endmacro
%macro LOG2 5 ; log_sum, src, cmp, shift, tmp
pcmpgtw %5, %2, %3
pandd %5, %4
paddw %1, %5
psrlw %2, %5
psrlw %4, 1
psrlw %3, %4
%endmacro
%macro LOG2 2 ; dst/src, offset
pextrw tmp0d, xm%1, %2
bsr tmp0d, tmp0d
pinsrw xm%1, tmp0d, %2
%endmacro
%macro LOG2 1 ; dst/src
LOG2 %1, 0
LOG2 %1, 1
LOG2 %1, 2
LOG2 %1, 3
LOG2 %1, 4
LOG2 %1, 5
LOG2 %1, 6
LOG2 %1, 7
%endmacro
; %1: 4 (sgx2, sgy2, sgxdi, gydi)
; %2: 4 (4sgxgy)
%macro BDOF_VX_VY 2 ;
pshufd m6, m%1, q0032
punpckldq m%1, m6
vextracti128 xm7, m%1, 1
punpcklqdq m8, m%1, m7 ; 4 (sgx2, sgy2)
punpckhqdq m9, m%1, m7 ; 4 (sgxdi, sgydi)
mova m10, m8
LOG2 10 ; 4 (log2(sgx2), log2(sgy2))
; Promote to dword since vpsrlvw is AVX-512 only
pmovsxwd m8, xm8
pmovsxwd m9, xm9
pmovsxwd m10, xm10
pslld m9, 2 ; 4 (log2(sgx2) << 2, log2(sgy2) << 2)
psignd m11, m9, m8
vpsravd m11, m11, m10
CLIPD m11, [pd_m15], [pd_15] ; 4 (vx, junk)
pshuflw m%1, m11, q0000
pshufhw m%1, m%1, q0000 ; 4 (2junk, 2vx)
psllq m6, m%2, 32
paddw m%2, m6
pmaddwd m%2, m%1 ; 4 (junk, vx * sgxgy)
psrad m%2, 1
psubd m9, m%2 ; 4 (junk, (sgydi << 2) - (vx * sgxgy >> 1))
psignd m9, m8
vpsravd m%2, m9, m10
CLIPD m%2, [pd_m15], [pd_15] ; 4 (junk, vy)
pshuflw m%2, m%2, q2222
pshufhw m%2, m%2, q2222 ; 4 (4vy)
%endmacro
%macro BDOF_MINI_BLOCKS 2 ; (block_num, last_block)
%if (%1) == 0
movu m0, [src0q + 0 * SRC_STRIDE + SRC_PS]
movu m1, [src0q + 1 * SRC_STRIDE + SRC_PS]
movu m3, [src1q + 0 * SRC_STRIDE + SRC_PS]
movu m4, [src1q + 1 * SRC_STRIDE + SRC_PS]
pxor m12, m12
pxor m13, m13
BDOF_PROF_GRAD 0, 0
%endif
mova m14, m12
mova m15, m13
pxor m12, m12
pxor m13, m13
BDOF_PROF_GRAD %1 * 4 + 1, 0
BDOF_PROF_GRAD %1 * 4 + 2, 0
paddw m14, m12
paddw m15, m13
pxor m12, m12
pxor m13, m13
BDOF_PROF_GRAD %1 * 4 + 3, %2
%if (%2) == 0
BDOF_PROF_GRAD %1 * 4 + 4, 0
%endif
paddw m14, m12
paddw m15, m13
BDOF_VX_VY 14, 15
APPLY_BDOF_MIN_BLOCK %1, m14, m15, bd
lea dstq, [dstq + 4 * dsq]
%endmacro
;void ff_vvc_apply_bdof_%1(uint8_t *dst, const ptrdiff_t dst_stride, int16_t *src0, int16_t *src1,
; const int w, const int h, const int int pixel_max)
%macro BDOF_AVX2 0
cglobal vvc_apply_bdof, 7, 10, 16, BDOF_STACK_SIZE*32, dst, ds, src0, src1, w, h, pixel_max, ds3, tmp0, tmp1
lea ds3q, [dsq * 3]
sub src0q, SRC_STRIDE + SRC_PS
sub src1q, SRC_STRIDE + SRC_PS
BDOF_MINI_BLOCKS 0, 0
cmp hd, 16
je .h16
BDOF_MINI_BLOCKS 1, 1
jmp .end
.h16:
BDOF_MINI_BLOCKS 1, 0
BDOF_MINI_BLOCKS 2, 0
BDOF_MINI_BLOCKS 3, 1
.end:
RET
%endmacro
%macro VVC_OF_AVX2 0
BDOF_AVX2
%endmacro
VVC_OF_AVX2
%endif ; HAVE_AVX2_EXTERNAL
%endif ; ARCH_X86_64
ffmpeg/libavcodec/x86/vvc/sad.asm
+134
View File
@@ -0,0 +1,134 @@
; /*
; * Provide SIMD DMVR SAD functions for VVC decoding
; *
; * Copyright (c) 2024 Stone Chen
; *
; * This file is part of FFmpeg.
; *
; * FFmpeg is free software; you can redistribute it and/or
; * modify it under the terms of the GNU Lesser General Public
; * License as published by the Free Software Foundation; either
; * version 2.1 of the License, or (at your option) any later version.
; *
; * FFmpeg is distributed in the hope that it will be useful,
; * but WITHOUT ANY WARRANTY; without even the implied warranty of
; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
; * Lesser General Public License for more details.
; *
; * You should have received a copy of the GNU Lesser General Public
; * License along with FFmpeg; if not, write to the Free Software
; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
; */
%include "libavutil/x86/x86util.asm"
%define MAX_PB_SIZE 128
%define ROWS 2
SECTION_RODATA
pw_1: times 2 dw 1
; DMVR SAD is only calculated on even rows to reduce complexity
; Additionally the only valid sizes are 8x16, 16x8, and 16x16
SECTION .text
%macro MIN_MAX_SAD 3
pminuw %3, %2, %1
pmaxuw %1, %2, %1
psubusw %1, %1, %3
%endmacro
%macro HORIZ_ADD 3 ; xm0, xm1, m1
vextracti128 %1, %3, q0001 ; 3 2 1 0
paddd %1, %2 ; xm0 (7 + 3) (6 + 2) (5 + 1) (4 + 0)
pshufd %2, %1, q0032 ; xm1 - - (7 + 3) (6 + 2)
paddd %1, %1, %2 ; xm0 _ _ (5 1 7 3) (4 0 6 2)
pshufd %2, %1, q0001 ; xm1 _ _ (5 1 7 3) (5 1 7 3)
paddd %1, %1, %2 ; (01234567)
%endmacro
%if ARCH_X86_64
%if HAVE_AVX2_EXTERNAL
INIT_YMM avx2
cglobal vvc_sad, 6, 9, 5, src1, src2, dx, dy, block_w, block_h, off1, off2, row_idx
movsxdifnidn dxq, dxd
movsxdifnidn dyq, dyd
sub dxq, 2
sub dyq, 2
mov off1q, 2
mov off2q, 2
add off1q, dyq
sub off2q, dyq
shl off1q, 7
shl off2q, 7
add off1q, dxq
sub off2q, dxq
lea src1q, [src1q + off1q * 2 + 2 * 2]
lea src2q, [src2q + off2q * 2 + 2 * 2]
pxor m3, m3
vpbroadcastd m4, [pw_1]
cmp block_wd, 16
je vvc_sad_16
vvc_sad_8:
.loop_height:
movu xm0, [src1q]
vinserti128 m0, m0, [src1q + MAX_PB_SIZE * ROWS * 2], 1
movu xm1, [src2q]
vinserti128 m1, m1, [src2q + MAX_PB_SIZE * ROWS * 2], 1
MIN_MAX_SAD m1, m0, m2
pmaddwd m1, m4
paddd m3, m1
add src1q, 2 * MAX_PB_SIZE * ROWS * 2
add src2q, 2 * MAX_PB_SIZE * ROWS * 2
sub block_hd, 4
jg .loop_height
HORIZ_ADD xm0, xm3, m3
movd eax, xm0
RET
vvc_sad_16:
sar block_wd, 4
.loop_height:
mov off1q, src1q
mov off2q, src2q
mov row_idxd, block_wd
.loop_width:
movu m0, [src1q]
movu m1, [src2q]
MIN_MAX_SAD m1, m0, m2
pmaddwd m1, m4
paddd m3, m1
add src1q, 32
add src2q, 32
dec row_idxd
jg .loop_width
lea src1q, [off1q + ROWS * MAX_PB_SIZE * 2]
lea src2q, [off2q + ROWS * MAX_PB_SIZE * 2]
sub block_hd, 2
jg .loop_height
HORIZ_ADD xm0, xm3, m3
movd eax, xm0
RET
%endif
%endif
ffmpeg/libavcodec/x86/vvc/sao.asm
+73
View File
@@ -0,0 +1,73 @@
;******************************************************************************
;* SIMD optimized SAO functions for VVC 8bit decoding
;*
;* Copyright (c) 2024 Shaun Loo
;* Copyright (c) 2024 Nuo Mi
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%define MAX_PB_SIZE 128
%include "libavcodec/x86/h26x/h2656_sao.asm"
%macro VVC_SAO_BAND_FILTER 2
H2656_SAO_BAND_FILTER vvc, %1, %2
%endmacro
%macro VVC_SAO_BAND_FILTER_FUNCS 0
VVC_SAO_BAND_FILTER 8, 0
VVC_SAO_BAND_FILTER 16, 1
VVC_SAO_BAND_FILTER 32, 2
VVC_SAO_BAND_FILTER 48, 2
VVC_SAO_BAND_FILTER 64, 4
VVC_SAO_BAND_FILTER 80, 4
VVC_SAO_BAND_FILTER 96, 6
VVC_SAO_BAND_FILTER 112, 6
VVC_SAO_BAND_FILTER 128, 8
%endmacro
%if HAVE_AVX2_EXTERNAL
INIT_XMM avx2
VVC_SAO_BAND_FILTER 8, 0
VVC_SAO_BAND_FILTER 16, 1
INIT_YMM avx2
VVC_SAO_BAND_FILTER 32, 1
VVC_SAO_BAND_FILTER 48, 1
VVC_SAO_BAND_FILTER 64, 2
VVC_SAO_BAND_FILTER 80, 2
VVC_SAO_BAND_FILTER 96, 3
VVC_SAO_BAND_FILTER 112, 3
VVC_SAO_BAND_FILTER 128, 4
%endif
%macro VVC_SAO_EDGE_FILTER 2-3
H2656_SAO_EDGE_FILTER vvc, %{1:-1}
%endmacro
%if HAVE_AVX2_EXTERNAL
INIT_XMM avx2
VVC_SAO_EDGE_FILTER 8, 0
VVC_SAO_EDGE_FILTER 16, 1, a
INIT_YMM avx2
VVC_SAO_EDGE_FILTER 32, 1, a
VVC_SAO_EDGE_FILTER 48, 1, u
VVC_SAO_EDGE_FILTER 64, 2, a
VVC_SAO_EDGE_FILTER 80, 2, u
VVC_SAO_EDGE_FILTER 96, 3, a
VVC_SAO_EDGE_FILTER 112, 3, u
VVC_SAO_EDGE_FILTER 128, 4, a
%endif
ffmpeg/libavcodec/x86/vvc/sao_10bit.asm
+113
View File
@@ -0,0 +1,113 @@
;******************************************************************************
;* SIMD optimized SAO functions for VVC 10/12bit decoding
;*
;* Copyright (c) 2024 Shaun Loo
;* Copyright (c) 2024 Nuo Mi
;*
;* This file is part of FFmpeg.
;*
;* FFmpeg is free software; you can redistribute it and/or
;* modify it under the terms of the GNU Lesser General Public
;* License as published by the Free Software Foundation; either
;* version 2.1 of the License, or (at your option) any later version.
;*
;* FFmpeg is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
;* Lesser General Public License for more details.
;*
;* You should have received a copy of the GNU Lesser General Public
;* License along with FFmpeg; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
;******************************************************************************
%define MAX_PB_SIZE 128
%include "libavcodec/x86/h26x/h2656_sao_10bit.asm"
%macro VVC_SAO_BAND_FILTER 3
H2656_SAO_BAND_FILTER vvc, %1, %2, %3
%endmacro
%macro VVC_SAO_BAND_FILTER_FUNCS 1
VVC_SAO_BAND_FILTER %1, 8, 1
VVC_SAO_BAND_FILTER %1, 16, 2
VVC_SAO_BAND_FILTER %1, 32, 4
VVC_SAO_BAND_FILTER %1, 48, 6
VVC_SAO_BAND_FILTER %1, 64, 8
VVC_SAO_BAND_FILTER %1, 80, 10
VVC_SAO_BAND_FILTER %1, 96, 12
VVC_SAO_BAND_FILTER %1, 112, 14
VVC_SAO_BAND_FILTER %1, 128, 16
%endmacro
%macro VVC_SAO_BAND_FILTER_FUNCS 0
VVC_SAO_BAND_FILTER_FUNCS 10
VVC_SAO_BAND_FILTER_FUNCS 12
%endmacro
INIT_XMM sse2
VVC_SAO_BAND_FILTER_FUNCS
INIT_XMM avx
VVC_SAO_BAND_FILTER_FUNCS
%if HAVE_AVX2_EXTERNAL
%macro VVC_SAO_BAND_FILTER_FUNCS_AVX2 1
INIT_XMM avx2
VVC_SAO_BAND_FILTER %1, 8, 1
INIT_YMM avx2
VVC_SAO_BAND_FILTER %1, 16, 1
VVC_SAO_BAND_FILTER %1, 32, 2
VVC_SAO_BAND_FILTER %1, 48, 3
VVC_SAO_BAND_FILTER %1, 64, 4
VVC_SAO_BAND_FILTER %1, 80, 5
VVC_SAO_BAND_FILTER %1, 96, 6
VVC_SAO_BAND_FILTER %1, 112, 7
VVC_SAO_BAND_FILTER %1, 128, 8
%endmacro
VVC_SAO_BAND_FILTER_FUNCS_AVX2 10
VVC_SAO_BAND_FILTER_FUNCS_AVX2 12
%endif ; HAVE_AVX2_EXTERNAL
%macro VVC_SAO_EDGE_FILTER 3
H2656_SAO_EDGE_FILTER vvc, %1, %2, %3
%endmacro
%macro VVC_SAO_EDGE_FILTER_FUNCS 1
VVC_SAO_EDGE_FILTER %1, 8, 1
VVC_SAO_EDGE_FILTER %1, 16, 2
VVC_SAO_EDGE_FILTER %1, 32, 4
VVC_SAO_EDGE_FILTER %1, 48, 6
VVC_SAO_EDGE_FILTER %1, 64, 8
VVC_SAO_EDGE_FILTER %1, 80, 10
VVC_SAO_EDGE_FILTER %1, 96, 12
VVC_SAO_EDGE_FILTER %1, 112, 14
VVC_SAO_EDGE_FILTER %1, 128, 16
%endmacro
INIT_XMM sse2
VVC_SAO_EDGE_FILTER_FUNCS 10
VVC_SAO_EDGE_FILTER_FUNCS 12
%if HAVE_AVX2_EXTERNAL
%macro VVC_SAO_EDGE_FILTER_FUNCS_AVX2 1
INIT_XMM avx2
VVC_SAO_EDGE_FILTER %1, 8, 1
INIT_YMM avx2
VVC_SAO_EDGE_FILTER %1, 16, 1
VVC_SAO_EDGE_FILTER %1, 32, 2
VVC_SAO_EDGE_FILTER %1, 48, 3
VVC_SAO_EDGE_FILTER %1, 64, 4
VVC_SAO_EDGE_FILTER %1, 80, 5
VVC_SAO_EDGE_FILTER %1, 96, 6
VVC_SAO_EDGE_FILTER %1, 112, 7
VVC_SAO_EDGE_FILTER %1, 128, 8
%endmacro
VVC_SAO_EDGE_FILTER_FUNCS_AVX2 10
VVC_SAO_EDGE_FILTER_FUNCS_AVX2 12
%endif ; HAVE_AVX2_EXTERNAL