...
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
OBJS += riscv/float_dsp_init.o \
|
||||
riscv/fixed_dsp_init.o \
|
||||
riscv/lls_init.o \
|
||||
riscv/cpu.o \
|
||||
riscv/cpu_common.o
|
||||
RVV-OBJS += riscv/float_dsp_rvv.o \
|
||||
riscv/fixed_dsp_rvv.o \
|
||||
riscv/lls_rvv.o
|
||||
@@ -0,0 +1,239 @@
|
||||
/*
|
||||
* Copyright © 2022 Rémi Denis-Courmont.
|
||||
* Loosely based on earlier work copyrighted by Måns Rullgård, 2008.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#if defined (__riscv_float_abi_soft)
|
||||
#define NOHWF
|
||||
#define NOHWD
|
||||
#define HWF #
|
||||
#define HWD #
|
||||
#elif defined (__riscv_float_abi_single)
|
||||
#define NOHWF #
|
||||
#define NOHWD
|
||||
#define HWF
|
||||
#define HWD #
|
||||
#else
|
||||
#define NOHWF #
|
||||
#define NOHWD #
|
||||
#define HWF
|
||||
#define HWD
|
||||
#endif
|
||||
|
||||
.macro archadd ext=, more:vararg
|
||||
.ifnb \ext
|
||||
.ifc \ext, b
|
||||
# B was defined later, is known to fewer assemblers.
|
||||
archadd zba, zbb, zbs
|
||||
.else
|
||||
.option arch, +\ext
|
||||
.endif
|
||||
archadd \more
|
||||
.endif
|
||||
.endm
|
||||
|
||||
.macro func sym, exts:vararg
|
||||
.text
|
||||
.option push
|
||||
archadd \exts
|
||||
|
||||
.global \sym
|
||||
.hidden \sym
|
||||
.type \sym, %function
|
||||
.option push
|
||||
.option norvc
|
||||
.align 2
|
||||
\sym:
|
||||
.option pop
|
||||
|
||||
.macro endfunc
|
||||
.size \sym, . - \sym
|
||||
.option pop
|
||||
.previous
|
||||
.purgem endfunc
|
||||
.endm
|
||||
.endm
|
||||
|
||||
.macro const sym, align=3, relocate=0
|
||||
.if \relocate
|
||||
.pushsection .data.rel.ro
|
||||
.else
|
||||
.pushsection .rodata
|
||||
.endif
|
||||
.align \align
|
||||
\sym:
|
||||
|
||||
.macro endconst
|
||||
.size \sym, . - \sym
|
||||
.popsection
|
||||
.purgem endconst
|
||||
.endm
|
||||
.endm
|
||||
|
||||
#if !defined (__riscv_zicfilp)
|
||||
.macro lpad lpl
|
||||
auipc zero, \lpl
|
||||
.endm
|
||||
#endif
|
||||
|
||||
#if defined (__riscv_v_elen)
|
||||
# define RV_V_ELEN __riscv_v_elen
|
||||
#else
|
||||
/* Run-time detection of the V extension implies ELEN >= 64. */
|
||||
# define RV_V_ELEN 64
|
||||
#endif
|
||||
#if RV_V_ELEN == 32
|
||||
# define VSEW_MAX 2
|
||||
#else
|
||||
# define VSEW_MAX 3
|
||||
#endif
|
||||
|
||||
.macro parse_vtype ew, tp, mp
|
||||
.ifc \ew,e8
|
||||
.equ vsew, 0
|
||||
.else
|
||||
.ifc \ew,e16
|
||||
.equ vsew, 1
|
||||
.else
|
||||
.ifc \ew,e32
|
||||
.equ vsew, 2
|
||||
.else
|
||||
.ifc \ew,e64
|
||||
.equ vsew, 3
|
||||
.else
|
||||
.error "Unknown element width \ew"
|
||||
.endif
|
||||
.endif
|
||||
.endif
|
||||
.endif
|
||||
|
||||
.ifc \tp,tu
|
||||
.equ tp, 0
|
||||
.else
|
||||
.ifc \tp,ta
|
||||
.equ tp, 1
|
||||
.else
|
||||
.error "Unknown tail policy \tp"
|
||||
.endif
|
||||
.endif
|
||||
|
||||
.ifc \mp,mu
|
||||
.equ mp, 0
|
||||
.else
|
||||
.ifc \mp,ma
|
||||
.equ mp, 1
|
||||
.else
|
||||
.error "Unknown mask policy \mp"
|
||||
.endif
|
||||
.endif
|
||||
.endm
|
||||
|
||||
/**
|
||||
* Gets the vector type with the smallest suitable LMUL value.
|
||||
* @param[out] rd vector type destination register
|
||||
* @param vl vector length constant
|
||||
* @param ew element width: e8, e16, e32 or e64
|
||||
* @param tp tail policy: tu or ta
|
||||
* @param mp mask policty: mu or ma
|
||||
*/
|
||||
.macro vtype_ivli rd, avl, ew, tp=tu, mp=mu
|
||||
.if \avl <= 1
|
||||
.equ log2vl, 0
|
||||
.elseif \avl <= 2
|
||||
.equ log2vl, 1
|
||||
.elseif \avl <= 4
|
||||
.equ log2vl, 2
|
||||
.elseif \avl <= 8
|
||||
.equ log2vl, 3
|
||||
.elseif \avl <= 16
|
||||
.equ log2vl, 4
|
||||
.elseif \avl <= 32
|
||||
.equ log2vl, 5
|
||||
.elseif \avl <= 64
|
||||
.equ log2vl, 6
|
||||
.elseif \avl <= 128
|
||||
.equ log2vl, 7
|
||||
.else
|
||||
.error "Vector length \avl out of range"
|
||||
.endif
|
||||
parse_vtype \ew, \tp, \mp
|
||||
csrr \rd, vlenb
|
||||
clz \rd, \rd
|
||||
addi \rd, \rd, log2vl + 1 + VSEW_MAX - __riscv_xlen
|
||||
max \rd, \rd, zero // VLMUL must be >= VSEW - VSEW_MAX
|
||||
.if vsew < VSEW_MAX
|
||||
addi \rd, \rd, vsew - VSEW_MAX
|
||||
andi \rd, \rd, 7
|
||||
.endif
|
||||
ori \rd, \rd, (vsew << 3) | (tp << 6) | (mp << 7)
|
||||
.endm
|
||||
|
||||
/**
|
||||
* Gets the vector type with the smallest suitable LMUL value.
|
||||
* @param[out] rd vector type destination register
|
||||
* @param rs vector length source register
|
||||
* @param[out] tmp temporary register to be clobbered
|
||||
* @param ew element width: e8, e16, e32 or e64
|
||||
* @param tp tail policy: tu or ta
|
||||
* @param mp mask policty: mu or ma
|
||||
* @param addend optional addend for the vector length register
|
||||
*/
|
||||
.macro vtype_vli rd, rs, tmp, ew, tp=tu, mp=mu, addend=0
|
||||
parse_vtype \ew, \tp, \mp
|
||||
/*
|
||||
* The difference between the CLZ's notionally equals the VLMUL value
|
||||
* for 4-bit elements. But we want the value for SEW_MAX-bit elements.
|
||||
*/
|
||||
slli \tmp, \rs, 1 + VSEW_MAX
|
||||
.if \addend - 1
|
||||
addi \tmp, \tmp, \addend - 1
|
||||
.endif
|
||||
csrr \rd, vlenb
|
||||
clz \tmp, \tmp
|
||||
clz \rd, \rd
|
||||
sub \rd, \rd, \tmp
|
||||
max \rd, \rd, zero // VLMUL must be >= VSEW - VSEW_MAX
|
||||
.if vsew < VSEW_MAX
|
||||
addi \rd, \rd, vsew - VSEW_MAX
|
||||
andi \rd, \rd, 7
|
||||
.endif
|
||||
ori \rd, \rd, (vsew << 3) | (tp << 6) | (mp << 7)
|
||||
.endm
|
||||
|
||||
/**
|
||||
* Widens a vector type.
|
||||
* @param[out] rd widened vector type destination register
|
||||
* @param rs vector type source register
|
||||
* @param n number of times to widen (once by default)
|
||||
*/
|
||||
.macro vwtypei rd, rs, n=1
|
||||
xori \rd, \rs, 4
|
||||
addi \rd, \rd, (\n) * 011
|
||||
xori \rd, \rd, 4
|
||||
.endm
|
||||
|
||||
/**
|
||||
* Narrows a vector type.
|
||||
* @param[out] rd narrowed vector type destination register
|
||||
* @param rs vector type source register
|
||||
* @param n number of times to narrow (once by default)
|
||||
*/
|
||||
.macro vntypei rd, rs, n=1
|
||||
vwtypei \rd, \rs, -(\n)
|
||||
.endm
|
||||
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#ifndef AVUTIL_RISCV_BSWAP_H
|
||||
#define AVUTIL_RISCV_BSWAP_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include "config.h"
|
||||
#include "libavutil/attributes.h"
|
||||
#include "libavutil/riscv/cpu.h"
|
||||
|
||||
#if defined (__GNUC__) || defined (__clang__)
|
||||
#define av_bswap16 __builtin_bswap16
|
||||
|
||||
static av_always_inline av_const uint32_t av_bswap32_rv(uint32_t x)
|
||||
{
|
||||
#if HAVE_RV && !defined(__riscv_zbb)
|
||||
if (!__builtin_constant_p(x) &&
|
||||
__builtin_expect(ff_rv_zbb_support(), 1)) {
|
||||
uintptr_t y;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +zbb\n"
|
||||
"rev8 %0, %1\n"
|
||||
".option pop" : "=r" (y) : "r" (x));
|
||||
return y >> (__riscv_xlen - 32);
|
||||
}
|
||||
#endif
|
||||
return __builtin_bswap32(x);
|
||||
}
|
||||
#define av_bswap32 av_bswap32_rv
|
||||
|
||||
#if __riscv_xlen >= 64
|
||||
static av_always_inline av_const uint64_t av_bswap64_rv(uint64_t x)
|
||||
{
|
||||
#if HAVE_RV && !defined(__riscv_zbb)
|
||||
if (!__builtin_constant_p(x) &&
|
||||
__builtin_expect(ff_rv_zbb_support(), 1)) {
|
||||
uintptr_t y;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +zbb\n"
|
||||
"rev8 %0, %1\n"
|
||||
".option pop" : "=r" (y) : "r" (x));
|
||||
return y >> (__riscv_xlen - 64);
|
||||
}
|
||||
#endif
|
||||
return __builtin_bswap64(x);
|
||||
}
|
||||
#define av_bswap64 av_bswap64_rv
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* AVUTIL_RISCV_BSWAP_H */
|
||||
@@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright © 2022 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#if (__riscv_xlen >= 64)
|
||||
.macro bswap32_rvb out, in, count
|
||||
andi t0, \count, 1
|
||||
beqz t0, 1f
|
||||
/* Align input to 64-bit */
|
||||
lwu t0, (\in)
|
||||
addi \out, \out, 4
|
||||
rev8 t0, t0
|
||||
addi \count, \count, -4
|
||||
srli t0, t0, __riscv_xlen - 32
|
||||
addi \in, \in, 4
|
||||
sw t0, -4(\out)
|
||||
1:
|
||||
andi t3, \count, -8
|
||||
add \count, \count, \out
|
||||
beqz t3, 3f
|
||||
add t3, t3, \out
|
||||
2: /* 2 elements (64 bits) at a time on a 64-bit boundary */
|
||||
ld t0, (\in)
|
||||
addi \out, \out, 8
|
||||
rev8 t0, t0
|
||||
#if (__riscv_xlen == 64)
|
||||
srli t2, t0, 32
|
||||
sw t0, -4(\out)
|
||||
#else
|
||||
srli t1, t0, __riscv_xlen - 64
|
||||
srli t2, t0, __riscv_xlen - 32
|
||||
sw t1, -4(\out)
|
||||
#endif
|
||||
addi \in, \in, 8
|
||||
sw t2, -8(\out)
|
||||
bne \out, t3, 2b
|
||||
3:
|
||||
beq \out, \count, 5f
|
||||
4: /* Process last element */
|
||||
lwu t0, (\in)
|
||||
addi \out, \out, 4
|
||||
rev8 t0, t0
|
||||
addi \in, \in, 4
|
||||
srli t0, t0, __riscv_xlen - 32
|
||||
sw t0, -4(\out)
|
||||
5:
|
||||
ret
|
||||
.endm
|
||||
#endif
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Copyright © 2022 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#define _GNU_SOURCE
|
||||
#include "libavutil/cpu.h"
|
||||
#include "libavutil/cpu_internal.h"
|
||||
#include "libavutil/macros.h"
|
||||
#include "libavutil/log.h"
|
||||
#include "config.h"
|
||||
|
||||
#if HAVE_GETAUXVAL || HAVE_ELF_AUX_INFO
|
||||
#include <sys/auxv.h>
|
||||
#define HWCAP_RV(letter) (1ul << ((letter) - 'A'))
|
||||
#endif
|
||||
#if HAVE_SYS_HWPROBE_H
|
||||
#include <sys/hwprobe.h>
|
||||
#elif HAVE_ASM_HWPROBE_H
|
||||
#include <asm/hwprobe.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <unistd.h>
|
||||
|
||||
static int __riscv_hwprobe(struct riscv_hwprobe *pairs, size_t pair_count,
|
||||
size_t cpu_count, unsigned long *cpus,
|
||||
unsigned int flags)
|
||||
{
|
||||
return syscall(__NR_riscv_hwprobe, pairs, pair_count, cpu_count, cpus,
|
||||
flags);
|
||||
}
|
||||
#endif
|
||||
|
||||
int ff_get_cpu_flags_riscv(void)
|
||||
{
|
||||
int ret = 0;
|
||||
#if HAVE_SYS_HWPROBE_H || HAVE_ASM_HWPROBE_H
|
||||
struct riscv_hwprobe pairs[] = {
|
||||
{ RISCV_HWPROBE_KEY_BASE_BEHAVIOR, 0 },
|
||||
{ RISCV_HWPROBE_KEY_IMA_EXT_0, 0 },
|
||||
{ RISCV_HWPROBE_KEY_CPUPERF_0, 0 },
|
||||
};
|
||||
|
||||
if (__riscv_hwprobe(pairs, FF_ARRAY_ELEMS(pairs), 0, NULL, 0) == 0) {
|
||||
if (pairs[0].value & RISCV_HWPROBE_BASE_BEHAVIOR_IMA)
|
||||
ret |= AV_CPU_FLAG_RVI;
|
||||
#ifdef RISCV_HWPROBE_IMA_V
|
||||
if (pairs[1].value & RISCV_HWPROBE_IMA_V)
|
||||
ret |= AV_CPU_FLAG_RVV_I32 | AV_CPU_FLAG_RVV_I64
|
||||
| AV_CPU_FLAG_RVV_F32 | AV_CPU_FLAG_RVV_F64;
|
||||
#endif
|
||||
#ifdef RISCV_HWPROBE_EXT_ZBB
|
||||
if (pairs[1].value & RISCV_HWPROBE_EXT_ZBB)
|
||||
ret |= AV_CPU_FLAG_RVB_BASIC;
|
||||
#if defined (RISCV_HWPROBE_EXT_ZBA) && defined (RISCV_HWPROBE_EXT_ZBS)
|
||||
if ((pairs[1].value & RISCV_HWPROBE_EXT_ZBA) &&
|
||||
(pairs[1].value & RISCV_HWPROBE_EXT_ZBB) &&
|
||||
(pairs[1].value & RISCV_HWPROBE_EXT_ZBS))
|
||||
ret |= AV_CPU_FLAG_RVB;
|
||||
#endif
|
||||
#endif
|
||||
#ifdef RISCV_HWPROBE_EXT_ZVBB
|
||||
if (pairs[1].value & RISCV_HWPROBE_EXT_ZVBB)
|
||||
ret |= AV_CPU_FLAG_RV_ZVBB;
|
||||
#endif
|
||||
switch (pairs[2].value & RISCV_HWPROBE_MISALIGNED_MASK) {
|
||||
case RISCV_HWPROBE_MISALIGNED_FAST:
|
||||
ret |= AV_CPU_FLAG_RV_MISALIGNED;
|
||||
break;
|
||||
default:
|
||||
}
|
||||
}
|
||||
#elif HAVE_GETAUXVAL || HAVE_ELF_AUX_INFO
|
||||
{
|
||||
const unsigned long hwcap = ff_getauxval(AT_HWCAP);
|
||||
|
||||
if (hwcap & HWCAP_RV('I'))
|
||||
ret |= AV_CPU_FLAG_RVI;
|
||||
if (hwcap & HWCAP_RV('B'))
|
||||
ret |= AV_CPU_FLAG_RVB_BASIC | AV_CPU_FLAG_RVB;
|
||||
|
||||
/* The V extension implies all Zve* functional subsets */
|
||||
if (hwcap & HWCAP_RV('V'))
|
||||
ret |= AV_CPU_FLAG_RVV_I32 | AV_CPU_FLAG_RVV_I64
|
||||
| AV_CPU_FLAG_RVV_F32 | AV_CPU_FLAG_RVV_F64;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef __riscv_i
|
||||
ret |= AV_CPU_FLAG_RVI;
|
||||
#endif
|
||||
|
||||
#ifdef __riscv_zbb
|
||||
ret |= AV_CPU_FLAG_RVB_BASIC;
|
||||
#endif
|
||||
#if defined (__riscv_b) || \
|
||||
(defined (__riscv_zba) && defined (__riscv_zbb) && defined (__riscv_zbs))
|
||||
ret |= AV_CPU_FLAG_RVB;
|
||||
#endif
|
||||
|
||||
/* If RV-V is enabled statically at compile-time, check the details. */
|
||||
#ifdef __riscv_vector
|
||||
ret |= AV_CPU_FLAG_RVV_I32;
|
||||
#if __riscv_v_elen >= 64
|
||||
ret |= AV_CPU_FLAG_RVV_I64;
|
||||
#endif
|
||||
#if __riscv_v_elen_fp >= 32
|
||||
ret |= AV_CPU_FLAG_RVV_F32;
|
||||
#if __riscv_v_elen_fp >= 64
|
||||
ret |= AV_CPU_FLAG_RVV_F64;
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
#ifdef __riscv_zvbb
|
||||
ret |= AV_CPU_FLAG_RV_ZVBB;
|
||||
#endif
|
||||
|
||||
return ret;
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright © 2022 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#ifndef AVUTIL_RISCV_CPU_H
|
||||
#define AVUTIL_RISCV_CPU_H
|
||||
|
||||
#include "config.h"
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include "libavutil/attributes_internal.h"
|
||||
#include "libavutil/cpu.h"
|
||||
|
||||
#ifndef __riscv_zbb
|
||||
extern attribute_visibility_hidden bool ff_rv_zbb_supported;
|
||||
#endif
|
||||
|
||||
static inline av_const bool ff_rv_zbb_support(void)
|
||||
{
|
||||
#ifndef __riscv_zbb
|
||||
return ff_rv_zbb_supported;
|
||||
#else
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
#if HAVE_RVV
|
||||
/**
|
||||
* Returns the vector size in bytes (always a power of two and at least 4).
|
||||
* This is undefined behaviour if vectors are not implemented.
|
||||
*/
|
||||
static inline size_t ff_get_rv_vlenb(void)
|
||||
{
|
||||
size_t vlenb;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +v\n"
|
||||
" csrr %0, vlenb\n"
|
||||
".option pop\n" : "=r" (vlenb));
|
||||
return vlenb;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks that the vector bit-size is at least the given value.
|
||||
* This is potentially undefined behaviour if vectors are not implemented.
|
||||
*/
|
||||
static inline bool ff_rv_vlen_least(unsigned int bits)
|
||||
{
|
||||
#ifdef __riscv_v_min_vlen
|
||||
if (bits <= __riscv_v_min_vlen)
|
||||
return true;
|
||||
#else
|
||||
/*
|
||||
* Vector lengths smaller than 128 bits are only possible in embedded cases
|
||||
* and cannot be run-time detected, so we can assume 128 bits at least.
|
||||
*/
|
||||
if (bits <= 128)
|
||||
return true;
|
||||
#endif
|
||||
return bits <= (8 * ff_get_rv_vlenb());
|
||||
}
|
||||
#endif
|
||||
#endif /* HAVE_RVV */
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright © 2024 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#include "libavutil/cpu.h"
|
||||
|
||||
#ifndef __riscv_zbb
|
||||
unsigned char ff_rv_zbb_supported = 0;
|
||||
|
||||
#ifdef __ELF__
|
||||
__attribute__((constructor))
|
||||
static void probe_zbb(void)
|
||||
{
|
||||
ff_rv_zbb_supported = (av_get_cpu_flags() & AV_CPU_FLAG_RVB_BASIC) != 0;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright © 2022 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "config.h"
|
||||
#include "libavutil/attributes.h"
|
||||
#include "libavutil/cpu.h"
|
||||
#include "libavutil/fixed_dsp.h"
|
||||
|
||||
void ff_vector_fmul_window_scaled_rvv(int16_t *dst, const int32_t *src0,
|
||||
const int32_t *src1, const int32_t *win,
|
||||
int len, uint8_t bits);
|
||||
void ff_vector_fmul_window_fixed_rvv(int32_t *dst, const int32_t *src0,
|
||||
const int32_t *src1, const int32_t *win,
|
||||
int len);
|
||||
void ff_vector_fmul_fixed_rvv(int *dst, const int *src0, const int *src1,
|
||||
int len);
|
||||
void ff_vector_fmul_reverse_fixed_rvv(int *dst, const int *src0,
|
||||
const int *src1, int len);
|
||||
void ff_vector_fmul_add_fixed_rvv(int *dst, const int *src0, const int *src1,
|
||||
const int *src2, int len);
|
||||
int ff_scalarproduct_fixed_rvv(const int *v1, const int *v2, int len);
|
||||
void ff_butterflies_fixed_rvv(int *v1, int *v2, int len);
|
||||
|
||||
av_cold void ff_fixed_dsp_init_riscv(AVFixedDSPContext *fdsp)
|
||||
{
|
||||
#if HAVE_RVV
|
||||
int flags = av_get_cpu_flags();
|
||||
|
||||
if ((flags & AV_CPU_FLAG_RVV_I32) && (flags & AV_CPU_FLAG_RVB)) {
|
||||
if (flags & AV_CPU_FLAG_RVV_I64) {
|
||||
fdsp->vector_fmul_window_scaled = ff_vector_fmul_window_scaled_rvv;
|
||||
fdsp->vector_fmul_window = ff_vector_fmul_window_fixed_rvv;
|
||||
}
|
||||
|
||||
fdsp->vector_fmul = ff_vector_fmul_fixed_rvv;
|
||||
fdsp->vector_fmul_reverse = ff_vector_fmul_reverse_fixed_rvv;
|
||||
fdsp->vector_fmul_add = ff_vector_fmul_add_fixed_rvv;
|
||||
|
||||
if (flags & AV_CPU_FLAG_RVV_I64)
|
||||
fdsp->scalarproduct_fixed = ff_scalarproduct_fixed_rvv;
|
||||
|
||||
fdsp->butterflies_fixed = ff_butterflies_fixed_rvv;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
/*
|
||||
* Copyright © 2022 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#include "asm.S"
|
||||
|
||||
func ff_vector_fmul_window_scaled_rvv, zve64x, zba
|
||||
lpad 0
|
||||
csrwi vxrm, 0
|
||||
vsetvli t0, zero, e16, m1, ta, ma
|
||||
sh2add a2, a4, a2
|
||||
vid.v v0
|
||||
sh3add t3, a4, a3
|
||||
vadd.vi v0, v0, 1
|
||||
sh2add t0, a4, a0
|
||||
1:
|
||||
vsetvli t2, a4, e16, m1, ta, ma
|
||||
slli t4, t2, 2
|
||||
slli t1, t2, 1
|
||||
vrsub.vx v2, v0, t2
|
||||
sub t3, t3, t4
|
||||
vsetvli zero, zero, e32, m2, ta, ma
|
||||
sub a2, a2, t4
|
||||
vle32.v v8, (t3)
|
||||
sub t0, t0, t1
|
||||
vle32.v v4, (a2)
|
||||
sub a4, a4, t2
|
||||
vrgatherei16.vv v28, v8, v2
|
||||
vle32.v v16, (a1)
|
||||
add a1, a1, t4
|
||||
vrgatherei16.vv v20, v4, v2
|
||||
vle32.v v24, (a3)
|
||||
add a3, a3, t4
|
||||
vwmul.vv v12, v16, v28
|
||||
vwmul.vv v8, v16, v24
|
||||
// vwnmsac.vv does _not_ exist so multiply & subtract separately
|
||||
vwmul.vv v4, v20, v24
|
||||
vwmacc.vv v8, v20, v28
|
||||
vsetvli zero, zero, e64, m4, ta, ma
|
||||
vsub.vv v12, v12, v4
|
||||
vsetvli zero, zero, e32, m2, ta, ma
|
||||
vnclip.wi v16, v8, 31
|
||||
vnclip.wi v20, v12, 31
|
||||
vsetvli zero, zero, e16, m1, ta, ma
|
||||
vnclip.wx v8, v16, a5
|
||||
vnclip.wx v12, v20, a5
|
||||
vrgatherei16.vv v16, v8, v2
|
||||
vse16.v v12, (a0)
|
||||
add a0, a0, t1
|
||||
vse16.v v16, (t0)
|
||||
bnez a4, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
func ff_vector_fmul_window_fixed_rvv, zve64x, zba
|
||||
lpad 0
|
||||
csrwi vxrm, 0
|
||||
vsetvli t0, zero, e16, m1, ta, ma
|
||||
sh2add a2, a4, a2
|
||||
vid.v v0
|
||||
sh3add t3, a4, a3
|
||||
vadd.vi v0, v0, 1
|
||||
sh3add t0, a4, a0
|
||||
1:
|
||||
vsetvli t2, a4, e16, m1, ta, ma
|
||||
slli t4, t2, 2
|
||||
vrsub.vx v2, v0, t2
|
||||
sub t3, t3, t4
|
||||
vsetvli zero, zero, e32, m2, ta, ma
|
||||
sub a2, a2, t4
|
||||
vle32.v v8, (t3)
|
||||
sub t0, t0, t4
|
||||
vle32.v v4, (a2)
|
||||
sub a4, a4, t2
|
||||
vrgatherei16.vv v28, v8, v2
|
||||
vle32.v v16, (a1)
|
||||
add a1, a1, t4
|
||||
vrgatherei16.vv v20, v4, v2
|
||||
vle32.v v24, (a3)
|
||||
add a3, a3, t4
|
||||
vwmul.vv v12, v16, v28
|
||||
vwmul.vv v8, v16, v24
|
||||
// vwnmsac.vv does _not_ exist so multiply & subtract separately
|
||||
vwmul.vv v4, v20, v24
|
||||
vwmacc.vv v8, v20, v28
|
||||
vsetvli zero, zero, e64, m4, ta, ma
|
||||
vsub.vv v12, v12, v4
|
||||
vsetvli zero, zero, e32, m2, ta, ma
|
||||
vnclip.wi v16, v8, 31
|
||||
vnclip.wi v20, v12, 31
|
||||
vrgatherei16.vv v8, v16, v2
|
||||
vse32.v v20, (a0)
|
||||
add a0, a0, t4
|
||||
vse32.v v8, (t0)
|
||||
bnez a4, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
func ff_vector_fmul_fixed_rvv, zve32x, zba
|
||||
lpad 0
|
||||
csrwi vxrm, 0
|
||||
1:
|
||||
vsetvli t0, a3, e32, m4, ta, ma
|
||||
vle32.v v16, (a1)
|
||||
sub a3, a3, t0
|
||||
vle32.v v24, (a2)
|
||||
sh2add a1, t0, a1
|
||||
vsmul.vv v8, v16, v24
|
||||
sh2add a2, t0, a2
|
||||
vse32.v v8, (a0)
|
||||
sh2add a0, t0, a0
|
||||
bnez a3, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
func ff_vector_fmul_reverse_fixed_rvv, zve32x, zba
|
||||
csrwi vxrm, 0
|
||||
// e16/m4 and e32/m8 are possible but slow the gathers down.
|
||||
vsetvli t0, zero, e16, m1, ta, ma
|
||||
sh2add a2, a3, a2
|
||||
vid.v v0
|
||||
vadd.vi v0, v0, 1
|
||||
1:
|
||||
vsetvli t0, a3, e16, m1, ta, ma
|
||||
slli t1, t0, 2
|
||||
vrsub.vx v4, v0, t0 // v4[i] = [VL-1, VL-2... 1, 0]
|
||||
sub a2, a2, t1
|
||||
vsetvli zero, zero, e32, m2, ta, ma
|
||||
vle32.v v8, (a2)
|
||||
sub a3, a3, t0
|
||||
vle32.v v16, (a1)
|
||||
add a1, a1, t1
|
||||
vrgatherei16.vv v24, v8, v4 // v24 = reverse(v8)
|
||||
vsmul.vv v16, v16, v24
|
||||
vse32.v v16, (a0)
|
||||
add a0, a0, t1
|
||||
bnez a3, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
func ff_vector_fmul_add_fixed_rvv, zve32x, zba
|
||||
lpad 0
|
||||
csrwi vxrm, 0
|
||||
1:
|
||||
vsetvli t0, a4, e32, m8, ta, ma
|
||||
vle32.v v16, (a1)
|
||||
sub a4, a4, t0
|
||||
vle32.v v24, (a2)
|
||||
sh2add a1, t0, a1
|
||||
vsmul.vv v8, v16, v24
|
||||
sh2add a2, t0, a2
|
||||
vle32.v v0,(a3)
|
||||
sh2add a3, t0, a3
|
||||
vadd.vv v8, v8, v0
|
||||
vse32.v v8, (a0)
|
||||
sh2add a0, t0, a0
|
||||
bnez a4, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
func ff_scalarproduct_fixed_rvv, zve64x, zba
|
||||
lpad 0
|
||||
li t1, 1 << 30
|
||||
vsetvli t0, zero, e64, m8, ta, ma
|
||||
vmv.v.x v8, zero
|
||||
vmv.s.x v0, t1
|
||||
1:
|
||||
vsetvli t0, a2, e32, m4, tu, ma
|
||||
vle32.v v16, (a0)
|
||||
sub a2, a2, t0
|
||||
vle32.v v20, (a1)
|
||||
sh2add a0, t0, a0
|
||||
vwmacc.vv v8, v16, v20
|
||||
sh2add a1, t0, a1
|
||||
bnez a2, 1b
|
||||
|
||||
vsetvli t0, zero, e64, m8, ta, ma
|
||||
vredsum.vs v0, v8, v0
|
||||
vmv.x.s a0, v0
|
||||
srai a0, a0, 31
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// (a0) = (a0) + (a1), (a1) = (a0) - (a1) [0..a2-1]
|
||||
func ff_butterflies_fixed_rvv, zve32x, zba
|
||||
lpad 0
|
||||
1:
|
||||
vsetvli t0, a2, e32, m4, ta, ma
|
||||
vle32.v v16, (a0)
|
||||
sub a2, a2, t0
|
||||
vle32.v v24, (a1)
|
||||
vadd.vv v0, v16, v24
|
||||
vsub.vv v8, v16, v24
|
||||
vse32.v v0, (a0)
|
||||
sh2add a0, t0, a0
|
||||
vse32.v v8, (a1)
|
||||
sh2add a1, t0, a1
|
||||
bnez a2, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Copyright © 2022 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include "config.h"
|
||||
#include "libavutil/attributes.h"
|
||||
#include "libavutil/cpu.h"
|
||||
#include "libavutil/float_dsp.h"
|
||||
|
||||
void ff_vector_fmul_rvv(float *dst, const float *src0, const float *src1,
|
||||
int len);
|
||||
void ff_vector_fmac_scalar_rvv(float *dst, const float *src, float mul,
|
||||
int len);
|
||||
void ff_vector_fmul_scalar_rvv(float *dst, const float *src, float mul,
|
||||
int len);
|
||||
void ff_vector_fmul_window_rvv(float *dst, const float *src0,
|
||||
const float *src1, const float *win, int len);
|
||||
void ff_vector_fmul_add_rvv(float *dst, const float *src0, const float *src1,
|
||||
const float *src2, int len);
|
||||
void ff_vector_fmul_reverse_rvv(float *dst, const float *src0,
|
||||
const float *src1, int len);
|
||||
void ff_butterflies_float_rvv(float *v1, float *v2, int len);
|
||||
float ff_scalarproduct_float_rvv(const float *v1, const float *v2, int len);
|
||||
|
||||
void ff_vector_dmul_rvv(double *dst, const double *src0, const double *src1,
|
||||
int len);
|
||||
void ff_vector_dmac_scalar_rvv(double *dst, const double *src, double mul,
|
||||
int len);
|
||||
void ff_vector_dmul_scalar_rvv(double *dst, const double *src, double mul,
|
||||
int len);
|
||||
double ff_scalarproduct_double_rvv(const double *v1, const double *v2,
|
||||
size_t len);
|
||||
|
||||
av_cold void ff_float_dsp_init_riscv(AVFloatDSPContext *fdsp)
|
||||
{
|
||||
#if HAVE_RVV
|
||||
int flags = av_get_cpu_flags();
|
||||
|
||||
if (flags & AV_CPU_FLAG_RVB) {
|
||||
if (flags & AV_CPU_FLAG_RVV_F32) {
|
||||
fdsp->vector_fmul = ff_vector_fmul_rvv;
|
||||
fdsp->vector_fmac_scalar = ff_vector_fmac_scalar_rvv;
|
||||
fdsp->vector_fmul_scalar = ff_vector_fmul_scalar_rvv;
|
||||
fdsp->vector_fmul_window = ff_vector_fmul_window_rvv;
|
||||
fdsp->vector_fmul_add = ff_vector_fmul_add_rvv;
|
||||
fdsp->vector_fmul_reverse = ff_vector_fmul_reverse_rvv;
|
||||
fdsp->butterflies_float = ff_butterflies_float_rvv;
|
||||
fdsp->scalarproduct_float = ff_scalarproduct_float_rvv;
|
||||
}
|
||||
|
||||
if (flags & AV_CPU_FLAG_RVV_F64) {
|
||||
fdsp->vector_dmul = ff_vector_dmul_rvv;
|
||||
fdsp->vector_dmac_scalar = ff_vector_dmac_scalar_rvv;
|
||||
fdsp->vector_dmul_scalar = ff_vector_dmul_scalar_rvv;
|
||||
fdsp->scalarproduct_double = ff_scalarproduct_double_rvv;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
/*
|
||||
* Copyright © 2022 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#include "asm.S"
|
||||
|
||||
// (a0) = (a1) * (a2) [0..a3-1]
|
||||
func ff_vector_fmul_rvv, zve32f, zba
|
||||
lpad 0
|
||||
1:
|
||||
vsetvli t0, a3, e32, m8, ta, ma
|
||||
vle32.v v16, (a1)
|
||||
sub a3, a3, t0
|
||||
vle32.v v24, (a2)
|
||||
sh2add a1, t0, a1
|
||||
vfmul.vv v16, v16, v24
|
||||
sh2add a2, t0, a2
|
||||
vse32.v v16, (a0)
|
||||
sh2add a0, t0, a0
|
||||
bnez a3, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// (a0) += (a1) * fa0 [0..a2-1]
|
||||
func ff_vector_fmac_scalar_rvv, zve32f, zba
|
||||
lpad 0
|
||||
NOHWF fmv.w.x fa0, a2
|
||||
NOHWF mv a2, a3
|
||||
1:
|
||||
vsetvli t0, a2, e32, m8, ta, ma
|
||||
slli t1, t0, 2
|
||||
vle32.v v24, (a1)
|
||||
sub a2, a2, t0
|
||||
vle32.v v16, (a0)
|
||||
sh2add a1, t0, a1
|
||||
vfmacc.vf v16, fa0, v24
|
||||
vse32.v v16, (a0)
|
||||
sh2add a0, t0, a0
|
||||
bnez a2, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// (a0) = (a1) * fa0 [0..a2-1]
|
||||
func ff_vector_fmul_scalar_rvv, zve32f, zba
|
||||
lpad 0
|
||||
NOHWF fmv.w.x fa0, a2
|
||||
NOHWF mv a2, a3
|
||||
1:
|
||||
vsetvli t0, a2, e32, m8, ta, ma
|
||||
vle32.v v16, (a1)
|
||||
sub a2, a2, t0
|
||||
vfmul.vf v16, v16, fa0
|
||||
sh2add a1, t0, a1
|
||||
vse32.v v16, (a0)
|
||||
sh2add a0, t0, a0
|
||||
bnez a2, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
func ff_vector_fmul_window_rvv, zve32f, zba
|
||||
lpad 0
|
||||
// a0: dst, a1: src0, a2: src1, a3: window, a4: length
|
||||
// e16/m2 and e32/m4 are possible but slower due to gather.
|
||||
vsetvli t0, zero, e16, m1, ta, ma
|
||||
sh2add a2, a4, a2
|
||||
vid.v v0
|
||||
sh3add t3, a4, a3
|
||||
vadd.vi v0, v0, 1
|
||||
sh3add t0, a4, a0
|
||||
1:
|
||||
vsetvli t2, a4, e16, m1, ta, ma
|
||||
slli t4, t2, 2
|
||||
vrsub.vx v2, v0, t2
|
||||
sub t3, t3, t4
|
||||
vsetvli zero, zero, e32, m2, ta, ma
|
||||
sub a2, a2, t4
|
||||
vle32.v v8, (t3)
|
||||
sub t0, t0, t4
|
||||
vle32.v v4, (a2)
|
||||
sub a4, a4, t2
|
||||
vrgatherei16.vv v28, v8, v2
|
||||
vle32.v v16, (a1)
|
||||
add a1, a1, t4
|
||||
vrgatherei16.vv v20, v4, v2
|
||||
vle32.v v24, (a3)
|
||||
add a3, a3, t4
|
||||
vfmul.vv v12, v16, v28
|
||||
vfmul.vv v16, v16, v24
|
||||
vfnmsac.vv v12, v20, v24
|
||||
vfmacc.vv v16, v20, v28
|
||||
vrgatherei16.vv v8, v16, v2
|
||||
vse32.v v12, (a0)
|
||||
add a0, a0, t4
|
||||
vse32.v v8, (t0)
|
||||
bnez a4, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// (a0) = (a1) * (a2) + (a3) [0..a4-1]
|
||||
func ff_vector_fmul_add_rvv, zve32f, zba
|
||||
lpad 0
|
||||
1:
|
||||
vsetvli t0, a4, e32, m8, ta, ma
|
||||
vle32.v v8, (a1)
|
||||
sub a4, a4, t0
|
||||
vle32.v v16, (a2)
|
||||
sh2add a1, t0, a1
|
||||
vle32.v v24, (a3)
|
||||
sh2add a2, t0, a2
|
||||
vfmadd.vv v8, v16, v24
|
||||
sh2add a3, t0, a3
|
||||
vse32.v v8, (a0)
|
||||
sh2add a0, t0, a0
|
||||
bnez a4, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// TODO factor vrsub, separate last iteration?
|
||||
// (a0) = (a1) * reverse(a2) [0..a3-1]
|
||||
func ff_vector_fmul_reverse_rvv, zve32f, zba
|
||||
lpad 0
|
||||
// e16/m4 and e32/m8 are possible but slower due to gather.
|
||||
vsetvli t0, zero, e16, m1, ta, ma
|
||||
sh2add a2, a3, a2
|
||||
vid.v v0
|
||||
vadd.vi v0, v0, 1
|
||||
1:
|
||||
vsetvli t0, a3, e16, m1, ta, ma
|
||||
slli t1, t0, 2
|
||||
vrsub.vx v4, v0, t0 // v4[i] = [VL-1, VL-2... 1, 0]
|
||||
sub a2, a2, t1
|
||||
vsetvli zero, zero, e32, m2, ta, ma
|
||||
vle32.v v8, (a2)
|
||||
sub a3, a3, t0
|
||||
vle32.v v16, (a1)
|
||||
add a1, a1, t1
|
||||
vrgatherei16.vv v24, v8, v4 // v24 = reverse(v8)
|
||||
vfmul.vv v16, v16, v24
|
||||
vse32.v v16, (a0)
|
||||
add a0, a0, t1
|
||||
bnez a3, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// (a0) = (a0) + (a1), (a1) = (a0) - (a1) [0..a2-1]
|
||||
func ff_butterflies_float_rvv, zve32f, zba
|
||||
lpad 0
|
||||
1:
|
||||
vsetvli t0, a2, e32, m8, ta, ma
|
||||
vle32.v v16, (a0)
|
||||
sub a2, a2, t0
|
||||
vle32.v v24, (a1)
|
||||
vfadd.vv v0, v16, v24
|
||||
vfsub.vv v8, v16, v24
|
||||
vse32.v v0, (a0)
|
||||
sh2add a0, t0, a0
|
||||
vse32.v v8, (a1)
|
||||
sh2add a1, t0, a1
|
||||
bnez a2, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// a0 = (a0).(a1) [0..a2-1]
|
||||
func ff_scalarproduct_float_rvv, zve32f, zba
|
||||
lpad 0
|
||||
vsetvli t0, zero, e32, m8, ta, ma
|
||||
vmv.v.x v8, zero
|
||||
vmv.s.x v0, zero
|
||||
1:
|
||||
vsetvli t0, a2, e32, m8, tu, ma
|
||||
vle32.v v16, (a0)
|
||||
sub a2, a2, t0
|
||||
vle32.v v24, (a1)
|
||||
sh2add a0, t0, a0
|
||||
vfmacc.vv v8, v16, v24
|
||||
sh2add a1, t0, a1
|
||||
bnez a2, 1b
|
||||
|
||||
vsetvli t0, zero, e32, m8, ta, ma
|
||||
vfredusum.vs v0, v8, v0
|
||||
vfmv.f.s fa0, v0
|
||||
NOHWF fmv.x.w a0, fa0
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// (a0) = (a1) * (a2) [0..a3-1]
|
||||
func ff_vector_dmul_rvv, zve64d, zba
|
||||
lpad 0
|
||||
1:
|
||||
vsetvli t0, a3, e64, m8, ta, ma
|
||||
vle64.v v16, (a1)
|
||||
sub a3, a3, t0
|
||||
vle64.v v24, (a2)
|
||||
sh3add a1, t0, a1
|
||||
vfmul.vv v16, v16, v24
|
||||
sh3add a2, t0, a2
|
||||
vse64.v v16, (a0)
|
||||
sh3add a0, t0, a0
|
||||
bnez a3, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// (a0) += (a1) * fa0 [0..a2-1]
|
||||
func ff_vector_dmac_scalar_rvv, zve64d, zba
|
||||
lpad 0
|
||||
#if __riscv_xlen >= 64
|
||||
NOHWD fmv.d.x fa0, a2
|
||||
#else
|
||||
NOHWD addi sp, sp, -16
|
||||
NOHWD sw a0, 0(sp)
|
||||
NOHWD sw a1, 4(sp)
|
||||
NOHWD fld fa0, (sp)
|
||||
NOHWD addi sp, sp, 16
|
||||
#endif
|
||||
NOHWD mv a2, a3
|
||||
1:
|
||||
vsetvli t0, a2, e64, m8, ta, ma
|
||||
vle64.v v24, (a1)
|
||||
sub a2, a2, t0
|
||||
vle64.v v16, (a0)
|
||||
sh3add a1, t0, a1
|
||||
vfmacc.vf v16, fa0, v24
|
||||
vse64.v v16, (a0)
|
||||
sh3add a0, t0, a0
|
||||
bnez a2, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
// (a0) = (a1) * fa0 [0..a2-1]
|
||||
func ff_vector_dmul_scalar_rvv, zve64d, zba
|
||||
lpad 0
|
||||
#if __riscv_xlen >= 64
|
||||
NOHWD fmv.d.x fa0, a2
|
||||
#else
|
||||
NOHWD addi sp, sp, -16
|
||||
NOHWD sw a0, 0(sp)
|
||||
NOHWD sw a1, 4(sp)
|
||||
NOHWD fld fa0, (sp)
|
||||
NOHWD addi sp, sp, 16
|
||||
#endif
|
||||
NOHWD mv a2, a3
|
||||
1:
|
||||
vsetvli t0, a2, e64, m8, ta, ma
|
||||
vle64.v v16, (a1)
|
||||
sub a2, a2, t0
|
||||
vfmul.vf v16, v16, fa0
|
||||
sh3add a1, t0, a1
|
||||
vse64.v v16, (a0)
|
||||
sh3add a0, t0, a0
|
||||
bnez a2, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
|
||||
func ff_scalarproduct_double_rvv, zve64f, zba
|
||||
lpad 0
|
||||
vsetvli t0, zero, e64, m8, ta, ma
|
||||
vmv.v.x v8, zero
|
||||
vmv.s.x v0, zero
|
||||
1:
|
||||
vsetvli t0, a2, e64, m8, tu, ma
|
||||
vle64.v v16, (a0)
|
||||
sub a2, a2, t0
|
||||
vle64.v v24, (a1)
|
||||
sh3add a0, t0, a0
|
||||
vfmacc.vv v8, v16, v24
|
||||
sh3add a1, t0, a1
|
||||
bnez a2, 1b
|
||||
|
||||
vsetvli t0, zero, e64, m8, ta, ma
|
||||
vfredusum.vs v0, v8, v0
|
||||
vfmv.f.s fa0, v0
|
||||
NOHWD fmv.x.w a0, fa0
|
||||
ret
|
||||
endfunc
|
||||
@@ -0,0 +1,266 @@
|
||||
/*
|
||||
* Copyright © 2022-2024 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#ifndef AVUTIL_RISCV_INTMATH_H
|
||||
#define AVUTIL_RISCV_INTMATH_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <math.h>
|
||||
|
||||
#include "config.h"
|
||||
#include "libavutil/attributes.h"
|
||||
#include "libavutil/riscv/cpu.h"
|
||||
|
||||
/*
|
||||
* The compiler is forced to sign-extend the result anyhow, so it is faster to
|
||||
* compute it explicitly and use it.
|
||||
*/
|
||||
#define av_clip_int8 av_clip_int8_rvi
|
||||
static av_always_inline av_const int8_t av_clip_int8_rvi(int a)
|
||||
{
|
||||
union { uint8_t u; int8_t s; } u = { .u = a };
|
||||
|
||||
if (a != u.s)
|
||||
a = ((a >> 31) ^ 0x7F);
|
||||
return a;
|
||||
}
|
||||
|
||||
#define av_clip_int16 av_clip_int16_rvi
|
||||
static av_always_inline av_const int16_t av_clip_int16_rvi(int a)
|
||||
{
|
||||
union { uint16_t u; int16_t s; } u = { .u = a };
|
||||
|
||||
if (a != u.s)
|
||||
a = ((a >> 31) ^ 0x7FFF);
|
||||
return a;
|
||||
}
|
||||
|
||||
#define av_clipl_int32 av_clipl_int32_rvi
|
||||
static av_always_inline av_const int32_t av_clipl_int32_rvi(int64_t a)
|
||||
{
|
||||
union { uint32_t u; int32_t s; } u = { .u = a };
|
||||
|
||||
if (a != u.s)
|
||||
a = ((a >> 63) ^ 0x7FFFFFFF);
|
||||
return a;
|
||||
}
|
||||
|
||||
#define av_clip_intp2 av_clip_intp2_rvi
|
||||
static av_always_inline av_const int av_clip_intp2_rvi(int a, int p)
|
||||
{
|
||||
const int shift = 31 - p;
|
||||
int b = ((int)(((unsigned)a) << shift)) >> shift;
|
||||
|
||||
if (a != b)
|
||||
b = (a >> 31) ^ ((1 << p) - 1);
|
||||
return b;
|
||||
}
|
||||
|
||||
#if defined (__riscv_f) || defined (__riscv_zfinx)
|
||||
#define av_clipf av_clipf_rvf
|
||||
static av_always_inline av_const float av_clipf_rvf(float a, float min,
|
||||
float max)
|
||||
{
|
||||
return fminf(fmaxf(a, min), max);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined (__riscv_d) || defined (__riscv_zdinx)
|
||||
#define av_clipd av_clipd_rvd
|
||||
static av_always_inline av_const double av_clipd_rvd(double a, double min,
|
||||
double max)
|
||||
{
|
||||
return fmin(fmax(a, min), max);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined (__GNUC__) || defined (__clang__)
|
||||
static inline av_const int ff_ctz_rv(int x)
|
||||
{
|
||||
#if HAVE_RV && !defined(__riscv_zbb)
|
||||
if (!__builtin_constant_p(x) &&
|
||||
__builtin_expect(ff_rv_zbb_support(), true)) {
|
||||
int y;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +zbb\n"
|
||||
#if __riscv_xlen >= 64
|
||||
"ctzw %0, %1\n"
|
||||
#else
|
||||
"ctz %0, %1\n"
|
||||
#endif
|
||||
".option pop" : "=r" (y) : "r" (x));
|
||||
if (y > 32)
|
||||
__builtin_unreachable();
|
||||
return y;
|
||||
}
|
||||
#endif
|
||||
return __builtin_ctz(x);
|
||||
}
|
||||
#define ff_ctz ff_ctz_rv
|
||||
|
||||
static inline av_const int ff_ctzll_rv(long long x)
|
||||
{
|
||||
#if HAVE_RV && !defined(__riscv_zbb) && __riscv_xlen == 64
|
||||
if (!__builtin_constant_p(x) &&
|
||||
__builtin_expect(ff_rv_zbb_support(), true)) {
|
||||
int y;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +zbb\n"
|
||||
"ctz %0, %1\n"
|
||||
".option pop" : "=r" (y) : "r" (x));
|
||||
if (y > 64)
|
||||
__builtin_unreachable();
|
||||
return y;
|
||||
}
|
||||
#endif
|
||||
return __builtin_ctzll(x);
|
||||
}
|
||||
#define ff_ctzll ff_ctzll_rv
|
||||
|
||||
static inline av_const int ff_clz_rv(int x)
|
||||
{
|
||||
#if HAVE_RV && !defined(__riscv_zbb)
|
||||
if (!__builtin_constant_p(x) &&
|
||||
__builtin_expect(ff_rv_zbb_support(), true)) {
|
||||
int y;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +zbb\n"
|
||||
#if __riscv_xlen >= 64
|
||||
"clzw %0, %1\n"
|
||||
#else
|
||||
"clz %0, %1\n"
|
||||
#endif
|
||||
".option pop" : "=r" (y) : "r" (x));
|
||||
if (y > 32)
|
||||
__builtin_unreachable();
|
||||
return y;
|
||||
}
|
||||
#endif
|
||||
return __builtin_clz(x);
|
||||
}
|
||||
#define ff_clz ff_clz_rv
|
||||
|
||||
#if __riscv_xlen == 64
|
||||
static inline av_const int ff_clzll_rv(long long x)
|
||||
{
|
||||
#if HAVE_RV && !defined(__riscv_zbb)
|
||||
if (!__builtin_constant_p(x) &&
|
||||
__builtin_expect(ff_rv_zbb_support(), true)) {
|
||||
int y;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +zbb\n"
|
||||
"clz %0, %1\n"
|
||||
".option pop" : "=r" (y) : "r" (x));
|
||||
if (y > 64)
|
||||
__builtin_unreachable();
|
||||
return y;
|
||||
}
|
||||
#endif
|
||||
return __builtin_clzll(x);
|
||||
}
|
||||
#define ff_clz ff_clz_rv
|
||||
#endif
|
||||
|
||||
static inline av_const int ff_log2_rv(unsigned int x)
|
||||
{
|
||||
return 31 - ff_clz_rv(x | 1);
|
||||
}
|
||||
#define ff_log2 ff_log2_rv
|
||||
#define ff_log2_16bit ff_log2_rv
|
||||
|
||||
static inline av_const int av_popcount_rv(unsigned int x)
|
||||
{
|
||||
#if HAVE_RV && !defined(__riscv_zbb)
|
||||
if (!__builtin_constant_p(x) &&
|
||||
__builtin_expect(ff_rv_zbb_support(), true)) {
|
||||
int y;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +zbb\n"
|
||||
#if __riscv_xlen >= 64
|
||||
"cpopw %0, %1\n"
|
||||
#else
|
||||
"cpop %0, %1\n"
|
||||
#endif
|
||||
".option pop" : "=r" (y) : "r" (x));
|
||||
if (y > 32)
|
||||
__builtin_unreachable();
|
||||
return y;
|
||||
}
|
||||
#endif
|
||||
return __builtin_popcount(x);
|
||||
}
|
||||
#define av_popcount av_popcount_rv
|
||||
|
||||
static inline av_const int av_popcount64_rv(uint64_t x)
|
||||
{
|
||||
#if HAVE_RV && !defined(__riscv_zbb) && __riscv_xlen >= 64
|
||||
if (!__builtin_constant_p(x) &&
|
||||
__builtin_expect(ff_rv_zbb_support(), true)) {
|
||||
int y;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +zbb\n"
|
||||
"cpop %0, %1\n"
|
||||
".option pop" : "=r" (y) : "r" (x));
|
||||
if (y > 64)
|
||||
__builtin_unreachable();
|
||||
return y;
|
||||
}
|
||||
#endif
|
||||
return __builtin_popcountl(x);
|
||||
}
|
||||
#define av_popcount64 av_popcount64_rv
|
||||
|
||||
static inline av_const int av_parity_rv(unsigned int x)
|
||||
{
|
||||
#if HAVE_RV && !defined(__riscv_zbb)
|
||||
if (!__builtin_constant_p(x) &&
|
||||
__builtin_expect(ff_rv_zbb_support(), true)) {
|
||||
int y;
|
||||
|
||||
__asm__ (
|
||||
".option push\n"
|
||||
".option arch, +zbb\n"
|
||||
#if __riscv_xlen >= 64
|
||||
"cpopw %0, %1\n"
|
||||
#else
|
||||
"cpop %0, %1\n"
|
||||
#endif
|
||||
".option pop" : "=r" (y) : "r" (x));
|
||||
return y & 1;
|
||||
}
|
||||
#endif
|
||||
return __builtin_parity(x);
|
||||
}
|
||||
#define av_parity av_parity_rv
|
||||
#endif
|
||||
|
||||
#endif /* AVUTIL_RISCV_INTMATH_H */
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright © 2024 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "config.h"
|
||||
#include "libavutil/attributes.h"
|
||||
#include "libavutil/cpu.h"
|
||||
#include "libavutil/riscv/cpu.h"
|
||||
#include "libavutil/lls.h"
|
||||
|
||||
void ff_lls_update_covariance_rvv(double covar[][36], const double *var,
|
||||
int count);
|
||||
double ff_scalarproduct_double_rvv(const double *, const double *, size_t);
|
||||
|
||||
static void ff_lls_update_rvv(struct LLSModel *m, const double *var)
|
||||
{
|
||||
ff_lls_update_covariance_rvv(m->covariance, var, m->indep_count + 1);
|
||||
}
|
||||
|
||||
static double ff_lls_evaluate_rvv(struct LLSModel *m, const double *var,
|
||||
int order)
|
||||
{
|
||||
return ff_scalarproduct_double_rvv(m->coeff[order], var, order + 1);
|
||||
}
|
||||
|
||||
av_cold void ff_init_lls_riscv(LLSModel *m)
|
||||
{
|
||||
#if HAVE_RVV
|
||||
int flags = av_get_cpu_flags();
|
||||
|
||||
if ((flags & AV_CPU_FLAG_RVB) && (flags & AV_CPU_FLAG_RVV_F64)) {
|
||||
if (ff_get_rv_vlenb() > m->indep_count)
|
||||
m->update_lls = ff_lls_update_rvv;
|
||||
m->evaluate_lls = ff_lls_evaluate_rvv;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright © 2024 Rémi Denis-Courmont.
|
||||
*
|
||||
* This file is part of FFmpeg.
|
||||
*
|
||||
* FFmpeg is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
* License as published by the Free Software Foundation; either
|
||||
* version 2.1 of the License, or (at your option) any later version.
|
||||
*
|
||||
* FFmpeg is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public
|
||||
* License along with FFmpeg; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
#include "asm.S"
|
||||
|
||||
func ff_lls_update_covariance_rvv, zve64d, zbb
|
||||
lpad 0
|
||||
vtype_vli t0, a2, t1, e64, ta, ma
|
||||
vsetvl zero, a2, t0
|
||||
vle64.v v8, (a1)
|
||||
1:
|
||||
vfmv.f.s ft0, v8
|
||||
vle64.v v16, (a0)
|
||||
vfmacc.vf v16, ft0, v8
|
||||
addi a2, a2, -1
|
||||
vslidedown.vi v8, v8, 1
|
||||
vse64.v v16, (a0)
|
||||
addi a0, a0, (36 + 1) * 8 # 1 row + 1 element
|
||||
bnez a2, 1b
|
||||
|
||||
ret
|
||||
endfunc
|
||||
Reference in New Issue
Block a user