mirror of git://sourceware.org/git/glibc.git
riscv: Add RVV memset for both multiarch and non-multiarch builds
This patch adds an RVV-optimized implementation of memset for RISC-V and enables it for both multiarch (IFUNC) and non-multiarch builds. The implementation integrates Hau Hsu's 2023 RVV work under a unified ifunc-based framework. A vectorized version (__memset_vector) is added alongside the generic fallback (__memset_generic). The runtime resolver selects the RVV variant when RISCV_HWPROBE_KEY_IMA_EXT_0 reports vector support (RVV). Currently, the resolver still selects the RVV variant even when the RVV extension is disabled via prctl(). As a consequence, any process that has RVV disabled via prctl() will receive SIGILL when calling memset(). Co-authored-by: Jerry Shih <jerry.shih@sifive.com> Co-authored-by: Jeff Law <jeffreyalaw@gmail.com> Signed-off-by: Yao Zihong <zihong.plct@isrc.iscas.ac.cn> Reviewed-by: Peter Bergner <bergner@tenstorrent.com>
This commit is contained in:
Yao Zihong
Peter Bergner
parent
65f55bb7e5
commit
0b8a996f44
|
|
@ -0,0 +1,26 @@
|
|||
/* Symbol redirection for loader/static initialization code.
|
||||
Copyright (C) 2025 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#ifndef _DL_IFUNC_GENERIC_H
|
||||
#define _DL_IFUNC_GENERIC_H
|
||||
|
||||
#ifndef SHARED
|
||||
asm ("memset = __memset_generic");
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/* Re-include the default memset implementation.
|
||||
Copyright (C) 2025 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#if IS_IN(libc)
|
||||
# define MEMSET __memset_generic
|
||||
# undef libc_hidden_builtin_def
|
||||
# define libc_hidden_builtin_def(x)
|
||||
#endif
|
||||
#include <string/memset.c>
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
|
||||
/* RISC-V RVV based memset.
|
||||
Copyright (C) 2025 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#if IS_IN(libc)
|
||||
# define MEMSET __memset_vector
|
||||
# undef libc_hidden_builtin_def
|
||||
# define libc_hidden_builtin_def(name)
|
||||
#include <sysdeps/riscv/rvv/memset.S>
|
||||
#endif
|
||||
|
|
@ -60,6 +60,7 @@ riscv*)
|
|||
version=`$CC $CFLAGS $CPPFLAGS -E -dM -xc /dev/null | sed -n 's/^#define __GNUC__ \(.*\)/\1/p'`
|
||||
test $version -lt 15 && as_fn_error 1 "glibc requires GCC 15 or later for the V extension" "$LINENO" 5
|
||||
test $vector -lt "1000000" && as_fn_error 1 "glibc requires at least RVV 1.0 for the V extension" "$LINENO" 5
|
||||
float_machine=rvv
|
||||
fi
|
||||
|
||||
base_machine=riscv
|
||||
|
|
|
|||
|
|
@ -60,6 +60,7 @@ riscv*)
|
|||
version=`$CC $CFLAGS $CPPFLAGS -E -dM -xc /dev/null | sed -n 's/^#define __GNUC__ \(.*\)/\1/p'`
|
||||
test $version -lt 15 && AC_MSG_ERROR([glibc requires GCC 15 or later for the V extension], [1])
|
||||
test $vector -lt "1000000" && AC_MSG_ERROR([glibc requires at least RVV 1.0 for the V extension], [1])
|
||||
float_machine=rvv
|
||||
fi
|
||||
|
||||
base_machine=riscv
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
riscv/rv32/rvd
|
||||
riscv/rvv
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
riscv/rv64/rvd
|
||||
riscv/rvv
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
/* RISC-V RVV based memset.
|
||||
Copyright (C) 2025 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<http://www.gnu.org/licenses/>. */
|
||||
|
||||
#include <sysdep.h>
|
||||
#include <sys/asm.h>
|
||||
|
||||
#ifndef MEMSET
|
||||
# define MEMSET memset
|
||||
#endif
|
||||
|
||||
#define dst a0
|
||||
#define value a1
|
||||
#define num a2
|
||||
|
||||
#define ivl a3
|
||||
#define dst_ptr a5
|
||||
|
||||
#define ELEM_LMUL_SETTING m8
|
||||
#define vdata v0
|
||||
|
||||
ENTRY (MEMSET)
|
||||
.option push
|
||||
.option arch, +v
|
||||
mv dst_ptr, dst
|
||||
|
||||
vsetvli ivl, num, e8, ELEM_LMUL_SETTING, ta, ma
|
||||
vmv.v.x vdata, value
|
||||
L(loop):
|
||||
vse8.v vdata, (dst_ptr)
|
||||
sub num, num, ivl
|
||||
add dst_ptr, dst_ptr, ivl
|
||||
vsetvli ivl, num, e8, ELEM_LMUL_SETTING, ta, ma
|
||||
bnez num, L(loop)
|
||||
|
||||
ret
|
||||
.option pop
|
||||
END (MEMSET)
|
||||
libc_hidden_builtin_def (memset)
|
||||
|
|
@ -3,6 +3,9 @@ sysdep_routines += \
|
|||
memcpy \
|
||||
memcpy-generic \
|
||||
memcpy_noalignment \
|
||||
memset \
|
||||
memset-generic \
|
||||
memset-vector \
|
||||
# sysdep_routines
|
||||
|
||||
CFLAGS-memcpy_noalignment.c += -mno-strict-align
|
||||
|
|
|
|||
|
|
@ -27,17 +27,31 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
|
|||
size_t i = max;
|
||||
|
||||
bool fast_unaligned = false;
|
||||
bool rvv_enabled = false;
|
||||
|
||||
struct riscv_hwprobe pair = { .key = RISCV_HWPROBE_KEY_CPUPERF_0 };
|
||||
if (__riscv_hwprobe (&pair, 1, 0, NULL, 0) == 0
|
||||
&& (pair.value & RISCV_HWPROBE_MISALIGNED_MASK)
|
||||
struct riscv_hwprobe pairs[2] = {
|
||||
{.key = RISCV_HWPROBE_KEY_CPUPERF_0},
|
||||
{.key = RISCV_HWPROBE_KEY_IMA_EXT_0}
|
||||
};
|
||||
|
||||
if (__riscv_hwprobe (pairs, 2, 0, NULL, 0) == 0) {
|
||||
if ((pairs[0].value & RISCV_HWPROBE_MISALIGNED_MASK)
|
||||
== RISCV_HWPROBE_MISALIGNED_FAST)
|
||||
fast_unaligned = true;
|
||||
fast_unaligned = true;
|
||||
|
||||
if (pairs[1].value & RISCV_HWPROBE_IMA_V)
|
||||
rvv_enabled = true;
|
||||
}
|
||||
|
||||
IFUNC_IMPL (i, name, memcpy,
|
||||
IFUNC_IMPL_ADD (array, i, memcpy, fast_unaligned,
|
||||
__memcpy_noalignment)
|
||||
IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_generic))
|
||||
|
||||
IFUNC_IMPL (i, name, memset,
|
||||
IFUNC_IMPL_ADD (array, i, memset, rvv_enabled,
|
||||
__memset_vector)
|
||||
IFUNC_IMPL_ADD (array, i, memset, 1, __memset_generic))
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
/* Multiple versions of memset.
|
||||
All versions must be listed in ifunc-impl-list.c.
|
||||
Copyright (C) 2025 Free Software Foundation, Inc.
|
||||
This file is part of the GNU C Library.
|
||||
|
||||
The GNU C Library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation; either
|
||||
version 2.1 of the License, or (at your option) any later version.
|
||||
|
||||
The GNU C Library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with the GNU C Library; if not, see
|
||||
<https://www.gnu.org/licenses/>. */
|
||||
|
||||
#if IS_IN (libc)
|
||||
/* Redefine memset so that the compiler won't complain about the type
|
||||
mismatch with the IFUNC selector in strong_alias, below. */
|
||||
# undef memset
|
||||
# define memset __redirect_memset
|
||||
# include <stdint.h>
|
||||
# include <string.h>
|
||||
# include <ifunc-init.h>
|
||||
# include <riscv-ifunc.h>
|
||||
# include <sys/hwprobe.h>
|
||||
|
||||
extern __typeof (__redirect_memset) __libc_memset;
|
||||
|
||||
extern __typeof (__redirect_memset) __memset_generic attribute_hidden;
|
||||
extern __typeof (__redirect_memset) __memset_vector attribute_hidden;
|
||||
|
||||
static inline __typeof (__redirect_memset) *
|
||||
select_memset_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func)
|
||||
{
|
||||
unsigned long long v;
|
||||
|
||||
if (__riscv_hwprobe_one (hwprobe_func, RISCV_HWPROBE_KEY_IMA_EXT_0, &v) == 0
|
||||
&& (v & RISCV_HWPROBE_IMA_V) == RISCV_HWPROBE_IMA_V)
|
||||
return __memset_vector;
|
||||
|
||||
return __memset_generic;
|
||||
}
|
||||
|
||||
riscv_libc_ifunc (__libc_memset, select_memset_ifunc);
|
||||
|
||||
# undef memset
|
||||
strong_alias (__libc_memset, memset);
|
||||
# ifdef SHARED
|
||||
__hidden_ver1 (memset, __GI_memset, __redirect_memset)
|
||||
__attribute__ ((visibility ("hidden"))) __attribute_copy__ (memset);
|
||||
# endif
|
||||
#else
|
||||
# include <string/memset.c>
|
||||
#endif
|
||||
Loading…
Reference in New Issue