diff --git a/lib/libc/aarch64/string/Makefile.inc b/lib/libc/aarch64/string/Makefile.inc --- a/lib/libc/aarch64/string/Makefile.inc +++ b/lib/libc/aarch64/string/Makefile.inc @@ -5,10 +5,8 @@ AARCH64_STRING_FUNCS= \ memcmp \ - memcpy \ memmove \ memrchr \ - memset \ stpcpy \ strchr \ strchrnul \ @@ -34,7 +32,12 @@ timingsafe_bcmp.S \ timingsafe_memcmp.S \ bcopy.c \ - bzero.c + bzero.c \ + memcpy.S \ + memcpy_resolver.c \ + memmove_resolver.c \ + memset.S \ + memset_resolver.c # # Add the above functions. Generate an asm file that includes the needed @@ -55,6 +58,8 @@ CFLAGS.${FUNC}.S+=-I${SRCTOP}/contrib/arm-optimized-routines/string .endfor -# memchr.S is a wrapper in the src tree for the implementation from +# Several files are wrappers in the src tree for the implementation from # arm-optimized-routines CFLAGS.memchr.S+=-I${SRCTOP}/contrib/arm-optimized-routines/string +CFLAGS.memcpy.S+=-I${SRCTOP}/contrib/arm-optimized-routines/string +CFLAGS.memset.S+=-I${SRCTOP}/contrib/arm-optimized-routines/string diff --git a/lib/libc/aarch64/string/memcpy.S b/lib/libc/aarch64/string/memcpy.S --- a/lib/libc/aarch64/string/memcpy.S +++ b/lib/libc/aarch64/string/memcpy.S @@ -1,3 +1,3 @@ -#define __memcpy_aarch64_simd memcpy -#define __memmove_aarch64_simd memmove #include "aarch64/memcpy-advsimd.S" +#include "aarch64/memcpy-mops.S" +#include "aarch64/memmove-mops.S" diff --git a/lib/libc/aarch64/string/memcpy_resolver.c b/lib/libc/aarch64/string/memcpy_resolver.c new file mode 100644 --- /dev/null +++ b/lib/libc/aarch64/string/memcpy_resolver.c @@ -0,0 +1,42 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include +#include + +#include + +void *__memcpy_aarch64_simd(void *, const void *, size_t); +void *__memcpy_aarch64_mops(void *, const void *, size_t); + +DEFINE_UIFUNC(, void *, memcpy, (void *, const void *, size_t)) +{ + if (ifunc_arg->_hwcap2 & HWCAP2_MOPS) + return (__memcpy_aarch64_mops); + + return (__memcpy_aarch64_simd); +} + diff --git a/lib/libc/aarch64/string/memmove_resolver.c b/lib/libc/aarch64/string/memmove_resolver.c new file mode 100644 --- /dev/null +++ b/lib/libc/aarch64/string/memmove_resolver.c @@ -0,0 +1,42 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include +#include + +#include + +void *__memmove_aarch64_simd(void *, const void *, size_t); +void *__memmove_aarch64_mops(void *, const void *, size_t); + +DEFINE_UIFUNC(, void *, memmove, (void *, const void *, size_t)) +{ + if (ifunc_arg->_hwcap2 & HWCAP2_MOPS) + return (__memmove_aarch64_mops); + + return (__memmove_aarch64_simd); +} + diff --git a/lib/libc/aarch64/string/memset.S b/lib/libc/aarch64/string/memset.S new file mode 100644 --- /dev/null +++ b/lib/libc/aarch64/string/memset.S @@ -0,0 +1,2 @@ +#include "aarch64/memset.S" +#include "aarch64/memset-mops.S" diff --git a/lib/libc/aarch64/string/memset_resolver.c b/lib/libc/aarch64/string/memset_resolver.c new file mode 100644 --- /dev/null +++ b/lib/libc/aarch64/string/memset_resolver.c @@ -0,0 +1,42 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include +#include + +#include + +void *__memset_aarch64(void *, int, size_t); +void *__memset_aarch64_mops(void *, int, size_t); + +DEFINE_UIFUNC(, void *, memset, (void *, int, size_t)) +{ + if (ifunc_arg->_hwcap2 & HWCAP2_MOPS) + return (__memset_aarch64_mops); + + return (__memset_aarch64); +} +