From 64b8b6516b3cba19dba4c8f4f9b97daa0556fd98 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Tue, 8 Nov 2022 17:38:40 -0800 Subject: [PATCH] x86: Add evex optimized functions for the wchar_t strcpy family Implemented: wcscat-evex (+ 905 bytes) wcscpy-evex (+ 674 bytes) wcpcpy-evex (+ 709 bytes) wcsncpy-evex (+1358 bytes) wcpncpy-evex (+1467 bytes) wcsncat-evex (+1213 bytes) Performance Changes: Times are from N = 10 runs of the benchmark suite and are reported as geometric mean of all ratios of New Implementation / Best Old Implementation. Best Old Implementation was determined with the highest ISA implementation. wcscat-evex -> 0.991 wcscpy-evex -> 0.587 wcpcpy-evex -> 0.695 wcsncpy-evex -> 0.719 wcpncpy-evex -> 0.694 wcsncat-evex -> 0.979 Code Size Changes: This change increase the size of libc.so by ~6.3kb bytes. For reference the patch optimizing the normal strcpy family functions decreases libc.so by ~5.7kb. Full check passes on x86-64 and build succeeds for all ISA levels w/ and w/o multiarch. --- sysdeps/x86_64/Makefile | 5 ++ sysdeps/x86_64/multiarch/Makefile | 14 ++++- sysdeps/x86_64/multiarch/ifunc-impl-list.c | 63 ++++++++++++++++++++-- sysdeps/x86_64/multiarch/ifunc-wcs.h | 48 +++++++++++++++++ sysdeps/x86_64/multiarch/wcpcpy-evex.S | 8 +++ .../{wcscpy-generic.c => wcpcpy-generic.c} | 12 ++--- .../multiarch/{wcscpy-generic.c => wcpcpy.c} | 22 +++++--- sysdeps/x86_64/multiarch/wcpncpy-evex.S | 8 +++ .../{wcscpy-generic.c => wcpncpy-generic.c} | 12 ++--- .../multiarch/{wcscpy-generic.c => wcpncpy.c} | 22 +++++--- sysdeps/x86_64/multiarch/wcscat-evex.S | 9 ++++ .../{wcscpy-generic.c => wcscat-generic.c} | 12 ++--- sysdeps/x86_64/multiarch/{wcscpy.c => wcscat.c} | 33 ++++-------- sysdeps/x86_64/multiarch/wcscpy-evex.S | 7 +++ sysdeps/x86_64/multiarch/wcscpy-generic.c | 3 +- sysdeps/x86_64/multiarch/wcscpy.c | 11 ++++ sysdeps/x86_64/multiarch/wcsncat-evex.S | 9 ++++ .../{wcscpy-generic.c => wcsncat-generic.c} | 12 ++--- .../multiarch/{wcscpy-generic.c => wcsncat.c} | 21 +++++--- sysdeps/x86_64/multiarch/wcsncpy-evex.S | 7 +++ .../{wcscpy-generic.c => wcsncpy-generic.c} | 12 ++--- sysdeps/x86_64/multiarch/{wcscpy.c => wcsncpy.c} | 33 ++++-------- .../wcscpy-generic.c => wcpcpy-generic.c} | 12 +++-- sysdeps/x86_64/{wcscpy.S => wcpcpy.S} | 22 ++++---- .../wcscpy-generic.c => wcpncpy-generic.c} | 12 +++-- sysdeps/x86_64/{wcscpy.S => wcpncpy.S} | 22 ++++---- .../wcscpy-generic.c => wcscat-generic.c} | 12 +++-- sysdeps/x86_64/{wcscpy.S => wcscat.S} | 22 ++++---- sysdeps/x86_64/wcscpy.S | 3 +- .../wcscpy-generic.c => wcsncat-generic.c} | 12 +++-- sysdeps/x86_64/{wcscpy.S => wcsncat.S} | 20 ++++--- .../wcscpy-generic.c => wcsncpy-generic.c} | 12 +++-- sysdeps/x86_64/{wcscpy.S => wcsncpy.S} | 22 ++++---- 33 files changed, 377 insertions(+), 177 deletions(-) create mode 100644 sysdeps/x86_64/multiarch/ifunc-wcs.h create mode 100644 sysdeps/x86_64/multiarch/wcpcpy-evex.S copy sysdeps/x86_64/multiarch/{wcscpy-generic.c => wcpcpy-generic.c} (80%) copy sysdeps/x86_64/multiarch/{wcscpy-generic.c => wcpcpy.c} (58%) create mode 100644 sysdeps/x86_64/multiarch/wcpncpy-evex.S copy sysdeps/x86_64/multiarch/{wcscpy-generic.c => wcpncpy-generic.c} (79%) copy sysdeps/x86_64/multiarch/{wcscpy-generic.c => wcpncpy.c} (57%) create mode 100644 sysdeps/x86_64/multiarch/wcscat-evex.S copy sysdeps/x86_64/multiarch/{wcscpy-generic.c => wcscat-generic.c} (80%) copy sysdeps/x86_64/multiarch/{wcscpy.c => wcscat.c} (59%) create mode 100644 sysdeps/x86_64/multiarch/wcscpy-evex.S create mode 100644 sysdeps/x86_64/multiarch/wcsncat-evex.S copy sysdeps/x86_64/multiarch/{wcscpy-generic.c => wcsncat-generic.c} (79%) copy sysdeps/x86_64/multiarch/{wcscpy-generic.c => wcsncat.c} (60%) create mode 100644 sysdeps/x86_64/multiarch/wcsncpy-evex.S copy sysdeps/x86_64/multiarch/{wcscpy-generic.c => wcsncpy-generic.c} (79%) copy sysdeps/x86_64/multiarch/{wcscpy.c => wcsncpy.c} (58%) copy sysdeps/x86_64/{multiarch/wcscpy-generic.c => wcpcpy-generic.c} (67%) copy sysdeps/x86_64/{wcscpy.S => wcpcpy.S} (65%) copy sysdeps/x86_64/{multiarch/wcscpy-generic.c => wcpncpy-generic.c} (66%) copy sysdeps/x86_64/{wcscpy.S => wcpncpy.S} (65%) copy sysdeps/x86_64/{multiarch/wcscpy-generic.c => wcscat-generic.c} (67%) copy sysdeps/x86_64/{wcscpy.S => wcscat.S} (65%) copy sysdeps/x86_64/{multiarch/wcscpy-generic.c => wcsncat-generic.c} (66%) copy sysdeps/x86_64/{wcscpy.S => wcsncat.S} (65%) copy sysdeps/x86_64/{multiarch/wcscpy-generic.c => wcsncpy-generic.c} (66%) copy sysdeps/x86_64/{wcscpy.S => wcsncpy.S} (65%) diff --git a/sysdeps/x86_64/Makefile b/sysdeps/x86_64/Makefile index 3627c5659f..688eb2d7c4 100644 --- a/sysdeps/x86_64/Makefile +++ b/sysdeps/x86_64/Makefile @@ -188,8 +188,13 @@ endif ifeq ($(subdir),wcsmbs) sysdep_routines += \ + wcpcpy-generic \ + wcpncpy-generic \ + wcscat-generic \ wcscpy-generic \ + wcsncat-generic \ wcsncmp-generic \ + wcsncpy-generic \ wcsnlen-generic \ # sysdep_routines diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index 066bfa48d9..d6e01940c3 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -131,6 +131,12 @@ endif ifeq ($(subdir),wcsmbs) sysdep_routines += \ + wcpcpy-evex \ + wcpcpy-generic \ + wcpncpy-evex \ + wcpncpy-generic \ + wcscat-evex \ + wcscat-generic \ wcschr-avx2 \ wcschr-avx2-rtm \ wcschr-evex \ @@ -140,6 +146,8 @@ sysdep_routines += \ wcscmp-avx2-rtm \ wcscmp-evex \ wcscmp-sse2 \ + wcscpy-evex \ + wcscpy-generic \ wcscpy-ssse3 \ wcslen-avx2 \ wcslen-avx2-rtm \ @@ -147,9 +155,13 @@ sysdep_routines += \ wcslen-evex512 \ wcslen-sse2 \ wcslen-sse4_1 \ + wcsncat-evex \ + wcsncat-generic \ wcsncmp-avx2 \ wcsncmp-avx2-rtm \ wcsncmp-evex \ + wcsncpy-evex \ + wcsncpy-generic \ wcsnlen-avx2 \ wcsnlen-avx2-rtm \ wcsnlen-evex \ @@ -163,8 +175,8 @@ sysdep_routines += \ wmemchr-avx2 \ wmemchr-avx2-rtm \ wmemchr-evex \ - wmemchr-evex512 \ wmemchr-evex-rtm \ + wmemchr-evex512 \ wmemchr-sse2 \ wmemcmp-avx2-movbe \ wmemcmp-avx2-movbe-rtm \ diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c index 7cebee7ec7..c908d6c158 100644 --- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c +++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c @@ -901,16 +901,73 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, /* Support sysdeps/x86_64/multiarch/wcscpy.c. */ IFUNC_IMPL (i, name, wcscpy, - /* ISA V4 wrapper for SSSE3 implementation because - the SSSE3 implementation is also used at ISA - level 3/4. */ X86_IFUNC_IMPL_ADD_V4 (array, i, wcscpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __wcscpy_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, wcscpy, CPU_FEATURE_USABLE (SSSE3), __wcscpy_ssse3) X86_IFUNC_IMPL_ADD_V1 (array, i, wcscpy, 1, __wcscpy_generic)) + /* Support sysdeps/x86_64/multiarch/wcsncpy.c. */ + IFUNC_IMPL (i, name, wcsncpy, + X86_IFUNC_IMPL_ADD_V4 (array, i, wcsncpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __wcsncpy_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, wcpncpy, + 1, + __wcsncpy_generic)) + + /* Support sysdeps/x86_64/multiarch/wcpcpy.c. */ + IFUNC_IMPL (i, name, wcpcpy, + X86_IFUNC_IMPL_ADD_V4 (array, i, wcpcpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __wcpcpy_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, wcpcpy, + 1, + __wcpcpy_generic)) + + /* Support sysdeps/x86_64/multiarch/wcpncpy.c. */ + IFUNC_IMPL (i, name, wcpncpy, + X86_IFUNC_IMPL_ADD_V4 (array, i, wcpncpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __wcpncpy_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncpy, + 1, + __wcpncpy_generic)) + + /* Support sysdeps/x86_64/multiarch/wcscat.c. */ + IFUNC_IMPL (i, name, wcscat, + X86_IFUNC_IMPL_ADD_V4 (array, i, wcscat, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __wcscat_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, wcscat, + 1, + __wcscat_generic)) + + /* Support sysdeps/x86_64/multiarch/wcsncat.c. */ + IFUNC_IMPL (i, name, wcsncat, + X86_IFUNC_IMPL_ADD_V4 (array, i, wcsncat, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (BMI2)), + __wcsncat_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncat, + 1, + __wcsncat_generic)) + /* Support sysdeps/x86_64/multiarch/wcslen.c. */ IFUNC_IMPL (i, name, wcslen, X86_IFUNC_IMPL_ADD_V4 (array, i, wcslen, diff --git a/sysdeps/x86_64/multiarch/ifunc-wcs.h b/sysdeps/x86_64/multiarch/ifunc-wcs.h new file mode 100644 index 0000000000..1d2a63458b --- /dev/null +++ b/sysdeps/x86_64/multiarch/ifunc-wcs.h @@ -0,0 +1,48 @@ +/* Common definition for ifunc selections optimized wide-character + string copy functions. + + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2022 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#include + +#ifndef GENERIC +# define GENERIC generic +#endif + +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + +extern __typeof (REDIRECT_NAME) OPTIMIZE (GENERIC) attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ + const struct cpu_features *cpu_features = __get_cpu_features (); + + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2) + && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, + AVX_Fast_Unaligned_Load, )) + { + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) + && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) + return OPTIMIZE (evex); + } + + return OPTIMIZE (GENERIC); +} diff --git a/sysdeps/x86_64/multiarch/wcpcpy-evex.S b/sysdeps/x86_64/multiarch/wcpcpy-evex.S new file mode 100644 index 0000000000..ac6429cc07 --- /dev/null +++ b/sysdeps/x86_64/multiarch/wcpcpy-evex.S @@ -0,0 +1,8 @@ +#ifndef WCPCPY +# define WCPCPY __wcpcpy_evex +#endif + +#define USE_AS_STPCPY +#define USE_AS_WCSCPY +#define STRCPY WCPCPY +#include "strcpy-evex.S" diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcpcpy-generic.c similarity index 80% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/multiarch/wcpcpy-generic.c index 93d314aaad..6039196a3e 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/multiarch/wcpcpy-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* wcpcpy. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,12 @@ License along with the GNU C Library; if not, see . */ - +/* We always need to build this implementation as strspn-sse4 needs to + be able to fallback to it. */ #include +#if ISA_SHOULD_BUILD (3) -#if ISA_SHOULD_BUILD (1) - -# define WCSCPY __wcscpy_generic -# include +# define WCPCPY __wcpcpy_generic +# include #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcpcpy.c similarity index 58% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/multiarch/wcpcpy.c index 93d314aaad..8f96ddbc99 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/multiarch/wcpcpy.c @@ -1,4 +1,5 @@ -/* wcscpy. +/* Multiple versions of wcpcpy. + All versions must be listed in ifunc-impl-list.c. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +17,21 @@ License along with the GNU C Library; if not, see . */ +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define __wcpcpy __redirect_wcpcpy +# include +# undef __wcpcpy -#include +# define SYMBOL_NAME wcpcpy +# include -#if ISA_SHOULD_BUILD (1) - -# define WCSCPY __wcscpy_generic -# include +# include "ifunc-wcs.h" +libc_ifunc_redirected (__redirect_wcpcpy, __wcpcpy, IFUNC_SELECTOR ()); +weak_alias (__wcpcpy, wcpcpy) +# ifdef SHARED +__hidden_ver1 (__wcpcpy, __GI___wcpcpy, __redirect_wcpcpy) + __attribute__((visibility ("hidden"))) __attribute_copy__ (wcpcpy); +# endif #endif diff --git a/sysdeps/x86_64/multiarch/wcpncpy-evex.S b/sysdeps/x86_64/multiarch/wcpncpy-evex.S new file mode 100644 index 0000000000..62ddb694fe --- /dev/null +++ b/sysdeps/x86_64/multiarch/wcpncpy-evex.S @@ -0,0 +1,8 @@ +#ifndef WCPNCPY +# define WCPNCPY __wcpncpy_evex +#endif + +#define USE_AS_WCSCPY +#define USE_AS_STPCPY +#define STRNCPY WCPNCPY +#include "strncpy-evex.S" diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcpncpy-generic.c similarity index 79% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/multiarch/wcpncpy-generic.c index 93d314aaad..de8d34320e 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/multiarch/wcpncpy-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* wcpncpy. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,12 @@ License along with the GNU C Library; if not, see . */ - +/* We always need to build this implementation as strspn-sse4 needs to + be able to fallback to it. */ #include +#if ISA_SHOULD_BUILD (3) -#if ISA_SHOULD_BUILD (1) - -# define WCSCPY __wcscpy_generic -# include +# define WCPNCPY __wcpncpy_generic +# include #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcpncpy.c similarity index 57% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/multiarch/wcpncpy.c index 93d314aaad..ed8f307e07 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/multiarch/wcpncpy.c @@ -1,4 +1,5 @@ -/* wcscpy. +/* Multiple versions of wcpncpy. + All versions must be listed in ifunc-impl-list.c. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +17,21 @@ License along with the GNU C Library; if not, see . */ +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define __wcpncpy __redirect_wcpncpy +# include +# undef __wcpncpy -#include +# define SYMBOL_NAME wcpncpy +# include -#if ISA_SHOULD_BUILD (1) - -# define WCSCPY __wcscpy_generic -# include +# include "ifunc-wcs.h" +libc_ifunc_redirected (__redirect_wcpncpy, __wcpncpy, IFUNC_SELECTOR ()); +weak_alias (__wcpncpy, wcpncpy) +# ifdef SHARED +__hidden_ver1 (__wcpncpy, __GI___wcpncpy, __redirect_wcpncpy) + __attribute__((visibility ("hidden"))) __attribute_copy__ (wcpncpy); +# endif #endif diff --git a/sysdeps/x86_64/multiarch/wcscat-evex.S b/sysdeps/x86_64/multiarch/wcscat-evex.S new file mode 100644 index 0000000000..1d017e4899 --- /dev/null +++ b/sysdeps/x86_64/multiarch/wcscat-evex.S @@ -0,0 +1,9 @@ +#ifndef WCSCAT +# define WCSCAT __wcscat_evex +#endif + +#define USE_AS_WCSCPY +#define USE_AS_STRCAT + +#define STRCPY WCSCAT +#include "strcpy-evex.S" diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcscat-generic.c similarity index 80% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/multiarch/wcscat-generic.c index 93d314aaad..d86b4d5c00 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/multiarch/wcscat-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* wcscat. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,12 @@ License along with the GNU C Library; if not, see . */ - +/* We always need to build this implementation as strspn-sse4 needs to + be able to fallback to it. */ #include +#if ISA_SHOULD_BUILD (3) -#if ISA_SHOULD_BUILD (1) - -# define WCSCPY __wcscpy_generic -# include +# define WCSCAT __wcscat_generic +# include #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy.c b/sysdeps/x86_64/multiarch/wcscat.c similarity index 59% copy from sysdeps/x86_64/multiarch/wcscpy.c copy to sysdeps/x86_64/multiarch/wcscat.c index 92c917b6b4..3277c44561 100644 --- a/sysdeps/x86_64/multiarch/wcscpy.c +++ b/sysdeps/x86_64/multiarch/wcscat.c @@ -1,6 +1,6 @@ -/* Multiple versions of wcscpy. +/* Multiple versions of wcscat. All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2017-2022 Free Software Foundation, Inc. + Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -19,32 +19,19 @@ /* Define multiple versions only for the definition in libc. */ #if IS_IN (libc) -# define __wcscpy __redirect_wcscpy +# define __wcscat __redirect_wcscat # include -# undef __wcscpy +# undef __wcscat -# define SYMBOL_NAME wcscpy +# define SYMBOL_NAME wcscat # include -extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; +# include "ifunc-wcs.h" -extern __typeof (REDIRECT_NAME) OPTIMIZE (generic) attribute_hidden; - -static inline void * -IFUNC_SELECTOR (void) -{ - const struct cpu_features* cpu_features = __get_cpu_features (); - - if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, SSSE3)) - return OPTIMIZE (ssse3); - - return OPTIMIZE (generic); -} - -libc_ifunc_redirected (__redirect_wcscpy, __wcscpy, IFUNC_SELECTOR ()); -weak_alias (__wcscpy, wcscpy) +libc_ifunc_redirected (__redirect_wcscat, __wcscat, IFUNC_SELECTOR ()); +weak_alias (__wcscat, wcscat) # ifdef SHARED -__hidden_ver1 (__wcscpy, __GI___wcscpy, __redirect_wcscpy) - __attribute__((visibility ("hidden"))) __attribute_copy__ (wcscpy); +__hidden_ver1 (__wcscat, __GI___wcscat, __redirect_wcscat) + __attribute__((visibility ("hidden"))) __attribute_copy__ (wcscat); # endif #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy-evex.S b/sysdeps/x86_64/multiarch/wcscpy-evex.S new file mode 100644 index 0000000000..1069a8e224 --- /dev/null +++ b/sysdeps/x86_64/multiarch/wcscpy-evex.S @@ -0,0 +1,7 @@ +#ifndef WCSCPY +# define WCSCPY __wcscpy_evex +#endif + +#define USE_AS_WCSCPY +#define STRCPY WCSCPY +#include "strcpy-evex.S" diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcscpy-generic.c index 93d314aaad..4a1fffae4b 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/multiarch/wcscpy-generic.c @@ -18,8 +18,7 @@ #include - -#if ISA_SHOULD_BUILD (1) +#if ISA_SHOULD_BUILD (3) # define WCSCPY __wcscpy_generic # include diff --git a/sysdeps/x86_64/multiarch/wcscpy.c b/sysdeps/x86_64/multiarch/wcscpy.c index 92c917b6b4..9ad77da8ac 100644 --- a/sysdeps/x86_64/multiarch/wcscpy.c +++ b/sysdeps/x86_64/multiarch/wcscpy.c @@ -26,6 +26,8 @@ # define SYMBOL_NAME wcscpy # include +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; extern __typeof (REDIRECT_NAME) OPTIMIZE (generic) attribute_hidden; @@ -35,6 +37,15 @@ IFUNC_SELECTOR (void) { const struct cpu_features* cpu_features = __get_cpu_features (); + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2) + && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load, )) + { + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) + && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) + return OPTIMIZE (evex); + } + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, SSSE3)) return OPTIMIZE (ssse3); diff --git a/sysdeps/x86_64/multiarch/wcsncat-evex.S b/sysdeps/x86_64/multiarch/wcsncat-evex.S new file mode 100644 index 0000000000..392215950a --- /dev/null +++ b/sysdeps/x86_64/multiarch/wcsncat-evex.S @@ -0,0 +1,9 @@ +#ifndef WCSCAT +# define WCSCAT __wcsncat_evex +#endif + +#define USE_AS_WCSCPY +#define USE_AS_STRCAT + +#define STRNCAT WCSCAT +#include "strncat-evex.S" diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcsncat-generic.c similarity index 79% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/multiarch/wcsncat-generic.c index 93d314aaad..4b55cb40bc 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/multiarch/wcsncat-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* wcsncat. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,12 @@ License along with the GNU C Library; if not, see . */ - +/* We always need to build this implementation as strspn-sse4 needs to + be able to fallback to it. */ #include +#if ISA_SHOULD_BUILD (3) -#if ISA_SHOULD_BUILD (1) - -# define WCSCPY __wcscpy_generic -# include +# define WCSNCAT __wcsncat_generic +# include #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcsncat.c similarity index 60% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/multiarch/wcsncat.c index 93d314aaad..49c46aef08 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/multiarch/wcsncat.c @@ -1,4 +1,5 @@ -/* wcscpy. +/* Multiple versions of wcsncat. + All versions must be listed in ifunc-impl-list.c. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +17,18 @@ License along with the GNU C Library; if not, see . */ +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define wcsncat __redirect_wcsncat +# include +# undef wcsncat -#include - -#if ISA_SHOULD_BUILD (1) - -# define WCSCPY __wcscpy_generic -# include +# define SYMBOL_NAME wcsncat +# include "ifunc-wcs.h" +libc_ifunc_redirected (__redirect_wcsncat, wcsncat, IFUNC_SELECTOR ()); +# ifdef SHARED +__hidden_ver1 (wcsncat, __GI_wcsncat, __redirect_wcsncat) + __attribute__((visibility ("hidden"))) __attribute_copy__ (wcsncat); +# endif #endif diff --git a/sysdeps/x86_64/multiarch/wcsncpy-evex.S b/sysdeps/x86_64/multiarch/wcsncpy-evex.S new file mode 100644 index 0000000000..2debb8fd6b --- /dev/null +++ b/sysdeps/x86_64/multiarch/wcsncpy-evex.S @@ -0,0 +1,7 @@ +#ifndef WCSNCPY +# define WCSNCPY __wcsncpy_evex +#endif + +#define USE_AS_WCSCPY +#define STRNCPY WCSNCPY +#include "strncpy-evex.S" diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcsncpy-generic.c similarity index 79% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/multiarch/wcsncpy-generic.c index 93d314aaad..d0e8a86605 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/multiarch/wcsncpy-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* wcsncpy. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,12 @@ License along with the GNU C Library; if not, see . */ - +/* We always need to build this implementation as strspn-sse4 needs to + be able to fallback to it. */ #include +#if ISA_SHOULD_BUILD (3) -#if ISA_SHOULD_BUILD (1) - -# define WCSCPY __wcscpy_generic -# include +# define WCSNCPY __wcsncpy_generic +# include #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy.c b/sysdeps/x86_64/multiarch/wcsncpy.c similarity index 58% copy from sysdeps/x86_64/multiarch/wcscpy.c copy to sysdeps/x86_64/multiarch/wcsncpy.c index 92c917b6b4..5b89dd4d27 100644 --- a/sysdeps/x86_64/multiarch/wcscpy.c +++ b/sysdeps/x86_64/multiarch/wcsncpy.c @@ -1,6 +1,6 @@ -/* Multiple versions of wcscpy. +/* Multiple versions of wcsncpy. All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2017-2022 Free Software Foundation, Inc. + Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -19,32 +19,19 @@ /* Define multiple versions only for the definition in libc. */ #if IS_IN (libc) -# define __wcscpy __redirect_wcscpy +# define __wcsncpy __redirect_wcsncpy # include -# undef __wcscpy +# undef __wcsncpy -# define SYMBOL_NAME wcscpy +# define SYMBOL_NAME wcsncpy # include -extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; +# include "ifunc-wcs.h" -extern __typeof (REDIRECT_NAME) OPTIMIZE (generic) attribute_hidden; - -static inline void * -IFUNC_SELECTOR (void) -{ - const struct cpu_features* cpu_features = __get_cpu_features (); - - if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, SSSE3)) - return OPTIMIZE (ssse3); - - return OPTIMIZE (generic); -} - -libc_ifunc_redirected (__redirect_wcscpy, __wcscpy, IFUNC_SELECTOR ()); -weak_alias (__wcscpy, wcscpy) +libc_ifunc_redirected (__redirect_wcsncpy, __wcsncpy, IFUNC_SELECTOR ()); +weak_alias (__wcsncpy, wcsncpy) # ifdef SHARED -__hidden_ver1 (__wcscpy, __GI___wcscpy, __redirect_wcscpy) - __attribute__((visibility ("hidden"))) __attribute_copy__ (wcscpy); +__hidden_ver1 (__wcsncpy, __GI___wcsncpy, __redirect_wcsncpy) + __attribute__((visibility ("hidden"))) __attribute_copy__ (wcsncpy); # endif #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/wcpcpy-generic.c similarity index 67% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/wcpcpy-generic.c index 93d314aaad..3ddc98872f 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/wcpcpy-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* ISA level static dispatch for wcpcpy .c files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,16 @@ License along with the GNU C Library; if not, see . */ +/* wcpcpy non-multiarch build is split into two files, + wcpcpy-generic.c and wcpcpy.S. The wcpcpy-generic.c build is + for ISA level <= 1 and just uses multiarch/wcpcpy-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if ISA_SHOULD_BUILD (1) +#if MINIMUM_X86_ISA_LEVEL <= 3 -# define WCSCPY __wcscpy_generic -# include +# include #endif diff --git a/sysdeps/x86_64/wcscpy.S b/sysdeps/x86_64/wcpcpy.S similarity index 65% copy from sysdeps/x86_64/wcscpy.S copy to sysdeps/x86_64/wcpcpy.S index 11d0bb4bab..4e4fca71eb 100644 --- a/sysdeps/x86_64/wcscpy.S +++ b/sysdeps/x86_64/wcpcpy.S @@ -1,4 +1,4 @@ -/* wcscpy dispatch for RTLD and non-multiarch .c files +/* ISA level static dispatch for wcpcpy .S files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,25 +16,25 @@ License along with the GNU C Library; if not, see . */ -/* wcscpy non-multiarch build is split into two files, - wcscpy-generic.c and wcscpy.S. The wcscpy.S build is for - ISA level >= 2 uses the optimized assembly implementations in - multiarch/wcscpy*.S. This must be split into two files because - we cannot include C code from assembly or vice versa. */ +/* wcpcpy non-multiarch build is split into two files, + wcpcpy-generic.c and wcpcpy.S. The wcpcpy-generic.c build is + for ISA level <= 1 and just uses multiarch/wcpcpy-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if MINIMUM_X86_ISA_LEVEL >= 2 +#if MINIMUM_X86_ISA_LEVEL >= 4 -# define WCSCPY __wcscpy +# define WCPCPY __wcpcpy -# define DEFAULT_IMPL_V2 "multiarch/wcscpy-ssse3.S" +# define DEFAULT_IMPL_V4 "multiarch/wcpcpy-evex.S" /* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it should never be used from here. */ # define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL" # include "isa-default-impl.h" -weak_alias (__wcscpy, wcscpy) -libc_hidden_def (__wcscpy) +weak_alias (__wcpcpy, wcpcpy) +libc_hidden_def (__wcpcpy) #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/wcpncpy-generic.c similarity index 66% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/wcpncpy-generic.c index 93d314aaad..0c76e5614c 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/wcpncpy-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* ISA level static dispatch for wcpncpy .c files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,16 @@ License along with the GNU C Library; if not, see . */ +/* wcpncpy non-multiarch build is split into two files, + wcpncpy-generic.c and wcpncpy.S. The wcpncpy-generic.c build is + for ISA level <= 1 and just uses multiarch/wcpncpy-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if ISA_SHOULD_BUILD (1) +#if MINIMUM_X86_ISA_LEVEL <= 3 -# define WCSCPY __wcscpy_generic -# include +# include #endif diff --git a/sysdeps/x86_64/wcscpy.S b/sysdeps/x86_64/wcpncpy.S similarity index 65% copy from sysdeps/x86_64/wcscpy.S copy to sysdeps/x86_64/wcpncpy.S index 11d0bb4bab..b4e531473e 100644 --- a/sysdeps/x86_64/wcscpy.S +++ b/sysdeps/x86_64/wcpncpy.S @@ -1,4 +1,4 @@ -/* wcscpy dispatch for RTLD and non-multiarch .c files +/* ISA level static dispatch for wcpcpy .S files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,25 +16,25 @@ License along with the GNU C Library; if not, see . */ -/* wcscpy non-multiarch build is split into two files, - wcscpy-generic.c and wcscpy.S. The wcscpy.S build is for - ISA level >= 2 uses the optimized assembly implementations in - multiarch/wcscpy*.S. This must be split into two files because - we cannot include C code from assembly or vice versa. */ +/* wcpncpy non-multiarch build is split into two files, + wcpncpy-generic.c and wcpncpy.S. The wcpncpy-generic.c build is + for ISA level <= 1 and just uses multiarch/wcpncpy-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if MINIMUM_X86_ISA_LEVEL >= 2 +#if MINIMUM_X86_ISA_LEVEL >= 4 -# define WCSCPY __wcscpy +# define WCPNCPY __wcpncpy -# define DEFAULT_IMPL_V2 "multiarch/wcscpy-ssse3.S" +# define DEFAULT_IMPL_V4 "multiarch/wcpncpy-evex.S" /* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it should never be used from here. */ # define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL" # include "isa-default-impl.h" -weak_alias (__wcscpy, wcscpy) -libc_hidden_def (__wcscpy) +weak_alias (__wcpncpy, wcpncpy) +libc_hidden_def (__wcpncpy) #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/wcscat-generic.c similarity index 67% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/wcscat-generic.c index 93d314aaad..512d0e4d43 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/wcscat-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* ISA level static dispatch for wcscat .c files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,16 @@ License along with the GNU C Library; if not, see . */ +/* wcscat non-multiarch build is split into two files, + wcscat-generic.c and wcscat.S. The wcscat-generic.c build is + for ISA level <= 1 and just uses multiarch/wcscat-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if ISA_SHOULD_BUILD (1) +#if MINIMUM_X86_ISA_LEVEL <= 3 -# define WCSCPY __wcscpy_generic -# include +# include #endif diff --git a/sysdeps/x86_64/wcscpy.S b/sysdeps/x86_64/wcscat.S similarity index 65% copy from sysdeps/x86_64/wcscpy.S copy to sysdeps/x86_64/wcscat.S index 11d0bb4bab..ee8360b6e8 100644 --- a/sysdeps/x86_64/wcscpy.S +++ b/sysdeps/x86_64/wcscat.S @@ -1,4 +1,4 @@ -/* wcscpy dispatch for RTLD and non-multiarch .c files +/* ISA level static dispatch for wcscat .S files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,25 +16,25 @@ License along with the GNU C Library; if not, see . */ -/* wcscpy non-multiarch build is split into two files, - wcscpy-generic.c and wcscpy.S. The wcscpy.S build is for - ISA level >= 2 uses the optimized assembly implementations in - multiarch/wcscpy*.S. This must be split into two files because - we cannot include C code from assembly or vice versa. */ +/* wcscat non-multiarch build is split into two files, + wcscat-generic.c and wcscat.S. The wcscat-generic.c build is + for ISA level <= 1 and just uses multiarch/wcscat-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if MINIMUM_X86_ISA_LEVEL >= 2 +#if MINIMUM_X86_ISA_LEVEL >= 4 -# define WCSCPY __wcscpy +# define WCSCAT __wcscat -# define DEFAULT_IMPL_V2 "multiarch/wcscpy-ssse3.S" +# define DEFAULT_IMPL_V4 "multiarch/wcscat-evex.S" /* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it should never be used from here. */ # define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL" # include "isa-default-impl.h" -weak_alias (__wcscpy, wcscpy) -libc_hidden_def (__wcscpy) +weak_alias (__wcscat, wcscat) +libc_hidden_def (__wcscat) #endif diff --git a/sysdeps/x86_64/wcscpy.S b/sysdeps/x86_64/wcscpy.S index 11d0bb4bab..e403579961 100644 --- a/sysdeps/x86_64/wcscpy.S +++ b/sysdeps/x86_64/wcscpy.S @@ -1,4 +1,4 @@ -/* wcscpy dispatch for RTLD and non-multiarch .c files +/* ISA level static dispatch for wcscpy .S files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -28,6 +28,7 @@ # define WCSCPY __wcscpy +# define DEFAULT_IMPL_V4 "multiarch/wcscpy-evex.S" # define DEFAULT_IMPL_V2 "multiarch/wcscpy-ssse3.S" /* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it should never be used from here. */ diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/wcsncat-generic.c similarity index 66% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/wcsncat-generic.c index 93d314aaad..86e20d9028 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/wcsncat-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* ISA level static dispatch for wcsncat .c files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,16 @@ License along with the GNU C Library; if not, see . */ +/* wcsncat non-multiarch build is split into two files, + wcsncat-generic.c and wcsncat.S. The wcsncat-generic.c build is + for ISA level <= 1 and just uses multiarch/wcsncat-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if ISA_SHOULD_BUILD (1) +#if MINIMUM_X86_ISA_LEVEL <= 3 -# define WCSCPY __wcscpy_generic -# include +# include #endif diff --git a/sysdeps/x86_64/wcscpy.S b/sysdeps/x86_64/wcsncat.S similarity index 65% copy from sysdeps/x86_64/wcscpy.S copy to sysdeps/x86_64/wcsncat.S index 11d0bb4bab..090055a1b8 100644 --- a/sysdeps/x86_64/wcscpy.S +++ b/sysdeps/x86_64/wcsncat.S @@ -1,4 +1,4 @@ -/* wcscpy dispatch for RTLD and non-multiarch .c files +/* ISA level static dispatch for wcsncat .S files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,25 +16,23 @@ License along with the GNU C Library; if not, see . */ -/* wcscpy non-multiarch build is split into two files, - wcscpy-generic.c and wcscpy.S. The wcscpy.S build is for - ISA level >= 2 uses the optimized assembly implementations in - multiarch/wcscpy*.S. This must be split into two files because - we cannot include C code from assembly or vice versa. */ +/* wcsncat non-multiarch build is split into two files, + wcsncat-generic.c and wcsncat.S. The wcsncat-generic.c build is + for ISA level <= 1 and just uses multiarch/wcsncat-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if MINIMUM_X86_ISA_LEVEL >= 2 +#if MINIMUM_X86_ISA_LEVEL >= 4 -# define WCSCPY __wcscpy +# define WCSNCAT wcsncat -# define DEFAULT_IMPL_V2 "multiarch/wcscpy-ssse3.S" +# define DEFAULT_IMPL_V4 "multiarch/wcsncat-evex.S" /* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it should never be used from here. */ # define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL" # include "isa-default-impl.h" -weak_alias (__wcscpy, wcscpy) -libc_hidden_def (__wcscpy) #endif diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/wcsncpy-generic.c similarity index 66% copy from sysdeps/x86_64/multiarch/wcscpy-generic.c copy to sysdeps/x86_64/wcsncpy-generic.c index 93d314aaad..0f0ee65b65 100644 --- a/sysdeps/x86_64/multiarch/wcscpy-generic.c +++ b/sysdeps/x86_64/wcsncpy-generic.c @@ -1,4 +1,4 @@ -/* wcscpy. +/* ISA level static dispatch for wcsncpy .c files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,12 +16,16 @@ License along with the GNU C Library; if not, see . */ +/* wcsncpy non-multiarch build is split into two files, + wcsncpy-generic.c and wcsncpy.S. The wcsncpy-generic.c build is + for ISA level <= 1 and just uses multiarch/wcsncpy-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if ISA_SHOULD_BUILD (1) +#if MINIMUM_X86_ISA_LEVEL <= 3 -# define WCSCPY __wcscpy_generic -# include +# include #endif diff --git a/sysdeps/x86_64/wcscpy.S b/sysdeps/x86_64/wcsncpy.S similarity index 65% copy from sysdeps/x86_64/wcscpy.S copy to sysdeps/x86_64/wcsncpy.S index 11d0bb4bab..32eaf1163b 100644 --- a/sysdeps/x86_64/wcscpy.S +++ b/sysdeps/x86_64/wcsncpy.S @@ -1,4 +1,4 @@ -/* wcscpy dispatch for RTLD and non-multiarch .c files +/* ISA level static dispatch for wcsncpy .S files. Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. @@ -16,25 +16,25 @@ License along with the GNU C Library; if not, see . */ -/* wcscpy non-multiarch build is split into two files, - wcscpy-generic.c and wcscpy.S. The wcscpy.S build is for - ISA level >= 2 uses the optimized assembly implementations in - multiarch/wcscpy*.S. This must be split into two files because - we cannot include C code from assembly or vice versa. */ +/* wcsncpy non-multiarch build is split into two files, + wcsncpy-generic.c and wcsncpy.S. The wcsncpy-generic.c build is + for ISA level <= 1 and just uses multiarch/wcsncpy-generic.c. + This must be split into two files because we cannot include C + code from assembly or vice versa. */ #include -#if MINIMUM_X86_ISA_LEVEL >= 2 +#if MINIMUM_X86_ISA_LEVEL >= 4 -# define WCSCPY __wcscpy +# define WCSNCPY __wcsncpy -# define DEFAULT_IMPL_V2 "multiarch/wcscpy-ssse3.S" +# define DEFAULT_IMPL_V4 "multiarch/wcsncpy-evex.S" /* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it should never be used from here. */ # define DEFAULT_IMPL_V1 "ERROR -- Invalid ISA IMPL" # include "isa-default-impl.h" -weak_alias (__wcscpy, wcscpy) -libc_hidden_def (__wcscpy) +weak_alias (__wcsncpy, wcsncpy) +libc_hidden_def (__wcsncpy) #endif -- 2.11.4.GIT