From 02cea47161c00969afa985e641f4ac376f84d35f Mon Sep 17 00:00:00 2001 From: "H.J. Lu" Date: Tue, 4 Aug 2009 12:13:43 -0700 Subject: [PATCH] Add x86 32-bit SSE4.2 string functions. This patch adds 32bit SSE4.2 string functions. It uses -16L instead of 0xfffffffffffffff0L, which works for both 32bit and 64bit long. Tested on 32bit Core i7 and Core 2. --- ChangeLog | 24 ++++++++ sysdeps/i386/i686/multiarch/Makefile | 11 ++++ sysdeps/i386/i686/multiarch/strcasestr-c.c | 2 + sysdeps/i386/i686/multiarch/strcasestr.c | 1 + sysdeps/i386/i686/multiarch/strcspn-c.c | 2 + sysdeps/i386/i686/multiarch/strcspn.S | 99 ++++++++++++++++++++++++++++++ sysdeps/i386/i686/multiarch/strpbrk-c.c | 2 + sysdeps/i386/i686/multiarch/strpbrk.S | 3 + sysdeps/i386/i686/multiarch/strspn-c.c | 2 + sysdeps/i386/i686/multiarch/strspn.S | 80 ++++++++++++++++++++++++ sysdeps/i386/i686/multiarch/strstr-c.c | 12 ++++ sysdeps/i386/i686/multiarch/strstr.c | 1 + sysdeps/x86_64/multiarch/strcspn-c.c | 4 +- sysdeps/x86_64/multiarch/strspn-c.c | 4 +- 14 files changed, 243 insertions(+), 4 deletions(-) create mode 100644 sysdeps/i386/i686/multiarch/strcasestr-c.c create mode 100644 sysdeps/i386/i686/multiarch/strcasestr.c create mode 100644 sysdeps/i386/i686/multiarch/strcspn-c.c create mode 100644 sysdeps/i386/i686/multiarch/strcspn.S create mode 100644 sysdeps/i386/i686/multiarch/strpbrk-c.c create mode 100644 sysdeps/i386/i686/multiarch/strpbrk.S create mode 100644 sysdeps/i386/i686/multiarch/strspn-c.c create mode 100644 sysdeps/i386/i686/multiarch/strspn.S create mode 100644 sysdeps/i386/i686/multiarch/strstr-c.c create mode 100644 sysdeps/i386/i686/multiarch/strstr.c diff --git a/ChangeLog b/ChangeLog index c011f59c5c..149f66f1ff 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,27 @@ +2009-08-03 H.J. Lu + + * sysdeps/i386/i686/multiarch/Makefile [subdir=string] + (sysdep_routines): Add strcspn-c, strpbrk-c, strspn-c, strstr-c, and + strcasestr-c. + (CFLAGS-strcspn-c.c): Define. + (CFLAGS-strpbrk-c.c): Define. + (CFLAGS-strspn-c.c): Define. + (CFLAGS-strstr.c): Define. + (CFLAGS-strcasestr.c): Define. + * sysdeps/i386/i686/multiarch/strcspn-c.c: New file. + * sysdeps/i386/i686/multiarch/strcspn.S: New file. + * sysdeps/i386/i686/multiarch/strpbrk-c.c: New file. + * sysdeps/i386/i686/multiarch/strpbrk.S: New file. + * sysdeps/i386/i686/multiarch/strspn-c.c: New file. + * sysdeps/i386/i686/multiarch/strspn.S: New file. + * sysdeps/i386/i686/multiarch/strstr-c.c: New file. + * sysdeps/i386/i686/multiarch/strstr.c: New file. + * sysdeps/i386/i686/multiarch/strcasestr-c.c: New file. + * sysdeps/i386/i686/multiarch/strcasestr.c: New file. + * sysdeps/x86_64/multiarch/strcspn-c.c (STRCSPN_SSE42): Use + -16L instead of 0xfffffffffffffff0L. + * sysdeps/x86_64/multiarch/strspn-c.c (__strspn_sse42): Likewise. + 2009-08-02 Ulrich Drepper * sysdeps/i386/configure.in: Add test for . diff --git a/sysdeps/i386/i686/multiarch/Makefile b/sysdeps/i386/i686/multiarch/Makefile index 33d98c36e6..e1553b284e 100644 --- a/sysdeps/i386/i686/multiarch/Makefile +++ b/sysdeps/i386/i686/multiarch/Makefile @@ -2,3 +2,14 @@ ifeq ($(subdir),csu) aux += init-arch gen-as-const-headers += ifunc-defines.sym endif + +ifeq ($(subdir),string) +ifeq (yes,$(config-cflags-sse4)) +sysdep_routines += strcspn-c strpbrk-c strspn-c strstr-c strcasestr-c +CFLAGS-strcspn-c.c += -msse4 +CFLAGS-strpbrk-c.c += -msse4 +CFLAGS-strspn-c.c += -msse4 +CFLAGS-strstr.c += -msse4 +CFLAGS-strcasestr.c += -msse4 +endif +endif diff --git a/sysdeps/i386/i686/multiarch/strcasestr-c.c b/sysdeps/i386/i686/multiarch/strcasestr-c.c new file mode 100644 index 0000000000..0d52b0e47a --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strcasestr-c.c @@ -0,0 +1,2 @@ +#define __strcasestr_sse2 __strcasestr_ia32 +#include diff --git a/sysdeps/i386/i686/multiarch/strcasestr.c b/sysdeps/i386/i686/multiarch/strcasestr.c new file mode 100644 index 0000000000..511bb29ede --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strcasestr.c @@ -0,0 +1 @@ +#include diff --git a/sysdeps/i386/i686/multiarch/strcspn-c.c b/sysdeps/i386/i686/multiarch/strcspn-c.c new file mode 100644 index 0000000000..6d61e190a8 --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strcspn-c.c @@ -0,0 +1,2 @@ +#define __strcspn_sse2 __strcspn_ia32 +#include diff --git a/sysdeps/i386/i686/multiarch/strcspn.S b/sysdeps/i386/i686/multiarch/strcspn.S new file mode 100644 index 0000000000..f5ca092340 --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strcspn.S @@ -0,0 +1,99 @@ +/* Multiple versions of strcspn + Copyright (C) 2009 Free Software Foundation, Inc. + Contributed by Intel Corporation. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include + +#ifdef HAVE_SSE4_SUPPORT + +#include +#include + +#ifdef USE_AS_STRPBRK +#define STRCSPN_SSE42 __strpbrk_sse42 +#define STRCSPN_IA32 __strpbrk_ia32 +#define __GI_STRCSPN __GI_strpbrk +#else +#ifndef STRCSPN +#define STRCSPN strcspn +#define STRCSPN_SSE42 __strcspn_sse42 +#define STRCSPN_IA32 __strcspn_ia32 +#define __GI_STRCSPN __GI_strcspn +#endif +#endif + +/* Define multiple versions only for the definition in libc. Don't + define multiple versions for strpbrk in static library since we + need strpbrk before the initialization happened. */ +#if (defined SHARED || !defined USE_AS_STRPBRK) && !defined NOT_IN_libc + .section .gnu.linkonce.t.__i686.get_pc_thunk.bx,"ax",@progbits + .globl __i686.get_pc_thunk.bx + .hidden __i686.get_pc_thunk.bx + .p2align 4 + .type __i686.get_pc_thunk.bx,@function +__i686.get_pc_thunk.bx: + movl (%esp), %ebx + ret + + .text +ENTRY(STRCSPN) + .type STRCSPN, @gnu_indirect_function + pushl %ebx + cfi_adjust_cfa_offset (4) + cfi_rel_offset (ebx, 0) + call __i686.get_pc_thunk.bx + addl $_GLOBAL_OFFSET_TABLE_, %ebx + cmpl $0, KIND_OFFSET+__cpu_features@GOTOFF(%ebx) + jne 1f + call __init_cpu_features +1: leal STRCSPN_IA32@GOTOFF(%ebx), %eax + testl $(1<<20), CPUID_OFFSET+COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_ECX_OFFSET+__cpu_features@GOTOFF(%ebx) + jz 2f + leal STRCSPN_SSE42@GOTOFF(%ebx), %eax +2: popl %ebx + cfi_adjust_cfa_offset (-4); + cfi_restore (ebx) + ret +END(STRCSPN) + +# undef ENTRY +# define ENTRY(name) \ + .type STRCSPN_IA32, @function; \ + .globl STRCSPN_IA32; \ + .p2align 4; \ + STRCSPN_IA32: cfi_startproc; \ + CALL_MCOUNT +# undef END +# define END(name) \ + cfi_endproc; .size STRCSPN_IA32, .-STRCSPN_IA32 +# undef libc_hidden_builtin_def +/* It doesn't make sense to send libc-internal strcspn calls through a PLT. + The speedup we get from using SSE4.2 instruction is likely eaten away + by the indirect call in the PLT. */ +# define libc_hidden_builtin_def(name) \ + .globl __GI_STRCSPN; __GI_STRCSPN = STRCSPN_IA32 +#endif + +#endif /* HAVE_SSE4_SUPPORT */ + +#ifdef USE_AS_STRPBRK +#include "../../strpbrk.S" +#else +#include "../../strcspn.S" +#endif diff --git a/sysdeps/i386/i686/multiarch/strpbrk-c.c b/sysdeps/i386/i686/multiarch/strpbrk-c.c new file mode 100644 index 0000000000..5db62053b3 --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strpbrk-c.c @@ -0,0 +1,2 @@ +#define __strpbrk_sse2 __strpbrk_ia32 +#include diff --git a/sysdeps/i386/i686/multiarch/strpbrk.S b/sysdeps/i386/i686/multiarch/strpbrk.S new file mode 100644 index 0000000000..ed5bca6a94 --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strpbrk.S @@ -0,0 +1,3 @@ +#define STRCSPN strpbrk +#define USE_AS_STRPBRK +#include "strcspn.S" diff --git a/sysdeps/i386/i686/multiarch/strspn-c.c b/sysdeps/i386/i686/multiarch/strspn-c.c new file mode 100644 index 0000000000..bea09dea71 --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strspn-c.c @@ -0,0 +1,2 @@ +#define __strspn_sse2 __strspn_ia32 +#include diff --git a/sysdeps/i386/i686/multiarch/strspn.S b/sysdeps/i386/i686/multiarch/strspn.S new file mode 100644 index 0000000000..53db13181d --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strspn.S @@ -0,0 +1,80 @@ +/* Multiple versions of strspn + Copyright (C) 2009 Free Software Foundation, Inc. + Contributed by Intel Corporation. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include + +#ifdef HAVE_SSE4_SUPPORT + +#include +#include + +/* Define multiple versions only for the definition in libc. */ +#ifndef NOT_IN_libc + .section .gnu.linkonce.t.__i686.get_pc_thunk.bx,"ax",@progbits + .globl __i686.get_pc_thunk.bx + .hidden __i686.get_pc_thunk.bx + .p2align 4 + .type __i686.get_pc_thunk.bx,@function +__i686.get_pc_thunk.bx: + movl (%esp), %ebx + ret + + .text +ENTRY(strspn) + .type strspn, @gnu_indirect_function + pushl %ebx + cfi_adjust_cfa_offset (4) + cfi_rel_offset (ebx, 0) + call __i686.get_pc_thunk.bx + addl $_GLOBAL_OFFSET_TABLE_, %ebx + cmpl $0, KIND_OFFSET+__cpu_features@GOTOFF(%ebx) + jne 1f + call __init_cpu_features +1: leal __strspn_ia32@GOTOFF(%ebx), %eax + testl $(1<<20), CPUID_OFFSET+COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_ECX_OFFSET+__cpu_features@GOTOFF(%ebx) + jz 2f + leal __strspn_sse42@GOTOFF(%ebx), %eax +2: popl %ebx + cfi_adjust_cfa_offset (-4); + cfi_restore (ebx) + ret +END(strspn) + +# undef ENTRY +# define ENTRY(name) \ + .type __strspn_ia32, @function; \ + .globl __strspn_ia32; \ + .p2align 4 + __strspn_ia32: cfi_startproc; \ + CALL_MCOUNT +# undef END +# define END(name) \ + cfi_endproc; .size __strspn_ia32, .-__strspn_ia32 +# undef libc_hidden_builtin_def +/* It doesn't make sense to send libc-internal strspn calls through a PLT. + The speedup we get from using SSE4.2 instruction is likely eaten away + by the indirect call in the PLT. */ +# define libc_hidden_builtin_def(name) \ + .globl __GI_strspn; __GI_strspn = __strspn_ia32 +#endif + +#endif /* HAVE_SSE4_SUPPORT */ + +#include "../../strspn.S" diff --git a/sysdeps/i386/i686/multiarch/strstr-c.c b/sysdeps/i386/i686/multiarch/strstr-c.c new file mode 100644 index 0000000000..7ef1157ce4 --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strstr-c.c @@ -0,0 +1,12 @@ +#include "init-arch.h" + +#define STRSTR __strstr_ia32 +#undef libc_hidden_builtin_def +#define libc_hidden_builtin_def(name) \ + __hidden_ver1 (__strstr_ia32, __GI_strstr, __strstr_ia32); + +#include "string/strstr.c" + +extern char *__strstr_sse42 (const char *, const char *); + +libc_ifunc (strstr, HAS_SSE4_2 ? __strstr_sse42 : __strstr_ia32); diff --git a/sysdeps/i386/i686/multiarch/strstr.c b/sysdeps/i386/i686/multiarch/strstr.c new file mode 100644 index 0000000000..a97428c125 --- /dev/null +++ b/sysdeps/i386/i686/multiarch/strstr.c @@ -0,0 +1 @@ +#include diff --git a/sysdeps/x86_64/multiarch/strcspn-c.c b/sysdeps/x86_64/multiarch/strcspn-c.c index 4512267d3f..8286d0372b 100644 --- a/sysdeps/x86_64/multiarch/strcspn-c.c +++ b/sysdeps/x86_64/multiarch/strcspn-c.c @@ -90,7 +90,7 @@ STRCSPN_SSE42 (const char *s, const char *a) if (offset != 0) { /* Load masks. */ - aligned = (const char *) ((size_t) a & 0xfffffffffffffff0L); + aligned = (const char *) ((size_t) a & -16L); __m128i mask0 = _mm_load_si128 ((__m128i *) aligned); switch (offset) @@ -229,7 +229,7 @@ STRCSPN_SSE42 (const char *s, const char *a) if (offset != 0) { /* Check partial string. */ - aligned = (const char *) ((size_t) s & 0xfffffffffffffff0L); + aligned = (const char *) ((size_t) s & -16L); __m128i value = _mm_load_si128 ((__m128i *) aligned); switch (offset) diff --git a/sysdeps/x86_64/multiarch/strspn-c.c b/sysdeps/x86_64/multiarch/strspn-c.c index 5b99f0d383..be9e8ac0a8 100644 --- a/sysdeps/x86_64/multiarch/strspn-c.c +++ b/sysdeps/x86_64/multiarch/strspn-c.c @@ -68,7 +68,7 @@ __strspn_sse42 (const char *s, const char *a) if (offset != 0) { /* Load masks. */ - aligned = (const char *) ((size_t) a & 0xfffffffffffffff0L); + aligned = (const char *) ((size_t) a & -16L); __m128i mask0 = _mm_load_si128 ((__m128i *) aligned); switch (offset) @@ -207,7 +207,7 @@ __strspn_sse42 (const char *s, const char *a) if (offset != 0) { /* Check partial string. */ - aligned = (const char *) ((size_t) s & 0xfffffffffffffff0L); + aligned = (const char *) ((size_t) s & -16L); __m128i value = _mm_load_si128 ((__m128i *) aligned); switch (offset) -- 2.11.4.GIT