third_party/highway/hwy/highway.h

   1 // Copyright 2020 Google LLC
   2 // SPDX-License-Identifier: Apache-2.0
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //      http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15
  16 // Main header required before using vector types.
  17
  18 // IWYU pragma: begin_exports
  19 // Export does not seem to be recursive, so re-export these (also in base.h)
  20 #include <stddef.h>
  21 #include <stdint.h>
  22
  23 #include "hwy/base.h"
  24 #include "hwy/detect_compiler_arch.h"
  25 #include "hwy/highway_export.h"
  26 #include "hwy/targets.h"
  27 // IWYU pragma: end_exports
  28
  29 // This include guard is checked by foreach_target, so avoid the usual _H_
  30 // suffix to prevent copybara from renaming it. NOTE: ops/*-inl.h are included
  31 // after/outside this include guard.
  32 #ifndef HWY_HIGHWAY_INCLUDED
  33 #define HWY_HIGHWAY_INCLUDED
  34
  35 namespace hwy {
  36
  37 // API version (https://semver.org/); keep in sync with CMakeLists.txt.
  38 #define HWY_MAJOR 1
  39 #define HWY_MINOR 0
  40 #define HWY_PATCH 4
  41
  42 //------------------------------------------------------------------------------
  43 // Shorthand for tags (defined in shared-inl.h) used to select overloads.
  44 // Note that ScalableTag<T> is preferred over HWY_FULL, and CappedTag<T, N> over
  45 // HWY_CAPPED(T, N).
  46
  47 // HWY_FULL(T[,LMUL=1]) is a native vector/group. LMUL is the number of
  48 // registers in the group, and is ignored on targets that do not support groups.
  49 #define HWY_FULL1(T) hwy::HWY_NAMESPACE::ScalableTag<T>
  50 #define HWY_FULL2(T, LMUL) \
  51   hwy::HWY_NAMESPACE::ScalableTag<T, hwy::CeilLog2(HWY_MAX(0, LMUL))>
  52 #define HWY_3TH_ARG(arg1, arg2, arg3, ...) arg3
  53 // Workaround for MSVC grouping __VA_ARGS__ into a single argument
  54 #define HWY_FULL_RECOMPOSER(args_with_paren) HWY_3TH_ARG args_with_paren
  55 // Trailing comma avoids -pedantic false alarm
  56 #define HWY_CHOOSE_FULL(...) \
  57   HWY_FULL_RECOMPOSER((__VA_ARGS__, HWY_FULL2, HWY_FULL1, ))
  58 #define HWY_FULL(...) HWY_CHOOSE_FULL(__VA_ARGS__())(__VA_ARGS__)
  59
  60 // Vector of up to MAX_N lanes. It's better to use full vectors where possible.
  61 #define HWY_CAPPED(T, MAX_N) hwy::HWY_NAMESPACE::CappedTag<T, MAX_N>
  62
  63 //------------------------------------------------------------------------------
  64 // Export user functions for static/dynamic dispatch
  65
  66 // Evaluates to 0 inside a translation unit if it is generating anything but the
  67 // static target (the last one if multiple targets are enabled). Used to prevent
  68 // redefinitions of HWY_EXPORT. Unless foreach_target.h is included, we only
  69 // compile once anyway, so this is 1 unless it is or has been included.
  70 #ifndef HWY_ONCE
  71 #define HWY_ONCE 1
  72 #endif
  73
  74 // HWY_STATIC_DISPATCH(FUNC_NAME) is the namespace-qualified FUNC_NAME for
  75 // HWY_STATIC_TARGET (the only defined namespace unless HWY_TARGET_INCLUDE is
  76 // defined), and can be used to deduce the return type of Choose*.
  77 #if HWY_STATIC_TARGET == HWY_SCALAR
  78 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SCALAR::FUNC_NAME
  79 #elif HWY_STATIC_TARGET == HWY_EMU128
  80 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_EMU128::FUNC_NAME
  81 #elif HWY_STATIC_TARGET == HWY_RVV
  82 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_RVV::FUNC_NAME
  83 #elif HWY_STATIC_TARGET == HWY_WASM_EMU256
  84 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM_EMU256::FUNC_NAME
  85 #elif HWY_STATIC_TARGET == HWY_WASM
  86 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM::FUNC_NAME
  87 #elif HWY_STATIC_TARGET == HWY_NEON_WITHOUT_AES
  88 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON_WITHOUT_AES::FUNC_NAME
  89 #elif HWY_STATIC_TARGET == HWY_NEON
  90 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON::FUNC_NAME
  91 #elif HWY_STATIC_TARGET == HWY_SVE
  92 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE::FUNC_NAME
  93 #elif HWY_STATIC_TARGET == HWY_SVE2
  94 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2::FUNC_NAME
  95 #elif HWY_STATIC_TARGET == HWY_SVE_256
  96 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE_256::FUNC_NAME
  97 #elif HWY_STATIC_TARGET == HWY_SVE2_128
  98 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2_128::FUNC_NAME
  99 #elif HWY_STATIC_TARGET == HWY_PPC8
 100 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC8::FUNC_NAME
 101 #elif HWY_STATIC_TARGET == HWY_PPC9
 102 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC9::FUNC_NAME
 103 #elif HWY_STATIC_TARGET == HWY_PPC10
 104 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC10::FUNC_NAME
 105 #elif HWY_STATIC_TARGET == HWY_SSE2
 106 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE2::FUNC_NAME
 107 #elif HWY_STATIC_TARGET == HWY_SSSE3
 108 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSSE3::FUNC_NAME
 109 #elif HWY_STATIC_TARGET == HWY_SSE4
 110 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE4::FUNC_NAME
 111 #elif HWY_STATIC_TARGET == HWY_AVX2
 112 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX2::FUNC_NAME
 113 #elif HWY_STATIC_TARGET == HWY_AVX3
 114 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3::FUNC_NAME
 115 #elif HWY_STATIC_TARGET == HWY_AVX3_DL
 116 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_DL::FUNC_NAME
 117 #elif HWY_STATIC_TARGET == HWY_AVX3_ZEN4
 118 #define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_ZEN4::FUNC_NAME
 119 #endif
 120
 121 // HWY_CHOOSE_*(FUNC_NAME) expands to the function pointer for that target or
 122 // nullptr is that target was not compiled.
 123 #if HWY_TARGETS & HWY_EMU128
 124 #define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_EMU128::FUNC_NAME
 125 #elif HWY_TARGETS & HWY_SCALAR
 126 #define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_SCALAR::FUNC_NAME
 127 #else
 128 // When HWY_SCALAR/HWY_EMU128 are not present and other targets were disabled at
 129 // runtime, fall back to the baseline with HWY_STATIC_DISPATCH().
 130 #define HWY_CHOOSE_FALLBACK(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME)
 131 #endif
 132
 133 #if HWY_TARGETS & HWY_WASM_EMU256
 134 #define HWY_CHOOSE_WASM_EMU256(FUNC_NAME) &N_WASM_EMU256::FUNC_NAME
 135 #else
 136 #define HWY_CHOOSE_WASM_EMU256(FUNC_NAME) nullptr
 137 #endif
 138
 139 #if HWY_TARGETS & HWY_WASM
 140 #define HWY_CHOOSE_WASM(FUNC_NAME) &N_WASM::FUNC_NAME
 141 #else
 142 #define HWY_CHOOSE_WASM(FUNC_NAME) nullptr
 143 #endif
 144
 145 #if HWY_TARGETS & HWY_RVV
 146 #define HWY_CHOOSE_RVV(FUNC_NAME) &N_RVV::FUNC_NAME
 147 #else
 148 #define HWY_CHOOSE_RVV(FUNC_NAME) nullptr
 149 #endif
 150
 151 #if HWY_TARGETS & HWY_NEON_WITHOUT_AES
 152 #define HWY_CHOOSE_NEON_WITHOUT_AES(FUNC_NAME) &N_NEON_WITHOUT_AES::FUNC_NAME
 153 #else
 154 #define HWY_CHOOSE_NEON_WITHOUT_AES(FUNC_NAME) nullptr
 155 #endif
 156
 157 #if HWY_TARGETS & HWY_NEON
 158 #define HWY_CHOOSE_NEON(FUNC_NAME) &N_NEON::FUNC_NAME
 159 #else
 160 #define HWY_CHOOSE_NEON(FUNC_NAME) nullptr
 161 #endif
 162
 163 #if HWY_TARGETS & HWY_SVE
 164 #define HWY_CHOOSE_SVE(FUNC_NAME) &N_SVE::FUNC_NAME
 165 #else
 166 #define HWY_CHOOSE_SVE(FUNC_NAME) nullptr
 167 #endif
 168
 169 #if HWY_TARGETS & HWY_SVE2
 170 #define HWY_CHOOSE_SVE2(FUNC_NAME) &N_SVE2::FUNC_NAME
 171 #else
 172 #define HWY_CHOOSE_SVE2(FUNC_NAME) nullptr
 173 #endif
 174
 175 #if HWY_TARGETS & HWY_SVE_256
 176 #define HWY_CHOOSE_SVE_256(FUNC_NAME) &N_SVE_256::FUNC_NAME
 177 #else
 178 #define HWY_CHOOSE_SVE_256(FUNC_NAME) nullptr
 179 #endif
 180
 181 #if HWY_TARGETS & HWY_SVE2_128
 182 #define HWY_CHOOSE_SVE2_128(FUNC_NAME) &N_SVE2_128::FUNC_NAME
 183 #else
 184 #define HWY_CHOOSE_SVE2_128(FUNC_NAME) nullptr
 185 #endif
 186
 187 #if HWY_TARGETS & HWY_PPC8
 188 #define HWY_CHOOSE_PPC8(FUNC_NAME) &N_PPC8::FUNC_NAME
 189 #else
 190 #define HWY_CHOOSE_PPC8(FUNC_NAME) nullptr
 191 #endif
 192
 193 #if HWY_TARGETS & HWY_PPC9
 194 #define HWY_CHOOSE_PPC9(FUNC_NAME) &N_PPC9::FUNC_NAME
 195 #else
 196 #define HWY_CHOOSE_PPC9(FUNC_NAME) nullptr
 197 #endif
 198
 199 #if HWY_TARGETS & HWY_PPC10
 200 #define HWY_CHOOSE_PPC10(FUNC_NAME) &N_PPC10::FUNC_NAME
 201 #else
 202 #define HWY_CHOOSE_PPC10(FUNC_NAME) nullptr
 203 #endif
 204
 205 #if HWY_TARGETS & HWY_SSE2
 206 #define HWY_CHOOSE_SSE2(FUNC_NAME) &N_SSE2::FUNC_NAME
 207 #else
 208 #define HWY_CHOOSE_SSE2(FUNC_NAME) nullptr
 209 #endif
 210
 211 #if HWY_TARGETS & HWY_SSSE3
 212 #define HWY_CHOOSE_SSSE3(FUNC_NAME) &N_SSSE3::FUNC_NAME
 213 #else
 214 #define HWY_CHOOSE_SSSE3(FUNC_NAME) nullptr
 215 #endif
 216
 217 #if HWY_TARGETS & HWY_SSE4
 218 #define HWY_CHOOSE_SSE4(FUNC_NAME) &N_SSE4::FUNC_NAME
 219 #else
 220 #define HWY_CHOOSE_SSE4(FUNC_NAME) nullptr
 221 #endif
 222
 223 #if HWY_TARGETS & HWY_AVX2
 224 #define HWY_CHOOSE_AVX2(FUNC_NAME) &N_AVX2::FUNC_NAME
 225 #else
 226 #define HWY_CHOOSE_AVX2(FUNC_NAME) nullptr
 227 #endif
 228
 229 #if HWY_TARGETS & HWY_AVX3
 230 #define HWY_CHOOSE_AVX3(FUNC_NAME) &N_AVX3::FUNC_NAME
 231 #else
 232 #define HWY_CHOOSE_AVX3(FUNC_NAME) nullptr
 233 #endif
 234
 235 #if HWY_TARGETS & HWY_AVX3_DL
 236 #define HWY_CHOOSE_AVX3_DL(FUNC_NAME) &N_AVX3_DL::FUNC_NAME
 237 #else
 238 #define HWY_CHOOSE_AVX3_DL(FUNC_NAME) nullptr
 239 #endif
 240
 241 #if HWY_TARGETS & HWY_AVX3_ZEN4
 242 #define HWY_CHOOSE_AVX3_ZEN4(FUNC_NAME) &N_AVX3_ZEN4::FUNC_NAME
 243 #else
 244 #define HWY_CHOOSE_AVX3_ZEN4(FUNC_NAME) nullptr
 245 #endif
 246
 247 // MSVC 2017 workaround: the non-type template parameter to ChooseAndCall
 248 // apparently cannot be an array. Use a function pointer instead, which has the
 249 // disadvantage that we call the static (not best) target on the first call to
 250 // any HWY_DYNAMIC_DISPATCH.
 251 #if HWY_COMPILER_MSVC && HWY_COMPILER_MSVC < 1915
 252 #define HWY_DISPATCH_WORKAROUND 1
 253 #else
 254 #define HWY_DISPATCH_WORKAROUND 0
 255 #endif
 256
 257 // Provides a static member function which is what is called during the first
 258 // HWY_DYNAMIC_DISPATCH, where GetIndex is still zero, and instantiations of
 259 // this function are the first entry in the tables created by HWY_EXPORT.
 260 template <typename RetType, typename... Args>
 261 struct FunctionCache {
 262  public:
 263   typedef RetType(FunctionType)(Args...);
 264
 265 #if HWY_DISPATCH_WORKAROUND
 266   template <FunctionType* const func>
 267   static RetType ChooseAndCall(Args... args) {
 268     ChosenTarget& chosen_target = GetChosenTarget();
 269     chosen_target.Update(SupportedTargets());
 270     return (*func)(args...);
 271   }
 272 #else
 273   // A template function that when instantiated has the same signature as the
 274   // function being called. This function initializes the bit array of targets
 275   // supported by the current CPU and then calls the appropriate entry within
 276   // the HWY_EXPORT table. Subsequent calls via HWY_DYNAMIC_DISPATCH to any
 277   // exported functions, even those defined by different translation units,
 278   // will dispatch directly to the best available target.
 279   template <FunctionType* const table[]>
 280   static RetType ChooseAndCall(Args... args) {
 281     ChosenTarget& chosen_target = GetChosenTarget();
 282     chosen_target.Update(SupportedTargets());
 283     return (table[chosen_target.GetIndex()])(args...);
 284   }
 285 #endif  // HWY_DISPATCH_WORKAROUND
 286 };
 287
 288 // Used to deduce the template parameters RetType and Args from a function.
 289 template <typename RetType, typename... Args>
 290 FunctionCache<RetType, Args...> DeduceFunctionCache(RetType (*)(Args...)) {
 291   return FunctionCache<RetType, Args...>();
 292 }
 293
 294 #define HWY_DISPATCH_TABLE(FUNC_NAME) \
 295   HWY_CONCAT(FUNC_NAME, HighwayDispatchTable)
 296
 297 // HWY_EXPORT(FUNC_NAME); expands to a static array that is used by
 298 // HWY_DYNAMIC_DISPATCH() to call the appropriate function at runtime. This
 299 // static array must be defined at the same namespace level as the function
 300 // it is exporting.
 301 // After being exported, it can be called from other parts of the same source
 302 // file using HWY_DYNAMIC_DISPATCH(), in particular from a function wrapper
 303 // like in the following example:
 304 //
 305 //   #include "hwy/highway.h"
 306 //   HWY_BEFORE_NAMESPACE();
 307 //   namespace skeleton {
 308 //   namespace HWY_NAMESPACE {
 309 //
 310 //   void MyFunction(int a, char b, const char* c) { ... }
 311 //
 312 //   // NOLINTNEXTLINE(google-readability-namespace-comments)
 313 //   }  // namespace HWY_NAMESPACE
 314 //   }  // namespace skeleton
 315 //   HWY_AFTER_NAMESPACE();
 316 //
 317 //   namespace skeleton {
 318 //   HWY_EXPORT(MyFunction);  // Defines the dispatch table in this scope.
 319 //
 320 //   void MyFunction(int a, char b, const char* c) {
 321 //     return HWY_DYNAMIC_DISPATCH(MyFunction)(a, b, c);
 322 //   }
 323 //   }  // namespace skeleton
 324 //
 325
 326 #if HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0)
 327
 328 // Simplified version for IDE or the dynamic dispatch case with only one target.
 329 // This case still uses a table, although of a single element, to provide the
 330 // same compile error conditions as with the dynamic dispatch case when multiple
 331 // targets are being compiled.
 332 #define HWY_EXPORT(FUNC_NAME)                                             \
 333   HWY_MAYBE_UNUSED static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const \
 334   HWY_DISPATCH_TABLE(FUNC_NAME)[1] = {&HWY_STATIC_DISPATCH(FUNC_NAME)}
 335 #define HWY_DYNAMIC_DISPATCH(FUNC_NAME) HWY_STATIC_DISPATCH(FUNC_NAME)
 336
 337 #else
 338
 339 // Simplified version for MSVC 2017: function pointer instead of table.
 340 #if HWY_DISPATCH_WORKAROUND
 341
 342 #define HWY_EXPORT(FUNC_NAME)                                                \
 343   static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \
 344       FUNC_NAME)[HWY_MAX_DYNAMIC_TARGETS + 2] = {                            \
 345       /* The first entry in the table initializes the global cache and       \
 346        * calls the function from HWY_STATIC_TARGET. */                       \
 347       &decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH(               \
 348           FUNC_NAME)))::ChooseAndCall<&HWY_STATIC_DISPATCH(FUNC_NAME)>,      \
 349       HWY_CHOOSE_TARGET_LIST(FUNC_NAME),                                     \
 350       HWY_CHOOSE_FALLBACK(FUNC_NAME),                                        \
 351   }
 352
 353 #else
 354
 355 // Dynamic dispatch case with one entry per dynamic target plus the fallback
 356 // target and the initialization wrapper.
 357 #define HWY_EXPORT(FUNC_NAME)                                                \
 358   static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \
 359       FUNC_NAME)[HWY_MAX_DYNAMIC_TARGETS + 2] = {                            \
 360       /* The first entry in the table initializes the global cache and       \
 361        * calls the appropriate function. */                                  \
 362       &decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH(               \
 363           FUNC_NAME)))::ChooseAndCall<HWY_DISPATCH_TABLE(FUNC_NAME)>,        \
 364       HWY_CHOOSE_TARGET_LIST(FUNC_NAME),                                     \
 365       HWY_CHOOSE_FALLBACK(FUNC_NAME),                                        \
 366   }
 367
 368 #endif  // HWY_DISPATCH_WORKAROUND
 369
 370 #define HWY_DYNAMIC_DISPATCH(FUNC_NAME) \
 371   (*(HWY_DISPATCH_TABLE(FUNC_NAME)[hwy::GetChosenTarget().GetIndex()]))
 372
 373 #endif  // HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0)
 374
 375 // DEPRECATED names; please use HWY_HAVE_* instead.
 376 #define HWY_CAP_INTEGER64 HWY_HAVE_INTEGER64
 377 #define HWY_CAP_FLOAT16 HWY_HAVE_FLOAT16
 378 #define HWY_CAP_FLOAT64 HWY_HAVE_FLOAT64
 379
 380 }  // namespace hwy
 381
 382 #endif  // HWY_HIGHWAY_INCLUDED
 383
 384 //------------------------------------------------------------------------------
 385
 386 // NOTE: the following definitions and ops/*.h depend on HWY_TARGET, so we want
 387 // to include them once per target, which is ensured by the toggle check.
 388 // Because ops/*.h are included under it, they do not need their own guard.
 389 #if defined(HWY_HIGHWAY_PER_TARGET) == defined(HWY_TARGET_TOGGLE)
 390 #ifdef HWY_HIGHWAY_PER_TARGET
 391 #undef HWY_HIGHWAY_PER_TARGET
 392 #else
 393 #define HWY_HIGHWAY_PER_TARGET
 394 #endif
 395
 396 // These define ops inside namespace hwy::HWY_NAMESPACE.
 397 #if HWY_TARGET == HWY_SSE2 || HWY_TARGET == HWY_SSSE3 || \
 398     HWY_TARGET == HWY_SSE4
 399 #include "hwy/ops/x86_128-inl.h"
 400 #elif HWY_TARGET == HWY_AVX2
 401 #include "hwy/ops/x86_256-inl.h"
 402 #elif HWY_TARGET == HWY_AVX3 || HWY_TARGET == HWY_AVX3_DL || \
 403     HWY_TARGET == HWY_AVX3_ZEN4
 404 #include "hwy/ops/x86_512-inl.h"
 405 #elif HWY_TARGET == HWY_PPC8 || HWY_TARGET == HWY_PPC9 || \
 406     HWY_TARGET == HWY_PPC10
 407 #include "hwy/ops/ppc_vsx-inl.h"
 408 #elif HWY_TARGET == HWY_NEON || HWY_TARGET == HWY_NEON_WITHOUT_AES
 409 #include "hwy/ops/arm_neon-inl.h"
 410 #elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 || \
 411     HWY_TARGET == HWY_SVE_256 || HWY_TARGET == HWY_SVE2_128
 412 #include "hwy/ops/arm_sve-inl.h"
 413 #elif HWY_TARGET == HWY_WASM_EMU256
 414 #include "hwy/ops/wasm_256-inl.h"
 415 #elif HWY_TARGET == HWY_WASM
 416 #include "hwy/ops/wasm_128-inl.h"
 417 #elif HWY_TARGET == HWY_RVV
 418 #include "hwy/ops/rvv-inl.h"
 419 #elif HWY_TARGET == HWY_EMU128
 420 #include "hwy/ops/emu128-inl.h"
 421 #elif HWY_TARGET == HWY_SCALAR
 422 #include "hwy/ops/scalar-inl.h"
 423 #else
 424 #pragma message("HWY_TARGET does not match any known target")
 425 #endif  // HWY_TARGET
 426
 427 #include "hwy/ops/generic_ops-inl.h"
 428
 429 #endif  // HWY_HIGHWAY_PER_TARGET