Headers/ppc_wrappers/xmmintrin.h

/*===---- xmmintrin.h - Implementation of SSE intrinsics on PowerPC --------===
 *
 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 * See https://llvm.org/LICENSE.txt for license information.
 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 *
 *===-----------------------------------------------------------------------===
 */

/* Implemented from the specification included in the Intel C++ Compiler
   User Guide and Reference, version 9.0.  */

#ifndef NO_WARN_X86_INTRINSICS
/* This header file is to help porting code using Intel intrinsics
   explicitly from x86_64 to powerpc64/powerpc64le.

   Since X86 SSE intrinsics mainly handles __m128 type, PowerPC
   VMX/VSX ISA is a good match for vector float SIMD operations.
   However scalar float operations in vector (XMM) registers require
   the POWER8 VSX ISA (2.07) level. There are differences for data
   format and placement of float scalars in the vector register, which
   require extra steps to match SSE scalar float semantics on POWER.

   It should be noted that there's much difference between X86_64's
   MXSCR and PowerISA's FPSCR/VSCR registers. It's recommended to use
   portable <fenv.h> instead of access MXSCR directly.

   Most SSE scalar float intrinsic operations can be performed more
   efficiently as C language float scalar operations or optimized to
   use vector SIMD operations. We recommend this for new applications. */
#error                                                                         \
    "Please read comment above. Use -DNO_WARN_X86_INTRINSICS to disable this error."
#endif

#ifndef XMMINTRIN_H_
#define XMMINTRIN_H_

#if defined(__powerpc64__) &&                                                  \
    (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX))

/* Define four value permute mask */
#define _MM_SHUFFLE(w, x, y, z) (((w) << 6) | ((x) << 4) | ((y) << 2) | (z))

#include <altivec.h>

/* Avoid collisions between altivec.h and strict adherence to C++ and
   C11 standards.  This should eventually be done inside altivec.h itself,
   but only after testing a full distro build.  */
#if defined(__STRICT_ANSI__) &&                                                \
    (defined(__cplusplus) ||                                                   \
     (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L))
#undef vector
#undef pixel
#undef bool
#endif

/* We need type definitions from the MMX header file.  */
#include <mmintrin.h>

/* Get _mm_malloc () and _mm_free ().  */
#if __STDC_HOSTED__
#include <mm_malloc.h>
#endif

/* The Intel API is flexible enough that we must allow aliasing with other
   vector types, and their scalar components.  */
typedef vector float __m128 __attribute__((__may_alias__));

/* Unaligned version of the same type.  */
typedef vector float __m128_u __attribute__((__may_alias__, __aligned__(1)));

/* Internal data types for implementing the intrinsics.  */
typedef vector float __v4sf;

/* Create an undefined vector.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_undefined_ps(void) {
  __m128 __Y = __Y;
  return __Y;
}

/* Create a vector of zeros.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_setzero_ps(void) {
  return __extension__(__m128){0.0f, 0.0f, 0.0f, 0.0f};
}

/* Load four SPFP values from P.  The address must be 16-byte aligned.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_load_ps(float const *__P) {
  return ((__m128)vec_ld(0, (__v4sf *)__P));
}

/* Load four SPFP values from P.  The address need not be 16-byte aligned.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_loadu_ps(float const *__P) {
  return (vec_vsx_ld(0, __P));
}

/* Load four SPFP values in reverse order.  The address must be aligned.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_loadr_ps(float const *__P) {
  __v4sf __tmp;
  __m128 __result;
  static const __vector unsigned char __permute_vector = {
      0x1C, 0x1D, 0x1E, 0x1F, 0x18, 0x19, 0x1A, 0x1B,
      0x14, 0x15, 0x16, 0x17, 0x10, 0x11, 0x12, 0x13};

  __tmp = vec_ld(0, (__v4sf *)__P);
  __result = (__m128)vec_perm(__tmp, __tmp, __permute_vector);
  return __result;
}

/* Create a vector with all four elements equal to F.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_set1_ps(float __F) {
  return __extension__(__m128)(__v4sf){__F, __F, __F, __F};
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_set_ps1(float __F) {
  return _mm_set1_ps(__F);
}

/* Create the vector [Z Y X W].  */
extern __inline __m128 __attribute__((__gnu_inline__, __always_inline__,
                                      __artificial__))
_mm_set_ps(const float __Z, const float __Y, const float __X, const float __W) {
  return __extension__(__m128)(__v4sf){__W, __X, __Y, __Z};
}

/* Create the vector [W X Y Z].  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_setr_ps(float __Z, float __Y, float __X, float __W) {
  return __extension__(__m128)(__v4sf){__Z, __Y, __X, __W};
}

/* Store four SPFP values.  The address must be 16-byte aligned.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_store_ps(float *__P, __m128 __A) {
  vec_st((__v4sf)__A, 0, (__v4sf *)__P);
}

/* Store four SPFP values.  The address need not be 16-byte aligned.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_storeu_ps(float *__P, __m128 __A) {
  *(__m128_u *)__P = __A;
}

/* Store four SPFP values in reverse order.  The address must be aligned.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_storer_ps(float *__P, __m128 __A) {
  __v4sf __tmp;
  static const __vector unsigned char __permute_vector = {
      0x1C, 0x1D, 0x1E, 0x1F, 0x18, 0x19, 0x1A, 0x1B,
      0x14, 0x15, 0x16, 0x17, 0x10, 0x11, 0x12, 0x13};

  __tmp = (__m128)vec_perm(__A, __A, __permute_vector);

  _mm_store_ps(__P, __tmp);
}

/* Store the lower SPFP value across four words.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_store1_ps(float *__P, __m128 __A) {
  __v4sf __va = vec_splat((__v4sf)__A, 0);
  _mm_store_ps(__P, __va);
}

extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_store_ps1(float *__P, __m128 __A) {
  _mm_store1_ps(__P, __A);
}

/* Create a vector with element 0 as F and the rest zero.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_set_ss(float __F) {
  return __extension__(__m128)(__v4sf){__F, 0.0f, 0.0f, 0.0f};
}

/* Sets the low SPFP value of A from the low value of B.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_move_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};

  return (vec_sel((__v4sf)__A, (__v4sf)__B, __mask));
}

/* Create a vector with element 0 as *P and the rest zero.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_load_ss(float const *__P) {
  return _mm_set_ss(*__P);
}

/* Stores the lower SPFP value.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_store_ss(float *__P, __m128 __A) {
  *__P = ((__v4sf)__A)[0];
}

/* Perform the respective operation on the lower SPFP (single-precision
   floating-point) values of A and B; the upper three SPFP values are
   passed through from A.  */

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_add_ss(__m128 __A, __m128 __B) {
#ifdef _ARCH_PWR7
  __m128 __a, __b, __c;
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  /* PowerISA VSX does not allow partial (for just lower double)
     results. So to insure we don't generate spurious exceptions
     (from the upper double values) we splat the lower double
     before we to the operation.  */
  __a = vec_splat(__A, 0);
  __b = vec_splat(__B, 0);
  __c = __a + __b;
  /* Then we merge the lower float result with the original upper
     float elements from __A.  */
  return (vec_sel(__A, __c, __mask));
#else
  __A[0] = __A[0] + __B[0];
  return (__A);
#endif
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_sub_ss(__m128 __A, __m128 __B) {
#ifdef _ARCH_PWR7
  __m128 __a, __b, __c;
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  /* PowerISA VSX does not allow partial (for just lower double)
     results. So to insure we don't generate spurious exceptions
     (from the upper double values) we splat the lower double
     before we to the operation.  */
  __a = vec_splat(__A, 0);
  __b = vec_splat(__B, 0);
  __c = __a - __b;
  /* Then we merge the lower float result with the original upper
     float elements from __A.  */
  return (vec_sel(__A, __c, __mask));
#else
  __A[0] = __A[0] - __B[0];
  return (__A);
#endif
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_mul_ss(__m128 __A, __m128 __B) {
#ifdef _ARCH_PWR7
  __m128 __a, __b, __c;
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  /* PowerISA VSX does not allow partial (for just lower double)
     results. So to insure we don't generate spurious exceptions
     (from the upper double values) we splat the lower double
     before we to the operation.  */
  __a = vec_splat(__A, 0);
  __b = vec_splat(__B, 0);
  __c = __a * __b;
  /* Then we merge the lower float result with the original upper
     float elements from __A.  */
  return (vec_sel(__A, __c, __mask));
#else
  __A[0] = __A[0] * __B[0];
  return (__A);
#endif
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_div_ss(__m128 __A, __m128 __B) {
#ifdef _ARCH_PWR7
  __m128 __a, __b, __c;
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  /* PowerISA VSX does not allow partial (for just lower double)
     results. So to insure we don't generate spurious exceptions
     (from the upper double values) we splat the lower double
     before we to the operation.  */
  __a = vec_splat(__A, 0);
  __b = vec_splat(__B, 0);
  __c = __a / __b;
  /* Then we merge the lower float result with the original upper
     float elements from __A.  */
  return (vec_sel(__A, __c, __mask));
#else
  __A[0] = __A[0] / __B[0];
  return (__A);
#endif
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_sqrt_ss(__m128 __A) {
  __m128 __a, __c;
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  /* PowerISA VSX does not allow partial (for just lower double)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper double values) we splat the lower double
   * before we to the operation. */
  __a = vec_splat(__A, 0);
  __c = vec_sqrt(__a);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return (vec_sel(__A, __c, __mask));
}

/* Perform the respective operation on the four SPFP values in A and B.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_add_ps(__m128 __A, __m128 __B) {
  return (__m128)((__v4sf)__A + (__v4sf)__B);
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_sub_ps(__m128 __A, __m128 __B) {
  return (__m128)((__v4sf)__A - (__v4sf)__B);
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_mul_ps(__m128 __A, __m128 __B) {
  return (__m128)((__v4sf)__A * (__v4sf)__B);
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_div_ps(__m128 __A, __m128 __B) {
  return (__m128)((__v4sf)__A / (__v4sf)__B);
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_sqrt_ps(__m128 __A) {
  return (vec_sqrt((__v4sf)__A));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_rcp_ps(__m128 __A) {
  return (vec_re((__v4sf)__A));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_rsqrt_ps(__m128 __A) {
  return (vec_rsqrte(__A));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_rcp_ss(__m128 __A) {
  __m128 __a, __c;
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  /* PowerISA VSX does not allow partial (for just lower double)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper double values) we splat the lower double
   * before we to the operation. */
  __a = vec_splat(__A, 0);
  __c = _mm_rcp_ps(__a);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return (vec_sel(__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_rsqrt_ss(__m128 __A) {
  __m128 __a, __c;
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  /* PowerISA VSX does not allow partial (for just lower double)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper double values) we splat the lower double
   * before we to the operation. */
  __a = vec_splat(__A, 0);
  __c = vec_rsqrte(__a);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return (vec_sel(__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_min_ss(__m128 __A, __m128 __B) {
  __v4sf __a, __b, __c;
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  /* PowerISA VSX does not allow partial (for just lower float)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper float values) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = vec_min(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return (vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_max_ss(__m128 __A, __m128 __B) {
  __v4sf __a, __b, __c;
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  /* PowerISA VSX does not allow partial (for just lower float)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper float values) we splat the lower float
   * before we to the operation. */
  __a = vec_splat(__A, 0);
  __b = vec_splat(__B, 0);
  __c = vec_max(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return (vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_min_ps(__m128 __A, __m128 __B) {
  __vector __bool int __m = vec_cmpgt((__v4sf)__B, (__v4sf)__A);
  return vec_sel(__B, __A, __m);
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_max_ps(__m128 __A, __m128 __B) {
  __vector __bool int __m = vec_cmpgt((__v4sf)__A, (__v4sf)__B);
  return vec_sel(__B, __A, __m);
}

/* Perform logical bit-wise operations on 128-bit values.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_and_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_and((__v4sf)__A, (__v4sf)__B));
  //  return __builtin_ia32_andps (__A, __B);
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_andnot_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_andc((__v4sf)__B, (__v4sf)__A));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_or_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_or((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_xor_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_xor((__v4sf)__A, (__v4sf)__B));
}

/* Perform a comparison on the four SPFP values of A and B.  For each
   element, if the comparison is true, place a mask of all ones in the
   result, otherwise a mask of zeros.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpeq_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_cmpeq((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmplt_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_cmplt((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmple_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_cmple((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpgt_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_cmpgt((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpge_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_cmpge((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpneq_ps(__m128 __A, __m128 __B) {
  __v4sf __temp = (__v4sf)vec_cmpeq((__v4sf)__A, (__v4sf)__B);
  return ((__m128)vec_nor(__temp, __temp));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpnlt_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_cmpge((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpnle_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_cmpgt((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpngt_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_cmple((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpnge_ps(__m128 __A, __m128 __B) {
  return ((__m128)vec_cmplt((__v4sf)__A, (__v4sf)__B));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpord_ps(__m128 __A, __m128 __B) {
  __vector unsigned int __a, __b;
  __vector unsigned int __c, __d;
  static const __vector unsigned int __float_exp_mask = {
      0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000};

  __a = (__vector unsigned int)vec_abs((__v4sf)__A);
  __b = (__vector unsigned int)vec_abs((__v4sf)__B);
  __c = (__vector unsigned int)vec_cmpgt(__float_exp_mask, __a);
  __d = (__vector unsigned int)vec_cmpgt(__float_exp_mask, __b);
  return ((__m128)vec_and(__c, __d));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpunord_ps(__m128 __A, __m128 __B) {
  __vector unsigned int __a, __b;
  __vector unsigned int __c, __d;
  static const __vector unsigned int __float_exp_mask = {
      0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000};

  __a = (__vector unsigned int)vec_abs((__v4sf)__A);
  __b = (__vector unsigned int)vec_abs((__v4sf)__B);
  __c = (__vector unsigned int)vec_cmpgt(__a, __float_exp_mask);
  __d = (__vector unsigned int)vec_cmpgt(__b, __float_exp_mask);
  return ((__m128)vec_or(__c, __d));
}

/* Perform a comparison on the lower SPFP values of A and B.  If the
   comparison is true, place a mask of all ones in the result, otherwise a
   mask of zeros.  The upper three SPFP values are passed through from A.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpeq_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmpeq(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmplt_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmplt(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmple_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmple(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpgt_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmpgt(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpge_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmpge(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpneq_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmpeq(__a, __b);
  __c = vec_nor(__c, __c);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpnlt_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmpge(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpnle_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmpgt(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpngt_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we to the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmple(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpnge_ss(__m128 __A, __m128 __B) {
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};
  __v4sf __a, __b, __c;
  /* PowerISA VMX does not allow partial (for just element 0)
   * results. So to insure we don't generate spurious exceptions
   * (from the upper elements) we splat the lower float
   * before we do the operation. */
  __a = vec_splat((__v4sf)__A, 0);
  __b = vec_splat((__v4sf)__B, 0);
  __c = (__v4sf)vec_cmplt(__a, __b);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, __c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpord_ss(__m128 __A, __m128 __B) {
  __vector unsigned int __a, __b;
  __vector unsigned int __c, __d;
  static const __vector unsigned int __float_exp_mask = {
      0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000};
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};

  __a = (__vector unsigned int)vec_abs((__v4sf)__A);
  __b = (__vector unsigned int)vec_abs((__v4sf)__B);
  __c = (__vector unsigned int)vec_cmpgt(__float_exp_mask, __a);
  __d = (__vector unsigned int)vec_cmpgt(__float_exp_mask, __b);
  __c = vec_and(__c, __d);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, (__v4sf)__c, __mask));
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cmpunord_ss(__m128 __A, __m128 __B) {
  __vector unsigned int __a, __b;
  __vector unsigned int __c, __d;
  static const __vector unsigned int __float_exp_mask = {
      0x7f800000, 0x7f800000, 0x7f800000, 0x7f800000};
  static const __vector unsigned int __mask = {0xffffffff, 0, 0, 0};

  __a = (__vector unsigned int)vec_abs((__v4sf)__A);
  __b = (__vector unsigned int)vec_abs((__v4sf)__B);
  __c = (__vector unsigned int)vec_cmpgt(__a, __float_exp_mask);
  __d = (__vector unsigned int)vec_cmpgt(__b, __float_exp_mask);
  __c = vec_or(__c, __d);
  /* Then we merge the lower float result with the original upper
   * float elements from __A.  */
  return ((__m128)vec_sel((__v4sf)__A, (__v4sf)__c, __mask));
}

/* Compare the lower SPFP values of A and B and return 1 if true
   and 0 if false.  */
extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_comieq_ss(__m128 __A, __m128 __B) {
  return (__A[0] == __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_comilt_ss(__m128 __A, __m128 __B) {
  return (__A[0] < __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_comile_ss(__m128 __A, __m128 __B) {
  return (__A[0] <= __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_comigt_ss(__m128 __A, __m128 __B) {
  return (__A[0] > __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_comige_ss(__m128 __A, __m128 __B) {
  return (__A[0] >= __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_comineq_ss(__m128 __A, __m128 __B) {
  return (__A[0] != __B[0]);
}

/* FIXME
 * The __mm_ucomi??_ss implementations below are exactly the same as
 * __mm_comi??_ss because GCC for PowerPC only generates unordered
 * compares (scalar and vector).
 * Technically __mm_comieq_ss et al should be using the ordered
 * compare and signal for QNaNs.
 * The __mm_ucomieq_sd et all should be OK, as is.
 */
extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_ucomieq_ss(__m128 __A, __m128 __B) {
  return (__A[0] == __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_ucomilt_ss(__m128 __A, __m128 __B) {
  return (__A[0] < __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_ucomile_ss(__m128 __A, __m128 __B) {
  return (__A[0] <= __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_ucomigt_ss(__m128 __A, __m128 __B) {
  return (__A[0] > __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_ucomige_ss(__m128 __A, __m128 __B) {
  return (__A[0] >= __B[0]);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_ucomineq_ss(__m128 __A, __m128 __B) {
  return (__A[0] != __B[0]);
}

extern __inline float
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtss_f32(__m128 __A) {
  return ((__v4sf)__A)[0];
}

/* Convert the lower SPFP value to a 32-bit integer according to the current
   rounding mode.  */
extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtss_si32(__m128 __A) {
  int __res;
#ifdef _ARCH_PWR8
  double __dtmp;
  __asm__(
#ifdef __LITTLE_ENDIAN__
      "xxsldwi %x0,%x0,%x0,3;\n"
#endif
      "xscvspdp %x2,%x0;\n"
      "fctiw  %2,%2;\n"
      "mfvsrd  %1,%x2;\n"
      : "+wa"(__A), "=r"(__res), "=f"(__dtmp)
      :);
#else
  __res = __builtin_rint(__A[0]);
#endif
  return __res;
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvt_ss2si(__m128 __A) {
  return _mm_cvtss_si32(__A);
}

/* Convert the lower SPFP value to a 32-bit integer according to the
   current rounding mode.  */

/* Intel intrinsic.  */
extern __inline long long
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtss_si64(__m128 __A) {
  long long __res;
#if defined(_ARCH_PWR8) && defined(__powerpc64__)
  double __dtmp;
  __asm__(
#ifdef __LITTLE_ENDIAN__
      "xxsldwi %x0,%x0,%x0,3;\n"
#endif
      "xscvspdp %x2,%x0;\n"
      "fctid  %2,%2;\n"
      "mfvsrd  %1,%x2;\n"
      : "+wa"(__A), "=r"(__res), "=f"(__dtmp)
      :);
#else
  __res = __builtin_llrint(__A[0]);
#endif
  return __res;
}

/* Microsoft intrinsic.  */
extern __inline long long
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtss_si64x(__m128 __A) {
  return _mm_cvtss_si64((__v4sf)__A);
}

/* Constants for use with _mm_prefetch.  */
enum _mm_hint {
  /* _MM_HINT_ET is _MM_HINT_T with set 3rd bit.  */
  _MM_HINT_ET0 = 7,
  _MM_HINT_ET1 = 6,
  _MM_HINT_T0 = 3,
  _MM_HINT_T1 = 2,
  _MM_HINT_T2 = 1,
  _MM_HINT_NTA = 0
};

/* Loads one cache line from address P to a location "closer" to the
   processor.  The selector I specifies the type of prefetch operation.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_prefetch(const void *__P, enum _mm_hint __I) {
  /* Current PowerPC will ignores the hint parameters.  */
  __builtin_prefetch(__P);
}

/* Convert the two lower SPFP values to 32-bit integers according to the
   current rounding mode.  Return the integers in packed form.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtps_pi32(__m128 __A) {
  /* Splat two lower SPFP values to both halves.  */
  __v4sf __temp, __rounded;
  __vector unsigned long long __result;

  /* Splat two lower SPFP values to both halves.  */
  __temp = (__v4sf)vec_splat((__vector long long)__A, 0);
  __rounded = vec_rint(__temp);
  __result = (__vector unsigned long long)vec_cts(__rounded, 0);

  return (__m64)((__vector long long)__result)[0];
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvt_ps2pi(__m128 __A) {
  return _mm_cvtps_pi32(__A);
}

/* Truncate the lower SPFP value to a 32-bit integer.  */
extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvttss_si32(__m128 __A) {
  /* Extract the lower float element.  */
  float __temp = __A[0];
  /* truncate to 32-bit integer and return.  */
  return __temp;
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtt_ss2si(__m128 __A) {
  return _mm_cvttss_si32(__A);
}

/* Intel intrinsic.  */
extern __inline long long
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvttss_si64(__m128 __A) {
  /* Extract the lower float element.  */
  float __temp = __A[0];
  /* truncate to 32-bit integer and return.  */
  return __temp;
}

/* Microsoft intrinsic.  */
extern __inline long long
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvttss_si64x(__m128 __A) {
  /* Extract the lower float element.  */
  float __temp = __A[0];
  /* truncate to 32-bit integer and return.  */
  return __temp;
}

/* Truncate the two lower SPFP values to 32-bit integers.  Return the
   integers in packed form.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvttps_pi32(__m128 __A) {
  __v4sf __temp;
  __vector unsigned long long __result;

  /* Splat two lower SPFP values to both halves.  */
  __temp = (__v4sf)vec_splat((__vector long long)__A, 0);
  __result = (__vector unsigned long long)vec_cts(__temp, 0);

  return (__m64)((__vector long long)__result)[0];
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtt_ps2pi(__m128 __A) {
  return _mm_cvttps_pi32(__A);
}

/* Convert B to a SPFP value and insert it as element zero in A.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtsi32_ss(__m128 __A, int __B) {
  float __temp = __B;
  __A[0] = __temp;

  return __A;
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvt_si2ss(__m128 __A, int __B) {
  return _mm_cvtsi32_ss(__A, __B);
}

/* Convert B to a SPFP value and insert it as element zero in A.  */
/* Intel intrinsic.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtsi64_ss(__m128 __A, long long __B) {
  float __temp = __B;
  __A[0] = __temp;

  return __A;
}

/* Microsoft intrinsic.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtsi64x_ss(__m128 __A, long long __B) {
  return _mm_cvtsi64_ss(__A, __B);
}

/* Convert the two 32-bit values in B to SPFP form and insert them
   as the two lower elements in A.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtpi32_ps(__m128 __A, __m64 __B) {
  __vector signed int __vm1;
  __vector float __vf1;

  __vm1 = (__vector signed int)(__vector unsigned long long){__B, __B};
  __vf1 = (__vector float)vec_ctf(__vm1, 0);

  return ((__m128)(__vector unsigned long long){
      ((__vector unsigned long long)__vf1)[0],
      ((__vector unsigned long long)__A)[1]});
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvt_pi2ps(__m128 __A, __m64 __B) {
  return _mm_cvtpi32_ps(__A, __B);
}

/* Convert the four signed 16-bit values in A to SPFP form.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtpi16_ps(__m64 __A) {
  __vector signed short __vs8;
  __vector signed int __vi4;
  __vector float __vf1;

  __vs8 = (__vector signed short)(__vector unsigned long long){__A, __A};
  __vi4 = vec_vupklsh(__vs8);
  __vf1 = (__vector float)vec_ctf(__vi4, 0);

  return (__m128)__vf1;
}

/* Convert the four unsigned 16-bit values in A to SPFP form.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtpu16_ps(__m64 __A) {
  const __vector unsigned short __zero = {0, 0, 0, 0, 0, 0, 0, 0};
  __vector unsigned short __vs8;
  __vector unsigned int __vi4;
  __vector float __vf1;

  __vs8 = (__vector unsigned short)(__vector unsigned long long){__A, __A};
  __vi4 = (__vector unsigned int)vec_mergel
#ifdef __LITTLE_ENDIAN__
      (__vs8, __zero);
#else
      (__zero, __vs8);
#endif
  __vf1 = (__vector float)vec_ctf(__vi4, 0);

  return (__m128)__vf1;
}

/* Convert the low four signed 8-bit values in A to SPFP form.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtpi8_ps(__m64 __A) {
  __vector signed char __vc16;
  __vector signed short __vs8;
  __vector signed int __vi4;
  __vector float __vf1;

  __vc16 = (__vector signed char)(__vector unsigned long long){__A, __A};
  __vs8 = vec_vupkhsb(__vc16);
  __vi4 = vec_vupkhsh(__vs8);
  __vf1 = (__vector float)vec_ctf(__vi4, 0);

  return (__m128)__vf1;
}

/* Convert the low four unsigned 8-bit values in A to SPFP form.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))

    _mm_cvtpu8_ps(__m64 __A) {
  const __vector unsigned char __zero = {0, 0, 0, 0, 0, 0, 0, 0};
  __vector unsigned char __vc16;
  __vector unsigned short __vs8;
  __vector unsigned int __vi4;
  __vector float __vf1;

  __vc16 = (__vector unsigned char)(__vector unsigned long long){__A, __A};
#ifdef __LITTLE_ENDIAN__
  __vs8 = (__vector unsigned short)vec_mergel(__vc16, __zero);
  __vi4 =
      (__vector unsigned int)vec_mergeh(__vs8, (__vector unsigned short)__zero);
#else
  __vs8 = (__vector unsigned short)vec_mergel(__zero, __vc16);
  __vi4 =
      (__vector unsigned int)vec_mergeh((__vector unsigned short)__zero, __vs8);
#endif
  __vf1 = (__vector float)vec_ctf(__vi4, 0);

  return (__m128)__vf1;
}

/* Convert the four signed 32-bit values in A and B to SPFP form.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtpi32x2_ps(__m64 __A, __m64 __B) {
  __vector signed int __vi4;
  __vector float __vf4;

  __vi4 = (__vector signed int)(__vector unsigned long long){__A, __B};
  __vf4 = (__vector float)vec_ctf(__vi4, 0);
  return (__m128)__vf4;
}

/* Convert the four SPFP values in A to four signed 16-bit integers.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtps_pi16(__m128 __A) {
  __v4sf __rounded;
  __vector signed int __temp;
  __vector unsigned long long __result;

  __rounded = vec_rint(__A);
  __temp = vec_cts(__rounded, 0);
  __result = (__vector unsigned long long)vec_pack(__temp, __temp);

  return (__m64)((__vector long long)__result)[0];
}

/* Convert the four SPFP values in A to four signed 8-bit integers.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_cvtps_pi8(__m128 __A) {
  __v4sf __rounded;
  __vector signed int __tmp_i;
  static const __vector signed int __zero = {0, 0, 0, 0};
  __vector signed short __tmp_s;
  __vector signed char __res_v;

  __rounded = vec_rint(__A);
  __tmp_i = vec_cts(__rounded, 0);
  __tmp_s = vec_pack(__tmp_i, __zero);
  __res_v = vec_pack(__tmp_s, __tmp_s);
  return (__m64)((__vector long long)__res_v)[0];
}

/* Selects four specific SPFP values from A and B based on MASK.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))

    _mm_shuffle_ps(__m128 __A, __m128 __B, int const __mask) {
  unsigned long __element_selector_10 = __mask & 0x03;
  unsigned long __element_selector_32 = (__mask >> 2) & 0x03;
  unsigned long __element_selector_54 = (__mask >> 4) & 0x03;
  unsigned long __element_selector_76 = (__mask >> 6) & 0x03;
  static const unsigned int __permute_selectors[4] = {
#ifdef __LITTLE_ENDIAN__
      0x03020100, 0x07060504, 0x0B0A0908, 0x0F0E0D0C
#else
      0x00010203, 0x04050607, 0x08090A0B, 0x0C0D0E0F
#endif
  };
  __vector unsigned int __t;

  __t[0] = __permute_selectors[__element_selector_10];
  __t[1] = __permute_selectors[__element_selector_32];
  __t[2] = __permute_selectors[__element_selector_54] + 0x10101010;
  __t[3] = __permute_selectors[__element_selector_76] + 0x10101010;
  return vec_perm((__v4sf)__A, (__v4sf)__B, (__vector unsigned char)__t);
}

/* Selects and interleaves the upper two SPFP values from A and B.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_unpackhi_ps(__m128 __A, __m128 __B) {
  return (__m128)vec_vmrglw((__v4sf)__A, (__v4sf)__B);
}

/* Selects and interleaves the lower two SPFP values from A and B.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_unpacklo_ps(__m128 __A, __m128 __B) {
  return (__m128)vec_vmrghw((__v4sf)__A, (__v4sf)__B);
}

/* Sets the upper two SPFP values with 64-bits of data loaded from P;
   the lower two values are passed through from A.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_loadh_pi(__m128 __A, __m64 const *__P) {
  __vector unsigned long long __a = (__vector unsigned long long)__A;
  __vector unsigned long long __p = vec_splats(*__P);
  __a[1] = __p[1];

  return (__m128)__a;
}

/* Stores the upper two SPFP values of A into P.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_storeh_pi(__m64 *__P, __m128 __A) {
  __vector unsigned long long __a = (__vector unsigned long long)__A;

  *__P = __a[1];
}

/* Moves the upper two values of B into the lower two values of A.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_movehl_ps(__m128 __A, __m128 __B) {
  return (__m128)vec_mergel((__vector unsigned long long)__B,
                            (__vector unsigned long long)__A);
}

/* Moves the lower two values of B into the upper two values of A.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_movelh_ps(__m128 __A, __m128 __B) {
  return (__m128)vec_mergeh((__vector unsigned long long)__A,
                            (__vector unsigned long long)__B);
}

/* Sets the lower two SPFP values with 64-bits of data loaded from P;
   the upper two values are passed through from A.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_loadl_pi(__m128 __A, __m64 const *__P) {
  __vector unsigned long long __a = (__vector unsigned long long)__A;
  __vector unsigned long long __p = vec_splats(*__P);
  __a[0] = __p[0];

  return (__m128)__a;
}

/* Stores the lower two SPFP values of A into P.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_storel_pi(__m64 *__P, __m128 __A) {
  __vector unsigned long long __a = (__vector unsigned long long)__A;

  *__P = __a[0];
}

#ifdef _ARCH_PWR8
/* Intrinsic functions that require PowerISA 2.07 minimum.  */

/* Creates a 4-bit mask from the most significant bits of the SPFP values.  */
extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_movemask_ps(__m128 __A) {
#ifdef _ARCH_PWR10
  return vec_extractm((__vector unsigned int)__A);
#else
  __vector unsigned long long __result;
  static const __vector unsigned int __perm_mask = {
#ifdef __LITTLE_ENDIAN__
      0x00204060, 0x80808080, 0x80808080, 0x80808080
#else
      0x80808080, 0x80808080, 0x80808080, 0x00204060
#endif
  };

  __result = ((__vector unsigned long long)vec_vbpermq(
      (__vector unsigned char)__A, (__vector unsigned char)__perm_mask));

#ifdef __LITTLE_ENDIAN__
  return __result[1];
#else
  return __result[0];
#endif
#endif /* !_ARCH_PWR10 */
}
#endif /* _ARCH_PWR8 */

/* Create a vector with all four elements equal to *P.  */
extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_load1_ps(float const *__P) {
  return _mm_set1_ps(*__P);
}

extern __inline __m128
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_load_ps1(float const *__P) {
  return _mm_load1_ps(__P);
}

/* Extracts one of the four words of A.  The selector N must be immediate.  */
extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_extract_pi16(__m64 const __A, int const __N) {
  unsigned int __shiftr = __N & 3;
#ifdef __BIG_ENDIAN__
  __shiftr = 3 - __shiftr;
#endif

  return ((__A >> (__shiftr * 16)) & 0xffff);
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pextrw(__m64 const __A, int const __N) {
  return _mm_extract_pi16(__A, __N);
}

/* Inserts word D into one of four words of A.  The selector N must be
   immediate.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_insert_pi16(__m64 const __A, int const __D, int const __N) {
  const int __shiftl = (__N & 3) * 16;
  const __m64 __shiftD = (const __m64)__D << __shiftl;
  const __m64 __mask = 0xffffUL << __shiftl;
  __m64 __result = (__A & (~__mask)) | (__shiftD & __mask);

  return __result;
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pinsrw(__m64 const __A, int const __D, int const __N) {
  return _mm_insert_pi16(__A, __D, __N);
}

/* Compute the element-wise maximum of signed 16-bit values.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))

    _mm_max_pi16(__m64 __A, __m64 __B) {
#if _ARCH_PWR8
  __vector signed short __a, __b, __r;
  __vector __bool short __c;

  __a = (__vector signed short)vec_splats(__A);
  __b = (__vector signed short)vec_splats(__B);
  __c = (__vector __bool short)vec_cmpgt(__a, __b);
  __r = vec_sel(__b, __a, __c);
  return (__m64)((__vector long long)__r)[0];
#else
  __m64_union __m1, __m2, __res;

  __m1.as_m64 = __A;
  __m2.as_m64 = __B;

  __res.as_short[0] = (__m1.as_short[0] > __m2.as_short[0]) ? __m1.as_short[0]
                                                            : __m2.as_short[0];
  __res.as_short[1] = (__m1.as_short[1] > __m2.as_short[1]) ? __m1.as_short[1]
                                                            : __m2.as_short[1];
  __res.as_short[2] = (__m1.as_short[2] > __m2.as_short[2]) ? __m1.as_short[2]
                                                            : __m2.as_short[2];
  __res.as_short[3] = (__m1.as_short[3] > __m2.as_short[3]) ? __m1.as_short[3]
                                                            : __m2.as_short[3];

  return (__m64)__res.as_m64;
#endif
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pmaxsw(__m64 __A, __m64 __B) {
  return _mm_max_pi16(__A, __B);
}

/* Compute the element-wise maximum of unsigned 8-bit values.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_max_pu8(__m64 __A, __m64 __B) {
#if _ARCH_PWR8
  __vector unsigned char __a, __b, __r;
  __vector __bool char __c;

  __a = (__vector unsigned char)vec_splats(__A);
  __b = (__vector unsigned char)vec_splats(__B);
  __c = (__vector __bool char)vec_cmpgt(__a, __b);
  __r = vec_sel(__b, __a, __c);
  return (__m64)((__vector long long)__r)[0];
#else
  __m64_union __m1, __m2, __res;
  long __i;

  __m1.as_m64 = __A;
  __m2.as_m64 = __B;

  for (__i = 0; __i < 8; __i++)
    __res.as_char[__i] =
        ((unsigned char)__m1.as_char[__i] > (unsigned char)__m2.as_char[__i])
            ? __m1.as_char[__i]
            : __m2.as_char[__i];

  return (__m64)__res.as_m64;
#endif
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pmaxub(__m64 __A, __m64 __B) {
  return _mm_max_pu8(__A, __B);
}

/* Compute the element-wise minimum of signed 16-bit values.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_min_pi16(__m64 __A, __m64 __B) {
#if _ARCH_PWR8
  __vector signed short __a, __b, __r;
  __vector __bool short __c;

  __a = (__vector signed short)vec_splats(__A);
  __b = (__vector signed short)vec_splats(__B);
  __c = (__vector __bool short)vec_cmplt(__a, __b);
  __r = vec_sel(__b, __a, __c);
  return (__m64)((__vector long long)__r)[0];
#else
  __m64_union __m1, __m2, __res;

  __m1.as_m64 = __A;
  __m2.as_m64 = __B;

  __res.as_short[0] = (__m1.as_short[0] < __m2.as_short[0]) ? __m1.as_short[0]
                                                            : __m2.as_short[0];
  __res.as_short[1] = (__m1.as_short[1] < __m2.as_short[1]) ? __m1.as_short[1]
                                                            : __m2.as_short[1];
  __res.as_short[2] = (__m1.as_short[2] < __m2.as_short[2]) ? __m1.as_short[2]
                                                            : __m2.as_short[2];
  __res.as_short[3] = (__m1.as_short[3] < __m2.as_short[3]) ? __m1.as_short[3]
                                                            : __m2.as_short[3];

  return (__m64)__res.as_m64;
#endif
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pminsw(__m64 __A, __m64 __B) {
  return _mm_min_pi16(__A, __B);
}

/* Compute the element-wise minimum of unsigned 8-bit values.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_min_pu8(__m64 __A, __m64 __B) {
#if _ARCH_PWR8
  __vector unsigned char __a, __b, __r;
  __vector __bool char __c;

  __a = (__vector unsigned char)vec_splats(__A);
  __b = (__vector unsigned char)vec_splats(__B);
  __c = (__vector __bool char)vec_cmplt(__a, __b);
  __r = vec_sel(__b, __a, __c);
  return (__m64)((__vector long long)__r)[0];
#else
  __m64_union __m1, __m2, __res;
  long __i;

  __m1.as_m64 = __A;
  __m2.as_m64 = __B;

  for (__i = 0; __i < 8; __i++)
    __res.as_char[__i] =
        ((unsigned char)__m1.as_char[__i] < (unsigned char)__m2.as_char[__i])
            ? __m1.as_char[__i]
            : __m2.as_char[__i];

  return (__m64)__res.as_m64;
#endif
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pminub(__m64 __A, __m64 __B) {
  return _mm_min_pu8(__A, __B);
}

/* Create an 8-bit mask of the signs of 8-bit values.  */
extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_movemask_pi8(__m64 __A) {
#ifdef __powerpc64__
  unsigned long long __p =
#ifdef __LITTLE_ENDIAN__
      0x0008101820283038UL; // permute control for sign bits
#else
      0x3830282018100800UL; // permute control for sign bits
#endif
  return __builtin_bpermd(__p, __A);
#else
#ifdef __LITTLE_ENDIAN__
  unsigned int __mask = 0x20283038UL;
  unsigned int __r1 = __builtin_bpermd(__mask, __A) & 0xf;
  unsigned int __r2 = __builtin_bpermd(__mask, __A >> 32) & 0xf;
#else
  unsigned int __mask = 0x38302820UL;
  unsigned int __r1 = __builtin_bpermd(__mask, __A >> 32) & 0xf;
  unsigned int __r2 = __builtin_bpermd(__mask, __A) & 0xf;
#endif
  return (__r2 << 4) | __r1;
#endif
}

extern __inline int
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pmovmskb(__m64 __A) {
  return _mm_movemask_pi8(__A);
}

/* Multiply four unsigned 16-bit values in A by four unsigned 16-bit values
   in B and produce the high 16 bits of the 32-bit results.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_mulhi_pu16(__m64 __A, __m64 __B) {
  __vector unsigned short __a, __b;
  __vector unsigned short __c;
  __vector unsigned int __w0, __w1;
  __vector unsigned char __xform1 = {
#ifdef __LITTLE_ENDIAN__
      0x02, 0x03, 0x12, 0x13, 0x06, 0x07, 0x16, 0x17, 0x0A,
      0x0B, 0x1A, 0x1B, 0x0E, 0x0F, 0x1E, 0x1F
#else
      0x00, 0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15, 0x00,
      0x01, 0x10, 0x11, 0x04, 0x05, 0x14, 0x15
#endif
  };

  __a = (__vector unsigned short)vec_splats(__A);
  __b = (__vector unsigned short)vec_splats(__B);

  __w0 = vec_vmuleuh(__a, __b);
  __w1 = vec_vmulouh(__a, __b);
  __c = (__vector unsigned short)vec_perm(__w0, __w1, __xform1);

  return (__m64)((__vector long long)__c)[0];
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pmulhuw(__m64 __A, __m64 __B) {
  return _mm_mulhi_pu16(__A, __B);
}

/* Return a combination of the four 16-bit values in A.  The selector
   must be an immediate.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_shuffle_pi16(__m64 __A, int const __N) {
  unsigned long __element_selector_10 = __N & 0x03;
  unsigned long __element_selector_32 = (__N >> 2) & 0x03;
  unsigned long __element_selector_54 = (__N >> 4) & 0x03;
  unsigned long __element_selector_76 = (__N >> 6) & 0x03;
  static const unsigned short __permute_selectors[4] = {
#ifdef __LITTLE_ENDIAN__
      0x0908, 0x0B0A, 0x0D0C, 0x0F0E
#else
      0x0607, 0x0405, 0x0203, 0x0001
#endif
  };
  __m64_union __t;
  __vector unsigned long long __a, __p, __r;

#ifdef __LITTLE_ENDIAN__
  __t.as_short[0] = __permute_selectors[__element_selector_10];
  __t.as_short[1] = __permute_selectors[__element_selector_32];
  __t.as_short[2] = __permute_selectors[__element_selector_54];
  __t.as_short[3] = __permute_selectors[__element_selector_76];
#else
  __t.as_short[3] = __permute_selectors[__element_selector_10];
  __t.as_short[2] = __permute_selectors[__element_selector_32];
  __t.as_short[1] = __permute_selectors[__element_selector_54];
  __t.as_short[0] = __permute_selectors[__element_selector_76];
#endif
  __p = vec_splats(__t.as_m64);
  __a = vec_splats(__A);
  __r = vec_perm(__a, __a, (__vector unsigned char)__p);
  return (__m64)((__vector long long)__r)[0];
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pshufw(__m64 __A, int const __N) {
  return _mm_shuffle_pi16(__A, __N);
}

/* Conditionally store byte elements of A into P.  The high bit of each
   byte in the selector N determines whether the corresponding byte from
   A is stored.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_maskmove_si64(__m64 __A, __m64 __N, char *__P) {
  __m64 __hibit = 0x8080808080808080UL;
  __m64 __mask, __tmp;
  __m64 *__p = (__m64 *)__P;

  __tmp = *__p;
  __mask = _mm_cmpeq_pi8((__N & __hibit), __hibit);
  __tmp = (__tmp & (~__mask)) | (__A & __mask);
  *__p = __tmp;
}

extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_maskmovq(__m64 __A, __m64 __N, char *__P) {
  _mm_maskmove_si64(__A, __N, __P);
}

/* Compute the rounded averages of the unsigned 8-bit values in A and B.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_avg_pu8(__m64 __A, __m64 __B) {
  __vector unsigned char __a, __b, __c;

  __a = (__vector unsigned char)vec_splats(__A);
  __b = (__vector unsigned char)vec_splats(__B);
  __c = vec_avg(__a, __b);
  return (__m64)((__vector long long)__c)[0];
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pavgb(__m64 __A, __m64 __B) {
  return _mm_avg_pu8(__A, __B);
}

/* Compute the rounded averages of the unsigned 16-bit values in A and B.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_avg_pu16(__m64 __A, __m64 __B) {
  __vector unsigned short __a, __b, __c;

  __a = (__vector unsigned short)vec_splats(__A);
  __b = (__vector unsigned short)vec_splats(__B);
  __c = vec_avg(__a, __b);
  return (__m64)((__vector long long)__c)[0];
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_pavgw(__m64 __A, __m64 __B) {
  return _mm_avg_pu16(__A, __B);
}

/* Compute the sum of the absolute differences of the unsigned 8-bit
   values in A and B.  Return the value in the lower 16-bit word; the
   upper words are cleared.  */
extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_sad_pu8(__m64 __A, __m64 __B) {
  __vector unsigned char __a, __b;
  __vector unsigned char __vmin, __vmax, __vabsdiff;
  __vector signed int __vsum;
  const __vector unsigned int __zero = {0, 0, 0, 0};
  __m64_union __result = {0};

  __a = (__vector unsigned char)(__vector unsigned long long){0UL, __A};
  __b = (__vector unsigned char)(__vector unsigned long long){0UL, __B};
  __vmin = vec_min(__a, __b);
  __vmax = vec_max(__a, __b);
  __vabsdiff = vec_sub(__vmax, __vmin);
  /* Sum four groups of bytes into integers.  */
  __vsum = (__vector signed int)vec_sum4s(__vabsdiff, __zero);
  /* Sum across four integers with integer result.  */
  __vsum = vec_sums(__vsum, (__vector signed int)__zero);
  /* The sum is in the right most 32-bits of the vector result.
     Transfer to a GPR and truncate to 16 bits.  */
  __result.as_short[0] = __vsum[3];
  return __result.as_m64;
}

extern __inline __m64
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _m_psadbw(__m64 __A, __m64 __B) {
  return _mm_sad_pu8(__A, __B);
}

/* Stores the data in A to the address P without polluting the caches.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_stream_pi(__m64 *__P, __m64 __A) {
  /* Use the data cache block touch for store transient.  */
  __asm__("	dcbtstt	0,%0" : : "b"(__P) : "memory");
  *__P = __A;
}

/* Likewise.  The address must be 16-byte aligned.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_stream_ps(float *__P, __m128 __A) {
  /* Use the data cache block touch for store transient.  */
  __asm__("	dcbtstt	0,%0" : : "b"(__P) : "memory");
  _mm_store_ps(__P, __A);
}

/* Guarantees that every preceding store is globally visible before
   any subsequent store.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_sfence(void) {
  /* Generate a light weight sync.  */
  __atomic_thread_fence(__ATOMIC_RELEASE);
}

/* The execution of the next instruction is delayed by an implementation
   specific amount of time.  The instruction does not modify the
   architectural state.  This is after the pop_options pragma because
   it does not require SSE support in the processor--the encoding is a
   nop on processors that do not support it.  */
extern __inline void
    __attribute__((__gnu_inline__, __always_inline__, __artificial__))
    _mm_pause(void) {
  /* There is no exact match with this construct, but the following is
     close to the desired effect.  */
#if _ARCH_PWR8
  /* On power8 and later processors we can depend on Program Priority
     (PRI) and associated "very low" PPI setting.  Since we don't know
     what PPI this thread is running at we: 1) save the current PRI
     from the PPR SPR into a local GRP, 2) set the PRI to "very low*
     via the special or 31,31,31 encoding. 3) issue an "isync" to
     insure the PRI change takes effect before we execute any more
     instructions.
     Now we can execute a lwsync (release barrier) while we execute
     this thread at "very low" PRI.  Finally we restore the original
     PRI and continue execution.  */
  unsigned long __PPR;

  __asm__ volatile("	mfppr	%0;"
                   "   or 31,31,31;"
                   "   isync;"
                   "   lwsync;"
                   "   isync;"
                   "   mtppr	%0;"
                   : "=r"(__PPR)
                   :
                   : "memory");
#else
  /* For older processor where we may not even have Program Priority
     controls we can only depend on Heavy Weight Sync.  */
  __atomic_thread_fence(__ATOMIC_SEQ_CST);
#endif
}

/* Transpose the 4x4 matrix composed of row[0-3].  */
#define _MM_TRANSPOSE4_PS(row0, row1, row2, row3)                              \
  do {                                                                         \
    __v4sf __r0 = (row0), __r1 = (row1), __r2 = (row2), __r3 = (row3);         \
    __v4sf __t0 = vec_vmrghw(__r0, __r1);                                      \
    __v4sf __t1 = vec_vmrghw(__r2, __r3);                                      \
    __v4sf __t2 = vec_vmrglw(__r0, __r1);                                      \
    __v4sf __t3 = vec_vmrglw(__r2, __r3);                                      \
    (row0) = (__v4sf)vec_mergeh((__vector long long)__t0,                      \
                                (__vector long long)__t1);                     \
    (row1) = (__v4sf)vec_mergel((__vector long long)__t0,                      \
                                (__vector long long)__t1);                     \
    (row2) = (__v4sf)vec_mergeh((__vector long long)__t2,                      \
                                (__vector long long)__t3);                     \
    (row3) = (__v4sf)vec_mergel((__vector long long)__t2,                      \
                                (__vector long long)__t3);                     \
  } while (0)

/* For backward source compatibility.  */
//# include <emmintrin.h>

#else
#include_next <xmmintrin.h>
#endif /* defined(__powerpc64__) &&                                            \
        *   (defined(__linux__) || defined(__FreeBSD__) || defined(_AIX)) */

#endif /* XMMINTRIN_H_ */