lib/Headers/vecintrin.h

/*===---- vecintrin.h - Vector intrinsics ----------------------------------===
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 *===-----------------------------------------------------------------------===
 */

#if defined(__s390x__) && defined(__VEC__)

#define __ATTRS_ai __attribute__((__always_inline__))
#define __ATTRS_o __attribute__((__overloadable__))
#define __ATTRS_o_ai __attribute__((__overloadable__, __always_inline__))

#define __constant(PARM) \
  __attribute__((__enable_if__ ((PARM) == (PARM), \
     "argument must be a constant integer")))
#define __constant_range(PARM, LOW, HIGH) \
  __attribute__((__enable_if__ ((PARM) >= (LOW) && (PARM) <= (HIGH), \
     "argument must be a constant integer from " #LOW " to " #HIGH)))
#define __constant_pow2_range(PARM, LOW, HIGH) \
  __attribute__((__enable_if__ ((PARM) >= (LOW) && (PARM) <= (HIGH) && \
                                ((PARM) & ((PARM) - 1)) == 0, \
     "argument must be a constant power of 2 from " #LOW " to " #HIGH)))

/*-- __lcbb -----------------------------------------------------------------*/

extern __ATTRS_o unsigned int
__lcbb(const void *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

#define __lcbb(X, Y) ((__typeof__((__lcbb)((X), (Y)))) \
  __builtin_s390_lcbb((X), __builtin_constant_p((Y))? \
                           ((Y) == 64 ? 0 : \
                            (Y) == 128 ? 1 : \
                            (Y) == 256 ? 2 : \
                            (Y) == 512 ? 3 : \
                            (Y) == 1024 ? 4 : \
                            (Y) == 2048 ? 5 : \
                            (Y) == 4096 ? 6 : 0) : 0))

/*-- vec_extract ------------------------------------------------------------*/

static inline __ATTRS_o_ai signed char
vec_extract(vector signed char __vec, int __index) {
  return __vec[__index & 15];
}

static inline __ATTRS_o_ai unsigned char
vec_extract(vector bool char __vec, int __index) {
  return __vec[__index & 15];
}

static inline __ATTRS_o_ai unsigned char
vec_extract(vector unsigned char __vec, int __index) {
  return __vec[__index & 15];
}

static inline __ATTRS_o_ai signed short
vec_extract(vector signed short __vec, int __index) {
  return __vec[__index & 7];
}

static inline __ATTRS_o_ai unsigned short
vec_extract(vector bool short __vec, int __index) {
  return __vec[__index & 7];
}

static inline __ATTRS_o_ai unsigned short
vec_extract(vector unsigned short __vec, int __index) {
  return __vec[__index & 7];
}

static inline __ATTRS_o_ai signed int
vec_extract(vector signed int __vec, int __index) {
  return __vec[__index & 3];
}

static inline __ATTRS_o_ai unsigned int
vec_extract(vector bool int __vec, int __index) {
  return __vec[__index & 3];
}

static inline __ATTRS_o_ai unsigned int
vec_extract(vector unsigned int __vec, int __index) {
  return __vec[__index & 3];
}

static inline __ATTRS_o_ai signed long long
vec_extract(vector signed long long __vec, int __index) {
  return __vec[__index & 1];
}

static inline __ATTRS_o_ai unsigned long long
vec_extract(vector bool long long __vec, int __index) {
  return __vec[__index & 1];
}

static inline __ATTRS_o_ai unsigned long long
vec_extract(vector unsigned long long __vec, int __index) {
  return __vec[__index & 1];
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai float
vec_extract(vector float __vec, int __index) {
  return __vec[__index & 3];
}
#endif

static inline __ATTRS_o_ai double
vec_extract(vector double __vec, int __index) {
  return __vec[__index & 1];
}

/*-- vec_insert -------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_insert(signed char __scalar, vector signed char __vec, int __index) {
  __vec[__index & 15] = __scalar;
  return __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned char
vec_insert(unsigned char __scalar, vector bool char __vec, int __index) {
  vector unsigned char __newvec = (vector unsigned char)__vec;
  __newvec[__index & 15] = (unsigned char)__scalar;
  return __newvec;
}

static inline __ATTRS_o_ai vector unsigned char
vec_insert(unsigned char __scalar, vector unsigned char __vec, int __index) {
  __vec[__index & 15] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector signed short
vec_insert(signed short __scalar, vector signed short __vec, int __index) {
  __vec[__index & 7] = __scalar;
  return __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned short
vec_insert(unsigned short __scalar, vector bool short __vec, int __index) {
  vector unsigned short __newvec = (vector unsigned short)__vec;
  __newvec[__index & 7] = (unsigned short)__scalar;
  return __newvec;
}

static inline __ATTRS_o_ai vector unsigned short
vec_insert(unsigned short __scalar, vector unsigned short __vec, int __index) {
  __vec[__index & 7] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector signed int
vec_insert(signed int __scalar, vector signed int __vec, int __index) {
  __vec[__index & 3] = __scalar;
  return __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned int
vec_insert(unsigned int __scalar, vector bool int __vec, int __index) {
  vector unsigned int __newvec = (vector unsigned int)__vec;
  __newvec[__index & 3] = __scalar;
  return __newvec;
}

static inline __ATTRS_o_ai vector unsigned int
vec_insert(unsigned int __scalar, vector unsigned int __vec, int __index) {
  __vec[__index & 3] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector signed long long
vec_insert(signed long long __scalar, vector signed long long __vec,
           int __index) {
  __vec[__index & 1] = __scalar;
  return __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned long long
vec_insert(unsigned long long __scalar, vector bool long long __vec,
           int __index) {
  vector unsigned long long __newvec = (vector unsigned long long)__vec;
  __newvec[__index & 1] = __scalar;
  return __newvec;
}

static inline __ATTRS_o_ai vector unsigned long long
vec_insert(unsigned long long __scalar, vector unsigned long long __vec,
           int __index) {
  __vec[__index & 1] = __scalar;
  return __vec;
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_insert(float __scalar, vector float __vec, int __index) {
  __vec[__index & 1] = __scalar;
  return __vec;
}
#endif

static inline __ATTRS_o_ai vector double
vec_insert(double __scalar, vector double __vec, int __index) {
  __vec[__index & 1] = __scalar;
  return __vec;
}

/*-- vec_promote ------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_promote(signed char __scalar, int __index) {
  const vector signed char __zero = (vector signed char)0;
  vector signed char __vec = __builtin_shufflevector(__zero, __zero,
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
  __vec[__index & 15] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned char
vec_promote(unsigned char __scalar, int __index) {
  const vector unsigned char __zero = (vector unsigned char)0;
  vector unsigned char __vec = __builtin_shufflevector(__zero, __zero,
    -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
  __vec[__index & 15] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector signed short
vec_promote(signed short __scalar, int __index) {
  const vector signed short __zero = (vector signed short)0;
  vector signed short __vec = __builtin_shufflevector(__zero, __zero,
                                -1, -1, -1, -1, -1, -1, -1, -1);
  __vec[__index & 7] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned short
vec_promote(unsigned short __scalar, int __index) {
  const vector unsigned short __zero = (vector unsigned short)0;
  vector unsigned short __vec = __builtin_shufflevector(__zero, __zero,
                                  -1, -1, -1, -1, -1, -1, -1, -1);
  __vec[__index & 7] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector signed int
vec_promote(signed int __scalar, int __index) {
  const vector signed int __zero = (vector signed int)0;
  vector signed int __vec = __builtin_shufflevector(__zero, __zero,
                                                    -1, -1, -1, -1);
  __vec[__index & 3] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned int
vec_promote(unsigned int __scalar, int __index) {
  const vector unsigned int __zero = (vector unsigned int)0;
  vector unsigned int __vec = __builtin_shufflevector(__zero, __zero,
                                                      -1, -1, -1, -1);
  __vec[__index & 3] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector signed long long
vec_promote(signed long long __scalar, int __index) {
  const vector signed long long __zero = (vector signed long long)0;
  vector signed long long __vec = __builtin_shufflevector(__zero, __zero,
                                                          -1, -1);
  __vec[__index & 1] = __scalar;
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned long long
vec_promote(unsigned long long __scalar, int __index) {
  const vector unsigned long long __zero = (vector unsigned long long)0;
  vector unsigned long long __vec = __builtin_shufflevector(__zero, __zero,
                                                            -1, -1);
  __vec[__index & 1] = __scalar;
  return __vec;
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_promote(float __scalar, int __index) {
  const vector float __zero = (vector float)0;
  vector float __vec = __builtin_shufflevector(__zero, __zero, -1, -1, -1, -1);
  __vec[__index & 3] = __scalar;
  return __vec;
}
#endif

static inline __ATTRS_o_ai vector double
vec_promote(double __scalar, int __index) {
  const vector double __zero = (vector double)0;
  vector double __vec = __builtin_shufflevector(__zero, __zero, -1, -1);
  __vec[__index & 1] = __scalar;
  return __vec;
}

/*-- vec_insert_and_zero ----------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_insert_and_zero(const signed char *__ptr) {
  vector signed char __vec = (vector signed char)0;
  __vec[7] = *__ptr;
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned char
vec_insert_and_zero(const unsigned char *__ptr) {
  vector unsigned char __vec = (vector unsigned char)0;
  __vec[7] = *__ptr;
  return __vec;
}

static inline __ATTRS_o_ai vector signed short
vec_insert_and_zero(const signed short *__ptr) {
  vector signed short __vec = (vector signed short)0;
  __vec[3] = *__ptr;
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned short
vec_insert_and_zero(const unsigned short *__ptr) {
  vector unsigned short __vec = (vector unsigned short)0;
  __vec[3] = *__ptr;
  return __vec;
}

static inline __ATTRS_o_ai vector signed int
vec_insert_and_zero(const signed int *__ptr) {
  vector signed int __vec = (vector signed int)0;
  __vec[1] = *__ptr;
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned int
vec_insert_and_zero(const unsigned int *__ptr) {
  vector unsigned int __vec = (vector unsigned int)0;
  __vec[1] = *__ptr;
  return __vec;
}

static inline __ATTRS_o_ai vector signed long long
vec_insert_and_zero(const signed long long *__ptr) {
  vector signed long long __vec = (vector signed long long)0;
  __vec[0] = *__ptr;
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned long long
vec_insert_and_zero(const unsigned long long *__ptr) {
  vector unsigned long long __vec = (vector unsigned long long)0;
  __vec[0] = *__ptr;
  return __vec;
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_insert_and_zero(const float *__ptr) {
  vector float __vec = (vector float)0;
  __vec[0] = *__ptr;
  return __vec;
}
#endif

static inline __ATTRS_o_ai vector double
vec_insert_and_zero(const double *__ptr) {
  vector double __vec = (vector double)0;
  __vec[0] = *__ptr;
  return __vec;
}

/*-- vec_perm ---------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_perm(vector signed char __a, vector signed char __b,
         vector unsigned char __c) {
  return (vector signed char)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector unsigned char
vec_perm(vector unsigned char __a, vector unsigned char __b,
         vector unsigned char __c) {
  return (vector unsigned char)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector bool char
vec_perm(vector bool char __a, vector bool char __b,
         vector unsigned char __c) {
  return (vector bool char)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector signed short
vec_perm(vector signed short __a, vector signed short __b,
         vector unsigned char __c) {
  return (vector signed short)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector unsigned short
vec_perm(vector unsigned short __a, vector unsigned short __b,
         vector unsigned char __c) {
  return (vector unsigned short)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector bool short
vec_perm(vector bool short __a, vector bool short __b,
         vector unsigned char __c) {
  return (vector bool short)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector signed int
vec_perm(vector signed int __a, vector signed int __b,
         vector unsigned char __c) {
  return (vector signed int)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector unsigned int
vec_perm(vector unsigned int __a, vector unsigned int __b,
         vector unsigned char __c) {
  return (vector unsigned int)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector bool int
vec_perm(vector bool int __a, vector bool int __b,
         vector unsigned char __c) {
  return (vector bool int)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector signed long long
vec_perm(vector signed long long __a, vector signed long long __b,
         vector unsigned char __c) {
  return (vector signed long long)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_perm(vector unsigned long long __a, vector unsigned long long __b,
         vector unsigned char __c) {
  return (vector unsigned long long)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

static inline __ATTRS_o_ai vector bool long long
vec_perm(vector bool long long __a, vector bool long long __b,
         vector unsigned char __c) {
  return (vector bool long long)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_perm(vector float __a, vector float __b,
         vector unsigned char __c) {
  return (vector float)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}
#endif

static inline __ATTRS_o_ai vector double
vec_perm(vector double __a, vector double __b,
         vector unsigned char __c) {
  return (vector double)__builtin_s390_vperm(
           (vector unsigned char)__a, (vector unsigned char)__b, __c);
}

/*-- vec_permi --------------------------------------------------------------*/

// This prototype is deprecated.
extern __ATTRS_o vector signed long long
vec_permi(vector signed long long __a, vector signed long long __b, int __c)
  __constant_range(__c, 0, 3);

// This prototype is deprecated.
extern __ATTRS_o vector unsigned long long
vec_permi(vector unsigned long long __a, vector unsigned long long __b, int __c)
  __constant_range(__c, 0, 3);

// This prototype is deprecated.
extern __ATTRS_o vector bool long long
vec_permi(vector bool long long __a, vector bool long long __b, int __c)
  __constant_range(__c, 0, 3);

// This prototype is deprecated.
extern __ATTRS_o vector double
vec_permi(vector double __a, vector double __b, int __c)
  __constant_range(__c, 0, 3);

#define vec_permi(X, Y, Z) ((__typeof__((vec_permi)((X), (Y), (Z)))) \
  __builtin_s390_vpdi((vector unsigned long long)(X), \
                      (vector unsigned long long)(Y), \
                      (((Z) & 2) << 1) | ((Z) & 1)))

/*-- vec_bperm_u128 ---------------------------------------------------------*/

#if __ARCH__ >= 12
static inline __ATTRS_ai vector unsigned long long
vec_bperm_u128(vector unsigned char __a, vector unsigned char __b) {
  return __builtin_s390_vbperm(__a, __b);
}
#endif

/*-- vec_sel ----------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_sel(vector signed char __a, vector signed char __b,
        vector unsigned char __c) {
  return ((vector signed char)__c & __b) | (~(vector signed char)__c & __a);
}

static inline __ATTRS_o_ai vector signed char
vec_sel(vector signed char __a, vector signed char __b, vector bool char __c) {
  return ((vector signed char)__c & __b) | (~(vector signed char)__c & __a);
}

static inline __ATTRS_o_ai vector bool char
vec_sel(vector bool char __a, vector bool char __b, vector unsigned char __c) {
  return ((vector bool char)__c & __b) | (~(vector bool char)__c & __a);
}

static inline __ATTRS_o_ai vector bool char
vec_sel(vector bool char __a, vector bool char __b, vector bool char __c) {
  return (__c & __b) | (~__c & __a);
}

static inline __ATTRS_o_ai vector unsigned char
vec_sel(vector unsigned char __a, vector unsigned char __b,
        vector unsigned char __c) {
  return (__c & __b) | (~__c & __a);
}

static inline __ATTRS_o_ai vector unsigned char
vec_sel(vector unsigned char __a, vector unsigned char __b,
        vector bool char __c) {
  return ((vector unsigned char)__c & __b) | (~(vector unsigned char)__c & __a);
}

static inline __ATTRS_o_ai vector signed short
vec_sel(vector signed short __a, vector signed short __b,
        vector unsigned short __c) {
  return ((vector signed short)__c & __b) | (~(vector signed short)__c & __a);
}

static inline __ATTRS_o_ai vector signed short
vec_sel(vector signed short __a, vector signed short __b,
        vector bool short __c) {
  return ((vector signed short)__c & __b) | (~(vector signed short)__c & __a);
}

static inline __ATTRS_o_ai vector bool short
vec_sel(vector bool short __a, vector bool short __b,
        vector unsigned short __c) {
  return ((vector bool short)__c & __b) | (~(vector bool short)__c & __a);
}

static inline __ATTRS_o_ai vector bool short
vec_sel(vector bool short __a, vector bool short __b, vector bool short __c) {
  return (__c & __b) | (~__c & __a);
}

static inline __ATTRS_o_ai vector unsigned short
vec_sel(vector unsigned short __a, vector unsigned short __b,
        vector unsigned short __c) {
  return (__c & __b) | (~__c & __a);
}

static inline __ATTRS_o_ai vector unsigned short
vec_sel(vector unsigned short __a, vector unsigned short __b,
        vector bool short __c) {
  return (((vector unsigned short)__c & __b) |
          (~(vector unsigned short)__c & __a));
}

static inline __ATTRS_o_ai vector signed int
vec_sel(vector signed int __a, vector signed int __b,
        vector unsigned int __c) {
  return ((vector signed int)__c & __b) | (~(vector signed int)__c & __a);
}

static inline __ATTRS_o_ai vector signed int
vec_sel(vector signed int __a, vector signed int __b, vector bool int __c) {
  return ((vector signed int)__c & __b) | (~(vector signed int)__c & __a);
}

static inline __ATTRS_o_ai vector bool int
vec_sel(vector bool int __a, vector bool int __b, vector unsigned int __c) {
  return ((vector bool int)__c & __b) | (~(vector bool int)__c & __a);
}

static inline __ATTRS_o_ai vector bool int
vec_sel(vector bool int __a, vector bool int __b, vector bool int __c) {
  return (__c & __b) | (~__c & __a);
}

static inline __ATTRS_o_ai vector unsigned int
vec_sel(vector unsigned int __a, vector unsigned int __b,
        vector unsigned int __c) {
  return (__c & __b) | (~__c & __a);
}

static inline __ATTRS_o_ai vector unsigned int
vec_sel(vector unsigned int __a, vector unsigned int __b, vector bool int __c) {
  return ((vector unsigned int)__c & __b) | (~(vector unsigned int)__c & __a);
}

static inline __ATTRS_o_ai vector signed long long
vec_sel(vector signed long long __a, vector signed long long __b,
        vector unsigned long long __c) {
  return (((vector signed long long)__c & __b) |
          (~(vector signed long long)__c & __a));
}

static inline __ATTRS_o_ai vector signed long long
vec_sel(vector signed long long __a, vector signed long long __b,
        vector bool long long __c) {
  return (((vector signed long long)__c & __b) |
          (~(vector signed long long)__c & __a));
}

static inline __ATTRS_o_ai vector bool long long
vec_sel(vector bool long long __a, vector bool long long __b,
        vector unsigned long long __c) {
  return (((vector bool long long)__c & __b) |
          (~(vector bool long long)__c & __a));
}

static inline __ATTRS_o_ai vector bool long long
vec_sel(vector bool long long __a, vector bool long long __b,
        vector bool long long __c) {
  return (__c & __b) | (~__c & __a);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_sel(vector unsigned long long __a, vector unsigned long long __b,
        vector unsigned long long __c) {
  return (__c & __b) | (~__c & __a);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_sel(vector unsigned long long __a, vector unsigned long long __b,
        vector bool long long __c) {
  return (((vector unsigned long long)__c & __b) |
          (~(vector unsigned long long)__c & __a));
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_sel(vector float __a, vector float __b, vector unsigned int __c) {
  return (vector float)((__c & (vector unsigned int)__b) |
                        (~__c & (vector unsigned int)__a));
}

static inline __ATTRS_o_ai vector float
vec_sel(vector float __a, vector float __b, vector bool int __c) {
  vector unsigned int __ac = (vector unsigned int)__a;
  vector unsigned int __bc = (vector unsigned int)__b;
  vector unsigned int __cc = (vector unsigned int)__c;
  return (vector float)((__cc & __bc) | (~__cc & __ac));
}
#endif

static inline __ATTRS_o_ai vector double
vec_sel(vector double __a, vector double __b, vector unsigned long long __c) {
  return (vector double)((__c & (vector unsigned long long)__b) |
                         (~__c & (vector unsigned long long)__a));
}

static inline __ATTRS_o_ai vector double
vec_sel(vector double __a, vector double __b, vector bool long long __c) {
  vector unsigned long long __ac = (vector unsigned long long)__a;
  vector unsigned long long __bc = (vector unsigned long long)__b;
  vector unsigned long long __cc = (vector unsigned long long)__c;
  return (vector double)((__cc & __bc) | (~__cc & __ac));
}

/*-- vec_gather_element -----------------------------------------------------*/

static inline __ATTRS_o_ai vector signed int
vec_gather_element(vector signed int __vec, vector unsigned int __offset,
                   const signed int *__ptr, int __index)
  __constant_range(__index, 0, 3) {
  __vec[__index] = *(const signed int *)(
    (__INTPTR_TYPE__)__ptr + (__INTPTR_TYPE__)__offset[__index]);
  return __vec;
}

static inline __ATTRS_o_ai vector bool int
vec_gather_element(vector bool int __vec, vector unsigned int __offset,
                   const unsigned int *__ptr, int __index)
  __constant_range(__index, 0, 3) {
  __vec[__index] = *(const unsigned int *)(
    (__INTPTR_TYPE__)__ptr + (__INTPTR_TYPE__)__offset[__index]);
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned int
vec_gather_element(vector unsigned int __vec, vector unsigned int __offset,
                   const unsigned int *__ptr, int __index)
  __constant_range(__index, 0, 3) {
  __vec[__index] = *(const unsigned int *)(
    (__INTPTR_TYPE__)__ptr + (__INTPTR_TYPE__)__offset[__index]);
  return __vec;
}

static inline __ATTRS_o_ai vector signed long long
vec_gather_element(vector signed long long __vec,
                   vector unsigned long long __offset,
                   const signed long long *__ptr, int __index)
  __constant_range(__index, 0, 1) {
  __vec[__index] = *(const signed long long *)(
    (__INTPTR_TYPE__)__ptr + (__INTPTR_TYPE__)__offset[__index]);
  return __vec;
}

static inline __ATTRS_o_ai vector bool long long
vec_gather_element(vector bool long long __vec,
                   vector unsigned long long __offset,
                   const unsigned long long *__ptr, int __index)
  __constant_range(__index, 0, 1) {
  __vec[__index] = *(const unsigned long long *)(
    (__INTPTR_TYPE__)__ptr + (__INTPTR_TYPE__)__offset[__index]);
  return __vec;
}

static inline __ATTRS_o_ai vector unsigned long long
vec_gather_element(vector unsigned long long __vec,
                   vector unsigned long long __offset,
                   const unsigned long long *__ptr, int __index)
  __constant_range(__index, 0, 1) {
  __vec[__index] = *(const unsigned long long *)(
    (__INTPTR_TYPE__)__ptr + (__INTPTR_TYPE__)__offset[__index]);
  return __vec;
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_gather_element(vector float __vec, vector unsigned int __offset,
                   const float *__ptr, int __index)
  __constant_range(__index, 0, 3) {
  __vec[__index] = *(const float *)(
    (__INTPTR_TYPE__)__ptr + (__INTPTR_TYPE__)__offset[__index]);
  return __vec;
}
#endif

static inline __ATTRS_o_ai vector double
vec_gather_element(vector double __vec, vector unsigned long long __offset,
                   const double *__ptr, int __index)
  __constant_range(__index, 0, 1) {
  __vec[__index] = *(const double *)(
    (__INTPTR_TYPE__)__ptr + (__INTPTR_TYPE__)__offset[__index]);
  return __vec;
}

/*-- vec_scatter_element ----------------------------------------------------*/

static inline __ATTRS_o_ai void
vec_scatter_element(vector signed int __vec, vector unsigned int __offset,
                    signed int *__ptr, int __index)
  __constant_range(__index, 0, 3) {
  *(signed int *)((__INTPTR_TYPE__)__ptr + __offset[__index]) =
    __vec[__index];
}

static inline __ATTRS_o_ai void
vec_scatter_element(vector bool int __vec, vector unsigned int __offset,
                    unsigned int *__ptr, int __index)
  __constant_range(__index, 0, 3) {
  *(unsigned int *)((__INTPTR_TYPE__)__ptr + __offset[__index]) =
    __vec[__index];
}

static inline __ATTRS_o_ai void
vec_scatter_element(vector unsigned int __vec, vector unsigned int __offset,
                    unsigned int *__ptr, int __index)
  __constant_range(__index, 0, 3) {
  *(unsigned int *)((__INTPTR_TYPE__)__ptr + __offset[__index]) =
    __vec[__index];
}

static inline __ATTRS_o_ai void
vec_scatter_element(vector signed long long __vec,
                    vector unsigned long long __offset,
                    signed long long *__ptr, int __index)
  __constant_range(__index, 0, 1) {
  *(signed long long *)((__INTPTR_TYPE__)__ptr + __offset[__index]) =
    __vec[__index];
}

static inline __ATTRS_o_ai void
vec_scatter_element(vector bool long long __vec,
                    vector unsigned long long __offset,
                    unsigned long long *__ptr, int __index)
  __constant_range(__index, 0, 1) {
  *(unsigned long long *)((__INTPTR_TYPE__)__ptr + __offset[__index]) =
    __vec[__index];
}

static inline __ATTRS_o_ai void
vec_scatter_element(vector unsigned long long __vec,
                    vector unsigned long long __offset,
                    unsigned long long *__ptr, int __index)
  __constant_range(__index, 0, 1) {
  *(unsigned long long *)((__INTPTR_TYPE__)__ptr + __offset[__index]) =
    __vec[__index];
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai void
vec_scatter_element(vector float __vec, vector unsigned int __offset,
                    float *__ptr, int __index)
  __constant_range(__index, 0, 3) {
  *(float *)((__INTPTR_TYPE__)__ptr + __offset[__index]) =
    __vec[__index];
}
#endif

static inline __ATTRS_o_ai void
vec_scatter_element(vector double __vec, vector unsigned long long __offset,
                    double *__ptr, int __index)
  __constant_range(__index, 0, 1) {
  *(double *)((__INTPTR_TYPE__)__ptr + __offset[__index]) =
    __vec[__index];
}

/*-- vec_xl -----------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_xl(long __offset, const signed char *__ptr) {
  return *(const vector signed char *)((__INTPTR_TYPE__)__ptr + __offset);
}

static inline __ATTRS_o_ai vector unsigned char
vec_xl(long __offset, const unsigned char *__ptr) {
  return *(const vector unsigned char *)((__INTPTR_TYPE__)__ptr + __offset);
}

static inline __ATTRS_o_ai vector signed short
vec_xl(long __offset, const signed short *__ptr) {
  return *(const vector signed short *)((__INTPTR_TYPE__)__ptr + __offset);
}

static inline __ATTRS_o_ai vector unsigned short
vec_xl(long __offset, const unsigned short *__ptr) {
  return *(const vector unsigned short *)((__INTPTR_TYPE__)__ptr + __offset);
}

static inline __ATTRS_o_ai vector signed int
vec_xl(long __offset, const signed int *__ptr) {
  return *(const vector signed int *)((__INTPTR_TYPE__)__ptr + __offset);
}

static inline __ATTRS_o_ai vector unsigned int
vec_xl(long __offset, const unsigned int *__ptr) {
  return *(const vector unsigned int *)((__INTPTR_TYPE__)__ptr + __offset);
}

static inline __ATTRS_o_ai vector signed long long
vec_xl(long __offset, const signed long long *__ptr) {
  return *(const vector signed long long *)((__INTPTR_TYPE__)__ptr + __offset);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_xl(long __offset, const unsigned long long *__ptr) {
  return *(const vector unsigned long long *)((__INTPTR_TYPE__)__ptr + __offset);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_xl(long __offset, const float *__ptr) {
  return *(const vector float *)((__INTPTR_TYPE__)__ptr + __offset);
}
#endif

static inline __ATTRS_o_ai vector double
vec_xl(long __offset, const double *__ptr) {
  return *(const vector double *)((__INTPTR_TYPE__)__ptr + __offset);
}

/*-- vec_xld2 ---------------------------------------------------------------*/

// This prototype is deprecated.
static inline __ATTRS_o_ai vector signed char
vec_xld2(long __offset, const signed char *__ptr) {
  return *(const vector signed char *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned char
vec_xld2(long __offset, const unsigned char *__ptr) {
  return *(const vector unsigned char *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector signed short
vec_xld2(long __offset, const signed short *__ptr) {
  return *(const vector signed short *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned short
vec_xld2(long __offset, const unsigned short *__ptr) {
  return *(const vector unsigned short *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector signed int
vec_xld2(long __offset, const signed int *__ptr) {
  return *(const vector signed int *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned int
vec_xld2(long __offset, const unsigned int *__ptr) {
  return *(const vector unsigned int *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector signed long long
vec_xld2(long __offset, const signed long long *__ptr) {
  return *(const vector signed long long *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned long long
vec_xld2(long __offset, const unsigned long long *__ptr) {
  return *(const vector unsigned long long *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector double
vec_xld2(long __offset, const double *__ptr) {
  return *(const vector double *)((__INTPTR_TYPE__)__ptr + __offset);
}

/*-- vec_xlw4 ---------------------------------------------------------------*/

// This prototype is deprecated.
static inline __ATTRS_o_ai vector signed char
vec_xlw4(long __offset, const signed char *__ptr) {
  return *(const vector signed char *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned char
vec_xlw4(long __offset, const unsigned char *__ptr) {
  return *(const vector unsigned char *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector signed short
vec_xlw4(long __offset, const signed short *__ptr) {
  return *(const vector signed short *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned short
vec_xlw4(long __offset, const unsigned short *__ptr) {
  return *(const vector unsigned short *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector signed int
vec_xlw4(long __offset, const signed int *__ptr) {
  return *(const vector signed int *)((__INTPTR_TYPE__)__ptr + __offset);
}

// This prototype is deprecated.
static inline __ATTRS_o_ai vector unsigned int
vec_xlw4(long __offset, const unsigned int *__ptr) {
  return *(const vector unsigned int *)((__INTPTR_TYPE__)__ptr + __offset);
}

/*-- vec_xst ----------------------------------------------------------------*/

static inline __ATTRS_o_ai void
vec_xst(vector signed char __vec, long __offset, signed char *__ptr) {
  *(vector signed char *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

static inline __ATTRS_o_ai void
vec_xst(vector unsigned char __vec, long __offset, unsigned char *__ptr) {
  *(vector unsigned char *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

static inline __ATTRS_o_ai void
vec_xst(vector signed short __vec, long __offset, signed short *__ptr) {
  *(vector signed short *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

static inline __ATTRS_o_ai void
vec_xst(vector unsigned short __vec, long __offset, unsigned short *__ptr) {
  *(vector unsigned short *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

static inline __ATTRS_o_ai void
vec_xst(vector signed int __vec, long __offset, signed int *__ptr) {
  *(vector signed int *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

static inline __ATTRS_o_ai void
vec_xst(vector unsigned int __vec, long __offset, unsigned int *__ptr) {
  *(vector unsigned int *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

static inline __ATTRS_o_ai void
vec_xst(vector signed long long __vec, long __offset,
          signed long long *__ptr) {
  *(vector signed long long *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

static inline __ATTRS_o_ai void
vec_xst(vector unsigned long long __vec, long __offset,
          unsigned long long *__ptr) {
  *(vector unsigned long long *)((__INTPTR_TYPE__)__ptr + __offset) =
    __vec;
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai void
vec_xst(vector float __vec, long __offset, float *__ptr) {
  *(vector float *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}
#endif

static inline __ATTRS_o_ai void
vec_xst(vector double __vec, long __offset, double *__ptr) {
  *(vector double *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

/*-- vec_xstd2 --------------------------------------------------------------*/

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstd2(vector signed char __vec, long __offset, signed char *__ptr) {
  *(vector signed char *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstd2(vector unsigned char __vec, long __offset, unsigned char *__ptr) {
  *(vector unsigned char *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstd2(vector signed short __vec, long __offset, signed short *__ptr) {
  *(vector signed short *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstd2(vector unsigned short __vec, long __offset, unsigned short *__ptr) {
  *(vector unsigned short *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstd2(vector signed int __vec, long __offset, signed int *__ptr) {
  *(vector signed int *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstd2(vector unsigned int __vec, long __offset, unsigned int *__ptr) {
  *(vector unsigned int *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstd2(vector signed long long __vec, long __offset,
          signed long long *__ptr) {
  *(vector signed long long *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstd2(vector unsigned long long __vec, long __offset,
          unsigned long long *__ptr) {
  *(vector unsigned long long *)((__INTPTR_TYPE__)__ptr + __offset) =
    __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstd2(vector double __vec, long __offset, double *__ptr) {
  *(vector double *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

/*-- vec_xstw4 --------------------------------------------------------------*/

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstw4(vector signed char __vec, long __offset, signed char *__ptr) {
  *(vector signed char *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstw4(vector unsigned char __vec, long __offset, unsigned char *__ptr) {
  *(vector unsigned char *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstw4(vector signed short __vec, long __offset, signed short *__ptr) {
  *(vector signed short *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstw4(vector unsigned short __vec, long __offset, unsigned short *__ptr) {
  *(vector unsigned short *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstw4(vector signed int __vec, long __offset, signed int *__ptr) {
  *(vector signed int *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai void
vec_xstw4(vector unsigned int __vec, long __offset, unsigned int *__ptr) {
  *(vector unsigned int *)((__INTPTR_TYPE__)__ptr + __offset) = __vec;
}

/*-- vec_load_bndry ---------------------------------------------------------*/

extern __ATTRS_o vector signed char
vec_load_bndry(const signed char *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

extern __ATTRS_o vector unsigned char
vec_load_bndry(const unsigned char *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

extern __ATTRS_o vector signed short
vec_load_bndry(const signed short *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

extern __ATTRS_o vector unsigned short
vec_load_bndry(const unsigned short *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

extern __ATTRS_o vector signed int
vec_load_bndry(const signed int *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

extern __ATTRS_o vector unsigned int
vec_load_bndry(const unsigned int *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

extern __ATTRS_o vector signed long long
vec_load_bndry(const signed long long *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

extern __ATTRS_o vector unsigned long long
vec_load_bndry(const unsigned long long *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

#if __ARCH__ >= 12
extern __ATTRS_o vector float
vec_load_bndry(const float *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);
#endif

extern __ATTRS_o vector double
vec_load_bndry(const double *__ptr, unsigned short __len)
  __constant_pow2_range(__len, 64, 4096);

#define vec_load_bndry(X, Y) ((__typeof__((vec_load_bndry)((X), (Y)))) \
  __builtin_s390_vlbb((X), ((Y) == 64 ? 0 : \
                            (Y) == 128 ? 1 : \
                            (Y) == 256 ? 2 : \
                            (Y) == 512 ? 3 : \
                            (Y) == 1024 ? 4 : \
                            (Y) == 2048 ? 5 : \
                            (Y) == 4096 ? 6 : -1)))

/*-- vec_load_len -----------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_load_len(const signed char *__ptr, unsigned int __len) {
  return (vector signed char)__builtin_s390_vll(__len, __ptr);
}

static inline __ATTRS_o_ai vector unsigned char
vec_load_len(const unsigned char *__ptr, unsigned int __len) {
  return (vector unsigned char)__builtin_s390_vll(__len, __ptr);
}

static inline __ATTRS_o_ai vector signed short
vec_load_len(const signed short *__ptr, unsigned int __len) {
  return (vector signed short)__builtin_s390_vll(__len, __ptr);
}

static inline __ATTRS_o_ai vector unsigned short
vec_load_len(const unsigned short *__ptr, unsigned int __len) {
  return (vector unsigned short)__builtin_s390_vll(__len, __ptr);
}

static inline __ATTRS_o_ai vector signed int
vec_load_len(const signed int *__ptr, unsigned int __len) {
  return (vector signed int)__builtin_s390_vll(__len, __ptr);
}

static inline __ATTRS_o_ai vector unsigned int
vec_load_len(const unsigned int *__ptr, unsigned int __len) {
  return (vector unsigned int)__builtin_s390_vll(__len, __ptr);
}

static inline __ATTRS_o_ai vector signed long long
vec_load_len(const signed long long *__ptr, unsigned int __len) {
  return (vector signed long long)__builtin_s390_vll(__len, __ptr);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_load_len(const unsigned long long *__ptr, unsigned int __len) {
  return (vector unsigned long long)__builtin_s390_vll(__len, __ptr);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_load_len(const float *__ptr, unsigned int __len) {
  return (vector float)__builtin_s390_vll(__len, __ptr);
}
#endif

static inline __ATTRS_o_ai vector double
vec_load_len(const double *__ptr, unsigned int __len) {
  return (vector double)__builtin_s390_vll(__len, __ptr);
}

/*-- vec_load_len_r ---------------------------------------------------------*/

#if __ARCH__ >= 12
static inline __ATTRS_ai vector unsigned char
vec_load_len_r(const unsigned char *__ptr, unsigned int __len) {
  return (vector unsigned char)__builtin_s390_vlrl(__len, __ptr);
}
#endif

/*-- vec_store_len ----------------------------------------------------------*/

static inline __ATTRS_o_ai void
vec_store_len(vector signed char __vec, signed char *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}

static inline __ATTRS_o_ai void
vec_store_len(vector unsigned char __vec, unsigned char *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}

static inline __ATTRS_o_ai void
vec_store_len(vector signed short __vec, signed short *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}

static inline __ATTRS_o_ai void
vec_store_len(vector unsigned short __vec, unsigned short *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}

static inline __ATTRS_o_ai void
vec_store_len(vector signed int __vec, signed int *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}

static inline __ATTRS_o_ai void
vec_store_len(vector unsigned int __vec, unsigned int *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}

static inline __ATTRS_o_ai void
vec_store_len(vector signed long long __vec, signed long long *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}

static inline __ATTRS_o_ai void
vec_store_len(vector unsigned long long __vec, unsigned long long *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai void
vec_store_len(vector float __vec, float *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}
#endif

static inline __ATTRS_o_ai void
vec_store_len(vector double __vec, double *__ptr,
              unsigned int __len) {
  __builtin_s390_vstl((vector signed char)__vec, __len, __ptr);
}

/*-- vec_store_len_r --------------------------------------------------------*/

#if __ARCH__ >= 12
static inline __ATTRS_ai void
vec_store_len_r(vector unsigned char __vec, unsigned char *__ptr,
                unsigned int __len) {
  __builtin_s390_vstrl((vector signed char)__vec, __len, __ptr);
}
#endif

/*-- vec_load_pair ----------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed long long
vec_load_pair(signed long long __a, signed long long __b) {
  return (vector signed long long)(__a, __b);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_load_pair(unsigned long long __a, unsigned long long __b) {
  return (vector unsigned long long)(__a, __b);
}

/*-- vec_genmask ------------------------------------------------------------*/

static inline __ATTRS_o_ai vector unsigned char
vec_genmask(unsigned short __mask)
  __constant(__mask) {
  return (vector unsigned char)(
    __mask & 0x8000 ? 0xff : 0,
    __mask & 0x4000 ? 0xff : 0,
    __mask & 0x2000 ? 0xff : 0,
    __mask & 0x1000 ? 0xff : 0,
    __mask & 0x0800 ? 0xff : 0,
    __mask & 0x0400 ? 0xff : 0,
    __mask & 0x0200 ? 0xff : 0,
    __mask & 0x0100 ? 0xff : 0,
    __mask & 0x0080 ? 0xff : 0,
    __mask & 0x0040 ? 0xff : 0,
    __mask & 0x0020 ? 0xff : 0,
    __mask & 0x0010 ? 0xff : 0,
    __mask & 0x0008 ? 0xff : 0,
    __mask & 0x0004 ? 0xff : 0,
    __mask & 0x0002 ? 0xff : 0,
    __mask & 0x0001 ? 0xff : 0);
}

/*-- vec_genmasks_* ---------------------------------------------------------*/

static inline __ATTRS_o_ai vector unsigned char
vec_genmasks_8(unsigned char __first, unsigned char __last)
  __constant(__first) __constant(__last) {
  unsigned char __bit1 = __first & 7;
  unsigned char __bit2 = __last & 7;
  unsigned char __mask1 = (unsigned char)(1U << (7 - __bit1) << 1) - 1;
  unsigned char __mask2 = (unsigned char)(1U << (7 - __bit2)) - 1;
  unsigned char __value = (__bit1 <= __bit2 ?
                           __mask1 & ~__mask2 :
                           __mask1 | ~__mask2);
  return (vector unsigned char)__value;
}

static inline __ATTRS_o_ai vector unsigned short
vec_genmasks_16(unsigned char __first, unsigned char __last)
  __constant(__first) __constant(__last) {
  unsigned char __bit1 = __first & 15;
  unsigned char __bit2 = __last & 15;
  unsigned short __mask1 = (unsigned short)(1U << (15 - __bit1) << 1) - 1;
  unsigned short __mask2 = (unsigned short)(1U << (15 - __bit2)) - 1;
  unsigned short __value = (__bit1 <= __bit2 ?
                            __mask1 & ~__mask2 :
                            __mask1 | ~__mask2);
  return (vector unsigned short)__value;
}

static inline __ATTRS_o_ai vector unsigned int
vec_genmasks_32(unsigned char __first, unsigned char __last)
  __constant(__first) __constant(__last) {
  unsigned char __bit1 = __first & 31;
  unsigned char __bit2 = __last & 31;
  unsigned int __mask1 = (1U << (31 - __bit1) << 1) - 1;
  unsigned int __mask2 = (1U << (31 - __bit2)) - 1;
  unsigned int __value = (__bit1 <= __bit2 ?
                          __mask1 & ~__mask2 :
                          __mask1 | ~__mask2);
  return (vector unsigned int)__value;
}

static inline __ATTRS_o_ai vector unsigned long long
vec_genmasks_64(unsigned char __first, unsigned char __last)
  __constant(__first) __constant(__last) {
  unsigned char __bit1 = __first & 63;
  unsigned char __bit2 = __last & 63;
  unsigned long long __mask1 = (1ULL << (63 - __bit1) << 1) - 1;
  unsigned long long __mask2 = (1ULL << (63 - __bit2)) - 1;
  unsigned long long __value = (__bit1 <= __bit2 ?
                                __mask1 & ~__mask2 :
                                __mask1 | ~__mask2);
  return (vector unsigned long long)__value;
}

/*-- vec_splat --------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_splat(vector signed char __vec, int __index)
  __constant_range(__index, 0, 15) {
  return (vector signed char)__vec[__index];
}

static inline __ATTRS_o_ai vector bool char
vec_splat(vector bool char __vec, int __index)
  __constant_range(__index, 0, 15) {
  return (vector bool char)(vector unsigned char)__vec[__index];
}

static inline __ATTRS_o_ai vector unsigned char
vec_splat(vector unsigned char __vec, int __index)
  __constant_range(__index, 0, 15) {
  return (vector unsigned char)__vec[__index];
}

static inline __ATTRS_o_ai vector signed short
vec_splat(vector signed short __vec, int __index)
  __constant_range(__index, 0, 7) {
  return (vector signed short)__vec[__index];
}

static inline __ATTRS_o_ai vector bool short
vec_splat(vector bool short __vec, int __index)
  __constant_range(__index, 0, 7) {
  return (vector bool short)(vector unsigned short)__vec[__index];
}

static inline __ATTRS_o_ai vector unsigned short
vec_splat(vector unsigned short __vec, int __index)
  __constant_range(__index, 0, 7) {
  return (vector unsigned short)__vec[__index];
}

static inline __ATTRS_o_ai vector signed int
vec_splat(vector signed int __vec, int __index)
  __constant_range(__index, 0, 3) {
  return (vector signed int)__vec[__index];
}

static inline __ATTRS_o_ai vector bool int
vec_splat(vector bool int __vec, int __index)
  __constant_range(__index, 0, 3) {
  return (vector bool int)(vector unsigned int)__vec[__index];
}

static inline __ATTRS_o_ai vector unsigned int
vec_splat(vector unsigned int __vec, int __index)
  __constant_range(__index, 0, 3) {
  return (vector unsigned int)__vec[__index];
}

static inline __ATTRS_o_ai vector signed long long
vec_splat(vector signed long long __vec, int __index)
  __constant_range(__index, 0, 1) {
  return (vector signed long long)__vec[__index];
}

static inline __ATTRS_o_ai vector bool long long
vec_splat(vector bool long long __vec, int __index)
  __constant_range(__index, 0, 1) {
  return (vector bool long long)(vector unsigned long long)__vec[__index];
}

static inline __ATTRS_o_ai vector unsigned long long
vec_splat(vector unsigned long long __vec, int __index)
  __constant_range(__index, 0, 1) {
  return (vector unsigned long long)__vec[__index];
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_splat(vector float __vec, int __index)
  __constant_range(__index, 0, 3) {
  return (vector float)__vec[__index];
}
#endif

static inline __ATTRS_o_ai vector double
vec_splat(vector double __vec, int __index)
  __constant_range(__index, 0, 1) {
  return (vector double)__vec[__index];
}

/*-- vec_splat_s* -----------------------------------------------------------*/

static inline __ATTRS_ai vector signed char
vec_splat_s8(signed char __scalar)
  __constant(__scalar) {
  return (vector signed char)__scalar;
}

static inline __ATTRS_ai vector signed short
vec_splat_s16(signed short __scalar)
  __constant(__scalar) {
  return (vector signed short)__scalar;
}

static inline __ATTRS_ai vector signed int
vec_splat_s32(signed short __scalar)
  __constant(__scalar) {
  return (vector signed int)(signed int)__scalar;
}

static inline __ATTRS_ai vector signed long long
vec_splat_s64(signed short __scalar)
  __constant(__scalar) {
  return (vector signed long long)(signed long)__scalar;
}

/*-- vec_splat_u* -----------------------------------------------------------*/

static inline __ATTRS_ai vector unsigned char
vec_splat_u8(unsigned char __scalar)
  __constant(__scalar) {
  return (vector unsigned char)__scalar;
}

static inline __ATTRS_ai vector unsigned short
vec_splat_u16(unsigned short __scalar)
  __constant(__scalar) {
  return (vector unsigned short)__scalar;
}

static inline __ATTRS_ai vector unsigned int
vec_splat_u32(signed short __scalar)
  __constant(__scalar) {
  return (vector unsigned int)(signed int)__scalar;
}

static inline __ATTRS_ai vector unsigned long long
vec_splat_u64(signed short __scalar)
  __constant(__scalar) {
  return (vector unsigned long long)(signed long long)__scalar;
}

/*-- vec_splats -------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_splats(signed char __scalar) {
  return (vector signed char)__scalar;
}

static inline __ATTRS_o_ai vector unsigned char
vec_splats(unsigned char __scalar) {
  return (vector unsigned char)__scalar;
}

static inline __ATTRS_o_ai vector signed short
vec_splats(signed short __scalar) {
  return (vector signed short)__scalar;
}

static inline __ATTRS_o_ai vector unsigned short
vec_splats(unsigned short __scalar) {
  return (vector unsigned short)__scalar;
}

static inline __ATTRS_o_ai vector signed int
vec_splats(signed int __scalar) {
  return (vector signed int)__scalar;
}

static inline __ATTRS_o_ai vector unsigned int
vec_splats(unsigned int __scalar) {
  return (vector unsigned int)__scalar;
}

static inline __ATTRS_o_ai vector signed long long
vec_splats(signed long long __scalar) {
  return (vector signed long long)__scalar;
}

static inline __ATTRS_o_ai vector unsigned long long
vec_splats(unsigned long long __scalar) {
  return (vector unsigned long long)__scalar;
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_splats(float __scalar) {
  return (vector float)__scalar;
}
#endif

static inline __ATTRS_o_ai vector double
vec_splats(double __scalar) {
  return (vector double)__scalar;
}

/*-- vec_extend_s64 ---------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed long long
vec_extend_s64(vector signed char __a) {
  return (vector signed long long)(__a[7], __a[15]);
}

static inline __ATTRS_o_ai vector signed long long
vec_extend_s64(vector signed short __a) {
  return (vector signed long long)(__a[3], __a[7]);
}

static inline __ATTRS_o_ai vector signed long long
vec_extend_s64(vector signed int __a) {
  return (vector signed long long)(__a[1], __a[3]);
}

/*-- vec_mergeh -------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_mergeh(vector signed char __a, vector signed char __b) {
  return (vector signed char)(
    __a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3],
    __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7]);
}

static inline __ATTRS_o_ai vector bool char
vec_mergeh(vector bool char __a, vector bool char __b) {
  return (vector bool char)(
    __a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3],
    __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7]);
}

static inline __ATTRS_o_ai vector unsigned char
vec_mergeh(vector unsigned char __a, vector unsigned char __b) {
  return (vector unsigned char)(
    __a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3],
    __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7]);
}

static inline __ATTRS_o_ai vector signed short
vec_mergeh(vector signed short __a, vector signed short __b) {
  return (vector signed short)(
    __a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3]);
}

static inline __ATTRS_o_ai vector bool short
vec_mergeh(vector bool short __a, vector bool short __b) {
  return (vector bool short)(
    __a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3]);
}

static inline __ATTRS_o_ai vector unsigned short
vec_mergeh(vector unsigned short __a, vector unsigned short __b) {
  return (vector unsigned short)(
    __a[0], __b[0], __a[1], __b[1], __a[2], __b[2], __a[3], __b[3]);
}

static inline __ATTRS_o_ai vector signed int
vec_mergeh(vector signed int __a, vector signed int __b) {
  return (vector signed int)(__a[0], __b[0], __a[1], __b[1]);
}

static inline __ATTRS_o_ai vector bool int
vec_mergeh(vector bool int __a, vector bool int __b) {
  return (vector bool int)(__a[0], __b[0], __a[1], __b[1]);
}

static inline __ATTRS_o_ai vector unsigned int
vec_mergeh(vector unsigned int __a, vector unsigned int __b) {
  return (vector unsigned int)(__a[0], __b[0], __a[1], __b[1]);
}

static inline __ATTRS_o_ai vector signed long long
vec_mergeh(vector signed long long __a, vector signed long long __b) {
  return (vector signed long long)(__a[0], __b[0]);
}

static inline __ATTRS_o_ai vector bool long long
vec_mergeh(vector bool long long __a, vector bool long long __b) {
  return (vector bool long long)(__a[0], __b[0]);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_mergeh(vector unsigned long long __a, vector unsigned long long __b) {
  return (vector unsigned long long)(__a[0], __b[0]);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_mergeh(vector float __a, vector float __b) {
  return (vector float)(__a[0], __b[0], __a[1], __b[1]);
}
#endif

static inline __ATTRS_o_ai vector double
vec_mergeh(vector double __a, vector double __b) {
  return (vector double)(__a[0], __b[0]);
}

/*-- vec_mergel -------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_mergel(vector signed char __a, vector signed char __b) {
  return (vector signed char)(
    __a[8], __b[8], __a[9], __b[9], __a[10], __b[10], __a[11], __b[11],
    __a[12], __b[12], __a[13], __b[13], __a[14], __b[14], __a[15], __b[15]);
}

static inline __ATTRS_o_ai vector bool char
vec_mergel(vector bool char __a, vector bool char __b) {
  return (vector bool char)(
    __a[8], __b[8], __a[9], __b[9], __a[10], __b[10], __a[11], __b[11],
    __a[12], __b[12], __a[13], __b[13], __a[14], __b[14], __a[15], __b[15]);
}

static inline __ATTRS_o_ai vector unsigned char
vec_mergel(vector unsigned char __a, vector unsigned char __b) {
  return (vector unsigned char)(
    __a[8], __b[8], __a[9], __b[9], __a[10], __b[10], __a[11], __b[11],
    __a[12], __b[12], __a[13], __b[13], __a[14], __b[14], __a[15], __b[15]);
}

static inline __ATTRS_o_ai vector signed short
vec_mergel(vector signed short __a, vector signed short __b) {
  return (vector signed short)(
    __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7]);
}

static inline __ATTRS_o_ai vector bool short
vec_mergel(vector bool short __a, vector bool short __b) {
  return (vector bool short)(
    __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7]);
}

static inline __ATTRS_o_ai vector unsigned short
vec_mergel(vector unsigned short __a, vector unsigned short __b) {
  return (vector unsigned short)(
    __a[4], __b[4], __a[5], __b[5], __a[6], __b[6], __a[7], __b[7]);
}

static inline __ATTRS_o_ai vector signed int
vec_mergel(vector signed int __a, vector signed int __b) {
  return (vector signed int)(__a[2], __b[2], __a[3], __b[3]);
}

static inline __ATTRS_o_ai vector bool int
vec_mergel(vector bool int __a, vector bool int __b) {
  return (vector bool int)(__a[2], __b[2], __a[3], __b[3]);
}

static inline __ATTRS_o_ai vector unsigned int
vec_mergel(vector unsigned int __a, vector unsigned int __b) {
  return (vector unsigned int)(__a[2], __b[2], __a[3], __b[3]);
}

static inline __ATTRS_o_ai vector signed long long
vec_mergel(vector signed long long __a, vector signed long long __b) {
  return (vector signed long long)(__a[1], __b[1]);
}

static inline __ATTRS_o_ai vector bool long long
vec_mergel(vector bool long long __a, vector bool long long __b) {
  return (vector bool long long)(__a[1], __b[1]);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_mergel(vector unsigned long long __a, vector unsigned long long __b) {
  return (vector unsigned long long)(__a[1], __b[1]);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector float
vec_mergel(vector float __a, vector float __b) {
  return (vector float)(__a[2], __b[2], __a[3], __b[3]);
}
#endif

static inline __ATTRS_o_ai vector double
vec_mergel(vector double __a, vector double __b) {
  return (vector double)(__a[1], __b[1]);
}

/*-- vec_pack ---------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_pack(vector signed short __a, vector signed short __b) {
  vector signed char __ac = (vector signed char)__a;
  vector signed char __bc = (vector signed char)__b;
  return (vector signed char)(
    __ac[1], __ac[3], __ac[5], __ac[7], __ac[9], __ac[11], __ac[13], __ac[15],
    __bc[1], __bc[3], __bc[5], __bc[7], __bc[9], __bc[11], __bc[13], __bc[15]);
}

static inline __ATTRS_o_ai vector bool char
vec_pack(vector bool short __a, vector bool short __b) {
  vector bool char __ac = (vector bool char)__a;
  vector bool char __bc = (vector bool char)__b;
  return (vector bool char)(
    __ac[1], __ac[3], __ac[5], __ac[7], __ac[9], __ac[11], __ac[13], __ac[15],
    __bc[1], __bc[3], __bc[5], __bc[7], __bc[9], __bc[11], __bc[13], __bc[15]);
}

static inline __ATTRS_o_ai vector unsigned char
vec_pack(vector unsigned short __a, vector unsigned short __b) {
  vector unsigned char __ac = (vector unsigned char)__a;
  vector unsigned char __bc = (vector unsigned char)__b;
  return (vector unsigned char)(
    __ac[1], __ac[3], __ac[5], __ac[7], __ac[9], __ac[11], __ac[13], __ac[15],
    __bc[1], __bc[3], __bc[5], __bc[7], __bc[9], __bc[11], __bc[13], __bc[15]);
}

static inline __ATTRS_o_ai vector signed short
vec_pack(vector signed int __a, vector signed int __b) {
  vector signed short __ac = (vector signed short)__a;
  vector signed short __bc = (vector signed short)__b;
  return (vector signed short)(
    __ac[1], __ac[3], __ac[5], __ac[7],
    __bc[1], __bc[3], __bc[5], __bc[7]);
}

static inline __ATTRS_o_ai vector bool short
vec_pack(vector bool int __a, vector bool int __b) {
  vector bool short __ac = (vector bool short)__a;
  vector bool short __bc = (vector bool short)__b;
  return (vector bool short)(
    __ac[1], __ac[3], __ac[5], __ac[7],
    __bc[1], __bc[3], __bc[5], __bc[7]);
}

static inline __ATTRS_o_ai vector unsigned short
vec_pack(vector unsigned int __a, vector unsigned int __b) {
  vector unsigned short __ac = (vector unsigned short)__a;
  vector unsigned short __bc = (vector unsigned short)__b;
  return (vector unsigned short)(
    __ac[1], __ac[3], __ac[5], __ac[7],
    __bc[1], __bc[3], __bc[5], __bc[7]);
}

static inline __ATTRS_o_ai vector signed int
vec_pack(vector signed long long __a, vector signed long long __b) {
  vector signed int __ac = (vector signed int)__a;
  vector signed int __bc = (vector signed int)__b;
  return (vector signed int)(__ac[1], __ac[3], __bc[1], __bc[3]);
}

static inline __ATTRS_o_ai vector bool int
vec_pack(vector bool long long __a, vector bool long long __b) {
  vector bool int __ac = (vector bool int)__a;
  vector bool int __bc = (vector bool int)__b;
  return (vector bool int)(__ac[1], __ac[3], __bc[1], __bc[3]);
}

static inline __ATTRS_o_ai vector unsigned int
vec_pack(vector unsigned long long __a, vector unsigned long long __b) {
  vector unsigned int __ac = (vector unsigned int)__a;
  vector unsigned int __bc = (vector unsigned int)__b;
  return (vector unsigned int)(__ac[1], __ac[3], __bc[1], __bc[3]);
}

/*-- vec_packs --------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_packs(vector signed short __a, vector signed short __b) {
  return __builtin_s390_vpksh(__a, __b);
}

static inline __ATTRS_o_ai vector unsigned char
vec_packs(vector unsigned short __a, vector unsigned short __b) {
  return __builtin_s390_vpklsh(__a, __b);
}

static inline __ATTRS_o_ai vector signed short
vec_packs(vector signed int __a, vector signed int __b) {
  return __builtin_s390_vpksf(__a, __b);
}

static inline __ATTRS_o_ai vector unsigned short
vec_packs(vector unsigned int __a, vector unsigned int __b) {
  return __builtin_s390_vpklsf(__a, __b);
}

static inline __ATTRS_o_ai vector signed int
vec_packs(vector signed long long __a, vector signed long long __b) {
  return __builtin_s390_vpksg(__a, __b);
}

static inline __ATTRS_o_ai vector unsigned int
vec_packs(vector unsigned long long __a, vector unsigned long long __b) {
  return __builtin_s390_vpklsg(__a, __b);
}

/*-- vec_packs_cc -----------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed char
vec_packs_cc(vector signed short __a, vector signed short __b, int *__cc) {
  return __builtin_s390_vpkshs(__a, __b, __cc);
}

static inline __ATTRS_o_ai vector unsigned char
vec_packs_cc(vector unsigned short __a, vector unsigned short __b, int *__cc) {
  return __builtin_s390_vpklshs(__a, __b, __cc);
}

static inline __ATTRS_o_ai vector signed short
vec_packs_cc(vector signed int __a, vector signed int __b, int *__cc) {
  return __builtin_s390_vpksfs(__a, __b, __cc);
}

static inline __ATTRS_o_ai vector unsigned short
vec_packs_cc(vector unsigned int __a, vector unsigned int __b, int *__cc) {
  return __builtin_s390_vpklsfs(__a, __b, __cc);
}

static inline __ATTRS_o_ai vector signed int
vec_packs_cc(vector signed long long __a, vector signed long long __b,
             int *__cc) {
  return __builtin_s390_vpksgs(__a, __b, __cc);
}

static inline __ATTRS_o_ai vector unsigned int
vec_packs_cc(vector unsigned long long __a, vector unsigned long long __b,
             int *__cc) {
  return __builtin_s390_vpklsgs(__a, __b, __cc);
}

/*-- vec_packsu -------------------------------------------------------------*/

static inline __ATTRS_o_ai vector unsigned char
vec_packsu(vector signed short __a, vector signed short __b) {
  const vector signed short __zero = (vector signed short)0;
  return __builtin_s390_vpklsh(
    (vector unsigned short)(__a >= __zero) & (vector unsigned short)__a,
    (vector unsigned short)(__b >= __zero) & (vector unsigned short)__b);
}

static inline __ATTRS_o_ai vector unsigned char
vec_packsu(vector unsigned short __a, vector unsigned short __b) {
  return __builtin_s390_vpklsh(__a, __b);
}

static inline __ATTRS_o_ai vector unsigned short
vec_packsu(vector signed int __a, vector signed int __b) {
  const vector signed int __zero = (vector signed int)0;
  return __builtin_s390_vpklsf(
    (vector unsigned int)(__a >= __zero) & (vector unsigned int)__a,
    (vector unsigned int)(__b >= __zero) & (vector unsigned int)__b);
}

static inline __ATTRS_o_ai vector unsigned short
vec_packsu(vector unsigned int __a, vector unsigned int __b) {
  return __builtin_s390_vpklsf(__a, __b);
}

static inline __ATTRS_o_ai vector unsigned int
vec_packsu(vector signed long long __a, vector signed long long __b) {
  const vector signed long long __zero = (vector signed long long)0;
  return __builtin_s390_vpklsg(
    (vector unsigned long long)(__a >= __zero) &
    (vector unsigned long long)__a,
    (vector unsigned long long)(__b >= __zero) &
    (vector unsigned long long)__b);
}

static inline __ATTRS_o_ai vector unsigned int
vec_packsu(vector unsigned long long __a, vector unsigned long long __b) {
  return __builtin_s390_vpklsg(__a, __b);
}

/*-- vec_packsu_cc ----------------------------------------------------------*/

static inline __ATTRS_o_ai vector unsigned char
vec_packsu_cc(vector unsigned short __a, vector unsigned short __b, int *__cc) {
  return __builtin_s390_vpklshs(__a, __b, __cc);
}

static inline __ATTRS_o_ai vector unsigned short
vec_packsu_cc(vector unsigned int __a, vector unsigned int __b, int *__cc) {
  return __builtin_s390_vpklsfs(__a, __b, __cc);
}

static inline __ATTRS_o_ai vector unsigned int
vec_packsu_cc(vector unsigned long long __a, vector unsigned long long __b,
              int *__cc) {
  return __builtin_s390_vpklsgs(__a, __b, __cc);
}

/*-- vec_unpackh ------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed short
vec_unpackh(vector signed char __a) {
  return __builtin_s390_vuphb(__a);
}

static inline __ATTRS_o_ai vector bool short
vec_unpackh(vector bool char __a) {
  return (vector bool short)__builtin_s390_vuphb((vector signed char)__a);
}

static inline __ATTRS_o_ai vector unsigned short
vec_unpackh(vector unsigned char __a) {
  return __builtin_s390_vuplhb(__a);
}

static inline __ATTRS_o_ai vector signed int
vec_unpackh(vector signed short __a) {
  return __builtin_s390_vuphh(__a);
}

static inline __ATTRS_o_ai vector bool int
vec_unpackh(vector bool short __a) {
  return (vector bool int)__builtin_s390_vuphh((vector signed short)__a);
}

static inline __ATTRS_o_ai vector unsigned int
vec_unpackh(vector unsigned short __a) {
  return __builtin_s390_vuplhh(__a);
}

static inline __ATTRS_o_ai vector signed long long
vec_unpackh(vector signed int __a) {
  return __builtin_s390_vuphf(__a);
}

static inline __ATTRS_o_ai vector bool long long
vec_unpackh(vector bool int __a) {
  return (vector bool long long)__builtin_s390_vuphf((vector signed int)__a);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_unpackh(vector unsigned int __a) {
  return __builtin_s390_vuplhf(__a);
}

/*-- vec_unpackl ------------------------------------------------------------*/

static inline __ATTRS_o_ai vector signed short
vec_unpackl(vector signed char __a) {
  return __builtin_s390_vuplb(__a);
}

static inline __ATTRS_o_ai vector bool short
vec_unpackl(vector bool char __a) {
  return (vector bool short)__builtin_s390_vuplb((vector signed char)__a);
}

static inline __ATTRS_o_ai vector unsigned short
vec_unpackl(vector unsigned char __a) {
  return __builtin_s390_vupllb(__a);
}

static inline __ATTRS_o_ai vector signed int
vec_unpackl(vector signed short __a) {
  return __builtin_s390_vuplhw(__a);
}

static inline __ATTRS_o_ai vector bool int
vec_unpackl(vector bool short __a) {
  return (vector bool int)__builtin_s390_vuplhw((vector signed short)__a);
}

static inline __ATTRS_o_ai vector unsigned int
vec_unpackl(vector unsigned short __a) {
  return __builtin_s390_vupllh(__a);
}

static inline __ATTRS_o_ai vector signed long long
vec_unpackl(vector signed int __a) {
  return __builtin_s390_vuplf(__a);
}

static inline __ATTRS_o_ai vector bool long long
vec_unpackl(vector bool int __a) {
  return (vector bool long long)__builtin_s390_vuplf((vector signed int)__a);
}

static inline __ATTRS_o_ai vector unsigned long long
vec_unpackl(vector unsigned int __a) {
  return __builtin_s390_vupllf(__a);
}

/*-- vec_cmpeq --------------------------------------------------------------*/

static inline __ATTRS_o_ai vector bool char
vec_cmpeq(vector bool char __a, vector bool char __b) {
  return (vector bool char)(__a == __b);
}

static inline __ATTRS_o_ai vector bool char
vec_cmpeq(vector signed char __a, vector signed char __b) {
  return (vector bool char)(__a == __b);
}

static inline __ATTRS_o_ai vector bool char
vec_cmpeq(vector unsigned char __a, vector unsigned char __b) {
  return (vector bool char)(__a == __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmpeq(vector bool short __a, vector bool short __b) {
  return (vector bool short)(__a == __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmpeq(vector signed short __a, vector signed short __b) {
  return (vector bool short)(__a == __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmpeq(vector unsigned short __a, vector unsigned short __b) {
  return (vector bool short)(__a == __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmpeq(vector bool int __a, vector bool int __b) {
  return (vector bool int)(__a == __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmpeq(vector signed int __a, vector signed int __b) {
  return (vector bool int)(__a == __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmpeq(vector unsigned int __a, vector unsigned int __b) {
  return (vector bool int)(__a == __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmpeq(vector bool long long __a, vector bool long long __b) {
  return (vector bool long long)(__a == __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmpeq(vector signed long long __a, vector signed long long __b) {
  return (vector bool long long)(__a == __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmpeq(vector unsigned long long __a, vector unsigned long long __b) {
  return (vector bool long long)(__a == __b);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector bool int
vec_cmpeq(vector float __a, vector float __b) {
  return (vector bool int)(__a == __b);
}
#endif

static inline __ATTRS_o_ai vector bool long long
vec_cmpeq(vector double __a, vector double __b) {
  return (vector bool long long)(__a == __b);
}

/*-- vec_cmpge --------------------------------------------------------------*/

static inline __ATTRS_o_ai vector bool char
vec_cmpge(vector signed char __a, vector signed char __b) {
  return (vector bool char)(__a >= __b);
}

static inline __ATTRS_o_ai vector bool char
vec_cmpge(vector unsigned char __a, vector unsigned char __b) {
  return (vector bool char)(__a >= __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmpge(vector signed short __a, vector signed short __b) {
  return (vector bool short)(__a >= __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmpge(vector unsigned short __a, vector unsigned short __b) {
  return (vector bool short)(__a >= __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmpge(vector signed int __a, vector signed int __b) {
  return (vector bool int)(__a >= __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmpge(vector unsigned int __a, vector unsigned int __b) {
  return (vector bool int)(__a >= __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmpge(vector signed long long __a, vector signed long long __b) {
  return (vector bool long long)(__a >= __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmpge(vector unsigned long long __a, vector unsigned long long __b) {
  return (vector bool long long)(__a >= __b);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector bool int
vec_cmpge(vector float __a, vector float __b) {
  return (vector bool int)(__a >= __b);
}
#endif

static inline __ATTRS_o_ai vector bool long long
vec_cmpge(vector double __a, vector double __b) {
  return (vector bool long long)(__a >= __b);
}

/*-- vec_cmpgt --------------------------------------------------------------*/

static inline __ATTRS_o_ai vector bool char
vec_cmpgt(vector signed char __a, vector signed char __b) {
  return (vector bool char)(__a > __b);
}

static inline __ATTRS_o_ai vector bool char
vec_cmpgt(vector unsigned char __a, vector unsigned char __b) {
  return (vector bool char)(__a > __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmpgt(vector signed short __a, vector signed short __b) {
  return (vector bool short)(__a > __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmpgt(vector unsigned short __a, vector unsigned short __b) {
  return (vector bool short)(__a > __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmpgt(vector signed int __a, vector signed int __b) {
  return (vector bool int)(__a > __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmpgt(vector unsigned int __a, vector unsigned int __b) {
  return (vector bool int)(__a > __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmpgt(vector signed long long __a, vector signed long long __b) {
  return (vector bool long long)(__a > __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmpgt(vector unsigned long long __a, vector unsigned long long __b) {
  return (vector bool long long)(__a > __b);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector bool int
vec_cmpgt(vector float __a, vector float __b) {
  return (vector bool int)(__a > __b);
}
#endif

static inline __ATTRS_o_ai vector bool long long
vec_cmpgt(vector double __a, vector double __b) {
  return (vector bool long long)(__a > __b);
}

/*-- vec_cmple --------------------------------------------------------------*/

static inline __ATTRS_o_ai vector bool char
vec_cmple(vector signed char __a, vector signed char __b) {
  return (vector bool char)(__a <= __b);
}

static inline __ATTRS_o_ai vector bool char
vec_cmple(vector unsigned char __a, vector unsigned char __b) {
  return (vector bool char)(__a <= __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmple(vector signed short __a, vector signed short __b) {
  return (vector bool short)(__a <= __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmple(vector unsigned short __a, vector unsigned short __b) {
  return (vector bool short)(__a <= __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmple(vector signed int __a, vector signed int __b) {
  return (vector bool int)(__a <= __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmple(vector unsigned int __a, vector unsigned int __b) {
  return (vector bool int)(__a <= __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmple(vector signed long long __a, vector signed long long __b) {
  return (vector bool long long)(__a <= __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmple(vector unsigned long long __a, vector unsigned long long __b) {
  return (vector bool long long)(__a <= __b);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector bool int
vec_cmple(vector float __a, vector float __b) {
  return (vector bool int)(__a <= __b);
}
#endif

static inline __ATTRS_o_ai vector bool long long
vec_cmple(vector double __a, vector double __b) {
  return (vector bool long long)(__a <= __b);
}

/*-- vec_cmplt --------------------------------------------------------------*/

static inline __ATTRS_o_ai vector bool char
vec_cmplt(vector signed char __a, vector signed char __b) {
  return (vector bool char)(__a < __b);
}

static inline __ATTRS_o_ai vector bool char
vec_cmplt(vector unsigned char __a, vector unsigned char __b) {
  return (vector bool char)(__a < __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmplt(vector signed short __a, vector signed short __b) {
  return (vector bool short)(__a < __b);
}

static inline __ATTRS_o_ai vector bool short
vec_cmplt(vector unsigned short __a, vector unsigned short __b) {
  return (vector bool short)(__a < __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmplt(vector signed int __a, vector signed int __b) {
  return (vector bool int)(__a < __b);
}

static inline __ATTRS_o_ai vector bool int
vec_cmplt(vector unsigned int __a, vector unsigned int __b) {
  return (vector bool int)(__a < __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmplt(vector signed long long __a, vector signed long long __b) {
  return (vector bool long long)(__a < __b);
}

static inline __ATTRS_o_ai vector bool long long
vec_cmplt(vector unsigned long long __a, vector unsigned long long __b) {
  return (vector bool long long)(__a < __b);
}

#if __ARCH__ >= 12
static inline __ATTRS_o_ai vector bool int
vec_cmplt(vector float __a, vector float __b) {
  return (vector bool int)(__a < __b);
}
#endif

static inline __ATTRS_o_ai vector bool long long
vec_cmplt(vector double __a, vector double __b) {
  return (vector bool long long)(__a < __b);
}

/*-- vec_all_eq -------------------------------------------------------------*/

static inline __ATTRS_o_ai int
vec_all_eq(vector signed char __a, vector signed char __b) {
  int __cc;
  __builtin_s390_vceqbs(__a, __b, &__cc);
  return __cc == 0;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai int
vec_all_eq(vector signed char __a, vector bool char __b) {
  int __cc;
  __builtin_s390_vceqbs(__a, (vector signed char)__b, &__cc);
  return __cc == 0;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai int
vec_all_eq(vector bool char __a, vector signed char __b) {
  int __cc;
  __builtin_s390_vceqbs((vector signed char)__a, __b, &__cc);
  return __cc == 0;
}

static inline __ATTRS_o_ai int
vec_all_eq(vector unsigned char __a, vector unsigned char __b) {
  int __cc;
  __builtin_s390_vceqbs((vector signed char)__a,
                        (vector signed char)__b, &__cc);
  return __cc == 0;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai int
vec_all_eq(vector unsigned char __a, vector bool char __b) {
  int __cc;
  __builtin_s390_vceqbs((vector signed char)__a,
                        (vector signed char)__b, &__cc);
  return __cc == 0;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai int
vec_all_eq(vector bool char __a, vector unsigned char __b) {
  int __cc;
  __builtin_s390_vceqbs((vector signed char)__a,
                        (vector signed char)__b, &__cc);
  return __cc == 0;
}

static inline __ATTRS_o_ai int
vec_all_eq(vector bool char __a, vector bool char __b) {
  int __cc;
  __builtin_s390_vceqbs((vector signed char)__a,
                        (vector signed char)__b, &__cc);
  return __cc == 0;
}

static inline __ATTRS_o_ai int
vec_all_eq(vector signed short __a, vector signed short __b) {
  int __cc;
  __builtin_s390_vceqhs(__a, __b, &__cc);
  return __cc == 0;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai int
vec_all_eq(vector signed short __a, vector bool short __b) {
  int __cc;
  __builtin_s390_vceqhs(__a, (vector signed short)__b, &__cc);
  return __cc == 0;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai int
vec_all_eq(vector bool short __a, vector signed short __b) {
  int __cc;
  __builtin_s390_vceqhs((vector signed short)__a, __b, &__cc);
  return __cc == 0;
}

static inline __ATTRS_o_ai int
vec_all_eq(vector unsigned short __a, vector unsigned short __b) {
  int __cc;
  __builtin_s390_vceqhs((vector signed short)__a,
                        (vector signed short)__b, &__cc);
  return __cc == 0;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai int
vec_all_eq(vector unsigned short __a, vector bool short __b) {
  int __cc;
  __builtin_s390_vceqhs((vector signed short)__a,
                        (vector signed short)__b, &__cc);
  return __cc == 0;
}

// This prototype is deprecated.
static inline __ATTRS_o_ai int
vec_all_eq(vector bool short __a, vector unsigned short __b) {
  int __cc;
  __builtin_s390_vceqhs((vector signed short)__a,
                        (vector signed short)__b, &__cc);
  return __cc == 0;
}

static inline __ATTRS_o_ai int
vec_all_eq(vector bool short __a,