update C language headers to LLVM 17

release/17.x branch, commit 8f4dd44097c9ae25dd203d5ac87f3b48f854bba8

lib/include/__clang_cuda_intrinsics.h

@@ -513,6 +513,197 @@ __device__ inline cuuint32_t __nvvm_get_smem_pointer(void *__ptr) {
   return __nv_cvta_generic_to_shared_impl(__ptr);
 }
 } // extern "C"
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
+__device__ inline unsigned __reduce_add_sync(unsigned __mask,
+                                             unsigned __value) {
+  return __nvvm_redux_sync_add(__mask, __value);
+}
+__device__ inline unsigned __reduce_min_sync(unsigned __mask,
+                                             unsigned __value) {
+  return __nvvm_redux_sync_umin(__mask, __value);
+}
+__device__ inline unsigned __reduce_max_sync(unsigned __mask,
+                                             unsigned __value) {
+  return __nvvm_redux_sync_umax(__mask, __value);
+}
+__device__ inline int __reduce_min_sync(unsigned __mask, int __value) {
+  return __nvvm_redux_sync_min(__mask, __value);
+}
+__device__ inline int __reduce_max_sync(unsigned __mask, int __value) {
+  return __nvvm_redux_sync_max(__mask, __value);
+}
+__device__ inline unsigned __reduce_or_sync(unsigned __mask, unsigned __value) {
+  return __nvvm_redux_sync_or(__mask, __value);
+}
+__device__ inline unsigned __reduce_and_sync(unsigned __mask,
+                                             unsigned __value) {
+  return __nvvm_redux_sync_and(__mask, __value);
+}
+__device__ inline unsigned __reduce_xor_sync(unsigned __mask,
+                                             unsigned __value) {
+  return __nvvm_redux_sync_xor(__mask, __value);
+}
+
+__device__ inline void __nv_memcpy_async_shared_global_4(void *__dst,
+                                                         const void *__src,
+                                                         unsigned __src_size) {
+  __nvvm_cp_async_ca_shared_global_4(
+      (void __attribute__((address_space(3))) *)__dst,
+      (const void __attribute__((address_space(1))) *)__src, __src_size);
+}
+__device__ inline void __nv_memcpy_async_shared_global_8(void *__dst,
+                                                         const void *__src,
+                                                         unsigned __src_size) {
+  __nvvm_cp_async_ca_shared_global_8(
+      (void __attribute__((address_space(3))) *)__dst,
+      (const void __attribute__((address_space(1))) *)__src, __src_size);
+}
+__device__ inline void __nv_memcpy_async_shared_global_16(void *__dst,
+                                                          const void *__src,
+                                                          unsigned __src_size) {
+  __nvvm_cp_async_ca_shared_global_16(
+      (void __attribute__((address_space(3))) *)__dst,
+      (const void __attribute__((address_space(1))) *)__src, __src_size);
+}
+
+__device__ inline void *
+__nv_associate_access_property(const void *__ptr, unsigned long long __prop) {
+  // TODO: it appears to provide compiler with some sort of a hint. We do not
+  // know what exactly it is supposed to do. However, CUDA headers suggest that
+  // just passing through __ptr should not affect correctness. They do so on
+  // pre-sm80 GPUs where this builtin is not available.
+  return (void*)__ptr;
+}
+#endif // !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800
+
+#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 900
+__device__ inline unsigned __isCtaShared(const void *ptr) {
+  return __isShared(ptr);
+}
+
+__device__ inline unsigned __isClusterShared(const void *__ptr) {
+  return __nvvm_isspacep_shared_cluster(__ptr);
+}
+
+__device__ inline void *__cluster_map_shared_rank(const void *__ptr,
+                                                  unsigned __rank) {
+  return __nvvm_mapa((void *)__ptr, __rank);
+}
+
+__device__ inline unsigned __cluster_query_shared_rank(const void *__ptr) {
+  return __nvvm_getctarank((void *)__ptr);
+}
+
+__device__ inline uint2
+__cluster_map_shared_multicast(const void *__ptr,
+                               unsigned int __cluster_cta_mask) {
+  return make_uint2((unsigned)__cvta_generic_to_shared(__ptr),
+                    __cluster_cta_mask);
+}
+
+__device__ inline unsigned __clusterDimIsSpecified() {
+  return __nvvm_is_explicit_cluster();
+}
+
+__device__ inline dim3 __clusterDim() {
+  return dim3(__nvvm_read_ptx_sreg_cluster_nctaid_x(),
+              __nvvm_read_ptx_sreg_cluster_nctaid_y(),
+              __nvvm_read_ptx_sreg_cluster_nctaid_z());
+}
+
+__device__ inline dim3 __clusterRelativeBlockIdx() {
+  return dim3(__nvvm_read_ptx_sreg_cluster_ctaid_x(),
+              __nvvm_read_ptx_sreg_cluster_ctaid_y(),
+              __nvvm_read_ptx_sreg_cluster_ctaid_z());
+}
+
+__device__ inline dim3 __clusterGridDimInClusters() {
+  return dim3(__nvvm_read_ptx_sreg_nclusterid_x(),
+              __nvvm_read_ptx_sreg_nclusterid_y(),
+              __nvvm_read_ptx_sreg_nclusterid_z());
+}
+
+__device__ inline dim3 __clusterIdx() {
+  return dim3(__nvvm_read_ptx_sreg_clusterid_x(),
+              __nvvm_read_ptx_sreg_clusterid_y(),
+              __nvvm_read_ptx_sreg_clusterid_z());
+}
+
+__device__ inline unsigned __clusterRelativeBlockRank() {
+  return __nvvm_read_ptx_sreg_cluster_ctarank();
+}
+
+__device__ inline unsigned __clusterSizeInBlocks() {
+  return __nvvm_read_ptx_sreg_cluster_nctarank();
+}
+
+__device__ inline void __cluster_barrier_arrive() {
+  __nvvm_barrier_cluster_arrive();
+}
+
+__device__ inline void __cluster_barrier_arrive_relaxed() {
+  __nvvm_barrier_cluster_arrive_relaxed();
+}
+
+__device__ inline void __cluster_barrier_wait() {
+  __nvvm_barrier_cluster_wait();
+}
+
+__device__ inline void __threadfence_cluster() { __nvvm_fence_sc_cluster(); }
+
+__device__ inline float2 atomicAdd(float2 *__ptr, float2 __val) {
+  float2 __ret;
+  __asm__("atom.add.v2.f32         {%0, %1}, [%2], {%3, %4};"
+          : "=f"(__ret.x), "=f"(__ret.y)
+          : "l"(__ptr), "f"(__val.x), "f"(__val.y));
+  return __ret;
+}
+
+__device__ inline float2 atomicAdd_block(float2 *__ptr, float2 __val) {
+  float2 __ret;
+  __asm__("atom.cta.add.v2.f32         {%0, %1}, [%2], {%3, %4};"
+          : "=f"(__ret.x), "=f"(__ret.y)
+          : "l"(__ptr), "f"(__val.x), "f"(__val.y));
+  return __ret;
+}
+
+__device__ inline float2 atomicAdd_system(float2 *__ptr, float2 __val) {
+  float2 __ret;
+  __asm__("atom.sys.add.v2.f32         {%0, %1}, [%2], {%3, %4};"
+          : "=f"(__ret.x), "=f"(__ret.y)
+          : "l"(__ptr), "f"(__val.x), "f"(__val.y));
+  return __ret;
+}
+
+__device__ inline float4 atomicAdd(float4 *__ptr, float4 __val) {
+  float4 __ret;
+  __asm__("atom.add.v4.f32         {%0, %1, %2, %3}, [%4], {%5, %6, %7, %8};"
+          : "=f"(__ret.x), "=f"(__ret.y), "=f"(__ret.z), "=f"(__ret.w)
+          : "l"(__ptr), "f"(__val.x), "f"(__val.y), "f"(__val.z), "f"(__val.w));
+  return __ret;
+}
+
+__device__ inline float4 atomicAdd_block(float4 *__ptr, float4 __val) {
+  float4 __ret;
+  __asm__(
+      "atom.cta.add.v4.f32         {%0, %1, %2, %3}, [%4], {%5, %6, %7, %8};"
+      : "=f"(__ret.x), "=f"(__ret.y), "=f"(__ret.z), "=f"(__ret.w)
+      : "l"(__ptr), "f"(__val.x), "f"(__val.y), "f"(__val.z), "f"(__val.w));
+  return __ret;
+}
+
+__device__ inline float4 atomicAdd_system(float4 *__ptr, float4 __val) {
+  float4 __ret;
+  __asm__(
+      "atom.sys.add.v4.f32         {%0, %1, %2, %3}, [%4], {%5, %6, %7, %8};"
+      : "=f"(__ret.x), "=f"(__ret.y), "=f"(__ret.z), "=f"(__ret.w)
+      : "l"(__ptr), "f"(__val.x), "f"(__val.y), "f"(__val.z), "f"(__val.w)
+      :);
+  return __ret;
+}
+
+#endif // !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 900
 #endif // CUDA_VERSION >= 11000
 
 #endif // defined(__CLANG_CUDA_INTRINSICS_H__)

lib/include/__clang_hip_cmath.h

@@ -171,7 +171,7 @@ __DEVICE__ __CONSTEXPR__ bool signbit(double __x) { return ::__signbit(__x); }
 // Other functions.
 __DEVICE__ __CONSTEXPR__ _Float16 fma(_Float16 __x, _Float16 __y,
                                       _Float16 __z) {
-  return __ocml_fma_f16(__x, __y, __z);
+  return __builtin_fmaf16(__x, __y, __z);
 }
 __DEVICE__ __CONSTEXPR__ _Float16 pow(_Float16 __base, int __iexp) {
   return __ocml_pown_f16(__base, __iexp);

lib/include/__clang_hip_libdevice_declares.h

@@ -10,6 +10,10 @@
 #ifndef __CLANG_HIP_LIBDEVICE_DECLARES_H__
 #define __CLANG_HIP_LIBDEVICE_DECLARES_H__
 
+#if !defined(__HIPCC_RTC__) && __has_include("hip/hip_version.h")
+#include "hip/hip_version.h"
+#endif // __has_include("hip/hip_version.h")
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -137,23 +141,6 @@ __device__ __attribute__((const)) float __ocml_fma_rte_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_fma_rtn_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_fma_rtp_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_fma_rtz_f32(float, float, float);
-
-__device__ inline __attribute__((const)) float
-__llvm_amdgcn_cos_f32(float __x) {
-  return __builtin_amdgcn_cosf(__x);
-}
-__device__ inline __attribute__((const)) float
-__llvm_amdgcn_rcp_f32(float __x) {
-  return __builtin_amdgcn_rcpf(__x);
-}
-__device__ inline __attribute__((const)) float
-__llvm_amdgcn_rsq_f32(float __x) {
-  return __builtin_amdgcn_rsqf(__x);
-}
-__device__ inline __attribute__((const)) float
-__llvm_amdgcn_sin_f32(float __x) {
-  return __builtin_amdgcn_sinf(__x);
-}
 // END INTRINSICS
 // END FLOAT
 
@@ -277,15 +264,6 @@ __device__ __attribute__((const)) double __ocml_fma_rtp_f64(double, double,
 __device__ __attribute__((const)) double __ocml_fma_rtz_f64(double, double,
                                                             double);
 
-__device__ inline __attribute__((const)) double
-__llvm_amdgcn_rcp_f64(double __x) {
-  return __builtin_amdgcn_rcp(__x);
-}
-__device__ inline __attribute__((const)) double
-__llvm_amdgcn_rsq_f64(double __x) {
-  return __builtin_amdgcn_rsq(__x);
-}
-
 __device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16);
 __device__ _Float16 __ocml_cos_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
@@ -305,7 +283,6 @@ __device__ __attribute__((const)) int __ocml_isnan_f16(_Float16);
 __device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16);
 __device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16);
 __device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16);
-__device__ __attribute__((const)) _Float16 __llvm_amdgcn_rcp_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16);
 __device__ _Float16 __ocml_sin_f16(_Float16);
@@ -316,8 +293,15 @@ __device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int);
 typedef _Float16 __2f16 __attribute__((ext_vector_type(2)));
 typedef short __2i16 __attribute__((ext_vector_type(2)));
 
+// We need to match C99's bool and get an i1 in the IR.
+#ifdef __cplusplus
+typedef bool __ockl_bool;
+#else
+typedef _Bool __ockl_bool;
+#endif
+
 __device__ __attribute__((const)) float __ockl_fdot2(__2f16 a, __2f16 b,
-                                                     float c, bool s);
+                                                     float c, __ockl_bool s);
 __device__ __attribute__((const)) __2f16 __ocml_ceil_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_fabs_2f16(__2f16);
 __device__ __2f16 __ocml_cos_2f16(__2f16);
@@ -332,11 +316,29 @@ __device__ __attribute__((const)) __2i16 __ocml_isnan_2f16(__2f16);
 __device__ __attribute__((pure)) __2f16 __ocml_log_2f16(__2f16);
 __device__ __attribute__((pure)) __2f16 __ocml_log10_2f16(__2f16);
 __device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16);
-__device__ inline __2f16
+
-__llvm_amdgcn_rcp_2f16(__2f16 __x) // Not currently exposed by ROCDL.
+#if HIP_VERSION_MAJOR * 100 + HIP_VERSION_MINOR >= 560
-{
+#define __DEPRECATED_SINCE_HIP_560(X) __attribute__((deprecated(X)))
-  return (__2f16)(__llvm_amdgcn_rcp_f16(__x.x), __llvm_amdgcn_rcp_f16(__x.y));
+#else
+#define __DEPRECATED_SINCE_HIP_560(X)
+#endif
+
+// Deprecated, should be removed when rocm releases using it are no longer
+// relevant.
+__DEPRECATED_SINCE_HIP_560("use ((_Float16)1.0) / ")
+__device__ inline _Float16 __llvm_amdgcn_rcp_f16(_Float16 x) {
+  return ((_Float16)1.0f) / x;
 }
+
+__DEPRECATED_SINCE_HIP_560("use ((__2f16)1.0) / ")
+__device__ inline __2f16
+__llvm_amdgcn_rcp_2f16(__2f16 __x)
+{
+  return ((__2f16)1.0f) / __x;
+}
+
+#undef __DEPRECATED_SINCE_HIP_560
+
 __device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16);
 __device__ __2f16 __ocml_sin_2f16(__2f16);

lib/include/__clang_hip_math.h

@@ -182,10 +182,10 @@ __DEVICE__
 float cbrtf(float __x) { return __ocml_cbrt_f32(__x); }
 
 __DEVICE__
-float ceilf(float __x) { return __ocml_ceil_f32(__x); }
+float ceilf(float __x) { return __builtin_ceilf(__x); }
 
 __DEVICE__
-float copysignf(float __x, float __y) { return __ocml_copysign_f32(__x, __y); }
+float copysignf(float __x, float __y) { return __builtin_copysignf(__x, __y); }
 
 __DEVICE__
 float cosf(float __x) { return __ocml_cos_f32(__x); }
@@ -221,10 +221,10 @@ __DEVICE__
 float exp10f(float __x) { return __ocml_exp10_f32(__x); }
 
 __DEVICE__
-float exp2f(float __x) { return __ocml_exp2_f32(__x); }
+float exp2f(float __x) { return __builtin_exp2f(__x); }
 
 __DEVICE__
-float expf(float __x) { return __ocml_exp_f32(__x); }
+float expf(float __x) { return __builtin_expf(__x); }
 
 __DEVICE__
 float expm1f(float __x) { return __ocml_expm1_f32(__x); }
@@ -239,33 +239,25 @@ __DEVICE__
 float fdividef(float __x, float __y) { return __x / __y; }
 
 __DEVICE__
-float floorf(float __x) { return __ocml_floor_f32(__x); }
+float floorf(float __x) { return __builtin_floorf(__x); }
 
 __DEVICE__
 float fmaf(float __x, float __y, float __z) {
-  return __ocml_fma_f32(__x, __y, __z);
+  return __builtin_fmaf(__x, __y, __z);
 }
 
 __DEVICE__
-float fmaxf(float __x, float __y) { return __ocml_fmax_f32(__x, __y); }
+float fmaxf(float __x, float __y) { return __builtin_fmaxf(__x, __y); }
 
 __DEVICE__
-float fminf(float __x, float __y) { return __ocml_fmin_f32(__x, __y); }
+float fminf(float __x, float __y) { return __builtin_fminf(__x, __y); }
 
 __DEVICE__
 float fmodf(float __x, float __y) { return __ocml_fmod_f32(__x, __y); }
 
 __DEVICE__
 float frexpf(float __x, int *__nptr) {
-  int __tmp;
+  return __builtin_frexpf(__x, __nptr);
-#ifdef __OPENMP_AMDGCN__
-#pragma omp allocate(__tmp) allocator(omp_thread_mem_alloc)
-#endif
-  float __r =
-      __ocml_frexp_f32(__x, (__attribute__((address_space(5))) int *)&__tmp);
-  *__nptr = __tmp;
-
-  return __r;
 }
 
 __DEVICE__
@@ -275,13 +267,13 @@ __DEVICE__
 int ilogbf(float __x) { return __ocml_ilogb_f32(__x); }
 
 __DEVICE__
-__RETURN_TYPE __finitef(float __x) { return __ocml_isfinite_f32(__x); }
+__RETURN_TYPE __finitef(float __x) { return __builtin_isfinite(__x); }
 
 __DEVICE__
-__RETURN_TYPE __isinff(float __x) { return __ocml_isinf_f32(__x); }
+__RETURN_TYPE __isinff(float __x) { return __builtin_isinf(__x); }
 
 __DEVICE__
-__RETURN_TYPE __isnanf(float __x) { return __ocml_isnan_f32(__x); }
+__RETURN_TYPE __isnanf(float __x) { return __builtin_isnan(__x); }
 
 __DEVICE__
 float j0f(float __x) { return __ocml_j0_f32(__x); }
@@ -311,37 +303,37 @@ float jnf(int __n, float __x) { // TODO: we could use Ahmes multiplication
 }
 
 __DEVICE__
-float ldexpf(float __x, int __e) { return __ocml_ldexp_f32(__x, __e); }
+float ldexpf(float __x, int __e) { return __builtin_amdgcn_ldexpf(__x, __e); }
 
 __DEVICE__
 float lgammaf(float __x) { return __ocml_lgamma_f32(__x); }
 
 __DEVICE__
-long long int llrintf(float __x) { return __ocml_rint_f32(__x); }
+long long int llrintf(float __x) { return __builtin_rintf(__x); }
 
 __DEVICE__
-long long int llroundf(float __x) { return __ocml_round_f32(__x); }
+long long int llroundf(float __x) { return __builtin_roundf(__x); }
 
 __DEVICE__
-float log10f(float __x) { return __ocml_log10_f32(__x); }
+float log10f(float __x) { return __builtin_log10f(__x); }
 
 __DEVICE__
 float log1pf(float __x) { return __ocml_log1p_f32(__x); }
 
 __DEVICE__
-float log2f(float __x) { return __ocml_log2_f32(__x); }
+float log2f(float __x) { return __builtin_log2f(__x); }
 
 __DEVICE__
 float logbf(float __x) { return __ocml_logb_f32(__x); }
 
 __DEVICE__
-float logf(float __x) { return __ocml_log_f32(__x); }
+float logf(float __x) { return __builtin_logf(__x); }
 
 __DEVICE__
-long int lrintf(float __x) { return __ocml_rint_f32(__x); }
+long int lrintf(float __x) { return __builtin_rintf(__x); }
 
 __DEVICE__
-long int lroundf(float __x) { return __ocml_round_f32(__x); }
+long int lroundf(float __x) { return __builtin_roundf(__x); }
 
 __DEVICE__
 float modff(float __x, float *__iptr) {
@@ -377,7 +369,7 @@ float nanf(const char *__tagp __attribute__((nonnull))) {
 }
 
 __DEVICE__
-float nearbyintf(float __x) { return __ocml_nearbyint_f32(__x); }
+float nearbyintf(float __x) { return __builtin_nearbyintf(__x); }
 
 __DEVICE__
 float nextafterf(float __x, float __y) {
@@ -443,7 +435,7 @@ __DEVICE__
 float rhypotf(float __x, float __y) { return __ocml_rhypot_f32(__x, __y); }
 
 __DEVICE__
-float rintf(float __x) { return __ocml_rint_f32(__x); }
+float rintf(float __x) { return __builtin_rintf(__x); }
 
 __DEVICE__
 float rnorm3df(float __x, float __y, float __z) {
@@ -468,22 +460,22 @@ float rnormf(int __dim,
 }
 
 __DEVICE__
-float roundf(float __x) { return __ocml_round_f32(__x); }
+float roundf(float __x) { return __builtin_roundf(__x); }
 
 __DEVICE__
 float rsqrtf(float __x) { return __ocml_rsqrt_f32(__x); }
 
 __DEVICE__
 float scalblnf(float __x, long int __n) {
-  return (__n < INT_MAX) ? __ocml_scalbn_f32(__x, __n)
+  return (__n < INT_MAX) ? __builtin_amdgcn_ldexpf(__x, __n)
                          : __ocml_scalb_f32(__x, __n);
 }
 
 __DEVICE__
-float scalbnf(float __x, int __n) { return __ocml_scalbn_f32(__x, __n); }
+float scalbnf(float __x, int __n) { return __builtin_amdgcn_ldexpf(__x, __n); }
 
 __DEVICE__
-__RETURN_TYPE __signbitf(float __x) { return __ocml_signbit_f32(__x); }
+__RETURN_TYPE __signbitf(float __x) { return __builtin_signbitf(__x); }
 
 __DEVICE__
 void sincosf(float __x, float *__sinptr, float *__cosptr) {
@@ -529,7 +521,7 @@ __DEVICE__
 float tgammaf(float __x) { return __ocml_tgamma_f32(__x); }
 
 __DEVICE__
-float truncf(float __x) { return __ocml_trunc_f32(__x); }
+float truncf(float __x) { return __builtin_truncf(__x); }
 
 __DEVICE__
 float y0f(float __x) { return __ocml_y0_f32(__x); }
@@ -621,7 +613,7 @@ float __fmaf_rz(float __x, float __y, float __z) {
 #else
 __DEVICE__
 float __fmaf_rn(float __x, float __y, float __z) {
-  return __ocml_fma_f32(__x, __y, __z);
+  return __builtin_fmaf(__x, __y, __z);
 }
 #endif
 
@@ -654,7 +646,7 @@ float __frcp_rn(float __x) { return 1.0f / __x; }
 #endif
 
 __DEVICE__
-float __frsqrt_rn(float __x) { return __llvm_amdgcn_rsq_f32(__x); }
+float __frsqrt_rn(float __x) { return __builtin_amdgcn_rsqf(__x); }
 
 #if defined OCML_BASIC_ROUNDED_OPERATIONS
 __DEVICE__
@@ -739,11 +731,11 @@ __DEVICE__
 double cbrt(double __x) { return __ocml_cbrt_f64(__x); }
 
 __DEVICE__
-double ceil(double __x) { return __ocml_ceil_f64(__x); }
+double ceil(double __x) { return __builtin_ceil(__x); }
 
 __DEVICE__
 double copysign(double __x, double __y) {
-  return __ocml_copysign_f64(__x, __y);
+  return __builtin_copysign(__x, __y);
 }
 
 __DEVICE__
@@ -795,32 +787,25 @@ __DEVICE__
 double fdim(double __x, double __y) { return __ocml_fdim_f64(__x, __y); }
 
 __DEVICE__
-double floor(double __x) { return __ocml_floor_f64(__x); }
+double floor(double __x) { return __builtin_floor(__x); }
 
 __DEVICE__
 double fma(double __x, double __y, double __z) {
-  return __ocml_fma_f64(__x, __y, __z);
+  return __builtin_fma(__x, __y, __z);
 }
 
 __DEVICE__
-double fmax(double __x, double __y) { return __ocml_fmax_f64(__x, __y); }
+double fmax(double __x, double __y) { return __builtin_fmax(__x, __y); }
 
 __DEVICE__
-double fmin(double __x, double __y) { return __ocml_fmin_f64(__x, __y); }
+double fmin(double __x, double __y) { return __builtin_fmin(__x, __y); }
 
 __DEVICE__
 double fmod(double __x, double __y) { return __ocml_fmod_f64(__x, __y); }
 
 __DEVICE__
 double frexp(double __x, int *__nptr) {
-  int __tmp;
+  return __builtin_frexp(__x, __nptr);
-#ifdef __OPENMP_AMDGCN__
-#pragma omp allocate(__tmp) allocator(omp_thread_mem_alloc)
-#endif
-  double __r =
-      __ocml_frexp_f64(__x, (__attribute__((address_space(5))) int *)&__tmp);
-  *__nptr = __tmp;
-  return __r;
 }
 
 __DEVICE__
@@ -830,13 +815,13 @@ __DEVICE__
 int ilogb(double __x) { return __ocml_ilogb_f64(__x); }
 
 __DEVICE__
-__RETURN_TYPE __finite(double __x) { return __ocml_isfinite_f64(__x); }
+__RETURN_TYPE __finite(double __x) { return __builtin_isfinite(__x); }
 
 __DEVICE__
-__RETURN_TYPE __isinf(double __x) { return __ocml_isinf_f64(__x); }
+__RETURN_TYPE __isinf(double __x) { return __builtin_isinf(__x); }
 
 __DEVICE__
-__RETURN_TYPE __isnan(double __x) { return __ocml_isnan_f64(__x); }
+__RETURN_TYPE __isnan(double __x) { return __builtin_isnan(__x); }
 
 __DEVICE__
 double j0(double __x) { return __ocml_j0_f64(__x); }
@@ -866,16 +851,16 @@ double jn(int __n, double __x) { // TODO: we could use Ahmes multiplication
 }
 
 __DEVICE__
-double ldexp(double __x, int __e) { return __ocml_ldexp_f64(__x, __e); }
+double ldexp(double __x, int __e) { return __builtin_amdgcn_ldexp(__x, __e); }
 
 __DEVICE__
 double lgamma(double __x) { return __ocml_lgamma_f64(__x); }
 
 __DEVICE__
-long long int llrint(double __x) { return __ocml_rint_f64(__x); }
+long long int llrint(double __x) { return __builtin_rint(__x); }
 
 __DEVICE__
-long long int llround(double __x) { return __ocml_round_f64(__x); }
+long long int llround(double __x) { return __builtin_round(__x); }
 
 __DEVICE__
 double log(double __x) { return __ocml_log_f64(__x); }
@@ -893,10 +878,10 @@ __DEVICE__
 double logb(double __x) { return __ocml_logb_f64(__x); }
 
 __DEVICE__
-long int lrint(double __x) { return __ocml_rint_f64(__x); }
+long int lrint(double __x) { return __builtin_rint(__x); }
 
 __DEVICE__
-long int lround(double __x) { return __ocml_round_f64(__x); }
+long int lround(double __x) { return __builtin_round(__x); }
 
 __DEVICE__
 double modf(double __x, double *__iptr) {
@@ -940,7 +925,7 @@ double nan(const char *__tagp) {
 }
 
 __DEVICE__
-double nearbyint(double __x) { return __ocml_nearbyint_f64(__x); }
+double nearbyint(double __x) { return __builtin_nearbyint(__x); }
 
 __DEVICE__
 double nextafter(double __x, double __y) {
@@ -1006,7 +991,7 @@ __DEVICE__
 double rhypot(double __x, double __y) { return __ocml_rhypot_f64(__x, __y); }
 
 __DEVICE__
-double rint(double __x) { return __ocml_rint_f64(__x); }
+double rint(double __x) { return __builtin_rint(__x); }
 
 __DEVICE__
 double rnorm(int __dim,
@@ -1031,21 +1016,21 @@ double rnorm4d(double __x, double __y, double __z, double __w) {
 }
 
 __DEVICE__
-double round(double __x) { return __ocml_round_f64(__x); }
+double round(double __x) { return __builtin_round(__x); }
 
 __DEVICE__
 double rsqrt(double __x) { return __ocml_rsqrt_f64(__x); }
 
 __DEVICE__
 double scalbln(double __x, long int __n) {
-  return (__n < INT_MAX) ? __ocml_scalbn_f64(__x, __n)
+  return (__n < INT_MAX) ? __builtin_amdgcn_ldexp(__x, __n)
                          : __ocml_scalb_f64(__x, __n);
 }
 __DEVICE__
-double scalbn(double __x, int __n) { return __ocml_scalbn_f64(__x, __n); }
+double scalbn(double __x, int __n) { return __builtin_amdgcn_ldexp(__x, __n); }
 
 __DEVICE__
-__RETURN_TYPE __signbit(double __x) { return __ocml_signbit_f64(__x); }
+__RETURN_TYPE __signbit(double __x) { return __builtin_signbit(__x); }
 
 __DEVICE__
 double sin(double __x) { return __ocml_sin_f64(__x); }
@@ -1091,7 +1076,7 @@ __DEVICE__
 double tgamma(double __x) { return __ocml_tgamma_f64(__x); }
 
 __DEVICE__
-double trunc(double __x) { return __ocml_trunc_f64(__x); }
+double trunc(double __x) { return __builtin_trunc(__x); }
 
 __DEVICE__
 double y0(double __x) { return __ocml_y0_f64(__x); }
@@ -1258,7 +1243,7 @@ double __fma_rz(double __x, double __y, double __z) {
 #else
 __DEVICE__
 double __fma_rn(double __x, double __y, double __z) {
-  return __ocml_fma_f64(__x, __y, __z);
+  return __builtin_fma(__x, __y, __z);
 }
 #endif
 // END INTRINSICS
@@ -1290,16 +1275,16 @@ __DEVICE__ int max(int __arg1, int __arg2) {
 }
 
 __DEVICE__
-float max(float __x, float __y) { return fmaxf(__x, __y); }
+float max(float __x, float __y) { return __builtin_fmaxf(__x, __y); }
 
 __DEVICE__
-double max(double __x, double __y) { return fmax(__x, __y); }
+double max(double __x, double __y) { return __builtin_fmax(__x, __y); }
 
 __DEVICE__
-float min(float __x, float __y) { return fminf(__x, __y); }
+float min(float __x, float __y) { return __builtin_fminf(__x, __y); }
 
 __DEVICE__
-double min(double __x, double __y) { return fmin(__x, __y); }
+double min(double __x, double __y) { return __builtin_fmin(__x, __y); }
 
 #if !defined(__HIPCC_RTC__) && !defined(__OPENMP_AMDGCN__)
 __host__ inline static int min(int __arg1, int __arg2) {

lib/include/__clang_hip_runtime_wrapper.h

@@ -80,12 +80,25 @@ extern "C" {
 #if HIP_VERSION_MAJOR * 100 + HIP_VERSION_MINOR >= 405
 extern "C" __device__ unsigned long long __ockl_dm_alloc(unsigned long long __size);
 extern "C" __device__ void __ockl_dm_dealloc(unsigned long long __addr);
+#if __has_feature(address_sanitizer)
+extern "C" __device__ unsigned long long __asan_malloc_impl(unsigned long long __size, unsigned long long __pc);
+extern "C" __device__ void __asan_free_impl(unsigned long long __addr, unsigned long long __pc);
+__attribute__((noinline, weak)) __device__ void *malloc(__hip_size_t __size) {
+  unsigned long long __pc = (unsigned long long)__builtin_return_address(0);
+  return (void *)__asan_malloc_impl(__size, __pc);
+}
+__attribute__((noinline, weak)) __device__ void free(void *__ptr) {
+  unsigned long long __pc = (unsigned long long)__builtin_return_address(0);
+  __asan_free_impl((unsigned long long)__ptr, __pc);
+}
+#else
 __attribute__((weak)) inline __device__ void *malloc(__hip_size_t __size) {
   return (void *) __ockl_dm_alloc(__size);
 }
 __attribute__((weak)) inline __device__ void free(void *__ptr) {
   __ockl_dm_dealloc((unsigned long long)__ptr);
 }
+#endif // __has_feature(address_sanitizer)
 #else  // HIP version check
 #if __HIP_ENABLE_DEVICE_MALLOC__
 __device__ void *__hip_malloc(__hip_size_t __size);

lib/include/adxintrin.h

@@ -17,56 +17,211 @@
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
 
-/* Intrinsics that are available only if __ADX__ defined */
+/* Use C++ inline semantics in C++, GNU inline for C mode. */
-static __inline unsigned char __attribute__((__always_inline__, __nodebug__, __target__("adx")))
+#if defined(__cplusplus)
+#define __INLINE __inline
+#else
+#define __INLINE static __inline
+#endif
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* Intrinsics that are available only if __ADX__ is defined. */
+
+/// Adds unsigned 32-bit integers \a __x and \a __y, plus 0 or 1 as indicated
+///    by the carry flag \a __cf. Stores the unsigned 32-bit sum in the memory
+///    at \a __p, and returns the 8-bit carry-out (carry flag).
+///
+/// \code{.operation}
+/// temp := (__cf == 0) ? 0 : 1
+/// Store32(__p, __x + __y + temp)
+/// result := CF
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c ADCX instruction.
+///
+/// \param __cf
+///    The 8-bit unsigned carry flag; any non-zero value indicates carry.
+/// \param __x
+///    A 32-bit unsigned addend.
+/// \param __y
+///    A 32-bit unsigned addend.
+/// \param __p
+///    Pointer to memory for storing the sum.
+/// \returns The 8-bit unsigned carry-out value.
+__INLINE unsigned char
+    __attribute__((__always_inline__, __nodebug__, __target__("adx")))
     _addcarryx_u32(unsigned char __cf, unsigned int __x, unsigned int __y,
-               unsigned int *__p)
+                   unsigned int *__p) {
-{
   return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p);
 }
 
 #ifdef __x86_64__
-static __inline unsigned char __attribute__((__always_inline__, __nodebug__, __target__("adx")))
+/// Adds unsigned 64-bit integers \a __x and \a __y, plus 0 or 1 as indicated
+///    by the carry flag \a __cf. Stores the unsigned 64-bit sum in the memory
+///    at \a __p, and returns the 8-bit carry-out (carry flag).
+///
+/// \code{.operation}
+/// temp := (__cf == 0) ? 0 : 1
+/// Store64(__p, __x + __y + temp)
+/// result := CF
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c ADCX instruction.
+///
+/// \param __cf
+///    The 8-bit unsigned carry flag; any non-zero value indicates carry.
+/// \param __x
+///    A 64-bit unsigned addend.
+/// \param __y
+///    A 64-bit unsigned addend.
+/// \param __p
+///    Pointer to memory for storing the sum.
+/// \returns The 8-bit unsigned carry-out value.
+__INLINE unsigned char
+    __attribute__((__always_inline__, __nodebug__, __target__("adx")))
     _addcarryx_u64(unsigned char __cf, unsigned long long __x,
-               unsigned long long __y, unsigned long long  *__p)
+                   unsigned long long __y, unsigned long long *__p) {
-{
   return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p);
 }
 #endif
 
-/* Intrinsics that are also available if __ADX__ undefined */
+/* Intrinsics that are also available if __ADX__ is undefined. */
-static __inline unsigned char __DEFAULT_FN_ATTRS
+
-_addcarry_u32(unsigned char __cf, unsigned int __x, unsigned int __y,
+/// Adds unsigned 32-bit integers \a __x and \a __y, plus 0 or 1 as indicated
-              unsigned int *__p)
+///    by the carry flag \a __cf. Stores the unsigned 32-bit sum in the memory
-{
+///    at \a __p, and returns the 8-bit carry-out (carry flag).
+///
+/// \code{.operation}
+/// temp := (__cf == 0) ? 0 : 1
+/// Store32(__p, __x + __y + temp)
+/// result := CF
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c ADC instruction.
+///
+/// \param __cf
+///    The 8-bit unsigned carry flag; any non-zero value indicates carry.
+/// \param __x
+///    A 32-bit unsigned addend.
+/// \param __y
+///    A 32-bit unsigned addend.
+/// \param __p
+///    Pointer to memory for storing the sum.
+/// \returns The 8-bit unsigned carry-out value.
+__INLINE unsigned char __DEFAULT_FN_ATTRS _addcarry_u32(unsigned char __cf,
+                                                        unsigned int __x,
+                                                        unsigned int __y,
+                                                        unsigned int *__p) {
   return __builtin_ia32_addcarryx_u32(__cf, __x, __y, __p);
 }
 
 #ifdef __x86_64__
-static __inline unsigned char __DEFAULT_FN_ATTRS
+/// Adds unsigned 64-bit integers \a __x and \a __y, plus 0 or 1 as indicated
+///    by the carry flag \a __cf. Stores the unsigned 64-bit sum in the memory
+///    at \a __p, and returns the 8-bit carry-out (carry flag).
+///
+/// \code{.operation}
+/// temp := (__cf == 0) ? 0 : 1
+/// Store64(__p, __x + __y + temp)
+/// result := CF
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c ADC instruction.
+///
+/// \param __cf
+///    The 8-bit unsigned carry flag; any non-zero value indicates carry.
+/// \param __x
+///    A 64-bit unsigned addend.
+/// \param __y
+///    A 64-bit unsigned addend.
+/// \param __p
+///    Pointer to memory for storing the sum.
+/// \returns The 8-bit unsigned carry-out value.
+__INLINE unsigned char __DEFAULT_FN_ATTRS
 _addcarry_u64(unsigned char __cf, unsigned long long __x,
-              unsigned long long __y, unsigned long long  *__p)
+              unsigned long long __y, unsigned long long *__p) {
-{
   return __builtin_ia32_addcarryx_u64(__cf, __x, __y, __p);
 }
 #endif
 
-static __inline unsigned char __DEFAULT_FN_ATTRS
+/// Adds unsigned 32-bit integer \a __y to 0 or 1 as indicated by the carry
-_subborrow_u32(unsigned char __cf, unsigned int __x, unsigned int __y,
+///    flag \a __cf, and subtracts the result from unsigned 32-bit integer
-              unsigned int *__p)
+///    \a __x. Stores the unsigned 32-bit difference in the memory at \a __p,
-{
+///    and returns the 8-bit carry-out (carry or overflow flag).
+///
+/// \code{.operation}
+/// temp := (__cf == 0) ? 0 : 1
+/// Store32(__p, __x - (__y + temp))
+/// result := CF
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c SBB instruction.
+///
+/// \param __cf
+///    The 8-bit unsigned carry flag; any non-zero value indicates carry.
+/// \param __x
+///    The 32-bit unsigned minuend.
+/// \param __y
+///    The 32-bit unsigned subtrahend.
+/// \param __p
+///    Pointer to memory for storing the difference.
+/// \returns The 8-bit unsigned carry-out value.
+__INLINE unsigned char __DEFAULT_FN_ATTRS _subborrow_u32(unsigned char __cf,
+                                                         unsigned int __x,
+                                                         unsigned int __y,
+                                                         unsigned int *__p) {
   return __builtin_ia32_subborrow_u32(__cf, __x, __y, __p);
 }
 
 #ifdef __x86_64__
-static __inline unsigned char __DEFAULT_FN_ATTRS
+/// Adds unsigned 64-bit integer \a __y to 0 or 1 as indicated by the carry
+///    flag \a __cf, and subtracts the result from unsigned 64-bit integer
+///    \a __x. Stores the unsigned 64-bit difference in the memory at \a __p,
+///    and returns the 8-bit carry-out (carry or overflow flag).
+///
+/// \code{.operation}
+/// temp := (__cf == 0) ? 0 : 1
+/// Store64(__p, __x - (__y + temp))
+/// result := CF
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c ADC instruction.
+///
+/// \param __cf
+///    The 8-bit unsigned carry flag; any non-zero value indicates carry.
+/// \param __x
+///    The 64-bit unsigned minuend.
+/// \param __y
+///    The 64-bit unsigned subtrahend.
+/// \param __p
+///    Pointer to memory for storing the difference.
+/// \returns The 8-bit unsigned carry-out value.
+__INLINE unsigned char __DEFAULT_FN_ATTRS
 _subborrow_u64(unsigned char __cf, unsigned long long __x,
-               unsigned long long __y, unsigned long long  *__p)
+               unsigned long long __y, unsigned long long *__p) {
-{
   return __builtin_ia32_subborrow_u64(__cf, __x, __y, __p);
 }
 #endif
 
+#if defined(__cplusplus)
+}
+#endif
+
 #undef __DEFAULT_FN_ATTRS
 
 #endif /* __ADXINTRIN_H */

lib/include/altivec.h

@@ -3202,70 +3202,78 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
 // the XL-compatible signatures are used for those functions.
 #ifdef __XL_COMPAT_ALTIVEC__
 #define vec_ctf(__a, __b)                                                      \
-  _Generic(                                                                    \
+  _Generic((__a),                                                              \
-      (__a), vector int                                                        \
+      vector int: (vector float)__builtin_altivec_vcfsx((vector int)(__a),     \
-      : (vector float)__builtin_altivec_vcfsx((vector int)(__a), (__b)),       \
+                                                        ((__b)&0x1F)),         \
-        vector unsigned int                                                    \
+      vector unsigned int: (vector float)__builtin_altivec_vcfux(              \
-      : (vector float)__builtin_altivec_vcfux((vector unsigned int)(__a),      \
+               (vector unsigned int)(__a), ((__b)&0x1F)),                      \
-                                              (__b)),                          \
+      vector unsigned long long: (                                             \
-        vector unsigned long long                                              \
+               vector float)(__builtin_vsx_xvcvuxdsp(                          \
-      : (vector float)(__builtin_vsx_xvcvuxdsp(                                \
                                  (vector unsigned long long)(__a)) *           \
-                       (vector float)(vector unsigned)((0x7f - (__b)) << 23)), \
+                             (vector float)(vector unsigned)((0x7f -           \
-        vector signed long long                                                \
+                                                              ((__b)&0x1F))    \
-      : (vector float)(__builtin_vsx_xvcvsxdsp(                                \
+                                                             << 23)),          \
+      vector signed long long: (                                               \
+               vector float)(__builtin_vsx_xvcvsxdsp(                          \
                                  (vector signed long long)(__a)) *             \
-                       (vector float)(vector unsigned)((0x7f - (__b)) << 23)))
+                             (vector float)(vector unsigned)((0x7f -           \
+                                                              ((__b)&0x1F))    \
+                                                             << 23)))
 #else // __XL_COMPAT_ALTIVEC__
 #define vec_ctf(__a, __b)                                                         \
   _Generic(                                                                       \
-      (__a), vector int                                                        \
+      (__a),                                                                      \
-      : (vector float)__builtin_altivec_vcfsx((vector int)(__a), (__b)),       \
+      vector int: (vector float)__builtin_altivec_vcfsx((vector int)(__a),        \
-        vector unsigned int                                                    \
+                                                        ((__b)&0x1F)),            \
-      : (vector float)__builtin_altivec_vcfux((vector unsigned int)(__a),      \
+      vector unsigned int: (vector float)__builtin_altivec_vcfux(                 \
-                                              (__b)),                          \
+          (vector unsigned int)(__a), ((__b)&0x1F)),                              \
-        vector unsigned long long                                              \
+      vector unsigned long long: (                                                \
-      : (vector float)(__builtin_convertvector(                                \
+          vector float)(__builtin_convertvector(                                  \
                             (vector unsigned long long)(__a), vector double) *    \
                         (vector double)(vector unsigned long long)((0x3ffULL -    \
-                                                                   (__b))      \
+                                                                    ((__b)&0x1F)) \
                                                                    << 52)),       \
-        vector signed long long                                                \
+      vector signed long long: (                                                  \
-      : (vector float)(__builtin_convertvector((vector signed long long)(__a), \
+          vector float)(__builtin_convertvector(                                  \
-                                               vector double) *                \
+                            (vector signed long long)(__a), vector double) *      \
                         (vector double)(vector unsigned long long)((0x3ffULL -    \
-                                                                   (__b))      \
+                                                                    ((__b)&0x1F)) \
                                                                    << 52)))
 #endif // __XL_COMPAT_ALTIVEC__
 #else
 #define vec_ctf(__a, __b)                                                      \
-  _Generic((__a), vector int                                                   \
+  _Generic((__a),                                                              \
-           : (vector float)__builtin_altivec_vcfsx((vector int)(__a), (__b)),  \
+      vector int: (vector float)__builtin_altivec_vcfsx((vector int)(__a),     \
-             vector unsigned int                                               \
+                                                        ((__b)&0x1F)),         \
-           : (vector float)__builtin_altivec_vcfux((vector unsigned int)(__a), \
+      vector unsigned int: (vector float)__builtin_altivec_vcfux(              \
-                                                   (__b)))
+               (vector unsigned int)(__a), ((__b)&0x1F)))
 #endif
 
 /* vec_ctd */
 #ifdef __VSX__
 #define vec_ctd(__a, __b)                                                      \
-  _Generic((__a), vector signed int                                            \
+  _Generic((__a),                                                              \
-           : (vec_doublee((vector signed int)(__a)) *                          \
+      vector signed int: (                                                     \
-              (vector double)(vector unsigned long long)((0x3ffULL - (__b))    \
+               vec_doublee((vector signed int)(__a)) *                         \
+               (vector double)(vector unsigned long long)((0x3ffULL -          \
+                                                           ((__b)&0x1F))       \
                                                           << 52)),             \
-             vector unsigned int                                               \
+      vector unsigned int: (                                                   \
-           : (vec_doublee((vector unsigned int)(__a)) *                        \
+               vec_doublee((vector unsigned int)(__a)) *                       \
-              (vector double)(vector unsigned long long)((0x3ffULL - (__b))    \
+               (vector double)(vector unsigned long long)((0x3ffULL -          \
+                                                           ((__b)&0x1F))       \
                                                           << 52)),             \
-             vector unsigned long long                                         \
+      vector unsigned long long: (                                             \
-           : (__builtin_convertvector((vector unsigned long long)(__a),        \
+               __builtin_convertvector((vector unsigned long long)(__a),       \
                                        vector double) *                        \
-              (vector double)(vector unsigned long long)((0x3ffULL - (__b))    \
+               (vector double)(vector unsigned long long)((0x3ffULL -          \
+                                                           ((__b)&0x1F))       \
                                                           << 52)),             \
-             vector signed long long                                           \
+      vector signed long long: (                                               \
-           : (__builtin_convertvector((vector signed long long)(__a),          \
+               __builtin_convertvector((vector signed long long)(__a),         \
                                        vector double) *                        \
-              (vector double)(vector unsigned long long)((0x3ffULL - (__b))    \
+               (vector double)(vector unsigned long long)((0x3ffULL -          \
+                                                           ((__b)&0x1F))       \
                                                           << 52)))
 #endif // __VSX__
 
@@ -3281,27 +3289,27 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
 #ifdef __VSX__
 #ifdef __XL_COMPAT_ALTIVEC__
 #define vec_cts(__a, __b)                                                      \
-  _Generic((__a), vector float                                                 \
+  _Generic((__a),                                                              \
-           : (vector signed int)__builtin_altivec_vctsxs((vector float)(__a),  \
+      vector float: (vector signed int)__builtin_altivec_vctsxs(               \
-                                                         (__b)),               \
+               (vector float)(__a), ((__b)&0x1F)),                             \
-             vector double                                                     \
+      vector double: __extension__({                                           \
-           : __extension__({                                                   \
              vector double __ret =                                             \
                  (vector double)(__a) *                                        \
-                 (vector double)(vector unsigned long long)((0x3ffULL + (__b)) \
+                 (vector double)(vector unsigned long long)((0x3ffULL +        \
+                                                             ((__b)&0x1F))     \
                                                             << 52);            \
              (vector signed long long)__builtin_vsx_xvcvdpsxws(__ret);         \
            }))
 #else // __XL_COMPAT_ALTIVEC__
 #define vec_cts(__a, __b)                                                      \
-  _Generic((__a), vector float                                                 \
+  _Generic((__a),                                                              \
-           : (vector signed int)__builtin_altivec_vctsxs((vector float)(__a),  \
+      vector float: (vector signed int)__builtin_altivec_vctsxs(               \
-                                                         (__b)),               \
+               (vector float)(__a), ((__b)&0x1F)),                             \
-             vector double                                                     \
+      vector double: __extension__({                                           \
-           : __extension__({                                                   \
              vector double __ret =                                             \
                  (vector double)(__a) *                                        \
-                 (vector double)(vector unsigned long long)((0x3ffULL + (__b)) \
+                 (vector double)(vector unsigned long long)((0x3ffULL +        \
+                                                             ((__b)&0x1F))     \
                                                             << 52);            \
              (vector signed long long)__builtin_convertvector(                 \
                  __ret, vector signed long long);                              \
@@ -3320,27 +3328,27 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
 #ifdef __VSX__
 #ifdef __XL_COMPAT_ALTIVEC__
 #define vec_ctu(__a, __b)                                                      \
-  _Generic((__a), vector float                                                 \
+  _Generic((__a),                                                              \
-           : (vector unsigned int)__builtin_altivec_vctuxs(                    \
+      vector float: (vector unsigned int)__builtin_altivec_vctuxs(             \
-                 (vector float)(__a), (__b)),                                  \
+               (vector float)(__a), ((__b)&0x1F)),                             \
-             vector double                                                     \
+      vector double: __extension__({                                           \
-           : __extension__({                                                   \
              vector double __ret =                                             \
                  (vector double)(__a) *                                        \
-                 (vector double)(vector unsigned long long)((0x3ffULL + __b)   \
+                 (vector double)(vector unsigned long long)((0x3ffULL +        \
+                                                             ((__b)&0x1F))     \
                                                             << 52);            \
              (vector unsigned long long)__builtin_vsx_xvcvdpuxws(__ret);       \
            }))
 #else // __XL_COMPAT_ALTIVEC__
 #define vec_ctu(__a, __b)                                                      \
-  _Generic((__a), vector float                                                 \
+  _Generic((__a),                                                              \
-           : (vector unsigned int)__builtin_altivec_vctuxs(                    \
+      vector float: (vector unsigned int)__builtin_altivec_vctuxs(             \
-                 (vector float)(__a), (__b)),                                  \
+               (vector float)(__a), ((__b)&0x1F)),                             \
-             vector double                                                     \
+      vector double: __extension__({                                           \
-           : __extension__({                                                   \
              vector double __ret =                                             \
                  (vector double)(__a) *                                        \
-                 (vector double)(vector unsigned long long)((0x3ffULL + __b)   \
+                 (vector double)(vector unsigned long long)((0x3ffULL +        \
+                                                             ((__b)&0x1F))     \
                                                             << 52);            \
              (vector unsigned long long)__builtin_convertvector(               \
                  __ret, vector unsigned long long);                            \
@@ -3355,19 +3363,20 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
 
 #ifdef __VSX__
 #define vec_ctsl(__a, __b)                                                     \
-  _Generic((__a), vector float                                                 \
+  _Generic(                                                                    \
+      (__a), vector float                                                      \
       : __extension__({                                                        \
           vector float __ret =                                                 \
               (vector float)(__a) *                                            \
-                   (vector float)(vector unsigned)((0x7f + (__b)) << 23);      \
+              (vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) << 23);    \
-               __builtin_vsx_xvcvspsxds(                                       \
+          __builtin_vsx_xvcvspsxds(__builtin_vsx_xxsldwi(__ret, __ret, 1));    \
-                   __builtin_vsx_xxsldwi(__ret, __ret, 1));                    \
         }),                                                                    \
         vector double                                                          \
       : __extension__({                                                        \
         vector double __ret =                                                  \
             (vector double)(__a) *                                             \
-                 (vector double)(vector unsigned long long)((0x3ffULL + __b)   \
+            (vector double)(vector unsigned long long)((0x3ffULL +             \
+                                                        ((__b)&0x1F))          \
                                                        << 52);                 \
         __builtin_convertvector(__ret, vector signed long long);               \
       }))
@@ -3375,19 +3384,20 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
 /* vec_ctul */
 
 #define vec_ctul(__a, __b)                                                     \
-  _Generic((__a), vector float                                                 \
+  _Generic(                                                                    \
+      (__a), vector float                                                      \
       : __extension__({                                                        \
           vector float __ret =                                                 \
               (vector float)(__a) *                                            \
-                   (vector float)(vector unsigned)((0x7f + (__b)) << 23);      \
+              (vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) << 23);    \
-               __builtin_vsx_xvcvspuxds(                                       \
+          __builtin_vsx_xvcvspuxds(__builtin_vsx_xxsldwi(__ret, __ret, 1));    \
-                   __builtin_vsx_xxsldwi(__ret, __ret, 1));                    \
         }),                                                                    \
         vector double                                                          \
       : __extension__({                                                        \
         vector double __ret =                                                  \
             (vector double)(__a) *                                             \
-                 (vector double)(vector unsigned long long)((0x3ffULL + __b)   \
+            (vector double)(vector unsigned long long)((0x3ffULL +             \
+                                                        ((__b)&0x1F))          \
                                                        << 52);                 \
         __builtin_convertvector(__ret, vector unsigned long long);             \
       }))
@@ -3397,18 +3407,18 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
 
 #ifdef __VSX__
 #define vec_ctsl(__a, __b)                                                     \
-  _Generic((__a), vector float                                                 \
+  _Generic((__a),                                                              \
-           : __extension__({                                                   \
+      vector float: __extension__({                                            \
              vector float __ret =                                              \
                  (vector float)(__a) *                                         \
-                   (vector float)(vector unsigned)((0x7f + (__b)) << 23);      \
+                 (vector float)(vector unsigned)((0x7f + ((__b)&0x1F)) << 23); \
              __builtin_vsx_xvcvspsxds(__ret);                                  \
            }),                                                                 \
-             vector double                                                     \
+      vector double: __extension__({                                           \
-           : __extension__({                                                   \
              vector double __ret =                                             \
                  (vector double)(__a) *                                        \
-                 (vector double)(vector unsigned long long)((0x3ffULL + __b)   \
+                 (vector double)(vector unsigned long long)((0x3ffULL +        \
+                                                             ((__b)&0x1F))     \
                                                             << 52);            \
              __builtin_convertvector(__ret, vector signed long long);          \
            }))
@@ -3420,14 +3430,16 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
            : __extension__({                                                   \
                vector float __ret =                                            \
                    (vector float)(__a) *                                       \
-                   (vector float)(vector unsigned)((0x7f + (__b)) << 23);      \
+                   (vector float)(vector unsigned)((0x7f + ((__b)&0x1F))       \
+                                                   << 23);                     \
                __builtin_vsx_xvcvspuxds(__ret);                                \
              }),                                                               \
              vector double                                                     \
            : __extension__({                                                   \
              vector double __ret =                                             \
                  (vector double)(__a) *                                        \
-                 (vector double)(vector unsigned long long)((0x3ffULL + __b)   \
+                 (vector double)(vector unsigned long long)((0x3ffULL +        \
+                                                             ((__b)&0x1F))     \
                                                             << 52);            \
              __builtin_convertvector(__ret, vector unsigned long long);        \
            }))

lib/include/amxcomplexintrin.h

@@ -0,0 +1,169 @@
+/*===--------- amxcomplexintrin.h - AMXCOMPLEX intrinsics -*- C++ -*---------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===------------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <amxcomplexintrin.h> directly; include <immintrin.h> instead."
+#endif // __IMMINTRIN_H
+
+#ifndef __AMX_COMPLEXINTRIN_H
+#define __AMX_COMPLEXINTRIN_H
+#ifdef __x86_64__
+
+#define __DEFAULT_FN_ATTRS_COMPLEX                                             \
+  __attribute__((__always_inline__, __nodebug__, __target__("amx-complex")))
+
+/// Perform matrix multiplication of two tiles containing complex elements and
+///    accumulate the results into a packed single precision tile. Each dword
+///    element in input tiles \a a and \a b is interpreted as a complex number
+///    with FP16 real part and FP16 imaginary part.
+/// Calculates the imaginary part of the result. For each possible combination
+///    of (row of \a a, column of \a b), it performs a set of multiplication
+///    and accumulations on all corresponding complex numbers (one from \a a
+///    and one from \a b). The imaginary part of the \a a element is multiplied
+///    with the real part of the corresponding \a b element, and the real part
+///    of the \a a element is multiplied with the imaginary part of the
+///    corresponding \a b elements. The two accumulated results are added, and
+///    then accumulated into the corresponding row and column of \a dst.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \code
+/// void _tile_cmmimfp16ps(__tile dst, __tile a, __tile b);
+/// \endcode
+///
+/// \code{.operation}
+/// FOR m := 0 TO dst.rows - 1
+///	tmp := dst.row[m]
+///	FOR k := 0 TO (a.colsb / 4) - 1
+///		FOR n := 0 TO (dst.colsb / 4) - 1
+///			tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+1])
+///			tmp.fp32[n] += FP32(a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+0])
+///		ENDFOR
+///	ENDFOR
+///	write_row_and_zero(dst, m, tmp, dst.colsb)
+/// ENDFOR
+/// zero_upper_rows(dst, dst.rows)
+/// zero_tileconfig_start()
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TCMMIMFP16PS instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param a
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param b
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_cmmimfp16ps(dst, a, b) __builtin_ia32_tcmmimfp16ps(dst, a, b)
+
+/// Perform matrix multiplication of two tiles containing complex elements and
+///    accumulate the results into a packed single precision tile. Each dword
+///    element in input tiles \a a and \a b is interpreted as a complex number
+///    with FP16 real part and FP16 imaginary part.
+/// Calculates the real part of the result. For each possible combination
+///    of (row of \a a, column of \a b), it performs a set of multiplication
+///    and accumulations on all corresponding complex numbers (one from \a a
+///    and one from \a b). The real part of the \a a element is multiplied
+///    with the real part of the corresponding \a b element, and the negated
+///    imaginary part of the \a a element is multiplied with the imaginary
+///    part of the corresponding \a b elements. The two accumulated results
+///    are added, and then accumulated into the corresponding row and column
+///    of \a dst.
+///
+/// \headerfile <x86intrin.h>
+///
+/// \code
+/// void _tile_cmmrlfp16ps(__tile dst, __tile a, __tile b);
+/// \endcode
+///
+/// \code{.operation}
+/// FOR m := 0 TO dst.rows - 1
+///	tmp := dst.row[m]
+///	FOR k := 0 TO (a.colsb / 4) - 1
+///		FOR n := 0 TO (dst.colsb / 4) - 1
+///			tmp.fp32[n] += FP32(a.row[m].fp16[2*k+0]) * FP32(b.row[k].fp16[2*n+0])
+///			tmp.fp32[n] += FP32(-a.row[m].fp16[2*k+1]) * FP32(b.row[k].fp16[2*n+1])
+///		ENDFOR
+///	ENDFOR
+///	write_row_and_zero(dst, m, tmp, dst.colsb)
+/// ENDFOR
+/// zero_upper_rows(dst, dst.rows)
+/// zero_tileconfig_start()
+/// \endcode
+///
+/// This intrinsic corresponds to the \c TCMMIMFP16PS instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param a
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param b
+///    The 2nd source tile. Max size is 1024 Bytes.
+#define _tile_cmmrlfp16ps(dst, a, b) __builtin_ia32_tcmmrlfp16ps(dst, a, b)
+
+static __inline__ _tile1024i __DEFAULT_FN_ATTRS_COMPLEX
+_tile_cmmimfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
+                           _tile1024i dst, _tile1024i src1, _tile1024i src2) {
+  return __builtin_ia32_tcmmimfp16ps_internal(m, n, k, dst, src1, src2);
+}
+
+static __inline__ _tile1024i __DEFAULT_FN_ATTRS_COMPLEX
+_tile_cmmrlfp16ps_internal(unsigned short m, unsigned short n, unsigned short k,
+                           _tile1024i dst, _tile1024i src1, _tile1024i src2) {
+  return __builtin_ia32_tcmmrlfp16ps_internal(m, n, k, dst, src1, src2);
+}
+
+/// Perform matrix multiplication of two tiles containing complex elements and
+/// accumulate the results into a packed single precision tile. Each dword
+/// element in input tiles src0 and src1 is interpreted as a complex number with
+/// FP16 real part and FP16 imaginary part.
+/// This function calculates the imaginary part of the result.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCMMIMFP16PS </c> instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param src0
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param src1
+///    The 2nd source tile. Max size is 1024 Bytes.
+__DEFAULT_FN_ATTRS_COMPLEX
+static void __tile_cmmimfp16ps(__tile1024i *dst, __tile1024i src0,
+                               __tile1024i src1) {
+  dst->tile = _tile_cmmimfp16ps_internal(src0.row, src1.col, src0.col,
+                                         dst->tile, src0.tile, src1.tile);
+}
+
+/// Perform matrix multiplication of two tiles containing complex elements and
+/// accumulate the results into a packed single precision tile. Each dword
+/// element in input tiles src0 and src1 is interpreted as a complex number with
+/// FP16 real part and FP16 imaginary part.
+/// This function calculates the real part of the result.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> TCMMRLFP16PS </c> instruction.
+///
+/// \param dst
+///    The destination tile. Max size is 1024 Bytes.
+/// \param src0
+///    The 1st source tile. Max size is 1024 Bytes.
+/// \param src1
+///    The 2nd source tile. Max size is 1024 Bytes.
+__DEFAULT_FN_ATTRS_COMPLEX
+static void __tile_cmmrlfp16ps(__tile1024i *dst, __tile1024i src0,
+                               __tile1024i src1) {
+  dst->tile = _tile_cmmrlfp16ps_internal(src0.row, src1.col, src0.col,
+                                         dst->tile, src0.tile, src1.tile);
+}
+
+#endif // __x86_64__
+#endif // __AMX_COMPLEXINTRIN_H

lib/include/arm_acle.h

@@ -138,28 +138,32 @@ __rorl(unsigned long __x, uint32_t __y) {
 
 
 /* CLZ */
-static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
 __clz(uint32_t __t) {
-  return (uint32_t)__builtin_clz(__t);
+  return __builtin_arm_clz(__t);
 }
 
-static __inline__ unsigned long __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
 __clzl(unsigned long __t) {
-  return (unsigned long)__builtin_clzl(__t);
+#if __SIZEOF_LONG__ == 4
+  return __builtin_arm_clz(__t);
+#else
+  return __builtin_arm_clz64(__t);
+#endif
 }
 
-static __inline__ uint64_t __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
 __clzll(uint64_t __t) {
-  return (uint64_t)__builtin_clzll(__t);
+  return __builtin_arm_clz64(__t);
 }
 
 /* CLS */
-static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
 __cls(uint32_t __t) {
   return __builtin_arm_cls(__t);
 }
 
-static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
 __clsl(unsigned long __t) {
 #if __SIZEOF_LONG__ == 4
   return __builtin_arm_cls(__t);
@@ -168,7 +172,7 @@ __clsl(unsigned long __t) {
 #endif
 }
 
-static __inline__ uint32_t __attribute__((__always_inline__, __nodebug__))
+static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
 __clsll(uint64_t __t) {
   return __builtin_arm_cls64(__t);
 }

lib/include/arm_neon.h

@@ -35,7 +35,6 @@
 #include <stdint.h>
 
 #include <arm_bf16.h>
-typedef __bf16 bfloat16_t;
 typedef float float32_t;
 typedef __fp16 float16_t;
 #ifdef __aarch64__
@@ -64938,6 +64937,170 @@ int8x16_t __reint_786 = __rev2_786; \
 })
 #endif
 
+#define vldap1_lane_p64(__p0, __p1, __p2) __extension__ ({ \
+  poly64x1_t __ret; \
+  poly64x1_t __s1 = __p1; \
+  __ret = (poly64x1_t) __builtin_neon_vldap1_lane_p64(__p0, (int8x8_t)__s1, __p2, 6); \
+  __ret; \
+})
+#ifdef __LITTLE_ENDIAN__
+#define vldap1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \
+  poly64x2_t __ret; \
+  poly64x2_t __s1 = __p1; \
+  __ret = (poly64x2_t) __builtin_neon_vldap1q_lane_p64(__p0, (int8x16_t)__s1, __p2, 38); \
+  __ret; \
+})
+#else
+#define vldap1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \
+  poly64x2_t __ret; \
+  poly64x2_t __s1 = __p1; \
+  poly64x2_t __rev1;  __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
+  __ret = (poly64x2_t) __builtin_neon_vldap1q_lane_p64(__p0, (int8x16_t)__rev1, __p2, 38); \
+  __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
+  __ret; \
+})
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+#define vldap1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \
+  uint64x2_t __ret; \
+  uint64x2_t __s1 = __p1; \
+  __ret = (uint64x2_t) __builtin_neon_vldap1q_lane_u64(__p0, (int8x16_t)__s1, __p2, 51); \
+  __ret; \
+})
+#else
+#define vldap1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \
+  uint64x2_t __ret; \
+  uint64x2_t __s1 = __p1; \
+  uint64x2_t __rev1;  __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
+  __ret = (uint64x2_t) __builtin_neon_vldap1q_lane_u64(__p0, (int8x16_t)__rev1, __p2, 51); \
+  __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
+  __ret; \
+})
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+#define vldap1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \
+  float64x2_t __ret; \
+  float64x2_t __s1 = __p1; \
+  __ret = (float64x2_t) __builtin_neon_vldap1q_lane_f64(__p0, (int8x16_t)__s1, __p2, 42); \
+  __ret; \
+})
+#else
+#define vldap1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \
+  float64x2_t __ret; \
+  float64x2_t __s1 = __p1; \
+  float64x2_t __rev1;  __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
+  __ret = (float64x2_t) __builtin_neon_vldap1q_lane_f64(__p0, (int8x16_t)__rev1, __p2, 42); \
+  __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
+  __ret; \
+})
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+#define vldap1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \
+  int64x2_t __ret; \
+  int64x2_t __s1 = __p1; \
+  __ret = (int64x2_t) __builtin_neon_vldap1q_lane_s64(__p0, (int8x16_t)__s1, __p2, 35); \
+  __ret; \
+})
+#else
+#define vldap1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \
+  int64x2_t __ret; \
+  int64x2_t __s1 = __p1; \
+  int64x2_t __rev1;  __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
+  __ret = (int64x2_t) __builtin_neon_vldap1q_lane_s64(__p0, (int8x16_t)__rev1, __p2, 35); \
+  __ret = __builtin_shufflevector(__ret, __ret, 1, 0); \
+  __ret; \
+})
+#endif
+
+#define vldap1_lane_u64(__p0, __p1, __p2) __extension__ ({ \
+  uint64x1_t __ret; \
+  uint64x1_t __s1 = __p1; \
+  __ret = (uint64x1_t) __builtin_neon_vldap1_lane_u64(__p0, (int8x8_t)__s1, __p2, 19); \
+  __ret; \
+})
+#define vldap1_lane_f64(__p0, __p1, __p2) __extension__ ({ \
+  float64x1_t __ret; \
+  float64x1_t __s1 = __p1; \
+  __ret = (float64x1_t) __builtin_neon_vldap1_lane_f64(__p0, (int8x8_t)__s1, __p2, 10); \
+  __ret; \
+})
+#define vldap1_lane_s64(__p0, __p1, __p2) __extension__ ({ \
+  int64x1_t __ret; \
+  int64x1_t __s1 = __p1; \
+  __ret = (int64x1_t) __builtin_neon_vldap1_lane_s64(__p0, (int8x8_t)__s1, __p2, 3); \
+  __ret; \
+})
+#define vstl1_lane_p64(__p0, __p1, __p2) __extension__ ({ \
+  poly64x1_t __s1 = __p1; \
+  __builtin_neon_vstl1_lane_p64(__p0, (int8x8_t)__s1, __p2, 6); \
+})
+#ifdef __LITTLE_ENDIAN__
+#define vstl1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \
+  poly64x2_t __s1 = __p1; \
+  __builtin_neon_vstl1q_lane_p64(__p0, (int8x16_t)__s1, __p2, 38); \
+})
+#else
+#define vstl1q_lane_p64(__p0, __p1, __p2) __extension__ ({ \
+  poly64x2_t __s1 = __p1; \
+  poly64x2_t __rev1;  __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
+  __builtin_neon_vstl1q_lane_p64(__p0, (int8x16_t)__rev1, __p2, 38); \
+})
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+#define vstl1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \
+  uint64x2_t __s1 = __p1; \
+  __builtin_neon_vstl1q_lane_u64(__p0, (int8x16_t)__s1, __p2, 51); \
+})
+#else
+#define vstl1q_lane_u64(__p0, __p1, __p2) __extension__ ({ \
+  uint64x2_t __s1 = __p1; \
+  uint64x2_t __rev1;  __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
+  __builtin_neon_vstl1q_lane_u64(__p0, (int8x16_t)__rev1, __p2, 51); \
+})
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+#define vstl1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \
+  float64x2_t __s1 = __p1; \
+  __builtin_neon_vstl1q_lane_f64(__p0, (int8x16_t)__s1, __p2, 42); \
+})
+#else
+#define vstl1q_lane_f64(__p0, __p1, __p2) __extension__ ({ \
+  float64x2_t __s1 = __p1; \
+  float64x2_t __rev1;  __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
+  __builtin_neon_vstl1q_lane_f64(__p0, (int8x16_t)__rev1, __p2, 42); \
+})
+#endif
+
+#ifdef __LITTLE_ENDIAN__
+#define vstl1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \
+  int64x2_t __s1 = __p1; \
+  __builtin_neon_vstl1q_lane_s64(__p0, (int8x16_t)__s1, __p2, 35); \
+})
+#else
+#define vstl1q_lane_s64(__p0, __p1, __p2) __extension__ ({ \
+  int64x2_t __s1 = __p1; \
+  int64x2_t __rev1;  __rev1 = __builtin_shufflevector(__s1, __s1, 1, 0); \
+  __builtin_neon_vstl1q_lane_s64(__p0, (int8x16_t)__rev1, __p2, 35); \
+})
+#endif
+
+#define vstl1_lane_u64(__p0, __p1, __p2) __extension__ ({ \
+  uint64x1_t __s1 = __p1; \
+  __builtin_neon_vstl1_lane_u64(__p0, (int8x8_t)__s1, __p2, 19); \
+})
+#define vstl1_lane_f64(__p0, __p1, __p2) __extension__ ({ \
+  float64x1_t __s1 = __p1; \
+  __builtin_neon_vstl1_lane_f64(__p0, (int8x8_t)__s1, __p2, 10); \
+})
+#define vstl1_lane_s64(__p0, __p1, __p2) __extension__ ({ \
+  int64x1_t __s1 = __p1; \
+  __builtin_neon_vstl1_lane_s64(__p0, (int8x8_t)__s1, __p2, 3); \
+})
 #ifdef __LITTLE_ENDIAN__
 __ai __attribute__((target("sha3"))) uint8x16_t vbcaxq_u8(uint8x16_t __p0, uint8x16_t __p1, uint8x16_t __p2) {
   uint8x16_t __ret;

lib/include/arm_sme_draft_spec_subject_to_change.h

@@ -0,0 +1,642 @@
+/*===---- arm_sme_draft_spec_subject_to_change.h - ARM SME intrinsics ------===
+ *
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __ARM_SME_H
+#define __ARM_SME_H
+
+#if !defined(__LITTLE_ENDIAN__)
+#error "Big endian is currently not supported for arm_sme_draft_spec_subject_to_change.h"
+#endif
+#include <arm_sve.h> 
+
+/* Function attributes */
+#define __ai static __inline__ __attribute__((__always_inline__, __nodebug__))
+
+#define __aio static __inline__ __attribute__((__always_inline__, __nodebug__, __overloadable__))
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_u32_m), arm_streaming, arm_shared_za))
+void svaddha_za32_u32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_s32_m), arm_streaming, arm_shared_za))
+void svaddha_za32_s32_m(uint64_t, svbool_t, svbool_t, svint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_u32_m), arm_streaming, arm_shared_za))
+void svaddva_za32_u32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_s32_m), arm_streaming, arm_shared_za))
+void svaddva_za32_s32_m(uint64_t, svbool_t, svbool_t, svint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsb), arm_streaming_compatible, arm_preserves_za))
+uint64_t svcntsb(void);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsd), arm_streaming_compatible, arm_preserves_za))
+uint64_t svcntsd(void);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsh), arm_streaming_compatible, arm_preserves_za))
+uint64_t svcntsh(void);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svcntsw), arm_streaming_compatible, arm_preserves_za))
+uint64_t svcntsw(void);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za128), arm_streaming, arm_shared_za))
+void svld1_hor_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za16), arm_streaming, arm_shared_za))
+void svld1_hor_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za32), arm_streaming, arm_shared_za))
+void svld1_hor_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za64), arm_streaming, arm_shared_za))
+void svld1_hor_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_vnum_za8), arm_streaming, arm_shared_za))
+void svld1_hor_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za128), arm_streaming, arm_shared_za))
+void svld1_hor_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za16), arm_streaming, arm_shared_za))
+void svld1_hor_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za32), arm_streaming, arm_shared_za))
+void svld1_hor_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za64), arm_streaming, arm_shared_za))
+void svld1_hor_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_hor_za8), arm_streaming, arm_shared_za))
+void svld1_hor_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za128), arm_streaming, arm_shared_za))
+void svld1_ver_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za16), arm_streaming, arm_shared_za))
+void svld1_ver_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za32), arm_streaming, arm_shared_za))
+void svld1_ver_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za64), arm_streaming, arm_shared_za))
+void svld1_ver_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_vnum_za8), arm_streaming, arm_shared_za))
+void svld1_ver_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za128), arm_streaming, arm_shared_za))
+void svld1_ver_za128(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za16), arm_streaming, arm_shared_za))
+void svld1_ver_za16(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za32), arm_streaming, arm_shared_za))
+void svld1_ver_za32(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za64), arm_streaming, arm_shared_za))
+void svld1_ver_za64(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svld1_ver_za8), arm_streaming, arm_shared_za))
+void svld1_ver_za8(uint64_t, uint32_t, uint64_t, svbool_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f16_m), arm_streaming, arm_shared_za))
+void svmopa_za32_f16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_bf16_m), arm_streaming, arm_shared_za))
+void svmopa_za32_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f32_m), arm_streaming, arm_shared_za))
+void svmopa_za32_f32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_s8_m), arm_streaming, arm_shared_za))
+void svmopa_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_u8_m), arm_streaming, arm_shared_za))
+void svmopa_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f16_m), arm_streaming, arm_shared_za))
+void svmops_za32_f16_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_bf16_m), arm_streaming, arm_shared_za))
+void svmops_za32_bf16_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f32_m), arm_streaming, arm_shared_za))
+void svmops_za32_f32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_s8_m), arm_streaming, arm_shared_za))
+void svmops_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_u8_m), arm_streaming, arm_shared_za))
+void svmops_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint8_t svread_hor_za128_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint32_t svread_hor_za128_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint64_t svread_hor_za128_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint16_t svread_hor_za128_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svbfloat16_t svread_hor_za128_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint8_t svread_hor_za128_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat64_t svread_hor_za128_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat32_t svread_hor_za128_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat16_t svread_hor_za128_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint32_t svread_hor_za128_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint64_t svread_hor_za128_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint16_t svread_hor_za128_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint16_t svread_hor_za16_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svbfloat16_t svread_hor_za16_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat16_t svread_hor_za16_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint16_t svread_hor_za16_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint32_t svread_hor_za32_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat32_t svread_hor_za32_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint32_t svread_hor_za32_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint64_t svread_hor_za64_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat64_t svread_hor_za64_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint64_t svread_hor_za64_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint8_t svread_hor_za8_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint8_t svread_hor_za8_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint8_t svread_ver_za128_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint32_t svread_ver_za128_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint64_t svread_ver_za128_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint16_t svread_ver_za128_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svbfloat16_t svread_ver_za128_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint8_t svread_ver_za128_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat64_t svread_ver_za128_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat32_t svread_ver_za128_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat16_t svread_ver_za128_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint32_t svread_ver_za128_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint64_t svread_ver_za128_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint16_t svread_ver_za128_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint16_t svread_ver_za16_u16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svbfloat16_t svread_ver_za16_bf16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat16_t svread_ver_za16_f16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint16_t svread_ver_za16_s16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint32_t svread_ver_za32_u32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat32_t svread_ver_za32_f32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint32_t svread_ver_za32_s32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint64_t svread_ver_za64_u64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat64_t svread_ver_za64_f64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint64_t svread_ver_za64_s64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint8_t svread_ver_za8_u8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint8_t svread_ver_za8_s8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za128), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za16), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za32), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za64), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_vnum_za8), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za128), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za16), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za32), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za64), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_hor_za8), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_hor_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za128), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_vnum_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za16), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_vnum_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za32), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_vnum_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za64), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_vnum_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_vnum_za8), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_vnum_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *, int64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za128), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_za128(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za16), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_za16(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za32), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_za32(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za64), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_za64(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svst1_ver_za8), arm_streaming, arm_shared_za, arm_preserves_za))
+void svst1_ver_za8(uint64_t, uint32_t, uint64_t, svbool_t, void *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za32_s8_m), arm_streaming, arm_shared_za))
+void svsumopa_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za32_s8_m), arm_streaming, arm_shared_za))
+void svsumops_za32_s8_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za32_u8_m), arm_streaming, arm_shared_za))
+void svusmopa_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za32_u8_m), arm_streaming, arm_shared_za))
+void svusmops_za32_u8_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u8_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_bf16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s8_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_u16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za16_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_bf16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za16_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_f16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za16_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_s16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za16_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_u32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za32_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_f32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za32_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_s32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za32_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_u64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za64_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_f64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za64_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_s64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za64_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_u8_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za8_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_s8_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za8_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u8_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_bf16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s8_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_u16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za16_u16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_bf16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za16_bf16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_f16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za16_f16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_s16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za16_s16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_u32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za32_u32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_f32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za32_f32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_s32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za32_s32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_u64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za64_u64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_f64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za64_f64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_s64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za64_s64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_u8_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za8_u8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_s8_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za8_s8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_mask_za), arm_streaming_compatible, arm_shared_za))
+void svzero_mask_za(uint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svzero_za), arm_streaming_compatible, arm_shared_za))
+void svzero_za();
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_u32_m), arm_streaming, arm_shared_za))
+void svaddha_za32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za32_s32_m), arm_streaming, arm_shared_za))
+void svaddha_za32_m(uint64_t, svbool_t, svbool_t, svint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_u32_m), arm_streaming, arm_shared_za))
+void svaddva_za32_m(uint64_t, svbool_t, svbool_t, svuint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za32_s32_m), arm_streaming, arm_shared_za))
+void svaddva_za32_m(uint64_t, svbool_t, svbool_t, svint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f16_m), arm_streaming, arm_shared_za))
+void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_bf16_m), arm_streaming, arm_shared_za))
+void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_f32_m), arm_streaming, arm_shared_za))
+void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_s8_m), arm_streaming, arm_shared_za))
+void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za32_u8_m), arm_streaming, arm_shared_za))
+void svmopa_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f16_m), arm_streaming, arm_shared_za))
+void svmops_za32_m(uint64_t, svbool_t, svbool_t, svfloat16_t, svfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_bf16_m), arm_streaming, arm_shared_za))
+void svmops_za32_m(uint64_t, svbool_t, svbool_t, svbfloat16_t, svbfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_f32_m), arm_streaming, arm_shared_za))
+void svmops_za32_m(uint64_t, svbool_t, svbool_t, svfloat32_t, svfloat32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_s8_m), arm_streaming, arm_shared_za))
+void svmops_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za32_u8_m), arm_streaming, arm_shared_za))
+void svmops_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint8_t svread_hor_za128_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint32_t svread_hor_za128_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint64_t svread_hor_za128_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint16_t svread_hor_za128_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svbfloat16_t svread_hor_za128_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint8_t svread_hor_za128_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat64_t svread_hor_za128_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat32_t svread_hor_za128_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat16_t svread_hor_za128_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint32_t svread_hor_za128_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint64_t svread_hor_za128_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint16_t svread_hor_za128_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint16_t svread_hor_za16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svbfloat16_t svread_hor_za16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat16_t svread_hor_za16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint16_t svread_hor_za16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint32_t svread_hor_za32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat32_t svread_hor_za32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint32_t svread_hor_za32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint64_t svread_hor_za64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat64_t svread_hor_za64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint64_t svread_hor_za64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint8_t svread_hor_za8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_hor_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint8_t svread_hor_za8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint8_t svread_ver_za128_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint32_t svread_ver_za128_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint64_t svread_ver_za128_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint16_t svread_ver_za128_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svbfloat16_t svread_ver_za128_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint8_t svread_ver_za128_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat64_t svread_ver_za128_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat32_t svread_ver_za128_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat16_t svread_ver_za128_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint32_t svread_ver_za128_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint64_t svread_ver_za128_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za128_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint16_t svread_ver_za128_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_u16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint16_t svread_ver_za16_m(svuint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_bf16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svbfloat16_t svread_ver_za16_m(svbfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_f16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat16_t svread_ver_za16_m(svfloat16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za16_s16_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint16_t svread_ver_za16_m(svint16_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_u32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint32_t svread_ver_za32_m(svuint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_f32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat32_t svread_ver_za32_m(svfloat32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za32_s32_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint32_t svread_ver_za32_m(svint32_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_u64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint64_t svread_ver_za64_m(svuint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_f64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svfloat64_t svread_ver_za64_m(svfloat64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za64_s64_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint64_t svread_ver_za64_m(svint64_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_u8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svuint8_t svread_ver_za8_m(svuint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svread_ver_za8_s8_m), arm_streaming, arm_shared_za, arm_preserves_za))
+svint8_t svread_ver_za8_m(svint8_t, svbool_t, uint64_t, uint32_t, uint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za32_s8_m), arm_streaming, arm_shared_za))
+void svsumopa_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za32_s8_m), arm_streaming, arm_shared_za))
+void svsumops_za32_m(uint64_t, svbool_t, svbool_t, svint8_t, svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za32_u8_m), arm_streaming, arm_shared_za))
+void svusmopa_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za32_u8_m), arm_streaming, arm_shared_za))
+void svusmops_za32_m(uint64_t, svbool_t, svbool_t, svuint8_t, svint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u8_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_u16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_bf16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s8_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_f16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za128_s16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_u16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_bf16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_f16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za16_s16_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_u32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_f32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za32_s32_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_u64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_f64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za64_s64_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_u8_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_hor_za8_s8_m), arm_streaming, arm_shared_za))
+void svwrite_hor_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u8_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_u16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_bf16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s8_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_f16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za128_s16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za128_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_u16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_bf16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svbfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_f16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za16_s16_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za16_m(uint64_t, uint32_t, uint64_t, svbool_t, svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_u32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_f32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za32_s32_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za32_m(uint64_t, uint32_t, uint64_t, svbool_t, svint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_u64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_f64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svfloat64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za64_s64_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za64_m(uint64_t, uint32_t, uint64_t, svbool_t, svint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_u8_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svwrite_ver_za8_s8_m), arm_streaming, arm_shared_za))
+void svwrite_ver_za8_m(uint64_t, uint32_t, uint64_t, svbool_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m), arm_streaming, arm_shared_za))
+void svmopa_za64_f64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m), arm_streaming, arm_shared_za))
+void svmops_za64_f64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_f64_m), arm_streaming, arm_shared_za))
+void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_f64_m), arm_streaming, arm_shared_za))
+void svmops_za64_m(uint64_t, svbool_t, svbool_t, svfloat64_t, svfloat64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_u64_m), arm_streaming, arm_shared_za))
+void svaddha_za64_u64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_s64_m), arm_streaming, arm_shared_za))
+void svaddha_za64_s64_m(uint64_t, svbool_t, svbool_t, svint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_u64_m), arm_streaming, arm_shared_za))
+void svaddva_za64_u64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_s64_m), arm_streaming, arm_shared_za))
+void svaddva_za64_s64_m(uint64_t, svbool_t, svbool_t, svint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_s16_m), arm_streaming, arm_shared_za))
+void svmopa_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_u16_m), arm_streaming, arm_shared_za))
+void svmopa_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_s16_m), arm_streaming, arm_shared_za))
+void svmops_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_u16_m), arm_streaming, arm_shared_za))
+void svmops_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za64_s16_m), arm_streaming, arm_shared_za))
+void svsumopa_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za64_s16_m), arm_streaming, arm_shared_za))
+void svsumops_za64_s16_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za64_u16_m), arm_streaming, arm_shared_za))
+void svusmopa_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za64_u16_m), arm_streaming, arm_shared_za))
+void svusmops_za64_u16_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_u64_m), arm_streaming, arm_shared_za))
+void svaddha_za64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddha_za64_s64_m), arm_streaming, arm_shared_za))
+void svaddha_za64_m(uint64_t, svbool_t, svbool_t, svint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_u64_m), arm_streaming, arm_shared_za))
+void svaddva_za64_m(uint64_t, svbool_t, svbool_t, svuint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svaddva_za64_s64_m), arm_streaming, arm_shared_za))
+void svaddva_za64_m(uint64_t, svbool_t, svbool_t, svint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_s16_m), arm_streaming, arm_shared_za))
+void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmopa_za64_u16_m), arm_streaming, arm_shared_za))
+void svmopa_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_s16_m), arm_streaming, arm_shared_za))
+void svmops_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svmops_za64_u16_m), arm_streaming, arm_shared_za))
+void svmops_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svuint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumopa_za64_s16_m), arm_streaming, arm_shared_za))
+void svsumopa_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svsumops_za64_s16_m), arm_streaming, arm_shared_za))
+void svsumops_za64_m(uint64_t, svbool_t, svbool_t, svint16_t, svuint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmopa_za64_u16_m), arm_streaming, arm_shared_za))
+void svusmopa_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sme_svusmops_za64_u16_m), arm_streaming, arm_shared_za))
+void svusmops_za64_m(uint64_t, svbool_t, svbool_t, svuint16_t, svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svldr_vnum_za), arm_streaming_compatible, arm_shared_za))
+void svldr_vnum_za(uint32_t, uint64_t, void const *);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sme_svstr_vnum_za), arm_streaming_compatible, arm_shared_za, arm_preserves_za))
+void svstr_vnum_za(uint32_t, uint64_t, void *);
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
+#undef __ai
+
+#endif /* __ARM_SME_H */

lib/include/arm_sve.h

@@ -37,7 +37,6 @@ typedef __SVFloat16_t svfloat16_t;
 
 typedef __SVBFloat16_t svbfloat16_t;
 #include <arm_bf16.h>
-typedef __bf16 bfloat16_t;
 typedef __SVFloat32_t svfloat32_t;
 typedef __SVFloat64_t svfloat64_t;
 typedef __clang_svint8x2_t svint8x2_t;
@@ -74,10 +73,14 @@ typedef __clang_svfloat16x4_t svfloat16x4_t;
 typedef __clang_svfloat32x4_t svfloat32x4_t;
 typedef __clang_svfloat64x4_t svfloat64x4_t;
 typedef __SVBool_t  svbool_t;
+typedef __clang_svboolx2_t  svboolx2_t;
+typedef __clang_svboolx4_t  svboolx4_t;
 
 typedef __clang_svbfloat16x2_t svbfloat16x2_t;
 typedef __clang_svbfloat16x3_t svbfloat16x3_t;
 typedef __clang_svbfloat16x4_t svbfloat16x4_t;
+typedef __SVCount_t svcount_t;
+
 enum svpattern
 {
   SV_POW2 = 0,
@@ -2914,6 +2917,10 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s64)))
 svint64_t svdup_lane_s64(svint64_t, uint64_t);
 __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s16)))
 svint16_t svdup_lane_s16(svint16_t, uint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8)))
+svuint8_t svdupq_n_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8)))
+svint8_t svdupq_n_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t);
 __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u16)))
 svuint16_t svdupq_n_u16(uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t);
 __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f16)))
@@ -2932,18 +2939,14 @@ __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f64)))
 svfloat64_t svdupq_n_f64(float64_t, float64_t);
 __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s64)))
 svint64_t svdupq_n_s64(int64_t, int64_t);
-__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8)))
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8)))
-svuint8_t svdupq_n_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t);
+svbool_t svdupq_n_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool);
-__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8)))
-svint8_t svdupq_n_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t);
 __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b16)))
 svbool_t svdupq_n_b16(bool, bool, bool, bool, bool, bool, bool, bool);
 __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b32)))
 svbool_t svdupq_n_b32(bool, bool, bool, bool);
 __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b64)))
 svbool_t svdupq_n_b64(bool, bool);
-__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8)))
-svbool_t svdupq_n_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool);
 __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u8)))
 svuint8_t svdupq_lane_u8(svuint8_t, uint64_t);
 __ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u32)))
@@ -10528,6 +10531,10 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s64)))
 svint64_t svdup_lane(svint64_t, uint64_t);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdup_lane_s16)))
 svint16_t svdup_lane(svint16_t, uint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8)))
+svuint8_t svdupq_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8)))
+svint8_t svdupq_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u16)))
 svuint16_t svdupq_u16(uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t, uint16_t);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f16)))
@@ -10546,18 +10553,14 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_f64)))
 svfloat64_t svdupq_f64(float64_t, float64_t);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s64)))
 svint64_t svdupq_s64(int64_t, int64_t);
-__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_u8)))
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8)))
-svuint8_t svdupq_u8(uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t, uint8_t);
+svbool_t svdupq_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool);
-__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_s8)))
-svint8_t svdupq_s8(int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t, int8_t);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b16)))
 svbool_t svdupq_b16(bool, bool, bool, bool, bool, bool, bool, bool);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b32)))
 svbool_t svdupq_b32(bool, bool, bool, bool);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b64)))
 svbool_t svdupq_b64(bool, bool);
-__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_n_b8)))
-svbool_t svdupq_b8(bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool, bool);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u8)))
 svuint8_t svdupq_lane(svuint8_t, uint64_t);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svdupq_lane_u32)))
@@ -23874,6 +23877,58 @@ __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4e_u32)))
 svuint32_t svsm4e(svuint32_t, svuint32_t);
 __aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svsm4ekey_u32)))
 svuint32_t svsm4ekey(svuint32_t, svuint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f64)))
+svfloat64_t svclamp_f64(svfloat64_t, svfloat64_t, svfloat64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f32)))
+svfloat32_t svclamp_f32(svfloat32_t, svfloat32_t, svfloat32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f16)))
+svfloat16_t svclamp_f16(svfloat16_t, svfloat16_t, svfloat16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s8)))
+svint8_t svclamp_s8(svint8_t, svint8_t, svint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s32)))
+svint32_t svclamp_s32(svint32_t, svint32_t, svint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s64)))
+svint64_t svclamp_s64(svint64_t, svint64_t, svint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s16)))
+svint16_t svclamp_s16(svint16_t, svint16_t, svint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u8)))
+svuint8_t svclamp_u8(svuint8_t, svuint8_t, svuint8_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u32)))
+svuint32_t svclamp_u32(svuint32_t, svuint32_t, svuint32_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u64)))
+svuint64_t svclamp_u64(svuint64_t, svuint64_t, svuint64_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u16)))
+svuint16_t svclamp_u16(svuint16_t, svuint16_t, svuint16_t);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c8)))
+svcount_t svptrue_c8(void);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c32)))
+svcount_t svptrue_c32(void);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c64)))
+svcount_t svptrue_c64(void);
+__ai __attribute__((__clang_arm_builtin_alias(__builtin_sve_svptrue_c16)))
+svcount_t svptrue_c16(void);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f64)))
+svfloat64_t svclamp(svfloat64_t, svfloat64_t, svfloat64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f32)))
+svfloat32_t svclamp(svfloat32_t, svfloat32_t, svfloat32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_f16)))
+svfloat16_t svclamp(svfloat16_t, svfloat16_t, svfloat16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s8)))
+svint8_t svclamp(svint8_t, svint8_t, svint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s32)))
+svint32_t svclamp(svint32_t, svint32_t, svint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s64)))
+svint64_t svclamp(svint64_t, svint64_t, svint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_s16)))
+svint16_t svclamp(svint16_t, svint16_t, svint16_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u8)))
+svuint8_t svclamp(svuint8_t, svuint8_t, svuint8_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u32)))
+svuint32_t svclamp(svuint32_t, svuint32_t, svuint32_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u64)))
+svuint64_t svclamp(svuint64_t, svuint64_t, svuint64_t);
+__aio __attribute__((__clang_arm_builtin_alias(__builtin_sve_svclamp_u16)))
+svuint16_t svclamp(svuint16_t, svuint16_t, svuint16_t);
 #define svcvtnt_bf16_x      svcvtnt_bf16_m
 #define svcvtnt_bf16_f32_x  svcvtnt_bf16_f32_m
 #define svcvtnt_f16_x      svcvtnt_f16_m

lib/include/avx512fintrin.h

@@ -397,14 +397,15 @@ _mm512_broadcastsd_pd(__m128d __A)
 static __inline __m512d __DEFAULT_FN_ATTRS512
 _mm512_castpd256_pd512(__m256d __a)
 {
-  return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, -1, -1, -1, -1);
+  return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), 0,
+                                 1, 2, 3, 4, 5, 6, 7);
 }
 
 static __inline __m512 __DEFAULT_FN_ATTRS512
 _mm512_castps256_ps512(__m256 __a)
 {
-  return __builtin_shufflevector(__a, __a, 0,  1,  2,  3,  4,  5,  6,  7,
+  return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), 0,
-                                          -1, -1, -1, -1, -1, -1, -1, -1);
+                                 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
 }
 
 static __inline __m128d __DEFAULT_FN_ATTRS512
@@ -446,7 +447,10 @@ _mm512_castpd_si512 (__m512d __A)
 static __inline__ __m512d __DEFAULT_FN_ATTRS512
 _mm512_castpd128_pd512 (__m128d __A)
 {
-  return __builtin_shufflevector( __A, __A, 0, 1, -1, -1, -1, -1, -1, -1);
+  __m256d __B = __builtin_nondeterministic_value(__B);
+  return __builtin_shufflevector(
+      __builtin_shufflevector(__A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3),
+      __B, 0, 1, 2, 3, 4, 5, 6, 7);
 }
 
 static __inline __m512d __DEFAULT_FN_ATTRS512
@@ -464,19 +468,25 @@ _mm512_castps_si512 (__m512 __A)
 static __inline__ __m512 __DEFAULT_FN_ATTRS512
 _mm512_castps128_ps512 (__m128 __A)
 {
-    return  __builtin_shufflevector( __A, __A, 0, 1, 2, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
+  __m256 __B = __builtin_nondeterministic_value(__B);
+  return __builtin_shufflevector(
+      __builtin_shufflevector(__A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3, 4, 5, 6, 7),
+      __B, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
 _mm512_castsi128_si512 (__m128i __A)
 {
-   return  __builtin_shufflevector( __A, __A, 0, 1, -1, -1, -1, -1, -1, -1);
+  __m256i __B = __builtin_nondeterministic_value(__B);
+  return __builtin_shufflevector(
+      __builtin_shufflevector(__A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3),
+      __B, 0, 1, 2, 3, 4, 5, 6, 7);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
 _mm512_castsi256_si512 (__m256i __A)
 {
-   return  __builtin_shufflevector( __A, __A, 0, 1, 2, 3, -1, -1, -1, -1);
+   return  __builtin_shufflevector( __A, __builtin_nondeterministic_value(__A), 0, 1, 2, 3, 4, 5, 6, 7);
 }
 
 static __inline __m512 __DEFAULT_FN_ATTRS512

lib/include/avx512fp16intrin.h

@@ -192,22 +192,26 @@ _mm512_castph512_ph256(__m512h __a) {
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_castph128_ph256(__m128h __a) {
-  return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1,
+  return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a),
-                                 -1, -1, -1, -1, -1);
+                                  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
 }
 
 static __inline__ __m512h __DEFAULT_FN_ATTRS512
 _mm512_castph128_ph512(__m128h __a) {
-  return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, -1, -1, -1,
+  __m256h __b = __builtin_nondeterministic_value(__b);
-                                 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+  return __builtin_shufflevector(
-                                 -1, -1, -1, -1, -1, -1, -1, -1, -1);
+      __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a),
+                              0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15),
+      __b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+      20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31);
 }
 
 static __inline__ __m512h __DEFAULT_FN_ATTRS512
 _mm512_castph256_ph512(__m256h __a) {
-  return __builtin_shufflevector(__a, __a, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11,
+  return __builtin_shufflevector(__a, __builtin_nondeterministic_value(__a), 0,
-                                 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1,
+                                 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
-                                 -1, -1, -1, -1, -1, -1, -1, -1);
+                                 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26,
+                                 27, 28, 29, 30, 31);
 }
 
 /// Constructs a 256-bit floating-point vector of [16 x half] from a

lib/include/avxintrin.h

@@ -3017,8 +3017,11 @@ _mm256_zeroupper(void)
 static __inline __m128 __DEFAULT_FN_ATTRS128
 _mm_broadcast_ss(float const *__a)
 {
-  float __f = *__a;
+  struct __mm_broadcast_ss_struct {
-  return __extension__ (__m128)(__v4sf){ __f, __f, __f, __f };
+    float __f;
+  } __attribute__((__packed__, __may_alias__));
+  float __f = ((const struct __mm_broadcast_ss_struct*)__a)->__f;
+  return __extension__ (__m128){ __f, __f, __f, __f };
 }
 
 /// Loads a scalar double-precision floating point value from the
@@ -3036,7 +3039,10 @@ _mm_broadcast_ss(float const *__a)
 static __inline __m256d __DEFAULT_FN_ATTRS
 _mm256_broadcast_sd(double const *__a)
 {
-  double __d = *__a;
+  struct __mm256_broadcast_sd_struct {
+    double __d;
+  } __attribute__((__packed__, __may_alias__));
+  double __d = ((const struct __mm256_broadcast_sd_struct*)__a)->__d;
   return __extension__ (__m256d)(__v4df){ __d, __d, __d, __d };
 }
 
@@ -3055,7 +3061,10 @@ _mm256_broadcast_sd(double const *__a)
 static __inline __m256 __DEFAULT_FN_ATTRS
 _mm256_broadcast_ss(float const *__a)
 {
-  float __f = *__a;
+  struct __mm256_broadcast_ss_struct {
+    float __f;
+  } __attribute__((__packed__, __may_alias__));
+  float __f = ((const struct __mm256_broadcast_ss_struct*)__a)->__f;
   return __extension__ (__m256)(__v8sf){ __f, __f, __f, __f, __f, __f, __f, __f };
 }
 
@@ -4499,7 +4508,8 @@ _mm256_castsi256_si128(__m256i __a)
 static __inline __m256d __DEFAULT_FN_ATTRS
 _mm256_castpd128_pd256(__m128d __a)
 {
-  return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 1, -1, -1);
+  return __builtin_shufflevector(
+      (__v2df)__a, (__v2df)__builtin_nondeterministic_value(__a), 0, 1, 2, 3);
 }
 
 /// Constructs a 256-bit floating-point vector of [8 x float] from a
@@ -4520,7 +4530,9 @@ _mm256_castpd128_pd256(__m128d __a)
 static __inline __m256 __DEFAULT_FN_ATTRS
 _mm256_castps128_ps256(__m128 __a)
 {
-  return __builtin_shufflevector((__v4sf)__a, (__v4sf)__a, 0, 1, 2, 3, -1, -1, -1, -1);
+  return __builtin_shufflevector((__v4sf)__a,
+                                 (__v4sf)__builtin_nondeterministic_value(__a),
+                                 0, 1, 2, 3, 4, 5, 6, 7);
 }
 
 /// Constructs a 256-bit integer vector from a 128-bit integer vector.
@@ -4539,7 +4551,8 @@ _mm256_castps128_ps256(__m128 __a)
 static __inline __m256i __DEFAULT_FN_ATTRS
 _mm256_castsi128_si256(__m128i __a)
 {
-  return __builtin_shufflevector((__v2di)__a, (__v2di)__a, 0, 1, -1, -1);
+  return __builtin_shufflevector(
+      (__v2di)__a, (__v2di)__builtin_nondeterministic_value(__a), 0, 1, 2, 3);
 }
 
 /// Constructs a 256-bit floating-point vector of [4 x double] from a

lib/include/avxvnniint16intrin.h

@@ -0,0 +1,473 @@
+/*===----------- avxvnniint16intrin.h - AVXVNNIINT16 intrinsics-------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error                                                                         \
+    "Never use <avxvnniint16intrin.h> directly; include <immintrin.h> instead."
+#endif // __IMMINTRIN_H
+
+#ifndef __AVXVNNIINT16INTRIN_H
+#define __AVXVNNIINT16INTRIN_H
+
+/* Define the default attributes for the functions in this file. */
+#define __DEFAULT_FN_ATTRS128                                                  \
+  __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"),   \
+                 __min_vector_width__(128)))
+#define __DEFAULT_FN_ATTRS256                                                  \
+  __attribute__((__always_inline__, __nodebug__, __target__("avxvnniint16"),   \
+                 __min_vector_width__(256)))
+
+/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
+///    corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_dpwsud_epi32(__m128i __W, __m128i __A, __m128i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWSUD instruction.
+///
+/// \param __W
+///    A 128-bit vector of [4 x int].
+/// \param __A
+///    A 128-bit vector of [8 x short].
+/// \param __B
+///    A 128-bit vector of [8 x unsigned short].
+/// \returns
+///    A 128-bit vector of [4 x int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 3
+/// 	tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])
+/// 	tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := __W.dword[j] + tmp1 + tmp2
+/// ENDFOR
+/// dst[MAX:128] := 0
+/// \endcode
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsud_epi32(__m128i __W,
+                                                                 __m128i __A,
+                                                                 __m128i __B) {
+  return (__m128i)__builtin_ia32_vpdpwsud128((__v4si)__W, (__v4si)__A,
+                                             (__v4si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
+///    corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWSUD instruction.
+///
+/// \param __W
+///    A 256-bit vector of [8 x int].
+/// \param __A
+///    A 256-bit vector of [16 x short].
+/// \param __B
+///    A 256-bit vector of [16 x unsigned short].
+/// \returns
+///    A 256-bit vector of [8 x int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 7
+/// 	tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])
+/// 	tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := __W.dword[j] + tmp1 + tmp2
+/// ENDFOR
+/// dst[MAX:256] := 0
+/// \endcode
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_dpwsud_epi32(__m256i __W, __m256i __A, __m256i __B) {
+  return (__m256i)__builtin_ia32_vpdpwsud256((__v8si)__W, (__v8si)__A,
+                                             (__v8si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
+///    corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W with signed saturation, and store the packed
+///    32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_dpwsuds_epi32(__m128i __W, __m128i __A, __m128i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
+///
+/// \param __W
+///    A 128-bit vector of [4 x int].
+/// \param __A
+///    A 128-bit vector of [8 x short].
+/// \param __B
+///    A 128-bit vector of [8 x unsigned short].
+/// \returns
+///    A 128-bit vector of [4 x int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 3
+/// 	tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])
+/// 	tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
+/// ENDFOR
+/// dst[MAX:128] := 0
+/// \endcode
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwsuds_epi32(__m128i __W,
+                                                                  __m128i __A,
+                                                                  __m128i __B) {
+  return (__m128i)__builtin_ia32_vpdpwsuds128((__v4si)__W, (__v4si)__A,
+                                              (__v4si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of signed 16-bit integers in \a __A with
+///    corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W with signed saturation, and store the packed
+///    32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
+///
+/// \param __W
+///    A 256-bit vector of [8 x int].
+/// \param __A
+///    A 256-bit vector of [16 x short].
+/// \param __B
+///    A 256-bit vector of [16 x unsigned short].
+/// \returns
+///    A 256-bit vector of [8 x int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 7
+/// 	tmp1.dword := SignExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])
+/// 	tmp2.dword := SignExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
+/// ENDFOR
+/// dst[MAX:256] := 0
+/// \endcode
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
+  return (__m256i)__builtin_ia32_vpdpwsuds256((__v8si)__W, (__v8si)__A,
+                                              (__v8si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
+///    corresponding signed 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_dpbusd_epi32(__m128i __W, __m128i __A, __m128i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWUSD instruction.
+///
+/// \param __W
+///    A 128-bit vector of [4 x int].
+/// \param __A
+///    A 128-bit vector of [8 x unsigned short].
+/// \param __B
+///    A 128-bit vector of [8 x short].
+/// \returns
+///    A 128-bit vector of [4 x int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 3
+/// 	tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])
+/// 	tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := __W.dword[j] + tmp1 + tmp2
+/// ENDFOR
+/// dst[MAX:128] := 0
+/// \endcode
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusd_epi32(__m128i __W,
+                                                                 __m128i __A,
+                                                                 __m128i __B) {
+  return (__m128i)__builtin_ia32_vpdpwusd128((__v4si)__W, (__v4si)__A,
+                                             (__v4si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
+///    corresponding signed 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWUSD instruction.
+///
+/// \param __W
+///    A 256-bit vector of [8 x int].
+/// \param __A
+///    A 256-bit vector of [16 x unsigned short].
+/// \param __B
+///    A 256-bit vector of [16 x short].
+/// \returns
+///    A 256-bit vector of [8 x int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 7
+/// 	tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])
+/// 	tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := __W.dword[j] + tmp1 + tmp2
+/// ENDFOR
+/// dst[MAX:256] := 0
+/// \endcode
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_dpwusd_epi32(__m256i __W, __m256i __A, __m256i __B) {
+  return (__m256i)__builtin_ia32_vpdpwusd256((__v8si)__W, (__v8si)__A,
+                                             (__v8si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
+///    corresponding signed 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W with signed saturation, and store the packed
+///    32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_dpwusds_epi32(__m128i __W, __m128i __A, __m128i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
+///
+/// \param __W
+///    A 128-bit vector of [4 x int].
+/// \param __A
+///    A 128-bit vector of [8 x unsigned short].
+/// \param __B
+///    A 128-bit vector of [8 x short].
+/// \returns
+///    A 128-bit vector of [4 x int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 3
+/// 	tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])
+/// 	tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
+/// ENDFOR
+/// dst[MAX:128] := 0
+/// \endcode
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwusds_epi32(__m128i __W,
+                                                                  __m128i __A,
+                                                                  __m128i __B) {
+  return (__m128i)__builtin_ia32_vpdpwusds128((__v4si)__W, (__v4si)__A,
+                                              (__v4si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
+///    corresponding signed 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W with signed saturation, and store the packed
+///    32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_dpwsuds_epi32(__m256i __W, __m256i __A, __m256i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
+///
+/// \param __W
+///    A 256-bit vector of [8 x int].
+/// \param __A
+///    A 256-bit vector of [16 x unsigned short].
+/// \param __B
+///    A 256-bit vector of [16 x short].
+/// \returns
+///    A 256-bit vector of [8 x int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 7
+/// 	tmp1.dword := ZeroExtend32(__A.word[2*j]) * SignExtend32(__B.word[2*j])
+/// 	tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * SignExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := SIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
+/// ENDFOR
+/// dst[MAX:256] := 0
+/// \endcode
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_dpwusds_epi32(__m256i __W, __m256i __A, __m256i __B) {
+  return (__m256i)__builtin_ia32_vpdpwusds256((__v8si)__W, (__v8si)__A,
+                                              (__v8si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
+///    corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_dpwuud_epi32(__m128i __W, __m128i __A, __m128i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWUUD instruction.
+///
+/// \param __W
+///    A 128-bit vector of [4 x unsigned int].
+/// \param __A
+///    A 128-bit vector of [8 x unsigned short].
+/// \param __B
+///    A 128-bit vector of [8 x unsigned short].
+/// \returns
+///    A 128-bit vector of [4 x unsigned int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 3
+/// 	tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])
+/// 	tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := __W.dword[j] + tmp1 + tmp2
+/// ENDFOR
+/// dst[MAX:128] := 0
+/// \endcode
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuud_epi32(__m128i __W,
+                                                                 __m128i __A,
+                                                                 __m128i __B) {
+  return (__m128i)__builtin_ia32_vpdpwuud128((__v4si)__W, (__v4si)__A,
+                                             (__v4si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
+///    corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W, and store the packed 32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWUUD instruction.
+///
+/// \param __W
+///    A 256-bit vector of [8 x unsigned int].
+/// \param __A
+///    A 256-bit vector of [16 x unsigned short].
+/// \param __B
+///    A 256-bit vector of [16 x unsigned short].
+/// \returns
+///    A 256-bit vector of [8 x unsigned int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 7
+/// 	tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])
+/// 	tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := __W.dword[j] + tmp1 + tmp2
+/// ENDFOR
+/// dst[MAX:256] := 0
+/// \endcode
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_dpwuud_epi32(__m256i __W, __m256i __A, __m256i __B) {
+  return (__m256i)__builtin_ia32_vpdpwuud256((__v8si)__W, (__v8si)__A,
+                                             (__v8si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
+///    corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W with signed saturation, and store the packed
+///    32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_dpwsuds_epi32(__m128i __W, __m128i __A, __m128i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
+///
+/// \param __W
+///    A 128-bit vector of [4 x unsigned int].
+/// \param __A
+///    A 128-bit vector of [8 x unsigned short].
+/// \param __B
+///    A 128-bit vector of [8 x unsigned short].
+/// \returns
+///    A 128-bit vector of [4 x unsigned int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 3
+/// 	tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])
+/// 	tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
+/// ENDFOR
+/// dst[MAX:128] := 0
+/// \endcode
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_dpwuuds_epi32(__m128i __W,
+                                                                  __m128i __A,
+                                                                  __m128i __B) {
+  return (__m128i)__builtin_ia32_vpdpwuuds128((__v4si)__W, (__v4si)__A,
+                                              (__v4si)__B);
+}
+
+/// Multiply groups of 2 adjacent pairs of unsigned 16-bit integers in \a __A with
+///    corresponding unsigned 16-bit integers in \a __B, producing 2 intermediate
+///    signed 16-bit results. Sum these 2 results with the corresponding
+///    32-bit integer in \a __W with signed saturation, and store the packed
+///    32-bit results in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VPDPWSUDS instruction.
+///
+/// \param __W
+///    A 256-bit vector of [8 x unsigned int].
+/// \param __A
+///    A 256-bit vector of [16 x unsigned short].
+/// \param __B
+///    A 256-bit vector of [16 x unsigned short].
+/// \returns
+///    A 256-bit vector of [8 x unsigned int].
+///
+/// \code{.operation}
+/// FOR j := 0 to 7
+/// 	tmp1.dword := ZeroExtend32(__A.word[2*j]) * ZeroExtend32(__B.word[2*j])
+/// 	tmp2.dword := ZeroExtend32(__A.word[2*j+1]) * ZeroExtend32(__B.word[2*j+1])
+/// 	dst.dword[j] := UNSIGNED_DWORD_SATURATE(__W.dword[j] + tmp1 + tmp2)
+/// ENDFOR
+/// dst[MAX:256] := 0
+/// \endcode
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_dpwuuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
+  return (__m256i)__builtin_ia32_vpdpwuuds256((__v8si)__W, (__v8si)__A,
+                                              (__v8si)__B);
+}
+
+#undef __DEFAULT_FN_ATTRS128
+#undef __DEFAULT_FN_ATTRS256
+
+#endif // __AVXVNNIINT16INTRIN_H

lib/include/bmi2intrin.h

@@ -7,8 +7,8 @@
  *===-----------------------------------------------------------------------===
  */
 
-#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H
+#ifndef __IMMINTRIN_H
-#error "Never use <bmi2intrin.h> directly; include <x86intrin.h> instead."
+#error "Never use <bmi2intrin.h> directly; include <immintrin.h> instead."
 #endif
 
 #ifndef __BMI2INTRIN_H
@@ -17,44 +17,228 @@
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("bmi2")))
 
+/// Copies the unsigned 32-bit integer \a __X and zeroes the upper bits
+///    starting at bit number \a __Y.
+///
+/// \code{.operation}
+/// i := __Y[7:0]
+/// result := __X
+/// IF i < 32
+///   result[31:i] := 0
+/// FI
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c BZHI instruction.
+///
+/// \param __X
+///    The 32-bit source value to copy.
+/// \param __Y
+///    The lower 8 bits specify the bit number of the lowest bit to zero.
+/// \returns The partially zeroed 32-bit value.
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 _bzhi_u32(unsigned int __X, unsigned int __Y)
 {
   return __builtin_ia32_bzhi_si(__X, __Y);
 }
 
+/// Deposit (scatter) low-order bits from the unsigned 32-bit integer \a __X
+///    into the 32-bit result, according to the mask in the unsigned 32-bit
+///    integer \a __Y. All other bits of the result are zero.
+///
+/// \code{.operation}
+/// i := 0
+/// result := 0
+/// FOR m := 0 TO 31
+///   IF __Y[m] == 1
+///     result[m] := __X[i]
+///     i := i + 1
+///   ENDIF
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c PDEP instruction.
+///
+/// \param __X
+///    The 32-bit source value to copy.
+/// \param __Y
+///    The 32-bit mask specifying where to deposit source bits.
+/// \returns The 32-bit result.
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 _pdep_u32(unsigned int __X, unsigned int __Y)
 {
   return __builtin_ia32_pdep_si(__X, __Y);
 }
 
+/// Extract (gather) bits from the unsigned 32-bit integer \a __X into the
+///    low-order bits of the 32-bit result, according to the mask in the
+///    unsigned 32-bit integer \a __Y. All other bits of the result are zero.
+///
+/// \code{.operation}
+/// i := 0
+/// result := 0
+/// FOR m := 0 TO 31
+///   IF __Y[m] == 1
+///     result[i] := __X[m]
+///     i := i + 1
+///   ENDIF
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c PEXT instruction.
+///
+/// \param __X
+///    The 32-bit source value to copy.
+/// \param __Y
+///    The 32-bit mask specifying which source bits to extract.
+/// \returns The 32-bit result.
 static __inline__ unsigned int __DEFAULT_FN_ATTRS
 _pext_u32(unsigned int __X, unsigned int __Y)
 {
   return __builtin_ia32_pext_si(__X, __Y);
 }
 
+/// Multiplies the unsigned 32-bit integers \a __X and \a __Y to form a
+///    64-bit product. Stores the upper 32 bits of the product in the
+///    memory at \a __P and returns the lower 32 bits.
+///
+/// \code{.operation}
+/// Store32(__P, (__X * __Y)[63:32])
+/// result := (__X * __Y)[31:0]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c MULX instruction.
+///
+/// \param __X
+///    An unsigned 32-bit multiplicand.
+/// \param __Y
+///    An unsigned 32-bit multiplicand.
+/// \param __P
+///    A pointer to memory for storing the upper half of the product.
+/// \returns The lower half of the product.
+static __inline__ unsigned int __DEFAULT_FN_ATTRS
+_mulx_u32(unsigned int __X, unsigned int __Y, unsigned int *__P)
+{
+  unsigned long long __res = (unsigned long long) __X * __Y;
+  *__P = (unsigned int)(__res >> 32);
+  return (unsigned int)__res;
+}
+
 #ifdef  __x86_64__
 
+/// Copies the unsigned 64-bit integer \a __X and zeroes the upper bits
+///    starting at bit number \a __Y.
+///
+/// \code{.operation}
+/// i := __Y[7:0]
+/// result := __X
+/// IF i < 64
+///   result[63:i] := 0
+/// FI
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c BZHI instruction.
+///
+/// \param __X
+///    The 64-bit source value to copy.
+/// \param __Y
+///    The lower 8 bits specify the bit number of the lowest bit to zero.
+/// \returns The partially zeroed 64-bit value.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 _bzhi_u64(unsigned long long __X, unsigned long long __Y)
 {
   return __builtin_ia32_bzhi_di(__X, __Y);
 }
 
+/// Deposit (scatter) low-order bits from the unsigned 64-bit integer \a __X
+///    into the 64-bit result, according to the mask in the unsigned 64-bit
+///    integer \a __Y. All other bits of the result are zero.
+///
+/// \code{.operation}
+/// i := 0
+/// result := 0
+/// FOR m := 0 TO 63
+///   IF __Y[m] == 1
+///     result[m] := __X[i]
+///     i := i + 1
+///   ENDIF
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c PDEP instruction.
+///
+/// \param __X
+///    The 64-bit source value to copy.
+/// \param __Y
+///    The 64-bit mask specifying where to deposit source bits.
+/// \returns The 64-bit result.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 _pdep_u64(unsigned long long __X, unsigned long long __Y)
 {
   return __builtin_ia32_pdep_di(__X, __Y);
 }
 
+/// Extract (gather) bits from the unsigned 64-bit integer \a __X into the
+///    low-order bits of the 64-bit result, according to the mask in the
+///    unsigned 64-bit integer \a __Y. All other bits of the result are zero.
+///
+/// \code{.operation}
+/// i := 0
+/// result := 0
+/// FOR m := 0 TO 63
+///   IF __Y[m] == 1
+///     result[i] := __X[m]
+///     i := i + 1
+///   ENDIF
+/// ENDFOR
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c PEXT instruction.
+///
+/// \param __X
+///    The 64-bit source value to copy.
+/// \param __Y
+///    The 64-bit mask specifying which source bits to extract.
+/// \returns The 64-bit result.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 _pext_u64(unsigned long long __X, unsigned long long __Y)
 {
   return __builtin_ia32_pext_di(__X, __Y);
 }
 
+/// Multiplies the unsigned 64-bit integers \a __X and \a __Y to form a
+///    128-bit product. Stores the upper 64 bits of the product to the
+///    memory addressed by \a __P and returns the lower 64 bits.
+///
+/// \code{.operation}
+/// Store64(__P, (__X * __Y)[127:64])
+/// result := (__X * __Y)[63:0]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c MULX instruction.
+///
+/// \param __X
+///    An unsigned 64-bit multiplicand.
+/// \param __Y
+///    An unsigned 64-bit multiplicand.
+/// \param __P
+///    A pointer to memory for storing the upper half of the product.
+/// \returns The lower half of the product.
 static __inline__ unsigned long long __DEFAULT_FN_ATTRS
 _mulx_u64 (unsigned long long __X, unsigned long long __Y,
 	   unsigned long long *__P)
@@ -64,17 +248,7 @@ _mulx_u64 (unsigned long long __X, unsigned long long __Y,
   return (unsigned long long) __res;
 }
 
-#else /* !__x86_64__ */
+#endif /* __x86_64__  */
-
-static __inline__ unsigned int __DEFAULT_FN_ATTRS
-_mulx_u32 (unsigned int __X, unsigned int __Y, unsigned int *__P)
-{
-  unsigned long long __res = (unsigned long long) __X * __Y;
-  *__P = (unsigned int) (__res >> 32);
-  return (unsigned int) __res;
-}
-
-#endif /* !__x86_64__  */
 
 #undef __DEFAULT_FN_ATTRS
 

lib/include/clflushoptintrin.h

@@ -17,6 +17,15 @@
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__,  __target__("clflushopt")))
 
+/// Invalidates all levels of the cache hierarchy and flushes modified data to
+///    memory for the cache line specified by the address \a __m.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c CLFLUSHOPT instruction.
+///
+/// \param __m
+///    An address within the cache line to flush and invalidate.
 static __inline__ void __DEFAULT_FN_ATTRS
 _mm_clflushopt(void const * __m) {
   __builtin_ia32_clflushopt(__m);

lib/include/clzerointrin.h

@@ -6,7 +6,7 @@
  *
  *===-----------------------------------------------------------------------===
  */
-#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H
+#ifndef __X86INTRIN_H
 #error "Never use <clzerointrin.h> directly; include <x86intrin.h> instead."
 #endif
 
@@ -17,14 +17,16 @@
 #define __DEFAULT_FN_ATTRS \
   __attribute__((__always_inline__, __nodebug__,  __target__("clzero")))
 
-/// Loads the cache line address and zero's out the cacheline
+/// Zeroes out the cache line for the address \a __line. This uses a
+///    non-temporal store. Calling \c _mm_sfence() afterward might be needed
+///    to enforce ordering.
 ///
-/// \headerfile <clzerointrin.h>
+/// \headerfile <x86intrin.h>
 ///
-/// This intrinsic corresponds to the <c> CLZERO </c> instruction.
+/// This intrinsic corresponds to the \c CLZERO instruction.
 ///
 /// \param __line
-///    A pointer to a cacheline which needs to be zeroed out.
+///    An address within the cache line to zero out.
 static __inline__ void __DEFAULT_FN_ATTRS
 _mm_clzero (void * __line)
 {

lib/include/cuda_wrappers/bits/shared_ptr_base.h

@@ -0,0 +1,9 @@
+// CUDA headers define __noinline__ which interferes with libstdc++'s use of
+// `__attribute((__noinline__))`. In order to avoid compilation error,
+// temporarily unset __noinline__ when we include affected libstdc++ header.
+
+#pragma push_macro("__noinline__")
+#undef __noinline__
+#include_next "bits/shared_ptr_base.h"
+
+#pragma pop_macro("__noinline__")

lib/include/fmaintrin.h

@@ -18,192 +18,756 @@
 #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(128)))
 #define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(256)))
 
+/// Computes a multiply-add of 128-bit vectors of [4 x float].
+///    For each element, computes <c> (__A * __B) + __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADD213PS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the addend.
+/// \returns A 128-bit vector of [4 x float] containing the result.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fmadd_ps(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
 }
 
+/// Computes a multiply-add of 128-bit vectors of [2 x double].
+///    For each element, computes <c> (__A * __B) + __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADD213PD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the addend.
+/// \returns A 128-bit [2 x double] vector containing the result.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fmadd_pd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, (__v2df)__C);
 }
 
+/// Computes a scalar multiply-add of the single-precision values in the
+///    low 32 bits of 128-bit vectors of [4 x float].
+/// \code
+/// result[31:0] = (__A[31:0] * __B[31:0]) + __C[31:0]
+/// result[127:32] = __A[127:32]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADD213SS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand in the low
+///    32 bits.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier in the low
+///    32 bits.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the addend in the low
+///    32 bits.
+/// \returns A 128-bit vector of [4 x float] containing the result in the low
+///    32 bits and a copy of \a __A[127:32] in the upper 96 bits.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fmadd_ss(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
 }
 
+/// Computes a scalar multiply-add of the double-precision values in the
+///    low 64 bits of 128-bit vectors of [2 x double].
+/// \code
+/// result[63:0] = (__A[63:0] * __B[63:0]) + __C[63:0]
+/// result[127:64] = __A[127:64]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADD213SD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand in the low
+///    64 bits.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier in the low
+///    64 bits.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the addend in the low
+///    64 bits.
+/// \returns A 128-bit vector of [2 x double] containing the result in the low
+///    64 bits and a copy of \a __A[127:64] in the upper 64 bits.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fmadd_sd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, (__v2df)__C);
 }
 
+/// Computes a multiply-subtract of 128-bit vectors of [4 x float].
+///    For each element, computes <c> (__A * __B) - __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMSUB213PS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the subtrahend.
+/// \returns A 128-bit vector of [4 x float] containing the result.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fmsub_ps(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
 }
 
+/// Computes a multiply-subtract of 128-bit vectors of [2 x double].
+///    For each element, computes <c> (__A * __B) - __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMSUB213PD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the addend.
+/// \returns A 128-bit vector of [2 x double] containing the result.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fmsub_pd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
 }
 
+/// Computes a scalar multiply-subtract of the single-precision values in
+///    the low 32 bits of 128-bit vectors of [4 x float].
+/// \code
+/// result[31:0] = (__A[31:0] * __B[31:0]) - __C[31:0]
+/// result[127:32] = __A[127:32]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMSUB213SS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand in the low
+///    32 bits.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier in the low
+///    32 bits.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the subtrahend in the low
+///   32 bits.
+/// \returns A 128-bit vector of [4 x float] containing the result in the low
+///    32 bits, and a copy of \a __A[127:32] in the upper 96 bits.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fmsub_ss(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
 }
 
+/// Computes a scalar multiply-subtract of the double-precision values in
+///    the low 64 bits of 128-bit vectors of [2 x double].
+/// \code
+/// result[63:0] = (__A[63:0] * __B[63:0]) - __C[63:0]
+/// result[127:64] = __A[127:64]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMSUB213SD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand in the low
+///    64 bits.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier in the low
+///    64 bits.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the subtrahend in the low
+///    64 bits.
+/// \returns A 128-bit vector of [2 x double] containing the result in the low
+///    64 bits, and a copy of \a __A[127:64] in the upper 64 bits.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fmsub_sd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, (__v2df)__B, -(__v2df)__C);
 }
 
+/// Computes a negated multiply-add of 128-bit vectors of [4 x float].
+///    For each element, computes <c> -(__A * __B) + __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMADD213DPS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the addend.
+/// \returns A 128-bit [4 x float] vector containing the result.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
 }
 
+/// Computes a negated multiply-add of 128-bit vectors of [2 x double].
+///    For each element, computes <c> -(__A * __B) + __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMADD213PD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the addend.
+/// \returns A 128-bit vector of [2 x double] containing the result.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, (__v2df)__C);
 }
 
+/// Computes a scalar negated multiply-add of the single-precision values in
+///    the low 32 bits of 128-bit vectors of [4 x float].
+/// \code
+/// result[31:0] = -(__A[31:0] * __B[31:0]) + __C[31:0]
+/// result[127:32] = __A[127:32]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMADD213SS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand in the low
+///    32 bits.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier in the low
+///    32 bits.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the addend in the low
+///    32 bits.
+/// \returns A 128-bit vector of [4 x float] containing the result in the low
+///    32 bits, and a copy of \a __A[127:32] in the upper 96 bits.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fnmadd_ss(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, (__v4sf)__C);
 }
 
+/// Computes a scalar negated multiply-add of the double-precision values
+///    in the low 64 bits of 128-bit vectors of [2 x double].
+/// \code
+/// result[63:0] = -(__A[63:0] * __B[63:0]) + __C[63:0]
+/// result[127:64] = __A[127:64]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMADD213SD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand in the low
+///    64 bits.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier in the low
+///    64 bits.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the addend in the low
+///    64 bits.
+/// \returns A 128-bit vector of [2 x double] containing the result in the low
+///    64 bits, and a copy of \a __A[127:64] in the upper 64 bits.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fnmadd_sd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, (__v2df)__C);
 }
 
+/// Computes a negated multiply-subtract of 128-bit vectors of [4 x float].
+///    For each element, computes <c> -(__A * __B) - __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMSUB213PS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the subtrahend.
+/// \returns A 128-bit vector of [4 x float] containing the result.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
 }
 
+/// Computes a negated multiply-subtract of 128-bit vectors of [2 x double].
+///    For each element, computes <c> -(__A * __B) - __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMSUB213PD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the subtrahend.
+/// \returns A 128-bit vector of [2 x double] containing the result.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, -(__v2df)__C);
 }
 
+/// Computes a scalar negated multiply-subtract of the single-precision
+///    values in the low 32 bits of 128-bit vectors of [4 x float].
+/// \code
+/// result[31:0] = -(__A[31:0] * __B[31:0]) - __C[31:0]
+/// result[127:32] = __A[127:32]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMSUB213SS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand in the low
+///    32 bits.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier in the low
+///    32 bits.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the subtrahend in the low
+///    32 bits.
+/// \returns A 128-bit vector of [4 x float] containing the result in the low
+///    32 bits, and a copy of \a __A[127:32] in the upper 96 bits.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fnmsub_ss(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddss3((__v4sf)__A, -(__v4sf)__B, -(__v4sf)__C);
 }
 
+/// Computes a scalar negated multiply-subtract of the double-precision
+///    values in the low 64 bits of 128-bit vectors of [2 x double].
+/// \code
+/// result[63:0] = -(__A[63:0] * __B[63:0]) - __C[63:0]
+/// result[127:64] = __A[127:64]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMSUB213SD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand in the low
+///    64 bits.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier in the low
+///    64 bits.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the subtrahend in the low
+///    64 bits.
+/// \returns A 128-bit vector of [2 x double] containing the result in the low
+///    64 bits, and a copy of \a __A[127:64] in the upper 64 bits.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fnmsub_sd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddsd3((__v2df)__A, -(__v2df)__B, -(__v2df)__C);
 }
 
+/// Computes a multiply with alternating add/subtract of 128-bit vectors of
+///    [4 x float].
+/// \code
+/// result[31:0]  = (__A[31:0] * __B[31:0]) - __C[31:0]
+/// result[63:32] = (__A[63:32] * __B[63:32]) + __C[63:32]
+/// result[95:64] = (__A[95:64] * __B[95:64]) - __C[95:64]
+/// result[127:96] = (__A[127:96] * __B[127:96]) + __C[127:96]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADDSUB213PS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the addend/subtrahend.
+/// \returns A 128-bit vector of [4 x float] containing the result.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fmaddsub_ps(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddsubps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
 }
 
+/// Computes a multiply with alternating add/subtract of 128-bit vectors of
+///    [2 x double].
+/// \code
+/// result[63:0]  = (__A[63:0] * __B[63:0]) - __C[63:0]
+/// result[127:64] = (__A[127:64] * __B[127:64]) + __C[127:64]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADDSUB213PD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the addend/subtrahend.
+/// \returns A 128-bit vector of [2 x double] containing the result.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fmaddsub_pd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, (__v2df)__C);
 }
 
+/// Computes a multiply with alternating add/subtract of 128-bit vectors of
+///    [4 x float].
+/// \code
+/// result[31:0]  = (__A[31:0] * __B[31:0]) + __C[31:0]
+/// result[63:32] = (__A[63:32] * __B[63:32]) - __C[63:32]
+/// result[95:64] = (__A[95:64] * __B[95:64]) + __C[95:64]
+/// result[127:96 = (__A[127:96] * __B[127:96]) - __C[127:96]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMSUBADD213PS instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x float] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [4 x float] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [4 x float] containing the addend/subtrahend.
+/// \returns A 128-bit vector of [4 x float] containing the result.
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_fmsubadd_ps(__m128 __A, __m128 __B, __m128 __C)
 {
   return (__m128)__builtin_ia32_vfmaddsubps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
 }
 
+/// Computes a multiply with alternating add/subtract of 128-bit vectors of
+///    [2 x double].
+/// \code
+/// result[63:0]  = (__A[63:0] * __B[63:0]) + __C[63:0]
+/// result[127:64] = (__A[127:64] * __B[127:64]) - __C[127:64]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADDSUB213PD instruction.
+///
+/// \param __A
+///    A 128-bit vector of [2 x double] containing the multiplicand.
+/// \param __B
+///    A 128-bit vector of [2 x double] containing the multiplier.
+/// \param __C
+///    A 128-bit vector of [2 x double] containing the addend/subtrahend.
+/// \returns A 128-bit vector of [2 x double] containing the result.
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_fmsubadd_pd(__m128d __A, __m128d __B, __m128d __C)
 {
   return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
 }
 
+/// Computes a multiply-add of 256-bit vectors of [8 x float].
+///    For each element, computes <c> (__A * __B) + __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADD213PS instruction.
+///
+/// \param __A
+///    A 256-bit vector of [8 x float] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [8 x float] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [8 x float] containing the addend.
+/// \returns A 256-bit vector of [8 x float] containing the result.
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_fmadd_ps(__m256 __A, __m256 __B, __m256 __C)
 {
   return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
 }
 
+/// Computes a multiply-add of 256-bit vectors of [4 x double].
+///    For each element, computes <c> (__A * __B) + __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADD213PD instruction.
+///
+/// \param __A
+///    A 256-bit vector of [4 x double] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [4 x double] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [4 x double] containing the addend.
+/// \returns A 256-bit vector of [4 x double] containing the result.
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_fmadd_pd(__m256d __A, __m256d __B, __m256d __C)
 {
   return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, (__v4df)__C);
 }
 
+/// Computes a multiply-subtract of 256-bit vectors of [8 x float].
+///    For each element, computes <c> (__A * __B) - __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMSUB213PS instruction.
+///
+/// \param __A
+///    A 256-bit vector of [8 x float] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [8 x float] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [8 x float] containing the subtrahend.
+/// \returns A 256-bit vector of [8 x float] containing the result.
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_fmsub_ps(__m256 __A, __m256 __B, __m256 __C)
 {
   return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
 }
 
+/// Computes a multiply-subtract of 256-bit vectors of [4 x double].
+///    For each element, computes <c> (__A * __B) - __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMSUB213PD instruction.
+///
+/// \param __A
+///    A 256-bit vector of [4 x double] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [4 x double] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [4 x double] containing the subtrahend.
+/// \returns A 256-bit vector of [4 x double] containing the result.
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_fmsub_pd(__m256d __A, __m256d __B, __m256d __C)
 {
   return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, -(__v4df)__C);
 }
 
+/// Computes a negated multiply-add of 256-bit vectors of [8 x float].
+///    For each element, computes <c> -(__A * __B) + __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMADD213PS instruction.
+///
+/// \param __A
+///    A 256-bit vector of [8 x float] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [8 x float] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [8 x float] containing the addend.
+/// \returns A 256-bit vector of [8 x float] containing the result.
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C)
 {
   return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
 }
 
+/// Computes a negated multiply-add of 256-bit vectors of [4 x double].
+///    For each element, computes <c> -(__A * __B) + __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMADD213PD instruction.
+///
+/// \param __A
+///    A 256-bit vector of [4 x double] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [4 x double] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [4 x double] containing the addend.
+/// \returns A 256-bit vector of [4 x double] containing the result.
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C)
 {
   return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, (__v4df)__C);
 }
 
+/// Computes a negated multiply-subtract of 256-bit vectors of [8 x float].
+///    For each element, computes <c> -(__A * __B) - __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMSUB213PS instruction.
+///
+/// \param __A
+///    A 256-bit vector of [8 x float] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [8 x float] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [8 x float] containing the subtrahend.
+/// \returns A 256-bit vector of [8 x float] containing the result.
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C)
 {
   return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
 }
 
+/// Computes a negated multiply-subtract of 256-bit vectors of [4 x double].
+///    For each element, computes <c> -(__A * __B) - __C </c>.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFNMSUB213PD instruction.
+///
+/// \param __A
+///    A 256-bit vector of [4 x double] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [4 x double] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [4 x double] containing the subtrahend.
+/// \returns A 256-bit vector of [4 x double] containing the result.
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C)
 {
   return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, -(__v4df)__C);
 }
 
+/// Computes a multiply with alternating add/subtract of 256-bit vectors of
+///    [8 x float].
+/// \code
+/// result[31:0] = (__A[31:0] * __B[31:0]) - __C[31:0]
+/// result[63:32] = (__A[63:32] * __B[63:32]) + __C[63:32]
+/// result[95:64] = (__A[95:64] * __B[95:64]) - __C[95:64]
+/// result[127:96] = (__A[127:96] * __B[127:96]) + __C[127:96]
+/// result[159:128] = (__A[159:128] * __B[159:128]) - __C[159:128]
+/// result[191:160] = (__A[191:160] * __B[191:160]) + __C[191:160]
+/// result[223:192] = (__A[223:192] * __B[223:192]) - __C[223:192]
+/// result[255:224] = (__A[255:224] * __B[255:224]) + __C[255:224]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADDSUB213PS instruction.
+///
+/// \param __A
+///    A 256-bit vector of [8 x float] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [8 x float] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [8 x float] containing the addend/subtrahend.
+/// \returns A 256-bit vector of [8 x float] containing the result.
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_fmaddsub_ps(__m256 __A, __m256 __B, __m256 __C)
 {
   return (__m256)__builtin_ia32_vfmaddsubps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
 }
 
+/// Computes a multiply with alternating add/subtract of 256-bit vectors of
+///    [4 x double].
+/// \code
+/// result[63:0] = (__A[63:0] * __B[63:0]) - __C[63:0]
+/// result[127:64] = (__A[127:64] * __B[127:64]) + __C[127:64]
+/// result[191:128] = (__A[191:128] * __B[191:128]) - __C[191:128]
+/// result[255:192] = (__A[255:192] * __B[255:192]) + __C[255:192]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMADDSUB213PD instruction.
+///
+/// \param __A
+///    A 256-bit vector of [4 x double] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [4 x double] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [4 x double] containing the addend/subtrahend.
+/// \returns A 256-bit vector of [4 x double] containing the result.
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_fmaddsub_pd(__m256d __A, __m256d __B, __m256d __C)
 {
   return (__m256d)__builtin_ia32_vfmaddsubpd256((__v4df)__A, (__v4df)__B, (__v4df)__C);
 }
 
+/// Computes a vector multiply with alternating add/subtract of 256-bit
+///    vectors of [8 x float].
+/// \code
+/// result[31:0] = (__A[31:0] * __B[31:0]) + __C[31:0]
+/// result[63:32] = (__A[63:32] * __B[63:32]) - __C[63:32]
+/// result[95:64] = (__A[95:64] * __B[95:64]) + __C[95:64]
+/// result[127:96] = (__A[127:96] * __B[127:96]) - __C[127:96]
+/// result[159:128] = (__A[159:128] * __B[159:128]) + __C[159:128]
+/// result[191:160] = (__A[191:160] * __B[191:160]) - __C[191:160]
+/// result[223:192] = (__A[223:192] * __B[223:192]) + __C[223:192]
+/// result[255:224] = (__A[255:224] * __B[255:224]) - __C[255:224]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMSUBADD213PS instruction.
+///
+/// \param __A
+///    A 256-bit vector of [8 x float] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [8 x float] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [8 x float] containing the addend/subtrahend.
+/// \returns A 256-bit vector of [8 x float] containing the result.
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_fmsubadd_ps(__m256 __A, __m256 __B, __m256 __C)
 {
   return (__m256)__builtin_ia32_vfmaddsubps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
 }
 
+/// Computes a vector multiply with alternating add/subtract of 256-bit
+///    vectors of [4 x double].
+/// \code
+/// result[63:0] = (__A[63:0] * __B[63:0]) + __C[63:0]
+/// result[127:64] = (__A[127:64] * __B[127:64]) - __C[127:64]
+/// result[191:128] = (__A[191:128] * __B[191:128]) + __C[191:128]
+/// result[255:192] = (__A[255:192] * __B[255:192]) - __C[255:192]
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c VFMSUBADD213PD instruction.
+///
+/// \param __A
+///    A 256-bit vector of [4 x double] containing the multiplicand.
+/// \param __B
+///    A 256-bit vector of [4 x double] containing the multiplier.
+/// \param __C
+///    A 256-bit vector of [4 x double] containing the addend/subtrahend.
+/// \returns A 256-bit vector of [4 x double] containing the result.
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_fmsubadd_pd(__m256d __A, __m256d __B, __m256d __C)
 {

lib/include/immintrin.h

@@ -269,6 +269,26 @@
 #include <avxneconvertintrin.h>
 #endif
 
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__SHA512__)
+#include <sha512intrin.h>
+#endif
+
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__SM3__)
+#include <sm3intrin.h>
+#endif
+
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__SM4__)
+#include <sm4intrin.h>
+#endif
+
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__AVXVNNIINT16__)
+#include <avxvnniint16intrin.h>
+#endif
+
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__RDPID__)
 /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
@@ -284,30 +304,53 @@ _rdpid_u32(void) {
 
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__RDRND__)
+/// Returns a 16-bit hardware-generated random value.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> RDRAND </c> instruction.
+///
+/// \param __p
+///    A pointer to a 16-bit memory location to place the random value.
+/// \returns 1 if the value was successfully generated, 0 otherwise.
 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 _rdrand16_step(unsigned short *__p)
 {
   return (int)__builtin_ia32_rdrand16_step(__p);
 }
 
+/// Returns a 32-bit hardware-generated random value.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> RDRAND </c> instruction.
+///
+/// \param __p
+///    A pointer to a 32-bit memory location to place the random value.
+/// \returns 1 if the value was successfully generated, 0 otherwise.
 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 _rdrand32_step(unsigned int *__p)
 {
   return (int)__builtin_ia32_rdrand32_step(__p);
 }
 
-#ifdef __x86_64__
+/// Returns a 64-bit hardware-generated random value.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> RDRAND </c> instruction.
+///
+/// \param __p
+///    A pointer to a 64-bit memory location to place the random value.
+/// \returns 1 if the value was successfully generated, 0 otherwise.
 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 _rdrand64_step(unsigned long long *__p)
 {
+#ifdef __x86_64__
   return (int)__builtin_ia32_rdrand64_step(__p);
-}
 #else
   // We need to emulate the functionality of 64-bit rdrand with 2 32-bit
   // rdrand instructions.
-static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
-_rdrand64_step(unsigned long long *__p)
-{
   unsigned int __lo, __hi;
   unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo);
   unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi);
@@ -318,55 +361,115 @@ _rdrand64_step(unsigned long long *__p)
     *__p = 0;
     return 0;
   }
-}
 #endif
+}
 #endif /* __RDRND__ */
 
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__FSGSBASE__)
 #ifdef __x86_64__
+/// Reads the FS base register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> RDFSBASE </c> instruction.
+///
+/// \returns The lower 32 bits of the FS base register.
 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 _readfsbase_u32(void)
 {
   return __builtin_ia32_rdfsbase32();
 }
 
+/// Reads the FS base register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> RDFSBASE </c> instruction.
+///
+/// \returns The contents of the FS base register.
 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 _readfsbase_u64(void)
 {
   return __builtin_ia32_rdfsbase64();
 }
 
+/// Reads the GS base register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> RDGSBASE </c> instruction.
+///
+/// \returns The lower 32 bits of the GS base register.
 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 _readgsbase_u32(void)
 {
   return __builtin_ia32_rdgsbase32();
 }
 
+/// Reads the GS base register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> RDGSBASE </c> instruction.
+///
+/// \returns The contents of the GS base register.
 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 _readgsbase_u64(void)
 {
   return __builtin_ia32_rdgsbase64();
 }
 
+/// Modifies the FS base register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.
+///
+/// \param __V
+///    Value to use for the lower 32 bits of the FS base register.
 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 _writefsbase_u32(unsigned int __V)
 {
   __builtin_ia32_wrfsbase32(__V);
 }
 
+/// Modifies the FS base register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.
+///
+/// \param __V
+///    Value to use for the FS base register.
 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 _writefsbase_u64(unsigned long long __V)
 {
   __builtin_ia32_wrfsbase64(__V);
 }
 
+/// Modifies the GS base register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> WRGSBASE </c> instruction.
+///
+/// \param __V
+///    Value to use for the lower 32 bits of the GS base register.
 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 _writegsbase_u32(unsigned int __V)
 {
   __builtin_ia32_wrgsbase32(__V);
 }
 
+/// Modifies the GS base register.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.
+///
+/// \param __V
+///    Value to use for GS base register.
 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 _writegsbase_u64(unsigned long long __V)
 {
@@ -538,6 +641,11 @@ _storebe_i64(void * __P, long long __D) {
 #include <amxintrin.h>
 #endif
 
+#if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
+    defined(__AMX_COMPLEX__)
+#include <amxcomplexintrin.h>
+#endif
+
 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
     defined(__AVX512VP2INTERSECT__)
 #include <avx512vp2intersectintrin.h>

lib/include/limits.h

@@ -52,7 +52,11 @@
 #define LONG_MIN  (-__LONG_MAX__ -1L)
 
 #define UCHAR_MAX (__SCHAR_MAX__*2  +1)
+#if __SHRT_WIDTH__ < __INT_WIDTH__
 #define USHRT_MAX (__SHRT_MAX__ * 2 + 1)
+#else
+#define USHRT_MAX (__SHRT_MAX__ * 2U + 1U)
+#endif
 #define UINT_MAX  (__INT_MAX__  *2U +1U)
 #define ULONG_MAX (__LONG_MAX__ *2UL+1UL)
 

lib/include/llvm_libc_wrappers/ctype.h

@@ -0,0 +1,85 @@
+//===-- Wrapper for C standard ctype.h declarations on the GPU ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__
+#define __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__
+
+#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
+#error "This file is for GPU offloading compilation only"
+#endif
+
+#include_next <ctype.h>
+
+#if __has_include(<llvm-libc-decls/ctype.h>)
+
+#if defined(__HIP__) || defined(__CUDA__)
+#define __LIBC_ATTRS __attribute__((device))
+#endif
+
+// The GNU headers like to provide these as macros, we need to undefine them so
+// they do not conflict with the following definitions for the GPU.
+
+#pragma push_macro("isalnum")
+#pragma push_macro("isalpha")
+#pragma push_macro("isblank")
+#pragma push_macro("iscntrl")
+#pragma push_macro("isdigit")
+#pragma push_macro("isgraph")
+#pragma push_macro("islower")
+#pragma push_macro("isprint")
+#pragma push_macro("ispunct")
+#pragma push_macro("isspace")
+#pragma push_macro("isupper")
+#pragma push_macro("isxdigit")
+#pragma push_macro("tolower")
+#pragma push_macro("toupper")
+
+#undef isalnum
+#undef isalpha
+#undef iscntrl
+#undef isdigit
+#undef islower
+#undef isgraph
+#undef isprint
+#undef ispunct
+#undef isspace
+#undef isupper
+#undef isblank
+#undef isxdigit
+#undef tolower
+#undef toupper
+
+#pragma omp begin declare target
+
+#include <llvm-libc-decls/ctype.h>
+
+#pragma omp end declare target
+
+// Restore the original macros when compiling on the host.
+#if !defined(__NVPTX__) && !defined(__AMDGPU__)
+#pragma pop_macro("isalnum")
+#pragma pop_macro("isalpha")
+#pragma pop_macro("isblank")
+#pragma pop_macro("iscntrl")
+#pragma pop_macro("isdigit")
+#pragma pop_macro("isgraph")
+#pragma pop_macro("islower")
+#pragma pop_macro("isprint")
+#pragma pop_macro("ispunct")
+#pragma pop_macro("isspace")
+#pragma pop_macro("isupper")
+#pragma pop_macro("isxdigit")
+#pragma pop_macro("tolower")
+#pragma pop_macro("toupper")
+#endif
+
+#undef __LIBC_ATTRS
+
+#endif
+
+#endif // __CLANG_LLVM_LIBC_WRAPPERS_CTYPE_H__

lib/include/llvm_libc_wrappers/inttypes.h

@@ -0,0 +1,34 @@
+//===-- Wrapper for C standard inttypes.h declarations on the GPU ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLANG_LLVM_LIBC_WRAPPERS_INTTYPES_H__
+#define __CLANG_LLVM_LIBC_WRAPPERS_INTTYPES_H__
+
+#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
+#error "This file is for GPU offloading compilation only"
+#endif
+
+#include_next <inttypes.h>
+
+#if __has_include(<llvm-libc-decls/inttypes.h>)
+
+#if defined(__HIP__) || defined(__CUDA__)
+#define __LIBC_ATTRS __attribute__((device))
+#endif
+
+#pragma omp begin declare target
+
+#include <llvm-libc-decls/inttypes.h>
+
+#pragma omp end declare target
+
+#undef __LIBC_ATTRS
+
+#endif
+
+#endif // __CLANG_LLVM_LIBC_WRAPPERS_INTTYPES_H__

lib/include/llvm_libc_wrappers/stdio.h

@@ -0,0 +1,34 @@
+//===-- Wrapper for C standard stdio.h declarations on the GPU ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
+#define __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__
+
+#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
+#error "This file is for GPU offloading compilation only"
+#endif
+
+#include_next <stdio.h>
+
+#if __has_include(<llvm-libc-decls/stdio.h>)
+
+#if defined(__HIP__) || defined(__CUDA__)
+#define __LIBC_ATTRS __attribute__((device))
+#endif
+
+#pragma omp begin declare target
+
+#include <llvm-libc-decls/stdio.h>
+
+#pragma omp end declare target
+
+#undef __LIBC_ATTRS
+
+#endif
+
+#endif // __CLANG_LLVM_LIBC_WRAPPERS_STDIO_H__

lib/include/llvm_libc_wrappers/stdlib.h

@@ -0,0 +1,42 @@
+//===-- Wrapper for C standard stdlib.h declarations on the GPU -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__
+#define __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__
+
+#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
+#error "This file is for GPU offloading compilation only"
+#endif
+
+#include_next <stdlib.h>
+
+#if __has_include(<llvm-libc-decls/stdlib.h>)
+
+#if defined(__HIP__) || defined(__CUDA__)
+#define __LIBC_ATTRS __attribute__((device))
+#endif
+
+#pragma omp begin declare target
+
+// The LLVM C library uses this type so we forward declare it.
+typedef void (*__atexithandler_t)(void);
+
+// Enforce ABI compatibility with the structs used by the LLVM C library.
+_Static_assert(__builtin_offsetof(div_t, quot) == 0, "ABI mismatch!");
+_Static_assert(__builtin_offsetof(ldiv_t, quot) == 0, "ABI mismatch!");
+_Static_assert(__builtin_offsetof(lldiv_t, quot) == 0, "ABI mismatch!");
+
+#include <llvm-libc-decls/stdlib.h>
+
+#pragma omp end declare target
+
+#undef __LIBC_ATTRS
+
+#endif
+
+#endif // __CLANG_LLVM_LIBC_WRAPPERS_STDLIB_H__

lib/include/llvm_libc_wrappers/string.h

@@ -0,0 +1,37 @@
+//===-- Wrapper for C standard string.h declarations on the GPU -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__
+#define __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__
+
+#if !defined(_OPENMP) && !defined(__HIP__) && !defined(__CUDA__)
+#error "This file is for GPU offloading compilation only"
+#endif
+
+// FIXME: The GNU headers provide C++ standard compliant headers when in C++
+// mode and the LLVM libc does not. We cannot enable memchr, strchr, strchrnul,
+// strpbrk, strrchr, strstr, or strcasestr until this is addressed.
+#include_next <string.h>
+
+#if __has_include(<llvm-libc-decls/string.h>)
+
+#if defined(__HIP__) || defined(__CUDA__)
+#define __LIBC_ATTRS __attribute__((device))
+#endif
+
+#pragma omp begin declare target
+
+#include <llvm-libc-decls/string.h>
+
+#pragma omp end declare target
+
+#undef __LIBC_ATTRS
+
+#endif
+
+#endif // __CLANG_LLVM_LIBC_WRAPPERS_STRING_H__

lib/include/mwaitxintrin.h

@@ -16,12 +16,41 @@
 
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__,  __target__("mwaitx")))
+
+/// Establishes a linear address memory range to be monitored and puts
+///    the processor in the monitor event pending state. Data stored in the
+///    monitored address range causes the processor to exit the pending state.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the \c MONITORX instruction.
+///
+/// \param __p
+///    The memory range to be monitored. The size of the range is determined by
+///    CPUID function 0000_0005h.
+/// \param __extensions
+///    Optional extensions for the monitoring state.
+/// \param __hints
+///    Optional hints for the monitoring state.
 static __inline__ void __DEFAULT_FN_ATTRS
 _mm_monitorx(void * __p, unsigned __extensions, unsigned __hints)
 {
   __builtin_ia32_monitorx(__p, __extensions, __hints);
 }
 
+/// Used with the \c MONITORX instruction to wait while the processor is in
+///    the monitor event pending state. Data stored in the monitored address
+///    range, or an interrupt, causes the processor to exit the pending state.
+///
+/// \headerfile <x86intrin.h>
+///
+/// This intrinsic corresponds to the \c MWAITX instruction.
+///
+/// \param __extensions
+///    Optional extensions for the monitoring state, which can vary by
+///    processor.
+/// \param __hints
+///    Optional hints for the monitoring state, which can vary by processor.
 static __inline__ void __DEFAULT_FN_ATTRS
 _mm_mwaitx(unsigned __extensions, unsigned __hints, unsigned __clock)
 {

lib/include/opencl-c-base.h

@@ -474,6 +474,9 @@ typedef enum memory_order
 #define CLK_HALF_FLOAT        0x10DD
 #define CLK_FLOAT             0x10DE
 #define CLK_UNORM_INT24       0x10DF
+#if __OPENCL_C_VERSION__ >= CL_VERSION_3_0
+#define CLK_UNORM_INT_101010_2 0x10E0
+#endif // __OPENCL_C_VERSION__ >= CL_VERSION_3_0
 
 // Channel order, numbering must be aligned with cl_channel_order in cl.h
 //

lib/include/openmp_wrappers/__clang_openmp_device_functions.h

@@ -40,7 +40,6 @@ extern "C" {
 
 // Import types which will be used by __clang_hip_libdevice_declares.h
 #ifndef __cplusplus
-#include <stdbool.h>
 #include <stdint.h>
 #endif
 

lib/include/openmp_wrappers/new

@@ -13,7 +13,7 @@
 // which do not use nothrow_t are provided without the <new> header.
 #include_next <new>
 
-#if defined(__NVPTX__) && defined(_OPENMP)
+#if (defined(__NVPTX__) || defined(__AMDGPU__)) && defined(_OPENMP)
 
 #include <cstdlib>
 

lib/include/pmmintrin.h

@@ -253,9 +253,12 @@ _mm_movedup_pd(__m128d __a)
 ///    the processor in the monitor event pending state. Data stored in the
 ///    monitored address range causes the processor to exit the pending state.
 ///
+/// The \c MONITOR instruction can be used in kernel mode, and in other modes
+/// if MSR <c> C001_0015h[MonMwaitUserEn] </c> is set.
+///
 /// \headerfile <x86intrin.h>
 ///
-/// This intrinsic corresponds to the <c> MONITOR </c> instruction.
+/// This intrinsic corresponds to the \c MONITOR instruction.
 ///
 /// \param __p
 ///    The memory range to be monitored. The size of the range is determined by
@@ -270,19 +273,22 @@ _mm_monitor(void const *__p, unsigned __extensions, unsigned __hints)
   __builtin_ia32_monitor(__p, __extensions, __hints);
 }
 
-/// Used with the MONITOR instruction to wait while the processor is in
+/// Used with the \c MONITOR instruction to wait while the processor is in
 ///    the monitor event pending state. Data stored in the monitored address
-///    range causes the processor to exit the pending state.
+///    range, or an interrupt, causes the processor to exit the pending state.
+///
+/// The \c MWAIT instruction can be used in kernel mode, and in other modes if
+/// MSR <c> C001_0015h[MonMwaitUserEn] </c> is set.
 ///
 /// \headerfile <x86intrin.h>
 ///
-/// This intrinsic corresponds to the <c> MWAIT </c> instruction.
+/// This intrinsic corresponds to the \c MWAIT instruction.
 ///
 /// \param __extensions
-///    Optional extensions for the monitoring state, which may vary by
+///    Optional extensions for the monitoring state, which can vary by
 ///    processor.
 /// \param __hints
-///    Optional hints for the monitoring state, which may vary by processor.
+///    Optional hints for the monitoring state, which can vary by processor.
 static __inline__ void __DEFAULT_FN_ATTRS
 _mm_mwait(unsigned __extensions, unsigned __hints)
 {

lib/include/ppc_wrappers/emmintrin.h

@@ -46,6 +46,7 @@
 
 /* SSE2 */
 typedef __vector double __v2df;
+typedef __vector float __v4f;
 typedef __vector long long __v2di;
 typedef __vector unsigned long long __v2du;
 typedef __vector int __v4si;
@@ -951,7 +952,7 @@ extern __inline __m128d
     _mm_cvtpi32_pd(__m64 __A) {
   __v4si __temp;
   __v2di __tmp2;
-  __v2df __result;
+  __v4f __result;
 
   __temp = (__v4si)vec_splats(__A);
   __tmp2 = (__v2di)vec_unpackl(__temp);

lib/include/ppc_wrappers/smmintrin.h

@@ -305,9 +305,9 @@ extern __inline int
 extern __inline __m128i
     __attribute__((__gnu_inline__, __always_inline__, __artificial__))
     _mm_blend_epi16(__m128i __A, __m128i __B, const int __imm8) {
-  __v16qi __charmask = vec_splats((signed char)__imm8);
+  __v16qu __charmask = vec_splats((unsigned char)__imm8);
   __charmask = vec_gb(__charmask);
-  __v8hu __shortmask = (__v8hu)vec_unpackh(__charmask);
+  __v8hu __shortmask = (__v8hu)vec_unpackh((__v16qi)__charmask);
 #ifdef __BIG_ENDIAN__
   __shortmask = vec_reve(__shortmask);
 #endif

lib/include/rdseedintrin.h

@@ -7,8 +7,8 @@
  *===-----------------------------------------------------------------------===
  */
 
-#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H
+#ifndef __IMMINTRIN_H
-#error "Never use <rdseedintrin.h> directly; include <x86intrin.h> instead."
+#error "Never use <rdseedintrin.h> directly; include <immintrin.h> instead."
 #endif
 
 #ifndef __RDSEEDINTRIN_H
@@ -17,12 +17,54 @@
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("rdseed")))
 
+/// Stores a hardware-generated 16-bit random value in the memory at \a __p.
+///
+///    The random number generator complies with NIST SP800-90B and SP800-90C.
+///
+/// \code{.operation}
+/// IF HW_NRND_GEN.ready == 1
+///   Store16(__p, HW_NRND_GEN.data)
+///   result := 1
+/// ELSE
+///   Store16(__p, 0)
+///   result := 0
+/// END
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c RDSEED instruction.
+///
+/// \param __p
+///    Pointer to memory for storing the 16-bit random number.
+/// \returns 1 if a random number was generated, 0 if not.
 static __inline__ int __DEFAULT_FN_ATTRS
 _rdseed16_step(unsigned short *__p)
 {
   return (int) __builtin_ia32_rdseed16_step(__p);
 }
 
+/// Stores a hardware-generated 32-bit random value in the memory at \a __p.
+///
+///    The random number generator complies with NIST SP800-90B and SP800-90C.
+///
+/// \code{.operation}
+/// IF HW_NRND_GEN.ready == 1
+///   Store32(__p, HW_NRND_GEN.data)
+///   result := 1
+/// ELSE
+///   Store32(__p, 0)
+///   result := 0
+/// END
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c RDSEED instruction.
+///
+/// \param __p
+///    Pointer to memory for storing the 32-bit random number.
+/// \returns 1 if a random number was generated, 0 if not.
 static __inline__ int __DEFAULT_FN_ATTRS
 _rdseed32_step(unsigned int *__p)
 {
@@ -30,6 +72,27 @@ _rdseed32_step(unsigned int *__p)
 }
 
 #ifdef __x86_64__
+/// Stores a hardware-generated 64-bit random value in the memory at \a __p.
+///
+///    The random number generator complies with NIST SP800-90B and SP800-90C.
+///
+/// \code{.operation}
+/// IF HW_NRND_GEN.ready == 1
+///   Store64(__p, HW_NRND_GEN.data)
+///   result := 1
+/// ELSE
+///   Store64(__p, 0)
+///   result := 0
+/// END
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c RDSEED instruction.
+///
+/// \param __p
+///    Pointer to memory for storing the 64-bit random number.
+/// \returns 1 if a random number was generated, 0 if not.
 static __inline__ int __DEFAULT_FN_ATTRS
 _rdseed64_step(unsigned long long *__p)
 {

lib/include/riscv_ntlh.h

@@ -0,0 +1,28 @@
+/*===---- riscv_ntlh.h - RISC-V NTLH intrinsics ----------------------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __RISCV_NTLH_H
+#define __RISCV_NTLH_H
+
+#ifndef __riscv_zihintntl
+#error "NTLH intrinsics require the NTLH extension."
+#endif
+
+enum {
+  __RISCV_NTLH_INNERMOST_PRIVATE = 2,
+  __RISCV_NTLH_ALL_PRIVATE,
+  __RISCV_NTLH_INNERMOST_SHARED,
+  __RISCV_NTLH_ALL
+};
+
+#define __riscv_ntl_load(PTR, DOMAIN) __builtin_riscv_ntl_load((PTR), (DOMAIN))
+#define __riscv_ntl_store(PTR, VAL, DOMAIN)                                    \
+  __builtin_riscv_ntl_store((PTR), (VAL), (DOMAIN))
+
+#endif

lib/include/riscv_vector.h

@@ -25,52 +25,15 @@ extern "C" {
 #pragma clang riscv intrinsic vector
 
 
-#define __riscv_vlenb() __builtin_rvv_vlenb()
+enum __RISCV_FRM {
-
+  __RISCV_FRM_RNE = 0,
-enum RVV_CSR {
+  __RISCV_FRM_RTZ = 1,
-  RVV_VSTART = 0,
+  __RISCV_FRM_RDN = 2,
-  RVV_VXSAT,
+  __RISCV_FRM_RUP = 3,
-  RVV_VXRM,
+  __RISCV_FRM_RMM = 4,
-  RVV_VCSR,
 };
 
-static __inline__ __attribute__((__always_inline__, __nodebug__))
+#define __riscv_vlenb() __builtin_rvv_vlenb()
-unsigned long __riscv_vread_csr(enum RVV_CSR __csr) {
-  unsigned long __rv = 0;
-  switch (__csr) {
-    case RVV_VSTART:
-      __asm__ __volatile__ ("csrr\t%0, vstart" : "=r"(__rv) : : "memory");
-      break;
-    case RVV_VXSAT:
-      __asm__ __volatile__ ("csrr\t%0, vxsat" : "=r"(__rv) : : "memory");
-      break;
-    case RVV_VXRM:
-      __asm__ __volatile__ ("csrr\t%0, vxrm" : "=r"(__rv) : : "memory");
-      break;
-    case RVV_VCSR:
-      __asm__ __volatile__ ("csrr\t%0, vcsr" : "=r"(__rv) : : "memory");
-      break;
-  }
-  return __rv;
-}
-
-static __inline__ __attribute__((__always_inline__, __nodebug__))
-void __riscv_vwrite_csr(enum RVV_CSR __csr, unsigned long __value) {
-  switch (__csr) {
-    case RVV_VSTART:
-      __asm__ __volatile__ ("csrw\tvstart, %z0" : : "rJ"(__value) : "memory");
-      break;
-    case RVV_VXSAT:
-      __asm__ __volatile__ ("csrw\tvxsat, %z0" : : "rJ"(__value) : "memory");
-      break;
-    case RVV_VXRM:
-      __asm__ __volatile__ ("csrw\tvxrm, %z0" : : "rJ"(__value) : "memory");
-      break;
-    case RVV_VCSR:
-      __asm__ __volatile__ ("csrw\tvcsr, %z0" : : "rJ"(__value) : "memory");
-      break;
-  }
-}
 
 #define __riscv_vsetvl_e8mf4(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 6)
 #define __riscv_vsetvl_e8mf2(avl) __builtin_rvv_vsetvli((size_t)(avl), 0, 7)
@@ -130,6 +93,13 @@ void __riscv_vwrite_csr(enum RVV_CSR __csr, unsigned long __value) {
 #define __riscv_vsetvlmax_e64m8() __builtin_rvv_vsetvlimax(3, 3)
 #endif
 
+
+enum __RISCV_VXRM {
+  __RISCV_VXRM_RNU = 0,
+  __RISCV_VXRM_RNE = 1,
+  __RISCV_VXRM_RDN = 2,
+  __RISCV_VXRM_ROD = 3,
+};
 typedef __rvv_bool64_t vbool64_t;
 typedef __rvv_bool32_t vbool32_t;
 typedef __rvv_bool16_t vbool16_t;
@@ -139,70 +109,289 @@ typedef __rvv_bool2_t vbool2_t;
 typedef __rvv_bool1_t vbool1_t;
 typedef __rvv_int8mf8_t vint8mf8_t;
 typedef __rvv_uint8mf8_t vuint8mf8_t;
+typedef __rvv_int8mf8x2_t vint8mf8x2_t;
+typedef __rvv_uint8mf8x2_t vuint8mf8x2_t;
+typedef __rvv_int8mf8x3_t vint8mf8x3_t;
+typedef __rvv_uint8mf8x3_t vuint8mf8x3_t;
+typedef __rvv_int8mf8x4_t vint8mf8x4_t;
+typedef __rvv_uint8mf8x4_t vuint8mf8x4_t;
+typedef __rvv_int8mf8x5_t vint8mf8x5_t;
+typedef __rvv_uint8mf8x5_t vuint8mf8x5_t;
+typedef __rvv_int8mf8x6_t vint8mf8x6_t;
+typedef __rvv_uint8mf8x6_t vuint8mf8x6_t;
+typedef __rvv_int8mf8x7_t vint8mf8x7_t;
+typedef __rvv_uint8mf8x7_t vuint8mf8x7_t;
+typedef __rvv_int8mf8x8_t vint8mf8x8_t;
+typedef __rvv_uint8mf8x8_t vuint8mf8x8_t;
 typedef __rvv_int8mf4_t vint8mf4_t;
 typedef __rvv_uint8mf4_t vuint8mf4_t;
+typedef __rvv_int8mf4x2_t vint8mf4x2_t;
+typedef __rvv_uint8mf4x2_t vuint8mf4x2_t;
+typedef __rvv_int8mf4x3_t vint8mf4x3_t;
+typedef __rvv_uint8mf4x3_t vuint8mf4x3_t;
+typedef __rvv_int8mf4x4_t vint8mf4x4_t;
+typedef __rvv_uint8mf4x4_t vuint8mf4x4_t;
+typedef __rvv_int8mf4x5_t vint8mf4x5_t;
+typedef __rvv_uint8mf4x5_t vuint8mf4x5_t;
+typedef __rvv_int8mf4x6_t vint8mf4x6_t;
+typedef __rvv_uint8mf4x6_t vuint8mf4x6_t;
+typedef __rvv_int8mf4x7_t vint8mf4x7_t;
+typedef __rvv_uint8mf4x7_t vuint8mf4x7_t;
+typedef __rvv_int8mf4x8_t vint8mf4x8_t;
+typedef __rvv_uint8mf4x8_t vuint8mf4x8_t;
 typedef __rvv_int8mf2_t vint8mf2_t;
 typedef __rvv_uint8mf2_t vuint8mf2_t;
+typedef __rvv_int8mf2x2_t vint8mf2x2_t;
+typedef __rvv_uint8mf2x2_t vuint8mf2x2_t;
+typedef __rvv_int8mf2x3_t vint8mf2x3_t;
+typedef __rvv_uint8mf2x3_t vuint8mf2x3_t;
+typedef __rvv_int8mf2x4_t vint8mf2x4_t;
+typedef __rvv_uint8mf2x4_t vuint8mf2x4_t;
+typedef __rvv_int8mf2x5_t vint8mf2x5_t;
+typedef __rvv_uint8mf2x5_t vuint8mf2x5_t;
+typedef __rvv_int8mf2x6_t vint8mf2x6_t;
+typedef __rvv_uint8mf2x6_t vuint8mf2x6_t;
+typedef __rvv_int8mf2x7_t vint8mf2x7_t;
+typedef __rvv_uint8mf2x7_t vuint8mf2x7_t;
+typedef __rvv_int8mf2x8_t vint8mf2x8_t;
+typedef __rvv_uint8mf2x8_t vuint8mf2x8_t;
 typedef __rvv_int8m1_t vint8m1_t;
 typedef __rvv_uint8m1_t vuint8m1_t;
+typedef __rvv_int8m1x2_t vint8m1x2_t;
+typedef __rvv_uint8m1x2_t vuint8m1x2_t;
+typedef __rvv_int8m1x3_t vint8m1x3_t;
+typedef __rvv_uint8m1x3_t vuint8m1x3_t;
+typedef __rvv_int8m1x4_t vint8m1x4_t;
+typedef __rvv_uint8m1x4_t vuint8m1x4_t;
+typedef __rvv_int8m1x5_t vint8m1x5_t;
+typedef __rvv_uint8m1x5_t vuint8m1x5_t;
+typedef __rvv_int8m1x6_t vint8m1x6_t;
+typedef __rvv_uint8m1x6_t vuint8m1x6_t;
+typedef __rvv_int8m1x7_t vint8m1x7_t;
+typedef __rvv_uint8m1x7_t vuint8m1x7_t;
+typedef __rvv_int8m1x8_t vint8m1x8_t;
+typedef __rvv_uint8m1x8_t vuint8m1x8_t;
 typedef __rvv_int8m2_t vint8m2_t;
 typedef __rvv_uint8m2_t vuint8m2_t;
+typedef __rvv_int8m2x2_t vint8m2x2_t;
+typedef __rvv_uint8m2x2_t vuint8m2x2_t;
+typedef __rvv_int8m2x3_t vint8m2x3_t;
+typedef __rvv_uint8m2x3_t vuint8m2x3_t;
+typedef __rvv_int8m2x4_t vint8m2x4_t;
+typedef __rvv_uint8m2x4_t vuint8m2x4_t;
 typedef __rvv_int8m4_t vint8m4_t;
 typedef __rvv_uint8m4_t vuint8m4_t;
+typedef __rvv_int8m4x2_t vint8m4x2_t;
+typedef __rvv_uint8m4x2_t vuint8m4x2_t;
 typedef __rvv_int8m8_t vint8m8_t;
 typedef __rvv_uint8m8_t vuint8m8_t;
 typedef __rvv_int16mf4_t vint16mf4_t;
 typedef __rvv_uint16mf4_t vuint16mf4_t;
+typedef __rvv_int16mf4x2_t vint16mf4x2_t;
+typedef __rvv_uint16mf4x2_t vuint16mf4x2_t;
+typedef __rvv_int16mf4x3_t vint16mf4x3_t;
+typedef __rvv_uint16mf4x3_t vuint16mf4x3_t;
+typedef __rvv_int16mf4x4_t vint16mf4x4_t;
+typedef __rvv_uint16mf4x4_t vuint16mf4x4_t;
+typedef __rvv_int16mf4x5_t vint16mf4x5_t;
+typedef __rvv_uint16mf4x5_t vuint16mf4x5_t;
+typedef __rvv_int16mf4x6_t vint16mf4x6_t;
+typedef __rvv_uint16mf4x6_t vuint16mf4x6_t;
+typedef __rvv_int16mf4x7_t vint16mf4x7_t;
+typedef __rvv_uint16mf4x7_t vuint16mf4x7_t;
+typedef __rvv_int16mf4x8_t vint16mf4x8_t;
+typedef __rvv_uint16mf4x8_t vuint16mf4x8_t;
 typedef __rvv_int16mf2_t vint16mf2_t;
 typedef __rvv_uint16mf2_t vuint16mf2_t;
+typedef __rvv_int16mf2x2_t vint16mf2x2_t;
+typedef __rvv_uint16mf2x2_t vuint16mf2x2_t;
+typedef __rvv_int16mf2x3_t vint16mf2x3_t;
+typedef __rvv_uint16mf2x3_t vuint16mf2x3_t;
+typedef __rvv_int16mf2x4_t vint16mf2x4_t;
+typedef __rvv_uint16mf2x4_t vuint16mf2x4_t;
+typedef __rvv_int16mf2x5_t vint16mf2x5_t;
+typedef __rvv_uint16mf2x5_t vuint16mf2x5_t;
+typedef __rvv_int16mf2x6_t vint16mf2x6_t;
+typedef __rvv_uint16mf2x6_t vuint16mf2x6_t;
+typedef __rvv_int16mf2x7_t vint16mf2x7_t;
+typedef __rvv_uint16mf2x7_t vuint16mf2x7_t;
+typedef __rvv_int16mf2x8_t vint16mf2x8_t;
+typedef __rvv_uint16mf2x8_t vuint16mf2x8_t;
 typedef __rvv_int16m1_t vint16m1_t;
 typedef __rvv_uint16m1_t vuint16m1_t;
+typedef __rvv_int16m1x2_t vint16m1x2_t;
+typedef __rvv_uint16m1x2_t vuint16m1x2_t;
+typedef __rvv_int16m1x3_t vint16m1x3_t;
+typedef __rvv_uint16m1x3_t vuint16m1x3_t;
+typedef __rvv_int16m1x4_t vint16m1x4_t;
+typedef __rvv_uint16m1x4_t vuint16m1x4_t;
+typedef __rvv_int16m1x5_t vint16m1x5_t;
+typedef __rvv_uint16m1x5_t vuint16m1x5_t;
+typedef __rvv_int16m1x6_t vint16m1x6_t;
+typedef __rvv_uint16m1x6_t vuint16m1x6_t;
+typedef __rvv_int16m1x7_t vint16m1x7_t;
+typedef __rvv_uint16m1x7_t vuint16m1x7_t;
+typedef __rvv_int16m1x8_t vint16m1x8_t;
+typedef __rvv_uint16m1x8_t vuint16m1x8_t;
 typedef __rvv_int16m2_t vint16m2_t;
 typedef __rvv_uint16m2_t vuint16m2_t;
+typedef __rvv_int16m2x2_t vint16m2x2_t;
+typedef __rvv_uint16m2x2_t vuint16m2x2_t;
+typedef __rvv_int16m2x3_t vint16m2x3_t;
+typedef __rvv_uint16m2x3_t vuint16m2x3_t;
+typedef __rvv_int16m2x4_t vint16m2x4_t;
+typedef __rvv_uint16m2x4_t vuint16m2x4_t;
 typedef __rvv_int16m4_t vint16m4_t;
 typedef __rvv_uint16m4_t vuint16m4_t;
+typedef __rvv_int16m4x2_t vint16m4x2_t;
+typedef __rvv_uint16m4x2_t vuint16m4x2_t;
 typedef __rvv_int16m8_t vint16m8_t;
 typedef __rvv_uint16m8_t vuint16m8_t;
 typedef __rvv_int32mf2_t vint32mf2_t;
 typedef __rvv_uint32mf2_t vuint32mf2_t;
+typedef __rvv_int32mf2x2_t vint32mf2x2_t;
+typedef __rvv_uint32mf2x2_t vuint32mf2x2_t;
+typedef __rvv_int32mf2x3_t vint32mf2x3_t;
+typedef __rvv_uint32mf2x3_t vuint32mf2x3_t;
+typedef __rvv_int32mf2x4_t vint32mf2x4_t;
+typedef __rvv_uint32mf2x4_t vuint32mf2x4_t;
+typedef __rvv_int32mf2x5_t vint32mf2x5_t;
+typedef __rvv_uint32mf2x5_t vuint32mf2x5_t;
+typedef __rvv_int32mf2x6_t vint32mf2x6_t;
+typedef __rvv_uint32mf2x6_t vuint32mf2x6_t;
+typedef __rvv_int32mf2x7_t vint32mf2x7_t;
+typedef __rvv_uint32mf2x7_t vuint32mf2x7_t;
+typedef __rvv_int32mf2x8_t vint32mf2x8_t;
+typedef __rvv_uint32mf2x8_t vuint32mf2x8_t;
 typedef __rvv_int32m1_t vint32m1_t;
 typedef __rvv_uint32m1_t vuint32m1_t;
+typedef __rvv_int32m1x2_t vint32m1x2_t;
+typedef __rvv_uint32m1x2_t vuint32m1x2_t;
+typedef __rvv_int32m1x3_t vint32m1x3_t;
+typedef __rvv_uint32m1x3_t vuint32m1x3_t;
+typedef __rvv_int32m1x4_t vint32m1x4_t;
+typedef __rvv_uint32m1x4_t vuint32m1x4_t;
+typedef __rvv_int32m1x5_t vint32m1x5_t;
+typedef __rvv_uint32m1x5_t vuint32m1x5_t;
+typedef __rvv_int32m1x6_t vint32m1x6_t;
+typedef __rvv_uint32m1x6_t vuint32m1x6_t;
+typedef __rvv_int32m1x7_t vint32m1x7_t;
+typedef __rvv_uint32m1x7_t vuint32m1x7_t;
+typedef __rvv_int32m1x8_t vint32m1x8_t;
+typedef __rvv_uint32m1x8_t vuint32m1x8_t;
 typedef __rvv_int32m2_t vint32m2_t;
 typedef __rvv_uint32m2_t vuint32m2_t;
+typedef __rvv_int32m2x2_t vint32m2x2_t;
+typedef __rvv_uint32m2x2_t vuint32m2x2_t;
+typedef __rvv_int32m2x3_t vint32m2x3_t;
+typedef __rvv_uint32m2x3_t vuint32m2x3_t;
+typedef __rvv_int32m2x4_t vint32m2x4_t;
+typedef __rvv_uint32m2x4_t vuint32m2x4_t;
 typedef __rvv_int32m4_t vint32m4_t;
 typedef __rvv_uint32m4_t vuint32m4_t;
+typedef __rvv_int32m4x2_t vint32m4x2_t;
+typedef __rvv_uint32m4x2_t vuint32m4x2_t;
 typedef __rvv_int32m8_t vint32m8_t;
 typedef __rvv_uint32m8_t vuint32m8_t;
 typedef __rvv_int64m1_t vint64m1_t;
 typedef __rvv_uint64m1_t vuint64m1_t;
+typedef __rvv_int64m1x2_t vint64m1x2_t;
+typedef __rvv_uint64m1x2_t vuint64m1x2_t;
+typedef __rvv_int64m1x3_t vint64m1x3_t;
+typedef __rvv_uint64m1x3_t vuint64m1x3_t;
+typedef __rvv_int64m1x4_t vint64m1x4_t;
+typedef __rvv_uint64m1x4_t vuint64m1x4_t;
+typedef __rvv_int64m1x5_t vint64m1x5_t;
+typedef __rvv_uint64m1x5_t vuint64m1x5_t;
+typedef __rvv_int64m1x6_t vint64m1x6_t;
+typedef __rvv_uint64m1x6_t vuint64m1x6_t;
+typedef __rvv_int64m1x7_t vint64m1x7_t;
+typedef __rvv_uint64m1x7_t vuint64m1x7_t;
+typedef __rvv_int64m1x8_t vint64m1x8_t;
+typedef __rvv_uint64m1x8_t vuint64m1x8_t;
 typedef __rvv_int64m2_t vint64m2_t;
 typedef __rvv_uint64m2_t vuint64m2_t;
+typedef __rvv_int64m2x2_t vint64m2x2_t;
+typedef __rvv_uint64m2x2_t vuint64m2x2_t;
+typedef __rvv_int64m2x3_t vint64m2x3_t;
+typedef __rvv_uint64m2x3_t vuint64m2x3_t;
+typedef __rvv_int64m2x4_t vint64m2x4_t;
+typedef __rvv_uint64m2x4_t vuint64m2x4_t;
 typedef __rvv_int64m4_t vint64m4_t;
 typedef __rvv_uint64m4_t vuint64m4_t;
+typedef __rvv_int64m4x2_t vint64m4x2_t;
+typedef __rvv_uint64m4x2_t vuint64m4x2_t;
 typedef __rvv_int64m8_t vint64m8_t;
 typedef __rvv_uint64m8_t vuint64m8_t;
-#if defined(__riscv_zvfh)
 typedef __rvv_float16mf4_t vfloat16mf4_t;
+typedef __rvv_float16mf4x2_t vfloat16mf4x2_t;
+typedef __rvv_float16mf4x3_t vfloat16mf4x3_t;
+typedef __rvv_float16mf4x4_t vfloat16mf4x4_t;
+typedef __rvv_float16mf4x5_t vfloat16mf4x5_t;
+typedef __rvv_float16mf4x6_t vfloat16mf4x6_t;
+typedef __rvv_float16mf4x7_t vfloat16mf4x7_t;
+typedef __rvv_float16mf4x8_t vfloat16mf4x8_t;
 typedef __rvv_float16mf2_t vfloat16mf2_t;
+typedef __rvv_float16mf2x2_t vfloat16mf2x2_t;
+typedef __rvv_float16mf2x3_t vfloat16mf2x3_t;
+typedef __rvv_float16mf2x4_t vfloat16mf2x4_t;
+typedef __rvv_float16mf2x5_t vfloat16mf2x5_t;
+typedef __rvv_float16mf2x6_t vfloat16mf2x6_t;
+typedef __rvv_float16mf2x7_t vfloat16mf2x7_t;
+typedef __rvv_float16mf2x8_t vfloat16mf2x8_t;
 typedef __rvv_float16m1_t vfloat16m1_t;
+typedef __rvv_float16m1x2_t vfloat16m1x2_t;
+typedef __rvv_float16m1x3_t vfloat16m1x3_t;
+typedef __rvv_float16m1x4_t vfloat16m1x4_t;
+typedef __rvv_float16m1x5_t vfloat16m1x5_t;
+typedef __rvv_float16m1x6_t vfloat16m1x6_t;
+typedef __rvv_float16m1x7_t vfloat16m1x7_t;
+typedef __rvv_float16m1x8_t vfloat16m1x8_t;
 typedef __rvv_float16m2_t vfloat16m2_t;
+typedef __rvv_float16m2x2_t vfloat16m2x2_t;
+typedef __rvv_float16m2x3_t vfloat16m2x3_t;
+typedef __rvv_float16m2x4_t vfloat16m2x4_t;
 typedef __rvv_float16m4_t vfloat16m4_t;
+typedef __rvv_float16m4x2_t vfloat16m4x2_t;
 typedef __rvv_float16m8_t vfloat16m8_t;
-#endif
-#if (__riscv_v_elen_fp >= 32)
 typedef __rvv_float32mf2_t vfloat32mf2_t;
+typedef __rvv_float32mf2x2_t vfloat32mf2x2_t;
+typedef __rvv_float32mf2x3_t vfloat32mf2x3_t;
+typedef __rvv_float32mf2x4_t vfloat32mf2x4_t;
+typedef __rvv_float32mf2x5_t vfloat32mf2x5_t;
+typedef __rvv_float32mf2x6_t vfloat32mf2x6_t;
+typedef __rvv_float32mf2x7_t vfloat32mf2x7_t;
+typedef __rvv_float32mf2x8_t vfloat32mf2x8_t;
 typedef __rvv_float32m1_t vfloat32m1_t;
+typedef __rvv_float32m1x2_t vfloat32m1x2_t;
+typedef __rvv_float32m1x3_t vfloat32m1x3_t;
+typedef __rvv_float32m1x4_t vfloat32m1x4_t;
+typedef __rvv_float32m1x5_t vfloat32m1x5_t;
+typedef __rvv_float32m1x6_t vfloat32m1x6_t;
+typedef __rvv_float32m1x7_t vfloat32m1x7_t;
+typedef __rvv_float32m1x8_t vfloat32m1x8_t;
 typedef __rvv_float32m2_t vfloat32m2_t;
+typedef __rvv_float32m2x2_t vfloat32m2x2_t;
+typedef __rvv_float32m2x3_t vfloat32m2x3_t;
+typedef __rvv_float32m2x4_t vfloat32m2x4_t;
 typedef __rvv_float32m4_t vfloat32m4_t;
+typedef __rvv_float32m4x2_t vfloat32m4x2_t;
 typedef __rvv_float32m8_t vfloat32m8_t;
-#endif
-#if (__riscv_v_elen_fp >= 64)
 typedef __rvv_float64m1_t vfloat64m1_t;
+typedef __rvv_float64m1x2_t vfloat64m1x2_t;
+typedef __rvv_float64m1x3_t vfloat64m1x3_t;
+typedef __rvv_float64m1x4_t vfloat64m1x4_t;
+typedef __rvv_float64m1x5_t vfloat64m1x5_t;
+typedef __rvv_float64m1x6_t vfloat64m1x6_t;
+typedef __rvv_float64m1x7_t vfloat64m1x7_t;
+typedef __rvv_float64m1x8_t vfloat64m1x8_t;
 typedef __rvv_float64m2_t vfloat64m2_t;
+typedef __rvv_float64m2x2_t vfloat64m2x2_t;
+typedef __rvv_float64m2x3_t vfloat64m2x3_t;
+typedef __rvv_float64m2x4_t vfloat64m2x4_t;
 typedef __rvv_float64m4_t vfloat64m4_t;
+typedef __rvv_float64m4x2_t vfloat64m4x2_t;
 typedef __rvv_float64m8_t vfloat64m8_t;
-#endif
-
 #define __riscv_v_intrinsic_overloading 1
 
 #ifdef __cplusplus

lib/include/sha512intrin.h

@@ -0,0 +1,200 @@
+/*===--------------- sha512intrin.h - SHA512 intrinsics -----------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <sha512intrin.h> directly; include <immintrin.h> instead."
+#endif // __IMMINTRIN_H
+
+#ifndef __SHA512INTRIN_H
+#define __SHA512INTRIN_H
+
+#define __DEFAULT_FN_ATTRS256                                                  \
+  __attribute__((__always_inline__, __nodebug__, __target__("sha512"),         \
+                 __min_vector_width__(256)))
+
+/// This intrinisc is one of the two SHA512 message scheduling instructions.
+///    The intrinsic performs an intermediate calculation for the next four
+///    SHA512 message qwords. The calculated results are stored in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_sha512msg1_epi64(__m256i __A, __m128i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VSHA512MSG1 instruction.
+///
+/// \param __A
+///    A 256-bit vector of [4 x long long].
+/// \param __B
+///    A 128-bit vector of [2 x long long].
+/// \returns
+///    A 256-bit vector of [4 x long long].
+///
+/// \code{.operation}
+/// DEFINE ROR64(qword, n) {
+/// 	count := n % 64
+/// 	dest := (qword >> count) | (qword << (64 - count))
+/// 	RETURN dest
+/// }
+/// DEFINE SHR64(qword, n) {
+/// 	RETURN qword >> n
+/// }
+/// DEFINE s0(qword):
+/// 	RETURN ROR64(qword,1) ^ ROR64(qword, 8) ^ SHR64(qword, 7)
+/// }
+/// W[4] := __B.qword[0]
+/// W[3] := __A.qword[3]
+/// W[2] := __A.qword[2]
+/// W[1] := __A.qword[1]
+/// W[0] := __A.qword[0]
+/// dst.qword[3] := W[3] + s0(W[4])
+/// dst.qword[2] := W[2] + s0(W[3])
+/// dst.qword[1] := W[1] + s0(W[2])
+/// dst.qword[0] := W[0] + s0(W[1])
+/// dst[MAX:256] := 0
+/// \endcode
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_sha512msg1_epi64(__m256i __A, __m128i __B) {
+  return (__m256i)__builtin_ia32_vsha512msg1((__v4du)__A, (__v2du)__B);
+}
+
+/// This intrinisc is one of the two SHA512 message scheduling instructions.
+///    The intrinsic performs the final calculation for the next four SHA512
+///    message qwords. The calculated results are stored in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_sha512msg2_epi64(__m256i __A, __m256i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VSHA512MSG2 instruction.
+///
+/// \param __A
+///    A 256-bit vector of [4 x long long].
+/// \param __B
+///    A 256-bit vector of [4 x long long].
+/// \returns
+///    A 256-bit vector of [4 x long long].
+///
+/// \code{.operation}
+/// DEFINE ROR64(qword, n) {
+/// 	count := n % 64
+/// 	dest := (qword >> count) | (qword << (64 - count))
+/// 	RETURN dest
+/// }
+/// DEFINE SHR64(qword, n) {
+/// 	RETURN qword >> n
+/// }
+/// DEFINE s1(qword) {
+/// 	RETURN ROR64(qword,19) ^ ROR64(qword, 61) ^ SHR64(qword, 6)
+/// }
+/// W[14] := __B.qword[2]
+/// W[15] := __B.qword[3]
+/// W[16] := __A.qword[0] + s1(W[14])
+/// W[17] := __A.qword[1] + s1(W[15])
+/// W[18] := __A.qword[2] + s1(W[16])
+/// W[19] := __A.qword[3] + s1(W[17])
+/// dst.qword[3] := W[19]
+/// dst.qword[2] := W[18]
+/// dst.qword[1] := W[17]
+/// dst.qword[0] := W[16]
+/// dst[MAX:256] := 0
+/// \endcode
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_sha512msg2_epi64(__m256i __A, __m256i __B) {
+  return (__m256i)__builtin_ia32_vsha512msg2((__v4du)__A, (__v4du)__B);
+}
+
+/// This intrinisc performs two rounds of SHA512 operation using initial SHA512
+///    state (C,D,G,H) from \a __A, an initial SHA512 state (A,B,E,F) from
+///    \a __A, and a pre-computed sum of the next two round message qwords and
+///    the corresponding round constants from \a __C (only the two lower qwords
+///    of the third operand). The updated SHA512 state (A,B,E,F) is written to
+///    \a __A, and \a __A can be used as the updated state (C,D,G,H) in later
+///    rounds.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_sha512rnds2_epi64(__m256i __A, __m256i __B, __m128i __C)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VSHA512RNDS2 instruction.
+///
+/// \param __A
+///    A 256-bit vector of [4 x long long].
+/// \param __B
+///    A 256-bit vector of [4 x long long].
+/// \param __C
+///    A 128-bit vector of [2 x long long].
+/// \returns
+///    A 256-bit vector of [4 x long long].
+///
+/// \code{.operation}
+/// DEFINE ROR64(qword, n) {
+/// 	count := n % 64
+/// 	dest := (qword >> count) | (qword << (64 - count))
+/// 	RETURN dest
+/// }
+/// DEFINE SHR64(qword, n) {
+/// 	RETURN qword >> n
+/// }
+/// DEFINE cap_sigma0(qword) {
+/// 	RETURN ROR64(qword,28) ^ ROR64(qword, 34) ^ ROR64(qword, 39)
+/// }
+/// DEFINE cap_sigma1(qword) {
+/// 	RETURN ROR64(qword,14) ^ ROR64(qword, 18) ^ ROR64(qword, 41)
+/// }
+/// DEFINE MAJ(a,b,c) {
+/// 	RETURN (a & b) ^ (a & c) ^ (b & c)
+/// }
+/// DEFINE CH(e,f,g) {
+/// 	RETURN (e & f) ^ (g & ~e)
+/// }
+/// A[0] := __B.qword[3]
+/// B[0] := __B.qword[2]
+/// C[0] := __C.qword[3]
+/// D[0] := __C.qword[2]
+/// E[0] := __B.qword[1]
+/// F[0] := __B.qword[0]
+/// G[0] := __C.qword[1]
+/// H[0] := __C.qword[0]
+/// WK[0]:= __A.qword[0]
+/// WK[1]:= __A.qword[1]
+/// FOR i := 0 to 1:
+/// 	A[i+1] := CH(E[i], F[i], G[i]) +
+/// 	cap_sigma1(E[i]) + WK[i] + H[i] +
+/// 	MAJ(A[i], B[i], C[i]) +
+/// 	cap_sigma0(A[i])
+/// 	B[i+1] := A[i]
+/// 	C[i+1] := B[i]
+/// 	D[i+1] := C[i]
+/// 	E[i+1] := CH(E[i], F[i], G[i]) +
+/// 	cap_sigma1(E[i]) + WK[i] + H[i] + D[i]
+/// 	F[i+1] := E[i]
+/// 	G[i+1] := F[i]
+/// 	H[i+1] := G[i]
+/// ENDFOR
+/// dst.qword[3] := A[2]
+/// dst.qword[2] := B[2]
+/// dst.qword[1] := E[2]
+/// dst.qword[0] := F[2]
+/// dst[MAX:256] := 0
+/// \endcode
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
+_mm256_sha512rnds2_epi64(__m256i __A, __m256i __B, __m128i __C) {
+  return (__m256i)__builtin_ia32_vsha512rnds2((__v4du)__A, (__v4du)__B,
+                                              (__v2du)__C);
+}
+
+#undef __DEFAULT_FN_ATTRS256
+
+#endif // __SHA512INTRIN_H

lib/include/shaintrin.h

@@ -17,39 +17,167 @@
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sha"), __min_vector_width__(128)))
 
+/// Performs four iterations of the inner loop of the SHA-1 message digest
+///    algorithm using the starting SHA-1 state (A, B, C, D) from the 128-bit
+///    vector of [4 x i32] in \a V1 and the next four 32-bit elements of the
+///    message from the 128-bit vector of [4 x i32] in \a V2. Note that the
+///    SHA-1 state variable E must have already been added to \a V2
+///    (\c _mm_sha1nexte_epu32() can perform this step). Returns the updated
+///    SHA-1 state (A, B, C, D) as a 128-bit vector of [4 x i32].
+///
+///    The SHA-1 algorithm has an inner loop of 80 iterations, twenty each
+///    with a different combining function and rounding constant. This
+///    intrinsic performs four iterations using a combining function and
+///    rounding constant selected by \a M[1:0].
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_sha1rnds4_epu32(__m128i V1, __m128i V2, const int M);
+/// \endcode
+///
+/// This intrinsic corresponds to the \c SHA1RNDS4 instruction.
+///
+/// \param V1
+///    A 128-bit vector of [4 x i32] containing the initial SHA-1 state.
+/// \param V2
+///    A 128-bit vector of [4 x i32] containing the next four elements of
+///    the message, plus SHA-1 state variable E.
+/// \param M
+///    An immediate value where bits [1:0] select among four possible
+///    combining functions and rounding constants (not specified here).
+/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1 state.
 #define _mm_sha1rnds4_epu32(V1, V2, M) \
   __builtin_ia32_sha1rnds4((__v4si)(__m128i)(V1), (__v4si)(__m128i)(V2), (M))
 
+/// Calculates the SHA-1 state variable E from the SHA-1 state variables in
+///    the 128-bit vector of [4 x i32] in \a __X, adds that to the next set of
+///    four message elements in the 128-bit vector of [4 x i32] in \a __Y, and
+///    returns the result.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c SHA1NEXTE instruction.
+///
+/// \param __X
+///    A 128-bit vector of [4 x i32] containing the current SHA-1 state.
+/// \param __Y
+///    A 128-bit vector of [4 x i32] containing the next four elements of the
+///    message.
+/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1
+///    values.
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_sha1nexte_epu32(__m128i __X, __m128i __Y)
 {
   return (__m128i)__builtin_ia32_sha1nexte((__v4si)__X, (__v4si)__Y);
 }
 
+/// Performs an intermediate calculation for deriving the next four SHA-1
+///    message elements using previous message elements from the 128-bit
+///    vectors of [4 x i32] in \a __X and \a __Y, and returns the result.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c SHA1MSG1 instruction.
+///
+/// \param __X
+///    A 128-bit vector of [4 x i32] containing previous message elements.
+/// \param __Y
+///    A 128-bit vector of [4 x i32] containing previous message elements.
+/// \returns A 128-bit vector of [4 x i32] containing the derived SHA-1
+///    elements.
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_sha1msg1_epu32(__m128i __X, __m128i __Y)
 {
   return (__m128i)__builtin_ia32_sha1msg1((__v4si)__X, (__v4si)__Y);
 }
 
+/// Performs the final calculation for deriving the next four SHA-1 message
+///    elements using previous message elements from the 128-bit vectors of
+///    [4 x i32] in \a __X and \a __Y, and returns the result.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c SHA1MSG2 instruction.
+///
+/// \param __X
+///    A 128-bit vector of [4 x i32] containing an intermediate result.
+/// \param __Y
+///    A 128-bit vector of [4 x i32] containing previous message values.
+/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1
+///    values.
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_sha1msg2_epu32(__m128i __X, __m128i __Y)
 {
   return (__m128i)__builtin_ia32_sha1msg2((__v4si)__X, (__v4si)__Y);
 }
 
+/// Performs two rounds of SHA-256 operation using the following inputs: a
+///    starting SHA-256 state (C, D, G, H) from the 128-bit vector of
+///    [4 x i32] in \a __X; a starting SHA-256 state (A, B, E, F) from the
+///    128-bit vector of [4 x i32] in \a __Y; and a pre-computed sum of the
+///    next two message elements (unsigned 32-bit integers) and corresponding
+///    rounding constants from the 128-bit vector of [4 x i32] in \a __Z.
+///    Returns the updated SHA-256 state (A, B, E, F) as a 128-bit vector of
+///    [4 x i32].
+///
+///    The SHA-256 algorithm has a core loop of 64 iterations. This intrinsic
+///    performs two of those iterations.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c SHA256RNDS2 instruction.
+///
+/// \param __X
+///    A 128-bit vector of [4 x i32] containing part of the initial SHA-256
+///    state.
+/// \param __Y
+///    A 128-bit vector of [4 x i32] containing part of the initial SHA-256
+///    state.
+/// \param __Z
+///    A 128-bit vector of [4 x i32] containing additional input to the
+///    SHA-256 operation.
+/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-1 state.
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_sha256rnds2_epu32(__m128i __X, __m128i __Y, __m128i __Z)
 {
   return (__m128i)__builtin_ia32_sha256rnds2((__v4si)__X, (__v4si)__Y, (__v4si)__Z);
 }
 
+/// Performs an intermediate calculation for deriving the next four SHA-256
+///    message elements using previous message elements from the 128-bit
+///    vectors of [4 x i32] in \a __X and \a __Y, and returns the result.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c SHA256MSG1 instruction.
+///
+/// \param __X
+///    A 128-bit vector of [4 x i32] containing previous message elements.
+/// \param __Y
+///    A 128-bit vector of [4 x i32] containing previous message elements.
+/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-256
+///    values.
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_sha256msg1_epu32(__m128i __X, __m128i __Y)
 {
   return (__m128i)__builtin_ia32_sha256msg1((__v4si)__X, (__v4si)__Y);
 }
 
+/// Performs the final calculation for deriving the next four SHA-256 message
+///    elements using previous message elements from the 128-bit vectors of
+///    [4 x i32] in \a __X and \a __Y, and returns the result.
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c SHA256MSG2 instruction.
+///
+/// \param __X
+///    A 128-bit vector of [4 x i32] containing an intermediate result.
+/// \param __Y
+///    A 128-bit vector of [4 x i32] containing previous message values.
+/// \returns A 128-bit vector of [4 x i32] containing the updated SHA-256
+///    values.
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_sha256msg2_epu32(__m128i __X, __m128i __Y)
 {

lib/include/sifive_vector.h

@@ -0,0 +1,16 @@
+//===----- sifive_vector.h - SiFive Vector definitions --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _SIFIVE_VECTOR_H_
+#define _SIFIVE_VECTOR_H_
+
+#include "riscv_vector.h"
+
+#pragma clang riscv intrinsic sifive_vector
+
+#endif //_SIFIVE_VECTOR_H_

lib/include/sm3intrin.h

@@ -0,0 +1,238 @@
+/*===-------------------- sm3intrin.h - SM3 intrinsics ---------------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <sm3intrin.h> directly; include <immintrin.h> instead."
+#endif // __IMMINTRIN_H
+
+#ifndef __SM3INTRIN_H
+#define __SM3INTRIN_H
+
+#define __DEFAULT_FN_ATTRS128                                                  \
+  __attribute__((__always_inline__, __nodebug__, __target__("sm3"),            \
+                 __min_vector_width__(128)))
+
+/// This intrinisc is one of the two SM3 message scheduling intrinsics. The
+///    intrinsic performs an initial calculation for the next four SM3 message
+///    words. The calculated results are stored in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_sm3msg1_epi32(__m128i __A, __m128i __B, __m128i __C)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VSM3MSG1 instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x int].
+/// \param __B
+///    A 128-bit vector of [4 x int].
+/// \param __C
+///    A 128-bit vector of [4 x int].
+/// \returns
+///    A 128-bit vector of [4 x int].
+///
+/// \code{.operation}
+/// DEFINE ROL32(dword, n) {
+/// 	count := n % 32
+/// 	dest := (dword << count) | (dword >> (32 - count))
+/// 	RETURN dest
+/// }
+/// DEFINE P1(x) {
+/// 	RETURN x ^ ROL32(x, 15) ^ ROL32(x, 23)
+/// }
+/// W[0] := __C.dword[0]
+/// W[1] := __C.dword[1]
+/// W[2] := __C.dword[2]
+/// W[3] := __C.dword[3]
+/// W[7] := __A.dword[0]
+/// W[8] := __A.dword[1]
+/// W[9] := __A.dword[2]
+/// W[10] := __A.dword[3]
+/// W[13] := __B.dword[0]
+/// W[14] := __B.dword[1]
+/// W[15] := __B.dword[2]
+/// TMP0 := W[7] ^ W[0] ^ ROL32(W[13], 15)
+/// TMP1 := W[8] ^ W[1] ^ ROL32(W[14], 15)
+/// TMP2 := W[9] ^ W[2] ^ ROL32(W[15], 15)
+/// TMP3 := W[10] ^ W[3]
+/// dst.dword[0] := P1(TMP0)
+/// dst.dword[1] := P1(TMP1)
+/// dst.dword[2] := P1(TMP2)
+/// dst.dword[3] := P1(TMP3)
+/// dst[MAX:128] := 0
+/// \endcode
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_sm3msg1_epi32(__m128i __A,
+                                                                  __m128i __B,
+                                                                  __m128i __C) {
+  return (__m128i)__builtin_ia32_vsm3msg1((__v4su)__A, (__v4su)__B,
+                                          (__v4su)__C);
+}
+
+/// This intrinisc is one of the two SM3 message scheduling intrinsics. The
+///    intrinsic performs the final calculation for the next four SM3 message
+///    words. The calculated results are stored in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_sm3msg2_epi32(__m128i __A, __m128i __B, __m128i __C)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VSM3MSG2 instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x int].
+/// \param __B
+///    A 128-bit vector of [4 x int].
+/// \param __C
+///    A 128-bit vector of [4 x int].
+/// \returns
+///    A 128-bit vector of [4 x int].
+///
+/// \code{.operation}
+/// DEFINE ROL32(dword, n) {
+/// 	count := n % 32
+/// 	dest := (dword << count) | (dword >> (32-count))
+/// 	RETURN dest
+/// }
+/// WTMP[0] := __A.dword[0]
+/// WTMP[1] := __A.dword[1]
+/// WTMP[2] := __A.dword[2]
+/// WTMP[3] := __A.dword[3]
+/// W[3] := __B.dword[0]
+/// W[4] := __B.dword[1]
+/// W[5] := __B.dword[2]
+/// W[6] := __B.dword[3]
+/// W[10] := __C.dword[0]
+/// W[11] := __C.dword[1]
+/// W[12] := __C.dword[2]
+/// W[13] := __C.dword[3]
+/// W[16] := ROL32(W[3], 7) ^ W[10] ^ WTMP[0]
+/// W[17] := ROL32(W[4], 7) ^ W[11] ^ WTMP[1]
+/// W[18] := ROL32(W[5], 7) ^ W[12] ^ WTMP[2]
+/// W[19] := ROL32(W[6], 7) ^ W[13] ^ WTMP[3]
+/// W[19] := W[19] ^ ROL32(W[16], 6) ^ ROL32(W[16], 15) ^ ROL32(W[16], 30)
+/// dst.dword[0] := W[16]
+/// dst.dword[1] := W[17]
+/// dst.dword[2] := W[18]
+/// dst.dword[3] := W[19]
+/// dst[MAX:128] := 0
+/// \endcode
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_sm3msg2_epi32(__m128i __A,
+                                                                  __m128i __B,
+                                                                  __m128i __C) {
+  return (__m128i)__builtin_ia32_vsm3msg2((__v4su)__A, (__v4su)__B,
+                                          (__v4su)__C);
+}
+
+/// This intrinsic performs two rounds of SM3 operation using initial SM3 state
+///    (C, D, G, H) from \a __A, an initial SM3 states (A, B, E, F)
+///    from \a __B and a pre-computed words from the \a __C. \a __A with
+///    initial SM3 state of (C, D, G, H) assumes input of non-rotated left
+///    variables from previous state. The updated SM3 state (A, B, E, F) is
+///    written to \a __A. The \a imm8 should contain the even round number
+///    for the first of the two rounds computed by this instruction. The
+///    computation masks the \a imm8 value by AND’ing it with 0x3E so that only
+///    even round numbers from 0 through 62 are used for this operation. The
+///    calculated results are stored in \a dst.
+///
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_sm3rnds2_epi32(__m128i __A, __m128i __B, __m128i __C, const int
+/// imm8) \endcode
+///
+/// This intrinsic corresponds to the \c VSM3RNDS2 instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x int].
+/// \param __B
+///    A 128-bit vector of [4 x int].
+/// \param __C
+///    A 128-bit vector of [4 x int].
+/// \param imm8
+///    A 8-bit constant integer.
+/// \returns
+///    A 128-bit vector of [4 x int].
+///
+/// \code{.operation}
+/// DEFINE ROL32(dword, n) {
+/// 	count := n % 32
+/// 	dest := (dword << count) | (dword >> (32-count))
+/// 	RETURN dest
+/// }
+/// DEFINE P0(dword) {
+/// 	RETURN dword ^ ROL32(dword, 9) ^ ROL32(dword, 17)
+/// }
+/// DEFINE FF(x,y,z, round){
+/// 	IF round < 16
+/// 		RETURN (x ^ y ^ z)
+/// 	ELSE
+/// 		RETURN (x & y) | (x & z) | (y & z)
+/// 	FI
+/// }
+/// DEFINE GG(x, y, z, round){
+///   IF round < 16
+///   	RETURN (x ^ y ^ z)
+///   ELSE
+///   	RETURN (x & y) | (~x & z)
+///   FI
+/// }
+/// A[0] := __B.dword[3]
+/// B[0] := __B.dword[2]
+/// C[0] := __A.dword[3]
+/// D[0] := __A.dword[2]
+/// E[0] := __B.dword[1]
+/// F[0] := __B.dword[0]
+/// G[0] := __A.dword[1]
+/// H[0] := __A.dword[0]
+/// W[0] := __C.dword[0]
+/// W[1] := __C.dword[1]
+/// W[4] := __C.dword[2]
+/// W[5] := __C.dword[3]
+/// C[0] := ROL32(C[0], 9)
+/// D[0] := ROL32(D[0], 9)
+/// G[0] := ROL32(G[0], 19)
+/// H[0] := ROL32(H[0], 19)
+/// ROUND := __D & 0x3E
+/// IF ROUND < 16
+/// 	CONST := 0x79CC4519
+/// ELSE
+/// 	CONST := 0x7A879D8A
+/// FI
+/// CONST := ROL32(CONST,ROUND)
+/// FOR i:= 0 to 1
+/// 	S1 := ROL32((ROL32(A[i], 12) + E[i] + CONST), 7)
+/// 	S2 := S1 ^ ROL32(A[i], 12)
+/// 	T1 := FF(A[i], B[i], C[i], ROUND) + D[i] + S2 + (W[i] ^ W[i+4])
+/// 	T2 := GG(E[i], F[i], G[i], ROUND) + H[i] + S1 + W[i]
+/// 	D[i+1] := C[i]
+/// 	C[i+1] := ROL32(B[i],9)
+/// 	B[i+1] := A[i]
+/// 	A[i+1] := T1
+/// 	H[i+1] := G[i]
+/// 	G[i+1] := ROL32(F[i], 19)
+/// 	F[i+1] := E[i]
+/// 	E[i+1] := P0(T2)
+/// 	CONST := ROL32(CONST, 1)
+/// ENDFOR
+/// dst.dword[3] := A[2]
+/// dst.dword[2] := B[2]
+/// dst.dword[1] := E[2]
+/// dst.dword[0] := F[2]
+/// dst[MAX:128] := 0
+/// \endcode
+#define _mm_sm3rnds2_epi32(A, B, C, D)                                         \
+  (__m128i) __builtin_ia32_vsm3rnds2((__v4su)A, (__v4su)B, (__v4su)C, (int)D)
+
+#undef __DEFAULT_FN_ATTRS128
+
+#endif // __SM3INTRIN_H

lib/include/sm4intrin.h

@@ -0,0 +1,269 @@
+/*===--------------- sm4intrin.h - SM4 intrinsics -----------------===
+ *
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <sm4intrin.h> directly; include <immintrin.h> instead."
+#endif // __IMMINTRIN_H
+
+#ifndef __SM4INTRIN_H
+#define __SM4INTRIN_H
+
+/// This intrinsic performs four rounds of SM4 key expansion. The intrinsic
+///    operates on independent 128-bit lanes. The calculated results are
+///    stored in \a dst.
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_sm4key4_epi32(__m128i __A, __m128i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VSM4KEY4 instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x int].
+/// \param __B
+///    A 128-bit vector of [4 x int].
+/// \returns
+///    A 128-bit vector of [4 x int].
+///
+/// \code{.operation}
+/// DEFINE ROL32(dword, n) {
+/// 	count := n % 32
+/// 	dest := (dword << count) | (dword >> (32-count))
+/// 	RETURN dest
+/// }
+/// DEFINE SBOX_BYTE(dword, i) {
+/// 	RETURN sbox[dword.byte[i]]
+/// }
+/// DEFINE lower_t(dword) {
+/// 	tmp.byte[0] := SBOX_BYTE(dword, 0)
+/// 	tmp.byte[1] := SBOX_BYTE(dword, 1)
+/// 	tmp.byte[2] := SBOX_BYTE(dword, 2)
+/// 	tmp.byte[3] := SBOX_BYTE(dword, 3)
+/// 	RETURN tmp
+/// }
+/// DEFINE L_KEY(dword) {
+/// 	RETURN dword ^ ROL32(dword, 13) ^ ROL32(dword, 23)
+/// }
+/// DEFINE T_KEY(dword) {
+/// 	RETURN L_KEY(lower_t(dword))
+/// }
+/// DEFINE F_KEY(X0, X1, X2, X3, round_key) {
+/// 	RETURN X0 ^ T_KEY(X1 ^ X2 ^ X3 ^ round_key)
+/// }
+/// FOR i:= 0 to 0
+/// 	P[0] := __B.xmm[i].dword[0]
+/// 	P[1] := __B.xmm[i].dword[1]
+/// 	P[2] := __B.xmm[i].dword[2]
+/// 	P[3] := __B.xmm[i].dword[3]
+/// 	C[0] := F_KEY(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0])
+/// 	C[1] := F_KEY(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1])
+/// 	C[2] := F_KEY(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2])
+/// 	C[3] := F_KEY(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3])
+/// 	DEST.xmm[i].dword[0] := C[0]
+/// 	DEST.xmm[i].dword[1] := C[1]
+/// 	DEST.xmm[i].dword[2] := C[2]
+/// 	DEST.xmm[i].dword[3] := C[3]
+/// ENDFOR
+/// DEST[MAX:128] := 0
+/// \endcode
+#define _mm_sm4key4_epi32(A, B)                                                \
+  (__m128i) __builtin_ia32_vsm4key4128((__v4su)A, (__v4su)B)
+
+/// This intrinsic performs four rounds of SM4 key expansion. The intrinsic
+///    operates on independent 128-bit lanes. The calculated results are
+///    stored in \a dst.
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_sm4key4_epi32(__m256i __A, __m256i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VSM4KEY4 instruction.
+///
+/// \param __A
+///    A 256-bit vector of [8 x int].
+/// \param __B
+///    A 256-bit vector of [8 x int].
+/// \returns
+///    A 256-bit vector of [8 x int].
+///
+/// \code{.operation}
+/// DEFINE ROL32(dword, n) {
+/// 	count := n % 32
+/// 	dest := (dword << count) | (dword >> (32-count))
+/// 	RETURN dest
+/// }
+/// DEFINE SBOX_BYTE(dword, i) {
+/// 	RETURN sbox[dword.byte[i]]
+/// }
+/// DEFINE lower_t(dword) {
+/// 	tmp.byte[0] := SBOX_BYTE(dword, 0)
+/// 	tmp.byte[1] := SBOX_BYTE(dword, 1)
+/// 	tmp.byte[2] := SBOX_BYTE(dword, 2)
+/// 	tmp.byte[3] := SBOX_BYTE(dword, 3)
+/// 	RETURN tmp
+/// }
+/// DEFINE L_KEY(dword) {
+/// 	RETURN dword ^ ROL32(dword, 13) ^ ROL32(dword, 23)
+/// }
+/// DEFINE T_KEY(dword) {
+/// 	RETURN L_KEY(lower_t(dword))
+/// }
+/// DEFINE F_KEY(X0, X1, X2, X3, round_key) {
+/// 	RETURN X0 ^ T_KEY(X1 ^ X2 ^ X3 ^ round_key)
+/// }
+/// FOR i:= 0 to 1
+/// 	P[0] := __B.xmm[i].dword[0]
+/// 	P[1] := __B.xmm[i].dword[1]
+/// 	P[2] := __B.xmm[i].dword[2]
+/// 	P[3] := __B.xmm[i].dword[3]
+/// 	C[0] := F_KEY(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0])
+/// 	C[1] := F_KEY(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1])
+/// 	C[2] := F_KEY(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2])
+/// 	C[3] := F_KEY(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3])
+/// 	DEST.xmm[i].dword[0] := C[0]
+/// 	DEST.xmm[i].dword[1] := C[1]
+/// 	DEST.xmm[i].dword[2] := C[2]
+/// 	DEST.xmm[i].dword[3] := C[3]
+/// ENDFOR
+/// DEST[MAX:256] := 0
+/// \endcode
+#define _mm256_sm4key4_epi32(A, B)                                             \
+  (__m256i) __builtin_ia32_vsm4key4256((__v8su)A, (__v8su)B)
+
+/// This intrinisc performs four rounds of SM4 encryption. The intrinisc
+///    operates on independent 128-bit lanes. The calculated results are
+///    stored in \a dst.
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m128i _mm_sm4rnds4_epi32(__m128i __A, __m128i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VSM4RNDS4 instruction.
+///
+/// \param __A
+///    A 128-bit vector of [4 x int].
+/// \param __B
+///    A 128-bit vector of [4 x int].
+/// \returns
+///    A 128-bit vector of [4 x int].
+///
+/// \code{.operation}
+/// DEFINE ROL32(dword, n) {
+/// 	count := n % 32
+/// 	dest := (dword << count) | (dword >> (32-count))
+/// 	RETURN dest
+/// }
+/// DEFINE lower_t(dword) {
+/// 	tmp.byte[0] := SBOX_BYTE(dword, 0)
+/// 	tmp.byte[1] := SBOX_BYTE(dword, 1)
+/// 	tmp.byte[2] := SBOX_BYTE(dword, 2)
+/// 	tmp.byte[3] := SBOX_BYTE(dword, 3)
+/// 	RETURN tmp
+/// }
+/// DEFINE L_RND(dword) {
+/// 	tmp := dword
+/// 	tmp := tmp ^ ROL32(dword, 2)
+/// 	tmp := tmp ^ ROL32(dword, 10)
+/// 	tmp := tmp ^ ROL32(dword, 18)
+/// 	tmp := tmp ^ ROL32(dword, 24)
+///   RETURN tmp
+/// }
+/// DEFINE T_RND(dword) {
+/// 	RETURN L_RND(lower_t(dword))
+/// }
+/// DEFINE F_RND(X0, X1, X2, X3, round_key) {
+/// 	RETURN X0 ^ T_RND(X1 ^ X2 ^ X3 ^ round_key)
+/// }
+/// FOR i:= 0 to 0
+/// 	P[0] := __B.xmm[i].dword[0]
+/// 	P[1] := __B.xmm[i].dword[1]
+/// 	P[2] := __B.xmm[i].dword[2]
+/// 	P[3] := __B.xmm[i].dword[3]
+/// 	C[0] := F_RND(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0])
+/// 	C[1] := F_RND(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1])
+/// 	C[2] := F_RND(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2])
+/// 	C[3] := F_RND(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3])
+/// 	DEST.xmm[i].dword[0] := C[0]
+/// 	DEST.xmm[i].dword[1] := C[1]
+/// 	DEST.xmm[i].dword[2] := C[2]
+/// 	DEST.xmm[i].dword[3] := C[3]
+/// ENDFOR
+/// DEST[MAX:128] := 0
+/// \endcode
+#define _mm_sm4rnds4_epi32(A, B)                                               \
+  (__m128i) __builtin_ia32_vsm4rnds4128((__v4su)A, (__v4su)B)
+
+/// This intrinisc performs four rounds of SM4 encryption. The intrinisc
+///    operates on independent 128-bit lanes. The calculated results are
+///    stored in \a dst.
+/// \headerfile <immintrin.h>
+///
+/// \code
+/// __m256i _mm256_sm4rnds4_epi32(__m256i __A, __m256i __B)
+/// \endcode
+///
+/// This intrinsic corresponds to the \c VSM4RNDS4 instruction.
+///
+/// \param __A
+///    A 256-bit vector of [8 x int].
+/// \param __B
+///    A 256-bit vector of [8 x int].
+/// \returns
+///    A 256-bit vector of [8 x int].
+///
+/// \code{.operation}
+/// DEFINE ROL32(dword, n) {
+/// 	count := n % 32
+/// 	dest := (dword << count) | (dword >> (32-count))
+/// 	RETURN dest
+/// }
+/// DEFINE lower_t(dword) {
+/// 	tmp.byte[0] := SBOX_BYTE(dword, 0)
+/// 	tmp.byte[1] := SBOX_BYTE(dword, 1)
+/// 	tmp.byte[2] := SBOX_BYTE(dword, 2)
+/// 	tmp.byte[3] := SBOX_BYTE(dword, 3)
+/// 	RETURN tmp
+/// }
+/// DEFINE L_RND(dword) {
+/// 	tmp := dword
+/// 	tmp := tmp ^ ROL32(dword, 2)
+/// 	tmp := tmp ^ ROL32(dword, 10)
+/// 	tmp := tmp ^ ROL32(dword, 18)
+/// 	tmp := tmp ^ ROL32(dword, 24)
+///   RETURN tmp
+/// }
+/// DEFINE T_RND(dword) {
+/// 	RETURN L_RND(lower_t(dword))
+/// }
+/// DEFINE F_RND(X0, X1, X2, X3, round_key) {
+/// 	RETURN X0 ^ T_RND(X1 ^ X2 ^ X3 ^ round_key)
+/// }
+/// FOR i:= 0 to 0
+/// 	P[0] := __B.xmm[i].dword[0]
+/// 	P[1] := __B.xmm[i].dword[1]
+/// 	P[2] := __B.xmm[i].dword[2]
+/// 	P[3] := __B.xmm[i].dword[3]
+/// 	C[0] := F_RND(P[0], P[1], P[2], P[3], __A.xmm[i].dword[0])
+/// 	C[1] := F_RND(P[1], P[2], P[3], C[0], __A.xmm[i].dword[1])
+/// 	C[2] := F_RND(P[2], P[3], C[0], C[1], __A.xmm[i].dword[2])
+/// 	C[3] := F_RND(P[3], C[0], C[1], C[2], __A.xmm[i].dword[3])
+/// 	DEST.xmm[i].dword[0] := C[0]
+/// 	DEST.xmm[i].dword[1] := C[1]
+/// 	DEST.xmm[i].dword[2] := C[2]
+/// 	DEST.xmm[i].dword[3] := C[3]
+/// ENDFOR
+/// DEST[MAX:256] := 0
+/// \endcode
+#define _mm256_sm4rnds4_epi32(A, B)                                            \
+  (__m256i) __builtin_ia32_vsm4rnds4256((__v8su)A, (__v8su)B)
+
+#endif // __SM4INTRIN_H

lib/include/stdalign.h

@@ -10,6 +10,10 @@
 #ifndef __STDALIGN_H
 #define __STDALIGN_H
 
+/* FIXME: This is using the placeholder dates Clang produces for these macros
+   in C2x mode; switch to the correct values once they've been published. */
+#if defined(__cplusplus) ||                                                    \
+    (defined(__STDC_VERSION__) && __STDC_VERSION__ < 202000L)
 #ifndef __cplusplus
 #define alignas _Alignas
 #define alignof _Alignof
@@ -17,5 +21,6 @@
 
 #define __alignas_is_defined 1
 #define __alignof_is_defined 1
+#endif /* __STDC_VERSION__ */
 
 #endif /* __STDALIGN_H */

lib/include/stdatomic.h

@@ -45,9 +45,16 @@ extern "C" {
 #define ATOMIC_POINTER_LOCK_FREE    __CLANG_ATOMIC_POINTER_LOCK_FREE
 
 /* 7.17.2 Initialization */
-
+/* FIXME: This is using the placeholder dates Clang produces for these macros
+   in C2x mode; switch to the correct values once they've been published. */
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ < 202000L) ||               \
+    defined(__cplusplus)
+/* ATOMIC_VAR_INIT was removed in C2x, but still remains in C++23. */
 #define ATOMIC_VAR_INIT(value) (value)
-#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201710L) ||             \
+#endif
+
+#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201710L &&              \
+      __STDC_VERSION__ < 202000L) ||                                           \
      (defined(__cplusplus) && __cplusplus >= 202002L)) &&                      \
     !defined(_CLANG_DISABLE_CRT_DEPRECATION_WARNINGS)
 /* ATOMIC_VAR_INIT was deprecated in C17 and C++20. */

lib/include/stddef.h

@@ -103,6 +103,11 @@ using ::std::nullptr_t;
 typedef typeof(nullptr) nullptr_t;
 #endif /* defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202000L */
 
+#if defined(__need_STDDEF_H_misc) && defined(__STDC_VERSION__) &&              \
+    __STDC_VERSION__ >= 202000L
+#define unreachable() __builtin_unreachable()
+#endif /* defined(__need_STDDEF_H_misc) && >= C23 */
+
 #if defined(__need_STDDEF_H_misc)
 #if (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) ||              \
     (defined(__cplusplus) && __cplusplus >= 201103L)

lib/include/wasm_simd128.h

@@ -961,17 +961,17 @@ static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_popcnt(v128_t __a) {
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shl(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__i8x16)__a << __b);
+  return (v128_t)((__i8x16)__a << (__b & 0x7));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_shr(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__i8x16)__a >> __b);
+  return (v128_t)((__i8x16)__a >> (__b & 0x7));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u8x16_shr(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__u8x16)__a >> __b);
+  return (v128_t)((__u8x16)__a >> (__b & 0x7));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i8x16_add(v128_t __a,
@@ -1047,17 +1047,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i16x8_bitmask(v128_t __a) {
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shl(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__i16x8)__a << __b);
+  return (v128_t)((__i16x8)__a << (__b & 0xF));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_shr(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__i16x8)__a >> __b);
+  return (v128_t)((__i16x8)__a >> (__b & 0xF));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u16x8_shr(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__u16x8)__a >> __b);
+  return (v128_t)((__u16x8)__a >> (__b & 0xF));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i16x8_add(v128_t __a,
@@ -1138,17 +1138,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i32x4_bitmask(v128_t __a) {
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shl(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__i32x4)__a << __b);
+  return (v128_t)((__i32x4)__a << (__b & 0x1F));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_shr(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__i32x4)__a >> __b);
+  return (v128_t)((__i32x4)__a >> (__b & 0x1F));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u32x4_shr(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__u32x4)__a >> __b);
+  return (v128_t)((__u32x4)__a >> (__b & 0x1F));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i32x4_add(v128_t __a,
@@ -1209,17 +1209,17 @@ static __inline__ uint32_t __DEFAULT_FN_ATTRS wasm_i64x2_bitmask(v128_t __a) {
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shl(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__i64x2)__a << (int64_t)__b);
+  return (v128_t)((__i64x2)__a << ((int64_t)__b & 0x3F));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_shr(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__i64x2)__a >> (int64_t)__b);
+  return (v128_t)((__i64x2)__a >> ((int64_t)__b & 0x3F));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_u64x2_shr(v128_t __a,
                                                            uint32_t __b) {
-  return (v128_t)((__u64x2)__a >> (int64_t)__b);
+  return (v128_t)((__u64x2)__a >> ((int64_t)__b & 0x3F));
 }
 
 static __inline__ v128_t __DEFAULT_FN_ATTRS wasm_i64x2_add(v128_t __a,
@@ -1760,6 +1760,126 @@ wasm_u64x2_load_32x2(const void *__mem) {
   __DEPRECATED_WASM_MACRO("wasm_v64x2_shuffle", "wasm_i64x2_shuffle")          \
   wasm_i64x2_shuffle(__a, __b, __c0, __c1)
 
+// Relaxed SIMD intrinsics
+
+#define __RELAXED_FN_ATTRS                                                     \
+  __attribute__((__always_inline__, __nodebug__, __target__("relaxed-simd"),   \
+                 __min_vector_width__(128)))
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_f32x4_relaxed_madd(v128_t __a, v128_t __b, v128_t __c) {
+  return (v128_t)__builtin_wasm_relaxed_madd_f32x4((__f32x4)__a, (__f32x4)__b,
+                                                   (__f32x4)__c);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_f32x4_relaxed_nmadd(v128_t __a, v128_t __b, v128_t __c) {
+  return (v128_t)__builtin_wasm_relaxed_nmadd_f32x4((__f32x4)__a, (__f32x4)__b,
+                                                    (__f32x4)__c);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_f64x2_relaxed_madd(v128_t __a, v128_t __b, v128_t __c) {
+  return (v128_t)__builtin_wasm_relaxed_madd_f64x2((__f64x2)__a, (__f64x2)__b,
+                                                   (__f64x2)__c);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_f64x2_relaxed_nmadd(v128_t __a, v128_t __b, v128_t __c) {
+  return (v128_t)__builtin_wasm_relaxed_nmadd_f64x2((__f64x2)__a, (__f64x2)__b,
+                                                    (__f64x2)__c);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i8x16_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) {
+  return (v128_t)__builtin_wasm_relaxed_laneselect_i8x16(
+      (__i8x16)__a, (__i8x16)__b, (__i8x16)__m);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i16x8_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) {
+  return (v128_t)__builtin_wasm_relaxed_laneselect_i16x8(
+      (__i16x8)__a, (__i16x8)__b, (__i16x8)__m);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i32x4_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) {
+  return (v128_t)__builtin_wasm_relaxed_laneselect_i32x4(
+      (__i32x4)__a, (__i32x4)__b, (__i32x4)__m);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i64x2_relaxed_laneselect(v128_t __a, v128_t __b, v128_t __m) {
+  return (v128_t)__builtin_wasm_relaxed_laneselect_i64x2(
+      (__i64x2)__a, (__i64x2)__b, (__i64x2)__m);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i8x16_relaxed_swizzle(v128_t __a, v128_t __s) {
+  return (v128_t)__builtin_wasm_relaxed_swizzle_i8x16((__i8x16)__a,
+                                                      (__i8x16)__s);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f32x4_relaxed_min(v128_t __a,
+                                                                   v128_t __b) {
+  return (v128_t)__builtin_wasm_relaxed_min_f32x4((__f32x4)__a, (__f32x4)__b);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f32x4_relaxed_max(v128_t __a,
+                                                                   v128_t __b) {
+  return (v128_t)__builtin_wasm_relaxed_max_f32x4((__f32x4)__a, (__f32x4)__b);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f64x2_relaxed_min(v128_t __a,
+                                                                   v128_t __b) {
+  return (v128_t)__builtin_wasm_relaxed_min_f64x2((__f64x2)__a, (__f64x2)__b);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS wasm_f64x2_relaxed_max(v128_t __a,
+                                                                   v128_t __b) {
+  return (v128_t)__builtin_wasm_relaxed_max_f64x2((__f64x2)__a, (__f64x2)__b);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i32x4_relaxed_trunc_f32x4(v128_t __a) {
+  return (v128_t)__builtin_wasm_relaxed_trunc_s_i32x4_f32x4((__f32x4)__a);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_u32x4_relaxed_trunc_f32x4(v128_t __a) {
+  return (v128_t)__builtin_wasm_relaxed_trunc_u_i32x4_f32x4((__f32x4)__a);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i32x4_relaxed_trunc_f64x2_zero(v128_t __a) {
+  return (v128_t)__builtin_wasm_relaxed_trunc_s_zero_i32x4_f64x2((__f64x2)__a);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_u32x4_relaxed_trunc_f64x2_zero(v128_t __a) {
+  return (v128_t)__builtin_wasm_relaxed_trunc_u_zero_i32x4_f64x2((__f64x2)__a);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i16x8_relaxed_q15mulr(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_relaxed_q15mulr_s_i16x8((__i16x8)__a,
+                                                        (__i16x8)__b);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i16x8_relaxed_dot_i8x16_i7x16(v128_t __a, v128_t __b) {
+  return (v128_t)__builtin_wasm_relaxed_dot_i8x16_i7x16_s_i16x8((__i8x16)__a,
+                                                                (__i8x16)__b);
+}
+
+static __inline__ v128_t __RELAXED_FN_ATTRS
+wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v128_t __a, v128_t __b, v128_t __c) {
+  return (v128_t)__builtin_wasm_relaxed_dot_i8x16_i7x16_add_s_i32x4(
+      (__i8x16)__a, (__i8x16)__b, (__i32x4)__c);
+}
+
+// Deprecated intrinsics
+
 static __inline__ v128_t __DEPRECATED_FN_ATTRS("wasm_i8x16_swizzle")
 wasm_v8x16_swizzle(v128_t __a, v128_t __b) {
   return wasm_i8x16_swizzle(__a, __b);

lib/include/xsavecintrin.h

@@ -17,12 +17,62 @@
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__,  __target__("xsavec")))
 
+/// Performs a full or partial save of processor state to the memory at
+///    \a __p. The exact state saved depends on the 64-bit mask \a __m and
+///    processor control register \c XCR0.
+///
+/// \code{.operation}
+/// mask[62:0] := __m[62:0] AND XCR0[62:0]
+/// FOR i := 0 TO 62
+///   IF mask[i] == 1
+///     CASE (i) OF
+///     0: save X87 FPU state
+///     1: save SSE state
+///     DEFAULT: __p.Ext_Save_Area[i] := ProcessorState[i]
+///   FI
+/// ENDFOR
+/// __p.Header.XSTATE_BV[62:0] := INIT_FUNCTION(mask[62:0])
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c XSAVEC instruction.
+///
+/// \param __p
+///    Pointer to the save area; must be 64-byte aligned.
+/// \param __m
+///    A 64-bit mask indicating what state should be saved.
 static __inline__ void __DEFAULT_FN_ATTRS
 _xsavec(void *__p, unsigned long long __m) {
   __builtin_ia32_xsavec(__p, __m);
 }
 
 #ifdef __x86_64__
+/// Performs a full or partial save of processor state to the memory at
+///    \a __p. The exact state saved depends on the 64-bit mask \a __m and
+///    processor control register \c XCR0.
+///
+/// \code{.operation}
+/// mask[62:0] := __m[62:0] AND XCR0[62:0]
+/// FOR i := 0 TO 62
+///   IF mask[i] == 1
+///     CASE (i) OF
+///     0: save X87 FPU state
+///     1: save SSE state
+///     DEFAULT: __p.Ext_Save_Area[i] := ProcessorState[i]
+///   FI
+/// ENDFOR
+/// __p.Header.XSTATE_BV[62:0] := INIT_FUNCTION(mask[62:0])
+/// \endcode
+///
+/// \headerfile <immintrin.h>
+///
+/// This intrinsic corresponds to the \c XSAVEC64 instruction.
+///
+/// \param __p
+///    Pointer to the save area; must be 64-byte aligned.
+/// \param __m
+///    A 64-bit mask indicating what state should be saved.
 static __inline__ void __DEFAULT_FN_ATTRS
 _xsavec64(void *__p, unsigned long long __m) {
   __builtin_ia32_xsavec64(__p, __m);