core/stdarch/crates/core_arch/src/s390x/
vector.rs

1//! s390x vector intrinsics.
2//!
3//! For more info see the [Reference Summary] or the online [IBM docs].
4//!
5//! [Reference Summary]: https://www.ibm.com/support/pages/sites/default/files/2021-05/SA22-7871-10.pdf
6//! [IBM docs]: https://www.ibm.com/docs/en/zos/2.4.0?topic=support-vector-built-in-functions
7
8#![allow(non_camel_case_types)]
9
10use crate::{core_arch::simd::*, intrinsics::simd::*, mem::MaybeUninit, mem::transmute};
11
12#[cfg(test)]
13use stdarch_test::assert_instr;
14
15use super::macros::*;
16
17types! {
18    #![unstable(feature = "stdarch_s390x", issue = "135681")]
19
20    /// s390x-specific 128-bit wide vector of sixteen packed `i8`
21    pub struct vector_signed_char(16 x i8);
22    /// s390x-specific 128-bit wide vector of sixteen packed `u8`
23    pub struct vector_unsigned_char(16 x u8);
24    /// s390x-specific 128-bit wide vector mask of sixteen packed elements
25    pub struct vector_bool_char(16 x i8);
26
27    /// s390x-specific 128-bit wide vector of eight packed `i16`
28    pub struct vector_signed_short(8 x i16);
29    /// s390x-specific 128-bit wide vector of eight packed `u16`
30    pub struct vector_unsigned_short(8 x u16);
31    /// s390x-specific 128-bit wide vector mask of eight packed elements
32    pub struct vector_bool_short(8 x i16);
33
34    /// s390x-specific 128-bit wide vector of four packed `i32`
35    pub struct vector_signed_int(4 x i32);
36    /// s390x-specific 128-bit wide vector of four packed `u32`
37    pub struct vector_unsigned_int(4 x u32);
38    /// s390x-specific 128-bit wide vector mask of four packed elements
39    pub struct vector_bool_int(4 x i32);
40
41    /// s390x-specific 128-bit wide vector of two packed `i64`
42    pub struct vector_signed_long_long(2 x i64);
43    /// s390x-specific 128-bit wide vector of two packed `u64`
44    pub struct vector_unsigned_long_long(2 x u64);
45    /// s390x-specific 128-bit wide vector mask of two packed elements
46    pub struct vector_bool_long_long(2 x i64);
47
48    /// s390x-specific 128-bit wide vector of four packed `f32`
49    pub struct vector_float(4 x f32);
50    /// s390x-specific 128-bit wide vector of two packed `f64`
51    pub struct vector_double(2 x f64);
52}
53
54#[unstable(feature = "stdarch_s390x", issue = "135681")]
55impl From<m8x16> for vector_bool_char {
56    #[inline]
57    fn from(value: m8x16) -> Self {
58        unsafe { transmute(value) }
59    }
60}
61
62#[unstable(feature = "stdarch_s390x", issue = "135681")]
63impl From<vector_bool_char> for m8x16 {
64    #[inline]
65    fn from(value: vector_bool_char) -> Self {
66        unsafe { transmute(value) }
67    }
68}
69
70#[unstable(feature = "stdarch_s390x", issue = "135681")]
71impl From<m16x8> for vector_bool_short {
72    #[inline]
73    fn from(value: m16x8) -> Self {
74        unsafe { transmute(value) }
75    }
76}
77
78#[unstable(feature = "stdarch_s390x", issue = "135681")]
79impl From<vector_bool_short> for m16x8 {
80    #[inline]
81    fn from(value: vector_bool_short) -> Self {
82        unsafe { transmute(value) }
83    }
84}
85
86#[unstable(feature = "stdarch_s390x", issue = "135681")]
87impl From<m32x4> for vector_bool_int {
88    #[inline]
89    fn from(value: m32x4) -> Self {
90        unsafe { transmute(value) }
91    }
92}
93
94#[unstable(feature = "stdarch_s390x", issue = "135681")]
95impl From<vector_bool_int> for m32x4 {
96    #[inline]
97    fn from(value: vector_bool_int) -> Self {
98        unsafe { transmute(value) }
99    }
100}
101
102#[repr(C, packed)]
103struct PackedTuple<T, U> {
104    x: T,
105    y: U,
106}
107
108#[allow(improper_ctypes)]
109#[rustfmt::skip]
110unsafe extern "unadjusted" {
111    #[link_name = "llvm.nearbyint.v4f32"] fn nearbyint_v4f32(a: vector_float) -> vector_float;
112    #[link_name = "llvm.nearbyint.v2f64"] fn nearbyint_v2f64(a: vector_double) -> vector_double;
113
114    #[link_name = "llvm.roundeven.v4f32"] fn roundeven_v4f32(a: vector_float) -> vector_float;
115    #[link_name = "llvm.roundeven.v2f64"] fn roundeven_v2f64(a: vector_double) -> vector_double;
116
117    #[link_name = "llvm.s390.vsra"] fn vsra(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
118    #[link_name = "llvm.s390.vsrl"] fn vsrl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
119    #[link_name = "llvm.s390.vsl"] fn vsl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
120
121    #[link_name = "llvm.s390.vsrab"] fn vsrab(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
122    #[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
123    #[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
124
125    #[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16;
126
127    #[link_name = "llvm.s390.verimb"] fn verimb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char, d: i32) -> vector_signed_char;
128    #[link_name = "llvm.s390.verimh"] fn verimh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short, d: i32) -> vector_signed_short;
129    #[link_name = "llvm.s390.verimf"] fn verimf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int, d: i32) -> vector_signed_int;
130    #[link_name = "llvm.s390.verimg"] fn verimg(a: vector_signed_long_long, b: vector_signed_long_long, c: vector_signed_long_long, d: i32) -> vector_signed_long_long;
131
132    #[link_name = "llvm.s390.vperm"] fn vperm(a: vector_signed_char, b: vector_signed_char, c: vector_unsigned_char) -> vector_signed_char;
133
134    #[link_name = "llvm.s390.vsumb"] fn vsumb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int;
135    #[link_name = "llvm.s390.vsumh"] fn vsumh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
136
137    #[link_name = "llvm.s390.vsumgh"] fn vsumgh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long;
138    #[link_name = "llvm.s390.vsumgf"] fn vsumgf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
139
140    #[link_name = "llvm.s390.vsumqf"] fn vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128;
141    #[link_name = "llvm.s390.vsumqg"] fn vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
142
143    #[link_name = "llvm.s390.vaccq"] fn vaccq(a: u128, b: u128) -> u128;
144    #[link_name = "llvm.s390.vacccq"] fn vacccq(a: u128, b: u128, c: u128) -> u128;
145
146    #[link_name = "llvm.s390.vscbiq"] fn vscbiq(a: u128, b: u128) -> u128;
147    #[link_name = "llvm.s390.vsbiq"] fn vsbiq(a: u128, b: u128, c: u128) -> u128;
148    #[link_name = "llvm.s390.vsbcbiq"] fn vsbcbiq(a: u128, b: u128, c: u128) -> u128;
149
150    #[link_name = "llvm.s390.vacq"] fn vacq(a: u128, b: u128, c: u128) -> u128;
151
152    #[link_name = "llvm.s390.vscbib"] fn vscbib(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
153    #[link_name = "llvm.s390.vscbih"] fn vscbih(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
154    #[link_name = "llvm.s390.vscbif"] fn vscbif(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
155    #[link_name = "llvm.s390.vscbig"] fn vscbig(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
156
157    #[link_name = "llvm.s390.vfaeb"] fn vfaeb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
158    #[link_name = "llvm.s390.vfaeh"] fn vfaeh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
159    #[link_name = "llvm.s390.vfaef"] fn vfaef(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
160
161    #[link_name = "llvm.s390.vfaezb"] fn vfaezb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
162    #[link_name = "llvm.s390.vfaezh"] fn vfaezh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
163    #[link_name = "llvm.s390.vfaezf"] fn vfaezf(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
164
165    #[link_name = "llvm.s390.vfaebs"] fn vfaebs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
166    #[link_name = "llvm.s390.vfaehs"] fn vfaehs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
167    #[link_name = "llvm.s390.vfaefs"] fn vfaefs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
168
169    #[link_name = "llvm.s390.vfaezbs"] fn vfaezbs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
170    #[link_name = "llvm.s390.vfaezhs"] fn vfaezhs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
171    #[link_name = "llvm.s390.vfaezfs"] fn vfaezfs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
172
173    #[link_name = "llvm.s390.vll"] fn vll(a: u32, b: *const u8) -> vector_signed_char;
174    #[link_name = "llvm.s390.vstl"] fn vstl(a: vector_signed_char, b: u32, c: *mut u8);
175
176    #[link_name = "llvm.s390.vlrl"] fn vlrl(a: u32, b: *const u8) -> vector_unsigned_char;
177    #[link_name = "llvm.s390.vstrl"] fn vstrl(a: vector_unsigned_char, b: u32, c: *mut u8);
178
179    #[link_name = "llvm.s390.lcbb"] fn lcbb(a: *const u8, b: u32) -> u32;
180    #[link_name = "llvm.s390.vlbb"] fn vlbb(a: *const u8, b: u32) -> MaybeUninit<vector_signed_char>;
181
182    #[link_name = "llvm.s390.vpksh"] fn vpksh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_char;
183    #[link_name = "llvm.s390.vpksf"] fn vpksf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_short;
184    #[link_name = "llvm.s390.vpksg"] fn vpksg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_int;
185
186    #[link_name = "llvm.s390.vpklsh"] fn vpklsh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char;
187    #[link_name = "llvm.s390.vpklsf"] fn vpklsf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short;
188    #[link_name = "llvm.s390.vpklsg"] fn vpklsg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_unsigned_int;
189
190    #[link_name = "llvm.s390.vpkshs"] fn vpkshs(a: vector_signed_short, b: vector_signed_short) -> PackedTuple<vector_signed_char, i32>;
191    #[link_name = "llvm.s390.vpksfs"] fn vpksfs(a: vector_signed_int, b: vector_signed_int) -> PackedTuple<vector_signed_short, i32>;
192    #[link_name = "llvm.s390.vpksgs"] fn vpksgs(a: vector_signed_long_long, b: vector_signed_long_long) -> PackedTuple<vector_signed_int, i32>;
193
194    #[link_name = "llvm.s390.vpklshs"] fn vpklshs(a: vector_unsigned_short, b: vector_unsigned_short) -> PackedTuple<vector_unsigned_char, i32>;
195    #[link_name = "llvm.s390.vpklsfs"] fn vpklsfs(a: vector_unsigned_int, b: vector_unsigned_int) -> PackedTuple<vector_unsigned_short, i32>;
196    #[link_name = "llvm.s390.vpklsgs"] fn vpklsgs(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> PackedTuple<vector_unsigned_int, i32>;
197
198    #[link_name = "llvm.s390.vavgb"] fn vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
199    #[link_name = "llvm.s390.vavgh"] fn vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
200    #[link_name = "llvm.s390.vavgf"] fn vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
201    #[link_name = "llvm.s390.vavgg"] fn vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
202
203    #[link_name = "llvm.s390.vavglb"] fn vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
204    #[link_name = "llvm.s390.vavglh"] fn vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
205    #[link_name = "llvm.s390.vavglf"] fn vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
206    #[link_name = "llvm.s390.vavglg"] fn vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
207
208    #[link_name = "llvm.s390.vcksm"] fn vcksm(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
209
210    #[link_name = "llvm.s390.vmhb"] fn vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
211    #[link_name = "llvm.s390.vmhh"] fn vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
212    #[link_name = "llvm.s390.vmhf"] fn vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
213
214    #[link_name = "llvm.s390.vmlhb"] fn vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
215    #[link_name = "llvm.s390.vmlhh"] fn vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
216    #[link_name = "llvm.s390.vmlhf"] fn vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
217
218    #[link_name = "llvm.s390.vmaeb"] fn vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
219    #[link_name = "llvm.s390.vmaeh"] fn vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
220    #[link_name = "llvm.s390.vmaef"] fn vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
221
222    #[link_name = "llvm.s390.vmaleb"] fn vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
223    #[link_name = "llvm.s390.vmaleh"] fn vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
224    #[link_name = "llvm.s390.vmalef"] fn vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
225
226    #[link_name = "llvm.s390.vmaob"] fn vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
227    #[link_name = "llvm.s390.vmaoh"] fn vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
228    #[link_name = "llvm.s390.vmaof"] fn vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
229
230    #[link_name = "llvm.s390.vmalob"] fn vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
231    #[link_name = "llvm.s390.vmaloh"] fn vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
232    #[link_name = "llvm.s390.vmalof"] fn vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
233
234    #[link_name = "llvm.s390.vmahb"] fn vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
235    #[link_name = "llvm.s390.vmahh"] fn vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
236    #[link_name = "llvm.s390.vmahf"] fn vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
237
238    #[link_name = "llvm.s390.vmalhb"] fn vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
239    #[link_name = "llvm.s390.vmalhh"] fn vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
240    #[link_name = "llvm.s390.vmalhf"] fn vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
241
242    #[link_name = "llvm.s390.vmalb"] fn vmalb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
243    #[link_name = "llvm.s390.vmalh"] fn vmalh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
244    #[link_name = "llvm.s390.vmalf"] fn vmalf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
245
246    #[link_name = "llvm.s390.vmallb"] fn vmallb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
247    #[link_name = "llvm.s390.vmallh"] fn vmallh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
248    #[link_name = "llvm.s390.vmallf"] fn vmallf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
249
250    #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
251    #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
252    #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
253    #[link_name = "llvm.s390.vgfmg"] fn vgfmg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
254
255    #[link_name = "llvm.s390.vgfmab"] fn vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
256    #[link_name = "llvm.s390.vgfmah"] fn vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
257    #[link_name = "llvm.s390.vgfmaf"] fn vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
258    #[link_name = "llvm.s390.vgfmag"] fn vgfmag(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128) -> u128;
259
260    #[link_name = "llvm.s390.vbperm"] fn vbperm(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_long_long;
261
262    #[link_name = "llvm.s390.vftcisb"] fn vftcisb(a: vector_float, b: u32) -> PackedTuple<vector_bool_int, i32>;
263    #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple<vector_bool_long_long, i32>;
264
265    #[link_name = "llvm.s390.vtm"] fn vtm(a: i8x16, b: i8x16) -> i32;
266
267    #[link_name = "llvm.s390.vstrsb"] fn vstrsb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
268    #[link_name = "llvm.s390.vstrsh"] fn vstrsh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
269    #[link_name = "llvm.s390.vstrsf"] fn vstrsf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
270
271    #[link_name = "llvm.s390.vstrszb"] fn vstrszb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
272    #[link_name = "llvm.s390.vstrszh"] fn vstrszh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
273    #[link_name = "llvm.s390.vstrszf"] fn vstrszf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
274
275    #[link_name = "llvm.s390.vistrb"] fn vistrb(a: vector_unsigned_char) -> vector_unsigned_char;
276    #[link_name = "llvm.s390.vistrh"] fn vistrh(a: vector_unsigned_short) -> vector_unsigned_short;
277    #[link_name = "llvm.s390.vistrf"] fn vistrf(a: vector_unsigned_int) -> vector_unsigned_int;
278
279    #[link_name = "llvm.s390.vistrbs"] fn vistrbs(a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
280    #[link_name = "llvm.s390.vistrhs"] fn vistrhs(a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32>;
281    #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32>;
282
283    #[link_name = "llvm.s390.vmslg"] fn vmslg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128, d: u32) -> u128;
284
285    #[link_name = "llvm.s390.vstrcb"] fn vstrcb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
286    #[link_name = "llvm.s390.vstrch"] fn vstrch(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
287    #[link_name = "llvm.s390.vstrcf"] fn vstrcf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
288
289    #[link_name = "llvm.s390.vstrcbs"] fn vstrcbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
290    #[link_name = "llvm.s390.vstrchs"] fn vstrchs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
291    #[link_name = "llvm.s390.vstrcfs"] fn vstrcfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
292
293    #[link_name = "llvm.s390.vstrczb"] fn vstrczb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
294    #[link_name = "llvm.s390.vstrczh"] fn vstrczh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
295    #[link_name = "llvm.s390.vstrczf"] fn vstrczf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
296
297    #[link_name = "llvm.s390.vstrczbs"] fn vstrczbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
298    #[link_name = "llvm.s390.vstrczhs"] fn vstrczhs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
299    #[link_name = "llvm.s390.vstrczfs"] fn vstrczfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
300
301    #[link_name = "llvm.s390.vfeeb"] fn vfeeb(a: i8x16, b: i8x16) -> i8x16;
302    #[link_name = "llvm.s390.vfeeh"] fn vfeeh(a: i16x8, b: i16x8) -> i16x8;
303    #[link_name = "llvm.s390.vfeef"] fn vfeef(a: i32x4, b: i32x4) -> i32x4;
304
305    #[link_name = "llvm.s390.vfeezb"] fn vfeezb(a: i8x16, b: i8x16) -> i8x16;
306    #[link_name = "llvm.s390.vfeezh"] fn vfeezh(a: i16x8, b: i16x8) -> i16x8;
307    #[link_name = "llvm.s390.vfeezf"] fn vfeezf(a: i32x4, b: i32x4) -> i32x4;
308
309    #[link_name = "llvm.s390.vfeebs"] fn vfeebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
310    #[link_name = "llvm.s390.vfeehs"] fn vfeehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
311    #[link_name = "llvm.s390.vfeefs"] fn vfeefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
312
313    #[link_name = "llvm.s390.vfeezbs"] fn vfeezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
314    #[link_name = "llvm.s390.vfeezhs"] fn vfeezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
315    #[link_name = "llvm.s390.vfeezfs"] fn vfeezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
316
317    #[link_name = "llvm.s390.vfeneb"] fn vfeneb(a: i8x16, b: i8x16) -> i8x16;
318    #[link_name = "llvm.s390.vfeneh"] fn vfeneh(a: i16x8, b: i16x8) -> i16x8;
319    #[link_name = "llvm.s390.vfenef"] fn vfenef(a: i32x4, b: i32x4) -> i32x4;
320
321    #[link_name = "llvm.s390.vfenezb"] fn vfenezb(a: i8x16, b: i8x16) -> i8x16;
322    #[link_name = "llvm.s390.vfenezh"] fn vfenezh(a: i16x8, b: i16x8) -> i16x8;
323    #[link_name = "llvm.s390.vfenezf"] fn vfenezf(a: i32x4, b: i32x4) -> i32x4;
324
325    #[link_name = "llvm.s390.vfenebs"] fn vfenebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
326    #[link_name = "llvm.s390.vfenehs"] fn vfenehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
327    #[link_name = "llvm.s390.vfenefs"] fn vfenefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
328
329    #[link_name = "llvm.s390.vfenezbs"] fn vfenezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
330    #[link_name = "llvm.s390.vfenezhs"] fn vfenezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
331    #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
332
333    #[link_name = "llvm.s390.vclfnhs"] fn vclfnhs(a: vector_signed_short, immarg: i32) -> vector_float;
334    #[link_name = "llvm.s390.vclfnls"] fn vclfnls(a: vector_signed_short, immarg: i32) -> vector_float;
335    #[link_name = "llvm.s390.vcfn"] fn vcfn(a: vector_signed_short, immarg: i32) -> vector_signed_short;
336    #[link_name = "llvm.s390.vcnf"] fn vcnf(a: vector_signed_short, immarg: i32) -> vector_signed_short;
337    #[link_name = "llvm.s390.vcrnfs"] fn vcrnfs(a: vector_float, b: vector_float, immarg: i32) -> vector_signed_short;
338}
339
340#[repr(simd)]
341struct ShuffleMask<const N: usize>([u32; N]);
342
343impl<const N: usize> ShuffleMask<N> {
344    const fn reverse() -> Self {
345        let mut index = [0; N];
346        let mut i = 0;
347        while i < N {
348            index[i] = (N - i - 1) as u32;
349            i += 1;
350        }
351        ShuffleMask(index)
352    }
353
354    const fn merge_low() -> Self {
355        let mut mask = [0; N];
356        let mut i = N / 2;
357        let mut index = 0;
358        while index < N {
359            mask[index] = i as u32;
360            mask[index + 1] = (i + N) as u32;
361
362            i += 1;
363            index += 2;
364        }
365        ShuffleMask(mask)
366    }
367
368    const fn merge_high() -> Self {
369        let mut mask = [0; N];
370        let mut i = 0;
371        let mut index = 0;
372        while index < N {
373            mask[index] = i as u32;
374            mask[index + 1] = (i + N) as u32;
375
376            i += 1;
377            index += 2;
378        }
379        ShuffleMask(mask)
380    }
381
382    const fn even() -> Self {
383        let mut mask = [0; N];
384        let mut i = 0;
385        let mut index = 0;
386        while index < N {
387            mask[index] = i as u32;
388
389            i += 2;
390            index += 1;
391        }
392        ShuffleMask(mask)
393    }
394
395    const fn odd() -> Self {
396        let mut mask = [0; N];
397        let mut i = 1;
398        let mut index = 0;
399        while index < N {
400            mask[index] = i as u32;
401
402            i += 2;
403            index += 1;
404        }
405        ShuffleMask(mask)
406    }
407
408    const fn pack() -> Self {
409        Self::odd()
410    }
411
412    const fn unpack_low() -> Self {
413        let mut mask = [0; N];
414        let mut i = 0;
415        while i < N {
416            mask[i] = (N + i) as u32;
417            i += 1;
418        }
419        ShuffleMask(mask)
420    }
421
422    const fn unpack_high() -> Self {
423        let mut mask = [0; N];
424        let mut i = 0;
425        while i < N {
426            mask[i] = i as u32;
427            i += 1;
428        }
429        ShuffleMask(mask)
430    }
431}
432
433const fn genmask<const MASK: u16>() -> [u8; 16] {
434    let mut bits = MASK;
435    let mut elements = [0u8; 16];
436
437    let mut i = 0;
438    while i < 16 {
439        elements[i] = match bits & (1u16 << 15) {
440            0 => 0,
441            _ => 0xFF,
442        };
443
444        bits <<= 1;
445        i += 1;
446    }
447
448    elements
449}
450
451const fn genmasks(bit_width: u32, a: u8, b: u8) -> u64 {
452    let bit_width = bit_width as u8;
453    let a = a % bit_width;
454    let mut b = b % bit_width;
455    if a > b {
456        b = bit_width - 1;
457    }
458
459    // of course these indices start from the left
460    let a = (bit_width - 1) - a;
461    let b = (bit_width - 1) - b;
462
463    ((1u64.wrapping_shl(a as u32 + 1)) - 1) & !((1u64.wrapping_shl(b as u32)) - 1)
464}
465
466const fn validate_block_boundary(block_boundary: u16) -> u32 {
467    assert!(
468        block_boundary.is_power_of_two() && block_boundary >= 64 && block_boundary <= 4096,
469        "block boundary must be a constant power of 2 from 64 to 4096",
470    );
471
472    // so that 64 is encoded as 0, 128 as 1, ect.
473    block_boundary as u32 >> 7
474}
475
476enum FindImm {
477    Eq = 4,
478    Ne = 12,
479    EqIdx = 0,
480    NeIdx = 8,
481}
482
483#[macro_use]
484mod sealed {
485    use super::*;
486
487    #[unstable(feature = "stdarch_s390x", issue = "135681")]
488    pub trait VectorNeg {
489        unsafe fn vec_neg(self) -> Self;
490    }
491
492    macro_rules! impl_neg {
493        ($($v:ty)*) => {
494            $(
495                #[unstable(feature = "stdarch_s390x", issue = "135681")]
496                impl VectorNeg for $v {
497                    #[inline]
498                    #[target_feature(enable = "vector")]
499                    unsafe fn vec_neg(self) -> Self {
500                        simd_neg(self)
501                    }
502                }
503            )*
504        }
505    }
506
507    impl_neg! {
508        vector_signed_char
509        vector_unsigned_char
510
511        vector_signed_short
512        vector_unsigned_short
513
514        vector_signed_int
515        vector_unsigned_int
516
517        vector_signed_long_long
518        vector_unsigned_long_long
519
520        vector_float
521        vector_double
522    }
523
524    #[unstable(feature = "stdarch_s390x", issue = "135681")]
525    pub trait VectorAdd<Other> {
526        type Result;
527        unsafe fn vec_add(self, other: Other) -> Self::Result;
528    }
529
530    macro_rules! impl_add {
531        ($name:ident, $a:ty, $instr:ident) => {
532            impl_add!($name, $a, $a, $a, $instr);
533        };
534        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
535            #[inline]
536            #[target_feature(enable = "vector")]
537            #[cfg_attr(test, assert_instr($instr))]
538            pub unsafe fn $name(a: $a, b: $b) -> $c {
539                transmute(simd_add(transmute(a), b))
540            }
541
542            #[unstable(feature = "stdarch_s390x", issue = "135681")]
543            impl VectorAdd<$b> for $a {
544                type Result = $c;
545
546                #[inline]
547                #[target_feature(enable = "vector")]
548                unsafe fn vec_add(self, other: $b) -> Self::Result {
549                    $name(self, other)
550                }
551            }
552        };
553    }
554
555    #[rustfmt::skip]
556    mod impl_add {
557        use super::*;
558
559        impl_add!(va_sc, vector_signed_char, vab);
560        impl_add!(va_uc, vector_unsigned_char, vab);
561        impl_add!(va_sh, vector_signed_short, vah);
562        impl_add!(va_uh, vector_unsigned_short, vah);
563        impl_add!(va_sf, vector_signed_int, vaf);
564        impl_add!(va_uf, vector_unsigned_int, vaf);
565        impl_add!(va_sg, vector_signed_long_long, vag);
566        impl_add!(va_ug, vector_unsigned_long_long, vag);
567
568        impl_add!(va_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vab);
569        impl_add!(va_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vab);
570        impl_add!(va_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vah);
571        impl_add!(va_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vah);
572        impl_add!(va_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vaf);
573        impl_add!(va_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vaf);
574        impl_add!(va_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vag);
575        impl_add!(va_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vag);
576
577        impl_add!(va_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vab);
578        impl_add!(va_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vab);
579        impl_add!(va_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vah);
580        impl_add!(va_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vah);
581        impl_add!(va_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vaf);
582        impl_add!(va_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vaf);
583        impl_add!(va_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vag);
584        impl_add!(va_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vag);
585
586        impl_add!(va_double, vector_double, vfadb);
587
588        #[inline]
589        #[target_feature(enable = "vector")]
590        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfasb))]
591        pub unsafe fn va_float(a: vector_float, b: vector_float) -> vector_float {
592            transmute(simd_add(a, b))
593        }
594
595        #[unstable(feature = "stdarch_s390x", issue = "135681")]
596        impl VectorAdd<Self> for vector_float {
597            type Result = Self;
598
599            #[inline]
600            #[target_feature(enable = "vector")]
601            unsafe fn vec_add(self, other: Self) -> Self::Result {
602                va_float(self, other)
603            }
604        }
605    }
606
607    #[unstable(feature = "stdarch_s390x", issue = "135681")]
608    pub trait VectorSub<Other> {
609        type Result;
610        unsafe fn vec_sub(self, other: Other) -> Self::Result;
611    }
612
613    macro_rules! impl_sub {
614        ($name:ident, $a:ty, $instr:ident) => {
615            impl_sub!($name, $a, $a, $a, $instr);
616        };
617        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
618            #[inline]
619            #[target_feature(enable = "vector")]
620            #[cfg_attr(test, assert_instr($instr))]
621            pub unsafe fn $name(a: $a, b: $b) -> $c {
622                transmute(simd_sub(transmute(a), b))
623            }
624
625            #[unstable(feature = "stdarch_s390x", issue = "135681")]
626            impl VectorSub<$b> for $a {
627                type Result = $c;
628
629                #[inline]
630                #[target_feature(enable = "vector")]
631                unsafe fn vec_sub(self, other: $b) -> Self::Result {
632                    $name(self, other)
633                }
634            }
635        };
636    }
637
638    #[rustfmt::skip]
639    mod impl_sub {
640        use super::*;
641
642        impl_sub!(vs_sc, vector_signed_char, vsb);
643        impl_sub!(vs_uc, vector_unsigned_char, vsb);
644        impl_sub!(vs_sh, vector_signed_short, vsh);
645        impl_sub!(vs_uh, vector_unsigned_short, vsh);
646        impl_sub!(vs_sf, vector_signed_int, vsf);
647        impl_sub!(vs_uf, vector_unsigned_int, vsf);
648        impl_sub!(vs_sg, vector_signed_long_long, vsg);
649        impl_sub!(vs_ug, vector_unsigned_long_long, vsg);
650
651        impl_sub!(vs_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vsb);
652        impl_sub!(vs_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vsb);
653        impl_sub!(vs_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vsh);
654        impl_sub!(vs_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vsh);
655        impl_sub!(vs_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vsf);
656        impl_sub!(vs_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vsf);
657        impl_sub!(vs_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vsg);
658        impl_sub!(vs_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vsg);
659
660        impl_sub!(vs_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vsb);
661        impl_sub!(vs_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vsb);
662        impl_sub!(vs_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vsh);
663        impl_sub!(vs_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vsh);
664        impl_sub!(vs_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vsf);
665        impl_sub!(vs_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vsf);
666        impl_sub!(vs_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vsg);
667        impl_sub!(vs_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vsg);
668
669        impl_sub!(vs_double, vector_double, vfsdb);
670
671        #[inline]
672        #[target_feature(enable = "vector")]
673        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfssb))]
674        pub unsafe fn vs_float(a: vector_float, b: vector_float) -> vector_float {
675            transmute(simd_sub(a, b))
676        }
677
678        #[unstable(feature = "stdarch_s390x", issue = "135681")]
679        impl VectorSub<Self> for vector_float {
680            type Result = Self;
681
682            #[inline]
683            #[target_feature(enable = "vector")]
684            unsafe fn vec_sub(self, other: Self) -> Self::Result {
685                vs_float(self, other)
686            }
687        }
688    }
689
690    #[unstable(feature = "stdarch_s390x", issue = "135681")]
691    pub trait VectorMul {
692        unsafe fn vec_mul(self, b: Self) -> Self;
693    }
694
695    macro_rules! impl_mul {
696        ($name:ident, $a:ty, std_simd) => {
697            #[unstable(feature = "stdarch_s390x", issue = "135681")]
698            impl VectorMul for $a {
699                #[inline]
700                #[target_feature(enable = "vector")]
701                unsafe fn vec_mul(self, other: Self) -> Self {
702                    transmute(simd_mul(transmute(self), other))
703                }
704            }
705        };
706        ($name:ident, $a:ty, $instr:ident) => {
707            #[inline]
708            #[target_feature(enable = "vector")]
709            #[cfg_attr(test, assert_instr($instr))]
710            pub unsafe fn $name(a: $a, b: $a) -> $a {
711                transmute(simd_mul(transmute(a), b))
712            }
713
714            #[unstable(feature = "stdarch_s390x", issue = "135681")]
715            impl VectorMul for $a {
716                #[inline]
717                #[target_feature(enable = "vector")]
718                unsafe fn vec_mul(self, other: Self) -> Self {
719                    $name(self, other)
720                }
721            }
722        };
723    }
724
725    #[rustfmt::skip]
726    mod impl_mul {
727        use super::*;
728
729        impl_mul!(vml_sc, vector_signed_char, vmlb);
730        impl_mul!(vml_uc, vector_unsigned_char, vmlb);
731        impl_mul!(vml_sh, vector_signed_short, vmlhw);
732        impl_mul!(vml_uh, vector_unsigned_short, vmlhw);
733        impl_mul!(vml_sf, vector_signed_int, vmlf);
734        impl_mul!(vml_uf, vector_unsigned_int, vmlf);
735        impl_mul!(vml_sg, vector_signed_long_long, std_simd);
736        impl_mul!(vml_ug, vector_unsigned_long_long, std_simd);
737
738        impl_mul!(vml_float, vector_float, std_simd);
739        impl_mul!(vml_double, vector_double, vfmdb);
740    }
741
742    #[unstable(feature = "stdarch_s390x", issue = "135681")]
743    pub trait VectorMax<Other> {
744        type Result;
745        unsafe fn vec_max(self, b: Other) -> Self::Result;
746    }
747
748    macro_rules! impl_max {
749        ($name:ident, $a:ty, $instr:ident) => {
750            #[inline]
751            #[target_feature(enable = "vector")]
752            #[cfg_attr(test, assert_instr($instr))]
753            pub unsafe fn $name(a: $a, b: $a) -> $a {
754                simd_select(simd_ge::<_, $a>(a, b), a, b)
755            }
756
757            #[unstable(feature = "stdarch_s390x", issue = "135681")]
758            impl VectorMax<Self> for $a {
759                type Result = Self;
760
761                #[inline]
762                #[target_feature(enable = "vector")]
763                unsafe fn vec_max(self, other: Self) -> Self {
764                    $name(self, other)
765                }
766            }
767        };
768    }
769
770    mod impl_max {
771        use super::*;
772
773        impl_max!(vec_vmxsc, vector_signed_char, vmxb);
774        impl_max!(vec_vmxslc, vector_unsigned_char, vmxlb);
775        impl_max!(vec_vmxsh, vector_signed_short, vmxh);
776        impl_max!(vec_vmxslh, vector_unsigned_short, vmxlh);
777        impl_max!(vec_vmxsf, vector_signed_int, vmxf);
778        impl_max!(vec_vmxslf, vector_unsigned_int, vmxlf);
779        impl_max!(vec_vmxsg, vector_signed_long_long, vmxg);
780        impl_max!(vec_vmxslg, vector_unsigned_long_long, vmxlg);
781    }
782
783    test_impl! { vec_vfmaxsb (a: vector_float, b: vector_float) -> vector_float [simd_fmax, "vector-enhancements-1" vfmaxsb ] }
784    test_impl! { vec_vfmaxdb (a: vector_double, b: vector_double) -> vector_double [simd_fmax, "vector-enhancements-1" vfmaxdb] }
785
786    impl_vec_trait!([VectorMax vec_max] vec_vfmaxsb (vector_float, vector_float) -> vector_float);
787    impl_vec_trait!([VectorMax vec_max] vec_vfmaxdb (vector_double, vector_double) -> vector_double);
788
789    #[unstable(feature = "stdarch_s390x", issue = "135681")]
790    pub trait VectorMin<Other> {
791        type Result;
792        unsafe fn vec_min(self, b: Other) -> Self::Result;
793    }
794
795    macro_rules! impl_min {
796        ($name:ident, $a:ty, $instr:ident) => {
797            #[inline]
798            #[target_feature(enable = "vector")]
799            #[cfg_attr(test, assert_instr($instr))]
800            pub unsafe fn $name(a: $a, b: $a) -> $a {
801                simd_select(simd_le::<_, $a>(a, b), a, b)
802            }
803
804            #[unstable(feature = "stdarch_s390x", issue = "135681")]
805            impl VectorMin<Self> for $a {
806                type Result = Self;
807
808                #[inline]
809                #[target_feature(enable = "vector")]
810                unsafe fn vec_min(self, other: Self) -> Self {
811                    $name(self, other)
812                }
813            }
814        };
815    }
816
817    mod impl_min {
818        use super::*;
819
820        impl_min!(vec_vmnsc, vector_signed_char, vmnb);
821        impl_min!(vec_vmnslc, vector_unsigned_char, vmnlb);
822        impl_min!(vec_vmnsh, vector_signed_short, vmnh);
823        impl_min!(vec_vmnslh, vector_unsigned_short, vmnlh);
824        impl_min!(vec_vmnsf, vector_signed_int, vmnf);
825        impl_min!(vec_vmnslf, vector_unsigned_int, vmnlf);
826        impl_min!(vec_vmnsg, vector_signed_long_long, vmng);
827        impl_min!(vec_vmnslg, vector_unsigned_long_long, vmnlg);
828    }
829
830    test_impl! { vec_vfminsb (a: vector_float, b: vector_float) -> vector_float [simd_fmin, "vector-enhancements-1" vfminsb]  }
831    test_impl! { vec_vfmindb (a: vector_double, b: vector_double) -> vector_double [simd_fmin, "vector-enhancements-1" vfmindb]  }
832
833    impl_vec_trait!([VectorMin vec_min] vec_vfminsb (vector_float, vector_float) -> vector_float);
834    impl_vec_trait!([VectorMin vec_min] vec_vfmindb (vector_double, vector_double) -> vector_double);
835
836    #[unstable(feature = "stdarch_s390x", issue = "135681")]
837    pub trait VectorAbs {
838        unsafe fn vec_abs(self) -> Self;
839    }
840
841    macro_rules! impl_abs {
842        ($name:ident, $ty:ident) => {
843            #[inline]
844            #[target_feature(enable = "vector")]
845            unsafe fn $name(v: s_t_l!($ty)) -> s_t_l!($ty) {
846                v.vec_max(simd_neg(v))
847            }
848
849            impl_vec_trait! { [VectorAbs vec_abs] $name (s_t_l!($ty)) }
850        };
851    }
852
853    impl_abs! { vec_abs_i8, i8x16 }
854    impl_abs! { vec_abs_i16, i16x8 }
855    impl_abs! { vec_abs_i32, i32x4 }
856    impl_abs! { vec_abs_i64, i64x2 }
857
858    test_impl! { vec_abs_f32 (v: vector_float) -> vector_float [ simd_fabs, "vector-enhancements-1" vflpsb ] }
859    test_impl! { vec_abs_f64 (v: vector_double) -> vector_double [ simd_fabs, vflpdb ] }
860
861    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f32 (vector_float) }
862    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f64 (vector_double) }
863
864    #[unstable(feature = "stdarch_s390x", issue = "135681")]
865    pub trait VectorNabs {
866        unsafe fn vec_nabs(self) -> Self;
867    }
868
869    #[inline]
870    #[target_feature(enable = "vector")]
871    #[cfg_attr(
872        all(test, target_feature = "vector-enhancements-1"),
873        assert_instr(vflnsb)
874    )]
875    unsafe fn vec_nabs_f32(a: vector_float) -> vector_float {
876        simd_neg(simd_fabs(a))
877    }
878
879    #[inline]
880    #[target_feature(enable = "vector")]
881    #[cfg_attr(test, assert_instr(vflndb))]
882    unsafe fn vec_nabs_f64(a: vector_double) -> vector_double {
883        simd_neg(simd_fabs(a))
884    }
885
886    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f32 (vector_float) }
887    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f64 (vector_double) }
888
889    #[unstable(feature = "stdarch_s390x", issue = "135681")]
890    pub trait VectorNmsub {
891        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self;
892    }
893
894    #[inline]
895    #[target_feature(enable = "vector")]
896    #[cfg_attr(
897        all(test, target_feature = "vector-enhancements-2"),
898        assert_instr(vfnmssb)
899    )]
900    unsafe fn vec_nmsub_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
901        simd_neg(simd_fma(a, b, simd_neg(c)))
902    }
903
904    #[unstable(feature = "stdarch_s390x", issue = "135681")]
905    impl VectorNmsub for vector_float {
906        #[target_feature(enable = "vector")]
907        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
908            vec_nmsub_f32(self, b, c)
909        }
910    }
911
912    #[inline]
913    #[target_feature(enable = "vector")]
914    #[cfg_attr(
915        all(test, target_feature = "vector-enhancements-2"),
916        assert_instr(vfnmsdb)
917    )]
918    unsafe fn vec_nmsub_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
919        simd_neg(simd_fma(a, b, simd_neg(c)))
920    }
921
922    #[unstable(feature = "stdarch_s390x", issue = "135681")]
923    impl VectorNmsub for vector_double {
924        #[target_feature(enable = "vector")]
925        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
926            vec_nmsub_f64(self, b, c)
927        }
928    }
929
930    #[unstable(feature = "stdarch_s390x", issue = "135681")]
931    pub trait VectorNmadd {
932        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self;
933    }
934
935    #[inline]
936    #[target_feature(enable = "vector")]
937    #[cfg_attr(
938        all(test, target_feature = "vector-enhancements-2"),
939        assert_instr(vfnmasb)
940    )]
941    unsafe fn vec_nmadd_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
942        simd_neg(simd_fma(a, b, c))
943    }
944
945    #[unstable(feature = "stdarch_s390x", issue = "135681")]
946    impl VectorNmadd for vector_float {
947        #[target_feature(enable = "vector")]
948        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
949            vec_nmadd_f32(self, b, c)
950        }
951    }
952
953    #[inline]
954    #[target_feature(enable = "vector")]
955    #[cfg_attr(
956        all(test, target_feature = "vector-enhancements-2"),
957        assert_instr(vfnmadb)
958    )]
959    unsafe fn vec_nmadd_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
960        simd_neg(simd_fma(a, b, c))
961    }
962
963    #[unstable(feature = "stdarch_s390x", issue = "135681")]
964    impl VectorNmadd for vector_double {
965        #[target_feature(enable = "vector")]
966        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
967            vec_nmadd_f64(self, b, c)
968        }
969    }
970
971    #[unstable(feature = "stdarch_s390x", issue = "135681")]
972    pub trait VectorSplat {
973        unsafe fn vec_splat<const IMM: u32>(self) -> Self;
974    }
975
976    #[inline]
977    #[target_feature(enable = "vector")]
978    #[cfg_attr(test, assert_instr(vrepb, IMM2 = 1))]
979    unsafe fn vrepb<const IMM2: u32>(a: vector_signed_char) -> vector_signed_char {
980        static_assert_uimm_bits!(IMM2, 4);
981        simd_shuffle!(a, a, [IMM2; 16])
982    }
983
984    #[inline]
985    #[target_feature(enable = "vector")]
986    #[cfg_attr(test, assert_instr(vreph, IMM2 = 1))]
987    unsafe fn vreph<const IMM2: u32>(a: vector_signed_short) -> vector_signed_short {
988        static_assert_uimm_bits!(IMM2, 3);
989        simd_shuffle!(a, a, [IMM2; 8])
990    }
991
992    #[inline]
993    #[target_feature(enable = "vector")]
994    #[cfg_attr(test, assert_instr(vrepf, IMM2 = 1))]
995    unsafe fn vrepf<const IMM2: u32>(a: vector_signed_int) -> vector_signed_int {
996        static_assert_uimm_bits!(IMM2, 2);
997        simd_shuffle!(a, a, [IMM2; 4])
998    }
999
1000    #[inline]
1001    #[target_feature(enable = "vector")]
1002    #[cfg_attr(test, assert_instr(vrepg, IMM2 = 1))]
1003    unsafe fn vrepg<const IMM2: u32>(a: vector_signed_long_long) -> vector_signed_long_long {
1004        static_assert_uimm_bits!(IMM2, 1);
1005        simd_shuffle!(a, a, [IMM2; 2])
1006    }
1007
1008    macro_rules! impl_vec_splat {
1009        ($ty:ty, $fun:ident) => {
1010            #[unstable(feature = "stdarch_s390x", issue = "135681")]
1011            impl VectorSplat for $ty {
1012                #[inline]
1013                #[target_feature(enable = "vector")]
1014                unsafe fn vec_splat<const IMM: u32>(self) -> Self {
1015                    transmute($fun::<IMM>(transmute(self)))
1016                }
1017            }
1018        };
1019    }
1020
1021    impl_vec_splat! { vector_signed_char, vrepb }
1022    impl_vec_splat! { vector_unsigned_char, vrepb }
1023    impl_vec_splat! { vector_bool_char, vrepb }
1024    impl_vec_splat! { vector_signed_short, vreph }
1025    impl_vec_splat! { vector_unsigned_short, vreph }
1026    impl_vec_splat! { vector_bool_short, vreph }
1027    impl_vec_splat! { vector_signed_int, vrepf }
1028    impl_vec_splat! { vector_unsigned_int, vrepf }
1029    impl_vec_splat! { vector_bool_int, vrepf }
1030    impl_vec_splat! { vector_signed_long_long, vrepg }
1031    impl_vec_splat! { vector_unsigned_long_long, vrepg }
1032    impl_vec_splat! { vector_bool_long_long, vrepg }
1033
1034    impl_vec_splat! { vector_float, vrepf }
1035    impl_vec_splat! { vector_double, vrepg }
1036
1037    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1038    pub trait VectorSplats<Output> {
1039        unsafe fn vec_splats(self) -> Output;
1040    }
1041
1042    macro_rules! impl_vec_splats {
1043        ($(($fn:ident ($ty:ty, $shortty:tt) $instr:ident)),*) => {
1044            $(
1045                #[inline]
1046                #[target_feature(enable = "vector")]
1047                #[cfg_attr(test, assert_instr($instr))]
1048                pub unsafe fn $fn(v: $ty) -> s_t_l!($shortty) {
1049                    transmute($shortty::splat(v))
1050                }
1051
1052                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1053                impl VectorSplats<s_t_l!($shortty)> for $ty {
1054                    #[inline]
1055                    #[target_feature(enable = "vector")]
1056                    unsafe fn vec_splats(self) -> s_t_l!($shortty) {
1057                        $fn (self)
1058                    }
1059                }
1060            )*
1061        }
1062    }
1063
1064    impl_vec_splats! {
1065        (vec_splats_u8 (u8, u8x16) vrepb),
1066        (vec_splats_i8 (i8, i8x16) vrepb),
1067        (vec_splats_u16 (u16, u16x8) vreph),
1068        (vec_splats_i16 (i16, i16x8) vreph),
1069        (vec_splats_u32 (u32, u32x4) vrepf),
1070        (vec_splats_i32 (i32, i32x4) vrepf),
1071        (vec_splats_u64 (u64, u64x2) vlvgp),
1072        (vec_splats_i64 (i64, i64x2) vlvgp),
1073        (vec_splats_f32 (f32, f32x4) vrepf),
1074        (vec_splats_f64 (f64, f64x2) vrepg)
1075    }
1076
1077    macro_rules! impl_bool_vec_splats {
1078        ($(($ty:ty, $shortty:tt, $boolty:ty)),*) => {
1079            $(
1080                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1081                impl VectorSplats<$boolty> for $ty {
1082                    #[inline]
1083                    #[target_feature(enable = "vector")]
1084                    unsafe fn vec_splats(self) -> $boolty {
1085                        transmute($shortty::splat(self))
1086                    }
1087                }
1088            )*
1089        }
1090    }
1091
1092    impl_bool_vec_splats! {
1093        (u8, u8x16, vector_bool_char),
1094        (i8, i8x16, vector_bool_char),
1095        (u16, u16x8, vector_bool_short),
1096        (i16, i16x8, vector_bool_short),
1097        (u32, u32x4, vector_bool_int),
1098        (i32, i32x4, vector_bool_int),
1099        (u64, u64x2, vector_bool_long_long),
1100        (i64, i64x2, vector_bool_long_long)
1101    }
1102
1103    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1104    pub trait CountBits {
1105        type Result;
1106
1107        unsafe fn vec_cntlz(self) -> Self::Result;
1108        unsafe fn vec_cnttz(self) -> Self::Result;
1109        unsafe fn vec_popcnt(self) -> Self::Result;
1110    }
1111
1112    macro_rules! impl_count_bits {
1113        ($ty:tt) => {
1114            #[unstable(feature = "stdarch_s390x", issue = "135681")]
1115            impl CountBits for $ty {
1116                type Result = t_u!($ty);
1117
1118                #[inline]
1119                #[target_feature(enable = "vector")]
1120                unsafe fn vec_cntlz(self) -> Self::Result {
1121                    transmute(simd_ctlz(self))
1122                }
1123
1124                #[inline]
1125                #[target_feature(enable = "vector")]
1126                unsafe fn vec_cnttz(self) -> Self::Result {
1127                    transmute(simd_cttz(self))
1128                }
1129
1130                #[inline]
1131                #[target_feature(enable = "vector")]
1132                unsafe fn vec_popcnt(self) -> Self::Result {
1133                    transmute(simd_ctpop(self))
1134                }
1135            }
1136        };
1137    }
1138
1139    impl_count_bits!(vector_signed_char);
1140    impl_count_bits!(vector_unsigned_char);
1141    impl_count_bits!(vector_signed_short);
1142    impl_count_bits!(vector_unsigned_short);
1143    impl_count_bits!(vector_signed_int);
1144    impl_count_bits!(vector_unsigned_int);
1145    impl_count_bits!(vector_signed_long_long);
1146    impl_count_bits!(vector_unsigned_long_long);
1147
1148    test_impl! { vec_clzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1149    test_impl! { vec_clzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1150    test_impl! { vec_clzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1151    test_impl! { vec_clzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1152
1153    test_impl! { vec_clzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1154    test_impl! { vec_clzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1155    test_impl! { vec_clzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1156    test_impl! { vec_clzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1157
1158    test_impl! { vec_ctzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1159    test_impl! { vec_ctzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1160    test_impl! { vec_ctzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1161    test_impl! { vec_ctzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1162
1163    test_impl! { vec_ctzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1164    test_impl! { vec_ctzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1165    test_impl! { vec_ctzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1166    test_impl! { vec_ctzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1167
1168    test_impl! { vec_vpopctb_signed +(a: vector_signed_char) -> vector_signed_char [simd_ctpop, vpopctb] }
1169    test_impl! { vec_vpopcth_signed +(a: vector_signed_short) -> vector_signed_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1170    test_impl! { vec_vpopctf_signed +(a: vector_signed_int) -> vector_signed_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1171    test_impl! { vec_vpopctg_signed +(a: vector_signed_long_long) -> vector_signed_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1172
1173    test_impl! { vec_vpopctb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctpop, vpopctb] }
1174    test_impl! { vec_vpopcth_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1175    test_impl! { vec_vpopctf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1176    test_impl! { vec_vpopctg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1177
1178    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1179    pub trait VectorAnd<Other> {
1180        type Result;
1181        unsafe fn vec_and(self, b: Other) -> Self::Result;
1182    }
1183
1184    impl_vec_trait! { [VectorAnd vec_and] ~(simd_and) }
1185
1186    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1187    pub trait VectorOr<Other> {
1188        type Result;
1189        unsafe fn vec_or(self, b: Other) -> Self::Result;
1190    }
1191
1192    impl_vec_trait! { [VectorOr vec_or] ~(simd_or) }
1193
1194    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1195    pub trait VectorXor<Other> {
1196        type Result;
1197        unsafe fn vec_xor(self, b: Other) -> Self::Result;
1198    }
1199
1200    impl_vec_trait! { [VectorXor vec_xor] ~(simd_xor) }
1201
1202    #[inline]
1203    #[target_feature(enable = "vector")]
1204    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vno))]
1205    unsafe fn nor(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1206        let a: u8x16 = transmute(a);
1207        let b: u8x16 = transmute(b);
1208        transmute(simd_xor(simd_or(a, b), u8x16::splat(0xff)))
1209    }
1210
1211    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1212    pub trait VectorNor<Other> {
1213        type Result;
1214        unsafe fn vec_nor(self, b: Other) -> Self::Result;
1215    }
1216
1217    impl_vec_trait! { [VectorNor vec_nor]+ 2c (nor) }
1218
1219    #[inline]
1220    #[target_feature(enable = "vector")]
1221    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnn))]
1222    unsafe fn nand(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1223        let a: u8x16 = transmute(a);
1224        let b: u8x16 = transmute(b);
1225        transmute(simd_xor(simd_and(a, b), u8x16::splat(0xff)))
1226    }
1227
1228    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1229    pub trait VectorNand<Other> {
1230        type Result;
1231        unsafe fn vec_nand(self, b: Other) -> Self::Result;
1232    }
1233
1234    impl_vec_trait! { [VectorNand vec_nand]+ 2c (nand) }
1235
1236    #[inline]
1237    #[target_feature(enable = "vector")]
1238    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnx))]
1239    unsafe fn eqv(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1240        let a: u8x16 = transmute(a);
1241        let b: u8x16 = transmute(b);
1242        transmute(simd_xor(simd_xor(a, b), u8x16::splat(0xff)))
1243    }
1244
1245    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1246    pub trait VectorEqv<Other> {
1247        type Result;
1248        unsafe fn vec_eqv(self, b: Other) -> Self::Result;
1249    }
1250
1251    impl_vec_trait! { [VectorEqv vec_eqv]+ 2c (eqv) }
1252
1253    #[inline]
1254    #[target_feature(enable = "vector")]
1255    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnc))]
1256    unsafe fn andc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1257        let a = transmute(a);
1258        let b = transmute(b);
1259        transmute(simd_and(simd_xor(u8x16::splat(0xff), b), a))
1260    }
1261
1262    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1263    pub trait VectorAndc<Other> {
1264        type Result;
1265        unsafe fn vec_andc(self, b: Other) -> Self::Result;
1266    }
1267
1268    impl_vec_trait! { [VectorAndc vec_andc]+ 2c (andc) }
1269
1270    #[inline]
1271    #[target_feature(enable = "vector")]
1272    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(voc))]
1273    unsafe fn orc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1274        let a = transmute(a);
1275        let b = transmute(b);
1276        transmute(simd_or(simd_xor(u8x16::splat(0xff), b), a))
1277    }
1278
1279    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1280    pub trait VectorOrc<Other> {
1281        type Result;
1282        unsafe fn vec_orc(self, b: Other) -> Self::Result;
1283    }
1284
1285    impl_vec_trait! { [VectorOrc vec_orc]+ 2c (orc) }
1286
1287    // Z vector intrinsic      C23 math.h  LLVM IR         ISO/IEC 60559 operation        inexact  vfidb parameters
1288    //
1289    // vec_rint                rint        llvm.rint       roundToIntegralExact           yes      0, 0
1290    // vec_roundc              nearbyint   llvm.nearbyint  n/a                            no       4, 0
1291    // vec_floor / vec_roundm  floor       llvm.floor      roundToIntegralTowardNegative  no       4, 7
1292    // vec_ceil / vec_roundp   ceil        llvm.ceil       roundToIntegralTowardPositive  no       4, 6
1293    // vec_trunc / vec_roundz  trunc       llvm.trunc      roundToIntegralTowardZero      no       4, 5
1294    // vec_round               roundeven   llvm.roundeven  roundToIntegralTiesToEven      no       4, 4
1295    // n/a                     round       llvm.round      roundToIntegralTiesAway        no       4, 1
1296
1297    // `simd_round_ties_even` is implemented as `llvm.rint`.
1298    test_impl! { vec_rint_f32 (a: vector_float) -> vector_float [simd_round_ties_even, "vector-enhancements-1" vfisb] }
1299    test_impl! { vec_rint_f64 (a: vector_double) -> vector_double [simd_round_ties_even, vfidb] }
1300
1301    test_impl! { vec_roundc_f32 (a: vector_float) -> vector_float [nearbyint_v4f32,  "vector-enhancements-1" vfisb] }
1302    test_impl! { vec_roundc_f64 (a: vector_double) -> vector_double [nearbyint_v2f64, vfidb] }
1303
1304    test_impl! { vec_round_f32 (a: vector_float) -> vector_float [roundeven_v4f32, "vector-enhancements-1" vfisb] }
1305    test_impl! { vec_round_f64 (a: vector_double) -> vector_double [roundeven_v2f64, vfidb] }
1306
1307    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1308    pub trait VectorRoundc {
1309        unsafe fn vec_roundc(self) -> Self;
1310    }
1311
1312    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1313    pub trait VectorRound {
1314        unsafe fn vec_round(self) -> Self;
1315    }
1316
1317    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1318    pub trait VectorRint {
1319        unsafe fn vec_rint(self) -> Self;
1320    }
1321
1322    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f32 (vector_float) }
1323    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f64 (vector_double) }
1324
1325    impl_vec_trait! { [VectorRound vec_round] vec_round_f32 (vector_float) }
1326    impl_vec_trait! { [VectorRound vec_round] vec_round_f64 (vector_double) }
1327
1328    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_float) }
1329    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_double) }
1330
1331    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1332    pub trait VectorTrunc {
1333        // same as vec_roundz
1334        unsafe fn vec_trunc(self) -> Self;
1335    }
1336
1337    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1338    pub trait VectorCeil {
1339        // same as vec_roundp
1340        unsafe fn vec_ceil(self) -> Self;
1341    }
1342
1343    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1344    pub trait VectorFloor {
1345        // same as vec_roundm
1346        unsafe fn vec_floor(self) -> Self;
1347    }
1348
1349    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_float) }
1350    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_double) }
1351
1352    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_float) }
1353    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_double) }
1354
1355    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_float) }
1356    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_double) }
1357
1358    macro_rules! impl_vec_shift {
1359        ([$Trait:ident $m:ident] ($b:ident, $h:ident, $w:ident, $g:ident)) => {
1360            impl_vec_trait!{ [$Trait $m]+ $b (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1361            impl_vec_trait!{ [$Trait $m]+ $b (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1362            impl_vec_trait!{ [$Trait $m]+ $h (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1363            impl_vec_trait!{ [$Trait $m]+ $h (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1364            impl_vec_trait!{ [$Trait $m]+ $w (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1365            impl_vec_trait!{ [$Trait $m]+ $w (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1366            impl_vec_trait!{ [$Trait $m]+ $g (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1367            impl_vec_trait!{ [$Trait $m]+ $g (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1368        };
1369    }
1370
1371    macro_rules! impl_shift {
1372        ($fun:ident $intr:ident $ty:ident) => {
1373            #[inline]
1374            #[target_feature(enable = "vector")]
1375            #[cfg_attr(test, assert_instr($fun))]
1376            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1377                let a = transmute(a);
1378                // use the remainder of b by the width of a's elements to prevent UB
1379                let b = simd_rem(transmute(b), <t_t_s!($ty)>::splat($ty::BITS as $ty));
1380
1381                transmute($intr(a, b))
1382            }
1383        };
1384    }
1385
1386    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1387    pub trait VectorSl<Other> {
1388        type Result;
1389        unsafe fn vec_sl(self, b: Other) -> Self::Result;
1390    }
1391
1392    impl_shift! { veslvb simd_shl u8 }
1393    impl_shift! { veslvh simd_shl u16 }
1394    impl_shift! { veslvf simd_shl u32 }
1395    impl_shift! { veslvg simd_shl u64 }
1396
1397    impl_vec_shift! { [VectorSl vec_sl] (veslvb, veslvh, veslvf, veslvg) }
1398
1399    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1400    pub trait VectorSr<Other> {
1401        type Result;
1402        unsafe fn vec_sr(self, b: Other) -> Self::Result;
1403    }
1404
1405    impl_shift! { vesrlvb simd_shr u8 }
1406    impl_shift! { vesrlvh simd_shr u16 }
1407    impl_shift! { vesrlvf simd_shr u32 }
1408    impl_shift! { vesrlvg simd_shr u64 }
1409
1410    impl_vec_shift! { [VectorSr vec_sr] (vesrlvb, vesrlvh, vesrlvf, vesrlvg) }
1411
1412    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1413    pub trait VectorSra<Other> {
1414        type Result;
1415        unsafe fn vec_sra(self, b: Other) -> Self::Result;
1416    }
1417
1418    impl_shift! { vesravb simd_shr i8 }
1419    impl_shift! { vesravh simd_shr i16 }
1420    impl_shift! { vesravf simd_shr i32 }
1421    impl_shift! { vesravg simd_shr i64 }
1422
1423    impl_vec_shift! { [VectorSra vec_sra] (vesravb, vesravh, vesravf, vesravg) }
1424
1425    macro_rules! impl_vec_shift_byte {
1426        ([$trait:ident $m:ident] ($f:ident)) => {
1427            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_signed_char) -> vector_unsigned_char }
1428            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1429            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_signed_char) -> vector_signed_char }
1430            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1431            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_signed_short) -> vector_unsigned_short }
1432            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1433            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_signed_short) -> vector_signed_short }
1434            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1435            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_signed_int) -> vector_unsigned_int }
1436            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1437            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_signed_int) -> vector_signed_int }
1438            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1439            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_signed_long_long) -> vector_unsigned_long_long }
1440            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1441            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_signed_long_long) -> vector_signed_long_long }
1442            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1443            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_signed_int) -> vector_float }
1444            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_unsigned_int) -> vector_float }
1445            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_signed_long_long) -> vector_double }
1446            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_unsigned_long_long) -> vector_double }
1447        };
1448    }
1449
1450    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1451    pub trait VectorSlb<Other> {
1452        type Result;
1453        unsafe fn vec_slb(self, b: Other) -> Self::Result;
1454    }
1455
1456    impl_vec_shift_byte! { [VectorSlb vec_slb] (vslb) }
1457
1458    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1459    pub trait VectorSrab<Other> {
1460        type Result;
1461        unsafe fn vec_srab(self, b: Other) -> Self::Result;
1462    }
1463
1464    impl_vec_shift_byte! { [VectorSrab vec_srab] (vsrab) }
1465
1466    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1467    pub trait VectorSrb<Other> {
1468        type Result;
1469        unsafe fn vec_srb(self, b: Other) -> Self::Result;
1470    }
1471
1472    impl_vec_shift_byte! { [VectorSrb vec_srb] (vsrlb) }
1473
1474    macro_rules! impl_vec_shift_long {
1475        ([$trait:ident $m:ident] ($f:ident)) => {
1476            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1477            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1478            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_char) -> vector_unsigned_short }
1479            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_char) -> vector_signed_short }
1480            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_char) -> vector_unsigned_int }
1481            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_char) -> vector_signed_int }
1482            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_char) -> vector_unsigned_long_long }
1483            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_char) -> vector_signed_long_long }
1484        };
1485    }
1486
1487    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1488    pub trait VectorSrl<Other> {
1489        type Result;
1490        unsafe fn vec_srl(self, b: Other) -> Self::Result;
1491    }
1492
1493    impl_vec_shift_long! { [VectorSrl vec_srl] (vsrl) }
1494
1495    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1496    pub trait VectorSral<Other> {
1497        type Result;
1498        unsafe fn vec_sral(self, b: Other) -> Self::Result;
1499    }
1500
1501    impl_vec_shift_long! { [VectorSral vec_sral] (vsra) }
1502
1503    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1504    pub trait VectorSll<Other> {
1505        type Result;
1506        unsafe fn vec_sll(self, b: Other) -> Self::Result;
1507    }
1508
1509    impl_vec_shift_long! { [VectorSll vec_sll] (vsl) }
1510
1511    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1512    pub trait VectorRl<Other> {
1513        type Result;
1514        unsafe fn vec_rl(self, b: Other) -> Self::Result;
1515    }
1516
1517    macro_rules! impl_rot {
1518        ($fun:ident $ty:ident) => {
1519            #[inline]
1520            #[target_feature(enable = "vector")]
1521            #[cfg_attr(test, assert_instr($fun))]
1522            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1523                simd_funnel_shl(a, a, b)
1524            }
1525        };
1526    }
1527
1528    impl_rot! { verllvb u8 }
1529    impl_rot! { verllvh u16 }
1530    impl_rot! { verllvf u32 }
1531    impl_rot! { verllvg u64 }
1532
1533    impl_vec_shift! { [VectorRl vec_rl] (verllvb, verllvh, verllvf, verllvg) }
1534
1535    macro_rules! test_rot_imm {
1536        ($fun:ident $instr:ident $ty:ident) => {
1537            #[inline]
1538            #[target_feature(enable = "vector")]
1539            #[cfg_attr(test, assert_instr($instr))]
1540            unsafe fn $fun(a: t_t_l!($ty), bits: core::ffi::c_ulong) -> t_t_l!($ty) {
1541                // mod by the number of bits in a's element type to prevent UB
1542                let bits = (bits % $ty::BITS as core::ffi::c_ulong) as $ty;
1543                let b = <t_t_s!($ty)>::splat(bits);
1544
1545                simd_funnel_shl(a, a, transmute(b))
1546            }
1547        };
1548    }
1549
1550    test_rot_imm! { verllvb_imm verllb u8 }
1551    test_rot_imm! { verllvh_imm verllh u16 }
1552    test_rot_imm! { verllvf_imm verllf u32 }
1553    test_rot_imm! { verllvg_imm verllg u64 }
1554
1555    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1556    pub trait VectorRli {
1557        unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self;
1558    }
1559
1560    macro_rules! impl_rot_imm {
1561        ($($ty:ident, $intr:ident),*) => {
1562            $(
1563                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1564                impl VectorRli for $ty {
1565                    #[inline]
1566                    #[target_feature(enable = "vector")]
1567                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1568                        transmute($intr(transmute(self), bits))
1569                    }
1570                }
1571
1572                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1573                impl VectorRli for t_u!($ty) {
1574                    #[inline]
1575                    #[target_feature(enable = "vector")]
1576                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1577                        $intr(self, bits)
1578                    }
1579                }
1580            )*
1581        }
1582    }
1583
1584    impl_rot_imm! {
1585        vector_signed_char, verllvb_imm,
1586        vector_signed_short, verllvh_imm,
1587        vector_signed_int, verllvf_imm,
1588        vector_signed_long_long, verllvg_imm
1589    }
1590
1591    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1592    pub trait VectorRlMask<Other> {
1593        unsafe fn vec_rl_mask<const IMM8: u8>(self, other: Other) -> Self;
1594    }
1595
1596    macro_rules! impl_rl_mask {
1597        ($($ty:ident, $intr:ident, $fun:ident),*) => {
1598            $(
1599                #[inline]
1600                #[target_feature(enable = "vector")]
1601                #[cfg_attr(test, assert_instr($intr, IMM8 = 6))]
1602                unsafe fn $fun<const IMM8: u8>(a: $ty, b: t_u!($ty)) -> $ty {
1603                    // mod by the number of bits in a's element type to prevent UB
1604                    $intr(a, a, transmute(b), const { (IMM8 % <l_t_t!($ty)>::BITS as u8) as i32 })
1605                }
1606
1607                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1608                impl VectorRlMask<t_u!($ty)> for $ty {
1609                    #[inline]
1610                    #[target_feature(enable = "vector")]
1611                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1612                        $fun::<IMM8>(self, other)
1613                    }
1614                }
1615
1616                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1617                impl VectorRlMask<t_u!($ty)> for t_u!($ty) {
1618                    #[inline]
1619                    #[target_feature(enable = "vector")]
1620                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1621                        transmute($fun::<IMM8>(transmute(self), transmute(other)))
1622                    }
1623                }
1624            )*
1625        }
1626    }
1627
1628    impl_rl_mask! {
1629        vector_signed_char, verimb, test_verimb,
1630        vector_signed_short, verimh, test_verimh,
1631        vector_signed_int, verimf, test_verimf,
1632        vector_signed_long_long, verimg, test_verimg
1633    }
1634
1635    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1636    pub trait VectorReve {
1637        unsafe fn vec_reve(self) -> Self;
1638    }
1639
1640    #[repr(simd)]
1641    struct ReverseMask<const N: usize>([u32; N]);
1642
1643    impl<const N: usize> ReverseMask<N> {
1644        const fn new() -> Self {
1645            let mut index = [0; N];
1646            let mut i = 0;
1647            while i < N {
1648                index[i] = (N - i - 1) as u32;
1649                i += 1;
1650            }
1651            ReverseMask(index)
1652        }
1653    }
1654
1655    macro_rules! impl_reve {
1656        ($($ty:ident, $fun:ident, $instr:ident),*) => {
1657            $(
1658                #[inline]
1659                #[target_feature(enable = "vector")]
1660                #[cfg_attr(test, assert_instr($instr))]
1661                unsafe fn $fun(a: $ty) -> $ty {
1662                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1663                    simd_shuffle(a, a, const { ShuffleMask::<N>::reverse() })
1664                }
1665
1666                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1667                impl VectorReve for $ty {
1668                    #[inline]
1669                    #[target_feature(enable = "vector")]
1670                    unsafe fn vec_reve(self) -> Self {
1671                        $fun(self)
1672                    }
1673                }
1674
1675                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1676                impl VectorReve for t_u!($ty) {
1677                    #[inline]
1678                    #[target_feature(enable = "vector")]
1679                    unsafe fn vec_reve(self) -> Self {
1680                        transmute($fun(transmute(self)))
1681                    }
1682                }
1683
1684                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1685                impl VectorReve for t_b!($ty) {
1686                    #[inline]
1687                    #[target_feature(enable = "vector")]
1688                    unsafe fn vec_reve(self) -> Self {
1689                        transmute($fun(transmute(self)))
1690                    }
1691                }
1692            )*
1693        }
1694    }
1695
1696    impl_reve! {
1697        vector_signed_char, reveb, vperm,
1698        vector_signed_short, reveh, vperm,
1699        vector_signed_int, revef, vperm,
1700        vector_signed_long_long, reveg, vpdi
1701    }
1702
1703    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1704    impl VectorReve for vector_float {
1705        #[inline]
1706        #[target_feature(enable = "vector")]
1707        unsafe fn vec_reve(self) -> Self {
1708            transmute(transmute::<_, vector_signed_int>(self).vec_reve())
1709        }
1710    }
1711
1712    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1713    impl VectorReve for vector_double {
1714        #[inline]
1715        #[target_feature(enable = "vector")]
1716        unsafe fn vec_reve(self) -> Self {
1717            transmute(transmute::<_, vector_signed_long_long>(self).vec_reve())
1718        }
1719    }
1720
1721    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1722    pub trait VectorRevb {
1723        unsafe fn vec_revb(self) -> Self;
1724    }
1725
1726    test_impl! { bswapb (a: vector_signed_char) -> vector_signed_char [simd_bswap, _] }
1727    test_impl! { bswaph (a: vector_signed_short) -> vector_signed_short [simd_bswap, vperm] }
1728    test_impl! { bswapf (a: vector_signed_int) -> vector_signed_int [simd_bswap, vperm] }
1729    test_impl! { bswapg (a: vector_signed_long_long) -> vector_signed_long_long [simd_bswap, vperm] }
1730
1731    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_unsigned_char) }
1732    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_signed_char) }
1733    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_unsigned_short) }
1734    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_signed_short) }
1735    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_unsigned_int) }
1736    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_signed_int) }
1737    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_unsigned_long_long) }
1738    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_signed_long_long) }
1739
1740    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1741    impl VectorRevb for vector_float {
1742        #[inline]
1743        #[target_feature(enable = "vector")]
1744        unsafe fn vec_revb(self) -> Self {
1745            transmute(transmute::<_, vector_signed_int>(self).vec_revb())
1746        }
1747    }
1748
1749    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1750    impl VectorRevb for vector_double {
1751        #[inline]
1752        #[target_feature(enable = "vector")]
1753        unsafe fn vec_revb(self) -> Self {
1754            transmute(transmute::<_, vector_signed_long_long>(self).vec_revb())
1755        }
1756    }
1757
1758    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1759    pub trait VectorMergel {
1760        unsafe fn vec_mergel(self, other: Self) -> Self;
1761    }
1762
1763    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1764    pub trait VectorMergeh {
1765        unsafe fn vec_mergeh(self, other: Self) -> Self;
1766    }
1767
1768    macro_rules! impl_merge {
1769        ($($ty:ident, $mergel:ident, $mergeh:ident),*) => {
1770            $(
1771                #[inline]
1772                #[target_feature(enable = "vector")]
1773                #[cfg_attr(test, assert_instr($mergel))]
1774                unsafe fn $mergel(a: $ty, b: $ty) -> $ty {
1775                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1776                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_low() })
1777                }
1778
1779                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1780                impl VectorMergel for $ty {
1781                    #[inline]
1782                    #[target_feature(enable = "vector")]
1783                    unsafe fn vec_mergel(self, other: Self) -> Self {
1784                        $mergel(self, other)
1785                    }
1786                }
1787
1788                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1789                impl VectorMergel for t_u!($ty) {
1790                    #[inline]
1791                    #[target_feature(enable = "vector")]
1792                    unsafe fn vec_mergel(self, other: Self) -> Self {
1793                        transmute($mergel(transmute(self), transmute(other)))
1794                    }
1795                }
1796
1797                #[inline]
1798                #[target_feature(enable = "vector")]
1799                #[cfg_attr(test, assert_instr($mergeh))]
1800                unsafe fn $mergeh(a: $ty, b: $ty) -> $ty {
1801                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1802                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_high() })
1803                }
1804
1805                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1806                impl VectorMergeh for $ty {
1807                    #[inline]
1808                    #[target_feature(enable = "vector")]
1809                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1810                        $mergeh(self, other)
1811                    }
1812                }
1813
1814                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1815                impl VectorMergeh for t_u!($ty) {
1816                    #[inline]
1817                    #[target_feature(enable = "vector")]
1818                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1819                        transmute($mergeh(transmute(self), transmute(other)))
1820                    }
1821                }
1822            )*
1823        }
1824    }
1825
1826    impl_merge! {
1827        vector_signed_char, vmrlb, vmrhb,
1828        vector_signed_short, vmrlh, vmrhh,
1829        vector_signed_int, vmrlf, vmrhf,
1830        vector_signed_long_long, vmrlg, vmrhg
1831    }
1832
1833    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1834    pub trait VectorPerm {
1835        unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self;
1836    }
1837
1838    macro_rules! impl_merge {
1839        ($($ty:ident),*) => {
1840            $(
1841                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1842                impl VectorPerm for $ty {
1843                    #[inline]
1844                    #[target_feature(enable = "vector")]
1845                    unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self {
1846                        transmute(vperm(transmute(self), transmute(other), c))
1847                    }
1848                }
1849            )*
1850        }
1851    }
1852
1853    impl_merge! {
1854        vector_signed_char,
1855        vector_signed_short,
1856        vector_signed_int,
1857        vector_signed_long_long,
1858        vector_unsigned_char,
1859        vector_unsigned_short,
1860        vector_unsigned_int,
1861        vector_unsigned_long_long,
1862        vector_bool_char,
1863        vector_bool_short,
1864        vector_bool_int,
1865        vector_bool_long_long,
1866        vector_float,
1867        vector_double
1868    }
1869
1870    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1871    pub trait VectorSumU128 {
1872        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char;
1873    }
1874
1875    #[inline]
1876    #[target_feature(enable = "vector")]
1877    #[cfg_attr(test, assert_instr(vsumqf))]
1878    pub unsafe fn vec_vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128 {
1879        transmute(vsumqf(a, b))
1880    }
1881
1882    #[inline]
1883    #[target_feature(enable = "vector")]
1884    #[cfg_attr(test, assert_instr(vsumqg))]
1885    pub unsafe fn vec_vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128 {
1886        transmute(vsumqg(a, b))
1887    }
1888
1889    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1890    impl VectorSumU128 for vector_unsigned_int {
1891        #[inline]
1892        #[target_feature(enable = "vector")]
1893        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1894            transmute(vec_vsumqf(self, other))
1895        }
1896    }
1897
1898    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1899    impl VectorSumU128 for vector_unsigned_long_long {
1900        #[inline]
1901        #[target_feature(enable = "vector")]
1902        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1903            transmute(vec_vsumqg(self, other))
1904        }
1905    }
1906
1907    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1908    pub trait VectorSum2 {
1909        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long;
1910    }
1911
1912    test_impl! { vec_vsumgh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long [vsumgh, vsumgh] }
1913    test_impl! { vec_vsumgf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [vsumgf, vsumgf] }
1914
1915    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1916    impl VectorSum2 for vector_unsigned_short {
1917        #[inline]
1918        #[target_feature(enable = "vector")]
1919        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1920            vec_vsumgh(self, other)
1921        }
1922    }
1923
1924    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1925    impl VectorSum2 for vector_unsigned_int {
1926        #[inline]
1927        #[target_feature(enable = "vector")]
1928        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1929            vec_vsumgf(self, other)
1930        }
1931    }
1932
1933    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1934    pub trait VectorSum4 {
1935        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int;
1936    }
1937
1938    test_impl! { vec_vsumb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int [vsumb, vsumb] }
1939    test_impl! { vec_vsumh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int [vsumh, vsumh] }
1940
1941    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1942    impl VectorSum4 for vector_unsigned_char {
1943        #[inline]
1944        #[target_feature(enable = "vector")]
1945        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1946            vec_vsumb(self, other)
1947        }
1948    }
1949
1950    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1951    impl VectorSum4 for vector_unsigned_short {
1952        #[inline]
1953        #[target_feature(enable = "vector")]
1954        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1955            vec_vsumh(self, other)
1956        }
1957    }
1958
1959    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1960    pub trait VectorSubc<Other> {
1961        type Result;
1962        unsafe fn vec_subc(self, b: Other) -> Self::Result;
1963    }
1964
1965    test_impl! { vec_vscbib (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vscbib, vscbib] }
1966    test_impl! { vec_vscbih (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vscbih, vscbih] }
1967    test_impl! { vec_vscbif (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vscbif, vscbif] }
1968    test_impl! { vec_vscbig (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vscbig, vscbig] }
1969
1970    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbib (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1971    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbih (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1972    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbif (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1973    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbig (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1974
1975    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1976    pub trait VectorSqrt {
1977        unsafe fn vec_sqrt(self) -> Self;
1978    }
1979
1980    test_impl! { vec_sqrt_f32 (v: vector_float) -> vector_float [ simd_fsqrt, "vector-enhancements-1" vfsqsb ] }
1981    test_impl! { vec_sqrt_f64 (v: vector_double) -> vector_double [ simd_fsqrt, vfsqdb ] }
1982
1983    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f32 (vector_float) }
1984    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f64 (vector_double) }
1985
1986    macro_rules! vfae_wrapper {
1987        ($($name:ident $ty:ident)*) => {
1988            $(
1989                #[inline]
1990                #[target_feature(enable = "vector")]
1991                #[cfg_attr(test, assert_instr($name, IMM = 0))]
1992                unsafe fn $name<const IMM: i32>(
1993                    a: $ty,
1994                    b: $ty,
1995                ) -> $ty {
1996                    super::$name(a, b, IMM)
1997                }
1998            )*
1999        }
2000     }
2001
2002    vfae_wrapper! {
2003       vfaeb vector_signed_char
2004       vfaeh vector_signed_short
2005       vfaef vector_signed_int
2006
2007       vfaezb vector_signed_char
2008       vfaezh vector_signed_short
2009       vfaezf vector_signed_int
2010    }
2011
2012    macro_rules! impl_vfae {
2013        ([idx_cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
2014            impl_vfae! { [idx_cc $Trait $m] $imm
2015                $b vector_signed_char vector_signed_char
2016                $b vector_unsigned_char vector_unsigned_char
2017                $b vector_bool_char vector_unsigned_char
2018
2019                $h vector_signed_short vector_signed_short
2020                $h vector_unsigned_short vector_unsigned_short
2021                $h vector_bool_short vector_unsigned_short
2022
2023                $f vector_signed_int vector_signed_int
2024                $f vector_unsigned_int vector_unsigned_int
2025                $f vector_bool_int vector_unsigned_int
2026            }
2027        };
2028        ([idx_cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
2029            $(
2030                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2031                impl $Trait<Self> for $ty {
2032                    type Result = $r;
2033                    #[inline]
2034                    #[target_feature(enable = "vector")]
2035                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
2036                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
2037                        (transmute(x), y)
2038                    }
2039                }
2040            )*
2041        };
2042        ([cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
2043            impl_vfae! { [cc $Trait $m] $imm
2044                $b vector_signed_char
2045                $b vector_unsigned_char
2046                $b vector_bool_char
2047
2048                $h vector_signed_short
2049                $h vector_unsigned_short
2050                $h vector_bool_short
2051
2052                $f vector_signed_int
2053                $f vector_unsigned_int
2054                $f vector_bool_int
2055            }
2056        };
2057        ([cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
2058            $(
2059                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2060                impl $Trait<Self> for $ty {
2061                    type Result = t_b!($ty);
2062                    #[inline]
2063                    #[target_feature(enable = "vector")]
2064                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
2065                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
2066                        (transmute(x), y)
2067                    }
2068                }
2069            )*
2070        };
2071        ([idx $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
2072            impl_vfae! { [idx $Trait $m] $imm
2073                $b vector_signed_char vector_signed_char
2074                $b vector_unsigned_char vector_unsigned_char
2075                $b vector_bool_char vector_unsigned_char
2076
2077                $h vector_signed_short vector_signed_short
2078                $h vector_unsigned_short vector_unsigned_short
2079                $h vector_bool_short vector_unsigned_short
2080
2081                $f vector_signed_int vector_signed_int
2082                $f vector_unsigned_int vector_unsigned_int
2083                $f vector_bool_int vector_unsigned_int
2084            }
2085        };
2086        ([idx $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
2087            $(
2088                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2089                impl $Trait<Self> for $ty {
2090                    type Result = $r;
2091                    #[inline]
2092                    #[target_feature(enable = "vector")]
2093                    unsafe fn $m(self, b: Self) -> Self::Result {
2094                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
2095                    }
2096                }
2097            )*
2098        };
2099        ([$Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
2100            impl_vfae! { [$Trait $m] $imm
2101                $b vector_signed_char
2102                $b vector_unsigned_char
2103                $b vector_bool_char
2104
2105                $h vector_signed_short
2106                $h vector_unsigned_short
2107                $h vector_bool_short
2108
2109                $f vector_signed_int
2110                $f vector_unsigned_int
2111                $f vector_bool_int
2112            }
2113        };
2114        ([$Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
2115            $(
2116                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2117                impl $Trait<Self> for $ty {
2118                    type Result = t_b!($ty);
2119                    #[inline]
2120                    #[target_feature(enable = "vector")]
2121                    unsafe fn $m(self, b: Self) -> Self::Result {
2122                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
2123                    }
2124                }
2125            )*
2126        };
2127    }
2128
2129    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2130    pub trait VectorFindAnyEq<Other> {
2131        type Result;
2132        unsafe fn vec_find_any_eq(self, other: Other) -> Self::Result;
2133    }
2134
2135    impl_vfae! { [VectorFindAnyEq vec_find_any_eq] Eq vfaeb vfaeh vfaef }
2136
2137    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2138    pub trait VectorFindAnyNe<Other> {
2139        type Result;
2140        unsafe fn vec_find_any_ne(self, other: Other) -> Self::Result;
2141    }
2142
2143    impl_vfae! { [VectorFindAnyNe vec_find_any_ne] Ne vfaeb vfaeh vfaef }
2144
2145    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2146    pub trait VectorFindAnyEqOrZeroIdx<Other> {
2147        type Result;
2148        unsafe fn vec_find_any_eq_or_0_idx(self, other: Other) -> Self::Result;
2149    }
2150
2151    impl_vfae! { [idx VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx] EqIdx
2152        vfaezb vector_signed_char vector_signed_char
2153        vfaezb vector_unsigned_char vector_unsigned_char
2154        vfaezb vector_bool_char vector_unsigned_char
2155
2156        vfaezh vector_signed_short vector_signed_short
2157        vfaezh vector_unsigned_short vector_unsigned_short
2158        vfaezh vector_bool_short vector_unsigned_short
2159
2160        vfaezf vector_signed_int vector_signed_int
2161        vfaezf vector_unsigned_int vector_unsigned_int
2162        vfaezf vector_bool_int vector_unsigned_int
2163    }
2164
2165    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2166    pub trait VectorFindAnyNeOrZeroIdx<Other> {
2167        type Result;
2168        unsafe fn vec_find_any_ne_or_0_idx(self, other: Other) -> Self::Result;
2169    }
2170
2171    impl_vfae! { [idx VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx] NeIdx
2172        vfaezb vector_signed_char vector_signed_char
2173        vfaezb vector_unsigned_char vector_unsigned_char
2174        vfaezb vector_bool_char vector_unsigned_char
2175
2176        vfaezh vector_signed_short vector_signed_short
2177        vfaezh vector_unsigned_short vector_unsigned_short
2178        vfaezh vector_bool_short vector_unsigned_short
2179
2180        vfaezf vector_signed_int vector_signed_int
2181        vfaezf vector_unsigned_int vector_unsigned_int
2182        vfaezf vector_bool_int vector_unsigned_int
2183    }
2184
2185    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2186    pub trait VectorFindAnyEqIdx<Other> {
2187        type Result;
2188        unsafe fn vec_find_any_eq_idx(self, other: Other) -> Self::Result;
2189    }
2190
2191    impl_vfae! { [idx VectorFindAnyEqIdx vec_find_any_eq_idx] EqIdx vfaeb vfaeh vfaef }
2192
2193    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2194    pub trait VectorFindAnyNeIdx<Other> {
2195        type Result;
2196        unsafe fn vec_find_any_ne_idx(self, other: Other) -> Self::Result;
2197    }
2198
2199    impl_vfae! { [idx VectorFindAnyNeIdx vec_find_any_ne_idx] NeIdx vfaeb vfaeh vfaef }
2200
2201    macro_rules! vfaes_wrapper {
2202        ($($name:ident $ty:ident)*) => {
2203            $(
2204                #[inline]
2205                #[target_feature(enable = "vector")]
2206                #[cfg_attr(test, assert_instr($name, IMM = 0))]
2207                unsafe fn $name<const IMM: i32>(
2208                    a: $ty,
2209                    b: $ty,
2210                ) -> PackedTuple<$ty, i32> {
2211                    super::$name(a, b, IMM)
2212                }
2213            )*
2214        }
2215     }
2216
2217    vfaes_wrapper! {
2218        vfaebs vector_signed_char
2219        vfaehs vector_signed_short
2220        vfaefs vector_signed_int
2221
2222        vfaezbs vector_signed_char
2223        vfaezhs vector_signed_short
2224        vfaezfs vector_signed_int
2225    }
2226
2227    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2228    pub trait VectorFindAnyEqCC<Other> {
2229        type Result;
2230        unsafe fn vec_find_any_eq_cc(self, other: Other) -> (Self::Result, i32);
2231    }
2232
2233    impl_vfae! { [cc VectorFindAnyEqCC vec_find_any_eq_cc] Eq vfaebs vfaehs vfaefs }
2234
2235    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2236    pub trait VectorFindAnyNeCC<Other> {
2237        type Result;
2238        unsafe fn vec_find_any_ne_cc(self, other: Other) -> (Self::Result, i32);
2239    }
2240
2241    impl_vfae! { [cc VectorFindAnyNeCC vec_find_any_ne_cc] Ne vfaebs vfaehs vfaefs }
2242
2243    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2244    pub trait VectorFindAnyEqIdxCC<Other> {
2245        type Result;
2246        unsafe fn vec_find_any_eq_idx_cc(self, other: Other) -> (Self::Result, i32);
2247    }
2248
2249    impl_vfae! { [idx_cc VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc] EqIdx vfaebs vfaehs vfaefs }
2250
2251    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2252    pub trait VectorFindAnyNeIdxCC<Other> {
2253        type Result;
2254        unsafe fn vec_find_any_ne_idx_cc(self, other: Other) -> (Self::Result, i32);
2255    }
2256
2257    impl_vfae! { [idx_cc VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc] NeIdx vfaebs vfaehs vfaefs }
2258
2259    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2260    pub trait VectorFindAnyEqOrZeroIdxCC<Other> {
2261        type Result;
2262        unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2263    }
2264
2265    impl_vfae! { [idx_cc VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc] EqIdx vfaezbs vfaezhs vfaezfs }
2266
2267    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2268    pub trait VectorFindAnyNeOrZeroIdxCC<Other> {
2269        type Result;
2270        unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2271    }
2272
2273    impl_vfae! { [idx_cc VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc] NeIdx vfaezbs vfaezhs vfaezfs }
2274
2275    #[inline]
2276    #[target_feature(enable = "vector")]
2277    #[cfg_attr(test, assert_instr(vl))]
2278    unsafe fn test_vector_load(offset: isize, ptr: *const i32) -> vector_signed_int {
2279        ptr.byte_offset(offset)
2280            .cast::<vector_signed_int>()
2281            .read_unaligned()
2282    }
2283
2284    #[inline]
2285    #[target_feature(enable = "vector")]
2286    #[cfg_attr(test, assert_instr(vst))]
2287    unsafe fn test_vector_store(vector: vector_signed_int, offset: isize, ptr: *mut i32) {
2288        ptr.byte_offset(offset)
2289            .cast::<vector_signed_int>()
2290            .write_unaligned(vector)
2291    }
2292
2293    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2294    pub trait VectorLoad: Sized {
2295        type ElementType;
2296
2297        #[inline]
2298        #[target_feature(enable = "vector")]
2299        unsafe fn vec_xl(offset: isize, ptr: *const Self::ElementType) -> Self {
2300            ptr.byte_offset(offset).cast::<Self>().read_unaligned()
2301        }
2302
2303        unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self;
2304
2305        unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(
2306            ptr: *const Self::ElementType,
2307        ) -> MaybeUninit<Self>;
2308    }
2309
2310    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2311    pub trait VectorStore: Sized {
2312        type ElementType;
2313
2314        #[inline]
2315        #[target_feature(enable = "vector")]
2316        unsafe fn vec_xst(self, offset: isize, ptr: *mut Self::ElementType) {
2317            ptr.byte_offset(offset).cast::<Self>().write_unaligned(self)
2318        }
2319
2320        unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32);
2321    }
2322
2323    macro_rules! impl_load_store {
2324        ($($ty:ident)*) => {
2325            $(
2326                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2327                impl VectorLoad for t_t_l!($ty) {
2328                    type ElementType = $ty;
2329
2330                    #[inline]
2331                    #[target_feature(enable = "vector")]
2332                    unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self {
2333                        transmute(vll( byte_count, ptr.cast(),))
2334                    }
2335
2336                    #[inline]
2337                    #[target_feature(enable = "vector")]
2338                    unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(ptr: *const Self::ElementType) -> MaybeUninit<Self> {
2339                        transmute(vlbb(ptr.cast(), const { validate_block_boundary(BLOCK_BOUNDARY) }))
2340                    }
2341
2342                }
2343
2344                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2345                impl VectorStore for t_t_l!($ty) {
2346                    type ElementType = $ty;
2347
2348                    #[inline]
2349                    #[target_feature(enable = "vector")]
2350                    unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32) {
2351                        vstl(transmute(self), byte_count, ptr.cast())
2352                    }
2353                }
2354            )*
2355        }
2356    }
2357
2358    impl_load_store! { i8 u8 i16 u16 i32 u32 i64 u64 f32 f64 }
2359
2360    #[inline]
2361    #[target_feature(enable = "vector")]
2362    #[cfg_attr(test, assert_instr(vll))]
2363    unsafe fn test_vec_load_len(ptr: *const i32, byte_count: u32) -> vector_signed_int {
2364        vector_signed_int::vec_load_len(ptr, byte_count)
2365    }
2366
2367    #[inline]
2368    #[target_feature(enable = "vector")]
2369    #[cfg_attr(test, assert_instr(vlbb))]
2370    unsafe fn test_vec_load_bndry(ptr: *const i32) -> MaybeUninit<vector_signed_int> {
2371        vector_signed_int::vec_load_bndry::<512>(ptr)
2372    }
2373
2374    #[inline]
2375    #[target_feature(enable = "vector")]
2376    #[cfg_attr(test, assert_instr(vstl))]
2377    unsafe fn test_vec_store_len(vector: vector_signed_int, ptr: *mut i32, byte_count: u32) {
2378        vector.vec_store_len(ptr, byte_count)
2379    }
2380
2381    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2382    pub trait VectorLoadPair: Sized {
2383        type ElementType;
2384
2385        unsafe fn vec_load_pair(a: Self::ElementType, b: Self::ElementType) -> Self;
2386    }
2387
2388    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2389    impl VectorLoadPair for vector_signed_long_long {
2390        type ElementType = i64;
2391
2392        #[inline]
2393        #[target_feature(enable = "vector")]
2394        unsafe fn vec_load_pair(a: i64, b: i64) -> Self {
2395            vector_signed_long_long([a, b])
2396        }
2397    }
2398
2399    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2400    impl VectorLoadPair for vector_unsigned_long_long {
2401        type ElementType = u64;
2402
2403        #[inline]
2404        #[target_feature(enable = "vector")]
2405        unsafe fn vec_load_pair(a: u64, b: u64) -> Self {
2406            vector_unsigned_long_long([a, b])
2407        }
2408    }
2409
2410    #[inline]
2411    #[target_feature(enable = "vector")]
2412    unsafe fn pack<T, const N: usize>(a: T, b: T) -> T {
2413        simd_shuffle(a, b, const { ShuffleMask::<N>::pack() })
2414    }
2415
2416    #[inline]
2417    #[target_feature(enable = "vector")]
2418    #[cfg_attr(test, assert_instr(vpkh))]
2419    unsafe fn vpkh(a: i16x8, b: i16x8) -> i8x16 {
2420        let a: i8x16 = transmute(a);
2421        let b: i8x16 = transmute(b);
2422        simd_shuffle(a, b, const { ShuffleMask::<16>::pack() })
2423    }
2424    #[inline]
2425    #[target_feature(enable = "vector")]
2426    #[cfg_attr(test, assert_instr(vpkf))]
2427    unsafe fn vpkf(a: i32x4, b: i32x4) -> i16x8 {
2428        let a: i16x8 = transmute(a);
2429        let b: i16x8 = transmute(b);
2430        simd_shuffle(a, b, const { ShuffleMask::<8>::pack() })
2431    }
2432    #[inline]
2433    #[target_feature(enable = "vector")]
2434    #[cfg_attr(test, assert_instr(vpkg))]
2435    unsafe fn vpkg(a: i64x2, b: i64x2) -> i32x4 {
2436        let a: i32x4 = transmute(a);
2437        let b: i32x4 = transmute(b);
2438        simd_shuffle(a, b, const { ShuffleMask::<4>::pack() })
2439    }
2440
2441    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2442    pub trait VectorPack<Other> {
2443        type Result;
2444        unsafe fn vec_pack(self, b: Other) -> Self::Result;
2445    }
2446
2447    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2448    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2449    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_bool_short, vector_bool_short) -> vector_bool_char }
2450    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2451    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2452    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_bool_int, vector_bool_int) -> vector_bool_short }
2453    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2454    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2455    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_bool_long_long, vector_bool_long_long) -> vector_bool_int }
2456
2457    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2458    pub trait VectorPacks<Other> {
2459        type Result;
2460        unsafe fn vec_packs(self, b: Other) -> Self::Result;
2461    }
2462
2463    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/153655
2464    // Other targets can use a min/max for the saturation + a truncation.
2465
2466    impl_vec_trait! { [VectorPacks vec_packs] vpksh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2467    impl_vec_trait! { [VectorPacks vec_packs] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2468    impl_vec_trait! { [VectorPacks vec_packs] vpksf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2469    impl_vec_trait! { [VectorPacks vec_packs] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2470    impl_vec_trait! { [VectorPacks vec_packs] vpksg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2471    impl_vec_trait! { [VectorPacks vec_packs] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2472
2473    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2474    pub trait VectorPacksu<Other> {
2475        type Result;
2476        unsafe fn vec_packsu(self, b: Other) -> Self::Result;
2477    }
2478
2479    #[inline]
2480    #[target_feature(enable = "vector")]
2481    #[cfg_attr(test, assert_instr(vpklsh))]
2482    unsafe fn vpacksuh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char {
2483        vpklsh(
2484            vec_max(a, vector_signed_short([0; 8])),
2485            vec_max(b, vector_signed_short([0; 8])),
2486        )
2487    }
2488    #[inline]
2489    #[target_feature(enable = "vector")]
2490    #[cfg_attr(test, assert_instr(vpklsf))]
2491    unsafe fn vpacksuf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short {
2492        vpklsf(
2493            vec_max(a, vector_signed_int([0; 4])),
2494            vec_max(b, vector_signed_int([0; 4])),
2495        )
2496    }
2497    #[inline]
2498    #[target_feature(enable = "vector")]
2499    #[cfg_attr(test, assert_instr(vpklsg))]
2500    unsafe fn vpacksug(
2501        a: vector_signed_long_long,
2502        b: vector_signed_long_long,
2503    ) -> vector_unsigned_int {
2504        vpklsg(
2505            vec_max(a, vector_signed_long_long([0; 2])),
2506            vec_max(b, vector_signed_long_long([0; 2])),
2507        )
2508    }
2509
2510    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuh (vector_signed_short, vector_signed_short) -> vector_unsigned_char }
2511    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2512    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuf (vector_signed_int, vector_signed_int) -> vector_unsigned_short }
2513    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2514    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksug (vector_signed_long_long, vector_signed_long_long) -> vector_unsigned_int }
2515    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2516
2517    macro_rules! impl_vector_packs_cc {
2518        ($($intr:ident $ty:ident $outty:ident)*) => {
2519            $(
2520                #[inline]
2521                #[target_feature(enable = "vector")]
2522                #[cfg_attr(test, assert_instr($intr))]
2523                unsafe fn $intr(
2524                    a: $ty,
2525                    b: $ty,
2526                ) -> ($outty, i32) {
2527                    let PackedTuple { x, y } = super::$intr(a, b);
2528                    (x, y)
2529                }
2530
2531                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2532                impl VectorPacksCC for $ty {
2533                    type Result = $outty;
2534
2535                    #[inline]
2536                    #[target_feature(enable = "vector")]
2537                    unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32) {
2538                        $intr(self, b)
2539                    }
2540                }
2541            )*
2542        }
2543    }
2544
2545    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2546    pub trait VectorPacksCC {
2547        type Result;
2548        unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32);
2549    }
2550
2551    impl_vector_packs_cc! {
2552        vpkshs vector_signed_short vector_signed_char
2553        vpklshs vector_unsigned_short vector_unsigned_char
2554        vpksfs vector_signed_int vector_signed_short
2555        vpklsfs vector_unsigned_int vector_unsigned_short
2556        vpksgs vector_signed_long_long vector_signed_int
2557        vpklsgs vector_unsigned_long_long vector_unsigned_int
2558    }
2559
2560    macro_rules! impl_vector_packsu_cc {
2561        ($($intr:ident $ty:ident $outty:ident)*) => {
2562            $(
2563                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2564                impl VectorPacksuCC for $ty {
2565                    type Result = $outty;
2566
2567                    #[inline]
2568                    #[target_feature(enable = "vector")]
2569                    unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32) {
2570                        $intr(self, b)
2571                    }
2572                }
2573            )*
2574        }
2575    }
2576
2577    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2578    pub trait VectorPacksuCC {
2579        type Result;
2580        unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32);
2581    }
2582
2583    impl_vector_packsu_cc! {
2584        vpklshs vector_unsigned_short vector_unsigned_char
2585        vpklsfs vector_unsigned_int vector_unsigned_short
2586        vpklsgs vector_unsigned_long_long vector_unsigned_int
2587    }
2588
2589    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2590    pub trait VectorMadd {
2591        unsafe fn vec_madd(self, b: Self, c: Self) -> Self;
2592        unsafe fn vec_msub(self, b: Self, c: Self) -> Self;
2593    }
2594
2595    test_impl! { vfmasb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fma, "vector-enhancements-1" vfmasb] }
2596    test_impl! { vfmadb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fma, vfmadb] }
2597
2598    #[inline]
2599    unsafe fn simd_fms<T>(a: T, b: T, c: T) -> T {
2600        simd_fma(a, b, simd_neg(c))
2601    }
2602
2603    test_impl! { vfmssb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fms, "vector-enhancements-1" vfmssb] }
2604    test_impl! { vfmsdb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fms, vfmsdb] }
2605
2606    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2607    impl VectorMadd for vector_float {
2608        #[inline]
2609        #[target_feature(enable = "vector")]
2610        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2611            vfmasb(self, b, c)
2612        }
2613
2614        #[inline]
2615        #[target_feature(enable = "vector")]
2616        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2617            vfmssb(self, b, c)
2618        }
2619    }
2620
2621    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2622    impl VectorMadd for vector_double {
2623        #[inline]
2624        #[target_feature(enable = "vector")]
2625        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2626            vfmadb(self, b, c)
2627        }
2628
2629        #[inline]
2630        #[target_feature(enable = "vector")]
2631        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2632            vfmsdb(self, b, c)
2633        }
2634    }
2635
2636    macro_rules! impl_vec_unpack {
2637        ($mask:ident $instr:ident $src:ident $shuffled:ident $dst:ident $width:literal) => {
2638            #[inline]
2639            #[target_feature(enable = "vector")]
2640            #[cfg_attr(test, assert_instr($instr))]
2641            unsafe fn $instr(a: $src) -> $dst {
2642                simd_as(simd_shuffle::<_, _, $shuffled>(
2643                    a,
2644                    a,
2645                    const { ShuffleMask::<$width>::$mask() },
2646                ))
2647            }
2648        };
2649    }
2650
2651    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2652    pub trait VectorUnpackh {
2653        type Result;
2654        unsafe fn vec_unpackh(self) -> Self::Result;
2655    }
2656
2657    impl_vec_unpack!(unpack_high vuphb vector_signed_char i8x8 vector_signed_short 8);
2658    impl_vec_unpack!(unpack_high vuphh vector_signed_short i16x4 vector_signed_int 4);
2659    impl_vec_unpack!(unpack_high vuphf vector_signed_int i32x2 vector_signed_long_long 2);
2660
2661    impl_vec_unpack!(unpack_high vuplhb vector_unsigned_char u8x8 vector_unsigned_short 8);
2662    impl_vec_unpack!(unpack_high vuplhh vector_unsigned_short u16x4 vector_unsigned_int 4);
2663    impl_vec_unpack!(unpack_high vuplhf vector_unsigned_int u32x2 vector_unsigned_long_long 2);
2664
2665    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphb (vector_signed_char) -> vector_signed_short}
2666    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphh (vector_signed_short) -> vector_signed_int}
2667    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphf (vector_signed_int) -> vector_signed_long_long}
2668
2669    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhb (vector_unsigned_char) -> vector_unsigned_short}
2670    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhh (vector_unsigned_short) -> vector_unsigned_int}
2671    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhf (vector_unsigned_int) -> vector_unsigned_long_long}
2672
2673    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhb (vector_bool_char) -> vector_bool_short}
2674    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhh (vector_bool_short) -> vector_bool_int}
2675    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhf (vector_bool_int) -> vector_bool_long_long}
2676
2677    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2678    pub trait VectorUnpackl {
2679        type Result;
2680        unsafe fn vec_unpackl(self) -> Self::Result;
2681    }
2682
2683    // NOTE: `vuplh` is used for "unpack logical high", hence `vuplhw`.
2684    impl_vec_unpack!(unpack_low vuplb vector_signed_char i8x8 vector_signed_short 8);
2685    impl_vec_unpack!(unpack_low vuplhw vector_signed_short i16x4 vector_signed_int 4);
2686    impl_vec_unpack!(unpack_low vuplf vector_signed_int i32x2 vector_signed_long_long 2);
2687
2688    impl_vec_unpack!(unpack_low vupllb vector_unsigned_char u8x8 vector_unsigned_short 8);
2689    impl_vec_unpack!(unpack_low vupllh vector_unsigned_short u16x4 vector_unsigned_int 4);
2690    impl_vec_unpack!(unpack_low vupllf vector_unsigned_int u32x2 vector_unsigned_long_long 2);
2691
2692    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplb (vector_signed_char) -> vector_signed_short}
2693    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplhw (vector_signed_short) -> vector_signed_int}
2694    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplf (vector_signed_int) -> vector_signed_long_long}
2695
2696    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllb (vector_unsigned_char) -> vector_unsigned_short}
2697    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllh (vector_unsigned_short) -> vector_unsigned_int}
2698    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllf (vector_unsigned_int) -> vector_unsigned_long_long}
2699
2700    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllb (vector_bool_char) -> vector_bool_short}
2701    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllh (vector_bool_short) -> vector_bool_int}
2702    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllf (vector_bool_int) -> vector_bool_long_long}
2703
2704    test_impl! { vec_vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vavgb, vavgb ] }
2705    test_impl! { vec_vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vavgh, vavgh ] }
2706    test_impl! { vec_vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vavgf, vavgf ] }
2707    test_impl! { vec_vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [ vavgg, vavgg ] }
2708
2709    test_impl! { vec_vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vavglb, vavglb ] }
2710    test_impl! { vec_vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vavglh, vavglh ] }
2711    test_impl! { vec_vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vavglf, vavglf ] }
2712    test_impl! { vec_vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [ vavglg, vavglg ] }
2713
2714    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2715    pub trait VectorAvg<Other> {
2716        type Result;
2717        unsafe fn vec_avg(self, b: Other) -> Self::Result;
2718    }
2719
2720    impl_vec_trait! { [VectorAvg vec_avg] 2 (vec_vavglb, vec_vavgb, vec_vavglh, vec_vavgh, vec_vavglf, vec_vavgf, vec_vavglg, vec_vavgg) }
2721
2722    macro_rules! impl_mul {
2723        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty) -> $r:ty) => {
2724            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2725            impl $Trait<$r> for $a {
2726                #[inline]
2727                #[target_feature(enable = "vector")]
2728                unsafe fn $m(self, b: $b) -> $r {
2729                    $fun(transmute(self), transmute(b))
2730                }
2731            }
2732        };
2733        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty, $c:ty) -> $r:ty) => {
2734            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2735            impl $Trait for $a {
2736                type Result = $r;
2737                #[inline]
2738                #[target_feature(enable = "vector")]
2739                unsafe fn $m(self, b: $b, c: $c) -> $r {
2740                    $fun(self, b, c)
2741                }
2742            }
2743        };
2744    }
2745
2746    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2747    pub trait VectorMule<Result> {
2748        unsafe fn vec_mule(self, b: Self) -> Result;
2749    }
2750
2751    macro_rules! impl_vec_mul_even_odd {
2752        ($mask:ident $instr:ident $src:ident $shuffled:ident $dst:ident $width:literal) => {
2753            #[inline]
2754            #[target_feature(enable = "vector")]
2755            #[cfg_attr(test, assert_instr($instr))]
2756            unsafe fn $instr(a: $src, b: $src) -> $dst {
2757                let elems_a: $dst = simd_as(simd_shuffle::<_, _, $shuffled>(
2758                    a,
2759                    a, // this argument is ignored entirely.
2760                    const { ShuffleMask::<$width>::$mask() },
2761                ));
2762
2763                let elems_b: $dst = simd_as(simd_shuffle::<_, _, $shuffled>(
2764                    b,
2765                    b, // this argument is ignored entirely.
2766                    const { ShuffleMask::<$width>::$mask() },
2767                ));
2768
2769                simd_mul(elems_a, elems_b)
2770            }
2771        };
2772    }
2773
2774    impl_vec_mul_even_odd! { even vmeb vector_signed_char i8x8 vector_signed_short 8 }
2775    impl_vec_mul_even_odd! { even vmeh vector_signed_short i16x4 vector_signed_int 4 }
2776    impl_vec_mul_even_odd! { even vmef vector_signed_int i32x2 vector_signed_long_long 2 }
2777
2778    impl_vec_mul_even_odd! { even vmleb vector_unsigned_char u8x8 vector_unsigned_short 8 }
2779    impl_vec_mul_even_odd! { even vmleh vector_unsigned_short u16x4 vector_unsigned_int 4 }
2780    impl_vec_mul_even_odd! { even vmlef vector_unsigned_int u32x2 vector_unsigned_long_long 2 }
2781
2782    impl_mul!([VectorMule vec_mule] vmeb (vector_signed_char, vector_signed_char) -> vector_signed_short );
2783    impl_mul!([VectorMule vec_mule] vmeh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2784    impl_mul!([VectorMule vec_mule] vmef (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2785
2786    impl_mul!([VectorMule vec_mule] vmleb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2787    impl_mul!([VectorMule vec_mule] vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2788    impl_mul!([VectorMule vec_mule] vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2789
2790    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2791    pub trait VectorMulo<Result> {
2792        unsafe fn vec_mulo(self, b: Self) -> Result;
2793    }
2794
2795    impl_vec_mul_even_odd! { odd vmob vector_signed_char i8x8 vector_signed_short 8 }
2796    impl_vec_mul_even_odd! { odd vmoh vector_signed_short i16x4 vector_signed_int 4 }
2797    impl_vec_mul_even_odd! { odd vmof vector_signed_int i32x2 vector_signed_long_long 2 }
2798
2799    impl_vec_mul_even_odd! { odd vmlob vector_unsigned_char u8x8 vector_unsigned_short 8 }
2800    impl_vec_mul_even_odd! { odd vmloh vector_unsigned_short u16x4 vector_unsigned_int 4 }
2801    impl_vec_mul_even_odd! { odd vmlof vector_unsigned_int u32x2 vector_unsigned_long_long 2 }
2802
2803    impl_mul!([VectorMulo vec_mulo] vmob (vector_signed_char, vector_signed_char) -> vector_signed_short );
2804    impl_mul!([VectorMulo vec_mulo] vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2805    impl_mul!([VectorMulo vec_mulo] vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2806
2807    impl_mul!([VectorMulo vec_mulo] vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2808    impl_mul!([VectorMulo vec_mulo] vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2809    impl_mul!([VectorMulo vec_mulo] vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2810
2811    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2812    pub trait VectorMulh<Result> {
2813        unsafe fn vec_mulh(self, b: Self) -> Result;
2814    }
2815
2816    test_impl! { vec_vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vmhb, vmhb ] }
2817    test_impl! { vec_vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vmhh, vmhh ] }
2818    test_impl! { vec_vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vmhf, vmhf ] }
2819
2820    test_impl! { vec_vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vmlhb, vmlhb ] }
2821    test_impl! { vec_vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vmlhh, vmlhh ] }
2822    test_impl! { vec_vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vmlhf, vmlhf ] }
2823
2824    impl_mul!([VectorMulh vec_mulh] vec_vmhb (vector_signed_char, vector_signed_char) -> vector_signed_char);
2825    impl_mul!([VectorMulh vec_mulh] vec_vmhh (vector_signed_short, vector_signed_short) -> vector_signed_short);
2826    impl_mul!([VectorMulh vec_mulh] vec_vmhf (vector_signed_int, vector_signed_int) -> vector_signed_int);
2827
2828    impl_mul!([VectorMulh vec_mulh] vec_vmlhb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char);
2829    impl_mul!([VectorMulh vec_mulh] vec_vmlhh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2830    impl_mul!([VectorMulh vec_mulh] vec_vmlhf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int);
2831
2832    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2833    pub trait VectorMeadd {
2834        type Result;
2835        unsafe fn vec_meadd(self, b: Self, c: Self::Result) -> Self::Result;
2836    }
2837
2838    test_impl! { vec_vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaeb, vmaeb ] }
2839    test_impl! { vec_vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaeh, vmaeh ] }
2840    test_impl! { vec_vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaef, vmaef ] }
2841
2842    test_impl! { vec_vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmaleb, vmaleb ] }
2843    test_impl! { vec_vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaleh, vmaleh ] }
2844    test_impl! { vec_vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalef, vmalef ] }
2845
2846    impl_mul!([VectorMeadd vec_meadd] vec_vmaeb (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2847    impl_mul!([VectorMeadd vec_meadd] vec_vmaeh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2848    impl_mul!([VectorMeadd vec_meadd] vec_vmaef (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2849
2850    impl_mul!([VectorMeadd vec_meadd] vec_vmaleb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2851    impl_mul!([VectorMeadd vec_meadd] vec_vmaleh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2852    impl_mul!([VectorMeadd vec_meadd] vec_vmalef (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2853
2854    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2855    pub trait VectorMoadd {
2856        type Result;
2857        unsafe fn vec_moadd(self, b: Self, c: Self::Result) -> Self::Result;
2858    }
2859
2860    test_impl! { vec_vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaob, vmaob ] }
2861    test_impl! { vec_vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaoh, vmaoh ] }
2862    test_impl! { vec_vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaof, vmaof ] }
2863
2864    test_impl! { vec_vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmalob, vmalob ] }
2865    test_impl! { vec_vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaloh, vmaloh ] }
2866    test_impl! { vec_vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalof, vmalof ] }
2867
2868    impl_mul!([VectorMoadd vec_moadd] vec_vmaob (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2869    impl_mul!([VectorMoadd vec_moadd] vec_vmaoh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2870    impl_mul!([VectorMoadd vec_moadd] vec_vmaof (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2871
2872    impl_mul!([VectorMoadd vec_moadd] vec_vmalob (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2873    impl_mul!([VectorMoadd vec_moadd] vec_vmaloh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2874    impl_mul!([VectorMoadd vec_moadd] vec_vmalof (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2875
2876    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2877    pub trait VectorMhadd {
2878        type Result;
2879        unsafe fn vec_mhadd(self, b: Self, c: Self::Result) -> Self::Result;
2880    }
2881
2882    test_impl! { vec_vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [ vmahb, vmahb ] }
2883    test_impl! { vec_vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[ vmahh, vmahh ] }
2884    test_impl! { vec_vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [ vmahf, vmahf ] }
2885
2886    test_impl! { vec_vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [ vmalhb, vmalhb ] }
2887    test_impl! { vec_vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[ vmalhh, vmalhh ] }
2888    test_impl! { vec_vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [ vmalhf, vmalhf ] }
2889
2890    impl_mul!([VectorMhadd vec_mhadd] vec_vmahb (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2891    impl_mul!([VectorMhadd vec_mhadd] vec_vmahh (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2892    impl_mul!([VectorMhadd vec_mhadd] vec_vmahf (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2893
2894    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2895    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2896    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2897
2898    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2899    pub trait VectorMladd {
2900        type Result;
2901        unsafe fn vec_mladd(self, b: Self, c: Self::Result) -> Self::Result;
2902    }
2903
2904    #[inline]
2905    #[target_feature(enable = "vector")]
2906    unsafe fn simd_mladd<T>(a: T, b: T, c: T) -> T {
2907        simd_add(simd_mul(a, b), c)
2908    }
2909
2910    test_impl! { vec_vmal_ib(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [simd_mladd, vmalb ] }
2911    test_impl! { vec_vmal_ih(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[simd_mladd, vmalhw ] }
2912    test_impl! { vec_vmal_if(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [simd_mladd, vmalf ] }
2913
2914    test_impl! { vec_vmal_ub(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [simd_mladd, vmalb ] }
2915    test_impl! { vec_vmal_uh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[simd_mladd, vmalhw ] }
2916    test_impl! { vec_vmal_uf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [simd_mladd, vmalf ] }
2917
2918    impl_mul!([VectorMladd vec_mladd] vec_vmal_ib (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2919    impl_mul!([VectorMladd vec_mladd] vec_vmal_ih (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2920    impl_mul!([VectorMladd vec_mladd] vec_vmal_if (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2921
2922    impl_mul!([VectorMladd vec_mladd] vec_vmal_ub (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2923    impl_mul!([VectorMladd vec_mladd] vec_vmal_uh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2924    impl_mul!([VectorMladd vec_mladd] vec_vmal_uf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2925
2926    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2927    pub trait VectorGfmsum<Result> {
2928        unsafe fn vec_gfmsum(self, b: Self) -> Result;
2929    }
2930
2931    test_impl! { vec_vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vgfmb, vgfmb ] }
2932    test_impl! { vec_vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vgfmh, vgfmh] }
2933    test_impl! { vec_vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vgfmf, vgfmf ] }
2934
2935    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2936    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2937    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2938
2939    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2940    pub trait VectorGfmsumAccum {
2941        type Result;
2942        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result;
2943    }
2944
2945    test_impl! { vec_vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vgfmab, vgfmab ] }
2946    test_impl! { vec_vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vgfmah, vgfmah] }
2947    test_impl! { vec_vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vgfmaf, vgfmaf ] }
2948
2949    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2950    impl VectorGfmsumAccum for vector_unsigned_char {
2951        type Result = vector_unsigned_short;
2952        #[inline]
2953        #[target_feature(enable = "vector")]
2954        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2955            vec_vgfmab(self, b, c)
2956        }
2957    }
2958    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2959    impl VectorGfmsumAccum for vector_unsigned_short {
2960        type Result = vector_unsigned_int;
2961        #[inline]
2962        #[target_feature(enable = "vector")]
2963        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2964            vec_vgfmah(self, b, c)
2965        }
2966    }
2967    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2968    impl VectorGfmsumAccum for vector_unsigned_int {
2969        type Result = vector_unsigned_long_long;
2970        #[inline]
2971        #[target_feature(enable = "vector")]
2972        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2973            vec_vgfmaf(self, b, c)
2974        }
2975    }
2976
2977    #[inline]
2978    #[target_feature(enable = "vector")]
2979    #[cfg_attr(test, assert_instr(vgef, D = 3))]
2980    unsafe fn vgef<const D: u32>(
2981        a: vector_unsigned_int,
2982        b: vector_unsigned_int,
2983        c: *const u32,
2984    ) -> vector_unsigned_int {
2985        static_assert_uimm_bits!(D, 2);
2986        let offset: u32 = simd_extract(b, D);
2987        let ptr = c.byte_add(offset as usize);
2988        let value = ptr.read();
2989        simd_insert(a, D, value)
2990    }
2991
2992    #[inline]
2993    #[target_feature(enable = "vector")]
2994    #[cfg_attr(test, assert_instr(vgeg, D = 1))]
2995    unsafe fn vgeg<const D: u32>(
2996        a: vector_unsigned_long_long,
2997        b: vector_unsigned_long_long,
2998        c: *const u64,
2999    ) -> vector_unsigned_long_long {
3000        static_assert_uimm_bits!(D, 1);
3001        let offset: u64 = simd_extract(b, D);
3002        let ptr = c.byte_add(offset as usize);
3003        let value = ptr.read();
3004        simd_insert(a, D, value)
3005    }
3006
3007    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3008    pub trait VectorGatherElement {
3009        type Element;
3010        type Offset;
3011        unsafe fn vec_gather_element<const D: u32>(
3012            self,
3013            b: Self::Offset,
3014            c: *const Self::Element,
3015        ) -> Self;
3016    }
3017
3018    macro_rules! impl_vec_gather_element {
3019        ($($instr:ident $ty:ident)*) => {
3020            $(
3021                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3022                impl VectorGatherElement for $ty {
3023                    type Element = l_t_t!($ty);
3024                    type Offset = t_u!($ty);
3025
3026                    #[inline]
3027                    #[target_feature(enable = "vector")]
3028                    unsafe fn vec_gather_element<const D: u32>(self, b: Self::Offset, c: *const Self::Element) -> Self {
3029                        transmute($instr::<D>(transmute(self), b, c.cast()))
3030                    }
3031                }
3032            )*
3033        }
3034    }
3035
3036    impl_vec_gather_element! {
3037        vgef vector_signed_int
3038        vgef vector_bool_int
3039        vgef vector_unsigned_int
3040
3041        vgeg vector_signed_long_long
3042        vgeg vector_bool_long_long
3043        vgeg vector_unsigned_long_long
3044
3045        vgef vector_float
3046        vgeg vector_double
3047    }
3048
3049    #[inline]
3050    #[target_feature(enable = "vector")]
3051    #[cfg_attr(test, assert_instr(vscef, D = 3))]
3052    unsafe fn vscef<const D: u32>(a: vector_unsigned_int, b: vector_unsigned_int, c: *mut u32) {
3053        static_assert_uimm_bits!(D, 2);
3054        let value = simd_extract(a, D);
3055        let offset: u32 = simd_extract(b, D);
3056        let ptr = c.byte_add(offset as usize);
3057        ptr.write(value);
3058    }
3059
3060    #[inline]
3061    #[target_feature(enable = "vector")]
3062    #[cfg_attr(test, assert_instr(vsceg, D = 1))]
3063    unsafe fn vsceg<const D: u32>(
3064        a: vector_unsigned_long_long,
3065        b: vector_unsigned_long_long,
3066        c: *mut u64,
3067    ) {
3068        static_assert_uimm_bits!(D, 1);
3069        let value = simd_extract(a, D);
3070        let offset: u64 = simd_extract(b, D);
3071        let ptr = c.byte_add(offset as usize);
3072        ptr.write(value);
3073    }
3074
3075    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3076    pub trait VectorScatterElement {
3077        type Element;
3078        type Offset;
3079        unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element);
3080    }
3081
3082    macro_rules! impl_vec_scatter_element {
3083        ($($instr:ident $ty:ident)*) => {
3084            $(
3085                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3086                impl VectorScatterElement for $ty {
3087                    type Element = l_t_t!($ty);
3088                    type Offset = t_u!($ty);
3089
3090                    #[inline]
3091                    #[target_feature(enable = "vector")]
3092                    unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element) {
3093                        $instr::<D>(transmute(self), b, c.cast())
3094                    }
3095                }
3096            )*
3097        }
3098    }
3099
3100    impl_vec_scatter_element! {
3101        vscef vector_signed_int
3102        vscef vector_bool_int
3103        vscef vector_unsigned_int
3104
3105        vsceg vector_signed_long_long
3106        vsceg vector_bool_long_long
3107        vsceg vector_unsigned_long_long
3108
3109        vscef vector_float
3110        vsceg vector_double
3111    }
3112
3113    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3114    pub trait VectorSel<Mask>: Sized {
3115        unsafe fn vec_sel(self, b: Self, c: Mask) -> Self;
3116    }
3117
3118    macro_rules! impl_vec_sel {
3119        ($($ty:ident)*) => {
3120            $(
3121                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3122                impl VectorSel<t_u!($ty)> for $ty {
3123                    #[inline]
3124                    #[target_feature(enable = "vector")]
3125                    unsafe fn vec_sel(self, b: Self, c: t_u!($ty)) -> Self {
3126                        let b = simd_and(transmute(b), c);
3127                        let a = simd_and(transmute(self), simd_xor(c, transmute(vector_signed_char([!0; 16]))));
3128                        transmute(simd_or(a, b))
3129                    }
3130                }
3131
3132                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3133                impl VectorSel<t_b!($ty)> for $ty {
3134                    #[inline]
3135                    #[target_feature(enable = "vector")]
3136                    unsafe fn vec_sel(self, b: Self, c: t_b!($ty)) -> Self {
3137                        // defer to the implementation with an unsigned mask
3138                        self.vec_sel(b, transmute::<_, t_u!($ty)>(c))
3139                    }
3140                }
3141            )*
3142        }
3143    }
3144
3145    impl_vec_sel! {
3146        vector_signed_char
3147        vector_signed_short
3148        vector_signed_int
3149        vector_signed_long_long
3150
3151        vector_unsigned_char
3152        vector_unsigned_short
3153        vector_unsigned_int
3154        vector_unsigned_long_long
3155
3156        vector_bool_char
3157        vector_bool_short
3158        vector_bool_int
3159        vector_bool_long_long
3160
3161        vector_float
3162        vector_double
3163    }
3164
3165    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3166    pub trait VectorFpTestDataClass {
3167        type Result;
3168        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32);
3169    }
3170
3171    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3172    impl VectorFpTestDataClass for vector_float {
3173        type Result = vector_bool_int;
3174
3175        #[inline]
3176        #[target_feature(enable = "vector")]
3177        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3178            let PackedTuple { x, y } = vftcisb(self, CLASS);
3179            (x, y)
3180        }
3181    }
3182
3183    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3184    impl VectorFpTestDataClass for vector_double {
3185        type Result = vector_bool_long_long;
3186
3187        #[inline]
3188        #[target_feature(enable = "vector")]
3189        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3190            let PackedTuple { x, y } = vftcidb(self, CLASS);
3191            (x, y)
3192        }
3193    }
3194
3195    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3196    pub trait VectorCompare {
3197        unsafe fn vec_all_lt(self, other: Self) -> i32;
3198        unsafe fn vec_all_le(self, other: Self) -> i32;
3199        unsafe fn vec_all_gt(self, other: Self) -> i32;
3200        unsafe fn vec_all_ge(self, other: Self) -> i32;
3201    }
3202
3203    // NOTE: this implementation is currently non-optimal, but it does work for floats even with
3204    // only `vector` enabled.
3205    //
3206    // - https://github.com/llvm/llvm-project/issues/129434
3207    // - https://github.com/llvm/llvm-project/issues/130424
3208    macro_rules! impl_vec_compare {
3209        ($($ty:ident)*) => {
3210            $(
3211                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3212                impl VectorCompare for $ty {
3213                    #[inline]
3214                    #[target_feature(enable = "vector")]
3215                    unsafe fn vec_all_lt(self, other: Self) -> i32 {
3216                        simd_reduce_all(simd_lt::<_, t_b!($ty)>(self, other)) as i32
3217                    }
3218                    #[inline]
3219                    #[target_feature(enable = "vector")]
3220                    unsafe fn vec_all_le(self, other: Self) -> i32 {
3221                        simd_reduce_all(simd_le::<_, t_b!($ty)>(self, other)) as i32
3222                    }
3223                    #[inline]
3224                    #[target_feature(enable = "vector")]
3225                    unsafe fn vec_all_gt(self, other: Self) -> i32 {
3226                        simd_reduce_all(simd_gt::<_, t_b!($ty)>(self, other)) as i32
3227                    }
3228                    #[inline]
3229                    #[target_feature(enable = "vector")]
3230                    unsafe fn vec_all_ge(self, other: Self) -> i32 {
3231                        simd_reduce_all(simd_ge::<_, t_b!($ty)>(self, other)) as i32
3232                    }
3233                }
3234            )*
3235        }
3236    }
3237
3238    impl_vec_compare! {
3239        vector_signed_char
3240        vector_unsigned_char
3241
3242        vector_signed_short
3243        vector_unsigned_short
3244
3245        vector_signed_int
3246        vector_unsigned_int
3247        vector_float
3248
3249        vector_signed_long_long
3250        vector_unsigned_long_long
3251        vector_double
3252    }
3253
3254    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3255    pub trait VectorTestMask {
3256        type Mask;
3257        unsafe fn vec_test_mask(self, other: Self::Mask) -> i32;
3258    }
3259
3260    macro_rules! impl_vec_test_mask {
3261        ($($instr:ident $ty:ident)*) => {
3262            $(
3263                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3264                impl VectorTestMask for $ty {
3265                    type Mask = t_u!($ty);
3266
3267                    #[inline]
3268                    #[target_feature(enable = "vector")]
3269                    unsafe fn vec_test_mask(self, other: Self::Mask) -> i32 {
3270                        vtm(transmute(self), transmute(other))
3271                    }
3272                }
3273            )*
3274        }
3275    }
3276
3277    impl_vec_test_mask! {
3278        vector_signed_char
3279        vector_signed_short
3280        vector_signed_int
3281        vector_signed_long_long
3282
3283        vector_unsigned_char
3284        vector_unsigned_short
3285        vector_unsigned_int
3286        vector_unsigned_long_long
3287
3288        vector_float
3289        vector_double
3290    }
3291
3292    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3293    pub trait VectorSearchString {
3294        unsafe fn vec_search_string_cc(
3295            self,
3296            b: Self,
3297            c: vector_unsigned_char,
3298        ) -> (vector_unsigned_char, i32);
3299
3300        unsafe fn vec_search_string_until_zero_cc(
3301            self,
3302            b: Self,
3303            c: vector_unsigned_char,
3304        ) -> (vector_unsigned_char, i32);
3305    }
3306
3307    macro_rules! impl_vec_search_string{
3308        ($($intr_s:ident $intr_sz:ident $ty:ident)*) => {
3309            $(
3310                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3311                impl VectorSearchString for $ty {
3312                    #[inline]
3313                    #[target_feature(enable = "vector-enhancements-2")]
3314                    unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3315                        let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c);
3316                        (x, y)
3317                    }
3318
3319                    #[inline]
3320                    #[target_feature(enable = "vector-enhancements-2")]
3321                    unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3322                        let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c);
3323                        (x, y)
3324                    }
3325                }
3326
3327            )*
3328        }
3329    }
3330
3331    impl_vec_search_string! {
3332        vstrsb vstrszb vector_signed_char
3333        vstrsb vstrszb vector_bool_char
3334        vstrsb vstrszb vector_unsigned_char
3335
3336        vstrsh vstrszh vector_signed_short
3337        vstrsh vstrszh vector_bool_short
3338        vstrsh vstrszh vector_unsigned_short
3339
3340        vstrsf vstrszf vector_signed_int
3341        vstrsf vstrszf vector_bool_int
3342        vstrsf vstrszf vector_unsigned_int
3343    }
3344
3345    #[inline]
3346    #[target_feature(enable = "vector")]
3347    #[cfg_attr(test, assert_instr(vcdgb))]
3348    pub unsafe fn vcdgb(a: vector_signed_long_long) -> vector_double {
3349        simd_as(a)
3350    }
3351
3352    #[inline]
3353    #[target_feature(enable = "vector")]
3354    #[cfg_attr(test, assert_instr(vcdlgb))]
3355    pub unsafe fn vcdlgb(a: vector_unsigned_long_long) -> vector_double {
3356        simd_as(a)
3357    }
3358
3359    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3360    pub trait VectorDouble {
3361        unsafe fn vec_double(self) -> vector_double;
3362    }
3363
3364    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3365    impl VectorDouble for vector_signed_long_long {
3366        #[inline]
3367        #[target_feature(enable = "vector")]
3368        unsafe fn vec_double(self) -> vector_double {
3369            vcdgb(self)
3370        }
3371    }
3372
3373    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3374    impl VectorDouble for vector_unsigned_long_long {
3375        #[inline]
3376        #[target_feature(enable = "vector")]
3377        unsafe fn vec_double(self) -> vector_double {
3378            vcdlgb(self)
3379        }
3380    }
3381
3382    #[inline]
3383    #[target_feature(enable = "vector")]
3384    #[cfg_attr(
3385        all(test, target_feature = "vector-enhancements-2"),
3386        assert_instr(vcefb)
3387    )]
3388    pub unsafe fn vcefb(a: vector_signed_int) -> vector_float {
3389        simd_as(a)
3390    }
3391
3392    #[inline]
3393    #[target_feature(enable = "vector")]
3394    #[cfg_attr(
3395        all(test, target_feature = "vector-enhancements-2"),
3396        assert_instr(vcelfb)
3397    )]
3398    pub unsafe fn vcelfb(a: vector_unsigned_int) -> vector_float {
3399        simd_as(a)
3400    }
3401
3402    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3403    pub trait VectorFloat {
3404        unsafe fn vec_float(self) -> vector_float;
3405    }
3406
3407    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3408    impl VectorFloat for vector_signed_int {
3409        #[inline]
3410        #[target_feature(enable = "vector")]
3411        unsafe fn vec_float(self) -> vector_float {
3412            vcefb(self)
3413        }
3414    }
3415
3416    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3417    impl VectorFloat for vector_unsigned_int {
3418        #[inline]
3419        #[target_feature(enable = "vector")]
3420        unsafe fn vec_float(self) -> vector_float {
3421            vcelfb(self)
3422        }
3423    }
3424
3425    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3426    pub trait VectorExtendSigned64 {
3427        unsafe fn vec_extend_s64(self) -> vector_signed_long_long;
3428    }
3429
3430    #[inline]
3431    #[target_feature(enable = "vector")]
3432    #[cfg_attr(test, assert_instr(vsegb))]
3433    pub unsafe fn vsegb(a: vector_signed_char) -> vector_signed_long_long {
3434        simd_as(simd_shuffle::<_, _, i8x2>(
3435            a,
3436            a,
3437            const { u32x2::from_array([7, 15]) },
3438        ))
3439    }
3440
3441    #[inline]
3442    #[target_feature(enable = "vector")]
3443    #[cfg_attr(test, assert_instr(vsegh))]
3444    pub unsafe fn vsegh(a: vector_signed_short) -> vector_signed_long_long {
3445        simd_as(simd_shuffle::<_, _, i16x2>(
3446            a,
3447            a,
3448            const { u32x2::from_array([3, 7]) },
3449        ))
3450    }
3451
3452    #[inline]
3453    #[target_feature(enable = "vector")]
3454    #[cfg_attr(test, assert_instr(vsegf))]
3455    pub unsafe fn vsegf(a: vector_signed_int) -> vector_signed_long_long {
3456        simd_as(simd_shuffle::<_, _, i32x2>(
3457            a,
3458            a,
3459            const { u32x2::from_array([1, 3]) },
3460        ))
3461    }
3462
3463    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3464    impl VectorExtendSigned64 for vector_signed_char {
3465        #[inline]
3466        #[target_feature(enable = "vector")]
3467        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3468            vsegb(self)
3469        }
3470    }
3471    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3472    impl VectorExtendSigned64 for vector_signed_short {
3473        #[inline]
3474        #[target_feature(enable = "vector")]
3475        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3476            vsegh(self)
3477        }
3478    }
3479    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3480    impl VectorExtendSigned64 for vector_signed_int {
3481        #[inline]
3482        #[target_feature(enable = "vector")]
3483        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3484            vsegf(self)
3485        }
3486    }
3487
3488    // NOTE: VectorSigned and VectorUnsigned make strong safety assumptions around floats.
3489    // This is what C provides, but even IBM does not clearly document these constraints.
3490    //
3491    // https://doc.rust-lang.org/std/intrinsics/simd/fn.simd_cast.html
3492
3493    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3494    pub trait VectorSigned {
3495        type Result;
3496        unsafe fn vec_signed(self) -> Self::Result;
3497    }
3498
3499    test_impl! { vcgsb (a: vector_float) -> vector_signed_int [simd_cast, "vector-enhancements-2" vcgsb] }
3500    test_impl! { vcgdb (a: vector_double) -> vector_signed_long_long [simd_cast, vcgdb] }
3501
3502    impl_vec_trait! { [VectorSigned vec_signed] vcgsb (vector_float) -> vector_signed_int }
3503    impl_vec_trait! { [VectorSigned vec_signed] vcgdb (vector_double) -> vector_signed_long_long }
3504
3505    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3506    pub trait VectorUnsigned {
3507        type Result;
3508        unsafe fn vec_unsigned(self) -> Self::Result;
3509    }
3510
3511    test_impl! { vclgsb (a: vector_float) -> vector_unsigned_int [simd_cast, "vector-enhancements-2" vclgsb] }
3512    test_impl! { vclgdb (a: vector_double) -> vector_unsigned_long_long [simd_cast, vclgdb] }
3513
3514    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgsb (vector_float) -> vector_unsigned_int }
3515    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgdb (vector_double) -> vector_unsigned_long_long }
3516
3517    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3518    pub trait VectorCopyUntilZero {
3519        unsafe fn vec_cp_until_zero(self) -> Self;
3520    }
3521
3522    test_impl! { vec_vistrb (a: vector_unsigned_char) -> vector_unsigned_char [vistrb, vistrb] }
3523    test_impl! { vec_vistrh (a: vector_unsigned_short) -> vector_unsigned_short [vistrh, vistrh] }
3524    test_impl! { vec_vistrf (a: vector_unsigned_int) -> vector_unsigned_int [vistrf, vistrf] }
3525
3526    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_signed_char) }
3527    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_bool_char) }
3528    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_unsigned_char) }
3529
3530    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_signed_short) }
3531    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_bool_short) }
3532    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_unsigned_short) }
3533
3534    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_signed_int) }
3535    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_bool_int) }
3536    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_unsigned_int) }
3537
3538    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3539    pub trait VectorCopyUntilZeroCC: Sized {
3540        unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32);
3541    }
3542
3543    test_impl! { vec_vistrbs (a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32> [vistrbs, vistrbs] }
3544    test_impl! { vec_vistrhs (a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32> [vistrhs, vistrhs] }
3545    test_impl! { vec_vistrfs (a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32> [vistrfs, vistrfs] }
3546
3547    macro_rules! impl_vec_copy_until_zero_cc {
3548        ($($intr:ident $ty:ident)*) => {
3549            $(
3550                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3551                impl VectorCopyUntilZeroCC for $ty {
3552                    #[inline]
3553                    #[target_feature(enable = "vector")]
3554                    unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32) {
3555                        let PackedTuple { x,y } = $intr(transmute(self));
3556                        (transmute(x), y)
3557                    }
3558                }
3559
3560            )*
3561        }
3562    }
3563
3564    impl_vec_copy_until_zero_cc! {
3565        vec_vistrbs vector_signed_char
3566        vec_vistrbs vector_bool_char
3567        vec_vistrbs vector_unsigned_char
3568
3569        vec_vistrhs vector_signed_short
3570        vec_vistrhs vector_bool_short
3571        vec_vistrhs vector_unsigned_short
3572
3573        vec_vistrfs vector_signed_int
3574        vec_vistrfs vector_bool_int
3575        vec_vistrfs vector_unsigned_int
3576    }
3577
3578    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3579    pub trait VectorSrdb {
3580        unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self;
3581    }
3582
3583    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3584    pub trait VectorSld {
3585        unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self;
3586
3587        unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self;
3588
3589        unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self;
3590    }
3591
3592    #[inline]
3593    #[target_feature(enable = "vector")]
3594    #[cfg_attr(test, assert_instr(vsldb))]
3595    unsafe fn test_vec_sld(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
3596        a.vec_sld::<13>(b)
3597    }
3598
3599    #[inline]
3600    #[target_feature(enable = "vector")]
3601    #[cfg_attr(test, assert_instr(vsldb))]
3602    unsafe fn test_vec_sldw(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
3603        a.vec_sldw::<3>(b)
3604    }
3605
3606    #[inline]
3607    #[target_feature(enable = "vector-enhancements-2")]
3608    #[cfg_attr(test, assert_instr(vsld))]
3609    unsafe fn test_vec_sldb(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
3610        a.vec_sldb::<7>(b)
3611    }
3612
3613    #[inline]
3614    #[target_feature(enable = "vector-enhancements-2")]
3615    #[cfg_attr(test, assert_instr(vsrd))]
3616    unsafe fn test_vec_srdb(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
3617        a.vec_srdb::<7>(b)
3618    }
3619
3620    macro_rules! impl_vec_sld {
3621        ($($ty:ident)*) => {
3622            $(
3623                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3624                impl VectorSld for $ty {
3625                    #[inline]
3626                    #[target_feature(enable = "vector")]
3627                    unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self {
3628                        static_assert_uimm_bits!(C, 4);
3629                        transmute(u128::funnel_shl(transmute(self), transmute(b), C  * 8))
3630                    }
3631
3632                    #[inline]
3633                    #[target_feature(enable = "vector")]
3634                    unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self {
3635                        static_assert_uimm_bits!(C, 2);
3636                        transmute(u128::funnel_shl(transmute(self), transmute(b), C * 4 * 8))
3637                    }
3638
3639                    #[inline]
3640                    #[target_feature(enable = "vector-enhancements-2")]
3641                    unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self {
3642                        static_assert_uimm_bits!(C, 3);
3643                        transmute(u128::funnel_shl(transmute(self), transmute(b), C))
3644                    }
3645                }
3646
3647                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3648                impl VectorSrdb for $ty {
3649                    #[inline]
3650                    #[target_feature(enable = "vector-enhancements-2")]
3651                    unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self {
3652                        static_assert_uimm_bits!(C, 3);
3653                        transmute(vsrd(transmute(self), transmute(b), C))
3654                        // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129955#issuecomment-3207488190
3655                        // LLVM currently rewrites `fshr` to `fshl`, and the logic in the s390x
3656                        // backend cannot deal with that yet.
3657                        // #[link_name = "llvm.fshr.i128"] fn fshr_i128(a: u128, b: u128, c: u128) -> u128;
3658                        // transmute(fshr_i128(transmute(self), transmute(b), const { C as u128 }))
3659                    }
3660                }
3661            )*
3662        }
3663    }
3664
3665    impl_vec_sld! {
3666        vector_signed_char
3667        vector_bool_char
3668        vector_unsigned_char
3669
3670        vector_signed_short
3671        vector_bool_short
3672        vector_unsigned_short
3673
3674        vector_signed_int
3675        vector_bool_int
3676        vector_unsigned_int
3677
3678        vector_signed_long_long
3679        vector_bool_long_long
3680        vector_unsigned_long_long
3681
3682        vector_float
3683        vector_double
3684    }
3685
3686    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3687    pub trait VectorCompareRange: Sized {
3688        type Result;
3689
3690        unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3691        unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3692        unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3693        unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3694    }
3695
3696    const fn validate_compare_range_imm(imm: u32) {
3697        if !matches!(imm, 0 | 4 | 8 | 12) {
3698            panic!("IMM needs to be one of 0, 4, 8, 12");
3699        }
3700    }
3701
3702    macro_rules! impl_compare_range {
3703        ($($ty:ident $vstrc:ident $vstrcs:ident $vstrcz:ident $vstrczs:ident)*) => {
3704            $(
3705                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3706                impl VectorCompareRange for $ty {
3707                    type Result = t_b!($ty);
3708
3709                    #[inline]
3710                    #[target_feature(enable = "vector")]
3711                    unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3712                        const { validate_compare_range_imm };
3713                        $vstrc(self, b, c, IMM)
3714                    }
3715
3716                    #[inline]
3717                    #[target_feature(enable = "vector")]
3718                    unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3719                        const { validate_compare_range_imm };
3720                        $vstrcz(self, b, c, IMM)
3721                    }
3722
3723                    #[inline]
3724                    #[target_feature(enable = "vector")]
3725                    unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3726                        const { validate_compare_range_imm };
3727                        let PackedTuple { x, y } = $vstrcs(self, b, c, IMM);
3728                        (x,y)
3729                    }
3730
3731                    #[inline]
3732                    #[target_feature(enable = "vector")]
3733                    unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3734                        const { validate_compare_range_imm };
3735                        let PackedTuple { x, y } = $vstrczs(self, b, c, IMM);
3736                        (x,y)
3737                    }
3738                }
3739            )*
3740        }
3741    }
3742
3743    impl_compare_range! {
3744        vector_unsigned_char    vstrcb vstrcbs vstrczb vstrczbs
3745        vector_unsigned_short   vstrch vstrchs vstrczh vstrczhs
3746        vector_unsigned_int     vstrcf vstrcfs vstrczf vstrczfs
3747    }
3748
3749    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3750    pub trait VectorComparePredicate: Sized {
3751        type Result;
3752
3753        #[inline]
3754        #[target_feature(enable = "vector")]
3755        unsafe fn vec_cmpgt(self, other: Self) -> Self::Result {
3756            simd_gt(self, other)
3757        }
3758
3759        #[inline]
3760        #[target_feature(enable = "vector")]
3761        unsafe fn vec_cmpge(self, other: Self) -> Self::Result {
3762            simd_ge(self, other)
3763        }
3764
3765        #[inline]
3766        #[target_feature(enable = "vector")]
3767        unsafe fn vec_cmplt(self, other: Self) -> Self::Result {
3768            simd_lt(self, other)
3769        }
3770
3771        #[inline]
3772        #[target_feature(enable = "vector")]
3773        unsafe fn vec_cmple(self, other: Self) -> Self::Result {
3774            simd_le(self, other)
3775        }
3776    }
3777
3778    macro_rules! impl_compare_predicate {
3779        ($($ty:ident)*) => {
3780            $(
3781                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3782                impl VectorComparePredicate for $ty {
3783                    type Result = t_b!($ty);
3784                }
3785            )*
3786        }
3787    }
3788
3789    impl_compare_predicate! {
3790        vector_signed_char
3791        vector_unsigned_char
3792
3793        vector_signed_short
3794        vector_unsigned_short
3795
3796        vector_signed_int
3797        vector_unsigned_int
3798        vector_float
3799
3800        vector_signed_long_long
3801        vector_unsigned_long_long
3802        vector_double
3803    }
3804
3805    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3806    pub trait VectorEquality: Sized {
3807        type Result;
3808
3809        #[inline]
3810        #[target_feature(enable = "vector")]
3811        unsafe fn vec_cmpeq(self, other: Self) -> Self::Result {
3812            simd_eq(self, other)
3813        }
3814
3815        #[inline]
3816        #[target_feature(enable = "vector")]
3817        unsafe fn vec_cmpne(self, other: Self) -> Self::Result {
3818            simd_ne(self, other)
3819        }
3820    }
3821
3822    macro_rules! impl_compare_equality {
3823        ($($ty:ident)*) => {
3824            $(
3825                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3826                impl VectorEquality for $ty {
3827                    type Result = t_b!($ty);
3828                }
3829            )*
3830        }
3831    }
3832
3833    impl_compare_equality! {
3834        vector_bool_char
3835        vector_signed_char
3836        vector_unsigned_char
3837
3838        vector_bool_short
3839        vector_signed_short
3840        vector_unsigned_short
3841
3842        vector_bool_int
3843        vector_signed_int
3844        vector_unsigned_int
3845        vector_float
3846
3847        vector_bool_long_long
3848        vector_signed_long_long
3849        vector_unsigned_long_long
3850        vector_double
3851    }
3852
3853    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3854    pub trait VectorEqualityIdx: Sized {
3855        type Result;
3856
3857        unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result;
3858        unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result;
3859
3860        unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32);
3861        unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32);
3862
3863        unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result;
3864        unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result;
3865
3866        unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3867        unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3868    }
3869
3870    macro_rules! impl_compare_equality_idx {
3871        ($($ty:ident $ret:ident
3872                $cmpeq:ident $cmpne:ident
3873                $cmpeq_or_0:ident $cmpne_or_0:ident
3874                $cmpeq_cc:ident $cmpne_cc:ident
3875                $cmpeq_or_0_cc:ident $cmpne_or_0_cc:ident
3876        )*) => {
3877            $(
3878                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3879                impl VectorEqualityIdx for $ty {
3880                    type Result = $ret;
3881
3882                    #[inline]
3883                    #[target_feature(enable = "vector")]
3884                    unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result {
3885                        transmute($cmpeq(transmute(self), transmute(other)))
3886                    }
3887
3888                    #[inline]
3889                    #[target_feature(enable = "vector")]
3890                    unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result {
3891                        transmute($cmpne(transmute(self), transmute(other)))
3892                    }
3893
3894                    #[inline]
3895                    #[target_feature(enable = "vector")]
3896                    unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result {
3897                        transmute($cmpeq_or_0(transmute(self), transmute(other)))
3898                    }
3899
3900                    #[inline]
3901                    #[target_feature(enable = "vector")]
3902                    unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result {
3903                        transmute($cmpne_or_0(transmute(self), transmute(other)))
3904                    }
3905
3906                    #[inline]
3907                    #[target_feature(enable = "vector")]
3908                    unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32) {
3909                        let PackedTuple { x, y } = $cmpeq_cc(transmute(self), transmute(other));
3910                        (transmute(x), y)
3911                    }
3912
3913                    #[inline]
3914                    #[target_feature(enable = "vector")]
3915                    unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32) {
3916                        let PackedTuple { x, y } = $cmpne_cc(transmute(self), transmute(other));
3917                        (transmute(x),y)
3918                    }
3919
3920                    #[inline]
3921                    #[target_feature(enable = "vector")]
3922                    unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3923                        let PackedTuple { x, y } = $cmpeq_or_0_cc(transmute(self), transmute(other));
3924                        (transmute(x), y)
3925                    }
3926
3927                    #[inline]
3928                    #[target_feature(enable = "vector")]
3929                    unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3930                        let PackedTuple { x, y } = $cmpne_or_0_cc(transmute(self), transmute(other));
3931                        (transmute(x),y)
3932                    }
3933                }
3934            )*
3935        }
3936    }
3937
3938    impl_compare_equality_idx! {
3939        vector_signed_char vector_signed_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3940        vector_bool_char vector_unsigned_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3941        vector_unsigned_char vector_unsigned_char           vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3942        vector_signed_short vector_signed_short             vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3943        vector_bool_short  vector_unsigned_short            vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3944        vector_unsigned_short vector_unsigned_short         vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3945        vector_signed_int vector_signed_int                 vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3946        vector_bool_int  vector_unsigned_int                vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3947        vector_unsigned_int vector_unsigned_int             vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3948    }
3949
3950    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3951    pub trait VectorExtract {
3952        type ElementType;
3953
3954        unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType;
3955    }
3956
3957    #[inline]
3958    #[target_feature(enable = "vector")]
3959    #[cfg_attr(test, assert_instr(vlgvb))]
3960    unsafe fn vlgvb(a: vector_unsigned_char, b: i32) -> u8 {
3961        simd_extract_dyn(a, b as u32 % 16)
3962    }
3963
3964    #[inline]
3965    #[target_feature(enable = "vector")]
3966    #[cfg_attr(test, assert_instr(vlgvh))]
3967    unsafe fn vlgvh(a: vector_unsigned_short, b: i32) -> u16 {
3968        simd_extract_dyn(a, b as u32 % 8)
3969    }
3970
3971    #[inline]
3972    #[target_feature(enable = "vector")]
3973    #[cfg_attr(test, assert_instr(vlgvf))]
3974    unsafe fn vlgvf(a: vector_unsigned_int, b: i32) -> u32 {
3975        simd_extract_dyn(a, b as u32 % 4)
3976    }
3977
3978    #[inline]
3979    #[target_feature(enable = "vector")]
3980    #[cfg_attr(test, assert_instr(vlgvg))]
3981    unsafe fn vlgvg(a: vector_unsigned_long_long, b: i32) -> u64 {
3982        simd_extract_dyn(a, b as u32 % 2)
3983    }
3984
3985    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3986    pub trait VectorInsert {
3987        type ElementType;
3988
3989        unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self;
3990    }
3991
3992    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3993    pub trait VectorPromote: Sized {
3994        type ElementType;
3995
3996        unsafe fn vec_promote(a: Self::ElementType, b: i32) -> MaybeUninit<Self>;
3997    }
3998
3999    #[inline]
4000    #[target_feature(enable = "vector")]
4001    #[cfg_attr(test, assert_instr(vlvgb))]
4002    unsafe fn vlvgb(a: u8, b: vector_unsigned_char, c: i32) -> vector_unsigned_char {
4003        simd_insert_dyn(b, c as u32 % 16, a)
4004    }
4005
4006    #[inline]
4007    #[target_feature(enable = "vector")]
4008    #[cfg_attr(test, assert_instr(vlvgh))]
4009    unsafe fn vlvgh(a: u16, b: vector_unsigned_short, c: i32) -> vector_unsigned_short {
4010        simd_insert_dyn(b, c as u32 % 8, a)
4011    }
4012
4013    #[inline]
4014    #[target_feature(enable = "vector")]
4015    #[cfg_attr(test, assert_instr(vlvgf))]
4016    unsafe fn vlvgf(a: u32, b: vector_unsigned_int, c: i32) -> vector_unsigned_int {
4017        simd_insert_dyn(b, c as u32 % 4, a)
4018    }
4019
4020    #[inline]
4021    #[target_feature(enable = "vector")]
4022    #[cfg_attr(test, assert_instr(vlvgg))]
4023    unsafe fn vlvgg(a: u64, b: vector_unsigned_long_long, c: i32) -> vector_unsigned_long_long {
4024        simd_insert_dyn(b, c as u32 % 2, a)
4025    }
4026
4027    #[unstable(feature = "stdarch_s390x", issue = "135681")]
4028    pub trait VectorInsertAndZero {
4029        type ElementType;
4030
4031        unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self;
4032    }
4033
4034    #[inline]
4035    #[target_feature(enable = "vector")]
4036    #[cfg_attr(test, assert_instr(vllezb))]
4037    unsafe fn vllezb(x: *const u8) -> vector_unsigned_char {
4038        vector_unsigned_char([0, 0, 0, 0, 0, 0, 0, *x, 0, 0, 0, 0, 0, 0, 0, 0])
4039    }
4040
4041    #[inline]
4042    #[target_feature(enable = "vector")]
4043    #[cfg_attr(test, assert_instr(vllezh))]
4044    unsafe fn vllezh(x: *const u16) -> vector_unsigned_short {
4045        vector_unsigned_short([0, 0, 0, *x, 0, 0, 0, 0])
4046    }
4047
4048    #[inline]
4049    #[target_feature(enable = "vector")]
4050    #[cfg_attr(test, assert_instr(vllezf))]
4051    unsafe fn vllezf(x: *const u32) -> vector_unsigned_int {
4052        vector_unsigned_int([0, *x, 0, 0])
4053    }
4054
4055    #[inline]
4056    #[target_feature(enable = "vector")]
4057    #[cfg_attr(test, assert_instr(vllezg))]
4058    unsafe fn vllezg(x: *const u64) -> vector_unsigned_long_long {
4059        vector_unsigned_long_long([*x, 0])
4060    }
4061
4062    macro_rules! impl_extract_insert {
4063        ($($ty:ident $extract_intr:ident $insert_intr:ident $insert_and_zero_intr:ident)*) => {
4064            $(
4065                #[unstable(feature = "stdarch_s390x", issue = "135681")]
4066                impl VectorExtract for $ty {
4067                    type ElementType = l_t_t!($ty);
4068
4069                    #[inline]
4070                    #[target_feature(enable = "vector")]
4071                    unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType {
4072                        transmute($extract_intr(transmute(a), b))
4073                    }
4074                }
4075
4076                #[unstable(feature = "stdarch_s390x", issue = "135681")]
4077                impl VectorInsert for $ty {
4078                    type ElementType = l_t_t!($ty);
4079
4080                    #[inline]
4081                    #[target_feature(enable = "vector")]
4082                    unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self {
4083                        transmute($insert_intr(transmute(a), transmute(b), c))
4084                    }
4085                }
4086
4087                #[unstable(feature = "stdarch_s390x", issue = "135681")]
4088                impl VectorInsertAndZero for $ty {
4089                    type ElementType = l_t_t!($ty);
4090
4091                    #[inline]
4092                    #[target_feature(enable = "vector")]
4093                    unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self {
4094                        transmute($insert_and_zero_intr(a.cast()))
4095                    }
4096                }
4097
4098                #[unstable(feature = "stdarch_s390x", issue = "135681")]
4099                impl VectorPromote for $ty {
4100                    type ElementType = l_t_t!($ty);
4101
4102                    #[inline]
4103                    #[target_feature(enable = "vector")]
4104                    unsafe fn vec_promote(a: Self::ElementType, c: i32) -> MaybeUninit<Self> {
4105                        // Rust does not currently support `MaybeUninit` element types to simd
4106                        // vectors. In C/LLVM that is allowed (using poison values). So rust will
4107                        // use an extra instruction to zero the memory.
4108                        let b = MaybeUninit::<$ty>::zeroed();
4109                        MaybeUninit::new(transmute($insert_intr(transmute(a), transmute(b), c)))
4110                    }
4111                }
4112            )*
4113        }
4114
4115    }
4116
4117    impl_extract_insert! {
4118        vector_signed_char          vlgvb vlvgb vllezb
4119        vector_unsigned_char        vlgvb vlvgb vllezb
4120        vector_signed_short         vlgvh vlvgh vllezh
4121        vector_unsigned_short       vlgvh vlvgh vllezh
4122        vector_signed_int           vlgvf vlvgf vllezf
4123        vector_unsigned_int         vlgvf vlvgf vllezf
4124        vector_signed_long_long     vlgvg vlvgg vllezg
4125        vector_unsigned_long_long   vlgvg vlvgg vllezg
4126        vector_float                vlgvf vlvgf vllezf
4127        vector_double               vlgvg vlvgg vllezg
4128    }
4129}
4130
4131/// Load Count to Block Boundary
4132#[inline]
4133#[target_feature(enable = "vector")]
4134#[unstable(feature = "stdarch_s390x", issue = "135681")]
4135#[cfg_attr(test, assert_instr(lcbb, BLOCK_BOUNDARY = 512))]
4136unsafe fn __lcbb<const BLOCK_BOUNDARY: u16>(ptr: *const u8) -> u32 {
4137    lcbb(ptr, const { validate_block_boundary(BLOCK_BOUNDARY) })
4138}
4139
4140/// Vector Negate
4141#[inline]
4142#[target_feature(enable = "vector")]
4143#[unstable(feature = "stdarch_s390x", issue = "135681")]
4144pub unsafe fn vec_neg<T: sealed::VectorNeg>(a: T) -> T {
4145    a.vec_neg()
4146}
4147
4148/// Vector Add
4149#[inline]
4150#[target_feature(enable = "vector")]
4151#[unstable(feature = "stdarch_s390x", issue = "135681")]
4152pub unsafe fn vec_add<T: sealed::VectorAdd<U>, U>(a: T, b: U) -> T::Result {
4153    a.vec_add(b)
4154}
4155
4156/// Vector Subtract
4157#[inline]
4158#[target_feature(enable = "vector")]
4159#[unstable(feature = "stdarch_s390x", issue = "135681")]
4160pub unsafe fn vec_sub<T: sealed::VectorSub<U>, U>(a: T, b: U) -> T::Result {
4161    a.vec_sub(b)
4162}
4163
4164/// Vector Multiply
4165///
4166/// ## Purpose
4167/// Compute the products of corresponding elements of two vectors.
4168///
4169/// ## Result value
4170/// Each element of r receives the product of the corresponding elements of a and b.
4171#[inline]
4172#[target_feature(enable = "vector")]
4173#[unstable(feature = "stdarch_s390x", issue = "135681")]
4174pub unsafe fn vec_mul<T: sealed::VectorMul>(a: T, b: T) -> T {
4175    a.vec_mul(b)
4176}
4177
4178/// Vector Count Leading Zeros
4179#[inline]
4180#[target_feature(enable = "vector")]
4181#[unstable(feature = "stdarch_s390x", issue = "135681")]
4182pub unsafe fn vec_cntlz<T: sealed::CountBits>(a: T) -> T::Result {
4183    a.vec_cntlz()
4184}
4185
4186/// Vector Count Trailing Zeros
4187#[inline]
4188#[target_feature(enable = "vector")]
4189#[unstable(feature = "stdarch_s390x", issue = "135681")]
4190pub unsafe fn vec_cnttz<T: sealed::CountBits>(a: T) -> T::Result {
4191    a.vec_cnttz()
4192}
4193
4194/// Vector Population Count
4195///
4196/// Computes the population count (number of set bits) in each element of the input.
4197#[inline]
4198#[target_feature(enable = "vector")]
4199#[unstable(feature = "stdarch_s390x", issue = "135681")]
4200pub unsafe fn vec_popcnt<T: sealed::CountBits>(a: T) -> T::Result {
4201    a.vec_popcnt()
4202}
4203
4204/// Vector Maximum
4205#[inline]
4206#[target_feature(enable = "vector")]
4207#[unstable(feature = "stdarch_s390x", issue = "135681")]
4208pub unsafe fn vec_max<T: sealed::VectorMax<U>, U>(a: T, b: U) -> T::Result {
4209    a.vec_max(b)
4210}
4211
4212/// Vector  Minimum
4213#[inline]
4214#[target_feature(enable = "vector")]
4215#[unstable(feature = "stdarch_s390x", issue = "135681")]
4216pub unsafe fn vec_min<T: sealed::VectorMin<U>, U>(a: T, b: U) -> T::Result {
4217    a.vec_min(b)
4218}
4219
4220/// Vector Absolute
4221#[inline]
4222#[target_feature(enable = "vector")]
4223#[unstable(feature = "stdarch_s390x", issue = "135681")]
4224pub unsafe fn vec_abs<T: sealed::VectorAbs>(a: T) -> T {
4225    a.vec_abs()
4226}
4227
4228/// Vector Negative Absolute
4229#[inline]
4230#[target_feature(enable = "vector")]
4231#[unstable(feature = "stdarch_s390x", issue = "135681")]
4232pub unsafe fn vec_nabs<T: sealed::VectorNabs>(a: T) -> T {
4233    a.vec_nabs()
4234}
4235
4236/// Vector Negative Multiply Add
4237#[inline]
4238#[target_feature(enable = "vector")]
4239#[unstable(feature = "stdarch_s390x", issue = "135681")]
4240pub unsafe fn vec_nmadd<T: sealed::VectorNmadd>(a: T, b: T, c: T) -> T {
4241    a.vec_nmadd(b, c)
4242}
4243
4244/// Vector Negative Multiply Subtract
4245#[inline]
4246#[target_feature(enable = "vector")]
4247#[unstable(feature = "stdarch_s390x", issue = "135681")]
4248pub unsafe fn vec_nmsub<T: sealed::VectorNmsub>(a: T, b: T, c: T) -> T {
4249    a.vec_nmsub(b, c)
4250}
4251
4252/// Vector Square Root
4253#[inline]
4254#[target_feature(enable = "vector")]
4255#[unstable(feature = "stdarch_s390x", issue = "135681")]
4256pub unsafe fn vec_sqrt<T: sealed::VectorSqrt>(a: T) -> T {
4257    a.vec_sqrt()
4258}
4259
4260/// Vector Splat
4261#[inline]
4262#[target_feature(enable = "vector")]
4263#[unstable(feature = "stdarch_s390x", issue = "135681")]
4264pub unsafe fn vec_splat<T: sealed::VectorSplat, const IMM: u32>(a: T) -> T {
4265    a.vec_splat::<IMM>()
4266}
4267
4268/// Vector Splats
4269#[inline]
4270#[target_feature(enable = "vector")]
4271#[unstable(feature = "stdarch_s390x", issue = "135681")]
4272pub unsafe fn vec_splats<T: sealed::VectorSplats<U>, U>(a: T) -> U {
4273    a.vec_splats()
4274}
4275
4276/// Vector AND
4277#[inline]
4278#[target_feature(enable = "vector")]
4279#[unstable(feature = "stdarch_s390x", issue = "135681")]
4280pub unsafe fn vec_and<T: sealed::VectorAnd<U>, U>(a: T, b: U) -> T::Result {
4281    a.vec_and(b)
4282}
4283
4284/// Vector OR
4285#[inline]
4286#[target_feature(enable = "vector")]
4287#[unstable(feature = "stdarch_s390x", issue = "135681")]
4288pub unsafe fn vec_or<T: sealed::VectorOr<U>, U>(a: T, b: U) -> T::Result {
4289    a.vec_or(b)
4290}
4291
4292/// Vector XOR
4293#[inline]
4294#[target_feature(enable = "vector")]
4295#[unstable(feature = "stdarch_s390x", issue = "135681")]
4296pub unsafe fn vec_xor<T: sealed::VectorXor<U>, U>(a: T, b: U) -> T::Result {
4297    a.vec_xor(b)
4298}
4299
4300/// Vector NOR
4301#[inline]
4302#[target_feature(enable = "vector")]
4303#[unstable(feature = "stdarch_s390x", issue = "135681")]
4304pub unsafe fn vec_nor<T: sealed::VectorNor<U>, U>(a: T, b: U) -> T::Result {
4305    a.vec_nor(b)
4306}
4307
4308/// Vector NAND
4309#[inline]
4310#[target_feature(enable = "vector")]
4311#[unstable(feature = "stdarch_s390x", issue = "135681")]
4312pub unsafe fn vec_nand<T: sealed::VectorNand<U>, U>(a: T, b: U) -> T::Result {
4313    a.vec_nand(b)
4314}
4315
4316/// Vector XNOR
4317#[inline]
4318#[target_feature(enable = "vector")]
4319#[unstable(feature = "stdarch_s390x", issue = "135681")]
4320pub unsafe fn vec_eqv<T: sealed::VectorEqv<U>, U>(a: T, b: U) -> T::Result {
4321    a.vec_eqv(b)
4322}
4323
4324/// Vector ANDC
4325#[inline]
4326#[target_feature(enable = "vector")]
4327#[unstable(feature = "stdarch_s390x", issue = "135681")]
4328pub unsafe fn vec_andc<T: sealed::VectorAndc<U>, U>(a: T, b: U) -> T::Result {
4329    a.vec_andc(b)
4330}
4331
4332/// Vector OR with Complement
4333///
4334/// ## Purpose
4335/// Performs a bitwise OR of the first vector with the bitwise-complemented second vector.
4336///
4337/// ## Result value
4338/// r is the bitwise OR of a and the bitwise complement of b.
4339#[inline]
4340#[target_feature(enable = "vector")]
4341#[unstable(feature = "stdarch_s390x", issue = "135681")]
4342pub unsafe fn vec_orc<T: sealed::VectorOrc<U>, U>(a: T, b: U) -> T::Result {
4343    a.vec_orc(b)
4344}
4345
4346/// Vector Floor
4347#[inline]
4348#[target_feature(enable = "vector")]
4349#[unstable(feature = "stdarch_s390x", issue = "135681")]
4350pub unsafe fn vec_floor<T: sealed::VectorFloor>(a: T) -> T {
4351    a.vec_floor()
4352}
4353
4354/// Vector Ceil
4355#[inline]
4356#[target_feature(enable = "vector")]
4357#[unstable(feature = "stdarch_s390x", issue = "135681")]
4358pub unsafe fn vec_ceil<T: sealed::VectorCeil>(a: T) -> T {
4359    a.vec_ceil()
4360}
4361
4362/// Vector Truncate
4363///
4364/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4365/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4366#[inline]
4367#[target_feature(enable = "vector")]
4368#[unstable(feature = "stdarch_s390x", issue = "135681")]
4369pub unsafe fn vec_trunc<T: sealed::VectorTrunc>(a: T) -> T {
4370    a.vec_trunc()
4371}
4372
4373/// Vector Round
4374///
4375/// Returns a vector containing the rounded values to the nearest representable floating-point integer,
4376/// using IEEE round-to-nearest rounding, of the corresponding elements of the given vector
4377#[inline]
4378#[target_feature(enable = "vector")]
4379#[unstable(feature = "stdarch_s390x", issue = "135681")]
4380pub unsafe fn vec_round<T: sealed::VectorRound>(a: T) -> T {
4381    a.vec_round()
4382}
4383
4384/// Vector Round to Current
4385///
4386/// Returns a vector by using the current rounding mode to round every
4387/// floating-point element in the given vector to integer.
4388#[inline]
4389#[target_feature(enable = "vector")]
4390#[unstable(feature = "stdarch_s390x", issue = "135681")]
4391pub unsafe fn vec_roundc<T: sealed::VectorRoundc>(a: T) -> T {
4392    a.vec_roundc()
4393}
4394
4395/// Vector Round toward Negative Infinity
4396///
4397/// Returns a vector containing the largest representable floating-point integral values less
4398/// than or equal to the values of the corresponding elements of the given vector.
4399#[inline]
4400#[target_feature(enable = "vector")]
4401#[unstable(feature = "stdarch_s390x", issue = "135681")]
4402pub unsafe fn vec_roundm<T: sealed::VectorFloor>(a: T) -> T {
4403    // the IBM docs note
4404    //
4405    // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception.
4406    //
4407    // but in practice `vec_floor` also does not trigger that exception, so both are equivalent
4408    a.vec_floor()
4409}
4410
4411/// Vector Round toward Positive Infinity
4412///
4413/// Returns a vector containing the smallest representable floating-point integral values greater
4414/// than or equal to the values of the corresponding elements of the given vector.
4415#[inline]
4416#[target_feature(enable = "vector")]
4417#[unstable(feature = "stdarch_s390x", issue = "135681")]
4418pub unsafe fn vec_roundp<T: sealed::VectorCeil>(a: T) -> T {
4419    // the IBM docs note
4420    //
4421    // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception.
4422    //
4423    // but in practice `vec_ceil` also does not trigger that exception, so both are equivalent
4424    a.vec_ceil()
4425}
4426
4427/// Vector Round toward Zero
4428///
4429/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4430/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4431#[inline]
4432#[target_feature(enable = "vector")]
4433#[unstable(feature = "stdarch_s390x", issue = "135681")]
4434pub unsafe fn vec_roundz<T: sealed::VectorTrunc>(a: T) -> T {
4435    // the IBM docs note
4436    //
4437    // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception.
4438    //
4439    // but in practice `vec_trunc` also does not trigger that exception, so both are equivalent
4440    a.vec_trunc()
4441}
4442
4443/// Vector Round to Integer
4444///
4445/// Returns a vector by using the current rounding mode to round every floating-point element in the given vector to integer.
4446#[inline]
4447#[target_feature(enable = "vector")]
4448#[unstable(feature = "stdarch_s390x", issue = "135681")]
4449pub unsafe fn vec_rint<T: sealed::VectorRint>(a: T) -> T {
4450    a.vec_rint()
4451}
4452
4453/// Vector Average
4454#[inline]
4455#[target_feature(enable = "vector")]
4456#[unstable(feature = "stdarch_s390x", issue = "135681")]
4457pub unsafe fn vec_avg<T: sealed::VectorAvg<U>, U>(a: T, b: U) -> T::Result {
4458    a.vec_avg(b)
4459}
4460
4461/// Vector Shift Left
4462#[inline]
4463#[target_feature(enable = "vector")]
4464#[unstable(feature = "stdarch_s390x", issue = "135681")]
4465pub unsafe fn vec_sl<T: sealed::VectorSl<U>, U>(a: T, b: U) -> T::Result {
4466    a.vec_sl(b)
4467}
4468
4469/// Vector Shift Right
4470#[inline]
4471#[target_feature(enable = "vector")]
4472#[unstable(feature = "stdarch_s390x", issue = "135681")]
4473pub unsafe fn vec_sr<T: sealed::VectorSr<U>, U>(a: T, b: U) -> T::Result {
4474    a.vec_sr(b)
4475}
4476
4477/// Vector Shift Right Algebraic
4478#[inline]
4479#[target_feature(enable = "vector")]
4480#[unstable(feature = "stdarch_s390x", issue = "135681")]
4481pub unsafe fn vec_sra<T: sealed::VectorSra<U>, U>(a: T, b: U) -> T::Result {
4482    a.vec_sra(b)
4483}
4484
4485/// Vector Shift Left by Byte
4486#[inline]
4487#[target_feature(enable = "vector")]
4488#[unstable(feature = "stdarch_s390x", issue = "135681")]
4489pub unsafe fn vec_slb<T: sealed::VectorSlb<U>, U>(a: T, b: U) -> T::Result {
4490    a.vec_slb(b)
4491}
4492
4493/// Vector Shift Right by Byte
4494#[inline]
4495#[target_feature(enable = "vector")]
4496#[unstable(feature = "stdarch_s390x", issue = "135681")]
4497pub unsafe fn vec_srb<T: sealed::VectorSrb<U>, U>(a: T, b: U) -> T::Result {
4498    a.vec_srb(b)
4499}
4500
4501/// Vector Shift Right Algebraic by Byte
4502#[inline]
4503#[target_feature(enable = "vector")]
4504#[unstable(feature = "stdarch_s390x", issue = "135681")]
4505pub unsafe fn vec_srab<T: sealed::VectorSrab<U>, U>(a: T, b: U) -> T::Result {
4506    a.vec_srab(b)
4507}
4508
4509/// Vector Element Rotate Left
4510#[inline]
4511#[target_feature(enable = "vector")]
4512#[unstable(feature = "stdarch_s390x", issue = "135681")]
4513pub unsafe fn vec_rl<T: sealed::VectorRl<U>, U>(a: T, b: U) -> T::Result {
4514    a.vec_rl(b)
4515}
4516
4517/// Vector Shift Left
4518///
4519/// Performs a left shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4520/// element of a left by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4521#[inline]
4522#[target_feature(enable = "vector")]
4523#[unstable(feature = "stdarch_s390x", issue = "135681")]
4524pub unsafe fn vec_sll<T>(a: T, b: vector_unsigned_char) -> T
4525where
4526    T: sealed::VectorSll<vector_unsigned_char, Result = T>,
4527{
4528    a.vec_sll(b)
4529}
4530
4531/// Vector Shift Right
4532///
4533/// Performs a right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4534/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4535#[inline]
4536#[target_feature(enable = "vector")]
4537#[unstable(feature = "stdarch_s390x", issue = "135681")]
4538pub unsafe fn vec_srl<T>(a: T, b: vector_unsigned_char) -> T
4539where
4540    T: sealed::VectorSrl<vector_unsigned_char, Result = T>,
4541{
4542    a.vec_srl(b)
4543}
4544
4545/// Vector Shift Right Arithmetic
4546///
4547/// Performs an algebraic right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4548/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by copies of
4549/// the most significant bit of the element of a.
4550#[inline]
4551#[target_feature(enable = "vector")]
4552#[unstable(feature = "stdarch_s390x", issue = "135681")]
4553pub unsafe fn vec_sral<T>(a: T, b: vector_unsigned_char) -> T
4554where
4555    T: sealed::VectorSral<vector_unsigned_char, Result = T>,
4556{
4557    a.vec_sral(b)
4558}
4559
4560/// Vector Element Rotate Left Immediate
4561///
4562/// Rotates each element of a vector left by a given number of bits. Each element of the result is obtained by rotating the corresponding element
4563/// of a left by the number of bits specified by b, modulo the number of bits in the element.
4564#[inline]
4565#[target_feature(enable = "vector")]
4566#[unstable(feature = "stdarch_s390x", issue = "135681")]
4567pub unsafe fn vec_rli<T: sealed::VectorRli>(a: T, bits: core::ffi::c_ulong) -> T {
4568    a.vec_rli(bits)
4569}
4570
4571/// Vector Reverse Elements
4572///
4573/// Returns a vector with the elements of the input vector in reversed order.
4574#[inline]
4575#[target_feature(enable = "vector")]
4576#[unstable(feature = "stdarch_s390x", issue = "135681")]
4577pub unsafe fn vec_reve<T: sealed::VectorReve>(a: T) -> T {
4578    a.vec_reve()
4579}
4580
4581/// Vector Byte Reverse
4582///
4583/// Returns a vector where each vector element contains the corresponding byte-reversed vector element of the input vector.
4584#[inline]
4585#[target_feature(enable = "vector")]
4586#[unstable(feature = "stdarch_s390x", issue = "135681")]
4587pub unsafe fn vec_revb<T: sealed::VectorRevb>(a: T) -> T {
4588    a.vec_revb()
4589}
4590
4591/// Vector Merge High
4592///
4593/// Merges the most significant ("high") halves of two vectors.
4594#[inline]
4595#[target_feature(enable = "vector")]
4596#[unstable(feature = "stdarch_s390x", issue = "135681")]
4597pub unsafe fn vec_mergeh<T: sealed::VectorMergeh>(a: T, b: T) -> T {
4598    a.vec_mergeh(b)
4599}
4600
4601/// Vector Merge Low
4602///
4603/// Merges the least significant ("low") halves of two vectors.
4604#[inline]
4605#[target_feature(enable = "vector")]
4606#[unstable(feature = "stdarch_s390x", issue = "135681")]
4607pub unsafe fn vec_mergel<T: sealed::VectorMergel>(a: T, b: T) -> T {
4608    a.vec_mergel(b)
4609}
4610
4611/// Vector Pack
4612#[inline]
4613#[target_feature(enable = "vector")]
4614#[unstable(feature = "stdarch_s390x", issue = "135681")]
4615pub unsafe fn vec_pack<T: sealed::VectorPack<U>, U>(a: T, b: U) -> T::Result {
4616    a.vec_pack(b)
4617}
4618
4619/// Vector Pack Saturated
4620#[inline]
4621#[target_feature(enable = "vector")]
4622#[unstable(feature = "stdarch_s390x", issue = "135681")]
4623pub unsafe fn vec_packs<T: sealed::VectorPacks<U>, U>(a: T, b: U) -> T::Result {
4624    a.vec_packs(b)
4625}
4626
4627/// Vector Pack Saturated Condition Code
4628#[inline]
4629#[target_feature(enable = "vector")]
4630#[unstable(feature = "stdarch_s390x", issue = "135681")]
4631pub unsafe fn vec_packs_cc<T: sealed::VectorPacksCC>(a: T, b: T) -> (T::Result, i32) {
4632    a.vec_packs_cc(b)
4633}
4634
4635/// Vector Pack Saturated Unsigned
4636#[inline]
4637#[target_feature(enable = "vector")]
4638#[unstable(feature = "stdarch_s390x", issue = "135681")]
4639pub unsafe fn vec_packsu<T: sealed::VectorPacksu<U>, U>(a: T, b: U) -> T::Result {
4640    a.vec_packsu(b)
4641}
4642
4643/// Vector Pack Saturated Unsigned Condition Code
4644#[inline]
4645#[target_feature(enable = "vector")]
4646#[unstable(feature = "stdarch_s390x", issue = "135681")]
4647pub unsafe fn vec_packsu_cc<T: sealed::VectorPacksuCC>(a: T, b: T) -> (T::Result, i32) {
4648    a.vec_packsu_cc(b)
4649}
4650
4651/// Vector Unpack High
4652#[inline]
4653#[target_feature(enable = "vector")]
4654#[unstable(feature = "stdarch_s390x", issue = "135681")]
4655pub unsafe fn vec_unpackh<T: sealed::VectorUnpackh>(a: T) -> <T as sealed::VectorUnpackh>::Result {
4656    a.vec_unpackh()
4657}
4658
4659/// Vector Unpack Low
4660#[inline]
4661#[target_feature(enable = "vector")]
4662#[unstable(feature = "stdarch_s390x", issue = "135681")]
4663pub unsafe fn vec_unpackl<T: sealed::VectorUnpackl>(a: T) -> <T as sealed::VectorUnpackl>::Result {
4664    a.vec_unpackl()
4665}
4666
4667/// Vector Generate Byte Mask
4668///
4669/// Generates byte masks for elements in the return vector. For each bit in a, if the bit is one, all bit positions
4670/// in the corresponding byte element of d are set to ones. Otherwise, if the bit is zero, the corresponding byte element is set to zero.
4671#[inline]
4672#[target_feature(enable = "vector")]
4673#[unstable(feature = "stdarch_s390x", issue = "135681")]
4674#[cfg_attr(test, assert_instr(vgbm, MASK = 0x00FF))]
4675pub unsafe fn vec_genmask<const MASK: u16>() -> vector_unsigned_char {
4676    vector_unsigned_char(const { genmask::<MASK>() })
4677}
4678
4679/// Vector Generate Mask (Byte)
4680#[inline]
4681#[target_feature(enable = "vector")]
4682#[unstable(feature = "stdarch_s390x", issue = "135681")]
4683#[cfg_attr(test, assert_instr(vrepib, L = 3, H = 5))]
4684pub unsafe fn vec_genmasks_8<const L: u8, const H: u8>() -> vector_unsigned_char {
4685    vector_unsigned_char(const { [genmasks(u8::BITS, L, H) as u8; 16] })
4686}
4687
4688/// Vector Generate Mask (Halfword)
4689#[inline]
4690#[target_feature(enable = "vector")]
4691#[unstable(feature = "stdarch_s390x", issue = "135681")]
4692#[cfg_attr(test, assert_instr(vrepih, L = 3, H = 5))]
4693pub unsafe fn vec_genmasks_16<const L: u8, const H: u8>() -> vector_unsigned_short {
4694    vector_unsigned_short(const { [genmasks(u16::BITS, L, H) as u16; 8] })
4695}
4696
4697/// Vector Generate Mask (Word)
4698#[inline]
4699#[target_feature(enable = "vector")]
4700#[unstable(feature = "stdarch_s390x", issue = "135681")]
4701#[cfg_attr(test, assert_instr(vgmf, L = 3, H = 5))]
4702pub unsafe fn vec_genmasks_32<const L: u8, const H: u8>() -> vector_unsigned_int {
4703    vector_unsigned_int(const { [genmasks(u32::BITS, L, H) as u32; 4] })
4704}
4705
4706/// Vector Generate Mask (Doubleword)
4707#[inline]
4708#[target_feature(enable = "vector")]
4709#[unstable(feature = "stdarch_s390x", issue = "135681")]
4710#[cfg_attr(test, assert_instr(vgmg, L = 3, H = 5))]
4711pub unsafe fn vec_genmasks_64<const L: u8, const H: u8>() -> vector_unsigned_long_long {
4712    vector_unsigned_long_long(const { [genmasks(u64::BITS, L, H); 2] })
4713}
4714
4715/// Vector Permute
4716///
4717/// Returns a vector that contains some elements of two vectors, in the order specified by a third vector.
4718/// Each byte of the result is selected by using the least significant 5 bits of the corresponding byte of c as an index into the concatenated bytes of a and b.
4719/// Note: The vector generate mask built-in function [`vec_genmask`] could help generate the mask c.
4720#[inline]
4721#[target_feature(enable = "vector")]
4722#[unstable(feature = "stdarch_s390x", issue = "135681")]
4723pub unsafe fn vec_perm<T: sealed::VectorPerm>(a: T, b: T, c: vector_unsigned_char) -> T {
4724    a.vec_perm(b, c)
4725}
4726
4727/// Vector Sum Across Quadword
4728///
4729/// Returns a vector containing the results of performing a sum across all the elements in each of the quadword of vector a,
4730/// and the rightmost word or doubleword element of the b. The result is an unsigned 128-bit integer.
4731#[inline]
4732#[target_feature(enable = "vector")]
4733#[unstable(feature = "stdarch_s390x", issue = "135681")]
4734pub unsafe fn vec_sum_u128<T: sealed::VectorSumU128>(a: T, b: T) -> vector_unsigned_char {
4735    a.vec_sum_u128(b)
4736}
4737
4738/// Vector Sum Across Doubleword
4739///
4740/// Returns a vector containing the results of performing a sum across all the elements in each of the doubleword of vector a,
4741/// and the rightmost sub-element of the corresponding doubleword of b.
4742#[inline]
4743#[target_feature(enable = "vector")]
4744#[unstable(feature = "stdarch_s390x", issue = "135681")]
4745pub unsafe fn vec_sum2<T: sealed::VectorSum2>(a: T, b: T) -> vector_unsigned_long_long {
4746    a.vec_sum2(b)
4747}
4748
4749/// Vector Sum Across Word
4750///
4751/// Returns a vector containing the results of performing a sum across all the elements in each of the word of vector a,
4752/// and the rightmost sub-element of the corresponding word of b.
4753#[inline]
4754#[target_feature(enable = "vector")]
4755#[unstable(feature = "stdarch_s390x", issue = "135681")]
4756pub unsafe fn vec_sum4<T: sealed::VectorSum4>(a: T, b: T) -> vector_unsigned_int {
4757    a.vec_sum4(b)
4758}
4759
4760/// Vector Addition unsigned 128-bits
4761///
4762/// Adds unsigned quadword values.
4763///
4764/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a + b.
4765#[inline]
4766#[target_feature(enable = "vector")]
4767#[unstable(feature = "stdarch_s390x", issue = "135681")]
4768#[cfg_attr(test, assert_instr(vaq))]
4769pub unsafe fn vec_add_u128(
4770    a: vector_unsigned_char,
4771    b: vector_unsigned_char,
4772) -> vector_unsigned_char {
4773    let a: u128 = transmute(a);
4774    let b: u128 = transmute(b);
4775    transmute(a.wrapping_add(b))
4776}
4777
4778/// Vector Subtract unsigned 128-bits
4779///
4780/// Subtracts unsigned quadword values.
4781///
4782/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a - b.
4783#[inline]
4784#[target_feature(enable = "vector")]
4785#[unstable(feature = "stdarch_s390x", issue = "135681")]
4786#[cfg_attr(test, assert_instr(vsq))]
4787pub unsafe fn vec_sub_u128(
4788    a: vector_unsigned_char,
4789    b: vector_unsigned_char,
4790) -> vector_unsigned_char {
4791    let a: u128 = transmute(a);
4792    let b: u128 = transmute(b);
4793
4794    transmute(a.wrapping_sub(b))
4795}
4796
4797/// Vector Subtract Carryout
4798///
4799/// Returns a vector containing the borrow produced by subtracting each of corresponding elements of b from a.
4800///
4801/// On each resulting element, the value is 0 if a borrow occurred, or 1 if no borrow occurred.
4802#[inline]
4803#[target_feature(enable = "vector")]
4804#[unstable(feature = "stdarch_s390x", issue = "135681")]
4805pub unsafe fn vec_subc<T: sealed::VectorSubc<U>, U>(a: T, b: U) -> T::Result {
4806    a.vec_subc(b)
4807}
4808
4809/// Vector Subtract Carryout unsigned 128-bits
4810///
4811/// Gets the carry bit of the 128-bit subtraction of two quadword values.
4812/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the borrow produced by subtracting b from a, as unsigned 128-bits integers.
4813/// If no borrow occurred, the bit 127 of d is 1; otherwise it is set to 0. All other bits of d are 0.
4814#[inline]
4815#[target_feature(enable = "vector")]
4816#[unstable(feature = "stdarch_s390x", issue = "135681")]
4817#[cfg_attr(test, assert_instr(vscbiq))]
4818pub unsafe fn vec_subc_u128(
4819    a: vector_unsigned_char,
4820    b: vector_unsigned_char,
4821) -> vector_unsigned_char {
4822    let a: u128 = transmute(a);
4823    let b: u128 = transmute(b);
4824    transmute(!a.overflowing_sub(b).1 as u128)
4825}
4826
4827/// Vector Add Compute Carryout unsigned 128-bits
4828#[inline]
4829#[target_feature(enable = "vector")]
4830#[unstable(feature = "stdarch_s390x", issue = "135681")]
4831#[cfg_attr(test, assert_instr(vaccq))]
4832pub unsafe fn vec_addc_u128(
4833    a: vector_unsigned_char,
4834    b: vector_unsigned_char,
4835) -> vector_unsigned_char {
4836    let a: u128 = transmute(a);
4837    let b: u128 = transmute(b);
4838    // FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557
4839    // transmute(a.overflowing_add(b).1 as u128)
4840    transmute(vaccq(a, b))
4841}
4842
4843/// Vector Add With Carry unsigned 128-bits
4844#[inline]
4845#[target_feature(enable = "vector")]
4846#[unstable(feature = "stdarch_s390x", issue = "135681")]
4847#[cfg_attr(test, assert_instr(vacq))]
4848pub unsafe fn vec_adde_u128(
4849    a: vector_unsigned_char,
4850    b: vector_unsigned_char,
4851    c: vector_unsigned_char,
4852) -> vector_unsigned_char {
4853    let a: u128 = transmute(a);
4854    let b: u128 = transmute(b);
4855    let c: u128 = transmute(c);
4856    // FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557
4857    //     let (d, _carry) = a.carrying_add(b, c & 1 != 0);
4858    //     transmute(d)
4859    transmute(vacq(a, b, c))
4860}
4861
4862/// Vector Add With Carry Compute Carry unsigned 128-bits
4863#[inline]
4864#[target_feature(enable = "vector")]
4865#[unstable(feature = "stdarch_s390x", issue = "135681")]
4866#[cfg_attr(test, assert_instr(vacccq))]
4867pub unsafe fn vec_addec_u128(
4868    a: vector_unsigned_char,
4869    b: vector_unsigned_char,
4870    c: vector_unsigned_char,
4871) -> vector_unsigned_char {
4872    let a: u128 = transmute(a);
4873    let b: u128 = transmute(b);
4874    let c: u128 = transmute(c);
4875    // FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557
4876    // let (_d, carry) = a.carrying_add(b, c & 1 != 0);
4877    // transmute(carry as u128)
4878    transmute(vacccq(a, b, c))
4879}
4880
4881/// Vector Subtract with Carryout
4882///
4883/// Subtracts unsigned quadword values with carry bit from a previous operation.
4884///
4885/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the result of subtracting of b from a,
4886/// and the carryout bit from a previous operation.
4887///
4888/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4889#[inline]
4890#[target_feature(enable = "vector")]
4891#[unstable(feature = "stdarch_s390x", issue = "135681")]
4892#[cfg_attr(test, assert_instr(vsbiq))]
4893pub unsafe fn vec_sube_u128(
4894    a: vector_unsigned_char,
4895    b: vector_unsigned_char,
4896    c: vector_unsigned_char,
4897) -> vector_unsigned_char {
4898    transmute(vsbiq(transmute(a), transmute(b), transmute(c)))
4899}
4900
4901/// Vector Subtract with Carryout, Carryout
4902///
4903/// Gets the carry bit of the 128-bit subtraction of two quadword values with carry bit from the previous operation.
4904///
4905/// It returns a vector containing the carryout produced from the result of subtracting of b from a,
4906/// and the carryout bit from a previous operation. If no borrow occurred, the 127-bit of d is 1, otherwise 0.
4907/// All other bits of d are 0.
4908///
4909/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4910#[inline]
4911#[target_feature(enable = "vector")]
4912#[unstable(feature = "stdarch_s390x", issue = "135681")]
4913#[cfg_attr(test, assert_instr(vsbcbiq))]
4914pub unsafe fn vec_subec_u128(
4915    a: vector_unsigned_char,
4916    b: vector_unsigned_char,
4917    c: vector_unsigned_char,
4918) -> vector_unsigned_char {
4919    transmute(vsbcbiq(transmute(a), transmute(b), transmute(c)))
4920}
4921
4922/// Vector Splat Signed Byte
4923#[inline]
4924#[target_feature(enable = "vector")]
4925#[unstable(feature = "stdarch_s390x", issue = "135681")]
4926#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4927pub unsafe fn vec_splat_s8<const IMM: i8>() -> vector_signed_char {
4928    vector_signed_char([IMM; 16])
4929}
4930
4931/// Vector Splat Signed Halfword
4932#[inline]
4933#[target_feature(enable = "vector")]
4934#[unstable(feature = "stdarch_s390x", issue = "135681")]
4935#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4936pub unsafe fn vec_splat_s16<const IMM: i16>() -> vector_signed_short {
4937    vector_signed_short([IMM; 8])
4938}
4939
4940/// Vector Splat Signed Word
4941#[inline]
4942#[target_feature(enable = "vector")]
4943#[unstable(feature = "stdarch_s390x", issue = "135681")]
4944#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4945pub unsafe fn vec_splat_s32<const IMM: i16>() -> vector_signed_int {
4946    vector_signed_int([IMM as i32; 4])
4947}
4948
4949/// Vector Splat Signed Doubleword
4950#[inline]
4951#[target_feature(enable = "vector")]
4952#[unstable(feature = "stdarch_s390x", issue = "135681")]
4953#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4954pub unsafe fn vec_splat_s64<const IMM: i16>() -> vector_signed_long_long {
4955    vector_signed_long_long([IMM as i64; 2])
4956}
4957
4958/// Vector Splat Unsigned Byte
4959#[inline]
4960#[target_feature(enable = "vector")]
4961#[unstable(feature = "stdarch_s390x", issue = "135681")]
4962#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4963pub unsafe fn vec_splat_u8<const IMM: u8>() -> vector_unsigned_char {
4964    vector_unsigned_char([IMM; 16])
4965}
4966
4967/// Vector Splat Unsigned Halfword
4968#[inline]
4969#[target_feature(enable = "vector")]
4970#[unstable(feature = "stdarch_s390x", issue = "135681")]
4971#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4972pub unsafe fn vec_splat_u16<const IMM: i16>() -> vector_unsigned_short {
4973    vector_unsigned_short([IMM as u16; 8])
4974}
4975
4976/// Vector Splat Unsigned Word
4977#[inline]
4978#[target_feature(enable = "vector")]
4979#[unstable(feature = "stdarch_s390x", issue = "135681")]
4980#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4981pub unsafe fn vec_splat_u32<const IMM: i16>() -> vector_unsigned_int {
4982    vector_unsigned_int([IMM as u32; 4])
4983}
4984
4985/// Vector Splat Unsigned Doubleword
4986#[inline]
4987#[target_feature(enable = "vector")]
4988#[unstable(feature = "stdarch_s390x", issue = "135681")]
4989#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4990pub unsafe fn vec_splat_u64<const IMM: i16>() -> vector_unsigned_long_long {
4991    vector_unsigned_long_long([IMM as u64; 2])
4992}
4993
4994macro_rules! vec_find_any {
4995    ($($Trait:ident $fun:ident $doc:literal)*) => {
4996        $(
4997            #[inline]
4998            #[target_feature(enable = "vector")]
4999            #[unstable(feature = "stdarch_s390x", issue = "135681")]
5000            #[doc = $doc]
5001            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> T::Result {
5002                a.$fun(b)
5003            }
5004        )*
5005    }
5006}
5007
5008vec_find_any! {
5009    VectorFindAnyEq vec_find_any_eq "Vector Find Any Element Equal with Condition Code"
5010    VectorFindAnyNe vec_find_any_ne "Vector Find Any Element Not Equal with Condition Code"
5011    VectorFindAnyEqIdx vec_find_any_eq_idx "Vector Find Any Element Equal Index with Condition Code"
5012    VectorFindAnyNeIdx vec_find_any_ne_idx "Vector Find Any Element Not Equal Index with Condition Code"
5013    VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx "Vector Find Any Element Equal or Zero Index with Condition Code"
5014    VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx "Vector Find Any Element Not Equal or Zero Index with Condition Code"
5015}
5016
5017macro_rules! vec_find_any_cc {
5018    ($($Trait:ident $fun:ident $doc:literal)*) => {
5019        $(
5020            #[inline]
5021            #[target_feature(enable = "vector")]
5022            #[unstable(feature = "stdarch_s390x", issue = "135681")]
5023            #[doc = $doc]
5024            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> (T::Result, i32) {
5025                a.$fun(b)
5026            }
5027        )*
5028    }
5029}
5030
5031vec_find_any_cc! {
5032    VectorFindAnyEqCC vec_find_any_eq_cc "Vector Find Any Element Equal with Condition Code"
5033    VectorFindAnyNeCC vec_find_any_ne_cc "Vector Find Any Element Not Equal with Condition Code"
5034    VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc "Vector Find Any Element Equal Index with Condition Code"
5035    VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc "Vector Find Any Element Not Equal Index with Condition Code"
5036    VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc "Vector Find Any Element Equal or Zero Index with Condition Code"
5037    VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc "Vector Find Any Element Not Equal or Zero Index with Condition Code"
5038}
5039
5040/// Vector Load
5041#[inline]
5042#[target_feature(enable = "vector")]
5043#[unstable(feature = "stdarch_s390x", issue = "135681")]
5044pub unsafe fn vec_xl<T: sealed::VectorLoad>(offset: isize, ptr: *const T::ElementType) -> T {
5045    T::vec_xl(offset, ptr)
5046}
5047
5048/// Vector Load Pair
5049#[inline]
5050#[target_feature(enable = "vector")]
5051#[unstable(feature = "stdarch_s390x", issue = "135681")]
5052pub unsafe fn vec_load_pair<T: sealed::VectorLoadPair>(a: T::ElementType, b: T::ElementType) -> T {
5053    T::vec_load_pair(a, b)
5054}
5055
5056/// Vector Load to Block Boundary
5057#[inline]
5058#[target_feature(enable = "vector")]
5059#[unstable(feature = "stdarch_s390x", issue = "135681")]
5060pub unsafe fn vec_load_bndry<T: sealed::VectorLoad, const BLOCK_BOUNDARY: u16>(
5061    ptr: *const T::ElementType,
5062) -> MaybeUninit<T> {
5063    T::vec_load_bndry::<BLOCK_BOUNDARY>(ptr)
5064}
5065
5066/// Vector Store
5067#[inline]
5068#[target_feature(enable = "vector")]
5069#[unstable(feature = "stdarch_s390x", issue = "135681")]
5070pub unsafe fn vec_xst<T: sealed::VectorStore>(vector: T, offset: isize, ptr: *mut T::ElementType) {
5071    vector.vec_xst(offset, ptr)
5072}
5073
5074/// Vector Load with Length
5075#[inline]
5076#[target_feature(enable = "vector")]
5077#[unstable(feature = "stdarch_s390x", issue = "135681")]
5078pub unsafe fn vec_load_len<T: sealed::VectorLoad>(
5079    ptr: *const T::ElementType,
5080    byte_count: u32,
5081) -> T {
5082    T::vec_load_len(ptr, byte_count)
5083}
5084
5085/// Vector Store with Length
5086#[inline]
5087#[target_feature(enable = "vector")]
5088#[unstable(feature = "stdarch_s390x", issue = "135681")]
5089pub unsafe fn vec_store_len<T: sealed::VectorStore>(
5090    vector: T,
5091    ptr: *mut T::ElementType,
5092    byte_count: u32,
5093) {
5094    vector.vec_store_len(ptr, byte_count)
5095}
5096
5097/// Vector Load Rightmost with Length
5098#[inline]
5099#[target_feature(enable = "vector-packed-decimal")]
5100#[unstable(feature = "stdarch_s390x", issue = "135681")]
5101#[cfg_attr(test, assert_instr(vlrlr))]
5102pub unsafe fn vec_load_len_r(ptr: *const u8, byte_count: u32) -> vector_unsigned_char {
5103    vlrl(byte_count, ptr)
5104}
5105
5106/// Vector Store Rightmost with Length
5107#[inline]
5108#[target_feature(enable = "vector-packed-decimal")]
5109#[unstable(feature = "stdarch_s390x", issue = "135681")]
5110#[cfg_attr(test, assert_instr(vstrlr))]
5111pub unsafe fn vec_store_len_r(vector: vector_unsigned_char, ptr: *mut u8, byte_count: u32) {
5112    vstrl(vector, byte_count, ptr)
5113}
5114
5115/// Vector Multiply Add
5116#[inline]
5117#[target_feature(enable = "vector-packed-decimal")]
5118#[unstable(feature = "stdarch_s390x", issue = "135681")]
5119pub unsafe fn vec_madd<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
5120    a.vec_madd(b, c)
5121}
5122
5123/// Vector Multiply Add
5124#[inline]
5125#[target_feature(enable = "vector-packed-decimal")]
5126#[unstable(feature = "stdarch_s390x", issue = "135681")]
5127pub unsafe fn vec_msub<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
5128    a.vec_msub(b, c)
5129}
5130
5131/// Vector Multiply and Add Even
5132#[inline]
5133#[target_feature(enable = "vector-packed-decimal")]
5134#[unstable(feature = "stdarch_s390x", issue = "135681")]
5135pub unsafe fn vec_meadd<T: sealed::VectorMeadd>(a: T, b: T, c: T::Result) -> T::Result {
5136    a.vec_meadd(b, c)
5137}
5138
5139/// Vector Multiply and Add Odd
5140#[inline]
5141#[target_feature(enable = "vector-packed-decimal")]
5142#[unstable(feature = "stdarch_s390x", issue = "135681")]
5143pub unsafe fn vec_moadd<T: sealed::VectorMoadd>(a: T, b: T, c: T::Result) -> T::Result {
5144    a.vec_moadd(b, c)
5145}
5146
5147/// Vector Multiply and Add High
5148#[inline]
5149#[target_feature(enable = "vector-packed-decimal")]
5150#[unstable(feature = "stdarch_s390x", issue = "135681")]
5151pub unsafe fn vec_mhadd<T: sealed::VectorMhadd>(a: T, b: T, c: T::Result) -> T::Result {
5152    a.vec_mhadd(b, c)
5153}
5154
5155/// Vector Multiply and Add Low
5156#[inline]
5157#[target_feature(enable = "vector-packed-decimal")]
5158#[unstable(feature = "stdarch_s390x", issue = "135681")]
5159pub unsafe fn vec_mladd<T: sealed::VectorMladd>(a: T, b: T, c: T::Result) -> T::Result {
5160    a.vec_mladd(b, c)
5161}
5162
5163/// Vector Checksum
5164#[inline]
5165#[target_feature(enable = "vector")]
5166#[unstable(feature = "stdarch_s390x", issue = "135681")]
5167#[cfg_attr(test, assert_instr(vcksm))]
5168pub unsafe fn vec_checksum(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int {
5169    vcksm(a, b)
5170}
5171
5172/// Vector Multiply Even
5173#[inline]
5174#[target_feature(enable = "vector")]
5175#[unstable(feature = "stdarch_s390x", issue = "135681")]
5176pub unsafe fn vec_mule<T: sealed::VectorMule<U>, U>(a: T, b: T) -> U {
5177    a.vec_mule(b)
5178}
5179
5180/// Vector Multiply Odd
5181#[inline]
5182#[target_feature(enable = "vector")]
5183#[unstable(feature = "stdarch_s390x", issue = "135681")]
5184pub unsafe fn vec_mulo<T: sealed::VectorMulo<U>, U>(a: T, b: T) -> U {
5185    a.vec_mulo(b)
5186}
5187
5188/// Vector Multiply High
5189#[inline]
5190#[target_feature(enable = "vector")]
5191#[unstable(feature = "stdarch_s390x", issue = "135681")]
5192pub unsafe fn vec_mulh<T: sealed::VectorMulh<U>, U>(a: T, b: T) -> U {
5193    a.vec_mulh(b)
5194}
5195
5196/// Vector Galois Field Multiply Sum
5197#[inline]
5198#[target_feature(enable = "vector")]
5199#[unstable(feature = "stdarch_s390x", issue = "135681")]
5200pub unsafe fn vec_gfmsum<T: sealed::VectorGfmsum<U>, U>(a: T, b: T) -> U {
5201    a.vec_gfmsum(b)
5202}
5203
5204/// Vector Galois Field Multiply Sum
5205#[inline]
5206#[target_feature(enable = "vector")]
5207#[unstable(feature = "stdarch_s390x", issue = "135681")]
5208pub unsafe fn vec_gfmsum_accum<T: sealed::VectorGfmsumAccum>(
5209    a: T,
5210    b: T,
5211    c: T::Result,
5212) -> T::Result {
5213    a.vec_gfmsum_accum(b, c)
5214}
5215
5216/// Vector Galois Field Multiply Sum 128-bits
5217#[inline]
5218#[target_feature(enable = "vector")]
5219#[unstable(feature = "stdarch_s390x", issue = "135681")]
5220#[cfg_attr(test, assert_instr(vgfmg))]
5221pub unsafe fn vec_gfmsum_128(
5222    a: vector_unsigned_long_long,
5223    b: vector_unsigned_long_long,
5224) -> vector_unsigned_char {
5225    transmute(vgfmg(a, b))
5226}
5227
5228/// Vector Galois Field Multiply Sum and Accumulate 128-bits
5229#[inline]
5230#[target_feature(enable = "vector")]
5231#[unstable(feature = "stdarch_s390x", issue = "135681")]
5232#[cfg_attr(test, assert_instr(vgfmag))]
5233pub unsafe fn vec_gfmsum_accum_128(
5234    a: vector_unsigned_long_long,
5235    b: vector_unsigned_long_long,
5236    c: vector_unsigned_char,
5237) -> vector_unsigned_char {
5238    transmute(vgfmag(a, b, transmute(c)))
5239}
5240
5241/// Vector Bit Permute
5242#[inline]
5243#[target_feature(enable = "vector-enhancements-1")]
5244#[unstable(feature = "stdarch_s390x", issue = "135681")]
5245#[cfg_attr(test, assert_instr(vbperm))]
5246pub unsafe fn vec_bperm_u128(
5247    a: vector_unsigned_char,
5248    b: vector_unsigned_char,
5249) -> vector_unsigned_long_long {
5250    vbperm(a, b)
5251}
5252
5253/// Vector Gather Element
5254#[inline]
5255#[target_feature(enable = "vector")]
5256#[unstable(feature = "stdarch_s390x", issue = "135681")]
5257pub unsafe fn vec_gather_element<T: sealed::VectorGatherElement, const D: u32>(
5258    a: T,
5259    b: T::Offset,
5260    c: *const T::Element,
5261) -> T {
5262    a.vec_gather_element::<D>(b, c)
5263}
5264
5265/// Vector Select
5266#[inline]
5267#[target_feature(enable = "vector")]
5268#[unstable(feature = "stdarch_s390x", issue = "135681")]
5269pub unsafe fn vec_sel<T: sealed::VectorSel<U>, U>(a: T, b: T, c: U) -> T {
5270    a.vec_sel(b, c)
5271}
5272
5273#[unstable(feature = "stdarch_s390x", issue = "135681")]
5274pub const __VEC_CLASS_FP_ZERO_P: u32 = 1 << 11;
5275#[unstable(feature = "stdarch_s390x", issue = "135681")]
5276pub const __VEC_CLASS_FP_ZERO_N: u32 = 1 << 10;
5277#[unstable(feature = "stdarch_s390x", issue = "135681")]
5278pub const __VEC_CLASS_FP_ZERO: u32 = __VEC_CLASS_FP_ZERO_P | __VEC_CLASS_FP_ZERO_N;
5279#[unstable(feature = "stdarch_s390x", issue = "135681")]
5280pub const __VEC_CLASS_FP_NORMAL_P: u32 = 1 << 9;
5281#[unstable(feature = "stdarch_s390x", issue = "135681")]
5282pub const __VEC_CLASS_FP_NORMAL_N: u32 = 1 << 8;
5283#[unstable(feature = "stdarch_s390x", issue = "135681")]
5284pub const __VEC_CLASS_FP_NORMAL: u32 = __VEC_CLASS_FP_NORMAL_P | __VEC_CLASS_FP_NORMAL_N;
5285#[unstable(feature = "stdarch_s390x", issue = "135681")]
5286pub const __VEC_CLASS_FP_SUBNORMAL_P: u32 = 1 << 7;
5287#[unstable(feature = "stdarch_s390x", issue = "135681")]
5288pub const __VEC_CLASS_FP_SUBNORMAL_N: u32 = 1 << 6;
5289#[unstable(feature = "stdarch_s390x", issue = "135681")]
5290pub const __VEC_CLASS_FP_SUBNORMAL: u32 = __VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N;
5291#[unstable(feature = "stdarch_s390x", issue = "135681")]
5292pub const __VEC_CLASS_FP_INFINITY_P: u32 = 1 << 5;
5293#[unstable(feature = "stdarch_s390x", issue = "135681")]
5294pub const __VEC_CLASS_FP_INFINITY_N: u32 = 1 << 4;
5295#[unstable(feature = "stdarch_s390x", issue = "135681")]
5296pub const __VEC_CLASS_FP_INFINITY: u32 = __VEC_CLASS_FP_INFINITY_P | __VEC_CLASS_FP_INFINITY_N;
5297#[unstable(feature = "stdarch_s390x", issue = "135681")]
5298pub const __VEC_CLASS_FP_QNAN_P: u32 = 1 << 3;
5299#[unstable(feature = "stdarch_s390x", issue = "135681")]
5300pub const __VEC_CLASS_FP_QNAN_N: u32 = 1 << 2;
5301#[unstable(feature = "stdarch_s390x", issue = "135681")]
5302pub const __VEC_CLASS_FP_QNAN: u32 = __VEC_CLASS_FP_QNAN_P | __VEC_CLASS_FP_QNAN_N;
5303#[unstable(feature = "stdarch_s390x", issue = "135681")]
5304pub const __VEC_CLASS_FP_SNAN_P: u32 = 1 << 1;
5305#[unstable(feature = "stdarch_s390x", issue = "135681")]
5306pub const __VEC_CLASS_FP_SNAN_N: u32 = 1 << 0;
5307#[unstable(feature = "stdarch_s390x", issue = "135681")]
5308pub const __VEC_CLASS_FP_SNAN: u32 = __VEC_CLASS_FP_SNAN_P | __VEC_CLASS_FP_SNAN_N;
5309#[unstable(feature = "stdarch_s390x", issue = "135681")]
5310pub const __VEC_CLASS_FP_NAN: u32 = __VEC_CLASS_FP_QNAN | __VEC_CLASS_FP_SNAN;
5311#[unstable(feature = "stdarch_s390x", issue = "135681")]
5312pub const __VEC_CLASS_FP_NOT_NORMAL: u32 =
5313    __VEC_CLASS_FP_NAN | __VEC_CLASS_FP_SUBNORMAL | __VEC_CLASS_FP_ZERO | __VEC_CLASS_FP_INFINITY;
5314
5315/// Vector Floating-Point Test Data Class
5316///
5317/// You can use the `__VEC_CLASS_FP_*` constants as the argument for this operand
5318#[inline]
5319#[target_feature(enable = "vector")]
5320#[unstable(feature = "stdarch_s390x", issue = "135681")]
5321pub unsafe fn vec_fp_test_data_class<T: sealed::VectorFpTestDataClass, const CLASS: u32>(
5322    a: T,
5323    c: *mut i32,
5324) -> T::Result {
5325    let (x, y) = a.vec_fp_test_data_class::<CLASS>();
5326    c.write(y);
5327    x
5328}
5329
5330/// All Elements Not a Number
5331#[inline]
5332#[target_feature(enable = "vector")]
5333#[unstable(feature = "stdarch_s390x", issue = "135681")]
5334pub unsafe fn vec_all_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5335    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 0)
5336}
5337
5338/// All Elements Numeric
5339#[inline]
5340#[target_feature(enable = "vector")]
5341#[unstable(feature = "stdarch_s390x", issue = "135681")]
5342pub unsafe fn vec_all_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5343    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 3)
5344}
5345
5346/// Any Elements Not a Number
5347#[inline]
5348#[target_feature(enable = "vector")]
5349#[unstable(feature = "stdarch_s390x", issue = "135681")]
5350pub unsafe fn vec_any_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5351    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 3)
5352}
5353
5354/// Any Elements Numeric
5355#[inline]
5356#[target_feature(enable = "vector")]
5357#[unstable(feature = "stdarch_s390x", issue = "135681")]
5358pub unsafe fn vec_any_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5359    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 0)
5360}
5361
5362/// Vector Test under Mask
5363#[inline]
5364#[target_feature(enable = "vector")]
5365#[unstable(feature = "stdarch_s390x", issue = "135681")]
5366pub unsafe fn vec_test_mask<T: sealed::VectorTestMask>(a: T, b: T::Mask) -> i32 {
5367    // I can't find much information about this, but this might just be a check for whether the
5368    // bitwise and of a and b is non-zero?
5369    a.vec_test_mask(b)
5370}
5371
5372/// Vector Search String
5373#[inline]
5374#[target_feature(enable = "vector")]
5375#[unstable(feature = "stdarch_s390x", issue = "135681")]
5376pub unsafe fn vec_search_string_cc<T: sealed::VectorSearchString>(
5377    a: T,
5378    b: T,
5379    c: vector_unsigned_char,
5380) -> (vector_unsigned_char, i32) {
5381    a.vec_search_string_cc(b, c)
5382}
5383
5384/// Vector Search String Until Zero
5385#[inline]
5386#[target_feature(enable = "vector")]
5387#[unstable(feature = "stdarch_s390x", issue = "135681")]
5388pub unsafe fn vec_search_string_until_zero_cc<T: sealed::VectorSearchString>(
5389    a: T,
5390    b: T,
5391    c: vector_unsigned_char,
5392) -> (vector_unsigned_char, i32) {
5393    a.vec_search_string_until_zero_cc(b, c)
5394}
5395
5396/// Vector Convert from float (even elements) to double
5397#[inline]
5398#[target_feature(enable = "vector-enhancements-1")]
5399#[unstable(feature = "stdarch_s390x", issue = "135681")]
5400// NOTE: `vflls` and `vldeb` are equivalent; our disassmbler prefers vflls.
5401#[cfg_attr(
5402    all(test, target_feature = "vector-enhancements-1"),
5403    assert_instr(vflls)
5404)]
5405pub unsafe fn vec_doublee(a: vector_float) -> vector_double {
5406    simd_as::<f32x2, vector_double>(simd_shuffle!(a, a, [0, 2]))
5407}
5408
5409/// Vector Convert from double to float (even elements)
5410#[inline]
5411#[target_feature(enable = "vector-enhancements-1")]
5412#[unstable(feature = "stdarch_s390x", issue = "135681")]
5413// FIXME: the C version uses a shuffle mask with poison; we can't do that
5414// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vledb))]
5415pub unsafe fn vec_floate(a: vector_double) -> vector_float {
5416    let truncated: f32x2 = simd_as(a);
5417    simd_shuffle!(truncated, truncated, [0, 0, 1, 1])
5418}
5419
5420/// Vector Convert from int to float
5421#[inline]
5422#[target_feature(enable = "vector")]
5423#[unstable(feature = "stdarch_s390x", issue = "135681")]
5424pub unsafe fn vec_float(a: impl sealed::VectorFloat) -> vector_float {
5425    a.vec_float()
5426}
5427
5428/// Vector Convert from long long to double
5429#[inline]
5430#[target_feature(enable = "vector")]
5431#[unstable(feature = "stdarch_s390x", issue = "135681")]
5432pub unsafe fn vec_double(a: impl sealed::VectorDouble) -> vector_double {
5433    a.vec_double()
5434}
5435
5436/// Vector Sign Extend to Doubleword
5437#[inline]
5438#[target_feature(enable = "vector")]
5439#[unstable(feature = "stdarch_s390x", issue = "135681")]
5440pub unsafe fn vec_extend_s64(a: impl sealed::VectorExtendSigned64) -> vector_signed_long_long {
5441    a.vec_extend_s64()
5442}
5443
5444/// Vector Convert floating point to signed
5445#[inline]
5446#[target_feature(enable = "vector")]
5447#[unstable(feature = "stdarch_s390x", issue = "135681")]
5448pub unsafe fn vec_signed<T: sealed::VectorSigned>(a: T) -> T::Result {
5449    a.vec_signed()
5450}
5451
5452/// Vector Convert floating point to unsigned
5453#[inline]
5454#[target_feature(enable = "vector")]
5455#[unstable(feature = "stdarch_s390x", issue = "135681")]
5456pub unsafe fn vec_unsigned<T: sealed::VectorUnsigned>(a: T) -> T::Result {
5457    a.vec_unsigned()
5458}
5459
5460/// Vector Copy Until Zero
5461#[inline]
5462#[target_feature(enable = "vector")]
5463#[unstable(feature = "stdarch_s390x", issue = "135681")]
5464pub unsafe fn vec_cp_until_zero<T: sealed::VectorCopyUntilZero>(a: T) -> T {
5465    a.vec_cp_until_zero()
5466}
5467
5468/// Vector Copy Until Zero
5469#[inline]
5470#[target_feature(enable = "vector")]
5471#[unstable(feature = "stdarch_s390x", issue = "135681")]
5472pub unsafe fn vec_cp_until_zero_cc<T: sealed::VectorCopyUntilZeroCC>(a: T) -> (T, i32) {
5473    a.vec_cp_until_zero_cc()
5474}
5475
5476/// Vector Multiply Sum Logical
5477#[inline]
5478#[target_feature(enable = "vector-enhancements-1")]
5479#[unstable(feature = "stdarch_s390x", issue = "135681")]
5480#[cfg_attr(
5481    all(test, target_feature = "vector-enhancements-1"),
5482    assert_instr(vmslg, D = 4)
5483)]
5484pub unsafe fn vec_msum_u128<const D: u32>(
5485    a: vector_unsigned_long_long,
5486    b: vector_unsigned_long_long,
5487    c: vector_unsigned_char,
5488) -> vector_unsigned_char {
5489    const {
5490        if !matches!(D, 0 | 4 | 8 | 12) {
5491            panic!("D needs to be one of 0, 4, 8, 12");
5492        }
5493    };
5494    transmute(vmslg(a, b, transmute(c), D))
5495}
5496
5497/// Vector Shift Left Double by Byte
5498#[inline]
5499#[target_feature(enable = "vector")]
5500#[unstable(feature = "stdarch_s390x", issue = "135681")]
5501pub unsafe fn vec_sld<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5502    static_assert_uimm_bits!(C, 4);
5503    a.vec_sld::<C>(b)
5504}
5505
5506/// Vector Shift Left Double by Word
5507#[inline]
5508#[target_feature(enable = "vector")]
5509#[unstable(feature = "stdarch_s390x", issue = "135681")]
5510pub unsafe fn vec_sldw<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5511    static_assert_uimm_bits!(C, 2);
5512    a.vec_sldw::<C>(b)
5513}
5514
5515/// Vector Shift Left Double by Bit
5516#[inline]
5517#[target_feature(enable = "vector-enhancements-2")]
5518#[unstable(feature = "stdarch_s390x", issue = "135681")]
5519pub unsafe fn vec_sldb<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5520    static_assert_uimm_bits!(C, 3);
5521    a.vec_sldb::<C>(b)
5522}
5523
5524/// Vector Shift Right Double by Bit
5525#[inline]
5526#[target_feature(enable = "vector-enhancements-2")]
5527#[unstable(feature = "stdarch_s390x", issue = "135681")]
5528pub unsafe fn vec_srdb<T: sealed::VectorSrdb, const C: u32>(a: T, b: T) -> T {
5529    static_assert_uimm_bits!(C, 3);
5530    a.vec_srdb::<C>(b)
5531}
5532
5533/// Vector Compare Ranges
5534#[inline]
5535#[target_feature(enable = "vector")]
5536#[unstable(feature = "stdarch_s390x", issue = "135681")]
5537pub unsafe fn vec_cmprg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5538    a.vstrc::<{ FindImm::Eq as u32 }>(b, c)
5539}
5540
5541/// Vector Compare Not in Ranges
5542#[inline]
5543#[target_feature(enable = "vector")]
5544#[unstable(feature = "stdarch_s390x", issue = "135681")]
5545pub unsafe fn vec_cmpnrg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5546    a.vstrc::<{ FindImm::Ne as u32 }>(b, c)
5547}
5548
5549/// Vector Compare Ranges Index
5550#[inline]
5551#[target_feature(enable = "vector")]
5552#[unstable(feature = "stdarch_s390x", issue = "135681")]
5553pub unsafe fn vec_cmprg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5554    a.vstrc::<{ FindImm::EqIdx as u32 }>(b, c)
5555}
5556
5557/// Vector Compare Not in Ranges Index
5558#[inline]
5559#[target_feature(enable = "vector")]
5560#[unstable(feature = "stdarch_s390x", issue = "135681")]
5561pub unsafe fn vec_cmpnrg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5562    a.vstrc::<{ FindImm::NeIdx as u32 }>(b, c)
5563}
5564
5565/// Vector Compare Ranges with Condition Code
5566#[inline]
5567#[target_feature(enable = "vector")]
5568#[unstable(feature = "stdarch_s390x", issue = "135681")]
5569pub unsafe fn vec_cmprg_cc<T: sealed::VectorCompareRange>(
5570    a: T,
5571    b: T,
5572    c: T,
5573    d: *mut i32,
5574) -> T::Result {
5575    let (x, y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c);
5576    d.write(y);
5577    x
5578}
5579
5580/// Vector Compare Not in Ranges with Condition Code
5581#[inline]
5582#[target_feature(enable = "vector")]
5583#[unstable(feature = "stdarch_s390x", issue = "135681")]
5584pub unsafe fn vec_cmpnrg_cc<T: sealed::VectorCompareRange>(
5585    a: T,
5586    b: T,
5587    c: T,
5588    d: *mut i32,
5589) -> T::Result {
5590    let (x, y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c);
5591    d.write(y);
5592    x
5593}
5594
5595/// Vector Compare Ranges Index with Condition Code
5596#[inline]
5597#[target_feature(enable = "vector")]
5598#[unstable(feature = "stdarch_s390x", issue = "135681")]
5599pub unsafe fn vec_cmprg_idx_cc<T: sealed::VectorCompareRange>(
5600    a: T,
5601    b: T,
5602    c: T,
5603    d: *mut i32,
5604) -> T::Result {
5605    let (x, y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c);
5606    d.write(y);
5607    x
5608}
5609
5610/// Vector Compare Not in Ranges Index with Condition Code
5611#[inline]
5612#[target_feature(enable = "vector")]
5613#[unstable(feature = "stdarch_s390x", issue = "135681")]
5614pub unsafe fn vec_cmpnrg_idx_cc<T: sealed::VectorCompareRange>(
5615    a: T,
5616    b: T,
5617    c: T,
5618    d: *mut i32,
5619) -> T::Result {
5620    let (x, y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c);
5621    d.write(y);
5622    x
5623}
5624
5625/// Vector Compare Ranges or Zero Index
5626#[inline]
5627#[target_feature(enable = "vector")]
5628#[unstable(feature = "stdarch_s390x", issue = "135681")]
5629pub unsafe fn vec_cmprg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5630    a.vstrcz::<{ FindImm::EqIdx as u32 }>(b, c)
5631}
5632
5633/// Vector Compare Not in Ranges or Zero Index
5634#[inline]
5635#[target_feature(enable = "vector")]
5636#[unstable(feature = "stdarch_s390x", issue = "135681")]
5637pub unsafe fn vec_cmpnrg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5638    a.vstrcz::<{ FindImm::NeIdx as u32 }>(b, c)
5639}
5640
5641/// Vector Compare Ranges or Zero Index with Condition Code
5642#[inline]
5643#[target_feature(enable = "vector")]
5644#[unstable(feature = "stdarch_s390x", issue = "135681")]
5645pub unsafe fn vec_cmprg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5646    a: T,
5647    b: T,
5648    c: T,
5649    d: *mut i32,
5650) -> T::Result {
5651    let (x, y) = a.vstrczs::<{ FindImm::EqIdx as u32 }>(b, c);
5652    d.write(y);
5653    x
5654}
5655
5656/// Vector Compare Not in Ranges or Zero Index with Condition Code
5657#[inline]
5658#[target_feature(enable = "vector")]
5659#[unstable(feature = "stdarch_s390x", issue = "135681")]
5660pub unsafe fn vec_cmpnrg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5661    a: T,
5662    b: T,
5663    c: T,
5664    d: *mut i32,
5665) -> T::Result {
5666    let (x, y) = a.vstrczs::<{ FindImm::NeIdx as u32 }>(b, c);
5667    d.write(y);
5668    x
5669}
5670
5671/// Vector Compare Equal
5672#[inline]
5673#[target_feature(enable = "vector")]
5674#[unstable(feature = "stdarch_s390x", issue = "135681")]
5675pub unsafe fn vec_cmpeq<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5676    a.vec_cmpeq(b)
5677}
5678
5679/// Vector Compare Not Equal
5680#[inline]
5681#[target_feature(enable = "vector")]
5682#[unstable(feature = "stdarch_s390x", issue = "135681")]
5683pub unsafe fn vec_cmpne<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5684    a.vec_cmpne(b)
5685}
5686
5687/// Vector Compare Greater Than
5688#[inline]
5689#[target_feature(enable = "vector")]
5690#[unstable(feature = "stdarch_s390x", issue = "135681")]
5691pub unsafe fn vec_cmpgt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5692    a.vec_cmpgt(b)
5693}
5694
5695/// Vector Compare Greater Than or Equal
5696#[inline]
5697#[target_feature(enable = "vector")]
5698#[unstable(feature = "stdarch_s390x", issue = "135681")]
5699pub unsafe fn vec_cmpge<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5700    a.vec_cmpge(b)
5701}
5702
5703/// Vector Compare Less
5704#[inline]
5705#[target_feature(enable = "vector")]
5706#[unstable(feature = "stdarch_s390x", issue = "135681")]
5707pub unsafe fn vec_cmplt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5708    a.vec_cmplt(b)
5709}
5710
5711/// Vector Compare Less Than or Equal
5712#[inline]
5713#[target_feature(enable = "vector")]
5714#[unstable(feature = "stdarch_s390x", issue = "135681")]
5715pub unsafe fn vec_cmple<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5716    a.vec_cmple(b)
5717}
5718
5719/// Vector Compare Equal Index
5720#[inline]
5721#[target_feature(enable = "vector")]
5722#[unstable(feature = "stdarch_s390x", issue = "135681")]
5723pub unsafe fn vec_cmpeq_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5724    a.vec_cmpeq_idx(b)
5725}
5726/// Vector Compare Not Equal Index
5727#[inline]
5728#[target_feature(enable = "vector")]
5729#[unstable(feature = "stdarch_s390x", issue = "135681")]
5730pub unsafe fn vec_cmpne_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5731    a.vec_cmpne_idx(b)
5732}
5733/// Vector Compare Equal Index with Condition Code
5734#[inline]
5735#[target_feature(enable = "vector")]
5736#[unstable(feature = "stdarch_s390x", issue = "135681")]
5737pub unsafe fn vec_cmpeq_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5738    a.vec_cmpeq_idx_cc(b)
5739}
5740/// Vector Compare Not Equal Index with Condition Code
5741#[inline]
5742#[target_feature(enable = "vector")]
5743#[unstable(feature = "stdarch_s390x", issue = "135681")]
5744pub unsafe fn vec_cmpne_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5745    a.vec_cmpne_idx_cc(b)
5746}
5747/// Vector Compare Equal or Zero Index
5748#[inline]
5749#[target_feature(enable = "vector")]
5750#[unstable(feature = "stdarch_s390x", issue = "135681")]
5751pub unsafe fn vec_cmpeq_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5752    a.vec_cmpeq_or_0_idx(b)
5753}
5754/// Vector Compare Not Equal or Zero Index
5755#[inline]
5756#[target_feature(enable = "vector")]
5757#[unstable(feature = "stdarch_s390x", issue = "135681")]
5758pub unsafe fn vec_cmpne_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5759    a.vec_cmpne_or_0_idx(b)
5760}
5761/// Vector Compare Equal or Zero Index with Condition Code
5762#[inline]
5763#[target_feature(enable = "vector")]
5764#[unstable(feature = "stdarch_s390x", issue = "135681")]
5765pub unsafe fn vec_cmpeq_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5766    a.vec_cmpeq_or_0_idx_cc(b)
5767}
5768/// Vector Compare Not Equal or Zero Index with Condition Code
5769#[inline]
5770#[target_feature(enable = "vector")]
5771#[unstable(feature = "stdarch_s390x", issue = "135681")]
5772pub unsafe fn vec_cmpne_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5773    a.vec_cmpne_or_0_idx_cc(b)
5774}
5775
5776/// All Elements Equal
5777#[inline]
5778#[target_feature(enable = "vector")]
5779#[unstable(feature = "stdarch_s390x", issue = "135681")]
5780pub unsafe fn vec_all_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5781    simd_reduce_all(vec_cmpeq(a, b)) as i32 as i32
5782}
5783
5784/// All Elements Not Equal
5785#[inline]
5786#[target_feature(enable = "vector")]
5787#[unstable(feature = "stdarch_s390x", issue = "135681")]
5788pub unsafe fn vec_all_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5789    simd_reduce_all(vec_cmpne(a, b)) as i32
5790}
5791
5792/// Any Element Equal
5793#[inline]
5794#[target_feature(enable = "vector")]
5795#[unstable(feature = "stdarch_s390x", issue = "135681")]
5796pub unsafe fn vec_any_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5797    simd_reduce_any(vec_cmpeq(a, b)) as i32
5798}
5799
5800/// Any Element Not Equal
5801#[inline]
5802#[target_feature(enable = "vector")]
5803#[unstable(feature = "stdarch_s390x", issue = "135681")]
5804pub unsafe fn vec_any_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5805    simd_reduce_any(vec_cmpne(a, b)) as i32
5806}
5807
5808/// All Elements Less Than
5809#[inline]
5810#[target_feature(enable = "vector")]
5811#[unstable(feature = "stdarch_s390x", issue = "135681")]
5812pub unsafe fn vec_all_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5813    a.vec_all_lt(b)
5814}
5815
5816/// All Elements Less Than or Equal
5817#[inline]
5818#[target_feature(enable = "vector")]
5819#[unstable(feature = "stdarch_s390x", issue = "135681")]
5820pub unsafe fn vec_all_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5821    a.vec_all_le(b)
5822}
5823
5824/// All Elements Greater Than
5825#[inline]
5826#[target_feature(enable = "vector")]
5827#[unstable(feature = "stdarch_s390x", issue = "135681")]
5828pub unsafe fn vec_all_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5829    a.vec_all_gt(b)
5830}
5831
5832/// All Elements Greater Than or Equal
5833#[inline]
5834#[target_feature(enable = "vector")]
5835#[unstable(feature = "stdarch_s390x", issue = "135681")]
5836pub unsafe fn vec_all_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5837    a.vec_all_ge(b)
5838}
5839
5840/// All Elements Not Less Than
5841#[inline]
5842#[target_feature(enable = "vector")]
5843#[unstable(feature = "stdarch_s390x", issue = "135681")]
5844pub unsafe fn vec_all_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5845    vec_all_ge(a, b)
5846}
5847
5848/// All Elements Not Less Than or Equal
5849#[inline]
5850#[target_feature(enable = "vector")]
5851#[unstable(feature = "stdarch_s390x", issue = "135681")]
5852pub unsafe fn vec_all_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5853    vec_all_gt(a, b)
5854}
5855
5856/// All Elements Not Greater Than
5857#[inline]
5858#[target_feature(enable = "vector")]
5859#[unstable(feature = "stdarch_s390x", issue = "135681")]
5860pub unsafe fn vec_all_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5861    vec_all_le(a, b)
5862}
5863
5864/// All Elements Not Greater Than or Equal
5865#[inline]
5866#[target_feature(enable = "vector")]
5867#[unstable(feature = "stdarch_s390x", issue = "135681")]
5868pub unsafe fn vec_all_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5869    vec_all_lt(a, b)
5870}
5871
5872/// Any Elements Less Than
5873#[inline]
5874#[target_feature(enable = "vector")]
5875#[unstable(feature = "stdarch_s390x", issue = "135681")]
5876pub unsafe fn vec_any_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5877    !vec_all_ge(a, b)
5878}
5879
5880/// Any Elements Less Than or Equal
5881#[inline]
5882#[target_feature(enable = "vector")]
5883#[unstable(feature = "stdarch_s390x", issue = "135681")]
5884pub unsafe fn vec_any_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5885    !vec_all_gt(a, b)
5886}
5887
5888/// Any Elements Greater Than
5889#[inline]
5890#[target_feature(enable = "vector")]
5891#[unstable(feature = "stdarch_s390x", issue = "135681")]
5892pub unsafe fn vec_any_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5893    !vec_all_le(a, b)
5894}
5895
5896/// Any Elements Greater Than or Equal
5897#[inline]
5898#[target_feature(enable = "vector")]
5899#[unstable(feature = "stdarch_s390x", issue = "135681")]
5900pub unsafe fn vec_any_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5901    !vec_all_lt(a, b)
5902}
5903
5904/// Any Elements Not Less Than
5905#[inline]
5906#[target_feature(enable = "vector")]
5907#[unstable(feature = "stdarch_s390x", issue = "135681")]
5908pub unsafe fn vec_any_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5909    vec_any_ge(a, b)
5910}
5911
5912/// Any Elements Not Less Than or Equal
5913#[inline]
5914#[target_feature(enable = "vector")]
5915#[unstable(feature = "stdarch_s390x", issue = "135681")]
5916pub unsafe fn vec_any_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5917    vec_any_gt(a, b)
5918}
5919
5920/// Any Elements Not Greater Than
5921#[inline]
5922#[target_feature(enable = "vector")]
5923#[unstable(feature = "stdarch_s390x", issue = "135681")]
5924pub unsafe fn vec_any_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5925    vec_any_le(a, b)
5926}
5927
5928/// Any Elements Not Greater Than or Equal
5929#[inline]
5930#[target_feature(enable = "vector")]
5931#[unstable(feature = "stdarch_s390x", issue = "135681")]
5932pub unsafe fn vec_any_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5933    vec_any_lt(a, b)
5934}
5935
5936/// Vector Extract
5937#[inline]
5938#[target_feature(enable = "vector")]
5939#[unstable(feature = "stdarch_s390x", issue = "135681")]
5940pub unsafe fn vec_extract<T: sealed::VectorExtract>(a: T, b: i32) -> T::ElementType {
5941    T::vec_extract(a, b)
5942}
5943
5944/// Vector Insert
5945#[inline]
5946#[target_feature(enable = "vector")]
5947#[unstable(feature = "stdarch_s390x", issue = "135681")]
5948pub unsafe fn vec_insert<T: sealed::VectorInsert>(a: T::ElementType, b: T, c: i32) -> T {
5949    T::vec_insert(a, b, c)
5950}
5951
5952/// Vector Insert and Zero
5953#[inline]
5954#[target_feature(enable = "vector")]
5955#[unstable(feature = "stdarch_s390x", issue = "135681")]
5956pub unsafe fn vec_insert_and_zero<T: sealed::VectorInsertAndZero>(a: *const T::ElementType) -> T {
5957    T::vec_insert_and_zero(a)
5958}
5959
5960/// Vector Promote
5961#[inline]
5962#[target_feature(enable = "vector")]
5963#[unstable(feature = "stdarch_s390x", issue = "135681")]
5964pub unsafe fn vec_promote<T: sealed::VectorPromote>(a: T::ElementType, b: i32) -> MaybeUninit<T> {
5965    T::vec_promote(a, b)
5966}
5967
5968/// Converts the left-most half of `a` to a vector of single-precision numbers.
5969/// The format of the source vector elements is specified by `B`.
5970#[inline]
5971#[target_feature(enable = "nnp-assist")]
5972#[cfg_attr(test, assert_instr(vclfnh, B = 0))]
5973#[unstable(feature = "stdarch_s390x", issue = "135681")]
5974pub unsafe fn vec_extend_to_fp32_hi<const B: i32>(a: vector_signed_short) -> vector_float {
5975    // On processors implementing the IBM z16 architecture, only the value 0 is supported.
5976    static_assert_uimm_bits!(B, 4);
5977
5978    vclfnhs(a, B)
5979}
5980
5981/// Converts the right-most half of `a` to a vector of single-precision numbers.
5982/// The format of the source vector elements is specified by `B`.
5983#[inline]
5984#[target_feature(enable = "nnp-assist")]
5985#[cfg_attr(test, assert_instr(vclfnl, B = 0))]
5986#[unstable(feature = "stdarch_s390x", issue = "135681")]
5987pub unsafe fn vec_extend_to_fp32_lo<const B: i32>(a: vector_signed_short) -> vector_float {
5988    // On processors implementing the IBM z16 architecture, only the value 0 is supported.
5989    static_assert_uimm_bits!(B, 4);
5990
5991    vclfnls(a, B)
5992}
5993
5994/// Converts the elements of vector `a` to the 16-bit IEEE floating point format.
5995/// The format of the source vector elements is specified by `B`.
5996#[inline]
5997#[target_feature(enable = "nnp-assist")]
5998#[cfg_attr(test, assert_instr(vcfn, B = 0))]
5999#[unstable(feature = "stdarch_s390x", issue = "135681")]
6000pub unsafe fn vec_convert_to_fp16<const B: i32>(a: vector_signed_short) -> vector_signed_short {
6001    // On processors implementing the IBM z16 architecture, only the value 0 is supported.
6002    static_assert_uimm_bits!(B, 4);
6003
6004    vcfn(a, B)
6005}
6006
6007/// Converts the elements of vector `a` to an internal floating point format.
6008/// The format of the target vector elements is specified by `B`.
6009#[inline]
6010#[target_feature(enable = "nnp-assist")]
6011#[cfg_attr(test, assert_instr(vcnf, B = 0))]
6012#[unstable(feature = "stdarch_s390x", issue = "135681")]
6013pub unsafe fn vec_convert_from_fp16<const B: i32>(a: vector_signed_short) -> vector_signed_short {
6014    // On processors implementing the IBM z16 architecture, only the value 0 is supported.
6015    static_assert_uimm_bits!(B, 4);
6016
6017    vcnf(a, B)
6018}
6019
6020/// Converts the elements of single-precision vectors `a` and `b` to an internal floating point
6021/// format with 16-bit sized elements. The format of the target vector elements is specified by `C`.
6022#[inline]
6023#[target_feature(enable = "nnp-assist")]
6024#[unstable(feature = "stdarch_s390x", issue = "135681")]
6025#[cfg_attr(test, assert_instr(vcrnf, C = 0))]
6026pub unsafe fn vec_round_from_fp32<const C: i32>(
6027    a: vector_float,
6028    b: vector_float,
6029) -> vector_signed_short {
6030    // On processors implementing the IBM z16 architecture, only the value 0 is supported.
6031    static_assert_uimm_bits!(C, 4);
6032
6033    vcrnfs(a, b, C)
6034}
6035
6036#[cfg(test)]
6037mod tests {
6038    use super::*;
6039
6040    use std::mem::transmute;
6041
6042    use crate::core_arch::simd::*;
6043    use stdarch_test::simd_test;
6044
6045    impl<const N: usize> ShuffleMask<N> {
6046        fn as_array(&self) -> &[u32; N] {
6047            unsafe { std::mem::transmute(self) }
6048        }
6049    }
6050
6051    #[test]
6052    fn reverse_mask() {
6053        assert_eq!(ShuffleMask::<4>::reverse().as_array(), &[3, 2, 1, 0]);
6054    }
6055
6056    #[test]
6057    fn mergel_mask() {
6058        assert_eq!(ShuffleMask::<4>::merge_low().as_array(), &[2, 6, 3, 7]);
6059    }
6060
6061    #[test]
6062    fn mergeh_mask() {
6063        assert_eq!(ShuffleMask::<4>::merge_high().as_array(), &[0, 4, 1, 5]);
6064    }
6065
6066    #[test]
6067    fn pack_mask() {
6068        assert_eq!(ShuffleMask::<4>::pack().as_array(), &[1, 3, 5, 7]);
6069    }
6070
6071    #[test]
6072    fn test_vec_mask() {
6073        assert_eq!(
6074            genmask::<0x00FF>(),
6075            [
6076                0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
6077            ]
6078        );
6079    }
6080
6081    #[test]
6082    fn test_genmasks() {
6083        assert_eq!(genmasks(u8::BITS, 3, 5), 28);
6084        assert_eq!(genmasks(u8::BITS, 3, 7), 31);
6085
6086        // If a or b is greater than 8, the operation is performed as if the value gets modulo by 8.
6087        assert_eq!(genmasks(u8::BITS, 3 + 8, 7 + 8), 31);
6088        // If a is greater than b, the operation is perform as if b equals 7.
6089        assert_eq!(genmasks(u8::BITS, 5, 4), genmasks(u8::BITS, 5, 7));
6090
6091        assert_eq!(
6092            genmasks(u16::BITS, 4, 12) as u16,
6093            u16::from_be_bytes([15, -8i8 as u8])
6094        );
6095        assert_eq!(
6096            genmasks(u32::BITS, 4, 29) as u32,
6097            u32::from_be_bytes([15, 0xFF, 0xFF, -4i8 as u8])
6098        );
6099    }
6100
6101    macro_rules! test_vec_1 {
6102        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($d:expr),+] } => {
6103            test_vec_1! { $name, $fn, $ty -> $ty, [$($a),+], [$($d),+] }
6104        };
6105        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($d:expr),+] } => {
6106            #[simd_test(enable = "vector")]
6107            fn $name() {
6108                let a: s_t_l!($ty) = $ty::new($($a),+).into();
6109
6110                let d = $ty_out::new($($d),+);
6111                let r = $ty_out::from(unsafe { $fn(a) });
6112                assert_eq!(d, r);
6113            }
6114        }
6115    }
6116
6117    macro_rules! test_vec_2 {
6118        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
6119            test_vec_2! { $name, $fn, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
6120        };
6121        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
6122            test_vec_2! { $name, $fn, $ty, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
6123         };
6124        { $name: ident, $fn:ident, $ty1: ident, $ty2: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
6125            #[simd_test(enable = "vector")]
6126            fn $name() {
6127                let a: s_t_l!($ty1) = $ty1::new($($a),+).into();
6128                let b: s_t_l!($ty2) = $ty2::new($($b),+).into();
6129
6130                let d = $ty_out::new($($d),+);
6131                let r = $ty_out::from(unsafe { $fn(a, b) });
6132                assert_eq!(d, r);
6133            }
6134         };
6135   }
6136
6137    #[simd_test(enable = "vector")]
6138    fn vec_add_i32x4_i32x4() {
6139        let x = vector_signed_int::from(i32x4::new(1, 2, 3, 4));
6140        let y = vector_signed_int::from(i32x4::new(4, 3, 2, 1));
6141        let z = unsafe { vec_add(x, y) };
6142        assert_eq!(i32x4::splat(5), i32x4::from(z));
6143    }
6144
6145    macro_rules! test_vec_sub {
6146        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
6147            test_vec_2! {$name, vec_sub, $ty, [$($a),+], [$($b),+], [$($d),+] }
6148        }
6149    }
6150
6151    test_vec_sub! { test_vec_sub_f32x4, f32x4,
6152    [-1.0, 0.0, 1.0, 2.0],
6153    [2.0, 1.0, -1.0, -2.0],
6154    [-3.0, -1.0, 2.0, 4.0] }
6155
6156    test_vec_sub! { test_vec_sub_f64x2, f64x2,
6157    [-1.0, 0.0],
6158    [2.0, 1.0],
6159    [-3.0, -1.0] }
6160
6161    test_vec_sub! { test_vec_sub_i64x2, i64x2,
6162    [-1, 0],
6163    [2, 1],
6164    [-3, -1] }
6165
6166    test_vec_sub! { test_vec_sub_u64x2, u64x2,
6167    [0, 1],
6168    [1, 0],
6169    [u64::MAX, 1] }
6170
6171    test_vec_sub! { test_vec_sub_i32x4, i32x4,
6172    [-1, 0, 1, 2],
6173    [2, 1, -1, -2],
6174    [-3, -1, 2, 4] }
6175
6176    test_vec_sub! { test_vec_sub_u32x4, u32x4,
6177    [0, 0, 1, 2],
6178    [2, 1, 0, 0],
6179    [4294967294, 4294967295, 1, 2] }
6180
6181    test_vec_sub! { test_vec_sub_i16x8, i16x8,
6182    [-1, 0, 1, 2, -1, 0, 1, 2],
6183    [2, 1, -1, -2, 2, 1, -1, -2],
6184    [-3, -1, 2, 4, -3, -1, 2, 4] }
6185
6186    test_vec_sub! { test_vec_sub_u16x8, u16x8,
6187    [0, 0, 1, 2, 0, 0, 1, 2],
6188    [2, 1, 0, 0, 2, 1, 0, 0],
6189    [65534, 65535, 1, 2, 65534, 65535, 1, 2] }
6190
6191    test_vec_sub! { test_vec_sub_i8x16, i8x16,
6192    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
6193    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
6194    [-3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4] }
6195
6196    test_vec_sub! { test_vec_sub_u8x16, u8x16,
6197    [0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2],
6198    [2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0],
6199    [254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2] }
6200
6201    macro_rules! test_vec_mul {
6202        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
6203            test_vec_2! {$name, vec_mul, $ty, [$($a),+], [$($b),+], [$($d),+] }
6204        }
6205    }
6206
6207    test_vec_mul! { test_vec_mul_f32x4, f32x4,
6208    [-1.0, 0.0, 1.0, 2.0],
6209    [2.0, 1.0, -1.0, -2.0],
6210    [-2.0, 0.0, -1.0, -4.0] }
6211
6212    test_vec_mul! { test_vec_mul_f64x2, f64x2,
6213    [-1.0, 0.0],
6214    [2.0, 1.0],
6215    [-2.0, 0.0] }
6216
6217    test_vec_mul! { test_vec_mul_i64x2, i64x2,
6218    [i64::MAX, -4],
6219    [2, 3],
6220    [i64::MAX.wrapping_mul(2), -12] }
6221
6222    test_vec_mul! { test_vec_mul_u64x2, u64x2,
6223    [u64::MAX, 4],
6224    [2, 3],
6225    [u64::MAX.wrapping_mul(2), 12] }
6226
6227    test_vec_mul! { test_vec_mul_i32x4, i32x4,
6228    [-1, 0, 1, 2],
6229    [2, 1, -1, -2],
6230    [-2, 0, -1, -4] }
6231
6232    test_vec_mul! { test_vec_mul_u32x4, u32x4,
6233    [0, u32::MAX - 1, 1, 2],
6234    [5, 6, 7, 8],
6235    [0, 4294967284, 7, 16] }
6236
6237    test_vec_mul! { test_vec_mul_i16x8, i16x8,
6238    [-1, 0, 1, 2, -1, 0, 1, 2],
6239    [2, 1, -1, -2, 2, 1, -1, -2],
6240    [-2, 0, -1, -4, -2, 0, -1, -4] }
6241
6242    test_vec_mul! { test_vec_mul_u16x8, u16x8,
6243    [0, u16::MAX - 1, 1, 2, 3, 4, 5, 6],
6244    [5, 6, 7, 8, 9, 8, 7, 6],
6245    [0, 65524, 7, 16, 27, 32, 35, 36] }
6246
6247    test_vec_mul! { test_vec_mul_i8x16, i8x16,
6248    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
6249    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
6250    [-2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4] }
6251
6252    test_vec_mul! { test_vec_mul_u8x16, u8x16,
6253    [0, u8::MAX - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4],
6254    [5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 0, u8::MAX, 1, 2, 3, 4],
6255    [0, 244, 7, 16, 27, 32, 35, 36, 35, 32, 0, 248, 7, 12, 15, 16] }
6256
6257    macro_rules! test_vec_abs {
6258        { $name: ident, $ty: ident, $a: expr, $d: expr } => {
6259            #[simd_test(enable = "vector")]
6260            fn $name() {
6261                let a: s_t_l!($ty) = unsafe { vec_splats($a) };
6262                let a: s_t_l!($ty) = unsafe { vec_abs(a) };
6263                let d = $ty::splat($d);
6264                assert_eq!(d, $ty::from(a));
6265            }
6266        }
6267    }
6268
6269    test_vec_abs! { test_vec_abs_i8, i8x16, -42i8, 42i8 }
6270    test_vec_abs! { test_vec_abs_i16, i16x8, -42i16, 42i16 }
6271    test_vec_abs! { test_vec_abs_i32, i32x4, -42i32, 42i32 }
6272    test_vec_abs! { test_vec_abs_i64, i64x2, -42i64, 42i64 }
6273    test_vec_abs! { test_vec_abs_f32, f32x4, -42f32, 42f32 }
6274    test_vec_abs! { test_vec_abs_f64, f64x2, -42f64, 42f64 }
6275
6276    test_vec_1! { test_vec_nabs, vec_nabs, f32x4,
6277    [core::f32::consts::PI, 1.0, 0.0, -1.0],
6278    [-core::f32::consts::PI, -1.0, 0.0, -1.0] }
6279
6280    test_vec_2! { test_vec_andc, vec_andc, i32x4,
6281    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6282    [0b00110011, 0b11110011, 0b00001100, 0b10000000],
6283    [0b11001100, 0b00001100, 0b11000000, 0b01001100] }
6284
6285    test_vec_2! { test_vec_and, vec_and, i32x4,
6286    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6287    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6288    [0b00000000, 0b11000000, 0b00001100, 0b00000000] }
6289
6290    test_vec_2! { test_vec_nand, vec_nand, i32x4,
6291    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6292    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6293    [!0b00000000, !0b11000000, !0b00001100, !0b00000000] }
6294
6295    test_vec_2! { test_vec_orc, vec_orc, u32x4,
6296    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6297    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6298    [0b11001100 | !0b00110011, 0b11001100 | !0b11110011, 0b11001100 | !0b00001100, 0b11001100 | !0b00000000] }
6299
6300    test_vec_2! { test_vec_or, vec_or, i32x4,
6301    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6302    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6303    [0b11111111, 0b11111111, 0b11001100, 0b11001100] }
6304
6305    test_vec_2! { test_vec_nor, vec_nor, i32x4,
6306    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6307    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6308    [!0b11111111, !0b11111111, !0b11001100, !0b11001100] }
6309
6310    test_vec_2! { test_vec_xor, vec_xor, i32x4,
6311    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6312    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6313    [0b11111111, 0b00111111, 0b11000000, 0b11001100] }
6314
6315    test_vec_2! { test_vec_eqv, vec_eqv, i32x4,
6316    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6317    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6318    [!0b11111111, !0b00111111, !0b11000000, !0b11001100] }
6319
6320    test_vec_1! { test_vec_floor_f32, vec_floor, f32x4,
6321        [1.1, 1.9, -0.5, -0.9],
6322        [1.0, 1.0, -1.0, -1.0]
6323    }
6324
6325    test_vec_1! { test_vec_floor_f64_1, vec_floor, f64x2,
6326        [1.1, 1.9],
6327        [1.0, 1.0]
6328    }
6329    test_vec_1! { test_vec_floor_f64_2, vec_floor, f64x2,
6330        [-0.5, -0.9],
6331        [-1.0, -1.0]
6332    }
6333
6334    test_vec_1! { test_vec_ceil_f32, vec_ceil, f32x4,
6335        [0.1, 0.5, 0.6, 0.9],
6336        [1.0, 1.0, 1.0, 1.0]
6337    }
6338    test_vec_1! { test_vec_ceil_f64_1, vec_ceil, f64x2,
6339        [0.1, 0.5],
6340        [1.0, 1.0]
6341    }
6342    test_vec_1! { test_vec_ceil_f64_2, vec_ceil, f64x2,
6343        [0.6, 0.9],
6344        [1.0, 1.0]
6345    }
6346
6347    test_vec_1! { test_vec_round_f32, vec_round, f32x4,
6348        [0.1, 0.5, 0.6, 0.9],
6349        [0.0, 0.0, 1.0, 1.0]
6350    }
6351
6352    test_vec_1! { test_vec_round_f32_even_odd, vec_round, f32x4,
6353        [0.5, 1.5, 2.5, 3.5],
6354        [0.0, 2.0, 2.0, 4.0]
6355    }
6356
6357    test_vec_1! { test_vec_round_f64_1, vec_round, f64x2,
6358        [0.1, 0.5],
6359        [0.0, 0.0]
6360    }
6361    test_vec_1! { test_vec_round_f64_2, vec_round, f64x2,
6362        [0.6, 0.9],
6363        [1.0, 1.0]
6364    }
6365
6366    test_vec_1! { test_vec_roundc_f32, vec_roundc, f32x4,
6367        [0.1, 0.5, 0.6, 0.9],
6368        [0.0, 0.0, 1.0, 1.0]
6369    }
6370
6371    test_vec_1! { test_vec_roundc_f32_even_odd, vec_roundc, f32x4,
6372        [0.5, 1.5, 2.5, 3.5],
6373        [0.0, 2.0, 2.0, 4.0]
6374    }
6375
6376    test_vec_1! { test_vec_roundc_f64_1, vec_roundc, f64x2,
6377        [0.1, 0.5],
6378        [0.0, 0.0]
6379    }
6380    test_vec_1! { test_vec_roundc_f64_2, vec_roundc, f64x2,
6381        [0.6, 0.9],
6382        [1.0, 1.0]
6383    }
6384
6385    test_vec_1! { test_vec_rint_f32, vec_rint, f32x4,
6386        [0.1, 0.5, 0.6, 0.9],
6387        [0.0, 0.0, 1.0, 1.0]
6388    }
6389
6390    test_vec_1! { test_vec_rint_f32_even_odd, vec_rint, f32x4,
6391        [0.5, 1.5, 2.5, 3.5],
6392        [0.0, 2.0, 2.0, 4.0]
6393    }
6394
6395    test_vec_1! { test_vec_rint_f64_1, vec_rint, f64x2,
6396        [0.1, 0.5],
6397        [0.0, 0.0]
6398    }
6399    test_vec_1! { test_vec_rint_f64_2, vec_rint, f64x2,
6400        [0.6, 0.9],
6401        [1.0, 1.0]
6402    }
6403
6404    test_vec_2! { test_vec_sll, vec_sll, i32x4, u8x16 -> i32x4,
6405    [1, 1, 1, 1],
6406    [0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8],
6407    [1 << 2, 1 << 3, 1 << 4, 1] }
6408
6409    test_vec_2! { test_vec_srl, vec_srl, i32x4, u8x16 -> i32x4,
6410    [0b1000, 0b1000, 0b1000, 0b1000],
6411    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6412    [4, 2, 1, 8] }
6413
6414    test_vec_2! { test_vec_sral_pos, vec_sral, u32x4, u8x16 -> u32x4,
6415    [0b1000, 0b1000, 0b1000, 0b1000],
6416    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6417    [4, 2, 1, 8] }
6418
6419    test_vec_2! { test_vec_sral_neg, vec_sral, i32x4, u8x16 -> i32x4,
6420    [-8, -8, -8, -8],
6421    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6422    [-4, -2, -1, -8] }
6423
6424    test_vec_1! { test_vec_reve_f32, vec_reve, f32x4,
6425        [0.1, 0.5, 0.6, 0.9],
6426        [0.9, 0.6, 0.5, 0.1]
6427    }
6428
6429    test_vec_1! { test_vec_revb_u32, vec_revb, u32x4,
6430        [0xAABBCCDD, 0xEEFF0011, 0x22334455, 0x66778899],
6431        [0xDDCCBBAA, 0x1100FFEE, 0x55443322, 0x99887766]
6432    }
6433
6434    test_vec_2! { test_vec_mergeh_u32, vec_mergeh, u32x4,
6435        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6436        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6437        [0xAAAAAAAA, 0x00000000, 0xBBBBBBBB, 0x11111111]
6438    }
6439
6440    test_vec_2! { test_vec_mergel_u32, vec_mergel, u32x4,
6441        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6442        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6443        [0xCCCCCCCC, 0x22222222, 0xDDDDDDDD, 0x33333333]
6444    }
6445
6446    macro_rules! test_vec_perm {
6447        {$name:ident,
6448         $shorttype:ident, $longtype:ident,
6449         [$($a:expr),+], [$($b:expr),+], [$($c:expr),+], [$($d:expr),+]} => {
6450            #[simd_test(enable = "vector")]
6451            fn $name() {
6452                let a = $longtype::from($shorttype::new($($a),+));
6453                let b = $longtype::from($shorttype::new($($b),+));
6454                let c = vector_unsigned_char::from(u8x16::new($($c),+));
6455                let d = $shorttype::new($($d),+);
6456
6457                let r = $shorttype::from(unsafe { vec_perm(a, b, c) });
6458                assert_eq!(d, r);
6459            }
6460        }
6461    }
6462
6463    test_vec_perm! {test_vec_perm_u8x16,
6464    u8x16, vector_unsigned_char,
6465    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6466    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6467    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6468     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6469    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6470    test_vec_perm! {test_vec_perm_i8x16,
6471    i8x16, vector_signed_char,
6472    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6473    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6474    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6475     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6476    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6477
6478    test_vec_perm! {test_vec_perm_m8x16,
6479    m8x16, vector_bool_char,
6480    [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false],
6481    [true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true],
6482    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6483     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6484    [false, false, true, true, false, false, true, true, false, false, true, true, false, false, true, true]}
6485    test_vec_perm! {test_vec_perm_u16x8,
6486    u16x8, vector_unsigned_short,
6487    [0, 1, 2, 3, 4, 5, 6, 7],
6488    [10, 11, 12, 13, 14, 15, 16, 17],
6489    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6490     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6491    [0, 10, 1, 11, 2, 12, 3, 13]}
6492    test_vec_perm! {test_vec_perm_i16x8,
6493    i16x8, vector_signed_short,
6494    [0, 1, 2, 3, 4, 5, 6, 7],
6495    [10, 11, 12, 13, 14, 15, 16, 17],
6496    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6497     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6498    [0, 10, 1, 11, 2, 12, 3, 13]}
6499    test_vec_perm! {test_vec_perm_m16x8,
6500    m16x8, vector_bool_short,
6501    [false, false, false, false, false, false, false, false],
6502    [true, true, true, true, true, true, true, true],
6503    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6504     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6505    [false, true, false, true, false, true, false, true]}
6506
6507    test_vec_perm! {test_vec_perm_u32x4,
6508    u32x4, vector_unsigned_int,
6509    [0, 1, 2, 3],
6510    [10, 11, 12, 13],
6511    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6512     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6513    [0, 10, 1, 11]}
6514    test_vec_perm! {test_vec_perm_i32x4,
6515    i32x4, vector_signed_int,
6516    [0, 1, 2, 3],
6517    [10, 11, 12, 13],
6518    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6519     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6520    [0, 10, 1, 11]}
6521    test_vec_perm! {test_vec_perm_m32x4,
6522    m32x4, vector_bool_int,
6523    [false, false, false, false],
6524    [true, true, true, true],
6525    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6526     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6527    [false, true, false, true]}
6528    test_vec_perm! {test_vec_perm_f32x4,
6529    f32x4, vector_float,
6530    [0.0, 1.0, 2.0, 3.0],
6531    [1.0, 1.1, 1.2, 1.3],
6532    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6533     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6534    [0.0, 1.0, 1.0, 1.1]}
6535
6536    test_vec_1! { test_vec_sqrt, vec_sqrt, f32x4,
6537    [core::f32::consts::PI, 1.0, 25.0, 2.0],
6538    [core::f32::consts::PI.sqrt(), 1.0, 5.0, core::f32::consts::SQRT_2] }
6539
6540    test_vec_2! { test_vec_find_any_eq, vec_find_any_eq, i32x4, i32x4 -> i32x4,
6541        [1, -2, 3, -4],
6542        [-5, 3, -7, 8],
6543        [0, 0, !0, 0]
6544    }
6545
6546    test_vec_2! { test_vec_find_any_ne, vec_find_any_ne, i32x4, i32x4 -> i32x4,
6547        [1, -2, 3, -4],
6548        [-5, 3, -7, 8],
6549        [!0, !0, 0, !0]
6550    }
6551
6552    test_vec_2! { test_vec_find_any_eq_idx_1, vec_find_any_eq_idx, i32x4, i32x4 -> i32x4,
6553        [1, 2, 3, 4],
6554        [5, 3, 7, 8],
6555        [0, 8, 0, 0]
6556    }
6557    test_vec_2! { test_vec_find_any_eq_idx_2, vec_find_any_eq_idx, i32x4, i32x4 -> i32x4,
6558        [1, 2, 3, 4],
6559        [5, 6, 7, 8],
6560        [0, 16, 0, 0]
6561    }
6562
6563    test_vec_2! { test_vec_find_any_ne_idx_1, vec_find_any_ne_idx, i32x4, i32x4 -> i32x4,
6564        [1, 2, 3, 4],
6565        [1, 5, 3, 4],
6566        [0, 4, 0, 0]
6567    }
6568    test_vec_2! { test_vec_find_any_ne_idx_2, vec_find_any_ne_idx, i32x4, i32x4 -> i32x4,
6569        [1, 2, 3, 4],
6570        [1, 2, 3, 4],
6571        [0, 16, 0, 0]
6572    }
6573
6574    test_vec_2! { test_vec_find_any_eq_or_0_idx_1, vec_find_any_eq_or_0_idx, i32x4, i32x4 -> i32x4,
6575        [1, 2, 0, 4],
6576        [5, 6, 7, 8],
6577        [0, 8, 0, 0]
6578    }
6579    test_vec_2! { test_vec_find_any_ne_or_0_idx_1, vec_find_any_ne_or_0_idx, i32x4, i32x4 -> i32x4,
6580        [1, 2, 0, 4],
6581        [1, 2, 3, 4],
6582        [0, 8, 0, 0]
6583    }
6584
6585    #[simd_test(enable = "vector")]
6586    fn test_vec_find_any_eq_cc() {
6587        let a = vector_unsigned_int([1, 2, 3, 4]);
6588        let b = vector_unsigned_int([5, 3, 7, 8]);
6589
6590        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6591        assert_eq!(c, 1);
6592        assert_eq!(d.as_array(), &[0, 0, -1, 0]);
6593
6594        let a = vector_unsigned_int([1, 2, 3, 4]);
6595        let b = vector_unsigned_int([5, 6, 7, 8]);
6596        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6597        assert_eq!(c, 3);
6598        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6599    }
6600
6601    #[simd_test(enable = "vector")]
6602    fn test_vec_find_any_ne_cc() {
6603        let a = vector_unsigned_int([1, 2, 3, 4]);
6604        let b = vector_unsigned_int([5, 3, 7, 8]);
6605
6606        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6607        assert_eq!(c, 1);
6608        assert_eq!(d.as_array(), &[-1, -1, 0, -1]);
6609
6610        let a = vector_unsigned_int([1, 2, 3, 4]);
6611        let b = vector_unsigned_int([1, 2, 3, 4]);
6612        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6613        assert_eq!(c, 3);
6614        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6615    }
6616
6617    #[simd_test(enable = "vector")]
6618    fn test_vec_find_any_eq_idx_cc() {
6619        let a = vector_unsigned_int([1, 2, 3, 4]);
6620        let b = vector_unsigned_int([5, 3, 7, 8]);
6621
6622        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6623        assert_eq!(c, 1);
6624        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
6625
6626        let a = vector_unsigned_int([1, 2, 3, 4]);
6627        let b = vector_unsigned_int([5, 6, 7, 8]);
6628        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6629        assert_eq!(c, 3);
6630        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6631    }
6632
6633    #[simd_test(enable = "vector")]
6634    fn test_vec_find_any_ne_idx_cc() {
6635        let a = vector_unsigned_int([5, 2, 3, 4]);
6636        let b = vector_unsigned_int([5, 3, 7, 8]);
6637
6638        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6639        assert_eq!(c, 1);
6640        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6641
6642        let a = vector_unsigned_int([1, 2, 3, 4]);
6643        let b = vector_unsigned_int([1, 2, 3, 4]);
6644        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6645        assert_eq!(c, 3);
6646        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6647    }
6648
6649    #[simd_test(enable = "vector")]
6650    fn test_vec_find_any_eq_or_0_idx_cc() {
6651        // if no element of a matches any element of b with an equal value, and there is at least one element from a with a value of 0
6652        let a = vector_unsigned_int([0, 1, 2, 3]);
6653        let b = vector_unsigned_int([4, 5, 6, 7]);
6654        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6655        assert_eq!(c, 0);
6656        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6657
6658        // if at least one element of a matches any element of b with an equal value, and no elements of a with a value of 0
6659        let a = vector_unsigned_int([1, 2, 3, 4]);
6660        let b = vector_unsigned_int([5, 2, 3, 4]);
6661        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6662        assert_eq!(c, 1);
6663        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6664
6665        // if at least one element of a matches any element of b with an equal value, and there is at least one element from a has a value of 0
6666        let a = vector_unsigned_int([1, 2, 3, 0]);
6667        let b = vector_unsigned_int([1, 2, 3, 4]);
6668        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6669        assert_eq!(c, 2);
6670        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6671
6672        // if no element of a matches any element of b with an equal value, and there is no element from a with a value of 0.
6673        let a = vector_unsigned_int([1, 2, 3, 4]);
6674        let b = vector_unsigned_int([5, 6, 7, 8]);
6675        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6676        assert_eq!(c, 3);
6677        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6678    }
6679
6680    #[simd_test(enable = "vector")]
6681    fn test_vec_find_any_ne_or_0_idx_cc() {
6682        // if no element of a matches any element of b with a not equal value, and there is at least one element from a with a value of 0.
6683        let a = vector_unsigned_int([0, 1, 2, 3]);
6684        let b = vector_unsigned_int([4, 1, 2, 3]);
6685        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6686        assert_eq!(c, 0);
6687        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6688
6689        // if at least one element of a matches any element of b with a not equal value, and no elements of a with a value of 0.
6690        let a = vector_unsigned_int([4, 2, 3, 4]);
6691        let b = vector_unsigned_int([4, 5, 6, 7]);
6692        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6693        assert_eq!(c, 1);
6694        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6695
6696        // if at least one element of a matches any element of b with a not equal value, and there is at least one element from a has a value of 0.
6697        let a = vector_unsigned_int([1, 0, 1, 1]);
6698        let b = vector_unsigned_int([4, 5, 6, 7]);
6699        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6700        assert_eq!(c, 2);
6701        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6702
6703        // if no element of a matches any element of b with a not equal value, and there is no element from a with a value of 0.
6704        let a = vector_unsigned_int([4, 4, 4, 4]);
6705        let b = vector_unsigned_int([4, 5, 6, 7]);
6706        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6707        assert_eq!(c, 3);
6708        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6709    }
6710
6711    #[simd_test(enable = "vector")]
6712    fn test_vector_load() {
6713        let expected = [0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD];
6714
6715        let source: [u32; 8] = [
6716            0xAAAA_AAAA,
6717            0xBBBB_BBBB,
6718            0xCCCC_CCCC,
6719            0xDDDD_DDDD,
6720            0,
6721            0,
6722            0,
6723            0,
6724        ];
6725        assert_eq!(
6726            unsafe { vec_xl::<vector_unsigned_int>(0, source.as_ptr()) }.as_array(),
6727            &expected
6728        );
6729
6730        // offset is in bytes
6731        let source: [u32; 8] = [
6732            0x0000_AAAA,
6733            0xAAAA_BBBB,
6734            0xBBBB_CCCC,
6735            0xCCCC_DDDD,
6736            0xDDDD_0000,
6737            0,
6738            0,
6739            0,
6740        ];
6741        assert_eq!(
6742            unsafe { vec_xl::<vector_unsigned_int>(2, source.as_ptr()) }.as_array(),
6743            &expected
6744        );
6745    }
6746
6747    #[simd_test(enable = "vector")]
6748    fn test_vector_store() {
6749        let vec = vector_unsigned_int([0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD]);
6750
6751        let mut dest = [0u32; 8];
6752        unsafe { vec_xst(vec, 0, dest.as_mut_ptr()) };
6753        assert_eq!(
6754            dest,
6755            [
6756                0xAAAA_AAAA,
6757                0xBBBB_BBBB,
6758                0xCCCC_CCCC,
6759                0xDDDD_DDDD,
6760                0,
6761                0,
6762                0,
6763                0
6764            ]
6765        );
6766
6767        // offset is in bytes
6768        let mut dest = [0u32; 8];
6769        unsafe { vec_xst(vec, 2, dest.as_mut_ptr()) };
6770        assert_eq!(
6771            dest,
6772            [
6773                0x0000_AAAA,
6774                0xAAAA_BBBB,
6775                0xBBBB_CCCC,
6776                0xCCCC_DDDD,
6777                0xDDDD_0000,
6778                0,
6779                0,
6780                0,
6781            ]
6782        );
6783    }
6784
6785    #[simd_test(enable = "vector")]
6786    fn test_vector_lcbb() {
6787        #[repr(align(64))]
6788        struct Align64<T>(T);
6789
6790        static ARRAY: Align64<[u8; 128]> = Align64([0; 128]);
6791
6792        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[64..].as_ptr()) }, 16);
6793        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[63..].as_ptr()) }, 1);
6794        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[56..].as_ptr()) }, 8);
6795        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[48..].as_ptr()) }, 16);
6796    }
6797
6798    test_vec_2! { test_vec_pack, vec_pack, i16x8, i16x8 -> i8x16,
6799        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6800        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6801        [0, 1, -1, 42, -1, 0, 48, -48, -1, 0, 57, -57, 0, 1, -1, 42]
6802    }
6803
6804    test_vec_2! { test_vec_packs, vec_packs, i16x8, i16x8 -> i8x16,
6805        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6806        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6807        [0, 1, -1, 42, 127, -128, 127, -128, 127, -128, 127, -128, 0, 1, -1, 42]
6808    }
6809
6810    test_vec_2! { test_vec_packsu_signed, vec_packsu, i16x8, i16x8 -> u8x16,
6811        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6812        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6813        [0, 1, 0, 42, 255, 0, 255, 0, 255, 0, 255, 0, 0, 1, 0, 42]
6814    }
6815
6816    test_vec_2! { test_vec_packsu_unsigned, vec_packsu, u16x8, u16x8 -> u8x16,
6817        [65535, 32768, 1234, 5678, 16, 8, 4, 2],
6818        [30000, 25000, 20000, 15000, 31, 63, 127, 255],
6819        [255, 255, 255, 255, 16, 8, 4, 2, 255, 255, 255, 255, 31, 63, 127, 255]
6820    }
6821
6822    test_vec_2! { test_vec_rl, vec_rl, u32x4,
6823        [0x12345678, 0x9ABCDEF0, 0x0F0F0F0F, 0x12345678],
6824        [4, 8, 12, 68],
6825        [0x23456781, 0xBCDEF09A, 0xF0F0F0F0, 0x23456781]
6826    }
6827
6828    test_vec_1! { test_vec_unpackh_i, vec_unpackh, i16x8 -> i32x4,
6829        [0x1234, -2, 0x0F0F, -32768, 0, 0, 0, 0],
6830        [0x1234, -2, 0x0F0F, -32768]
6831    }
6832
6833    test_vec_1! { test_vec_unpackh_u, vec_unpackh, u16x8 -> u32x4,
6834        [0x1234, 0xFFFF, 0x0F0F, 0x8000, 0, 0, 0, 0],
6835        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6836    }
6837
6838    test_vec_1! { test_vec_unpackl_i, vec_unpackl, i16x8 -> i32x4,
6839        [0, 0, 0, 0, 0x1234, -2, 0x0F0F, -32768],
6840        [0x1234, -2, 0x0F0F, -32768]
6841    }
6842
6843    test_vec_1! { test_vec_unpackl_u, vec_unpackl, u16x8 -> u32x4,
6844        [0, 0, 0, 0, 0x1234, 0xFFFF, 0x0F0F, 0x8000],
6845        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6846    }
6847
6848    test_vec_2! { test_vec_avg, vec_avg, u32x4,
6849        [2, 1, u32::MAX, 0],
6850        [4, 2, 2, 0],
6851        [3, (1u32 + 2).div_ceil(2), (u32::MAX as u64 + 2u64).div_ceil(2) as u32, 0]
6852    }
6853
6854    test_vec_2! { test_vec_checksum, vec_checksum, u32x4,
6855        [1, 2, 3, u32::MAX],
6856        [5, 6, 7, 8],
6857        [0, 12, 0, 0]
6858    }
6859
6860    test_vec_2! { test_vec_add_u128, vec_add_u128, u8x16,
6861        [0x01, 0x05, 0x0F, 0x1A, 0x2F, 0x3F, 0x50, 0x65,
6862                              0x7A, 0x8F, 0x9A, 0xAD, 0xB0, 0xC3, 0xD5, 0xE8],
6863        [0xF0, 0xEF, 0xC3, 0xB1, 0x92, 0x71, 0x5A, 0x43,
6864                              0x3B, 0x29, 0x13, 0x04, 0xD7, 0xA1, 0x8C, 0x76],
6865        [0xF1, 0xF4, 0xD2, 0xCB, 0xC1, 0xB0, 0xAA, 0xA8, 0xB5, 0xB8, 0xAD, 0xB2, 0x88, 0x65, 0x62, 0x5E]
6866    }
6867
6868    #[simd_test(enable = "vector")]
6869    fn test_vec_addc_u128() {
6870        unsafe {
6871            let a = u128::MAX;
6872            let b = 1u128;
6873
6874            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6875            assert!(a.checked_add(b).is_none());
6876            assert_eq!(d, 1);
6877
6878            let a = 1u128;
6879            let b = 1u128;
6880
6881            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6882            assert!(a.checked_add(b).is_some());
6883            assert_eq!(d, 0);
6884        }
6885    }
6886
6887    #[simd_test(enable = "vector")]
6888    fn test_vec_subc_u128() {
6889        unsafe {
6890            let a = 0u128;
6891            let b = 1u128;
6892
6893            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6894            assert!(a.checked_sub(b).is_none());
6895            assert_eq!(d, 0);
6896
6897            let a = 1u128;
6898            let b = 1u128;
6899
6900            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6901            assert!(a.checked_sub(b).is_some());
6902            assert_eq!(d, 1);
6903        }
6904    }
6905
6906    test_vec_2! { test_vec_mule_u, vec_mule, u16x8, u16x8 -> u32x4,
6907        [0xFFFF, 0, 2, 0, 2, 0, 1, 0],
6908        [0xFFFF, 0, 4, 0, 0xFFFF, 0, 2, 0],
6909        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6910    }
6911
6912    test_vec_2! { test_vec_mule_i, vec_mule, i16x8, i16x8 -> i32x4,
6913        [i16::MIN, 0, -2, 0, 2, 0, 1, 0],
6914        [i16::MIN, 0, 4, 0, i16::MAX, 0, 2, 0],
6915        [0x4000_0000, -8, 0xFFFE, 2]
6916    }
6917
6918    test_vec_2! { test_vec_mulo_u, vec_mulo, u16x8, u16x8 -> u32x4,
6919        [0, 0xFFFF, 0, 2, 0, 2, 0, 1],
6920        [0, 0xFFFF, 0, 4, 0, 0xFFFF, 0, 2],
6921        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6922    }
6923
6924    test_vec_2! { test_vec_mulo_i, vec_mulo, i16x8, i16x8 -> i32x4,
6925        [0, i16::MIN, 0, -2, 0, 2, 0, 1],
6926        [0, i16::MIN, 0, 4, 0, i16::MAX, 0, 2],
6927        [0x4000_0000, -8, 0xFFFE, 2]
6928    }
6929
6930    test_vec_2! { test_vec_mulh_u, vec_mulh, u32x4, u32x4 -> u32x4,
6931        [u32::MAX, 2, 2, 1],
6932        [u32::MAX, 4, u32::MAX, 2],
6933        [u32::MAX - 1, 0, 1, 0]
6934    }
6935
6936    test_vec_2! { test_vec_mulh_i, vec_mulh, i32x4, i32x4 -> i32x4,
6937        [i32::MIN, -2, 2, 1],
6938        [i32::MIN, 4, i32::MAX, 2],
6939        [0x4000_0000, -1, 0, 0]
6940    }
6941
6942    test_vec_2! { test_vec_gfmsum_1, vec_gfmsum, u16x8, u16x8 -> u32x4,
6943        [0x1234, 0x5678, 0x9ABC, 0xDEF0, 0x1357, 0x2468, 0xACE0, 0xBDF0],
6944        [0xFFFF, 0x0001, 0x8000, 0x7FFF, 0xAAAA, 0x5555, 0x1234, 0x5678],
6945        [0xE13A794, 0x68764A50, 0x94AA3E, 0x2C93F300]
6946    }
6947
6948    test_vec_2! { test_vec_gfmsum_2, vec_gfmsum, u16x8, u16x8 -> u32x4,
6949        [0x0000, 0xFFFF, 0xAAAA, 0x5555, 0x1234, 0x5678, 0x9ABC, 0xDEF0],
6950        [0xFFFF, 0x0000, 0x5555, 0xAAAA, 0x0001, 0x8000, 0x7FFF, 0x1357],
6951        [0, 0, 0x2B3C1234, 0x3781D244]
6952    }
6953
6954    #[simd_test(enable = "vector")]
6955    fn test_vec_gfmsum_128() {
6956        let a = vector_unsigned_long_long([1, 2]);
6957        let b = vector_unsigned_long_long([3, 4]);
6958
6959        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6960        assert_eq!(d, 11);
6961
6962        let a = vector_unsigned_long_long([0x0101010101010101, 0x0202020202020202]);
6963        let b = vector_unsigned_long_long([0x0404040404040404, 0x0505050505050505]);
6964
6965        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6966        assert_eq!(d, 0xE000E000E000E000E000E000E000E);
6967    }
6968
6969    #[simd_test(enable = "vector-enhancements-1")]
6970    fn test_vec_bperm_u128() {
6971        let a = vector_unsigned_char([65, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]);
6972        let b = vector_unsigned_char([
6973            0, 0, 0, 0, 1, 1, 1, 1, 128, 128, 128, 128, 255, 255, 255, 255,
6974        ]);
6975        let d = unsafe { vec_bperm_u128(a, b) };
6976        assert_eq!(d.as_array(), &[0xF00, 0]);
6977    }
6978
6979    #[simd_test(enable = "vector")]
6980    fn test_vec_sel() {
6981        let a = vector_signed_int([1, 2, 3, 4]);
6982        let b = vector_signed_int([5, 6, 7, 8]);
6983
6984        let e = vector_unsigned_int([9, 10, 11, 12]);
6985        let f = vector_unsigned_int([9, 9, 11, 11]);
6986
6987        let c: vector_bool_int = unsafe { simd_eq(e, f) };
6988        assert_eq!(c.as_array(), &[!0, 0, !0, 0]);
6989        let d: vector_signed_int = unsafe { vec_sel(a, b, c) };
6990        assert_eq!(d.as_array(), &[5, 2, 7, 4]);
6991    }
6992
6993    #[simd_test(enable = "vector")]
6994    fn test_vec_gather_element() {
6995        let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19];
6996        let a2: [u32; 10] = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29];
6997
6998        let v1 = vector_unsigned_int([1, 2, 3, 4]);
6999        let v2 = vector_unsigned_int([1, 2, 3, 4]);
7000
7001        let sizeof_int = core::mem::size_of::<u32>() as u32;
7002        let v3 = vector_unsigned_int([
7003            5 * sizeof_int,
7004            8 * sizeof_int,
7005            9 * sizeof_int,
7006            6 * sizeof_int,
7007        ]);
7008
7009        unsafe {
7010            let d1 = vec_gather_element::<_, 0>(v1, v3, a1.as_ptr());
7011            assert_eq!(d1.as_array(), &[15, 2, 3, 4]);
7012            let d2 = vec_gather_element::<_, 0>(v2, v3, a2.as_ptr());
7013            assert_eq!(d2.as_array(), &[25, 2, 3, 4]);
7014        }
7015    }
7016
7017    #[simd_test(enable = "vector")]
7018    fn test_vec_fp_test_data_class() {
7019        let mut cc = 42;
7020
7021        let v1 = vector_double([0.0, f64::NAN]);
7022        let v2 = vector_double([f64::INFINITY, 1.0]);
7023        let v3 = vector_double([1.0, 2.0]);
7024
7025        unsafe {
7026            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_ZERO>(v1, &mut cc);
7027            assert_eq!(cc, 1);
7028            assert_eq!(d.as_array(), &[!0, 0]);
7029
7030            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NAN>(v1, &mut cc);
7031            assert_eq!(cc, 1);
7032            assert_eq!(d.as_array(), &[0, !0]);
7033
7034            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY>(v2, &mut cc);
7035            assert_eq!(cc, 1);
7036            assert_eq!(d.as_array(), &[!0, 0]);
7037
7038            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY_N>(v2, &mut cc);
7039            assert_eq!(cc, 3);
7040            assert_eq!(d.as_array(), &[0, 0]);
7041
7042            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v2, &mut cc);
7043            assert_eq!(cc, 1);
7044            assert_eq!(d.as_array(), &[0, !0]);
7045
7046            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v3, &mut cc);
7047            assert_eq!(cc, 0);
7048            assert_eq!(d.as_array(), &[!0, !0]);
7049        }
7050    }
7051
7052    #[simd_test(enable = "vector")]
7053    fn test_vec_fp_any_all_nan_numeric() {
7054        unsafe {
7055            assert_eq!(
7056                vec_all_nan(vector_double([f64::NAN, f64::NAN])),
7057                i32::from(true)
7058            );
7059            assert_eq!(
7060                vec_all_nan(vector_double([f64::NAN, 1.0])),
7061                i32::from(false)
7062            );
7063            assert_eq!(vec_all_nan(vector_double([0.0, 1.0])), i32::from(false));
7064
7065            assert_eq!(
7066                vec_any_nan(vector_double([f64::NAN, f64::NAN])),
7067                i32::from(true)
7068            );
7069            assert_eq!(vec_any_nan(vector_double([f64::NAN, 1.0])), i32::from(true));
7070            assert_eq!(vec_any_nan(vector_double([0.0, 1.0])), i32::from(false));
7071
7072            assert_eq!(
7073                vec_all_numeric(vector_double([f64::NAN, f64::NAN])),
7074                i32::from(false)
7075            );
7076            assert_eq!(
7077                vec_all_numeric(vector_double([f64::NAN, 1.0])),
7078                i32::from(false)
7079            );
7080            assert_eq!(vec_all_numeric(vector_double([0.0, 1.0])), i32::from(true));
7081
7082            assert_eq!(
7083                vec_any_numeric(vector_double([f64::NAN, f64::NAN])),
7084                i32::from(false)
7085            );
7086            assert_eq!(
7087                vec_any_numeric(vector_double([f64::NAN, 1.0])),
7088                i32::from(true)
7089            );
7090            assert_eq!(vec_any_numeric(vector_double([0.0, 1.0])), i32::from(true));
7091
7092            // "numeric" means "not NaN". infinities are numeric
7093            assert_eq!(
7094                vec_all_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
7095                i32::from(true)
7096            );
7097            assert_eq!(
7098                vec_any_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
7099                i32::from(true)
7100            );
7101        }
7102    }
7103
7104    #[simd_test(enable = "vector")]
7105    fn test_vec_test_mask() {
7106        unsafe {
7107            let v = vector_unsigned_long_long([0xFF00FF00FF00FF00; 2]);
7108            let m = vector_unsigned_long_long([0x0000FF000000FF00; 2]);
7109            assert_eq!(vec_test_mask(v, m), 3);
7110
7111            let v = vector_unsigned_long_long([u64::MAX; 2]);
7112            let m = vector_unsigned_long_long([0; 2]);
7113            assert_eq!(vec_test_mask(v, m), 0);
7114
7115            let v = vector_unsigned_long_long([0; 2]);
7116            let m = vector_unsigned_long_long([u64::MAX; 2]);
7117            assert_eq!(vec_test_mask(v, m), 0);
7118
7119            let v = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
7120            let m = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
7121            assert_eq!(vec_test_mask(v, m), 3);
7122        }
7123    }
7124
7125    #[simd_test(enable = "vector-enhancements-2")]
7126    fn test_vec_search_string_cc() {
7127        unsafe {
7128            let b = vector_unsigned_char(*b"ABCD------------");
7129            let c = vector_unsigned_char([4; 16]);
7130
7131            let haystack = vector_unsigned_char(*b"__ABCD__________");
7132            let (result, d) = vec_search_string_cc(haystack, b, c);
7133            assert_eq!(result.as_array()[7], 2);
7134            assert_eq!(d, 2);
7135
7136            let haystack = vector_unsigned_char(*b"___ABCD_________");
7137            let (result, d) = vec_search_string_cc(haystack, b, c);
7138            assert_eq!(result.as_array()[7], 3);
7139            assert_eq!(d, 2);
7140
7141            let haystack = vector_unsigned_char(*b"________________");
7142            let (result, d) = vec_search_string_cc(haystack, b, c);
7143            assert_eq!(result.as_array()[7], 16);
7144            assert_eq!(d, 0);
7145
7146            let haystack = vector_unsigned_char(*b"______\0_________");
7147            let (result, d) = vec_search_string_cc(haystack, b, c);
7148            assert_eq!(result.as_array()[7], 16);
7149            assert_eq!(d, 0);
7150
7151            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
7152            let (result, d) = vec_search_string_cc(haystack, b, c);
7153            assert_eq!(result.as_array()[7], 9);
7154            assert_eq!(d, 2);
7155        }
7156    }
7157
7158    #[simd_test(enable = "vector-enhancements-2")]
7159    fn test_vec_search_string_until_zero_cc() {
7160        unsafe {
7161            let b = vector_unsigned_char(*b"ABCD\0\0\0\0\0\0\0\0\0\0\0\0");
7162            let c = vector_unsigned_char([16; 16]);
7163
7164            let haystack = vector_unsigned_char(*b"__ABCD__________");
7165            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7166            assert_eq!(result.as_array()[7], 2);
7167            assert_eq!(d, 2);
7168
7169            let haystack = vector_unsigned_char(*b"___ABCD_________");
7170            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7171            assert_eq!(result.as_array()[7], 3);
7172            assert_eq!(d, 2);
7173
7174            let haystack = vector_unsigned_char(*b"________________");
7175            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7176            assert_eq!(result.as_array()[7], 16);
7177            assert_eq!(d, 0);
7178
7179            let haystack = vector_unsigned_char(*b"______\0_________");
7180            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7181            assert_eq!(result.as_array()[7], 16);
7182            assert_eq!(d, 1);
7183
7184            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
7185            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
7186            assert_eq!(result.as_array()[7], 16);
7187            assert_eq!(d, 1);
7188        }
7189    }
7190
7191    #[simd_test(enable = "vector")]
7192    fn test_vec_doublee() {
7193        unsafe {
7194            let v = vector_float([1.0, 2.0, 3.0, 4.0]);
7195            assert_eq!(vec_doublee(v).as_array(), &[1.0, 3.0]);
7196
7197            let v = vector_float([f32::NAN, 2.0, f32::INFINITY, 4.0]);
7198            let d = vec_doublee(v);
7199            assert!(d.as_array()[0].is_nan());
7200            assert_eq!(d.as_array()[1], f64::INFINITY);
7201        }
7202    }
7203
7204    #[simd_test(enable = "vector")]
7205    fn test_vec_floate() {
7206        // NOTE: indices 1 and 3 can have an arbitrary value. With the C version
7207        // these are poison values, our version initializes the memory but its
7208        // value still should not be relied upon by application code.
7209        unsafe {
7210            let v = vector_double([1.0, 2.0]);
7211            let d = vec_floate(v);
7212            assert_eq!(d.as_array()[0], 1.0);
7213            assert_eq!(d.as_array()[2], 2.0);
7214
7215            let v = vector_double([f64::NAN, f64::INFINITY]);
7216            let d = vec_floate(v);
7217            assert!(d.as_array()[0].is_nan());
7218            assert_eq!(d.as_array()[2], f32::INFINITY);
7219
7220            let v = vector_double([f64::MIN, f64::MAX]);
7221            let d = vec_floate(v);
7222            assert_eq!(d.as_array()[0], f64::MIN as f32);
7223            assert_eq!(d.as_array()[2], f64::MAX as f32);
7224        }
7225    }
7226
7227    #[simd_test(enable = "vector")]
7228    fn test_vec_extend_s64() {
7229        unsafe {
7230            let v = vector_signed_char([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
7231            assert_eq!(vec_extend_s64(v).as_array(), &[7, 15]);
7232
7233            let v = vector_signed_short([0, 1, 2, 3, 4, 5, 6, 7]);
7234            assert_eq!(vec_extend_s64(v).as_array(), &[3, 7]);
7235
7236            let v = vector_signed_int([0, 1, 2, 3]);
7237            assert_eq!(vec_extend_s64(v).as_array(), &[1, 3]);
7238        }
7239    }
7240
7241    #[simd_test(enable = "vector")]
7242    fn test_vec_signed() {
7243        unsafe {
7244            let v = vector_float([1.0, 2.5, -2.5, -0.0]);
7245            assert_eq!(vec_signed(v).as_array(), &[1, 2, -2, 0]);
7246
7247            let v = vector_double([2.5, -2.5]);
7248            assert_eq!(vec_signed(v).as_array(), &[2, -2]);
7249        }
7250    }
7251
7252    #[simd_test(enable = "vector")]
7253    fn test_vec_unsigned() {
7254        // NOTE: converting a negative floating point value is UB!
7255        unsafe {
7256            let v = vector_float([1.0, 2.5, 3.5, 0.0]);
7257            assert_eq!(vec_unsigned(v).as_array(), &[1, 2, 3, 0]);
7258
7259            let v = vector_double([2.5, 3.5]);
7260            assert_eq!(vec_unsigned(v).as_array(), &[2, 3]);
7261        }
7262    }
7263
7264    #[simd_test(enable = "vector")]
7265    fn test_vec_cp_until_zero() {
7266        unsafe {
7267            let v = vector_signed_int([1, 2, 3, 4]);
7268            let d = vec_cp_until_zero(v);
7269            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7270
7271            let v = vector_signed_int([1, 2, 0, 4]);
7272            let d = vec_cp_until_zero(v);
7273            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7274        }
7275    }
7276
7277    #[simd_test(enable = "vector")]
7278    fn test_vec_cp_until_zero_cc() {
7279        unsafe {
7280            let v = vector_signed_int([1, 2, 3, 4]);
7281            let (d, cc) = vec_cp_until_zero_cc(v);
7282            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7283            assert_eq!(cc, 3);
7284
7285            let v = vector_signed_int([1, 2, 0, 4]);
7286            let (d, cc) = vec_cp_until_zero_cc(v);
7287            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7288            assert_eq!(cc, 0);
7289        }
7290    }
7291
7292    #[simd_test(enable = "vector-enhancements-1")]
7293    fn test_vec_msum_u128() {
7294        let a = vector_unsigned_long_long([1, 2]);
7295        let b = vector_unsigned_long_long([3, 4]);
7296
7297        unsafe {
7298            let c: vector_unsigned_char = transmute(100u128);
7299
7300            let d: u128 = transmute(vec_msum_u128::<0>(a, b, c));
7301            assert_eq!(d, (1 * 3) + (2 * 4) + 100);
7302
7303            let d: u128 = transmute(vec_msum_u128::<4>(a, b, c));
7304            assert_eq!(d, (1 * 3) + (2 * 4) * 2 + 100);
7305
7306            let d: u128 = transmute(vec_msum_u128::<8>(a, b, c));
7307            assert_eq!(d, (1 * 3) * 2 + (2 * 4) + 100);
7308
7309            let d: u128 = transmute(vec_msum_u128::<12>(a, b, c));
7310            assert_eq!(d, (1 * 3) * 2 + (2 * 4) * 2 + 100);
7311        }
7312    }
7313
7314    #[simd_test(enable = "vector")]
7315    fn test_vec_sld() {
7316        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7317        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7318
7319        unsafe {
7320            let d = vec_sld::<_, 4>(a, b);
7321            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7322        }
7323    }
7324
7325    #[simd_test(enable = "vector")]
7326    fn test_vec_sldw() {
7327        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7328        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7329
7330        unsafe {
7331            let d = vec_sldw::<_, 1>(a, b);
7332            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7333        }
7334    }
7335
7336    #[simd_test(enable = "vector-enhancements-2")]
7337    fn test_vec_sldb() {
7338        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7339        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7340
7341        unsafe {
7342            let d = vec_sldb::<_, 4>(a, b);
7343            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAB]);
7344        }
7345    }
7346
7347    #[simd_test(enable = "vector-enhancements-2")]
7348    fn test_vec_srdb() {
7349        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7350        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7351
7352        unsafe {
7353            let d = vec_srdb::<_, 4>(a, b);
7354            assert_eq!(d.as_array(), &[0xABBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7355        }
7356    }
7357
7358    const GT: u32 = 0x20000000;
7359    const LT: u32 = 0x40000000;
7360    const EQ: u32 = 0x80000000;
7361
7362    #[simd_test(enable = "vector")]
7363    fn test_vec_cmprg() {
7364        let a = vector_unsigned_int([11, 22, 33, 44]);
7365        let b = vector_unsigned_int([10, 20, 30, 40]);
7366
7367        let c = vector_unsigned_int([GT, LT, GT, LT]);
7368        let d = unsafe { vec_cmprg(a, b, c) };
7369        assert_eq!(d.as_array(), &[!0, 0, !0, 0]);
7370
7371        let c = vector_unsigned_int([GT, LT, 0, 0]);
7372        let d = unsafe { vec_cmprg(a, b, c) };
7373        assert_eq!(d.as_array(), &[!0, 0, 0, 0]);
7374
7375        let a = vector_unsigned_int([11, 22, 33, 30]);
7376        let b = vector_unsigned_int([10, 20, 30, 30]);
7377
7378        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7379        let d = unsafe { vec_cmprg(a, b, c) };
7380        assert_eq!(d.as_array(), &[!0, 0, 0, !0]);
7381    }
7382
7383    #[simd_test(enable = "vector")]
7384    fn test_vec_cmpnrg() {
7385        let a = vector_unsigned_int([11, 22, 33, 44]);
7386        let b = vector_unsigned_int([10, 20, 30, 40]);
7387
7388        let c = vector_unsigned_int([GT, LT, GT, LT]);
7389        let d = unsafe { vec_cmpnrg(a, b, c) };
7390        assert_eq!(d.as_array(), &[0, !0, 0, !0]);
7391
7392        let c = vector_unsigned_int([GT, LT, 0, 0]);
7393        let d = unsafe { vec_cmpnrg(a, b, c) };
7394        assert_eq!(d.as_array(), &[0, !0, !0, !0]);
7395
7396        let a = vector_unsigned_int([11, 22, 33, 30]);
7397        let b = vector_unsigned_int([10, 20, 30, 30]);
7398
7399        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7400        let d = unsafe { vec_cmpnrg(a, b, c) };
7401        assert_eq!(d.as_array(), &[0, !0, !0, 0]);
7402    }
7403
7404    #[simd_test(enable = "vector")]
7405    fn test_vec_cmprg_idx() {
7406        let a = vector_unsigned_int([1, 11, 22, 33]);
7407        let b = vector_unsigned_int([10, 20, 30, 40]);
7408
7409        let c = vector_unsigned_int([GT, LT, GT, LT]);
7410        let d = unsafe { vec_cmprg_idx(a, b, c) };
7411        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7412    }
7413
7414    #[simd_test(enable = "vector")]
7415    fn test_vec_cmpnrg_idx() {
7416        let a = vector_unsigned_int([1, 11, 22, 33]);
7417        let b = vector_unsigned_int([10, 20, 30, 40]);
7418
7419        let c = vector_unsigned_int([GT, LT, GT, LT]);
7420        let d = unsafe { vec_cmpnrg_idx(a, b, c) };
7421        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
7422    }
7423
7424    #[simd_test(enable = "vector")]
7425    fn test_vec_cmprg_or_0_idx() {
7426        let a = vector_unsigned_int([1, 0, 22, 33]);
7427        let b = vector_unsigned_int([10, 20, 30, 40]);
7428
7429        let c = vector_unsigned_int([GT, LT, GT, LT]);
7430        let d = unsafe { vec_cmprg_or_0_idx(a, b, c) };
7431        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7432    }
7433
7434    #[simd_test(enable = "vector")]
7435    fn test_vec_cmpnrg_or_0_idx() {
7436        let a = vector_unsigned_int([11, 33, 0, 22]);
7437        let b = vector_unsigned_int([10, 20, 30, 40]);
7438
7439        let c = vector_unsigned_int([GT, LT, GT, LT]);
7440        let d = unsafe { vec_cmpnrg_or_0_idx(a, b, c) };
7441        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
7442    }
7443
7444    test_vec_2! { test_vec_cmpgt, vec_cmpgt, f32x4, f32x4 -> i32x4,
7445        [1.0, f32::NAN, f32::NAN, 3.14],
7446        [2.0, f32::NAN, 5.0, 2.0],
7447        [0, 0, 0, !0]
7448    }
7449
7450    test_vec_2! { test_vec_cmpge, vec_cmpge, f32x4, f32x4 -> i32x4,
7451        [1.0, f32::NAN, f32::NAN, 3.14],
7452        [1.0, f32::NAN, 5.0, 2.0],
7453        [!0, 0, 0, !0]
7454    }
7455
7456    test_vec_2! { test_vec_cmplt, vec_cmplt, f32x4, f32x4 -> i32x4,
7457        [1.0, f32::NAN, f32::NAN, 2.0],
7458        [2.0, f32::NAN, 5.0, 2.0],
7459        [!0, 0, 0, 0]
7460    }
7461
7462    test_vec_2! { test_vec_cmple, vec_cmple, f32x4, f32x4 -> i32x4,
7463        [1.0, f32::NAN, f32::NAN, 2.0],
7464        [1.0, f32::NAN, 5.0, 3.14],
7465        [!0, 0, 0, !0]
7466    }
7467
7468    test_vec_2! { test_vec_cmpeq, vec_cmpeq, f32x4, f32x4 -> i32x4,
7469        [1.0, f32::NAN, f32::NAN, 2.0],
7470        [1.0, f32::NAN, 5.0, 3.14],
7471        [!0, 0, 0, 0]
7472    }
7473
7474    test_vec_2! { test_vec_cmpne, vec_cmpne, f32x4, f32x4 -> i32x4,
7475        [1.0, f32::NAN, f32::NAN, 2.0],
7476        [1.0, f32::NAN, 5.0, 3.14],
7477        [0, !0, !0, !0]
7478    }
7479
7480    #[simd_test(enable = "vector")]
7481    fn test_vec_meadd() {
7482        let a = vector_unsigned_short([1, 0, 2, 0, 3, 0, 4, 0]);
7483        let b = vector_unsigned_short([5, 0, 6, 0, 7, 0, 8, 0]);
7484        let c = vector_unsigned_int([2, 2, 2, 2]);
7485
7486        let d = unsafe { vec_meadd(a, b, c) };
7487        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7488
7489        let a = vector_signed_short([1, 0, 2, 0, 3, 0, 4, 0]);
7490        let b = vector_signed_short([5, 0, 6, 0, 7, 0, 8, 0]);
7491        let c = vector_signed_int([2, -2, 2, -2]);
7492
7493        let d = unsafe { vec_meadd(a, b, c) };
7494        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7495    }
7496
7497    #[simd_test(enable = "vector")]
7498    fn test_vec_moadd() {
7499        let a = vector_unsigned_short([0, 1, 0, 2, 0, 3, 0, 4]);
7500        let b = vector_unsigned_short([0, 5, 0, 6, 0, 7, 0, 8]);
7501        let c = vector_unsigned_int([2, 2, 2, 2]);
7502
7503        let d = unsafe { vec_moadd(a, b, c) };
7504        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7505
7506        let a = vector_signed_short([0, 1, 0, 2, 0, 3, 0, 4]);
7507        let b = vector_signed_short([0, 5, 0, 6, 0, 7, 0, 8]);
7508        let c = vector_signed_int([2, -2, 2, -2]);
7509
7510        let d = unsafe { vec_moadd(a, b, c) };
7511        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7512    }
7513
7514    #[simd_test(enable = "vector")]
7515    fn test_vec_mhadd() {
7516        let a = vector_unsigned_int([1, 2, 3, 4]);
7517        let b = vector_unsigned_int([5, 6, 7, 8]);
7518        let c = vector_unsigned_int([u32::MAX; 4]);
7519
7520        let d = unsafe { vec_mhadd(a, b, c) };
7521        assert_eq!(d.as_array(), &[1, 1, 1, 1]);
7522
7523        let a = vector_signed_int([-1, -2, -3, -4]);
7524        let b = vector_signed_int([5, 6, 7, 8]);
7525        let c = vector_signed_int([i32::MIN; 4]);
7526
7527        let d = unsafe { vec_mhadd(a, b, c) };
7528        assert_eq!(d.as_array(), &[-1, -1, -1, -1]);
7529    }
7530
7531    #[simd_test(enable = "vector")]
7532    fn test_vec_mladd() {
7533        let a = vector_unsigned_int([1, 2, 3, 4]);
7534        let b = vector_unsigned_int([5, 6, 7, 8]);
7535        let c = vector_unsigned_int([2, 2, 2, 2]);
7536
7537        let d = unsafe { vec_mladd(a, b, c) };
7538        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7539
7540        let a = vector_signed_int([-1, -2, -3, -4]);
7541        let b = vector_signed_int([5, 6, 7, 8]);
7542        let c = vector_signed_int([2, 2, 2, 2]);
7543
7544        let d = unsafe { vec_mladd(a, b, c) };
7545        assert_eq!(d.as_array(), &[-3, -10, -19, -30]);
7546    }
7547
7548    #[simd_test(enable = "vector")]
7549    fn test_vec_extract() {
7550        let v = vector_unsigned_int([1, 2, 3, 4]);
7551
7552        assert_eq!(unsafe { vec_extract(v, 1) }, 2);
7553        assert_eq!(unsafe { vec_extract(v, 4 + 2) }, 3);
7554    }
7555
7556    #[simd_test(enable = "vector")]
7557    fn test_vec_insert() {
7558        let mut v = vector_unsigned_int([1, 2, 3, 4]);
7559
7560        v = unsafe { vec_insert(42, v, 1) };
7561        assert_eq!(v.as_array(), &[1, 42, 3, 4]);
7562
7563        v = unsafe { vec_insert(64, v, 6) };
7564        assert_eq!(v.as_array(), &[1, 42, 64, 4]);
7565    }
7566
7567    #[simd_test(enable = "vector")]
7568    fn test_vec_promote() {
7569        let v: vector_unsigned_int = unsafe { vec_promote(42, 1).assume_init() };
7570        assert_eq!(v.as_array(), &[0, 42, 0, 0]);
7571    }
7572
7573    #[simd_test(enable = "vector")]
7574    fn test_vec_insert_and_zero() {
7575        let v = unsafe { vec_insert_and_zero::<vector_unsigned_int>(&42u32) };
7576        assert_eq!(v.as_array(), vector_unsigned_int([0, 42, 0, 0]).as_array());
7577    }
7578}
core/stdarch/crates/core_arch/src/s390x/vector.rs

core/stdarch/crates/core_arch/src/s390x/
vector.rs
