core/stdarch/crates/core_arch/src/x86/
kl.rs

1//! AES Key Locker Intrinsics
2//!
3//! The Intrinsics here correspond to those in the `keylockerintrin.h` C header.
4
5use crate::core_arch::x86::__m128i;
6use crate::ptr;
7
8#[cfg(test)]
9use stdarch_test::assert_instr;
10
11#[repr(C, packed)]
12struct EncodeKey128Output(u32, __m128i, __m128i, __m128i, __m128i, __m128i, __m128i);
13
14#[repr(C, packed)]
15struct EncodeKey256Output(
16    u32,
17    __m128i,
18    __m128i,
19    __m128i,
20    __m128i,
21    __m128i,
22    __m128i,
23    __m128i,
24);
25
26#[repr(C, packed)]
27struct AesOutput(u8, __m128i);
28
29#[repr(C, packed)]
30struct WideAesOutput(
31    u8,
32    __m128i,
33    __m128i,
34    __m128i,
35    __m128i,
36    __m128i,
37    __m128i,
38    __m128i,
39    __m128i,
40);
41
42#[allow(improper_ctypes)]
43unsafe extern "unadjusted" {
44    #[link_name = "llvm.x86.loadiwkey"]
45    fn loadiwkey(integrity_key: __m128i, key_lo: __m128i, key_hi: __m128i, control: u32);
46
47    #[link_name = "llvm.x86.encodekey128"]
48    fn encodekey128(key_metadata: u32, key: __m128i) -> EncodeKey128Output;
49    #[link_name = "llvm.x86.encodekey256"]
50    fn encodekey256(key_metadata: u32, key_lo: __m128i, key_hi: __m128i) -> EncodeKey256Output;
51
52    #[link_name = "llvm.x86.aesenc128kl"]
53    fn aesenc128kl(data: __m128i, handle: *const u8) -> AesOutput;
54    #[link_name = "llvm.x86.aesdec128kl"]
55    fn aesdec128kl(data: __m128i, handle: *const u8) -> AesOutput;
56    #[link_name = "llvm.x86.aesenc256kl"]
57    fn aesenc256kl(data: __m128i, handle: *const u8) -> AesOutput;
58    #[link_name = "llvm.x86.aesdec256kl"]
59    fn aesdec256kl(data: __m128i, handle: *const u8) -> AesOutput;
60
61    #[link_name = "llvm.x86.aesencwide128kl"]
62    fn aesencwide128kl(
63        handle: *const u8,
64        i0: __m128i,
65        i1: __m128i,
66        i2: __m128i,
67        i3: __m128i,
68        i4: __m128i,
69        i5: __m128i,
70        i6: __m128i,
71        i7: __m128i,
72    ) -> WideAesOutput;
73    #[link_name = "llvm.x86.aesdecwide128kl"]
74    fn aesdecwide128kl(
75        handle: *const u8,
76        i0: __m128i,
77        i1: __m128i,
78        i2: __m128i,
79        i3: __m128i,
80        i4: __m128i,
81        i5: __m128i,
82        i6: __m128i,
83        i7: __m128i,
84    ) -> WideAesOutput;
85    #[link_name = "llvm.x86.aesencwide256kl"]
86    fn aesencwide256kl(
87        handle: *const u8,
88        i0: __m128i,
89        i1: __m128i,
90        i2: __m128i,
91        i3: __m128i,
92        i4: __m128i,
93        i5: __m128i,
94        i6: __m128i,
95        i7: __m128i,
96    ) -> WideAesOutput;
97    #[link_name = "llvm.x86.aesdecwide256kl"]
98    fn aesdecwide256kl(
99        handle: *const u8,
100        i0: __m128i,
101        i1: __m128i,
102        i2: __m128i,
103        i3: __m128i,
104        i4: __m128i,
105        i5: __m128i,
106        i6: __m128i,
107        i7: __m128i,
108    ) -> WideAesOutput;
109}
110
111/// Load internal wrapping key (IWKey). The 32-bit unsigned integer `control` specifies IWKey's KeySource
112/// and whether backing up the key is permitted. IWKey's 256-bit encryption key is loaded from `key_lo`
113/// and `key_hi`.
114///
115///  - `control[0]`: NoBackup bit. If set, the IWKey cannot be backed up.
116///  - `control[1:4]`: KeySource bits. These bits specify the encoding method of the IWKey. The only
117///    allowed values are `0` (AES GCM SIV wrapping algorithm with the specified key) and `1` (AES GCM
118///    SIV wrapping algorithm with random keys enforced by hardware). After calling `_mm_loadiwkey` with
119///    KeySource set to `1`, software must check `ZF` to ensure that the key was loaded successfully.
120///    Using any other value may result in a General Protection Exception.
121///  - `control[5:31]`: Reserved for future use, must be set to `0`.
122///
123/// Note that setting the NoBackup bit and using the KeySource value `1` requires hardware support. These
124/// permissions can be found by calling `__cpuid(0x19)` and checking the `ECX[0:1]` bits. Failing to follow
125/// these restrictions may result in a General Protection Exception.
126///
127/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_loadiwkey)
128#[inline]
129#[target_feature(enable = "kl")]
130#[stable(feature = "keylocker_x86", since = "1.89.0")]
131#[cfg_attr(test, assert_instr(loadiwkey))]
132pub unsafe fn _mm_loadiwkey(
133    control: u32,
134    integrity_key: __m128i,
135    key_lo: __m128i,
136    key_hi: __m128i,
137) {
138    loadiwkey(integrity_key, key_lo, key_hi, control);
139}
140
141/// Wrap a 128-bit AES key into a 384-bit key handle and stores it in `handle`. Returns the `control`
142/// parameter used to create the IWKey.
143///
144///  - `key_params[0]`: If set, this key can only be used by the Kernel.
145///  - `key_params[1]`: If set, this key can not be used to encrypt.
146///  - `key_params[2]`: If set, this key can not be used to decrypt.
147///  - `key_params[31:3]`: Reserved for future use, must be set to `0`.
148///
149/// Note that these restrictions need hardware support, and the supported restrictions can be found by
150/// calling `__cpuid(0x19)` and checking the `EAX[0:2]` bits. Failing to follow these restrictions may
151/// result in a General Protection Exception.
152///
153/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_encodekey128_u32)
154#[inline]
155#[target_feature(enable = "kl")]
156#[stable(feature = "keylocker_x86", since = "1.89.0")]
157#[cfg_attr(test, assert_instr(encodekey128))]
158pub unsafe fn _mm_encodekey128_u32(key_params: u32, key: __m128i, handle: *mut u8) -> u32 {
159    let EncodeKey128Output(control, key0, key1, key2, _, _, _) = encodekey128(key_params, key);
160    ptr::write_unaligned(handle.cast(), [key0, key1, key2]);
161    control
162}
163
164/// Wrap a 256-bit AES key into a 512-bit key handle and stores it in `handle`. Returns the `control`
165/// parameter used to create the IWKey.
166///
167///  - `key_params[0]`: If set, this key can only be used by the Kernel.
168///  - `key_params[1]`: If set, this key can not be used to encrypt.
169///  - `key_params[2]`: If set, this key can not be used to decrypt.
170///  - `key_params[31:3]`: Reserved for future use, must be set to `0`.
171///
172/// Note that these restrictions need hardware support, and the supported restrictions can be found by
173/// calling `__cpuid(0x19)` and checking the `EAX[0:2]` bits. Failing to follow these restrictions may
174/// result in a General Protection Exception.
175///
176/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_encodekey256_u32)
177#[inline]
178#[target_feature(enable = "kl")]
179#[stable(feature = "keylocker_x86", since = "1.89.0")]
180#[cfg_attr(test, assert_instr(encodekey256))]
181pub unsafe fn _mm_encodekey256_u32(
182    key_params: u32,
183    key_lo: __m128i,
184    key_hi: __m128i,
185    handle: *mut u8,
186) -> u32 {
187    let EncodeKey256Output(control, key0, key1, key2, key3, _, _, _) =
188        encodekey256(key_params, key_lo, key_hi);
189    ptr::write_unaligned(handle.cast(), [key0, key1, key2, key3]);
190    control
191}
192
193/// Encrypt 10 rounds of unsigned 8-bit integers in `input` using 128-bit AES key specified in the
194/// 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
195/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
196/// due to a handle violation.
197///
198/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenc128kl_u8)
199#[inline]
200#[target_feature(enable = "kl")]
201#[stable(feature = "keylocker_x86", since = "1.89.0")]
202#[cfg_attr(test, assert_instr(aesenc128kl))]
203pub unsafe fn _mm_aesenc128kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
204    let AesOutput(status, result) = aesenc128kl(input, handle);
205    *output = result;
206    status
207}
208
209/// Decrypt 10 rounds of unsigned 8-bit integers in `input` using 128-bit AES key specified in the
210/// 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
211/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
212/// due to a handle violation.
213///
214/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec128kl_u8)
215#[inline]
216#[target_feature(enable = "kl")]
217#[stable(feature = "keylocker_x86", since = "1.89.0")]
218#[cfg_attr(test, assert_instr(aesdec128kl))]
219pub unsafe fn _mm_aesdec128kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
220    let AesOutput(status, result) = aesdec128kl(input, handle);
221    *output = result;
222    status
223}
224
225/// Encrypt 14 rounds of unsigned 8-bit integers in `input` using 256-bit AES key specified in the
226/// 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
227/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
228/// due to a handle violation.
229///
230/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenc256kl_u8)
231#[inline]
232#[target_feature(enable = "kl")]
233#[stable(feature = "keylocker_x86", since = "1.89.0")]
234#[cfg_attr(test, assert_instr(aesenc256kl))]
235pub unsafe fn _mm_aesenc256kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
236    let AesOutput(status, result) = aesenc256kl(input, handle);
237    *output = result;
238    status
239}
240
241/// Decrypt 14 rounds of unsigned 8-bit integers in `input` using 256-bit AES key specified in the
242/// 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
243/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
244/// due to a handle violation.
245///
246/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec256kl_u8)
247#[inline]
248#[target_feature(enable = "kl")]
249#[stable(feature = "keylocker_x86", since = "1.89.0")]
250#[cfg_attr(test, assert_instr(aesdec256kl))]
251pub unsafe fn _mm_aesdec256kl_u8(output: *mut __m128i, input: __m128i, handle: *const u8) -> u8 {
252    let AesOutput(status, result) = aesdec256kl(input, handle);
253    *output = result;
254    status
255}
256
257/// Encrypt 10 rounds of 8 groups of unsigned 8-bit integers in `input` using 128-bit AES key specified
258/// in the 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
259/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
260/// due to a handle violation.
261///
262/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesencwide128kl_u8)
263#[inline]
264#[target_feature(enable = "widekl")]
265#[stable(feature = "keylocker_x86", since = "1.89.0")]
266#[cfg_attr(test, assert_instr(aesencwide128kl))]
267pub unsafe fn _mm_aesencwide128kl_u8(
268    output: *mut __m128i,
269    input: *const __m128i,
270    handle: *const u8,
271) -> u8 {
272    let input = &*ptr::slice_from_raw_parts(input, 8);
273    let WideAesOutput(status, out0, out1, out2, out3, out4, out5, out6, out7) = aesencwide128kl(
274        handle, input[0], input[1], input[2], input[3], input[4], input[5], input[6], input[7],
275    );
276    *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
277    status
278}
279
280/// Decrypt 10 rounds of 8 groups of unsigned 8-bit integers in `input` using 128-bit AES key specified
281/// in the 384-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
282/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
283/// due to a handle violation.
284///
285/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdecwide128kl_u8)
286#[inline]
287#[target_feature(enable = "widekl")]
288#[stable(feature = "keylocker_x86", since = "1.89.0")]
289#[cfg_attr(test, assert_instr(aesdecwide128kl))]
290pub unsafe fn _mm_aesdecwide128kl_u8(
291    output: *mut __m128i,
292    input: *const __m128i,
293    handle: *const u8,
294) -> u8 {
295    let input = &*ptr::slice_from_raw_parts(input, 8);
296    let WideAesOutput(status, out0, out1, out2, out3, out4, out5, out6, out7) = aesdecwide128kl(
297        handle, input[0], input[1], input[2], input[3], input[4], input[5], input[6], input[7],
298    );
299    *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
300    status
301}
302
303/// Encrypt 14 rounds of 8 groups of unsigned 8-bit integers in `input` using 256-bit AES key specified
304/// in the 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
305/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
306/// due to a handle violation.
307///
308/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesencwide256kl_u8)
309#[inline]
310#[target_feature(enable = "widekl")]
311#[stable(feature = "keylocker_x86", since = "1.89.0")]
312#[cfg_attr(test, assert_instr(aesencwide256kl))]
313pub unsafe fn _mm_aesencwide256kl_u8(
314    output: *mut __m128i,
315    input: *const __m128i,
316    handle: *const u8,
317) -> u8 {
318    let input = &*ptr::slice_from_raw_parts(input, 8);
319    let WideAesOutput(status, out0, out1, out2, out3, out4, out5, out6, out7) = aesencwide256kl(
320        handle, input[0], input[1], input[2], input[3], input[4], input[5], input[6], input[7],
321    );
322    *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
323    status
324}
325
326/// Decrypt 14 rounds of 8 groups of unsigned 8-bit integers in `input` using 256-bit AES key specified
327/// in the 512-bit key handle `handle`. Store the resulting unsigned 8-bit integers into the corresponding
328/// elements of `output`. Returns `0` if the operation was successful, and `1` if the operation failed
329/// due to a handle violation.
330///
331/// [Intel's documentation](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdecwide256kl_u8)
332#[inline]
333#[target_feature(enable = "widekl")]
334#[stable(feature = "keylocker_x86", since = "1.89.0")]
335#[cfg_attr(test, assert_instr(aesdecwide256kl))]
336pub unsafe fn _mm_aesdecwide256kl_u8(
337    output: *mut __m128i,
338    input: *const __m128i,
339    handle: *const u8,
340) -> u8 {
341    let input = &*ptr::slice_from_raw_parts(input, 8);
342    let WideAesOutput(status, out0, out1, out2, out3, out4, out5, out6, out7) = aesdecwide256kl(
343        handle, input[0], input[1], input[2], input[3], input[4], input[5], input[6], input[7],
344    );
345    *output.cast() = [out0, out1, out2, out3, out4, out5, out6, out7];
346    status
347}
348
349#[cfg(test)]
350mod tests {
351    use crate::core_arch::x86::*;
352    use stdarch_test::simd_test;
353
354    #[target_feature(enable = "kl")]
355    fn encodekey128() -> [u8; 48] {
356        let mut handle = [0; 48];
357        let _ = unsafe { _mm_encodekey128_u32(0, _mm_setzero_si128(), handle.as_mut_ptr()) };
358        handle
359    }
360
361    #[target_feature(enable = "kl")]
362    fn encodekey256() -> [u8; 64] {
363        let mut handle = [0; 64];
364        let _ = unsafe {
365            _mm_encodekey256_u32(
366                0,
367                _mm_setzero_si128(),
368                _mm_setzero_si128(),
369                handle.as_mut_ptr(),
370            )
371        };
372        handle
373    }
374
375    #[simd_test(enable = "kl")]
376    fn test_mm_encodekey128_u32() {
377        encodekey128();
378    }
379
380    #[simd_test(enable = "kl")]
381    fn test_mm_encodekey256_u32() {
382        encodekey256();
383    }
384
385    #[simd_test(enable = "kl")]
386    fn test_mm_aesenc128kl_u8() {
387        let mut buffer = _mm_setzero_si128();
388        let key = encodekey128();
389
390        for _ in 0..100 {
391            let status = unsafe { _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr()) };
392            assert_eq!(status, 0);
393        }
394        for _ in 0..100 {
395            let status = unsafe { _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr()) };
396            assert_eq!(status, 0);
397        }
398
399        assert_eq_m128i(buffer, _mm_setzero_si128());
400    }
401
402    #[simd_test(enable = "kl")]
403    fn test_mm_aesdec128kl_u8() {
404        let mut buffer = _mm_setzero_si128();
405        let key = encodekey128();
406
407        for _ in 0..100 {
408            let status = unsafe { _mm_aesdec128kl_u8(&mut buffer, buffer, key.as_ptr()) };
409            assert_eq!(status, 0);
410        }
411        for _ in 0..100 {
412            let status = unsafe { _mm_aesenc128kl_u8(&mut buffer, buffer, key.as_ptr()) };
413            assert_eq!(status, 0);
414        }
415
416        assert_eq_m128i(buffer, _mm_setzero_si128());
417    }
418
419    #[simd_test(enable = "kl")]
420    fn test_mm_aesenc256kl_u8() {
421        let mut buffer = _mm_setzero_si128();
422        let key = encodekey256();
423
424        for _ in 0..100 {
425            let status = unsafe { _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr()) };
426            assert_eq!(status, 0);
427        }
428        for _ in 0..100 {
429            let status = unsafe { _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr()) };
430            assert_eq!(status, 0);
431        }
432
433        assert_eq_m128i(buffer, _mm_setzero_si128());
434    }
435
436    #[simd_test(enable = "kl")]
437    fn test_mm_aesdec256kl_u8() {
438        let mut buffer = _mm_setzero_si128();
439        let key = encodekey256();
440
441        for _ in 0..100 {
442            let status = unsafe { _mm_aesdec256kl_u8(&mut buffer, buffer, key.as_ptr()) };
443            assert_eq!(status, 0);
444        }
445        for _ in 0..100 {
446            let status = unsafe { _mm_aesenc256kl_u8(&mut buffer, buffer, key.as_ptr()) };
447            assert_eq!(status, 0);
448        }
449
450        assert_eq_m128i(buffer, _mm_setzero_si128());
451    }
452
453    #[simd_test(enable = "widekl")]
454    fn test_mm_aesencwide128kl_u8() {
455        let mut buffer = [_mm_setzero_si128(); 8];
456        let key = encodekey128();
457
458        for _ in 0..100 {
459            let status = unsafe {
460                _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
461            };
462            assert_eq!(status, 0);
463        }
464        for _ in 0..100 {
465            let status = unsafe {
466                _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
467            };
468            assert_eq!(status, 0);
469        }
470
471        for elem in buffer {
472            assert_eq_m128i(elem, _mm_setzero_si128());
473        }
474    }
475
476    #[simd_test(enable = "widekl")]
477    fn test_mm_aesdecwide128kl_u8() {
478        let mut buffer = [_mm_setzero_si128(); 8];
479        let key = encodekey128();
480
481        for _ in 0..100 {
482            let status = unsafe {
483                _mm_aesdecwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
484            };
485            assert_eq!(status, 0);
486        }
487        for _ in 0..100 {
488            let status = unsafe {
489                _mm_aesencwide128kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
490            };
491            assert_eq!(status, 0);
492        }
493
494        for elem in buffer {
495            assert_eq_m128i(elem, _mm_setzero_si128());
496        }
497    }
498
499    #[simd_test(enable = "widekl")]
500    fn test_mm_aesencwide256kl_u8() {
501        let mut buffer = [_mm_setzero_si128(); 8];
502        let key = encodekey256();
503
504        for _ in 0..100 {
505            let status = unsafe {
506                _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
507            };
508            assert_eq!(status, 0);
509        }
510        for _ in 0..100 {
511            let status = unsafe {
512                _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
513            };
514            assert_eq!(status, 0);
515        }
516
517        for elem in buffer {
518            assert_eq_m128i(elem, _mm_setzero_si128());
519        }
520    }
521
522    #[simd_test(enable = "widekl")]
523    fn test_mm_aesdecwide256kl_u8() {
524        let mut buffer = [_mm_setzero_si128(); 8];
525        let key = encodekey256();
526
527        for _ in 0..100 {
528            let status = unsafe {
529                _mm_aesdecwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
530            };
531            assert_eq!(status, 0);
532        }
533        for _ in 0..100 {
534            let status = unsafe {
535                _mm_aesencwide256kl_u8(buffer.as_mut_ptr(), buffer.as_ptr(), key.as_ptr())
536            };
537            assert_eq!(status, 0);
538        }
539
540        for elem in buffer {
541            assert_eq_m128i(elem, _mm_setzero_si128());
542        }
543    }
544}
core/stdarch/crates/core_arch/src/x86/kl.rs

core/stdarch/crates/core_arch/src/x86/
kl.rs
