1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * AVX-512 optimized implementation of xor_gen()
4 *
5 * Copyright 2026 Google LLC
6 */
7
8 #include <linux/types.h>
9 #include <asm/fpu/api.h>
10 #include "xor_impl.h"
11 #include "xor_arch.h"
12
13 /*
14 * Implementation notes:
15 *
16 * Unrolling by the number of buffers (2-5) is very important.
17 *
18 * Unrolling by length is less important, especially when using register-indexed
19 * addressing with negative indices from the end of the buffers. That approach
20 * results in just two loop control instructions being needed per iteration,
21 * regardless of the number of buffers.
22 *
23 * In fact, benchmarks showed that the 2 and 3 buffer cases require only 2x
24 * unrolling by length, while the 4 and 5 buffer cases don't require any
25 * unrolling by length. Benchmarks also showed that the register-indexed
26 * addressing isn't a bottleneck either; i.e., we can't do any better by
27 * incrementing the pointers as we go along, even with more unrolling.
28 */
29
xor_avx512_2(long bytes,u8 * p1,const u8 * p2)30 static void xor_avx512_2(long bytes, u8 *p1, const u8 *p2)
31 {
32 long i = -bytes;
33
34 asm volatile("1: vmovdqa64 (%1,%0), %%zmm0\n"
35 "vmovdqa64 64(%1,%0), %%zmm1\n"
36 "vpxorq (%2,%0), %%zmm0, %%zmm0\n"
37 "vpxorq 64(%2,%0), %%zmm1, %%zmm1\n"
38 "vmovdqa64 %%zmm0, (%1,%0)\n"
39 "vmovdqa64 %%zmm1, 64(%1,%0)\n"
40 "add $128, %0\n"
41 "jnz 1b\n"
42 : "+&r"(i)
43 : "r"(p1 + bytes), "r"(p2 + bytes)
44 : "memory", "cc");
45 }
46
xor_avx512_3(long bytes,u8 * p1,const u8 * p2,const u8 * p3)47 static void xor_avx512_3(long bytes, u8 *p1, const u8 *p2, const u8 *p3)
48 {
49 long i = -bytes;
50
51 asm volatile("1: vmovdqa64 (%1,%0), %%zmm0\n"
52 "vmovdqa64 64(%1,%0), %%zmm1\n"
53 "vmovdqa64 (%2,%0), %%zmm2\n"
54 "vmovdqa64 64(%2,%0), %%zmm3\n"
55 "vpternlogq $0x96, (%3,%0), %%zmm2, %%zmm0\n"
56 "vpternlogq $0x96, 64(%3,%0), %%zmm3, %%zmm1\n"
57 "vmovdqa64 %%zmm0, (%1,%0)\n"
58 "vmovdqa64 %%zmm1, 64(%1,%0)\n"
59 "add $128, %0\n"
60 "jnz 1b\n"
61 : "+&r"(i)
62 : "r"(p1 + bytes), "r"(p2 + bytes), "r"(p3 + bytes)
63 : "memory", "cc");
64 }
65
xor_avx512_4(long bytes,u8 * p1,const u8 * p2,const u8 * p3,const u8 * p4)66 static void xor_avx512_4(long bytes, u8 *p1, const u8 *p2, const u8 *p3,
67 const u8 *p4)
68 {
69 long i = -bytes;
70
71 asm volatile("1: vmovdqa64 (%1,%0), %%zmm0\n"
72 "vmovdqa64 (%2,%0), %%zmm1\n"
73 "vpxorq (%3,%0), %%zmm0, %%zmm0\n"
74 "vpternlogq $0x96, (%4,%0), %%zmm1, %%zmm0\n"
75 "vmovdqa64 %%zmm0, (%1,%0)\n"
76 "add $64, %0\n"
77 "jnz 1b\n"
78 : "+&r"(i)
79 : "r"(p1 + bytes), "r"(p2 + bytes), "r"(p3 + bytes),
80 "r"(p4 + bytes)
81 : "memory", "cc");
82 }
83
xor_avx512_5(long bytes,u8 * p1,const u8 * p2,const u8 * p3,const u8 * p4,const u8 * p5)84 static void xor_avx512_5(long bytes, u8 *p1, const u8 *p2, const u8 *p3,
85 const u8 *p4, const u8 *p5)
86 {
87 long i = -bytes;
88
89 asm volatile("1: vmovdqa64 (%1,%0), %%zmm0\n"
90 "vmovdqa64 (%2,%0), %%zmm1\n"
91 "vpternlogq $0x96, (%3,%0), %%zmm1, %%zmm0\n"
92 "vmovdqa64 (%4,%0), %%zmm1\n"
93 "vpternlogq $0x96, (%5,%0), %%zmm1, %%zmm0\n"
94 "vmovdqa64 %%zmm0, (%1,%0)\n"
95 "add $64, %0\n"
96 "jnz 1b\n"
97 : "+&r"(i)
98 : "r"(p1 + bytes), "r"(p2 + bytes), "r"(p3 + bytes),
99 "r"(p4 + bytes), "r"(p5 + bytes)
100 : "memory", "cc");
101 }
102
103 DO_XOR_BLOCKS(avx512_inner, xor_avx512_2, xor_avx512_3, xor_avx512_4,
104 xor_avx512_5);
105
106 /*
107 * Preconditions: bytes is a nonzero multiple of 512, and all buffers are
108 * 64-byte aligned.
109 */
xor_gen_avx512(void * dest,void ** srcs,unsigned int src_cnt,unsigned int bytes)110 static void xor_gen_avx512(void *dest, void **srcs, unsigned int src_cnt,
111 unsigned int bytes)
112 {
113 kernel_fpu_begin();
114 xor_gen_avx512_inner(dest, srcs, src_cnt, bytes);
115 asm volatile("vzeroupper");
116 kernel_fpu_end();
117 }
118
119 struct xor_block_template xor_block_avx512 = {
120 .name = "avx512",
121 .xor_gen = xor_gen_avx512,
122 };
123