194 static u32 mul_by_x(u32 w)  in mul_by_x()  argument
196 	u32 x = w & 0x7f7f7f7f;  in mul_by_x()
197 	u32 y = w & 0x80808080;  in mul_by_x()
199 	/* multiply by polynomial 'x' (0b10) in GF(2^8) */  in mul_by_x()
203 static u32 mul_by_x2(u32 w)  in mul_by_x2()  argument
205 	u32 x = w & 0x3f3f3f3f;  in mul_by_x2()
206 	u32 y = w & 0x80808080;  in mul_by_x2()
207 	u32 z = w & 0x40404040;  in mul_by_x2()
209 	/* multiply by polynomial 'x^2' (0b100) in GF(2^8) */  in mul_by_x2()
210 	return (x << 2) ^ (y >> 7) * 0x36 ^ (z >> 6) * 0x1b;  in mul_by_x2()
216 	 * Perform the following matrix multiplication in GF(2^8)  in mix_columns()
220 	 * | 0x1 0x1 0x2 0x3 | x | x[2] |  in mix_columns()
231 	 * Perform the following matrix multiplication in GF(2^8)  in inv_mix_columns()
235 	 * | 0xd 0x9 0xe 0xb | x | x[2] |  in inv_mix_columns()
242 	 * | 0x1 0x1 0x2 0x3 | x | 0x4 0x0 0x5 0x0 | x | x[2] |  in inv_mix_columns()
273 		rko[2] = rko[1] ^ rki[2];  in aes_expandkey_generic()
274 		rko[3] = rko[2] ^ rki[3];  in aes_expandkey_generic()
300 		inv_rndkeys[2] = rndkeys[key_len + 26];  in aes_expandkey_generic()
306 			inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]);  in aes_expandkey_generic()
312 		inv_rndkeys[i + 2] = rndkeys[2];  in aes_expandkey_generic()
328 static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk)  in enc_quarterround()
330 	return rk ^ aes_enc_tab[(u8)w[i]] ^  in enc_quarterround()
331 	       rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^  in enc_quarterround()
332 	       rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^  in enc_quarterround()
333 	       rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24);  in enc_quarterround()
336 static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk)  in enclast_quarterround()
338 	return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^  in enclast_quarterround()
339 	       (aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^  in enclast_quarterround()
340 	       ((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^  in enclast_quarterround()
341 	       ((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16);  in enclast_quarterround()
350 	u32 w[4];  in aes_encrypt_generic()  local
352 	w[0] = get_unaligned_le32(&in[0]) ^ *rkp++;  in aes_encrypt_generic()
353 	w[1] = get_unaligned_le32(&in[4]) ^ *rkp++;  in aes_encrypt_generic()
354 	w[2] = get_unaligned_le32(&in[8]) ^ *rkp++;  in aes_encrypt_generic()
355 	w[3] = get_unaligned_le32(&in[12]) ^ *rkp++;  in aes_encrypt_generic()
368 		u32 w0 = enc_quarterround(w, 0, *rkp++);  in aes_encrypt_generic()
369 		u32 w1 = enc_quarterround(w, 1, *rkp++);  in aes_encrypt_generic()
370 		u32 w2 = enc_quarterround(w, 2, *rkp++);  in aes_encrypt_generic()
371 		u32 w3 = enc_quarterround(w, 3, *rkp++);  in aes_encrypt_generic()
373 		w[0] = w0;  in aes_encrypt_generic()
374 		w[1] = w1;  in aes_encrypt_generic()
375 		w[2] = w2;  in aes_encrypt_generic()
376 		w[3] = w3;  in aes_encrypt_generic()
379 	put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]);  in aes_encrypt_generic()
380 	put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]);  in aes_encrypt_generic()
381 	put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]);  in aes_encrypt_generic()
382 	put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]);  in aes_encrypt_generic()
385 static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk)  in dec_quarterround()
387 	return rk ^ aes_dec_tab[(u8)w[i]] ^  in dec_quarterround()
388 	       rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^  in dec_quarterround()
389 	       rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^  in dec_quarterround()
390 	       rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24);  in dec_quarterround()
393 static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk)  in declast_quarterround()
395 	return rk ^ aes_inv_sbox[(u8)w[i]] ^  in declast_quarterround()
396 	       ((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^  in declast_quarterround()
397 	       ((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^  in declast_quarterround()
398 	       ((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24);  in declast_quarterround()
408 	u32 w[4];  in aes_decrypt_generic()  local
410 	w[0] = get_unaligned_le32(&in[0]) ^ *rkp++;  in aes_decrypt_generic()
411 	w[1] = get_unaligned_le32(&in[4]) ^ *rkp++;  in aes_decrypt_generic()
412 	w[2] = get_unaligned_le32(&in[8]) ^ *rkp++;  in aes_decrypt_generic()
413 	w[3] = get_unaligned_le32(&in[12]) ^ *rkp++;  in aes_decrypt_generic()
418 		u32 w0 = dec_quarterround(w, 0, *rkp++);  in aes_decrypt_generic()
419 		u32 w1 = dec_quarterround(w, 1, *rkp++);  in aes_decrypt_generic()
420 		u32 w2 = dec_quarterround(w, 2, *rkp++);  in aes_decrypt_generic()
421 		u32 w3 = dec_quarterround(w, 3, *rkp++);  in aes_decrypt_generic()
423 		w[0] = w0;  in aes_decrypt_generic()
424 		w[1] = w1;  in aes_decrypt_generic()
425 		w[2] = w2;  in aes_decrypt_generic()
426 		w[3] = w3;  in aes_decrypt_generic()
430 	put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]);  in aes_decrypt_generic()
431 	put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]);  in aes_decrypt_generic()
432 	put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]);  in aes_decrypt_generic()
433 	put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]);  in aes_decrypt_generic()