xref: /linux/drivers/ras/amd/atl/dehash.c (revision c02ce1735b150cf7c3b43790b48e23dcd17c0d46)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * AMD Address Translation Library
4  *
5  * dehash.c : Functions to account for hashing bits
6  *
7  * Copyright (c) 2023, Advanced Micro Devices, Inc.
8  * All Rights Reserved.
9  *
10  * Author: Yazen Ghannam <Yazen.Ghannam@amd.com>
11  */
12 
13 #include "internal.h"
14 
15 /*
16  * Verify the interleave bits are correct in the different interleaving
17  * settings.
18  *
19  * If @num_intlv_dies and/or @num_intlv_sockets are 1, it means the
20  * respective interleaving is disabled.
21  */
22 static inline bool map_bits_valid(struct addr_ctx *ctx, u8 bit1, u8 bit2,
23 				  u8 num_intlv_dies, u8 num_intlv_sockets)
24 {
25 	if (!(ctx->map.intlv_bit_pos == bit1 || ctx->map.intlv_bit_pos == bit2)) {
26 		pr_debug("Invalid interleave bit: %u", ctx->map.intlv_bit_pos);
27 		return false;
28 	}
29 
30 	if (ctx->map.num_intlv_dies > num_intlv_dies) {
31 		pr_debug("Invalid number of interleave dies: %u", ctx->map.num_intlv_dies);
32 		return false;
33 	}
34 
35 	if (ctx->map.num_intlv_sockets > num_intlv_sockets) {
36 		pr_debug("Invalid number of interleave sockets: %u", ctx->map.num_intlv_sockets);
37 		return false;
38 	}
39 
40 	return true;
41 }
42 
43 static int df2_dehash_addr(struct addr_ctx *ctx)
44 {
45 	u8 hashed_bit, intlv_bit, intlv_bit_pos;
46 
47 	if (!map_bits_valid(ctx, 8, 9, 1, 1))
48 		return -EINVAL;
49 
50 	intlv_bit_pos = ctx->map.intlv_bit_pos;
51 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
52 
53 	hashed_bit = intlv_bit;
54 	hashed_bit ^= FIELD_GET(BIT_ULL(12), ctx->ret_addr);
55 	hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr);
56 	hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr);
57 	hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr);
58 
59 	if (hashed_bit != intlv_bit)
60 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
61 
62 	return 0;
63 }
64 
65 static int df3_dehash_addr(struct addr_ctx *ctx)
66 {
67 	bool hash_ctl_64k, hash_ctl_2M, hash_ctl_1G;
68 	u8 hashed_bit, intlv_bit, intlv_bit_pos;
69 
70 	if (!map_bits_valid(ctx, 8, 9, 1, 1))
71 		return -EINVAL;
72 
73 	hash_ctl_64k = FIELD_GET(DF3_HASH_CTL_64K, ctx->map.ctl);
74 	hash_ctl_2M  = FIELD_GET(DF3_HASH_CTL_2M, ctx->map.ctl);
75 	hash_ctl_1G  = FIELD_GET(DF3_HASH_CTL_1G, ctx->map.ctl);
76 
77 	intlv_bit_pos = ctx->map.intlv_bit_pos;
78 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
79 
80 	hashed_bit = intlv_bit;
81 	hashed_bit ^= FIELD_GET(BIT_ULL(14), ctx->ret_addr);
82 	hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k;
83 	hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M;
84 	hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G;
85 
86 	if (hashed_bit != intlv_bit)
87 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
88 
89 	/* Calculation complete for 2 channels. Continue for 4 and 8 channels. */
90 	if (ctx->map.intlv_mode == DF3_COD4_2CHAN_HASH)
91 		return 0;
92 
93 	intlv_bit = FIELD_GET(BIT_ULL(12), ctx->ret_addr);
94 
95 	hashed_bit = intlv_bit;
96 	hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k;
97 	hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M;
98 	hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G;
99 
100 	if (hashed_bit != intlv_bit)
101 		ctx->ret_addr ^= BIT_ULL(12);
102 
103 	/* Calculation complete for 4 channels. Continue for 8 channels. */
104 	if (ctx->map.intlv_mode == DF3_COD2_4CHAN_HASH)
105 		return 0;
106 
107 	intlv_bit = FIELD_GET(BIT_ULL(13), ctx->ret_addr);
108 
109 	hashed_bit = intlv_bit;
110 	hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k;
111 	hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M;
112 	hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G;
113 
114 	if (hashed_bit != intlv_bit)
115 		ctx->ret_addr ^= BIT_ULL(13);
116 
117 	return 0;
118 }
119 
120 static int df3_6chan_dehash_addr(struct addr_ctx *ctx)
121 {
122 	u8 intlv_bit_pos = ctx->map.intlv_bit_pos;
123 	u8 hashed_bit, intlv_bit, num_intlv_bits;
124 	bool hash_ctl_2M, hash_ctl_1G;
125 
126 	if (ctx->map.intlv_mode != DF3_6CHAN) {
127 		atl_debug_on_bad_intlv_mode(ctx);
128 		return -EINVAL;
129 	}
130 
131 	num_intlv_bits = ilog2(ctx->map.num_intlv_chan) + 1;
132 
133 	hash_ctl_2M = FIELD_GET(DF3_HASH_CTL_2M, ctx->map.ctl);
134 	hash_ctl_1G = FIELD_GET(DF3_HASH_CTL_1G, ctx->map.ctl);
135 
136 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
137 
138 	hashed_bit = intlv_bit;
139 	hashed_bit ^= !!(BIT_ULL(intlv_bit_pos + num_intlv_bits) & ctx->ret_addr);
140 	hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M;
141 	hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G;
142 
143 	if (hashed_bit != intlv_bit)
144 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
145 
146 	intlv_bit_pos++;
147 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
148 
149 	hashed_bit = intlv_bit;
150 	hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M;
151 	hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G;
152 
153 	if (hashed_bit != intlv_bit)
154 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
155 
156 	intlv_bit_pos++;
157 	intlv_bit = !!(BIT_ULL(intlv_bit_pos) & ctx->ret_addr);
158 
159 	hashed_bit = intlv_bit;
160 	hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M;
161 	hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G;
162 
163 	if (hashed_bit != intlv_bit)
164 		ctx->ret_addr ^= BIT_ULL(intlv_bit_pos);
165 
166 	return 0;
167 }
168 
169 static int df4_dehash_addr(struct addr_ctx *ctx)
170 {
171 	bool hash_ctl_64k, hash_ctl_2M, hash_ctl_1G;
172 	u8 hashed_bit, intlv_bit;
173 
174 	if (!map_bits_valid(ctx, 8, 8, 1, 2))
175 		return -EINVAL;
176 
177 	hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl);
178 	hash_ctl_2M  = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl);
179 	hash_ctl_1G  = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl);
180 
181 	intlv_bit = FIELD_GET(BIT_ULL(8), ctx->ret_addr);
182 
183 	hashed_bit = intlv_bit;
184 	hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k;
185 	hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M;
186 	hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G;
187 
188 	if (ctx->map.num_intlv_sockets == 1)
189 		hashed_bit ^= FIELD_GET(BIT_ULL(14), ctx->ret_addr);
190 
191 	if (hashed_bit != intlv_bit)
192 		ctx->ret_addr ^= BIT_ULL(8);
193 
194 	/*
195 	 * Hashing is possible with socket interleaving, so check the total number
196 	 * of channels in the system rather than DRAM map interleaving mode.
197 	 *
198 	 * Calculation complete for 2 channels. Continue for 4, 8, and 16 channels.
199 	 */
200 	if (ctx->map.total_intlv_chan <= 2)
201 		return 0;
202 
203 	intlv_bit = FIELD_GET(BIT_ULL(12), ctx->ret_addr);
204 
205 	hashed_bit = intlv_bit;
206 	hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k;
207 	hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M;
208 	hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G;
209 
210 	if (hashed_bit != intlv_bit)
211 		ctx->ret_addr ^= BIT_ULL(12);
212 
213 	/* Calculation complete for 4 channels. Continue for 8 and 16 channels. */
214 	if (ctx->map.total_intlv_chan <= 4)
215 		return 0;
216 
217 	intlv_bit = FIELD_GET(BIT_ULL(13), ctx->ret_addr);
218 
219 	hashed_bit = intlv_bit;
220 	hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k;
221 	hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M;
222 	hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G;
223 
224 	if (hashed_bit != intlv_bit)
225 		ctx->ret_addr ^= BIT_ULL(13);
226 
227 	/* Calculation complete for 8 channels. Continue for 16 channels. */
228 	if (ctx->map.total_intlv_chan <= 8)
229 		return 0;
230 
231 	intlv_bit = FIELD_GET(BIT_ULL(14), ctx->ret_addr);
232 
233 	hashed_bit = intlv_bit;
234 	hashed_bit ^= FIELD_GET(BIT_ULL(19), ctx->ret_addr) & hash_ctl_64k;
235 	hashed_bit ^= FIELD_GET(BIT_ULL(24), ctx->ret_addr) & hash_ctl_2M;
236 	hashed_bit ^= FIELD_GET(BIT_ULL(33), ctx->ret_addr) & hash_ctl_1G;
237 
238 	if (hashed_bit != intlv_bit)
239 		ctx->ret_addr ^= BIT_ULL(14);
240 
241 	return 0;
242 }
243 
244 static int df4p5_dehash_addr(struct addr_ctx *ctx)
245 {
246 	bool hash_ctl_64k, hash_ctl_2M, hash_ctl_1G, hash_ctl_1T;
247 	u8 hashed_bit, intlv_bit;
248 	u64 rehash_vector;
249 
250 	if (!map_bits_valid(ctx, 8, 8, 1, 2))
251 		return -EINVAL;
252 
253 	hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K, ctx->map.ctl);
254 	hash_ctl_2M  = FIELD_GET(DF4_HASH_CTL_2M, ctx->map.ctl);
255 	hash_ctl_1G  = FIELD_GET(DF4_HASH_CTL_1G, ctx->map.ctl);
256 	hash_ctl_1T  = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl);
257 
258 	/*
259 	 * Generate a unique address to determine which bits
260 	 * need to be dehashed.
261 	 *
262 	 * Start with a contiguous bitmask for the total
263 	 * number of channels starting at bit 8.
264 	 *
265 	 * Then make a gap in the proper place based on
266 	 * interleave mode.
267 	 */
268 	rehash_vector = ctx->map.total_intlv_chan - 1;
269 	rehash_vector <<= 8;
270 
271 	if (ctx->map.intlv_mode == DF4p5_NPS2_4CHAN_1K_HASH ||
272 	    ctx->map.intlv_mode == DF4p5_NPS1_8CHAN_1K_HASH ||
273 	    ctx->map.intlv_mode == DF4p5_NPS1_16CHAN_1K_HASH)
274 		rehash_vector = expand_bits(10, 2, rehash_vector);
275 	else
276 		rehash_vector = expand_bits(9, 3, rehash_vector);
277 
278 	if (rehash_vector & BIT_ULL(8)) {
279 		intlv_bit = FIELD_GET(BIT_ULL(8), ctx->ret_addr);
280 
281 		hashed_bit = intlv_bit;
282 		hashed_bit ^= FIELD_GET(BIT_ULL(16), ctx->ret_addr) & hash_ctl_64k;
283 		hashed_bit ^= FIELD_GET(BIT_ULL(21), ctx->ret_addr) & hash_ctl_2M;
284 		hashed_bit ^= FIELD_GET(BIT_ULL(30), ctx->ret_addr) & hash_ctl_1G;
285 		hashed_bit ^= FIELD_GET(BIT_ULL(40), ctx->ret_addr) & hash_ctl_1T;
286 
287 		if (hashed_bit != intlv_bit)
288 			ctx->ret_addr ^= BIT_ULL(8);
289 	}
290 
291 	if (rehash_vector & BIT_ULL(9)) {
292 		intlv_bit = FIELD_GET(BIT_ULL(9), ctx->ret_addr);
293 
294 		hashed_bit = intlv_bit;
295 		hashed_bit ^= FIELD_GET(BIT_ULL(17), ctx->ret_addr) & hash_ctl_64k;
296 		hashed_bit ^= FIELD_GET(BIT_ULL(22), ctx->ret_addr) & hash_ctl_2M;
297 		hashed_bit ^= FIELD_GET(BIT_ULL(31), ctx->ret_addr) & hash_ctl_1G;
298 		hashed_bit ^= FIELD_GET(BIT_ULL(41), ctx->ret_addr) & hash_ctl_1T;
299 
300 		if (hashed_bit != intlv_bit)
301 			ctx->ret_addr ^= BIT_ULL(9);
302 	}
303 
304 	if (rehash_vector & BIT_ULL(12)) {
305 		intlv_bit = FIELD_GET(BIT_ULL(12), ctx->ret_addr);
306 
307 		hashed_bit = intlv_bit;
308 		hashed_bit ^= FIELD_GET(BIT_ULL(18), ctx->ret_addr) & hash_ctl_64k;
309 		hashed_bit ^= FIELD_GET(BIT_ULL(23), ctx->ret_addr) & hash_ctl_2M;
310 		hashed_bit ^= FIELD_GET(BIT_ULL(32), ctx->ret_addr) & hash_ctl_1G;
311 		hashed_bit ^= FIELD_GET(BIT_ULL(42), ctx->ret_addr) & hash_ctl_1T;
312 
313 		if (hashed_bit != intlv_bit)
314 			ctx->ret_addr ^= BIT_ULL(12);
315 	}
316 
317 	if (rehash_vector & BIT_ULL(13)) {
318 		intlv_bit = FIELD_GET(BIT_ULL(13), ctx->ret_addr);
319 
320 		hashed_bit = intlv_bit;
321 		hashed_bit ^= FIELD_GET(BIT_ULL(19), ctx->ret_addr) & hash_ctl_64k;
322 		hashed_bit ^= FIELD_GET(BIT_ULL(24), ctx->ret_addr) & hash_ctl_2M;
323 		hashed_bit ^= FIELD_GET(BIT_ULL(33), ctx->ret_addr) & hash_ctl_1G;
324 		hashed_bit ^= FIELD_GET(BIT_ULL(43), ctx->ret_addr) & hash_ctl_1T;
325 
326 		if (hashed_bit != intlv_bit)
327 			ctx->ret_addr ^= BIT_ULL(13);
328 	}
329 
330 	if (rehash_vector & BIT_ULL(14)) {
331 		intlv_bit = FIELD_GET(BIT_ULL(14), ctx->ret_addr);
332 
333 		hashed_bit = intlv_bit;
334 		hashed_bit ^= FIELD_GET(BIT_ULL(20), ctx->ret_addr) & hash_ctl_64k;
335 		hashed_bit ^= FIELD_GET(BIT_ULL(25), ctx->ret_addr) & hash_ctl_2M;
336 		hashed_bit ^= FIELD_GET(BIT_ULL(34), ctx->ret_addr) & hash_ctl_1G;
337 		hashed_bit ^= FIELD_GET(BIT_ULL(44), ctx->ret_addr) & hash_ctl_1T;
338 
339 		if (hashed_bit != intlv_bit)
340 			ctx->ret_addr ^= BIT_ULL(14);
341 	}
342 
343 	return 0;
344 }
345 
346 /*
347  * MI300 hash bits
348  *					  4K 64K  2M  1G  1T  1T
349  * COH_ST_Select[0]	= XOR of addr{8,  12, 15, 22, 29, 36, 43}
350  * COH_ST_Select[1]	= XOR of addr{9,  13, 16, 23, 30, 37, 44}
351  * COH_ST_Select[2]	= XOR of addr{10, 14, 17, 24, 31, 38, 45}
352  * COH_ST_Select[3]	= XOR of addr{11,     18, 25, 32, 39, 46}
353  * COH_ST_Select[4]	= XOR of addr{14,     19, 26, 33, 40, 47} aka Stack
354  * DieID[0]		= XOR of addr{12,     20, 27, 34, 41    }
355  * DieID[1]		= XOR of addr{13,     21, 28, 35, 42    }
356  */
357 static int mi300_dehash_addr(struct addr_ctx *ctx)
358 {
359 	bool hash_ctl_4k, hash_ctl_64k, hash_ctl_2M, hash_ctl_1G, hash_ctl_1T;
360 	bool hashed_bit, intlv_bit, test_bit;
361 	u8 num_intlv_bits, base_bit, i;
362 
363 	if (!map_bits_valid(ctx, 8, 8, 4, 1))
364 		return -EINVAL;
365 
366 	hash_ctl_4k  = FIELD_GET(DF4p5_HASH_CTL_4K, ctx->map.ctl);
367 	hash_ctl_64k = FIELD_GET(DF4_HASH_CTL_64K,  ctx->map.ctl);
368 	hash_ctl_2M  = FIELD_GET(DF4_HASH_CTL_2M,   ctx->map.ctl);
369 	hash_ctl_1G  = FIELD_GET(DF4_HASH_CTL_1G,   ctx->map.ctl);
370 	hash_ctl_1T  = FIELD_GET(DF4p5_HASH_CTL_1T, ctx->map.ctl);
371 
372 	/* Channel bits */
373 	num_intlv_bits = ilog2(ctx->map.num_intlv_chan);
374 
375 	for (i = 0; i < num_intlv_bits; i++) {
376 		base_bit = 8 + i;
377 
378 		/* COH_ST_Select[4] jumps to a base bit of 14. */
379 		if (i == 4)
380 			base_bit = 14;
381 
382 		intlv_bit = BIT_ULL(base_bit) & ctx->ret_addr;
383 
384 		hashed_bit = intlv_bit;
385 
386 		/* 4k hash bit only applies to the first 3 bits. */
387 		if (i <= 2) {
388 			test_bit    = BIT_ULL(12 + i) & ctx->ret_addr;
389 			hashed_bit ^= test_bit & hash_ctl_4k;
390 		}
391 
392 		/* Use temporary 'test_bit' value to avoid Sparse warnings. */
393 		test_bit    = BIT_ULL(15 + i) & ctx->ret_addr;
394 		hashed_bit ^= test_bit & hash_ctl_64k;
395 		test_bit    = BIT_ULL(22 + i) & ctx->ret_addr;
396 		hashed_bit ^= test_bit & hash_ctl_2M;
397 		test_bit    = BIT_ULL(29 + i) & ctx->ret_addr;
398 		hashed_bit ^= test_bit & hash_ctl_1G;
399 		test_bit    = BIT_ULL(36 + i) & ctx->ret_addr;
400 		hashed_bit ^= test_bit & hash_ctl_1T;
401 		test_bit    = BIT_ULL(43 + i) & ctx->ret_addr;
402 		hashed_bit ^= test_bit & hash_ctl_1T;
403 
404 		if (hashed_bit != intlv_bit)
405 			ctx->ret_addr ^= BIT_ULL(base_bit);
406 	}
407 
408 	/* Die bits */
409 	num_intlv_bits = ilog2(ctx->map.num_intlv_dies);
410 
411 	for (i = 0; i < num_intlv_bits; i++) {
412 		base_bit = 12 + i;
413 
414 		intlv_bit = BIT_ULL(base_bit) & ctx->ret_addr;
415 
416 		hashed_bit = intlv_bit;
417 
418 		test_bit    = BIT_ULL(20 + i) & ctx->ret_addr;
419 		hashed_bit ^= test_bit & hash_ctl_64k;
420 		test_bit    = BIT_ULL(27 + i) & ctx->ret_addr;
421 		hashed_bit ^= test_bit & hash_ctl_2M;
422 		test_bit    = BIT_ULL(34 + i) & ctx->ret_addr;
423 		hashed_bit ^= test_bit & hash_ctl_1G;
424 		test_bit    = BIT_ULL(41 + i) & ctx->ret_addr;
425 		hashed_bit ^= test_bit & hash_ctl_1T;
426 
427 		if (hashed_bit != intlv_bit)
428 			ctx->ret_addr ^= BIT_ULL(base_bit);
429 	}
430 
431 	return 0;
432 }
433 
434 int dehash_address(struct addr_ctx *ctx)
435 {
436 	switch (ctx->map.intlv_mode) {
437 	/* No hashing cases. */
438 	case NONE:
439 	case NOHASH_2CHAN:
440 	case NOHASH_4CHAN:
441 	case NOHASH_8CHAN:
442 	case NOHASH_16CHAN:
443 	case NOHASH_32CHAN:
444 	/* Hashing bits handled earlier during CS ID calculation. */
445 	case DF4_NPS4_3CHAN_HASH:
446 	case DF4_NPS2_5CHAN_HASH:
447 	case DF4_NPS2_6CHAN_HASH:
448 	case DF4_NPS1_10CHAN_HASH:
449 	case DF4_NPS1_12CHAN_HASH:
450 	case DF4p5_NPS2_6CHAN_1K_HASH:
451 	case DF4p5_NPS2_6CHAN_2K_HASH:
452 	case DF4p5_NPS1_10CHAN_1K_HASH:
453 	case DF4p5_NPS1_10CHAN_2K_HASH:
454 	case DF4p5_NPS1_12CHAN_1K_HASH:
455 	case DF4p5_NPS1_12CHAN_2K_HASH:
456 	case DF4p5_NPS0_24CHAN_1K_HASH:
457 	case DF4p5_NPS0_24CHAN_2K_HASH:
458 	/* No hash physical address bits, so nothing to do. */
459 	case DF4p5_NPS4_3CHAN_1K_HASH:
460 	case DF4p5_NPS4_3CHAN_2K_HASH:
461 	case DF4p5_NPS2_5CHAN_1K_HASH:
462 	case DF4p5_NPS2_5CHAN_2K_HASH:
463 		return 0;
464 
465 	case DF2_2CHAN_HASH:
466 		return df2_dehash_addr(ctx);
467 
468 	case DF3_COD4_2CHAN_HASH:
469 	case DF3_COD2_4CHAN_HASH:
470 	case DF3_COD1_8CHAN_HASH:
471 		return df3_dehash_addr(ctx);
472 
473 	case DF3_6CHAN:
474 		return df3_6chan_dehash_addr(ctx);
475 
476 	case DF4_NPS4_2CHAN_HASH:
477 	case DF4_NPS2_4CHAN_HASH:
478 	case DF4_NPS1_8CHAN_HASH:
479 		return df4_dehash_addr(ctx);
480 
481 	case DF4p5_NPS4_2CHAN_1K_HASH:
482 	case DF4p5_NPS4_2CHAN_2K_HASH:
483 	case DF4p5_NPS2_4CHAN_2K_HASH:
484 	case DF4p5_NPS2_4CHAN_1K_HASH:
485 	case DF4p5_NPS1_8CHAN_1K_HASH:
486 	case DF4p5_NPS1_8CHAN_2K_HASH:
487 	case DF4p5_NPS1_16CHAN_1K_HASH:
488 	case DF4p5_NPS1_16CHAN_2K_HASH:
489 		return df4p5_dehash_addr(ctx);
490 
491 	case MI3_HASH_8CHAN:
492 	case MI3_HASH_16CHAN:
493 	case MI3_HASH_32CHAN:
494 		return mi300_dehash_addr(ctx);
495 
496 	default:
497 		atl_debug_on_bad_intlv_mode(ctx);
498 		return -EINVAL;
499 	}
500 }
501