xref: /linux/drivers/ras/amd/atl/internal.h (revision 7dfc15c47372e8bf8a693ca3dfaaec33a68ee116)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * AMD Address Translation Library
4  *
5  * internal.h : Helper functions and common defines
6  *
7  * Copyright (c) 2023, Advanced Micro Devices, Inc.
8  * All Rights Reserved.
9  *
10  * Author: Yazen Ghannam <Yazen.Ghannam@amd.com>
11  */
12 
13 #ifndef __AMD_ATL_INTERNAL_H__
14 #define __AMD_ATL_INTERNAL_H__
15 
16 #include <linux/bitfield.h>
17 #include <linux/bitops.h>
18 #include <linux/ras.h>
19 
20 #include <asm/amd_nb.h>
21 
22 #include "reg_fields.h"
23 
24 #undef pr_fmt
25 #define pr_fmt(fmt) "amd_atl: " fmt
26 
27 /* Maximum possible number of Coherent Stations within a single Data Fabric. */
28 #define MAX_COH_ST_CHANNELS		32
29 
30 /* PCI ID for Zen4 Server DF Function 0. */
31 #define DF_FUNC0_ID_ZEN4_SERVER		0x14AD1022
32 
33 /* PCI IDs for MI300 DF Function 0. */
34 #define DF_FUNC0_ID_MI300		0x15281022
35 
36 /* Shift needed for adjusting register values to true values. */
37 #define DF_DRAM_BASE_LIMIT_LSB		28
38 #define MI300_DRAM_LIMIT_LSB		20
39 
40 #define INVALID_SPA ~0ULL
41 
42 enum df_revisions {
43 	UNKNOWN,
44 	DF2,
45 	DF3,
46 	DF3p5,
47 	DF4,
48 	DF4p5,
49 };
50 
51 /* These are mapped 1:1 to the hardware values. Special cases are set at > 0x20. */
52 enum intlv_modes {
53 	NONE				= 0x00,
54 	NOHASH_2CHAN			= 0x01,
55 	NOHASH_4CHAN			= 0x03,
56 	NOHASH_8CHAN			= 0x05,
57 	DF3_6CHAN			= 0x06,
58 	NOHASH_16CHAN			= 0x07,
59 	NOHASH_32CHAN			= 0x08,
60 	DF3_COD4_2CHAN_HASH		= 0x0C,
61 	DF3_COD2_4CHAN_HASH		= 0x0D,
62 	DF3_COD1_8CHAN_HASH		= 0x0E,
63 	DF4_NPS4_2CHAN_HASH		= 0x10,
64 	DF4_NPS2_4CHAN_HASH		= 0x11,
65 	DF4_NPS1_8CHAN_HASH		= 0x12,
66 	DF4_NPS4_3CHAN_HASH		= 0x13,
67 	DF4_NPS2_6CHAN_HASH		= 0x14,
68 	DF4_NPS1_12CHAN_HASH		= 0x15,
69 	DF4_NPS2_5CHAN_HASH		= 0x16,
70 	DF4_NPS1_10CHAN_HASH		= 0x17,
71 	MI3_HASH_8CHAN			= 0x18,
72 	MI3_HASH_16CHAN			= 0x19,
73 	MI3_HASH_32CHAN			= 0x1A,
74 	DF2_2CHAN_HASH			= 0x21,
75 	/* DF4.5 modes are all IntLvNumChan + 0x20 */
76 	DF4p5_NPS1_16CHAN_1K_HASH	= 0x2C,
77 	DF4p5_NPS0_24CHAN_1K_HASH	= 0x2E,
78 	DF4p5_NPS4_2CHAN_1K_HASH	= 0x30,
79 	DF4p5_NPS2_4CHAN_1K_HASH	= 0x31,
80 	DF4p5_NPS1_8CHAN_1K_HASH	= 0x32,
81 	DF4p5_NPS4_3CHAN_1K_HASH	= 0x33,
82 	DF4p5_NPS2_6CHAN_1K_HASH	= 0x34,
83 	DF4p5_NPS1_12CHAN_1K_HASH	= 0x35,
84 	DF4p5_NPS2_5CHAN_1K_HASH	= 0x36,
85 	DF4p5_NPS1_10CHAN_1K_HASH	= 0x37,
86 	DF4p5_NPS4_2CHAN_2K_HASH	= 0x40,
87 	DF4p5_NPS2_4CHAN_2K_HASH	= 0x41,
88 	DF4p5_NPS1_8CHAN_2K_HASH	= 0x42,
89 	DF4p5_NPS1_16CHAN_2K_HASH	= 0x43,
90 	DF4p5_NPS4_3CHAN_2K_HASH	= 0x44,
91 	DF4p5_NPS2_6CHAN_2K_HASH	= 0x45,
92 	DF4p5_NPS1_12CHAN_2K_HASH	= 0x46,
93 	DF4p5_NPS0_24CHAN_2K_HASH	= 0x47,
94 	DF4p5_NPS2_5CHAN_2K_HASH	= 0x48,
95 	DF4p5_NPS1_10CHAN_2K_HASH	= 0x49,
96 };
97 
98 struct df4p5_denorm_ctx {
99 	/* Indicates the number of "lost" bits. This will be 1, 2, or 3. */
100 	u8 perm_shift;
101 
102 	/* A mask indicating the bits that need to be rehashed. */
103 	u16 rehash_vector;
104 
105 	/*
106 	 * Represents the value that the high bits of the normalized address
107 	 * are divided by during normalization. This value will be 3 for
108 	 * interleave modes with a number of channels divisible by 3 or the
109 	 * value will be 5 for interleave modes with a number of channels
110 	 * divisible by 5. Power-of-two interleave modes are handled
111 	 * separately.
112 	 */
113 	u8 mod_value;
114 
115 	/*
116 	 * Represents the bits that can be directly pulled from the normalized
117 	 * address. In each case, pass through bits [7:0] of the normalized
118 	 * address. The other bits depend on the interleave bit position which
119 	 * will be bit 10 for 1K interleave stripe cases and bit 11 for 2K
120 	 * interleave stripe cases.
121 	 */
122 	u64 base_denorm_addr;
123 
124 	/*
125 	 * Represents the high bits of the physical address that have been
126 	 * divided by the mod_value.
127 	 */
128 	u64 div_addr;
129 
130 	u64 current_spa;
131 	u64 resolved_spa;
132 
133 	u16 coh_st_fabric_id;
134 };
135 
136 struct df_flags {
137 	__u8	legacy_ficaa		: 1,
138 		socket_id_shift_quirk	: 1,
139 		heterogeneous		: 1,
140 		__reserved_0		: 5;
141 };
142 
143 struct df_config {
144 	enum df_revisions rev;
145 
146 	/*
147 	 * These masks operate on the 16-bit Coherent Station IDs,
148 	 * e.g. Instance, Fabric, Destination, etc.
149 	 */
150 	u16 component_id_mask;
151 	u16 die_id_mask;
152 	u16 node_id_mask;
153 	u16 socket_id_mask;
154 
155 	/*
156 	 * Least-significant bit of Node ID portion of the
157 	 * system-wide Coherent Station Fabric ID.
158 	 */
159 	u8 node_id_shift;
160 
161 	/*
162 	 * Least-significant bit of Die portion of the Node ID.
163 	 * Adjusted to include the Node ID shift in order to apply
164 	 * to the Coherent Station Fabric ID.
165 	 */
166 	u8 die_id_shift;
167 
168 	/*
169 	 * Least-significant bit of Socket portion of the Node ID.
170 	 * Adjusted to include the Node ID shift in order to apply
171 	 * to the Coherent Station Fabric ID.
172 	 */
173 	u8 socket_id_shift;
174 
175 	/* Number of DRAM Address maps visible in a Coherent Station. */
176 	u8 num_coh_st_maps;
177 
178 	u32 dram_hole_base;
179 
180 	/* Global flags to handle special cases. */
181 	struct df_flags flags;
182 };
183 
184 extern struct df_config df_cfg;
185 
186 struct dram_addr_map {
187 	/*
188 	 * Each DRAM Address Map can operate independently
189 	 * in different interleaving modes.
190 	 */
191 	enum intlv_modes intlv_mode;
192 
193 	/* System-wide number for this address map. */
194 	u8 num;
195 
196 	/* Raw register values */
197 	u32 base;
198 	u32 limit;
199 	u32 ctl;
200 	u32 intlv;
201 
202 	/*
203 	 * Logical to Physical Coherent Station Remapping array
204 	 *
205 	 * Index: Logical Coherent Station Instance ID
206 	 * Value: Physical Coherent Station Instance ID
207 	 *
208 	 * phys_coh_st_inst_id = remap_array[log_coh_st_inst_id]
209 	 */
210 	u8 remap_array[MAX_COH_ST_CHANNELS];
211 
212 	/*
213 	 * Number of bits covering DRAM Address map 0
214 	 * when interleaving is non-power-of-2.
215 	 *
216 	 * Used only for DF3_6CHAN.
217 	 */
218 	u8 np2_bits;
219 
220 	/* Position of the 'interleave bit'. */
221 	u8 intlv_bit_pos;
222 	/* Number of channels interleaved in this map. */
223 	u8 num_intlv_chan;
224 	/* Number of dies interleaved in this map. */
225 	u8 num_intlv_dies;
226 	/* Number of sockets interleaved in this map. */
227 	u8 num_intlv_sockets;
228 	/*
229 	 * Total number of channels interleaved accounting
230 	 * for die and socket interleaving.
231 	 */
232 	u8 total_intlv_chan;
233 	/* Total bits needed to cover 'total_intlv_chan'. */
234 	u8 total_intlv_bits;
235 };
236 
237 /* Original input values cached for debug printing. */
238 struct addr_ctx_inputs {
239 	u64 norm_addr;
240 	u8 socket_id;
241 	u8 die_id;
242 	u8 coh_st_inst_id;
243 };
244 
245 struct addr_ctx {
246 	u64 ret_addr;
247 
248 	struct addr_ctx_inputs inputs;
249 	struct dram_addr_map map;
250 
251 	/* AMD Node ID calculated from Socket and Die IDs. */
252 	u8 node_id;
253 
254 	/*
255 	 * Coherent Station Instance ID
256 	 * Local ID used within a 'node'.
257 	 */
258 	u16 inst_id;
259 
260 	/*
261 	 * Coherent Station Fabric ID
262 	 * System-wide ID that includes 'node' bits.
263 	 */
264 	u16 coh_st_fabric_id;
265 };
266 
267 int df_indirect_read_instance(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo);
268 int df_indirect_read_broadcast(u16 node, u8 func, u16 reg, u32 *lo);
269 
270 int get_df_system_info(void);
271 int determine_node_id(struct addr_ctx *ctx, u8 socket_num, u8 die_num);
272 int get_umc_info_mi300(void);
273 
274 int get_address_map(struct addr_ctx *ctx);
275 
276 int denormalize_address(struct addr_ctx *ctx);
277 int dehash_address(struct addr_ctx *ctx);
278 
279 unsigned long norm_to_sys_addr(u8 socket_id, u8 die_id, u8 coh_st_inst_id, unsigned long addr);
280 unsigned long convert_umc_mca_addr_to_sys_addr(struct atl_err *err);
281 
282 u64 add_base_and_hole(struct addr_ctx *ctx, u64 addr);
283 u64 remove_base_and_hole(struct addr_ctx *ctx, u64 addr);
284 
285 #ifdef CONFIG_AMD_ATL_PRM
286 unsigned long prm_umc_norm_to_sys_addr(u8 socket_id, u64 umc_bank_inst_id, unsigned long addr);
287 #else
prm_umc_norm_to_sys_addr(u8 socket_id,u64 umc_bank_inst_id,unsigned long addr)288 static inline unsigned long prm_umc_norm_to_sys_addr(u8 socket_id, u64 umc_bank_inst_id,
289 						     unsigned long addr)
290 {
291        return -ENODEV;
292 }
293 #endif
294 
295 /*
296  * Make a gap in @data that is @num_bits long starting at @bit_num.
297  * e.g. data		= 11111111'b
298  *	bit_num		= 3
299  *	num_bits	= 2
300  *	result		= 1111100111'b
301  */
expand_bits(u8 bit_num,u8 num_bits,u64 data)302 static inline u64 expand_bits(u8 bit_num, u8 num_bits, u64 data)
303 {
304 	u64 temp1, temp2;
305 
306 	if (!num_bits)
307 		return data;
308 
309 	if (!bit_num) {
310 		WARN_ON_ONCE(num_bits >= BITS_PER_LONG);
311 		return data << num_bits;
312 	}
313 
314 	WARN_ON_ONCE(bit_num >= BITS_PER_LONG);
315 
316 	temp1 = data & GENMASK_ULL(bit_num - 1, 0);
317 
318 	temp2 = data & GENMASK_ULL(63, bit_num);
319 	temp2 <<= num_bits;
320 
321 	return temp1 | temp2;
322 }
323 
324 /*
325  * Remove bits in @data between @low_bit and @high_bit inclusive.
326  * e.g. data		= XXXYYZZZ'b
327  *	low_bit		= 3
328  *	high_bit	= 4
329  *	result		= XXXZZZ'b
330  */
remove_bits(u8 low_bit,u8 high_bit,u64 data)331 static inline u64 remove_bits(u8 low_bit, u8 high_bit, u64 data)
332 {
333 	u64 temp1, temp2;
334 
335 	WARN_ON_ONCE(high_bit >= BITS_PER_LONG);
336 	WARN_ON_ONCE(low_bit  >= BITS_PER_LONG);
337 	WARN_ON_ONCE(low_bit  >  high_bit);
338 
339 	if (!low_bit)
340 		return data >> (high_bit++);
341 
342 	temp1 = GENMASK_ULL(low_bit - 1, 0) & data;
343 	temp2 = GENMASK_ULL(63, high_bit + 1) & data;
344 	temp2 >>= high_bit - low_bit + 1;
345 
346 	return temp1 | temp2;
347 }
348 
349 #define atl_debug(ctx, fmt, arg...) \
350 	pr_debug("socket_id=%u die_id=%u coh_st_inst_id=%u norm_addr=0x%016llx: " fmt,\
351 		 (ctx)->inputs.socket_id, (ctx)->inputs.die_id,\
352 		 (ctx)->inputs.coh_st_inst_id, (ctx)->inputs.norm_addr, ##arg)
353 
atl_debug_on_bad_df_rev(void)354 static inline void atl_debug_on_bad_df_rev(void)
355 {
356 	pr_debug("Unrecognized DF rev: %u", df_cfg.rev);
357 }
358 
atl_debug_on_bad_intlv_mode(struct addr_ctx * ctx)359 static inline void atl_debug_on_bad_intlv_mode(struct addr_ctx *ctx)
360 {
361 	atl_debug(ctx, "Unrecognized interleave mode: %u", ctx->map.intlv_mode);
362 }
363 
364 #endif /* __AMD_ATL_INTERNAL_H__ */
365