1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24 #include <linux/list.h>
25 #include "amdgpu.h"
26 #include "amdgpu_xgmi.h"
27 #include "amdgpu_ras.h"
28 #include "soc15.h"
29 #include "df/df_3_6_offset.h"
30 #include "xgmi/xgmi_4_0_0_smn.h"
31 #include "xgmi/xgmi_4_0_0_sh_mask.h"
32 #include "xgmi/xgmi_6_1_0_sh_mask.h"
33 #include "wafl/wafl2_4_0_0_smn.h"
34 #include "wafl/wafl2_4_0_0_sh_mask.h"
35
36 #include "amdgpu_reset.h"
37
38 #define smnPCS_XGMI3X16_PCS_ERROR_STATUS 0x11a0020c
39 #define smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK 0x11a00218
40 #define smnPCS_GOPX1_PCS_ERROR_STATUS 0x12200210
41 #define smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK 0x12200218
42
43 #define XGMI_STATE_DISABLE 0xD1
44 #define XGMI_STATE_LS0 0x81
45 #define XGMI_LINK_ACTIVE 1
46 #define XGMI_LINK_INACTIVE 0
47
48 static DEFINE_MUTEX(xgmi_mutex);
49
50 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4
51
52 static LIST_HEAD(xgmi_hive_list);
53
54 static const int xgmi_pcs_err_status_reg_vg20[] = {
55 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
56 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
57 };
58
59 static const int wafl_pcs_err_status_reg_vg20[] = {
60 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
61 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
62 };
63
64 static const int xgmi_pcs_err_status_reg_arct[] = {
65 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
66 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
67 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x500000,
68 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x600000,
69 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x700000,
70 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x800000,
71 };
72
73 /* same as vg20*/
74 static const int wafl_pcs_err_status_reg_arct[] = {
75 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
76 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
77 };
78
79 static const int xgmi3x16_pcs_err_status_reg_aldebaran[] = {
80 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
81 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000,
82 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x200000,
83 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x300000,
84 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x400000,
85 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x500000,
86 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x600000,
87 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x700000
88 };
89
90 static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
91 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
92 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000,
93 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x200000,
94 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x300000,
95 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x400000,
96 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x500000,
97 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x600000,
98 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x700000
99 };
100
101 static const int walf_pcs_err_status_reg_aldebaran[] = {
102 smnPCS_GOPX1_PCS_ERROR_STATUS,
103 smnPCS_GOPX1_PCS_ERROR_STATUS + 0x100000
104 };
105
106 static const int walf_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
107 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK,
108 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
109 };
110
111 static const int xgmi3x16_pcs_err_status_reg_v6_4[] = {
112 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
113 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000
114 };
115
116 static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[] = {
117 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
118 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
119 };
120
121 static const u64 xgmi_v6_4_0_mca_base_array[] = {
122 0x11a09200,
123 0x11b09200,
124 };
125
126 static const char *xgmi_v6_4_0_ras_error_code_ext[32] = {
127 [0x00] = "XGMI PCS DataLossErr",
128 [0x01] = "XGMI PCS TrainingErr",
129 [0x02] = "XGMI PCS FlowCtrlAckErr",
130 [0x03] = "XGMI PCS RxFifoUnderflowErr",
131 [0x04] = "XGMI PCS RxFifoOverflowErr",
132 [0x05] = "XGMI PCS CRCErr",
133 [0x06] = "XGMI PCS BERExceededErr",
134 [0x07] = "XGMI PCS TxMetaDataErr",
135 [0x08] = "XGMI PCS ReplayBufParityErr",
136 [0x09] = "XGMI PCS DataParityErr",
137 [0x0a] = "XGMI PCS ReplayFifoOverflowErr",
138 [0x0b] = "XGMI PCS ReplayFifoUnderflowErr",
139 [0x0c] = "XGMI PCS ElasticFifoOverflowErr",
140 [0x0d] = "XGMI PCS DeskewErr",
141 [0x0e] = "XGMI PCS FlowCtrlCRCErr",
142 [0x0f] = "XGMI PCS DataStartupLimitErr",
143 [0x10] = "XGMI PCS FCInitTimeoutErr",
144 [0x11] = "XGMI PCS RecoveryTimeoutErr",
145 [0x12] = "XGMI PCS ReadySerialTimeoutErr",
146 [0x13] = "XGMI PCS ReadySerialAttemptErr",
147 [0x14] = "XGMI PCS RecoveryAttemptErr",
148 [0x15] = "XGMI PCS RecoveryRelockAttemptErr",
149 [0x16] = "XGMI PCS ReplayAttemptErr",
150 [0x17] = "XGMI PCS SyncHdrErr",
151 [0x18] = "XGMI PCS TxReplayTimeoutErr",
152 [0x19] = "XGMI PCS RxReplayTimeoutErr",
153 [0x1a] = "XGMI PCS LinkSubTxTimeoutErr",
154 [0x1b] = "XGMI PCS LinkSubRxTimeoutErr",
155 [0x1c] = "XGMI PCS RxCMDPktErr",
156 };
157
158 static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = {
159 {"XGMI PCS DataLossErr",
160 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)},
161 {"XGMI PCS TrainingErr",
162 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TrainingErr)},
163 {"XGMI PCS CRCErr",
164 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, CRCErr)},
165 {"XGMI PCS BERExceededErr",
166 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, BERExceededErr)},
167 {"XGMI PCS TxMetaDataErr",
168 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TxMetaDataErr)},
169 {"XGMI PCS ReplayBufParityErr",
170 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayBufParityErr)},
171 {"XGMI PCS DataParityErr",
172 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataParityErr)},
173 {"XGMI PCS ReplayFifoOverflowErr",
174 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
175 {"XGMI PCS ReplayFifoUnderflowErr",
176 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
177 {"XGMI PCS ElasticFifoOverflowErr",
178 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
179 {"XGMI PCS DeskewErr",
180 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DeskewErr)},
181 {"XGMI PCS DataStartupLimitErr",
182 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataStartupLimitErr)},
183 {"XGMI PCS FCInitTimeoutErr",
184 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
185 {"XGMI PCS RecoveryTimeoutErr",
186 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
187 {"XGMI PCS ReadySerialTimeoutErr",
188 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
189 {"XGMI PCS ReadySerialAttemptErr",
190 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
191 {"XGMI PCS RecoveryAttemptErr",
192 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
193 {"XGMI PCS RecoveryRelockAttemptErr",
194 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
195 };
196
197 static const struct amdgpu_pcs_ras_field wafl_pcs_ras_fields[] = {
198 {"WAFL PCS DataLossErr",
199 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataLossErr)},
200 {"WAFL PCS TrainingErr",
201 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TrainingErr)},
202 {"WAFL PCS CRCErr",
203 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, CRCErr)},
204 {"WAFL PCS BERExceededErr",
205 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, BERExceededErr)},
206 {"WAFL PCS TxMetaDataErr",
207 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TxMetaDataErr)},
208 {"WAFL PCS ReplayBufParityErr",
209 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayBufParityErr)},
210 {"WAFL PCS DataParityErr",
211 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataParityErr)},
212 {"WAFL PCS ReplayFifoOverflowErr",
213 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
214 {"WAFL PCS ReplayFifoUnderflowErr",
215 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
216 {"WAFL PCS ElasticFifoOverflowErr",
217 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
218 {"WAFL PCS DeskewErr",
219 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DeskewErr)},
220 {"WAFL PCS DataStartupLimitErr",
221 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataStartupLimitErr)},
222 {"WAFL PCS FCInitTimeoutErr",
223 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, FCInitTimeoutErr)},
224 {"WAFL PCS RecoveryTimeoutErr",
225 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
226 {"WAFL PCS ReadySerialTimeoutErr",
227 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
228 {"WAFL PCS ReadySerialAttemptErr",
229 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
230 {"WAFL PCS RecoveryAttemptErr",
231 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryAttemptErr)},
232 {"WAFL PCS RecoveryRelockAttemptErr",
233 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
234 };
235
236 static const struct amdgpu_pcs_ras_field xgmi3x16_pcs_ras_fields[] = {
237 {"XGMI3X16 PCS DataLossErr",
238 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataLossErr)},
239 {"XGMI3X16 PCS TrainingErr",
240 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TrainingErr)},
241 {"XGMI3X16 PCS FlowCtrlAckErr",
242 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlAckErr)},
243 {"XGMI3X16 PCS RxFifoUnderflowErr",
244 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoUnderflowErr)},
245 {"XGMI3X16 PCS RxFifoOverflowErr",
246 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoOverflowErr)},
247 {"XGMI3X16 PCS CRCErr",
248 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, CRCErr)},
249 {"XGMI3X16 PCS BERExceededErr",
250 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, BERExceededErr)},
251 {"XGMI3X16 PCS TxVcidDataErr",
252 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxVcidDataErr)},
253 {"XGMI3X16 PCS ReplayBufParityErr",
254 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayBufParityErr)},
255 {"XGMI3X16 PCS DataParityErr",
256 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataParityErr)},
257 {"XGMI3X16 PCS ReplayFifoOverflowErr",
258 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
259 {"XGMI3X16 PCS ReplayFifoUnderflowErr",
260 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
261 {"XGMI3X16 PCS ElasticFifoOverflowErr",
262 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
263 {"XGMI3X16 PCS DeskewErr",
264 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DeskewErr)},
265 {"XGMI3X16 PCS FlowCtrlCRCErr",
266 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlCRCErr)},
267 {"XGMI3X16 PCS DataStartupLimitErr",
268 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataStartupLimitErr)},
269 {"XGMI3X16 PCS FCInitTimeoutErr",
270 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
271 {"XGMI3X16 PCS RecoveryTimeoutErr",
272 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
273 {"XGMI3X16 PCS ReadySerialTimeoutErr",
274 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
275 {"XGMI3X16 PCS ReadySerialAttemptErr",
276 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
277 {"XGMI3X16 PCS RecoveryAttemptErr",
278 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
279 {"XGMI3X16 PCS RecoveryRelockAttemptErr",
280 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
281 {"XGMI3X16 PCS ReplayAttemptErr",
282 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayAttemptErr)},
283 {"XGMI3X16 PCS SyncHdrErr",
284 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, SyncHdrErr)},
285 {"XGMI3X16 PCS TxReplayTimeoutErr",
286 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxReplayTimeoutErr)},
287 {"XGMI3X16 PCS RxReplayTimeoutErr",
288 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxReplayTimeoutErr)},
289 {"XGMI3X16 PCS LinkSubTxTimeoutErr",
290 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubTxTimeoutErr)},
291 {"XGMI3X16 PCS LinkSubRxTimeoutErr",
292 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubRxTimeoutErr)},
293 {"XGMI3X16 PCS RxCMDPktErr",
294 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxCMDPktErr)},
295 };
296
amdgpu_xgmi_get_ext_link(struct amdgpu_device * adev,int link_num)297 int amdgpu_xgmi_get_ext_link(struct amdgpu_device *adev, int link_num)
298 {
299 int link_map_6_4_x[8] = { 0, 3, 1, 2, 7, 6, 4, 5 };
300
301 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
302 case IP_VERSION(6, 4, 0):
303 case IP_VERSION(6, 4, 1):
304 if (link_num < ARRAY_SIZE(link_map_6_4_x))
305 return link_map_6_4_x[link_num];
306 break;
307 default:
308 return -EINVAL;
309 }
310
311 return -EINVAL;
312 }
313
xgmi_v6_4_get_link_status(struct amdgpu_device * adev,int global_link_num)314 static u32 xgmi_v6_4_get_link_status(struct amdgpu_device *adev, int global_link_num)
315 {
316 const u32 smn_xgmi_6_4_pcs_state_hist1[2] = { 0x11a00070, 0x11b00070 };
317 const u32 smn_xgmi_6_4_1_pcs_state_hist1[2] = { 0x12100070,
318 0x11b00070 };
319 u32 i, n;
320 u64 addr;
321
322 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
323 case IP_VERSION(6, 4, 0):
324 n = ARRAY_SIZE(smn_xgmi_6_4_pcs_state_hist1);
325 addr = smn_xgmi_6_4_pcs_state_hist1[global_link_num % n];
326 break;
327 case IP_VERSION(6, 4, 1):
328 n = ARRAY_SIZE(smn_xgmi_6_4_1_pcs_state_hist1);
329 addr = smn_xgmi_6_4_1_pcs_state_hist1[global_link_num % n];
330 break;
331 default:
332 return U32_MAX;
333 }
334
335 i = global_link_num / n;
336 addr += adev->asic_funcs->encode_ext_smn_addressing(i);
337
338 return RREG32_PCIE_EXT(addr);
339 }
340
amdgpu_get_xgmi_link_status(struct amdgpu_device * adev,int global_link_num)341 int amdgpu_get_xgmi_link_status(struct amdgpu_device *adev, int global_link_num)
342 {
343 u32 xgmi_state_reg_val;
344
345 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
346 case IP_VERSION(6, 4, 0):
347 case IP_VERSION(6, 4, 1):
348 xgmi_state_reg_val = xgmi_v6_4_get_link_status(adev, global_link_num);
349 break;
350 default:
351 return -EOPNOTSUPP;
352 }
353
354 if ((xgmi_state_reg_val & 0xFF) == XGMI_STATE_DISABLE)
355 return -ENOLINK;
356
357 if ((xgmi_state_reg_val & 0xFF) == XGMI_STATE_LS0)
358 return XGMI_LINK_ACTIVE;
359
360 return XGMI_LINK_INACTIVE;
361 }
362
363 /**
364 * DOC: AMDGPU XGMI Support
365 *
366 * XGMI is a high speed interconnect that joins multiple GPU cards
367 * into a homogeneous memory space that is organized by a collective
368 * hive ID and individual node IDs, both of which are 64-bit numbers.
369 *
370 * The file xgmi_device_id contains the unique per GPU device ID and
371 * is stored in the /sys/class/drm/card${cardno}/device/ directory.
372 *
373 * Inside the device directory a sub-directory 'xgmi_hive_info' is
374 * created which contains the hive ID and the list of nodes.
375 *
376 * The hive ID is stored in:
377 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
378 *
379 * The node information is stored in numbered directories:
380 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
381 *
382 * Each device has their own xgmi_hive_info direction with a mirror
383 * set of node sub-directories.
384 *
385 * The XGMI memory space is built by contiguously adding the power of
386 * two padded VRAM space from each node to each other.
387 *
388 */
389
390 static struct attribute amdgpu_xgmi_hive_id = {
391 .name = "xgmi_hive_id",
392 .mode = S_IRUGO
393 };
394
395 static struct attribute *amdgpu_xgmi_hive_attrs[] = {
396 &amdgpu_xgmi_hive_id,
397 NULL
398 };
399 ATTRIBUTE_GROUPS(amdgpu_xgmi_hive);
400
amdgpu_xgmi_show_attrs(struct kobject * kobj,struct attribute * attr,char * buf)401 static ssize_t amdgpu_xgmi_show_attrs(struct kobject *kobj,
402 struct attribute *attr, char *buf)
403 {
404 struct amdgpu_hive_info *hive = container_of(
405 kobj, struct amdgpu_hive_info, kobj);
406
407 if (attr == &amdgpu_xgmi_hive_id)
408 return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);
409
410 return 0;
411 }
412
amdgpu_xgmi_hive_release(struct kobject * kobj)413 static void amdgpu_xgmi_hive_release(struct kobject *kobj)
414 {
415 struct amdgpu_hive_info *hive = container_of(
416 kobj, struct amdgpu_hive_info, kobj);
417
418 amdgpu_reset_put_reset_domain(hive->reset_domain);
419 hive->reset_domain = NULL;
420
421 mutex_destroy(&hive->hive_lock);
422 kfree(hive);
423 }
424
425 static const struct sysfs_ops amdgpu_xgmi_hive_ops = {
426 .show = amdgpu_xgmi_show_attrs,
427 };
428
429 static const struct kobj_type amdgpu_xgmi_hive_type = {
430 .release = amdgpu_xgmi_hive_release,
431 .sysfs_ops = &amdgpu_xgmi_hive_ops,
432 .default_groups = amdgpu_xgmi_hive_groups,
433 };
434
amdgpu_xgmi_show_device_id(struct device * dev,struct device_attribute * attr,char * buf)435 static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
436 struct device_attribute *attr,
437 char *buf)
438 {
439 struct drm_device *ddev = dev_get_drvdata(dev);
440 struct amdgpu_device *adev = drm_to_adev(ddev);
441
442 return sysfs_emit(buf, "%llu\n", adev->gmc.xgmi.node_id);
443
444 }
445
amdgpu_xgmi_show_physical_id(struct device * dev,struct device_attribute * attr,char * buf)446 static ssize_t amdgpu_xgmi_show_physical_id(struct device *dev,
447 struct device_attribute *attr,
448 char *buf)
449 {
450 struct drm_device *ddev = dev_get_drvdata(dev);
451 struct amdgpu_device *adev = drm_to_adev(ddev);
452
453 return sysfs_emit(buf, "%u\n", adev->gmc.xgmi.physical_node_id);
454
455 }
456
amdgpu_xgmi_show_num_hops(struct device * dev,struct device_attribute * attr,char * buf)457 static ssize_t amdgpu_xgmi_show_num_hops(struct device *dev,
458 struct device_attribute *attr,
459 char *buf)
460 {
461 struct drm_device *ddev = dev_get_drvdata(dev);
462 struct amdgpu_device *adev = drm_to_adev(ddev);
463 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
464 int i;
465
466 for (i = 0; i < top->num_nodes; i++)
467 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_hops);
468
469 return sysfs_emit(buf, "%s\n", buf);
470 }
471
amdgpu_xgmi_show_num_links(struct device * dev,struct device_attribute * attr,char * buf)472 static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,
473 struct device_attribute *attr,
474 char *buf)
475 {
476 struct drm_device *ddev = dev_get_drvdata(dev);
477 struct amdgpu_device *adev = drm_to_adev(ddev);
478 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
479 int i;
480
481 for (i = 0; i < top->num_nodes; i++)
482 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_links);
483
484 return sysfs_emit(buf, "%s\n", buf);
485 }
486
amdgpu_xgmi_show_connected_port_num(struct device * dev,struct device_attribute * attr,char * buf)487 static ssize_t amdgpu_xgmi_show_connected_port_num(struct device *dev,
488 struct device_attribute *attr,
489 char *buf)
490 {
491 struct drm_device *ddev = dev_get_drvdata(dev);
492 struct amdgpu_device *adev = drm_to_adev(ddev);
493 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
494 int i, j, size = 0;
495 int current_node;
496 /*
497 * get the node id in the sysfs for the current socket and show
498 * it in the port num info output in the sysfs for easy reading.
499 * it is NOT the one retrieved from xgmi ta.
500 */
501 for (i = 0; i < top->num_nodes; i++) {
502 if (top->nodes[i].node_id == adev->gmc.xgmi.node_id) {
503 current_node = i;
504 break;
505 }
506 }
507
508 if (i == top->num_nodes)
509 return -EINVAL;
510
511 for (i = 0; i < top->num_nodes; i++) {
512 for (j = 0; j < top->nodes[i].num_links; j++)
513 /* node id in sysfs starts from 1 rather than 0 so +1 here */
514 size += sysfs_emit_at(buf, size, "%02x:%02x -> %02x:%02x\n", current_node + 1,
515 top->nodes[i].port_num[j].src_xgmi_port_num, i + 1,
516 top->nodes[i].port_num[j].dst_xgmi_port_num);
517 }
518
519 return size;
520 }
521
522 #define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801)
amdgpu_xgmi_show_error(struct device * dev,struct device_attribute * attr,char * buf)523 static ssize_t amdgpu_xgmi_show_error(struct device *dev,
524 struct device_attribute *attr,
525 char *buf)
526 {
527 struct drm_device *ddev = dev_get_drvdata(dev);
528 struct amdgpu_device *adev = drm_to_adev(ddev);
529 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;
530 uint64_t fica_out;
531 unsigned int error_count = 0;
532
533 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);
534 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);
535
536 if ((!adev->df.funcs) ||
537 (!adev->df.funcs->get_fica) ||
538 (!adev->df.funcs->set_fica))
539 return -EINVAL;
540
541 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_ctl_in);
542 if (fica_out != 0x1f)
543 pr_err("xGMI error counters not enabled!\n");
544
545 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_status_in);
546
547 if ((fica_out & 0xffff) == 2)
548 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);
549
550 adev->df.funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);
551
552 return sysfs_emit(buf, "%u\n", error_count);
553 }
554
555
556 static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
557 static DEVICE_ATTR(xgmi_physical_id, 0444, amdgpu_xgmi_show_physical_id, NULL);
558 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
559 static DEVICE_ATTR(xgmi_num_hops, S_IRUGO, amdgpu_xgmi_show_num_hops, NULL);
560 static DEVICE_ATTR(xgmi_num_links, S_IRUGO, amdgpu_xgmi_show_num_links, NULL);
561 static DEVICE_ATTR(xgmi_port_num, S_IRUGO, amdgpu_xgmi_show_connected_port_num, NULL);
562
amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)563 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
564 struct amdgpu_hive_info *hive)
565 {
566 int ret = 0;
567 char node[10] = { 0 };
568
569 /* Create xgmi device id file */
570 ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);
571 if (ret) {
572 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
573 return ret;
574 }
575
576 ret = device_create_file(adev->dev, &dev_attr_xgmi_physical_id);
577 if (ret) {
578 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_physical_id\n");
579 return ret;
580 }
581
582 /* Create xgmi error file */
583 ret = device_create_file(adev->dev, &dev_attr_xgmi_error);
584 if (ret)
585 pr_err("failed to create xgmi_error\n");
586
587 /* Create xgmi num hops file */
588 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_hops);
589 if (ret)
590 pr_err("failed to create xgmi_num_hops\n");
591
592 /* Create xgmi num links file */
593 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_links);
594 if (ret)
595 pr_err("failed to create xgmi_num_links\n");
596
597 /* Create xgmi port num file if supported */
598 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
599 ret = device_create_file(adev->dev, &dev_attr_xgmi_port_num);
600 if (ret)
601 dev_err(adev->dev, "failed to create xgmi_port_num\n");
602 }
603
604 /* Create sysfs link to hive info folder on the first device */
605 if (hive->kobj.parent != (&adev->dev->kobj)) {
606 ret = sysfs_create_link(&adev->dev->kobj, &hive->kobj,
607 "xgmi_hive_info");
608 if (ret) {
609 dev_err(adev->dev, "XGMI: Failed to create link to hive info");
610 goto remove_file;
611 }
612 }
613
614 sprintf(node, "node%d", atomic_read(&hive->number_devices));
615 /* Create sysfs link form the hive folder to yourself */
616 ret = sysfs_create_link(&hive->kobj, &adev->dev->kobj, node);
617 if (ret) {
618 dev_err(adev->dev, "XGMI: Failed to create link from hive info");
619 goto remove_link;
620 }
621
622 goto success;
623
624
625 remove_link:
626 sysfs_remove_link(&adev->dev->kobj, adev_to_drm(adev)->unique);
627
628 remove_file:
629 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
630 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
631 device_remove_file(adev->dev, &dev_attr_xgmi_error);
632 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
633 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
634 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
635 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
636
637 success:
638 return ret;
639 }
640
amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)641 static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
642 struct amdgpu_hive_info *hive)
643 {
644 char node[10];
645 memset(node, 0, sizeof(node));
646
647 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
648 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
649 device_remove_file(adev->dev, &dev_attr_xgmi_error);
650 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
651 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
652 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
653 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
654
655 if (hive->kobj.parent != (&adev->dev->kobj))
656 sysfs_remove_link(&adev->dev->kobj,"xgmi_hive_info");
657
658 sprintf(node, "node%d", atomic_read(&hive->number_devices));
659 sysfs_remove_link(&hive->kobj, node);
660
661 }
662
663
664
amdgpu_get_xgmi_hive(struct amdgpu_device * adev)665 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
666 {
667 struct amdgpu_hive_info *hive = NULL;
668 int ret;
669
670 if (!adev->gmc.xgmi.hive_id)
671 return NULL;
672
673 if (adev->hive) {
674 kobject_get(&adev->hive->kobj);
675 return adev->hive;
676 }
677
678 mutex_lock(&xgmi_mutex);
679
680 list_for_each_entry(hive, &xgmi_hive_list, node) {
681 if (hive->hive_id == adev->gmc.xgmi.hive_id)
682 goto pro_end;
683 }
684
685 hive = kzalloc(sizeof(*hive), GFP_KERNEL);
686 if (!hive) {
687 dev_err(adev->dev, "XGMI: allocation failed\n");
688 ret = -ENOMEM;
689 hive = NULL;
690 goto pro_end;
691 }
692
693 /* initialize new hive if not exist */
694 ret = kobject_init_and_add(&hive->kobj,
695 &amdgpu_xgmi_hive_type,
696 &adev->dev->kobj,
697 "%s", "xgmi_hive_info");
698 if (ret) {
699 dev_err(adev->dev, "XGMI: failed initializing kobject for xgmi hive\n");
700 kobject_put(&hive->kobj);
701 hive = NULL;
702 goto pro_end;
703 }
704
705 /**
706 * Only init hive->reset_domain for none SRIOV configuration. For SRIOV,
707 * Host driver decide how to reset the GPU either through FLR or chain reset.
708 * Guest side will get individual notifications from the host for the FLR
709 * if necessary.
710 */
711 if (!amdgpu_sriov_vf(adev)) {
712 /**
713 * Avoid recreating reset domain when hive is reconstructed for the case
714 * of reset the devices in the XGMI hive during probe for passthrough GPU
715 * See https://www.spinics.net/lists/amd-gfx/msg58836.html
716 */
717 if (adev->reset_domain->type != XGMI_HIVE) {
718 hive->reset_domain =
719 amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive");
720 if (!hive->reset_domain) {
721 dev_err(adev->dev, "XGMI: failed initializing reset domain for xgmi hive\n");
722 ret = -ENOMEM;
723 kobject_put(&hive->kobj);
724 hive = NULL;
725 goto pro_end;
726 }
727 } else {
728 amdgpu_reset_get_reset_domain(adev->reset_domain);
729 hive->reset_domain = adev->reset_domain;
730 }
731 }
732
733 hive->hive_id = adev->gmc.xgmi.hive_id;
734 INIT_LIST_HEAD(&hive->device_list);
735 INIT_LIST_HEAD(&hive->node);
736 mutex_init(&hive->hive_lock);
737 atomic_set(&hive->number_devices, 0);
738 task_barrier_init(&hive->tb);
739 hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN;
740 hive->hi_req_gpu = NULL;
741 atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);
742
743 /*
744 * hive pstate on boot is high in vega20 so we have to go to low
745 * pstate on after boot.
746 */
747 hive->hi_req_count = AMDGPU_MAX_XGMI_DEVICE_PER_HIVE;
748 list_add_tail(&hive->node, &xgmi_hive_list);
749
750 pro_end:
751 if (hive)
752 kobject_get(&hive->kobj);
753 mutex_unlock(&xgmi_mutex);
754 return hive;
755 }
756
amdgpu_put_xgmi_hive(struct amdgpu_hive_info * hive)757 void amdgpu_put_xgmi_hive(struct amdgpu_hive_info *hive)
758 {
759 if (hive)
760 kobject_put(&hive->kobj);
761 }
762
amdgpu_xgmi_set_pstate(struct amdgpu_device * adev,int pstate)763 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
764 {
765 int ret = 0;
766 struct amdgpu_hive_info *hive;
767 struct amdgpu_device *request_adev;
768 bool is_hi_req = pstate == AMDGPU_XGMI_PSTATE_MAX_VEGA20;
769 bool init_low;
770
771 hive = amdgpu_get_xgmi_hive(adev);
772 if (!hive)
773 return 0;
774
775 request_adev = hive->hi_req_gpu ? hive->hi_req_gpu : adev;
776 init_low = hive->pstate == AMDGPU_XGMI_PSTATE_UNKNOWN;
777 amdgpu_put_xgmi_hive(hive);
778 /* fw bug so temporarily disable pstate switching */
779 return 0;
780
781 if (!hive || adev->asic_type != CHIP_VEGA20)
782 return 0;
783
784 mutex_lock(&hive->hive_lock);
785
786 if (is_hi_req)
787 hive->hi_req_count++;
788 else
789 hive->hi_req_count--;
790
791 /*
792 * Vega20 only needs single peer to request pstate high for the hive to
793 * go high but all peers must request pstate low for the hive to go low
794 */
795 if (hive->pstate == pstate ||
796 (!is_hi_req && hive->hi_req_count && !init_low))
797 goto out;
798
799 dev_dbg(request_adev->dev, "Set xgmi pstate %d.\n", pstate);
800
801 ret = amdgpu_dpm_set_xgmi_pstate(request_adev, pstate);
802 if (ret) {
803 dev_err(request_adev->dev,
804 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
805 request_adev->gmc.xgmi.node_id,
806 request_adev->gmc.xgmi.hive_id, ret);
807 goto out;
808 }
809
810 if (init_low)
811 hive->pstate = hive->hi_req_count ?
812 hive->pstate : AMDGPU_XGMI_PSTATE_MIN;
813 else {
814 hive->pstate = pstate;
815 hive->hi_req_gpu = pstate != AMDGPU_XGMI_PSTATE_MIN ?
816 adev : NULL;
817 }
818 out:
819 mutex_unlock(&hive->hive_lock);
820 return ret;
821 }
822
amdgpu_xgmi_update_topology(struct amdgpu_hive_info * hive,struct amdgpu_device * adev)823 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
824 {
825 int ret;
826
827 if (amdgpu_sriov_vf(adev))
828 return 0;
829
830 /* Each psp need to set the latest topology */
831 ret = psp_xgmi_set_topology_info(&adev->psp,
832 atomic_read(&hive->number_devices),
833 &adev->psp.xgmi_context.top_info);
834 if (ret)
835 dev_err(adev->dev,
836 "XGMI: Set topology failure on device %llx, hive %llx, ret %d",
837 adev->gmc.xgmi.node_id,
838 adev->gmc.xgmi.hive_id, ret);
839
840 return ret;
841 }
842
843
844 /*
845 * NOTE psp_xgmi_node_info.num_hops layout is as follows:
846 * num_hops[7:6] = link type (0 = xGMI2, 1 = xGMI3, 2/3 = reserved)
847 * num_hops[5:3] = reserved
848 * num_hops[2:0] = number of hops
849 */
amdgpu_xgmi_get_hops_count(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)850 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
851 struct amdgpu_device *peer_adev)
852 {
853 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
854 uint8_t num_hops_mask = 0x7;
855 int i;
856
857 if (!adev->gmc.xgmi.supported)
858 return 0;
859
860 for (i = 0 ; i < top->num_nodes; ++i)
861 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
862 return top->nodes[i].num_hops & num_hops_mask;
863
864 dev_err(adev->dev, "Failed to get xgmi hops count for peer %d.\n",
865 peer_adev->gmc.xgmi.physical_node_id);
866
867 return 0;
868 }
869
amdgpu_xgmi_get_bandwidth(struct amdgpu_device * adev,struct amdgpu_device * peer_adev,enum amdgpu_xgmi_bw_mode bw_mode,enum amdgpu_xgmi_bw_unit bw_unit,uint32_t * min_bw,uint32_t * max_bw)870 int amdgpu_xgmi_get_bandwidth(struct amdgpu_device *adev, struct amdgpu_device *peer_adev,
871 enum amdgpu_xgmi_bw_mode bw_mode, enum amdgpu_xgmi_bw_unit bw_unit,
872 uint32_t *min_bw, uint32_t *max_bw)
873 {
874 bool peer_mode = bw_mode == AMDGPU_XGMI_BW_MODE_PER_PEER;
875 int unit_scale = bw_unit == AMDGPU_XGMI_BW_UNIT_MBYTES ? 1000 : 1;
876 int num_lanes = adev->gmc.xgmi.max_width;
877 int speed = adev->gmc.xgmi.max_speed;
878 int num_links = !peer_mode ? 1 : -1;
879
880 if (!(min_bw && max_bw))
881 return -EINVAL;
882
883 *min_bw = 0;
884 *max_bw = 0;
885
886 if (!adev->gmc.xgmi.supported)
887 return -ENODATA;
888
889 if (peer_mode && !peer_adev)
890 return -EINVAL;
891
892 if (peer_mode) {
893 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
894 int i;
895
896 for (i = 0 ; i < top->num_nodes; ++i) {
897 if (top->nodes[i].node_id != peer_adev->gmc.xgmi.node_id)
898 continue;
899
900 num_links = top->nodes[i].num_links;
901 break;
902 }
903 }
904
905 if (num_links == -1) {
906 dev_err(adev->dev, "Failed to get number of xgmi links for peer %d.\n",
907 peer_adev->gmc.xgmi.physical_node_id);
908 } else if (num_links) {
909 int per_link_bw = (speed * num_lanes * unit_scale)/BITS_PER_BYTE;
910
911 *min_bw = per_link_bw;
912 *max_bw = num_links * per_link_bw;
913 }
914
915 return 0;
916 }
917
amdgpu_xgmi_get_is_sharing_enabled(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)918 bool amdgpu_xgmi_get_is_sharing_enabled(struct amdgpu_device *adev,
919 struct amdgpu_device *peer_adev)
920 {
921 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
922 int i;
923
924 /* Sharing should always be enabled for non-SRIOV. */
925 if (!amdgpu_sriov_vf(adev))
926 return true;
927
928 for (i = 0 ; i < top->num_nodes; ++i)
929 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
930 return !!top->nodes[i].is_sharing_enabled;
931
932 return false;
933 }
934
935 /*
936 * Devices that support extended data require the entire hive to initialize with
937 * the shared memory buffer flag set.
938 *
939 * Hive locks and conditions apply - see amdgpu_xgmi_add_device
940 */
amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info * hive,bool set_extended_data)941 static int amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info *hive,
942 bool set_extended_data)
943 {
944 struct amdgpu_device *tmp_adev;
945 int ret;
946
947 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
948 ret = psp_xgmi_initialize(&tmp_adev->psp, set_extended_data, false);
949 if (ret) {
950 dev_err(tmp_adev->dev,
951 "XGMI: Failed to initialize xgmi session for data partition %i\n",
952 set_extended_data);
953 return ret;
954 }
955
956 }
957
958 return 0;
959 }
960
amdgpu_xgmi_fill_topology_info(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)961 static void amdgpu_xgmi_fill_topology_info(struct amdgpu_device *adev,
962 struct amdgpu_device *peer_adev)
963 {
964 struct psp_xgmi_topology_info *top_info = &adev->psp.xgmi_context.top_info;
965 struct psp_xgmi_topology_info *peer_info = &peer_adev->psp.xgmi_context.top_info;
966
967 for (int i = 0; i < peer_info->num_nodes; i++) {
968 if (peer_info->nodes[i].node_id == adev->gmc.xgmi.node_id) {
969 for (int j = 0; j < top_info->num_nodes; j++) {
970 if (top_info->nodes[j].node_id == peer_adev->gmc.xgmi.node_id) {
971 peer_info->nodes[i].num_hops = top_info->nodes[j].num_hops;
972 peer_info->nodes[i].is_sharing_enabled =
973 top_info->nodes[j].is_sharing_enabled;
974 peer_info->nodes[i].num_links =
975 top_info->nodes[j].num_links;
976 return;
977 }
978 }
979 }
980 }
981 }
982
amdgpu_xgmi_add_device(struct amdgpu_device * adev)983 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
984 {
985 struct psp_xgmi_topology_info *top_info;
986 struct amdgpu_hive_info *hive;
987 struct amdgpu_xgmi *entry;
988 struct amdgpu_device *tmp_adev = NULL;
989
990 int count = 0, ret = 0;
991
992 if (!adev->gmc.xgmi.supported)
993 return 0;
994
995 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
996 ret = psp_xgmi_initialize(&adev->psp, false, true);
997 if (ret) {
998 dev_err(adev->dev,
999 "XGMI: Failed to initialize xgmi session\n");
1000 return ret;
1001 }
1002
1003 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id);
1004 if (ret) {
1005 dev_err(adev->dev,
1006 "XGMI: Failed to get hive id\n");
1007 return ret;
1008 }
1009
1010 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id);
1011 if (ret) {
1012 dev_err(adev->dev,
1013 "XGMI: Failed to get node id\n");
1014 return ret;
1015 }
1016 } else {
1017 adev->gmc.xgmi.hive_id = 16;
1018 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;
1019 }
1020
1021 hive = amdgpu_get_xgmi_hive(adev);
1022 if (!hive) {
1023 ret = -EINVAL;
1024 dev_err(adev->dev,
1025 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",
1026 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);
1027 goto exit;
1028 }
1029 mutex_lock(&hive->hive_lock);
1030
1031 top_info = &adev->psp.xgmi_context.top_info;
1032
1033 list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);
1034 list_for_each_entry(entry, &hive->device_list, head)
1035 top_info->nodes[count++].node_id = entry->node_id;
1036 top_info->num_nodes = count;
1037 atomic_set(&hive->number_devices, count);
1038
1039 task_barrier_add_task(&hive->tb);
1040
1041 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
1042 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1043 /* update node list for other device in the hive */
1044 if (tmp_adev != adev) {
1045 top_info = &tmp_adev->psp.xgmi_context.top_info;
1046 top_info->nodes[count - 1].node_id =
1047 adev->gmc.xgmi.node_id;
1048 top_info->num_nodes = count;
1049 }
1050 ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
1051 if (ret)
1052 goto exit_unlock;
1053 }
1054
1055 if (amdgpu_sriov_vf(adev) &&
1056 adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
1057 /* only get topology for VF being init if it can support full duplex */
1058 ret = psp_xgmi_get_topology_info(&adev->psp, count,
1059 &adev->psp.xgmi_context.top_info, false);
1060 if (ret) {
1061 dev_err(adev->dev,
1062 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
1063 adev->gmc.xgmi.node_id,
1064 adev->gmc.xgmi.hive_id, ret);
1065 /* To do: continue with some node failed or disable the whole hive*/
1066 goto exit_unlock;
1067 }
1068
1069 /* fill the topology info for peers instead of getting from PSP */
1070 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1071 amdgpu_xgmi_fill_topology_info(adev, tmp_adev);
1072 }
1073 } else {
1074 /* get latest topology info for each device from psp */
1075 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1076 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
1077 &tmp_adev->psp.xgmi_context.top_info, false);
1078 if (ret) {
1079 dev_err(tmp_adev->dev,
1080 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
1081 tmp_adev->gmc.xgmi.node_id,
1082 tmp_adev->gmc.xgmi.hive_id, ret);
1083 /* To do : continue with some node failed or disable the whole hive */
1084 goto exit_unlock;
1085 }
1086 }
1087 }
1088
1089 /* get topology again for hives that support extended data */
1090 if (adev->psp.xgmi_context.supports_extended_data) {
1091
1092 /* initialize the hive to get extended data. */
1093 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, true);
1094 if (ret)
1095 goto exit_unlock;
1096
1097 /* get the extended data. */
1098 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1099 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
1100 &tmp_adev->psp.xgmi_context.top_info, true);
1101 if (ret) {
1102 dev_err(tmp_adev->dev,
1103 "XGMI: Get topology for extended data failure on device %llx, hive %llx, ret %d",
1104 tmp_adev->gmc.xgmi.node_id,
1105 tmp_adev->gmc.xgmi.hive_id, ret);
1106 goto exit_unlock;
1107 }
1108 }
1109
1110 /* initialize the hive to get non-extended data for the next round. */
1111 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, false);
1112 if (ret)
1113 goto exit_unlock;
1114
1115 }
1116 }
1117
1118 if (!ret)
1119 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
1120
1121 exit_unlock:
1122 mutex_unlock(&hive->hive_lock);
1123 exit:
1124 if (!ret) {
1125 adev->hive = hive;
1126 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
1127 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
1128 } else {
1129 amdgpu_put_xgmi_hive(hive);
1130 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
1131 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
1132 ret);
1133 }
1134
1135 return ret;
1136 }
1137
amdgpu_xgmi_remove_device(struct amdgpu_device * adev)1138 int amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
1139 {
1140 struct amdgpu_hive_info *hive = adev->hive;
1141
1142 if (!adev->gmc.xgmi.supported)
1143 return -EINVAL;
1144
1145 if (!hive)
1146 return -EINVAL;
1147
1148 mutex_lock(&hive->hive_lock);
1149 task_barrier_rem_task(&hive->tb);
1150 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
1151 if (hive->hi_req_gpu == adev)
1152 hive->hi_req_gpu = NULL;
1153 list_del(&adev->gmc.xgmi.head);
1154 mutex_unlock(&hive->hive_lock);
1155
1156 amdgpu_put_xgmi_hive(hive);
1157 adev->hive = NULL;
1158
1159 if (atomic_dec_return(&hive->number_devices) == 0) {
1160 /* Remove the hive from global hive list */
1161 mutex_lock(&xgmi_mutex);
1162 list_del(&hive->node);
1163 mutex_unlock(&xgmi_mutex);
1164
1165 amdgpu_put_xgmi_hive(hive);
1166 }
1167
1168 return 0;
1169 }
1170
xgmi_v6_4_0_aca_bank_parser(struct aca_handle * handle,struct aca_bank * bank,enum aca_smu_type type,void * data)1171 static int xgmi_v6_4_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
1172 enum aca_smu_type type, void *data)
1173 {
1174 struct amdgpu_device *adev = handle->adev;
1175 struct aca_bank_info info;
1176 const char *error_str;
1177 u64 status, count;
1178 int ret, ext_error_code;
1179
1180 ret = aca_bank_info_decode(bank, &info);
1181 if (ret)
1182 return ret;
1183
1184 status = bank->regs[ACA_REG_IDX_STATUS];
1185 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1186
1187 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1188 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1189 if (error_str)
1190 dev_info(adev->dev, "%s detected\n", error_str);
1191
1192 count = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]);
1193
1194 switch (type) {
1195 case ACA_SMU_TYPE_UE:
1196 if (ext_error_code != 0 && ext_error_code != 9)
1197 count = 0ULL;
1198
1199 bank->aca_err_type = ACA_ERROR_TYPE_UE;
1200 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, count);
1201 break;
1202 case ACA_SMU_TYPE_CE:
1203 count = ext_error_code == 6 ? count : 0ULL;
1204 bank->aca_err_type = ACA_ERROR_TYPE_CE;
1205 ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, count);
1206 break;
1207 default:
1208 return -EINVAL;
1209 }
1210
1211 return ret;
1212 }
1213
1214 static const struct aca_bank_ops xgmi_v6_4_0_aca_bank_ops = {
1215 .aca_bank_parser = xgmi_v6_4_0_aca_bank_parser,
1216 };
1217
1218 static const struct aca_info xgmi_v6_4_0_aca_info = {
1219 .hwip = ACA_HWIP_TYPE_PCS_XGMI,
1220 .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
1221 .bank_ops = &xgmi_v6_4_0_aca_bank_ops,
1222 };
1223
amdgpu_xgmi_ras_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)1224 static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1225 {
1226 int r;
1227
1228 if (!adev->gmc.xgmi.supported ||
1229 adev->gmc.xgmi.num_physical_nodes == 0)
1230 return 0;
1231
1232 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1233
1234 r = amdgpu_ras_block_late_init(adev, ras_block);
1235 if (r)
1236 return r;
1237
1238 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1239 case IP_VERSION(6, 4, 0):
1240 case IP_VERSION(6, 4, 1):
1241 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL,
1242 &xgmi_v6_4_0_aca_info, NULL);
1243 if (r)
1244 goto late_fini;
1245 break;
1246 default:
1247 break;
1248 }
1249
1250 return 0;
1251
1252 late_fini:
1253 amdgpu_ras_block_late_fini(adev, ras_block);
1254
1255 return r;
1256 }
1257
amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device * adev,uint64_t addr)1258 uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
1259 uint64_t addr)
1260 {
1261 struct amdgpu_xgmi *xgmi = &adev->gmc.xgmi;
1262 return (addr + xgmi->physical_node_id * xgmi->node_segment_size);
1263 }
1264
pcs_clear_status(struct amdgpu_device * adev,uint32_t pcs_status_reg)1265 static void pcs_clear_status(struct amdgpu_device *adev, uint32_t pcs_status_reg)
1266 {
1267 WREG32_PCIE(pcs_status_reg, 0xFFFFFFFF);
1268 WREG32_PCIE(pcs_status_reg, 0);
1269 }
1270
amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device * adev)1271 static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev)
1272 {
1273 uint32_t i;
1274
1275 switch (adev->asic_type) {
1276 case CHIP_ARCTURUS:
1277 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++)
1278 pcs_clear_status(adev,
1279 xgmi_pcs_err_status_reg_arct[i]);
1280 break;
1281 case CHIP_VEGA20:
1282 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++)
1283 pcs_clear_status(adev,
1284 xgmi_pcs_err_status_reg_vg20[i]);
1285 break;
1286 case CHIP_ALDEBARAN:
1287 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++)
1288 pcs_clear_status(adev,
1289 xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1290 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++)
1291 pcs_clear_status(adev,
1292 walf_pcs_err_status_reg_aldebaran[i]);
1293 break;
1294 default:
1295 break;
1296 }
1297
1298 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1299 case IP_VERSION(6, 4, 0):
1300 case IP_VERSION(6, 4, 1):
1301 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++)
1302 pcs_clear_status(adev,
1303 xgmi3x16_pcs_err_status_reg_v6_4[i]);
1304 break;
1305 default:
1306 break;
1307 }
1308 }
1309
__xgmi_v6_4_0_reset_error_count(struct amdgpu_device * adev,int xgmi_inst,u64 mca_base)1310 static void __xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst, u64 mca_base)
1311 {
1312 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1313 }
1314
xgmi_v6_4_0_reset_error_count(struct amdgpu_device * adev,int xgmi_inst)1315 static void xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst)
1316 {
1317 int i;
1318
1319 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1320 __xgmi_v6_4_0_reset_error_count(adev, xgmi_inst, xgmi_v6_4_0_mca_base_array[i]);
1321 }
1322
xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device * adev)1323 static void xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device *adev)
1324 {
1325 int i;
1326
1327 for_each_inst(i, adev->aid_mask)
1328 xgmi_v6_4_0_reset_error_count(adev, i);
1329 }
1330
amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device * adev)1331 static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev)
1332 {
1333 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1334 case IP_VERSION(6, 4, 0):
1335 case IP_VERSION(6, 4, 1):
1336 xgmi_v6_4_0_reset_ras_error_count(adev);
1337 break;
1338 default:
1339 amdgpu_xgmi_legacy_reset_ras_error_count(adev);
1340 break;
1341 }
1342 }
1343
amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device * adev,uint32_t value,uint32_t mask_value,uint32_t * ue_count,uint32_t * ce_count,bool is_xgmi_pcs,bool check_mask)1344 static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,
1345 uint32_t value,
1346 uint32_t mask_value,
1347 uint32_t *ue_count,
1348 uint32_t *ce_count,
1349 bool is_xgmi_pcs,
1350 bool check_mask)
1351 {
1352 int i;
1353 int ue_cnt = 0;
1354 const struct amdgpu_pcs_ras_field *pcs_ras_fields = NULL;
1355 uint32_t field_array_size = 0;
1356
1357 if (is_xgmi_pcs) {
1358 if (amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1359 IP_VERSION(6, 1, 0) ||
1360 amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1361 IP_VERSION(6, 4, 0) ||
1362 amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1363 IP_VERSION(6, 4, 1)) {
1364 pcs_ras_fields = &xgmi3x16_pcs_ras_fields[0];
1365 field_array_size = ARRAY_SIZE(xgmi3x16_pcs_ras_fields);
1366 } else {
1367 pcs_ras_fields = &xgmi_pcs_ras_fields[0];
1368 field_array_size = ARRAY_SIZE(xgmi_pcs_ras_fields);
1369 }
1370 } else {
1371 pcs_ras_fields = &wafl_pcs_ras_fields[0];
1372 field_array_size = ARRAY_SIZE(wafl_pcs_ras_fields);
1373 }
1374
1375 if (check_mask)
1376 value = value & ~mask_value;
1377
1378 /* query xgmi/walf pcs error status,
1379 * only ue is supported */
1380 for (i = 0; value && i < field_array_size; i++) {
1381 ue_cnt = (value &
1382 pcs_ras_fields[i].pcs_err_mask) >>
1383 pcs_ras_fields[i].pcs_err_shift;
1384 if (ue_cnt) {
1385 dev_info(adev->dev, "%s detected\n",
1386 pcs_ras_fields[i].err_name);
1387 *ue_count += ue_cnt;
1388 }
1389
1390 /* reset bit value if the bit is checked */
1391 value &= ~(pcs_ras_fields[i].pcs_err_mask);
1392 }
1393
1394 return 0;
1395 }
1396
amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1397 static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev,
1398 void *ras_error_status)
1399 {
1400 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1401 int i, supported = 1;
1402 uint32_t data, mask_data = 0;
1403 uint32_t ue_cnt = 0, ce_cnt = 0;
1404
1405 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL))
1406 return ;
1407
1408 err_data->ue_count = 0;
1409 err_data->ce_count = 0;
1410
1411 switch (adev->asic_type) {
1412 case CHIP_ARCTURUS:
1413 /* check xgmi pcs error */
1414 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++) {
1415 data = RREG32_PCIE(xgmi_pcs_err_status_reg_arct[i]);
1416 if (data)
1417 amdgpu_xgmi_query_pcs_error_status(adev, data,
1418 mask_data, &ue_cnt, &ce_cnt, true, false);
1419 }
1420 /* check wafl pcs error */
1421 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_arct); i++) {
1422 data = RREG32_PCIE(wafl_pcs_err_status_reg_arct[i]);
1423 if (data)
1424 amdgpu_xgmi_query_pcs_error_status(adev, data,
1425 mask_data, &ue_cnt, &ce_cnt, false, false);
1426 }
1427 break;
1428 case CHIP_VEGA20:
1429 /* check xgmi pcs error */
1430 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++) {
1431 data = RREG32_PCIE(xgmi_pcs_err_status_reg_vg20[i]);
1432 if (data)
1433 amdgpu_xgmi_query_pcs_error_status(adev, data,
1434 mask_data, &ue_cnt, &ce_cnt, true, false);
1435 }
1436 /* check wafl pcs error */
1437 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_vg20); i++) {
1438 data = RREG32_PCIE(wafl_pcs_err_status_reg_vg20[i]);
1439 if (data)
1440 amdgpu_xgmi_query_pcs_error_status(adev, data,
1441 mask_data, &ue_cnt, &ce_cnt, false, false);
1442 }
1443 break;
1444 case CHIP_ALDEBARAN:
1445 /* check xgmi3x16 pcs error */
1446 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++) {
1447 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1448 mask_data =
1449 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1450 if (data)
1451 amdgpu_xgmi_query_pcs_error_status(adev, data,
1452 mask_data, &ue_cnt, &ce_cnt, true, true);
1453 }
1454 /* check wafl pcs error */
1455 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++) {
1456 data = RREG32_PCIE(walf_pcs_err_status_reg_aldebaran[i]);
1457 mask_data =
1458 RREG32_PCIE(walf_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1459 if (data)
1460 amdgpu_xgmi_query_pcs_error_status(adev, data,
1461 mask_data, &ue_cnt, &ce_cnt, false, true);
1462 }
1463 break;
1464 default:
1465 supported = 0;
1466 break;
1467 }
1468
1469 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1470 case IP_VERSION(6, 4, 0):
1471 case IP_VERSION(6, 4, 1):
1472 /* check xgmi3x16 pcs error */
1473 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++) {
1474 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_v6_4[i]);
1475 mask_data =
1476 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[i]);
1477 if (data)
1478 amdgpu_xgmi_query_pcs_error_status(adev, data,
1479 mask_data, &ue_cnt, &ce_cnt, true, true);
1480 }
1481 break;
1482 default:
1483 if (!supported)
1484 dev_warn(adev->dev, "XGMI RAS error query not supported");
1485 break;
1486 }
1487
1488 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1489
1490 err_data->ue_count += ue_cnt;
1491 err_data->ce_count += ce_cnt;
1492 }
1493
xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device * adev,u64 status)1494 static enum aca_error_type xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device *adev, u64 status)
1495 {
1496 const char *error_str;
1497 int ext_error_code;
1498
1499 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1500
1501 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1502 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1503 if (error_str)
1504 dev_info(adev->dev, "%s detected\n", error_str);
1505
1506 switch (ext_error_code) {
1507 case 0:
1508 return ACA_ERROR_TYPE_UE;
1509 case 6:
1510 return ACA_ERROR_TYPE_CE;
1511 default:
1512 return -EINVAL;
1513 }
1514
1515 return -EINVAL;
1516 }
1517
__xgmi_v6_4_0_query_error_count(struct amdgpu_device * adev,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 mca_base,struct ras_err_data * err_data)1518 static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct amdgpu_smuio_mcm_config_info *mcm_info,
1519 u64 mca_base, struct ras_err_data *err_data)
1520 {
1521 int xgmi_inst = mcm_info->die_id;
1522 u64 status = 0;
1523
1524 status = RREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS);
1525 if (!ACA_REG__STATUS__VAL(status))
1526 return;
1527
1528 switch (xgmi_v6_4_0_pcs_mca_get_error_type(adev, status)) {
1529 case ACA_ERROR_TYPE_UE:
1530 amdgpu_ras_error_statistic_ue_count(err_data, mcm_info, 1ULL);
1531 break;
1532 case ACA_ERROR_TYPE_CE:
1533 amdgpu_ras_error_statistic_ce_count(err_data, mcm_info, 1ULL);
1534 break;
1535 default:
1536 break;
1537 }
1538
1539 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1540 }
1541
xgmi_v6_4_0_query_error_count(struct amdgpu_device * adev,int xgmi_inst,struct ras_err_data * err_data)1542 static void xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, int xgmi_inst, struct ras_err_data *err_data)
1543 {
1544 struct amdgpu_smuio_mcm_config_info mcm_info = {
1545 .socket_id = adev->smuio.funcs->get_socket_id(adev),
1546 .die_id = xgmi_inst,
1547 };
1548 int i;
1549
1550 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1551 __xgmi_v6_4_0_query_error_count(adev, &mcm_info, xgmi_v6_4_0_mca_base_array[i], err_data);
1552 }
1553
xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1554 static void xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status)
1555 {
1556 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1557 int i;
1558
1559 for_each_inst(i, adev->aid_mask)
1560 xgmi_v6_4_0_query_error_count(adev, i, err_data);
1561 }
1562
amdgpu_xgmi_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1563 static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev,
1564 void *ras_error_status)
1565 {
1566 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1567 case IP_VERSION(6, 4, 0):
1568 case IP_VERSION(6, 4, 1):
1569 xgmi_v6_4_0_query_ras_error_count(adev, ras_error_status);
1570 break;
1571 default:
1572 amdgpu_xgmi_legacy_query_ras_error_count(adev, ras_error_status);
1573 break;
1574 }
1575 }
1576
1577 /* Trigger XGMI/WAFL error */
amdgpu_ras_error_inject_xgmi(struct amdgpu_device * adev,void * inject_if,uint32_t instance_mask)1578 static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev,
1579 void *inject_if, uint32_t instance_mask)
1580 {
1581 int ret1, ret2;
1582 struct ta_ras_trigger_error_input *block_info =
1583 (struct ta_ras_trigger_error_input *)inject_if;
1584
1585 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
1586 dev_warn(adev->dev, "Failed to disallow df cstate");
1587
1588 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DISALLOW);
1589 if (ret1 && ret1 != -EOPNOTSUPP)
1590 dev_warn(adev->dev, "Failed to disallow XGMI power down");
1591
1592 ret2 = psp_ras_trigger_error(&adev->psp, block_info, instance_mask);
1593
1594 if (amdgpu_ras_intr_triggered())
1595 return ret2;
1596
1597 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DEFAULT);
1598 if (ret1 && ret1 != -EOPNOTSUPP)
1599 dev_warn(adev->dev, "Failed to allow XGMI power down");
1600
1601 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW))
1602 dev_warn(adev->dev, "Failed to allow df cstate");
1603
1604 return ret2;
1605 }
1606
1607 struct amdgpu_ras_block_hw_ops xgmi_ras_hw_ops = {
1608 .query_ras_error_count = amdgpu_xgmi_query_ras_error_count,
1609 .reset_ras_error_count = amdgpu_xgmi_reset_ras_error_count,
1610 .ras_error_inject = amdgpu_ras_error_inject_xgmi,
1611 };
1612
1613 struct amdgpu_xgmi_ras xgmi_ras = {
1614 .ras_block = {
1615 .hw_ops = &xgmi_ras_hw_ops,
1616 .ras_late_init = amdgpu_xgmi_ras_late_init,
1617 },
1618 };
1619
amdgpu_xgmi_ras_sw_init(struct amdgpu_device * adev)1620 int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev)
1621 {
1622 int err;
1623 struct amdgpu_xgmi_ras *ras;
1624
1625 if (!adev->gmc.xgmi.ras)
1626 return 0;
1627
1628 ras = adev->gmc.xgmi.ras;
1629 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
1630 if (err) {
1631 dev_err(adev->dev, "Failed to register xgmi_wafl_pcs ras block!\n");
1632 return err;
1633 }
1634
1635 strcpy(ras->ras_block.ras_comm.name, "xgmi_wafl");
1636 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__XGMI_WAFL;
1637 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
1638 adev->gmc.xgmi.ras_if = &ras->ras_block.ras_comm;
1639
1640 return 0;
1641 }
1642
amdgpu_xgmi_reset_on_init_work(struct work_struct * work)1643 static void amdgpu_xgmi_reset_on_init_work(struct work_struct *work)
1644 {
1645 struct amdgpu_hive_info *hive =
1646 container_of(work, struct amdgpu_hive_info, reset_on_init_work);
1647 struct amdgpu_reset_context reset_context;
1648 struct amdgpu_device *tmp_adev;
1649 struct list_head device_list;
1650 int r;
1651
1652 mutex_lock(&hive->hive_lock);
1653
1654 INIT_LIST_HEAD(&device_list);
1655 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
1656 list_add_tail(&tmp_adev->reset_list, &device_list);
1657
1658 tmp_adev = list_first_entry(&device_list, struct amdgpu_device,
1659 reset_list);
1660 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
1661
1662 reset_context.method = AMD_RESET_METHOD_ON_INIT;
1663 reset_context.reset_req_dev = tmp_adev;
1664 reset_context.hive = hive;
1665 reset_context.reset_device_list = &device_list;
1666 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1667 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
1668
1669 amdgpu_reset_do_xgmi_reset_on_init(&reset_context);
1670 mutex_unlock(&hive->hive_lock);
1671 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
1672
1673 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1674 r = amdgpu_ras_init_badpage_info(tmp_adev);
1675 if (r && r != -EHWPOISON)
1676 dev_err(tmp_adev->dev,
1677 "error during bad page data initialization");
1678 }
1679 }
1680
amdgpu_xgmi_schedule_reset_on_init(struct amdgpu_hive_info * hive)1681 static void amdgpu_xgmi_schedule_reset_on_init(struct amdgpu_hive_info *hive)
1682 {
1683 INIT_WORK(&hive->reset_on_init_work, amdgpu_xgmi_reset_on_init_work);
1684 amdgpu_reset_domain_schedule(hive->reset_domain,
1685 &hive->reset_on_init_work);
1686 }
1687
amdgpu_xgmi_reset_on_init(struct amdgpu_device * adev)1688 int amdgpu_xgmi_reset_on_init(struct amdgpu_device *adev)
1689 {
1690 struct amdgpu_hive_info *hive;
1691 bool reset_scheduled;
1692 int num_devs;
1693
1694 hive = amdgpu_get_xgmi_hive(adev);
1695 if (!hive)
1696 return -EINVAL;
1697
1698 mutex_lock(&hive->hive_lock);
1699 num_devs = atomic_read(&hive->number_devices);
1700 reset_scheduled = false;
1701 if (num_devs == adev->gmc.xgmi.num_physical_nodes) {
1702 amdgpu_xgmi_schedule_reset_on_init(hive);
1703 reset_scheduled = true;
1704 }
1705
1706 mutex_unlock(&hive->hive_lock);
1707 amdgpu_put_xgmi_hive(hive);
1708
1709 if (reset_scheduled)
1710 flush_work(&hive->reset_on_init_work);
1711
1712 return 0;
1713 }
1714
amdgpu_xgmi_request_nps_change(struct amdgpu_device * adev,struct amdgpu_hive_info * hive,int req_nps_mode)1715 int amdgpu_xgmi_request_nps_change(struct amdgpu_device *adev,
1716 struct amdgpu_hive_info *hive,
1717 int req_nps_mode)
1718 {
1719 struct amdgpu_device *tmp_adev;
1720 int cur_nps_mode, r;
1721
1722 /* This is expected to be called only during unload of driver. The
1723 * request needs to be placed only once for all devices in the hive. If
1724 * one of them fail, revert the request for previous successful devices.
1725 * After placing the request, make hive mode as UNKNOWN so that other
1726 * devices don't request anymore.
1727 */
1728 mutex_lock(&hive->hive_lock);
1729 if (atomic_read(&hive->requested_nps_mode) ==
1730 UNKNOWN_MEMORY_PARTITION_MODE) {
1731 dev_dbg(adev->dev, "Unexpected entry for hive NPS change");
1732 mutex_unlock(&hive->hive_lock);
1733 return 0;
1734 }
1735 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
1736 r = adev->gmc.gmc_funcs->request_mem_partition_mode(
1737 tmp_adev, req_nps_mode);
1738 if (r)
1739 break;
1740 }
1741 if (r) {
1742 /* Request back current mode if one of the requests failed */
1743 cur_nps_mode =
1744 adev->gmc.gmc_funcs->query_mem_partition_mode(tmp_adev);
1745 list_for_each_entry_continue_reverse(
1746 tmp_adev, &hive->device_list, gmc.xgmi.head)
1747 adev->gmc.gmc_funcs->request_mem_partition_mode(
1748 tmp_adev, cur_nps_mode);
1749 }
1750 /* Set to UNKNOWN so that other devices don't request anymore */
1751 atomic_set(&hive->requested_nps_mode, UNKNOWN_MEMORY_PARTITION_MODE);
1752 mutex_unlock(&hive->hive_lock);
1753
1754 return r;
1755 }
1756
amdgpu_xgmi_same_hive(struct amdgpu_device * adev,struct amdgpu_device * bo_adev)1757 bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
1758 struct amdgpu_device *bo_adev)
1759 {
1760 return (amdgpu_use_xgmi_p2p && adev != bo_adev &&
1761 adev->gmc.xgmi.hive_id &&
1762 adev->gmc.xgmi.hive_id == bo_adev->gmc.xgmi.hive_id);
1763 }
1764
amdgpu_xgmi_early_init(struct amdgpu_device * adev)1765 void amdgpu_xgmi_early_init(struct amdgpu_device *adev)
1766 {
1767 if (!adev->gmc.xgmi.supported)
1768 return;
1769
1770 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1771 case IP_VERSION(9, 4, 0):
1772 case IP_VERSION(9, 4, 1):
1773 case IP_VERSION(9, 4, 2):
1774 /* 25 GT/s */
1775 adev->gmc.xgmi.max_speed = 25;
1776 adev->gmc.xgmi.max_width = 16;
1777 break;
1778 case IP_VERSION(9, 4, 3):
1779 case IP_VERSION(9, 4, 4):
1780 case IP_VERSION(9, 5, 0):
1781 /* 32 GT/s */
1782 adev->gmc.xgmi.max_speed = 32;
1783 adev->gmc.xgmi.max_width = 16;
1784 break;
1785 default:
1786 break;
1787 }
1788 }
1789
amgpu_xgmi_set_max_speed_width(struct amdgpu_device * adev,uint16_t max_speed,uint8_t max_width)1790 void amgpu_xgmi_set_max_speed_width(struct amdgpu_device *adev,
1791 uint16_t max_speed, uint8_t max_width)
1792 {
1793 adev->gmc.xgmi.max_speed = max_speed;
1794 adev->gmc.xgmi.max_width = max_width;
1795 }
1796