1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24 #include <linux/list.h>
25 #include "amdgpu.h"
26 #include "amdgpu_xgmi.h"
27 #include "amdgpu_ras.h"
28 #include "soc15.h"
29 #include "df/df_3_6_offset.h"
30 #include "xgmi/xgmi_4_0_0_smn.h"
31 #include "xgmi/xgmi_4_0_0_sh_mask.h"
32 #include "xgmi/xgmi_6_1_0_sh_mask.h"
33 #include "wafl/wafl2_4_0_0_smn.h"
34 #include "wafl/wafl2_4_0_0_sh_mask.h"
35
36 #include "amdgpu_reset.h"
37
38 #define smnPCS_XGMI3X16_PCS_ERROR_STATUS 0x11a0020c
39 #define smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK 0x11a00218
40 #define smnPCS_GOPX1_PCS_ERROR_STATUS 0x12200210
41 #define smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK 0x12200218
42
43 static DEFINE_MUTEX(xgmi_mutex);
44
45 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4
46
47 static LIST_HEAD(xgmi_hive_list);
48
49 static const int xgmi_pcs_err_status_reg_vg20[] = {
50 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
51 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
52 };
53
54 static const int wafl_pcs_err_status_reg_vg20[] = {
55 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
56 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
57 };
58
59 static const int xgmi_pcs_err_status_reg_arct[] = {
60 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,
61 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,
62 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x500000,
63 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x600000,
64 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x700000,
65 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x800000,
66 };
67
68 /* same as vg20*/
69 static const int wafl_pcs_err_status_reg_arct[] = {
70 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,
71 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,
72 };
73
74 static const int xgmi3x16_pcs_err_status_reg_aldebaran[] = {
75 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
76 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000,
77 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x200000,
78 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x300000,
79 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x400000,
80 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x500000,
81 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x600000,
82 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x700000
83 };
84
85 static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
86 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
87 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000,
88 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x200000,
89 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x300000,
90 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x400000,
91 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x500000,
92 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x600000,
93 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x700000
94 };
95
96 static const int walf_pcs_err_status_reg_aldebaran[] = {
97 smnPCS_GOPX1_PCS_ERROR_STATUS,
98 smnPCS_GOPX1_PCS_ERROR_STATUS + 0x100000
99 };
100
101 static const int walf_pcs_err_noncorrectable_mask_reg_aldebaran[] = {
102 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK,
103 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
104 };
105
106 static const int xgmi3x16_pcs_err_status_reg_v6_4[] = {
107 smnPCS_XGMI3X16_PCS_ERROR_STATUS,
108 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000
109 };
110
111 static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[] = {
112 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,
113 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000
114 };
115
116 static const u64 xgmi_v6_4_0_mca_base_array[] = {
117 0x11a09200,
118 0x11b09200,
119 };
120
121 static const char *xgmi_v6_4_0_ras_error_code_ext[32] = {
122 [0x00] = "XGMI PCS DataLossErr",
123 [0x01] = "XGMI PCS TrainingErr",
124 [0x02] = "XGMI PCS FlowCtrlAckErr",
125 [0x03] = "XGMI PCS RxFifoUnderflowErr",
126 [0x04] = "XGMI PCS RxFifoOverflowErr",
127 [0x05] = "XGMI PCS CRCErr",
128 [0x06] = "XGMI PCS BERExceededErr",
129 [0x07] = "XGMI PCS TxMetaDataErr",
130 [0x08] = "XGMI PCS ReplayBufParityErr",
131 [0x09] = "XGMI PCS DataParityErr",
132 [0x0a] = "XGMI PCS ReplayFifoOverflowErr",
133 [0x0b] = "XGMI PCS ReplayFifoUnderflowErr",
134 [0x0c] = "XGMI PCS ElasticFifoOverflowErr",
135 [0x0d] = "XGMI PCS DeskewErr",
136 [0x0e] = "XGMI PCS FlowCtrlCRCErr",
137 [0x0f] = "XGMI PCS DataStartupLimitErr",
138 [0x10] = "XGMI PCS FCInitTimeoutErr",
139 [0x11] = "XGMI PCS RecoveryTimeoutErr",
140 [0x12] = "XGMI PCS ReadySerialTimeoutErr",
141 [0x13] = "XGMI PCS ReadySerialAttemptErr",
142 [0x14] = "XGMI PCS RecoveryAttemptErr",
143 [0x15] = "XGMI PCS RecoveryRelockAttemptErr",
144 [0x16] = "XGMI PCS ReplayAttemptErr",
145 [0x17] = "XGMI PCS SyncHdrErr",
146 [0x18] = "XGMI PCS TxReplayTimeoutErr",
147 [0x19] = "XGMI PCS RxReplayTimeoutErr",
148 [0x1a] = "XGMI PCS LinkSubTxTimeoutErr",
149 [0x1b] = "XGMI PCS LinkSubRxTimeoutErr",
150 [0x1c] = "XGMI PCS RxCMDPktErr",
151 };
152
153 static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = {
154 {"XGMI PCS DataLossErr",
155 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)},
156 {"XGMI PCS TrainingErr",
157 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TrainingErr)},
158 {"XGMI PCS CRCErr",
159 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, CRCErr)},
160 {"XGMI PCS BERExceededErr",
161 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, BERExceededErr)},
162 {"XGMI PCS TxMetaDataErr",
163 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TxMetaDataErr)},
164 {"XGMI PCS ReplayBufParityErr",
165 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayBufParityErr)},
166 {"XGMI PCS DataParityErr",
167 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataParityErr)},
168 {"XGMI PCS ReplayFifoOverflowErr",
169 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
170 {"XGMI PCS ReplayFifoUnderflowErr",
171 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
172 {"XGMI PCS ElasticFifoOverflowErr",
173 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
174 {"XGMI PCS DeskewErr",
175 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DeskewErr)},
176 {"XGMI PCS DataStartupLimitErr",
177 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataStartupLimitErr)},
178 {"XGMI PCS FCInitTimeoutErr",
179 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
180 {"XGMI PCS RecoveryTimeoutErr",
181 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
182 {"XGMI PCS ReadySerialTimeoutErr",
183 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
184 {"XGMI PCS ReadySerialAttemptErr",
185 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
186 {"XGMI PCS RecoveryAttemptErr",
187 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
188 {"XGMI PCS RecoveryRelockAttemptErr",
189 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
190 };
191
192 static const struct amdgpu_pcs_ras_field wafl_pcs_ras_fields[] = {
193 {"WAFL PCS DataLossErr",
194 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataLossErr)},
195 {"WAFL PCS TrainingErr",
196 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TrainingErr)},
197 {"WAFL PCS CRCErr",
198 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, CRCErr)},
199 {"WAFL PCS BERExceededErr",
200 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, BERExceededErr)},
201 {"WAFL PCS TxMetaDataErr",
202 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TxMetaDataErr)},
203 {"WAFL PCS ReplayBufParityErr",
204 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayBufParityErr)},
205 {"WAFL PCS DataParityErr",
206 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataParityErr)},
207 {"WAFL PCS ReplayFifoOverflowErr",
208 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
209 {"WAFL PCS ReplayFifoUnderflowErr",
210 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
211 {"WAFL PCS ElasticFifoOverflowErr",
212 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
213 {"WAFL PCS DeskewErr",
214 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DeskewErr)},
215 {"WAFL PCS DataStartupLimitErr",
216 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataStartupLimitErr)},
217 {"WAFL PCS FCInitTimeoutErr",
218 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, FCInitTimeoutErr)},
219 {"WAFL PCS RecoveryTimeoutErr",
220 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
221 {"WAFL PCS ReadySerialTimeoutErr",
222 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
223 {"WAFL PCS ReadySerialAttemptErr",
224 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
225 {"WAFL PCS RecoveryAttemptErr",
226 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryAttemptErr)},
227 {"WAFL PCS RecoveryRelockAttemptErr",
228 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
229 };
230
231 static const struct amdgpu_pcs_ras_field xgmi3x16_pcs_ras_fields[] = {
232 {"XGMI3X16 PCS DataLossErr",
233 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataLossErr)},
234 {"XGMI3X16 PCS TrainingErr",
235 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TrainingErr)},
236 {"XGMI3X16 PCS FlowCtrlAckErr",
237 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlAckErr)},
238 {"XGMI3X16 PCS RxFifoUnderflowErr",
239 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoUnderflowErr)},
240 {"XGMI3X16 PCS RxFifoOverflowErr",
241 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoOverflowErr)},
242 {"XGMI3X16 PCS CRCErr",
243 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, CRCErr)},
244 {"XGMI3X16 PCS BERExceededErr",
245 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, BERExceededErr)},
246 {"XGMI3X16 PCS TxVcidDataErr",
247 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxVcidDataErr)},
248 {"XGMI3X16 PCS ReplayBufParityErr",
249 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayBufParityErr)},
250 {"XGMI3X16 PCS DataParityErr",
251 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataParityErr)},
252 {"XGMI3X16 PCS ReplayFifoOverflowErr",
253 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},
254 {"XGMI3X16 PCS ReplayFifoUnderflowErr",
255 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},
256 {"XGMI3X16 PCS ElasticFifoOverflowErr",
257 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},
258 {"XGMI3X16 PCS DeskewErr",
259 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DeskewErr)},
260 {"XGMI3X16 PCS FlowCtrlCRCErr",
261 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlCRCErr)},
262 {"XGMI3X16 PCS DataStartupLimitErr",
263 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataStartupLimitErr)},
264 {"XGMI3X16 PCS FCInitTimeoutErr",
265 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FCInitTimeoutErr)},
266 {"XGMI3X16 PCS RecoveryTimeoutErr",
267 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},
268 {"XGMI3X16 PCS ReadySerialTimeoutErr",
269 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},
270 {"XGMI3X16 PCS ReadySerialAttemptErr",
271 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},
272 {"XGMI3X16 PCS RecoveryAttemptErr",
273 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryAttemptErr)},
274 {"XGMI3X16 PCS RecoveryRelockAttemptErr",
275 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},
276 {"XGMI3X16 PCS ReplayAttemptErr",
277 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayAttemptErr)},
278 {"XGMI3X16 PCS SyncHdrErr",
279 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, SyncHdrErr)},
280 {"XGMI3X16 PCS TxReplayTimeoutErr",
281 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxReplayTimeoutErr)},
282 {"XGMI3X16 PCS RxReplayTimeoutErr",
283 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxReplayTimeoutErr)},
284 {"XGMI3X16 PCS LinkSubTxTimeoutErr",
285 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubTxTimeoutErr)},
286 {"XGMI3X16 PCS LinkSubRxTimeoutErr",
287 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubRxTimeoutErr)},
288 {"XGMI3X16 PCS RxCMDPktErr",
289 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxCMDPktErr)},
290 };
291
292 /**
293 * DOC: AMDGPU XGMI Support
294 *
295 * XGMI is a high speed interconnect that joins multiple GPU cards
296 * into a homogeneous memory space that is organized by a collective
297 * hive ID and individual node IDs, both of which are 64-bit numbers.
298 *
299 * The file xgmi_device_id contains the unique per GPU device ID and
300 * is stored in the /sys/class/drm/card${cardno}/device/ directory.
301 *
302 * Inside the device directory a sub-directory 'xgmi_hive_info' is
303 * created which contains the hive ID and the list of nodes.
304 *
305 * The hive ID is stored in:
306 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
307 *
308 * The node information is stored in numbered directories:
309 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
310 *
311 * Each device has their own xgmi_hive_info direction with a mirror
312 * set of node sub-directories.
313 *
314 * The XGMI memory space is built by contiguously adding the power of
315 * two padded VRAM space from each node to each other.
316 *
317 */
318
319 static struct attribute amdgpu_xgmi_hive_id = {
320 .name = "xgmi_hive_id",
321 .mode = S_IRUGO
322 };
323
324 static struct attribute *amdgpu_xgmi_hive_attrs[] = {
325 &amdgpu_xgmi_hive_id,
326 NULL
327 };
328 ATTRIBUTE_GROUPS(amdgpu_xgmi_hive);
329
amdgpu_xgmi_show_attrs(struct kobject * kobj,struct attribute * attr,char * buf)330 static ssize_t amdgpu_xgmi_show_attrs(struct kobject *kobj,
331 struct attribute *attr, char *buf)
332 {
333 struct amdgpu_hive_info *hive = container_of(
334 kobj, struct amdgpu_hive_info, kobj);
335
336 if (attr == &amdgpu_xgmi_hive_id)
337 return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);
338
339 return 0;
340 }
341
amdgpu_xgmi_hive_release(struct kobject * kobj)342 static void amdgpu_xgmi_hive_release(struct kobject *kobj)
343 {
344 struct amdgpu_hive_info *hive = container_of(
345 kobj, struct amdgpu_hive_info, kobj);
346
347 amdgpu_reset_put_reset_domain(hive->reset_domain);
348 hive->reset_domain = NULL;
349
350 mutex_destroy(&hive->hive_lock);
351 kfree(hive);
352 }
353
354 static const struct sysfs_ops amdgpu_xgmi_hive_ops = {
355 .show = amdgpu_xgmi_show_attrs,
356 };
357
358 static const struct kobj_type amdgpu_xgmi_hive_type = {
359 .release = amdgpu_xgmi_hive_release,
360 .sysfs_ops = &amdgpu_xgmi_hive_ops,
361 .default_groups = amdgpu_xgmi_hive_groups,
362 };
363
amdgpu_xgmi_show_device_id(struct device * dev,struct device_attribute * attr,char * buf)364 static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
365 struct device_attribute *attr,
366 char *buf)
367 {
368 struct drm_device *ddev = dev_get_drvdata(dev);
369 struct amdgpu_device *adev = drm_to_adev(ddev);
370
371 return sysfs_emit(buf, "%llu\n", adev->gmc.xgmi.node_id);
372
373 }
374
amdgpu_xgmi_show_physical_id(struct device * dev,struct device_attribute * attr,char * buf)375 static ssize_t amdgpu_xgmi_show_physical_id(struct device *dev,
376 struct device_attribute *attr,
377 char *buf)
378 {
379 struct drm_device *ddev = dev_get_drvdata(dev);
380 struct amdgpu_device *adev = drm_to_adev(ddev);
381
382 return sysfs_emit(buf, "%u\n", adev->gmc.xgmi.physical_node_id);
383
384 }
385
amdgpu_xgmi_show_num_hops(struct device * dev,struct device_attribute * attr,char * buf)386 static ssize_t amdgpu_xgmi_show_num_hops(struct device *dev,
387 struct device_attribute *attr,
388 char *buf)
389 {
390 struct drm_device *ddev = dev_get_drvdata(dev);
391 struct amdgpu_device *adev = drm_to_adev(ddev);
392 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
393 int i;
394
395 for (i = 0; i < top->num_nodes; i++)
396 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_hops);
397
398 return sysfs_emit(buf, "%s\n", buf);
399 }
400
amdgpu_xgmi_show_num_links(struct device * dev,struct device_attribute * attr,char * buf)401 static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,
402 struct device_attribute *attr,
403 char *buf)
404 {
405 struct drm_device *ddev = dev_get_drvdata(dev);
406 struct amdgpu_device *adev = drm_to_adev(ddev);
407 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
408 int i;
409
410 for (i = 0; i < top->num_nodes; i++)
411 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_links);
412
413 return sysfs_emit(buf, "%s\n", buf);
414 }
415
amdgpu_xgmi_show_connected_port_num(struct device * dev,struct device_attribute * attr,char * buf)416 static ssize_t amdgpu_xgmi_show_connected_port_num(struct device *dev,
417 struct device_attribute *attr,
418 char *buf)
419 {
420 struct drm_device *ddev = dev_get_drvdata(dev);
421 struct amdgpu_device *adev = drm_to_adev(ddev);
422 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
423 int i, j, size = 0;
424 int current_node;
425 /*
426 * get the node id in the sysfs for the current socket and show
427 * it in the port num info output in the sysfs for easy reading.
428 * it is NOT the one retrieved from xgmi ta.
429 */
430 for (i = 0; i < top->num_nodes; i++) {
431 if (top->nodes[i].node_id == adev->gmc.xgmi.node_id) {
432 current_node = i;
433 break;
434 }
435 }
436
437 if (i == top->num_nodes)
438 return -EINVAL;
439
440 for (i = 0; i < top->num_nodes; i++) {
441 for (j = 0; j < top->nodes[i].num_links; j++)
442 /* node id in sysfs starts from 1 rather than 0 so +1 here */
443 size += sysfs_emit_at(buf, size, "%02x:%02x -> %02x:%02x\n", current_node + 1,
444 top->nodes[i].port_num[j].src_xgmi_port_num, i + 1,
445 top->nodes[i].port_num[j].dst_xgmi_port_num);
446 }
447
448 return size;
449 }
450
451 #define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801)
amdgpu_xgmi_show_error(struct device * dev,struct device_attribute * attr,char * buf)452 static ssize_t amdgpu_xgmi_show_error(struct device *dev,
453 struct device_attribute *attr,
454 char *buf)
455 {
456 struct drm_device *ddev = dev_get_drvdata(dev);
457 struct amdgpu_device *adev = drm_to_adev(ddev);
458 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;
459 uint64_t fica_out;
460 unsigned int error_count = 0;
461
462 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);
463 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);
464
465 if ((!adev->df.funcs) ||
466 (!adev->df.funcs->get_fica) ||
467 (!adev->df.funcs->set_fica))
468 return -EINVAL;
469
470 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_ctl_in);
471 if (fica_out != 0x1f)
472 pr_err("xGMI error counters not enabled!\n");
473
474 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_status_in);
475
476 if ((fica_out & 0xffff) == 2)
477 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);
478
479 adev->df.funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);
480
481 return sysfs_emit(buf, "%u\n", error_count);
482 }
483
484
485 static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
486 static DEVICE_ATTR(xgmi_physical_id, 0444, amdgpu_xgmi_show_physical_id, NULL);
487 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
488 static DEVICE_ATTR(xgmi_num_hops, S_IRUGO, amdgpu_xgmi_show_num_hops, NULL);
489 static DEVICE_ATTR(xgmi_num_links, S_IRUGO, amdgpu_xgmi_show_num_links, NULL);
490 static DEVICE_ATTR(xgmi_port_num, S_IRUGO, amdgpu_xgmi_show_connected_port_num, NULL);
491
amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)492 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
493 struct amdgpu_hive_info *hive)
494 {
495 int ret = 0;
496 char node[10] = { 0 };
497
498 /* Create xgmi device id file */
499 ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);
500 if (ret) {
501 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
502 return ret;
503 }
504
505 ret = device_create_file(adev->dev, &dev_attr_xgmi_physical_id);
506 if (ret) {
507 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_physical_id\n");
508 return ret;
509 }
510
511 /* Create xgmi error file */
512 ret = device_create_file(adev->dev, &dev_attr_xgmi_error);
513 if (ret)
514 pr_err("failed to create xgmi_error\n");
515
516 /* Create xgmi num hops file */
517 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_hops);
518 if (ret)
519 pr_err("failed to create xgmi_num_hops\n");
520
521 /* Create xgmi num links file */
522 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_links);
523 if (ret)
524 pr_err("failed to create xgmi_num_links\n");
525
526 /* Create xgmi port num file if supported */
527 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
528 ret = device_create_file(adev->dev, &dev_attr_xgmi_port_num);
529 if (ret)
530 dev_err(adev->dev, "failed to create xgmi_port_num\n");
531 }
532
533 /* Create sysfs link to hive info folder on the first device */
534 if (hive->kobj.parent != (&adev->dev->kobj)) {
535 ret = sysfs_create_link(&adev->dev->kobj, &hive->kobj,
536 "xgmi_hive_info");
537 if (ret) {
538 dev_err(adev->dev, "XGMI: Failed to create link to hive info");
539 goto remove_file;
540 }
541 }
542
543 sprintf(node, "node%d", atomic_read(&hive->number_devices));
544 /* Create sysfs link form the hive folder to yourself */
545 ret = sysfs_create_link(&hive->kobj, &adev->dev->kobj, node);
546 if (ret) {
547 dev_err(adev->dev, "XGMI: Failed to create link from hive info");
548 goto remove_link;
549 }
550
551 goto success;
552
553
554 remove_link:
555 sysfs_remove_link(&adev->dev->kobj, adev_to_drm(adev)->unique);
556
557 remove_file:
558 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
559 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
560 device_remove_file(adev->dev, &dev_attr_xgmi_error);
561 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
562 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
563 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
564 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
565
566 success:
567 return ret;
568 }
569
amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)570 static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
571 struct amdgpu_hive_info *hive)
572 {
573 char node[10];
574 memset(node, 0, sizeof(node));
575
576 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
577 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);
578 device_remove_file(adev->dev, &dev_attr_xgmi_error);
579 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);
580 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);
581 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)
582 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);
583
584 if (hive->kobj.parent != (&adev->dev->kobj))
585 sysfs_remove_link(&adev->dev->kobj,"xgmi_hive_info");
586
587 sprintf(node, "node%d", atomic_read(&hive->number_devices));
588 sysfs_remove_link(&hive->kobj, node);
589
590 }
591
592
593
amdgpu_get_xgmi_hive(struct amdgpu_device * adev)594 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
595 {
596 struct amdgpu_hive_info *hive = NULL;
597 int ret;
598
599 if (!adev->gmc.xgmi.hive_id)
600 return NULL;
601
602 if (adev->hive) {
603 kobject_get(&adev->hive->kobj);
604 return adev->hive;
605 }
606
607 mutex_lock(&xgmi_mutex);
608
609 list_for_each_entry(hive, &xgmi_hive_list, node) {
610 if (hive->hive_id == adev->gmc.xgmi.hive_id)
611 goto pro_end;
612 }
613
614 hive = kzalloc(sizeof(*hive), GFP_KERNEL);
615 if (!hive) {
616 dev_err(adev->dev, "XGMI: allocation failed\n");
617 ret = -ENOMEM;
618 hive = NULL;
619 goto pro_end;
620 }
621
622 /* initialize new hive if not exist */
623 ret = kobject_init_and_add(&hive->kobj,
624 &amdgpu_xgmi_hive_type,
625 &adev->dev->kobj,
626 "%s", "xgmi_hive_info");
627 if (ret) {
628 dev_err(adev->dev, "XGMI: failed initializing kobject for xgmi hive\n");
629 kobject_put(&hive->kobj);
630 hive = NULL;
631 goto pro_end;
632 }
633
634 /**
635 * Only init hive->reset_domain for none SRIOV configuration. For SRIOV,
636 * Host driver decide how to reset the GPU either through FLR or chain reset.
637 * Guest side will get individual notifications from the host for the FLR
638 * if necessary.
639 */
640 if (!amdgpu_sriov_vf(adev)) {
641 /**
642 * Avoid recreating reset domain when hive is reconstructed for the case
643 * of reset the devices in the XGMI hive during probe for passthrough GPU
644 * See https://www.spinics.net/lists/amd-gfx/msg58836.html
645 */
646 if (adev->reset_domain->type != XGMI_HIVE) {
647 hive->reset_domain =
648 amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive");
649 if (!hive->reset_domain) {
650 dev_err(adev->dev, "XGMI: failed initializing reset domain for xgmi hive\n");
651 ret = -ENOMEM;
652 kobject_put(&hive->kobj);
653 hive = NULL;
654 goto pro_end;
655 }
656 } else {
657 amdgpu_reset_get_reset_domain(adev->reset_domain);
658 hive->reset_domain = adev->reset_domain;
659 }
660 }
661
662 hive->hive_id = adev->gmc.xgmi.hive_id;
663 INIT_LIST_HEAD(&hive->device_list);
664 INIT_LIST_HEAD(&hive->node);
665 mutex_init(&hive->hive_lock);
666 atomic_set(&hive->number_devices, 0);
667 task_barrier_init(&hive->tb);
668 hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN;
669 hive->hi_req_gpu = NULL;
670
671 /*
672 * hive pstate on boot is high in vega20 so we have to go to low
673 * pstate on after boot.
674 */
675 hive->hi_req_count = AMDGPU_MAX_XGMI_DEVICE_PER_HIVE;
676 list_add_tail(&hive->node, &xgmi_hive_list);
677
678 pro_end:
679 if (hive)
680 kobject_get(&hive->kobj);
681 mutex_unlock(&xgmi_mutex);
682 return hive;
683 }
684
amdgpu_put_xgmi_hive(struct amdgpu_hive_info * hive)685 void amdgpu_put_xgmi_hive(struct amdgpu_hive_info *hive)
686 {
687 if (hive)
688 kobject_put(&hive->kobj);
689 }
690
amdgpu_xgmi_set_pstate(struct amdgpu_device * adev,int pstate)691 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
692 {
693 int ret = 0;
694 struct amdgpu_hive_info *hive;
695 struct amdgpu_device *request_adev;
696 bool is_hi_req = pstate == AMDGPU_XGMI_PSTATE_MAX_VEGA20;
697 bool init_low;
698
699 hive = amdgpu_get_xgmi_hive(adev);
700 if (!hive)
701 return 0;
702
703 request_adev = hive->hi_req_gpu ? hive->hi_req_gpu : adev;
704 init_low = hive->pstate == AMDGPU_XGMI_PSTATE_UNKNOWN;
705 amdgpu_put_xgmi_hive(hive);
706 /* fw bug so temporarily disable pstate switching */
707 return 0;
708
709 if (!hive || adev->asic_type != CHIP_VEGA20)
710 return 0;
711
712 mutex_lock(&hive->hive_lock);
713
714 if (is_hi_req)
715 hive->hi_req_count++;
716 else
717 hive->hi_req_count--;
718
719 /*
720 * Vega20 only needs single peer to request pstate high for the hive to
721 * go high but all peers must request pstate low for the hive to go low
722 */
723 if (hive->pstate == pstate ||
724 (!is_hi_req && hive->hi_req_count && !init_low))
725 goto out;
726
727 dev_dbg(request_adev->dev, "Set xgmi pstate %d.\n", pstate);
728
729 ret = amdgpu_dpm_set_xgmi_pstate(request_adev, pstate);
730 if (ret) {
731 dev_err(request_adev->dev,
732 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
733 request_adev->gmc.xgmi.node_id,
734 request_adev->gmc.xgmi.hive_id, ret);
735 goto out;
736 }
737
738 if (init_low)
739 hive->pstate = hive->hi_req_count ?
740 hive->pstate : AMDGPU_XGMI_PSTATE_MIN;
741 else {
742 hive->pstate = pstate;
743 hive->hi_req_gpu = pstate != AMDGPU_XGMI_PSTATE_MIN ?
744 adev : NULL;
745 }
746 out:
747 mutex_unlock(&hive->hive_lock);
748 return ret;
749 }
750
amdgpu_xgmi_update_topology(struct amdgpu_hive_info * hive,struct amdgpu_device * adev)751 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
752 {
753 int ret;
754
755 if (amdgpu_sriov_vf(adev))
756 return 0;
757
758 /* Each psp need to set the latest topology */
759 ret = psp_xgmi_set_topology_info(&adev->psp,
760 atomic_read(&hive->number_devices),
761 &adev->psp.xgmi_context.top_info);
762 if (ret)
763 dev_err(adev->dev,
764 "XGMI: Set topology failure on device %llx, hive %llx, ret %d",
765 adev->gmc.xgmi.node_id,
766 adev->gmc.xgmi.hive_id, ret);
767
768 return ret;
769 }
770
771
772 /*
773 * NOTE psp_xgmi_node_info.num_hops layout is as follows:
774 * num_hops[7:6] = link type (0 = xGMI2, 1 = xGMI3, 2/3 = reserved)
775 * num_hops[5:3] = reserved
776 * num_hops[2:0] = number of hops
777 */
amdgpu_xgmi_get_hops_count(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)778 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
779 struct amdgpu_device *peer_adev)
780 {
781 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
782 uint8_t num_hops_mask = 0x7;
783 int i;
784
785 for (i = 0 ; i < top->num_nodes; ++i)
786 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
787 return top->nodes[i].num_hops & num_hops_mask;
788 return -EINVAL;
789 }
790
amdgpu_xgmi_get_num_links(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)791 int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
792 struct amdgpu_device *peer_adev)
793 {
794 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
795 int i;
796
797 for (i = 0 ; i < top->num_nodes; ++i)
798 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
799 return top->nodes[i].num_links;
800 return -EINVAL;
801 }
802
803 /*
804 * Devices that support extended data require the entire hive to initialize with
805 * the shared memory buffer flag set.
806 *
807 * Hive locks and conditions apply - see amdgpu_xgmi_add_device
808 */
amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info * hive,bool set_extended_data)809 static int amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info *hive,
810 bool set_extended_data)
811 {
812 struct amdgpu_device *tmp_adev;
813 int ret;
814
815 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
816 ret = psp_xgmi_initialize(&tmp_adev->psp, set_extended_data, false);
817 if (ret) {
818 dev_err(tmp_adev->dev,
819 "XGMI: Failed to initialize xgmi session for data partition %i\n",
820 set_extended_data);
821 return ret;
822 }
823
824 }
825
826 return 0;
827 }
828
amdgpu_xgmi_fill_topology_info(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)829 static void amdgpu_xgmi_fill_topology_info(struct amdgpu_device *adev,
830 struct amdgpu_device *peer_adev)
831 {
832 struct psp_xgmi_topology_info *top_info = &adev->psp.xgmi_context.top_info;
833 struct psp_xgmi_topology_info *peer_info = &peer_adev->psp.xgmi_context.top_info;
834
835 for (int i = 0; i < peer_info->num_nodes; i++) {
836 if (peer_info->nodes[i].node_id == adev->gmc.xgmi.node_id) {
837 for (int j = 0; j < top_info->num_nodes; j++) {
838 if (top_info->nodes[j].node_id == peer_adev->gmc.xgmi.node_id) {
839 peer_info->nodes[i].num_hops = top_info->nodes[j].num_hops;
840 peer_info->nodes[i].is_sharing_enabled =
841 top_info->nodes[j].is_sharing_enabled;
842 peer_info->nodes[i].num_links =
843 top_info->nodes[j].num_links;
844 return;
845 }
846 }
847 }
848 }
849 }
850
amdgpu_xgmi_add_device(struct amdgpu_device * adev)851 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
852 {
853 struct psp_xgmi_topology_info *top_info;
854 struct amdgpu_hive_info *hive;
855 struct amdgpu_xgmi *entry;
856 struct amdgpu_device *tmp_adev = NULL;
857
858 int count = 0, ret = 0;
859
860 if (!adev->gmc.xgmi.supported)
861 return 0;
862
863 if (!adev->gmc.xgmi.pending_reset &&
864 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
865 ret = psp_xgmi_initialize(&adev->psp, false, true);
866 if (ret) {
867 dev_err(adev->dev,
868 "XGMI: Failed to initialize xgmi session\n");
869 return ret;
870 }
871
872 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id);
873 if (ret) {
874 dev_err(adev->dev,
875 "XGMI: Failed to get hive id\n");
876 return ret;
877 }
878
879 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id);
880 if (ret) {
881 dev_err(adev->dev,
882 "XGMI: Failed to get node id\n");
883 return ret;
884 }
885 } else {
886 adev->gmc.xgmi.hive_id = 16;
887 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;
888 }
889
890 hive = amdgpu_get_xgmi_hive(adev);
891 if (!hive) {
892 ret = -EINVAL;
893 dev_err(adev->dev,
894 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",
895 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);
896 goto exit;
897 }
898 mutex_lock(&hive->hive_lock);
899
900 top_info = &adev->psp.xgmi_context.top_info;
901
902 list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);
903 list_for_each_entry(entry, &hive->device_list, head)
904 top_info->nodes[count++].node_id = entry->node_id;
905 top_info->num_nodes = count;
906 atomic_set(&hive->number_devices, count);
907
908 task_barrier_add_task(&hive->tb);
909
910 if (!adev->gmc.xgmi.pending_reset &&
911 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
912 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
913 /* update node list for other device in the hive */
914 if (tmp_adev != adev) {
915 top_info = &tmp_adev->psp.xgmi_context.top_info;
916 top_info->nodes[count - 1].node_id =
917 adev->gmc.xgmi.node_id;
918 top_info->num_nodes = count;
919 }
920 ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
921 if (ret)
922 goto exit_unlock;
923 }
924
925 if (amdgpu_sriov_vf(adev) &&
926 adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {
927 /* only get topology for VF being init if it can support full duplex */
928 ret = psp_xgmi_get_topology_info(&adev->psp, count,
929 &adev->psp.xgmi_context.top_info, false);
930 if (ret) {
931 dev_err(adev->dev,
932 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
933 adev->gmc.xgmi.node_id,
934 adev->gmc.xgmi.hive_id, ret);
935 /* To do: continue with some node failed or disable the whole hive*/
936 goto exit_unlock;
937 }
938
939 /* fill the topology info for peers instead of getting from PSP */
940 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
941 amdgpu_xgmi_fill_topology_info(adev, tmp_adev);
942 }
943 } else {
944 /* get latest topology info for each device from psp */
945 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
946 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
947 &tmp_adev->psp.xgmi_context.top_info, false);
948 if (ret) {
949 dev_err(tmp_adev->dev,
950 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
951 tmp_adev->gmc.xgmi.node_id,
952 tmp_adev->gmc.xgmi.hive_id, ret);
953 /* To do : continue with some node failed or disable the whole hive */
954 goto exit_unlock;
955 }
956 }
957 }
958
959 /* get topology again for hives that support extended data */
960 if (adev->psp.xgmi_context.supports_extended_data) {
961
962 /* initialize the hive to get extended data. */
963 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, true);
964 if (ret)
965 goto exit_unlock;
966
967 /* get the extended data. */
968 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
969 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
970 &tmp_adev->psp.xgmi_context.top_info, true);
971 if (ret) {
972 dev_err(tmp_adev->dev,
973 "XGMI: Get topology for extended data failure on device %llx, hive %llx, ret %d",
974 tmp_adev->gmc.xgmi.node_id,
975 tmp_adev->gmc.xgmi.hive_id, ret);
976 goto exit_unlock;
977 }
978 }
979
980 /* initialize the hive to get non-extended data for the next round. */
981 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, false);
982 if (ret)
983 goto exit_unlock;
984
985 }
986 }
987
988 if (!ret && !adev->gmc.xgmi.pending_reset)
989 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
990
991 exit_unlock:
992 mutex_unlock(&hive->hive_lock);
993 exit:
994 if (!ret) {
995 adev->hive = hive;
996 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
997 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
998 } else {
999 amdgpu_put_xgmi_hive(hive);
1000 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
1001 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
1002 ret);
1003 }
1004
1005 return ret;
1006 }
1007
amdgpu_xgmi_remove_device(struct amdgpu_device * adev)1008 int amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
1009 {
1010 struct amdgpu_hive_info *hive = adev->hive;
1011
1012 if (!adev->gmc.xgmi.supported)
1013 return -EINVAL;
1014
1015 if (!hive)
1016 return -EINVAL;
1017
1018 mutex_lock(&hive->hive_lock);
1019 task_barrier_rem_task(&hive->tb);
1020 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
1021 if (hive->hi_req_gpu == adev)
1022 hive->hi_req_gpu = NULL;
1023 list_del(&adev->gmc.xgmi.head);
1024 mutex_unlock(&hive->hive_lock);
1025
1026 amdgpu_put_xgmi_hive(hive);
1027 adev->hive = NULL;
1028
1029 if (atomic_dec_return(&hive->number_devices) == 0) {
1030 /* Remove the hive from global hive list */
1031 mutex_lock(&xgmi_mutex);
1032 list_del(&hive->node);
1033 mutex_unlock(&xgmi_mutex);
1034
1035 amdgpu_put_xgmi_hive(hive);
1036 }
1037
1038 return 0;
1039 }
1040
xgmi_v6_4_0_aca_bank_parser(struct aca_handle * handle,struct aca_bank * bank,enum aca_smu_type type,void * data)1041 static int xgmi_v6_4_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,
1042 enum aca_smu_type type, void *data)
1043 {
1044 struct amdgpu_device *adev = handle->adev;
1045 struct aca_bank_info info;
1046 const char *error_str;
1047 u64 status, count;
1048 int ret, ext_error_code;
1049
1050 ret = aca_bank_info_decode(bank, &info);
1051 if (ret)
1052 return ret;
1053
1054 status = bank->regs[ACA_REG_IDX_STATUS];
1055 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1056
1057 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1058 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1059 if (error_str)
1060 dev_info(adev->dev, "%s detected\n", error_str);
1061
1062 count = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]);
1063
1064 switch (type) {
1065 case ACA_SMU_TYPE_UE:
1066 if (ext_error_code != 0 && ext_error_code != 9)
1067 count = 0ULL;
1068
1069 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, count);
1070 break;
1071 case ACA_SMU_TYPE_CE:
1072 count = ext_error_code == 6 ? count : 0ULL;
1073 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_CE, count);
1074 break;
1075 default:
1076 return -EINVAL;
1077 }
1078
1079 return ret;
1080 }
1081
1082 static const struct aca_bank_ops xgmi_v6_4_0_aca_bank_ops = {
1083 .aca_bank_parser = xgmi_v6_4_0_aca_bank_parser,
1084 };
1085
1086 static const struct aca_info xgmi_v6_4_0_aca_info = {
1087 .hwip = ACA_HWIP_TYPE_PCS_XGMI,
1088 .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,
1089 .bank_ops = &xgmi_v6_4_0_aca_bank_ops,
1090 };
1091
amdgpu_xgmi_ras_late_init(struct amdgpu_device * adev,struct ras_common_if * ras_block)1092 static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)
1093 {
1094 int r;
1095
1096 if (!adev->gmc.xgmi.supported ||
1097 adev->gmc.xgmi.num_physical_nodes == 0)
1098 return 0;
1099
1100 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1101
1102 r = amdgpu_ras_block_late_init(adev, ras_block);
1103 if (r)
1104 return r;
1105
1106 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1107 case IP_VERSION(6, 4, 0):
1108 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL,
1109 &xgmi_v6_4_0_aca_info, NULL);
1110 if (r)
1111 goto late_fini;
1112 break;
1113 default:
1114 break;
1115 }
1116
1117 return 0;
1118
1119 late_fini:
1120 amdgpu_ras_block_late_fini(adev, ras_block);
1121
1122 return r;
1123 }
1124
amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device * adev,uint64_t addr)1125 uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
1126 uint64_t addr)
1127 {
1128 struct amdgpu_xgmi *xgmi = &adev->gmc.xgmi;
1129 return (addr + xgmi->physical_node_id * xgmi->node_segment_size);
1130 }
1131
pcs_clear_status(struct amdgpu_device * adev,uint32_t pcs_status_reg)1132 static void pcs_clear_status(struct amdgpu_device *adev, uint32_t pcs_status_reg)
1133 {
1134 WREG32_PCIE(pcs_status_reg, 0xFFFFFFFF);
1135 WREG32_PCIE(pcs_status_reg, 0);
1136 }
1137
amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device * adev)1138 static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev)
1139 {
1140 uint32_t i;
1141
1142 switch (adev->asic_type) {
1143 case CHIP_ARCTURUS:
1144 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++)
1145 pcs_clear_status(adev,
1146 xgmi_pcs_err_status_reg_arct[i]);
1147 break;
1148 case CHIP_VEGA20:
1149 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++)
1150 pcs_clear_status(adev,
1151 xgmi_pcs_err_status_reg_vg20[i]);
1152 break;
1153 case CHIP_ALDEBARAN:
1154 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++)
1155 pcs_clear_status(adev,
1156 xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1157 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++)
1158 pcs_clear_status(adev,
1159 walf_pcs_err_status_reg_aldebaran[i]);
1160 break;
1161 default:
1162 break;
1163 }
1164
1165 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1166 case IP_VERSION(6, 4, 0):
1167 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++)
1168 pcs_clear_status(adev,
1169 xgmi3x16_pcs_err_status_reg_v6_4[i]);
1170 break;
1171 default:
1172 break;
1173 }
1174 }
1175
__xgmi_v6_4_0_reset_error_count(struct amdgpu_device * adev,int xgmi_inst,u64 mca_base)1176 static void __xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst, u64 mca_base)
1177 {
1178 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1179 }
1180
xgmi_v6_4_0_reset_error_count(struct amdgpu_device * adev,int xgmi_inst)1181 static void xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst)
1182 {
1183 int i;
1184
1185 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1186 __xgmi_v6_4_0_reset_error_count(adev, xgmi_inst, xgmi_v6_4_0_mca_base_array[i]);
1187 }
1188
xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device * adev)1189 static void xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device *adev)
1190 {
1191 int i;
1192
1193 for_each_inst(i, adev->aid_mask)
1194 xgmi_v6_4_0_reset_error_count(adev, i);
1195 }
1196
amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device * adev)1197 static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev)
1198 {
1199 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1200 case IP_VERSION(6, 4, 0):
1201 xgmi_v6_4_0_reset_ras_error_count(adev);
1202 break;
1203 default:
1204 amdgpu_xgmi_legacy_reset_ras_error_count(adev);
1205 break;
1206 }
1207 }
1208
amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device * adev,uint32_t value,uint32_t mask_value,uint32_t * ue_count,uint32_t * ce_count,bool is_xgmi_pcs,bool check_mask)1209 static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,
1210 uint32_t value,
1211 uint32_t mask_value,
1212 uint32_t *ue_count,
1213 uint32_t *ce_count,
1214 bool is_xgmi_pcs,
1215 bool check_mask)
1216 {
1217 int i;
1218 int ue_cnt = 0;
1219 const struct amdgpu_pcs_ras_field *pcs_ras_fields = NULL;
1220 uint32_t field_array_size = 0;
1221
1222 if (is_xgmi_pcs) {
1223 if (amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1224 IP_VERSION(6, 1, 0) ||
1225 amdgpu_ip_version(adev, XGMI_HWIP, 0) ==
1226 IP_VERSION(6, 4, 0)) {
1227 pcs_ras_fields = &xgmi3x16_pcs_ras_fields[0];
1228 field_array_size = ARRAY_SIZE(xgmi3x16_pcs_ras_fields);
1229 } else {
1230 pcs_ras_fields = &xgmi_pcs_ras_fields[0];
1231 field_array_size = ARRAY_SIZE(xgmi_pcs_ras_fields);
1232 }
1233 } else {
1234 pcs_ras_fields = &wafl_pcs_ras_fields[0];
1235 field_array_size = ARRAY_SIZE(wafl_pcs_ras_fields);
1236 }
1237
1238 if (check_mask)
1239 value = value & ~mask_value;
1240
1241 /* query xgmi/walf pcs error status,
1242 * only ue is supported */
1243 for (i = 0; value && i < field_array_size; i++) {
1244 ue_cnt = (value &
1245 pcs_ras_fields[i].pcs_err_mask) >>
1246 pcs_ras_fields[i].pcs_err_shift;
1247 if (ue_cnt) {
1248 dev_info(adev->dev, "%s detected\n",
1249 pcs_ras_fields[i].err_name);
1250 *ue_count += ue_cnt;
1251 }
1252
1253 /* reset bit value if the bit is checked */
1254 value &= ~(pcs_ras_fields[i].pcs_err_mask);
1255 }
1256
1257 return 0;
1258 }
1259
amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1260 static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev,
1261 void *ras_error_status)
1262 {
1263 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1264 int i, supported = 1;
1265 uint32_t data, mask_data = 0;
1266 uint32_t ue_cnt = 0, ce_cnt = 0;
1267
1268 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL))
1269 return ;
1270
1271 err_data->ue_count = 0;
1272 err_data->ce_count = 0;
1273
1274 switch (adev->asic_type) {
1275 case CHIP_ARCTURUS:
1276 /* check xgmi pcs error */
1277 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++) {
1278 data = RREG32_PCIE(xgmi_pcs_err_status_reg_arct[i]);
1279 if (data)
1280 amdgpu_xgmi_query_pcs_error_status(adev, data,
1281 mask_data, &ue_cnt, &ce_cnt, true, false);
1282 }
1283 /* check wafl pcs error */
1284 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_arct); i++) {
1285 data = RREG32_PCIE(wafl_pcs_err_status_reg_arct[i]);
1286 if (data)
1287 amdgpu_xgmi_query_pcs_error_status(adev, data,
1288 mask_data, &ue_cnt, &ce_cnt, false, false);
1289 }
1290 break;
1291 case CHIP_VEGA20:
1292 /* check xgmi pcs error */
1293 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++) {
1294 data = RREG32_PCIE(xgmi_pcs_err_status_reg_vg20[i]);
1295 if (data)
1296 amdgpu_xgmi_query_pcs_error_status(adev, data,
1297 mask_data, &ue_cnt, &ce_cnt, true, false);
1298 }
1299 /* check wafl pcs error */
1300 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_vg20); i++) {
1301 data = RREG32_PCIE(wafl_pcs_err_status_reg_vg20[i]);
1302 if (data)
1303 amdgpu_xgmi_query_pcs_error_status(adev, data,
1304 mask_data, &ue_cnt, &ce_cnt, false, false);
1305 }
1306 break;
1307 case CHIP_ALDEBARAN:
1308 /* check xgmi3x16 pcs error */
1309 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++) {
1310 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_aldebaran[i]);
1311 mask_data =
1312 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1313 if (data)
1314 amdgpu_xgmi_query_pcs_error_status(adev, data,
1315 mask_data, &ue_cnt, &ce_cnt, true, true);
1316 }
1317 /* check wafl pcs error */
1318 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++) {
1319 data = RREG32_PCIE(walf_pcs_err_status_reg_aldebaran[i]);
1320 mask_data =
1321 RREG32_PCIE(walf_pcs_err_noncorrectable_mask_reg_aldebaran[i]);
1322 if (data)
1323 amdgpu_xgmi_query_pcs_error_status(adev, data,
1324 mask_data, &ue_cnt, &ce_cnt, false, true);
1325 }
1326 break;
1327 default:
1328 supported = 0;
1329 break;
1330 }
1331
1332 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1333 case IP_VERSION(6, 4, 0):
1334 /* check xgmi3x16 pcs error */
1335 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++) {
1336 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_v6_4[i]);
1337 mask_data =
1338 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[i]);
1339 if (data)
1340 amdgpu_xgmi_query_pcs_error_status(adev, data,
1341 mask_data, &ue_cnt, &ce_cnt, true, true);
1342 }
1343 break;
1344 default:
1345 if (!supported)
1346 dev_warn(adev->dev, "XGMI RAS error query not supported");
1347 break;
1348 }
1349
1350 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);
1351
1352 err_data->ue_count += ue_cnt;
1353 err_data->ce_count += ce_cnt;
1354 }
1355
xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device * adev,u64 status)1356 static enum aca_error_type xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device *adev, u64 status)
1357 {
1358 const char *error_str;
1359 int ext_error_code;
1360
1361 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);
1362
1363 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?
1364 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;
1365 if (error_str)
1366 dev_info(adev->dev, "%s detected\n", error_str);
1367
1368 switch (ext_error_code) {
1369 case 0:
1370 return ACA_ERROR_TYPE_UE;
1371 case 6:
1372 return ACA_ERROR_TYPE_CE;
1373 default:
1374 return -EINVAL;
1375 }
1376
1377 return -EINVAL;
1378 }
1379
__xgmi_v6_4_0_query_error_count(struct amdgpu_device * adev,struct amdgpu_smuio_mcm_config_info * mcm_info,u64 mca_base,struct ras_err_data * err_data)1380 static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct amdgpu_smuio_mcm_config_info *mcm_info,
1381 u64 mca_base, struct ras_err_data *err_data)
1382 {
1383 int xgmi_inst = mcm_info->die_id;
1384 u64 status = 0;
1385
1386 status = RREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS);
1387 if (!ACA_REG__STATUS__VAL(status))
1388 return;
1389
1390 switch (xgmi_v6_4_0_pcs_mca_get_error_type(adev, status)) {
1391 case ACA_ERROR_TYPE_UE:
1392 amdgpu_ras_error_statistic_ue_count(err_data, mcm_info, 1ULL);
1393 break;
1394 case ACA_ERROR_TYPE_CE:
1395 amdgpu_ras_error_statistic_ce_count(err_data, mcm_info, 1ULL);
1396 break;
1397 default:
1398 break;
1399 }
1400
1401 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);
1402 }
1403
xgmi_v6_4_0_query_error_count(struct amdgpu_device * adev,int xgmi_inst,struct ras_err_data * err_data)1404 static void xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, int xgmi_inst, struct ras_err_data *err_data)
1405 {
1406 struct amdgpu_smuio_mcm_config_info mcm_info = {
1407 .socket_id = adev->smuio.funcs->get_socket_id(adev),
1408 .die_id = xgmi_inst,
1409 };
1410 int i;
1411
1412 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)
1413 __xgmi_v6_4_0_query_error_count(adev, &mcm_info, xgmi_v6_4_0_mca_base_array[i], err_data);
1414 }
1415
xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1416 static void xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status)
1417 {
1418 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
1419 int i;
1420
1421 for_each_inst(i, adev->aid_mask)
1422 xgmi_v6_4_0_query_error_count(adev, i, err_data);
1423 }
1424
amdgpu_xgmi_query_ras_error_count(struct amdgpu_device * adev,void * ras_error_status)1425 static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev,
1426 void *ras_error_status)
1427 {
1428 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {
1429 case IP_VERSION(6, 4, 0):
1430 xgmi_v6_4_0_query_ras_error_count(adev, ras_error_status);
1431 break;
1432 default:
1433 amdgpu_xgmi_legacy_query_ras_error_count(adev, ras_error_status);
1434 break;
1435 }
1436 }
1437
1438 /* Trigger XGMI/WAFL error */
amdgpu_ras_error_inject_xgmi(struct amdgpu_device * adev,void * inject_if,uint32_t instance_mask)1439 static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev,
1440 void *inject_if, uint32_t instance_mask)
1441 {
1442 int ret1, ret2;
1443 struct ta_ras_trigger_error_input *block_info =
1444 (struct ta_ras_trigger_error_input *)inject_if;
1445
1446 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
1447 dev_warn(adev->dev, "Failed to disallow df cstate");
1448
1449 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DISALLOW);
1450 if (ret1 && ret1 != -EOPNOTSUPP)
1451 dev_warn(adev->dev, "Failed to disallow XGMI power down");
1452
1453 ret2 = psp_ras_trigger_error(&adev->psp, block_info, instance_mask);
1454
1455 if (amdgpu_ras_intr_triggered())
1456 return ret2;
1457
1458 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DEFAULT);
1459 if (ret1 && ret1 != -EOPNOTSUPP)
1460 dev_warn(adev->dev, "Failed to allow XGMI power down");
1461
1462 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW))
1463 dev_warn(adev->dev, "Failed to allow df cstate");
1464
1465 return ret2;
1466 }
1467
1468 struct amdgpu_ras_block_hw_ops xgmi_ras_hw_ops = {
1469 .query_ras_error_count = amdgpu_xgmi_query_ras_error_count,
1470 .reset_ras_error_count = amdgpu_xgmi_reset_ras_error_count,
1471 .ras_error_inject = amdgpu_ras_error_inject_xgmi,
1472 };
1473
1474 struct amdgpu_xgmi_ras xgmi_ras = {
1475 .ras_block = {
1476 .hw_ops = &xgmi_ras_hw_ops,
1477 .ras_late_init = amdgpu_xgmi_ras_late_init,
1478 },
1479 };
1480
amdgpu_xgmi_ras_sw_init(struct amdgpu_device * adev)1481 int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev)
1482 {
1483 int err;
1484 struct amdgpu_xgmi_ras *ras;
1485
1486 if (!adev->gmc.xgmi.ras)
1487 return 0;
1488
1489 ras = adev->gmc.xgmi.ras;
1490 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);
1491 if (err) {
1492 dev_err(adev->dev, "Failed to register xgmi_wafl_pcs ras block!\n");
1493 return err;
1494 }
1495
1496 strcpy(ras->ras_block.ras_comm.name, "xgmi_wafl");
1497 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__XGMI_WAFL;
1498 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
1499 adev->gmc.xgmi.ras_if = &ras->ras_block.ras_comm;
1500
1501 return 0;
1502 }
1503