1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include "amdgpu.h" 26 #include "amdgpu_sdma.h" 27 #include "amdgpu_ras.h" 28 29 #define AMDGPU_CSA_SDMA_SIZE 64 30 /* SDMA CSA reside in the 3rd page of CSA */ 31 #define AMDGPU_CSA_SDMA_OFFSET (4096 * 2) 32 33 /* 34 * GPU SDMA IP block helpers function. 35 */ 36 37 struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring) 38 { 39 struct amdgpu_device *adev = ring->adev; 40 int i; 41 42 for (i = 0; i < adev->sdma.num_instances; i++) 43 if (ring == &adev->sdma.instance[i].ring || 44 ring == &adev->sdma.instance[i].page) 45 return &adev->sdma.instance[i]; 46 47 return NULL; 48 } 49 50 int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index) 51 { 52 struct amdgpu_device *adev = ring->adev; 53 int i; 54 55 for (i = 0; i < adev->sdma.num_instances; i++) { 56 if (ring == &adev->sdma.instance[i].ring || 57 ring == &adev->sdma.instance[i].page) { 58 *index = i; 59 return 0; 60 } 61 } 62 63 return -EINVAL; 64 } 65 66 uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring, 67 unsigned int vmid) 68 { 69 struct amdgpu_device *adev = ring->adev; 70 uint64_t csa_mc_addr; 71 uint32_t index = 0; 72 int r; 73 74 /* don't enable OS preemption on SDMA under SRIOV */ 75 if (amdgpu_sriov_vf(adev) || vmid == 0 || !adev->gfx.mcbp) 76 return 0; 77 78 if (ring->is_mes_queue) { 79 uint32_t offset = 0; 80 81 offset = offsetof(struct amdgpu_mes_ctx_meta_data, 82 sdma[ring->idx].sdma_meta_data); 83 csa_mc_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset); 84 } else { 85 r = amdgpu_sdma_get_index_from_ring(ring, &index); 86 87 if (r || index > 31) 88 csa_mc_addr = 0; 89 else 90 csa_mc_addr = amdgpu_csa_vaddr(adev) + 91 AMDGPU_CSA_SDMA_OFFSET + 92 index * AMDGPU_CSA_SDMA_SIZE; 93 } 94 95 return csa_mc_addr; 96 } 97 98 int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev, 99 struct ras_common_if *ras_block) 100 { 101 int r, i; 102 103 r = amdgpu_ras_block_late_init(adev, ras_block); 104 if (r) 105 return r; 106 107 if (amdgpu_ras_is_supported(adev, ras_block->block)) { 108 for (i = 0; i < adev->sdma.num_instances; i++) { 109 r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq, 110 AMDGPU_SDMA_IRQ_INSTANCE0 + i); 111 if (r) 112 goto late_fini; 113 } 114 } 115 116 return 0; 117 118 late_fini: 119 amdgpu_ras_block_late_fini(adev, ras_block); 120 return r; 121 } 122 123 int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev, 124 void *err_data, 125 struct amdgpu_iv_entry *entry) 126 { 127 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 128 129 if (amdgpu_sriov_vf(adev)) 130 return AMDGPU_RAS_SUCCESS; 131 132 amdgpu_ras_reset_gpu(adev); 133 134 return AMDGPU_RAS_SUCCESS; 135 } 136 137 int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev, 138 struct amdgpu_irq_src *source, 139 struct amdgpu_iv_entry *entry) 140 { 141 struct ras_common_if *ras_if = adev->sdma.ras_if; 142 struct ras_dispatch_if ih_data = { 143 .entry = entry, 144 }; 145 146 if (!ras_if) 147 return 0; 148 149 ih_data.head = *ras_if; 150 151 amdgpu_ras_interrupt_dispatch(adev, &ih_data); 152 return 0; 153 } 154 155 static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst) 156 { 157 uint16_t version_major; 158 const struct common_firmware_header *header = NULL; 159 const struct sdma_firmware_header_v1_0 *hdr; 160 const struct sdma_firmware_header_v2_0 *hdr_v2; 161 const struct sdma_firmware_header_v3_0 *hdr_v3; 162 163 header = (const struct common_firmware_header *) 164 sdma_inst->fw->data; 165 version_major = le16_to_cpu(header->header_version_major); 166 167 switch (version_major) { 168 case 1: 169 hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data; 170 sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version); 171 sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version); 172 break; 173 case 2: 174 hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data; 175 sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version); 176 sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version); 177 break; 178 case 3: 179 hdr_v3 = (const struct sdma_firmware_header_v3_0 *)sdma_inst->fw->data; 180 sdma_inst->fw_version = le32_to_cpu(hdr_v3->header.ucode_version); 181 sdma_inst->feature_version = le32_to_cpu(hdr_v3->ucode_feature_version); 182 break; 183 default: 184 return -EINVAL; 185 } 186 187 if (sdma_inst->feature_version >= 20) 188 sdma_inst->burst_nop = true; 189 190 return 0; 191 } 192 193 void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev, 194 bool duplicate) 195 { 196 int i; 197 198 for (i = 0; i < adev->sdma.num_instances; i++) { 199 amdgpu_ucode_release(&adev->sdma.instance[i].fw); 200 if (duplicate) 201 break; 202 } 203 204 memset((void *)adev->sdma.instance, 0, 205 sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES); 206 } 207 208 int amdgpu_sdma_init_microcode(struct amdgpu_device *adev, 209 u32 instance, bool duplicate) 210 { 211 struct amdgpu_firmware_info *info = NULL; 212 const struct common_firmware_header *header = NULL; 213 int err, i; 214 const struct sdma_firmware_header_v2_0 *sdma_hdr; 215 const struct sdma_firmware_header_v3_0 *sdma_hv3; 216 uint16_t version_major; 217 char ucode_prefix[30]; 218 219 amdgpu_ucode_ip_version_decode(adev, SDMA0_HWIP, ucode_prefix, sizeof(ucode_prefix)); 220 if (instance == 0) 221 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw, 222 AMDGPU_UCODE_REQUIRED, 223 "amdgpu/%s.bin", ucode_prefix); 224 else 225 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw, 226 AMDGPU_UCODE_REQUIRED, 227 "amdgpu/%s%d.bin", ucode_prefix, instance); 228 if (err) 229 goto out; 230 231 header = (const struct common_firmware_header *) 232 adev->sdma.instance[instance].fw->data; 233 version_major = le16_to_cpu(header->header_version_major); 234 235 if ((duplicate && instance) || (!duplicate && version_major > 1)) { 236 err = -EINVAL; 237 goto out; 238 } 239 240 err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]); 241 if (err) 242 goto out; 243 244 if (duplicate) { 245 for (i = 1; i < adev->sdma.num_instances; i++) 246 memcpy((void *)&adev->sdma.instance[i], 247 (void *)&adev->sdma.instance[0], 248 sizeof(struct amdgpu_sdma_instance)); 249 } 250 251 DRM_DEBUG("psp_load == '%s'\n", 252 adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false"); 253 254 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 255 switch (version_major) { 256 case 1: 257 for (i = 0; i < adev->sdma.num_instances; i++) { 258 if (!duplicate && (instance != i)) 259 continue; 260 else { 261 /* Use a single copy per SDMA firmware type. PSP uses the same instance for all 262 * groups of SDMAs */ 263 if ((amdgpu_ip_version(adev, SDMA0_HWIP, 0) == 264 IP_VERSION(4, 4, 2) || 265 amdgpu_ip_version(adev, SDMA0_HWIP, 0) == 266 IP_VERSION(4, 4, 4) || 267 amdgpu_ip_version(adev, SDMA0_HWIP, 0) == 268 IP_VERSION(4, 4, 5)) && 269 adev->firmware.load_type == 270 AMDGPU_FW_LOAD_PSP && 271 adev->sdma.num_inst_per_aid == i) { 272 break; 273 } 274 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i]; 275 info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i; 276 info->fw = adev->sdma.instance[i].fw; 277 adev->firmware.fw_size += 278 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 279 } 280 } 281 break; 282 case 2: 283 sdma_hdr = (const struct sdma_firmware_header_v2_0 *) 284 adev->sdma.instance[0].fw->data; 285 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0]; 286 info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0; 287 info->fw = adev->sdma.instance[0].fw; 288 adev->firmware.fw_size += 289 ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE); 290 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1]; 291 info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1; 292 info->fw = adev->sdma.instance[0].fw; 293 adev->firmware.fw_size += 294 ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE); 295 break; 296 case 3: 297 sdma_hv3 = (const struct sdma_firmware_header_v3_0 *) 298 adev->sdma.instance[0].fw->data; 299 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_RS64]; 300 info->ucode_id = AMDGPU_UCODE_ID_SDMA_RS64; 301 info->fw = adev->sdma.instance[0].fw; 302 adev->firmware.fw_size += 303 ALIGN(le32_to_cpu(sdma_hv3->ucode_size_bytes), PAGE_SIZE); 304 break; 305 default: 306 err = -EINVAL; 307 } 308 } 309 310 out: 311 if (err) 312 amdgpu_sdma_destroy_inst_ctx(adev, duplicate); 313 return err; 314 } 315 316 int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev) 317 { 318 int err = 0; 319 struct amdgpu_sdma_ras *ras = NULL; 320 321 /* adev->sdma.ras is NULL, which means sdma does not 322 * support ras function, then do nothing here. 323 */ 324 if (!adev->sdma.ras) 325 return 0; 326 327 ras = adev->sdma.ras; 328 329 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block); 330 if (err) { 331 dev_err(adev->dev, "Failed to register sdma ras block!\n"); 332 return err; 333 } 334 335 strcpy(ras->ras_block.ras_comm.name, "sdma"); 336 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__SDMA; 337 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE; 338 adev->sdma.ras_if = &ras->ras_block.ras_comm; 339 340 /* If not define special ras_late_init function, use default ras_late_init */ 341 if (!ras->ras_block.ras_late_init) 342 ras->ras_block.ras_late_init = amdgpu_sdma_ras_late_init; 343 344 /* If not defined special ras_cb function, use default ras_cb */ 345 if (!ras->ras_block.ras_cb) 346 ras->ras_block.ras_cb = amdgpu_sdma_process_ras_data_cb; 347 348 return 0; 349 } 350 351 /* 352 * debugfs for to enable/disable sdma job submission to specific core. 353 */ 354 #if defined(CONFIG_DEBUG_FS) 355 static int amdgpu_debugfs_sdma_sched_mask_set(void *data, u64 val) 356 { 357 struct amdgpu_device *adev = (struct amdgpu_device *)data; 358 u32 i; 359 u64 mask = 0; 360 struct amdgpu_ring *ring; 361 362 if (!adev) 363 return -ENODEV; 364 365 mask = BIT_ULL(adev->sdma.num_instances) - 1; 366 if ((val & mask) == 0) 367 return -EINVAL; 368 369 for (i = 0; i < adev->sdma.num_instances; ++i) { 370 ring = &adev->sdma.instance[i].ring; 371 if (val & BIT_ULL(i)) 372 ring->sched.ready = true; 373 else 374 ring->sched.ready = false; 375 } 376 /* publish sched.ready flag update effective immediately across smp */ 377 smp_rmb(); 378 return 0; 379 } 380 381 static int amdgpu_debugfs_sdma_sched_mask_get(void *data, u64 *val) 382 { 383 struct amdgpu_device *adev = (struct amdgpu_device *)data; 384 u32 i; 385 u64 mask = 0; 386 struct amdgpu_ring *ring; 387 388 if (!adev) 389 return -ENODEV; 390 for (i = 0; i < adev->sdma.num_instances; ++i) { 391 ring = &adev->sdma.instance[i].ring; 392 if (ring->sched.ready) 393 mask |= 1 << i; 394 } 395 396 *val = mask; 397 return 0; 398 } 399 400 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_sdma_sched_mask_fops, 401 amdgpu_debugfs_sdma_sched_mask_get, 402 amdgpu_debugfs_sdma_sched_mask_set, "%llx\n"); 403 404 #endif 405 406 void amdgpu_debugfs_sdma_sched_mask_init(struct amdgpu_device *adev) 407 { 408 #if defined(CONFIG_DEBUG_FS) 409 struct drm_minor *minor = adev_to_drm(adev)->primary; 410 struct dentry *root = minor->debugfs_root; 411 char name[32]; 412 413 if (!(adev->sdma.num_instances > 1)) 414 return; 415 sprintf(name, "amdgpu_sdma_sched_mask"); 416 debugfs_create_file(name, 0600, root, adev, 417 &amdgpu_debugfs_sdma_sched_mask_fops); 418 #endif 419 } 420 421 static ssize_t amdgpu_get_sdma_reset_mask(struct device *dev, 422 struct device_attribute *attr, 423 char *buf) 424 { 425 struct drm_device *ddev = dev_get_drvdata(dev); 426 struct amdgpu_device *adev = drm_to_adev(ddev); 427 428 if (!adev) 429 return -ENODEV; 430 431 return amdgpu_show_reset_mask(buf, adev->sdma.supported_reset); 432 } 433 434 static DEVICE_ATTR(sdma_reset_mask, 0444, 435 amdgpu_get_sdma_reset_mask, NULL); 436 437 int amdgpu_sdma_sysfs_reset_mask_init(struct amdgpu_device *adev) 438 { 439 int r = 0; 440 441 if (!amdgpu_gpu_recovery) 442 return r; 443 444 if (adev->sdma.num_instances) { 445 r = device_create_file(adev->dev, &dev_attr_sdma_reset_mask); 446 if (r) 447 return r; 448 } 449 450 return r; 451 } 452 453 void amdgpu_sdma_sysfs_reset_mask_fini(struct amdgpu_device *adev) 454 { 455 if (!amdgpu_gpu_recovery) 456 return; 457 458 if (adev->dev->kobj.sd) { 459 if (adev->sdma.num_instances) 460 device_remove_file(adev->dev, &dev_attr_sdma_reset_mask); 461 } 462 } 463