1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include "amdgpu.h" 26 27 /** 28 * amdgpu_mm_rdoorbell - read a doorbell dword 29 * 30 * @adev: amdgpu_device pointer 31 * @index: doorbell index 32 * 33 * Returns the value in the doorbell aperture at the 34 * requested doorbell index (CIK). 35 */ 36 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) 37 { 38 if (amdgpu_device_skip_hw_access(adev)) 39 return 0; 40 41 if (index < adev->doorbell.num_kernel_doorbells) 42 return readl(adev->doorbell.cpu_addr + index); 43 44 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 45 return 0; 46 } 47 48 /** 49 * amdgpu_mm_wdoorbell - write a doorbell dword 50 * 51 * @adev: amdgpu_device pointer 52 * @index: doorbell index 53 * @v: value to write 54 * 55 * Writes @v to the doorbell aperture at the 56 * requested doorbell index (CIK). 57 */ 58 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) 59 { 60 if (amdgpu_device_skip_hw_access(adev)) 61 return; 62 63 if (index < adev->doorbell.num_kernel_doorbells) 64 writel(v, adev->doorbell.cpu_addr + index); 65 else 66 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 67 } 68 69 /** 70 * amdgpu_mm_rdoorbell64 - read a doorbell Qword 71 * 72 * @adev: amdgpu_device pointer 73 * @index: doorbell index 74 * 75 * Returns the value in the doorbell aperture at the 76 * requested doorbell index (VEGA10+). 77 */ 78 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) 79 { 80 if (amdgpu_device_skip_hw_access(adev)) 81 return 0; 82 83 if (index < adev->doorbell.num_kernel_doorbells) 84 return atomic64_read((atomic64_t *)(adev->doorbell.cpu_addr + index)); 85 86 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 87 return 0; 88 } 89 90 /** 91 * amdgpu_mm_wdoorbell64 - write a doorbell Qword 92 * 93 * @adev: amdgpu_device pointer 94 * @index: doorbell index 95 * @v: value to write 96 * 97 * Writes @v to the doorbell aperture at the 98 * requested doorbell index (VEGA10+). 99 */ 100 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) 101 { 102 if (amdgpu_device_skip_hw_access(adev)) 103 return; 104 105 if (index < adev->doorbell.num_kernel_doorbells) 106 atomic64_set((atomic64_t *)(adev->doorbell.cpu_addr + index), v); 107 else 108 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 109 } 110 111 /** 112 * amdgpu_doorbell_index_on_bar - Find doorbell's absolute offset in BAR 113 * 114 * @adev: amdgpu_device pointer 115 * @db_bo: doorbell object's bo 116 * @doorbell_index: doorbell relative index in this doorbell object 117 * @db_size: doorbell size is in byte 118 * 119 * returns doorbell's absolute index in BAR 120 */ 121 uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev, 122 struct amdgpu_bo *db_bo, 123 uint32_t doorbell_index, 124 uint32_t db_size) 125 { 126 int db_bo_offset; 127 128 db_bo_offset = amdgpu_bo_gpu_offset_no_check(db_bo); 129 130 /* doorbell index is 32 bit but doorbell's size can be 32 bit 131 * or 64 bit, so *db_size(in byte)/4 for alignment. 132 */ 133 return db_bo_offset / sizeof(u32) + doorbell_index * 134 DIV_ROUND_UP(db_size, 4); 135 } 136 137 /** 138 * amdgpu_doorbell_create_kernel_doorbells - Create kernel doorbells for graphics 139 * 140 * @adev: amdgpu_device pointer 141 * 142 * Creates doorbells for graphics driver usages. 143 * returns 0 on success, error otherwise. 144 */ 145 int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev) 146 { 147 int r; 148 int size; 149 150 /* SI HW does not have doorbells, skip allocation */ 151 if (adev->doorbell.num_kernel_doorbells == 0) 152 return 0; 153 154 /* Reserve first num_kernel_doorbells (page-aligned) for kernel ops */ 155 size = ALIGN(adev->doorbell.num_kernel_doorbells * sizeof(u32), PAGE_SIZE); 156 157 /* Allocate an extra page for MES kernel usages (ring test) */ 158 adev->mes.db_start_dw_offset = size / sizeof(u32); 159 size += PAGE_SIZE; 160 161 r = amdgpu_bo_create_kernel(adev, 162 size, 163 PAGE_SIZE, 164 AMDGPU_GEM_DOMAIN_DOORBELL, 165 &adev->doorbell.kernel_doorbells, 166 NULL, 167 (void **)&adev->doorbell.cpu_addr); 168 if (r) { 169 DRM_ERROR("Failed to allocate kernel doorbells, err=%d\n", r); 170 return r; 171 } 172 173 adev->doorbell.num_kernel_doorbells = size / sizeof(u32); 174 return 0; 175 } 176 177 /* 178 * GPU doorbell aperture helpers function. 179 */ 180 /** 181 * amdgpu_doorbell_init - Init doorbell driver information. 182 * 183 * @adev: amdgpu_device pointer 184 * 185 * Init doorbell driver information (CIK) 186 * Returns 0 on success, error on failure. 187 */ 188 int amdgpu_doorbell_init(struct amdgpu_device *adev) 189 { 190 191 /* No doorbell on SI hardware generation */ 192 if (adev->asic_type < CHIP_BONAIRE) { 193 adev->doorbell.base = 0; 194 adev->doorbell.size = 0; 195 adev->doorbell.num_kernel_doorbells = 0; 196 return 0; 197 } 198 199 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) 200 return -EINVAL; 201 202 amdgpu_asic_init_doorbell_index(adev); 203 204 /* doorbell bar mapping */ 205 adev->doorbell.base = pci_resource_start(adev->pdev, 2); 206 adev->doorbell.size = pci_resource_len(adev->pdev, 2); 207 208 adev->doorbell.num_kernel_doorbells = 209 min_t(u32, adev->doorbell.size / sizeof(u32), 210 adev->doorbell_index.max_assignment + 1); 211 if (adev->doorbell.num_kernel_doorbells == 0) 212 return -EINVAL; 213 214 /* 215 * For Vega, reserve and map two pages on doorbell BAR since SDMA 216 * paging queue doorbell use the second page. The 217 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the 218 * doorbells are in the first page. So with paging queue enabled, 219 * the max num_kernel_doorbells should + 1 page (0x400 in dword) 220 */ 221 if (adev->asic_type >= CHIP_VEGA10) 222 adev->doorbell.num_kernel_doorbells += 0x400; 223 224 return 0; 225 } 226 227 /** 228 * amdgpu_doorbell_fini - Tear down doorbell driver information. 229 * 230 * @adev: amdgpu_device pointer 231 * 232 * Tear down doorbell driver information (CIK) 233 */ 234 void amdgpu_doorbell_fini(struct amdgpu_device *adev) 235 { 236 amdgpu_bo_free_kernel(&adev->doorbell.kernel_doorbells, 237 NULL, 238 (void **)&adev->doorbell.cpu_addr); 239 } 240