1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #include "amdgpu.h" 26 27 /** 28 * amdgpu_mm_rdoorbell - read a doorbell dword 29 * 30 * @adev: amdgpu_device pointer 31 * @index: doorbell index 32 * 33 * Returns the value in the doorbell aperture at the 34 * requested doorbell index (CIK). 35 */ 36 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) 37 { 38 if (amdgpu_device_skip_hw_access(adev)) 39 return 0; 40 41 if (index < adev->doorbell.num_kernel_doorbells) 42 return readl(adev->doorbell.cpu_addr + index); 43 44 dev_err(adev->dev, "reading beyond doorbell aperture: 0x%08x!\n", 45 index); 46 return 0; 47 } 48 49 /** 50 * amdgpu_mm_wdoorbell - write a doorbell dword 51 * 52 * @adev: amdgpu_device pointer 53 * @index: doorbell index 54 * @v: value to write 55 * 56 * Writes @v to the doorbell aperture at the 57 * requested doorbell index (CIK). 58 */ 59 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) 60 { 61 if (amdgpu_device_skip_hw_access(adev)) 62 return; 63 64 if (index < adev->doorbell.num_kernel_doorbells) 65 writel(v, adev->doorbell.cpu_addr + index); 66 else 67 dev_err(adev->dev, 68 "writing beyond doorbell aperture: 0x%08x!\n", index); 69 } 70 71 /** 72 * amdgpu_mm_rdoorbell64 - read a doorbell Qword 73 * 74 * @adev: amdgpu_device pointer 75 * @index: doorbell index 76 * 77 * Returns the value in the doorbell aperture at the 78 * requested doorbell index (VEGA10+). 79 */ 80 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) 81 { 82 if (amdgpu_device_skip_hw_access(adev)) 83 return 0; 84 85 if (index < adev->doorbell.num_kernel_doorbells) 86 return atomic64_read((atomic64_t *)(adev->doorbell.cpu_addr + index)); 87 88 dev_err(adev->dev, "reading beyond doorbell aperture: 0x%08x!\n", 89 index); 90 return 0; 91 } 92 93 /** 94 * amdgpu_mm_wdoorbell64 - write a doorbell Qword 95 * 96 * @adev: amdgpu_device pointer 97 * @index: doorbell index 98 * @v: value to write 99 * 100 * Writes @v to the doorbell aperture at the 101 * requested doorbell index (VEGA10+). 102 */ 103 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) 104 { 105 if (amdgpu_device_skip_hw_access(adev)) 106 return; 107 108 if (index < adev->doorbell.num_kernel_doorbells) 109 atomic64_set((atomic64_t *)(adev->doorbell.cpu_addr + index), v); 110 else 111 dev_err(adev->dev, 112 "writing beyond doorbell aperture: 0x%08x!\n", index); 113 } 114 115 /** 116 * amdgpu_doorbell_index_on_bar - Find doorbell's absolute offset in BAR 117 * 118 * @adev: amdgpu_device pointer 119 * @db_bo: doorbell object's bo 120 * @doorbell_index: doorbell relative index in this doorbell object 121 * @db_size: doorbell size is in byte 122 * 123 * returns doorbell's absolute index in BAR 124 */ 125 uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev, 126 struct amdgpu_bo *db_bo, 127 uint32_t doorbell_index, 128 uint32_t db_size) 129 { 130 int db_bo_offset; 131 132 db_bo_offset = amdgpu_bo_gpu_offset_no_check(db_bo); 133 134 /* doorbell index is 32 bit but doorbell's size can be 32 bit 135 * or 64 bit, so *db_size(in byte)/4 for alignment. 136 */ 137 return db_bo_offset / sizeof(u32) + doorbell_index * 138 DIV_ROUND_UP(db_size, 4); 139 } 140 141 /** 142 * amdgpu_doorbell_create_kernel_doorbells - Create kernel doorbells for graphics 143 * 144 * @adev: amdgpu_device pointer 145 * 146 * Creates doorbells for graphics driver usages. 147 * returns 0 on success, error otherwise. 148 */ 149 int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev) 150 { 151 int r; 152 int size; 153 154 /* SI HW does not have doorbells, skip allocation */ 155 if (adev->doorbell.num_kernel_doorbells == 0) 156 return 0; 157 158 /* Reserve first num_kernel_doorbells (page-aligned) for kernel ops */ 159 size = ALIGN(adev->doorbell.num_kernel_doorbells * sizeof(u32), PAGE_SIZE); 160 161 /* Allocate an extra page for MES kernel usages (ring test) */ 162 adev->mes.db_start_dw_offset = size / sizeof(u32); 163 size += PAGE_SIZE; 164 165 r = amdgpu_bo_create_kernel(adev, 166 size, 167 PAGE_SIZE, 168 AMDGPU_GEM_DOMAIN_DOORBELL, 169 &adev->doorbell.kernel_doorbells, 170 NULL, 171 (void **)&adev->doorbell.cpu_addr); 172 if (r) { 173 dev_err(adev->dev, 174 "Failed to allocate kernel doorbells, err=%d\n", r); 175 return r; 176 } 177 178 adev->doorbell.num_kernel_doorbells = size / sizeof(u32); 179 return 0; 180 } 181 182 /* 183 * GPU doorbell aperture helpers function. 184 */ 185 /** 186 * amdgpu_doorbell_init - Init doorbell driver information. 187 * 188 * @adev: amdgpu_device pointer 189 * 190 * Init doorbell driver information (CIK) 191 * Returns 0 on success, error on failure. 192 */ 193 int amdgpu_doorbell_init(struct amdgpu_device *adev) 194 { 195 196 /* No doorbell on SI hardware generation */ 197 if (adev->asic_type < CHIP_BONAIRE) { 198 adev->doorbell.base = 0; 199 adev->doorbell.size = 0; 200 adev->doorbell.num_kernel_doorbells = 0; 201 return 0; 202 } 203 204 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) 205 return -EINVAL; 206 207 amdgpu_asic_init_doorbell_index(adev); 208 209 /* doorbell bar mapping */ 210 adev->doorbell.base = pci_resource_start(adev->pdev, 2); 211 adev->doorbell.size = pci_resource_len(adev->pdev, 2); 212 213 adev->doorbell.num_kernel_doorbells = 214 min_t(u32, adev->doorbell.size / sizeof(u32), 215 adev->doorbell_index.max_assignment + 1); 216 if (adev->doorbell.num_kernel_doorbells == 0) 217 return -EINVAL; 218 219 /* 220 * For Vega, reserve and map two pages on doorbell BAR since SDMA 221 * paging queue doorbell use the second page. The 222 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the 223 * doorbells are in the first page. So with paging queue enabled, 224 * the max num_kernel_doorbells should + 1 page (0x400 in dword) 225 */ 226 if (adev->asic_type >= CHIP_VEGA10) 227 adev->doorbell.num_kernel_doorbells += 0x400; 228 229 return 0; 230 } 231 232 /** 233 * amdgpu_doorbell_fini - Tear down doorbell driver information. 234 * 235 * @adev: amdgpu_device pointer 236 * 237 * Tear down doorbell driver information (CIK) 238 */ 239 void amdgpu_doorbell_fini(struct amdgpu_device *adev) 240 { 241 amdgpu_bo_free_kernel(&adev->doorbell.kernel_doorbells, 242 NULL, 243 (void **)&adev->doorbell.cpu_addr); 244 } 245