1 /* 2 * Copyright 2011 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/slab.h> 26 #include <linux/module.h> 27 #include <drm/drmP.h> 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include <drm/radeon_drm.h> 31 #include "sid.h" 32 #include "atom.h" 33 #include "si_blit_shaders.h" 34 #include "clearstate_si.h" 35 #include "radeon_ucode.h" 36 37 38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin"); 39 MODULE_FIRMWARE("radeon/TAHITI_me.bin"); 40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin"); 41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin"); 42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin"); 43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin"); 44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin"); 45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin"); 46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin"); 47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin"); 48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin"); 49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin"); 50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin"); 51 MODULE_FIRMWARE("radeon/VERDE_me.bin"); 52 MODULE_FIRMWARE("radeon/VERDE_ce.bin"); 53 MODULE_FIRMWARE("radeon/VERDE_mc.bin"); 54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin"); 55 MODULE_FIRMWARE("radeon/VERDE_smc.bin"); 56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin"); 57 MODULE_FIRMWARE("radeon/OLAND_me.bin"); 58 MODULE_FIRMWARE("radeon/OLAND_ce.bin"); 59 MODULE_FIRMWARE("radeon/OLAND_mc.bin"); 60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin"); 61 MODULE_FIRMWARE("radeon/OLAND_smc.bin"); 62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin"); 63 MODULE_FIRMWARE("radeon/HAINAN_me.bin"); 64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin"); 65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin"); 66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin"); 67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin"); 68 69 static void si_pcie_gen3_enable(struct radeon_device *rdev); 70 static void si_program_aspm(struct radeon_device *rdev); 71 extern void sumo_rlc_fini(struct radeon_device *rdev); 72 extern int sumo_rlc_init(struct radeon_device *rdev); 73 extern int r600_ih_ring_alloc(struct radeon_device *rdev); 74 extern void r600_ih_ring_fini(struct radeon_device *rdev); 75 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev); 76 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save); 77 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save); 78 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev); 79 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev); 80 extern bool evergreen_is_display_hung(struct radeon_device *rdev); 81 extern void si_dma_vm_set_page(struct radeon_device *rdev, 82 struct radeon_ib *ib, 83 uint64_t pe, 84 uint64_t addr, unsigned count, 85 uint32_t incr, uint32_t flags); 86 87 static const u32 verde_rlc_save_restore_register_list[] = 88 { 89 (0x8000 << 16) | (0x98f4 >> 2), 90 0x00000000, 91 (0x8040 << 16) | (0x98f4 >> 2), 92 0x00000000, 93 (0x8000 << 16) | (0xe80 >> 2), 94 0x00000000, 95 (0x8040 << 16) | (0xe80 >> 2), 96 0x00000000, 97 (0x8000 << 16) | (0x89bc >> 2), 98 0x00000000, 99 (0x8040 << 16) | (0x89bc >> 2), 100 0x00000000, 101 (0x8000 << 16) | (0x8c1c >> 2), 102 0x00000000, 103 (0x8040 << 16) | (0x8c1c >> 2), 104 0x00000000, 105 (0x9c00 << 16) | (0x98f0 >> 2), 106 0x00000000, 107 (0x9c00 << 16) | (0xe7c >> 2), 108 0x00000000, 109 (0x8000 << 16) | (0x9148 >> 2), 110 0x00000000, 111 (0x8040 << 16) | (0x9148 >> 2), 112 0x00000000, 113 (0x9c00 << 16) | (0x9150 >> 2), 114 0x00000000, 115 (0x9c00 << 16) | (0x897c >> 2), 116 0x00000000, 117 (0x9c00 << 16) | (0x8d8c >> 2), 118 0x00000000, 119 (0x9c00 << 16) | (0xac54 >> 2), 120 0X00000000, 121 0x3, 122 (0x9c00 << 16) | (0x98f8 >> 2), 123 0x00000000, 124 (0x9c00 << 16) | (0x9910 >> 2), 125 0x00000000, 126 (0x9c00 << 16) | (0x9914 >> 2), 127 0x00000000, 128 (0x9c00 << 16) | (0x9918 >> 2), 129 0x00000000, 130 (0x9c00 << 16) | (0x991c >> 2), 131 0x00000000, 132 (0x9c00 << 16) | (0x9920 >> 2), 133 0x00000000, 134 (0x9c00 << 16) | (0x9924 >> 2), 135 0x00000000, 136 (0x9c00 << 16) | (0x9928 >> 2), 137 0x00000000, 138 (0x9c00 << 16) | (0x992c >> 2), 139 0x00000000, 140 (0x9c00 << 16) | (0x9930 >> 2), 141 0x00000000, 142 (0x9c00 << 16) | (0x9934 >> 2), 143 0x00000000, 144 (0x9c00 << 16) | (0x9938 >> 2), 145 0x00000000, 146 (0x9c00 << 16) | (0x993c >> 2), 147 0x00000000, 148 (0x9c00 << 16) | (0x9940 >> 2), 149 0x00000000, 150 (0x9c00 << 16) | (0x9944 >> 2), 151 0x00000000, 152 (0x9c00 << 16) | (0x9948 >> 2), 153 0x00000000, 154 (0x9c00 << 16) | (0x994c >> 2), 155 0x00000000, 156 (0x9c00 << 16) | (0x9950 >> 2), 157 0x00000000, 158 (0x9c00 << 16) | (0x9954 >> 2), 159 0x00000000, 160 (0x9c00 << 16) | (0x9958 >> 2), 161 0x00000000, 162 (0x9c00 << 16) | (0x995c >> 2), 163 0x00000000, 164 (0x9c00 << 16) | (0x9960 >> 2), 165 0x00000000, 166 (0x9c00 << 16) | (0x9964 >> 2), 167 0x00000000, 168 (0x9c00 << 16) | (0x9968 >> 2), 169 0x00000000, 170 (0x9c00 << 16) | (0x996c >> 2), 171 0x00000000, 172 (0x9c00 << 16) | (0x9970 >> 2), 173 0x00000000, 174 (0x9c00 << 16) | (0x9974 >> 2), 175 0x00000000, 176 (0x9c00 << 16) | (0x9978 >> 2), 177 0x00000000, 178 (0x9c00 << 16) | (0x997c >> 2), 179 0x00000000, 180 (0x9c00 << 16) | (0x9980 >> 2), 181 0x00000000, 182 (0x9c00 << 16) | (0x9984 >> 2), 183 0x00000000, 184 (0x9c00 << 16) | (0x9988 >> 2), 185 0x00000000, 186 (0x9c00 << 16) | (0x998c >> 2), 187 0x00000000, 188 (0x9c00 << 16) | (0x8c00 >> 2), 189 0x00000000, 190 (0x9c00 << 16) | (0x8c14 >> 2), 191 0x00000000, 192 (0x9c00 << 16) | (0x8c04 >> 2), 193 0x00000000, 194 (0x9c00 << 16) | (0x8c08 >> 2), 195 0x00000000, 196 (0x8000 << 16) | (0x9b7c >> 2), 197 0x00000000, 198 (0x8040 << 16) | (0x9b7c >> 2), 199 0x00000000, 200 (0x8000 << 16) | (0xe84 >> 2), 201 0x00000000, 202 (0x8040 << 16) | (0xe84 >> 2), 203 0x00000000, 204 (0x8000 << 16) | (0x89c0 >> 2), 205 0x00000000, 206 (0x8040 << 16) | (0x89c0 >> 2), 207 0x00000000, 208 (0x8000 << 16) | (0x914c >> 2), 209 0x00000000, 210 (0x8040 << 16) | (0x914c >> 2), 211 0x00000000, 212 (0x8000 << 16) | (0x8c20 >> 2), 213 0x00000000, 214 (0x8040 << 16) | (0x8c20 >> 2), 215 0x00000000, 216 (0x8000 << 16) | (0x9354 >> 2), 217 0x00000000, 218 (0x8040 << 16) | (0x9354 >> 2), 219 0x00000000, 220 (0x9c00 << 16) | (0x9060 >> 2), 221 0x00000000, 222 (0x9c00 << 16) | (0x9364 >> 2), 223 0x00000000, 224 (0x9c00 << 16) | (0x9100 >> 2), 225 0x00000000, 226 (0x9c00 << 16) | (0x913c >> 2), 227 0x00000000, 228 (0x8000 << 16) | (0x90e0 >> 2), 229 0x00000000, 230 (0x8000 << 16) | (0x90e4 >> 2), 231 0x00000000, 232 (0x8000 << 16) | (0x90e8 >> 2), 233 0x00000000, 234 (0x8040 << 16) | (0x90e0 >> 2), 235 0x00000000, 236 (0x8040 << 16) | (0x90e4 >> 2), 237 0x00000000, 238 (0x8040 << 16) | (0x90e8 >> 2), 239 0x00000000, 240 (0x9c00 << 16) | (0x8bcc >> 2), 241 0x00000000, 242 (0x9c00 << 16) | (0x8b24 >> 2), 243 0x00000000, 244 (0x9c00 << 16) | (0x88c4 >> 2), 245 0x00000000, 246 (0x9c00 << 16) | (0x8e50 >> 2), 247 0x00000000, 248 (0x9c00 << 16) | (0x8c0c >> 2), 249 0x00000000, 250 (0x9c00 << 16) | (0x8e58 >> 2), 251 0x00000000, 252 (0x9c00 << 16) | (0x8e5c >> 2), 253 0x00000000, 254 (0x9c00 << 16) | (0x9508 >> 2), 255 0x00000000, 256 (0x9c00 << 16) | (0x950c >> 2), 257 0x00000000, 258 (0x9c00 << 16) | (0x9494 >> 2), 259 0x00000000, 260 (0x9c00 << 16) | (0xac0c >> 2), 261 0x00000000, 262 (0x9c00 << 16) | (0xac10 >> 2), 263 0x00000000, 264 (0x9c00 << 16) | (0xac14 >> 2), 265 0x00000000, 266 (0x9c00 << 16) | (0xae00 >> 2), 267 0x00000000, 268 (0x9c00 << 16) | (0xac08 >> 2), 269 0x00000000, 270 (0x9c00 << 16) | (0x88d4 >> 2), 271 0x00000000, 272 (0x9c00 << 16) | (0x88c8 >> 2), 273 0x00000000, 274 (0x9c00 << 16) | (0x88cc >> 2), 275 0x00000000, 276 (0x9c00 << 16) | (0x89b0 >> 2), 277 0x00000000, 278 (0x9c00 << 16) | (0x8b10 >> 2), 279 0x00000000, 280 (0x9c00 << 16) | (0x8a14 >> 2), 281 0x00000000, 282 (0x9c00 << 16) | (0x9830 >> 2), 283 0x00000000, 284 (0x9c00 << 16) | (0x9834 >> 2), 285 0x00000000, 286 (0x9c00 << 16) | (0x9838 >> 2), 287 0x00000000, 288 (0x9c00 << 16) | (0x9a10 >> 2), 289 0x00000000, 290 (0x8000 << 16) | (0x9870 >> 2), 291 0x00000000, 292 (0x8000 << 16) | (0x9874 >> 2), 293 0x00000000, 294 (0x8001 << 16) | (0x9870 >> 2), 295 0x00000000, 296 (0x8001 << 16) | (0x9874 >> 2), 297 0x00000000, 298 (0x8040 << 16) | (0x9870 >> 2), 299 0x00000000, 300 (0x8040 << 16) | (0x9874 >> 2), 301 0x00000000, 302 (0x8041 << 16) | (0x9870 >> 2), 303 0x00000000, 304 (0x8041 << 16) | (0x9874 >> 2), 305 0x00000000, 306 0x00000000 307 }; 308 309 static const u32 tahiti_golden_rlc_registers[] = 310 { 311 0xc424, 0xffffffff, 0x00601005, 312 0xc47c, 0xffffffff, 0x10104040, 313 0xc488, 0xffffffff, 0x0100000a, 314 0xc314, 0xffffffff, 0x00000800, 315 0xc30c, 0xffffffff, 0x800000f4, 316 0xf4a8, 0xffffffff, 0x00000000 317 }; 318 319 static const u32 tahiti_golden_registers[] = 320 { 321 0x9a10, 0x00010000, 0x00018208, 322 0x9830, 0xffffffff, 0x00000000, 323 0x9834, 0xf00fffff, 0x00000400, 324 0x9838, 0x0002021c, 0x00020200, 325 0xc78, 0x00000080, 0x00000000, 326 0xd030, 0x000300c0, 0x00800040, 327 0xd830, 0x000300c0, 0x00800040, 328 0x5bb0, 0x000000f0, 0x00000070, 329 0x5bc0, 0x00200000, 0x50100000, 330 0x7030, 0x31000311, 0x00000011, 331 0x277c, 0x00000003, 0x000007ff, 332 0x240c, 0x000007ff, 0x00000000, 333 0x8a14, 0xf000001f, 0x00000007, 334 0x8b24, 0xffffffff, 0x00ffffff, 335 0x8b10, 0x0000ff0f, 0x00000000, 336 0x28a4c, 0x07ffffff, 0x4e000000, 337 0x28350, 0x3f3f3fff, 0x2a00126a, 338 0x30, 0x000000ff, 0x0040, 339 0x34, 0x00000040, 0x00004040, 340 0x9100, 0x07ffffff, 0x03000000, 341 0x8e88, 0x01ff1f3f, 0x00000000, 342 0x8e84, 0x01ff1f3f, 0x00000000, 343 0x9060, 0x0000007f, 0x00000020, 344 0x9508, 0x00010000, 0x00010000, 345 0xac14, 0x00000200, 0x000002fb, 346 0xac10, 0xffffffff, 0x0000543b, 347 0xac0c, 0xffffffff, 0xa9210876, 348 0x88d0, 0xffffffff, 0x000fff40, 349 0x88d4, 0x0000001f, 0x00000010, 350 0x1410, 0x20000000, 0x20fffed8, 351 0x15c0, 0x000c0fc0, 0x000c0400 352 }; 353 354 static const u32 tahiti_golden_registers2[] = 355 { 356 0xc64, 0x00000001, 0x00000001 357 }; 358 359 static const u32 pitcairn_golden_rlc_registers[] = 360 { 361 0xc424, 0xffffffff, 0x00601004, 362 0xc47c, 0xffffffff, 0x10102020, 363 0xc488, 0xffffffff, 0x01000020, 364 0xc314, 0xffffffff, 0x00000800, 365 0xc30c, 0xffffffff, 0x800000a4 366 }; 367 368 static const u32 pitcairn_golden_registers[] = 369 { 370 0x9a10, 0x00010000, 0x00018208, 371 0x9830, 0xffffffff, 0x00000000, 372 0x9834, 0xf00fffff, 0x00000400, 373 0x9838, 0x0002021c, 0x00020200, 374 0xc78, 0x00000080, 0x00000000, 375 0xd030, 0x000300c0, 0x00800040, 376 0xd830, 0x000300c0, 0x00800040, 377 0x5bb0, 0x000000f0, 0x00000070, 378 0x5bc0, 0x00200000, 0x50100000, 379 0x7030, 0x31000311, 0x00000011, 380 0x2ae4, 0x00073ffe, 0x000022a2, 381 0x240c, 0x000007ff, 0x00000000, 382 0x8a14, 0xf000001f, 0x00000007, 383 0x8b24, 0xffffffff, 0x00ffffff, 384 0x8b10, 0x0000ff0f, 0x00000000, 385 0x28a4c, 0x07ffffff, 0x4e000000, 386 0x28350, 0x3f3f3fff, 0x2a00126a, 387 0x30, 0x000000ff, 0x0040, 388 0x34, 0x00000040, 0x00004040, 389 0x9100, 0x07ffffff, 0x03000000, 390 0x9060, 0x0000007f, 0x00000020, 391 0x9508, 0x00010000, 0x00010000, 392 0xac14, 0x000003ff, 0x000000f7, 393 0xac10, 0xffffffff, 0x00000000, 394 0xac0c, 0xffffffff, 0x32761054, 395 0x88d4, 0x0000001f, 0x00000010, 396 0x15c0, 0x000c0fc0, 0x000c0400 397 }; 398 399 static const u32 verde_golden_rlc_registers[] = 400 { 401 0xc424, 0xffffffff, 0x033f1005, 402 0xc47c, 0xffffffff, 0x10808020, 403 0xc488, 0xffffffff, 0x00800008, 404 0xc314, 0xffffffff, 0x00001000, 405 0xc30c, 0xffffffff, 0x80010014 406 }; 407 408 static const u32 verde_golden_registers[] = 409 { 410 0x9a10, 0x00010000, 0x00018208, 411 0x9830, 0xffffffff, 0x00000000, 412 0x9834, 0xf00fffff, 0x00000400, 413 0x9838, 0x0002021c, 0x00020200, 414 0xc78, 0x00000080, 0x00000000, 415 0xd030, 0x000300c0, 0x00800040, 416 0xd030, 0x000300c0, 0x00800040, 417 0xd830, 0x000300c0, 0x00800040, 418 0xd830, 0x000300c0, 0x00800040, 419 0x5bb0, 0x000000f0, 0x00000070, 420 0x5bc0, 0x00200000, 0x50100000, 421 0x7030, 0x31000311, 0x00000011, 422 0x2ae4, 0x00073ffe, 0x000022a2, 423 0x2ae4, 0x00073ffe, 0x000022a2, 424 0x2ae4, 0x00073ffe, 0x000022a2, 425 0x240c, 0x000007ff, 0x00000000, 426 0x240c, 0x000007ff, 0x00000000, 427 0x240c, 0x000007ff, 0x00000000, 428 0x8a14, 0xf000001f, 0x00000007, 429 0x8a14, 0xf000001f, 0x00000007, 430 0x8a14, 0xf000001f, 0x00000007, 431 0x8b24, 0xffffffff, 0x00ffffff, 432 0x8b10, 0x0000ff0f, 0x00000000, 433 0x28a4c, 0x07ffffff, 0x4e000000, 434 0x28350, 0x3f3f3fff, 0x0000124a, 435 0x28350, 0x3f3f3fff, 0x0000124a, 436 0x28350, 0x3f3f3fff, 0x0000124a, 437 0x30, 0x000000ff, 0x0040, 438 0x34, 0x00000040, 0x00004040, 439 0x9100, 0x07ffffff, 0x03000000, 440 0x9100, 0x07ffffff, 0x03000000, 441 0x8e88, 0x01ff1f3f, 0x00000000, 442 0x8e88, 0x01ff1f3f, 0x00000000, 443 0x8e88, 0x01ff1f3f, 0x00000000, 444 0x8e84, 0x01ff1f3f, 0x00000000, 445 0x8e84, 0x01ff1f3f, 0x00000000, 446 0x8e84, 0x01ff1f3f, 0x00000000, 447 0x9060, 0x0000007f, 0x00000020, 448 0x9508, 0x00010000, 0x00010000, 449 0xac14, 0x000003ff, 0x00000003, 450 0xac14, 0x000003ff, 0x00000003, 451 0xac14, 0x000003ff, 0x00000003, 452 0xac10, 0xffffffff, 0x00000000, 453 0xac10, 0xffffffff, 0x00000000, 454 0xac10, 0xffffffff, 0x00000000, 455 0xac0c, 0xffffffff, 0x00001032, 456 0xac0c, 0xffffffff, 0x00001032, 457 0xac0c, 0xffffffff, 0x00001032, 458 0x88d4, 0x0000001f, 0x00000010, 459 0x88d4, 0x0000001f, 0x00000010, 460 0x88d4, 0x0000001f, 0x00000010, 461 0x15c0, 0x000c0fc0, 0x000c0400 462 }; 463 464 static const u32 oland_golden_rlc_registers[] = 465 { 466 0xc424, 0xffffffff, 0x00601005, 467 0xc47c, 0xffffffff, 0x10104040, 468 0xc488, 0xffffffff, 0x0100000a, 469 0xc314, 0xffffffff, 0x00000800, 470 0xc30c, 0xffffffff, 0x800000f4 471 }; 472 473 static const u32 oland_golden_registers[] = 474 { 475 0x9a10, 0x00010000, 0x00018208, 476 0x9830, 0xffffffff, 0x00000000, 477 0x9834, 0xf00fffff, 0x00000400, 478 0x9838, 0x0002021c, 0x00020200, 479 0xc78, 0x00000080, 0x00000000, 480 0xd030, 0x000300c0, 0x00800040, 481 0xd830, 0x000300c0, 0x00800040, 482 0x5bb0, 0x000000f0, 0x00000070, 483 0x5bc0, 0x00200000, 0x50100000, 484 0x7030, 0x31000311, 0x00000011, 485 0x2ae4, 0x00073ffe, 0x000022a2, 486 0x240c, 0x000007ff, 0x00000000, 487 0x8a14, 0xf000001f, 0x00000007, 488 0x8b24, 0xffffffff, 0x00ffffff, 489 0x8b10, 0x0000ff0f, 0x00000000, 490 0x28a4c, 0x07ffffff, 0x4e000000, 491 0x28350, 0x3f3f3fff, 0x00000082, 492 0x30, 0x000000ff, 0x0040, 493 0x34, 0x00000040, 0x00004040, 494 0x9100, 0x07ffffff, 0x03000000, 495 0x9060, 0x0000007f, 0x00000020, 496 0x9508, 0x00010000, 0x00010000, 497 0xac14, 0x000003ff, 0x000000f3, 498 0xac10, 0xffffffff, 0x00000000, 499 0xac0c, 0xffffffff, 0x00003210, 500 0x88d4, 0x0000001f, 0x00000010, 501 0x15c0, 0x000c0fc0, 0x000c0400 502 }; 503 504 static const u32 hainan_golden_registers[] = 505 { 506 0x9a10, 0x00010000, 0x00018208, 507 0x9830, 0xffffffff, 0x00000000, 508 0x9834, 0xf00fffff, 0x00000400, 509 0x9838, 0x0002021c, 0x00020200, 510 0xd0c0, 0xff000fff, 0x00000100, 511 0xd030, 0x000300c0, 0x00800040, 512 0xd8c0, 0xff000fff, 0x00000100, 513 0xd830, 0x000300c0, 0x00800040, 514 0x2ae4, 0x00073ffe, 0x000022a2, 515 0x240c, 0x000007ff, 0x00000000, 516 0x8a14, 0xf000001f, 0x00000007, 517 0x8b24, 0xffffffff, 0x00ffffff, 518 0x8b10, 0x0000ff0f, 0x00000000, 519 0x28a4c, 0x07ffffff, 0x4e000000, 520 0x28350, 0x3f3f3fff, 0x00000000, 521 0x30, 0x000000ff, 0x0040, 522 0x34, 0x00000040, 0x00004040, 523 0x9100, 0x03e00000, 0x03600000, 524 0x9060, 0x0000007f, 0x00000020, 525 0x9508, 0x00010000, 0x00010000, 526 0xac14, 0x000003ff, 0x000000f1, 527 0xac10, 0xffffffff, 0x00000000, 528 0xac0c, 0xffffffff, 0x00003210, 529 0x88d4, 0x0000001f, 0x00000010, 530 0x15c0, 0x000c0fc0, 0x000c0400 531 }; 532 533 static const u32 hainan_golden_registers2[] = 534 { 535 0x98f8, 0xffffffff, 0x02010001 536 }; 537 538 static const u32 tahiti_mgcg_cgcg_init[] = 539 { 540 0xc400, 0xffffffff, 0xfffffffc, 541 0x802c, 0xffffffff, 0xe0000000, 542 0x9a60, 0xffffffff, 0x00000100, 543 0x92a4, 0xffffffff, 0x00000100, 544 0xc164, 0xffffffff, 0x00000100, 545 0x9774, 0xffffffff, 0x00000100, 546 0x8984, 0xffffffff, 0x06000100, 547 0x8a18, 0xffffffff, 0x00000100, 548 0x92a0, 0xffffffff, 0x00000100, 549 0xc380, 0xffffffff, 0x00000100, 550 0x8b28, 0xffffffff, 0x00000100, 551 0x9144, 0xffffffff, 0x00000100, 552 0x8d88, 0xffffffff, 0x00000100, 553 0x8d8c, 0xffffffff, 0x00000100, 554 0x9030, 0xffffffff, 0x00000100, 555 0x9034, 0xffffffff, 0x00000100, 556 0x9038, 0xffffffff, 0x00000100, 557 0x903c, 0xffffffff, 0x00000100, 558 0xad80, 0xffffffff, 0x00000100, 559 0xac54, 0xffffffff, 0x00000100, 560 0x897c, 0xffffffff, 0x06000100, 561 0x9868, 0xffffffff, 0x00000100, 562 0x9510, 0xffffffff, 0x00000100, 563 0xaf04, 0xffffffff, 0x00000100, 564 0xae04, 0xffffffff, 0x00000100, 565 0x949c, 0xffffffff, 0x00000100, 566 0x802c, 0xffffffff, 0xe0000000, 567 0x9160, 0xffffffff, 0x00010000, 568 0x9164, 0xffffffff, 0x00030002, 569 0x9168, 0xffffffff, 0x00040007, 570 0x916c, 0xffffffff, 0x00060005, 571 0x9170, 0xffffffff, 0x00090008, 572 0x9174, 0xffffffff, 0x00020001, 573 0x9178, 0xffffffff, 0x00040003, 574 0x917c, 0xffffffff, 0x00000007, 575 0x9180, 0xffffffff, 0x00060005, 576 0x9184, 0xffffffff, 0x00090008, 577 0x9188, 0xffffffff, 0x00030002, 578 0x918c, 0xffffffff, 0x00050004, 579 0x9190, 0xffffffff, 0x00000008, 580 0x9194, 0xffffffff, 0x00070006, 581 0x9198, 0xffffffff, 0x000a0009, 582 0x919c, 0xffffffff, 0x00040003, 583 0x91a0, 0xffffffff, 0x00060005, 584 0x91a4, 0xffffffff, 0x00000009, 585 0x91a8, 0xffffffff, 0x00080007, 586 0x91ac, 0xffffffff, 0x000b000a, 587 0x91b0, 0xffffffff, 0x00050004, 588 0x91b4, 0xffffffff, 0x00070006, 589 0x91b8, 0xffffffff, 0x0008000b, 590 0x91bc, 0xffffffff, 0x000a0009, 591 0x91c0, 0xffffffff, 0x000d000c, 592 0x91c4, 0xffffffff, 0x00060005, 593 0x91c8, 0xffffffff, 0x00080007, 594 0x91cc, 0xffffffff, 0x0000000b, 595 0x91d0, 0xffffffff, 0x000a0009, 596 0x91d4, 0xffffffff, 0x000d000c, 597 0x91d8, 0xffffffff, 0x00070006, 598 0x91dc, 0xffffffff, 0x00090008, 599 0x91e0, 0xffffffff, 0x0000000c, 600 0x91e4, 0xffffffff, 0x000b000a, 601 0x91e8, 0xffffffff, 0x000e000d, 602 0x91ec, 0xffffffff, 0x00080007, 603 0x91f0, 0xffffffff, 0x000a0009, 604 0x91f4, 0xffffffff, 0x0000000d, 605 0x91f8, 0xffffffff, 0x000c000b, 606 0x91fc, 0xffffffff, 0x000f000e, 607 0x9200, 0xffffffff, 0x00090008, 608 0x9204, 0xffffffff, 0x000b000a, 609 0x9208, 0xffffffff, 0x000c000f, 610 0x920c, 0xffffffff, 0x000e000d, 611 0x9210, 0xffffffff, 0x00110010, 612 0x9214, 0xffffffff, 0x000a0009, 613 0x9218, 0xffffffff, 0x000c000b, 614 0x921c, 0xffffffff, 0x0000000f, 615 0x9220, 0xffffffff, 0x000e000d, 616 0x9224, 0xffffffff, 0x00110010, 617 0x9228, 0xffffffff, 0x000b000a, 618 0x922c, 0xffffffff, 0x000d000c, 619 0x9230, 0xffffffff, 0x00000010, 620 0x9234, 0xffffffff, 0x000f000e, 621 0x9238, 0xffffffff, 0x00120011, 622 0x923c, 0xffffffff, 0x000c000b, 623 0x9240, 0xffffffff, 0x000e000d, 624 0x9244, 0xffffffff, 0x00000011, 625 0x9248, 0xffffffff, 0x0010000f, 626 0x924c, 0xffffffff, 0x00130012, 627 0x9250, 0xffffffff, 0x000d000c, 628 0x9254, 0xffffffff, 0x000f000e, 629 0x9258, 0xffffffff, 0x00100013, 630 0x925c, 0xffffffff, 0x00120011, 631 0x9260, 0xffffffff, 0x00150014, 632 0x9264, 0xffffffff, 0x000e000d, 633 0x9268, 0xffffffff, 0x0010000f, 634 0x926c, 0xffffffff, 0x00000013, 635 0x9270, 0xffffffff, 0x00120011, 636 0x9274, 0xffffffff, 0x00150014, 637 0x9278, 0xffffffff, 0x000f000e, 638 0x927c, 0xffffffff, 0x00110010, 639 0x9280, 0xffffffff, 0x00000014, 640 0x9284, 0xffffffff, 0x00130012, 641 0x9288, 0xffffffff, 0x00160015, 642 0x928c, 0xffffffff, 0x0010000f, 643 0x9290, 0xffffffff, 0x00120011, 644 0x9294, 0xffffffff, 0x00000015, 645 0x9298, 0xffffffff, 0x00140013, 646 0x929c, 0xffffffff, 0x00170016, 647 0x9150, 0xffffffff, 0x96940200, 648 0x8708, 0xffffffff, 0x00900100, 649 0xc478, 0xffffffff, 0x00000080, 650 0xc404, 0xffffffff, 0x0020003f, 651 0x30, 0xffffffff, 0x0000001c, 652 0x34, 0x000f0000, 0x000f0000, 653 0x160c, 0xffffffff, 0x00000100, 654 0x1024, 0xffffffff, 0x00000100, 655 0x102c, 0x00000101, 0x00000000, 656 0x20a8, 0xffffffff, 0x00000104, 657 0x264c, 0x000c0000, 0x000c0000, 658 0x2648, 0x000c0000, 0x000c0000, 659 0x55e4, 0xff000fff, 0x00000100, 660 0x55e8, 0x00000001, 0x00000001, 661 0x2f50, 0x00000001, 0x00000001, 662 0x30cc, 0xc0000fff, 0x00000104, 663 0xc1e4, 0x00000001, 0x00000001, 664 0xd0c0, 0xfffffff0, 0x00000100, 665 0xd8c0, 0xfffffff0, 0x00000100 666 }; 667 668 static const u32 pitcairn_mgcg_cgcg_init[] = 669 { 670 0xc400, 0xffffffff, 0xfffffffc, 671 0x802c, 0xffffffff, 0xe0000000, 672 0x9a60, 0xffffffff, 0x00000100, 673 0x92a4, 0xffffffff, 0x00000100, 674 0xc164, 0xffffffff, 0x00000100, 675 0x9774, 0xffffffff, 0x00000100, 676 0x8984, 0xffffffff, 0x06000100, 677 0x8a18, 0xffffffff, 0x00000100, 678 0x92a0, 0xffffffff, 0x00000100, 679 0xc380, 0xffffffff, 0x00000100, 680 0x8b28, 0xffffffff, 0x00000100, 681 0x9144, 0xffffffff, 0x00000100, 682 0x8d88, 0xffffffff, 0x00000100, 683 0x8d8c, 0xffffffff, 0x00000100, 684 0x9030, 0xffffffff, 0x00000100, 685 0x9034, 0xffffffff, 0x00000100, 686 0x9038, 0xffffffff, 0x00000100, 687 0x903c, 0xffffffff, 0x00000100, 688 0xad80, 0xffffffff, 0x00000100, 689 0xac54, 0xffffffff, 0x00000100, 690 0x897c, 0xffffffff, 0x06000100, 691 0x9868, 0xffffffff, 0x00000100, 692 0x9510, 0xffffffff, 0x00000100, 693 0xaf04, 0xffffffff, 0x00000100, 694 0xae04, 0xffffffff, 0x00000100, 695 0x949c, 0xffffffff, 0x00000100, 696 0x802c, 0xffffffff, 0xe0000000, 697 0x9160, 0xffffffff, 0x00010000, 698 0x9164, 0xffffffff, 0x00030002, 699 0x9168, 0xffffffff, 0x00040007, 700 0x916c, 0xffffffff, 0x00060005, 701 0x9170, 0xffffffff, 0x00090008, 702 0x9174, 0xffffffff, 0x00020001, 703 0x9178, 0xffffffff, 0x00040003, 704 0x917c, 0xffffffff, 0x00000007, 705 0x9180, 0xffffffff, 0x00060005, 706 0x9184, 0xffffffff, 0x00090008, 707 0x9188, 0xffffffff, 0x00030002, 708 0x918c, 0xffffffff, 0x00050004, 709 0x9190, 0xffffffff, 0x00000008, 710 0x9194, 0xffffffff, 0x00070006, 711 0x9198, 0xffffffff, 0x000a0009, 712 0x919c, 0xffffffff, 0x00040003, 713 0x91a0, 0xffffffff, 0x00060005, 714 0x91a4, 0xffffffff, 0x00000009, 715 0x91a8, 0xffffffff, 0x00080007, 716 0x91ac, 0xffffffff, 0x000b000a, 717 0x91b0, 0xffffffff, 0x00050004, 718 0x91b4, 0xffffffff, 0x00070006, 719 0x91b8, 0xffffffff, 0x0008000b, 720 0x91bc, 0xffffffff, 0x000a0009, 721 0x91c0, 0xffffffff, 0x000d000c, 722 0x9200, 0xffffffff, 0x00090008, 723 0x9204, 0xffffffff, 0x000b000a, 724 0x9208, 0xffffffff, 0x000c000f, 725 0x920c, 0xffffffff, 0x000e000d, 726 0x9210, 0xffffffff, 0x00110010, 727 0x9214, 0xffffffff, 0x000a0009, 728 0x9218, 0xffffffff, 0x000c000b, 729 0x921c, 0xffffffff, 0x0000000f, 730 0x9220, 0xffffffff, 0x000e000d, 731 0x9224, 0xffffffff, 0x00110010, 732 0x9228, 0xffffffff, 0x000b000a, 733 0x922c, 0xffffffff, 0x000d000c, 734 0x9230, 0xffffffff, 0x00000010, 735 0x9234, 0xffffffff, 0x000f000e, 736 0x9238, 0xffffffff, 0x00120011, 737 0x923c, 0xffffffff, 0x000c000b, 738 0x9240, 0xffffffff, 0x000e000d, 739 0x9244, 0xffffffff, 0x00000011, 740 0x9248, 0xffffffff, 0x0010000f, 741 0x924c, 0xffffffff, 0x00130012, 742 0x9250, 0xffffffff, 0x000d000c, 743 0x9254, 0xffffffff, 0x000f000e, 744 0x9258, 0xffffffff, 0x00100013, 745 0x925c, 0xffffffff, 0x00120011, 746 0x9260, 0xffffffff, 0x00150014, 747 0x9150, 0xffffffff, 0x96940200, 748 0x8708, 0xffffffff, 0x00900100, 749 0xc478, 0xffffffff, 0x00000080, 750 0xc404, 0xffffffff, 0x0020003f, 751 0x30, 0xffffffff, 0x0000001c, 752 0x34, 0x000f0000, 0x000f0000, 753 0x160c, 0xffffffff, 0x00000100, 754 0x1024, 0xffffffff, 0x00000100, 755 0x102c, 0x00000101, 0x00000000, 756 0x20a8, 0xffffffff, 0x00000104, 757 0x55e4, 0xff000fff, 0x00000100, 758 0x55e8, 0x00000001, 0x00000001, 759 0x2f50, 0x00000001, 0x00000001, 760 0x30cc, 0xc0000fff, 0x00000104, 761 0xc1e4, 0x00000001, 0x00000001, 762 0xd0c0, 0xfffffff0, 0x00000100, 763 0xd8c0, 0xfffffff0, 0x00000100 764 }; 765 766 static const u32 verde_mgcg_cgcg_init[] = 767 { 768 0xc400, 0xffffffff, 0xfffffffc, 769 0x802c, 0xffffffff, 0xe0000000, 770 0x9a60, 0xffffffff, 0x00000100, 771 0x92a4, 0xffffffff, 0x00000100, 772 0xc164, 0xffffffff, 0x00000100, 773 0x9774, 0xffffffff, 0x00000100, 774 0x8984, 0xffffffff, 0x06000100, 775 0x8a18, 0xffffffff, 0x00000100, 776 0x92a0, 0xffffffff, 0x00000100, 777 0xc380, 0xffffffff, 0x00000100, 778 0x8b28, 0xffffffff, 0x00000100, 779 0x9144, 0xffffffff, 0x00000100, 780 0x8d88, 0xffffffff, 0x00000100, 781 0x8d8c, 0xffffffff, 0x00000100, 782 0x9030, 0xffffffff, 0x00000100, 783 0x9034, 0xffffffff, 0x00000100, 784 0x9038, 0xffffffff, 0x00000100, 785 0x903c, 0xffffffff, 0x00000100, 786 0xad80, 0xffffffff, 0x00000100, 787 0xac54, 0xffffffff, 0x00000100, 788 0x897c, 0xffffffff, 0x06000100, 789 0x9868, 0xffffffff, 0x00000100, 790 0x9510, 0xffffffff, 0x00000100, 791 0xaf04, 0xffffffff, 0x00000100, 792 0xae04, 0xffffffff, 0x00000100, 793 0x949c, 0xffffffff, 0x00000100, 794 0x802c, 0xffffffff, 0xe0000000, 795 0x9160, 0xffffffff, 0x00010000, 796 0x9164, 0xffffffff, 0x00030002, 797 0x9168, 0xffffffff, 0x00040007, 798 0x916c, 0xffffffff, 0x00060005, 799 0x9170, 0xffffffff, 0x00090008, 800 0x9174, 0xffffffff, 0x00020001, 801 0x9178, 0xffffffff, 0x00040003, 802 0x917c, 0xffffffff, 0x00000007, 803 0x9180, 0xffffffff, 0x00060005, 804 0x9184, 0xffffffff, 0x00090008, 805 0x9188, 0xffffffff, 0x00030002, 806 0x918c, 0xffffffff, 0x00050004, 807 0x9190, 0xffffffff, 0x00000008, 808 0x9194, 0xffffffff, 0x00070006, 809 0x9198, 0xffffffff, 0x000a0009, 810 0x919c, 0xffffffff, 0x00040003, 811 0x91a0, 0xffffffff, 0x00060005, 812 0x91a4, 0xffffffff, 0x00000009, 813 0x91a8, 0xffffffff, 0x00080007, 814 0x91ac, 0xffffffff, 0x000b000a, 815 0x91b0, 0xffffffff, 0x00050004, 816 0x91b4, 0xffffffff, 0x00070006, 817 0x91b8, 0xffffffff, 0x0008000b, 818 0x91bc, 0xffffffff, 0x000a0009, 819 0x91c0, 0xffffffff, 0x000d000c, 820 0x9200, 0xffffffff, 0x00090008, 821 0x9204, 0xffffffff, 0x000b000a, 822 0x9208, 0xffffffff, 0x000c000f, 823 0x920c, 0xffffffff, 0x000e000d, 824 0x9210, 0xffffffff, 0x00110010, 825 0x9214, 0xffffffff, 0x000a0009, 826 0x9218, 0xffffffff, 0x000c000b, 827 0x921c, 0xffffffff, 0x0000000f, 828 0x9220, 0xffffffff, 0x000e000d, 829 0x9224, 0xffffffff, 0x00110010, 830 0x9228, 0xffffffff, 0x000b000a, 831 0x922c, 0xffffffff, 0x000d000c, 832 0x9230, 0xffffffff, 0x00000010, 833 0x9234, 0xffffffff, 0x000f000e, 834 0x9238, 0xffffffff, 0x00120011, 835 0x923c, 0xffffffff, 0x000c000b, 836 0x9240, 0xffffffff, 0x000e000d, 837 0x9244, 0xffffffff, 0x00000011, 838 0x9248, 0xffffffff, 0x0010000f, 839 0x924c, 0xffffffff, 0x00130012, 840 0x9250, 0xffffffff, 0x000d000c, 841 0x9254, 0xffffffff, 0x000f000e, 842 0x9258, 0xffffffff, 0x00100013, 843 0x925c, 0xffffffff, 0x00120011, 844 0x9260, 0xffffffff, 0x00150014, 845 0x9150, 0xffffffff, 0x96940200, 846 0x8708, 0xffffffff, 0x00900100, 847 0xc478, 0xffffffff, 0x00000080, 848 0xc404, 0xffffffff, 0x0020003f, 849 0x30, 0xffffffff, 0x0000001c, 850 0x34, 0x000f0000, 0x000f0000, 851 0x160c, 0xffffffff, 0x00000100, 852 0x1024, 0xffffffff, 0x00000100, 853 0x102c, 0x00000101, 0x00000000, 854 0x20a8, 0xffffffff, 0x00000104, 855 0x264c, 0x000c0000, 0x000c0000, 856 0x2648, 0x000c0000, 0x000c0000, 857 0x55e4, 0xff000fff, 0x00000100, 858 0x55e8, 0x00000001, 0x00000001, 859 0x2f50, 0x00000001, 0x00000001, 860 0x30cc, 0xc0000fff, 0x00000104, 861 0xc1e4, 0x00000001, 0x00000001, 862 0xd0c0, 0xfffffff0, 0x00000100, 863 0xd8c0, 0xfffffff0, 0x00000100 864 }; 865 866 static const u32 oland_mgcg_cgcg_init[] = 867 { 868 0xc400, 0xffffffff, 0xfffffffc, 869 0x802c, 0xffffffff, 0xe0000000, 870 0x9a60, 0xffffffff, 0x00000100, 871 0x92a4, 0xffffffff, 0x00000100, 872 0xc164, 0xffffffff, 0x00000100, 873 0x9774, 0xffffffff, 0x00000100, 874 0x8984, 0xffffffff, 0x06000100, 875 0x8a18, 0xffffffff, 0x00000100, 876 0x92a0, 0xffffffff, 0x00000100, 877 0xc380, 0xffffffff, 0x00000100, 878 0x8b28, 0xffffffff, 0x00000100, 879 0x9144, 0xffffffff, 0x00000100, 880 0x8d88, 0xffffffff, 0x00000100, 881 0x8d8c, 0xffffffff, 0x00000100, 882 0x9030, 0xffffffff, 0x00000100, 883 0x9034, 0xffffffff, 0x00000100, 884 0x9038, 0xffffffff, 0x00000100, 885 0x903c, 0xffffffff, 0x00000100, 886 0xad80, 0xffffffff, 0x00000100, 887 0xac54, 0xffffffff, 0x00000100, 888 0x897c, 0xffffffff, 0x06000100, 889 0x9868, 0xffffffff, 0x00000100, 890 0x9510, 0xffffffff, 0x00000100, 891 0xaf04, 0xffffffff, 0x00000100, 892 0xae04, 0xffffffff, 0x00000100, 893 0x949c, 0xffffffff, 0x00000100, 894 0x802c, 0xffffffff, 0xe0000000, 895 0x9160, 0xffffffff, 0x00010000, 896 0x9164, 0xffffffff, 0x00030002, 897 0x9168, 0xffffffff, 0x00040007, 898 0x916c, 0xffffffff, 0x00060005, 899 0x9170, 0xffffffff, 0x00090008, 900 0x9174, 0xffffffff, 0x00020001, 901 0x9178, 0xffffffff, 0x00040003, 902 0x917c, 0xffffffff, 0x00000007, 903 0x9180, 0xffffffff, 0x00060005, 904 0x9184, 0xffffffff, 0x00090008, 905 0x9188, 0xffffffff, 0x00030002, 906 0x918c, 0xffffffff, 0x00050004, 907 0x9190, 0xffffffff, 0x00000008, 908 0x9194, 0xffffffff, 0x00070006, 909 0x9198, 0xffffffff, 0x000a0009, 910 0x919c, 0xffffffff, 0x00040003, 911 0x91a0, 0xffffffff, 0x00060005, 912 0x91a4, 0xffffffff, 0x00000009, 913 0x91a8, 0xffffffff, 0x00080007, 914 0x91ac, 0xffffffff, 0x000b000a, 915 0x91b0, 0xffffffff, 0x00050004, 916 0x91b4, 0xffffffff, 0x00070006, 917 0x91b8, 0xffffffff, 0x0008000b, 918 0x91bc, 0xffffffff, 0x000a0009, 919 0x91c0, 0xffffffff, 0x000d000c, 920 0x91c4, 0xffffffff, 0x00060005, 921 0x91c8, 0xffffffff, 0x00080007, 922 0x91cc, 0xffffffff, 0x0000000b, 923 0x91d0, 0xffffffff, 0x000a0009, 924 0x91d4, 0xffffffff, 0x000d000c, 925 0x9150, 0xffffffff, 0x96940200, 926 0x8708, 0xffffffff, 0x00900100, 927 0xc478, 0xffffffff, 0x00000080, 928 0xc404, 0xffffffff, 0x0020003f, 929 0x30, 0xffffffff, 0x0000001c, 930 0x34, 0x000f0000, 0x000f0000, 931 0x160c, 0xffffffff, 0x00000100, 932 0x1024, 0xffffffff, 0x00000100, 933 0x102c, 0x00000101, 0x00000000, 934 0x20a8, 0xffffffff, 0x00000104, 935 0x264c, 0x000c0000, 0x000c0000, 936 0x2648, 0x000c0000, 0x000c0000, 937 0x55e4, 0xff000fff, 0x00000100, 938 0x55e8, 0x00000001, 0x00000001, 939 0x2f50, 0x00000001, 0x00000001, 940 0x30cc, 0xc0000fff, 0x00000104, 941 0xc1e4, 0x00000001, 0x00000001, 942 0xd0c0, 0xfffffff0, 0x00000100, 943 0xd8c0, 0xfffffff0, 0x00000100 944 }; 945 946 static const u32 hainan_mgcg_cgcg_init[] = 947 { 948 0xc400, 0xffffffff, 0xfffffffc, 949 0x802c, 0xffffffff, 0xe0000000, 950 0x9a60, 0xffffffff, 0x00000100, 951 0x92a4, 0xffffffff, 0x00000100, 952 0xc164, 0xffffffff, 0x00000100, 953 0x9774, 0xffffffff, 0x00000100, 954 0x8984, 0xffffffff, 0x06000100, 955 0x8a18, 0xffffffff, 0x00000100, 956 0x92a0, 0xffffffff, 0x00000100, 957 0xc380, 0xffffffff, 0x00000100, 958 0x8b28, 0xffffffff, 0x00000100, 959 0x9144, 0xffffffff, 0x00000100, 960 0x8d88, 0xffffffff, 0x00000100, 961 0x8d8c, 0xffffffff, 0x00000100, 962 0x9030, 0xffffffff, 0x00000100, 963 0x9034, 0xffffffff, 0x00000100, 964 0x9038, 0xffffffff, 0x00000100, 965 0x903c, 0xffffffff, 0x00000100, 966 0xad80, 0xffffffff, 0x00000100, 967 0xac54, 0xffffffff, 0x00000100, 968 0x897c, 0xffffffff, 0x06000100, 969 0x9868, 0xffffffff, 0x00000100, 970 0x9510, 0xffffffff, 0x00000100, 971 0xaf04, 0xffffffff, 0x00000100, 972 0xae04, 0xffffffff, 0x00000100, 973 0x949c, 0xffffffff, 0x00000100, 974 0x802c, 0xffffffff, 0xe0000000, 975 0x9160, 0xffffffff, 0x00010000, 976 0x9164, 0xffffffff, 0x00030002, 977 0x9168, 0xffffffff, 0x00040007, 978 0x916c, 0xffffffff, 0x00060005, 979 0x9170, 0xffffffff, 0x00090008, 980 0x9174, 0xffffffff, 0x00020001, 981 0x9178, 0xffffffff, 0x00040003, 982 0x917c, 0xffffffff, 0x00000007, 983 0x9180, 0xffffffff, 0x00060005, 984 0x9184, 0xffffffff, 0x00090008, 985 0x9188, 0xffffffff, 0x00030002, 986 0x918c, 0xffffffff, 0x00050004, 987 0x9190, 0xffffffff, 0x00000008, 988 0x9194, 0xffffffff, 0x00070006, 989 0x9198, 0xffffffff, 0x000a0009, 990 0x919c, 0xffffffff, 0x00040003, 991 0x91a0, 0xffffffff, 0x00060005, 992 0x91a4, 0xffffffff, 0x00000009, 993 0x91a8, 0xffffffff, 0x00080007, 994 0x91ac, 0xffffffff, 0x000b000a, 995 0x91b0, 0xffffffff, 0x00050004, 996 0x91b4, 0xffffffff, 0x00070006, 997 0x91b8, 0xffffffff, 0x0008000b, 998 0x91bc, 0xffffffff, 0x000a0009, 999 0x91c0, 0xffffffff, 0x000d000c, 1000 0x91c4, 0xffffffff, 0x00060005, 1001 0x91c8, 0xffffffff, 0x00080007, 1002 0x91cc, 0xffffffff, 0x0000000b, 1003 0x91d0, 0xffffffff, 0x000a0009, 1004 0x91d4, 0xffffffff, 0x000d000c, 1005 0x9150, 0xffffffff, 0x96940200, 1006 0x8708, 0xffffffff, 0x00900100, 1007 0xc478, 0xffffffff, 0x00000080, 1008 0xc404, 0xffffffff, 0x0020003f, 1009 0x30, 0xffffffff, 0x0000001c, 1010 0x34, 0x000f0000, 0x000f0000, 1011 0x160c, 0xffffffff, 0x00000100, 1012 0x1024, 0xffffffff, 0x00000100, 1013 0x20a8, 0xffffffff, 0x00000104, 1014 0x264c, 0x000c0000, 0x000c0000, 1015 0x2648, 0x000c0000, 0x000c0000, 1016 0x2f50, 0x00000001, 0x00000001, 1017 0x30cc, 0xc0000fff, 0x00000104, 1018 0xc1e4, 0x00000001, 0x00000001, 1019 0xd0c0, 0xfffffff0, 0x00000100, 1020 0xd8c0, 0xfffffff0, 0x00000100 1021 }; 1022 1023 static u32 verde_pg_init[] = 1024 { 1025 0x353c, 0xffffffff, 0x40000, 1026 0x3538, 0xffffffff, 0x200010ff, 1027 0x353c, 0xffffffff, 0x0, 1028 0x353c, 0xffffffff, 0x0, 1029 0x353c, 0xffffffff, 0x0, 1030 0x353c, 0xffffffff, 0x0, 1031 0x353c, 0xffffffff, 0x0, 1032 0x353c, 0xffffffff, 0x7007, 1033 0x3538, 0xffffffff, 0x300010ff, 1034 0x353c, 0xffffffff, 0x0, 1035 0x353c, 0xffffffff, 0x0, 1036 0x353c, 0xffffffff, 0x0, 1037 0x353c, 0xffffffff, 0x0, 1038 0x353c, 0xffffffff, 0x0, 1039 0x353c, 0xffffffff, 0x400000, 1040 0x3538, 0xffffffff, 0x100010ff, 1041 0x353c, 0xffffffff, 0x0, 1042 0x353c, 0xffffffff, 0x0, 1043 0x353c, 0xffffffff, 0x0, 1044 0x353c, 0xffffffff, 0x0, 1045 0x353c, 0xffffffff, 0x0, 1046 0x353c, 0xffffffff, 0x120200, 1047 0x3538, 0xffffffff, 0x500010ff, 1048 0x353c, 0xffffffff, 0x0, 1049 0x353c, 0xffffffff, 0x0, 1050 0x353c, 0xffffffff, 0x0, 1051 0x353c, 0xffffffff, 0x0, 1052 0x353c, 0xffffffff, 0x0, 1053 0x353c, 0xffffffff, 0x1e1e16, 1054 0x3538, 0xffffffff, 0x600010ff, 1055 0x353c, 0xffffffff, 0x0, 1056 0x353c, 0xffffffff, 0x0, 1057 0x353c, 0xffffffff, 0x0, 1058 0x353c, 0xffffffff, 0x0, 1059 0x353c, 0xffffffff, 0x0, 1060 0x353c, 0xffffffff, 0x171f1e, 1061 0x3538, 0xffffffff, 0x700010ff, 1062 0x353c, 0xffffffff, 0x0, 1063 0x353c, 0xffffffff, 0x0, 1064 0x353c, 0xffffffff, 0x0, 1065 0x353c, 0xffffffff, 0x0, 1066 0x353c, 0xffffffff, 0x0, 1067 0x353c, 0xffffffff, 0x0, 1068 0x3538, 0xffffffff, 0x9ff, 1069 0x3500, 0xffffffff, 0x0, 1070 0x3504, 0xffffffff, 0x10000800, 1071 0x3504, 0xffffffff, 0xf, 1072 0x3504, 0xffffffff, 0xf, 1073 0x3500, 0xffffffff, 0x4, 1074 0x3504, 0xffffffff, 0x1000051e, 1075 0x3504, 0xffffffff, 0xffff, 1076 0x3504, 0xffffffff, 0xffff, 1077 0x3500, 0xffffffff, 0x8, 1078 0x3504, 0xffffffff, 0x80500, 1079 0x3500, 0xffffffff, 0x12, 1080 0x3504, 0xffffffff, 0x9050c, 1081 0x3500, 0xffffffff, 0x1d, 1082 0x3504, 0xffffffff, 0xb052c, 1083 0x3500, 0xffffffff, 0x2a, 1084 0x3504, 0xffffffff, 0x1053e, 1085 0x3500, 0xffffffff, 0x2d, 1086 0x3504, 0xffffffff, 0x10546, 1087 0x3500, 0xffffffff, 0x30, 1088 0x3504, 0xffffffff, 0xa054e, 1089 0x3500, 0xffffffff, 0x3c, 1090 0x3504, 0xffffffff, 0x1055f, 1091 0x3500, 0xffffffff, 0x3f, 1092 0x3504, 0xffffffff, 0x10567, 1093 0x3500, 0xffffffff, 0x42, 1094 0x3504, 0xffffffff, 0x1056f, 1095 0x3500, 0xffffffff, 0x45, 1096 0x3504, 0xffffffff, 0x10572, 1097 0x3500, 0xffffffff, 0x48, 1098 0x3504, 0xffffffff, 0x20575, 1099 0x3500, 0xffffffff, 0x4c, 1100 0x3504, 0xffffffff, 0x190801, 1101 0x3500, 0xffffffff, 0x67, 1102 0x3504, 0xffffffff, 0x1082a, 1103 0x3500, 0xffffffff, 0x6a, 1104 0x3504, 0xffffffff, 0x1b082d, 1105 0x3500, 0xffffffff, 0x87, 1106 0x3504, 0xffffffff, 0x310851, 1107 0x3500, 0xffffffff, 0xba, 1108 0x3504, 0xffffffff, 0x891, 1109 0x3500, 0xffffffff, 0xbc, 1110 0x3504, 0xffffffff, 0x893, 1111 0x3500, 0xffffffff, 0xbe, 1112 0x3504, 0xffffffff, 0x20895, 1113 0x3500, 0xffffffff, 0xc2, 1114 0x3504, 0xffffffff, 0x20899, 1115 0x3500, 0xffffffff, 0xc6, 1116 0x3504, 0xffffffff, 0x2089d, 1117 0x3500, 0xffffffff, 0xca, 1118 0x3504, 0xffffffff, 0x8a1, 1119 0x3500, 0xffffffff, 0xcc, 1120 0x3504, 0xffffffff, 0x8a3, 1121 0x3500, 0xffffffff, 0xce, 1122 0x3504, 0xffffffff, 0x308a5, 1123 0x3500, 0xffffffff, 0xd3, 1124 0x3504, 0xffffffff, 0x6d08cd, 1125 0x3500, 0xffffffff, 0x142, 1126 0x3504, 0xffffffff, 0x2000095a, 1127 0x3504, 0xffffffff, 0x1, 1128 0x3500, 0xffffffff, 0x144, 1129 0x3504, 0xffffffff, 0x301f095b, 1130 0x3500, 0xffffffff, 0x165, 1131 0x3504, 0xffffffff, 0xc094d, 1132 0x3500, 0xffffffff, 0x173, 1133 0x3504, 0xffffffff, 0xf096d, 1134 0x3500, 0xffffffff, 0x184, 1135 0x3504, 0xffffffff, 0x15097f, 1136 0x3500, 0xffffffff, 0x19b, 1137 0x3504, 0xffffffff, 0xc0998, 1138 0x3500, 0xffffffff, 0x1a9, 1139 0x3504, 0xffffffff, 0x409a7, 1140 0x3500, 0xffffffff, 0x1af, 1141 0x3504, 0xffffffff, 0xcdc, 1142 0x3500, 0xffffffff, 0x1b1, 1143 0x3504, 0xffffffff, 0x800, 1144 0x3508, 0xffffffff, 0x6c9b2000, 1145 0x3510, 0xfc00, 0x2000, 1146 0x3544, 0xffffffff, 0xfc0, 1147 0x28d4, 0x00000100, 0x100 1148 }; 1149 1150 static void si_init_golden_registers(struct radeon_device *rdev) 1151 { 1152 switch (rdev->family) { 1153 case CHIP_TAHITI: 1154 radeon_program_register_sequence(rdev, 1155 tahiti_golden_registers, 1156 (const u32)ARRAY_SIZE(tahiti_golden_registers)); 1157 radeon_program_register_sequence(rdev, 1158 tahiti_golden_rlc_registers, 1159 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers)); 1160 radeon_program_register_sequence(rdev, 1161 tahiti_mgcg_cgcg_init, 1162 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init)); 1163 radeon_program_register_sequence(rdev, 1164 tahiti_golden_registers2, 1165 (const u32)ARRAY_SIZE(tahiti_golden_registers2)); 1166 break; 1167 case CHIP_PITCAIRN: 1168 radeon_program_register_sequence(rdev, 1169 pitcairn_golden_registers, 1170 (const u32)ARRAY_SIZE(pitcairn_golden_registers)); 1171 radeon_program_register_sequence(rdev, 1172 pitcairn_golden_rlc_registers, 1173 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers)); 1174 radeon_program_register_sequence(rdev, 1175 pitcairn_mgcg_cgcg_init, 1176 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init)); 1177 break; 1178 case CHIP_VERDE: 1179 radeon_program_register_sequence(rdev, 1180 verde_golden_registers, 1181 (const u32)ARRAY_SIZE(verde_golden_registers)); 1182 radeon_program_register_sequence(rdev, 1183 verde_golden_rlc_registers, 1184 (const u32)ARRAY_SIZE(verde_golden_rlc_registers)); 1185 radeon_program_register_sequence(rdev, 1186 verde_mgcg_cgcg_init, 1187 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init)); 1188 radeon_program_register_sequence(rdev, 1189 verde_pg_init, 1190 (const u32)ARRAY_SIZE(verde_pg_init)); 1191 break; 1192 case CHIP_OLAND: 1193 radeon_program_register_sequence(rdev, 1194 oland_golden_registers, 1195 (const u32)ARRAY_SIZE(oland_golden_registers)); 1196 radeon_program_register_sequence(rdev, 1197 oland_golden_rlc_registers, 1198 (const u32)ARRAY_SIZE(oland_golden_rlc_registers)); 1199 radeon_program_register_sequence(rdev, 1200 oland_mgcg_cgcg_init, 1201 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init)); 1202 break; 1203 case CHIP_HAINAN: 1204 radeon_program_register_sequence(rdev, 1205 hainan_golden_registers, 1206 (const u32)ARRAY_SIZE(hainan_golden_registers)); 1207 radeon_program_register_sequence(rdev, 1208 hainan_golden_registers2, 1209 (const u32)ARRAY_SIZE(hainan_golden_registers2)); 1210 radeon_program_register_sequence(rdev, 1211 hainan_mgcg_cgcg_init, 1212 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init)); 1213 break; 1214 default: 1215 break; 1216 } 1217 } 1218 1219 #define PCIE_BUS_CLK 10000 1220 #define TCLK (PCIE_BUS_CLK / 10) 1221 1222 /** 1223 * si_get_xclk - get the xclk 1224 * 1225 * @rdev: radeon_device pointer 1226 * 1227 * Returns the reference clock used by the gfx engine 1228 * (SI). 1229 */ 1230 u32 si_get_xclk(struct radeon_device *rdev) 1231 { 1232 u32 reference_clock = rdev->clock.spll.reference_freq; 1233 u32 tmp; 1234 1235 tmp = RREG32(CG_CLKPIN_CNTL_2); 1236 if (tmp & MUX_TCLK_TO_XCLK) 1237 return TCLK; 1238 1239 tmp = RREG32(CG_CLKPIN_CNTL); 1240 if (tmp & XTALIN_DIVIDE) 1241 return reference_clock / 4; 1242 1243 return reference_clock; 1244 } 1245 1246 /* get temperature in millidegrees */ 1247 int si_get_temp(struct radeon_device *rdev) 1248 { 1249 u32 temp; 1250 int actual_temp = 0; 1251 1252 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >> 1253 CTF_TEMP_SHIFT; 1254 1255 if (temp & 0x200) 1256 actual_temp = 255; 1257 else 1258 actual_temp = temp & 0x1ff; 1259 1260 actual_temp = (actual_temp * 1000); 1261 1262 return actual_temp; 1263 } 1264 1265 #define TAHITI_IO_MC_REGS_SIZE 36 1266 1267 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1268 {0x0000006f, 0x03044000}, 1269 {0x00000070, 0x0480c018}, 1270 {0x00000071, 0x00000040}, 1271 {0x00000072, 0x01000000}, 1272 {0x00000074, 0x000000ff}, 1273 {0x00000075, 0x00143400}, 1274 {0x00000076, 0x08ec0800}, 1275 {0x00000077, 0x040000cc}, 1276 {0x00000079, 0x00000000}, 1277 {0x0000007a, 0x21000409}, 1278 {0x0000007c, 0x00000000}, 1279 {0x0000007d, 0xe8000000}, 1280 {0x0000007e, 0x044408a8}, 1281 {0x0000007f, 0x00000003}, 1282 {0x00000080, 0x00000000}, 1283 {0x00000081, 0x01000000}, 1284 {0x00000082, 0x02000000}, 1285 {0x00000083, 0x00000000}, 1286 {0x00000084, 0xe3f3e4f4}, 1287 {0x00000085, 0x00052024}, 1288 {0x00000087, 0x00000000}, 1289 {0x00000088, 0x66036603}, 1290 {0x00000089, 0x01000000}, 1291 {0x0000008b, 0x1c0a0000}, 1292 {0x0000008c, 0xff010000}, 1293 {0x0000008e, 0xffffefff}, 1294 {0x0000008f, 0xfff3efff}, 1295 {0x00000090, 0xfff3efbf}, 1296 {0x00000094, 0x00101101}, 1297 {0x00000095, 0x00000fff}, 1298 {0x00000096, 0x00116fff}, 1299 {0x00000097, 0x60010000}, 1300 {0x00000098, 0x10010000}, 1301 {0x00000099, 0x00006000}, 1302 {0x0000009a, 0x00001000}, 1303 {0x0000009f, 0x00a77400} 1304 }; 1305 1306 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1307 {0x0000006f, 0x03044000}, 1308 {0x00000070, 0x0480c018}, 1309 {0x00000071, 0x00000040}, 1310 {0x00000072, 0x01000000}, 1311 {0x00000074, 0x000000ff}, 1312 {0x00000075, 0x00143400}, 1313 {0x00000076, 0x08ec0800}, 1314 {0x00000077, 0x040000cc}, 1315 {0x00000079, 0x00000000}, 1316 {0x0000007a, 0x21000409}, 1317 {0x0000007c, 0x00000000}, 1318 {0x0000007d, 0xe8000000}, 1319 {0x0000007e, 0x044408a8}, 1320 {0x0000007f, 0x00000003}, 1321 {0x00000080, 0x00000000}, 1322 {0x00000081, 0x01000000}, 1323 {0x00000082, 0x02000000}, 1324 {0x00000083, 0x00000000}, 1325 {0x00000084, 0xe3f3e4f4}, 1326 {0x00000085, 0x00052024}, 1327 {0x00000087, 0x00000000}, 1328 {0x00000088, 0x66036603}, 1329 {0x00000089, 0x01000000}, 1330 {0x0000008b, 0x1c0a0000}, 1331 {0x0000008c, 0xff010000}, 1332 {0x0000008e, 0xffffefff}, 1333 {0x0000008f, 0xfff3efff}, 1334 {0x00000090, 0xfff3efbf}, 1335 {0x00000094, 0x00101101}, 1336 {0x00000095, 0x00000fff}, 1337 {0x00000096, 0x00116fff}, 1338 {0x00000097, 0x60010000}, 1339 {0x00000098, 0x10010000}, 1340 {0x00000099, 0x00006000}, 1341 {0x0000009a, 0x00001000}, 1342 {0x0000009f, 0x00a47400} 1343 }; 1344 1345 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1346 {0x0000006f, 0x03044000}, 1347 {0x00000070, 0x0480c018}, 1348 {0x00000071, 0x00000040}, 1349 {0x00000072, 0x01000000}, 1350 {0x00000074, 0x000000ff}, 1351 {0x00000075, 0x00143400}, 1352 {0x00000076, 0x08ec0800}, 1353 {0x00000077, 0x040000cc}, 1354 {0x00000079, 0x00000000}, 1355 {0x0000007a, 0x21000409}, 1356 {0x0000007c, 0x00000000}, 1357 {0x0000007d, 0xe8000000}, 1358 {0x0000007e, 0x044408a8}, 1359 {0x0000007f, 0x00000003}, 1360 {0x00000080, 0x00000000}, 1361 {0x00000081, 0x01000000}, 1362 {0x00000082, 0x02000000}, 1363 {0x00000083, 0x00000000}, 1364 {0x00000084, 0xe3f3e4f4}, 1365 {0x00000085, 0x00052024}, 1366 {0x00000087, 0x00000000}, 1367 {0x00000088, 0x66036603}, 1368 {0x00000089, 0x01000000}, 1369 {0x0000008b, 0x1c0a0000}, 1370 {0x0000008c, 0xff010000}, 1371 {0x0000008e, 0xffffefff}, 1372 {0x0000008f, 0xfff3efff}, 1373 {0x00000090, 0xfff3efbf}, 1374 {0x00000094, 0x00101101}, 1375 {0x00000095, 0x00000fff}, 1376 {0x00000096, 0x00116fff}, 1377 {0x00000097, 0x60010000}, 1378 {0x00000098, 0x10010000}, 1379 {0x00000099, 0x00006000}, 1380 {0x0000009a, 0x00001000}, 1381 {0x0000009f, 0x00a37400} 1382 }; 1383 1384 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1385 {0x0000006f, 0x03044000}, 1386 {0x00000070, 0x0480c018}, 1387 {0x00000071, 0x00000040}, 1388 {0x00000072, 0x01000000}, 1389 {0x00000074, 0x000000ff}, 1390 {0x00000075, 0x00143400}, 1391 {0x00000076, 0x08ec0800}, 1392 {0x00000077, 0x040000cc}, 1393 {0x00000079, 0x00000000}, 1394 {0x0000007a, 0x21000409}, 1395 {0x0000007c, 0x00000000}, 1396 {0x0000007d, 0xe8000000}, 1397 {0x0000007e, 0x044408a8}, 1398 {0x0000007f, 0x00000003}, 1399 {0x00000080, 0x00000000}, 1400 {0x00000081, 0x01000000}, 1401 {0x00000082, 0x02000000}, 1402 {0x00000083, 0x00000000}, 1403 {0x00000084, 0xe3f3e4f4}, 1404 {0x00000085, 0x00052024}, 1405 {0x00000087, 0x00000000}, 1406 {0x00000088, 0x66036603}, 1407 {0x00000089, 0x01000000}, 1408 {0x0000008b, 0x1c0a0000}, 1409 {0x0000008c, 0xff010000}, 1410 {0x0000008e, 0xffffefff}, 1411 {0x0000008f, 0xfff3efff}, 1412 {0x00000090, 0xfff3efbf}, 1413 {0x00000094, 0x00101101}, 1414 {0x00000095, 0x00000fff}, 1415 {0x00000096, 0x00116fff}, 1416 {0x00000097, 0x60010000}, 1417 {0x00000098, 0x10010000}, 1418 {0x00000099, 0x00006000}, 1419 {0x0000009a, 0x00001000}, 1420 {0x0000009f, 0x00a17730} 1421 }; 1422 1423 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = { 1424 {0x0000006f, 0x03044000}, 1425 {0x00000070, 0x0480c018}, 1426 {0x00000071, 0x00000040}, 1427 {0x00000072, 0x01000000}, 1428 {0x00000074, 0x000000ff}, 1429 {0x00000075, 0x00143400}, 1430 {0x00000076, 0x08ec0800}, 1431 {0x00000077, 0x040000cc}, 1432 {0x00000079, 0x00000000}, 1433 {0x0000007a, 0x21000409}, 1434 {0x0000007c, 0x00000000}, 1435 {0x0000007d, 0xe8000000}, 1436 {0x0000007e, 0x044408a8}, 1437 {0x0000007f, 0x00000003}, 1438 {0x00000080, 0x00000000}, 1439 {0x00000081, 0x01000000}, 1440 {0x00000082, 0x02000000}, 1441 {0x00000083, 0x00000000}, 1442 {0x00000084, 0xe3f3e4f4}, 1443 {0x00000085, 0x00052024}, 1444 {0x00000087, 0x00000000}, 1445 {0x00000088, 0x66036603}, 1446 {0x00000089, 0x01000000}, 1447 {0x0000008b, 0x1c0a0000}, 1448 {0x0000008c, 0xff010000}, 1449 {0x0000008e, 0xffffefff}, 1450 {0x0000008f, 0xfff3efff}, 1451 {0x00000090, 0xfff3efbf}, 1452 {0x00000094, 0x00101101}, 1453 {0x00000095, 0x00000fff}, 1454 {0x00000096, 0x00116fff}, 1455 {0x00000097, 0x60010000}, 1456 {0x00000098, 0x10010000}, 1457 {0x00000099, 0x00006000}, 1458 {0x0000009a, 0x00001000}, 1459 {0x0000009f, 0x00a07730} 1460 }; 1461 1462 /* ucode loading */ 1463 static int si_mc_load_microcode(struct radeon_device *rdev) 1464 { 1465 const __be32 *fw_data; 1466 u32 running, blackout = 0; 1467 u32 *io_mc_regs; 1468 int i, ucode_size, regs_size; 1469 1470 if (!rdev->mc_fw) 1471 return -EINVAL; 1472 1473 switch (rdev->family) { 1474 case CHIP_TAHITI: 1475 io_mc_regs = (u32 *)&tahiti_io_mc_regs; 1476 ucode_size = SI_MC_UCODE_SIZE; 1477 regs_size = TAHITI_IO_MC_REGS_SIZE; 1478 break; 1479 case CHIP_PITCAIRN: 1480 io_mc_regs = (u32 *)&pitcairn_io_mc_regs; 1481 ucode_size = SI_MC_UCODE_SIZE; 1482 regs_size = TAHITI_IO_MC_REGS_SIZE; 1483 break; 1484 case CHIP_VERDE: 1485 default: 1486 io_mc_regs = (u32 *)&verde_io_mc_regs; 1487 ucode_size = SI_MC_UCODE_SIZE; 1488 regs_size = TAHITI_IO_MC_REGS_SIZE; 1489 break; 1490 case CHIP_OLAND: 1491 io_mc_regs = (u32 *)&oland_io_mc_regs; 1492 ucode_size = OLAND_MC_UCODE_SIZE; 1493 regs_size = TAHITI_IO_MC_REGS_SIZE; 1494 break; 1495 case CHIP_HAINAN: 1496 io_mc_regs = (u32 *)&hainan_io_mc_regs; 1497 ucode_size = OLAND_MC_UCODE_SIZE; 1498 regs_size = TAHITI_IO_MC_REGS_SIZE; 1499 break; 1500 } 1501 1502 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK; 1503 1504 if (running == 0) { 1505 if (running) { 1506 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL); 1507 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1); 1508 } 1509 1510 /* reset the engine and set to writable */ 1511 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1512 WREG32(MC_SEQ_SUP_CNTL, 0x00000010); 1513 1514 /* load mc io regs */ 1515 for (i = 0; i < regs_size; i++) { 1516 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]); 1517 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]); 1518 } 1519 /* load the MC ucode */ 1520 fw_data = (const __be32 *)rdev->mc_fw->data; 1521 for (i = 0; i < ucode_size; i++) 1522 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++)); 1523 1524 /* put the engine back into the active state */ 1525 WREG32(MC_SEQ_SUP_CNTL, 0x00000008); 1526 WREG32(MC_SEQ_SUP_CNTL, 0x00000004); 1527 WREG32(MC_SEQ_SUP_CNTL, 0x00000001); 1528 1529 /* wait for training to complete */ 1530 for (i = 0; i < rdev->usec_timeout; i++) { 1531 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0) 1532 break; 1533 udelay(1); 1534 } 1535 for (i = 0; i < rdev->usec_timeout; i++) { 1536 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1) 1537 break; 1538 udelay(1); 1539 } 1540 1541 if (running) 1542 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout); 1543 } 1544 1545 return 0; 1546 } 1547 1548 static int si_init_microcode(struct radeon_device *rdev) 1549 { 1550 const char *chip_name; 1551 const char *rlc_chip_name; 1552 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size; 1553 size_t smc_req_size; 1554 char fw_name[30]; 1555 int err; 1556 1557 DRM_DEBUG("\n"); 1558 1559 switch (rdev->family) { 1560 case CHIP_TAHITI: 1561 chip_name = "TAHITI"; 1562 rlc_chip_name = "TAHITI"; 1563 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1564 me_req_size = SI_PM4_UCODE_SIZE * 4; 1565 ce_req_size = SI_CE_UCODE_SIZE * 4; 1566 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1567 mc_req_size = SI_MC_UCODE_SIZE * 4; 1568 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4); 1569 break; 1570 case CHIP_PITCAIRN: 1571 chip_name = "PITCAIRN"; 1572 rlc_chip_name = "PITCAIRN"; 1573 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1574 me_req_size = SI_PM4_UCODE_SIZE * 4; 1575 ce_req_size = SI_CE_UCODE_SIZE * 4; 1576 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1577 mc_req_size = SI_MC_UCODE_SIZE * 4; 1578 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4); 1579 break; 1580 case CHIP_VERDE: 1581 chip_name = "VERDE"; 1582 rlc_chip_name = "VERDE"; 1583 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1584 me_req_size = SI_PM4_UCODE_SIZE * 4; 1585 ce_req_size = SI_CE_UCODE_SIZE * 4; 1586 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1587 mc_req_size = SI_MC_UCODE_SIZE * 4; 1588 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4); 1589 break; 1590 case CHIP_OLAND: 1591 chip_name = "OLAND"; 1592 rlc_chip_name = "OLAND"; 1593 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1594 me_req_size = SI_PM4_UCODE_SIZE * 4; 1595 ce_req_size = SI_CE_UCODE_SIZE * 4; 1596 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1597 mc_req_size = OLAND_MC_UCODE_SIZE * 4; 1598 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4); 1599 break; 1600 case CHIP_HAINAN: 1601 chip_name = "HAINAN"; 1602 rlc_chip_name = "HAINAN"; 1603 pfp_req_size = SI_PFP_UCODE_SIZE * 4; 1604 me_req_size = SI_PM4_UCODE_SIZE * 4; 1605 ce_req_size = SI_CE_UCODE_SIZE * 4; 1606 rlc_req_size = SI_RLC_UCODE_SIZE * 4; 1607 mc_req_size = OLAND_MC_UCODE_SIZE * 4; 1608 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4); 1609 break; 1610 default: BUG(); 1611 } 1612 1613 DRM_INFO("Loading %s Microcode\n", chip_name); 1614 1615 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name); 1616 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); 1617 if (err) 1618 goto out; 1619 if (rdev->pfp_fw->size != pfp_req_size) { 1620 printk(KERN_ERR 1621 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1622 rdev->pfp_fw->size, fw_name); 1623 err = -EINVAL; 1624 goto out; 1625 } 1626 1627 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name); 1628 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); 1629 if (err) 1630 goto out; 1631 if (rdev->me_fw->size != me_req_size) { 1632 printk(KERN_ERR 1633 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1634 rdev->me_fw->size, fw_name); 1635 err = -EINVAL; 1636 } 1637 1638 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name); 1639 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); 1640 if (err) 1641 goto out; 1642 if (rdev->ce_fw->size != ce_req_size) { 1643 printk(KERN_ERR 1644 "si_cp: Bogus length %zu in firmware \"%s\"\n", 1645 rdev->ce_fw->size, fw_name); 1646 err = -EINVAL; 1647 } 1648 1649 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name); 1650 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); 1651 if (err) 1652 goto out; 1653 if (rdev->rlc_fw->size != rlc_req_size) { 1654 printk(KERN_ERR 1655 "si_rlc: Bogus length %zu in firmware \"%s\"\n", 1656 rdev->rlc_fw->size, fw_name); 1657 err = -EINVAL; 1658 } 1659 1660 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name); 1661 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); 1662 if (err) 1663 goto out; 1664 if (rdev->mc_fw->size != mc_req_size) { 1665 printk(KERN_ERR 1666 "si_mc: Bogus length %zu in firmware \"%s\"\n", 1667 rdev->mc_fw->size, fw_name); 1668 err = -EINVAL; 1669 } 1670 1671 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name); 1672 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); 1673 if (err) { 1674 printk(KERN_ERR 1675 "smc: error loading firmware \"%s\"\n", 1676 fw_name); 1677 release_firmware(rdev->smc_fw); 1678 rdev->smc_fw = NULL; 1679 } else if (rdev->smc_fw->size != smc_req_size) { 1680 printk(KERN_ERR 1681 "si_smc: Bogus length %zu in firmware \"%s\"\n", 1682 rdev->smc_fw->size, fw_name); 1683 err = -EINVAL; 1684 } 1685 1686 out: 1687 if (err) { 1688 if (err != -EINVAL) 1689 printk(KERN_ERR 1690 "si_cp: Failed to load firmware \"%s\"\n", 1691 fw_name); 1692 release_firmware(rdev->pfp_fw); 1693 rdev->pfp_fw = NULL; 1694 release_firmware(rdev->me_fw); 1695 rdev->me_fw = NULL; 1696 release_firmware(rdev->ce_fw); 1697 rdev->ce_fw = NULL; 1698 release_firmware(rdev->rlc_fw); 1699 rdev->rlc_fw = NULL; 1700 release_firmware(rdev->mc_fw); 1701 rdev->mc_fw = NULL; 1702 release_firmware(rdev->smc_fw); 1703 rdev->smc_fw = NULL; 1704 } 1705 return err; 1706 } 1707 1708 /* watermark setup */ 1709 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev, 1710 struct radeon_crtc *radeon_crtc, 1711 struct drm_display_mode *mode, 1712 struct drm_display_mode *other_mode) 1713 { 1714 u32 tmp, buffer_alloc, i; 1715 u32 pipe_offset = radeon_crtc->crtc_id * 0x20; 1716 /* 1717 * Line Buffer Setup 1718 * There are 3 line buffers, each one shared by 2 display controllers. 1719 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 1720 * the display controllers. The paritioning is done via one of four 1721 * preset allocations specified in bits 21:20: 1722 * 0 - half lb 1723 * 2 - whole lb, other crtc must be disabled 1724 */ 1725 /* this can get tricky if we have two large displays on a paired group 1726 * of crtcs. Ideally for multiple large displays we'd assign them to 1727 * non-linked crtcs for maximum line buffer allocation. 1728 */ 1729 if (radeon_crtc->base.enabled && mode) { 1730 if (other_mode) { 1731 tmp = 0; /* 1/2 */ 1732 buffer_alloc = 1; 1733 } else { 1734 tmp = 2; /* whole */ 1735 buffer_alloc = 2; 1736 } 1737 } else { 1738 tmp = 0; 1739 buffer_alloc = 0; 1740 } 1741 1742 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, 1743 DC_LB_MEMORY_CONFIG(tmp)); 1744 1745 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset, 1746 DMIF_BUFFERS_ALLOCATED(buffer_alloc)); 1747 for (i = 0; i < rdev->usec_timeout; i++) { 1748 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) & 1749 DMIF_BUFFERS_ALLOCATED_COMPLETED) 1750 break; 1751 udelay(1); 1752 } 1753 1754 if (radeon_crtc->base.enabled && mode) { 1755 switch (tmp) { 1756 case 0: 1757 default: 1758 return 4096 * 2; 1759 case 2: 1760 return 8192 * 2; 1761 } 1762 } 1763 1764 /* controller not enabled, so no lb used */ 1765 return 0; 1766 } 1767 1768 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev) 1769 { 1770 u32 tmp = RREG32(MC_SHARED_CHMAP); 1771 1772 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 1773 case 0: 1774 default: 1775 return 1; 1776 case 1: 1777 return 2; 1778 case 2: 1779 return 4; 1780 case 3: 1781 return 8; 1782 case 4: 1783 return 3; 1784 case 5: 1785 return 6; 1786 case 6: 1787 return 10; 1788 case 7: 1789 return 12; 1790 case 8: 1791 return 16; 1792 } 1793 } 1794 1795 struct dce6_wm_params { 1796 u32 dram_channels; /* number of dram channels */ 1797 u32 yclk; /* bandwidth per dram data pin in kHz */ 1798 u32 sclk; /* engine clock in kHz */ 1799 u32 disp_clk; /* display clock in kHz */ 1800 u32 src_width; /* viewport width */ 1801 u32 active_time; /* active display time in ns */ 1802 u32 blank_time; /* blank time in ns */ 1803 bool interlaced; /* mode is interlaced */ 1804 fixed20_12 vsc; /* vertical scale ratio */ 1805 u32 num_heads; /* number of active crtcs */ 1806 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 1807 u32 lb_size; /* line buffer allocated to pipe */ 1808 u32 vtaps; /* vertical scaler taps */ 1809 }; 1810 1811 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm) 1812 { 1813 /* Calculate raw DRAM Bandwidth */ 1814 fixed20_12 dram_efficiency; /* 0.7 */ 1815 fixed20_12 yclk, dram_channels, bandwidth; 1816 fixed20_12 a; 1817 1818 a.full = dfixed_const(1000); 1819 yclk.full = dfixed_const(wm->yclk); 1820 yclk.full = dfixed_div(yclk, a); 1821 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1822 a.full = dfixed_const(10); 1823 dram_efficiency.full = dfixed_const(7); 1824 dram_efficiency.full = dfixed_div(dram_efficiency, a); 1825 bandwidth.full = dfixed_mul(dram_channels, yclk); 1826 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 1827 1828 return dfixed_trunc(bandwidth); 1829 } 1830 1831 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm) 1832 { 1833 /* Calculate DRAM Bandwidth and the part allocated to display. */ 1834 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 1835 fixed20_12 yclk, dram_channels, bandwidth; 1836 fixed20_12 a; 1837 1838 a.full = dfixed_const(1000); 1839 yclk.full = dfixed_const(wm->yclk); 1840 yclk.full = dfixed_div(yclk, a); 1841 dram_channels.full = dfixed_const(wm->dram_channels * 4); 1842 a.full = dfixed_const(10); 1843 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 1844 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 1845 bandwidth.full = dfixed_mul(dram_channels, yclk); 1846 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 1847 1848 return dfixed_trunc(bandwidth); 1849 } 1850 1851 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm) 1852 { 1853 /* Calculate the display Data return Bandwidth */ 1854 fixed20_12 return_efficiency; /* 0.8 */ 1855 fixed20_12 sclk, bandwidth; 1856 fixed20_12 a; 1857 1858 a.full = dfixed_const(1000); 1859 sclk.full = dfixed_const(wm->sclk); 1860 sclk.full = dfixed_div(sclk, a); 1861 a.full = dfixed_const(10); 1862 return_efficiency.full = dfixed_const(8); 1863 return_efficiency.full = dfixed_div(return_efficiency, a); 1864 a.full = dfixed_const(32); 1865 bandwidth.full = dfixed_mul(a, sclk); 1866 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 1867 1868 return dfixed_trunc(bandwidth); 1869 } 1870 1871 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm) 1872 { 1873 return 32; 1874 } 1875 1876 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm) 1877 { 1878 /* Calculate the DMIF Request Bandwidth */ 1879 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 1880 fixed20_12 disp_clk, sclk, bandwidth; 1881 fixed20_12 a, b1, b2; 1882 u32 min_bandwidth; 1883 1884 a.full = dfixed_const(1000); 1885 disp_clk.full = dfixed_const(wm->disp_clk); 1886 disp_clk.full = dfixed_div(disp_clk, a); 1887 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2); 1888 b1.full = dfixed_mul(a, disp_clk); 1889 1890 a.full = dfixed_const(1000); 1891 sclk.full = dfixed_const(wm->sclk); 1892 sclk.full = dfixed_div(sclk, a); 1893 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm)); 1894 b2.full = dfixed_mul(a, sclk); 1895 1896 a.full = dfixed_const(10); 1897 disp_clk_request_efficiency.full = dfixed_const(8); 1898 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 1899 1900 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2)); 1901 1902 a.full = dfixed_const(min_bandwidth); 1903 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency); 1904 1905 return dfixed_trunc(bandwidth); 1906 } 1907 1908 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm) 1909 { 1910 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 1911 u32 dram_bandwidth = dce6_dram_bandwidth(wm); 1912 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm); 1913 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm); 1914 1915 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 1916 } 1917 1918 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm) 1919 { 1920 /* Calculate the display mode Average Bandwidth 1921 * DisplayMode should contain the source and destination dimensions, 1922 * timing, etc. 1923 */ 1924 fixed20_12 bpp; 1925 fixed20_12 line_time; 1926 fixed20_12 src_width; 1927 fixed20_12 bandwidth; 1928 fixed20_12 a; 1929 1930 a.full = dfixed_const(1000); 1931 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 1932 line_time.full = dfixed_div(line_time, a); 1933 bpp.full = dfixed_const(wm->bytes_per_pixel); 1934 src_width.full = dfixed_const(wm->src_width); 1935 bandwidth.full = dfixed_mul(src_width, bpp); 1936 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 1937 bandwidth.full = dfixed_div(bandwidth, line_time); 1938 1939 return dfixed_trunc(bandwidth); 1940 } 1941 1942 static u32 dce6_latency_watermark(struct dce6_wm_params *wm) 1943 { 1944 /* First calcualte the latency in ns */ 1945 u32 mc_latency = 2000; /* 2000 ns. */ 1946 u32 available_bandwidth = dce6_available_bandwidth(wm); 1947 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 1948 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 1949 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 1950 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 1951 (wm->num_heads * cursor_line_pair_return_time); 1952 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 1953 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 1954 u32 tmp, dmif_size = 12288; 1955 fixed20_12 a, b, c; 1956 1957 if (wm->num_heads == 0) 1958 return 0; 1959 1960 a.full = dfixed_const(2); 1961 b.full = dfixed_const(1); 1962 if ((wm->vsc.full > a.full) || 1963 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 1964 (wm->vtaps >= 5) || 1965 ((wm->vsc.full >= a.full) && wm->interlaced)) 1966 max_src_lines_per_dst_line = 4; 1967 else 1968 max_src_lines_per_dst_line = 2; 1969 1970 a.full = dfixed_const(available_bandwidth); 1971 b.full = dfixed_const(wm->num_heads); 1972 a.full = dfixed_div(a, b); 1973 1974 b.full = dfixed_const(mc_latency + 512); 1975 c.full = dfixed_const(wm->disp_clk); 1976 b.full = dfixed_div(b, c); 1977 1978 c.full = dfixed_const(dmif_size); 1979 b.full = dfixed_div(c, b); 1980 1981 tmp = min(dfixed_trunc(a), dfixed_trunc(b)); 1982 1983 b.full = dfixed_const(1000); 1984 c.full = dfixed_const(wm->disp_clk); 1985 b.full = dfixed_div(c, b); 1986 c.full = dfixed_const(wm->bytes_per_pixel); 1987 b.full = dfixed_mul(b, c); 1988 1989 lb_fill_bw = min(tmp, dfixed_trunc(b)); 1990 1991 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 1992 b.full = dfixed_const(1000); 1993 c.full = dfixed_const(lb_fill_bw); 1994 b.full = dfixed_div(c, b); 1995 a.full = dfixed_div(a, b); 1996 line_fill_time = dfixed_trunc(a); 1997 1998 if (line_fill_time < wm->active_time) 1999 return latency; 2000 else 2001 return latency + (line_fill_time - wm->active_time); 2002 2003 } 2004 2005 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm) 2006 { 2007 if (dce6_average_bandwidth(wm) <= 2008 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads)) 2009 return true; 2010 else 2011 return false; 2012 }; 2013 2014 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm) 2015 { 2016 if (dce6_average_bandwidth(wm) <= 2017 (dce6_available_bandwidth(wm) / wm->num_heads)) 2018 return true; 2019 else 2020 return false; 2021 }; 2022 2023 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm) 2024 { 2025 u32 lb_partitions = wm->lb_size / wm->src_width; 2026 u32 line_time = wm->active_time + wm->blank_time; 2027 u32 latency_tolerant_lines; 2028 u32 latency_hiding; 2029 fixed20_12 a; 2030 2031 a.full = dfixed_const(1); 2032 if (wm->vsc.full > a.full) 2033 latency_tolerant_lines = 1; 2034 else { 2035 if (lb_partitions <= (wm->vtaps + 1)) 2036 latency_tolerant_lines = 1; 2037 else 2038 latency_tolerant_lines = 2; 2039 } 2040 2041 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 2042 2043 if (dce6_latency_watermark(wm) <= latency_hiding) 2044 return true; 2045 else 2046 return false; 2047 } 2048 2049 static void dce6_program_watermarks(struct radeon_device *rdev, 2050 struct radeon_crtc *radeon_crtc, 2051 u32 lb_size, u32 num_heads) 2052 { 2053 struct drm_display_mode *mode = &radeon_crtc->base.mode; 2054 struct dce6_wm_params wm_low, wm_high; 2055 u32 dram_channels; 2056 u32 pixel_period; 2057 u32 line_time = 0; 2058 u32 latency_watermark_a = 0, latency_watermark_b = 0; 2059 u32 priority_a_mark = 0, priority_b_mark = 0; 2060 u32 priority_a_cnt = PRIORITY_OFF; 2061 u32 priority_b_cnt = PRIORITY_OFF; 2062 u32 tmp, arb_control3; 2063 fixed20_12 a, b, c; 2064 2065 if (radeon_crtc->base.enabled && num_heads && mode) { 2066 pixel_period = 1000000 / (u32)mode->clock; 2067 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 2068 priority_a_cnt = 0; 2069 priority_b_cnt = 0; 2070 2071 if (rdev->family == CHIP_ARUBA) 2072 dram_channels = evergreen_get_number_of_dram_channels(rdev); 2073 else 2074 dram_channels = si_get_number_of_dram_channels(rdev); 2075 2076 /* watermark for high clocks */ 2077 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2078 wm_high.yclk = 2079 radeon_dpm_get_mclk(rdev, false) * 10; 2080 wm_high.sclk = 2081 radeon_dpm_get_sclk(rdev, false) * 10; 2082 } else { 2083 wm_high.yclk = rdev->pm.current_mclk * 10; 2084 wm_high.sclk = rdev->pm.current_sclk * 10; 2085 } 2086 2087 wm_high.disp_clk = mode->clock; 2088 wm_high.src_width = mode->crtc_hdisplay; 2089 wm_high.active_time = mode->crtc_hdisplay * pixel_period; 2090 wm_high.blank_time = line_time - wm_high.active_time; 2091 wm_high.interlaced = false; 2092 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2093 wm_high.interlaced = true; 2094 wm_high.vsc = radeon_crtc->vsc; 2095 wm_high.vtaps = 1; 2096 if (radeon_crtc->rmx_type != RMX_OFF) 2097 wm_high.vtaps = 2; 2098 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2099 wm_high.lb_size = lb_size; 2100 wm_high.dram_channels = dram_channels; 2101 wm_high.num_heads = num_heads; 2102 2103 /* watermark for low clocks */ 2104 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) { 2105 wm_low.yclk = 2106 radeon_dpm_get_mclk(rdev, true) * 10; 2107 wm_low.sclk = 2108 radeon_dpm_get_sclk(rdev, true) * 10; 2109 } else { 2110 wm_low.yclk = rdev->pm.current_mclk * 10; 2111 wm_low.sclk = rdev->pm.current_sclk * 10; 2112 } 2113 2114 wm_low.disp_clk = mode->clock; 2115 wm_low.src_width = mode->crtc_hdisplay; 2116 wm_low.active_time = mode->crtc_hdisplay * pixel_period; 2117 wm_low.blank_time = line_time - wm_low.active_time; 2118 wm_low.interlaced = false; 2119 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 2120 wm_low.interlaced = true; 2121 wm_low.vsc = radeon_crtc->vsc; 2122 wm_low.vtaps = 1; 2123 if (radeon_crtc->rmx_type != RMX_OFF) 2124 wm_low.vtaps = 2; 2125 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */ 2126 wm_low.lb_size = lb_size; 2127 wm_low.dram_channels = dram_channels; 2128 wm_low.num_heads = num_heads; 2129 2130 /* set for high clocks */ 2131 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535); 2132 /* set for low clocks */ 2133 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535); 2134 2135 /* possibly force display priority to high */ 2136 /* should really do this at mode validation time... */ 2137 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) || 2138 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) || 2139 !dce6_check_latency_hiding(&wm_high) || 2140 (rdev->disp_priority == 2)) { 2141 DRM_DEBUG_KMS("force priority to high\n"); 2142 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2143 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2144 } 2145 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) || 2146 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) || 2147 !dce6_check_latency_hiding(&wm_low) || 2148 (rdev->disp_priority == 2)) { 2149 DRM_DEBUG_KMS("force priority to high\n"); 2150 priority_a_cnt |= PRIORITY_ALWAYS_ON; 2151 priority_b_cnt |= PRIORITY_ALWAYS_ON; 2152 } 2153 2154 a.full = dfixed_const(1000); 2155 b.full = dfixed_const(mode->clock); 2156 b.full = dfixed_div(b, a); 2157 c.full = dfixed_const(latency_watermark_a); 2158 c.full = dfixed_mul(c, b); 2159 c.full = dfixed_mul(c, radeon_crtc->hsc); 2160 c.full = dfixed_div(c, a); 2161 a.full = dfixed_const(16); 2162 c.full = dfixed_div(c, a); 2163 priority_a_mark = dfixed_trunc(c); 2164 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 2165 2166 a.full = dfixed_const(1000); 2167 b.full = dfixed_const(mode->clock); 2168 b.full = dfixed_div(b, a); 2169 c.full = dfixed_const(latency_watermark_b); 2170 c.full = dfixed_mul(c, b); 2171 c.full = dfixed_mul(c, radeon_crtc->hsc); 2172 c.full = dfixed_div(c, a); 2173 a.full = dfixed_const(16); 2174 c.full = dfixed_div(c, a); 2175 priority_b_mark = dfixed_trunc(c); 2176 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 2177 } 2178 2179 /* select wm A */ 2180 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 2181 tmp = arb_control3; 2182 tmp &= ~LATENCY_WATERMARK_MASK(3); 2183 tmp |= LATENCY_WATERMARK_MASK(1); 2184 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 2185 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 2186 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 2187 LATENCY_HIGH_WATERMARK(line_time))); 2188 /* select wm B */ 2189 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset); 2190 tmp &= ~LATENCY_WATERMARK_MASK(3); 2191 tmp |= LATENCY_WATERMARK_MASK(2); 2192 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp); 2193 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset, 2194 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 2195 LATENCY_HIGH_WATERMARK(line_time))); 2196 /* restore original selection */ 2197 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3); 2198 2199 /* write the priority marks */ 2200 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 2201 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 2202 2203 /* save values for DPM */ 2204 radeon_crtc->line_time = line_time; 2205 radeon_crtc->wm_high = latency_watermark_a; 2206 radeon_crtc->wm_low = latency_watermark_b; 2207 } 2208 2209 void dce6_bandwidth_update(struct radeon_device *rdev) 2210 { 2211 struct drm_display_mode *mode0 = NULL; 2212 struct drm_display_mode *mode1 = NULL; 2213 u32 num_heads = 0, lb_size; 2214 int i; 2215 2216 radeon_update_display_priority(rdev); 2217 2218 for (i = 0; i < rdev->num_crtc; i++) { 2219 if (rdev->mode_info.crtcs[i]->base.enabled) 2220 num_heads++; 2221 } 2222 for (i = 0; i < rdev->num_crtc; i += 2) { 2223 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 2224 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 2225 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 2226 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 2227 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 2228 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 2229 } 2230 } 2231 2232 /* 2233 * Core functions 2234 */ 2235 static void si_tiling_mode_table_init(struct radeon_device *rdev) 2236 { 2237 const u32 num_tile_mode_states = 32; 2238 u32 reg_offset, gb_tile_moden, split_equal_to_row_size; 2239 2240 switch (rdev->config.si.mem_row_size_in_kb) { 2241 case 1: 2242 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB; 2243 break; 2244 case 2: 2245 default: 2246 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB; 2247 break; 2248 case 4: 2249 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB; 2250 break; 2251 } 2252 2253 if ((rdev->family == CHIP_TAHITI) || 2254 (rdev->family == CHIP_PITCAIRN)) { 2255 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2256 switch (reg_offset) { 2257 case 0: /* non-AA compressed depth or any compressed stencil */ 2258 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2259 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2261 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2262 NUM_BANKS(ADDR_SURF_16_BANK) | 2263 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2264 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2265 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2266 break; 2267 case 1: /* 2xAA/4xAA compressed depth only */ 2268 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2269 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2270 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2272 NUM_BANKS(ADDR_SURF_16_BANK) | 2273 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2274 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2275 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2276 break; 2277 case 2: /* 8xAA compressed depth only */ 2278 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2279 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2281 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2282 NUM_BANKS(ADDR_SURF_16_BANK) | 2283 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2284 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2285 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2286 break; 2287 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 2288 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2289 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2290 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2291 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2292 NUM_BANKS(ADDR_SURF_16_BANK) | 2293 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2294 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2295 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2296 break; 2297 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 2298 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2299 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2300 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2301 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2302 NUM_BANKS(ADDR_SURF_16_BANK) | 2303 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2304 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2305 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2306 break; 2307 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2309 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2310 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2311 TILE_SPLIT(split_equal_to_row_size) | 2312 NUM_BANKS(ADDR_SURF_16_BANK) | 2313 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2314 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2315 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2316 break; 2317 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 2318 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2319 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2320 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2321 TILE_SPLIT(split_equal_to_row_size) | 2322 NUM_BANKS(ADDR_SURF_16_BANK) | 2323 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2324 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2325 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2326 break; 2327 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 2328 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2329 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2331 TILE_SPLIT(split_equal_to_row_size) | 2332 NUM_BANKS(ADDR_SURF_16_BANK) | 2333 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2334 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2335 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2336 break; 2337 case 8: /* 1D and 1D Array Surfaces */ 2338 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2339 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2340 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2341 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2342 NUM_BANKS(ADDR_SURF_16_BANK) | 2343 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2346 break; 2347 case 9: /* Displayable maps. */ 2348 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2349 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2350 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2351 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2352 NUM_BANKS(ADDR_SURF_16_BANK) | 2353 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2354 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2355 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2356 break; 2357 case 10: /* Display 8bpp. */ 2358 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2359 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2360 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2361 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2362 NUM_BANKS(ADDR_SURF_16_BANK) | 2363 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2364 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2365 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2366 break; 2367 case 11: /* Display 16bpp. */ 2368 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2369 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2370 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2371 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2372 NUM_BANKS(ADDR_SURF_16_BANK) | 2373 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2376 break; 2377 case 12: /* Display 32bpp. */ 2378 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2379 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2380 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2381 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2382 NUM_BANKS(ADDR_SURF_16_BANK) | 2383 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2384 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2385 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2386 break; 2387 case 13: /* Thin. */ 2388 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2389 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2390 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2391 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2392 NUM_BANKS(ADDR_SURF_16_BANK) | 2393 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2394 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2395 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2396 break; 2397 case 14: /* Thin 8 bpp. */ 2398 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2399 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2400 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2401 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2402 NUM_BANKS(ADDR_SURF_16_BANK) | 2403 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2406 break; 2407 case 15: /* Thin 16 bpp. */ 2408 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2409 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2410 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2411 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2412 NUM_BANKS(ADDR_SURF_16_BANK) | 2413 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2414 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2415 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2416 break; 2417 case 16: /* Thin 32 bpp. */ 2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2419 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2420 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2421 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2422 NUM_BANKS(ADDR_SURF_16_BANK) | 2423 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2424 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2425 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2426 break; 2427 case 17: /* Thin 64 bpp. */ 2428 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2429 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2430 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2431 TILE_SPLIT(split_equal_to_row_size) | 2432 NUM_BANKS(ADDR_SURF_16_BANK) | 2433 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2434 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2435 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2436 break; 2437 case 21: /* 8 bpp PRT. */ 2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2439 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2440 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2442 NUM_BANKS(ADDR_SURF_16_BANK) | 2443 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2446 break; 2447 case 22: /* 16 bpp PRT */ 2448 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2449 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2450 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2451 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2452 NUM_BANKS(ADDR_SURF_16_BANK) | 2453 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2456 break; 2457 case 23: /* 32 bpp PRT */ 2458 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2459 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2460 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2461 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2462 NUM_BANKS(ADDR_SURF_16_BANK) | 2463 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2466 break; 2467 case 24: /* 64 bpp PRT */ 2468 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2469 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2470 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2471 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2472 NUM_BANKS(ADDR_SURF_16_BANK) | 2473 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2476 break; 2477 case 25: /* 128 bpp PRT */ 2478 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2479 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2480 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2481 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 2482 NUM_BANKS(ADDR_SURF_8_BANK) | 2483 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2484 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2485 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2486 break; 2487 default: 2488 gb_tile_moden = 0; 2489 break; 2490 } 2491 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; 2492 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2493 } 2494 } else if ((rdev->family == CHIP_VERDE) || 2495 (rdev->family == CHIP_OLAND) || 2496 (rdev->family == CHIP_HAINAN)) { 2497 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) { 2498 switch (reg_offset) { 2499 case 0: /* non-AA compressed depth or any compressed stencil */ 2500 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2501 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2502 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2503 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2504 NUM_BANKS(ADDR_SURF_16_BANK) | 2505 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2508 break; 2509 case 1: /* 2xAA/4xAA compressed depth only */ 2510 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2511 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2512 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2513 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2514 NUM_BANKS(ADDR_SURF_16_BANK) | 2515 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2516 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2517 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2518 break; 2519 case 2: /* 8xAA compressed depth only */ 2520 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2521 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2522 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2523 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2524 NUM_BANKS(ADDR_SURF_16_BANK) | 2525 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2526 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2527 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2528 break; 2529 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */ 2530 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2531 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2532 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2533 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2534 NUM_BANKS(ADDR_SURF_16_BANK) | 2535 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2536 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2537 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2538 break; 2539 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */ 2540 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2541 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2542 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2543 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2544 NUM_BANKS(ADDR_SURF_16_BANK) | 2545 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2546 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2547 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2548 break; 2549 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */ 2550 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2551 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2552 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2553 TILE_SPLIT(split_equal_to_row_size) | 2554 NUM_BANKS(ADDR_SURF_16_BANK) | 2555 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2556 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2557 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2558 break; 2559 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */ 2560 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2561 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2562 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2563 TILE_SPLIT(split_equal_to_row_size) | 2564 NUM_BANKS(ADDR_SURF_16_BANK) | 2565 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2566 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2567 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2568 break; 2569 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */ 2570 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2571 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) | 2572 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2573 TILE_SPLIT(split_equal_to_row_size) | 2574 NUM_BANKS(ADDR_SURF_16_BANK) | 2575 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2576 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2577 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2578 break; 2579 case 8: /* 1D and 1D Array Surfaces */ 2580 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2581 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2582 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2583 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2584 NUM_BANKS(ADDR_SURF_16_BANK) | 2585 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2586 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2587 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2588 break; 2589 case 9: /* Displayable maps. */ 2590 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2591 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2592 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2593 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2594 NUM_BANKS(ADDR_SURF_16_BANK) | 2595 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2596 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2597 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2598 break; 2599 case 10: /* Display 8bpp. */ 2600 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2601 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2602 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2603 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2604 NUM_BANKS(ADDR_SURF_16_BANK) | 2605 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2606 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2607 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2608 break; 2609 case 11: /* Display 16bpp. */ 2610 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2611 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2612 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2613 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2614 NUM_BANKS(ADDR_SURF_16_BANK) | 2615 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2616 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2617 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2618 break; 2619 case 12: /* Display 32bpp. */ 2620 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2621 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) | 2622 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2623 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2624 NUM_BANKS(ADDR_SURF_16_BANK) | 2625 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2626 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2627 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2628 break; 2629 case 13: /* Thin. */ 2630 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2631 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2632 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2633 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2634 NUM_BANKS(ADDR_SURF_16_BANK) | 2635 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2638 break; 2639 case 14: /* Thin 8 bpp. */ 2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2641 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2642 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2643 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2644 NUM_BANKS(ADDR_SURF_16_BANK) | 2645 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2646 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2647 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2648 break; 2649 case 15: /* Thin 16 bpp. */ 2650 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2651 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2652 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2653 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2654 NUM_BANKS(ADDR_SURF_16_BANK) | 2655 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2656 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2657 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2658 break; 2659 case 16: /* Thin 32 bpp. */ 2660 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2661 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2662 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2664 NUM_BANKS(ADDR_SURF_16_BANK) | 2665 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2666 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2667 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2668 break; 2669 case 17: /* Thin 64 bpp. */ 2670 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2671 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2672 PIPE_CONFIG(ADDR_SURF_P4_8x16) | 2673 TILE_SPLIT(split_equal_to_row_size) | 2674 NUM_BANKS(ADDR_SURF_16_BANK) | 2675 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2676 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2677 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2678 break; 2679 case 21: /* 8 bpp PRT. */ 2680 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2681 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2682 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2683 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2684 NUM_BANKS(ADDR_SURF_16_BANK) | 2685 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2686 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2687 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2688 break; 2689 case 22: /* 16 bpp PRT */ 2690 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2691 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2692 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2693 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2694 NUM_BANKS(ADDR_SURF_16_BANK) | 2695 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2696 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2697 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4)); 2698 break; 2699 case 23: /* 32 bpp PRT */ 2700 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2701 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2702 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2703 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2704 NUM_BANKS(ADDR_SURF_16_BANK) | 2705 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2706 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2707 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2708 break; 2709 case 24: /* 64 bpp PRT */ 2710 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2711 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2712 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2713 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2714 NUM_BANKS(ADDR_SURF_16_BANK) | 2715 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2716 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2717 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2)); 2718 break; 2719 case 25: /* 128 bpp PRT */ 2720 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2721 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) | 2722 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) | 2723 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) | 2724 NUM_BANKS(ADDR_SURF_8_BANK) | 2725 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1)); 2728 break; 2729 default: 2730 gb_tile_moden = 0; 2731 break; 2732 } 2733 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden; 2734 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden); 2735 } 2736 } else 2737 DRM_ERROR("unknown asic: 0x%x\n", rdev->family); 2738 } 2739 2740 static void si_select_se_sh(struct radeon_device *rdev, 2741 u32 se_num, u32 sh_num) 2742 { 2743 u32 data = INSTANCE_BROADCAST_WRITES; 2744 2745 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) 2746 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES; 2747 else if (se_num == 0xffffffff) 2748 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num); 2749 else if (sh_num == 0xffffffff) 2750 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num); 2751 else 2752 data |= SH_INDEX(sh_num) | SE_INDEX(se_num); 2753 WREG32(GRBM_GFX_INDEX, data); 2754 } 2755 2756 static u32 si_create_bitmask(u32 bit_width) 2757 { 2758 u32 i, mask = 0; 2759 2760 for (i = 0; i < bit_width; i++) { 2761 mask <<= 1; 2762 mask |= 1; 2763 } 2764 return mask; 2765 } 2766 2767 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh) 2768 { 2769 u32 data, mask; 2770 2771 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 2772 if (data & 1) 2773 data &= INACTIVE_CUS_MASK; 2774 else 2775 data = 0; 2776 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG); 2777 2778 data >>= INACTIVE_CUS_SHIFT; 2779 2780 mask = si_create_bitmask(cu_per_sh); 2781 2782 return ~data & mask; 2783 } 2784 2785 static void si_setup_spi(struct radeon_device *rdev, 2786 u32 se_num, u32 sh_per_se, 2787 u32 cu_per_sh) 2788 { 2789 int i, j, k; 2790 u32 data, mask, active_cu; 2791 2792 for (i = 0; i < se_num; i++) { 2793 for (j = 0; j < sh_per_se; j++) { 2794 si_select_se_sh(rdev, i, j); 2795 data = RREG32(SPI_STATIC_THREAD_MGMT_3); 2796 active_cu = si_get_cu_enabled(rdev, cu_per_sh); 2797 2798 mask = 1; 2799 for (k = 0; k < 16; k++) { 2800 mask <<= k; 2801 if (active_cu & mask) { 2802 data &= ~mask; 2803 WREG32(SPI_STATIC_THREAD_MGMT_3, data); 2804 break; 2805 } 2806 } 2807 } 2808 } 2809 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2810 } 2811 2812 static u32 si_get_rb_disabled(struct radeon_device *rdev, 2813 u32 max_rb_num, u32 se_num, 2814 u32 sh_per_se) 2815 { 2816 u32 data, mask; 2817 2818 data = RREG32(CC_RB_BACKEND_DISABLE); 2819 if (data & 1) 2820 data &= BACKEND_DISABLE_MASK; 2821 else 2822 data = 0; 2823 data |= RREG32(GC_USER_RB_BACKEND_DISABLE); 2824 2825 data >>= BACKEND_DISABLE_SHIFT; 2826 2827 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se); 2828 2829 return data & mask; 2830 } 2831 2832 static void si_setup_rb(struct radeon_device *rdev, 2833 u32 se_num, u32 sh_per_se, 2834 u32 max_rb_num) 2835 { 2836 int i, j; 2837 u32 data, mask; 2838 u32 disabled_rbs = 0; 2839 u32 enabled_rbs = 0; 2840 2841 for (i = 0; i < se_num; i++) { 2842 for (j = 0; j < sh_per_se; j++) { 2843 si_select_se_sh(rdev, i, j); 2844 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se); 2845 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH); 2846 } 2847 } 2848 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2849 2850 mask = 1; 2851 for (i = 0; i < max_rb_num; i++) { 2852 if (!(disabled_rbs & mask)) 2853 enabled_rbs |= mask; 2854 mask <<= 1; 2855 } 2856 2857 for (i = 0; i < se_num; i++) { 2858 si_select_se_sh(rdev, i, 0xffffffff); 2859 data = 0; 2860 for (j = 0; j < sh_per_se; j++) { 2861 switch (enabled_rbs & 3) { 2862 case 1: 2863 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2); 2864 break; 2865 case 2: 2866 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2); 2867 break; 2868 case 3: 2869 default: 2870 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2); 2871 break; 2872 } 2873 enabled_rbs >>= 2; 2874 } 2875 WREG32(PA_SC_RASTER_CONFIG, data); 2876 } 2877 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 2878 } 2879 2880 static void si_gpu_init(struct radeon_device *rdev) 2881 { 2882 u32 gb_addr_config = 0; 2883 u32 mc_shared_chmap, mc_arb_ramcfg; 2884 u32 sx_debug_1; 2885 u32 hdp_host_path_cntl; 2886 u32 tmp; 2887 int i, j; 2888 2889 switch (rdev->family) { 2890 case CHIP_TAHITI: 2891 rdev->config.si.max_shader_engines = 2; 2892 rdev->config.si.max_tile_pipes = 12; 2893 rdev->config.si.max_cu_per_sh = 8; 2894 rdev->config.si.max_sh_per_se = 2; 2895 rdev->config.si.max_backends_per_se = 4; 2896 rdev->config.si.max_texture_channel_caches = 12; 2897 rdev->config.si.max_gprs = 256; 2898 rdev->config.si.max_gs_threads = 32; 2899 rdev->config.si.max_hw_contexts = 8; 2900 2901 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2902 rdev->config.si.sc_prim_fifo_size_backend = 0x100; 2903 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2904 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2905 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 2906 break; 2907 case CHIP_PITCAIRN: 2908 rdev->config.si.max_shader_engines = 2; 2909 rdev->config.si.max_tile_pipes = 8; 2910 rdev->config.si.max_cu_per_sh = 5; 2911 rdev->config.si.max_sh_per_se = 2; 2912 rdev->config.si.max_backends_per_se = 4; 2913 rdev->config.si.max_texture_channel_caches = 8; 2914 rdev->config.si.max_gprs = 256; 2915 rdev->config.si.max_gs_threads = 32; 2916 rdev->config.si.max_hw_contexts = 8; 2917 2918 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2919 rdev->config.si.sc_prim_fifo_size_backend = 0x100; 2920 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2921 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2922 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN; 2923 break; 2924 case CHIP_VERDE: 2925 default: 2926 rdev->config.si.max_shader_engines = 1; 2927 rdev->config.si.max_tile_pipes = 4; 2928 rdev->config.si.max_cu_per_sh = 5; 2929 rdev->config.si.max_sh_per_se = 2; 2930 rdev->config.si.max_backends_per_se = 4; 2931 rdev->config.si.max_texture_channel_caches = 4; 2932 rdev->config.si.max_gprs = 256; 2933 rdev->config.si.max_gs_threads = 32; 2934 rdev->config.si.max_hw_contexts = 8; 2935 2936 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2937 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 2938 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2939 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2940 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; 2941 break; 2942 case CHIP_OLAND: 2943 rdev->config.si.max_shader_engines = 1; 2944 rdev->config.si.max_tile_pipes = 4; 2945 rdev->config.si.max_cu_per_sh = 6; 2946 rdev->config.si.max_sh_per_se = 1; 2947 rdev->config.si.max_backends_per_se = 2; 2948 rdev->config.si.max_texture_channel_caches = 4; 2949 rdev->config.si.max_gprs = 256; 2950 rdev->config.si.max_gs_threads = 16; 2951 rdev->config.si.max_hw_contexts = 8; 2952 2953 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2954 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 2955 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2956 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2957 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN; 2958 break; 2959 case CHIP_HAINAN: 2960 rdev->config.si.max_shader_engines = 1; 2961 rdev->config.si.max_tile_pipes = 4; 2962 rdev->config.si.max_cu_per_sh = 5; 2963 rdev->config.si.max_sh_per_se = 1; 2964 rdev->config.si.max_backends_per_se = 1; 2965 rdev->config.si.max_texture_channel_caches = 2; 2966 rdev->config.si.max_gprs = 256; 2967 rdev->config.si.max_gs_threads = 16; 2968 rdev->config.si.max_hw_contexts = 8; 2969 2970 rdev->config.si.sc_prim_fifo_size_frontend = 0x20; 2971 rdev->config.si.sc_prim_fifo_size_backend = 0x40; 2972 rdev->config.si.sc_hiz_tile_fifo_size = 0x30; 2973 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130; 2974 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN; 2975 break; 2976 } 2977 2978 /* Initialize HDP */ 2979 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 2980 WREG32((0x2c14 + j), 0x00000000); 2981 WREG32((0x2c18 + j), 0x00000000); 2982 WREG32((0x2c1c + j), 0x00000000); 2983 WREG32((0x2c20 + j), 0x00000000); 2984 WREG32((0x2c24 + j), 0x00000000); 2985 } 2986 2987 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 2988 2989 evergreen_fix_pci_max_read_req_size(rdev); 2990 2991 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN); 2992 2993 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 2994 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 2995 2996 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes; 2997 rdev->config.si.mem_max_burst_length_bytes = 256; 2998 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT; 2999 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 3000 if (rdev->config.si.mem_row_size_in_kb > 4) 3001 rdev->config.si.mem_row_size_in_kb = 4; 3002 /* XXX use MC settings? */ 3003 rdev->config.si.shader_engine_tile_size = 32; 3004 rdev->config.si.num_gpus = 1; 3005 rdev->config.si.multi_gpu_tile_size = 64; 3006 3007 /* fix up row size */ 3008 gb_addr_config &= ~ROW_SIZE_MASK; 3009 switch (rdev->config.si.mem_row_size_in_kb) { 3010 case 1: 3011 default: 3012 gb_addr_config |= ROW_SIZE(0); 3013 break; 3014 case 2: 3015 gb_addr_config |= ROW_SIZE(1); 3016 break; 3017 case 4: 3018 gb_addr_config |= ROW_SIZE(2); 3019 break; 3020 } 3021 3022 /* setup tiling info dword. gb_addr_config is not adequate since it does 3023 * not have bank info, so create a custom tiling dword. 3024 * bits 3:0 num_pipes 3025 * bits 7:4 num_banks 3026 * bits 11:8 group_size 3027 * bits 15:12 row_size 3028 */ 3029 rdev->config.si.tile_config = 0; 3030 switch (rdev->config.si.num_tile_pipes) { 3031 case 1: 3032 rdev->config.si.tile_config |= (0 << 0); 3033 break; 3034 case 2: 3035 rdev->config.si.tile_config |= (1 << 0); 3036 break; 3037 case 4: 3038 rdev->config.si.tile_config |= (2 << 0); 3039 break; 3040 case 8: 3041 default: 3042 /* XXX what about 12? */ 3043 rdev->config.si.tile_config |= (3 << 0); 3044 break; 3045 } 3046 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) { 3047 case 0: /* four banks */ 3048 rdev->config.si.tile_config |= 0 << 4; 3049 break; 3050 case 1: /* eight banks */ 3051 rdev->config.si.tile_config |= 1 << 4; 3052 break; 3053 case 2: /* sixteen banks */ 3054 default: 3055 rdev->config.si.tile_config |= 2 << 4; 3056 break; 3057 } 3058 rdev->config.si.tile_config |= 3059 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8; 3060 rdev->config.si.tile_config |= 3061 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12; 3062 3063 WREG32(GB_ADDR_CONFIG, gb_addr_config); 3064 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 3065 WREG32(DMIF_ADDR_CALC, gb_addr_config); 3066 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 3067 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config); 3068 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config); 3069 if (rdev->has_uvd) { 3070 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config); 3071 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config); 3072 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config); 3073 } 3074 3075 si_tiling_mode_table_init(rdev); 3076 3077 si_setup_rb(rdev, rdev->config.si.max_shader_engines, 3078 rdev->config.si.max_sh_per_se, 3079 rdev->config.si.max_backends_per_se); 3080 3081 si_setup_spi(rdev, rdev->config.si.max_shader_engines, 3082 rdev->config.si.max_sh_per_se, 3083 rdev->config.si.max_cu_per_sh); 3084 3085 3086 /* set HW defaults for 3D engine */ 3087 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 3088 ROQ_IB2_START(0x2b))); 3089 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60)); 3090 3091 sx_debug_1 = RREG32(SX_DEBUG_1); 3092 WREG32(SX_DEBUG_1, sx_debug_1); 3093 3094 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 3095 3096 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) | 3097 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) | 3098 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) | 3099 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size))); 3100 3101 WREG32(VGT_NUM_INSTANCES, 1); 3102 3103 WREG32(CP_PERFMON_CNTL, 0); 3104 3105 WREG32(SQ_CONFIG, 0); 3106 3107 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 3108 FORCE_EOV_MAX_REZ_CNT(255))); 3109 3110 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) | 3111 AUTO_INVLD_EN(ES_AND_GS_AUTO)); 3112 3113 WREG32(VGT_GS_VERTEX_REUSE, 16); 3114 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 3115 3116 WREG32(CB_PERFCOUNTER0_SELECT0, 0); 3117 WREG32(CB_PERFCOUNTER0_SELECT1, 0); 3118 WREG32(CB_PERFCOUNTER1_SELECT0, 0); 3119 WREG32(CB_PERFCOUNTER1_SELECT1, 0); 3120 WREG32(CB_PERFCOUNTER2_SELECT0, 0); 3121 WREG32(CB_PERFCOUNTER2_SELECT1, 0); 3122 WREG32(CB_PERFCOUNTER3_SELECT0, 0); 3123 WREG32(CB_PERFCOUNTER3_SELECT1, 0); 3124 3125 tmp = RREG32(HDP_MISC_CNTL); 3126 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 3127 WREG32(HDP_MISC_CNTL, tmp); 3128 3129 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 3130 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 3131 3132 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 3133 3134 udelay(50); 3135 } 3136 3137 /* 3138 * GPU scratch registers helpers function. 3139 */ 3140 static void si_scratch_init(struct radeon_device *rdev) 3141 { 3142 int i; 3143 3144 rdev->scratch.num_reg = 7; 3145 rdev->scratch.reg_base = SCRATCH_REG0; 3146 for (i = 0; i < rdev->scratch.num_reg; i++) { 3147 rdev->scratch.free[i] = true; 3148 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); 3149 } 3150 } 3151 3152 void si_fence_ring_emit(struct radeon_device *rdev, 3153 struct radeon_fence *fence) 3154 { 3155 struct radeon_ring *ring = &rdev->ring[fence->ring]; 3156 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; 3157 3158 /* flush read cache over gart */ 3159 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3160 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 3161 radeon_ring_write(ring, 0); 3162 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 3163 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 3164 PACKET3_TC_ACTION_ENA | 3165 PACKET3_SH_KCACHE_ACTION_ENA | 3166 PACKET3_SH_ICACHE_ACTION_ENA); 3167 radeon_ring_write(ring, 0xFFFFFFFF); 3168 radeon_ring_write(ring, 0); 3169 radeon_ring_write(ring, 10); /* poll interval */ 3170 /* EVENT_WRITE_EOP - flush caches, send int */ 3171 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 3172 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5)); 3173 radeon_ring_write(ring, addr & 0xffffffff); 3174 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2)); 3175 radeon_ring_write(ring, fence->seq); 3176 radeon_ring_write(ring, 0); 3177 } 3178 3179 /* 3180 * IB stuff 3181 */ 3182 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 3183 { 3184 struct radeon_ring *ring = &rdev->ring[ib->ring]; 3185 u32 header; 3186 3187 if (ib->is_const_ib) { 3188 /* set switch buffer packet before const IB */ 3189 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 3190 radeon_ring_write(ring, 0); 3191 3192 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 3193 } else { 3194 u32 next_rptr; 3195 if (ring->rptr_save_reg) { 3196 next_rptr = ring->wptr + 3 + 4 + 8; 3197 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3198 radeon_ring_write(ring, ((ring->rptr_save_reg - 3199 PACKET3_SET_CONFIG_REG_START) >> 2)); 3200 radeon_ring_write(ring, next_rptr); 3201 } else if (rdev->wb.enabled) { 3202 next_rptr = ring->wptr + 5 + 4 + 8; 3203 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3204 radeon_ring_write(ring, (1 << 8)); 3205 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc); 3206 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff); 3207 radeon_ring_write(ring, next_rptr); 3208 } 3209 3210 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 3211 } 3212 3213 radeon_ring_write(ring, header); 3214 radeon_ring_write(ring, 3215 #ifdef __BIG_ENDIAN 3216 (2 << 0) | 3217 #endif 3218 (ib->gpu_addr & 0xFFFFFFFC)); 3219 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 3220 radeon_ring_write(ring, ib->length_dw | 3221 (ib->vm ? (ib->vm->id << 24) : 0)); 3222 3223 if (!ib->is_const_ib) { 3224 /* flush read cache over gart for this vmid */ 3225 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3226 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2); 3227 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0); 3228 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 3229 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 3230 PACKET3_TC_ACTION_ENA | 3231 PACKET3_SH_KCACHE_ACTION_ENA | 3232 PACKET3_SH_ICACHE_ACTION_ENA); 3233 radeon_ring_write(ring, 0xFFFFFFFF); 3234 radeon_ring_write(ring, 0); 3235 radeon_ring_write(ring, 10); /* poll interval */ 3236 } 3237 } 3238 3239 /* 3240 * CP. 3241 */ 3242 static void si_cp_enable(struct radeon_device *rdev, bool enable) 3243 { 3244 if (enable) 3245 WREG32(CP_ME_CNTL, 0); 3246 else { 3247 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); 3248 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT)); 3249 WREG32(SCRATCH_UMSK, 0); 3250 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3251 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3252 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3253 } 3254 udelay(50); 3255 } 3256 3257 static int si_cp_load_microcode(struct radeon_device *rdev) 3258 { 3259 const __be32 *fw_data; 3260 int i; 3261 3262 if (!rdev->me_fw || !rdev->pfp_fw) 3263 return -EINVAL; 3264 3265 si_cp_enable(rdev, false); 3266 3267 /* PFP */ 3268 fw_data = (const __be32 *)rdev->pfp_fw->data; 3269 WREG32(CP_PFP_UCODE_ADDR, 0); 3270 for (i = 0; i < SI_PFP_UCODE_SIZE; i++) 3271 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 3272 WREG32(CP_PFP_UCODE_ADDR, 0); 3273 3274 /* CE */ 3275 fw_data = (const __be32 *)rdev->ce_fw->data; 3276 WREG32(CP_CE_UCODE_ADDR, 0); 3277 for (i = 0; i < SI_CE_UCODE_SIZE; i++) 3278 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++)); 3279 WREG32(CP_CE_UCODE_ADDR, 0); 3280 3281 /* ME */ 3282 fw_data = (const __be32 *)rdev->me_fw->data; 3283 WREG32(CP_ME_RAM_WADDR, 0); 3284 for (i = 0; i < SI_PM4_UCODE_SIZE; i++) 3285 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 3286 WREG32(CP_ME_RAM_WADDR, 0); 3287 3288 WREG32(CP_PFP_UCODE_ADDR, 0); 3289 WREG32(CP_CE_UCODE_ADDR, 0); 3290 WREG32(CP_ME_RAM_WADDR, 0); 3291 WREG32(CP_ME_RAM_RADDR, 0); 3292 return 0; 3293 } 3294 3295 static int si_cp_start(struct radeon_device *rdev) 3296 { 3297 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3298 int r, i; 3299 3300 r = radeon_ring_lock(rdev, ring, 7 + 4); 3301 if (r) { 3302 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3303 return r; 3304 } 3305 /* init the CP */ 3306 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5)); 3307 radeon_ring_write(ring, 0x1); 3308 radeon_ring_write(ring, 0x0); 3309 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1); 3310 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 3311 radeon_ring_write(ring, 0); 3312 radeon_ring_write(ring, 0); 3313 3314 /* init the CE partitions */ 3315 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3316 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3317 radeon_ring_write(ring, 0xc000); 3318 radeon_ring_write(ring, 0xe000); 3319 radeon_ring_unlock_commit(rdev, ring); 3320 3321 si_cp_enable(rdev, true); 3322 3323 r = radeon_ring_lock(rdev, ring, si_default_size + 10); 3324 if (r) { 3325 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 3326 return r; 3327 } 3328 3329 /* setup clear context state */ 3330 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3331 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3332 3333 for (i = 0; i < si_default_size; i++) 3334 radeon_ring_write(ring, si_default_state[i]); 3335 3336 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3337 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3338 3339 /* set clear context state */ 3340 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3341 radeon_ring_write(ring, 0); 3342 3343 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 3344 radeon_ring_write(ring, 0x00000316); 3345 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 3346 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */ 3347 3348 radeon_ring_unlock_commit(rdev, ring); 3349 3350 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) { 3351 ring = &rdev->ring[i]; 3352 r = radeon_ring_lock(rdev, ring, 2); 3353 3354 /* clear the compute context state */ 3355 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0)); 3356 radeon_ring_write(ring, 0); 3357 3358 radeon_ring_unlock_commit(rdev, ring); 3359 } 3360 3361 return 0; 3362 } 3363 3364 static void si_cp_fini(struct radeon_device *rdev) 3365 { 3366 struct radeon_ring *ring; 3367 si_cp_enable(rdev, false); 3368 3369 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3370 radeon_ring_fini(rdev, ring); 3371 radeon_scratch_free(rdev, ring->rptr_save_reg); 3372 3373 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 3374 radeon_ring_fini(rdev, ring); 3375 radeon_scratch_free(rdev, ring->rptr_save_reg); 3376 3377 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 3378 radeon_ring_fini(rdev, ring); 3379 radeon_scratch_free(rdev, ring->rptr_save_reg); 3380 } 3381 3382 static int si_cp_resume(struct radeon_device *rdev) 3383 { 3384 struct radeon_ring *ring; 3385 u32 tmp; 3386 u32 rb_bufsz; 3387 int r; 3388 3389 WREG32(CP_SEM_WAIT_TIMER, 0x0); 3390 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0); 3391 3392 /* Set the write pointer delay */ 3393 WREG32(CP_RB_WPTR_DELAY, 0); 3394 3395 WREG32(CP_DEBUG, 0); 3396 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 3397 3398 /* ring 0 - compute and gfx */ 3399 /* Set ring buffer size */ 3400 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 3401 rb_bufsz = order_base_2(ring->ring_size / 8); 3402 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3403 #ifdef __BIG_ENDIAN 3404 tmp |= BUF_SWAP_32BIT; 3405 #endif 3406 WREG32(CP_RB0_CNTL, tmp); 3407 3408 /* Initialize the ring buffer's read and write pointers */ 3409 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA); 3410 ring->wptr = 0; 3411 WREG32(CP_RB0_WPTR, ring->wptr); 3412 3413 /* set the wb address whether it's enabled or not */ 3414 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); 3415 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 3416 3417 if (rdev->wb.enabled) 3418 WREG32(SCRATCH_UMSK, 0xff); 3419 else { 3420 tmp |= RB_NO_UPDATE; 3421 WREG32(SCRATCH_UMSK, 0); 3422 } 3423 3424 mdelay(1); 3425 WREG32(CP_RB0_CNTL, tmp); 3426 3427 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8); 3428 3429 ring->rptr = RREG32(CP_RB0_RPTR); 3430 3431 /* ring1 - compute only */ 3432 /* Set ring buffer size */ 3433 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 3434 rb_bufsz = order_base_2(ring->ring_size / 8); 3435 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3436 #ifdef __BIG_ENDIAN 3437 tmp |= BUF_SWAP_32BIT; 3438 #endif 3439 WREG32(CP_RB1_CNTL, tmp); 3440 3441 /* Initialize the ring buffer's read and write pointers */ 3442 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA); 3443 ring->wptr = 0; 3444 WREG32(CP_RB1_WPTR, ring->wptr); 3445 3446 /* set the wb address whether it's enabled or not */ 3447 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC); 3448 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF); 3449 3450 mdelay(1); 3451 WREG32(CP_RB1_CNTL, tmp); 3452 3453 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8); 3454 3455 ring->rptr = RREG32(CP_RB1_RPTR); 3456 3457 /* ring2 - compute only */ 3458 /* Set ring buffer size */ 3459 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 3460 rb_bufsz = order_base_2(ring->ring_size / 8); 3461 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 3462 #ifdef __BIG_ENDIAN 3463 tmp |= BUF_SWAP_32BIT; 3464 #endif 3465 WREG32(CP_RB2_CNTL, tmp); 3466 3467 /* Initialize the ring buffer's read and write pointers */ 3468 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA); 3469 ring->wptr = 0; 3470 WREG32(CP_RB2_WPTR, ring->wptr); 3471 3472 /* set the wb address whether it's enabled or not */ 3473 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC); 3474 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF); 3475 3476 mdelay(1); 3477 WREG32(CP_RB2_CNTL, tmp); 3478 3479 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8); 3480 3481 ring->rptr = RREG32(CP_RB2_RPTR); 3482 3483 /* start the rings */ 3484 si_cp_start(rdev); 3485 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; 3486 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true; 3487 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true; 3488 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); 3489 if (r) { 3490 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; 3491 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3492 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3493 return r; 3494 } 3495 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); 3496 if (r) { 3497 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; 3498 } 3499 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); 3500 if (r) { 3501 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; 3502 } 3503 3504 return 0; 3505 } 3506 3507 u32 si_gpu_check_soft_reset(struct radeon_device *rdev) 3508 { 3509 u32 reset_mask = 0; 3510 u32 tmp; 3511 3512 /* GRBM_STATUS */ 3513 tmp = RREG32(GRBM_STATUS); 3514 if (tmp & (PA_BUSY | SC_BUSY | 3515 BCI_BUSY | SX_BUSY | 3516 TA_BUSY | VGT_BUSY | 3517 DB_BUSY | CB_BUSY | 3518 GDS_BUSY | SPI_BUSY | 3519 IA_BUSY | IA_BUSY_NO_DMA)) 3520 reset_mask |= RADEON_RESET_GFX; 3521 3522 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING | 3523 CP_BUSY | CP_COHERENCY_BUSY)) 3524 reset_mask |= RADEON_RESET_CP; 3525 3526 if (tmp & GRBM_EE_BUSY) 3527 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP; 3528 3529 /* GRBM_STATUS2 */ 3530 tmp = RREG32(GRBM_STATUS2); 3531 if (tmp & (RLC_RQ_PENDING | RLC_BUSY)) 3532 reset_mask |= RADEON_RESET_RLC; 3533 3534 /* DMA_STATUS_REG 0 */ 3535 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET); 3536 if (!(tmp & DMA_IDLE)) 3537 reset_mask |= RADEON_RESET_DMA; 3538 3539 /* DMA_STATUS_REG 1 */ 3540 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET); 3541 if (!(tmp & DMA_IDLE)) 3542 reset_mask |= RADEON_RESET_DMA1; 3543 3544 /* SRBM_STATUS2 */ 3545 tmp = RREG32(SRBM_STATUS2); 3546 if (tmp & DMA_BUSY) 3547 reset_mask |= RADEON_RESET_DMA; 3548 3549 if (tmp & DMA1_BUSY) 3550 reset_mask |= RADEON_RESET_DMA1; 3551 3552 /* SRBM_STATUS */ 3553 tmp = RREG32(SRBM_STATUS); 3554 3555 if (tmp & IH_BUSY) 3556 reset_mask |= RADEON_RESET_IH; 3557 3558 if (tmp & SEM_BUSY) 3559 reset_mask |= RADEON_RESET_SEM; 3560 3561 if (tmp & GRBM_RQ_PENDING) 3562 reset_mask |= RADEON_RESET_GRBM; 3563 3564 if (tmp & VMC_BUSY) 3565 reset_mask |= RADEON_RESET_VMC; 3566 3567 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY | 3568 MCC_BUSY | MCD_BUSY)) 3569 reset_mask |= RADEON_RESET_MC; 3570 3571 if (evergreen_is_display_hung(rdev)) 3572 reset_mask |= RADEON_RESET_DISPLAY; 3573 3574 /* VM_L2_STATUS */ 3575 tmp = RREG32(VM_L2_STATUS); 3576 if (tmp & L2_BUSY) 3577 reset_mask |= RADEON_RESET_VMC; 3578 3579 /* Skip MC reset as it's mostly likely not hung, just busy */ 3580 if (reset_mask & RADEON_RESET_MC) { 3581 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask); 3582 reset_mask &= ~RADEON_RESET_MC; 3583 } 3584 3585 return reset_mask; 3586 } 3587 3588 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) 3589 { 3590 struct evergreen_mc_save save; 3591 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 3592 u32 tmp; 3593 3594 if (reset_mask == 0) 3595 return; 3596 3597 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); 3598 3599 evergreen_print_gpu_status_regs(rdev); 3600 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 3601 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR)); 3602 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 3603 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS)); 3604 3605 /* Disable CP parsing/prefetching */ 3606 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT); 3607 3608 if (reset_mask & RADEON_RESET_DMA) { 3609 /* dma0 */ 3610 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET); 3611 tmp &= ~DMA_RB_ENABLE; 3612 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp); 3613 } 3614 if (reset_mask & RADEON_RESET_DMA1) { 3615 /* dma1 */ 3616 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET); 3617 tmp &= ~DMA_RB_ENABLE; 3618 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp); 3619 } 3620 3621 udelay(50); 3622 3623 evergreen_mc_stop(rdev, &save); 3624 if (evergreen_mc_wait_for_idle(rdev)) { 3625 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3626 } 3627 3628 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) { 3629 grbm_soft_reset = SOFT_RESET_CB | 3630 SOFT_RESET_DB | 3631 SOFT_RESET_GDS | 3632 SOFT_RESET_PA | 3633 SOFT_RESET_SC | 3634 SOFT_RESET_BCI | 3635 SOFT_RESET_SPI | 3636 SOFT_RESET_SX | 3637 SOFT_RESET_TC | 3638 SOFT_RESET_TA | 3639 SOFT_RESET_VGT | 3640 SOFT_RESET_IA; 3641 } 3642 3643 if (reset_mask & RADEON_RESET_CP) { 3644 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT; 3645 3646 srbm_soft_reset |= SOFT_RESET_GRBM; 3647 } 3648 3649 if (reset_mask & RADEON_RESET_DMA) 3650 srbm_soft_reset |= SOFT_RESET_DMA; 3651 3652 if (reset_mask & RADEON_RESET_DMA1) 3653 srbm_soft_reset |= SOFT_RESET_DMA1; 3654 3655 if (reset_mask & RADEON_RESET_DISPLAY) 3656 srbm_soft_reset |= SOFT_RESET_DC; 3657 3658 if (reset_mask & RADEON_RESET_RLC) 3659 grbm_soft_reset |= SOFT_RESET_RLC; 3660 3661 if (reset_mask & RADEON_RESET_SEM) 3662 srbm_soft_reset |= SOFT_RESET_SEM; 3663 3664 if (reset_mask & RADEON_RESET_IH) 3665 srbm_soft_reset |= SOFT_RESET_IH; 3666 3667 if (reset_mask & RADEON_RESET_GRBM) 3668 srbm_soft_reset |= SOFT_RESET_GRBM; 3669 3670 if (reset_mask & RADEON_RESET_VMC) 3671 srbm_soft_reset |= SOFT_RESET_VMC; 3672 3673 if (reset_mask & RADEON_RESET_MC) 3674 srbm_soft_reset |= SOFT_RESET_MC; 3675 3676 if (grbm_soft_reset) { 3677 tmp = RREG32(GRBM_SOFT_RESET); 3678 tmp |= grbm_soft_reset; 3679 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 3680 WREG32(GRBM_SOFT_RESET, tmp); 3681 tmp = RREG32(GRBM_SOFT_RESET); 3682 3683 udelay(50); 3684 3685 tmp &= ~grbm_soft_reset; 3686 WREG32(GRBM_SOFT_RESET, tmp); 3687 tmp = RREG32(GRBM_SOFT_RESET); 3688 } 3689 3690 if (srbm_soft_reset) { 3691 tmp = RREG32(SRBM_SOFT_RESET); 3692 tmp |= srbm_soft_reset; 3693 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 3694 WREG32(SRBM_SOFT_RESET, tmp); 3695 tmp = RREG32(SRBM_SOFT_RESET); 3696 3697 udelay(50); 3698 3699 tmp &= ~srbm_soft_reset; 3700 WREG32(SRBM_SOFT_RESET, tmp); 3701 tmp = RREG32(SRBM_SOFT_RESET); 3702 } 3703 3704 /* Wait a little for things to settle down */ 3705 udelay(50); 3706 3707 evergreen_mc_resume(rdev, &save); 3708 udelay(50); 3709 3710 evergreen_print_gpu_status_regs(rdev); 3711 } 3712 3713 int si_asic_reset(struct radeon_device *rdev) 3714 { 3715 u32 reset_mask; 3716 3717 reset_mask = si_gpu_check_soft_reset(rdev); 3718 3719 if (reset_mask) 3720 r600_set_bios_scratch_engine_hung(rdev, true); 3721 3722 si_gpu_soft_reset(rdev, reset_mask); 3723 3724 reset_mask = si_gpu_check_soft_reset(rdev); 3725 3726 if (!reset_mask) 3727 r600_set_bios_scratch_engine_hung(rdev, false); 3728 3729 return 0; 3730 } 3731 3732 /** 3733 * si_gfx_is_lockup - Check if the GFX engine is locked up 3734 * 3735 * @rdev: radeon_device pointer 3736 * @ring: radeon_ring structure holding ring information 3737 * 3738 * Check if the GFX engine is locked up. 3739 * Returns true if the engine appears to be locked up, false if not. 3740 */ 3741 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) 3742 { 3743 u32 reset_mask = si_gpu_check_soft_reset(rdev); 3744 3745 if (!(reset_mask & (RADEON_RESET_GFX | 3746 RADEON_RESET_COMPUTE | 3747 RADEON_RESET_CP))) { 3748 radeon_ring_lockup_update(ring); 3749 return false; 3750 } 3751 /* force CP activities */ 3752 radeon_ring_force_activity(rdev, ring); 3753 return radeon_ring_test_lockup(rdev, ring); 3754 } 3755 3756 /* MC */ 3757 static void si_mc_program(struct radeon_device *rdev) 3758 { 3759 struct evergreen_mc_save save; 3760 u32 tmp; 3761 int i, j; 3762 3763 /* Initialize HDP */ 3764 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 3765 WREG32((0x2c14 + j), 0x00000000); 3766 WREG32((0x2c18 + j), 0x00000000); 3767 WREG32((0x2c1c + j), 0x00000000); 3768 WREG32((0x2c20 + j), 0x00000000); 3769 WREG32((0x2c24 + j), 0x00000000); 3770 } 3771 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 3772 3773 evergreen_mc_stop(rdev, &save); 3774 if (radeon_mc_wait_for_idle(rdev)) { 3775 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3776 } 3777 if (!ASIC_IS_NODCE(rdev)) 3778 /* Lockout access through VGA aperture*/ 3779 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 3780 /* Update configuration */ 3781 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 3782 rdev->mc.vram_start >> 12); 3783 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 3784 rdev->mc.vram_end >> 12); 3785 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 3786 rdev->vram_scratch.gpu_addr >> 12); 3787 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 3788 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 3789 WREG32(MC_VM_FB_LOCATION, tmp); 3790 /* XXX double check these! */ 3791 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 3792 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 3793 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 3794 WREG32(MC_VM_AGP_BASE, 0); 3795 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 3796 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 3797 if (radeon_mc_wait_for_idle(rdev)) { 3798 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 3799 } 3800 evergreen_mc_resume(rdev, &save); 3801 if (!ASIC_IS_NODCE(rdev)) { 3802 /* we need to own VRAM, so turn off the VGA renderer here 3803 * to stop it overwriting our objects */ 3804 rv515_vga_render_disable(rdev); 3805 } 3806 } 3807 3808 void si_vram_gtt_location(struct radeon_device *rdev, 3809 struct radeon_mc *mc) 3810 { 3811 if (mc->mc_vram_size > 0xFFC0000000ULL) { 3812 /* leave room for at least 1024M GTT */ 3813 dev_warn(rdev->dev, "limiting VRAM\n"); 3814 mc->real_vram_size = 0xFFC0000000ULL; 3815 mc->mc_vram_size = 0xFFC0000000ULL; 3816 } 3817 radeon_vram_location(rdev, &rdev->mc, 0); 3818 rdev->mc.gtt_base_align = 0; 3819 radeon_gtt_location(rdev, mc); 3820 } 3821 3822 static int si_mc_init(struct radeon_device *rdev) 3823 { 3824 u32 tmp; 3825 int chansize, numchan; 3826 3827 /* Get VRAM informations */ 3828 rdev->mc.vram_is_ddr = true; 3829 tmp = RREG32(MC_ARB_RAMCFG); 3830 if (tmp & CHANSIZE_OVERRIDE) { 3831 chansize = 16; 3832 } else if (tmp & CHANSIZE_MASK) { 3833 chansize = 64; 3834 } else { 3835 chansize = 32; 3836 } 3837 tmp = RREG32(MC_SHARED_CHMAP); 3838 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 3839 case 0: 3840 default: 3841 numchan = 1; 3842 break; 3843 case 1: 3844 numchan = 2; 3845 break; 3846 case 2: 3847 numchan = 4; 3848 break; 3849 case 3: 3850 numchan = 8; 3851 break; 3852 case 4: 3853 numchan = 3; 3854 break; 3855 case 5: 3856 numchan = 6; 3857 break; 3858 case 6: 3859 numchan = 10; 3860 break; 3861 case 7: 3862 numchan = 12; 3863 break; 3864 case 8: 3865 numchan = 16; 3866 break; 3867 } 3868 rdev->mc.vram_width = numchan * chansize; 3869 /* Could aper size report 0 ? */ 3870 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 3871 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 3872 /* size in MB on si */ 3873 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3874 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; 3875 rdev->mc.visible_vram_size = rdev->mc.aper_size; 3876 si_vram_gtt_location(rdev, &rdev->mc); 3877 radeon_update_bandwidth_info(rdev); 3878 3879 return 0; 3880 } 3881 3882 /* 3883 * GART 3884 */ 3885 void si_pcie_gart_tlb_flush(struct radeon_device *rdev) 3886 { 3887 /* flush hdp cache */ 3888 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 3889 3890 /* bits 0-15 are the VM contexts0-15 */ 3891 WREG32(VM_INVALIDATE_REQUEST, 1); 3892 } 3893 3894 static int si_pcie_gart_enable(struct radeon_device *rdev) 3895 { 3896 int r, i; 3897 3898 if (rdev->gart.robj == NULL) { 3899 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 3900 return -EINVAL; 3901 } 3902 r = radeon_gart_table_vram_pin(rdev); 3903 if (r) 3904 return r; 3905 radeon_gart_restore(rdev); 3906 /* Setup TLB control */ 3907 WREG32(MC_VM_MX_L1_TLB_CNTL, 3908 (0xA << 7) | 3909 ENABLE_L1_TLB | 3910 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 3911 ENABLE_ADVANCED_DRIVER_MODEL | 3912 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 3913 /* Setup L2 cache */ 3914 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | 3915 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 3916 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 3917 EFFECTIVE_L2_QUEUE_SIZE(7) | 3918 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 3919 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE); 3920 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 3921 L2_CACHE_BIGK_FRAGMENT_SIZE(0)); 3922 /* setup context0 */ 3923 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 3924 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 3925 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 3926 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 3927 (u32)(rdev->dummy_page.addr >> 12)); 3928 WREG32(VM_CONTEXT0_CNTL2, 0); 3929 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 3930 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT)); 3931 3932 WREG32(0x15D4, 0); 3933 WREG32(0x15D8, 0); 3934 WREG32(0x15DC, 0); 3935 3936 /* empty context1-15 */ 3937 /* set vm size, must be a multiple of 4 */ 3938 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0); 3939 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn); 3940 /* Assign the pt base to something valid for now; the pts used for 3941 * the VMs are determined by the application and setup and assigned 3942 * on the fly in the vm part of radeon_gart.c 3943 */ 3944 for (i = 1; i < 16; i++) { 3945 if (i < 8) 3946 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2), 3947 rdev->gart.table_addr >> 12); 3948 else 3949 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2), 3950 rdev->gart.table_addr >> 12); 3951 } 3952 3953 /* enable context1-15 */ 3954 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR, 3955 (u32)(rdev->dummy_page.addr >> 12)); 3956 WREG32(VM_CONTEXT1_CNTL2, 4); 3957 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) | 3958 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 3959 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT | 3960 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT | 3961 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT | 3962 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT | 3963 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT | 3964 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT | 3965 VALID_PROTECTION_FAULT_ENABLE_DEFAULT | 3966 READ_PROTECTION_FAULT_ENABLE_INTERRUPT | 3967 READ_PROTECTION_FAULT_ENABLE_DEFAULT | 3968 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT | 3969 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT); 3970 3971 si_pcie_gart_tlb_flush(rdev); 3972 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 3973 (unsigned)(rdev->mc.gtt_size >> 20), 3974 (unsigned long long)rdev->gart.table_addr); 3975 rdev->gart.ready = true; 3976 return 0; 3977 } 3978 3979 static void si_pcie_gart_disable(struct radeon_device *rdev) 3980 { 3981 /* Disable all tables */ 3982 WREG32(VM_CONTEXT0_CNTL, 0); 3983 WREG32(VM_CONTEXT1_CNTL, 0); 3984 /* Setup TLB control */ 3985 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS | 3986 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU); 3987 /* Setup L2 cache */ 3988 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 3989 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE | 3990 EFFECTIVE_L2_QUEUE_SIZE(7) | 3991 CONTEXT1_IDENTITY_ACCESS_MODE(1)); 3992 WREG32(VM_L2_CNTL2, 0); 3993 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY | 3994 L2_CACHE_BIGK_FRAGMENT_SIZE(0)); 3995 radeon_gart_table_vram_unpin(rdev); 3996 } 3997 3998 static void si_pcie_gart_fini(struct radeon_device *rdev) 3999 { 4000 si_pcie_gart_disable(rdev); 4001 radeon_gart_table_vram_free(rdev); 4002 radeon_gart_fini(rdev); 4003 } 4004 4005 /* vm parser */ 4006 static bool si_vm_reg_valid(u32 reg) 4007 { 4008 /* context regs are fine */ 4009 if (reg >= 0x28000) 4010 return true; 4011 4012 /* check config regs */ 4013 switch (reg) { 4014 case GRBM_GFX_INDEX: 4015 case CP_STRMOUT_CNTL: 4016 case VGT_VTX_VECT_EJECT_REG: 4017 case VGT_CACHE_INVALIDATION: 4018 case VGT_ESGS_RING_SIZE: 4019 case VGT_GSVS_RING_SIZE: 4020 case VGT_GS_VERTEX_REUSE: 4021 case VGT_PRIMITIVE_TYPE: 4022 case VGT_INDEX_TYPE: 4023 case VGT_NUM_INDICES: 4024 case VGT_NUM_INSTANCES: 4025 case VGT_TF_RING_SIZE: 4026 case VGT_HS_OFFCHIP_PARAM: 4027 case VGT_TF_MEMORY_BASE: 4028 case PA_CL_ENHANCE: 4029 case PA_SU_LINE_STIPPLE_VALUE: 4030 case PA_SC_LINE_STIPPLE_STATE: 4031 case PA_SC_ENHANCE: 4032 case SQC_CACHES: 4033 case SPI_STATIC_THREAD_MGMT_1: 4034 case SPI_STATIC_THREAD_MGMT_2: 4035 case SPI_STATIC_THREAD_MGMT_3: 4036 case SPI_PS_MAX_WAVE_ID: 4037 case SPI_CONFIG_CNTL: 4038 case SPI_CONFIG_CNTL_1: 4039 case TA_CNTL_AUX: 4040 return true; 4041 default: 4042 DRM_ERROR("Invalid register 0x%x in CS\n", reg); 4043 return false; 4044 } 4045 } 4046 4047 static int si_vm_packet3_ce_check(struct radeon_device *rdev, 4048 u32 *ib, struct radeon_cs_packet *pkt) 4049 { 4050 switch (pkt->opcode) { 4051 case PACKET3_NOP: 4052 case PACKET3_SET_BASE: 4053 case PACKET3_SET_CE_DE_COUNTERS: 4054 case PACKET3_LOAD_CONST_RAM: 4055 case PACKET3_WRITE_CONST_RAM: 4056 case PACKET3_WRITE_CONST_RAM_OFFSET: 4057 case PACKET3_DUMP_CONST_RAM: 4058 case PACKET3_INCREMENT_CE_COUNTER: 4059 case PACKET3_WAIT_ON_DE_COUNTER: 4060 case PACKET3_CE_WRITE: 4061 break; 4062 default: 4063 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode); 4064 return -EINVAL; 4065 } 4066 return 0; 4067 } 4068 4069 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx) 4070 { 4071 u32 start_reg, reg, i; 4072 u32 command = ib[idx + 4]; 4073 u32 info = ib[idx + 1]; 4074 u32 idx_value = ib[idx]; 4075 if (command & PACKET3_CP_DMA_CMD_SAS) { 4076 /* src address space is register */ 4077 if (((info & 0x60000000) >> 29) == 0) { 4078 start_reg = idx_value << 2; 4079 if (command & PACKET3_CP_DMA_CMD_SAIC) { 4080 reg = start_reg; 4081 if (!si_vm_reg_valid(reg)) { 4082 DRM_ERROR("CP DMA Bad SRC register\n"); 4083 return -EINVAL; 4084 } 4085 } else { 4086 for (i = 0; i < (command & 0x1fffff); i++) { 4087 reg = start_reg + (4 * i); 4088 if (!si_vm_reg_valid(reg)) { 4089 DRM_ERROR("CP DMA Bad SRC register\n"); 4090 return -EINVAL; 4091 } 4092 } 4093 } 4094 } 4095 } 4096 if (command & PACKET3_CP_DMA_CMD_DAS) { 4097 /* dst address space is register */ 4098 if (((info & 0x00300000) >> 20) == 0) { 4099 start_reg = ib[idx + 2]; 4100 if (command & PACKET3_CP_DMA_CMD_DAIC) { 4101 reg = start_reg; 4102 if (!si_vm_reg_valid(reg)) { 4103 DRM_ERROR("CP DMA Bad DST register\n"); 4104 return -EINVAL; 4105 } 4106 } else { 4107 for (i = 0; i < (command & 0x1fffff); i++) { 4108 reg = start_reg + (4 * i); 4109 if (!si_vm_reg_valid(reg)) { 4110 DRM_ERROR("CP DMA Bad DST register\n"); 4111 return -EINVAL; 4112 } 4113 } 4114 } 4115 } 4116 } 4117 return 0; 4118 } 4119 4120 static int si_vm_packet3_gfx_check(struct radeon_device *rdev, 4121 u32 *ib, struct radeon_cs_packet *pkt) 4122 { 4123 int r; 4124 u32 idx = pkt->idx + 1; 4125 u32 idx_value = ib[idx]; 4126 u32 start_reg, end_reg, reg, i; 4127 4128 switch (pkt->opcode) { 4129 case PACKET3_NOP: 4130 case PACKET3_SET_BASE: 4131 case PACKET3_CLEAR_STATE: 4132 case PACKET3_INDEX_BUFFER_SIZE: 4133 case PACKET3_DISPATCH_DIRECT: 4134 case PACKET3_DISPATCH_INDIRECT: 4135 case PACKET3_ALLOC_GDS: 4136 case PACKET3_WRITE_GDS_RAM: 4137 case PACKET3_ATOMIC_GDS: 4138 case PACKET3_ATOMIC: 4139 case PACKET3_OCCLUSION_QUERY: 4140 case PACKET3_SET_PREDICATION: 4141 case PACKET3_COND_EXEC: 4142 case PACKET3_PRED_EXEC: 4143 case PACKET3_DRAW_INDIRECT: 4144 case PACKET3_DRAW_INDEX_INDIRECT: 4145 case PACKET3_INDEX_BASE: 4146 case PACKET3_DRAW_INDEX_2: 4147 case PACKET3_CONTEXT_CONTROL: 4148 case PACKET3_INDEX_TYPE: 4149 case PACKET3_DRAW_INDIRECT_MULTI: 4150 case PACKET3_DRAW_INDEX_AUTO: 4151 case PACKET3_DRAW_INDEX_IMMD: 4152 case PACKET3_NUM_INSTANCES: 4153 case PACKET3_DRAW_INDEX_MULTI_AUTO: 4154 case PACKET3_STRMOUT_BUFFER_UPDATE: 4155 case PACKET3_DRAW_INDEX_OFFSET_2: 4156 case PACKET3_DRAW_INDEX_MULTI_ELEMENT: 4157 case PACKET3_DRAW_INDEX_INDIRECT_MULTI: 4158 case PACKET3_MPEG_INDEX: 4159 case PACKET3_WAIT_REG_MEM: 4160 case PACKET3_MEM_WRITE: 4161 case PACKET3_PFP_SYNC_ME: 4162 case PACKET3_SURFACE_SYNC: 4163 case PACKET3_EVENT_WRITE: 4164 case PACKET3_EVENT_WRITE_EOP: 4165 case PACKET3_EVENT_WRITE_EOS: 4166 case PACKET3_SET_CONTEXT_REG: 4167 case PACKET3_SET_CONTEXT_REG_INDIRECT: 4168 case PACKET3_SET_SH_REG: 4169 case PACKET3_SET_SH_REG_OFFSET: 4170 case PACKET3_INCREMENT_DE_COUNTER: 4171 case PACKET3_WAIT_ON_CE_COUNTER: 4172 case PACKET3_WAIT_ON_AVAIL_BUFFER: 4173 case PACKET3_ME_WRITE: 4174 break; 4175 case PACKET3_COPY_DATA: 4176 if ((idx_value & 0xf00) == 0) { 4177 reg = ib[idx + 3] * 4; 4178 if (!si_vm_reg_valid(reg)) 4179 return -EINVAL; 4180 } 4181 break; 4182 case PACKET3_WRITE_DATA: 4183 if ((idx_value & 0xf00) == 0) { 4184 start_reg = ib[idx + 1] * 4; 4185 if (idx_value & 0x10000) { 4186 if (!si_vm_reg_valid(start_reg)) 4187 return -EINVAL; 4188 } else { 4189 for (i = 0; i < (pkt->count - 2); i++) { 4190 reg = start_reg + (4 * i); 4191 if (!si_vm_reg_valid(reg)) 4192 return -EINVAL; 4193 } 4194 } 4195 } 4196 break; 4197 case PACKET3_COND_WRITE: 4198 if (idx_value & 0x100) { 4199 reg = ib[idx + 5] * 4; 4200 if (!si_vm_reg_valid(reg)) 4201 return -EINVAL; 4202 } 4203 break; 4204 case PACKET3_COPY_DW: 4205 if (idx_value & 0x2) { 4206 reg = ib[idx + 3] * 4; 4207 if (!si_vm_reg_valid(reg)) 4208 return -EINVAL; 4209 } 4210 break; 4211 case PACKET3_SET_CONFIG_REG: 4212 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START; 4213 end_reg = 4 * pkt->count + start_reg - 4; 4214 if ((start_reg < PACKET3_SET_CONFIG_REG_START) || 4215 (start_reg >= PACKET3_SET_CONFIG_REG_END) || 4216 (end_reg >= PACKET3_SET_CONFIG_REG_END)) { 4217 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n"); 4218 return -EINVAL; 4219 } 4220 for (i = 0; i < pkt->count; i++) { 4221 reg = start_reg + (4 * i); 4222 if (!si_vm_reg_valid(reg)) 4223 return -EINVAL; 4224 } 4225 break; 4226 case PACKET3_CP_DMA: 4227 r = si_vm_packet3_cp_dma_check(ib, idx); 4228 if (r) 4229 return r; 4230 break; 4231 default: 4232 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode); 4233 return -EINVAL; 4234 } 4235 return 0; 4236 } 4237 4238 static int si_vm_packet3_compute_check(struct radeon_device *rdev, 4239 u32 *ib, struct radeon_cs_packet *pkt) 4240 { 4241 int r; 4242 u32 idx = pkt->idx + 1; 4243 u32 idx_value = ib[idx]; 4244 u32 start_reg, reg, i; 4245 4246 switch (pkt->opcode) { 4247 case PACKET3_NOP: 4248 case PACKET3_SET_BASE: 4249 case PACKET3_CLEAR_STATE: 4250 case PACKET3_DISPATCH_DIRECT: 4251 case PACKET3_DISPATCH_INDIRECT: 4252 case PACKET3_ALLOC_GDS: 4253 case PACKET3_WRITE_GDS_RAM: 4254 case PACKET3_ATOMIC_GDS: 4255 case PACKET3_ATOMIC: 4256 case PACKET3_OCCLUSION_QUERY: 4257 case PACKET3_SET_PREDICATION: 4258 case PACKET3_COND_EXEC: 4259 case PACKET3_PRED_EXEC: 4260 case PACKET3_CONTEXT_CONTROL: 4261 case PACKET3_STRMOUT_BUFFER_UPDATE: 4262 case PACKET3_WAIT_REG_MEM: 4263 case PACKET3_MEM_WRITE: 4264 case PACKET3_PFP_SYNC_ME: 4265 case PACKET3_SURFACE_SYNC: 4266 case PACKET3_EVENT_WRITE: 4267 case PACKET3_EVENT_WRITE_EOP: 4268 case PACKET3_EVENT_WRITE_EOS: 4269 case PACKET3_SET_CONTEXT_REG: 4270 case PACKET3_SET_CONTEXT_REG_INDIRECT: 4271 case PACKET3_SET_SH_REG: 4272 case PACKET3_SET_SH_REG_OFFSET: 4273 case PACKET3_INCREMENT_DE_COUNTER: 4274 case PACKET3_WAIT_ON_CE_COUNTER: 4275 case PACKET3_WAIT_ON_AVAIL_BUFFER: 4276 case PACKET3_ME_WRITE: 4277 break; 4278 case PACKET3_COPY_DATA: 4279 if ((idx_value & 0xf00) == 0) { 4280 reg = ib[idx + 3] * 4; 4281 if (!si_vm_reg_valid(reg)) 4282 return -EINVAL; 4283 } 4284 break; 4285 case PACKET3_WRITE_DATA: 4286 if ((idx_value & 0xf00) == 0) { 4287 start_reg = ib[idx + 1] * 4; 4288 if (idx_value & 0x10000) { 4289 if (!si_vm_reg_valid(start_reg)) 4290 return -EINVAL; 4291 } else { 4292 for (i = 0; i < (pkt->count - 2); i++) { 4293 reg = start_reg + (4 * i); 4294 if (!si_vm_reg_valid(reg)) 4295 return -EINVAL; 4296 } 4297 } 4298 } 4299 break; 4300 case PACKET3_COND_WRITE: 4301 if (idx_value & 0x100) { 4302 reg = ib[idx + 5] * 4; 4303 if (!si_vm_reg_valid(reg)) 4304 return -EINVAL; 4305 } 4306 break; 4307 case PACKET3_COPY_DW: 4308 if (idx_value & 0x2) { 4309 reg = ib[idx + 3] * 4; 4310 if (!si_vm_reg_valid(reg)) 4311 return -EINVAL; 4312 } 4313 break; 4314 case PACKET3_CP_DMA: 4315 r = si_vm_packet3_cp_dma_check(ib, idx); 4316 if (r) 4317 return r; 4318 break; 4319 default: 4320 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode); 4321 return -EINVAL; 4322 } 4323 return 0; 4324 } 4325 4326 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 4327 { 4328 int ret = 0; 4329 u32 idx = 0; 4330 struct radeon_cs_packet pkt; 4331 4332 do { 4333 pkt.idx = idx; 4334 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]); 4335 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]); 4336 pkt.one_reg_wr = 0; 4337 switch (pkt.type) { 4338 case RADEON_PACKET_TYPE0: 4339 dev_err(rdev->dev, "Packet0 not allowed!\n"); 4340 ret = -EINVAL; 4341 break; 4342 case RADEON_PACKET_TYPE2: 4343 idx += 1; 4344 break; 4345 case RADEON_PACKET_TYPE3: 4346 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]); 4347 if (ib->is_const_ib) 4348 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt); 4349 else { 4350 switch (ib->ring) { 4351 case RADEON_RING_TYPE_GFX_INDEX: 4352 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt); 4353 break; 4354 case CAYMAN_RING_TYPE_CP1_INDEX: 4355 case CAYMAN_RING_TYPE_CP2_INDEX: 4356 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt); 4357 break; 4358 default: 4359 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring); 4360 ret = -EINVAL; 4361 break; 4362 } 4363 } 4364 idx += pkt.count + 2; 4365 break; 4366 default: 4367 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type); 4368 ret = -EINVAL; 4369 break; 4370 } 4371 if (ret) 4372 break; 4373 } while (idx < ib->length_dw); 4374 4375 return ret; 4376 } 4377 4378 /* 4379 * vm 4380 */ 4381 int si_vm_init(struct radeon_device *rdev) 4382 { 4383 /* number of VMs */ 4384 rdev->vm_manager.nvm = 16; 4385 /* base offset of vram pages */ 4386 rdev->vm_manager.vram_base_offset = 0; 4387 4388 return 0; 4389 } 4390 4391 void si_vm_fini(struct radeon_device *rdev) 4392 { 4393 } 4394 4395 /** 4396 * si_vm_decode_fault - print human readable fault info 4397 * 4398 * @rdev: radeon_device pointer 4399 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value 4400 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value 4401 * 4402 * Print human readable fault information (SI). 4403 */ 4404 static void si_vm_decode_fault(struct radeon_device *rdev, 4405 u32 status, u32 addr) 4406 { 4407 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT; 4408 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT; 4409 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT; 4410 char *block; 4411 4412 if (rdev->family == CHIP_TAHITI) { 4413 switch (mc_id) { 4414 case 160: 4415 case 144: 4416 case 96: 4417 case 80: 4418 case 224: 4419 case 208: 4420 case 32: 4421 case 16: 4422 block = "CB"; 4423 break; 4424 case 161: 4425 case 145: 4426 case 97: 4427 case 81: 4428 case 225: 4429 case 209: 4430 case 33: 4431 case 17: 4432 block = "CB_FMASK"; 4433 break; 4434 case 162: 4435 case 146: 4436 case 98: 4437 case 82: 4438 case 226: 4439 case 210: 4440 case 34: 4441 case 18: 4442 block = "CB_CMASK"; 4443 break; 4444 case 163: 4445 case 147: 4446 case 99: 4447 case 83: 4448 case 227: 4449 case 211: 4450 case 35: 4451 case 19: 4452 block = "CB_IMMED"; 4453 break; 4454 case 164: 4455 case 148: 4456 case 100: 4457 case 84: 4458 case 228: 4459 case 212: 4460 case 36: 4461 case 20: 4462 block = "DB"; 4463 break; 4464 case 165: 4465 case 149: 4466 case 101: 4467 case 85: 4468 case 229: 4469 case 213: 4470 case 37: 4471 case 21: 4472 block = "DB_HTILE"; 4473 break; 4474 case 167: 4475 case 151: 4476 case 103: 4477 case 87: 4478 case 231: 4479 case 215: 4480 case 39: 4481 case 23: 4482 block = "DB_STEN"; 4483 break; 4484 case 72: 4485 case 68: 4486 case 64: 4487 case 8: 4488 case 4: 4489 case 0: 4490 case 136: 4491 case 132: 4492 case 128: 4493 case 200: 4494 case 196: 4495 case 192: 4496 block = "TC"; 4497 break; 4498 case 112: 4499 case 48: 4500 block = "CP"; 4501 break; 4502 case 49: 4503 case 177: 4504 case 50: 4505 case 178: 4506 block = "SH"; 4507 break; 4508 case 53: 4509 case 190: 4510 block = "VGT"; 4511 break; 4512 case 117: 4513 block = "IH"; 4514 break; 4515 case 51: 4516 case 115: 4517 block = "RLC"; 4518 break; 4519 case 119: 4520 case 183: 4521 block = "DMA0"; 4522 break; 4523 case 61: 4524 block = "DMA1"; 4525 break; 4526 case 248: 4527 case 120: 4528 block = "HDP"; 4529 break; 4530 default: 4531 block = "unknown"; 4532 break; 4533 } 4534 } else { 4535 switch (mc_id) { 4536 case 32: 4537 case 16: 4538 case 96: 4539 case 80: 4540 case 160: 4541 case 144: 4542 case 224: 4543 case 208: 4544 block = "CB"; 4545 break; 4546 case 33: 4547 case 17: 4548 case 97: 4549 case 81: 4550 case 161: 4551 case 145: 4552 case 225: 4553 case 209: 4554 block = "CB_FMASK"; 4555 break; 4556 case 34: 4557 case 18: 4558 case 98: 4559 case 82: 4560 case 162: 4561 case 146: 4562 case 226: 4563 case 210: 4564 block = "CB_CMASK"; 4565 break; 4566 case 35: 4567 case 19: 4568 case 99: 4569 case 83: 4570 case 163: 4571 case 147: 4572 case 227: 4573 case 211: 4574 block = "CB_IMMED"; 4575 break; 4576 case 36: 4577 case 20: 4578 case 100: 4579 case 84: 4580 case 164: 4581 case 148: 4582 case 228: 4583 case 212: 4584 block = "DB"; 4585 break; 4586 case 37: 4587 case 21: 4588 case 101: 4589 case 85: 4590 case 165: 4591 case 149: 4592 case 229: 4593 case 213: 4594 block = "DB_HTILE"; 4595 break; 4596 case 39: 4597 case 23: 4598 case 103: 4599 case 87: 4600 case 167: 4601 case 151: 4602 case 231: 4603 case 215: 4604 block = "DB_STEN"; 4605 break; 4606 case 72: 4607 case 68: 4608 case 8: 4609 case 4: 4610 case 136: 4611 case 132: 4612 case 200: 4613 case 196: 4614 block = "TC"; 4615 break; 4616 case 112: 4617 case 48: 4618 block = "CP"; 4619 break; 4620 case 49: 4621 case 177: 4622 case 50: 4623 case 178: 4624 block = "SH"; 4625 break; 4626 case 53: 4627 block = "VGT"; 4628 break; 4629 case 117: 4630 block = "IH"; 4631 break; 4632 case 51: 4633 case 115: 4634 block = "RLC"; 4635 break; 4636 case 119: 4637 case 183: 4638 block = "DMA0"; 4639 break; 4640 case 61: 4641 block = "DMA1"; 4642 break; 4643 case 248: 4644 case 120: 4645 block = "HDP"; 4646 break; 4647 default: 4648 block = "unknown"; 4649 break; 4650 } 4651 } 4652 4653 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n", 4654 protections, vmid, addr, 4655 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read", 4656 block, mc_id); 4657 } 4658 4659 /** 4660 * si_vm_set_page - update the page tables using the CP 4661 * 4662 * @rdev: radeon_device pointer 4663 * @ib: indirect buffer to fill with commands 4664 * @pe: addr of the page entry 4665 * @addr: dst addr to write into pe 4666 * @count: number of page entries to update 4667 * @incr: increase next addr by incr bytes 4668 * @flags: access flags 4669 * 4670 * Update the page tables using the CP (SI). 4671 */ 4672 void si_vm_set_page(struct radeon_device *rdev, 4673 struct radeon_ib *ib, 4674 uint64_t pe, 4675 uint64_t addr, unsigned count, 4676 uint32_t incr, uint32_t flags) 4677 { 4678 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags); 4679 uint64_t value; 4680 unsigned ndw; 4681 4682 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) { 4683 while (count) { 4684 ndw = 2 + count * 2; 4685 if (ndw > 0x3FFE) 4686 ndw = 0x3FFE; 4687 4688 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw); 4689 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) | 4690 WRITE_DATA_DST_SEL(1)); 4691 ib->ptr[ib->length_dw++] = pe; 4692 ib->ptr[ib->length_dw++] = upper_32_bits(pe); 4693 for (; ndw > 2; ndw -= 2, --count, pe += 8) { 4694 if (flags & RADEON_VM_PAGE_SYSTEM) { 4695 value = radeon_vm_map_gart(rdev, addr); 4696 value &= 0xFFFFFFFFFFFFF000ULL; 4697 } else if (flags & RADEON_VM_PAGE_VALID) { 4698 value = addr; 4699 } else { 4700 value = 0; 4701 } 4702 addr += incr; 4703 value |= r600_flags; 4704 ib->ptr[ib->length_dw++] = value; 4705 ib->ptr[ib->length_dw++] = upper_32_bits(value); 4706 } 4707 } 4708 } else { 4709 /* DMA */ 4710 si_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags); 4711 } 4712 } 4713 4714 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm) 4715 { 4716 struct radeon_ring *ring = &rdev->ring[ridx]; 4717 4718 if (vm == NULL) 4719 return; 4720 4721 /* write new base address */ 4722 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4723 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 4724 WRITE_DATA_DST_SEL(0))); 4725 4726 if (vm->id < 8) { 4727 radeon_ring_write(ring, 4728 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2); 4729 } else { 4730 radeon_ring_write(ring, 4731 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2); 4732 } 4733 radeon_ring_write(ring, 0); 4734 radeon_ring_write(ring, vm->pd_gpu_addr >> 12); 4735 4736 /* flush hdp cache */ 4737 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4738 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 4739 WRITE_DATA_DST_SEL(0))); 4740 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2); 4741 radeon_ring_write(ring, 0); 4742 radeon_ring_write(ring, 0x1); 4743 4744 /* bits 0-15 are the VM contexts0-15 */ 4745 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 4746 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 4747 WRITE_DATA_DST_SEL(0))); 4748 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2); 4749 radeon_ring_write(ring, 0); 4750 radeon_ring_write(ring, 1 << vm->id); 4751 4752 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 4753 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 4754 radeon_ring_write(ring, 0x0); 4755 } 4756 4757 /* 4758 * Power and clock gating 4759 */ 4760 static void si_wait_for_rlc_serdes(struct radeon_device *rdev) 4761 { 4762 int i; 4763 4764 for (i = 0; i < rdev->usec_timeout; i++) { 4765 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0) 4766 break; 4767 udelay(1); 4768 } 4769 4770 for (i = 0; i < rdev->usec_timeout; i++) { 4771 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0) 4772 break; 4773 udelay(1); 4774 } 4775 } 4776 4777 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev, 4778 bool enable) 4779 { 4780 u32 tmp = RREG32(CP_INT_CNTL_RING0); 4781 u32 mask; 4782 int i; 4783 4784 if (enable) 4785 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4786 else 4787 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 4788 WREG32(CP_INT_CNTL_RING0, tmp); 4789 4790 if (!enable) { 4791 /* read a gfx register */ 4792 tmp = RREG32(DB_DEPTH_INFO); 4793 4794 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS; 4795 for (i = 0; i < rdev->usec_timeout; i++) { 4796 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS)) 4797 break; 4798 udelay(1); 4799 } 4800 } 4801 } 4802 4803 static void si_set_uvd_dcm(struct radeon_device *rdev, 4804 bool sw_mode) 4805 { 4806 u32 tmp, tmp2; 4807 4808 tmp = RREG32(UVD_CGC_CTRL); 4809 tmp &= ~(CLK_OD_MASK | CG_DT_MASK); 4810 tmp |= DCM | CG_DT(1) | CLK_OD(4); 4811 4812 if (sw_mode) { 4813 tmp &= ~0x7ffff800; 4814 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7); 4815 } else { 4816 tmp |= 0x7ffff800; 4817 tmp2 = 0; 4818 } 4819 4820 WREG32(UVD_CGC_CTRL, tmp); 4821 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2); 4822 } 4823 4824 void si_init_uvd_internal_cg(struct radeon_device *rdev) 4825 { 4826 bool hw_mode = true; 4827 4828 if (hw_mode) { 4829 si_set_uvd_dcm(rdev, false); 4830 } else { 4831 u32 tmp = RREG32(UVD_CGC_CTRL); 4832 tmp &= ~DCM; 4833 WREG32(UVD_CGC_CTRL, tmp); 4834 } 4835 } 4836 4837 static u32 si_halt_rlc(struct radeon_device *rdev) 4838 { 4839 u32 data, orig; 4840 4841 orig = data = RREG32(RLC_CNTL); 4842 4843 if (data & RLC_ENABLE) { 4844 data &= ~RLC_ENABLE; 4845 WREG32(RLC_CNTL, data); 4846 4847 si_wait_for_rlc_serdes(rdev); 4848 } 4849 4850 return orig; 4851 } 4852 4853 static void si_update_rlc(struct radeon_device *rdev, u32 rlc) 4854 { 4855 u32 tmp; 4856 4857 tmp = RREG32(RLC_CNTL); 4858 if (tmp != rlc) 4859 WREG32(RLC_CNTL, rlc); 4860 } 4861 4862 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable) 4863 { 4864 u32 data, orig; 4865 4866 orig = data = RREG32(DMA_PG); 4867 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA)) 4868 data |= PG_CNTL_ENABLE; 4869 else 4870 data &= ~PG_CNTL_ENABLE; 4871 if (orig != data) 4872 WREG32(DMA_PG, data); 4873 } 4874 4875 static void si_init_dma_pg(struct radeon_device *rdev) 4876 { 4877 u32 tmp; 4878 4879 WREG32(DMA_PGFSM_WRITE, 0x00002000); 4880 WREG32(DMA_PGFSM_CONFIG, 0x100010ff); 4881 4882 for (tmp = 0; tmp < 5; tmp++) 4883 WREG32(DMA_PGFSM_WRITE, 0); 4884 } 4885 4886 static void si_enable_gfx_cgpg(struct radeon_device *rdev, 4887 bool enable) 4888 { 4889 u32 tmp; 4890 4891 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG)) { 4892 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10); 4893 WREG32(RLC_TTOP_D, tmp); 4894 4895 tmp = RREG32(RLC_PG_CNTL); 4896 tmp |= GFX_PG_ENABLE; 4897 WREG32(RLC_PG_CNTL, tmp); 4898 4899 tmp = RREG32(RLC_AUTO_PG_CTRL); 4900 tmp |= AUTO_PG_EN; 4901 WREG32(RLC_AUTO_PG_CTRL, tmp); 4902 } else { 4903 tmp = RREG32(RLC_AUTO_PG_CTRL); 4904 tmp &= ~AUTO_PG_EN; 4905 WREG32(RLC_AUTO_PG_CTRL, tmp); 4906 4907 tmp = RREG32(DB_RENDER_CONTROL); 4908 } 4909 } 4910 4911 static void si_init_gfx_cgpg(struct radeon_device *rdev) 4912 { 4913 u32 tmp; 4914 4915 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 4916 4917 tmp = RREG32(RLC_PG_CNTL); 4918 tmp |= GFX_PG_SRC; 4919 WREG32(RLC_PG_CNTL, tmp); 4920 4921 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 4922 4923 tmp = RREG32(RLC_AUTO_PG_CTRL); 4924 4925 tmp &= ~GRBM_REG_SGIT_MASK; 4926 tmp |= GRBM_REG_SGIT(0x700); 4927 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK; 4928 WREG32(RLC_AUTO_PG_CTRL, tmp); 4929 } 4930 4931 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) 4932 { 4933 u32 mask = 0, tmp, tmp1; 4934 int i; 4935 4936 si_select_se_sh(rdev, se, sh); 4937 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG); 4938 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG); 4939 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 4940 4941 tmp &= 0xffff0000; 4942 4943 tmp |= tmp1; 4944 tmp >>= 16; 4945 4946 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) { 4947 mask <<= 1; 4948 mask |= 1; 4949 } 4950 4951 return (~tmp) & mask; 4952 } 4953 4954 static void si_init_ao_cu_mask(struct radeon_device *rdev) 4955 { 4956 u32 i, j, k, active_cu_number = 0; 4957 u32 mask, counter, cu_bitmap; 4958 u32 tmp = 0; 4959 4960 for (i = 0; i < rdev->config.si.max_shader_engines; i++) { 4961 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) { 4962 mask = 1; 4963 cu_bitmap = 0; 4964 counter = 0; 4965 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) { 4966 if (si_get_cu_active_bitmap(rdev, i, j) & mask) { 4967 if (counter < 2) 4968 cu_bitmap |= mask; 4969 counter++; 4970 } 4971 mask <<= 1; 4972 } 4973 4974 active_cu_number += counter; 4975 tmp |= (cu_bitmap << (i * 16 + j * 8)); 4976 } 4977 } 4978 4979 WREG32(RLC_PG_AO_CU_MASK, tmp); 4980 4981 tmp = RREG32(RLC_MAX_PG_CU); 4982 tmp &= ~MAX_PU_CU_MASK; 4983 tmp |= MAX_PU_CU(active_cu_number); 4984 WREG32(RLC_MAX_PG_CU, tmp); 4985 } 4986 4987 static void si_enable_cgcg(struct radeon_device *rdev, 4988 bool enable) 4989 { 4990 u32 data, orig, tmp; 4991 4992 orig = data = RREG32(RLC_CGCG_CGLS_CTRL); 4993 4994 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { 4995 si_enable_gui_idle_interrupt(rdev, true); 4996 4997 WREG32(RLC_GCPM_GENERAL_3, 0x00000080); 4998 4999 tmp = si_halt_rlc(rdev); 5000 5001 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5002 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5003 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff); 5004 5005 si_wait_for_rlc_serdes(rdev); 5006 5007 si_update_rlc(rdev, tmp); 5008 5009 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff); 5010 5011 data |= CGCG_EN | CGLS_EN; 5012 } else { 5013 si_enable_gui_idle_interrupt(rdev, false); 5014 5015 RREG32(CB_CGTT_SCLK_CTRL); 5016 RREG32(CB_CGTT_SCLK_CTRL); 5017 RREG32(CB_CGTT_SCLK_CTRL); 5018 RREG32(CB_CGTT_SCLK_CTRL); 5019 5020 data &= ~(CGCG_EN | CGLS_EN); 5021 } 5022 5023 if (orig != data) 5024 WREG32(RLC_CGCG_CGLS_CTRL, data); 5025 } 5026 5027 static void si_enable_mgcg(struct radeon_device *rdev, 5028 bool enable) 5029 { 5030 u32 data, orig, tmp = 0; 5031 5032 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { 5033 orig = data = RREG32(CGTS_SM_CTRL_REG); 5034 data = 0x96940200; 5035 if (orig != data) 5036 WREG32(CGTS_SM_CTRL_REG, data); 5037 5038 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { 5039 orig = data = RREG32(CP_MEM_SLP_CNTL); 5040 data |= CP_MEM_LS_EN; 5041 if (orig != data) 5042 WREG32(CP_MEM_SLP_CNTL, data); 5043 } 5044 5045 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5046 data &= 0xffffffc0; 5047 if (orig != data) 5048 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5049 5050 tmp = si_halt_rlc(rdev); 5051 5052 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5053 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5054 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff); 5055 5056 si_update_rlc(rdev, tmp); 5057 } else { 5058 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE); 5059 data |= 0x00000003; 5060 if (orig != data) 5061 WREG32(RLC_CGTT_MGCG_OVERRIDE, data); 5062 5063 data = RREG32(CP_MEM_SLP_CNTL); 5064 if (data & CP_MEM_LS_EN) { 5065 data &= ~CP_MEM_LS_EN; 5066 WREG32(CP_MEM_SLP_CNTL, data); 5067 } 5068 orig = data = RREG32(CGTS_SM_CTRL_REG); 5069 data |= LS_OVERRIDE | OVERRIDE; 5070 if (orig != data) 5071 WREG32(CGTS_SM_CTRL_REG, data); 5072 5073 tmp = si_halt_rlc(rdev); 5074 5075 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff); 5076 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff); 5077 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff); 5078 5079 si_update_rlc(rdev, tmp); 5080 } 5081 } 5082 5083 static void si_enable_uvd_mgcg(struct radeon_device *rdev, 5084 bool enable) 5085 { 5086 u32 orig, data, tmp; 5087 5088 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { 5089 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 5090 tmp |= 0x3fff; 5091 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); 5092 5093 orig = data = RREG32(UVD_CGC_CTRL); 5094 data |= DCM; 5095 if (orig != data) 5096 WREG32(UVD_CGC_CTRL, data); 5097 5098 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0); 5099 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0); 5100 } else { 5101 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL); 5102 tmp &= ~0x3fff; 5103 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp); 5104 5105 orig = data = RREG32(UVD_CGC_CTRL); 5106 data &= ~DCM; 5107 if (orig != data) 5108 WREG32(UVD_CGC_CTRL, data); 5109 5110 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff); 5111 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff); 5112 } 5113 } 5114 5115 static const u32 mc_cg_registers[] = 5116 { 5117 MC_HUB_MISC_HUB_CG, 5118 MC_HUB_MISC_SIP_CG, 5119 MC_HUB_MISC_VM_CG, 5120 MC_XPB_CLK_GAT, 5121 ATC_MISC_CG, 5122 MC_CITF_MISC_WR_CG, 5123 MC_CITF_MISC_RD_CG, 5124 MC_CITF_MISC_VM_CG, 5125 VM_L2_CG, 5126 }; 5127 5128 static void si_enable_mc_ls(struct radeon_device *rdev, 5129 bool enable) 5130 { 5131 int i; 5132 u32 orig, data; 5133 5134 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 5135 orig = data = RREG32(mc_cg_registers[i]); 5136 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) 5137 data |= MC_LS_ENABLE; 5138 else 5139 data &= ~MC_LS_ENABLE; 5140 if (data != orig) 5141 WREG32(mc_cg_registers[i], data); 5142 } 5143 } 5144 5145 static void si_enable_mc_mgcg(struct radeon_device *rdev, 5146 bool enable) 5147 { 5148 int i; 5149 u32 orig, data; 5150 5151 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) { 5152 orig = data = RREG32(mc_cg_registers[i]); 5153 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) 5154 data |= MC_CG_ENABLE; 5155 else 5156 data &= ~MC_CG_ENABLE; 5157 if (data != orig) 5158 WREG32(mc_cg_registers[i], data); 5159 } 5160 } 5161 5162 static void si_enable_dma_mgcg(struct radeon_device *rdev, 5163 bool enable) 5164 { 5165 u32 orig, data, offset; 5166 int i; 5167 5168 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { 5169 for (i = 0; i < 2; i++) { 5170 if (i == 0) 5171 offset = DMA0_REGISTER_OFFSET; 5172 else 5173 offset = DMA1_REGISTER_OFFSET; 5174 orig = data = RREG32(DMA_POWER_CNTL + offset); 5175 data &= ~MEM_POWER_OVERRIDE; 5176 if (data != orig) 5177 WREG32(DMA_POWER_CNTL + offset, data); 5178 WREG32(DMA_CLK_CTRL + offset, 0x00000100); 5179 } 5180 } else { 5181 for (i = 0; i < 2; i++) { 5182 if (i == 0) 5183 offset = DMA0_REGISTER_OFFSET; 5184 else 5185 offset = DMA1_REGISTER_OFFSET; 5186 orig = data = RREG32(DMA_POWER_CNTL + offset); 5187 data |= MEM_POWER_OVERRIDE; 5188 if (data != orig) 5189 WREG32(DMA_POWER_CNTL + offset, data); 5190 5191 orig = data = RREG32(DMA_CLK_CTRL + offset); 5192 data = 0xff000000; 5193 if (data != orig) 5194 WREG32(DMA_CLK_CTRL + offset, data); 5195 } 5196 } 5197 } 5198 5199 static void si_enable_bif_mgls(struct radeon_device *rdev, 5200 bool enable) 5201 { 5202 u32 orig, data; 5203 5204 orig = data = RREG32_PCIE(PCIE_CNTL2); 5205 5206 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) 5207 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | 5208 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN; 5209 else 5210 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN | 5211 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN); 5212 5213 if (orig != data) 5214 WREG32_PCIE(PCIE_CNTL2, data); 5215 } 5216 5217 static void si_enable_hdp_mgcg(struct radeon_device *rdev, 5218 bool enable) 5219 { 5220 u32 orig, data; 5221 5222 orig = data = RREG32(HDP_HOST_PATH_CNTL); 5223 5224 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) 5225 data &= ~CLOCK_GATING_DIS; 5226 else 5227 data |= CLOCK_GATING_DIS; 5228 5229 if (orig != data) 5230 WREG32(HDP_HOST_PATH_CNTL, data); 5231 } 5232 5233 static void si_enable_hdp_ls(struct radeon_device *rdev, 5234 bool enable) 5235 { 5236 u32 orig, data; 5237 5238 orig = data = RREG32(HDP_MEM_POWER_LS); 5239 5240 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) 5241 data |= HDP_LS_ENABLE; 5242 else 5243 data &= ~HDP_LS_ENABLE; 5244 5245 if (orig != data) 5246 WREG32(HDP_MEM_POWER_LS, data); 5247 } 5248 5249 void si_update_cg(struct radeon_device *rdev, 5250 u32 block, bool enable) 5251 { 5252 if (block & RADEON_CG_BLOCK_GFX) { 5253 /* order matters! */ 5254 if (enable) { 5255 si_enable_mgcg(rdev, true); 5256 si_enable_cgcg(rdev, true); 5257 } else { 5258 si_enable_cgcg(rdev, false); 5259 si_enable_mgcg(rdev, false); 5260 } 5261 } 5262 5263 if (block & RADEON_CG_BLOCK_MC) { 5264 si_enable_mc_mgcg(rdev, enable); 5265 si_enable_mc_ls(rdev, enable); 5266 } 5267 5268 if (block & RADEON_CG_BLOCK_SDMA) { 5269 si_enable_dma_mgcg(rdev, enable); 5270 } 5271 5272 if (block & RADEON_CG_BLOCK_BIF) { 5273 si_enable_bif_mgls(rdev, enable); 5274 } 5275 5276 if (block & RADEON_CG_BLOCK_UVD) { 5277 if (rdev->has_uvd) { 5278 si_enable_uvd_mgcg(rdev, enable); 5279 } 5280 } 5281 5282 if (block & RADEON_CG_BLOCK_HDP) { 5283 si_enable_hdp_mgcg(rdev, enable); 5284 si_enable_hdp_ls(rdev, enable); 5285 } 5286 } 5287 5288 static void si_init_cg(struct radeon_device *rdev) 5289 { 5290 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | 5291 RADEON_CG_BLOCK_MC | 5292 RADEON_CG_BLOCK_SDMA | 5293 RADEON_CG_BLOCK_BIF | 5294 RADEON_CG_BLOCK_HDP), true); 5295 if (rdev->has_uvd) { 5296 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true); 5297 si_init_uvd_internal_cg(rdev); 5298 } 5299 } 5300 5301 static void si_fini_cg(struct radeon_device *rdev) 5302 { 5303 if (rdev->has_uvd) { 5304 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false); 5305 } 5306 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX | 5307 RADEON_CG_BLOCK_MC | 5308 RADEON_CG_BLOCK_SDMA | 5309 RADEON_CG_BLOCK_BIF | 5310 RADEON_CG_BLOCK_HDP), false); 5311 } 5312 5313 u32 si_get_csb_size(struct radeon_device *rdev) 5314 { 5315 u32 count = 0; 5316 const struct cs_section_def *sect = NULL; 5317 const struct cs_extent_def *ext = NULL; 5318 5319 if (rdev->rlc.cs_data == NULL) 5320 return 0; 5321 5322 /* begin clear state */ 5323 count += 2; 5324 /* context control state */ 5325 count += 3; 5326 5327 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 5328 for (ext = sect->section; ext->extent != NULL; ++ext) { 5329 if (sect->id == SECT_CONTEXT) 5330 count += 2 + ext->reg_count; 5331 else 5332 return 0; 5333 } 5334 } 5335 /* pa_sc_raster_config */ 5336 count += 3; 5337 /* end clear state */ 5338 count += 2; 5339 /* clear state */ 5340 count += 2; 5341 5342 return count; 5343 } 5344 5345 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) 5346 { 5347 u32 count = 0, i; 5348 const struct cs_section_def *sect = NULL; 5349 const struct cs_extent_def *ext = NULL; 5350 5351 if (rdev->rlc.cs_data == NULL) 5352 return; 5353 if (buffer == NULL) 5354 return; 5355 5356 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0); 5357 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE; 5358 5359 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1); 5360 buffer[count++] = 0x80000000; 5361 buffer[count++] = 0x80000000; 5362 5363 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { 5364 for (ext = sect->section; ext->extent != NULL; ++ext) { 5365 if (sect->id == SECT_CONTEXT) { 5366 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count); 5367 buffer[count++] = ext->reg_index - 0xa000; 5368 for (i = 0; i < ext->reg_count; i++) 5369 buffer[count++] = ext->extent[i]; 5370 } else { 5371 return; 5372 } 5373 } 5374 } 5375 5376 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 1); 5377 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START; 5378 switch (rdev->family) { 5379 case CHIP_TAHITI: 5380 case CHIP_PITCAIRN: 5381 buffer[count++] = 0x2a00126a; 5382 break; 5383 case CHIP_VERDE: 5384 buffer[count++] = 0x0000124a; 5385 break; 5386 case CHIP_OLAND: 5387 buffer[count++] = 0x00000082; 5388 break; 5389 case CHIP_HAINAN: 5390 buffer[count++] = 0x00000000; 5391 break; 5392 default: 5393 buffer[count++] = 0x00000000; 5394 break; 5395 } 5396 5397 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0); 5398 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE; 5399 5400 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0); 5401 buffer[count++] = 0; 5402 } 5403 5404 static void si_init_pg(struct radeon_device *rdev) 5405 { 5406 if (rdev->pg_flags) { 5407 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) { 5408 si_init_dma_pg(rdev); 5409 } 5410 si_init_ao_cu_mask(rdev); 5411 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG) { 5412 si_init_gfx_cgpg(rdev); 5413 } 5414 si_enable_dma_pg(rdev, true); 5415 si_enable_gfx_cgpg(rdev, true); 5416 } else { 5417 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); 5418 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8); 5419 } 5420 } 5421 5422 static void si_fini_pg(struct radeon_device *rdev) 5423 { 5424 if (rdev->pg_flags) { 5425 si_enable_dma_pg(rdev, false); 5426 si_enable_gfx_cgpg(rdev, false); 5427 } 5428 } 5429 5430 /* 5431 * RLC 5432 */ 5433 void si_rlc_reset(struct radeon_device *rdev) 5434 { 5435 u32 tmp = RREG32(GRBM_SOFT_RESET); 5436 5437 tmp |= SOFT_RESET_RLC; 5438 WREG32(GRBM_SOFT_RESET, tmp); 5439 udelay(50); 5440 tmp &= ~SOFT_RESET_RLC; 5441 WREG32(GRBM_SOFT_RESET, tmp); 5442 udelay(50); 5443 } 5444 5445 static void si_rlc_stop(struct radeon_device *rdev) 5446 { 5447 WREG32(RLC_CNTL, 0); 5448 5449 si_enable_gui_idle_interrupt(rdev, false); 5450 5451 si_wait_for_rlc_serdes(rdev); 5452 } 5453 5454 static void si_rlc_start(struct radeon_device *rdev) 5455 { 5456 WREG32(RLC_CNTL, RLC_ENABLE); 5457 5458 si_enable_gui_idle_interrupt(rdev, true); 5459 5460 udelay(50); 5461 } 5462 5463 static bool si_lbpw_supported(struct radeon_device *rdev) 5464 { 5465 u32 tmp; 5466 5467 /* Enable LBPW only for DDR3 */ 5468 tmp = RREG32(MC_SEQ_MISC0); 5469 if ((tmp & 0xF0000000) == 0xB0000000) 5470 return true; 5471 return false; 5472 } 5473 5474 static void si_enable_lbpw(struct radeon_device *rdev, bool enable) 5475 { 5476 u32 tmp; 5477 5478 tmp = RREG32(RLC_LB_CNTL); 5479 if (enable) 5480 tmp |= LOAD_BALANCE_ENABLE; 5481 else 5482 tmp &= ~LOAD_BALANCE_ENABLE; 5483 WREG32(RLC_LB_CNTL, tmp); 5484 5485 if (!enable) { 5486 si_select_se_sh(rdev, 0xffffffff, 0xffffffff); 5487 WREG32(SPI_LB_CU_MASK, 0x00ff); 5488 } 5489 } 5490 5491 static int si_rlc_resume(struct radeon_device *rdev) 5492 { 5493 u32 i; 5494 const __be32 *fw_data; 5495 5496 if (!rdev->rlc_fw) 5497 return -EINVAL; 5498 5499 si_rlc_stop(rdev); 5500 5501 si_rlc_reset(rdev); 5502 5503 si_init_pg(rdev); 5504 5505 si_init_cg(rdev); 5506 5507 WREG32(RLC_RL_BASE, 0); 5508 WREG32(RLC_RL_SIZE, 0); 5509 WREG32(RLC_LB_CNTL, 0); 5510 WREG32(RLC_LB_CNTR_MAX, 0xffffffff); 5511 WREG32(RLC_LB_CNTR_INIT, 0); 5512 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff); 5513 5514 WREG32(RLC_MC_CNTL, 0); 5515 WREG32(RLC_UCODE_CNTL, 0); 5516 5517 fw_data = (const __be32 *)rdev->rlc_fw->data; 5518 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) { 5519 WREG32(RLC_UCODE_ADDR, i); 5520 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++)); 5521 } 5522 WREG32(RLC_UCODE_ADDR, 0); 5523 5524 si_enable_lbpw(rdev, si_lbpw_supported(rdev)); 5525 5526 si_rlc_start(rdev); 5527 5528 return 0; 5529 } 5530 5531 static void si_enable_interrupts(struct radeon_device *rdev) 5532 { 5533 u32 ih_cntl = RREG32(IH_CNTL); 5534 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 5535 5536 ih_cntl |= ENABLE_INTR; 5537 ih_rb_cntl |= IH_RB_ENABLE; 5538 WREG32(IH_CNTL, ih_cntl); 5539 WREG32(IH_RB_CNTL, ih_rb_cntl); 5540 rdev->ih.enabled = true; 5541 } 5542 5543 static void si_disable_interrupts(struct radeon_device *rdev) 5544 { 5545 u32 ih_rb_cntl = RREG32(IH_RB_CNTL); 5546 u32 ih_cntl = RREG32(IH_CNTL); 5547 5548 ih_rb_cntl &= ~IH_RB_ENABLE; 5549 ih_cntl &= ~ENABLE_INTR; 5550 WREG32(IH_RB_CNTL, ih_rb_cntl); 5551 WREG32(IH_CNTL, ih_cntl); 5552 /* set rptr, wptr to 0 */ 5553 WREG32(IH_RB_RPTR, 0); 5554 WREG32(IH_RB_WPTR, 0); 5555 rdev->ih.enabled = false; 5556 rdev->ih.rptr = 0; 5557 } 5558 5559 static void si_disable_interrupt_state(struct radeon_device *rdev) 5560 { 5561 u32 tmp; 5562 5563 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 5564 WREG32(CP_INT_CNTL_RING1, 0); 5565 WREG32(CP_INT_CNTL_RING2, 0); 5566 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 5567 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp); 5568 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 5569 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp); 5570 WREG32(GRBM_INT_CNTL, 0); 5571 if (rdev->num_crtc >= 2) { 5572 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 5573 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 5574 } 5575 if (rdev->num_crtc >= 4) { 5576 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 5577 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 5578 } 5579 if (rdev->num_crtc >= 6) { 5580 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 5581 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 5582 } 5583 5584 if (rdev->num_crtc >= 2) { 5585 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 5586 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 5587 } 5588 if (rdev->num_crtc >= 4) { 5589 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 5590 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 5591 } 5592 if (rdev->num_crtc >= 6) { 5593 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 5594 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 5595 } 5596 5597 if (!ASIC_IS_NODCE(rdev)) { 5598 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 5599 5600 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5601 WREG32(DC_HPD1_INT_CONTROL, tmp); 5602 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5603 WREG32(DC_HPD2_INT_CONTROL, tmp); 5604 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5605 WREG32(DC_HPD3_INT_CONTROL, tmp); 5606 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5607 WREG32(DC_HPD4_INT_CONTROL, tmp); 5608 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5609 WREG32(DC_HPD5_INT_CONTROL, tmp); 5610 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 5611 WREG32(DC_HPD6_INT_CONTROL, tmp); 5612 } 5613 } 5614 5615 static int si_irq_init(struct radeon_device *rdev) 5616 { 5617 int ret = 0; 5618 int rb_bufsz; 5619 u32 interrupt_cntl, ih_cntl, ih_rb_cntl; 5620 5621 /* allocate ring */ 5622 ret = r600_ih_ring_alloc(rdev); 5623 if (ret) 5624 return ret; 5625 5626 /* disable irqs */ 5627 si_disable_interrupts(rdev); 5628 5629 /* init rlc */ 5630 ret = si_rlc_resume(rdev); 5631 if (ret) { 5632 r600_ih_ring_fini(rdev); 5633 return ret; 5634 } 5635 5636 /* setup interrupt control */ 5637 /* set dummy read address to ring address */ 5638 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8); 5639 interrupt_cntl = RREG32(INTERRUPT_CNTL); 5640 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi 5641 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN 5642 */ 5643 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE; 5644 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */ 5645 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN; 5646 WREG32(INTERRUPT_CNTL, interrupt_cntl); 5647 5648 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); 5649 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); 5650 5651 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE | 5652 IH_WPTR_OVERFLOW_CLEAR | 5653 (rb_bufsz << 1)); 5654 5655 if (rdev->wb.enabled) 5656 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE; 5657 5658 /* set the writeback address whether it's enabled or not */ 5659 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); 5660 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); 5661 5662 WREG32(IH_RB_CNTL, ih_rb_cntl); 5663 5664 /* set rptr, wptr to 0 */ 5665 WREG32(IH_RB_RPTR, 0); 5666 WREG32(IH_RB_WPTR, 0); 5667 5668 /* Default settings for IH_CNTL (disabled at first) */ 5669 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0); 5670 /* RPTR_REARM only works if msi's are enabled */ 5671 if (rdev->msi_enabled) 5672 ih_cntl |= RPTR_REARM; 5673 WREG32(IH_CNTL, ih_cntl); 5674 5675 /* force the active interrupt state to all disabled */ 5676 si_disable_interrupt_state(rdev); 5677 5678 pci_set_master(rdev->pdev); 5679 5680 /* enable irqs */ 5681 si_enable_interrupts(rdev); 5682 5683 return ret; 5684 } 5685 5686 int si_irq_set(struct radeon_device *rdev) 5687 { 5688 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 5689 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0; 5690 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 5691 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0; 5692 u32 grbm_int_cntl = 0; 5693 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; 5694 u32 dma_cntl, dma_cntl1; 5695 u32 thermal_int = 0; 5696 5697 if (!rdev->irq.installed) { 5698 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 5699 return -EINVAL; 5700 } 5701 /* don't enable anything if the ih is disabled */ 5702 if (!rdev->ih.enabled) { 5703 si_disable_interrupts(rdev); 5704 /* force the active interrupt state to all disabled */ 5705 si_disable_interrupt_state(rdev); 5706 return 0; 5707 } 5708 5709 if (!ASIC_IS_NODCE(rdev)) { 5710 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 5711 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 5712 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 5713 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 5714 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 5715 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 5716 } 5717 5718 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE; 5719 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE; 5720 5721 thermal_int = RREG32(CG_THERMAL_INT) & 5722 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW); 5723 5724 /* enable CP interrupts on all rings */ 5725 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { 5726 DRM_DEBUG("si_irq_set: sw int gfx\n"); 5727 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 5728 } 5729 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { 5730 DRM_DEBUG("si_irq_set: sw int cp1\n"); 5731 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE; 5732 } 5733 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { 5734 DRM_DEBUG("si_irq_set: sw int cp2\n"); 5735 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE; 5736 } 5737 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { 5738 DRM_DEBUG("si_irq_set: sw int dma\n"); 5739 dma_cntl |= TRAP_ENABLE; 5740 } 5741 5742 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { 5743 DRM_DEBUG("si_irq_set: sw int dma1\n"); 5744 dma_cntl1 |= TRAP_ENABLE; 5745 } 5746 if (rdev->irq.crtc_vblank_int[0] || 5747 atomic_read(&rdev->irq.pflip[0])) { 5748 DRM_DEBUG("si_irq_set: vblank 0\n"); 5749 crtc1 |= VBLANK_INT_MASK; 5750 } 5751 if (rdev->irq.crtc_vblank_int[1] || 5752 atomic_read(&rdev->irq.pflip[1])) { 5753 DRM_DEBUG("si_irq_set: vblank 1\n"); 5754 crtc2 |= VBLANK_INT_MASK; 5755 } 5756 if (rdev->irq.crtc_vblank_int[2] || 5757 atomic_read(&rdev->irq.pflip[2])) { 5758 DRM_DEBUG("si_irq_set: vblank 2\n"); 5759 crtc3 |= VBLANK_INT_MASK; 5760 } 5761 if (rdev->irq.crtc_vblank_int[3] || 5762 atomic_read(&rdev->irq.pflip[3])) { 5763 DRM_DEBUG("si_irq_set: vblank 3\n"); 5764 crtc4 |= VBLANK_INT_MASK; 5765 } 5766 if (rdev->irq.crtc_vblank_int[4] || 5767 atomic_read(&rdev->irq.pflip[4])) { 5768 DRM_DEBUG("si_irq_set: vblank 4\n"); 5769 crtc5 |= VBLANK_INT_MASK; 5770 } 5771 if (rdev->irq.crtc_vblank_int[5] || 5772 atomic_read(&rdev->irq.pflip[5])) { 5773 DRM_DEBUG("si_irq_set: vblank 5\n"); 5774 crtc6 |= VBLANK_INT_MASK; 5775 } 5776 if (rdev->irq.hpd[0]) { 5777 DRM_DEBUG("si_irq_set: hpd 1\n"); 5778 hpd1 |= DC_HPDx_INT_EN; 5779 } 5780 if (rdev->irq.hpd[1]) { 5781 DRM_DEBUG("si_irq_set: hpd 2\n"); 5782 hpd2 |= DC_HPDx_INT_EN; 5783 } 5784 if (rdev->irq.hpd[2]) { 5785 DRM_DEBUG("si_irq_set: hpd 3\n"); 5786 hpd3 |= DC_HPDx_INT_EN; 5787 } 5788 if (rdev->irq.hpd[3]) { 5789 DRM_DEBUG("si_irq_set: hpd 4\n"); 5790 hpd4 |= DC_HPDx_INT_EN; 5791 } 5792 if (rdev->irq.hpd[4]) { 5793 DRM_DEBUG("si_irq_set: hpd 5\n"); 5794 hpd5 |= DC_HPDx_INT_EN; 5795 } 5796 if (rdev->irq.hpd[5]) { 5797 DRM_DEBUG("si_irq_set: hpd 6\n"); 5798 hpd6 |= DC_HPDx_INT_EN; 5799 } 5800 5801 WREG32(CP_INT_CNTL_RING0, cp_int_cntl); 5802 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1); 5803 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2); 5804 5805 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl); 5806 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1); 5807 5808 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 5809 5810 if (rdev->irq.dpm_thermal) { 5811 DRM_DEBUG("dpm thermal\n"); 5812 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW; 5813 } 5814 5815 if (rdev->num_crtc >= 2) { 5816 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 5817 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 5818 } 5819 if (rdev->num_crtc >= 4) { 5820 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 5821 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 5822 } 5823 if (rdev->num_crtc >= 6) { 5824 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 5825 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 5826 } 5827 5828 if (rdev->num_crtc >= 2) { 5829 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1); 5830 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2); 5831 } 5832 if (rdev->num_crtc >= 4) { 5833 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3); 5834 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4); 5835 } 5836 if (rdev->num_crtc >= 6) { 5837 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5); 5838 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6); 5839 } 5840 5841 if (!ASIC_IS_NODCE(rdev)) { 5842 WREG32(DC_HPD1_INT_CONTROL, hpd1); 5843 WREG32(DC_HPD2_INT_CONTROL, hpd2); 5844 WREG32(DC_HPD3_INT_CONTROL, hpd3); 5845 WREG32(DC_HPD4_INT_CONTROL, hpd4); 5846 WREG32(DC_HPD5_INT_CONTROL, hpd5); 5847 WREG32(DC_HPD6_INT_CONTROL, hpd6); 5848 } 5849 5850 WREG32(CG_THERMAL_INT, thermal_int); 5851 5852 return 0; 5853 } 5854 5855 static inline void si_irq_ack(struct radeon_device *rdev) 5856 { 5857 u32 tmp; 5858 5859 if (ASIC_IS_NODCE(rdev)) 5860 return; 5861 5862 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 5863 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 5864 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 5865 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 5866 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 5867 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 5868 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 5869 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 5870 if (rdev->num_crtc >= 4) { 5871 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 5872 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 5873 } 5874 if (rdev->num_crtc >= 6) { 5875 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 5876 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 5877 } 5878 5879 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 5880 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5881 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 5882 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5883 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 5884 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 5885 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 5886 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 5887 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 5888 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 5889 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 5890 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 5891 5892 if (rdev->num_crtc >= 4) { 5893 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 5894 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5895 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 5896 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5897 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 5898 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 5899 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 5900 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 5901 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 5902 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 5903 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 5904 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 5905 } 5906 5907 if (rdev->num_crtc >= 6) { 5908 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 5909 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5910 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 5911 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 5912 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 5913 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 5914 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 5915 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 5916 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 5917 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 5918 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 5919 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 5920 } 5921 5922 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 5923 tmp = RREG32(DC_HPD1_INT_CONTROL); 5924 tmp |= DC_HPDx_INT_ACK; 5925 WREG32(DC_HPD1_INT_CONTROL, tmp); 5926 } 5927 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 5928 tmp = RREG32(DC_HPD2_INT_CONTROL); 5929 tmp |= DC_HPDx_INT_ACK; 5930 WREG32(DC_HPD2_INT_CONTROL, tmp); 5931 } 5932 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 5933 tmp = RREG32(DC_HPD3_INT_CONTROL); 5934 tmp |= DC_HPDx_INT_ACK; 5935 WREG32(DC_HPD3_INT_CONTROL, tmp); 5936 } 5937 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 5938 tmp = RREG32(DC_HPD4_INT_CONTROL); 5939 tmp |= DC_HPDx_INT_ACK; 5940 WREG32(DC_HPD4_INT_CONTROL, tmp); 5941 } 5942 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 5943 tmp = RREG32(DC_HPD5_INT_CONTROL); 5944 tmp |= DC_HPDx_INT_ACK; 5945 WREG32(DC_HPD5_INT_CONTROL, tmp); 5946 } 5947 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 5948 tmp = RREG32(DC_HPD5_INT_CONTROL); 5949 tmp |= DC_HPDx_INT_ACK; 5950 WREG32(DC_HPD6_INT_CONTROL, tmp); 5951 } 5952 } 5953 5954 static void si_irq_disable(struct radeon_device *rdev) 5955 { 5956 si_disable_interrupts(rdev); 5957 /* Wait and acknowledge irq */ 5958 mdelay(1); 5959 si_irq_ack(rdev); 5960 si_disable_interrupt_state(rdev); 5961 } 5962 5963 static void si_irq_suspend(struct radeon_device *rdev) 5964 { 5965 si_irq_disable(rdev); 5966 si_rlc_stop(rdev); 5967 } 5968 5969 static void si_irq_fini(struct radeon_device *rdev) 5970 { 5971 si_irq_suspend(rdev); 5972 r600_ih_ring_fini(rdev); 5973 } 5974 5975 static inline u32 si_get_ih_wptr(struct radeon_device *rdev) 5976 { 5977 u32 wptr, tmp; 5978 5979 if (rdev->wb.enabled) 5980 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 5981 else 5982 wptr = RREG32(IH_RB_WPTR); 5983 5984 if (wptr & RB_OVERFLOW) { 5985 /* When a ring buffer overflow happen start parsing interrupt 5986 * from the last not overwritten vector (wptr + 16). Hopefully 5987 * this should allow us to catchup. 5988 */ 5989 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", 5990 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); 5991 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 5992 tmp = RREG32(IH_RB_CNTL); 5993 tmp |= IH_WPTR_OVERFLOW_CLEAR; 5994 WREG32(IH_RB_CNTL, tmp); 5995 } 5996 return (wptr & rdev->ih.ptr_mask); 5997 } 5998 5999 /* SI IV Ring 6000 * Each IV ring entry is 128 bits: 6001 * [7:0] - interrupt source id 6002 * [31:8] - reserved 6003 * [59:32] - interrupt source data 6004 * [63:60] - reserved 6005 * [71:64] - RINGID 6006 * [79:72] - VMID 6007 * [127:80] - reserved 6008 */ 6009 int si_irq_process(struct radeon_device *rdev) 6010 { 6011 u32 wptr; 6012 u32 rptr; 6013 u32 src_id, src_data, ring_id; 6014 u32 ring_index; 6015 bool queue_hotplug = false; 6016 bool queue_thermal = false; 6017 u32 status, addr; 6018 6019 if (!rdev->ih.enabled || rdev->shutdown) 6020 return IRQ_NONE; 6021 6022 wptr = si_get_ih_wptr(rdev); 6023 6024 restart_ih: 6025 /* is somebody else already processing irqs? */ 6026 if (atomic_xchg(&rdev->ih.lock, 1)) 6027 return IRQ_NONE; 6028 6029 rptr = rdev->ih.rptr; 6030 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 6031 6032 /* Order reading of wptr vs. reading of IH ring data */ 6033 rmb(); 6034 6035 /* display interrupts */ 6036 si_irq_ack(rdev); 6037 6038 while (rptr != wptr) { 6039 /* wptr/rptr are in bytes! */ 6040 ring_index = rptr / 4; 6041 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 6042 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 6043 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; 6044 6045 switch (src_id) { 6046 case 1: /* D1 vblank/vline */ 6047 switch (src_data) { 6048 case 0: /* D1 vblank */ 6049 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { 6050 if (rdev->irq.crtc_vblank_int[0]) { 6051 drm_handle_vblank(rdev->ddev, 0); 6052 rdev->pm.vblank_sync = true; 6053 wake_up(&rdev->irq.vblank_queue); 6054 } 6055 if (atomic_read(&rdev->irq.pflip[0])) 6056 radeon_crtc_handle_flip(rdev, 0); 6057 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 6058 DRM_DEBUG("IH: D1 vblank\n"); 6059 } 6060 break; 6061 case 1: /* D1 vline */ 6062 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { 6063 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 6064 DRM_DEBUG("IH: D1 vline\n"); 6065 } 6066 break; 6067 default: 6068 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6069 break; 6070 } 6071 break; 6072 case 2: /* D2 vblank/vline */ 6073 switch (src_data) { 6074 case 0: /* D2 vblank */ 6075 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 6076 if (rdev->irq.crtc_vblank_int[1]) { 6077 drm_handle_vblank(rdev->ddev, 1); 6078 rdev->pm.vblank_sync = true; 6079 wake_up(&rdev->irq.vblank_queue); 6080 } 6081 if (atomic_read(&rdev->irq.pflip[1])) 6082 radeon_crtc_handle_flip(rdev, 1); 6083 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 6084 DRM_DEBUG("IH: D2 vblank\n"); 6085 } 6086 break; 6087 case 1: /* D2 vline */ 6088 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 6089 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 6090 DRM_DEBUG("IH: D2 vline\n"); 6091 } 6092 break; 6093 default: 6094 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6095 break; 6096 } 6097 break; 6098 case 3: /* D3 vblank/vline */ 6099 switch (src_data) { 6100 case 0: /* D3 vblank */ 6101 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 6102 if (rdev->irq.crtc_vblank_int[2]) { 6103 drm_handle_vblank(rdev->ddev, 2); 6104 rdev->pm.vblank_sync = true; 6105 wake_up(&rdev->irq.vblank_queue); 6106 } 6107 if (atomic_read(&rdev->irq.pflip[2])) 6108 radeon_crtc_handle_flip(rdev, 2); 6109 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 6110 DRM_DEBUG("IH: D3 vblank\n"); 6111 } 6112 break; 6113 case 1: /* D3 vline */ 6114 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 6115 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 6116 DRM_DEBUG("IH: D3 vline\n"); 6117 } 6118 break; 6119 default: 6120 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6121 break; 6122 } 6123 break; 6124 case 4: /* D4 vblank/vline */ 6125 switch (src_data) { 6126 case 0: /* D4 vblank */ 6127 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 6128 if (rdev->irq.crtc_vblank_int[3]) { 6129 drm_handle_vblank(rdev->ddev, 3); 6130 rdev->pm.vblank_sync = true; 6131 wake_up(&rdev->irq.vblank_queue); 6132 } 6133 if (atomic_read(&rdev->irq.pflip[3])) 6134 radeon_crtc_handle_flip(rdev, 3); 6135 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 6136 DRM_DEBUG("IH: D4 vblank\n"); 6137 } 6138 break; 6139 case 1: /* D4 vline */ 6140 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 6141 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 6142 DRM_DEBUG("IH: D4 vline\n"); 6143 } 6144 break; 6145 default: 6146 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6147 break; 6148 } 6149 break; 6150 case 5: /* D5 vblank/vline */ 6151 switch (src_data) { 6152 case 0: /* D5 vblank */ 6153 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 6154 if (rdev->irq.crtc_vblank_int[4]) { 6155 drm_handle_vblank(rdev->ddev, 4); 6156 rdev->pm.vblank_sync = true; 6157 wake_up(&rdev->irq.vblank_queue); 6158 } 6159 if (atomic_read(&rdev->irq.pflip[4])) 6160 radeon_crtc_handle_flip(rdev, 4); 6161 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 6162 DRM_DEBUG("IH: D5 vblank\n"); 6163 } 6164 break; 6165 case 1: /* D5 vline */ 6166 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 6167 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 6168 DRM_DEBUG("IH: D5 vline\n"); 6169 } 6170 break; 6171 default: 6172 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6173 break; 6174 } 6175 break; 6176 case 6: /* D6 vblank/vline */ 6177 switch (src_data) { 6178 case 0: /* D6 vblank */ 6179 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 6180 if (rdev->irq.crtc_vblank_int[5]) { 6181 drm_handle_vblank(rdev->ddev, 5); 6182 rdev->pm.vblank_sync = true; 6183 wake_up(&rdev->irq.vblank_queue); 6184 } 6185 if (atomic_read(&rdev->irq.pflip[5])) 6186 radeon_crtc_handle_flip(rdev, 5); 6187 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 6188 DRM_DEBUG("IH: D6 vblank\n"); 6189 } 6190 break; 6191 case 1: /* D6 vline */ 6192 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 6193 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 6194 DRM_DEBUG("IH: D6 vline\n"); 6195 } 6196 break; 6197 default: 6198 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6199 break; 6200 } 6201 break; 6202 case 42: /* HPD hotplug */ 6203 switch (src_data) { 6204 case 0: 6205 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 6206 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 6207 queue_hotplug = true; 6208 DRM_DEBUG("IH: HPD1\n"); 6209 } 6210 break; 6211 case 1: 6212 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 6213 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 6214 queue_hotplug = true; 6215 DRM_DEBUG("IH: HPD2\n"); 6216 } 6217 break; 6218 case 2: 6219 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 6220 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 6221 queue_hotplug = true; 6222 DRM_DEBUG("IH: HPD3\n"); 6223 } 6224 break; 6225 case 3: 6226 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 6227 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 6228 queue_hotplug = true; 6229 DRM_DEBUG("IH: HPD4\n"); 6230 } 6231 break; 6232 case 4: 6233 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 6234 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 6235 queue_hotplug = true; 6236 DRM_DEBUG("IH: HPD5\n"); 6237 } 6238 break; 6239 case 5: 6240 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 6241 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 6242 queue_hotplug = true; 6243 DRM_DEBUG("IH: HPD6\n"); 6244 } 6245 break; 6246 default: 6247 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6248 break; 6249 } 6250 break; 6251 case 146: 6252 case 147: 6253 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR); 6254 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS); 6255 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); 6256 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", 6257 addr); 6258 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", 6259 status); 6260 si_vm_decode_fault(rdev, status, addr); 6261 /* reset addr and status */ 6262 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1); 6263 break; 6264 case 176: /* RINGID0 CP_INT */ 6265 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 6266 break; 6267 case 177: /* RINGID1 CP_INT */ 6268 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6269 break; 6270 case 178: /* RINGID2 CP_INT */ 6271 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6272 break; 6273 case 181: /* CP EOP event */ 6274 DRM_DEBUG("IH: CP EOP\n"); 6275 switch (ring_id) { 6276 case 0: 6277 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); 6278 break; 6279 case 1: 6280 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6281 break; 6282 case 2: 6283 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6284 break; 6285 } 6286 break; 6287 case 224: /* DMA trap event */ 6288 DRM_DEBUG("IH: DMA trap\n"); 6289 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); 6290 break; 6291 case 230: /* thermal low to high */ 6292 DRM_DEBUG("IH: thermal low to high\n"); 6293 rdev->pm.dpm.thermal.high_to_low = false; 6294 queue_thermal = true; 6295 break; 6296 case 231: /* thermal high to low */ 6297 DRM_DEBUG("IH: thermal high to low\n"); 6298 rdev->pm.dpm.thermal.high_to_low = true; 6299 queue_thermal = true; 6300 break; 6301 case 233: /* GUI IDLE */ 6302 DRM_DEBUG("IH: GUI idle\n"); 6303 break; 6304 case 244: /* DMA trap event */ 6305 DRM_DEBUG("IH: DMA1 trap\n"); 6306 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 6307 break; 6308 default: 6309 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 6310 break; 6311 } 6312 6313 /* wptr/rptr are in bytes! */ 6314 rptr += 16; 6315 rptr &= rdev->ih.ptr_mask; 6316 } 6317 if (queue_hotplug) 6318 schedule_work(&rdev->hotplug_work); 6319 if (queue_thermal && rdev->pm.dpm_enabled) 6320 schedule_work(&rdev->pm.dpm.thermal.work); 6321 rdev->ih.rptr = rptr; 6322 WREG32(IH_RB_RPTR, rdev->ih.rptr); 6323 atomic_set(&rdev->ih.lock, 0); 6324 6325 /* make sure wptr hasn't changed while processing */ 6326 wptr = si_get_ih_wptr(rdev); 6327 if (wptr != rptr) 6328 goto restart_ih; 6329 6330 return IRQ_HANDLED; 6331 } 6332 6333 /* 6334 * startup/shutdown callbacks 6335 */ 6336 static int si_startup(struct radeon_device *rdev) 6337 { 6338 struct radeon_ring *ring; 6339 int r; 6340 6341 /* enable pcie gen2/3 link */ 6342 si_pcie_gen3_enable(rdev); 6343 /* enable aspm */ 6344 si_program_aspm(rdev); 6345 6346 /* scratch needs to be initialized before MC */ 6347 r = r600_vram_scratch_init(rdev); 6348 if (r) 6349 return r; 6350 6351 si_mc_program(rdev); 6352 6353 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || 6354 !rdev->rlc_fw || !rdev->mc_fw) { 6355 r = si_init_microcode(rdev); 6356 if (r) { 6357 DRM_ERROR("Failed to load firmware!\n"); 6358 return r; 6359 } 6360 } 6361 6362 r = si_mc_load_microcode(rdev); 6363 if (r) { 6364 DRM_ERROR("Failed to load MC firmware!\n"); 6365 return r; 6366 } 6367 6368 r = si_pcie_gart_enable(rdev); 6369 if (r) 6370 return r; 6371 si_gpu_init(rdev); 6372 6373 /* allocate rlc buffers */ 6374 if (rdev->family == CHIP_VERDE) { 6375 rdev->rlc.reg_list = verde_rlc_save_restore_register_list; 6376 rdev->rlc.reg_list_size = 6377 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list); 6378 } 6379 rdev->rlc.cs_data = si_cs_data; 6380 r = sumo_rlc_init(rdev); 6381 if (r) { 6382 DRM_ERROR("Failed to init rlc BOs!\n"); 6383 return r; 6384 } 6385 6386 /* allocate wb buffer */ 6387 r = radeon_wb_init(rdev); 6388 if (r) 6389 return r; 6390 6391 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); 6392 if (r) { 6393 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6394 return r; 6395 } 6396 6397 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); 6398 if (r) { 6399 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6400 return r; 6401 } 6402 6403 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); 6404 if (r) { 6405 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); 6406 return r; 6407 } 6408 6409 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); 6410 if (r) { 6411 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 6412 return r; 6413 } 6414 6415 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); 6416 if (r) { 6417 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); 6418 return r; 6419 } 6420 6421 if (rdev->has_uvd) { 6422 r = uvd_v2_2_resume(rdev); 6423 if (!r) { 6424 r = radeon_fence_driver_start_ring(rdev, 6425 R600_RING_TYPE_UVD_INDEX); 6426 if (r) 6427 dev_err(rdev->dev, "UVD fences init error (%d).\n", r); 6428 } 6429 if (r) 6430 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; 6431 } 6432 6433 /* Enable IRQ */ 6434 if (!rdev->irq.installed) { 6435 r = radeon_irq_kms_init(rdev); 6436 if (r) 6437 return r; 6438 } 6439 6440 r = si_irq_init(rdev); 6441 if (r) { 6442 DRM_ERROR("radeon: IH init failed (%d).\n", r); 6443 radeon_irq_kms_fini(rdev); 6444 return r; 6445 } 6446 si_irq_set(rdev); 6447 6448 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6449 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, 6450 CP_RB0_RPTR, CP_RB0_WPTR, 6451 RADEON_CP_PACKET2); 6452 if (r) 6453 return r; 6454 6455 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 6456 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, 6457 CP_RB1_RPTR, CP_RB1_WPTR, 6458 RADEON_CP_PACKET2); 6459 if (r) 6460 return r; 6461 6462 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 6463 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, 6464 CP_RB2_RPTR, CP_RB2_WPTR, 6465 RADEON_CP_PACKET2); 6466 if (r) 6467 return r; 6468 6469 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 6470 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, 6471 DMA_RB_RPTR + DMA0_REGISTER_OFFSET, 6472 DMA_RB_WPTR + DMA0_REGISTER_OFFSET, 6473 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 6474 if (r) 6475 return r; 6476 6477 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 6478 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, 6479 DMA_RB_RPTR + DMA1_REGISTER_OFFSET, 6480 DMA_RB_WPTR + DMA1_REGISTER_OFFSET, 6481 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0)); 6482 if (r) 6483 return r; 6484 6485 r = si_cp_load_microcode(rdev); 6486 if (r) 6487 return r; 6488 r = si_cp_resume(rdev); 6489 if (r) 6490 return r; 6491 6492 r = cayman_dma_resume(rdev); 6493 if (r) 6494 return r; 6495 6496 if (rdev->has_uvd) { 6497 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 6498 if (ring->ring_size) { 6499 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 6500 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR, 6501 RADEON_CP_PACKET2); 6502 if (!r) 6503 r = uvd_v1_0_init(rdev); 6504 if (r) 6505 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r); 6506 } 6507 } 6508 6509 r = radeon_ib_pool_init(rdev); 6510 if (r) { 6511 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); 6512 return r; 6513 } 6514 6515 r = radeon_vm_manager_init(rdev); 6516 if (r) { 6517 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); 6518 return r; 6519 } 6520 6521 r = dce6_audio_init(rdev); 6522 if (r) 6523 return r; 6524 6525 return 0; 6526 } 6527 6528 int si_resume(struct radeon_device *rdev) 6529 { 6530 int r; 6531 6532 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 6533 * posting will perform necessary task to bring back GPU into good 6534 * shape. 6535 */ 6536 /* post card */ 6537 atom_asic_init(rdev->mode_info.atom_context); 6538 6539 /* init golden registers */ 6540 si_init_golden_registers(rdev); 6541 6542 rdev->accel_working = true; 6543 r = si_startup(rdev); 6544 if (r) { 6545 DRM_ERROR("si startup failed on resume\n"); 6546 rdev->accel_working = false; 6547 return r; 6548 } 6549 6550 return r; 6551 6552 } 6553 6554 int si_suspend(struct radeon_device *rdev) 6555 { 6556 dce6_audio_fini(rdev); 6557 radeon_vm_manager_fini(rdev); 6558 si_cp_enable(rdev, false); 6559 cayman_dma_stop(rdev); 6560 if (rdev->has_uvd) { 6561 uvd_v1_0_fini(rdev); 6562 radeon_uvd_suspend(rdev); 6563 } 6564 si_fini_pg(rdev); 6565 si_fini_cg(rdev); 6566 si_irq_suspend(rdev); 6567 radeon_wb_disable(rdev); 6568 si_pcie_gart_disable(rdev); 6569 return 0; 6570 } 6571 6572 /* Plan is to move initialization in that function and use 6573 * helper function so that radeon_device_init pretty much 6574 * do nothing more than calling asic specific function. This 6575 * should also allow to remove a bunch of callback function 6576 * like vram_info. 6577 */ 6578 int si_init(struct radeon_device *rdev) 6579 { 6580 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6581 int r; 6582 6583 /* Read BIOS */ 6584 if (!radeon_get_bios(rdev)) { 6585 if (ASIC_IS_AVIVO(rdev)) 6586 return -EINVAL; 6587 } 6588 /* Must be an ATOMBIOS */ 6589 if (!rdev->is_atom_bios) { 6590 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); 6591 return -EINVAL; 6592 } 6593 r = radeon_atombios_init(rdev); 6594 if (r) 6595 return r; 6596 6597 /* Post card if necessary */ 6598 if (!radeon_card_posted(rdev)) { 6599 if (!rdev->bios) { 6600 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 6601 return -EINVAL; 6602 } 6603 DRM_INFO("GPU not posted. posting now...\n"); 6604 atom_asic_init(rdev->mode_info.atom_context); 6605 } 6606 /* init golden registers */ 6607 si_init_golden_registers(rdev); 6608 /* Initialize scratch registers */ 6609 si_scratch_init(rdev); 6610 /* Initialize surface registers */ 6611 radeon_surface_init(rdev); 6612 /* Initialize clocks */ 6613 radeon_get_clock_info(rdev->ddev); 6614 6615 /* Fence driver */ 6616 r = radeon_fence_driver_init(rdev); 6617 if (r) 6618 return r; 6619 6620 /* initialize memory controller */ 6621 r = si_mc_init(rdev); 6622 if (r) 6623 return r; 6624 /* Memory manager */ 6625 r = radeon_bo_init(rdev); 6626 if (r) 6627 return r; 6628 6629 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; 6630 ring->ring_obj = NULL; 6631 r600_ring_init(rdev, ring, 1024 * 1024); 6632 6633 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; 6634 ring->ring_obj = NULL; 6635 r600_ring_init(rdev, ring, 1024 * 1024); 6636 6637 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; 6638 ring->ring_obj = NULL; 6639 r600_ring_init(rdev, ring, 1024 * 1024); 6640 6641 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; 6642 ring->ring_obj = NULL; 6643 r600_ring_init(rdev, ring, 64 * 1024); 6644 6645 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; 6646 ring->ring_obj = NULL; 6647 r600_ring_init(rdev, ring, 64 * 1024); 6648 6649 if (rdev->has_uvd) { 6650 r = radeon_uvd_init(rdev); 6651 if (!r) { 6652 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; 6653 ring->ring_obj = NULL; 6654 r600_ring_init(rdev, ring, 4096); 6655 } 6656 } 6657 6658 rdev->ih.ring_obj = NULL; 6659 r600_ih_ring_init(rdev, 64 * 1024); 6660 6661 r = r600_pcie_gart_init(rdev); 6662 if (r) 6663 return r; 6664 6665 rdev->accel_working = true; 6666 r = si_startup(rdev); 6667 if (r) { 6668 dev_err(rdev->dev, "disabling GPU acceleration\n"); 6669 si_cp_fini(rdev); 6670 cayman_dma_fini(rdev); 6671 si_irq_fini(rdev); 6672 sumo_rlc_fini(rdev); 6673 radeon_wb_fini(rdev); 6674 radeon_ib_pool_fini(rdev); 6675 radeon_vm_manager_fini(rdev); 6676 radeon_irq_kms_fini(rdev); 6677 si_pcie_gart_fini(rdev); 6678 rdev->accel_working = false; 6679 } 6680 6681 /* Don't start up if the MC ucode is missing. 6682 * The default clocks and voltages before the MC ucode 6683 * is loaded are not suffient for advanced operations. 6684 */ 6685 if (!rdev->mc_fw) { 6686 DRM_ERROR("radeon: MC ucode required for NI+.\n"); 6687 return -EINVAL; 6688 } 6689 6690 return 0; 6691 } 6692 6693 void si_fini(struct radeon_device *rdev) 6694 { 6695 si_cp_fini(rdev); 6696 cayman_dma_fini(rdev); 6697 si_fini_pg(rdev); 6698 si_fini_cg(rdev); 6699 si_irq_fini(rdev); 6700 sumo_rlc_fini(rdev); 6701 radeon_wb_fini(rdev); 6702 radeon_vm_manager_fini(rdev); 6703 radeon_ib_pool_fini(rdev); 6704 radeon_irq_kms_fini(rdev); 6705 if (rdev->has_uvd) { 6706 uvd_v1_0_fini(rdev); 6707 radeon_uvd_fini(rdev); 6708 } 6709 si_pcie_gart_fini(rdev); 6710 r600_vram_scratch_fini(rdev); 6711 radeon_gem_fini(rdev); 6712 radeon_fence_driver_fini(rdev); 6713 radeon_bo_fini(rdev); 6714 radeon_atombios_fini(rdev); 6715 kfree(rdev->bios); 6716 rdev->bios = NULL; 6717 } 6718 6719 /** 6720 * si_get_gpu_clock_counter - return GPU clock counter snapshot 6721 * 6722 * @rdev: radeon_device pointer 6723 * 6724 * Fetches a GPU clock counter snapshot (SI). 6725 * Returns the 64 bit clock counter snapshot. 6726 */ 6727 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev) 6728 { 6729 uint64_t clock; 6730 6731 mutex_lock(&rdev->gpu_clock_mutex); 6732 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1); 6733 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) | 6734 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 6735 mutex_unlock(&rdev->gpu_clock_mutex); 6736 return clock; 6737 } 6738 6739 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) 6740 { 6741 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0; 6742 int r; 6743 6744 /* bypass vclk and dclk with bclk */ 6745 WREG32_P(CG_UPLL_FUNC_CNTL_2, 6746 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1), 6747 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 6748 6749 /* put PLL in bypass mode */ 6750 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK); 6751 6752 if (!vclk || !dclk) { 6753 /* keep the Bypass mode, put PLL to sleep */ 6754 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 6755 return 0; 6756 } 6757 6758 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000, 6759 16384, 0x03FFFFFF, 0, 128, 5, 6760 &fb_div, &vclk_div, &dclk_div); 6761 if (r) 6762 return r; 6763 6764 /* set RESET_ANTI_MUX to 0 */ 6765 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK); 6766 6767 /* set VCO_MODE to 1 */ 6768 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK); 6769 6770 /* toggle UPLL_SLEEP to 1 then back to 0 */ 6771 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK); 6772 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK); 6773 6774 /* deassert UPLL_RESET */ 6775 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 6776 6777 mdelay(1); 6778 6779 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 6780 if (r) 6781 return r; 6782 6783 /* assert UPLL_RESET again */ 6784 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK); 6785 6786 /* disable spread spectrum. */ 6787 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK); 6788 6789 /* set feedback divider */ 6790 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK); 6791 6792 /* set ref divider to 0 */ 6793 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK); 6794 6795 if (fb_div < 307200) 6796 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9); 6797 else 6798 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9); 6799 6800 /* set PDIV_A and PDIV_B */ 6801 WREG32_P(CG_UPLL_FUNC_CNTL_2, 6802 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div), 6803 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK)); 6804 6805 /* give the PLL some time to settle */ 6806 mdelay(15); 6807 6808 /* deassert PLL_RESET */ 6809 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK); 6810 6811 mdelay(15); 6812 6813 /* switch from bypass mode to normal mode */ 6814 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK); 6815 6816 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL); 6817 if (r) 6818 return r; 6819 6820 /* switch VCLK and DCLK selection */ 6821 WREG32_P(CG_UPLL_FUNC_CNTL_2, 6822 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2), 6823 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK)); 6824 6825 mdelay(100); 6826 6827 return 0; 6828 } 6829 6830 static void si_pcie_gen3_enable(struct radeon_device *rdev) 6831 { 6832 struct pci_dev *root = rdev->pdev->bus->self; 6833 int bridge_pos, gpu_pos; 6834 u32 speed_cntl, mask, current_data_rate; 6835 int ret, i; 6836 u16 tmp16; 6837 6838 if (radeon_pcie_gen2 == 0) 6839 return; 6840 6841 if (rdev->flags & RADEON_IS_IGP) 6842 return; 6843 6844 if (!(rdev->flags & RADEON_IS_PCIE)) 6845 return; 6846 6847 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask); 6848 if (ret != 0) 6849 return; 6850 6851 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80))) 6852 return; 6853 6854 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 6855 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >> 6856 LC_CURRENT_DATA_RATE_SHIFT; 6857 if (mask & DRM_PCIE_SPEED_80) { 6858 if (current_data_rate == 2) { 6859 DRM_INFO("PCIE gen 3 link speeds already enabled\n"); 6860 return; 6861 } 6862 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n"); 6863 } else if (mask & DRM_PCIE_SPEED_50) { 6864 if (current_data_rate == 1) { 6865 DRM_INFO("PCIE gen 2 link speeds already enabled\n"); 6866 return; 6867 } 6868 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n"); 6869 } 6870 6871 bridge_pos = pci_pcie_cap(root); 6872 if (!bridge_pos) 6873 return; 6874 6875 gpu_pos = pci_pcie_cap(rdev->pdev); 6876 if (!gpu_pos) 6877 return; 6878 6879 if (mask & DRM_PCIE_SPEED_80) { 6880 /* re-try equalization if gen3 is not already enabled */ 6881 if (current_data_rate != 2) { 6882 u16 bridge_cfg, gpu_cfg; 6883 u16 bridge_cfg2, gpu_cfg2; 6884 u32 max_lw, current_lw, tmp; 6885 6886 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 6887 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 6888 6889 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD; 6890 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 6891 6892 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD; 6893 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 6894 6895 tmp = RREG32_PCIE(PCIE_LC_STATUS1); 6896 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT; 6897 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT; 6898 6899 if (current_lw < max_lw) { 6900 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 6901 if (tmp & LC_RENEGOTIATION_SUPPORT) { 6902 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS); 6903 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT); 6904 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW; 6905 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp); 6906 } 6907 } 6908 6909 for (i = 0; i < 10; i++) { 6910 /* check status */ 6911 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16); 6912 if (tmp16 & PCI_EXP_DEVSTA_TRPND) 6913 break; 6914 6915 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg); 6916 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg); 6917 6918 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2); 6919 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2); 6920 6921 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 6922 tmp |= LC_SET_QUIESCE; 6923 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 6924 6925 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 6926 tmp |= LC_REDO_EQ; 6927 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 6928 6929 mdelay(100); 6930 6931 /* linkctl */ 6932 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16); 6933 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 6934 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD); 6935 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16); 6936 6937 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16); 6938 tmp16 &= ~PCI_EXP_LNKCTL_HAWD; 6939 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD); 6940 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16); 6941 6942 /* linkctl2 */ 6943 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16); 6944 tmp16 &= ~((1 << 4) | (7 << 9)); 6945 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9))); 6946 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16); 6947 6948 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 6949 tmp16 &= ~((1 << 4) | (7 << 9)); 6950 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9))); 6951 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 6952 6953 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4); 6954 tmp &= ~LC_SET_QUIESCE; 6955 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp); 6956 } 6957 } 6958 } 6959 6960 /* set the link speed */ 6961 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE; 6962 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE; 6963 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 6964 6965 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16); 6966 tmp16 &= ~0xf; 6967 if (mask & DRM_PCIE_SPEED_80) 6968 tmp16 |= 3; /* gen3 */ 6969 else if (mask & DRM_PCIE_SPEED_50) 6970 tmp16 |= 2; /* gen2 */ 6971 else 6972 tmp16 |= 1; /* gen1 */ 6973 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16); 6974 6975 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 6976 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE; 6977 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl); 6978 6979 for (i = 0; i < rdev->usec_timeout; i++) { 6980 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL); 6981 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0) 6982 break; 6983 udelay(1); 6984 } 6985 } 6986 6987 static void si_program_aspm(struct radeon_device *rdev) 6988 { 6989 u32 data, orig; 6990 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false; 6991 bool disable_clkreq = false; 6992 6993 if (radeon_aspm == 0) 6994 return; 6995 6996 if (!(rdev->flags & RADEON_IS_PCIE)) 6997 return; 6998 6999 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 7000 data &= ~LC_XMIT_N_FTS_MASK; 7001 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN; 7002 if (orig != data) 7003 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data); 7004 7005 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3); 7006 data |= LC_GO_TO_RECOVERY; 7007 if (orig != data) 7008 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data); 7009 7010 orig = data = RREG32_PCIE(PCIE_P_CNTL); 7011 data |= P_IGNORE_EDB_ERR; 7012 if (orig != data) 7013 WREG32_PCIE(PCIE_P_CNTL, data); 7014 7015 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 7016 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK); 7017 data |= LC_PMI_TO_L1_DIS; 7018 if (!disable_l0s) 7019 data |= LC_L0S_INACTIVITY(7); 7020 7021 if (!disable_l1) { 7022 data |= LC_L1_INACTIVITY(7); 7023 data &= ~LC_PMI_TO_L1_DIS; 7024 if (orig != data) 7025 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7026 7027 if (!disable_plloff_in_l1) { 7028 bool clk_req_support; 7029 7030 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 7031 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 7032 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 7033 if (orig != data) 7034 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 7035 7036 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 7037 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 7038 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 7039 if (orig != data) 7040 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 7041 7042 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 7043 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK); 7044 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7); 7045 if (orig != data) 7046 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 7047 7048 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 7049 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK); 7050 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7); 7051 if (orig != data) 7052 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 7053 7054 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) { 7055 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0); 7056 data &= ~PLL_RAMP_UP_TIME_0_MASK; 7057 if (orig != data) 7058 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data); 7059 7060 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1); 7061 data &= ~PLL_RAMP_UP_TIME_1_MASK; 7062 if (orig != data) 7063 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data); 7064 7065 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2); 7066 data &= ~PLL_RAMP_UP_TIME_2_MASK; 7067 if (orig != data) 7068 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data); 7069 7070 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3); 7071 data &= ~PLL_RAMP_UP_TIME_3_MASK; 7072 if (orig != data) 7073 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data); 7074 7075 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0); 7076 data &= ~PLL_RAMP_UP_TIME_0_MASK; 7077 if (orig != data) 7078 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data); 7079 7080 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1); 7081 data &= ~PLL_RAMP_UP_TIME_1_MASK; 7082 if (orig != data) 7083 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data); 7084 7085 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2); 7086 data &= ~PLL_RAMP_UP_TIME_2_MASK; 7087 if (orig != data) 7088 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data); 7089 7090 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3); 7091 data &= ~PLL_RAMP_UP_TIME_3_MASK; 7092 if (orig != data) 7093 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data); 7094 } 7095 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL); 7096 data &= ~LC_DYN_LANES_PWR_STATE_MASK; 7097 data |= LC_DYN_LANES_PWR_STATE(3); 7098 if (orig != data) 7099 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data); 7100 7101 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL); 7102 data &= ~LS2_EXIT_TIME_MASK; 7103 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) 7104 data |= LS2_EXIT_TIME(5); 7105 if (orig != data) 7106 WREG32_PIF_PHY0(PB0_PIF_CNTL, data); 7107 7108 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL); 7109 data &= ~LS2_EXIT_TIME_MASK; 7110 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN)) 7111 data |= LS2_EXIT_TIME(5); 7112 if (orig != data) 7113 WREG32_PIF_PHY1(PB1_PIF_CNTL, data); 7114 7115 if (!disable_clkreq) { 7116 struct pci_dev *root = rdev->pdev->bus->self; 7117 u32 lnkcap; 7118 7119 clk_req_support = false; 7120 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap); 7121 if (lnkcap & PCI_EXP_LNKCAP_CLKPM) 7122 clk_req_support = true; 7123 } else { 7124 clk_req_support = false; 7125 } 7126 7127 if (clk_req_support) { 7128 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2); 7129 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23; 7130 if (orig != data) 7131 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data); 7132 7133 orig = data = RREG32(THM_CLK_CNTL); 7134 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK); 7135 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1); 7136 if (orig != data) 7137 WREG32(THM_CLK_CNTL, data); 7138 7139 orig = data = RREG32(MISC_CLK_CNTL); 7140 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK); 7141 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1); 7142 if (orig != data) 7143 WREG32(MISC_CLK_CNTL, data); 7144 7145 orig = data = RREG32(CG_CLKPIN_CNTL); 7146 data &= ~BCLK_AS_XCLK; 7147 if (orig != data) 7148 WREG32(CG_CLKPIN_CNTL, data); 7149 7150 orig = data = RREG32(CG_CLKPIN_CNTL_2); 7151 data &= ~FORCE_BIF_REFCLK_EN; 7152 if (orig != data) 7153 WREG32(CG_CLKPIN_CNTL_2, data); 7154 7155 orig = data = RREG32(MPLL_BYPASSCLK_SEL); 7156 data &= ~MPLL_CLKOUT_SEL_MASK; 7157 data |= MPLL_CLKOUT_SEL(4); 7158 if (orig != data) 7159 WREG32(MPLL_BYPASSCLK_SEL, data); 7160 7161 orig = data = RREG32(SPLL_CNTL_MODE); 7162 data &= ~SPLL_REFCLK_SEL_MASK; 7163 if (orig != data) 7164 WREG32(SPLL_CNTL_MODE, data); 7165 } 7166 } 7167 } else { 7168 if (orig != data) 7169 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7170 } 7171 7172 orig = data = RREG32_PCIE(PCIE_CNTL2); 7173 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN; 7174 if (orig != data) 7175 WREG32_PCIE(PCIE_CNTL2, data); 7176 7177 if (!disable_l0s) { 7178 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL); 7179 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) { 7180 data = RREG32_PCIE(PCIE_LC_STATUS1); 7181 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) { 7182 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL); 7183 data &= ~LC_L0S_INACTIVITY_MASK; 7184 if (orig != data) 7185 WREG32_PCIE_PORT(PCIE_LC_CNTL, data); 7186 } 7187 } 7188 } 7189 } 7190