1 /* 2 * Copyright 2010 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Alex Deucher 23 */ 24 #include <linux/firmware.h> 25 #include <linux/platform_device.h> 26 #include <linux/slab.h> 27 #include "drmP.h" 28 #include "radeon.h" 29 #include "radeon_asic.h" 30 #include "radeon_drm.h" 31 #include "evergreend.h" 32 #include "atom.h" 33 #include "avivod.h" 34 #include "evergreen_reg.h" 35 #include "evergreen_blit_shaders.h" 36 37 #define EVERGREEN_PFP_UCODE_SIZE 1120 38 #define EVERGREEN_PM4_UCODE_SIZE 1376 39 40 static void evergreen_gpu_init(struct radeon_device *rdev); 41 void evergreen_fini(struct radeon_device *rdev); 42 static void evergreen_pcie_gen2_enable(struct radeon_device *rdev); 43 44 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc) 45 { 46 /* enable the pflip int */ 47 radeon_irq_kms_pflip_irq_get(rdev, crtc); 48 } 49 50 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc) 51 { 52 /* disable the pflip int */ 53 radeon_irq_kms_pflip_irq_put(rdev, crtc); 54 } 55 56 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base) 57 { 58 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id]; 59 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset); 60 61 /* Lock the graphics update lock */ 62 tmp |= EVERGREEN_GRPH_UPDATE_LOCK; 63 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); 64 65 /* update the scanout addresses */ 66 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 67 upper_32_bits(crtc_base)); 68 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 69 (u32)crtc_base); 70 71 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset, 72 upper_32_bits(crtc_base)); 73 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset, 74 (u32)crtc_base); 75 76 /* Wait for update_pending to go high. */ 77 while (!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)); 78 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n"); 79 80 /* Unlock the lock, so double-buffering can take place inside vblank */ 81 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK; 82 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp); 83 84 /* Return current update_pending status: */ 85 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING; 86 } 87 88 /* get temperature in millidegrees */ 89 int evergreen_get_temp(struct radeon_device *rdev) 90 { 91 u32 temp, toffset; 92 int actual_temp = 0; 93 94 if (rdev->family == CHIP_JUNIPER) { 95 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >> 96 TOFFSET_SHIFT; 97 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >> 98 TS0_ADC_DOUT_SHIFT; 99 100 if (toffset & 0x100) 101 actual_temp = temp / 2 - (0x200 - toffset); 102 else 103 actual_temp = temp / 2 + toffset; 104 105 actual_temp = actual_temp * 1000; 106 107 } else { 108 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >> 109 ASIC_T_SHIFT; 110 111 if (temp & 0x400) 112 actual_temp = -256; 113 else if (temp & 0x200) 114 actual_temp = 255; 115 else if (temp & 0x100) { 116 actual_temp = temp & 0x1ff; 117 actual_temp |= ~0x1ff; 118 } else 119 actual_temp = temp & 0xff; 120 121 actual_temp = (actual_temp * 1000) / 2; 122 } 123 124 return actual_temp; 125 } 126 127 int sumo_get_temp(struct radeon_device *rdev) 128 { 129 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff; 130 int actual_temp = temp - 49; 131 132 return actual_temp * 1000; 133 } 134 135 void evergreen_pm_misc(struct radeon_device *rdev) 136 { 137 int req_ps_idx = rdev->pm.requested_power_state_index; 138 int req_cm_idx = rdev->pm.requested_clock_mode_index; 139 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx]; 140 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage; 141 142 if (voltage->type == VOLTAGE_SW) { 143 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) { 144 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC); 145 rdev->pm.current_vddc = voltage->voltage; 146 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage); 147 } 148 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) { 149 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI); 150 rdev->pm.current_vddci = voltage->vddci; 151 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci); 152 } 153 } 154 } 155 156 void evergreen_pm_prepare(struct radeon_device *rdev) 157 { 158 struct drm_device *ddev = rdev->ddev; 159 struct drm_crtc *crtc; 160 struct radeon_crtc *radeon_crtc; 161 u32 tmp; 162 163 /* disable any active CRTCs */ 164 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 165 radeon_crtc = to_radeon_crtc(crtc); 166 if (radeon_crtc->enabled) { 167 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 168 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 169 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 170 } 171 } 172 } 173 174 void evergreen_pm_finish(struct radeon_device *rdev) 175 { 176 struct drm_device *ddev = rdev->ddev; 177 struct drm_crtc *crtc; 178 struct radeon_crtc *radeon_crtc; 179 u32 tmp; 180 181 /* enable any active CRTCs */ 182 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) { 183 radeon_crtc = to_radeon_crtc(crtc); 184 if (radeon_crtc->enabled) { 185 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset); 186 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE; 187 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp); 188 } 189 } 190 } 191 192 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd) 193 { 194 bool connected = false; 195 196 switch (hpd) { 197 case RADEON_HPD_1: 198 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE) 199 connected = true; 200 break; 201 case RADEON_HPD_2: 202 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE) 203 connected = true; 204 break; 205 case RADEON_HPD_3: 206 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE) 207 connected = true; 208 break; 209 case RADEON_HPD_4: 210 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE) 211 connected = true; 212 break; 213 case RADEON_HPD_5: 214 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE) 215 connected = true; 216 break; 217 case RADEON_HPD_6: 218 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE) 219 connected = true; 220 break; 221 default: 222 break; 223 } 224 225 return connected; 226 } 227 228 void evergreen_hpd_set_polarity(struct radeon_device *rdev, 229 enum radeon_hpd_id hpd) 230 { 231 u32 tmp; 232 bool connected = evergreen_hpd_sense(rdev, hpd); 233 234 switch (hpd) { 235 case RADEON_HPD_1: 236 tmp = RREG32(DC_HPD1_INT_CONTROL); 237 if (connected) 238 tmp &= ~DC_HPDx_INT_POLARITY; 239 else 240 tmp |= DC_HPDx_INT_POLARITY; 241 WREG32(DC_HPD1_INT_CONTROL, tmp); 242 break; 243 case RADEON_HPD_2: 244 tmp = RREG32(DC_HPD2_INT_CONTROL); 245 if (connected) 246 tmp &= ~DC_HPDx_INT_POLARITY; 247 else 248 tmp |= DC_HPDx_INT_POLARITY; 249 WREG32(DC_HPD2_INT_CONTROL, tmp); 250 break; 251 case RADEON_HPD_3: 252 tmp = RREG32(DC_HPD3_INT_CONTROL); 253 if (connected) 254 tmp &= ~DC_HPDx_INT_POLARITY; 255 else 256 tmp |= DC_HPDx_INT_POLARITY; 257 WREG32(DC_HPD3_INT_CONTROL, tmp); 258 break; 259 case RADEON_HPD_4: 260 tmp = RREG32(DC_HPD4_INT_CONTROL); 261 if (connected) 262 tmp &= ~DC_HPDx_INT_POLARITY; 263 else 264 tmp |= DC_HPDx_INT_POLARITY; 265 WREG32(DC_HPD4_INT_CONTROL, tmp); 266 break; 267 case RADEON_HPD_5: 268 tmp = RREG32(DC_HPD5_INT_CONTROL); 269 if (connected) 270 tmp &= ~DC_HPDx_INT_POLARITY; 271 else 272 tmp |= DC_HPDx_INT_POLARITY; 273 WREG32(DC_HPD5_INT_CONTROL, tmp); 274 break; 275 case RADEON_HPD_6: 276 tmp = RREG32(DC_HPD6_INT_CONTROL); 277 if (connected) 278 tmp &= ~DC_HPDx_INT_POLARITY; 279 else 280 tmp |= DC_HPDx_INT_POLARITY; 281 WREG32(DC_HPD6_INT_CONTROL, tmp); 282 break; 283 default: 284 break; 285 } 286 } 287 288 void evergreen_hpd_init(struct radeon_device *rdev) 289 { 290 struct drm_device *dev = rdev->ddev; 291 struct drm_connector *connector; 292 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | 293 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN; 294 295 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 296 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 297 switch (radeon_connector->hpd.hpd) { 298 case RADEON_HPD_1: 299 WREG32(DC_HPD1_CONTROL, tmp); 300 rdev->irq.hpd[0] = true; 301 break; 302 case RADEON_HPD_2: 303 WREG32(DC_HPD2_CONTROL, tmp); 304 rdev->irq.hpd[1] = true; 305 break; 306 case RADEON_HPD_3: 307 WREG32(DC_HPD3_CONTROL, tmp); 308 rdev->irq.hpd[2] = true; 309 break; 310 case RADEON_HPD_4: 311 WREG32(DC_HPD4_CONTROL, tmp); 312 rdev->irq.hpd[3] = true; 313 break; 314 case RADEON_HPD_5: 315 WREG32(DC_HPD5_CONTROL, tmp); 316 rdev->irq.hpd[4] = true; 317 break; 318 case RADEON_HPD_6: 319 WREG32(DC_HPD6_CONTROL, tmp); 320 rdev->irq.hpd[5] = true; 321 break; 322 default: 323 break; 324 } 325 } 326 if (rdev->irq.installed) 327 evergreen_irq_set(rdev); 328 } 329 330 void evergreen_hpd_fini(struct radeon_device *rdev) 331 { 332 struct drm_device *dev = rdev->ddev; 333 struct drm_connector *connector; 334 335 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 336 struct radeon_connector *radeon_connector = to_radeon_connector(connector); 337 switch (radeon_connector->hpd.hpd) { 338 case RADEON_HPD_1: 339 WREG32(DC_HPD1_CONTROL, 0); 340 rdev->irq.hpd[0] = false; 341 break; 342 case RADEON_HPD_2: 343 WREG32(DC_HPD2_CONTROL, 0); 344 rdev->irq.hpd[1] = false; 345 break; 346 case RADEON_HPD_3: 347 WREG32(DC_HPD3_CONTROL, 0); 348 rdev->irq.hpd[2] = false; 349 break; 350 case RADEON_HPD_4: 351 WREG32(DC_HPD4_CONTROL, 0); 352 rdev->irq.hpd[3] = false; 353 break; 354 case RADEON_HPD_5: 355 WREG32(DC_HPD5_CONTROL, 0); 356 rdev->irq.hpd[4] = false; 357 break; 358 case RADEON_HPD_6: 359 WREG32(DC_HPD6_CONTROL, 0); 360 rdev->irq.hpd[5] = false; 361 break; 362 default: 363 break; 364 } 365 } 366 } 367 368 /* watermark setup */ 369 370 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev, 371 struct radeon_crtc *radeon_crtc, 372 struct drm_display_mode *mode, 373 struct drm_display_mode *other_mode) 374 { 375 u32 tmp; 376 /* 377 * Line Buffer Setup 378 * There are 3 line buffers, each one shared by 2 display controllers. 379 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between 380 * the display controllers. The paritioning is done via one of four 381 * preset allocations specified in bits 2:0: 382 * first display controller 383 * 0 - first half of lb (3840 * 2) 384 * 1 - first 3/4 of lb (5760 * 2) 385 * 2 - whole lb (7680 * 2), other crtc must be disabled 386 * 3 - first 1/4 of lb (1920 * 2) 387 * second display controller 388 * 4 - second half of lb (3840 * 2) 389 * 5 - second 3/4 of lb (5760 * 2) 390 * 6 - whole lb (7680 * 2), other crtc must be disabled 391 * 7 - last 1/4 of lb (1920 * 2) 392 */ 393 /* this can get tricky if we have two large displays on a paired group 394 * of crtcs. Ideally for multiple large displays we'd assign them to 395 * non-linked crtcs for maximum line buffer allocation. 396 */ 397 if (radeon_crtc->base.enabled && mode) { 398 if (other_mode) 399 tmp = 0; /* 1/2 */ 400 else 401 tmp = 2; /* whole */ 402 } else 403 tmp = 0; 404 405 /* second controller of the pair uses second half of the lb */ 406 if (radeon_crtc->crtc_id % 2) 407 tmp += 4; 408 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp); 409 410 if (radeon_crtc->base.enabled && mode) { 411 switch (tmp) { 412 case 0: 413 case 4: 414 default: 415 if (ASIC_IS_DCE5(rdev)) 416 return 4096 * 2; 417 else 418 return 3840 * 2; 419 case 1: 420 case 5: 421 if (ASIC_IS_DCE5(rdev)) 422 return 6144 * 2; 423 else 424 return 5760 * 2; 425 case 2: 426 case 6: 427 if (ASIC_IS_DCE5(rdev)) 428 return 8192 * 2; 429 else 430 return 7680 * 2; 431 case 3: 432 case 7: 433 if (ASIC_IS_DCE5(rdev)) 434 return 2048 * 2; 435 else 436 return 1920 * 2; 437 } 438 } 439 440 /* controller not enabled, so no lb used */ 441 return 0; 442 } 443 444 static u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev) 445 { 446 u32 tmp = RREG32(MC_SHARED_CHMAP); 447 448 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 449 case 0: 450 default: 451 return 1; 452 case 1: 453 return 2; 454 case 2: 455 return 4; 456 case 3: 457 return 8; 458 } 459 } 460 461 struct evergreen_wm_params { 462 u32 dram_channels; /* number of dram channels */ 463 u32 yclk; /* bandwidth per dram data pin in kHz */ 464 u32 sclk; /* engine clock in kHz */ 465 u32 disp_clk; /* display clock in kHz */ 466 u32 src_width; /* viewport width */ 467 u32 active_time; /* active display time in ns */ 468 u32 blank_time; /* blank time in ns */ 469 bool interlaced; /* mode is interlaced */ 470 fixed20_12 vsc; /* vertical scale ratio */ 471 u32 num_heads; /* number of active crtcs */ 472 u32 bytes_per_pixel; /* bytes per pixel display + overlay */ 473 u32 lb_size; /* line buffer allocated to pipe */ 474 u32 vtaps; /* vertical scaler taps */ 475 }; 476 477 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm) 478 { 479 /* Calculate DRAM Bandwidth and the part allocated to display. */ 480 fixed20_12 dram_efficiency; /* 0.7 */ 481 fixed20_12 yclk, dram_channels, bandwidth; 482 fixed20_12 a; 483 484 a.full = dfixed_const(1000); 485 yclk.full = dfixed_const(wm->yclk); 486 yclk.full = dfixed_div(yclk, a); 487 dram_channels.full = dfixed_const(wm->dram_channels * 4); 488 a.full = dfixed_const(10); 489 dram_efficiency.full = dfixed_const(7); 490 dram_efficiency.full = dfixed_div(dram_efficiency, a); 491 bandwidth.full = dfixed_mul(dram_channels, yclk); 492 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency); 493 494 return dfixed_trunc(bandwidth); 495 } 496 497 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 498 { 499 /* Calculate DRAM Bandwidth and the part allocated to display. */ 500 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */ 501 fixed20_12 yclk, dram_channels, bandwidth; 502 fixed20_12 a; 503 504 a.full = dfixed_const(1000); 505 yclk.full = dfixed_const(wm->yclk); 506 yclk.full = dfixed_div(yclk, a); 507 dram_channels.full = dfixed_const(wm->dram_channels * 4); 508 a.full = dfixed_const(10); 509 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */ 510 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a); 511 bandwidth.full = dfixed_mul(dram_channels, yclk); 512 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation); 513 514 return dfixed_trunc(bandwidth); 515 } 516 517 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm) 518 { 519 /* Calculate the display Data return Bandwidth */ 520 fixed20_12 return_efficiency; /* 0.8 */ 521 fixed20_12 sclk, bandwidth; 522 fixed20_12 a; 523 524 a.full = dfixed_const(1000); 525 sclk.full = dfixed_const(wm->sclk); 526 sclk.full = dfixed_div(sclk, a); 527 a.full = dfixed_const(10); 528 return_efficiency.full = dfixed_const(8); 529 return_efficiency.full = dfixed_div(return_efficiency, a); 530 a.full = dfixed_const(32); 531 bandwidth.full = dfixed_mul(a, sclk); 532 bandwidth.full = dfixed_mul(bandwidth, return_efficiency); 533 534 return dfixed_trunc(bandwidth); 535 } 536 537 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm) 538 { 539 /* Calculate the DMIF Request Bandwidth */ 540 fixed20_12 disp_clk_request_efficiency; /* 0.8 */ 541 fixed20_12 disp_clk, bandwidth; 542 fixed20_12 a; 543 544 a.full = dfixed_const(1000); 545 disp_clk.full = dfixed_const(wm->disp_clk); 546 disp_clk.full = dfixed_div(disp_clk, a); 547 a.full = dfixed_const(10); 548 disp_clk_request_efficiency.full = dfixed_const(8); 549 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a); 550 a.full = dfixed_const(32); 551 bandwidth.full = dfixed_mul(a, disp_clk); 552 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency); 553 554 return dfixed_trunc(bandwidth); 555 } 556 557 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm) 558 { 559 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */ 560 u32 dram_bandwidth = evergreen_dram_bandwidth(wm); 561 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm); 562 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm); 563 564 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth)); 565 } 566 567 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm) 568 { 569 /* Calculate the display mode Average Bandwidth 570 * DisplayMode should contain the source and destination dimensions, 571 * timing, etc. 572 */ 573 fixed20_12 bpp; 574 fixed20_12 line_time; 575 fixed20_12 src_width; 576 fixed20_12 bandwidth; 577 fixed20_12 a; 578 579 a.full = dfixed_const(1000); 580 line_time.full = dfixed_const(wm->active_time + wm->blank_time); 581 line_time.full = dfixed_div(line_time, a); 582 bpp.full = dfixed_const(wm->bytes_per_pixel); 583 src_width.full = dfixed_const(wm->src_width); 584 bandwidth.full = dfixed_mul(src_width, bpp); 585 bandwidth.full = dfixed_mul(bandwidth, wm->vsc); 586 bandwidth.full = dfixed_div(bandwidth, line_time); 587 588 return dfixed_trunc(bandwidth); 589 } 590 591 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm) 592 { 593 /* First calcualte the latency in ns */ 594 u32 mc_latency = 2000; /* 2000 ns. */ 595 u32 available_bandwidth = evergreen_available_bandwidth(wm); 596 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth; 597 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth; 598 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */ 599 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) + 600 (wm->num_heads * cursor_line_pair_return_time); 601 u32 latency = mc_latency + other_heads_data_return_time + dc_latency; 602 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time; 603 fixed20_12 a, b, c; 604 605 if (wm->num_heads == 0) 606 return 0; 607 608 a.full = dfixed_const(2); 609 b.full = dfixed_const(1); 610 if ((wm->vsc.full > a.full) || 611 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) || 612 (wm->vtaps >= 5) || 613 ((wm->vsc.full >= a.full) && wm->interlaced)) 614 max_src_lines_per_dst_line = 4; 615 else 616 max_src_lines_per_dst_line = 2; 617 618 a.full = dfixed_const(available_bandwidth); 619 b.full = dfixed_const(wm->num_heads); 620 a.full = dfixed_div(a, b); 621 622 b.full = dfixed_const(1000); 623 c.full = dfixed_const(wm->disp_clk); 624 b.full = dfixed_div(c, b); 625 c.full = dfixed_const(wm->bytes_per_pixel); 626 b.full = dfixed_mul(b, c); 627 628 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b)); 629 630 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel); 631 b.full = dfixed_const(1000); 632 c.full = dfixed_const(lb_fill_bw); 633 b.full = dfixed_div(c, b); 634 a.full = dfixed_div(a, b); 635 line_fill_time = dfixed_trunc(a); 636 637 if (line_fill_time < wm->active_time) 638 return latency; 639 else 640 return latency + (line_fill_time - wm->active_time); 641 642 } 643 644 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm) 645 { 646 if (evergreen_average_bandwidth(wm) <= 647 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads)) 648 return true; 649 else 650 return false; 651 }; 652 653 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm) 654 { 655 if (evergreen_average_bandwidth(wm) <= 656 (evergreen_available_bandwidth(wm) / wm->num_heads)) 657 return true; 658 else 659 return false; 660 }; 661 662 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm) 663 { 664 u32 lb_partitions = wm->lb_size / wm->src_width; 665 u32 line_time = wm->active_time + wm->blank_time; 666 u32 latency_tolerant_lines; 667 u32 latency_hiding; 668 fixed20_12 a; 669 670 a.full = dfixed_const(1); 671 if (wm->vsc.full > a.full) 672 latency_tolerant_lines = 1; 673 else { 674 if (lb_partitions <= (wm->vtaps + 1)) 675 latency_tolerant_lines = 1; 676 else 677 latency_tolerant_lines = 2; 678 } 679 680 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time); 681 682 if (evergreen_latency_watermark(wm) <= latency_hiding) 683 return true; 684 else 685 return false; 686 } 687 688 static void evergreen_program_watermarks(struct radeon_device *rdev, 689 struct radeon_crtc *radeon_crtc, 690 u32 lb_size, u32 num_heads) 691 { 692 struct drm_display_mode *mode = &radeon_crtc->base.mode; 693 struct evergreen_wm_params wm; 694 u32 pixel_period; 695 u32 line_time = 0; 696 u32 latency_watermark_a = 0, latency_watermark_b = 0; 697 u32 priority_a_mark = 0, priority_b_mark = 0; 698 u32 priority_a_cnt = PRIORITY_OFF; 699 u32 priority_b_cnt = PRIORITY_OFF; 700 u32 pipe_offset = radeon_crtc->crtc_id * 16; 701 u32 tmp, arb_control3; 702 fixed20_12 a, b, c; 703 704 if (radeon_crtc->base.enabled && num_heads && mode) { 705 pixel_period = 1000000 / (u32)mode->clock; 706 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535); 707 priority_a_cnt = 0; 708 priority_b_cnt = 0; 709 710 wm.yclk = rdev->pm.current_mclk * 10; 711 wm.sclk = rdev->pm.current_sclk * 10; 712 wm.disp_clk = mode->clock; 713 wm.src_width = mode->crtc_hdisplay; 714 wm.active_time = mode->crtc_hdisplay * pixel_period; 715 wm.blank_time = line_time - wm.active_time; 716 wm.interlaced = false; 717 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 718 wm.interlaced = true; 719 wm.vsc = radeon_crtc->vsc; 720 wm.vtaps = 1; 721 if (radeon_crtc->rmx_type != RMX_OFF) 722 wm.vtaps = 2; 723 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */ 724 wm.lb_size = lb_size; 725 wm.dram_channels = evergreen_get_number_of_dram_channels(rdev); 726 wm.num_heads = num_heads; 727 728 /* set for high clocks */ 729 latency_watermark_a = min(evergreen_latency_watermark(&wm), (u32)65535); 730 /* set for low clocks */ 731 /* wm.yclk = low clk; wm.sclk = low clk */ 732 latency_watermark_b = min(evergreen_latency_watermark(&wm), (u32)65535); 733 734 /* possibly force display priority to high */ 735 /* should really do this at mode validation time... */ 736 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm) || 737 !evergreen_average_bandwidth_vs_available_bandwidth(&wm) || 738 !evergreen_check_latency_hiding(&wm) || 739 (rdev->disp_priority == 2)) { 740 DRM_INFO("force priority to high\n"); 741 priority_a_cnt |= PRIORITY_ALWAYS_ON; 742 priority_b_cnt |= PRIORITY_ALWAYS_ON; 743 } 744 745 a.full = dfixed_const(1000); 746 b.full = dfixed_const(mode->clock); 747 b.full = dfixed_div(b, a); 748 c.full = dfixed_const(latency_watermark_a); 749 c.full = dfixed_mul(c, b); 750 c.full = dfixed_mul(c, radeon_crtc->hsc); 751 c.full = dfixed_div(c, a); 752 a.full = dfixed_const(16); 753 c.full = dfixed_div(c, a); 754 priority_a_mark = dfixed_trunc(c); 755 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK; 756 757 a.full = dfixed_const(1000); 758 b.full = dfixed_const(mode->clock); 759 b.full = dfixed_div(b, a); 760 c.full = dfixed_const(latency_watermark_b); 761 c.full = dfixed_mul(c, b); 762 c.full = dfixed_mul(c, radeon_crtc->hsc); 763 c.full = dfixed_div(c, a); 764 a.full = dfixed_const(16); 765 c.full = dfixed_div(c, a); 766 priority_b_mark = dfixed_trunc(c); 767 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK; 768 } 769 770 /* select wm A */ 771 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 772 tmp = arb_control3; 773 tmp &= ~LATENCY_WATERMARK_MASK(3); 774 tmp |= LATENCY_WATERMARK_MASK(1); 775 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 776 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 777 (LATENCY_LOW_WATERMARK(latency_watermark_a) | 778 LATENCY_HIGH_WATERMARK(line_time))); 779 /* select wm B */ 780 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset); 781 tmp &= ~LATENCY_WATERMARK_MASK(3); 782 tmp |= LATENCY_WATERMARK_MASK(2); 783 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp); 784 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset, 785 (LATENCY_LOW_WATERMARK(latency_watermark_b) | 786 LATENCY_HIGH_WATERMARK(line_time))); 787 /* restore original selection */ 788 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3); 789 790 /* write the priority marks */ 791 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt); 792 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt); 793 794 } 795 796 void evergreen_bandwidth_update(struct radeon_device *rdev) 797 { 798 struct drm_display_mode *mode0 = NULL; 799 struct drm_display_mode *mode1 = NULL; 800 u32 num_heads = 0, lb_size; 801 int i; 802 803 radeon_update_display_priority(rdev); 804 805 for (i = 0; i < rdev->num_crtc; i++) { 806 if (rdev->mode_info.crtcs[i]->base.enabled) 807 num_heads++; 808 } 809 for (i = 0; i < rdev->num_crtc; i += 2) { 810 mode0 = &rdev->mode_info.crtcs[i]->base.mode; 811 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode; 812 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1); 813 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); 814 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0); 815 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads); 816 } 817 } 818 819 int evergreen_mc_wait_for_idle(struct radeon_device *rdev) 820 { 821 unsigned i; 822 u32 tmp; 823 824 for (i = 0; i < rdev->usec_timeout; i++) { 825 /* read MC_STATUS */ 826 tmp = RREG32(SRBM_STATUS) & 0x1F00; 827 if (!tmp) 828 return 0; 829 udelay(1); 830 } 831 return -1; 832 } 833 834 /* 835 * GART 836 */ 837 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev) 838 { 839 unsigned i; 840 u32 tmp; 841 842 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1); 843 844 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1)); 845 for (i = 0; i < rdev->usec_timeout; i++) { 846 /* read MC_STATUS */ 847 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE); 848 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT; 849 if (tmp == 2) { 850 printk(KERN_WARNING "[drm] r600 flush TLB failed\n"); 851 return; 852 } 853 if (tmp) { 854 return; 855 } 856 udelay(1); 857 } 858 } 859 860 int evergreen_pcie_gart_enable(struct radeon_device *rdev) 861 { 862 u32 tmp; 863 int r; 864 865 if (rdev->gart.table.vram.robj == NULL) { 866 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); 867 return -EINVAL; 868 } 869 r = radeon_gart_table_vram_pin(rdev); 870 if (r) 871 return r; 872 radeon_gart_restore(rdev); 873 /* Setup L2 cache */ 874 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 875 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 876 EFFECTIVE_L2_QUEUE_SIZE(7)); 877 WREG32(VM_L2_CNTL2, 0); 878 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 879 /* Setup TLB control */ 880 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 881 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 882 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 883 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 884 if (rdev->flags & RADEON_IS_IGP) { 885 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp); 886 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp); 887 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp); 888 } else { 889 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 890 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 891 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 892 } 893 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 894 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 895 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 896 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 897 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); 898 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); 899 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); 900 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) | 901 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT); 902 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR, 903 (u32)(rdev->dummy_page.addr >> 12)); 904 WREG32(VM_CONTEXT1_CNTL, 0); 905 906 evergreen_pcie_gart_tlb_flush(rdev); 907 rdev->gart.ready = true; 908 return 0; 909 } 910 911 void evergreen_pcie_gart_disable(struct radeon_device *rdev) 912 { 913 u32 tmp; 914 int r; 915 916 /* Disable all tables */ 917 WREG32(VM_CONTEXT0_CNTL, 0); 918 WREG32(VM_CONTEXT1_CNTL, 0); 919 920 /* Setup L2 cache */ 921 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING | 922 EFFECTIVE_L2_QUEUE_SIZE(7)); 923 WREG32(VM_L2_CNTL2, 0); 924 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 925 /* Setup TLB control */ 926 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 927 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 928 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 929 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 930 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 931 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 932 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 933 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 934 if (rdev->gart.table.vram.robj) { 935 r = radeon_bo_reserve(rdev->gart.table.vram.robj, false); 936 if (likely(r == 0)) { 937 radeon_bo_kunmap(rdev->gart.table.vram.robj); 938 radeon_bo_unpin(rdev->gart.table.vram.robj); 939 radeon_bo_unreserve(rdev->gart.table.vram.robj); 940 } 941 } 942 } 943 944 void evergreen_pcie_gart_fini(struct radeon_device *rdev) 945 { 946 evergreen_pcie_gart_disable(rdev); 947 radeon_gart_table_vram_free(rdev); 948 radeon_gart_fini(rdev); 949 } 950 951 952 void evergreen_agp_enable(struct radeon_device *rdev) 953 { 954 u32 tmp; 955 956 /* Setup L2 cache */ 957 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING | 958 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE | 959 EFFECTIVE_L2_QUEUE_SIZE(7)); 960 WREG32(VM_L2_CNTL2, 0); 961 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2)); 962 /* Setup TLB control */ 963 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING | 964 SYSTEM_ACCESS_MODE_NOT_IN_SYS | 965 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU | 966 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5); 967 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp); 968 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp); 969 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp); 970 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp); 971 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp); 972 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp); 973 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp); 974 WREG32(VM_CONTEXT0_CNTL, 0); 975 WREG32(VM_CONTEXT1_CNTL, 0); 976 } 977 978 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save) 979 { 980 save->vga_control[0] = RREG32(D1VGA_CONTROL); 981 save->vga_control[1] = RREG32(D2VGA_CONTROL); 982 save->vga_control[2] = RREG32(EVERGREEN_D3VGA_CONTROL); 983 save->vga_control[3] = RREG32(EVERGREEN_D4VGA_CONTROL); 984 save->vga_control[4] = RREG32(EVERGREEN_D5VGA_CONTROL); 985 save->vga_control[5] = RREG32(EVERGREEN_D6VGA_CONTROL); 986 save->vga_render_control = RREG32(VGA_RENDER_CONTROL); 987 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL); 988 save->crtc_control[0] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET); 989 save->crtc_control[1] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET); 990 if (!(rdev->flags & RADEON_IS_IGP)) { 991 save->crtc_control[2] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET); 992 save->crtc_control[3] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET); 993 save->crtc_control[4] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET); 994 save->crtc_control[5] = RREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET); 995 } 996 997 /* Stop all video */ 998 WREG32(VGA_RENDER_CONTROL, 0); 999 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1); 1000 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1); 1001 if (!(rdev->flags & RADEON_IS_IGP)) { 1002 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1); 1003 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1); 1004 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1); 1005 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1); 1006 } 1007 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 1008 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 1009 if (!(rdev->flags & RADEON_IS_IGP)) { 1010 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 1011 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 1012 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 1013 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 1014 } 1015 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 1016 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 1017 if (!(rdev->flags & RADEON_IS_IGP)) { 1018 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 1019 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 1020 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 1021 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 1022 } 1023 1024 WREG32(D1VGA_CONTROL, 0); 1025 WREG32(D2VGA_CONTROL, 0); 1026 WREG32(EVERGREEN_D3VGA_CONTROL, 0); 1027 WREG32(EVERGREEN_D4VGA_CONTROL, 0); 1028 WREG32(EVERGREEN_D5VGA_CONTROL, 0); 1029 WREG32(EVERGREEN_D6VGA_CONTROL, 0); 1030 } 1031 1032 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save) 1033 { 1034 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET, 1035 upper_32_bits(rdev->mc.vram_start)); 1036 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC0_REGISTER_OFFSET, 1037 upper_32_bits(rdev->mc.vram_start)); 1038 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET, 1039 (u32)rdev->mc.vram_start); 1040 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC0_REGISTER_OFFSET, 1041 (u32)rdev->mc.vram_start); 1042 1043 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET, 1044 upper_32_bits(rdev->mc.vram_start)); 1045 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC1_REGISTER_OFFSET, 1046 upper_32_bits(rdev->mc.vram_start)); 1047 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET, 1048 (u32)rdev->mc.vram_start); 1049 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC1_REGISTER_OFFSET, 1050 (u32)rdev->mc.vram_start); 1051 1052 if (!(rdev->flags & RADEON_IS_IGP)) { 1053 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET, 1054 upper_32_bits(rdev->mc.vram_start)); 1055 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC2_REGISTER_OFFSET, 1056 upper_32_bits(rdev->mc.vram_start)); 1057 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET, 1058 (u32)rdev->mc.vram_start); 1059 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC2_REGISTER_OFFSET, 1060 (u32)rdev->mc.vram_start); 1061 1062 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET, 1063 upper_32_bits(rdev->mc.vram_start)); 1064 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC3_REGISTER_OFFSET, 1065 upper_32_bits(rdev->mc.vram_start)); 1066 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET, 1067 (u32)rdev->mc.vram_start); 1068 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC3_REGISTER_OFFSET, 1069 (u32)rdev->mc.vram_start); 1070 1071 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET, 1072 upper_32_bits(rdev->mc.vram_start)); 1073 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC4_REGISTER_OFFSET, 1074 upper_32_bits(rdev->mc.vram_start)); 1075 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET, 1076 (u32)rdev->mc.vram_start); 1077 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC4_REGISTER_OFFSET, 1078 (u32)rdev->mc.vram_start); 1079 1080 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET, 1081 upper_32_bits(rdev->mc.vram_start)); 1082 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + EVERGREEN_CRTC5_REGISTER_OFFSET, 1083 upper_32_bits(rdev->mc.vram_start)); 1084 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET, 1085 (u32)rdev->mc.vram_start); 1086 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + EVERGREEN_CRTC5_REGISTER_OFFSET, 1087 (u32)rdev->mc.vram_start); 1088 } 1089 1090 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start)); 1091 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start); 1092 /* Unlock host access */ 1093 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control); 1094 mdelay(1); 1095 /* Restore video state */ 1096 WREG32(D1VGA_CONTROL, save->vga_control[0]); 1097 WREG32(D2VGA_CONTROL, save->vga_control[1]); 1098 WREG32(EVERGREEN_D3VGA_CONTROL, save->vga_control[2]); 1099 WREG32(EVERGREEN_D4VGA_CONTROL, save->vga_control[3]); 1100 WREG32(EVERGREEN_D5VGA_CONTROL, save->vga_control[4]); 1101 WREG32(EVERGREEN_D6VGA_CONTROL, save->vga_control[5]); 1102 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 1); 1103 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 1); 1104 if (!(rdev->flags & RADEON_IS_IGP)) { 1105 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 1); 1106 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 1); 1107 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 1); 1108 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 1); 1109 } 1110 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, save->crtc_control[0]); 1111 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, save->crtc_control[1]); 1112 if (!(rdev->flags & RADEON_IS_IGP)) { 1113 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, save->crtc_control[2]); 1114 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, save->crtc_control[3]); 1115 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, save->crtc_control[4]); 1116 WREG32(EVERGREEN_CRTC_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, save->crtc_control[5]); 1117 } 1118 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 1119 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 1120 if (!(rdev->flags & RADEON_IS_IGP)) { 1121 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 1122 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 1123 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 1124 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 1125 } 1126 WREG32(VGA_RENDER_CONTROL, save->vga_render_control); 1127 } 1128 1129 void evergreen_mc_program(struct radeon_device *rdev) 1130 { 1131 struct evergreen_mc_save save; 1132 u32 tmp; 1133 int i, j; 1134 1135 /* Initialize HDP */ 1136 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 1137 WREG32((0x2c14 + j), 0x00000000); 1138 WREG32((0x2c18 + j), 0x00000000); 1139 WREG32((0x2c1c + j), 0x00000000); 1140 WREG32((0x2c20 + j), 0x00000000); 1141 WREG32((0x2c24 + j), 0x00000000); 1142 } 1143 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0); 1144 1145 evergreen_mc_stop(rdev, &save); 1146 if (evergreen_mc_wait_for_idle(rdev)) { 1147 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 1148 } 1149 /* Lockout access through VGA aperture*/ 1150 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE); 1151 /* Update configuration */ 1152 if (rdev->flags & RADEON_IS_AGP) { 1153 if (rdev->mc.vram_start < rdev->mc.gtt_start) { 1154 /* VRAM before AGP */ 1155 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 1156 rdev->mc.vram_start >> 12); 1157 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 1158 rdev->mc.gtt_end >> 12); 1159 } else { 1160 /* VRAM after AGP */ 1161 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 1162 rdev->mc.gtt_start >> 12); 1163 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 1164 rdev->mc.vram_end >> 12); 1165 } 1166 } else { 1167 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, 1168 rdev->mc.vram_start >> 12); 1169 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, 1170 rdev->mc.vram_end >> 12); 1171 } 1172 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, 0); 1173 if (rdev->flags & RADEON_IS_IGP) { 1174 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF; 1175 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24; 1176 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20; 1177 WREG32(MC_FUS_VM_FB_OFFSET, tmp); 1178 } 1179 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; 1180 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); 1181 WREG32(MC_VM_FB_LOCATION, tmp); 1182 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); 1183 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30)); 1184 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF); 1185 if (rdev->flags & RADEON_IS_AGP) { 1186 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16); 1187 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16); 1188 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22); 1189 } else { 1190 WREG32(MC_VM_AGP_BASE, 0); 1191 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF); 1192 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF); 1193 } 1194 if (evergreen_mc_wait_for_idle(rdev)) { 1195 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 1196 } 1197 evergreen_mc_resume(rdev, &save); 1198 /* we need to own VRAM, so turn off the VGA renderer here 1199 * to stop it overwriting our objects */ 1200 rv515_vga_render_disable(rdev); 1201 } 1202 1203 /* 1204 * CP. 1205 */ 1206 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) 1207 { 1208 /* set to DX10/11 mode */ 1209 radeon_ring_write(rdev, PACKET3(PACKET3_MODE_CONTROL, 0)); 1210 radeon_ring_write(rdev, 1); 1211 /* FIXME: implement */ 1212 radeon_ring_write(rdev, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 1213 radeon_ring_write(rdev, 1214 #ifdef __BIG_ENDIAN 1215 (2 << 0) | 1216 #endif 1217 (ib->gpu_addr & 0xFFFFFFFC)); 1218 radeon_ring_write(rdev, upper_32_bits(ib->gpu_addr) & 0xFF); 1219 radeon_ring_write(rdev, ib->length_dw); 1220 } 1221 1222 1223 static int evergreen_cp_load_microcode(struct radeon_device *rdev) 1224 { 1225 const __be32 *fw_data; 1226 int i; 1227 1228 if (!rdev->me_fw || !rdev->pfp_fw) 1229 return -EINVAL; 1230 1231 r700_cp_stop(rdev); 1232 WREG32(CP_RB_CNTL, 1233 #ifdef __BIG_ENDIAN 1234 BUF_SWAP_32BIT | 1235 #endif 1236 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3)); 1237 1238 fw_data = (const __be32 *)rdev->pfp_fw->data; 1239 WREG32(CP_PFP_UCODE_ADDR, 0); 1240 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++) 1241 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++)); 1242 WREG32(CP_PFP_UCODE_ADDR, 0); 1243 1244 fw_data = (const __be32 *)rdev->me_fw->data; 1245 WREG32(CP_ME_RAM_WADDR, 0); 1246 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++) 1247 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++)); 1248 1249 WREG32(CP_PFP_UCODE_ADDR, 0); 1250 WREG32(CP_ME_RAM_WADDR, 0); 1251 WREG32(CP_ME_RAM_RADDR, 0); 1252 return 0; 1253 } 1254 1255 static int evergreen_cp_start(struct radeon_device *rdev) 1256 { 1257 int r, i; 1258 uint32_t cp_me; 1259 1260 r = radeon_ring_lock(rdev, 7); 1261 if (r) { 1262 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1263 return r; 1264 } 1265 radeon_ring_write(rdev, PACKET3(PACKET3_ME_INITIALIZE, 5)); 1266 radeon_ring_write(rdev, 0x1); 1267 radeon_ring_write(rdev, 0x0); 1268 radeon_ring_write(rdev, rdev->config.evergreen.max_hw_contexts - 1); 1269 radeon_ring_write(rdev, PACKET3_ME_INITIALIZE_DEVICE_ID(1)); 1270 radeon_ring_write(rdev, 0); 1271 radeon_ring_write(rdev, 0); 1272 radeon_ring_unlock_commit(rdev); 1273 1274 cp_me = 0xff; 1275 WREG32(CP_ME_CNTL, cp_me); 1276 1277 r = radeon_ring_lock(rdev, evergreen_default_size + 19); 1278 if (r) { 1279 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r); 1280 return r; 1281 } 1282 1283 /* setup clear context state */ 1284 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1285 radeon_ring_write(rdev, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1286 1287 for (i = 0; i < evergreen_default_size; i++) 1288 radeon_ring_write(rdev, evergreen_default_state[i]); 1289 1290 radeon_ring_write(rdev, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1291 radeon_ring_write(rdev, PACKET3_PREAMBLE_END_CLEAR_STATE); 1292 1293 /* set clear context state */ 1294 radeon_ring_write(rdev, PACKET3(PACKET3_CLEAR_STATE, 0)); 1295 radeon_ring_write(rdev, 0); 1296 1297 /* SQ_VTX_BASE_VTX_LOC */ 1298 radeon_ring_write(rdev, 0xc0026f00); 1299 radeon_ring_write(rdev, 0x00000000); 1300 radeon_ring_write(rdev, 0x00000000); 1301 radeon_ring_write(rdev, 0x00000000); 1302 1303 /* Clear consts */ 1304 radeon_ring_write(rdev, 0xc0036f00); 1305 radeon_ring_write(rdev, 0x00000bc4); 1306 radeon_ring_write(rdev, 0xffffffff); 1307 radeon_ring_write(rdev, 0xffffffff); 1308 radeon_ring_write(rdev, 0xffffffff); 1309 1310 radeon_ring_write(rdev, 0xc0026900); 1311 radeon_ring_write(rdev, 0x00000316); 1312 radeon_ring_write(rdev, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */ 1313 radeon_ring_write(rdev, 0x00000010); /* */ 1314 1315 radeon_ring_unlock_commit(rdev); 1316 1317 return 0; 1318 } 1319 1320 int evergreen_cp_resume(struct radeon_device *rdev) 1321 { 1322 u32 tmp; 1323 u32 rb_bufsz; 1324 int r; 1325 1326 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */ 1327 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP | 1328 SOFT_RESET_PA | 1329 SOFT_RESET_SH | 1330 SOFT_RESET_VGT | 1331 SOFT_RESET_SX)); 1332 RREG32(GRBM_SOFT_RESET); 1333 mdelay(15); 1334 WREG32(GRBM_SOFT_RESET, 0); 1335 RREG32(GRBM_SOFT_RESET); 1336 1337 /* Set ring buffer size */ 1338 rb_bufsz = drm_order(rdev->cp.ring_size / 8); 1339 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz; 1340 #ifdef __BIG_ENDIAN 1341 tmp |= BUF_SWAP_32BIT; 1342 #endif 1343 WREG32(CP_RB_CNTL, tmp); 1344 WREG32(CP_SEM_WAIT_TIMER, 0x4); 1345 1346 /* Set the write pointer delay */ 1347 WREG32(CP_RB_WPTR_DELAY, 0); 1348 1349 /* Initialize the ring buffer's read and write pointers */ 1350 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA); 1351 WREG32(CP_RB_RPTR_WR, 0); 1352 WREG32(CP_RB_WPTR, 0); 1353 1354 /* set the wb address wether it's enabled or not */ 1355 WREG32(CP_RB_RPTR_ADDR, 1356 #ifdef __BIG_ENDIAN 1357 RB_RPTR_SWAP(2) | 1358 #endif 1359 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC)); 1360 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); 1361 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); 1362 1363 if (rdev->wb.enabled) 1364 WREG32(SCRATCH_UMSK, 0xff); 1365 else { 1366 tmp |= RB_NO_UPDATE; 1367 WREG32(SCRATCH_UMSK, 0); 1368 } 1369 1370 mdelay(1); 1371 WREG32(CP_RB_CNTL, tmp); 1372 1373 WREG32(CP_RB_BASE, rdev->cp.gpu_addr >> 8); 1374 WREG32(CP_DEBUG, (1 << 27) | (1 << 28)); 1375 1376 rdev->cp.rptr = RREG32(CP_RB_RPTR); 1377 rdev->cp.wptr = RREG32(CP_RB_WPTR); 1378 1379 evergreen_cp_start(rdev); 1380 rdev->cp.ready = true; 1381 r = radeon_ring_test(rdev); 1382 if (r) { 1383 rdev->cp.ready = false; 1384 return r; 1385 } 1386 return 0; 1387 } 1388 1389 /* 1390 * Core functions 1391 */ 1392 static u32 evergreen_get_tile_pipe_to_backend_map(struct radeon_device *rdev, 1393 u32 num_tile_pipes, 1394 u32 num_backends, 1395 u32 backend_disable_mask) 1396 { 1397 u32 backend_map = 0; 1398 u32 enabled_backends_mask = 0; 1399 u32 enabled_backends_count = 0; 1400 u32 cur_pipe; 1401 u32 swizzle_pipe[EVERGREEN_MAX_PIPES]; 1402 u32 cur_backend = 0; 1403 u32 i; 1404 bool force_no_swizzle; 1405 1406 if (num_tile_pipes > EVERGREEN_MAX_PIPES) 1407 num_tile_pipes = EVERGREEN_MAX_PIPES; 1408 if (num_tile_pipes < 1) 1409 num_tile_pipes = 1; 1410 if (num_backends > EVERGREEN_MAX_BACKENDS) 1411 num_backends = EVERGREEN_MAX_BACKENDS; 1412 if (num_backends < 1) 1413 num_backends = 1; 1414 1415 for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) { 1416 if (((backend_disable_mask >> i) & 1) == 0) { 1417 enabled_backends_mask |= (1 << i); 1418 ++enabled_backends_count; 1419 } 1420 if (enabled_backends_count == num_backends) 1421 break; 1422 } 1423 1424 if (enabled_backends_count == 0) { 1425 enabled_backends_mask = 1; 1426 enabled_backends_count = 1; 1427 } 1428 1429 if (enabled_backends_count != num_backends) 1430 num_backends = enabled_backends_count; 1431 1432 memset((uint8_t *)&swizzle_pipe[0], 0, sizeof(u32) * EVERGREEN_MAX_PIPES); 1433 switch (rdev->family) { 1434 case CHIP_CEDAR: 1435 case CHIP_REDWOOD: 1436 case CHIP_PALM: 1437 case CHIP_SUMO: 1438 case CHIP_SUMO2: 1439 case CHIP_TURKS: 1440 case CHIP_CAICOS: 1441 force_no_swizzle = false; 1442 break; 1443 case CHIP_CYPRESS: 1444 case CHIP_HEMLOCK: 1445 case CHIP_JUNIPER: 1446 case CHIP_BARTS: 1447 default: 1448 force_no_swizzle = true; 1449 break; 1450 } 1451 if (force_no_swizzle) { 1452 bool last_backend_enabled = false; 1453 1454 force_no_swizzle = false; 1455 for (i = 0; i < EVERGREEN_MAX_BACKENDS; ++i) { 1456 if (((enabled_backends_mask >> i) & 1) == 1) { 1457 if (last_backend_enabled) 1458 force_no_swizzle = true; 1459 last_backend_enabled = true; 1460 } else 1461 last_backend_enabled = false; 1462 } 1463 } 1464 1465 switch (num_tile_pipes) { 1466 case 1: 1467 case 3: 1468 case 5: 1469 case 7: 1470 DRM_ERROR("odd number of pipes!\n"); 1471 break; 1472 case 2: 1473 swizzle_pipe[0] = 0; 1474 swizzle_pipe[1] = 1; 1475 break; 1476 case 4: 1477 if (force_no_swizzle) { 1478 swizzle_pipe[0] = 0; 1479 swizzle_pipe[1] = 1; 1480 swizzle_pipe[2] = 2; 1481 swizzle_pipe[3] = 3; 1482 } else { 1483 swizzle_pipe[0] = 0; 1484 swizzle_pipe[1] = 2; 1485 swizzle_pipe[2] = 1; 1486 swizzle_pipe[3] = 3; 1487 } 1488 break; 1489 case 6: 1490 if (force_no_swizzle) { 1491 swizzle_pipe[0] = 0; 1492 swizzle_pipe[1] = 1; 1493 swizzle_pipe[2] = 2; 1494 swizzle_pipe[3] = 3; 1495 swizzle_pipe[4] = 4; 1496 swizzle_pipe[5] = 5; 1497 } else { 1498 swizzle_pipe[0] = 0; 1499 swizzle_pipe[1] = 2; 1500 swizzle_pipe[2] = 4; 1501 swizzle_pipe[3] = 1; 1502 swizzle_pipe[4] = 3; 1503 swizzle_pipe[5] = 5; 1504 } 1505 break; 1506 case 8: 1507 if (force_no_swizzle) { 1508 swizzle_pipe[0] = 0; 1509 swizzle_pipe[1] = 1; 1510 swizzle_pipe[2] = 2; 1511 swizzle_pipe[3] = 3; 1512 swizzle_pipe[4] = 4; 1513 swizzle_pipe[5] = 5; 1514 swizzle_pipe[6] = 6; 1515 swizzle_pipe[7] = 7; 1516 } else { 1517 swizzle_pipe[0] = 0; 1518 swizzle_pipe[1] = 2; 1519 swizzle_pipe[2] = 4; 1520 swizzle_pipe[3] = 6; 1521 swizzle_pipe[4] = 1; 1522 swizzle_pipe[5] = 3; 1523 swizzle_pipe[6] = 5; 1524 swizzle_pipe[7] = 7; 1525 } 1526 break; 1527 } 1528 1529 for (cur_pipe = 0; cur_pipe < num_tile_pipes; ++cur_pipe) { 1530 while (((1 << cur_backend) & enabled_backends_mask) == 0) 1531 cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS; 1532 1533 backend_map |= (((cur_backend & 0xf) << (swizzle_pipe[cur_pipe] * 4))); 1534 1535 cur_backend = (cur_backend + 1) % EVERGREEN_MAX_BACKENDS; 1536 } 1537 1538 return backend_map; 1539 } 1540 1541 static void evergreen_program_channel_remap(struct radeon_device *rdev) 1542 { 1543 u32 tcp_chan_steer_lo, tcp_chan_steer_hi, mc_shared_chremap, tmp; 1544 1545 tmp = RREG32(MC_SHARED_CHMAP); 1546 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 1547 case 0: 1548 case 1: 1549 case 2: 1550 case 3: 1551 default: 1552 /* default mapping */ 1553 mc_shared_chremap = 0x00fac688; 1554 break; 1555 } 1556 1557 switch (rdev->family) { 1558 case CHIP_HEMLOCK: 1559 case CHIP_CYPRESS: 1560 case CHIP_BARTS: 1561 tcp_chan_steer_lo = 0x54763210; 1562 tcp_chan_steer_hi = 0x0000ba98; 1563 break; 1564 case CHIP_JUNIPER: 1565 case CHIP_REDWOOD: 1566 case CHIP_CEDAR: 1567 case CHIP_PALM: 1568 case CHIP_SUMO: 1569 case CHIP_SUMO2: 1570 case CHIP_TURKS: 1571 case CHIP_CAICOS: 1572 default: 1573 tcp_chan_steer_lo = 0x76543210; 1574 tcp_chan_steer_hi = 0x0000ba98; 1575 break; 1576 } 1577 1578 WREG32(TCP_CHAN_STEER_LO, tcp_chan_steer_lo); 1579 WREG32(TCP_CHAN_STEER_HI, tcp_chan_steer_hi); 1580 WREG32(MC_SHARED_CHREMAP, mc_shared_chremap); 1581 } 1582 1583 static void evergreen_gpu_init(struct radeon_device *rdev) 1584 { 1585 u32 cc_rb_backend_disable = 0; 1586 u32 cc_gc_shader_pipe_config; 1587 u32 gb_addr_config = 0; 1588 u32 mc_shared_chmap, mc_arb_ramcfg; 1589 u32 gb_backend_map; 1590 u32 grbm_gfx_index; 1591 u32 sx_debug_1; 1592 u32 smx_dc_ctl0; 1593 u32 sq_config; 1594 u32 sq_lds_resource_mgmt; 1595 u32 sq_gpr_resource_mgmt_1; 1596 u32 sq_gpr_resource_mgmt_2; 1597 u32 sq_gpr_resource_mgmt_3; 1598 u32 sq_thread_resource_mgmt; 1599 u32 sq_thread_resource_mgmt_2; 1600 u32 sq_stack_resource_mgmt_1; 1601 u32 sq_stack_resource_mgmt_2; 1602 u32 sq_stack_resource_mgmt_3; 1603 u32 vgt_cache_invalidation; 1604 u32 hdp_host_path_cntl, tmp; 1605 int i, j, num_shader_engines, ps_thread_count; 1606 1607 switch (rdev->family) { 1608 case CHIP_CYPRESS: 1609 case CHIP_HEMLOCK: 1610 rdev->config.evergreen.num_ses = 2; 1611 rdev->config.evergreen.max_pipes = 4; 1612 rdev->config.evergreen.max_tile_pipes = 8; 1613 rdev->config.evergreen.max_simds = 10; 1614 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 1615 rdev->config.evergreen.max_gprs = 256; 1616 rdev->config.evergreen.max_threads = 248; 1617 rdev->config.evergreen.max_gs_threads = 32; 1618 rdev->config.evergreen.max_stack_entries = 512; 1619 rdev->config.evergreen.sx_num_of_sets = 4; 1620 rdev->config.evergreen.sx_max_export_size = 256; 1621 rdev->config.evergreen.sx_max_export_pos_size = 64; 1622 rdev->config.evergreen.sx_max_export_smx_size = 192; 1623 rdev->config.evergreen.max_hw_contexts = 8; 1624 rdev->config.evergreen.sq_num_cf_insts = 2; 1625 1626 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1627 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1628 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1629 break; 1630 case CHIP_JUNIPER: 1631 rdev->config.evergreen.num_ses = 1; 1632 rdev->config.evergreen.max_pipes = 4; 1633 rdev->config.evergreen.max_tile_pipes = 4; 1634 rdev->config.evergreen.max_simds = 10; 1635 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 1636 rdev->config.evergreen.max_gprs = 256; 1637 rdev->config.evergreen.max_threads = 248; 1638 rdev->config.evergreen.max_gs_threads = 32; 1639 rdev->config.evergreen.max_stack_entries = 512; 1640 rdev->config.evergreen.sx_num_of_sets = 4; 1641 rdev->config.evergreen.sx_max_export_size = 256; 1642 rdev->config.evergreen.sx_max_export_pos_size = 64; 1643 rdev->config.evergreen.sx_max_export_smx_size = 192; 1644 rdev->config.evergreen.max_hw_contexts = 8; 1645 rdev->config.evergreen.sq_num_cf_insts = 2; 1646 1647 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1648 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1649 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1650 break; 1651 case CHIP_REDWOOD: 1652 rdev->config.evergreen.num_ses = 1; 1653 rdev->config.evergreen.max_pipes = 4; 1654 rdev->config.evergreen.max_tile_pipes = 4; 1655 rdev->config.evergreen.max_simds = 5; 1656 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 1657 rdev->config.evergreen.max_gprs = 256; 1658 rdev->config.evergreen.max_threads = 248; 1659 rdev->config.evergreen.max_gs_threads = 32; 1660 rdev->config.evergreen.max_stack_entries = 256; 1661 rdev->config.evergreen.sx_num_of_sets = 4; 1662 rdev->config.evergreen.sx_max_export_size = 256; 1663 rdev->config.evergreen.sx_max_export_pos_size = 64; 1664 rdev->config.evergreen.sx_max_export_smx_size = 192; 1665 rdev->config.evergreen.max_hw_contexts = 8; 1666 rdev->config.evergreen.sq_num_cf_insts = 2; 1667 1668 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1669 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1670 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1671 break; 1672 case CHIP_CEDAR: 1673 default: 1674 rdev->config.evergreen.num_ses = 1; 1675 rdev->config.evergreen.max_pipes = 2; 1676 rdev->config.evergreen.max_tile_pipes = 2; 1677 rdev->config.evergreen.max_simds = 2; 1678 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 1679 rdev->config.evergreen.max_gprs = 256; 1680 rdev->config.evergreen.max_threads = 192; 1681 rdev->config.evergreen.max_gs_threads = 16; 1682 rdev->config.evergreen.max_stack_entries = 256; 1683 rdev->config.evergreen.sx_num_of_sets = 4; 1684 rdev->config.evergreen.sx_max_export_size = 128; 1685 rdev->config.evergreen.sx_max_export_pos_size = 32; 1686 rdev->config.evergreen.sx_max_export_smx_size = 96; 1687 rdev->config.evergreen.max_hw_contexts = 4; 1688 rdev->config.evergreen.sq_num_cf_insts = 1; 1689 1690 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1691 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1692 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1693 break; 1694 case CHIP_PALM: 1695 rdev->config.evergreen.num_ses = 1; 1696 rdev->config.evergreen.max_pipes = 2; 1697 rdev->config.evergreen.max_tile_pipes = 2; 1698 rdev->config.evergreen.max_simds = 2; 1699 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 1700 rdev->config.evergreen.max_gprs = 256; 1701 rdev->config.evergreen.max_threads = 192; 1702 rdev->config.evergreen.max_gs_threads = 16; 1703 rdev->config.evergreen.max_stack_entries = 256; 1704 rdev->config.evergreen.sx_num_of_sets = 4; 1705 rdev->config.evergreen.sx_max_export_size = 128; 1706 rdev->config.evergreen.sx_max_export_pos_size = 32; 1707 rdev->config.evergreen.sx_max_export_smx_size = 96; 1708 rdev->config.evergreen.max_hw_contexts = 4; 1709 rdev->config.evergreen.sq_num_cf_insts = 1; 1710 1711 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1712 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1713 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1714 break; 1715 case CHIP_SUMO: 1716 rdev->config.evergreen.num_ses = 1; 1717 rdev->config.evergreen.max_pipes = 4; 1718 rdev->config.evergreen.max_tile_pipes = 2; 1719 if (rdev->pdev->device == 0x9648) 1720 rdev->config.evergreen.max_simds = 3; 1721 else if ((rdev->pdev->device == 0x9647) || 1722 (rdev->pdev->device == 0x964a)) 1723 rdev->config.evergreen.max_simds = 4; 1724 else 1725 rdev->config.evergreen.max_simds = 5; 1726 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 1727 rdev->config.evergreen.max_gprs = 256; 1728 rdev->config.evergreen.max_threads = 248; 1729 rdev->config.evergreen.max_gs_threads = 32; 1730 rdev->config.evergreen.max_stack_entries = 256; 1731 rdev->config.evergreen.sx_num_of_sets = 4; 1732 rdev->config.evergreen.sx_max_export_size = 256; 1733 rdev->config.evergreen.sx_max_export_pos_size = 64; 1734 rdev->config.evergreen.sx_max_export_smx_size = 192; 1735 rdev->config.evergreen.max_hw_contexts = 8; 1736 rdev->config.evergreen.sq_num_cf_insts = 2; 1737 1738 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1739 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1740 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1741 break; 1742 case CHIP_SUMO2: 1743 rdev->config.evergreen.num_ses = 1; 1744 rdev->config.evergreen.max_pipes = 4; 1745 rdev->config.evergreen.max_tile_pipes = 4; 1746 rdev->config.evergreen.max_simds = 2; 1747 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 1748 rdev->config.evergreen.max_gprs = 256; 1749 rdev->config.evergreen.max_threads = 248; 1750 rdev->config.evergreen.max_gs_threads = 32; 1751 rdev->config.evergreen.max_stack_entries = 512; 1752 rdev->config.evergreen.sx_num_of_sets = 4; 1753 rdev->config.evergreen.sx_max_export_size = 256; 1754 rdev->config.evergreen.sx_max_export_pos_size = 64; 1755 rdev->config.evergreen.sx_max_export_smx_size = 192; 1756 rdev->config.evergreen.max_hw_contexts = 8; 1757 rdev->config.evergreen.sq_num_cf_insts = 2; 1758 1759 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1760 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1761 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1762 break; 1763 case CHIP_BARTS: 1764 rdev->config.evergreen.num_ses = 2; 1765 rdev->config.evergreen.max_pipes = 4; 1766 rdev->config.evergreen.max_tile_pipes = 8; 1767 rdev->config.evergreen.max_simds = 7; 1768 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses; 1769 rdev->config.evergreen.max_gprs = 256; 1770 rdev->config.evergreen.max_threads = 248; 1771 rdev->config.evergreen.max_gs_threads = 32; 1772 rdev->config.evergreen.max_stack_entries = 512; 1773 rdev->config.evergreen.sx_num_of_sets = 4; 1774 rdev->config.evergreen.sx_max_export_size = 256; 1775 rdev->config.evergreen.sx_max_export_pos_size = 64; 1776 rdev->config.evergreen.sx_max_export_smx_size = 192; 1777 rdev->config.evergreen.max_hw_contexts = 8; 1778 rdev->config.evergreen.sq_num_cf_insts = 2; 1779 1780 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1781 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1782 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1783 break; 1784 case CHIP_TURKS: 1785 rdev->config.evergreen.num_ses = 1; 1786 rdev->config.evergreen.max_pipes = 4; 1787 rdev->config.evergreen.max_tile_pipes = 4; 1788 rdev->config.evergreen.max_simds = 6; 1789 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses; 1790 rdev->config.evergreen.max_gprs = 256; 1791 rdev->config.evergreen.max_threads = 248; 1792 rdev->config.evergreen.max_gs_threads = 32; 1793 rdev->config.evergreen.max_stack_entries = 256; 1794 rdev->config.evergreen.sx_num_of_sets = 4; 1795 rdev->config.evergreen.sx_max_export_size = 256; 1796 rdev->config.evergreen.sx_max_export_pos_size = 64; 1797 rdev->config.evergreen.sx_max_export_smx_size = 192; 1798 rdev->config.evergreen.max_hw_contexts = 8; 1799 rdev->config.evergreen.sq_num_cf_insts = 2; 1800 1801 rdev->config.evergreen.sc_prim_fifo_size = 0x100; 1802 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1803 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1804 break; 1805 case CHIP_CAICOS: 1806 rdev->config.evergreen.num_ses = 1; 1807 rdev->config.evergreen.max_pipes = 4; 1808 rdev->config.evergreen.max_tile_pipes = 2; 1809 rdev->config.evergreen.max_simds = 2; 1810 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses; 1811 rdev->config.evergreen.max_gprs = 256; 1812 rdev->config.evergreen.max_threads = 192; 1813 rdev->config.evergreen.max_gs_threads = 16; 1814 rdev->config.evergreen.max_stack_entries = 256; 1815 rdev->config.evergreen.sx_num_of_sets = 4; 1816 rdev->config.evergreen.sx_max_export_size = 128; 1817 rdev->config.evergreen.sx_max_export_pos_size = 32; 1818 rdev->config.evergreen.sx_max_export_smx_size = 96; 1819 rdev->config.evergreen.max_hw_contexts = 4; 1820 rdev->config.evergreen.sq_num_cf_insts = 1; 1821 1822 rdev->config.evergreen.sc_prim_fifo_size = 0x40; 1823 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30; 1824 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130; 1825 break; 1826 } 1827 1828 /* Initialize HDP */ 1829 for (i = 0, j = 0; i < 32; i++, j += 0x18) { 1830 WREG32((0x2c14 + j), 0x00000000); 1831 WREG32((0x2c18 + j), 0x00000000); 1832 WREG32((0x2c1c + j), 0x00000000); 1833 WREG32((0x2c20 + j), 0x00000000); 1834 WREG32((0x2c24 + j), 0x00000000); 1835 } 1836 1837 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff)); 1838 1839 cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & ~2; 1840 1841 cc_gc_shader_pipe_config |= 1842 INACTIVE_QD_PIPES((EVERGREEN_MAX_PIPES_MASK << rdev->config.evergreen.max_pipes) 1843 & EVERGREEN_MAX_PIPES_MASK); 1844 cc_gc_shader_pipe_config |= 1845 INACTIVE_SIMDS((EVERGREEN_MAX_SIMDS_MASK << rdev->config.evergreen.max_simds) 1846 & EVERGREEN_MAX_SIMDS_MASK); 1847 1848 cc_rb_backend_disable = 1849 BACKEND_DISABLE((EVERGREEN_MAX_BACKENDS_MASK << rdev->config.evergreen.max_backends) 1850 & EVERGREEN_MAX_BACKENDS_MASK); 1851 1852 1853 mc_shared_chmap = RREG32(MC_SHARED_CHMAP); 1854 if (rdev->flags & RADEON_IS_IGP) 1855 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG); 1856 else 1857 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG); 1858 1859 switch (rdev->config.evergreen.max_tile_pipes) { 1860 case 1: 1861 default: 1862 gb_addr_config |= NUM_PIPES(0); 1863 break; 1864 case 2: 1865 gb_addr_config |= NUM_PIPES(1); 1866 break; 1867 case 4: 1868 gb_addr_config |= NUM_PIPES(2); 1869 break; 1870 case 8: 1871 gb_addr_config |= NUM_PIPES(3); 1872 break; 1873 } 1874 1875 gb_addr_config |= PIPE_INTERLEAVE_SIZE((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT); 1876 gb_addr_config |= BANK_INTERLEAVE_SIZE(0); 1877 gb_addr_config |= NUM_SHADER_ENGINES(rdev->config.evergreen.num_ses - 1); 1878 gb_addr_config |= SHADER_ENGINE_TILE_SIZE(1); 1879 gb_addr_config |= NUM_GPUS(0); /* Hemlock? */ 1880 gb_addr_config |= MULTI_GPU_TILE_SIZE(2); 1881 1882 if (((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT) > 2) 1883 gb_addr_config |= ROW_SIZE(2); 1884 else 1885 gb_addr_config |= ROW_SIZE((mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT); 1886 1887 if (rdev->ddev->pdev->device == 0x689e) { 1888 u32 efuse_straps_4; 1889 u32 efuse_straps_3; 1890 u8 efuse_box_bit_131_124; 1891 1892 WREG32(RCU_IND_INDEX, 0x204); 1893 efuse_straps_4 = RREG32(RCU_IND_DATA); 1894 WREG32(RCU_IND_INDEX, 0x203); 1895 efuse_straps_3 = RREG32(RCU_IND_DATA); 1896 efuse_box_bit_131_124 = (u8)(((efuse_straps_4 & 0xf) << 4) | ((efuse_straps_3 & 0xf0000000) >> 28)); 1897 1898 switch(efuse_box_bit_131_124) { 1899 case 0x00: 1900 gb_backend_map = 0x76543210; 1901 break; 1902 case 0x55: 1903 gb_backend_map = 0x77553311; 1904 break; 1905 case 0x56: 1906 gb_backend_map = 0x77553300; 1907 break; 1908 case 0x59: 1909 gb_backend_map = 0x77552211; 1910 break; 1911 case 0x66: 1912 gb_backend_map = 0x77443300; 1913 break; 1914 case 0x99: 1915 gb_backend_map = 0x66552211; 1916 break; 1917 case 0x5a: 1918 gb_backend_map = 0x77552200; 1919 break; 1920 case 0xaa: 1921 gb_backend_map = 0x66442200; 1922 break; 1923 case 0x95: 1924 gb_backend_map = 0x66553311; 1925 break; 1926 default: 1927 DRM_ERROR("bad backend map, using default\n"); 1928 gb_backend_map = 1929 evergreen_get_tile_pipe_to_backend_map(rdev, 1930 rdev->config.evergreen.max_tile_pipes, 1931 rdev->config.evergreen.max_backends, 1932 ((EVERGREEN_MAX_BACKENDS_MASK << 1933 rdev->config.evergreen.max_backends) & 1934 EVERGREEN_MAX_BACKENDS_MASK)); 1935 break; 1936 } 1937 } else if (rdev->ddev->pdev->device == 0x68b9) { 1938 u32 efuse_straps_3; 1939 u8 efuse_box_bit_127_124; 1940 1941 WREG32(RCU_IND_INDEX, 0x203); 1942 efuse_straps_3 = RREG32(RCU_IND_DATA); 1943 efuse_box_bit_127_124 = (u8)((efuse_straps_3 & 0xF0000000) >> 28); 1944 1945 switch(efuse_box_bit_127_124) { 1946 case 0x0: 1947 gb_backend_map = 0x00003210; 1948 break; 1949 case 0x5: 1950 case 0x6: 1951 case 0x9: 1952 case 0xa: 1953 gb_backend_map = 0x00003311; 1954 break; 1955 default: 1956 DRM_ERROR("bad backend map, using default\n"); 1957 gb_backend_map = 1958 evergreen_get_tile_pipe_to_backend_map(rdev, 1959 rdev->config.evergreen.max_tile_pipes, 1960 rdev->config.evergreen.max_backends, 1961 ((EVERGREEN_MAX_BACKENDS_MASK << 1962 rdev->config.evergreen.max_backends) & 1963 EVERGREEN_MAX_BACKENDS_MASK)); 1964 break; 1965 } 1966 } else { 1967 switch (rdev->family) { 1968 case CHIP_CYPRESS: 1969 case CHIP_HEMLOCK: 1970 case CHIP_BARTS: 1971 gb_backend_map = 0x66442200; 1972 break; 1973 case CHIP_JUNIPER: 1974 gb_backend_map = 0x00006420; 1975 break; 1976 default: 1977 gb_backend_map = 1978 evergreen_get_tile_pipe_to_backend_map(rdev, 1979 rdev->config.evergreen.max_tile_pipes, 1980 rdev->config.evergreen.max_backends, 1981 ((EVERGREEN_MAX_BACKENDS_MASK << 1982 rdev->config.evergreen.max_backends) & 1983 EVERGREEN_MAX_BACKENDS_MASK)); 1984 } 1985 } 1986 1987 /* setup tiling info dword. gb_addr_config is not adequate since it does 1988 * not have bank info, so create a custom tiling dword. 1989 * bits 3:0 num_pipes 1990 * bits 7:4 num_banks 1991 * bits 11:8 group_size 1992 * bits 15:12 row_size 1993 */ 1994 rdev->config.evergreen.tile_config = 0; 1995 switch (rdev->config.evergreen.max_tile_pipes) { 1996 case 1: 1997 default: 1998 rdev->config.evergreen.tile_config |= (0 << 0); 1999 break; 2000 case 2: 2001 rdev->config.evergreen.tile_config |= (1 << 0); 2002 break; 2003 case 4: 2004 rdev->config.evergreen.tile_config |= (2 << 0); 2005 break; 2006 case 8: 2007 rdev->config.evergreen.tile_config |= (3 << 0); 2008 break; 2009 } 2010 /* num banks is 8 on all fusion asics */ 2011 if (rdev->flags & RADEON_IS_IGP) 2012 rdev->config.evergreen.tile_config |= 8 << 4; 2013 else 2014 rdev->config.evergreen.tile_config |= 2015 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4; 2016 rdev->config.evergreen.tile_config |= 2017 ((mc_arb_ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT) << 8; 2018 rdev->config.evergreen.tile_config |= 2019 ((gb_addr_config & 0x30000000) >> 28) << 12; 2020 2021 WREG32(GB_BACKEND_MAP, gb_backend_map); 2022 WREG32(GB_ADDR_CONFIG, gb_addr_config); 2023 WREG32(DMIF_ADDR_CONFIG, gb_addr_config); 2024 WREG32(HDP_ADDR_CONFIG, gb_addr_config); 2025 2026 evergreen_program_channel_remap(rdev); 2027 2028 num_shader_engines = ((RREG32(GB_ADDR_CONFIG) & NUM_SHADER_ENGINES(3)) >> 12) + 1; 2029 grbm_gfx_index = INSTANCE_BROADCAST_WRITES; 2030 2031 for (i = 0; i < rdev->config.evergreen.num_ses; i++) { 2032 u32 rb = cc_rb_backend_disable | (0xf0 << 16); 2033 u32 sp = cc_gc_shader_pipe_config; 2034 u32 gfx = grbm_gfx_index | SE_INDEX(i); 2035 2036 if (i == num_shader_engines) { 2037 rb |= BACKEND_DISABLE(EVERGREEN_MAX_BACKENDS_MASK); 2038 sp |= INACTIVE_SIMDS(EVERGREEN_MAX_SIMDS_MASK); 2039 } 2040 2041 WREG32(GRBM_GFX_INDEX, gfx); 2042 WREG32(RLC_GFX_INDEX, gfx); 2043 2044 WREG32(CC_RB_BACKEND_DISABLE, rb); 2045 WREG32(CC_SYS_RB_BACKEND_DISABLE, rb); 2046 WREG32(GC_USER_RB_BACKEND_DISABLE, rb); 2047 WREG32(CC_GC_SHADER_PIPE_CONFIG, sp); 2048 } 2049 2050 grbm_gfx_index |= SE_BROADCAST_WRITES; 2051 WREG32(GRBM_GFX_INDEX, grbm_gfx_index); 2052 WREG32(RLC_GFX_INDEX, grbm_gfx_index); 2053 2054 WREG32(CGTS_SYS_TCC_DISABLE, 0); 2055 WREG32(CGTS_TCC_DISABLE, 0); 2056 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0); 2057 WREG32(CGTS_USER_TCC_DISABLE, 0); 2058 2059 /* set HW defaults for 3D engine */ 2060 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | 2061 ROQ_IB2_START(0x2b))); 2062 2063 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30)); 2064 2065 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | 2066 SYNC_GRADIENT | 2067 SYNC_WALKER | 2068 SYNC_ALIGNER)); 2069 2070 sx_debug_1 = RREG32(SX_DEBUG_1); 2071 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS; 2072 WREG32(SX_DEBUG_1, sx_debug_1); 2073 2074 2075 smx_dc_ctl0 = RREG32(SMX_DC_CTL0); 2076 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff); 2077 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets); 2078 WREG32(SMX_DC_CTL0, smx_dc_ctl0); 2079 2080 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) | 2081 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) | 2082 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1))); 2083 2084 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) | 2085 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) | 2086 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size))); 2087 2088 WREG32(VGT_NUM_INSTANCES, 1); 2089 WREG32(SPI_CONFIG_CNTL, 0); 2090 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4)); 2091 WREG32(CP_PERFMON_CNTL, 0); 2092 2093 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) | 2094 FETCH_FIFO_HIWATER(0x4) | 2095 DONE_FIFO_HIWATER(0xe0) | 2096 ALU_UPDATE_FIFO_HIWATER(0x8))); 2097 2098 sq_config = RREG32(SQ_CONFIG); 2099 sq_config &= ~(PS_PRIO(3) | 2100 VS_PRIO(3) | 2101 GS_PRIO(3) | 2102 ES_PRIO(3)); 2103 sq_config |= (VC_ENABLE | 2104 EXPORT_SRC_C | 2105 PS_PRIO(0) | 2106 VS_PRIO(1) | 2107 GS_PRIO(2) | 2108 ES_PRIO(3)); 2109 2110 switch (rdev->family) { 2111 case CHIP_CEDAR: 2112 case CHIP_PALM: 2113 case CHIP_SUMO: 2114 case CHIP_SUMO2: 2115 case CHIP_CAICOS: 2116 /* no vertex cache */ 2117 sq_config &= ~VC_ENABLE; 2118 break; 2119 default: 2120 break; 2121 } 2122 2123 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT); 2124 2125 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32); 2126 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32); 2127 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4); 2128 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 2129 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32); 2130 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 2131 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32); 2132 2133 switch (rdev->family) { 2134 case CHIP_CEDAR: 2135 case CHIP_PALM: 2136 case CHIP_SUMO: 2137 case CHIP_SUMO2: 2138 ps_thread_count = 96; 2139 break; 2140 default: 2141 ps_thread_count = 128; 2142 break; 2143 } 2144 2145 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count); 2146 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2147 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2148 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2149 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2150 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8); 2151 2152 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2153 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2154 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2155 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2156 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2157 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6); 2158 2159 WREG32(SQ_CONFIG, sq_config); 2160 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1); 2161 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2); 2162 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3); 2163 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt); 2164 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2); 2165 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1); 2166 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2); 2167 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3); 2168 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0); 2169 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt); 2170 2171 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) | 2172 FORCE_EOV_MAX_REZ_CNT(255))); 2173 2174 switch (rdev->family) { 2175 case CHIP_CEDAR: 2176 case CHIP_PALM: 2177 case CHIP_SUMO: 2178 case CHIP_SUMO2: 2179 case CHIP_CAICOS: 2180 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY); 2181 break; 2182 default: 2183 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC); 2184 break; 2185 } 2186 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO); 2187 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation); 2188 2189 WREG32(VGT_GS_VERTEX_REUSE, 16); 2190 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0); 2191 WREG32(PA_SC_LINE_STIPPLE_STATE, 0); 2192 2193 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14); 2194 WREG32(VGT_OUT_DEALLOC_CNTL, 16); 2195 2196 WREG32(CB_PERF_CTR0_SEL_0, 0); 2197 WREG32(CB_PERF_CTR0_SEL_1, 0); 2198 WREG32(CB_PERF_CTR1_SEL_0, 0); 2199 WREG32(CB_PERF_CTR1_SEL_1, 0); 2200 WREG32(CB_PERF_CTR2_SEL_0, 0); 2201 WREG32(CB_PERF_CTR2_SEL_1, 0); 2202 WREG32(CB_PERF_CTR3_SEL_0, 0); 2203 WREG32(CB_PERF_CTR3_SEL_1, 0); 2204 2205 /* clear render buffer base addresses */ 2206 WREG32(CB_COLOR0_BASE, 0); 2207 WREG32(CB_COLOR1_BASE, 0); 2208 WREG32(CB_COLOR2_BASE, 0); 2209 WREG32(CB_COLOR3_BASE, 0); 2210 WREG32(CB_COLOR4_BASE, 0); 2211 WREG32(CB_COLOR5_BASE, 0); 2212 WREG32(CB_COLOR6_BASE, 0); 2213 WREG32(CB_COLOR7_BASE, 0); 2214 WREG32(CB_COLOR8_BASE, 0); 2215 WREG32(CB_COLOR9_BASE, 0); 2216 WREG32(CB_COLOR10_BASE, 0); 2217 WREG32(CB_COLOR11_BASE, 0); 2218 2219 /* set the shader const cache sizes to 0 */ 2220 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4) 2221 WREG32(i, 0); 2222 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4) 2223 WREG32(i, 0); 2224 2225 tmp = RREG32(HDP_MISC_CNTL); 2226 tmp |= HDP_FLUSH_INVALIDATE_CACHE; 2227 WREG32(HDP_MISC_CNTL, tmp); 2228 2229 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL); 2230 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl); 2231 2232 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3)); 2233 2234 udelay(50); 2235 2236 } 2237 2238 int evergreen_mc_init(struct radeon_device *rdev) 2239 { 2240 u32 tmp; 2241 int chansize, numchan; 2242 2243 /* Get VRAM informations */ 2244 rdev->mc.vram_is_ddr = true; 2245 tmp = RREG32(MC_ARB_RAMCFG); 2246 if (tmp & CHANSIZE_OVERRIDE) { 2247 chansize = 16; 2248 } else if (tmp & CHANSIZE_MASK) { 2249 chansize = 64; 2250 } else { 2251 chansize = 32; 2252 } 2253 tmp = RREG32(MC_SHARED_CHMAP); 2254 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) { 2255 case 0: 2256 default: 2257 numchan = 1; 2258 break; 2259 case 1: 2260 numchan = 2; 2261 break; 2262 case 2: 2263 numchan = 4; 2264 break; 2265 case 3: 2266 numchan = 8; 2267 break; 2268 } 2269 rdev->mc.vram_width = numchan * chansize; 2270 /* Could aper size report 0 ? */ 2271 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); 2272 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); 2273 /* Setup GPU memory space */ 2274 if (rdev->flags & RADEON_IS_IGP) { 2275 /* size in bytes on fusion */ 2276 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE); 2277 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE); 2278 } else { 2279 /* size in MB on evergreen */ 2280 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024; 2281 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024; 2282 } 2283 rdev->mc.visible_vram_size = rdev->mc.aper_size; 2284 r700_vram_gtt_location(rdev, &rdev->mc); 2285 radeon_update_bandwidth_info(rdev); 2286 2287 return 0; 2288 } 2289 2290 bool evergreen_gpu_is_lockup(struct radeon_device *rdev) 2291 { 2292 u32 srbm_status; 2293 u32 grbm_status; 2294 u32 grbm_status_se0, grbm_status_se1; 2295 struct r100_gpu_lockup *lockup = &rdev->config.evergreen.lockup; 2296 int r; 2297 2298 srbm_status = RREG32(SRBM_STATUS); 2299 grbm_status = RREG32(GRBM_STATUS); 2300 grbm_status_se0 = RREG32(GRBM_STATUS_SE0); 2301 grbm_status_se1 = RREG32(GRBM_STATUS_SE1); 2302 if (!(grbm_status & GUI_ACTIVE)) { 2303 r100_gpu_lockup_update(lockup, &rdev->cp); 2304 return false; 2305 } 2306 /* force CP activities */ 2307 r = radeon_ring_lock(rdev, 2); 2308 if (!r) { 2309 /* PACKET2 NOP */ 2310 radeon_ring_write(rdev, 0x80000000); 2311 radeon_ring_write(rdev, 0x80000000); 2312 radeon_ring_unlock_commit(rdev); 2313 } 2314 rdev->cp.rptr = RREG32(CP_RB_RPTR); 2315 return r100_gpu_cp_is_lockup(rdev, lockup, &rdev->cp); 2316 } 2317 2318 static int evergreen_gpu_soft_reset(struct radeon_device *rdev) 2319 { 2320 struct evergreen_mc_save save; 2321 u32 grbm_reset = 0; 2322 2323 if (!(RREG32(GRBM_STATUS) & GUI_ACTIVE)) 2324 return 0; 2325 2326 dev_info(rdev->dev, "GPU softreset \n"); 2327 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 2328 RREG32(GRBM_STATUS)); 2329 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 2330 RREG32(GRBM_STATUS_SE0)); 2331 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 2332 RREG32(GRBM_STATUS_SE1)); 2333 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 2334 RREG32(SRBM_STATUS)); 2335 evergreen_mc_stop(rdev, &save); 2336 if (evergreen_mc_wait_for_idle(rdev)) { 2337 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); 2338 } 2339 /* Disable CP parsing/prefetching */ 2340 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT); 2341 2342 /* reset all the gfx blocks */ 2343 grbm_reset = (SOFT_RESET_CP | 2344 SOFT_RESET_CB | 2345 SOFT_RESET_DB | 2346 SOFT_RESET_PA | 2347 SOFT_RESET_SC | 2348 SOFT_RESET_SPI | 2349 SOFT_RESET_SH | 2350 SOFT_RESET_SX | 2351 SOFT_RESET_TC | 2352 SOFT_RESET_TA | 2353 SOFT_RESET_VC | 2354 SOFT_RESET_VGT); 2355 2356 dev_info(rdev->dev, " GRBM_SOFT_RESET=0x%08X\n", grbm_reset); 2357 WREG32(GRBM_SOFT_RESET, grbm_reset); 2358 (void)RREG32(GRBM_SOFT_RESET); 2359 udelay(50); 2360 WREG32(GRBM_SOFT_RESET, 0); 2361 (void)RREG32(GRBM_SOFT_RESET); 2362 /* Wait a little for things to settle down */ 2363 udelay(50); 2364 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", 2365 RREG32(GRBM_STATUS)); 2366 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", 2367 RREG32(GRBM_STATUS_SE0)); 2368 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", 2369 RREG32(GRBM_STATUS_SE1)); 2370 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", 2371 RREG32(SRBM_STATUS)); 2372 evergreen_mc_resume(rdev, &save); 2373 return 0; 2374 } 2375 2376 int evergreen_asic_reset(struct radeon_device *rdev) 2377 { 2378 return evergreen_gpu_soft_reset(rdev); 2379 } 2380 2381 /* Interrupts */ 2382 2383 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc) 2384 { 2385 switch (crtc) { 2386 case 0: 2387 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC0_REGISTER_OFFSET); 2388 case 1: 2389 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC1_REGISTER_OFFSET); 2390 case 2: 2391 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC2_REGISTER_OFFSET); 2392 case 3: 2393 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC3_REGISTER_OFFSET); 2394 case 4: 2395 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC4_REGISTER_OFFSET); 2396 case 5: 2397 return RREG32(CRTC_STATUS_FRAME_COUNT + EVERGREEN_CRTC5_REGISTER_OFFSET); 2398 default: 2399 return 0; 2400 } 2401 } 2402 2403 void evergreen_disable_interrupt_state(struct radeon_device *rdev) 2404 { 2405 u32 tmp; 2406 2407 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE); 2408 WREG32(GRBM_INT_CNTL, 0); 2409 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 2410 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 2411 if (!(rdev->flags & RADEON_IS_IGP)) { 2412 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 2413 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 2414 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 2415 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 2416 } 2417 2418 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0); 2419 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0); 2420 if (!(rdev->flags & RADEON_IS_IGP)) { 2421 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0); 2422 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0); 2423 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0); 2424 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0); 2425 } 2426 2427 WREG32(DACA_AUTODETECT_INT_CONTROL, 0); 2428 WREG32(DACB_AUTODETECT_INT_CONTROL, 0); 2429 2430 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2431 WREG32(DC_HPD1_INT_CONTROL, tmp); 2432 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2433 WREG32(DC_HPD2_INT_CONTROL, tmp); 2434 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2435 WREG32(DC_HPD3_INT_CONTROL, tmp); 2436 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2437 WREG32(DC_HPD4_INT_CONTROL, tmp); 2438 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2439 WREG32(DC_HPD5_INT_CONTROL, tmp); 2440 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY; 2441 WREG32(DC_HPD6_INT_CONTROL, tmp); 2442 2443 } 2444 2445 int evergreen_irq_set(struct radeon_device *rdev) 2446 { 2447 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE; 2448 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0; 2449 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6; 2450 u32 grbm_int_cntl = 0; 2451 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0; 2452 2453 if (!rdev->irq.installed) { 2454 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n"); 2455 return -EINVAL; 2456 } 2457 /* don't enable anything if the ih is disabled */ 2458 if (!rdev->ih.enabled) { 2459 r600_disable_interrupts(rdev); 2460 /* force the active interrupt state to all disabled */ 2461 evergreen_disable_interrupt_state(rdev); 2462 return 0; 2463 } 2464 2465 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN; 2466 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN; 2467 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN; 2468 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN; 2469 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN; 2470 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN; 2471 2472 if (rdev->irq.sw_int) { 2473 DRM_DEBUG("evergreen_irq_set: sw int\n"); 2474 cp_int_cntl |= RB_INT_ENABLE; 2475 cp_int_cntl |= TIME_STAMP_INT_ENABLE; 2476 } 2477 if (rdev->irq.crtc_vblank_int[0] || 2478 rdev->irq.pflip[0]) { 2479 DRM_DEBUG("evergreen_irq_set: vblank 0\n"); 2480 crtc1 |= VBLANK_INT_MASK; 2481 } 2482 if (rdev->irq.crtc_vblank_int[1] || 2483 rdev->irq.pflip[1]) { 2484 DRM_DEBUG("evergreen_irq_set: vblank 1\n"); 2485 crtc2 |= VBLANK_INT_MASK; 2486 } 2487 if (rdev->irq.crtc_vblank_int[2] || 2488 rdev->irq.pflip[2]) { 2489 DRM_DEBUG("evergreen_irq_set: vblank 2\n"); 2490 crtc3 |= VBLANK_INT_MASK; 2491 } 2492 if (rdev->irq.crtc_vblank_int[3] || 2493 rdev->irq.pflip[3]) { 2494 DRM_DEBUG("evergreen_irq_set: vblank 3\n"); 2495 crtc4 |= VBLANK_INT_MASK; 2496 } 2497 if (rdev->irq.crtc_vblank_int[4] || 2498 rdev->irq.pflip[4]) { 2499 DRM_DEBUG("evergreen_irq_set: vblank 4\n"); 2500 crtc5 |= VBLANK_INT_MASK; 2501 } 2502 if (rdev->irq.crtc_vblank_int[5] || 2503 rdev->irq.pflip[5]) { 2504 DRM_DEBUG("evergreen_irq_set: vblank 5\n"); 2505 crtc6 |= VBLANK_INT_MASK; 2506 } 2507 if (rdev->irq.hpd[0]) { 2508 DRM_DEBUG("evergreen_irq_set: hpd 1\n"); 2509 hpd1 |= DC_HPDx_INT_EN; 2510 } 2511 if (rdev->irq.hpd[1]) { 2512 DRM_DEBUG("evergreen_irq_set: hpd 2\n"); 2513 hpd2 |= DC_HPDx_INT_EN; 2514 } 2515 if (rdev->irq.hpd[2]) { 2516 DRM_DEBUG("evergreen_irq_set: hpd 3\n"); 2517 hpd3 |= DC_HPDx_INT_EN; 2518 } 2519 if (rdev->irq.hpd[3]) { 2520 DRM_DEBUG("evergreen_irq_set: hpd 4\n"); 2521 hpd4 |= DC_HPDx_INT_EN; 2522 } 2523 if (rdev->irq.hpd[4]) { 2524 DRM_DEBUG("evergreen_irq_set: hpd 5\n"); 2525 hpd5 |= DC_HPDx_INT_EN; 2526 } 2527 if (rdev->irq.hpd[5]) { 2528 DRM_DEBUG("evergreen_irq_set: hpd 6\n"); 2529 hpd6 |= DC_HPDx_INT_EN; 2530 } 2531 if (rdev->irq.gui_idle) { 2532 DRM_DEBUG("gui idle\n"); 2533 grbm_int_cntl |= GUI_IDLE_INT_ENABLE; 2534 } 2535 2536 WREG32(CP_INT_CNTL, cp_int_cntl); 2537 WREG32(GRBM_INT_CNTL, grbm_int_cntl); 2538 2539 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1); 2540 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2); 2541 if (!(rdev->flags & RADEON_IS_IGP)) { 2542 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3); 2543 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4); 2544 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5); 2545 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6); 2546 } 2547 2548 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1); 2549 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2); 2550 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3); 2551 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4); 2552 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5); 2553 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6); 2554 2555 WREG32(DC_HPD1_INT_CONTROL, hpd1); 2556 WREG32(DC_HPD2_INT_CONTROL, hpd2); 2557 WREG32(DC_HPD3_INT_CONTROL, hpd3); 2558 WREG32(DC_HPD4_INT_CONTROL, hpd4); 2559 WREG32(DC_HPD5_INT_CONTROL, hpd5); 2560 WREG32(DC_HPD6_INT_CONTROL, hpd6); 2561 2562 return 0; 2563 } 2564 2565 static inline void evergreen_irq_ack(struct radeon_device *rdev) 2566 { 2567 u32 tmp; 2568 2569 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS); 2570 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); 2571 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); 2572 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); 2573 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); 2574 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); 2575 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET); 2576 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET); 2577 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET); 2578 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET); 2579 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET); 2580 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET); 2581 2582 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED) 2583 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2584 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED) 2585 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2586 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED) 2587 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2588 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED) 2589 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2590 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED) 2591 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2592 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED) 2593 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR); 2594 2595 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) 2596 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK); 2597 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) 2598 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK); 2599 2600 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) 2601 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK); 2602 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) 2603 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK); 2604 2605 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) 2606 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK); 2607 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) 2608 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK); 2609 2610 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) 2611 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK); 2612 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) 2613 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK); 2614 2615 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) 2616 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK); 2617 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) 2618 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK); 2619 2620 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) 2621 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK); 2622 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) 2623 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK); 2624 2625 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 2626 tmp = RREG32(DC_HPD1_INT_CONTROL); 2627 tmp |= DC_HPDx_INT_ACK; 2628 WREG32(DC_HPD1_INT_CONTROL, tmp); 2629 } 2630 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 2631 tmp = RREG32(DC_HPD2_INT_CONTROL); 2632 tmp |= DC_HPDx_INT_ACK; 2633 WREG32(DC_HPD2_INT_CONTROL, tmp); 2634 } 2635 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 2636 tmp = RREG32(DC_HPD3_INT_CONTROL); 2637 tmp |= DC_HPDx_INT_ACK; 2638 WREG32(DC_HPD3_INT_CONTROL, tmp); 2639 } 2640 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 2641 tmp = RREG32(DC_HPD4_INT_CONTROL); 2642 tmp |= DC_HPDx_INT_ACK; 2643 WREG32(DC_HPD4_INT_CONTROL, tmp); 2644 } 2645 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 2646 tmp = RREG32(DC_HPD5_INT_CONTROL); 2647 tmp |= DC_HPDx_INT_ACK; 2648 WREG32(DC_HPD5_INT_CONTROL, tmp); 2649 } 2650 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 2651 tmp = RREG32(DC_HPD5_INT_CONTROL); 2652 tmp |= DC_HPDx_INT_ACK; 2653 WREG32(DC_HPD6_INT_CONTROL, tmp); 2654 } 2655 } 2656 2657 void evergreen_irq_disable(struct radeon_device *rdev) 2658 { 2659 r600_disable_interrupts(rdev); 2660 /* Wait and acknowledge irq */ 2661 mdelay(1); 2662 evergreen_irq_ack(rdev); 2663 evergreen_disable_interrupt_state(rdev); 2664 } 2665 2666 void evergreen_irq_suspend(struct radeon_device *rdev) 2667 { 2668 evergreen_irq_disable(rdev); 2669 r600_rlc_stop(rdev); 2670 } 2671 2672 static inline u32 evergreen_get_ih_wptr(struct radeon_device *rdev) 2673 { 2674 u32 wptr, tmp; 2675 2676 if (rdev->wb.enabled) 2677 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); 2678 else 2679 wptr = RREG32(IH_RB_WPTR); 2680 2681 if (wptr & RB_OVERFLOW) { 2682 /* When a ring buffer overflow happen start parsing interrupt 2683 * from the last not overwritten vector (wptr + 16). Hopefully 2684 * this should allow us to catchup. 2685 */ 2686 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n", 2687 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask); 2688 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; 2689 tmp = RREG32(IH_RB_CNTL); 2690 tmp |= IH_WPTR_OVERFLOW_CLEAR; 2691 WREG32(IH_RB_CNTL, tmp); 2692 } 2693 return (wptr & rdev->ih.ptr_mask); 2694 } 2695 2696 int evergreen_irq_process(struct radeon_device *rdev) 2697 { 2698 u32 wptr = evergreen_get_ih_wptr(rdev); 2699 u32 rptr = rdev->ih.rptr; 2700 u32 src_id, src_data; 2701 u32 ring_index; 2702 unsigned long flags; 2703 bool queue_hotplug = false; 2704 2705 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr); 2706 if (!rdev->ih.enabled) 2707 return IRQ_NONE; 2708 2709 spin_lock_irqsave(&rdev->ih.lock, flags); 2710 2711 if (rptr == wptr) { 2712 spin_unlock_irqrestore(&rdev->ih.lock, flags); 2713 return IRQ_NONE; 2714 } 2715 if (rdev->shutdown) { 2716 spin_unlock_irqrestore(&rdev->ih.lock, flags); 2717 return IRQ_NONE; 2718 } 2719 2720 restart_ih: 2721 /* display interrupts */ 2722 evergreen_irq_ack(rdev); 2723 2724 rdev->ih.wptr = wptr; 2725 while (rptr != wptr) { 2726 /* wptr/rptr are in bytes! */ 2727 ring_index = rptr / 4; 2728 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; 2729 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; 2730 2731 switch (src_id) { 2732 case 1: /* D1 vblank/vline */ 2733 switch (src_data) { 2734 case 0: /* D1 vblank */ 2735 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) { 2736 if (rdev->irq.crtc_vblank_int[0]) { 2737 drm_handle_vblank(rdev->ddev, 0); 2738 rdev->pm.vblank_sync = true; 2739 wake_up(&rdev->irq.vblank_queue); 2740 } 2741 if (rdev->irq.pflip[0]) 2742 radeon_crtc_handle_flip(rdev, 0); 2743 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT; 2744 DRM_DEBUG("IH: D1 vblank\n"); 2745 } 2746 break; 2747 case 1: /* D1 vline */ 2748 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) { 2749 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT; 2750 DRM_DEBUG("IH: D1 vline\n"); 2751 } 2752 break; 2753 default: 2754 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 2755 break; 2756 } 2757 break; 2758 case 2: /* D2 vblank/vline */ 2759 switch (src_data) { 2760 case 0: /* D2 vblank */ 2761 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) { 2762 if (rdev->irq.crtc_vblank_int[1]) { 2763 drm_handle_vblank(rdev->ddev, 1); 2764 rdev->pm.vblank_sync = true; 2765 wake_up(&rdev->irq.vblank_queue); 2766 } 2767 if (rdev->irq.pflip[1]) 2768 radeon_crtc_handle_flip(rdev, 1); 2769 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; 2770 DRM_DEBUG("IH: D2 vblank\n"); 2771 } 2772 break; 2773 case 1: /* D2 vline */ 2774 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) { 2775 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; 2776 DRM_DEBUG("IH: D2 vline\n"); 2777 } 2778 break; 2779 default: 2780 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 2781 break; 2782 } 2783 break; 2784 case 3: /* D3 vblank/vline */ 2785 switch (src_data) { 2786 case 0: /* D3 vblank */ 2787 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) { 2788 if (rdev->irq.crtc_vblank_int[2]) { 2789 drm_handle_vblank(rdev->ddev, 2); 2790 rdev->pm.vblank_sync = true; 2791 wake_up(&rdev->irq.vblank_queue); 2792 } 2793 if (rdev->irq.pflip[2]) 2794 radeon_crtc_handle_flip(rdev, 2); 2795 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; 2796 DRM_DEBUG("IH: D3 vblank\n"); 2797 } 2798 break; 2799 case 1: /* D3 vline */ 2800 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) { 2801 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; 2802 DRM_DEBUG("IH: D3 vline\n"); 2803 } 2804 break; 2805 default: 2806 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 2807 break; 2808 } 2809 break; 2810 case 4: /* D4 vblank/vline */ 2811 switch (src_data) { 2812 case 0: /* D4 vblank */ 2813 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) { 2814 if (rdev->irq.crtc_vblank_int[3]) { 2815 drm_handle_vblank(rdev->ddev, 3); 2816 rdev->pm.vblank_sync = true; 2817 wake_up(&rdev->irq.vblank_queue); 2818 } 2819 if (rdev->irq.pflip[3]) 2820 radeon_crtc_handle_flip(rdev, 3); 2821 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; 2822 DRM_DEBUG("IH: D4 vblank\n"); 2823 } 2824 break; 2825 case 1: /* D4 vline */ 2826 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) { 2827 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; 2828 DRM_DEBUG("IH: D4 vline\n"); 2829 } 2830 break; 2831 default: 2832 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 2833 break; 2834 } 2835 break; 2836 case 5: /* D5 vblank/vline */ 2837 switch (src_data) { 2838 case 0: /* D5 vblank */ 2839 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) { 2840 if (rdev->irq.crtc_vblank_int[4]) { 2841 drm_handle_vblank(rdev->ddev, 4); 2842 rdev->pm.vblank_sync = true; 2843 wake_up(&rdev->irq.vblank_queue); 2844 } 2845 if (rdev->irq.pflip[4]) 2846 radeon_crtc_handle_flip(rdev, 4); 2847 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; 2848 DRM_DEBUG("IH: D5 vblank\n"); 2849 } 2850 break; 2851 case 1: /* D5 vline */ 2852 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) { 2853 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; 2854 DRM_DEBUG("IH: D5 vline\n"); 2855 } 2856 break; 2857 default: 2858 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 2859 break; 2860 } 2861 break; 2862 case 6: /* D6 vblank/vline */ 2863 switch (src_data) { 2864 case 0: /* D6 vblank */ 2865 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) { 2866 if (rdev->irq.crtc_vblank_int[5]) { 2867 drm_handle_vblank(rdev->ddev, 5); 2868 rdev->pm.vblank_sync = true; 2869 wake_up(&rdev->irq.vblank_queue); 2870 } 2871 if (rdev->irq.pflip[5]) 2872 radeon_crtc_handle_flip(rdev, 5); 2873 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; 2874 DRM_DEBUG("IH: D6 vblank\n"); 2875 } 2876 break; 2877 case 1: /* D6 vline */ 2878 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) { 2879 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; 2880 DRM_DEBUG("IH: D6 vline\n"); 2881 } 2882 break; 2883 default: 2884 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 2885 break; 2886 } 2887 break; 2888 case 42: /* HPD hotplug */ 2889 switch (src_data) { 2890 case 0: 2891 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) { 2892 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT; 2893 queue_hotplug = true; 2894 DRM_DEBUG("IH: HPD1\n"); 2895 } 2896 break; 2897 case 1: 2898 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) { 2899 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT; 2900 queue_hotplug = true; 2901 DRM_DEBUG("IH: HPD2\n"); 2902 } 2903 break; 2904 case 2: 2905 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) { 2906 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; 2907 queue_hotplug = true; 2908 DRM_DEBUG("IH: HPD3\n"); 2909 } 2910 break; 2911 case 3: 2912 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) { 2913 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; 2914 queue_hotplug = true; 2915 DRM_DEBUG("IH: HPD4\n"); 2916 } 2917 break; 2918 case 4: 2919 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) { 2920 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; 2921 queue_hotplug = true; 2922 DRM_DEBUG("IH: HPD5\n"); 2923 } 2924 break; 2925 case 5: 2926 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) { 2927 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; 2928 queue_hotplug = true; 2929 DRM_DEBUG("IH: HPD6\n"); 2930 } 2931 break; 2932 default: 2933 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 2934 break; 2935 } 2936 break; 2937 case 176: /* CP_INT in ring buffer */ 2938 case 177: /* CP_INT in IB1 */ 2939 case 178: /* CP_INT in IB2 */ 2940 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data); 2941 radeon_fence_process(rdev); 2942 break; 2943 case 181: /* CP EOP event */ 2944 DRM_DEBUG("IH: CP EOP\n"); 2945 radeon_fence_process(rdev); 2946 break; 2947 case 233: /* GUI IDLE */ 2948 DRM_DEBUG("IH: GUI idle\n"); 2949 rdev->pm.gui_idle = true; 2950 wake_up(&rdev->irq.idle_queue); 2951 break; 2952 default: 2953 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data); 2954 break; 2955 } 2956 2957 /* wptr/rptr are in bytes! */ 2958 rptr += 16; 2959 rptr &= rdev->ih.ptr_mask; 2960 } 2961 /* make sure wptr hasn't changed while processing */ 2962 wptr = evergreen_get_ih_wptr(rdev); 2963 if (wptr != rdev->ih.wptr) 2964 goto restart_ih; 2965 if (queue_hotplug) 2966 schedule_work(&rdev->hotplug_work); 2967 rdev->ih.rptr = rptr; 2968 WREG32(IH_RB_RPTR, rdev->ih.rptr); 2969 spin_unlock_irqrestore(&rdev->ih.lock, flags); 2970 return IRQ_HANDLED; 2971 } 2972 2973 static int evergreen_startup(struct radeon_device *rdev) 2974 { 2975 int r; 2976 2977 /* enable pcie gen2 link */ 2978 if (!ASIC_IS_DCE5(rdev)) 2979 evergreen_pcie_gen2_enable(rdev); 2980 2981 if (ASIC_IS_DCE5(rdev)) { 2982 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) { 2983 r = ni_init_microcode(rdev); 2984 if (r) { 2985 DRM_ERROR("Failed to load firmware!\n"); 2986 return r; 2987 } 2988 } 2989 r = ni_mc_load_microcode(rdev); 2990 if (r) { 2991 DRM_ERROR("Failed to load MC firmware!\n"); 2992 return r; 2993 } 2994 } else { 2995 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { 2996 r = r600_init_microcode(rdev); 2997 if (r) { 2998 DRM_ERROR("Failed to load firmware!\n"); 2999 return r; 3000 } 3001 } 3002 } 3003 3004 evergreen_mc_program(rdev); 3005 if (rdev->flags & RADEON_IS_AGP) { 3006 evergreen_agp_enable(rdev); 3007 } else { 3008 r = evergreen_pcie_gart_enable(rdev); 3009 if (r) 3010 return r; 3011 } 3012 evergreen_gpu_init(rdev); 3013 3014 r = evergreen_blit_init(rdev); 3015 if (r) { 3016 evergreen_blit_fini(rdev); 3017 rdev->asic->copy = NULL; 3018 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r); 3019 } 3020 3021 /* allocate wb buffer */ 3022 r = radeon_wb_init(rdev); 3023 if (r) 3024 return r; 3025 3026 /* Enable IRQ */ 3027 r = r600_irq_init(rdev); 3028 if (r) { 3029 DRM_ERROR("radeon: IH init failed (%d).\n", r); 3030 radeon_irq_kms_fini(rdev); 3031 return r; 3032 } 3033 evergreen_irq_set(rdev); 3034 3035 r = radeon_ring_init(rdev, rdev->cp.ring_size); 3036 if (r) 3037 return r; 3038 r = evergreen_cp_load_microcode(rdev); 3039 if (r) 3040 return r; 3041 r = evergreen_cp_resume(rdev); 3042 if (r) 3043 return r; 3044 3045 return 0; 3046 } 3047 3048 int evergreen_resume(struct radeon_device *rdev) 3049 { 3050 int r; 3051 3052 /* reset the asic, the gfx blocks are often in a bad state 3053 * after the driver is unloaded or after a resume 3054 */ 3055 if (radeon_asic_reset(rdev)) 3056 dev_warn(rdev->dev, "GPU reset failed !\n"); 3057 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw, 3058 * posting will perform necessary task to bring back GPU into good 3059 * shape. 3060 */ 3061 /* post card */ 3062 atom_asic_init(rdev->mode_info.atom_context); 3063 3064 r = evergreen_startup(rdev); 3065 if (r) { 3066 DRM_ERROR("evergreen startup failed on resume\n"); 3067 return r; 3068 } 3069 3070 r = r600_ib_test(rdev); 3071 if (r) { 3072 DRM_ERROR("radeon: failed testing IB (%d).\n", r); 3073 return r; 3074 } 3075 3076 return r; 3077 3078 } 3079 3080 int evergreen_suspend(struct radeon_device *rdev) 3081 { 3082 int r; 3083 3084 /* FIXME: we should wait for ring to be empty */ 3085 r700_cp_stop(rdev); 3086 rdev->cp.ready = false; 3087 evergreen_irq_suspend(rdev); 3088 radeon_wb_disable(rdev); 3089 evergreen_pcie_gart_disable(rdev); 3090 3091 /* unpin shaders bo */ 3092 r = radeon_bo_reserve(rdev->r600_blit.shader_obj, false); 3093 if (likely(r == 0)) { 3094 radeon_bo_unpin(rdev->r600_blit.shader_obj); 3095 radeon_bo_unreserve(rdev->r600_blit.shader_obj); 3096 } 3097 3098 return 0; 3099 } 3100 3101 int evergreen_copy_blit(struct radeon_device *rdev, 3102 uint64_t src_offset, uint64_t dst_offset, 3103 unsigned num_pages, struct radeon_fence *fence) 3104 { 3105 int r; 3106 3107 mutex_lock(&rdev->r600_blit.mutex); 3108 rdev->r600_blit.vb_ib = NULL; 3109 r = evergreen_blit_prepare_copy(rdev, num_pages * RADEON_GPU_PAGE_SIZE); 3110 if (r) { 3111 if (rdev->r600_blit.vb_ib) 3112 radeon_ib_free(rdev, &rdev->r600_blit.vb_ib); 3113 mutex_unlock(&rdev->r600_blit.mutex); 3114 return r; 3115 } 3116 evergreen_kms_blit_copy(rdev, src_offset, dst_offset, num_pages * RADEON_GPU_PAGE_SIZE); 3117 evergreen_blit_done_copy(rdev, fence); 3118 mutex_unlock(&rdev->r600_blit.mutex); 3119 return 0; 3120 } 3121 3122 /* Plan is to move initialization in that function and use 3123 * helper function so that radeon_device_init pretty much 3124 * do nothing more than calling asic specific function. This 3125 * should also allow to remove a bunch of callback function 3126 * like vram_info. 3127 */ 3128 int evergreen_init(struct radeon_device *rdev) 3129 { 3130 int r; 3131 3132 /* This don't do much */ 3133 r = radeon_gem_init(rdev); 3134 if (r) 3135 return r; 3136 /* Read BIOS */ 3137 if (!radeon_get_bios(rdev)) { 3138 if (ASIC_IS_AVIVO(rdev)) 3139 return -EINVAL; 3140 } 3141 /* Must be an ATOMBIOS */ 3142 if (!rdev->is_atom_bios) { 3143 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n"); 3144 return -EINVAL; 3145 } 3146 r = radeon_atombios_init(rdev); 3147 if (r) 3148 return r; 3149 /* reset the asic, the gfx blocks are often in a bad state 3150 * after the driver is unloaded or after a resume 3151 */ 3152 if (radeon_asic_reset(rdev)) 3153 dev_warn(rdev->dev, "GPU reset failed !\n"); 3154 /* Post card if necessary */ 3155 if (!radeon_card_posted(rdev)) { 3156 if (!rdev->bios) { 3157 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); 3158 return -EINVAL; 3159 } 3160 DRM_INFO("GPU not posted. posting now...\n"); 3161 atom_asic_init(rdev->mode_info.atom_context); 3162 } 3163 /* Initialize scratch registers */ 3164 r600_scratch_init(rdev); 3165 /* Initialize surface registers */ 3166 radeon_surface_init(rdev); 3167 /* Initialize clocks */ 3168 radeon_get_clock_info(rdev->ddev); 3169 /* Fence driver */ 3170 r = radeon_fence_driver_init(rdev); 3171 if (r) 3172 return r; 3173 /* initialize AGP */ 3174 if (rdev->flags & RADEON_IS_AGP) { 3175 r = radeon_agp_init(rdev); 3176 if (r) 3177 radeon_agp_disable(rdev); 3178 } 3179 /* initialize memory controller */ 3180 r = evergreen_mc_init(rdev); 3181 if (r) 3182 return r; 3183 /* Memory manager */ 3184 r = radeon_bo_init(rdev); 3185 if (r) 3186 return r; 3187 3188 r = radeon_irq_kms_init(rdev); 3189 if (r) 3190 return r; 3191 3192 rdev->cp.ring_obj = NULL; 3193 r600_ring_init(rdev, 1024 * 1024); 3194 3195 rdev->ih.ring_obj = NULL; 3196 r600_ih_ring_init(rdev, 64 * 1024); 3197 3198 r = r600_pcie_gart_init(rdev); 3199 if (r) 3200 return r; 3201 3202 rdev->accel_working = true; 3203 r = evergreen_startup(rdev); 3204 if (r) { 3205 dev_err(rdev->dev, "disabling GPU acceleration\n"); 3206 r700_cp_fini(rdev); 3207 r600_irq_fini(rdev); 3208 radeon_wb_fini(rdev); 3209 radeon_irq_kms_fini(rdev); 3210 evergreen_pcie_gart_fini(rdev); 3211 rdev->accel_working = false; 3212 } 3213 if (rdev->accel_working) { 3214 r = radeon_ib_pool_init(rdev); 3215 if (r) { 3216 DRM_ERROR("radeon: failed initializing IB pool (%d).\n", r); 3217 rdev->accel_working = false; 3218 } 3219 r = r600_ib_test(rdev); 3220 if (r) { 3221 DRM_ERROR("radeon: failed testing IB (%d).\n", r); 3222 rdev->accel_working = false; 3223 } 3224 } 3225 return 0; 3226 } 3227 3228 void evergreen_fini(struct radeon_device *rdev) 3229 { 3230 evergreen_blit_fini(rdev); 3231 r700_cp_fini(rdev); 3232 r600_irq_fini(rdev); 3233 radeon_wb_fini(rdev); 3234 radeon_irq_kms_fini(rdev); 3235 evergreen_pcie_gart_fini(rdev); 3236 radeon_gem_fini(rdev); 3237 radeon_fence_driver_fini(rdev); 3238 radeon_agp_fini(rdev); 3239 radeon_bo_fini(rdev); 3240 radeon_atombios_fini(rdev); 3241 kfree(rdev->bios); 3242 rdev->bios = NULL; 3243 } 3244 3245 static void evergreen_pcie_gen2_enable(struct radeon_device *rdev) 3246 { 3247 u32 link_width_cntl, speed_cntl; 3248 3249 if (radeon_pcie_gen2 == 0) 3250 return; 3251 3252 if (rdev->flags & RADEON_IS_IGP) 3253 return; 3254 3255 if (!(rdev->flags & RADEON_IS_PCIE)) 3256 return; 3257 3258 /* x2 cards have a special sequence */ 3259 if (ASIC_IS_X2(rdev)) 3260 return; 3261 3262 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3263 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) || 3264 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) { 3265 3266 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); 3267 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 3268 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 3269 3270 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3271 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN; 3272 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); 3273 3274 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3275 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT; 3276 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); 3277 3278 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3279 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT; 3280 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); 3281 3282 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL); 3283 speed_cntl |= LC_GEN2_EN_STRAP; 3284 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl); 3285 3286 } else { 3287 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL); 3288 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */ 3289 if (1) 3290 link_width_cntl |= LC_UPCONFIGURE_DIS; 3291 else 3292 link_width_cntl &= ~LC_UPCONFIGURE_DIS; 3293 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl); 3294 } 3295 } 3296