144  * cik_get_allowed_info_register - fetch the register for the info ioctl
150  * Returns 0 for success or -EINVAL for an invalid register
170 		return 0;  in cik_get_allowed_info_register()
172 		return -EINVAL;  in cik_get_allowed_info_register()
184 	spin_lock_irqsave(&rdev->didt_idx_lock, flags);  in cik_didt_rreg()
187 	spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);  in cik_didt_rreg()
195 	spin_lock_irqsave(&rdev->didt_idx_lock, flags);  in cik_didt_wreg()
198 	spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);  in cik_didt_wreg()
205 	int actual_temp = 0;  in ci_get_temp()
210 	if (temp & 0x200)  in ci_get_temp()
213 		actual_temp = temp & 0x1ff;  in ci_get_temp()
222 	int actual_temp = 0;  in kv_get_temp()
224 	temp = RREG32_SMC(0xC0300E0C);  in kv_get_temp()
227 		actual_temp = (temp / 8) - 49;  in kv_get_temp()
229 		actual_temp = 0;  in kv_get_temp()
242 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);  in cik_pciep_rreg()
246 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);  in cik_pciep_rreg()
254 	spin_lock_irqsave(&rdev->pciep_idx_lock, flags);  in cik_pciep_wreg()
259 	spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);  in cik_pciep_wreg()
264 	(0x0e00 << 16) | (0xc12c >> 2),
265 	0x00000000,
266 	(0x0e00 << 16) | (0xc140 >> 2),
267 	0x00000000,
268 	(0x0e00 << 16) | (0xc150 >> 2),
269 	0x00000000,
270 	(0x0e00 << 16) | (0xc15c >> 2),
271 	0x00000000,
272 	(0x0e00 << 16) | (0xc168 >> 2),
273 	0x00000000,
274 	(0x0e00 << 16) | (0xc170 >> 2),
275 	0x00000000,
276 	(0x0e00 << 16) | (0xc178 >> 2),
277 	0x00000000,
278 	(0x0e00 << 16) | (0xc204 >> 2),
279 	0x00000000,
280 	(0x0e00 << 16) | (0xc2b4 >> 2),
281 	0x00000000,
282 	(0x0e00 << 16) | (0xc2b8 >> 2),
283 	0x00000000,
284 	(0x0e00 << 16) | (0xc2bc >> 2),
285 	0x00000000,
286 	(0x0e00 << 16) | (0xc2c0 >> 2),
287 	0x00000000,
288 	(0x0e00 << 16) | (0x8228 >> 2),
289 	0x00000000,
290 	(0x0e00 << 16) | (0x829c >> 2),
291 	0x00000000,
292 	(0x0e00 << 16) | (0x869c >> 2),
293 	0x00000000,
294 	(0x0600 << 16) | (0x98f4 >> 2),
295 	0x00000000,
296 	(0x0e00 << 16) | (0x98f8 >> 2),
297 	0x00000000,
298 	(0x0e00 << 16) | (0x9900 >> 2),
299 	0x00000000,
300 	(0x0e00 << 16) | (0xc260 >> 2),
301 	0x00000000,
302 	(0x0e00 << 16) | (0x90e8 >> 2),
303 	0x00000000,
304 	(0x0e00 << 16) | (0x3c000 >> 2),
305 	0x00000000,
306 	(0x0e00 << 16) | (0x3c00c >> 2),
307 	0x00000000,
308 	(0x0e00 << 16) | (0x8c1c >> 2),
309 	0x00000000,
310 	(0x0e00 << 16) | (0x9700 >> 2),
311 	0x00000000,
312 	(0x0e00 << 16) | (0xcd20 >> 2),
313 	0x00000000,
314 	(0x4e00 << 16) | (0xcd20 >> 2),
315 	0x00000000,
316 	(0x5e00 << 16) | (0xcd20 >> 2),
317 	0x00000000,
318 	(0x6e00 << 16) | (0xcd20 >> 2),
319 	0x00000000,
320 	(0x7e00 << 16) | (0xcd20 >> 2),
321 	0x00000000,
322 	(0x8e00 << 16) | (0xcd20 >> 2),
323 	0x00000000,
324 	(0x9e00 << 16) | (0xcd20 >> 2),
325 	0x00000000,
326 	(0xae00 << 16) | (0xcd20 >> 2),
327 	0x00000000,
328 	(0xbe00 << 16) | (0xcd20 >> 2),
329 	0x00000000,
330 	(0x0e00 << 16) | (0x89bc >> 2),
331 	0x00000000,
332 	(0x0e00 << 16) | (0x8900 >> 2),
333 	0x00000000,
334 	0x3,
335 	(0x0e00 << 16) | (0xc130 >> 2),
336 	0x00000000,
337 	(0x0e00 << 16) | (0xc134 >> 2),
338 	0x00000000,
339 	(0x0e00 << 16) | (0xc1fc >> 2),
340 	0x00000000,
341 	(0x0e00 << 16) | (0xc208 >> 2),
342 	0x00000000,
343 	(0x0e00 << 16) | (0xc264 >> 2),
344 	0x00000000,
345 	(0x0e00 << 16) | (0xc268 >> 2),
346 	0x00000000,
347 	(0x0e00 << 16) | (0xc26c >> 2),
348 	0x00000000,
349 	(0x0e00 << 16) | (0xc270 >> 2),
350 	0x00000000,
351 	(0x0e00 << 16) | (0xc274 >> 2),
352 	0x00000000,
353 	(0x0e00 << 16) | (0xc278 >> 2),
354 	0x00000000,
355 	(0x0e00 << 16) | (0xc27c >> 2),
356 	0x00000000,
357 	(0x0e00 << 16) | (0xc280 >> 2),
358 	0x00000000,
359 	(0x0e00 << 16) | (0xc284 >> 2),
360 	0x00000000,
361 	(0x0e00 << 16) | (0xc288 >> 2),
362 	0x00000000,
363 	(0x0e00 << 16) | (0xc28c >> 2),
364 	0x00000000,
365 	(0x0e00 << 16) | (0xc290 >> 2),
366 	0x00000000,
367 	(0x0e00 << 16) | (0xc294 >> 2),
368 	0x00000000,
369 	(0x0e00 << 16) | (0xc298 >> 2),
370 	0x00000000,
371 	(0x0e00 << 16) | (0xc29c >> 2),
372 	0x00000000,
373 	(0x0e00 << 16) | (0xc2a0 >> 2),
374 	0x00000000,
375 	(0x0e00 << 16) | (0xc2a4 >> 2),
376 	0x00000000,
377 	(0x0e00 << 16) | (0xc2a8 >> 2),
378 	0x00000000,
379 	(0x0e00 << 16) | (0xc2ac  >> 2),
380 	0x00000000,
381 	(0x0e00 << 16) | (0xc2b0 >> 2),
382 	0x00000000,
383 	(0x0e00 << 16) | (0x301d0 >> 2),
384 	0x00000000,
385 	(0x0e00 << 16) | (0x30238 >> 2),
386 	0x00000000,
387 	(0x0e00 << 16) | (0x30250 >> 2),
388 	0x00000000,
389 	(0x0e00 << 16) | (0x30254 >> 2),
390 	0x00000000,
391 	(0x0e00 << 16) | (0x30258 >> 2),
392 	0x00000000,
393 	(0x0e00 << 16) | (0x3025c >> 2),
394 	0x00000000,
395 	(0x4e00 << 16) | (0xc900 >> 2),
396 	0x00000000,
397 	(0x5e00 << 16) | (0xc900 >> 2),
398 	0x00000000,
399 	(0x6e00 << 16) | (0xc900 >> 2),
400 	0x00000000,
401 	(0x7e00 << 16) | (0xc900 >> 2),
402 	0x00000000,
403 	(0x8e00 << 16) | (0xc900 >> 2),
404 	0x00000000,
405 	(0x9e00 << 16) | (0xc900 >> 2),
406 	0x00000000,
407 	(0xae00 << 16) | (0xc900 >> 2),
408 	0x00000000,
409 	(0xbe00 << 16) | (0xc900 >> 2),
410 	0x00000000,
411 	(0x4e00 << 16) | (0xc904 >> 2),
412 	0x00000000,
413 	(0x5e00 << 16) | (0xc904 >> 2),
414 	0x00000000,
415 	(0x6e00 << 16) | (0xc904 >> 2),
416 	0x00000000,
417 	(0x7e00 << 16) | (0xc904 >> 2),
418 	0x00000000,
419 	(0x8e00 << 16) | (0xc904 >> 2),
420 	0x00000000,
421 	(0x9e00 << 16) | (0xc904 >> 2),
422 	0x00000000,
423 	(0xae00 << 16) | (0xc904 >> 2),
424 	0x00000000,
425 	(0xbe00 << 16) | (0xc904 >> 2),
426 	0x00000000,
427 	(0x4e00 << 16) | (0xc908 >> 2),
428 	0x00000000,
429 	(0x5e00 << 16) | (0xc908 >> 2),
430 	0x00000000,
431 	(0x6e00 << 16) | (0xc908 >> 2),
432 	0x00000000,
433 	(0x7e00 << 16) | (0xc908 >> 2),
434 	0x00000000,
435 	(0x8e00 << 16) | (0xc908 >> 2),
436 	0x00000000,
437 	(0x9e00 << 16) | (0xc908 >> 2),
438 	0x00000000,
439 	(0xae00 << 16) | (0xc908 >> 2),
440 	0x00000000,
441 	(0xbe00 << 16) | (0xc908 >> 2),
442 	0x00000000,
443 	(0x4e00 << 16) | (0xc90c >> 2),
444 	0x00000000,
445 	(0x5e00 << 16) | (0xc90c >> 2),
446 	0x00000000,
447 	(0x6e00 << 16) | (0xc90c >> 2),
448 	0x00000000,
449 	(0x7e00 << 16) | (0xc90c >> 2),
450 	0x00000000,
451 	(0x8e00 << 16) | (0xc90c >> 2),
452 	0x00000000,
453 	(0x9e00 << 16) | (0xc90c >> 2),
454 	0x00000000,
455 	(0xae00 << 16) | (0xc90c >> 2),
456 	0x00000000,
457 	(0xbe00 << 16) | (0xc90c >> 2),
458 	0x00000000,
459 	(0x4e00 << 16) | (0xc910 >> 2),
460 	0x00000000,
461 	(0x5e00 << 16) | (0xc910 >> 2),
462 	0x00000000,
463 	(0x6e00 << 16) | (0xc910 >> 2),
464 	0x00000000,
465 	(0x7e00 << 16) | (0xc910 >> 2),
466 	0x00000000,
467 	(0x8e00 << 16) | (0xc910 >> 2),
468 	0x00000000,
469 	(0x9e00 << 16) | (0xc910 >> 2),
470 	0x00000000,
471 	(0xae00 << 16) | (0xc910 >> 2),
472 	0x00000000,
473 	(0xbe00 << 16) | (0xc910 >> 2),
474 	0x00000000,
475 	(0x0e00 << 16) | (0xc99c >> 2),
476 	0x00000000,
477 	(0x0e00 << 16) | (0x9834 >> 2),
478 	0x00000000,
479 	(0x0000 << 16) | (0x30f00 >> 2),
480 	0x00000000,
481 	(0x0001 << 16) | (0x30f00 >> 2),
482 	0x00000000,
483 	(0x0000 << 16) | (0x30f04 >> 2),
484 	0x00000000,
485 	(0x0001 << 16) | (0x30f04 >> 2),
486 	0x00000000,
487 	(0x0000 << 16) | (0x30f08 >> 2),
488 	0x00000000,
489 	(0x0001 << 16) | (0x30f08 >> 2),
490 	0x00000000,
491 	(0x0000 << 16) | (0x30f0c >> 2),
492 	0x00000000,
493 	(0x0001 << 16) | (0x30f0c >> 2),
494 	0x00000000,
495 	(0x0600 << 16) | (0x9b7c >> 2),
496 	0x00000000,
497 	(0x0e00 << 16) | (0x8a14 >> 2),
498 	0x00000000,
499 	(0x0e00 << 16) | (0x8a18 >> 2),
500 	0x00000000,
501 	(0x0600 << 16) | (0x30a00 >> 2),
502 	0x00000000,
503 	(0x0e00 << 16) | (0x8bf0 >> 2),
504 	0x00000000,
505 	(0x0e00 << 16) | (0x8bcc >> 2),
506 	0x00000000,
507 	(0x0e00 << 16) | (0x8b24 >> 2),
508 	0x00000000,
509 	(0x0e00 << 16) | (0x30a04 >> 2),
510 	0x00000000,
511 	(0x0600 << 16) | (0x30a10 >> 2),
512 	0x00000000,
513 	(0x0600 << 16) | (0x30a14 >> 2),
514 	0x00000000,
515 	(0x0600 << 16) | (0x30a18 >> 2),
516 	0x00000000,
517 	(0x0600 << 16) | (0x30a2c >> 2),
518 	0x00000000,
519 	(0x0e00 << 16) | (0xc700 >> 2),
520 	0x00000000,
521 	(0x0e00 << 16) | (0xc704 >> 2),
522 	0x00000000,
523 	(0x0e00 << 16) | (0xc708 >> 2),
524 	0x00000000,
525 	(0x0e00 << 16) | (0xc768 >> 2),
526 	0x00000000,
527 	(0x0400 << 16) | (0xc770 >> 2),
528 	0x00000000,
529 	(0x0400 << 16) | (0xc774 >> 2),
530 	0x00000000,
531 	(0x0400 << 16) | (0xc778 >> 2),
532 	0x00000000,
533 	(0x0400 << 16) | (0xc77c >> 2),
534 	0x00000000,
535 	(0x0400 << 16) | (0xc780 >> 2),
536 	0x00000000,
537 	(0x0400 << 16) | (0xc784 >> 2),
538 	0x00000000,
539 	(0x0400 << 16) | (0xc788 >> 2),
540 	0x00000000,
541 	(0x0400 << 16) | (0xc78c >> 2),
542 	0x00000000,
543 	(0x0400 << 16) | (0xc798 >> 2),
544 	0x00000000,
545 	(0x0400 << 16) | (0xc79c >> 2),
546 	0x00000000,
547 	(0x0400 << 16) | (0xc7a0 >> 2),
548 	0x00000000,
549 	(0x0400 << 16) | (0xc7a4 >> 2),
550 	0x00000000,
551 	(0x0400 << 16) | (0xc7a8 >> 2),
552 	0x00000000,
553 	(0x0400 << 16) | (0xc7ac >> 2),
554 	0x00000000,
555 	(0x0400 << 16) | (0xc7b0 >> 2),
556 	0x00000000,
557 	(0x0400 << 16) | (0xc7b4 >> 2),
558 	0x00000000,
559 	(0x0e00 << 16) | (0x9100 >> 2),
560 	0x00000000,
561 	(0x0e00 << 16) | (0x3c010 >> 2),
562 	0x00000000,
563 	(0x0e00 << 16) | (0x92a8 >> 2),
564 	0x00000000,
565 	(0x0e00 << 16) | (0x92ac >> 2),
566 	0x00000000,
567 	(0x0e00 << 16) | (0x92b4 >> 2),
568 	0x00000000,
569 	(0x0e00 << 16) | (0x92b8 >> 2),
570 	0x00000000,
571 	(0x0e00 << 16) | (0x92bc >> 2),
572 	0x00000000,
573 	(0x0e00 << 16) | (0x92c0 >> 2),
574 	0x00000000,
575 	(0x0e00 << 16) | (0x92c4 >> 2),
576 	0x00000000,
577 	(0x0e00 << 16) | (0x92c8 >> 2),
578 	0x00000000,
579 	(0x0e00 << 16) | (0x92cc >> 2),
580 	0x00000000,
581 	(0x0e00 << 16) | (0x92d0 >> 2),
582 	0x00000000,
583 	(0x0e00 << 16) | (0x8c00 >> 2),
584 	0x00000000,
585 	(0x0e00 << 16) | (0x8c04 >> 2),
586 	0x00000000,
587 	(0x0e00 << 16) | (0x8c20 >> 2),
588 	0x00000000,
589 	(0x0e00 << 16) | (0x8c38 >> 2),
590 	0x00000000,
591 	(0x0e00 << 16) | (0x8c3c >> 2),
592 	0x00000000,
593 	(0x0e00 << 16) | (0xae00 >> 2),
594 	0x00000000,
595 	(0x0e00 << 16) | (0x9604 >> 2),
596 	0x00000000,
597 	(0x0e00 << 16) | (0xac08 >> 2),
598 	0x00000000,
599 	(0x0e00 << 16) | (0xac0c >> 2),
600 	0x00000000,
601 	(0x0e00 << 16) | (0xac10 >> 2),
602 	0x00000000,
603 	(0x0e00 << 16) | (0xac14 >> 2),
604 	0x00000000,
605 	(0x0e00 << 16) | (0xac58 >> 2),
606 	0x00000000,
607 	(0x0e00 << 16) | (0xac68 >> 2),
608 	0x00000000,
609 	(0x0e00 << 16) | (0xac6c >> 2),
610 	0x00000000,
611 	(0x0e00 << 16) | (0xac70 >> 2),
612 	0x00000000,
613 	(0x0e00 << 16) | (0xac74 >> 2),
614 	0x00000000,
615 	(0x0e00 << 16) | (0xac78 >> 2),
616 	0x00000000,
617 	(0x0e00 << 16) | (0xac7c >> 2),
618 	0x00000000,
619 	(0x0e00 << 16) | (0xac80 >> 2),
620 	0x00000000,
621 	(0x0e00 << 16) | (0xac84 >> 2),
622 	0x00000000,
623 	(0x0e00 << 16) | (0xac88 >> 2),
624 	0x00000000,
625 	(0x0e00 << 16) | (0xac8c >> 2),
626 	0x00000000,
627 	(0x0e00 << 16) | (0x970c >> 2),
628 	0x00000000,
629 	(0x0e00 << 16) | (0x9714 >> 2),
630 	0x00000000,
631 	(0x0e00 << 16) | (0x9718 >> 2),
632 	0x00000000,
633 	(0x0e00 << 16) | (0x971c >> 2),
634 	0x00000000,
635 	(0x0e00 << 16) | (0x31068 >> 2),
636 	0x00000000,
637 	(0x4e00 << 16) | (0x31068 >> 2),
638 	0x00000000,
639 	(0x5e00 << 16) | (0x31068 >> 2),
640 	0x00000000,
641 	(0x6e00 << 16) | (0x31068 >> 2),
642 	0x00000000,
643 	(0x7e00 << 16) | (0x31068 >> 2),
644 	0x00000000,
645 	(0x8e00 << 16) | (0x31068 >> 2),
646 	0x00000000,
647 	(0x9e00 << 16) | (0x31068 >> 2),
648 	0x00000000,
649 	(0xae00 << 16) | (0x31068 >> 2),
650 	0x00000000,
651 	(0xbe00 << 16) | (0x31068 >> 2),
652 	0x00000000,
653 	(0x0e00 << 16) | (0xcd10 >> 2),
654 	0x00000000,
655 	(0x0e00 << 16) | (0xcd14 >> 2),
656 	0x00000000,
657 	(0x0e00 << 16) | (0x88b0 >> 2),
658 	0x00000000,
659 	(0x0e00 << 16) | (0x88b4 >> 2),
660 	0x00000000,
661 	(0x0e00 << 16) | (0x88b8 >> 2),
662 	0x00000000,
663 	(0x0e00 << 16) | (0x88bc >> 2),
664 	0x00000000,
665 	(0x0400 << 16) | (0x89c0 >> 2),
666 	0x00000000,
667 	(0x0e00 << 16) | (0x88c4 >> 2),
668 	0x00000000,
669 	(0x0e00 << 16) | (0x88c8 >> 2),
670 	0x00000000,
671 	(0x0e00 << 16) | (0x88d0 >> 2),
672 	0x00000000,
673 	(0x0e00 << 16) | (0x88d4 >> 2),
674 	0x00000000,
675 	(0x0e00 << 16) | (0x88d8 >> 2),
676 	0x00000000,
677 	(0x0e00 << 16) | (0x8980 >> 2),
678 	0x00000000,
679 	(0x0e00 << 16) | (0x30938 >> 2),
680 	0x00000000,
681 	(0x0e00 << 16) | (0x3093c >> 2),
682 	0x00000000,
683 	(0x0e00 << 16) | (0x30940 >> 2),
684 	0x00000000,
685 	(0x0e00 << 16) | (0x89a0 >> 2),
686 	0x00000000,
687 	(0x0e00 << 16) | (0x30900 >> 2),
688 	0x00000000,
689 	(0x0e00 << 16) | (0x30904 >> 2),
690 	0x00000000,
691 	(0x0e00 << 16) | (0x89b4 >> 2),
692 	0x00000000,
693 	(0x0e00 << 16) | (0x3c210 >> 2),
694 	0x00000000,
695 	(0x0e00 << 16) | (0x3c214 >> 2),
696 	0x00000000,
697 	(0x0e00 << 16) | (0x3c218 >> 2),
698 	0x00000000,
699 	(0x0e00 << 16) | (0x8904 >> 2),
700 	0x00000000,
701 	0x5,
702 	(0x0e00 << 16) | (0x8c28 >> 2),
703 	(0x0e00 << 16) | (0x8c2c >> 2),
704 	(0x0e00 << 16) | (0x8c30 >> 2),
705 	(0x0e00 << 16) | (0x8c34 >> 2),
706 	(0x0e00 << 16) | (0x9600 >> 2),
711 	(0x0e00 << 16) | (0xc12c >> 2),
712 	0x00000000,
713 	(0x0e00 << 16) | (0xc140 >> 2),
714 	0x00000000,
715 	(0x0e00 << 16) | (0xc150 >> 2),
716 	0x00000000,
717 	(0x0e00 << 16) | (0xc15c >> 2),
718 	0x00000000,
719 	(0x0e00 << 16) | (0xc168 >> 2),
720 	0x00000000,
721 	(0x0e00 << 16) | (0xc170 >> 2),
722 	0x00000000,
723 	(0x0e00 << 16) | (0xc204 >> 2),
724 	0x00000000,
725 	(0x0e00 << 16) | (0xc2b4 >> 2),
726 	0x00000000,
727 	(0x0e00 << 16) | (0xc2b8 >> 2),
728 	0x00000000,
729 	(0x0e00 << 16) | (0xc2bc >> 2),
730 	0x00000000,
731 	(0x0e00 << 16) | (0xc2c0 >> 2),
732 	0x00000000,
733 	(0x0e00 << 16) | (0x8228 >> 2),
734 	0x00000000,
735 	(0x0e00 << 16) | (0x829c >> 2),
736 	0x00000000,
737 	(0x0e00 << 16) | (0x869c >> 2),
738 	0x00000000,
739 	(0x0600 << 16) | (0x98f4 >> 2),
740 	0x00000000,
741 	(0x0e00 << 16) | (0x98f8 >> 2),
742 	0x00000000,
743 	(0x0e00 << 16) | (0x9900 >> 2),
744 	0x00000000,
745 	(0x0e00 << 16) | (0xc260 >> 2),
746 	0x00000000,
747 	(0x0e00 << 16) | (0x90e8 >> 2),
748 	0x00000000,
749 	(0x0e00 << 16) | (0x3c000 >> 2),
750 	0x00000000,
751 	(0x0e00 << 16) | (0x3c00c >> 2),
752 	0x00000000,
753 	(0x0e00 << 16) | (0x8c1c >> 2),
754 	0x00000000,
755 	(0x0e00 << 16) | (0x9700 >> 2),
756 	0x00000000,
757 	(0x0e00 << 16) | (0xcd20 >> 2),
758 	0x00000000,
759 	(0x4e00 << 16) | (0xcd20 >> 2),
760 	0x00000000,
761 	(0x5e00 << 16) | (0xcd20 >> 2),
762 	0x00000000,
763 	(0x6e00 << 16) | (0xcd20 >> 2),
764 	0x00000000,
765 	(0x7e00 << 16) | (0xcd20 >> 2),
766 	0x00000000,
767 	(0x0e00 << 16) | (0x89bc >> 2),
768 	0x00000000,
769 	(0x0e00 << 16) | (0x8900 >> 2),
770 	0x00000000,
771 	0x3,
772 	(0x0e00 << 16) | (0xc130 >> 2),
773 	0x00000000,
774 	(0x0e00 << 16) | (0xc134 >> 2),
775 	0x00000000,
776 	(0x0e00 << 16) | (0xc1fc >> 2),
777 	0x00000000,
778 	(0x0e00 << 16) | (0xc208 >> 2),
779 	0x00000000,
780 	(0x0e00 << 16) | (0xc264 >> 2),
781 	0x00000000,
782 	(0x0e00 << 16) | (0xc268 >> 2),
783 	0x00000000,
784 	(0x0e00 << 16) | (0xc26c >> 2),
785 	0x00000000,
786 	(0x0e00 << 16) | (0xc270 >> 2),
787 	0x00000000,
788 	(0x0e00 << 16) | (0xc274 >> 2),
789 	0x00000000,
790 	(0x0e00 << 16) | (0xc28c >> 2),
791 	0x00000000,
792 	(0x0e00 << 16) | (0xc290 >> 2),
793 	0x00000000,
794 	(0x0e00 << 16) | (0xc294 >> 2),
795 	0x00000000,
796 	(0x0e00 << 16) | (0xc298 >> 2),
797 	0x00000000,
798 	(0x0e00 << 16) | (0xc2a0 >> 2),
799 	0x00000000,
800 	(0x0e00 << 16) | (0xc2a4 >> 2),
801 	0x00000000,
802 	(0x0e00 << 16) | (0xc2a8 >> 2),
803 	0x00000000,
804 	(0x0e00 << 16) | (0xc2ac >> 2),
805 	0x00000000,
806 	(0x0e00 << 16) | (0x301d0 >> 2),
807 	0x00000000,
808 	(0x0e00 << 16) | (0x30238 >> 2),
809 	0x00000000,
810 	(0x0e00 << 16) | (0x30250 >> 2),
811 	0x00000000,
812 	(0x0e00 << 16) | (0x30254 >> 2),
813 	0x00000000,
814 	(0x0e00 << 16) | (0x30258 >> 2),
815 	0x00000000,
816 	(0x0e00 << 16) | (0x3025c >> 2),
817 	0x00000000,
818 	(0x4e00 << 16) | (0xc900 >> 2),
819 	0x00000000,
820 	(0x5e00 << 16) | (0xc900 >> 2),
821 	0x00000000,
822 	(0x6e00 << 16) | (0xc900 >> 2),
823 	0x00000000,
824 	(0x7e00 << 16) | (0xc900 >> 2),
825 	0x00000000,
826 	(0x4e00 << 16) | (0xc904 >> 2),
827 	0x00000000,
828 	(0x5e00 << 16) | (0xc904 >> 2),
829 	0x00000000,
830 	(0x6e00 << 16) | (0xc904 >> 2),
831 	0x00000000,
832 	(0x7e00 << 16) | (0xc904 >> 2),
833 	0x00000000,
834 	(0x4e00 << 16) | (0xc908 >> 2),
835 	0x00000000,
836 	(0x5e00 << 16) | (0xc908 >> 2),
837 	0x00000000,
838 	(0x6e00 << 16) | (0xc908 >> 2),
839 	0x00000000,
840 	(0x7e00 << 16) | (0xc908 >> 2),
841 	0x00000000,
842 	(0x4e00 << 16) | (0xc90c >> 2),
843 	0x00000000,
844 	(0x5e00 << 16) | (0xc90c >> 2),
845 	0x00000000,
846 	(0x6e00 << 16) | (0xc90c >> 2),
847 	0x00000000,
848 	(0x7e00 << 16) | (0xc90c >> 2),
849 	0x00000000,
850 	(0x4e00 << 16) | (0xc910 >> 2),
851 	0x00000000,
852 	(0x5e00 << 16) | (0xc910 >> 2),
853 	0x00000000,
854 	(0x6e00 << 16) | (0xc910 >> 2),
855 	0x00000000,
856 	(0x7e00 << 16) | (0xc910 >> 2),
857 	0x00000000,
858 	(0x0e00 << 16) | (0xc99c >> 2),
859 	0x00000000,
860 	(0x0e00 << 16) | (0x9834 >> 2),
861 	0x00000000,
862 	(0x0000 << 16) | (0x30f00 >> 2),
863 	0x00000000,
864 	(0x0000 << 16) | (0x30f04 >> 2),
865 	0x00000000,
866 	(0x0000 << 16) | (0x30f08 >> 2),
867 	0x00000000,
868 	(0x0000 << 16) | (0x30f0c >> 2),
869 	0x00000000,
870 	(0x0600 << 16) | (0x9b7c >> 2),
871 	0x00000000,
872 	(0x0e00 << 16) | (0x8a14 >> 2),
873 	0x00000000,
874 	(0x0e00 << 16) | (0x8a18 >> 2),
875 	0x00000000,
876 	(0x0600 << 16) | (0x30a00 >> 2),
877 	0x00000000,
878 	(0x0e00 << 16) | (0x8bf0 >> 2),
879 	0x00000000,
880 	(0x0e00 << 16) | (0x8bcc >> 2),
881 	0x00000000,
882 	(0x0e00 << 16) | (0x8b24 >> 2),
883 	0x00000000,
884 	(0x0e00 << 16) | (0x30a04 >> 2),
885 	0x00000000,
886 	(0x0600 << 16) | (0x30a10 >> 2),
887 	0x00000000,
888 	(0x0600 << 16) | (0x30a14 >> 2),
889 	0x00000000,
890 	(0x0600 << 16) | (0x30a18 >> 2),
891 	0x00000000,
892 	(0x0600 << 16) | (0x30a2c >> 2),
893 	0x00000000,
894 	(0x0e00 << 16) | (0xc700 >> 2),
895 	0x00000000,
896 	(0x0e00 << 16) | (0xc704 >> 2),
897 	0x00000000,
898 	(0x0e00 << 16) | (0xc708 >> 2),
899 	0x00000000,
900 	(0x0e00 << 16) | (0xc768 >> 2),
901 	0x00000000,
902 	(0x0400 << 16) | (0xc770 >> 2),
903 	0x00000000,
904 	(0x0400 << 16) | (0xc774 >> 2),
905 	0x00000000,
906 	(0x0400 << 16) | (0xc798 >> 2),
907 	0x00000000,
908 	(0x0400 << 16) | (0xc79c >> 2),
909 	0x00000000,
910 	(0x0e00 << 16) | (0x9100 >> 2),
911 	0x00000000,
912 	(0x0e00 << 16) | (0x3c010 >> 2),
913 	0x00000000,
914 	(0x0e00 << 16) | (0x8c00 >> 2),
915 	0x00000000,
916 	(0x0e00 << 16) | (0x8c04 >> 2),
917 	0x00000000,
918 	(0x0e00 << 16) | (0x8c20 >> 2),
919 	0x00000000,
920 	(0x0e00 << 16) | (0x8c38 >> 2),
921 	0x00000000,
922 	(0x0e00 << 16) | (0x8c3c >> 2),
923 	0x00000000,
924 	(0x0e00 << 16) | (0xae00 >> 2),
925 	0x00000000,
926 	(0x0e00 << 16) | (0x9604 >> 2),
927 	0x00000000,
928 	(0x0e00 << 16) | (0xac08 >> 2),
929 	0x00000000,
930 	(0x0e00 << 16) | (0xac0c >> 2),
931 	0x00000000,
932 	(0x0e00 << 16) | (0xac10 >> 2),
933 	0x00000000,
934 	(0x0e00 << 16) | (0xac14 >> 2),
935 	0x00000000,
936 	(0x0e00 << 16) | (0xac58 >> 2),
937 	0x00000000,
938 	(0x0e00 << 16) | (0xac68 >> 2),
939 	0x00000000,
940 	(0x0e00 << 16) | (0xac6c >> 2),
941 	0x00000000,
942 	(0x0e00 << 16) | (0xac70 >> 2),
943 	0x00000000,
944 	(0x0e00 << 16) | (0xac74 >> 2),
945 	0x00000000,
946 	(0x0e00 << 16) | (0xac78 >> 2),
947 	0x00000000,
948 	(0x0e00 << 16) | (0xac7c >> 2),
949 	0x00000000,
950 	(0x0e00 << 16) | (0xac80 >> 2),
951 	0x00000000,
952 	(0x0e00 << 16) | (0xac84 >> 2),
953 	0x00000000,
954 	(0x0e00 << 16) | (0xac88 >> 2),
955 	0x00000000,
956 	(0x0e00 << 16) | (0xac8c >> 2),
957 	0x00000000,
958 	(0x0e00 << 16) | (0x970c >> 2),
959 	0x00000000,
960 	(0x0e00 << 16) | (0x9714 >> 2),
961 	0x00000000,
962 	(0x0e00 << 16) | (0x9718 >> 2),
963 	0x00000000,
964 	(0x0e00 << 16) | (0x971c >> 2),
965 	0x00000000,
966 	(0x0e00 << 16) | (0x31068 >> 2),
967 	0x00000000,
968 	(0x4e00 << 16) | (0x31068 >> 2),
969 	0x00000000,
970 	(0x5e00 << 16) | (0x31068 >> 2),
971 	0x00000000,
972 	(0x6e00 << 16) | (0x31068 >> 2),
973 	0x00000000,
974 	(0x7e00 << 16) | (0x31068 >> 2),
975 	0x00000000,
976 	(0x0e00 << 16) | (0xcd10 >> 2),
977 	0x00000000,
978 	(0x0e00 << 16) | (0xcd14 >> 2),
979 	0x00000000,
980 	(0x0e00 << 16) | (0x88b0 >> 2),
981 	0x00000000,
982 	(0x0e00 << 16) | (0x88b4 >> 2),
983 	0x00000000,
984 	(0x0e00 << 16) | (0x88b8 >> 2),
985 	0x00000000,
986 	(0x0e00 << 16) | (0x88bc >> 2),
987 	0x00000000,
988 	(0x0400 << 16) | (0x89c0 >> 2),
989 	0x00000000,
990 	(0x0e00 << 16) | (0x88c4 >> 2),
991 	0x00000000,
992 	(0x0e00 << 16) | (0x88c8 >> 2),
993 	0x00000000,
994 	(0x0e00 << 16) | (0x88d0 >> 2),
995 	0x00000000,
996 	(0x0e00 << 16) | (0x88d4 >> 2),
997 	0x00000000,
998 	(0x0e00 << 16) | (0x88d8 >> 2),
999 	0x00000000,
1000 	(0x0e00 << 16) | (0x8980 >> 2),
1001 	0x00000000,
1002 	(0x0e00 << 16) | (0x30938 >> 2),
1003 	0x00000000,
1004 	(0x0e00 << 16) | (0x3093c >> 2),
1005 	0x00000000,
1006 	(0x0e00 << 16) | (0x30940 >> 2),
1007 	0x00000000,
1008 	(0x0e00 << 16) | (0x89a0 >> 2),
1009 	0x00000000,
1010 	(0x0e00 << 16) | (0x30900 >> 2),
1011 	0x00000000,
1012 	(0x0e00 << 16) | (0x30904 >> 2),
1013 	0x00000000,
1014 	(0x0e00 << 16) | (0x89b4 >> 2),
1015 	0x00000000,
1016 	(0x0e00 << 16) | (0x3e1fc >> 2),
1017 	0x00000000,
1018 	(0x0e00 << 16) | (0x3c210 >> 2),
1019 	0x00000000,
1020 	(0x0e00 << 16) | (0x3c214 >> 2),
1021 	0x00000000,
1022 	(0x0e00 << 16) | (0x3c218 >> 2),
1023 	0x00000000,
1024 	(0x0e00 << 16) | (0x8904 >> 2),
1025 	0x00000000,
1026 	0x5,
1027 	(0x0e00 << 16) | (0x8c28 >> 2),
1028 	(0x0e00 << 16) | (0x8c2c >> 2),
1029 	(0x0e00 << 16) | (0x8c30 >> 2),
1030 	(0x0e00 << 16) | (0x8c34 >> 2),
1031 	(0x0e00 << 16) | (0x9600 >> 2),
1036 	0x30800, 0xe0ffffff, 0xe0000000
1041 	0xc770, 0xffffffff, 0x00000800,
1042 	0xc774, 0xffffffff, 0x00000800,
1043 	0xc798, 0xffffffff, 0x00007fbf,
1044 	0xc79c, 0xffffffff, 0x00007faf
1049 	0x3354, 0x00000333, 0x00000333,
1050 	0x3350, 0x000c0fc0, 0x00040200,
1051 	0x9a10, 0x00010000, 0x00058208,
1052 	0x3c000, 0xffff1fff, 0x00140000,
1053 	0x3c200, 0xfdfc0fff, 0x00000100,
1054 	0x3c234, 0x40000000, 0x40000200,
1055 	0x9830, 0xffffffff, 0x00000000,
1056 	0x9834, 0xf00fffff, 0x00000400,
1057 	0x9838, 0x0002021c, 0x00020200,
1058 	0xc78, 0x00000080, 0x00000000,
1059 	0x5bb0, 0x000000f0, 0x00000070,
1060 	0x5bc0, 0xf0311fff, 0x80300000,
1061 	0x98f8, 0x73773777, 0x12010001,
1062 	0x350c, 0x00810000, 0x408af000,
1063 	0x7030, 0x31000111, 0x00000011,
1064 	0x2f48, 0x73773777, 0x12010001,
1065 	0x220c, 0x00007fb6, 0x0021a1b1,
1066 	0x2210, 0x00007fb6, 0x002021b1,
1067 	0x2180, 0x00007fb6, 0x00002191,
1068 	0x2218, 0x00007fb6, 0x002121b1,
1069 	0x221c, 0x00007fb6, 0x002021b1,
1070 	0x21dc, 0x00007fb6, 0x00002191,
1071 	0x21e0, 0x00007fb6, 0x00002191,
1072 	0x3628, 0x0000003f, 0x0000000a,
1073 	0x362c, 0x0000003f, 0x0000000a,
1074 	0x2ae4, 0x00073ffe, 0x000022a2,
1075 	0x240c, 0x000007ff, 0x00000000,
1076 	0x8a14, 0xf000003f, 0x00000007,
1077 	0x8bf0, 0x00002001, 0x00000001,
1078 	0x8b24, 0xffffffff, 0x00ffffff,
1079 	0x30a04, 0x0000ff0f, 0x00000000,
1080 	0x28a4c, 0x07ffffff, 0x06000000,
1081 	0x4d8, 0x00000fff, 0x00000100,
1082 	0x3e78, 0x00000001, 0x00000002,
1083 	0x9100, 0x03000000, 0x0362c688,
1084 	0x8c00, 0x000000ff, 0x00000001,
1085 	0xe40, 0x00001fff, 0x00001fff,
1086 	0x9060, 0x0000007f, 0x00000020,
1087 	0x9508, 0x00010000, 0x00010000,
1088 	0xac14, 0x000003ff, 0x000000f3,
1089 	0xac0c, 0xffffffff, 0x00001032
1094 	0xc420, 0xffffffff, 0xfffffffc,
1095 	0x30800, 0xffffffff, 0xe0000000,
1096 	0x3c2a0, 0xffffffff, 0x00000100,
1097 	0x3c208, 0xffffffff, 0x00000100,
1098 	0x3c2c0, 0xffffffff, 0xc0000100,
1099 	0x3c2c8, 0xffffffff, 0xc0000100,
1100 	0x3c2c4, 0xffffffff, 0xc0000100,
1101 	0x55e4, 0xffffffff, 0x00600100,
1102 	0x3c280, 0xffffffff, 0x00000100,
1103 	0x3c214, 0xffffffff, 0x06000100,
1104 	0x3c220, 0xffffffff, 0x00000100,
1105 	0x3c218, 0xffffffff, 0x06000100,
1106 	0x3c204, 0xffffffff, 0x00000100,
1107 	0x3c2e0, 0xffffffff, 0x00000100,
1108 	0x3c224, 0xffffffff, 0x00000100,
1109 	0x3c200, 0xffffffff, 0x00000100,
1110 	0x3c230, 0xffffffff, 0x00000100,
1111 	0x3c234, 0xffffffff, 0x00000100,
1112 	0x3c250, 0xffffffff, 0x00000100,
1113 	0x3c254, 0xffffffff, 0x00000100,
1114 	0x3c258, 0xffffffff, 0x00000100,
1115 	0x3c25c, 0xffffffff, 0x00000100,
1116 	0x3c260, 0xffffffff, 0x00000100,
1117 	0x3c27c, 0xffffffff, 0x00000100,
1118 	0x3c278, 0xffffffff, 0x00000100,
1119 	0x3c210, 0xffffffff, 0x06000100,
1120 	0x3c290, 0xffffffff, 0x00000100,
1121 	0x3c274, 0xffffffff, 0x00000100,
1122 	0x3c2b4, 0xffffffff, 0x00000100,
1123 	0x3c2b0, 0xffffffff, 0x00000100,
1124 	0x3c270, 0xffffffff, 0x00000100,
1125 	0x30800, 0xffffffff, 0xe0000000,
1126 	0x3c020, 0xffffffff, 0x00010000,
1127 	0x3c024, 0xffffffff, 0x00030002,
1128 	0x3c028, 0xffffffff, 0x00040007,
1129 	0x3c02c, 0xffffffff, 0x00060005,
1130 	0x3c030, 0xffffffff, 0x00090008,
1131 	0x3c034, 0xffffffff, 0x00010000,
1132 	0x3c038, 0xffffffff, 0x00030002,
1133 	0x3c03c, 0xffffffff, 0x00040007,
1134 	0x3c040, 0xffffffff, 0x00060005,
1135 	0x3c044, 0xffffffff, 0x00090008,
1136 	0x3c048, 0xffffffff, 0x00010000,
1137 	0x3c04c, 0xffffffff, 0x00030002,
1138 	0x3c050, 0xffffffff, 0x00040007,
1139 	0x3c054, 0xffffffff, 0x00060005,
1140 	0x3c058, 0xffffffff, 0x00090008,
1141 	0x3c05c, 0xffffffff, 0x00010000,
1142 	0x3c060, 0xffffffff, 0x00030002,
1143 	0x3c064, 0xffffffff, 0x00040007,
1144 	0x3c068, 0xffffffff, 0x00060005,
1145 	0x3c06c, 0xffffffff, 0x00090008,
1146 	0x3c070, 0xffffffff, 0x00010000,
1147 	0x3c074, 0xffffffff, 0x00030002,
1148 	0x3c078, 0xffffffff, 0x00040007,
1149 	0x3c07c, 0xffffffff, 0x00060005,
1150 	0x3c080, 0xffffffff, 0x00090008,
1151 	0x3c084, 0xffffffff, 0x00010000,
1152 	0x3c088, 0xffffffff, 0x00030002,
1153 	0x3c08c, 0xffffffff, 0x00040007,
1154 	0x3c090, 0xffffffff, 0x00060005,
1155 	0x3c094, 0xffffffff, 0x00090008,
1156 	0x3c098, 0xffffffff, 0x00010000,
1157 	0x3c09c, 0xffffffff, 0x00030002,
1158 	0x3c0a0, 0xffffffff, 0x00040007,
1159 	0x3c0a4, 0xffffffff, 0x00060005,
1160 	0x3c0a8, 0xffffffff, 0x00090008,
1161 	0x3c000, 0xffffffff, 0x96e00200,
1162 	0x8708, 0xffffffff, 0x00900100,
1163 	0xc424, 0xffffffff, 0x0020003f,
1164 	0x38, 0xffffffff, 0x0140001c,
1165 	0x3c, 0x000f0000, 0x000f0000,
1166 	0x220, 0xffffffff, 0xC060000C,
1167 	0x224, 0xc0000fff, 0x00000100,
1168 	0xf90, 0xffffffff, 0x00000100,
1169 	0xf98, 0x00000101, 0x00000000,
1170 	0x20a8, 0xffffffff, 0x00000104,
1171 	0x55e4, 0xff000fff, 0x00000100,
1172 	0x30cc, 0xc0000fff, 0x00000104,
1173 	0xc1e4, 0x00000001, 0x00000001,
1174 	0xd00c, 0xff000ff0, 0x00000100,
1175 	0xd80c, 0xff000ff0, 0x00000100
1180 	0x30800, 0xe0ffffff, 0xe0000000
1185 	0xc770, 0xffffffff, 0x00000800,
1186 	0xc774, 0xffffffff, 0x00000800,
1187 	0xc798, 0xffffffff, 0x00007fbf,
1188 	0xc79c, 0xffffffff, 0x00007faf
1193 	0x3c000, 0xffff1fff, 0x96940200,
1194 	0x3c00c, 0xffff0001, 0xff000000,
1195 	0x3c200, 0xfffc0fff, 0x00000100,
1196 	0x6ed8, 0x00010101, 0x00010000,
1197 	0x9834, 0xf00fffff, 0x00000400,
1198 	0x9838, 0xfffffffc, 0x00020200,
1199 	0x5bb0, 0x000000f0, 0x00000070,
1200 	0x5bc0, 0xf0311fff, 0x80300000,
1201 	0x98f8, 0x73773777, 0x12010001,
1202 	0x9b7c, 0x00ff0000, 0x00fc0000,
1203 	0x2f48, 0x73773777, 0x12010001,
1204 	0x8a14, 0xf000003f, 0x00000007,
1205 	0x8b24, 0xffffffff, 0x00ffffff,
1206 	0x28350, 0x3f3f3fff, 0x00000082,
1207 	0x28354, 0x0000003f, 0x00000000,
1208 	0x3e78, 0x00000001, 0x00000002,
1209 	0x913c, 0xffff03df, 0x00000004,
1210 	0xc768, 0x00000008, 0x00000008,
1211 	0x8c00, 0x000008ff, 0x00000800,
1212 	0x9508, 0x00010000, 0x00010000,
1213 	0xac0c, 0xffffffff, 0x54763210,
1214 	0x214f8, 0x01ff01ff, 0x00000002,
1215 	0x21498, 0x007ff800, 0x00200000,
1216 	0x2015c, 0xffffffff, 0x00000f40,
1217 	0x30934, 0xffffffff, 0x00000001
1222 	0xc420, 0xffffffff, 0xfffffffc,
1223 	0x30800, 0xffffffff, 0xe0000000,
1224 	0x3c2a0, 0xffffffff, 0x00000100,
1225 	0x3c208, 0xffffffff, 0x00000100,
1226 	0x3c2c0, 0xffffffff, 0x00000100,
1227 	0x3c2c8, 0xffffffff, 0x00000100,
1228 	0x3c2c4, 0xffffffff, 0x00000100,
1229 	0x55e4, 0xffffffff, 0x00600100,
1230 	0x3c280, 0xffffffff, 0x00000100,
1231 	0x3c214, 0xffffffff, 0x06000100,
1232 	0x3c220, 0xffffffff, 0x00000100,
1233 	0x3c218, 0xffffffff, 0x06000100,
1234 	0x3c204, 0xffffffff, 0x00000100,
1235 	0x3c2e0, 0xffffffff, 0x00000100,
1236 	0x3c224, 0xffffffff, 0x00000100,
1237 	0x3c200, 0xffffffff, 0x00000100,
1238 	0x3c230, 0xffffffff, 0x00000100,
1239 	0x3c234, 0xffffffff, 0x00000100,
1240 	0x3c250, 0xffffffff, 0x00000100,
1241 	0x3c254, 0xffffffff, 0x00000100,
1242 	0x3c258, 0xffffffff, 0x00000100,
1243 	0x3c25c, 0xffffffff, 0x00000100,
1244 	0x3c260, 0xffffffff, 0x00000100,
1245 	0x3c27c, 0xffffffff, 0x00000100,
1246 	0x3c278, 0xffffffff, 0x00000100,
1247 	0x3c210, 0xffffffff, 0x06000100,
1248 	0x3c290, 0xffffffff, 0x00000100,
1249 	0x3c274, 0xffffffff, 0x00000100,
1250 	0x3c2b4, 0xffffffff, 0x00000100,
1251 	0x3c2b0, 0xffffffff, 0x00000100,
1252 	0x3c270, 0xffffffff, 0x00000100,
1253 	0x30800, 0xffffffff, 0xe0000000,
1254 	0x3c020, 0xffffffff, 0x00010000,
1255 	0x3c024, 0xffffffff, 0x00030002,
1256 	0x3c028, 0xffffffff, 0x00040007,
1257 	0x3c02c, 0xffffffff, 0x00060005,
1258 	0x3c030, 0xffffffff, 0x00090008,
1259 	0x3c034, 0xffffffff, 0x00010000,
1260 	0x3c038, 0xffffffff, 0x00030002,
1261 	0x3c03c, 0xffffffff, 0x00040007,
1262 	0x3c040, 0xffffffff, 0x00060005,
1263 	0x3c044, 0xffffffff, 0x00090008,
1264 	0x3c048, 0xffffffff, 0x00010000,
1265 	0x3c04c, 0xffffffff, 0x00030002,
1266 	0x3c050, 0xffffffff, 0x00040007,
1267 	0x3c054, 0xffffffff, 0x00060005,
1268 	0x3c058, 0xffffffff, 0x00090008,
1269 	0x3c05c, 0xffffffff, 0x00010000,
1270 	0x3c060, 0xffffffff, 0x00030002,
1271 	0x3c064, 0xffffffff, 0x00040007,
1272 	0x3c068, 0xffffffff, 0x00060005,
1273 	0x3c06c, 0xffffffff, 0x00090008,
1274 	0x3c070, 0xffffffff, 0x00010000,
1275 	0x3c074, 0xffffffff, 0x00030002,
1276 	0x3c078, 0xffffffff, 0x00040007,
1277 	0x3c07c, 0xffffffff, 0x00060005,
1278 	0x3c080, 0xffffffff, 0x00090008,
1279 	0x3c084, 0xffffffff, 0x00010000,
1280 	0x3c088, 0xffffffff, 0x00030002,
1281 	0x3c08c, 0xffffffff, 0x00040007,
1282 	0x3c090, 0xffffffff, 0x00060005,
1283 	0x3c094, 0xffffffff, 0x00090008,
1284 	0x3c098, 0xffffffff, 0x00010000,
1285 	0x3c09c, 0xffffffff, 0x00030002,
1286 	0x3c0a0, 0xffffffff, 0x00040007,
1287 	0x3c0a4, 0xffffffff, 0x00060005,
1288 	0x3c0a8, 0xffffffff, 0x00090008,
1289 	0x3c0ac, 0xffffffff, 0x00010000,
1290 	0x3c0b0, 0xffffffff, 0x00030002,
1291 	0x3c0b4, 0xffffffff, 0x00040007,
1292 	0x3c0b8, 0xffffffff, 0x00060005,
1293 	0x3c0bc, 0xffffffff, 0x00090008,
1294 	0x3c000, 0xffffffff, 0x96e00200,
1295 	0x8708, 0xffffffff, 0x00900100,
1296 	0xc424, 0xffffffff, 0x0020003f,
1297 	0x38, 0xffffffff, 0x0140001c,
1298 	0x3c, 0x000f0000, 0x000f0000,
1299 	0x220, 0xffffffff, 0xC060000C,
1300 	0x224, 0xc0000fff, 0x00000100,
1301 	0xf90, 0xffffffff, 0x00000100,
1302 	0xf98, 0x00000101, 0x00000000,
1303 	0x20a8, 0xffffffff, 0x00000104,
1304 	0x55e4, 0xff000fff, 0x00000100,
1305 	0x30cc, 0xc0000fff, 0x00000104,
1306 	0xc1e4, 0x00000001, 0x00000001,
1307 	0xd00c, 0xff000ff0, 0x00000100,
1308 	0xd80c, 0xff000ff0, 0x00000100
1313 	0x30800, 0xe0ffffff, 0xe0000000
1318 	0xc770, 0xffffffff, 0x00000800,
1319 	0xc774, 0xffffffff, 0x00000800,
1320 	0xc798, 0xffffffff, 0x00007fbf,
1321 	0xc79c, 0xffffffff, 0x00007faf
1326 	0x3c000, 0xffffdfff, 0x6e944040,
1327 	0x55e4, 0xff607fff, 0xfc000100,
1328 	0x3c220, 0xff000fff, 0x00000100,
1329 	0x3c224, 0xff000fff, 0x00000100,
1330 	0x3c200, 0xfffc0fff, 0x00000100,
1331 	0x6ed8, 0x00010101, 0x00010000,
1332 	0x9830, 0xffffffff, 0x00000000,
1333 	0x9834, 0xf00fffff, 0x00000400,
1334 	0x5bb0, 0x000000f0, 0x00000070,
1335 	0x5bc0, 0xf0311fff, 0x80300000,
1336 	0x98f8, 0x73773777, 0x12010001,
1337 	0x98fc, 0xffffffff, 0x00000010,
1338 	0x9b7c, 0x00ff0000, 0x00fc0000,
1339 	0x8030, 0x00001f0f, 0x0000100a,
1340 	0x2f48, 0x73773777, 0x12010001,
1341 	0x2408, 0x000fffff, 0x000c007f,
1342 	0x8a14, 0xf000003f, 0x00000007,
1343 	0x8b24, 0x3fff3fff, 0x00ffcfff,
1344 	0x30a04, 0x0000ff0f, 0x00000000,
1345 	0x28a4c, 0x07ffffff, 0x06000000,
1346 	0x4d8, 0x00000fff, 0x00000100,
1347 	0x3e78, 0x00000001, 0x00000002,
1348 	0xc768, 0x00000008, 0x00000008,
1349 	0x8c00, 0x000000ff, 0x00000003,
1350 	0x214f8, 0x01ff01ff, 0x00000002,
1351 	0x21498, 0x007ff800, 0x00200000,
1352 	0x2015c, 0xffffffff, 0x00000f40,
1353 	0x88c4, 0x001f3ae3, 0x00000082,
1354 	0x88d4, 0x0000001f, 0x00000010,
1355 	0x30934, 0xffffffff, 0x00000000
1360 	0xc420, 0xffffffff, 0xfffffffc,
1361 	0x30800, 0xffffffff, 0xe0000000,
1362 	0x3c2a0, 0xffffffff, 0x00000100,
1363 	0x3c208, 0xffffffff, 0x00000100,
1364 	0x3c2c0, 0xffffffff, 0x00000100,
1365 	0x3c2c8, 0xffffffff, 0x00000100,
1366 	0x3c2c4, 0xffffffff, 0x00000100,
1367 	0x55e4, 0xffffffff, 0x00600100,
1368 	0x3c280, 0xffffffff, 0x00000100,
1369 	0x3c214, 0xffffffff, 0x06000100,
1370 	0x3c220, 0xffffffff, 0x00000100,
1371 	0x3c218, 0xffffffff, 0x06000100,
1372 	0x3c204, 0xffffffff, 0x00000100,
1373 	0x3c2e0, 0xffffffff, 0x00000100,
1374 	0x3c224, 0xffffffff, 0x00000100,
1375 	0x3c200, 0xffffffff, 0x00000100,
1376 	0x3c230, 0xffffffff, 0x00000100,
1377 	0x3c234, 0xffffffff, 0x00000100,
1378 	0x3c250, 0xffffffff, 0x00000100,
1379 	0x3c254, 0xffffffff, 0x00000100,
1380 	0x3c258, 0xffffffff, 0x00000100,
1381 	0x3c25c, 0xffffffff, 0x00000100,
1382 	0x3c260, 0xffffffff, 0x00000100,
1383 	0x3c27c, 0xffffffff, 0x00000100,
1384 	0x3c278, 0xffffffff, 0x00000100,
1385 	0x3c210, 0xffffffff, 0x06000100,
1386 	0x3c290, 0xffffffff, 0x00000100,
1387 	0x3c274, 0xffffffff, 0x00000100,
1388 	0x3c2b4, 0xffffffff, 0x00000100,
1389 	0x3c2b0, 0xffffffff, 0x00000100,
1390 	0x3c270, 0xffffffff, 0x00000100,
1391 	0x30800, 0xffffffff, 0xe0000000,
1392 	0x3c020, 0xffffffff, 0x00010000,
1393 	0x3c024, 0xffffffff, 0x00030002,
1394 	0x3c028, 0xffffffff, 0x00040007,
1395 	0x3c02c, 0xffffffff, 0x00060005,
1396 	0x3c030, 0xffffffff, 0x00090008,
1397 	0x3c034, 0xffffffff, 0x00010000,
1398 	0x3c038, 0xffffffff, 0x00030002,
1399 	0x3c03c, 0xffffffff, 0x00040007,
1400 	0x3c040, 0xffffffff, 0x00060005,
1401 	0x3c044, 0xffffffff, 0x00090008,
1402 	0x3c000, 0xffffffff, 0x96e00200,
1403 	0x8708, 0xffffffff, 0x00900100,
1404 	0xc424, 0xffffffff, 0x0020003f,
1405 	0x38, 0xffffffff, 0x0140001c,
1406 	0x3c, 0x000f0000, 0x000f0000,
1407 	0x220, 0xffffffff, 0xC060000C,
1408 	0x224, 0xc0000fff, 0x00000100,
1409 	0x20a8, 0xffffffff, 0x00000104,
1410 	0x55e4, 0xff000fff, 0x00000100,
1411 	0x30cc, 0xc0000fff, 0x00000104,
1412 	0xc1e4, 0x00000001, 0x00000001,
1413 	0xd00c, 0xff000ff0, 0x00000100,
1414 	0xd80c, 0xff000ff0, 0x00000100
1419 	0x30800, 0xe0ffffff, 0xe0000000
1424 	0x30800, 0xffffffff, 0xe0000000,
1425 	0x28350, 0xffffffff, 0x3a00161a,
1426 	0x28354, 0xffffffff, 0x0000002e,
1427 	0x9a10, 0xffffffff, 0x00018208,
1428 	0x98f8, 0xffffffff, 0x12011003
1433 	0x3354, 0x00000333, 0x00000333,
1434 	0x9a10, 0x00010000, 0x00058208,
1435 	0x9830, 0xffffffff, 0x00000000,
1436 	0x9834, 0xf00fffff, 0x00000400,
1437 	0x9838, 0x0002021c, 0x00020200,
1438 	0xc78, 0x00000080, 0x00000000,
1439 	0x5bb0, 0x000000f0, 0x00000070,
1440 	0x5bc0, 0xf0311fff, 0x80300000,
1441 	0x350c, 0x00810000, 0x408af000,
1442 	0x7030, 0x31000111, 0x00000011,
1443 	0x2f48, 0x73773777, 0x12010001,
1444 	0x2120, 0x0000007f, 0x0000001b,
1445 	0x21dc, 0x00007fb6, 0x00002191,
1446 	0x3628, 0x0000003f, 0x0000000a,
1447 	0x362c, 0x0000003f, 0x0000000a,
1448 	0x2ae4, 0x00073ffe, 0x000022a2,
1449 	0x240c, 0x000007ff, 0x00000000,
1450 	0x8bf0, 0x00002001, 0x00000001,
1451 	0x8b24, 0xffffffff, 0x00ffffff,
1452 	0x30a04, 0x0000ff0f, 0x00000000,
1453 	0x28a4c, 0x07ffffff, 0x06000000,
1454 	0x3e78, 0x00000001, 0x00000002,
1455 	0xc768, 0x00000008, 0x00000008,
1456 	0xc770, 0x00000f00, 0x00000800,
1457 	0xc774, 0x00000f00, 0x00000800,
1458 	0xc798, 0x00ffffff, 0x00ff7fbf,
1459 	0xc79c, 0x00ffffff, 0x00ff7faf,
1460 	0x8c00, 0x000000ff, 0x00000800,
1461 	0xe40, 0x00001fff, 0x00001fff,
1462 	0x9060, 0x0000007f, 0x00000020,
1463 	0x9508, 0x00010000, 0x00010000,
1464 	0xae00, 0x00100000, 0x000ff07c,
1465 	0xac14, 0x000003ff, 0x0000000f,
1466 	0xac10, 0xffffffff, 0x7564fdec,
1467 	0xac0c, 0xffffffff, 0x3120b9a8,
1468 	0xac08, 0x20000000, 0x0f9c0000
1473 	0xc420, 0xffffffff, 0xfffffffd,
1474 	0x30800, 0xffffffff, 0xe0000000,
1475 	0x3c2a0, 0xffffffff, 0x00000100,
1476 	0x3c208, 0xffffffff, 0x00000100,
1477 	0x3c2c0, 0xffffffff, 0x00000100,
1478 	0x3c2c8, 0xffffffff, 0x00000100,
1479 	0x3c2c4, 0xffffffff, 0x00000100,
1480 	0x55e4, 0xffffffff, 0x00200100,
1481 	0x3c280, 0xffffffff, 0x00000100,
1482 	0x3c214, 0xffffffff, 0x06000100,
1483 	0x3c220, 0xffffffff, 0x00000100,
1484 	0x3c218, 0xffffffff, 0x06000100,
1485 	0x3c204, 0xffffffff, 0x00000100,
1486 	0x3c2e0, 0xffffffff, 0x00000100,
1487 	0x3c224, 0xffffffff, 0x00000100,
1488 	0x3c200, 0xffffffff, 0x00000100,
1489 	0x3c230, 0xffffffff, 0x00000100,
1490 	0x3c234, 0xffffffff, 0x00000100,
1491 	0x3c250, 0xffffffff, 0x00000100,
1492 	0x3c254, 0xffffffff, 0x00000100,
1493 	0x3c258, 0xffffffff, 0x00000100,
1494 	0x3c25c, 0xffffffff, 0x00000100,
1495 	0x3c260, 0xffffffff, 0x00000100,
1496 	0x3c27c, 0xffffffff, 0x00000100,
1497 	0x3c278, 0xffffffff, 0x00000100,
1498 	0x3c210, 0xffffffff, 0x06000100,
1499 	0x3c290, 0xffffffff, 0x00000100,
1500 	0x3c274, 0xffffffff, 0x00000100,
1501 	0x3c2b4, 0xffffffff, 0x00000100,
1502 	0x3c2b0, 0xffffffff, 0x00000100,
1503 	0x3c270, 0xffffffff, 0x00000100,
1504 	0x30800, 0xffffffff, 0xe0000000,
1505 	0x3c020, 0xffffffff, 0x00010000,
1506 	0x3c024, 0xffffffff, 0x00030002,
1507 	0x3c028, 0xffffffff, 0x00040007,
1508 	0x3c02c, 0xffffffff, 0x00060005,
1509 	0x3c030, 0xffffffff, 0x00090008,
1510 	0x3c034, 0xffffffff, 0x00010000,
1511 	0x3c038, 0xffffffff, 0x00030002,
1512 	0x3c03c, 0xffffffff, 0x00040007,
1513 	0x3c040, 0xffffffff, 0x00060005,
1514 	0x3c044, 0xffffffff, 0x00090008,
1515 	0x3c048, 0xffffffff, 0x00010000,
1516 	0x3c04c, 0xffffffff, 0x00030002,
1517 	0x3c050, 0xffffffff, 0x00040007,
1518 	0x3c054, 0xffffffff, 0x00060005,
1519 	0x3c058, 0xffffffff, 0x00090008,
1520 	0x3c05c, 0xffffffff, 0x00010000,
1521 	0x3c060, 0xffffffff, 0x00030002,
1522 	0x3c064, 0xffffffff, 0x00040007,
1523 	0x3c068, 0xffffffff, 0x00060005,
1524 	0x3c06c, 0xffffffff, 0x00090008,
1525 	0x3c070, 0xffffffff, 0x00010000,
1526 	0x3c074, 0xffffffff, 0x00030002,
1527 	0x3c078, 0xffffffff, 0x00040007,
1528 	0x3c07c, 0xffffffff, 0x00060005,
1529 	0x3c080, 0xffffffff, 0x00090008,
1530 	0x3c084, 0xffffffff, 0x00010000,
1531 	0x3c088, 0xffffffff, 0x00030002,
1532 	0x3c08c, 0xffffffff, 0x00040007,
1533 	0x3c090, 0xffffffff, 0x00060005,
1534 	0x3c094, 0xffffffff, 0x00090008,
1535 	0x3c098, 0xffffffff, 0x00010000,
1536 	0x3c09c, 0xffffffff, 0x00030002,
1537 	0x3c0a0, 0xffffffff, 0x00040007,
1538 	0x3c0a4, 0xffffffff, 0x00060005,
1539 	0x3c0a8, 0xffffffff, 0x00090008,
1540 	0x3c0ac, 0xffffffff, 0x00010000,
1541 	0x3c0b0, 0xffffffff, 0x00030002,
1542 	0x3c0b4, 0xffffffff, 0x00040007,
1543 	0x3c0b8, 0xffffffff, 0x00060005,
1544 	0x3c0bc, 0xffffffff, 0x00090008,
1545 	0x3c0c0, 0xffffffff, 0x00010000,
1546 	0x3c0c4, 0xffffffff, 0x00030002,
1547 	0x3c0c8, 0xffffffff, 0x00040007,
1548 	0x3c0cc, 0xffffffff, 0x00060005,
1549 	0x3c0d0, 0xffffffff, 0x00090008,
1550 	0x3c0d4, 0xffffffff, 0x00010000,
1551 	0x3c0d8, 0xffffffff, 0x00030002,
1552 	0x3c0dc, 0xffffffff, 0x00040007,
1553 	0x3c0e0, 0xffffffff, 0x00060005,
1554 	0x3c0e4, 0xffffffff, 0x00090008,
1555 	0x3c0e8, 0xffffffff, 0x00010000,
1556 	0x3c0ec, 0xffffffff, 0x00030002,
1557 	0x3c0f0, 0xffffffff, 0x00040007,
1558 	0x3c0f4, 0xffffffff, 0x00060005,
1559 	0x3c0f8, 0xffffffff, 0x00090008,
1560 	0xc318, 0xffffffff, 0x00020200,
1561 	0x3350, 0xffffffff, 0x00000200,
1562 	0x15c0, 0xffffffff, 0x00000400,
1563 	0x55e8, 0xffffffff, 0x00000000,
1564 	0x2f50, 0xffffffff, 0x00000902,
1565 	0x3c000, 0xffffffff, 0x96940200,
1566 	0x8708, 0xffffffff, 0x00900100,
1567 	0xc424, 0xffffffff, 0x0020003f,
1568 	0x38, 0xffffffff, 0x0140001c,
1569 	0x3c, 0x000f0000, 0x000f0000,
1570 	0x220, 0xffffffff, 0xc060000c,
1571 	0x224, 0xc0000fff, 0x00000100,
1572 	0xf90, 0xffffffff, 0x00000100,
1573 	0xf98, 0x00000101, 0x00000000,
1574 	0x20a8, 0xffffffff, 0x00000104,
1575 	0x55e4, 0xff000fff, 0x00000100,
1576 	0x30cc, 0xc0000fff, 0x00000104,
1577 	0xc1e4, 0x00000001, 0x00000001,
1578 	0xd00c, 0xff000ff0, 0x00000100,
1579 	0xd80c, 0xff000ff0, 0x00000100
1584 	0x55e4, 0xff607fff, 0xfc000100,
1585 	0x6ed8, 0x00010101, 0x00010000,
1586 	0x9830, 0xffffffff, 0x00000000,
1587 	0x98302, 0xf00fffff, 0x00000400,
1588 	0x6130, 0xffffffff, 0x00010000,
1589 	0x5bb0, 0x000000f0, 0x00000070,
1590 	0x5bc0, 0xf0311fff, 0x80300000,
1591 	0x98f8, 0x73773777, 0x12010001,
1592 	0x98fc, 0xffffffff, 0x00000010,
1593 	0x8030, 0x00001f0f, 0x0000100a,
1594 	0x2f48, 0x73773777, 0x12010001,
1595 	0x2408, 0x000fffff, 0x000c007f,
1596 	0x8a14, 0xf000003f, 0x00000007,
1597 	0x8b24, 0xffffffff, 0x00ff0fff,
1598 	0x30a04, 0x0000ff0f, 0x00000000,
1599 	0x28a4c, 0x07ffffff, 0x06000000,
1600 	0x4d8, 0x00000fff, 0x00000100,
1601 	0xd014, 0x00010000, 0x00810001,
1602 	0xd814, 0x00010000, 0x00810001,
1603 	0x3e78, 0x00000001, 0x00000002,
1604 	0xc768, 0x00000008, 0x00000008,
1605 	0xc770, 0x00000f00, 0x00000800,
1606 	0xc774, 0x00000f00, 0x00000800,
1607 	0xc798, 0x00ffffff, 0x00ff7fbf,
1608 	0xc79c, 0x00ffffff, 0x00ff7faf,
1609 	0x8c00, 0x000000ff, 0x00000001,
1610 	0x214f8, 0x01ff01ff, 0x00000002,
1611 	0x21498, 0x007ff800, 0x00200000,
1612 	0x2015c, 0xffffffff, 0x00000f40,
1613 	0x88c4, 0x001f3ae3, 0x00000082,
1614 	0x88d4, 0x0000001f, 0x00000010,
1615 	0x30934, 0xffffffff, 0x00000000
1621 	switch (rdev->family) {  in cik_init_golden_registers()
1698  * cik_get_xclk - get the xclk
1707 	u32 reference_clock = rdev->clock.spll.reference_freq;  in cik_get_xclk()
1709 	if (rdev->flags & RADEON_IS_IGP) {  in cik_get_xclk()
1720  * cik_mm_rdoorbell - read a doorbell dword
1730 	if (index < rdev->doorbell.num_doorbells) {  in cik_mm_rdoorbell()
1731 		return readl(rdev->doorbell.ptr + index);  in cik_mm_rdoorbell()
1733 		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);  in cik_mm_rdoorbell()
1734 		return 0;  in cik_mm_rdoorbell()
1739  * cik_mm_wdoorbell - write a doorbell dword
1750 	if (index < rdev->doorbell.num_doorbells) {  in cik_mm_wdoorbell()
1751 		writel(v, rdev->doorbell.ptr + index);  in cik_mm_wdoorbell()
1753 		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);  in cik_mm_wdoorbell()
1761 	{0x00000070, 0x04400000},
1762 	{0x00000071, 0x80c01803},
1763 	{0x00000072, 0x00004004},
1764 	{0x00000073, 0x00000100},
1765 	{0x00000074, 0x00ff0000},
1766 	{0x00000075, 0x34000000},
1767 	{0x00000076, 0x08000014},
1768 	{0x00000077, 0x00cc08ec},
1769 	{0x00000078, 0x00000400},
1770 	{0x00000079, 0x00000000},
1771 	{0x0000007a, 0x04090000},
1772 	{0x0000007c, 0x00000000},
1773 	{0x0000007e, 0x4408a8e8},
1774 	{0x0000007f, 0x00000304},
1775 	{0x00000080, 0x00000000},
1776 	{0x00000082, 0x00000001},
1777 	{0x00000083, 0x00000002},
1778 	{0x00000084, 0xf3e4f400},
1779 	{0x00000085, 0x052024e3},
1780 	{0x00000087, 0x00000000},
1781 	{0x00000088, 0x01000000},
1782 	{0x0000008a, 0x1c0a0000},
1783 	{0x0000008b, 0xff010000},
1784 	{0x0000008d, 0xffffefff},
1785 	{0x0000008e, 0xfff3efff},
1786 	{0x0000008f, 0xfff3efbf},
1787 	{0x00000092, 0xf7ffffff},
1788 	{0x00000093, 0xffffff7f},
1789 	{0x00000095, 0x00101101},
1790 	{0x00000096, 0x00000fff},
1791 	{0x00000097, 0x00116fff},
1792 	{0x00000098, 0x60010000},
1793 	{0x00000099, 0x10010000},
1794 	{0x0000009a, 0x00006000},
1795 	{0x0000009b, 0x00001000},
1796 	{0x0000009f, 0x00b48000}
1803 	{0x0000007d, 0x40000000},
1804 	{0x0000007e, 0x40180304},
1805 	{0x0000007f, 0x0000ff00},
1806 	{0x00000081, 0x00000000},
1807 	{0x00000083, 0x00000800},
1808 	{0x00000086, 0x00000000},
1809 	{0x00000087, 0x00000100},
1810 	{0x00000088, 0x00020100},
1811 	{0x00000089, 0x00000000},
1812 	{0x0000008b, 0x00040000},
1813 	{0x0000008c, 0x00000100},
1814 	{0x0000008e, 0xff010000},
1815 	{0x00000090, 0xffffefff},
1816 	{0x00000091, 0xfff3efff},
1817 	{0x00000092, 0xfff3efbf},
1818 	{0x00000093, 0xf7ffffff},
1819 	{0x00000094, 0xffffff7f},
1820 	{0x00000095, 0x00000fff},
1821 	{0x00000096, 0x00116fff},
1822 	{0x00000097, 0x60010000},
1823 	{0x00000098, 0x10010000},
1824 	{0x0000009f, 0x00c79000}
1829  * cik_srbm_select - select specific register instances
1844 	u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |  in cik_srbm_select()
1845 			     MEID(me & 0x3) |  in cik_srbm_select()
1846 			     VMID(vmid & 0xf) |  in cik_srbm_select()
1847 			     QUEUEID(queue & 0x7));  in cik_srbm_select()
1853  * ci_mc_load_microcode - load MC ucode into the hw
1858  * Returns 0 on success, error on failure.
1869 	if (!rdev->mc_fw)  in ci_mc_load_microcode()
1870 		return -EINVAL;  in ci_mc_load_microcode()
1872 	if (rdev->new_fw) {  in ci_mc_load_microcode()
1874 			(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;  in ci_mc_load_microcode()
1876 		radeon_ucode_print_mc_hdr(&hdr->header);  in ci_mc_load_microcode()
1878 		regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);  in ci_mc_load_microcode()
1880 			(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));  in ci_mc_load_microcode()
1881 		ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;  in ci_mc_load_microcode()
1883 			(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in ci_mc_load_microcode()
1885 		ucode_size = rdev->mc_fw->size / 4;  in ci_mc_load_microcode()
1887 		switch (rdev->family) {  in ci_mc_load_microcode()
1897 			return -EINVAL;  in ci_mc_load_microcode()
1899 		fw_data = (const __be32 *)rdev->mc_fw->data;  in ci_mc_load_microcode()
1904 	if (running == 0) {  in ci_mc_load_microcode()
1906 		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);  in ci_mc_load_microcode()
1907 		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);  in ci_mc_load_microcode()
1910 		for (i = 0; i < regs_size; i++) {  in ci_mc_load_microcode()
1911 			if (rdev->new_fw) {  in ci_mc_load_microcode()
1921 		if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {  in ci_mc_load_microcode()
1923 			WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);  in ci_mc_load_microcode()
1925 			WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);  in ci_mc_load_microcode()
1929 		for (i = 0; i < ucode_size; i++) {  in ci_mc_load_microcode()
1930 			if (rdev->new_fw)  in ci_mc_load_microcode()
1937 		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);  in ci_mc_load_microcode()
1938 		WREG32(MC_SEQ_SUP_CNTL, 0x00000004);  in ci_mc_load_microcode()
1939 		WREG32(MC_SEQ_SUP_CNTL, 0x00000001);  in ci_mc_load_microcode()
1942 		for (i = 0; i < rdev->usec_timeout; i++) {  in ci_mc_load_microcode()
1947 		for (i = 0; i < rdev->usec_timeout; i++) {  in ci_mc_load_microcode()
1954 	return 0;  in ci_mc_load_microcode()
1958  * cik_init_microcode - load ucode images from disk
1964  * Returns 0 on success, error on failure.
1971 		mec_req_size, rlc_req_size, mc_req_size = 0,  in cik_init_microcode()
1972 		sdma_req_size, smc_req_size = 0, mc2_req_size = 0;  in cik_init_microcode()
1974 	int new_fw = 0;  in cik_init_microcode()
1981 	switch (rdev->family) {  in cik_init_microcode()
1984 		if ((rdev->pdev->revision == 0x80) ||  in cik_init_microcode()
1985 		    (rdev->pdev->revision == 0x81) ||  in cik_init_microcode()
1986 		    (rdev->pdev->device == 0x665f))  in cik_init_microcode()
2002 		if (rdev->pdev->revision == 0x80)  in cik_init_microcode()
2055 	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);  in cik_init_microcode()
2058 		err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);  in cik_init_microcode()
2061 		if (rdev->pfp_fw->size != pfp_req_size) {  in cik_init_microcode()
2063 			       rdev->pfp_fw->size, fw_name);  in cik_init_microcode()
2064 			err = -EINVAL;  in cik_init_microcode()
2068 		err = radeon_ucode_validate(rdev->pfp_fw);  in cik_init_microcode()
2079 	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);  in cik_init_microcode()
2082 		err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);  in cik_init_microcode()
2085 		if (rdev->me_fw->size != me_req_size) {  in cik_init_microcode()
2087 			       rdev->me_fw->size, fw_name);  in cik_init_microcode()
2088 			err = -EINVAL;  in cik_init_microcode()
2091 		err = radeon_ucode_validate(rdev->me_fw);  in cik_init_microcode()
2102 	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);  in cik_init_microcode()
2105 		err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);  in cik_init_microcode()
2108 		if (rdev->ce_fw->size != ce_req_size) {  in cik_init_microcode()
2110 			       rdev->ce_fw->size, fw_name);  in cik_init_microcode()
2111 			err = -EINVAL;  in cik_init_microcode()
2114 		err = radeon_ucode_validate(rdev->ce_fw);  in cik_init_microcode()
2125 	err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);  in cik_init_microcode()
2128 		err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);  in cik_init_microcode()
2131 		if (rdev->mec_fw->size != mec_req_size) {  in cik_init_microcode()
2133 			       rdev->mec_fw->size, fw_name);  in cik_init_microcode()
2134 			err = -EINVAL;  in cik_init_microcode()
2137 		err = radeon_ucode_validate(rdev->mec_fw);  in cik_init_microcode()
2147 	if (rdev->family == CHIP_KAVERI) {  in cik_init_microcode()
2149 		err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);  in cik_init_microcode()
2153 			err = radeon_ucode_validate(rdev->mec2_fw);  in cik_init_microcode()
2163 	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2166 		err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2169 		if (rdev->rlc_fw->size != rlc_req_size) {  in cik_init_microcode()
2171 			       rdev->rlc_fw->size, fw_name);  in cik_init_microcode()
2172 			err = -EINVAL;  in cik_init_microcode()
2175 		err = radeon_ucode_validate(rdev->rlc_fw);  in cik_init_microcode()
2186 	err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);  in cik_init_microcode()
2189 		err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);  in cik_init_microcode()
2192 		if (rdev->sdma_fw->size != sdma_req_size) {  in cik_init_microcode()
2194 			       rdev->sdma_fw->size, fw_name);  in cik_init_microcode()
2195 			err = -EINVAL;  in cik_init_microcode()
2198 		err = radeon_ucode_validate(rdev->sdma_fw);  in cik_init_microcode()
2209 	if (!(rdev->flags & RADEON_IS_IGP)) {  in cik_init_microcode()
2211 		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2214 			err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2217 				err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2221 			if ((rdev->mc_fw->size != mc_req_size) &&  in cik_init_microcode()
2222 			    (rdev->mc_fw->size != mc2_req_size)){  in cik_init_microcode()
2224 				       rdev->mc_fw->size, fw_name);  in cik_init_microcode()
2225 				err = -EINVAL;  in cik_init_microcode()
2227 			DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);  in cik_init_microcode()
2229 			err = radeon_ucode_validate(rdev->mc_fw);  in cik_init_microcode()
2243 		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2246 			err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);  in cik_init_microcode()
2250 				release_firmware(rdev->smc_fw);  in cik_init_microcode()
2251 				rdev->smc_fw = NULL;  in cik_init_microcode()
2252 				err = 0;  in cik_init_microcode()
2253 			} else if (rdev->smc_fw->size != smc_req_size) {  in cik_init_microcode()
2255 				       rdev->smc_fw->size, fw_name);  in cik_init_microcode()
2256 				err = -EINVAL;  in cik_init_microcode()
2259 			err = radeon_ucode_validate(rdev->smc_fw);  in cik_init_microcode()
2270 	if (new_fw == 0) {  in cik_init_microcode()
2271 		rdev->new_fw = false;  in cik_init_microcode()
2274 		err = -EINVAL;  in cik_init_microcode()
2276 		rdev->new_fw = true;  in cik_init_microcode()
2281 		if (err != -EINVAL)  in cik_init_microcode()
2284 		release_firmware(rdev->pfp_fw);  in cik_init_microcode()
2285 		rdev->pfp_fw = NULL;  in cik_init_microcode()
2286 		release_firmware(rdev->me_fw);  in cik_init_microcode()
2287 		rdev->me_fw = NULL;  in cik_init_microcode()
2288 		release_firmware(rdev->ce_fw);  in cik_init_microcode()
2289 		rdev->ce_fw = NULL;  in cik_init_microcode()
2290 		release_firmware(rdev->mec_fw);  in cik_init_microcode()
2291 		rdev->mec_fw = NULL;  in cik_init_microcode()
2292 		release_firmware(rdev->mec2_fw);  in cik_init_microcode()
2293 		rdev->mec2_fw = NULL;  in cik_init_microcode()
2294 		release_firmware(rdev->rlc_fw);  in cik_init_microcode()
2295 		rdev->rlc_fw = NULL;  in cik_init_microcode()
2296 		release_firmware(rdev->sdma_fw);  in cik_init_microcode()
2297 		rdev->sdma_fw = NULL;  in cik_init_microcode()
2298 		release_firmware(rdev->mc_fw);  in cik_init_microcode()
2299 		rdev->mc_fw = NULL;  in cik_init_microcode()
2300 		release_firmware(rdev->smc_fw);  in cik_init_microcode()
2301 		rdev->smc_fw = NULL;  in cik_init_microcode()
2310  * cik_tiling_mode_table_init - init the hw tiling table
2322 	u32 *tile = rdev->config.cik.tile_mode_array;  in cik_tiling_mode_table_init()
2323 	u32 *macrotile = rdev->config.cik.macrotile_mode_array;  in cik_tiling_mode_table_init()
2325 			ARRAY_SIZE(rdev->config.cik.tile_mode_array);  in cik_tiling_mode_table_init()
2327 			ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);  in cik_tiling_mode_table_init()
2330 	u32 num_rbs = rdev->config.cik.max_backends_per_se *  in cik_tiling_mode_table_init()
2331 		rdev->config.cik.max_shader_engines;  in cik_tiling_mode_table_init()
2333 	switch (rdev->config.cik.mem_row_size_in_kb) {  in cik_tiling_mode_table_init()
2346 	num_pipe_configs = rdev->config.cik.max_tile_pipes;  in cik_tiling_mode_table_init()
2350 	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
2351 		tile[reg_offset] = 0;  in cik_tiling_mode_table_init()
2352 	for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
2353 		macrotile[reg_offset] = 0;  in cik_tiling_mode_table_init()
2357 		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |  in cik_tiling_mode_table_init()
2436 		macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |  in cik_tiling_mode_table_init()
2493 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
2495 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
2500 		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |  in cik_tiling_mode_table_init()
2579 		macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |  in cik_tiling_mode_table_init()
2636 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
2638 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
2644 		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |  in cik_tiling_mode_table_init()
2724 		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |  in cik_tiling_mode_table_init()
2804 		macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |  in cik_tiling_mode_table_init()
2861 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
2863 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
2868 		tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |  in cik_tiling_mode_table_init()
2947 		macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |  in cik_tiling_mode_table_init()
3004 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
3006 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)  in cik_tiling_mode_table_init()
3011 		DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);  in cik_tiling_mode_table_init()
3016  * cik_select_se_sh - select which SE, SH to address
3023  * registers are instanced per SE or SH.  0xffffffff means
3031 	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))  in cik_select_se_sh()
3033 	else if (se_num == 0xffffffff)  in cik_select_se_sh()
3035 	else if (sh_num == 0xffffffff)  in cik_select_se_sh()
3043  * cik_create_bitmask - create a bitmask
3052 	u32 i, mask = 0;  in cik_create_bitmask()
3054 	for (i = 0; i < bit_width; i++) {  in cik_create_bitmask()
3062  * cik_get_rb_disabled - computes the mask of disabled RBs
3081 		data = 0;  in cik_get_rb_disabled()
3092  * cik_setup_rb - setup the RBs on the asic
3099  * Configures per-SE/SH RB registers (CIK).
3107 	u32 disabled_rbs = 0;  in cik_setup_rb()
3108 	u32 enabled_rbs = 0;  in cik_setup_rb()
3110 	for (i = 0; i < se_num; i++) {  in cik_setup_rb()
3111 		for (j = 0; j < sh_per_se; j++) {  in cik_setup_rb()
3114 			if (rdev->family == CHIP_HAWAII)  in cik_setup_rb()
3120 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_setup_rb()
3123 	for (i = 0; i < max_rb_num_per_se * se_num; i++) {  in cik_setup_rb()
3129 	rdev->config.cik.backend_enable_mask = enabled_rbs;  in cik_setup_rb()
3131 	for (i = 0; i < se_num; i++) {  in cik_setup_rb()
3132 		cik_select_se_sh(rdev, i, 0xffffffff);  in cik_setup_rb()
3133 		data = 0;  in cik_setup_rb()
3134 		for (j = 0; j < sh_per_se; j++) {  in cik_setup_rb()
3136 			case 0:  in cik_setup_rb()
3137 				if (j == 0)  in cik_setup_rb()
3157 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_setup_rb()
3161  * cik_gpu_init - setup the 3D engine
3176 	switch (rdev->family) {  in cik_gpu_init()
3178 		rdev->config.cik.max_shader_engines = 2;  in cik_gpu_init()
3179 		rdev->config.cik.max_tile_pipes = 4;  in cik_gpu_init()
3180 		rdev->config.cik.max_cu_per_sh = 7;  in cik_gpu_init()
3181 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3182 		rdev->config.cik.max_backends_per_se = 2;  in cik_gpu_init()
3183 		rdev->config.cik.max_texture_channel_caches = 4;  in cik_gpu_init()
3184 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3185 		rdev->config.cik.max_gs_threads = 32;  in cik_gpu_init()
3186 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3188 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3189 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3190 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3191 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3195 		rdev->config.cik.max_shader_engines = 4;  in cik_gpu_init()
3196 		rdev->config.cik.max_tile_pipes = 16;  in cik_gpu_init()
3197 		rdev->config.cik.max_cu_per_sh = 11;  in cik_gpu_init()
3198 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3199 		rdev->config.cik.max_backends_per_se = 4;  in cik_gpu_init()
3200 		rdev->config.cik.max_texture_channel_caches = 16;  in cik_gpu_init()
3201 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3202 		rdev->config.cik.max_gs_threads = 32;  in cik_gpu_init()
3203 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3205 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3206 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3207 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3208 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3212 		rdev->config.cik.max_shader_engines = 1;  in cik_gpu_init()
3213 		rdev->config.cik.max_tile_pipes = 4;  in cik_gpu_init()
3214 		rdev->config.cik.max_cu_per_sh = 8;  in cik_gpu_init()
3215 		rdev->config.cik.max_backends_per_se = 2;  in cik_gpu_init()
3216 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3217 		rdev->config.cik.max_texture_channel_caches = 4;  in cik_gpu_init()
3218 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3219 		rdev->config.cik.max_gs_threads = 16;  in cik_gpu_init()
3220 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3222 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3223 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3224 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3225 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3231 		rdev->config.cik.max_shader_engines = 1;  in cik_gpu_init()
3232 		rdev->config.cik.max_tile_pipes = 2;  in cik_gpu_init()
3233 		rdev->config.cik.max_cu_per_sh = 2;  in cik_gpu_init()
3234 		rdev->config.cik.max_sh_per_se = 1;  in cik_gpu_init()
3235 		rdev->config.cik.max_backends_per_se = 1;  in cik_gpu_init()
3236 		rdev->config.cik.max_texture_channel_caches = 2;  in cik_gpu_init()
3237 		rdev->config.cik.max_gprs = 256;  in cik_gpu_init()
3238 		rdev->config.cik.max_gs_threads = 16;  in cik_gpu_init()
3239 		rdev->config.cik.max_hw_contexts = 8;  in cik_gpu_init()
3241 		rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;  in cik_gpu_init()
3242 		rdev->config.cik.sc_prim_fifo_size_backend = 0x100;  in cik_gpu_init()
3243 		rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;  in cik_gpu_init()
3244 		rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;  in cik_gpu_init()
3250 	for (i = 0, j = 0; i < 32; i++, j += 0x18) {  in cik_gpu_init()
3251 		WREG32((0x2c14 + j), 0x00000000);  in cik_gpu_init()
3252 		WREG32((0x2c18 + j), 0x00000000);  in cik_gpu_init()
3253 		WREG32((0x2c1c + j), 0x00000000);  in cik_gpu_init()
3254 		WREG32((0x2c20 + j), 0x00000000);  in cik_gpu_init()
3255 		WREG32((0x2c24 + j), 0x00000000);  in cik_gpu_init()
3258 	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));  in cik_gpu_init()
3259 	WREG32(SRBM_INT_CNTL, 0x1);  in cik_gpu_init()
3260 	WREG32(SRBM_INT_ACK, 0x1);  in cik_gpu_init()
3267 	rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;  in cik_gpu_init()
3268 	rdev->config.cik.mem_max_burst_length_bytes = 256;  in cik_gpu_init()
3270 	rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;  in cik_gpu_init()
3271 	if (rdev->config.cik.mem_row_size_in_kb > 4)  in cik_gpu_init()
3272 		rdev->config.cik.mem_row_size_in_kb = 4;  in cik_gpu_init()
3274 	rdev->config.cik.shader_engine_tile_size = 32;  in cik_gpu_init()
3275 	rdev->config.cik.num_gpus = 1;  in cik_gpu_init()
3276 	rdev->config.cik.multi_gpu_tile_size = 64;  in cik_gpu_init()
3280 	switch (rdev->config.cik.mem_row_size_in_kb) {  in cik_gpu_init()
3283 		gb_addr_config |= ROW_SIZE(0);  in cik_gpu_init()
3295 	 * bits 3:0   num_pipes  in cik_gpu_init()
3300 	rdev->config.cik.tile_config = 0;  in cik_gpu_init()
3301 	switch (rdev->config.cik.num_tile_pipes) {  in cik_gpu_init()
3303 		rdev->config.cik.tile_config |= (0 << 0);  in cik_gpu_init()
3306 		rdev->config.cik.tile_config |= (1 << 0);  in cik_gpu_init()
3309 		rdev->config.cik.tile_config |= (2 << 0);  in cik_gpu_init()
3314 		rdev->config.cik.tile_config |= (3 << 0);  in cik_gpu_init()
3317 	rdev->config.cik.tile_config |=  in cik_gpu_init()
3319 	rdev->config.cik.tile_config |=  in cik_gpu_init()
3321 	rdev->config.cik.tile_config |=  in cik_gpu_init()
3327 	WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);  in cik_gpu_init()
3328 	WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);  in cik_gpu_init()
3335 	cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,  in cik_gpu_init()
3336 		     rdev->config.cik.max_sh_per_se,  in cik_gpu_init()
3337 		     rdev->config.cik.max_backends_per_se);  in cik_gpu_init()
3339 	rdev->config.cik.active_cus = 0;  in cik_gpu_init()
3340 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {  in cik_gpu_init()
3341 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {  in cik_gpu_init()
3342 			rdev->config.cik.active_cus +=  in cik_gpu_init()
3348 	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));  in cik_gpu_init()
3350 	WREG32(SX_DEBUG_1, 0x20);  in cik_gpu_init()
3352 	WREG32(TA_CNTL_AUX, 0x00010000);  in cik_gpu_init()
3355 	tmp |= 0x03000000;  in cik_gpu_init()
3360 	WREG32(DB_DEBUG, 0);  in cik_gpu_init()
3362 	tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;  in cik_gpu_init()
3363 	tmp |= 0x00000400;  in cik_gpu_init()
3366 	tmp = RREG32(DB_DEBUG3) & ~0x0002021c;  in cik_gpu_init()
3367 	tmp |= 0x00020200;  in cik_gpu_init()
3370 	tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;  in cik_gpu_init()
3371 	tmp |= 0x00018208;  in cik_gpu_init()
3376 	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |  in cik_gpu_init()
3377 				 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |  in cik_gpu_init()
3378 				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |  in cik_gpu_init()
3379 				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));  in cik_gpu_init()
3383 	WREG32(CP_PERFMON_CNTL, 0);  in cik_gpu_init()
3385 	WREG32(SQ_CONFIG, 0);  in cik_gpu_init()
3394 	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);  in cik_gpu_init()
3413  * cik_scratch_init - setup driver info for CP scratch regs
3426 	rdev->scratch.num_reg = 7;  in cik_scratch_init()
3427 	rdev->scratch.reg_base = SCRATCH_REG0;  in cik_scratch_init()
3428 	for (i = 0; i < rdev->scratch.num_reg; i++) {  in cik_scratch_init()
3429 		rdev->scratch.free[i] = true;  in cik_scratch_init()
3430 		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);  in cik_scratch_init()
3435  * cik_ring_test - basic gfx ring test
3443  * Returns 0 on success, error on failure.
3448 	uint32_t tmp = 0;  in cik_ring_test()
3457 	WREG32(scratch, 0xCAFEDEAD);  in cik_ring_test()
3460 		DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);  in cik_ring_test()
3465 	radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));  in cik_ring_test()
3466 	radeon_ring_write(ring, 0xDEADBEEF);  in cik_ring_test()
3469 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_ring_test()
3471 		if (tmp == 0xDEADBEEF)  in cik_ring_test()
3475 	if (i < rdev->usec_timeout) {  in cik_ring_test()
3476 		DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);  in cik_ring_test()
3478 		DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",  in cik_ring_test()
3479 			  ring->idx, scratch, tmp);  in cik_ring_test()
3480 		r = -EINVAL;  in cik_ring_test()
3487  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3497 	struct radeon_ring *ring = &rdev->ring[ridx];  in cik_hdp_flush_cp_ring_emit()
3500 	switch (ring->idx) {  in cik_hdp_flush_cp_ring_emit()
3504 		switch (ring->me) {  in cik_hdp_flush_cp_ring_emit()
3505 		case 0:  in cik_hdp_flush_cp_ring_emit()
3506 			ref_and_mask = CP2 << ring->pipe;  in cik_hdp_flush_cp_ring_emit()
3509 			ref_and_mask = CP6 << ring->pipe;  in cik_hdp_flush_cp_ring_emit()
3528 	radeon_ring_write(ring, 0x20); /* poll interval */  in cik_hdp_flush_cp_ring_emit()
3532  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3543 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cik_fence_gfx_ring_emit()
3544 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cik_fence_gfx_ring_emit()
3554 	radeon_ring_write(ring, addr & 0xfffffffc);  in cik_fence_gfx_ring_emit()
3555 	radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |  in cik_fence_gfx_ring_emit()
3556 				DATA_SEL(1) | INT_SEL(0));  in cik_fence_gfx_ring_emit()
3557 	radeon_ring_write(ring, fence->seq - 1);  in cik_fence_gfx_ring_emit()
3558 	radeon_ring_write(ring, 0);  in cik_fence_gfx_ring_emit()
3566 	radeon_ring_write(ring, addr & 0xfffffffc);  in cik_fence_gfx_ring_emit()
3567 	radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));  in cik_fence_gfx_ring_emit()
3568 	radeon_ring_write(ring, fence->seq);  in cik_fence_gfx_ring_emit()
3569 	radeon_ring_write(ring, 0);  in cik_fence_gfx_ring_emit()
3573  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3584 	struct radeon_ring *ring = &rdev->ring[fence->ring];  in cik_fence_compute_ring_emit()
3585 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;  in cik_fence_compute_ring_emit()
3587 	/* RELEASE_MEM - flush caches, send int */  in cik_fence_compute_ring_emit()
3594 	radeon_ring_write(ring, addr & 0xfffffffc);  in cik_fence_compute_ring_emit()
3596 	radeon_ring_write(ring, fence->seq);  in cik_fence_compute_ring_emit()
3597 	radeon_ring_write(ring, 0);  in cik_fence_compute_ring_emit()
3601  * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3616 	uint64_t addr = semaphore->gpu_addr;  in cik_semaphore_ring_emit()
3621 	radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);  in cik_semaphore_ring_emit()
3623 	if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {  in cik_semaphore_ring_emit()
3625 		radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));  in cik_semaphore_ring_emit()
3626 		radeon_ring_write(ring, 0x0);  in cik_semaphore_ring_emit()
3633  * cik_copy_cpdma - copy pages using the CP DMA engine
3652 	int ring_index = rdev->asic->copy.blit_ring_index;  in cik_copy_cpdma()
3653 	struct radeon_ring *ring = &rdev->ring[ring_index];  in cik_copy_cpdma()
3656 	int r = 0;  in cik_copy_cpdma()
3661 	num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);  in cik_copy_cpdma()
3670 	radeon_sync_rings(rdev, &sync, ring->idx);  in cik_copy_cpdma()
3672 	for (i = 0; i < num_loops; i++) {  in cik_copy_cpdma()
3674 		if (cur_size_in_bytes > 0x1fffff)  in cik_copy_cpdma()
3675 			cur_size_in_bytes = 0x1fffff;  in cik_copy_cpdma()
3676 		size_in_bytes -= cur_size_in_bytes;  in cik_copy_cpdma()
3677 		control = 0;  in cik_copy_cpdma()
3678 		if (size_in_bytes == 0)  in cik_copy_cpdma()
3691 	r = radeon_fence_emit(rdev, &fence, ring->idx);  in cik_copy_cpdma()
3708  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3721 	struct radeon_ring *ring = &rdev->ring[ib->ring];  in cik_ring_ib_execute()
3722 	unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;  in cik_ring_ib_execute()
3725 	if (ib->is_const_ib) {  in cik_ring_ib_execute()
3727 		radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));  in cik_ring_ib_execute()
3728 		radeon_ring_write(ring, 0);  in cik_ring_ib_execute()
3733 		if (ring->rptr_save_reg) {  in cik_ring_ib_execute()
3734 			next_rptr = ring->wptr + 3 + 4;  in cik_ring_ib_execute()
3736 			radeon_ring_write(ring, ((ring->rptr_save_reg -  in cik_ring_ib_execute()
3739 		} else if (rdev->wb.enabled) {  in cik_ring_ib_execute()
3740 			next_rptr = ring->wptr + 5 + 4;  in cik_ring_ib_execute()
3743 			radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);  in cik_ring_ib_execute()
3744 			radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));  in cik_ring_ib_execute()
3751 	control |= ib->length_dw | (vm_id << 24);  in cik_ring_ib_execute()
3754 	radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));  in cik_ring_ib_execute()
3755 	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);  in cik_ring_ib_execute()
3760  * cik_ib_test - basic gfx ring IB test
3767  * Returns 0 on success, error on failure.
3773 	uint32_t tmp = 0;  in cik_ib_test()
3782 	WREG32(scratch, 0xCAFEDEAD);  in cik_ib_test()
3783 	r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);  in cik_ib_test()
3789 	ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);  in cik_ib_test()
3790 	ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);  in cik_ib_test()
3791 	ib.ptr[2] = 0xDEADBEEF;  in cik_ib_test()
3802 	if (r < 0) {  in cik_ib_test()
3807 	} else if (r == 0) {  in cik_ib_test()
3811 		return -ETIMEDOUT;  in cik_ib_test()
3813 	r = 0;  in cik_ib_test()
3814 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_ib_test()
3816 		if (tmp == 0xDEADBEEF)  in cik_ib_test()
3820 	if (i < rdev->usec_timeout) {  in cik_ib_test()
3821 		DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);  in cik_ib_test()
3823 		DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",  in cik_ib_test()
3825 		r = -EINVAL;  in cik_ib_test()
3839  * PFP - Pre-Fetch Parser
3840  * ME - Micro Engine
3841  * CE - Constant Engine
3849  * MEC1 - Compute MicroEngine 1
3850  * MEC2 - Compute MicroEngine 2
3856  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3866 		WREG32(CP_ME_CNTL, 0);  in cik_cp_gfx_enable()
3868 		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in cik_cp_gfx_enable()
3869 			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);  in cik_cp_gfx_enable()
3871 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cik_cp_gfx_enable()
3877  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3882  * Returns 0 for success, -EINVAL if the ucode is not available.
3888 	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)  in cik_cp_gfx_load_microcode()
3889 		return -EINVAL;  in cik_cp_gfx_load_microcode()
3893 	if (rdev->new_fw) {  in cik_cp_gfx_load_microcode()
3895 			(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;  in cik_cp_gfx_load_microcode()
3897 			(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;  in cik_cp_gfx_load_microcode()
3899 			(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;  in cik_cp_gfx_load_microcode()
3903 		radeon_ucode_print_gfx_hdr(&pfp_hdr->header);  in cik_cp_gfx_load_microcode()
3904 		radeon_ucode_print_gfx_hdr(&ce_hdr->header);  in cik_cp_gfx_load_microcode()
3905 		radeon_ucode_print_gfx_hdr(&me_hdr->header);  in cik_cp_gfx_load_microcode()
3909 			(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));  in cik_cp_gfx_load_microcode()
3910 		fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;  in cik_cp_gfx_load_microcode()
3911 		WREG32(CP_PFP_UCODE_ADDR, 0);  in cik_cp_gfx_load_microcode()
3912 		for (i = 0; i < fw_size; i++)  in cik_cp_gfx_load_microcode()
3914 		WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));  in cik_cp_gfx_load_microcode()
3918 			(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));  in cik_cp_gfx_load_microcode()
3919 		fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;  in cik_cp_gfx_load_microcode()
3920 		WREG32(CP_CE_UCODE_ADDR, 0);  in cik_cp_gfx_load_microcode()
3921 		for (i = 0; i < fw_size; i++)  in cik_cp_gfx_load_microcode()
3923 		WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));  in cik_cp_gfx_load_microcode()
3927 			(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));  in cik_cp_gfx_load_microcode()
3928 		fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;  in cik_cp_gfx_load_microcode()
3929 		WREG32(CP_ME_RAM_WADDR, 0);  in cik_cp_gfx_load_microcode()
3930 		for (i = 0; i < fw_size; i++)  in cik_cp_gfx_load_microcode()
3932 		WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));  in cik_cp_gfx_load_microcode()
3933 		WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));  in cik_cp_gfx_load_microcode()
3938 		fw_data = (const __be32 *)rdev->pfp_fw->data;  in cik_cp_gfx_load_microcode()
3939 		WREG32(CP_PFP_UCODE_ADDR, 0);  in cik_cp_gfx_load_microcode()
3940 		for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)  in cik_cp_gfx_load_microcode()
3942 		WREG32(CP_PFP_UCODE_ADDR, 0);  in cik_cp_gfx_load_microcode()
3945 		fw_data = (const __be32 *)rdev->ce_fw->data;  in cik_cp_gfx_load_microcode()
3946 		WREG32(CP_CE_UCODE_ADDR, 0);  in cik_cp_gfx_load_microcode()
3947 		for (i = 0; i < CIK_CE_UCODE_SIZE; i++)  in cik_cp_gfx_load_microcode()
3949 		WREG32(CP_CE_UCODE_ADDR, 0);  in cik_cp_gfx_load_microcode()
3952 		fw_data = (const __be32 *)rdev->me_fw->data;  in cik_cp_gfx_load_microcode()
3953 		WREG32(CP_ME_RAM_WADDR, 0);  in cik_cp_gfx_load_microcode()
3954 		for (i = 0; i < CIK_ME_UCODE_SIZE; i++)  in cik_cp_gfx_load_microcode()
3956 		WREG32(CP_ME_RAM_WADDR, 0);  in cik_cp_gfx_load_microcode()
3959 	return 0;  in cik_cp_gfx_load_microcode()
3963  * cik_cp_gfx_start - start the gfx ring
3969  * Returns 0 for success, error for failure.
3973 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_cp_gfx_start()
3977 	WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);  in cik_cp_gfx_start()
3978 	WREG32(CP_ENDIAN_SWAP, 0);  in cik_cp_gfx_start()
3992 	radeon_ring_write(ring, 0x8000);  in cik_cp_gfx_start()
3993 	radeon_ring_write(ring, 0x8000);  in cik_cp_gfx_start()
3996 	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in cik_cp_gfx_start()
4000 	radeon_ring_write(ring, 0x80000000);  in cik_cp_gfx_start()
4001 	radeon_ring_write(ring, 0x80000000);  in cik_cp_gfx_start()
4003 	for (i = 0; i < cik_default_size; i++)  in cik_cp_gfx_start()
4006 	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in cik_cp_gfx_start()
4010 	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));  in cik_cp_gfx_start()
4011 	radeon_ring_write(ring, 0);  in cik_cp_gfx_start()
4014 	radeon_ring_write(ring, 0x00000316);  in cik_cp_gfx_start()
4015 	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */  in cik_cp_gfx_start()
4016 	radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */  in cik_cp_gfx_start()
4020 	return 0;  in cik_cp_gfx_start()
4024  * cik_cp_gfx_fini - stop the gfx ring
4034 	radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);  in cik_cp_gfx_fini()
4038  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4044  * Returns 0 for success, error for failure.
4054 	WREG32(CP_SEM_WAIT_TIMER, 0x0);  in cik_cp_gfx_resume()
4055 	if (rdev->family != CHIP_HAWAII)  in cik_cp_gfx_resume()
4056 		WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);  in cik_cp_gfx_resume()
4059 	WREG32(CP_RB_WPTR_DELAY, 0);  in cik_cp_gfx_resume()
4061 	/* set the RB to use vmid 0 */  in cik_cp_gfx_resume()
4062 	WREG32(CP_RB_VMID, 0);  in cik_cp_gfx_resume()
4064 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);  in cik_cp_gfx_resume()
4066 	/* ring 0 - compute and gfx */  in cik_cp_gfx_resume()
4068 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_cp_gfx_resume()
4069 	rb_bufsz = order_base_2(ring->ring_size / 8);  in cik_cp_gfx_resume()
4078 	ring->wptr = 0;  in cik_cp_gfx_resume()
4079 	WREG32(CP_RB0_WPTR, ring->wptr);  in cik_cp_gfx_resume()
4082 	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);  in cik_cp_gfx_resume()
4083 	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);  in cik_cp_gfx_resume()
4086 	WREG32(SCRATCH_UMSK, 0);  in cik_cp_gfx_resume()
4088 	if (!rdev->wb.enabled)  in cik_cp_gfx_resume()
4094 	rb_addr = ring->gpu_addr >> 8;  in cik_cp_gfx_resume()
4100 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;  in cik_cp_gfx_resume()
4101 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);  in cik_cp_gfx_resume()
4103 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;  in cik_cp_gfx_resume()
4107 	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)  in cik_cp_gfx_resume()
4108 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);  in cik_cp_gfx_resume()
4110 	return 0;  in cik_cp_gfx_resume()
4118 	if (rdev->wb.enabled)  in cik_gfx_get_rptr()
4119 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in cik_gfx_get_rptr()
4135 	WREG32(CP_RB0_WPTR, ring->wptr);  in cik_gfx_set_wptr()
4144 	if (rdev->wb.enabled) {  in cik_compute_get_rptr()
4145 		rptr = rdev->wb.wb[ring->rptr_offs/4];  in cik_compute_get_rptr()
4147 		mutex_lock(&rdev->srbm_mutex);  in cik_compute_get_rptr()
4148 		cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);  in cik_compute_get_rptr()
4150 		cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_compute_get_rptr()
4151 		mutex_unlock(&rdev->srbm_mutex);  in cik_compute_get_rptr()
4162 	if (rdev->wb.enabled) {  in cik_compute_get_wptr()
4164 		wptr = rdev->wb.wb[ring->wptr_offs/4];  in cik_compute_get_wptr()
4166 		mutex_lock(&rdev->srbm_mutex);  in cik_compute_get_wptr()
4167 		cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);  in cik_compute_get_wptr()
4169 		cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_compute_get_wptr()
4170 		mutex_unlock(&rdev->srbm_mutex);  in cik_compute_get_wptr()
4180 	rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;  in cik_compute_set_wptr()
4181 	WDOORBELL32(ring->doorbell_index, ring->wptr);  in cik_compute_set_wptr()
4189 	cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);  in cik_compute_stop()
4197 		for (j = 0; j < rdev->usec_timeout; j++) {  in cik_compute_stop()
4202 		WREG32(CP_HQD_DEQUEUE_REQUEST, 0);  in cik_compute_stop()
4203 		WREG32(CP_HQD_PQ_RPTR, 0);  in cik_compute_stop()
4204 		WREG32(CP_HQD_PQ_WPTR, 0);  in cik_compute_stop()
4206 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_compute_stop()
4210  * cik_cp_compute_enable - enable/disable the compute CP MEs
4220 		WREG32(CP_MEC_CNTL, 0);  in cik_cp_compute_enable()
4226 		mutex_lock(&rdev->srbm_mutex);  in cik_cp_compute_enable()
4227 		cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);  in cik_cp_compute_enable()
4228 		cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);  in cik_cp_compute_enable()
4229 		mutex_unlock(&rdev->srbm_mutex);  in cik_cp_compute_enable()
4232 		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;  in cik_cp_compute_enable()
4233 		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;  in cik_cp_compute_enable()
4239  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4244  * Returns 0 for success, -EINVAL if the ucode is not available.
4250 	if (!rdev->mec_fw)  in cik_cp_compute_load_microcode()
4251 		return -EINVAL;  in cik_cp_compute_load_microcode()
4255 	if (rdev->new_fw) {  in cik_cp_compute_load_microcode()
4257 			(const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;  in cik_cp_compute_load_microcode()
4261 		radeon_ucode_print_gfx_hdr(&mec_hdr->header);  in cik_cp_compute_load_microcode()
4265 			(rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));  in cik_cp_compute_load_microcode()
4266 		fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;  in cik_cp_compute_load_microcode()
4267 		WREG32(CP_MEC_ME1_UCODE_ADDR, 0);  in cik_cp_compute_load_microcode()
4268 		for (i = 0; i < fw_size; i++)  in cik_cp_compute_load_microcode()
4270 		WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));  in cik_cp_compute_load_microcode()
4273 		if (rdev->family == CHIP_KAVERI) {  in cik_cp_compute_load_microcode()
4275 				(const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;  in cik_cp_compute_load_microcode()
4278 				(rdev->mec2_fw->data +  in cik_cp_compute_load_microcode()
4279 				 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));  in cik_cp_compute_load_microcode()
4280 			fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;  in cik_cp_compute_load_microcode()
4281 			WREG32(CP_MEC_ME2_UCODE_ADDR, 0);  in cik_cp_compute_load_microcode()
4282 			for (i = 0; i < fw_size; i++)  in cik_cp_compute_load_microcode()
4284 			WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));  in cik_cp_compute_load_microcode()
4290 		fw_data = (const __be32 *)rdev->mec_fw->data;  in cik_cp_compute_load_microcode()
4291 		WREG32(CP_MEC_ME1_UCODE_ADDR, 0);  in cik_cp_compute_load_microcode()
4292 		for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)  in cik_cp_compute_load_microcode()
4294 		WREG32(CP_MEC_ME1_UCODE_ADDR, 0);  in cik_cp_compute_load_microcode()
4296 		if (rdev->family == CHIP_KAVERI) {  in cik_cp_compute_load_microcode()
4298 			fw_data = (const __be32 *)rdev->mec_fw->data;  in cik_cp_compute_load_microcode()
4299 			WREG32(CP_MEC_ME2_UCODE_ADDR, 0);  in cik_cp_compute_load_microcode()
4300 			for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)  in cik_cp_compute_load_microcode()
4302 			WREG32(CP_MEC_ME2_UCODE_ADDR, 0);  in cik_cp_compute_load_microcode()
4306 	return 0;  in cik_cp_compute_load_microcode()
4310  * cik_cp_compute_start - start the compute queues
4315  * Returns 0 for success, error for failure.
4321 	return 0;  in cik_cp_compute_start()
4325  * cik_cp_compute_fini - stop the compute queues
4338 	for (i = 0; i < 2; i++) {  in cik_cp_compute_fini()
4339 		if (i == 0)  in cik_cp_compute_fini()
4344 		if (rdev->ring[idx].mqd_obj) {  in cik_cp_compute_fini()
4345 			r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);  in cik_cp_compute_fini()
4346 			if (unlikely(r != 0))  in cik_cp_compute_fini()
4347 				dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);  in cik_cp_compute_fini()
4349 			radeon_bo_unpin(rdev->ring[idx].mqd_obj);  in cik_cp_compute_fini()
4350 			radeon_bo_unreserve(rdev->ring[idx].mqd_obj);  in cik_cp_compute_fini()
4352 			radeon_bo_unref(&rdev->ring[idx].mqd_obj);  in cik_cp_compute_fini()
4353 			rdev->ring[idx].mqd_obj = NULL;  in cik_cp_compute_fini()
4362 	if (rdev->mec.hpd_eop_obj) {  in cik_mec_fini()
4363 		r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);  in cik_mec_fini()
4364 		if (unlikely(r != 0))  in cik_mec_fini()
4365 			dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);  in cik_mec_fini()
4366 		radeon_bo_unpin(rdev->mec.hpd_eop_obj);  in cik_mec_fini()
4367 		radeon_bo_unreserve(rdev->mec.hpd_eop_obj);  in cik_mec_fini()
4369 		radeon_bo_unref(&rdev->mec.hpd_eop_obj);  in cik_mec_fini()
4370 		rdev->mec.hpd_eop_obj = NULL;  in cik_mec_fini()
4382 	 * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total  in cik_mec_init()
4383 	 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total  in cik_mec_init()
4385 	if (rdev->family == CHIP_KAVERI)  in cik_mec_init()
4386 		rdev->mec.num_mec = 2;  in cik_mec_init()
4388 		rdev->mec.num_mec = 1;  in cik_mec_init()
4389 	rdev->mec.num_pipe = 4;  in cik_mec_init()
4390 	rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;  in cik_mec_init()
4392 	if (rdev->mec.hpd_eop_obj == NULL) {  in cik_mec_init()
4394 				     rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,  in cik_mec_init()
4396 				     RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,  in cik_mec_init()
4397 				     &rdev->mec.hpd_eop_obj);  in cik_mec_init()
4399 			dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);  in cik_mec_init()
4404 	r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);  in cik_mec_init()
4405 	if (unlikely(r != 0)) {  in cik_mec_init()
4409 	r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,  in cik_mec_init()
4410 			  &rdev->mec.hpd_eop_gpu_addr);  in cik_mec_init()
4412 		dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);  in cik_mec_init()
4416 	r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);  in cik_mec_init()
4418 		dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);  in cik_mec_init()
4424 	memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);  in cik_mec_init()
4426 	radeon_bo_kunmap(rdev->mec.hpd_eop_obj);  in cik_mec_init()
4427 	radeon_bo_unreserve(rdev->mec.hpd_eop_obj);  in cik_mec_init()
4429 	return 0;  in cik_mec_init()
4500  * cik_cp_compute_resume - setup the compute queue registers
4506  * Returns 0 for success, error for failure.
4530 	mutex_lock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4532 	for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); ++i) {  in cik_cp_compute_resume()
4534 		int pipe = (i < 4) ? i : (i - 4);  in cik_cp_compute_resume()
4536 		cik_srbm_select(rdev, me, pipe, 0, 0);  in cik_cp_compute_resume()
4538 		eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;  in cik_cp_compute_resume()
4544 		WREG32(CP_HPD_EOP_VMID, 0);  in cik_cp_compute_resume()
4553 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_cp_compute_resume()
4554 	mutex_unlock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4557 	for (i = 0; i < 2; i++) {  in cik_cp_compute_resume()
4558 		if (i == 0)  in cik_cp_compute_resume()
4563 		if (rdev->ring[idx].mqd_obj == NULL) {  in cik_cp_compute_resume()
4567 					     RADEON_GEM_DOMAIN_GTT, 0, NULL,  in cik_cp_compute_resume()
4568 					     NULL, &rdev->ring[idx].mqd_obj);  in cik_cp_compute_resume()
4570 				dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);  in cik_cp_compute_resume()
4575 		r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);  in cik_cp_compute_resume()
4576 		if (unlikely(r != 0)) {  in cik_cp_compute_resume()
4580 		r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,  in cik_cp_compute_resume()
4583 			dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);  in cik_cp_compute_resume()
4587 		r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);  in cik_cp_compute_resume()
4589 			dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);  in cik_cp_compute_resume()
4595 		memset(buf, 0, sizeof(struct bonaire_mqd));  in cik_cp_compute_resume()
4598 		mqd->header = 0xC0310800;  in cik_cp_compute_resume()
4599 		mqd->static_thread_mgmt01[0] = 0xffffffff;  in cik_cp_compute_resume()
4600 		mqd->static_thread_mgmt01[1] = 0xffffffff;  in cik_cp_compute_resume()
4601 		mqd->static_thread_mgmt23[0] = 0xffffffff;  in cik_cp_compute_resume()
4602 		mqd->static_thread_mgmt23[1] = 0xffffffff;  in cik_cp_compute_resume()
4604 		mutex_lock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4605 		cik_srbm_select(rdev, rdev->ring[idx].me,  in cik_cp_compute_resume()
4606 				rdev->ring[idx].pipe,  in cik_cp_compute_resume()
4607 				rdev->ring[idx].queue, 0);  in cik_cp_compute_resume()
4615 		mqd->queue_state.cp_hqd_pq_doorbell_control =  in cik_cp_compute_resume()
4618 			mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;  in cik_cp_compute_resume()
4620 			mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;  in cik_cp_compute_resume()
4622 		       mqd->queue_state.cp_hqd_pq_doorbell_control);  in cik_cp_compute_resume()
4625 		mqd->queue_state.cp_hqd_dequeue_request = 0;  in cik_cp_compute_resume()
4626 		mqd->queue_state.cp_hqd_pq_rptr = 0;  in cik_cp_compute_resume()
4627 		mqd->queue_state.cp_hqd_pq_wptr= 0;  in cik_cp_compute_resume()
4630 			for (j = 0; j < rdev->usec_timeout; j++) {  in cik_cp_compute_resume()
4635 			WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);  in cik_cp_compute_resume()
4636 			WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);  in cik_cp_compute_resume()
4637 			WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);  in cik_cp_compute_resume()
4641 		mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;  in cik_cp_compute_resume()
4642 		mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);  in cik_cp_compute_resume()
4643 		WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);  in cik_cp_compute_resume()
4644 		WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);  in cik_cp_compute_resume()
4645 		/* set MQD vmid to 0 */  in cik_cp_compute_resume()
4646 		mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);  in cik_cp_compute_resume()
4647 		mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;  in cik_cp_compute_resume()
4648 		WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);  in cik_cp_compute_resume()
4651 		hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;  in cik_cp_compute_resume()
4652 		mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;  in cik_cp_compute_resume()
4653 		mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);  in cik_cp_compute_resume()
4654 		WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);  in cik_cp_compute_resume()
4655 		WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);  in cik_cp_compute_resume()
4658 		mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);  in cik_cp_compute_resume()
4659 		mqd->queue_state.cp_hqd_pq_control &=  in cik_cp_compute_resume()
4662 		mqd->queue_state.cp_hqd_pq_control |=  in cik_cp_compute_resume()
4663 			order_base_2(rdev->ring[idx].ring_size / 8);  in cik_cp_compute_resume()
4664 		mqd->queue_state.cp_hqd_pq_control |=  in cik_cp_compute_resume()
4667 		mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;  in cik_cp_compute_resume()
4669 		mqd->queue_state.cp_hqd_pq_control &=  in cik_cp_compute_resume()
4671 		mqd->queue_state.cp_hqd_pq_control |=  in cik_cp_compute_resume()
4673 		WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);  in cik_cp_compute_resume()
4676 		if (i == 0)  in cik_cp_compute_resume()
4677 			wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;  in cik_cp_compute_resume()
4679 			wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;  in cik_cp_compute_resume()
4680 		mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;  in cik_cp_compute_resume()
4681 		mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;  in cik_cp_compute_resume()
4682 		WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);  in cik_cp_compute_resume()
4684 		       mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);  in cik_cp_compute_resume()
4687 		if (i == 0)  in cik_cp_compute_resume()
4688 			wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;  in cik_cp_compute_resume()
4690 			wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;  in cik_cp_compute_resume()
4691 		mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;  in cik_cp_compute_resume()
4692 		mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =  in cik_cp_compute_resume()
4693 			upper_32_bits(wb_gpu_addr) & 0xffff;  in cik_cp_compute_resume()
4695 		       mqd->queue_state.cp_hqd_pq_rptr_report_addr);  in cik_cp_compute_resume()
4697 		       mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);  in cik_cp_compute_resume()
4701 			mqd->queue_state.cp_hqd_pq_doorbell_control =  in cik_cp_compute_resume()
4703 			mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;  in cik_cp_compute_resume()
4704 			mqd->queue_state.cp_hqd_pq_doorbell_control |=  in cik_cp_compute_resume()
4705 				DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);  in cik_cp_compute_resume()
4706 			mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;  in cik_cp_compute_resume()
4707 			mqd->queue_state.cp_hqd_pq_doorbell_control &=  in cik_cp_compute_resume()
4711 			mqd->queue_state.cp_hqd_pq_doorbell_control = 0;  in cik_cp_compute_resume()
4714 		       mqd->queue_state.cp_hqd_pq_doorbell_control);  in cik_cp_compute_resume()
4717 		rdev->ring[idx].wptr = 0;  in cik_cp_compute_resume()
4718 		mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;  in cik_cp_compute_resume()
4719 		WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);  in cik_cp_compute_resume()
4720 		mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);  in cik_cp_compute_resume()
4723 		mqd->queue_state.cp_hqd_vmid = 0;  in cik_cp_compute_resume()
4724 		WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);  in cik_cp_compute_resume()
4727 		mqd->queue_state.cp_hqd_active = 1;  in cik_cp_compute_resume()
4728 		WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);  in cik_cp_compute_resume()
4730 		cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_cp_compute_resume()
4731 		mutex_unlock(&rdev->srbm_mutex);  in cik_cp_compute_resume()
4733 		radeon_bo_kunmap(rdev->ring[idx].mqd_obj);  in cik_cp_compute_resume()
4734 		radeon_bo_unreserve(rdev->ring[idx].mqd_obj);  in cik_cp_compute_resume()
4736 		rdev->ring[idx].ready = true;  in cik_cp_compute_resume()
4737 		r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);  in cik_cp_compute_resume()
4739 			rdev->ring[idx].ready = false;  in cik_cp_compute_resume()
4742 	return 0;  in cik_cp_compute_resume()
4762 	return 0;  in cik_cp_load_microcode()
4790 	return 0;  in cik_cp_resume()
4795 	dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",  in cik_print_gpu_status_regs()
4797 	dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",  in cik_print_gpu_status_regs()
4799 	dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",  in cik_print_gpu_status_regs()
4801 	dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",  in cik_print_gpu_status_regs()
4803 	dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",  in cik_print_gpu_status_regs()
4805 	dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",  in cik_print_gpu_status_regs()
4807 	dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",  in cik_print_gpu_status_regs()
4809 	dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",  in cik_print_gpu_status_regs()
4811 	dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",  in cik_print_gpu_status_regs()
4813 	dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",  in cik_print_gpu_status_regs()
4815 	dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));  in cik_print_gpu_status_regs()
4816 	dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",  in cik_print_gpu_status_regs()
4818 	dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",  in cik_print_gpu_status_regs()
4820 	dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",  in cik_print_gpu_status_regs()
4822 	dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",  in cik_print_gpu_status_regs()
4824 	dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",  in cik_print_gpu_status_regs()
4826 	dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));  in cik_print_gpu_status_regs()
4827 	dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));  in cik_print_gpu_status_regs()
4828 	dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",  in cik_print_gpu_status_regs()
4830 	dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));  in cik_print_gpu_status_regs()
4834  * cik_gpu_check_soft_reset - check which blocks are busy
4844 	u32 reset_mask = 0;  in cik_gpu_check_soft_reset()
4907 		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);  in cik_gpu_check_soft_reset()
4915  * cik_gpu_soft_reset - soft reset GPU
4925 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;  in cik_gpu_soft_reset()
4928 	if (reset_mask == 0)  in cik_gpu_soft_reset()
4931 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);  in cik_gpu_soft_reset()
4934 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in cik_gpu_soft_reset()
4936 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in cik_gpu_soft_reset()
4967 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cik_gpu_soft_reset()
5003 	if (!(rdev->flags & RADEON_IS_IGP)) {  in cik_gpu_soft_reset()
5011 		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);  in cik_gpu_soft_reset()
5025 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);  in cik_gpu_soft_reset()
5054 	save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);  in kv_save_regs_for_reset()
5055 	save->gmcon_misc = RREG32(GMCON_MISC);  in kv_save_regs_for_reset()
5056 	save->gmcon_misc3 = RREG32(GMCON_MISC3);  in kv_save_regs_for_reset()
5058 	WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);  in kv_save_regs_for_reset()
5059 	WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |  in kv_save_regs_for_reset()
5068 	WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5069 	WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);  in kv_restore_regs_for_reset()
5071 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5072 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5074 	WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5075 	WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);  in kv_restore_regs_for_reset()
5077 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5078 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5080 	WREG32(GMCON_PGFSM_WRITE, 0x210000);  in kv_restore_regs_for_reset()
5081 	WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);  in kv_restore_regs_for_reset()
5083 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5084 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5086 	WREG32(GMCON_PGFSM_WRITE, 0x21003);  in kv_restore_regs_for_reset()
5087 	WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);  in kv_restore_regs_for_reset()
5089 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5090 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5092 	WREG32(GMCON_PGFSM_WRITE, 0x2b00);  in kv_restore_regs_for_reset()
5093 	WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);  in kv_restore_regs_for_reset()
5095 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5096 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5098 	WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5099 	WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);  in kv_restore_regs_for_reset()
5101 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5102 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5104 	WREG32(GMCON_PGFSM_WRITE, 0x420000);  in kv_restore_regs_for_reset()
5105 	WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);  in kv_restore_regs_for_reset()
5107 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5108 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5110 	WREG32(GMCON_PGFSM_WRITE, 0x120202);  in kv_restore_regs_for_reset()
5111 	WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);  in kv_restore_regs_for_reset()
5113 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5114 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5116 	WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);  in kv_restore_regs_for_reset()
5117 	WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);  in kv_restore_regs_for_reset()
5119 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5120 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5122 	WREG32(GMCON_PGFSM_WRITE, 0x373f3e);  in kv_restore_regs_for_reset()
5123 	WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);  in kv_restore_regs_for_reset()
5125 	for (i = 0; i < 5; i++)  in kv_restore_regs_for_reset()
5126 		WREG32(GMCON_PGFSM_WRITE, 0);  in kv_restore_regs_for_reset()
5128 	WREG32(GMCON_PGFSM_WRITE, 0x3e1332);  in kv_restore_regs_for_reset()
5129 	WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);  in kv_restore_regs_for_reset()
5131 	WREG32(GMCON_MISC3, save->gmcon_misc3);  in kv_restore_regs_for_reset()
5132 	WREG32(GMCON_MISC, save->gmcon_misc);  in kv_restore_regs_for_reset()
5133 	WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);  in kv_restore_regs_for_reset()
5139 	struct kv_reset_save_regs kv_save = { 0 };  in cik_gpu_pci_config_reset()
5142 	dev_info(rdev->dev, "GPU pci config reset\n");  in cik_gpu_pci_config_reset()
5174 		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");  in cik_gpu_pci_config_reset()
5177 	if (rdev->flags & RADEON_IS_IGP)  in cik_gpu_pci_config_reset()
5181 	pci_clear_master(rdev->pdev);  in cik_gpu_pci_config_reset()
5188 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_gpu_pci_config_reset()
5189 		if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)  in cik_gpu_pci_config_reset()
5195 	if (rdev->flags & RADEON_IS_IGP)  in cik_gpu_pci_config_reset()
5200  * cik_asic_reset - soft reset GPU
5207  * Returns 0 for success.
5215 		return 0;  in cik_asic_reset()
5237 	return 0;  in cik_asic_reset()
5241  * cik_gfx_is_lockup - check if the 3D engine is locked up
5264  * cik_mc_program - program the GPU memory controller
5278 	for (i = 0, j = 0; i < 32; i++, j += 0x18) {  in cik_mc_program()
5279 		WREG32((0x2c14 + j), 0x00000000);  in cik_mc_program()
5280 		WREG32((0x2c18 + j), 0x00000000);  in cik_mc_program()
5281 		WREG32((0x2c1c + j), 0x00000000);  in cik_mc_program()
5282 		WREG32((0x2c20 + j), 0x00000000);  in cik_mc_program()
5283 		WREG32((0x2c24 + j), 0x00000000);  in cik_mc_program()
5285 	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);  in cik_mc_program()
5289 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cik_mc_program()
5295 	       rdev->mc.vram_start >> 12);  in cik_mc_program()
5297 	       rdev->mc.vram_end >> 12);  in cik_mc_program()
5299 	       rdev->vram_scratch.gpu_addr >> 12);  in cik_mc_program()
5300 	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;  in cik_mc_program()
5301 	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);  in cik_mc_program()
5304 	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));  in cik_mc_program()
5306 	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);  in cik_mc_program()
5307 	WREG32(MC_VM_AGP_BASE, 0);  in cik_mc_program()
5308 	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);  in cik_mc_program()
5309 	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);  in cik_mc_program()
5311 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");  in cik_mc_program()
5320  * cik_mc_init - initialize the memory controller driver params
5326  * Returns 0 for success.
5334 	rdev->mc.vram_is_ddr = true;  in cik_mc_init()
5343 	case 0:  in cik_mc_init()
5372 	rdev->mc.vram_width = numchan * chansize;  in cik_mc_init()
5373 	/* Could aper size report 0 ? */  in cik_mc_init()
5374 	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);  in cik_mc_init()
5375 	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);  in cik_mc_init()
5377 	rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in cik_mc_init()
5378 	rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;  in cik_mc_init()
5379 	rdev->mc.visible_vram_size = rdev->mc.aper_size;  in cik_mc_init()
5380 	si_vram_gtt_location(rdev, &rdev->mc);  in cik_mc_init()
5383 	return 0;  in cik_mc_init()
5388  * VMID 0 is the physical GPU addresses as used by the kernel.
5389  * VMIDs 1-15 are used for userspace clients and are handled
5393  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5397  * Flush the TLB for the VMID 0 page table (CIK).
5402 	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);  in cik_pcie_gart_tlb_flush()
5404 	/* bits 0-15 are the VM contexts0-15 */  in cik_pcie_gart_tlb_flush()
5405 	WREG32(VM_INVALIDATE_REQUEST, 0x1);  in cik_pcie_gart_tlb_flush()
5409  * cik_pcie_gart_enable - gart enable
5414  * sets up the hw for VMIDs 1-15 which are allocated on
5417  * Returns 0 for success, errors for failure.
5423 	if (rdev->gart.robj == NULL) {  in cik_pcie_gart_enable()
5424 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");  in cik_pcie_gart_enable()
5425 		return -EINVAL;  in cik_pcie_gart_enable()
5432 	       (0xA << 7) |  in cik_pcie_gart_enable()
5450 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);  in cik_pcie_gart_enable()
5451 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);  in cik_pcie_gart_enable()
5452 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);  in cik_pcie_gart_enable()
5454 			(u32)(rdev->dummy_page.addr >> 12));  in cik_pcie_gart_enable()
5455 	WREG32(VM_CONTEXT0_CNTL2, 0);  in cik_pcie_gart_enable()
5456 	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |  in cik_pcie_gart_enable()
5459 	WREG32(0x15D4, 0);  in cik_pcie_gart_enable()
5460 	WREG32(0x15D8, 0);  in cik_pcie_gart_enable()
5461 	WREG32(0x15DC, 0);  in cik_pcie_gart_enable()
5463 	/* restore context1-15 */  in cik_pcie_gart_enable()
5465 	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);  in cik_pcie_gart_enable()
5466 	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);  in cik_pcie_gart_enable()
5470 			       rdev->vm_manager.saved_table_addr[i]);  in cik_pcie_gart_enable()
5472 			WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),  in cik_pcie_gart_enable()
5473 			       rdev->vm_manager.saved_table_addr[i]);  in cik_pcie_gart_enable()
5476 	/* enable context1-15 */  in cik_pcie_gart_enable()
5478 	       (u32)(rdev->dummy_page.addr >> 12));  in cik_pcie_gart_enable()
5481 				PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |  in cik_pcie_gart_enable()
5495 	if (rdev->family == CHIP_KAVERI) {  in cik_pcie_gart_enable()
5503 	mutex_lock(&rdev->srbm_mutex);  in cik_pcie_gart_enable()
5504 	for (i = 0; i < 16; i++) {  in cik_pcie_gart_enable()
5505 		cik_srbm_select(rdev, 0, 0, 0, i);  in cik_pcie_gart_enable()
5509 		WREG32(SH_MEM_APE1_LIMIT, 0);  in cik_pcie_gart_enable()
5510 		WREG32(SH_MEM_BASES, 0);  in cik_pcie_gart_enable()
5512 		WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);  in cik_pcie_gart_enable()
5513 		WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);  in cik_pcie_gart_enable()
5514 		WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);  in cik_pcie_gart_enable()
5515 		WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);  in cik_pcie_gart_enable()
5516 		/* XXX SDMA RLC - todo */  in cik_pcie_gart_enable()
5518 	cik_srbm_select(rdev, 0, 0, 0, 0);  in cik_pcie_gart_enable()
5519 	mutex_unlock(&rdev->srbm_mutex);  in cik_pcie_gart_enable()
5522 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",  in cik_pcie_gart_enable()
5523 		 (unsigned)(rdev->mc.gtt_size >> 20),  in cik_pcie_gart_enable()
5524 		 (unsigned long long)rdev->gart.table_addr);  in cik_pcie_gart_enable()
5525 	rdev->gart.ready = true;  in cik_pcie_gart_enable()
5526 	return 0;  in cik_pcie_gart_enable()
5530  * cik_pcie_gart_disable - gart disable
5545 			reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);  in cik_pcie_gart_disable()
5546 		rdev->vm_manager.saved_table_addr[i] = RREG32(reg);  in cik_pcie_gart_disable()
5550 	WREG32(VM_CONTEXT0_CNTL, 0);  in cik_pcie_gart_disable()
5551 	WREG32(VM_CONTEXT1_CNTL, 0);  in cik_pcie_gart_disable()
5562 	WREG32(VM_L2_CNTL2, 0);  in cik_pcie_gart_disable()
5569  * cik_pcie_gart_fini - vm fini callback
5584  * cik_ib_parse - vm ib_parse callback
5593 	return 0;  in cik_ib_parse()
5598  * VMID 0 is the physical GPU addresses as used by the kernel.
5599  * VMIDs 1-15 are used for userspace clients and are handled
5603  * cik_vm_init - cik vm init callback
5608  * VMIDs 1-15) (CIK).
5609  * Returns 0 for success.
5615 	 * VMID 0 is reserved for System  in cik_vm_init()
5616 	 * radeon graphics/compute will use VMIDs 1-15  in cik_vm_init()
5618 	rdev->vm_manager.nvm = 16;  in cik_vm_init()
5620 	if (rdev->flags & RADEON_IS_IGP) {  in cik_vm_init()
5623 		rdev->vm_manager.vram_base_offset = tmp;  in cik_vm_init()
5625 		rdev->vm_manager.vram_base_offset = 0;  in cik_vm_init()
5627 	return 0;  in cik_vm_init()
5631  * cik_vm_fini - cik vm fini callback
5642  * cik_vm_decode_fault - print human readable fault info
5657 	char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,  in cik_vm_decode_fault()
5658 		(mc_client >> 8) & 0xff, mc_client & 0xff, 0 };  in cik_vm_decode_fault()
5660 	if (rdev->family == CHIP_HAWAII)  in cik_vm_decode_fault()
5665 	printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",  in cik_vm_decode_fault()
5672  * cik_vm_flush - cik vm flush using the CP
5680 	int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);  in cik_vm_flush()
5684 				 WRITE_DATA_DST_SEL(0)));  in cik_vm_flush()
5690 				  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);  in cik_vm_flush()
5692 	radeon_ring_write(ring, 0);  in cik_vm_flush()
5698 				 WRITE_DATA_DST_SEL(0)));  in cik_vm_flush()
5700 	radeon_ring_write(ring, 0);  in cik_vm_flush()
5705 				 WRITE_DATA_DST_SEL(0)));  in cik_vm_flush()
5707 	radeon_ring_write(ring, 0);  in cik_vm_flush()
5709 	radeon_ring_write(ring, 0); /* SH_MEM_BASES */  in cik_vm_flush()
5712 	radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */  in cik_vm_flush()
5716 				 WRITE_DATA_DST_SEL(0)));  in cik_vm_flush()
5718 	radeon_ring_write(ring, 0);  in cik_vm_flush()
5719 	radeon_ring_write(ring, VMID(0));  in cik_vm_flush()
5722 	cik_hdp_flush_cp_ring_emit(rdev, ring->idx);  in cik_vm_flush()
5724 	/* bits 0-15 are the VM contexts0-15 */  in cik_vm_flush()
5727 				 WRITE_DATA_DST_SEL(0)));  in cik_vm_flush()
5729 	radeon_ring_write(ring, 0);  in cik_vm_flush()
5734 	radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */  in cik_vm_flush()
5735 				 WAIT_REG_MEM_FUNCTION(0) |  /* always */  in cik_vm_flush()
5736 				 WAIT_REG_MEM_ENGINE(0))); /* me */  in cik_vm_flush()
5738 	radeon_ring_write(ring, 0);  in cik_vm_flush()
5739 	radeon_ring_write(ring, 0); /* ref */  in cik_vm_flush()
5740 	radeon_ring_write(ring, 0); /* mask */  in cik_vm_flush()
5741 	radeon_ring_write(ring, 0x20); /* poll interval */  in cik_vm_flush()
5746 		radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));  in cik_vm_flush()
5747 		radeon_ring_write(ring, 0x0);  in cik_vm_flush()
5753  * The RLC is a multi-purpose microengine that handles a
5786 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {  in cik_wait_for_rlc_serdes()
5787 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {  in cik_wait_for_rlc_serdes()
5789 			for (k = 0; k < rdev->usec_timeout; k++) {  in cik_wait_for_rlc_serdes()
5790 				if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)  in cik_wait_for_rlc_serdes()
5796 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_wait_for_rlc_serdes()
5799 	for (k = 0; k < rdev->usec_timeout; k++) {  in cik_wait_for_rlc_serdes()
5800 		if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)  in cik_wait_for_rlc_serdes()
5827 		for (i = 0; i < rdev->usec_timeout; i++) {  in cik_halt_rlc()
5828 			if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)  in cik_halt_rlc()
5847 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_enter_rlc_safe_mode()
5853 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_enter_rlc_safe_mode()
5854 		if ((RREG32(RLC_GPR_REG2) & REQ) == 0)  in cik_enter_rlc_safe_mode()
5869  * cik_rlc_stop - stop the RLC ME
5877 	WREG32(RLC_CNTL, 0);  in cik_rlc_stop()
5885  * cik_rlc_start - start the RLC ME
5901  * cik_rlc_resume - setup the RLC hw
5907  * Returns 0 for success, -EINVAL if the ucode is not available.
5913 	if (!rdev->rlc_fw)  in cik_rlc_resume()
5914 		return -EINVAL;  in cik_rlc_resume()
5919 	tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;  in cik_rlc_resume()
5928 	WREG32(RLC_LB_CNTR_INIT, 0);  in cik_rlc_resume()
5929 	WREG32(RLC_LB_CNTR_MAX, 0x00008000);  in cik_rlc_resume()
5931 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_rlc_resume()
5932 	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);  in cik_rlc_resume()
5933 	WREG32(RLC_LB_PARAMS, 0x00600408);  in cik_rlc_resume()
5934 	WREG32(RLC_LB_CNTL, 0x80000004);  in cik_rlc_resume()
5936 	WREG32(RLC_MC_CNTL, 0);  in cik_rlc_resume()
5937 	WREG32(RLC_UCODE_CNTL, 0);  in cik_rlc_resume()
5939 	if (rdev->new_fw) {  in cik_rlc_resume()
5941 			(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;  in cik_rlc_resume()
5943 			(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_rlc_resume()
5945 		radeon_ucode_print_rlc_hdr(&hdr->header);  in cik_rlc_resume()
5947 		size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;  in cik_rlc_resume()
5948 		WREG32(RLC_GPM_UCODE_ADDR, 0);  in cik_rlc_resume()
5949 		for (i = 0; i < size; i++)  in cik_rlc_resume()
5951 		WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));  in cik_rlc_resume()
5955 		switch (rdev->family) {  in cik_rlc_resume()
5972 		fw_data = (const __be32 *)rdev->rlc_fw->data;  in cik_rlc_resume()
5973 		WREG32(RLC_GPM_UCODE_ADDR, 0);  in cik_rlc_resume()
5974 		for (i = 0; i < size; i++)  in cik_rlc_resume()
5976 		WREG32(RLC_GPM_UCODE_ADDR, 0);  in cik_rlc_resume()
5979 	/* XXX - find out what chips support lbpw */  in cik_rlc_resume()
5982 	if (rdev->family == CHIP_BONAIRE)  in cik_rlc_resume()
5983 		WREG32(RLC_DRIVER_DMA_STATUS, 0);  in cik_rlc_resume()
5987 	return 0;  in cik_rlc_resume()
5996 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {  in cik_enable_cgcg()
6001 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_enable_cgcg()
6002 		WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);  in cik_enable_cgcg()
6003 		WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);  in cik_enable_cgcg()
6028 	u32 data, orig, tmp = 0;  in cik_enable_mgcg()
6030 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {  in cik_enable_mgcg()
6031 		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {  in cik_enable_mgcg()
6032 			if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {  in cik_enable_mgcg()
6041 		data |= 0x00000001;  in cik_enable_mgcg()
6042 		data &= 0xfffffffd;  in cik_enable_mgcg()
6048 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_enable_mgcg()
6049 		WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);  in cik_enable_mgcg()
6050 		WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);  in cik_enable_mgcg()
6056 		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {  in cik_enable_mgcg()
6059 			data |= SM_MODE(0x2);  in cik_enable_mgcg()
6062 			if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&  in cik_enable_mgcg()
6063 			    (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))  in cik_enable_mgcg()
6067 			data |= ON_MONITOR_ADD(0x96);  in cik_enable_mgcg()
6073 		data |= 0x00000003;  in cik_enable_mgcg()
6096 		cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_enable_mgcg()
6097 		WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);  in cik_enable_mgcg()
6098 		WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);  in cik_enable_mgcg()
6125 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {  in cik_enable_mc_ls()
6127 		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))  in cik_enable_mc_ls()
6142 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {  in cik_enable_mc_mgcg()
6144 		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))  in cik_enable_mc_mgcg()
6158 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {  in cik_enable_sdma_mgcg()
6159 		WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);  in cik_enable_sdma_mgcg()
6160 		WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);  in cik_enable_sdma_mgcg()
6163 		data |= 0xff000000;  in cik_enable_sdma_mgcg()
6168 		data |= 0xff000000;  in cik_enable_sdma_mgcg()
6179 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {  in cik_enable_sdma_mgls()
6181 		data |= 0x100;  in cik_enable_sdma_mgls()
6186 		data |= 0x100;  in cik_enable_sdma_mgls()
6191 		data &= ~0x100;  in cik_enable_sdma_mgls()
6196 		data &= ~0x100;  in cik_enable_sdma_mgls()
6207 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {  in cik_enable_uvd_mgcg()
6209 		data = 0xfff;  in cik_enable_uvd_mgcg()
6218 		data &= ~0xfff;  in cik_enable_uvd_mgcg()
6235 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))  in cik_enable_bif_mgls()
6253 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))  in cik_enable_hdp_mgcg()
6269 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))  in cik_enable_hdp_ls()
6296 		if (!(rdev->flags & RADEON_IS_IGP)) {  in cik_update_cg()
6312 		if (rdev->has_uvd)  in cik_update_cg()
6331 	if (rdev->has_uvd)  in cik_init_cg()
6358 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))  in cik_enable_sck_slowdown_on_pu()
6372 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))  in cik_enable_sck_slowdown_on_pd()
6385 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))  in cik_enable_cp_pg()
6398 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))  in cik_enable_gds_pg()
6414 	u32 bo_offset = 0;  in cik_init_cp_pg_table()
6417 	if (rdev->family == CHIP_KAVERI)  in cik_init_cp_pg_table()
6420 	if (rdev->rlc.cp_table_ptr == NULL)  in cik_init_cp_pg_table()
6424 	dst_ptr = rdev->rlc.cp_table_ptr;  in cik_init_cp_pg_table()
6425 	for (me = 0; me < max_me; me++) {  in cik_init_cp_pg_table()
6426 		if (rdev->new_fw) {  in cik_init_cp_pg_table()
6430 			if (me == 0) {  in cik_init_cp_pg_table()
6431 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;  in cik_init_cp_pg_table()
6433 					(rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6434 				table_offset = le32_to_cpu(hdr->jt_offset);  in cik_init_cp_pg_table()
6435 				table_size = le32_to_cpu(hdr->jt_size);  in cik_init_cp_pg_table()
6437 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;  in cik_init_cp_pg_table()
6439 					(rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6440 				table_offset = le32_to_cpu(hdr->jt_offset);  in cik_init_cp_pg_table()
6441 				table_size = le32_to_cpu(hdr->jt_size);  in cik_init_cp_pg_table()
6443 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;  in cik_init_cp_pg_table()
6445 					(rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6446 				table_offset = le32_to_cpu(hdr->jt_offset);  in cik_init_cp_pg_table()
6447 				table_size = le32_to_cpu(hdr->jt_size);  in cik_init_cp_pg_table()
6449 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;  in cik_init_cp_pg_table()
6451 					(rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6452 				table_offset = le32_to_cpu(hdr->jt_offset);  in cik_init_cp_pg_table()
6453 				table_size = le32_to_cpu(hdr->jt_size);  in cik_init_cp_pg_table()
6455 				hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;  in cik_init_cp_pg_table()
6457 					(rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));  in cik_init_cp_pg_table()
6458 				table_offset = le32_to_cpu(hdr->jt_offset);  in cik_init_cp_pg_table()
6459 				table_size = le32_to_cpu(hdr->jt_size);  in cik_init_cp_pg_table()
6462 			for (i = 0; i < table_size; i ++) {  in cik_init_cp_pg_table()
6471 			if (me == 0) {  in cik_init_cp_pg_table()
6472 				fw_data = (const __be32 *)rdev->ce_fw->data;  in cik_init_cp_pg_table()
6475 				fw_data = (const __be32 *)rdev->pfp_fw->data;  in cik_init_cp_pg_table()
6478 				fw_data = (const __be32 *)rdev->me_fw->data;  in cik_init_cp_pg_table()
6481 				fw_data = (const __be32 *)rdev->mec_fw->data;  in cik_init_cp_pg_table()
6485 			for (i = 0; i < table_size; i ++) {  in cik_init_cp_pg_table()
6499 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {  in cik_enable_gfx_cgpg()
6526 	u32 mask = 0, tmp, tmp1;  in cik_get_cu_active_bitmap()
6532 	cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);  in cik_get_cu_active_bitmap()
6534 	tmp &= 0xffff0000;  in cik_get_cu_active_bitmap()
6539 	for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {  in cik_get_cu_active_bitmap()
6549 	u32 i, j, k, active_cu_number = 0;  in cik_init_ao_cu_mask()
6551 	u32 tmp = 0;  in cik_init_ao_cu_mask()
6553 	for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {  in cik_init_ao_cu_mask()
6554 		for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {  in cik_init_ao_cu_mask()
6556 			cu_bitmap = 0;  in cik_init_ao_cu_mask()
6557 			counter = 0;  in cik_init_ao_cu_mask()
6558 			for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {  in cik_init_ao_cu_mask()
6586 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))  in cik_enable_gfx_static_mgpg()
6600 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))  in cik_enable_gfx_dynamic_mgpg()
6608 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6609 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6616 	if (rdev->rlc.cs_data) {  in cik_init_gfx_cgpg()
6618 		WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));  in cik_init_gfx_cgpg()
6619 		WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));  in cik_init_gfx_cgpg()
6620 		WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);  in cik_init_gfx_cgpg()
6623 		for (i = 0; i < 3; i++)  in cik_init_gfx_cgpg()
6624 			WREG32(RLC_GPM_SCRATCH_DATA, 0);  in cik_init_gfx_cgpg()
6626 	if (rdev->rlc.reg_list) {  in cik_init_gfx_cgpg()
6628 		for (i = 0; i < rdev->rlc.reg_list_size; i++)  in cik_init_gfx_cgpg()
6629 			WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);  in cik_init_gfx_cgpg()
6637 	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);  in cik_init_gfx_cgpg()
6638 	WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);  in cik_init_gfx_cgpg()
6642 	data |= IDLE_POLL_COUNT(0x60);  in cik_init_gfx_cgpg()
6645 	data = 0x10101010;  in cik_init_gfx_cgpg()
6649 	data &= ~0xff;  in cik_init_gfx_cgpg()
6650 	data |= 0x3;  in cik_init_gfx_cgpg()
6655 	data |= GRBM_REG_SGIT(0x700);  in cik_init_gfx_cgpg()
6669 	u32 count = 0;  in cik_get_csb_size()
6673 	if (rdev->rlc.cs_data == NULL)  in cik_get_csb_size()
6674 		return 0;  in cik_get_csb_size()
6681 	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {  in cik_get_csb_size()
6682 		for (ext = sect->section; ext->extent != NULL; ++ext) {  in cik_get_csb_size()
6683 			if (sect->id == SECT_CONTEXT)  in cik_get_csb_size()
6684 				count += 2 + ext->reg_count;  in cik_get_csb_size()
6686 				return 0;  in cik_get_csb_size()
6701 	u32 count = 0, i;  in cik_get_csb_buffer()
6705 	if (rdev->rlc.cs_data == NULL)  in cik_get_csb_buffer()
6710 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in cik_get_csb_buffer()
6714 	buffer[count++] = cpu_to_le32(0x80000000);  in cik_get_csb_buffer()
6715 	buffer[count++] = cpu_to_le32(0x80000000);  in cik_get_csb_buffer()
6717 	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {  in cik_get_csb_buffer()
6718 		for (ext = sect->section; ext->extent != NULL; ++ext) {  in cik_get_csb_buffer()
6719 			if (sect->id == SECT_CONTEXT) {  in cik_get_csb_buffer()
6721 					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));  in cik_get_csb_buffer()
6722 				buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);  in cik_get_csb_buffer()
6723 				for (i = 0; i < ext->reg_count; i++)  in cik_get_csb_buffer()
6724 					buffer[count++] = cpu_to_le32(ext->extent[i]);  in cik_get_csb_buffer()
6732 	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);  in cik_get_csb_buffer()
6733 	switch (rdev->family) {  in cik_get_csb_buffer()
6735 		buffer[count++] = cpu_to_le32(0x16000012);  in cik_get_csb_buffer()
6736 		buffer[count++] = cpu_to_le32(0x00000000);  in cik_get_csb_buffer()
6739 		buffer[count++] = cpu_to_le32(0x00000000); /* XXX */  in cik_get_csb_buffer()
6740 		buffer[count++] = cpu_to_le32(0x00000000);  in cik_get_csb_buffer()
6744 		buffer[count++] = cpu_to_le32(0x00000000); /* XXX */  in cik_get_csb_buffer()
6745 		buffer[count++] = cpu_to_le32(0x00000000);  in cik_get_csb_buffer()
6748 		buffer[count++] = cpu_to_le32(0x3a00161a);  in cik_get_csb_buffer()
6749 		buffer[count++] = cpu_to_le32(0x0000002e);  in cik_get_csb_buffer()
6752 		buffer[count++] = cpu_to_le32(0x00000000);  in cik_get_csb_buffer()
6753 		buffer[count++] = cpu_to_le32(0x00000000);  in cik_get_csb_buffer()
6757 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));  in cik_get_csb_buffer()
6760 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));  in cik_get_csb_buffer()
6761 	buffer[count++] = cpu_to_le32(0);  in cik_get_csb_buffer()
6766 	if (rdev->pg_flags) {  in cik_init_pg()
6769 		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {  in cik_init_pg()
6781 	if (rdev->pg_flags) {  in cik_fini_pg()
6783 		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {  in cik_fini_pg()
6806  * cik_enable_interrupts - Enable the interrupt ring buffer
6821 	rdev->ih.enabled = true;  in cik_enable_interrupts()
6825  * cik_disable_interrupts - Disable the interrupt ring buffer
6840 	/* set rptr, wptr to 0 */  in cik_disable_interrupts()
6841 	WREG32(IH_RB_RPTR, 0);  in cik_disable_interrupts()
6842 	WREG32(IH_RB_WPTR, 0);  in cik_disable_interrupts()
6843 	rdev->ih.enabled = false;  in cik_disable_interrupts()
6844 	rdev->ih.rptr = 0;  in cik_disable_interrupts()
6848  * cik_disable_interrupt_state - Disable all interrupt sources
6868 	WREG32(CP_ME1_PIPE0_INT_CNTL, 0);  in cik_disable_interrupt_state()
6869 	WREG32(CP_ME1_PIPE1_INT_CNTL, 0);  in cik_disable_interrupt_state()
6870 	WREG32(CP_ME1_PIPE2_INT_CNTL, 0);  in cik_disable_interrupt_state()
6871 	WREG32(CP_ME1_PIPE3_INT_CNTL, 0);  in cik_disable_interrupt_state()
6872 	WREG32(CP_ME2_PIPE0_INT_CNTL, 0);  in cik_disable_interrupt_state()
6873 	WREG32(CP_ME2_PIPE1_INT_CNTL, 0);  in cik_disable_interrupt_state()
6874 	WREG32(CP_ME2_PIPE2_INT_CNTL, 0);  in cik_disable_interrupt_state()
6875 	WREG32(CP_ME2_PIPE3_INT_CNTL, 0);  in cik_disable_interrupt_state()
6877 	WREG32(GRBM_INT_CNTL, 0);  in cik_disable_interrupt_state()
6879 	WREG32(SRBM_INT_CNTL, 0);  in cik_disable_interrupt_state()
6881 	WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6882 	WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6883 	if (rdev->num_crtc >= 4) {  in cik_disable_interrupt_state()
6884 		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6885 		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6887 	if (rdev->num_crtc >= 6) {  in cik_disable_interrupt_state()
6888 		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6889 		WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6892 	if (rdev->num_crtc >= 2) {  in cik_disable_interrupt_state()
6893 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6894 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6896 	if (rdev->num_crtc >= 4) {  in cik_disable_interrupt_state()
6897 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6898 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6900 	if (rdev->num_crtc >= 6) {  in cik_disable_interrupt_state()
6901 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6902 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);  in cik_disable_interrupt_state()
6906 	WREG32(DAC_AUTODETECT_INT_CONTROL, 0);  in cik_disable_interrupt_state()
6925  * cik_irq_init - init and enable the interrupt ring
6933  * Returns 0 for success, errors for failure.
6937 	int ret = 0;  in cik_irq_init()
6958 	WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);  in cik_irq_init()
6960 	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi  in cik_irq_init()
6961 	 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN  in cik_irq_init()
6964 	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */  in cik_irq_init()
6968 	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);  in cik_irq_init()
6969 	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);  in cik_irq_init()
6975 	if (rdev->wb.enabled)  in cik_irq_init()
6979 	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);  in cik_irq_init()
6980 	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);  in cik_irq_init()
6984 	/* set rptr, wptr to 0 */  in cik_irq_init()
6985 	WREG32(IH_RB_RPTR, 0);  in cik_irq_init()
6986 	WREG32(IH_RB_WPTR, 0);  in cik_irq_init()
6989 	ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);  in cik_irq_init()
6991 	if (rdev->msi_enabled)  in cik_irq_init()
6998 	pci_set_master(rdev->pdev);  in cik_irq_init()
7007  * cik_irq_set - enable/disable interrupt sources
7013  * Returns 0 for success, errors for failure.
7020 	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;  in cik_irq_set()
7022 	u32 grbm_int_cntl = 0;  in cik_irq_set()
7025 	if (!rdev->irq.installed) {  in cik_irq_set()
7027 		return -EINVAL;  in cik_irq_set()
7030 	if (!rdev->ih.enabled) {  in cik_irq_set()
7034 		return 0;  in cik_irq_set()
7061 	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {  in cik_irq_set()
7065 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {  in cik_irq_set()
7066 		struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_irq_set()
7068 		if (ring->me == 1) {  in cik_irq_set()
7069 			switch (ring->pipe) {  in cik_irq_set()
7070 			case 0:  in cik_irq_set()
7083 				DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);  in cik_irq_set()
7086 		} else if (ring->me == 2) {  in cik_irq_set()
7087 			switch (ring->pipe) {  in cik_irq_set()
7088 			case 0:  in cik_irq_set()
7101 				DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);  in cik_irq_set()
7105 			DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);  in cik_irq_set()
7108 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {  in cik_irq_set()
7109 		struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_irq_set()
7111 		if (ring->me == 1) {  in cik_irq_set()
7112 			switch (ring->pipe) {  in cik_irq_set()
7113 			case 0:  in cik_irq_set()
7126 				DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);  in cik_irq_set()
7129 		} else if (ring->me == 2) {  in cik_irq_set()
7130 			switch (ring->pipe) {  in cik_irq_set()
7131 			case 0:  in cik_irq_set()
7144 				DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);  in cik_irq_set()
7148 			DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);  in cik_irq_set()
7152 	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {  in cik_irq_set()
7157 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {  in cik_irq_set()
7162 	if (rdev->irq.crtc_vblank_int[0] ||  in cik_irq_set()
7163 	    atomic_read(&rdev->irq.pflip[0])) {  in cik_irq_set()
7164 		DRM_DEBUG("cik_irq_set: vblank 0\n");  in cik_irq_set()
7167 	if (rdev->irq.crtc_vblank_int[1] ||  in cik_irq_set()
7168 	    atomic_read(&rdev->irq.pflip[1])) {  in cik_irq_set()
7172 	if (rdev->irq.crtc_vblank_int[2] ||  in cik_irq_set()
7173 	    atomic_read(&rdev->irq.pflip[2])) {  in cik_irq_set()
7177 	if (rdev->irq.crtc_vblank_int[3] ||  in cik_irq_set()
7178 	    atomic_read(&rdev->irq.pflip[3])) {  in cik_irq_set()
7182 	if (rdev->irq.crtc_vblank_int[4] ||  in cik_irq_set()
7183 	    atomic_read(&rdev->irq.pflip[4])) {  in cik_irq_set()
7187 	if (rdev->irq.crtc_vblank_int[5] ||  in cik_irq_set()
7188 	    atomic_read(&rdev->irq.pflip[5])) {  in cik_irq_set()
7192 	if (rdev->irq.hpd[0]) {  in cik_irq_set()
7196 	if (rdev->irq.hpd[1]) {  in cik_irq_set()
7200 	if (rdev->irq.hpd[2]) {  in cik_irq_set()
7204 	if (rdev->irq.hpd[3]) {  in cik_irq_set()
7208 	if (rdev->irq.hpd[4]) {  in cik_irq_set()
7212 	if (rdev->irq.hpd[5]) {  in cik_irq_set()
7235 	if (rdev->num_crtc >= 4) {  in cik_irq_set()
7239 	if (rdev->num_crtc >= 6) {  in cik_irq_set()
7244 	if (rdev->num_crtc >= 2) {  in cik_irq_set()
7250 	if (rdev->num_crtc >= 4) {  in cik_irq_set()
7256 	if (rdev->num_crtc >= 6) {  in cik_irq_set()
7273 	return 0;  in cik_irq_set()
7277  * cik_irq_ack - ack interrupt sources
7289 	rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);  in cik_irq_ack()
7290 	rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);  in cik_irq_ack()
7291 	rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);  in cik_irq_ack()
7292 	rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);  in cik_irq_ack()
7293 	rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);  in cik_irq_ack()
7294 	rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);  in cik_irq_ack()
7295 	rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);  in cik_irq_ack()
7297 	rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7299 	rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7301 	if (rdev->num_crtc >= 4) {  in cik_irq_ack()
7302 		rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7304 		rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7307 	if (rdev->num_crtc >= 6) {  in cik_irq_ack()
7308 		rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7310 		rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +  in cik_irq_ack()
7314 	if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7317 	if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7320 	if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)  in cik_irq_ack()
7322 	if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)  in cik_irq_ack()
7324 	if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)  in cik_irq_ack()
7326 	if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)  in cik_irq_ack()
7329 	if (rdev->num_crtc >= 4) {  in cik_irq_ack()
7330 		if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7333 		if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7336 		if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)  in cik_irq_ack()
7338 		if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)  in cik_irq_ack()
7340 		if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)  in cik_irq_ack()
7342 		if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)  in cik_irq_ack()
7346 	if (rdev->num_crtc >= 6) {  in cik_irq_ack()
7347 		if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7350 		if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)  in cik_irq_ack()
7353 		if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)  in cik_irq_ack()
7355 		if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)  in cik_irq_ack()
7357 		if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)  in cik_irq_ack()
7359 		if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)  in cik_irq_ack()
7363 	if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {  in cik_irq_ack()
7368 	if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {  in cik_irq_ack()
7373 	if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {  in cik_irq_ack()
7378 	if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {  in cik_irq_ack()
7383 	if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {  in cik_irq_ack()
7388 	if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {  in cik_irq_ack()
7393 	if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {  in cik_irq_ack()
7398 	if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {  in cik_irq_ack()
7403 	if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {  in cik_irq_ack()
7408 	if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {  in cik_irq_ack()
7413 	if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {  in cik_irq_ack()
7418 	if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {  in cik_irq_ack()
7426  * cik_irq_disable - disable interrupts
7442  * cik_irq_suspend - disable interrupts for suspend
7456  * cik_irq_fini - tear down interrupt support
7471  * cik_get_ih_wptr - get the IH ring buffer wptr
7485 	if (rdev->wb.enabled)  in cik_get_ih_wptr()
7486 		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);  in cik_get_ih_wptr()
7496 		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",  in cik_get_ih_wptr()
7497 			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);  in cik_get_ih_wptr()
7498 		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;  in cik_get_ih_wptr()
7503 	return (wptr & rdev->ih.ptr_mask);  in cik_get_ih_wptr()
7508  * [7:0]    - interrupt source id
7509  * [31:8]   - reserved
7510  * [59:32]  - interrupt source data
7511  * [63:60]  - reserved
7512  * [71:64]  - RINGID
7514  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7515  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7516  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7517  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7518  *            PIPE_ID - ME0 0=3D
7519  *                    - ME1&2 compute dispatcher (4 pipes each)
7521  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7522  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7523  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7524  * [79:72]  - VMID
7525  * [95:80]  - PASID
7526  * [127:96] - reserved
7529  * cik_irq_process - interrupt handler
7540 	struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_irq_process()
7541 	struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_irq_process()
7553 	if (!rdev->ih.enabled || rdev->shutdown)  in cik_irq_process()
7560 	if (atomic_xchg(&rdev->ih.lock, 1))  in cik_irq_process()
7563 	rptr = rdev->ih.rptr;  in cik_irq_process()
7576 		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;  in cik_irq_process()
7577 		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;  in cik_irq_process()
7578 		ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;  in cik_irq_process()
7583 			case 0: /* D1 vblank */  in cik_irq_process()
7584 				if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))  in cik_irq_process()
7587 				if (rdev->irq.crtc_vblank_int[0]) {  in cik_irq_process()
7588 					drm_handle_vblank(rdev_to_drm(rdev), 0);  in cik_irq_process()
7589 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7590 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7592 				if (atomic_read(&rdev->irq.pflip[0]))  in cik_irq_process()
7593 					radeon_crtc_handle_vblank(rdev, 0);  in cik_irq_process()
7594 				rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;  in cik_irq_process()
7599 				if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))  in cik_irq_process()
7602 				rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;  in cik_irq_process()
7613 			case 0: /* D2 vblank */  in cik_irq_process()
7614 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))  in cik_irq_process()
7617 				if (rdev->irq.crtc_vblank_int[1]) {  in cik_irq_process()
7619 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7620 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7622 				if (atomic_read(&rdev->irq.pflip[1]))  in cik_irq_process()
7624 				rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;  in cik_irq_process()
7629 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))  in cik_irq_process()
7632 				rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;  in cik_irq_process()
7643 			case 0: /* D3 vblank */  in cik_irq_process()
7644 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))  in cik_irq_process()
7647 				if (rdev->irq.crtc_vblank_int[2]) {  in cik_irq_process()
7649 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7650 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7652 				if (atomic_read(&rdev->irq.pflip[2]))  in cik_irq_process()
7654 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;  in cik_irq_process()
7659 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))  in cik_irq_process()
7662 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;  in cik_irq_process()
7673 			case 0: /* D4 vblank */  in cik_irq_process()
7674 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))  in cik_irq_process()
7677 				if (rdev->irq.crtc_vblank_int[3]) {  in cik_irq_process()
7679 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7680 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7682 				if (atomic_read(&rdev->irq.pflip[3]))  in cik_irq_process()
7684 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;  in cik_irq_process()
7689 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))  in cik_irq_process()
7692 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;  in cik_irq_process()
7703 			case 0: /* D5 vblank */  in cik_irq_process()
7704 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))  in cik_irq_process()
7707 				if (rdev->irq.crtc_vblank_int[4]) {  in cik_irq_process()
7709 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7710 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7712 				if (atomic_read(&rdev->irq.pflip[4]))  in cik_irq_process()
7714 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;  in cik_irq_process()
7719 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))  in cik_irq_process()
7722 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;  in cik_irq_process()
7733 			case 0: /* D6 vblank */  in cik_irq_process()
7734 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))  in cik_irq_process()
7737 				if (rdev->irq.crtc_vblank_int[5]) {  in cik_irq_process()
7739 					rdev->pm.vblank_sync = true;  in cik_irq_process()
7740 					wake_up(&rdev->irq.vblank_queue);  in cik_irq_process()
7742 				if (atomic_read(&rdev->irq.pflip[5]))  in cik_irq_process()
7744 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;  in cik_irq_process()
7749 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))  in cik_irq_process()
7752 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;  in cik_irq_process()
7767 			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);  in cik_irq_process()
7768 			if (radeon_use_pflipirq > 0)  in cik_irq_process()
7769 				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);  in cik_irq_process()
7773 			case 0:  in cik_irq_process()
7774 				if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))  in cik_irq_process()
7777 				rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;  in cik_irq_process()
7783 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))  in cik_irq_process()
7786 				rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;  in cik_irq_process()
7792 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))  in cik_irq_process()
7795 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;  in cik_irq_process()
7801 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))  in cik_irq_process()
7804 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;  in cik_irq_process()
7810 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))  in cik_irq_process()
7813 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;  in cik_irq_process()
7819 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))  in cik_irq_process()
7822 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;  in cik_irq_process()
7828 				if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))  in cik_irq_process()
7831 				rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;  in cik_irq_process()
7837 				if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))  in cik_irq_process()
7840 				rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;  in cik_irq_process()
7846 				if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))  in cik_irq_process()
7849 				rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;  in cik_irq_process()
7855 				if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))  in cik_irq_process()
7858 				rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;  in cik_irq_process()
7864 				if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))  in cik_irq_process()
7867 				rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;  in cik_irq_process()
7873 				if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))  in cik_irq_process()
7876 				rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;  in cik_irq_process()
7887 			DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));  in cik_irq_process()
7888 			WREG32(SRBM_INT_ACK, 0x1);  in cik_irq_process()
7891 			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);  in cik_irq_process()
7901 			if (addr == 0x0 && status == 0x0)  in cik_irq_process()
7903 			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);  in cik_irq_process()
7904 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",  in cik_irq_process()
7906 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",  in cik_irq_process()
7911 			DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);  in cik_irq_process()
7913 			case 0:  in cik_irq_process()
7931 			me_id = (ring_id & 0x60) >> 5;  in cik_irq_process()
7932 			pipe_id = (ring_id & 0x18) >> 3;  in cik_irq_process()
7933 			queue_id = (ring_id & 0x7) >> 0;  in cik_irq_process()
7935 			case 0:  in cik_irq_process()
7940 				if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))  in cik_irq_process()
7942 				if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))  in cik_irq_process()
7950 			me_id = (ring_id & 0x60) >> 5;  in cik_irq_process()
7952 			case 0:  in cik_irq_process()
7971 			me_id = (ring_id & 0x60) >> 5;  in cik_irq_process()
7973 			case 0:  in cik_irq_process()
7991 			me_id = (ring_id & 0x3) >> 0;  in cik_irq_process()
7992 			queue_id = (ring_id & 0xc) >> 2;  in cik_irq_process()
7995 			case 0:  in cik_irq_process()
7997 				case 0:  in cik_irq_process()
8010 				case 0:  in cik_irq_process()
8025 			rdev->pm.dpm.thermal.high_to_low = false;  in cik_irq_process()
8030 			rdev->pm.dpm.thermal.high_to_low = true;  in cik_irq_process()
8040 			me_id = (ring_id & 0x3) >> 0;  in cik_irq_process()
8041 			queue_id = (ring_id & 0xc) >> 2;  in cik_irq_process()
8043 			case 0:  in cik_irq_process()
8045 				case 0:  in cik_irq_process()
8060 				case 0:  in cik_irq_process()
8082 		rptr &= rdev->ih.ptr_mask;  in cik_irq_process()
8086 		schedule_work(&rdev->dp_work);  in cik_irq_process()
8088 		schedule_delayed_work(&rdev->hotplug_work, 0);  in cik_irq_process()
8090 		rdev->needs_reset = true;  in cik_irq_process()
8091 		wake_up_all(&rdev->fence_queue);  in cik_irq_process()
8094 		schedule_work(&rdev->pm.dpm.thermal.work);  in cik_irq_process()
8095 	rdev->ih.rptr = rptr;  in cik_irq_process()
8096 	atomic_set(&rdev->ih.lock, 0);  in cik_irq_process()
8113 	if (!rdev->has_uvd)  in cik_uvd_init()
8118 		dev_err(rdev->dev, "failed UVD (%d) init.\n", r);  in cik_uvd_init()
8120 		 * At this point rdev->uvd.vcpu_bo is NULL which trickles down  in cik_uvd_init()
8125 		rdev->has_uvd = false;  in cik_uvd_init()
8128 	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;  in cik_uvd_init()
8129 	r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);  in cik_uvd_init()
8136 	if (!rdev->has_uvd)  in cik_uvd_start()
8141 		dev_err(rdev->dev, "failed UVD resume (%d).\n", r);  in cik_uvd_start()
8146 		dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);  in cik_uvd_start()
8151 		dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);  in cik_uvd_start()
8157 	rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;  in cik_uvd_start()
8165 	if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)  in cik_uvd_resume()
8168 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];  in cik_uvd_resume()
8169 	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));  in cik_uvd_resume()
8171 		dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);  in cik_uvd_resume()
8176 		dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);  in cik_uvd_resume()
8185 	if (!rdev->has_vce)  in cik_vce_init()
8190 		dev_err(rdev->dev, "failed VCE (%d) init.\n", r);  in cik_vce_init()
8192 		 * At this point rdev->vce.vcpu_bo is NULL which trickles down  in cik_vce_init()
8197 		rdev->has_vce = false;  in cik_vce_init()
8200 	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;  in cik_vce_init()
8201 	r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);  in cik_vce_init()
8202 	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;  in cik_vce_init()
8203 	r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);  in cik_vce_init()
8210 	if (!rdev->has_vce)  in cik_vce_start()
8215 		dev_err(rdev->dev, "failed VCE resume (%d).\n", r);  in cik_vce_start()
8220 		dev_err(rdev->dev, "failed VCE resume (%d).\n", r);  in cik_vce_start()
8225 		dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);  in cik_vce_start()
8230 		dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);  in cik_vce_start()
8236 	rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;  in cik_vce_start()
8237 	rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;  in cik_vce_start()
8245 	if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)  in cik_vce_resume()
8248 	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];  in cik_vce_resume()
8249 	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);  in cik_vce_resume()
8251 		dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);  in cik_vce_resume()
8254 	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];  in cik_vce_resume()
8255 	r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);  in cik_vce_resume()
8257 		dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);  in cik_vce_resume()
8262 		dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);  in cik_vce_resume()
8268  * cik_startup - program the asic to a functional state
8274  * Returns 0 for success, error for failure.
8294 	if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {  in cik_startup()
8308 	if (rdev->flags & RADEON_IS_IGP) {  in cik_startup()
8309 		if (rdev->family == CHIP_KAVERI) {  in cik_startup()
8310 			rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;  in cik_startup()
8311 			rdev->rlc.reg_list_size =  in cik_startup()
8314 			rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;  in cik_startup()
8315 			rdev->rlc.reg_list_size =  in cik_startup()
8319 	rdev->rlc.cs_data = ci_cs_data;  in cik_startup()
8320 	rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */  in cik_startup()
8321 	rdev->rlc.cp_table_size += 64 * 1024; /* GDS */  in cik_startup()
8342 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cik_startup()
8348 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cik_startup()
8354 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);  in cik_startup()
8360 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in cik_startup()
8366 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);  in cik_startup()
8374 	if (!rdev->irq.installed) {  in cik_startup()
8388 	if (rdev->family == CHIP_HAWAII) {  in cik_startup()
8389 		if (rdev->new_fw)  in cik_startup()
8390 			nop = PACKET3(PACKET3_NOP, 0x3FFF);  in cik_startup()
8394 		nop = PACKET3(PACKET3_NOP, 0x3FFF);  in cik_startup()
8397 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_startup()
8398 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,  in cik_startup()
8404 	/* type-2 packets are deprecated on MEC, use type-3 instead */  in cik_startup()
8405 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_startup()
8406 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,  in cik_startup()
8410 	ring->me = 1; /* first MEC */  in cik_startup()
8411 	ring->pipe = 0; /* first pipe */  in cik_startup()
8412 	ring->queue = 0; /* first queue */  in cik_startup()
8413 	ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;  in cik_startup()
8415 	/* type-2 packets are deprecated on MEC, use type-3 instead */  in cik_startup()
8416 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_startup()
8417 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,  in cik_startup()
8422 	ring->me = 1; /* first MEC */  in cik_startup()
8423 	ring->pipe = 0; /* first pipe */  in cik_startup()
8424 	ring->queue = 1; /* second queue */  in cik_startup()
8425 	ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;  in cik_startup()
8427 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cik_startup()
8428 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,  in cik_startup()
8429 			     SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));  in cik_startup()
8433 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cik_startup()
8434 	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,  in cik_startup()
8435 			     SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));  in cik_startup()
8452 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);  in cik_startup()
8458 		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);  in cik_startup()
8466 	return 0;  in cik_startup()
8470  * cik_resume - resume the asic to a functional state
8476  * Returns 0 for success, error for failure.
8483 	atom_asic_init(rdev->mode_info.atom_context);  in cik_resume()
8488 	if (rdev->pm.pm_method == PM_METHOD_DPM)  in cik_resume()
8491 	rdev->accel_working = true;  in cik_resume()
8495 		rdev->accel_working = false;  in cik_resume()
8504  * cik_suspend - suspend the asic
8510  * Returns 0 for success.
8519 	if (rdev->has_uvd) {  in cik_suspend()
8523 	if (rdev->has_vce)  in cik_suspend()
8530 	return 0;  in cik_suspend()
8540  * cik_init - asic specific driver and hw init
8547  * Returns 0 for success, errors for failure.
8557 			return -EINVAL;  in cik_init()
8560 	if (!rdev->is_atom_bios) {  in cik_init()
8561 		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");  in cik_init()
8562 		return -EINVAL;  in cik_init()
8570 		if (!rdev->bios) {  in cik_init()
8571 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");  in cik_init()
8572 			return -EINVAL;  in cik_init()
8575 		atom_asic_init(rdev->mode_info.atom_context);  in cik_init()
8598 	if (rdev->flags & RADEON_IS_IGP) {  in cik_init()
8599 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||  in cik_init()
8600 		    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {  in cik_init()
8608 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||  in cik_init()
8609 		    !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||  in cik_init()
8610 		    !rdev->mc_fw) {  in cik_init()
8622 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];  in cik_init()
8623 	ring->ring_obj = NULL;  in cik_init()
8626 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];  in cik_init()
8627 	ring->ring_obj = NULL;  in cik_init()
8629 	r = radeon_doorbell_get(rdev, &ring->doorbell_index);  in cik_init()
8633 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];  in cik_init()
8634 	ring->ring_obj = NULL;  in cik_init()
8636 	r = radeon_doorbell_get(rdev, &ring->doorbell_index);  in cik_init()
8640 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];  in cik_init()
8641 	ring->ring_obj = NULL;  in cik_init()
8644 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];  in cik_init()
8645 	ring->ring_obj = NULL;  in cik_init()
8651 	rdev->ih.ring_obj = NULL;  in cik_init()
8658 	rdev->accel_working = true;  in cik_init()
8661 		dev_err(rdev->dev, "disabling GPU acceleration\n");  in cik_init()
8672 		rdev->accel_working = false;  in cik_init()
8679 	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {  in cik_init()
8681 		return -EINVAL;  in cik_init()
8684 	return 0;  in cik_init()
8688  * cik_fini - asic specific driver and hw fini
8719 	kfree(rdev->bios);  in cik_fini()
8720 	rdev->bios = NULL;  in cik_fini()
8725 	struct drm_device *dev = encoder->dev;  in dce8_program_fmt()
8726 	struct radeon_device *rdev = dev->dev_private;  in dce8_program_fmt()
8728 	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);  in dce8_program_fmt()
8730 	int bpc = 0;  in dce8_program_fmt()
8731 	u32 tmp = 0;  in dce8_program_fmt()
8737 		dither = radeon_connector->dither;  in dce8_program_fmt()
8741 	if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)  in dce8_program_fmt()
8745 	if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||  in dce8_program_fmt()
8746 	    (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))  in dce8_program_fmt()
8749 	if (bpc == 0)  in dce8_program_fmt()
8757 				FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));  in dce8_program_fmt()
8759 			tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));  in dce8_program_fmt()
8784 	WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);  in dce8_program_fmt()
8789  * dce8_line_buffer_adjust - Set up the line buffer
8805 	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;  in dce8_line_buffer_adjust()
8814 	if (radeon_crtc->base.enabled && mode) {  in dce8_line_buffer_adjust()
8815 		if (mode->crtc_hdisplay < 1920) {  in dce8_line_buffer_adjust()
8818 		} else if (mode->crtc_hdisplay < 2560) {  in dce8_line_buffer_adjust()
8821 		} else if (mode->crtc_hdisplay < 4096) {  in dce8_line_buffer_adjust()
8822 			tmp = 0;  in dce8_line_buffer_adjust()
8823 			buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;  in dce8_line_buffer_adjust()
8826 			tmp = 0;  in dce8_line_buffer_adjust()
8827 			buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;  in dce8_line_buffer_adjust()
8831 		buffer_alloc = 0;  in dce8_line_buffer_adjust()
8834 	WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,  in dce8_line_buffer_adjust()
8835 	       LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));  in dce8_line_buffer_adjust()
8839 	for (i = 0; i < rdev->usec_timeout; i++) {  in dce8_line_buffer_adjust()
8846 	if (radeon_crtc->base.enabled && mode) {  in dce8_line_buffer_adjust()
8848 		case 0:  in dce8_line_buffer_adjust()
8859 	return 0;  in dce8_line_buffer_adjust()
8863  * cik_get_number_of_dram_channels - get the number of dram channels
8876 	case 0:  in cik_get_number_of_dram_channels()
8915  * dce8_dram_bandwidth - get the dram bandwidth
8931 	yclk.full = dfixed_const(wm->yclk);  in dce8_dram_bandwidth()
8933 	dram_channels.full = dfixed_const(wm->dram_channels * 4);  in dce8_dram_bandwidth()
8944  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8960 	yclk.full = dfixed_const(wm->yclk);  in dce8_dram_bandwidth_for_display()
8962 	dram_channels.full = dfixed_const(wm->dram_channels * 4);  in dce8_dram_bandwidth_for_display()
8973  * dce8_data_return_bandwidth - get the data return bandwidth
8989 	sclk.full = dfixed_const(wm->sclk);  in dce8_data_return_bandwidth()
9002  * dce8_dmif_request_bandwidth - get the dmif bandwidth
9018 	disp_clk.full = dfixed_const(wm->disp_clk);  in dce8_dmif_request_bandwidth()
9033  * dce8_available_bandwidth - get the min available bandwidth
9052  * dce8_average_bandwidth - get the average available bandwidth
9073 	line_time.full = dfixed_const(wm->active_time + wm->blank_time);  in dce8_average_bandwidth()
9075 	bpp.full = dfixed_const(wm->bytes_per_pixel);  in dce8_average_bandwidth()
9076 	src_width.full = dfixed_const(wm->src_width);  in dce8_average_bandwidth()
9078 	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);  in dce8_average_bandwidth()
9085  * dce8_latency_watermark - get the latency watermark
9100 	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */  in dce8_latency_watermark()
9101 	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +  in dce8_latency_watermark()
9102 		(wm->num_heads * cursor_line_pair_return_time);  in dce8_latency_watermark()
9108 	if (wm->num_heads == 0)  in dce8_latency_watermark()
9109 		return 0;  in dce8_latency_watermark()
9113 	if ((wm->vsc.full > a.full) ||  in dce8_latency_watermark()
9114 	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||  in dce8_latency_watermark()
9115 	    (wm->vtaps >= 5) ||  in dce8_latency_watermark()
9116 	    ((wm->vsc.full >= a.full) && wm->interlaced))  in dce8_latency_watermark()
9122 	b.full = dfixed_const(wm->num_heads);  in dce8_latency_watermark()
9124 	tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);  in dce8_latency_watermark()
9127 	lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);  in dce8_latency_watermark()
9129 	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);  in dce8_latency_watermark()
9136 	if (line_fill_time < wm->active_time)  in dce8_latency_watermark()
9139 		return latency + (line_fill_time - wm->active_time);  in dce8_latency_watermark()
9144  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9157 	    (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))  in dce8_average_bandwidth_vs_dram_bandwidth_for_display()
9164  * dce8_average_bandwidth_vs_available_bandwidth - check
9177 	    (dce8_available_bandwidth(wm) / wm->num_heads))  in dce8_average_bandwidth_vs_available_bandwidth()
9184  * dce8_check_latency_hiding - check latency hiding
9194 	u32 lb_partitions = wm->lb_size / wm->src_width;  in dce8_check_latency_hiding()
9195 	u32 line_time = wm->active_time + wm->blank_time;  in dce8_check_latency_hiding()
9201 	if (wm->vsc.full > a.full)  in dce8_check_latency_hiding()
9204 		if (lb_partitions <= (wm->vtaps + 1))  in dce8_check_latency_hiding()
9210 	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);  in dce8_check_latency_hiding()
9219  * dce8_program_watermarks - program display watermarks
9233 	struct drm_display_mode *mode = &radeon_crtc->base.mode;  in dce8_program_watermarks()
9236 	u32 line_time = 0;  in dce8_program_watermarks()
9237 	u32 latency_watermark_a = 0, latency_watermark_b = 0;  in dce8_program_watermarks()
9240 	if (radeon_crtc->base.enabled && num_heads && mode) {  in dce8_program_watermarks()
9241 		active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,  in dce8_program_watermarks()
9242 					    (u32)mode->clock);  in dce8_program_watermarks()
9243 		line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,  in dce8_program_watermarks()
9244 					  (u32)mode->clock);  in dce8_program_watermarks()
9248 		if ((rdev->pm.pm_method == PM_METHOD_DPM) &&  in dce8_program_watermarks()
9249 		    rdev->pm.dpm_enabled) {  in dce8_program_watermarks()
9255 			wm_high.yclk = rdev->pm.current_mclk * 10;  in dce8_program_watermarks()
9256 			wm_high.sclk = rdev->pm.current_sclk * 10;  in dce8_program_watermarks()
9259 		wm_high.disp_clk = mode->clock;  in dce8_program_watermarks()
9260 		wm_high.src_width = mode->crtc_hdisplay;  in dce8_program_watermarks()
9262 		wm_high.blank_time = line_time - wm_high.active_time;  in dce8_program_watermarks()
9264 		if (mode->flags & DRM_MODE_FLAG_INTERLACE)  in dce8_program_watermarks()
9266 		wm_high.vsc = radeon_crtc->vsc;  in dce8_program_watermarks()
9268 		if (radeon_crtc->rmx_type != RMX_OFF)  in dce8_program_watermarks()
9283 		    (rdev->disp_priority == 2)) {  in dce8_program_watermarks()
9288 		if ((rdev->pm.pm_method == PM_METHOD_DPM) &&  in dce8_program_watermarks()
9289 		    rdev->pm.dpm_enabled) {  in dce8_program_watermarks()
9295 			wm_low.yclk = rdev->pm.current_mclk * 10;  in dce8_program_watermarks()
9296 			wm_low.sclk = rdev->pm.current_sclk * 10;  in dce8_program_watermarks()
9299 		wm_low.disp_clk = mode->clock;  in dce8_program_watermarks()
9300 		wm_low.src_width = mode->crtc_hdisplay;  in dce8_program_watermarks()
9302 		wm_low.blank_time = line_time - wm_low.active_time;  in dce8_program_watermarks()
9304 		if (mode->flags & DRM_MODE_FLAG_INTERLACE)  in dce8_program_watermarks()
9306 		wm_low.vsc = radeon_crtc->vsc;  in dce8_program_watermarks()
9308 		if (radeon_crtc->rmx_type != RMX_OFF)  in dce8_program_watermarks()
9323 		    (rdev->disp_priority == 2)) {  in dce8_program_watermarks()
9328 		radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);  in dce8_program_watermarks()
9332 	wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);  in dce8_program_watermarks()
9336 	WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);  in dce8_program_watermarks()
9337 	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,  in dce8_program_watermarks()
9341 	tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);  in dce8_program_watermarks()
9344 	WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);  in dce8_program_watermarks()
9345 	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,  in dce8_program_watermarks()
9349 	WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);  in dce8_program_watermarks()
9352 	radeon_crtc->line_time = line_time;  in dce8_program_watermarks()
9353 	radeon_crtc->wm_high = latency_watermark_a;  in dce8_program_watermarks()
9354 	radeon_crtc->wm_low = latency_watermark_b;  in dce8_program_watermarks()
9358  * dce8_bandwidth_update - program display watermarks
9368 	u32 num_heads = 0, lb_size;  in dce8_bandwidth_update()
9371 	if (!rdev->mode_info.mode_config_initialized)  in dce8_bandwidth_update()
9376 	for (i = 0; i < rdev->num_crtc; i++) {  in dce8_bandwidth_update()
9377 		if (rdev->mode_info.crtcs[i]->base.enabled)  in dce8_bandwidth_update()
9380 	for (i = 0; i < rdev->num_crtc; i++) {  in dce8_bandwidth_update()
9381 		mode = &rdev->mode_info.crtcs[i]->base.mode;  in dce8_bandwidth_update()
9382 		lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);  in dce8_bandwidth_update()
9383 		dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);  in dce8_bandwidth_update()
9388  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9399 	mutex_lock(&rdev->gpu_clock_mutex);  in cik_get_gpu_clock_counter()
9403 	mutex_unlock(&rdev->gpu_clock_mutex);  in cik_get_gpu_clock_counter()
9424 	for (i = 0; i < 100; i++) {  in cik_set_uvd_clock()
9430 		return -ETIMEDOUT;  in cik_set_uvd_clock()
9432 	return 0;  in cik_set_uvd_clock()
9437 	int r = 0;  in cik_set_uvd_clocks()
9458 	for (i = 0; i < 100; i++) {  in cik_set_vce_clocks()
9464 		return -ETIMEDOUT;  in cik_set_vce_clocks()
9471 	for (i = 0; i < 100; i++) {  in cik_set_vce_clocks()
9477 		return -ETIMEDOUT;  in cik_set_vce_clocks()
9479 	return 0;  in cik_set_vce_clocks()
9484 	struct pci_dev *root = rdev->pdev->bus->self;  in cik_pcie_gen3_enable()
9490 	if (pci_is_root_bus(rdev->pdev->bus))  in cik_pcie_gen3_enable()
9493 	if (radeon_pcie_gen2 == 0)  in cik_pcie_gen3_enable()
9496 	if (rdev->flags & RADEON_IS_IGP)  in cik_pcie_gen3_enable()
9499 	if (!(rdev->flags & RADEON_IS_PCIE))  in cik_pcie_gen3_enable()
9518 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");  in cik_pcie_gen3_enable()
9524 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");  in cik_pcie_gen3_enable()
9527 	if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))  in cik_pcie_gen3_enable()
9531 		/* re-try equalization if gen3 is not already enabled */  in cik_pcie_gen3_enable()
9538 			pcie_capability_set_word(rdev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);  in cik_pcie_gen3_enable()
9554 			for (i = 0; i < 10; i++) {  in cik_pcie_gen3_enable()
9556 				pcie_capability_read_word(rdev->pdev,  in cik_pcie_gen3_enable()
9564 				pcie_capability_read_word(rdev->pdev,  in cik_pcie_gen3_enable()
9570 				pcie_capability_read_word(rdev->pdev,  in cik_pcie_gen3_enable()
9589 				pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL,  in cik_pcie_gen3_enable()
9601 				pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL2,  in cik_pcie_gen3_enable()
9620 	tmp16 = 0;  in cik_pcie_gen3_enable()
9627 	pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL2,  in cik_pcie_gen3_enable()
9634 	for (i = 0; i < rdev->usec_timeout; i++) {  in cik_pcie_gen3_enable()
9636 		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)  in cik_pcie_gen3_enable()
9648 	if (radeon_aspm == 0)  in cik_program_aspm()
9652 	if (rdev->flags & RADEON_IS_IGP)  in cik_program_aspm()
9655 	if (!(rdev->flags & RADEON_IS_PCIE))  in cik_program_aspm()
9660 	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;  in cik_program_aspm()
9720 			    !pci_is_root_bus(rdev->pdev->bus)) {  in cik_program_aspm()
9721 				struct pci_dev *root = rdev->pdev->bus->self;  in cik_program_aspm()