grufault.c (2ce4d4c937ce4c14704f02ae60e2e07e3906c68a) grufault.c (5958ab88f721d312b531d17705fc3ed54102fa05)
1/*
2 * SN Platform GRU Driver
3 *
4 * FAULT HANDLER FOR GRU DETECTED TLB MISSES
5 *
6 * This file contains code that handles TLB misses within the GRU.
7 * These misses are reported either via interrupts or user polling of
8 * the user CB.

--- 424 unchanged lines hidden (view full) ---

433 }
434
435 if (unlikely(cbe) && pageshift == PAGE_SHIFT) {
436 gru_preload_tlb(gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe);
437 gru_flush_cache_cbe(cbe);
438 }
439
440 gru_cb_set_istatus_active(cbk);
1/*
2 * SN Platform GRU Driver
3 *
4 * FAULT HANDLER FOR GRU DETECTED TLB MISSES
5 *
6 * This file contains code that handles TLB misses within the GRU.
7 * These misses are reported either via interrupts or user polling of
8 * the user CB.

--- 424 unchanged lines hidden (view full) ---

433 }
434
435 if (unlikely(cbe) && pageshift == PAGE_SHIFT) {
436 gru_preload_tlb(gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe);
437 gru_flush_cache_cbe(cbe);
438 }
439
440 gru_cb_set_istatus_active(cbk);
441 gts->ustats.tlbdropin++;
441 tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
442 GRU_PAGESIZE(pageshift));
443 gru_dbg(grudev,
444 "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, indexway 0x%x,"
445 " rw %d, ps %d, gpa 0x%lx\n",
446 atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, vaddr, asid,
447 indexway, write, pageshift, gpa);
448 STAT(tlb_dropin);

--- 126 unchanged lines hidden (view full) ---

575 STAT(intr_spurious);
576 continue;
577 }
578
579 /*
580 * This is running in interrupt context. Trylock the mmap_sem.
581 * If it fails, retry the fault in user context.
582 */
442 tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
443 GRU_PAGESIZE(pageshift));
444 gru_dbg(grudev,
445 "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, indexway 0x%x,"
446 " rw %d, ps %d, gpa 0x%lx\n",
447 atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, vaddr, asid,
448 indexway, write, pageshift, gpa);
449 STAT(tlb_dropin);

--- 126 unchanged lines hidden (view full) ---

576 STAT(intr_spurious);
577 continue;
578 }
579
580 /*
581 * This is running in interrupt context. Trylock the mmap_sem.
582 * If it fails, retry the fault in user context.
583 */
584 gts->ustats.fmm_tlbmiss++;
583 if (!gts->ts_force_cch_reload &&
584 down_read_trylock(&gts->ts_mm->mmap_sem)) {
585 if (!gts->ts_force_cch_reload &&
586 down_read_trylock(&gts->ts_mm->mmap_sem)) {
585 gts->ustats.fmm_tlbdropin++;
586 gru_try_dropin(gru, gts, tfh, NULL);
587 up_read(&gts->ts_mm->mmap_sem);
588 } else {
589 tfh_user_polling_mode(tfh);
590 STAT(intr_mm_lock_failed);
591 }
592 }
593 return IRQ_HANDLED;

--- 25 unchanged lines hidden (view full) ---

619
620static int gru_user_dropin(struct gru_thread_state *gts,
621 struct gru_tlb_fault_handle *tfh,
622 void *cb)
623{
624 struct gru_mm_struct *gms = gts->ts_gms;
625 int ret;
626
587 gru_try_dropin(gru, gts, tfh, NULL);
588 up_read(&gts->ts_mm->mmap_sem);
589 } else {
590 tfh_user_polling_mode(tfh);
591 STAT(intr_mm_lock_failed);
592 }
593 }
594 return IRQ_HANDLED;

--- 25 unchanged lines hidden (view full) ---

620
621static int gru_user_dropin(struct gru_thread_state *gts,
622 struct gru_tlb_fault_handle *tfh,
623 void *cb)
624{
625 struct gru_mm_struct *gms = gts->ts_gms;
626 int ret;
627
627 gts->ustats.upm_tlbdropin++;
628 gts->ustats.upm_tlbmiss++;
628 while (1) {
629 wait_event(gms->ms_wait_queue,
630 atomic_read(&gms->ms_range_active) == 0);
631 prefetchw(tfh); /* Helps on hdw, required for emulator */
632 ret = gru_try_dropin(gts->ts_gru, gts, tfh, cb);
633 if (ret <= 0)
634 return ret;
635 STAT(call_os_wait_queue);

--- 265 unchanged lines hidden --- 		629 while (1) {
630 wait_event(gms->ms_wait_queue,
631 atomic_read(&gms->ms_range_active) == 0);
632 prefetchw(tfh); /* Helps on hdw, required for emulator */
633 ret = gru_try_dropin(gts->ts_gru, gts, tfh, cb);
634 if (ret <= 0)
635 return ret;
636 STAT(call_os_wait_queue);

--- 265 unchanged lines hidden ---