1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5#include <asm/asm.h> 6#include <asm/loongarch.h> 7#include <asm/page.h> 8#include <asm/pgtable.h> 9#include <asm/regdef.h> 10#include <asm/stackframe.h> 11 12#define INVTLB_ADDR_GFALSE_AND_ASID 5 13 14#define PTRS_PER_PGD_BITS (PAGE_SHIFT - PTRLOG) 15#define PTRS_PER_PUD_BITS (PAGE_SHIFT - PTRLOG) 16#define PTRS_PER_PMD_BITS (PAGE_SHIFT - PTRLOG) 17#define PTRS_PER_PTE_BITS (PAGE_SHIFT - PTRLOG) 18 19#ifdef CONFIG_32BIT 20#define PTE_LL ll.w 21#define PTE_SC sc.w 22#else 23#define PTE_LL ll.d 24#define PTE_SC sc.d 25#endif 26 27 .macro tlb_do_page_fault, write 28 SYM_CODE_START(tlb_do_page_fault_\write) 29 UNWIND_HINT_UNDEFINED 30 SAVE_ALL 31 csrrd a2, LOONGARCH_CSR_BADV 32 move a0, sp 33 REG_S a2, sp, PT_BVADDR 34 li.w a1, \write 35 bl do_page_fault 36 RESTORE_ALL_AND_RET 37 SYM_CODE_END(tlb_do_page_fault_\write) 38 .endm 39 40 tlb_do_page_fault 0 41 tlb_do_page_fault 1 42 43SYM_CODE_START(handle_tlb_protect) 44 UNWIND_HINT_UNDEFINED 45 BACKUP_T0T1 46 SAVE_ALL 47 move a0, sp 48 move a1, zero 49 csrrd a2, LOONGARCH_CSR_BADV 50 REG_S a2, sp, PT_BVADDR 51 la_abs t0, do_page_fault 52 jirl ra, t0, 0 53 RESTORE_ALL_AND_RET 54SYM_CODE_END(handle_tlb_protect) 55 56SYM_CODE_START(handle_tlb_load) 57 UNWIND_HINT_UNDEFINED 58 csrwr t0, EXCEPTION_KS0 59 csrwr t1, EXCEPTION_KS1 60 csrwr ra, EXCEPTION_KS2 61 62 /* 63 * The vmalloc handling is not in the hotpath. 64 */ 65 csrrd t0, LOONGARCH_CSR_BADV 66 bltz t0, vmalloc_load 67 csrrd t1, LOONGARCH_CSR_PGDL 68 69vmalloc_done_load: 70 /* Get PGD offset in bytes */ 71#ifdef CONFIG_32BIT 72 PTR_BSTRPICK ra, t0, 31, PGDIR_SHIFT 73#else 74 PTR_BSTRPICK ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 75#endif 76 PTR_ALSL t1, ra, t1, _PGD_T_LOG2 77 78#if CONFIG_PGTABLE_LEVELS > 3 79 PTR_L t1, t1, 0 80 PTR_BSTRPICK ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 81 PTR_ALSL t1, ra, t1, _PMD_T_LOG2 82 83#endif 84#if CONFIG_PGTABLE_LEVELS > 2 85 PTR_L t1, t1, 0 86 PTR_BSTRPICK ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 87 PTR_ALSL t1, ra, t1, _PMD_T_LOG2 88 89#endif 90 PTR_L ra, t1, 0 91 92 /* 93 * For huge tlb entries, pmde doesn't contain an address but 94 * instead contains the tlb pte. Check the PAGE_HUGE bit and 95 * see if we need to jump to huge tlb processing. 96 */ 97 PTR_ROTRI ra, ra, _PAGE_HUGE_SHIFT + 1 98 bltz ra, tlb_huge_update_load 99 100 PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 101 PTR_BSTRPICK t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 102 PTR_ALSL t1, t0, ra, _PTE_T_LOG2 103 104#ifdef CONFIG_SMP 105smp_pgtable_change_load: 106 PTE_LL t0, t1, 0 107#else 108 PTR_L t0, t1, 0 109#endif 110 andi ra, t0, _PAGE_PRESENT 111 beqz ra, nopage_tlb_load 112 113 ori t0, t0, _PAGE_VALID 114 115#ifdef CONFIG_SMP 116 PTE_SC t0, t1, 0 117 beqz t0, smp_pgtable_change_load 118#else 119 PTR_S t0, t1, 0 120#endif 121 122 tlbsrch 123 PTR_BSTRINS t1, zero, _PTE_T_LOG2, _PTE_T_LOG2 124 PTR_L t0, t1, 0 125 PTR_L t1, t1, _PTE_T_SIZE 126 csrwr t0, LOONGARCH_CSR_TLBELO0 127 csrwr t1, LOONGARCH_CSR_TLBELO1 128 tlbwr 129 130 csrrd t0, EXCEPTION_KS0 131 csrrd t1, EXCEPTION_KS1 132 csrrd ra, EXCEPTION_KS2 133 ertn 134 135vmalloc_load: 136 la_abs t1, swapper_pg_dir 137 b vmalloc_done_load 138 139 /* This is the entry point of a huge page. */ 140tlb_huge_update_load: 141#ifdef CONFIG_SMP 142 PTE_LL ra, t1, 0 143#else 144 PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 145#endif 146 andi t0, ra, _PAGE_PRESENT 147 beqz t0, nopage_tlb_load 148 149#ifdef CONFIG_SMP 150 ori t0, ra, _PAGE_VALID 151 PTE_SC t0, t1, 0 152 beqz t0, tlb_huge_update_load 153 ori t0, ra, _PAGE_VALID 154#else 155 ori t0, ra, _PAGE_VALID 156 PTR_S t0, t1, 0 157#endif 158 csrrd ra, LOONGARCH_CSR_ASID 159 csrrd t1, LOONGARCH_CSR_BADV 160 andi ra, ra, CSR_ASID_ASID 161 invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 162 163 /* 164 * A huge PTE describes an area the size of the 165 * configured huge page size. This is twice the 166 * of the large TLB entry size we intend to use. 167 * A TLB entry half the size of the configured 168 * huge page size is configured into entrylo0 169 * and entrylo1 to cover the contiguous huge PTE 170 * address space. 171 */ 172 /* Huge page: Move Global bit */ 173 xori t0, t0, _PAGE_HUGE 174 lu12i.w t1, _PAGE_HGLOBAL >> 12 175 and t1, t0, t1 176 PTR_SRLI t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 177 or t0, t0, t1 178 179 move ra, t0 180 csrwr ra, LOONGARCH_CSR_TLBELO0 181 182 /* Convert to entrylo1 */ 183 PTR_ADDI t1, zero, 1 184 PTR_SLLI t1, t1, (HPAGE_SHIFT - 1) 185 PTR_ADD t0, t0, t1 186 csrwr t0, LOONGARCH_CSR_TLBELO1 187 188 /* Set huge page tlb entry size */ 189 PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 190 PTR_LI t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT)) 191 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 192 193 tlbfill 194 195 PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 196 PTR_LI t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT)) 197 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 198 199 csrrd t0, EXCEPTION_KS0 200 csrrd t1, EXCEPTION_KS1 201 csrrd ra, EXCEPTION_KS2 202 ertn 203 204nopage_tlb_load: 205 dbar 0x700 206 csrrd ra, EXCEPTION_KS2 207 la_abs t0, tlb_do_page_fault_0 208 jr t0 209SYM_CODE_END(handle_tlb_load) 210 211SYM_CODE_START(handle_tlb_load_ptw) 212 UNWIND_HINT_UNDEFINED 213 csrwr t0, LOONGARCH_CSR_KS0 214 csrwr t1, LOONGARCH_CSR_KS1 215 la_abs t0, tlb_do_page_fault_0 216 jr t0 217SYM_CODE_END(handle_tlb_load_ptw) 218 219SYM_CODE_START(handle_tlb_store) 220 UNWIND_HINT_UNDEFINED 221 csrwr t0, EXCEPTION_KS0 222 csrwr t1, EXCEPTION_KS1 223 csrwr ra, EXCEPTION_KS2 224 225 /* 226 * The vmalloc handling is not in the hotpath. 227 */ 228 csrrd t0, LOONGARCH_CSR_BADV 229 bltz t0, vmalloc_store 230 csrrd t1, LOONGARCH_CSR_PGDL 231 232vmalloc_done_store: 233 /* Get PGD offset in bytes */ 234#ifdef CONFIG_32BIT 235 PTR_BSTRPICK ra, t0, 31, PGDIR_SHIFT 236#else 237 PTR_BSTRPICK ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 238#endif 239 PTR_ALSL t1, ra, t1, _PGD_T_LOG2 240 241#if CONFIG_PGTABLE_LEVELS > 3 242 PTR_L t1, t1, 0 243 PTR_BSTRPICK ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 244 PTR_ALSL t1, ra, t1, _PMD_T_LOG2 245#endif 246#if CONFIG_PGTABLE_LEVELS > 2 247 PTR_L t1, t1, 0 248 PTR_BSTRPICK ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 249 PTR_ALSL t1, ra, t1, _PMD_T_LOG2 250#endif 251 PTR_L ra, t1, 0 252 253 /* 254 * For huge tlb entries, pmde doesn't contain an address but 255 * instead contains the tlb pte. Check the PAGE_HUGE bit and 256 * see if we need to jump to huge tlb processing. 257 */ 258 PTR_ROTRI ra, ra, _PAGE_HUGE_SHIFT + 1 259 bltz ra, tlb_huge_update_store 260 261 PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 262 PTR_BSTRPICK t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 263 PTR_ALSL t1, t0, ra, _PTE_T_LOG2 264 265#ifdef CONFIG_SMP 266smp_pgtable_change_store: 267 PTE_LL t0, t1, 0 268#else 269 PTR_L t0, t1, 0 270#endif 271 272#ifdef CONFIG_64BIT 273 andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE 274 xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE 275#else 276 PTR_LI ra, _PAGE_PRESENT | _PAGE_WRITE 277 and ra, ra, t0 278 nor ra, ra, zero 279#endif 280 bnez ra, nopage_tlb_store 281 282#ifdef CONFIG_64BIT 283 ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 284#else 285 PTR_LI ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 286 or t0, ra, t0 287#endif 288 289#ifdef CONFIG_SMP 290 PTE_SC t0, t1, 0 291 beqz t0, smp_pgtable_change_store 292#else 293 PTR_S t0, t1, 0 294#endif 295 tlbsrch 296 PTR_BSTRINS t1, zero, _PTE_T_LOG2, _PTE_T_LOG2 297 PTR_L t0, t1, 0 298 PTR_L t1, t1, _PTE_T_SIZE 299 csrwr t0, LOONGARCH_CSR_TLBELO0 300 csrwr t1, LOONGARCH_CSR_TLBELO1 301 tlbwr 302 303 csrrd t0, EXCEPTION_KS0 304 csrrd t1, EXCEPTION_KS1 305 csrrd ra, EXCEPTION_KS2 306 ertn 307 308vmalloc_store: 309 la_abs t1, swapper_pg_dir 310 b vmalloc_done_store 311 312 /* This is the entry point of a huge page. */ 313tlb_huge_update_store: 314#ifdef CONFIG_SMP 315 PTE_LL ra, t1, 0 316#else 317 PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 318#endif 319 320#ifdef CONFIG_64BIT 321 andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE 322 xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE 323#else 324 PTR_LI t0, _PAGE_PRESENT | _PAGE_WRITE 325 and t0, t0, ra 326 nor t0, t0, zero 327#endif 328 329 bnez t0, nopage_tlb_store 330 331#ifdef CONFIG_SMP 332 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 333 PTE_SC t0, t1, 0 334 beqz t0, tlb_huge_update_store 335 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 336#else 337#ifdef CONFIG_64BIT 338 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 339#else 340 PTR_LI t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 341 or t0, ra, t0 342#endif 343 PTR_S t0, t1, 0 344#endif 345 csrrd ra, LOONGARCH_CSR_ASID 346 csrrd t1, LOONGARCH_CSR_BADV 347 andi ra, ra, CSR_ASID_ASID 348 invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 349 350 /* 351 * A huge PTE describes an area the size of the 352 * configured huge page size. This is twice the 353 * of the large TLB entry size we intend to use. 354 * A TLB entry half the size of the configured 355 * huge page size is configured into entrylo0 356 * and entrylo1 to cover the contiguous huge PTE 357 * address space. 358 */ 359 /* Huge page: Move Global bit */ 360 xori t0, t0, _PAGE_HUGE 361 lu12i.w t1, _PAGE_HGLOBAL >> 12 362 and t1, t0, t1 363 PTR_SRLI t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 364 or t0, t0, t1 365 366 move ra, t0 367 csrwr ra, LOONGARCH_CSR_TLBELO0 368 369 /* Convert to entrylo1 */ 370 PTR_ADDI t1, zero, 1 371 PTR_SLLI t1, t1, (HPAGE_SHIFT - 1) 372 PTR_ADD t0, t0, t1 373 csrwr t0, LOONGARCH_CSR_TLBELO1 374 375 /* Set huge page tlb entry size */ 376 PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 377 PTR_LI t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT)) 378 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 379 380 tlbfill 381 382 /* Reset default page size */ 383 PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 384 PTR_LI t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT)) 385 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 386 387 csrrd t0, EXCEPTION_KS0 388 csrrd t1, EXCEPTION_KS1 389 csrrd ra, EXCEPTION_KS2 390 ertn 391 392nopage_tlb_store: 393 dbar 0x700 394 csrrd ra, EXCEPTION_KS2 395 la_abs t0, tlb_do_page_fault_1 396 jr t0 397SYM_CODE_END(handle_tlb_store) 398 399SYM_CODE_START(handle_tlb_store_ptw) 400 UNWIND_HINT_UNDEFINED 401 csrwr t0, LOONGARCH_CSR_KS0 402 csrwr t1, LOONGARCH_CSR_KS1 403 la_abs t0, tlb_do_page_fault_1 404 jr t0 405SYM_CODE_END(handle_tlb_store_ptw) 406 407SYM_CODE_START(handle_tlb_modify) 408 UNWIND_HINT_UNDEFINED 409 csrwr t0, EXCEPTION_KS0 410 csrwr t1, EXCEPTION_KS1 411 csrwr ra, EXCEPTION_KS2 412 413 /* 414 * The vmalloc handling is not in the hotpath. 415 */ 416 csrrd t0, LOONGARCH_CSR_BADV 417 bltz t0, vmalloc_modify 418 csrrd t1, LOONGARCH_CSR_PGDL 419 420vmalloc_done_modify: 421 /* Get PGD offset in bytes */ 422#ifdef CONFIG_32BIT 423 PTR_BSTRPICK ra, t0, 31, PGDIR_SHIFT 424#else 425 PTR_BSTRPICK ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 426#endif 427 PTR_ALSL t1, ra, t1, _PGD_T_LOG2 428 429#if CONFIG_PGTABLE_LEVELS > 3 430 PTR_L t1, t1, 0 431 PTR_BSTRPICK ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 432 PTR_ALSL t1, ra, t1, _PMD_T_LOG2 433#endif 434#if CONFIG_PGTABLE_LEVELS > 2 435 PTR_L t1, t1, 0 436 PTR_BSTRPICK ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 437 PTR_ALSL t1, ra, t1, _PMD_T_LOG2 438#endif 439 PTR_L ra, t1, 0 440 441 /* 442 * For huge tlb entries, pmde doesn't contain an address but 443 * instead contains the tlb pte. Check the PAGE_HUGE bit and 444 * see if we need to jump to huge tlb processing. 445 */ 446 PTR_ROTRI ra, ra, _PAGE_HUGE_SHIFT + 1 447 bltz ra, tlb_huge_update_modify 448 449 PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 450 PTR_BSTRPICK t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 451 PTR_ALSL t1, t0, ra, _PTE_T_LOG2 452 453#ifdef CONFIG_SMP 454smp_pgtable_change_modify: 455 PTE_LL t0, t1, 0 456#else 457 PTR_L t0, t1, 0 458#endif 459#ifdef CONFIG_64BIT 460 andi ra, t0, _PAGE_WRITE 461#else 462 PTR_LI ra, _PAGE_WRITE 463 and ra, t0, ra 464#endif 465 466 beqz ra, nopage_tlb_modify 467 468#ifdef CONFIG_64BIT 469 ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 470#else 471 PTR_LI ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 472 or t0, ra, t0 473#endif 474 475#ifdef CONFIG_SMP 476 PTE_SC t0, t1, 0 477 beqz t0, smp_pgtable_change_modify 478#else 479 PTR_S t0, t1, 0 480#endif 481 tlbsrch 482 PTR_BSTRINS t1, zero, _PTE_T_LOG2, _PTE_T_LOG2 483 PTR_L t0, t1, 0 484 PTR_L t1, t1, _PTE_T_SIZE 485 csrwr t0, LOONGARCH_CSR_TLBELO0 486 csrwr t1, LOONGARCH_CSR_TLBELO1 487 tlbwr 488 489 csrrd t0, EXCEPTION_KS0 490 csrrd t1, EXCEPTION_KS1 491 csrrd ra, EXCEPTION_KS2 492 ertn 493 494vmalloc_modify: 495 la_abs t1, swapper_pg_dir 496 b vmalloc_done_modify 497 498 /* This is the entry point of a huge page. */ 499tlb_huge_update_modify: 500#ifdef CONFIG_SMP 501 PTE_LL ra, t1, 0 502#else 503 PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 504#endif 505 506#ifdef CONFIG_64BIT 507 andi t0, ra, _PAGE_WRITE 508#else 509 PTR_LI t0, _PAGE_WRITE 510 and t0, ra, t0 511#endif 512 513 beqz t0, nopage_tlb_modify 514 515#ifdef CONFIG_SMP 516 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 517 PTE_SC t0, t1, 0 518 beqz t0, tlb_huge_update_modify 519 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 520#else 521#ifdef CONFIG_64BIT 522 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 523#else 524 PTR_LI t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 525 or t0, ra, t0 526#endif 527 PTR_S t0, t1, 0 528#endif 529 csrrd ra, LOONGARCH_CSR_ASID 530 csrrd t1, LOONGARCH_CSR_BADV 531 andi ra, ra, CSR_ASID_ASID 532 invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 533 534 /* 535 * A huge PTE describes an area the size of the 536 * configured huge page size. This is twice the 537 * of the large TLB entry size we intend to use. 538 * A TLB entry half the size of the configured 539 * huge page size is configured into entrylo0 540 * and entrylo1 to cover the contiguous huge PTE 541 * address space. 542 */ 543 /* Huge page: Move Global bit */ 544 xori t0, t0, _PAGE_HUGE 545 lu12i.w t1, _PAGE_HGLOBAL >> 12 546 and t1, t0, t1 547 PTR_SRLI t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 548 or t0, t0, t1 549 550 move ra, t0 551 csrwr ra, LOONGARCH_CSR_TLBELO0 552 553 /* Convert to entrylo1 */ 554 PTR_ADDI t1, zero, 1 555 PTR_SLLI t1, t1, (HPAGE_SHIFT - 1) 556 PTR_ADD t0, t0, t1 557 csrwr t0, LOONGARCH_CSR_TLBELO1 558 559 /* Set huge page tlb entry size */ 560 PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 561 PTR_LI t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT)) 562 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 563 564 tlbfill 565 566 /* Reset default page size */ 567 PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 568 PTR_LI t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT)) 569 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 570 571 csrrd t0, EXCEPTION_KS0 572 csrrd t1, EXCEPTION_KS1 573 csrrd ra, EXCEPTION_KS2 574 ertn 575 576nopage_tlb_modify: 577 dbar 0x700 578 csrrd ra, EXCEPTION_KS2 579 la_abs t0, tlb_do_page_fault_1 580 jr t0 581SYM_CODE_END(handle_tlb_modify) 582 583SYM_CODE_START(handle_tlb_modify_ptw) 584 UNWIND_HINT_UNDEFINED 585 csrwr t0, LOONGARCH_CSR_KS0 586 csrwr t1, LOONGARCH_CSR_KS1 587 la_abs t0, tlb_do_page_fault_1 588 jr t0 589SYM_CODE_END(handle_tlb_modify_ptw) 590 591#ifdef CONFIG_32BIT 592SYM_CODE_START(handle_tlb_refill) 593 UNWIND_HINT_UNDEFINED 594 csrwr t0, EXCEPTION_KS0 595 csrwr t1, EXCEPTION_KS1 596 csrwr ra, EXCEPTION_KS2 597 li.w ra, 0x1fffffff 598 599 csrrd t0, LOONGARCH_CSR_PGD 600 csrrd t1, LOONGARCH_CSR_TLBRBADV 601 srli.w t1, t1, PGDIR_SHIFT 602 slli.w t1, t1, 0x2 603 add.w t0, t0, t1 604 and t0, t0, ra 605 606 ld.w t0, t0, 0 607 csrrd t1, LOONGARCH_CSR_TLBRBADV 608 slli.w t1, t1, (32 - PGDIR_SHIFT) 609 srli.w t1, t1, (32 - PGDIR_SHIFT + PAGE_SHIFT + 1) 610 slli.w t1, t1, (0x2 + 1) 611 add.w t0, t0, t1 612 and t0, t0, ra 613 614 ld.w t1, t0, 0x0 615 csrwr t1, LOONGARCH_CSR_TLBRELO0 616 617 ld.w t1, t0, 0x4 618 csrwr t1, LOONGARCH_CSR_TLBRELO1 619 620 tlbfill 621 csrrd t0, EXCEPTION_KS0 622 csrrd t1, EXCEPTION_KS1 623 csrrd ra, EXCEPTION_KS2 624 ertn 625SYM_CODE_END(handle_tlb_refill) 626#endif 627 628#ifdef CONFIG_64BIT 629SYM_CODE_START(handle_tlb_refill) 630 UNWIND_HINT_UNDEFINED 631 csrwr t0, LOONGARCH_CSR_TLBRSAVE 632 csrrd t0, LOONGARCH_CSR_PGD 633 lddir t0, t0, 3 634#if CONFIG_PGTABLE_LEVELS > 3 635 lddir t0, t0, 2 636#endif 637#if CONFIG_PGTABLE_LEVELS > 2 638 lddir t0, t0, 1 639#endif 640 ldpte t0, 0 641 ldpte t0, 1 642 tlbfill 643 csrrd t0, LOONGARCH_CSR_TLBRSAVE 644 ertn 645SYM_CODE_END(handle_tlb_refill) 646#endif 647