1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KVM guest address space mapping code 4 * 5 * Copyright IBM Corp. 2007, 2020, 2024 6 * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> 7 * Martin Schwidefsky <schwidefsky@de.ibm.com> 8 * David Hildenbrand <david@redhat.com> 9 * Janosch Frank <frankja@linux.ibm.com> 10 */ 11 12 #include <linux/kernel.h> 13 #include <linux/pagewalk.h> 14 #include <linux/swap.h> 15 #include <linux/smp.h> 16 #include <linux/spinlock.h> 17 #include <linux/slab.h> 18 #include <linux/swapops.h> 19 #include <linux/ksm.h> 20 #include <linux/mm.h> 21 #include <linux/mman.h> 22 #include <linux/pgtable.h> 23 #include <linux/kvm_types.h> 24 #include <linux/kvm_host.h> 25 #include <linux/pgalloc.h> 26 27 #include <asm/page-states.h> 28 #include <asm/tlb.h> 29 #include "dat.h" 30 31 int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc) 32 { 33 void *o; 34 35 for ( ; mc->n_crsts < KVM_S390_MMU_CACHE_N_CRSTS; mc->n_crsts++) { 36 o = (void *)__get_free_pages(GFP_KERNEL_ACCOUNT | __GFP_COMP, CRST_ALLOC_ORDER); 37 if (!o) 38 return -ENOMEM; 39 mc->crsts[mc->n_crsts] = o; 40 } 41 for ( ; mc->n_pts < KVM_S390_MMU_CACHE_N_PTS; mc->n_pts++) { 42 o = (void *)__get_free_page(GFP_KERNEL_ACCOUNT); 43 if (!o) 44 return -ENOMEM; 45 mc->pts[mc->n_pts] = o; 46 } 47 for ( ; mc->n_rmaps < KVM_S390_MMU_CACHE_N_RMAPS; mc->n_rmaps++) { 48 o = kzalloc_obj(struct vsie_rmap, GFP_KERNEL_ACCOUNT); 49 if (!o) 50 return -ENOMEM; 51 mc->rmaps[mc->n_rmaps] = o; 52 } 53 return 0; 54 } 55 56 static inline struct page_table *dat_alloc_pt_noinit(struct kvm_s390_mmu_cache *mc) 57 { 58 struct page_table *res; 59 60 res = kvm_s390_mmu_cache_alloc_pt(mc); 61 if (res) 62 __arch_set_page_dat(res, 1); 63 return res; 64 } 65 66 static inline struct crst_table *dat_alloc_crst_noinit(struct kvm_s390_mmu_cache *mc) 67 { 68 struct crst_table *res; 69 70 res = kvm_s390_mmu_cache_alloc_crst(mc); 71 if (res) 72 __arch_set_page_dat(res, 1UL << CRST_ALLOC_ORDER); 73 return res; 74 } 75 76 struct crst_table *dat_alloc_crst_sleepable(unsigned long init) 77 { 78 struct page *page; 79 void *virt; 80 81 page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_COMP, CRST_ALLOC_ORDER); 82 if (!page) 83 return NULL; 84 virt = page_to_virt(page); 85 __arch_set_page_dat(virt, 1UL << CRST_ALLOC_ORDER); 86 crst_table_init(virt, init); 87 return virt; 88 } 89 90 void dat_free_level(struct crst_table *table, bool owns_ptes) 91 { 92 unsigned int i; 93 94 for (i = 0; i < _CRST_ENTRIES; i++) { 95 if (table->crstes[i].h.fc || table->crstes[i].h.i) 96 continue; 97 if (!is_pmd(table->crstes[i])) 98 dat_free_level(dereference_crste(table->crstes[i]), owns_ptes); 99 else if (owns_ptes) 100 dat_free_pt(dereference_pmd(table->crstes[i].pmd)); 101 } 102 dat_free_crst(table); 103 } 104 105 int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype) 106 { 107 struct crst_table *table; 108 union crste crste; 109 110 while (asce->dt > newtype) { 111 table = dereference_asce(*asce); 112 crste = table->crstes[0]; 113 if (crste.h.fc) 114 return 0; 115 if (!crste.h.i) { 116 asce->rsto = crste.h.fc0.to; 117 dat_free_crst(table); 118 } else { 119 crste.h.tt--; 120 crst_table_init((void *)table, crste.val); 121 } 122 asce->dt--; 123 } 124 while (asce->dt < newtype) { 125 crste = _crste_fc0(asce->rsto, asce->dt + 1); 126 table = dat_alloc_crst_noinit(mc); 127 if (!table) 128 return -ENOMEM; 129 crst_table_init((void *)table, _CRSTE_HOLE(crste.h.tt).val); 130 table->crstes[0] = crste; 131 asce->rsto = __pa(table) >> PAGE_SHIFT; 132 asce->dt++; 133 } 134 return 0; 135 } 136 137 /** 138 * dat_crstep_xchg_atomic() - Atomically exchange a gmap CRSTE with another. 139 * @crstep: Pointer to the CRST entry. 140 * @old: Expected old value. 141 * @new: Replacement entry. 142 * @gfn: The affected guest address. 143 * @asce: The asce of the address space. 144 * 145 * This function is needed to atomically exchange a CRSTE that potentially 146 * maps a prefix area, without having to invalidate it inbetween. 147 * 148 * Context: This function is assumed to be called with kvm->mmu_lock held. 149 * 150 * Return: %true if the exchange was successful. 151 */ 152 bool __must_check dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, 153 gfn_t gfn, union asce asce) 154 { 155 if (old.h.i) 156 return arch_try_cmpxchg((long *)crstep, &old.val, new.val); 157 if (cpu_has_edat2()) 158 return crdte_crste(crstep, old, new, gfn, asce); 159 return cspg_crste(crstep, old, new); 160 } 161 162 static void dat_set_storage_key_from_pgste(union pte pte, union pgste pgste) 163 { 164 union skey nkey = { .acc = pgste.acc, .fp = pgste.fp }; 165 166 page_set_storage_key(pte_origin(pte), nkey.skey, 0); 167 } 168 169 static void dat_move_storage_key(union pte old, union pte new) 170 { 171 page_set_storage_key(pte_origin(new), page_get_storage_key(pte_origin(old)), 1); 172 } 173 174 static union pgste dat_save_storage_key_into_pgste(union pte pte, union pgste pgste) 175 { 176 union skey skey; 177 178 skey.skey = page_get_storage_key(pte_origin(pte)); 179 180 pgste.acc = skey.acc; 181 pgste.fp = skey.fp; 182 pgste.gr |= skey.r; 183 pgste.gc |= skey.c; 184 185 return pgste; 186 } 187 188 union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, gfn_t gfn, 189 union asce asce, bool uses_skeys) 190 { 191 union pte old = READ_ONCE(*ptep); 192 193 /* Updating only the software bits while holding the pgste lock. */ 194 if (!((ptep->val ^ new.val) & ~_PAGE_SW_BITS)) { 195 WRITE_ONCE(ptep->swbyte, new.swbyte); 196 return pgste; 197 } 198 199 if (!old.h.i) { 200 unsigned long opts = IPTE_GUEST_ASCE | (pgste.nodat ? IPTE_NODAT : 0); 201 202 if (machine_has_tlb_guest()) 203 __ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, opts, asce.val, IPTE_GLOBAL); 204 else 205 __ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, 0, 0, IPTE_GLOBAL); 206 } 207 208 if (uses_skeys) { 209 if (old.h.i && !new.h.i) 210 /* Invalid to valid: restore storage keys from PGSTE. */ 211 dat_set_storage_key_from_pgste(new, pgste); 212 else if (!old.h.i && new.h.i) 213 /* Valid to invalid: save storage keys to PGSTE. */ 214 pgste = dat_save_storage_key_into_pgste(old, pgste); 215 else if (!old.h.i && !new.h.i) 216 /* Valid to valid: move storage keys. */ 217 if (old.h.pfra != new.h.pfra) 218 dat_move_storage_key(old, new); 219 /* Invalid to invalid: nothing to do. */ 220 } 221 222 WRITE_ONCE(*ptep, new); 223 return pgste; 224 } 225 226 /* 227 * dat_split_ste() - Split a segment table entry into page table entries. 228 * 229 * Context: This function is assumed to be called with kvm->mmu_lock held. 230 * 231 * Return: 0 in case of success, -ENOMEM if running out of memory. 232 */ 233 static int dat_split_ste(struct kvm_s390_mmu_cache *mc, union pmd *pmdp, gfn_t gfn, 234 union asce asce, bool uses_skeys) 235 { 236 union pgste pgste_init; 237 struct page_table *pt; 238 union pmd new, old; 239 union pte init; 240 int i; 241 242 BUG_ON(!mc); 243 old = READ_ONCE(*pmdp); 244 245 /* Already split, nothing to do. */ 246 if (!old.h.i && !old.h.fc) 247 return 0; 248 249 pt = dat_alloc_pt_noinit(mc); 250 if (!pt) 251 return -ENOMEM; 252 new.val = virt_to_phys(pt); 253 254 while (old.h.i || old.h.fc) { 255 init.val = pmd_origin_large(old); 256 init.h.p = old.h.p; 257 init.h.i = old.h.i; 258 init.s.d = old.s.fc1.d; 259 init.s.w = old.s.fc1.w; 260 init.s.y = old.s.fc1.y; 261 init.s.sd = old.s.fc1.sd; 262 init.s.pr = old.s.fc1.pr; 263 pgste_init.val = 0; 264 if (old.h.fc) { 265 for (i = 0; i < _PAGE_ENTRIES; i++) 266 pt->ptes[i].val = init.val | i * PAGE_SIZE; 267 /* No need to take locks as the page table is not installed yet. */ 268 pgste_init.prefix_notif = old.s.fc1.prefix_notif; 269 pgste_init.vsie_notif = old.s.fc1.vsie_notif; 270 pgste_init.vsie_gmem = old.s.fc1.vsie_notif; 271 pgste_init.pcl = uses_skeys && init.h.i; 272 dat_init_pgstes(pt, pgste_init.val); 273 } else { 274 dat_init_page_table(pt, init.val, 0); 275 } 276 277 if (dat_pmdp_xchg_atomic(pmdp, old, new, gfn, asce)) { 278 if (!pgste_init.pcl) 279 return 0; 280 for (i = 0; i < _PAGE_ENTRIES; i++) { 281 union pgste pgste = pt->pgstes[i]; 282 283 pgste = dat_save_storage_key_into_pgste(pt->ptes[i], pgste); 284 pgste_set_unlock(pt->ptes + i, pgste); 285 } 286 return 0; 287 } 288 old = READ_ONCE(*pmdp); 289 } 290 291 dat_free_pt(pt); 292 return 0; 293 } 294 295 /* 296 * dat_split_crste() - Split a crste into smaller crstes. 297 * 298 * Context: This function is assumed to be called with kvm->mmu_lock held. 299 * 300 * Return: %0 in case of success, %-ENOMEM if running out of memory. 301 */ 302 static int dat_split_crste(struct kvm_s390_mmu_cache *mc, union crste *crstep, 303 gfn_t gfn, union asce asce, bool uses_skeys) 304 { 305 struct crst_table *table; 306 union crste old, new, init; 307 int i; 308 309 old = READ_ONCE(*crstep); 310 if (is_pmd(old)) 311 return dat_split_ste(mc, &crstep->pmd, gfn, asce, uses_skeys); 312 313 BUG_ON(!mc); 314 315 /* Already split, nothing to do. */ 316 if (!old.h.i && !old.h.fc) 317 return 0; 318 319 table = dat_alloc_crst_noinit(mc); 320 if (!table) 321 return -ENOMEM; 322 323 new.val = virt_to_phys(table); 324 new.h.tt = old.h.tt; 325 new.h.fc0.tl = _REGION_ENTRY_LENGTH; 326 327 while (old.h.i || old.h.fc) { 328 init = old; 329 init.h.tt--; 330 if (old.h.fc) { 331 for (i = 0; i < _CRST_ENTRIES; i++) 332 table->crstes[i].val = init.val | i * HPAGE_SIZE; 333 } else { 334 crst_table_init((void *)table, init.val); 335 } 336 if (dat_crstep_xchg_atomic(crstep, old, new, gfn, asce)) 337 return 0; 338 old = READ_ONCE(*crstep); 339 } 340 341 dat_free_crst(table); 342 return 0; 343 } 344 345 /** 346 * dat_entry_walk() - Walk the gmap page tables. 347 * @mc: Cache to use to allocate dat tables, if needed; can be NULL if neither 348 * %DAT_WALK_SPLIT or %DAT_WALK_ALLOC is specified in @flags. 349 * @gfn: Guest frame. 350 * @asce: The ASCE of the address space. 351 * @flags: Flags from WALK_* macros. 352 * @walk_level: Level to walk to, from LEVEL_* macros. 353 * @last: Will be filled the last visited non-pte DAT entry. 354 * @ptepp: Will be filled the last visited pte entry, if any, otherwise NULL. 355 * 356 * Returns a table entry pointer for the given guest address and @walk_level. 357 * 358 * The @flags have the following meanings: 359 * * %DAT_WALK_IGN_HOLES: consider holes as normal table entries 360 * * %DAT_WALK_ALLOC: allocate new tables to reach the requested level, if needed 361 * * %DAT_WALK_SPLIT: split existing large pages to reach the requested level, if needed 362 * * %DAT_WALK_LEAF: return successfully whenever a large page is encountered 363 * * %DAT_WALK_ANY: return successfully even if the requested level could not be reached 364 * * %DAT_WALK_CONTINUE: walk to the requested level with the specified flags, and then try to 365 * continue walking to ptes with only DAT_WALK_ANY 366 * * %DAT_WALK_USES_SKEYS: storage keys are in use 367 * 368 * Context: called with kvm->mmu_lock held. 369 * 370 * Return: 371 * * %PGM_ADDRESSING if the requested address lies outside memory 372 * * a PIC number if the requested address lies in a memory hole of type _DAT_TOKEN_PIC 373 * * %-EFAULT if the requested address lies inside a memory hole of a different type 374 * * %-EINVAL if the given ASCE is not compatible with the requested level 375 * * %-EFBIG if the requested level could not be reached because a larger frame was found 376 * * %-ENOENT if the requested level could not be reached for other reasons 377 * * %-ENOMEM if running out of memory while allocating or splitting a table 378 */ 379 int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags, 380 int walk_level, union crste **last, union pte **ptepp) 381 { 382 union vaddress vaddr = { .addr = gfn_to_gpa(gfn) }; 383 bool continue_anyway = flags & DAT_WALK_CONTINUE; 384 bool uses_skeys = flags & DAT_WALK_USES_SKEYS; 385 bool ign_holes = flags & DAT_WALK_IGN_HOLES; 386 bool allocate = flags & DAT_WALK_ALLOC; 387 bool split = flags & DAT_WALK_SPLIT; 388 bool leaf = flags & DAT_WALK_LEAF; 389 bool any = flags & DAT_WALK_ANY; 390 struct page_table *pgtable; 391 struct crst_table *table; 392 union crste entry; 393 int rc; 394 395 *last = NULL; 396 *ptepp = NULL; 397 if (WARN_ON_ONCE(unlikely(!asce.val))) 398 return -EINVAL; 399 if (WARN_ON_ONCE(unlikely(walk_level > asce.dt))) 400 return -EINVAL; 401 if (!asce_contains_gfn(asce, gfn)) 402 return PGM_ADDRESSING; 403 404 table = dereference_asce(asce); 405 if (asce.dt >= ASCE_TYPE_REGION1) { 406 *last = table->crstes + vaddr.rfx; 407 entry = READ_ONCE(**last); 408 if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION1)) 409 return -EINVAL; 410 if (crste_hole(entry) && !ign_holes) 411 return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; 412 if (walk_level == TABLE_TYPE_REGION1) 413 return 0; 414 if (entry.pgd.h.i) { 415 if (!allocate) 416 return any ? 0 : -ENOENT; 417 rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); 418 if (rc) 419 return rc; 420 entry = READ_ONCE(**last); 421 } 422 table = dereference_crste(entry.pgd); 423 } 424 425 if (asce.dt >= ASCE_TYPE_REGION2) { 426 *last = table->crstes + vaddr.rsx; 427 entry = READ_ONCE(**last); 428 if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION2)) 429 return -EINVAL; 430 if (crste_hole(entry) && !ign_holes) 431 return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; 432 if (walk_level == TABLE_TYPE_REGION2) 433 return 0; 434 if (entry.p4d.h.i) { 435 if (!allocate) 436 return any ? 0 : -ENOENT; 437 rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); 438 if (rc) 439 return rc; 440 entry = READ_ONCE(**last); 441 } 442 table = dereference_crste(entry.p4d); 443 } 444 445 if (asce.dt >= ASCE_TYPE_REGION3) { 446 *last = table->crstes + vaddr.rtx; 447 entry = READ_ONCE(**last); 448 if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION3)) 449 return -EINVAL; 450 if (crste_hole(entry) && !ign_holes) 451 return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; 452 if (walk_level == TABLE_TYPE_REGION3 && 453 continue_anyway && !entry.pud.h.fc && !entry.h.i) { 454 walk_level = TABLE_TYPE_PAGE_TABLE; 455 allocate = false; 456 } 457 if (walk_level == TABLE_TYPE_REGION3 || ((leaf || any) && entry.pud.h.fc)) 458 return 0; 459 if (entry.pud.h.i && !entry.pud.h.fc) { 460 if (!allocate) 461 return any ? 0 : -ENOENT; 462 rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); 463 if (rc) 464 return rc; 465 entry = READ_ONCE(**last); 466 } 467 if (walk_level <= TABLE_TYPE_SEGMENT && entry.pud.h.fc) { 468 if (!split) 469 return -EFBIG; 470 rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); 471 if (rc) 472 return rc; 473 entry = READ_ONCE(**last); 474 } 475 table = dereference_crste(entry.pud); 476 } 477 478 *last = table->crstes + vaddr.sx; 479 entry = READ_ONCE(**last); 480 if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_SEGMENT)) 481 return -EINVAL; 482 if (crste_hole(entry) && !ign_holes) 483 return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; 484 if (continue_anyway && !entry.pmd.h.fc && !entry.h.i) { 485 walk_level = TABLE_TYPE_PAGE_TABLE; 486 allocate = false; 487 } 488 if (walk_level == TABLE_TYPE_SEGMENT || ((leaf || any) && entry.pmd.h.fc)) 489 return 0; 490 491 if (entry.pmd.h.i && !entry.pmd.h.fc) { 492 if (!allocate) 493 return any ? 0 : -ENOENT; 494 rc = dat_split_ste(mc, &(*last)->pmd, gfn, asce, uses_skeys); 495 if (rc) 496 return rc; 497 entry = READ_ONCE(**last); 498 } 499 if (walk_level <= TABLE_TYPE_PAGE_TABLE && entry.pmd.h.fc) { 500 if (!split) 501 return -EFBIG; 502 rc = dat_split_ste(mc, &(*last)->pmd, gfn, asce, uses_skeys); 503 if (rc) 504 return rc; 505 entry = READ_ONCE(**last); 506 } 507 pgtable = dereference_pmd(entry.pmd); 508 *ptepp = pgtable->ptes + vaddr.px; 509 if (pte_hole(**ptepp) && !ign_holes) 510 return (*ptepp)->tok.type == _DAT_TOKEN_PIC ? (*ptepp)->tok.par : -EFAULT; 511 return 0; 512 } 513 514 static long dat_pte_walk_range(gfn_t gfn, gfn_t end, struct page_table *table, struct dat_walk *w) 515 { 516 unsigned int idx = gfn & (_PAGE_ENTRIES - 1); 517 long rc = 0; 518 519 for ( ; gfn < end; idx++, gfn++) { 520 if (pte_hole(READ_ONCE(table->ptes[idx]))) { 521 if (!(w->flags & DAT_WALK_IGN_HOLES)) 522 return -EFAULT; 523 if (!(w->flags & DAT_WALK_ANY)) 524 continue; 525 } 526 527 rc = w->ops->pte_entry(table->ptes + idx, gfn, gfn + 1, w); 528 if (rc) 529 break; 530 } 531 return rc; 532 } 533 534 static long dat_crste_walk_range(gfn_t start, gfn_t end, struct crst_table *table, 535 struct dat_walk *walk) 536 { 537 unsigned long idx, cur_shift, cur_size; 538 dat_walk_op the_op; 539 union crste crste; 540 gfn_t cur, next; 541 long rc = 0; 542 543 cur_shift = 8 + table->crstes[0].h.tt * 11; 544 idx = (start >> cur_shift) & (_CRST_ENTRIES - 1); 545 cur_size = 1UL << cur_shift; 546 547 for (cur = ALIGN_DOWN(start, cur_size); cur < end; idx++, cur = next) { 548 next = cur + cur_size; 549 walk->last = table->crstes + idx; 550 crste = READ_ONCE(*walk->last); 551 552 if (crste_hole(crste)) { 553 if (!(walk->flags & DAT_WALK_IGN_HOLES)) 554 return -EFAULT; 555 if (!(walk->flags & DAT_WALK_ANY)) 556 continue; 557 } 558 559 the_op = walk->ops->crste_ops[crste.h.tt]; 560 if (the_op) { 561 rc = the_op(walk->last, cur, next, walk); 562 crste = READ_ONCE(*walk->last); 563 } 564 if (rc) 565 break; 566 if (!crste.h.i && !crste.h.fc) { 567 if (!is_pmd(crste)) 568 rc = dat_crste_walk_range(max(start, cur), min(end, next), 569 _dereference_crste(crste), walk); 570 else if (walk->ops->pte_entry) 571 rc = dat_pte_walk_range(max(start, cur), min(end, next), 572 dereference_pmd(crste.pmd), walk); 573 if (rc) 574 break; 575 } 576 } 577 return rc; 578 } 579 580 /** 581 * _dat_walk_gfn_range() - Walk DAT tables. 582 * @start: The first guest page frame to walk. 583 * @end: The guest page frame immediately after the last one to walk. 584 * @asce: The ASCE of the guest mapping. 585 * @ops: The gmap_walk_ops that will be used to perform the walk. 586 * @flags: Flags from WALK_* (currently only WALK_IGN_HOLES is supported). 587 * @priv: Will be passed as-is to the callbacks. 588 * 589 * Any callback returning non-zero causes the walk to stop immediately. 590 * 591 * Return: %-EINVAL in case of error, %-EFAULT if @start is too high for the 592 * given ASCE unless the DAT_WALK_IGN_HOLES flag is specified, 593 * otherwise it returns whatever the callbacks return. 594 */ 595 long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce, 596 const struct dat_walk_ops *ops, int flags, void *priv) 597 { 598 struct crst_table *table = dereference_asce(asce); 599 struct dat_walk walk = { 600 .ops = ops, 601 .asce = asce, 602 .priv = priv, 603 .flags = flags, 604 .start = start, 605 .end = end, 606 }; 607 608 if (WARN_ON_ONCE(unlikely(!asce.val))) 609 return -EINVAL; 610 if (!asce_contains_gfn(asce, start)) 611 return (flags & DAT_WALK_IGN_HOLES) ? 0 : -EFAULT; 612 613 return dat_crste_walk_range(start, min(end, asce_end(asce)), table, &walk); 614 } 615 616 int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey) 617 { 618 union crste *crstep; 619 union pgste pgste; 620 union pte *ptep; 621 int rc; 622 623 skey->skey = 0; 624 rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); 625 if (rc) 626 return rc; 627 628 if (!ptep) { 629 union crste crste; 630 631 crste = READ_ONCE(*crstep); 632 if (!crste.h.fc || !crste.s.fc1.pr) 633 return 0; 634 skey->skey = page_get_storage_key(large_crste_to_phys(crste, gfn)); 635 return 0; 636 } 637 pgste = pgste_get_lock(ptep); 638 if (ptep->h.i) { 639 skey->acc = pgste.acc; 640 skey->fp = pgste.fp; 641 } else { 642 skey->skey = page_get_storage_key(pte_origin(*ptep)); 643 } 644 skey->r |= pgste.gr; 645 skey->c |= pgste.gc; 646 pgste_set_unlock(ptep, pgste); 647 return 0; 648 } 649 650 static void dat_update_ptep_sd(union pgste old, union pgste pgste, union pte *ptep) 651 { 652 if (pgste.acc != old.acc || pgste.fp != old.fp || pgste.gr != old.gr || pgste.gc != old.gc) 653 __atomic64_or(_PAGE_SD, &ptep->val); 654 } 655 656 int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, 657 union skey skey, bool nq) 658 { 659 union pgste pgste, old; 660 union crste *crstep; 661 union pte *ptep; 662 int rc; 663 664 rc = dat_entry_walk(mc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, 665 &crstep, &ptep); 666 if (rc) 667 return rc; 668 669 if (!ptep) { 670 page_set_storage_key(large_crste_to_phys(*crstep, gfn), skey.skey, !nq); 671 return 0; 672 } 673 674 old = pgste_get_lock(ptep); 675 pgste = old; 676 677 pgste.acc = skey.acc; 678 pgste.fp = skey.fp; 679 pgste.gc = skey.c; 680 pgste.gr = skey.r; 681 682 if (!ptep->h.i) { 683 union skey old_skey; 684 685 old_skey.skey = page_get_storage_key(pte_origin(*ptep)); 686 pgste.hc |= old_skey.c; 687 pgste.hr |= old_skey.r; 688 old_skey.c = old.gc; 689 old_skey.r = old.gr; 690 skey.r = 0; 691 skey.c = 0; 692 page_set_storage_key(pte_origin(*ptep), skey.skey, !nq); 693 } 694 695 dat_update_ptep_sd(old, pgste, ptep); 696 pgste_set_unlock(ptep, pgste); 697 return 0; 698 } 699 700 static bool page_cond_set_storage_key(phys_addr_t paddr, union skey skey, union skey *oldkey, 701 bool nq, bool mr, bool mc) 702 { 703 oldkey->skey = page_get_storage_key(paddr); 704 if (oldkey->acc == skey.acc && oldkey->fp == skey.fp && 705 (oldkey->r == skey.r || mr) && (oldkey->c == skey.c || mc)) 706 return false; 707 page_set_storage_key(paddr, skey.skey, !nq); 708 return true; 709 } 710 711 int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn, 712 union skey skey, union skey *oldkey, bool nq, bool mr, bool mc) 713 { 714 union pgste pgste, old; 715 union crste *crstep; 716 union skey prev; 717 union pte *ptep; 718 int rc; 719 720 rc = dat_entry_walk(mmc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, 721 &crstep, &ptep); 722 if (rc) 723 return rc; 724 725 if (!ptep) 726 return page_cond_set_storage_key(large_crste_to_phys(*crstep, gfn), skey, oldkey, 727 nq, mr, mc); 728 729 old = pgste_get_lock(ptep); 730 pgste = old; 731 732 rc = 1; 733 pgste.acc = skey.acc; 734 pgste.fp = skey.fp; 735 pgste.gc = skey.c; 736 pgste.gr = skey.r; 737 738 if (!ptep->h.i) { 739 rc = page_cond_set_storage_key(pte_origin(*ptep), skey, &prev, nq, mr, mc); 740 pgste.hc |= prev.c; 741 pgste.hr |= prev.r; 742 prev.c |= old.gc; 743 prev.r |= old.gr; 744 } else { 745 prev.acc = old.acc; 746 prev.fp = old.fp; 747 prev.c = old.gc; 748 prev.r = old.gr; 749 } 750 if (oldkey) 751 *oldkey = prev; 752 753 dat_update_ptep_sd(old, pgste, ptep); 754 pgste_set_unlock(ptep, pgste); 755 return rc; 756 } 757 758 int dat_reset_reference_bit(union asce asce, gfn_t gfn) 759 { 760 union pgste pgste, old; 761 union crste *crstep; 762 union pte *ptep; 763 int rc; 764 765 rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); 766 if (rc) 767 return rc; 768 769 if (!ptep) { 770 union crste crste = READ_ONCE(*crstep); 771 772 if (!crste.h.fc || !crste.s.fc1.pr) 773 return 0; 774 return page_reset_referenced(large_crste_to_phys(*crstep, gfn)); 775 } 776 old = pgste_get_lock(ptep); 777 pgste = old; 778 779 if (!ptep->h.i) { 780 rc = page_reset_referenced(pte_origin(*ptep)); 781 pgste.hr = rc >> 1; 782 } 783 rc |= (pgste.gr << 1) | pgste.gc; 784 pgste.gr = 0; 785 786 dat_update_ptep_sd(old, pgste, ptep); 787 pgste_set_unlock(ptep, pgste); 788 return rc; 789 } 790 791 static long dat_reset_skeys_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 792 { 793 union pgste pgste; 794 795 pgste = pgste_get_lock(ptep); 796 pgste.acc = 0; 797 pgste.fp = 0; 798 pgste.gr = 0; 799 pgste.gc = 0; 800 if (ptep->s.pr) 801 page_set_storage_key(pte_origin(*ptep), PAGE_DEFAULT_KEY, 1); 802 pgste_set_unlock(ptep, pgste); 803 804 if (need_resched()) 805 return next; 806 return 0; 807 } 808 809 static long dat_reset_skeys_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 810 { 811 phys_addr_t addr, end, origin = crste_origin_large(*crstep); 812 813 if (!crstep->h.fc || !crstep->s.fc1.pr) 814 return 0; 815 816 addr = ((max(gfn, walk->start) - gfn) << PAGE_SHIFT) + origin; 817 end = ((min(next, walk->end) - gfn) << PAGE_SHIFT) + origin; 818 while (ALIGN(addr + 1, _SEGMENT_SIZE) <= end) 819 addr = sske_frame(addr, PAGE_DEFAULT_KEY); 820 for ( ; addr < end; addr += PAGE_SIZE) 821 page_set_storage_key(addr, PAGE_DEFAULT_KEY, 1); 822 823 if (need_resched()) 824 return next; 825 return 0; 826 } 827 828 long dat_reset_skeys(union asce asce, gfn_t start) 829 { 830 const struct dat_walk_ops ops = { 831 .pte_entry = dat_reset_skeys_pte, 832 .pmd_entry = dat_reset_skeys_crste, 833 .pud_entry = dat_reset_skeys_crste, 834 }; 835 836 return _dat_walk_gfn_range(start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, NULL); 837 } 838 839 struct slot_priv { 840 unsigned long token; 841 struct kvm_s390_mmu_cache *mc; 842 }; 843 844 static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 845 { 846 struct slot_priv *p = walk->priv; 847 union crste dummy = { .val = p->token }; 848 union pte new_pte, pte = READ_ONCE(*ptep); 849 850 new_pte = _PTE_TOK(dummy.tok.type, dummy.tok.par); 851 852 /* Table entry already in the desired state. */ 853 if (pte.val == new_pte.val) 854 return 0; 855 856 dat_ptep_xchg(ptep, new_pte, gfn, walk->asce, false); 857 return 0; 858 } 859 860 static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 861 { 862 union crste new_crste, crste = READ_ONCE(*crstep); 863 struct slot_priv *p = walk->priv; 864 865 new_crste.val = p->token; 866 new_crste.h.tt = crste.h.tt; 867 868 /* Table entry already in the desired state. */ 869 if (crste.val == new_crste.val) 870 return 0; 871 872 /* This table entry needs to be updated. */ 873 if (walk->start <= gfn && walk->end >= next) { 874 if (!dat_crstep_xchg_atomic(crstep, crste, new_crste, gfn, walk->asce)) 875 return -EINVAL; 876 /* A lower level table was present, needs to be freed. */ 877 if (!crste.h.fc && !crste.h.i) { 878 if (is_pmd(crste)) 879 dat_free_pt(dereference_pmd(crste.pmd)); 880 else 881 dat_free_level(dereference_crste(crste), true); 882 } 883 return 0; 884 } 885 886 /* A lower level table is present, things will handled there. */ 887 if (!crste.h.fc && !crste.h.i) 888 return 0; 889 /* Split (install a lower level table), and handle things there. */ 890 return dat_split_crste(p->mc, crstep, gfn, walk->asce, false); 891 } 892 893 static const struct dat_walk_ops dat_slot_ops = { 894 .pte_entry = _dat_slot_pte, 895 .crste_ops = { _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, }, 896 }; 897 898 int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, 899 u16 type, u16 param) 900 { 901 struct slot_priv priv = { 902 .token = _CRSTE_TOK(0, type, param).val, 903 .mc = mc, 904 }; 905 906 return _dat_walk_gfn_range(start, end, asce, &dat_slot_ops, 907 DAT_WALK_IGN_HOLES | DAT_WALK_ANY, &priv); 908 } 909 910 static void pgste_set_unlock_multiple(union pte *first, int n, union pgste *pgstes) 911 { 912 int i; 913 914 for (i = 0; i < n; i++) { 915 if (!pgstes[i].pcl) 916 break; 917 pgste_set_unlock(first + i, pgstes[i]); 918 } 919 } 920 921 static bool pgste_get_trylock_multiple(union pte *first, int n, union pgste *pgstes) 922 { 923 int i; 924 925 for (i = 0; i < n; i++) { 926 if (!pgste_get_trylock(first + i, pgstes + i)) 927 break; 928 } 929 if (i == n) 930 return true; 931 pgste_set_unlock_multiple(first, n, pgstes); 932 return false; 933 } 934 935 unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param) 936 { 937 union pgste pgstes[4] = {}; 938 unsigned long res = 0; 939 int i, n; 940 941 n = param.len + 1; 942 943 while (!pgste_get_trylock_multiple(table->ptes + param.offset, n, pgstes)) 944 cpu_relax(); 945 946 for (i = 0; i < n; i++) 947 res = res << 16 | pgstes[i].val16; 948 949 pgste_set_unlock_multiple(table->ptes + param.offset, n, pgstes); 950 return res; 951 } 952 953 void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val) 954 { 955 union pgste pgstes[4] = {}; 956 int i, n; 957 958 n = param.len + 1; 959 960 while (!pgste_get_trylock_multiple(table->ptes + param.offset, n, pgstes)) 961 cpu_relax(); 962 963 for (i = param.len; i >= 0; i--) { 964 pgstes[i].val16 = val; 965 val = val >> 16; 966 } 967 968 pgste_set_unlock_multiple(table->ptes + param.offset, n, pgstes); 969 } 970 971 static long _dat_test_young_pte(union pte *ptep, gfn_t start, gfn_t end, struct dat_walk *walk) 972 { 973 return ptep->s.y; 974 } 975 976 static long _dat_test_young_crste(union crste *crstep, gfn_t start, gfn_t end, 977 struct dat_walk *walk) 978 { 979 return crstep->h.fc && crstep->s.fc1.y; 980 } 981 982 static const struct dat_walk_ops test_age_ops = { 983 .pte_entry = _dat_test_young_pte, 984 .pmd_entry = _dat_test_young_crste, 985 .pud_entry = _dat_test_young_crste, 986 }; 987 988 /** 989 * dat_test_age_gfn() - Test young. 990 * @asce: The ASCE whose address range is to be tested. 991 * @start: The first guest frame of the range to check. 992 * @end: The guest frame after the last in the range. 993 * 994 * Context: called by KVM common code with the kvm mmu write lock held. 995 * 996 * Return: %true if any page in the given range is young, otherwise %false. 997 */ 998 bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end) 999 { 1000 return _dat_walk_gfn_range(start, end, asce, &test_age_ops, 0, NULL) > 0; 1001 } 1002 1003 static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1004 { 1005 union crste newcrste, oldcrste; 1006 int *n = walk->priv; 1007 1008 do { 1009 oldcrste = READ_ONCE(*crstep); 1010 if (!oldcrste.h.fc || oldcrste.h.i || oldcrste.h.p) 1011 return 0; 1012 if (oldcrste.s.fc1.prefix_notif) 1013 break; 1014 newcrste = oldcrste; 1015 newcrste.s.fc1.prefix_notif = 1; 1016 } while (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, walk->asce)); 1017 *n = 2; 1018 return 0; 1019 } 1020 1021 static long dat_set_pn_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1022 { 1023 int *n = walk->priv; 1024 union pgste pgste; 1025 1026 pgste = pgste_get_lock(ptep); 1027 if (!ptep->h.i && !ptep->h.p) { 1028 pgste.prefix_notif = 1; 1029 *n += 1; 1030 } 1031 pgste_set_unlock(ptep, pgste); 1032 return 0; 1033 } 1034 1035 int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn) 1036 { 1037 static const struct dat_walk_ops ops = { 1038 .pte_entry = dat_set_pn_pte, 1039 .pmd_entry = dat_set_pn_crste, 1040 .pud_entry = dat_set_pn_crste, 1041 }; 1042 1043 int n = 0; 1044 1045 _dat_walk_gfn_range(gfn, gfn + 2, asce, &ops, DAT_WALK_IGN_HOLES, &n); 1046 if (n != 2) 1047 return -EAGAIN; 1048 return 0; 1049 } 1050 1051 /** 1052 * dat_perform_essa() - Perform ESSA actions on the PGSTE. 1053 * @asce: The asce to operate on. 1054 * @gfn: The guest page frame to operate on. 1055 * @orc: The specific action to perform, see the ESSA_SET_* macros. 1056 * @state: The storage attributes to be returned to the guest. 1057 * @dirty: Returns whether the function dirtied a previously clean entry. 1058 * 1059 * Context: Called with kvm->mmu_lock held. 1060 * 1061 * Return: 1062 * * %1 if the page state has been altered and the page is to be added to the CBRL 1063 * * %0 if the page state has been altered, but the page is not to be added to the CBRL 1064 * * %-1 if the page state has not been altered and the page is not to be added to the CBRL 1065 */ 1066 int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty) 1067 { 1068 union crste *crstep; 1069 union pgste pgste; 1070 union pte *ptep; 1071 int res = 0; 1072 1073 if (dat_entry_walk(NULL, gfn, asce, 0, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep)) { 1074 *state = (union essa_state) { .exception = 1 }; 1075 return -1; 1076 } 1077 1078 pgste = pgste_get_lock(ptep); 1079 1080 *state = (union essa_state) { 1081 .content = (ptep->h.i << 1) + (ptep->h.i && pgste.zero), 1082 .nodat = pgste.nodat, 1083 .usage = pgste.usage, 1084 }; 1085 1086 switch (orc) { 1087 case ESSA_GET_STATE: 1088 res = -1; 1089 break; 1090 case ESSA_SET_STABLE: 1091 pgste.usage = PGSTE_GPS_USAGE_STABLE; 1092 pgste.nodat = 0; 1093 break; 1094 case ESSA_SET_UNUSED: 1095 pgste.usage = PGSTE_GPS_USAGE_UNUSED; 1096 if (ptep->h.i) 1097 res = 1; 1098 break; 1099 case ESSA_SET_VOLATILE: 1100 pgste.usage = PGSTE_GPS_USAGE_VOLATILE; 1101 if (ptep->h.i) 1102 res = 1; 1103 break; 1104 case ESSA_SET_POT_VOLATILE: 1105 if (!ptep->h.i) { 1106 pgste.usage = PGSTE_GPS_USAGE_POT_VOLATILE; 1107 } else if (pgste.zero) { 1108 pgste.usage = PGSTE_GPS_USAGE_VOLATILE; 1109 } else if (!pgste.gc) { 1110 pgste.usage = PGSTE_GPS_USAGE_VOLATILE; 1111 res = 1; 1112 } 1113 break; 1114 case ESSA_SET_STABLE_RESIDENT: 1115 pgste.usage = PGSTE_GPS_USAGE_STABLE; 1116 /* 1117 * Since the resident state can go away any time after this 1118 * call, we will not make this page resident. We can revisit 1119 * this decision if a guest will ever start using this. 1120 */ 1121 break; 1122 case ESSA_SET_STABLE_IF_RESIDENT: 1123 if (!ptep->h.i) 1124 pgste.usage = PGSTE_GPS_USAGE_STABLE; 1125 break; 1126 case ESSA_SET_STABLE_NODAT: 1127 pgste.usage = PGSTE_GPS_USAGE_STABLE; 1128 pgste.nodat = 1; 1129 break; 1130 default: 1131 WARN_ONCE(1, "Invalid ORC!"); 1132 res = -1; 1133 break; 1134 } 1135 /* If we are discarding a page, set it to logical zero. */ 1136 pgste.zero = res == 1; 1137 if (orc > 0) { 1138 *dirty = !pgste.cmma_d; 1139 pgste.cmma_d = 1; 1140 } 1141 1142 pgste_set_unlock(ptep, pgste); 1143 1144 return res; 1145 } 1146 1147 static long dat_reset_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1148 { 1149 union pgste pgste; 1150 1151 pgste = pgste_get_lock(ptep); 1152 pgste.usage = 0; 1153 pgste.nodat = 0; 1154 pgste.cmma_d = 0; 1155 pgste_set_unlock(ptep, pgste); 1156 if (need_resched()) 1157 return next; 1158 return 0; 1159 } 1160 1161 long dat_reset_cmma(union asce asce, gfn_t start) 1162 { 1163 const struct dat_walk_ops dat_reset_cmma_ops = { 1164 .pte_entry = dat_reset_cmma_pte, 1165 }; 1166 1167 return _dat_walk_gfn_range(start, asce_end(asce), asce, &dat_reset_cmma_ops, 1168 DAT_WALK_IGN_HOLES, NULL); 1169 } 1170 1171 struct dat_get_cmma_state { 1172 gfn_t start; 1173 gfn_t end; 1174 unsigned int count; 1175 u8 *values; 1176 atomic64_t *remaining; 1177 }; 1178 1179 static long __dat_peek_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1180 { 1181 struct dat_get_cmma_state *state = walk->priv; 1182 union pgste pgste; 1183 1184 pgste = pgste_get_lock(ptep); 1185 state->values[gfn - walk->start] = pgste.usage | (pgste.nodat << 6); 1186 pgste_set_unlock(ptep, pgste); 1187 state->end = next; 1188 1189 return 0; 1190 } 1191 1192 static long __dat_peek_cmma_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1193 { 1194 struct dat_get_cmma_state *state = walk->priv; 1195 1196 if (crstep->h.i) 1197 state->end = min(walk->end, next); 1198 return 0; 1199 } 1200 1201 int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values) 1202 { 1203 const struct dat_walk_ops ops = { 1204 .pte_entry = __dat_peek_cmma_pte, 1205 .pmd_entry = __dat_peek_cmma_crste, 1206 .pud_entry = __dat_peek_cmma_crste, 1207 .p4d_entry = __dat_peek_cmma_crste, 1208 .pgd_entry = __dat_peek_cmma_crste, 1209 }; 1210 struct dat_get_cmma_state state = { .values = values, }; 1211 int rc; 1212 1213 rc = _dat_walk_gfn_range(start, start + *count, asce, &ops, DAT_WALK_DEFAULT, &state); 1214 *count = state.end >= start ? state.end - start : 0; 1215 /* Return success if at least one value was saved, otherwise an error. */ 1216 return (rc == -EFAULT && *count > 0) ? 0 : rc; 1217 } 1218 1219 static long __dat_get_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1220 { 1221 struct dat_get_cmma_state *state = walk->priv; 1222 union pgste pgste; 1223 1224 if (state->start != -1) { 1225 if ((gfn - state->end) > KVM_S390_MAX_BIT_DISTANCE) 1226 return 1; 1227 if (gfn - state->start >= state->count) 1228 return 1; 1229 } 1230 1231 if (!READ_ONCE(*pgste_of(ptep)).cmma_d) 1232 return 0; 1233 1234 pgste = pgste_get_lock(ptep); 1235 if (pgste.cmma_d) { 1236 if (state->start == -1) 1237 state->start = gfn; 1238 pgste.cmma_d = 0; 1239 atomic64_dec(state->remaining); 1240 state->values[gfn - state->start] = pgste.usage | pgste.nodat << 6; 1241 state->end = next; 1242 } 1243 pgste_set_unlock(ptep, pgste); 1244 return 0; 1245 } 1246 1247 int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem) 1248 { 1249 const struct dat_walk_ops ops = { .pte_entry = __dat_get_cmma_pte, }; 1250 struct dat_get_cmma_state state = { 1251 .remaining = rem, 1252 .values = values, 1253 .count = *count, 1254 .start = -1, 1255 }; 1256 1257 _dat_walk_gfn_range(*start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, &state); 1258 /* If no dirty pages were found, wrap around and continue searching */ 1259 if (*start && state.start == -1) 1260 _dat_walk_gfn_range(0, *start, asce, &ops, DAT_WALK_IGN_HOLES, &state); 1261 1262 if (state.start == -1) { 1263 *count = 0; 1264 } else { 1265 *count = state.end - state.start; 1266 *start = state.start; 1267 } 1268 1269 return 0; 1270 } 1271 1272 struct dat_set_cmma_state { 1273 unsigned long mask; 1274 const u8 *bits; 1275 }; 1276 1277 static long __dat_set_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1278 { 1279 struct dat_set_cmma_state *state = walk->priv; 1280 union pgste pgste, tmp; 1281 1282 tmp.val = (state->bits[gfn - walk->start] << 24) & state->mask; 1283 1284 pgste = pgste_get_lock(ptep); 1285 pgste.usage = tmp.usage; 1286 pgste.nodat = tmp.nodat; 1287 pgste_set_unlock(ptep, pgste); 1288 1289 return 0; 1290 } 1291 1292 /** 1293 * dat_set_cmma_bits() - Set CMMA bits for a range of guest pages. 1294 * @mc: Cache used for allocations. 1295 * @asce: The ASCE of the guest. 1296 * @gfn: The guest frame of the fist page whose CMMA bits are to set. 1297 * @count: How many pages need to be processed. 1298 * @mask: Which PGSTE bits should be set. 1299 * @bits: Points to an array with the CMMA attributes. 1300 * 1301 * This function sets the CMMA attributes for the given pages. If the input 1302 * buffer has zero length, no action is taken, otherwise the attributes are 1303 * set and the mm->context.uses_cmm flag is set. 1304 * 1305 * Each byte in @bits contains new values for bits 32-39 of the PGSTE. 1306 * Currently, only the fields NT and US are applied. 1307 * 1308 * Return: %0 in case of success, a negative error value otherwise. 1309 */ 1310 int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, 1311 unsigned long count, unsigned long mask, const uint8_t *bits) 1312 { 1313 const struct dat_walk_ops ops = { .pte_entry = __dat_set_cmma_pte, }; 1314 struct dat_set_cmma_state state = { .mask = mask, .bits = bits, }; 1315 union crste *crstep; 1316 union pte *ptep; 1317 gfn_t cur; 1318 int rc; 1319 1320 for (cur = ALIGN_DOWN(gfn, _PAGE_ENTRIES); cur < gfn + count; cur += _PAGE_ENTRIES) { 1321 rc = dat_entry_walk(mc, cur, asce, DAT_WALK_ALLOC, TABLE_TYPE_PAGE_TABLE, 1322 &crstep, &ptep); 1323 if (rc) 1324 return rc; 1325 } 1326 return _dat_walk_gfn_range(gfn, gfn + count, asce, &ops, DAT_WALK_IGN_HOLES, &state); 1327 } 1328