1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KVM guest address space mapping code 4 * 5 * Copyright IBM Corp. 2007, 2020, 2024 6 * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> 7 * Martin Schwidefsky <schwidefsky@de.ibm.com> 8 * David Hildenbrand <david@redhat.com> 9 * Janosch Frank <frankja@linux.ibm.com> 10 */ 11 12 #include <linux/kernel.h> 13 #include <linux/pagewalk.h> 14 #include <linux/swap.h> 15 #include <linux/smp.h> 16 #include <linux/spinlock.h> 17 #include <linux/slab.h> 18 #include <linux/swapops.h> 19 #include <linux/ksm.h> 20 #include <linux/mm.h> 21 #include <linux/mman.h> 22 #include <linux/pgtable.h> 23 #include <linux/kvm_types.h> 24 #include <linux/kvm_host.h> 25 #include <linux/pgalloc.h> 26 27 #include <asm/page-states.h> 28 #include <asm/tlb.h> 29 #include "dat.h" 30 31 int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc) 32 { 33 void *o; 34 35 for ( ; mc->n_crsts < KVM_S390_MMU_CACHE_N_CRSTS; mc->n_crsts++) { 36 o = (void *)__get_free_pages(GFP_KERNEL_ACCOUNT | __GFP_COMP, CRST_ALLOC_ORDER); 37 if (!o) 38 return -ENOMEM; 39 mc->crsts[mc->n_crsts] = o; 40 } 41 for ( ; mc->n_pts < KVM_S390_MMU_CACHE_N_PTS; mc->n_pts++) { 42 o = (void *)__get_free_page(GFP_KERNEL_ACCOUNT); 43 if (!o) 44 return -ENOMEM; 45 mc->pts[mc->n_pts] = o; 46 } 47 for ( ; mc->n_rmaps < KVM_S390_MMU_CACHE_N_RMAPS; mc->n_rmaps++) { 48 o = kzalloc_obj(struct vsie_rmap, GFP_KERNEL_ACCOUNT); 49 if (!o) 50 return -ENOMEM; 51 mc->rmaps[mc->n_rmaps] = o; 52 } 53 return 0; 54 } 55 56 static inline struct page_table *dat_alloc_pt_noinit(struct kvm_s390_mmu_cache *mc) 57 { 58 struct page_table *res; 59 60 res = kvm_s390_mmu_cache_alloc_pt(mc); 61 if (res) 62 __arch_set_page_dat(res, 1); 63 return res; 64 } 65 66 static inline struct crst_table *dat_alloc_crst_noinit(struct kvm_s390_mmu_cache *mc) 67 { 68 struct crst_table *res; 69 70 res = kvm_s390_mmu_cache_alloc_crst(mc); 71 if (res) 72 __arch_set_page_dat(res, 1UL << CRST_ALLOC_ORDER); 73 return res; 74 } 75 76 struct crst_table *dat_alloc_crst_sleepable(unsigned long init) 77 { 78 struct page *page; 79 void *virt; 80 81 page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_COMP, CRST_ALLOC_ORDER); 82 if (!page) 83 return NULL; 84 virt = page_to_virt(page); 85 __arch_set_page_dat(virt, 1UL << CRST_ALLOC_ORDER); 86 crst_table_init(virt, init); 87 return virt; 88 } 89 90 void dat_free_level(struct crst_table *table, bool owns_ptes) 91 { 92 unsigned int i; 93 94 for (i = 0; i < _CRST_ENTRIES; i++) { 95 if (table->crstes[i].h.fc || table->crstes[i].h.i) 96 continue; 97 if (!is_pmd(table->crstes[i])) 98 dat_free_level(dereference_crste(table->crstes[i]), owns_ptes); 99 else if (owns_ptes) 100 dat_free_pt(dereference_pmd(table->crstes[i].pmd)); 101 } 102 dat_free_crst(table); 103 } 104 105 int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype) 106 { 107 struct crst_table *table; 108 union crste crste; 109 110 while (asce->dt > newtype) { 111 table = dereference_asce(*asce); 112 crste = table->crstes[0]; 113 if (crste.h.fc) 114 return 0; 115 if (!crste.h.i) { 116 asce->rsto = crste.h.fc0.to; 117 dat_free_crst(table); 118 } else { 119 crste.h.tt--; 120 crst_table_init((void *)table, crste.val); 121 } 122 asce->dt--; 123 } 124 while (asce->dt < newtype) { 125 crste = _crste_fc0(asce->rsto, asce->dt + 1); 126 table = dat_alloc_crst_noinit(mc); 127 if (!table) 128 return -ENOMEM; 129 crst_table_init((void *)table, _CRSTE_HOLE(crste.h.tt).val); 130 table->crstes[0] = crste; 131 asce->rsto = __pa(table) >> PAGE_SHIFT; 132 asce->dt++; 133 } 134 return 0; 135 } 136 137 /** 138 * dat_crstep_xchg_atomic() - Atomically exchange a gmap CRSTE with another. 139 * @crstep: Pointer to the CRST entry. 140 * @old: Expected old value. 141 * @new: Replacement entry. 142 * @gfn: The affected guest address. 143 * @asce: The asce of the address space. 144 * 145 * This function is needed to atomically exchange a CRSTE that potentially 146 * maps a prefix area, without having to invalidate it inbetween. 147 * 148 * Context: This function is assumed to be called with kvm->mmu_lock held. 149 * 150 * Return: %true if the exchange was successful. 151 */ 152 bool __must_check dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, 153 gfn_t gfn, union asce asce) 154 { 155 if (old.h.i) 156 return arch_try_cmpxchg((long *)crstep, &old.val, new.val); 157 if (cpu_has_edat2()) 158 return crdte_crste(crstep, old, new, gfn, asce); 159 return cspg_crste(crstep, old, new); 160 } 161 162 static void dat_set_storage_key_from_pgste(union pte pte, union pgste pgste) 163 { 164 union skey nkey = { .acc = pgste.acc, .fp = pgste.fp }; 165 166 page_set_storage_key(pte_origin(pte), nkey.skey, 0); 167 } 168 169 static void dat_move_storage_key(union pte old, union pte new) 170 { 171 page_set_storage_key(pte_origin(new), page_get_storage_key(pte_origin(old)), 1); 172 } 173 174 static union pgste dat_save_storage_key_into_pgste(union pte pte, union pgste pgste) 175 { 176 union skey skey; 177 178 skey.skey = page_get_storage_key(pte_origin(pte)); 179 180 pgste.acc = skey.acc; 181 pgste.fp = skey.fp; 182 pgste.gr |= skey.r; 183 pgste.gc |= skey.c; 184 185 return pgste; 186 } 187 188 union pgste __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, gfn_t gfn, 189 union asce asce, bool uses_skeys) 190 { 191 union pte old = READ_ONCE(*ptep); 192 193 /* Updating only the software bits while holding the pgste lock. */ 194 if (!((ptep->val ^ new.val) & ~_PAGE_SW_BITS)) { 195 WRITE_ONCE(ptep->swbyte, new.swbyte); 196 return pgste; 197 } 198 199 if (!old.h.i) { 200 unsigned long opts = IPTE_GUEST_ASCE | (pgste.nodat ? IPTE_NODAT : 0); 201 202 if (machine_has_tlb_guest()) 203 __ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, opts, asce.val, IPTE_GLOBAL); 204 else 205 __ptep_ipte(gfn_to_gpa(gfn), (void *)ptep, 0, 0, IPTE_GLOBAL); 206 } 207 208 if (uses_skeys) { 209 if (old.h.i && !new.h.i) 210 /* Invalid to valid: restore storage keys from PGSTE. */ 211 dat_set_storage_key_from_pgste(new, pgste); 212 else if (!old.h.i && new.h.i) 213 /* Valid to invalid: save storage keys to PGSTE. */ 214 pgste = dat_save_storage_key_into_pgste(old, pgste); 215 else if (!old.h.i && !new.h.i) 216 /* Valid to valid: move storage keys. */ 217 if (old.h.pfra != new.h.pfra) 218 dat_move_storage_key(old, new); 219 /* Invalid to invalid: nothing to do. */ 220 } 221 222 WRITE_ONCE(*ptep, new); 223 return pgste; 224 } 225 226 /* 227 * dat_split_ste() - Split a segment table entry into page table entries. 228 * 229 * Context: This function is assumed to be called with kvm->mmu_lock held. 230 * 231 * Return: 0 in case of success, -ENOMEM if running out of memory. 232 */ 233 static int dat_split_ste(struct kvm_s390_mmu_cache *mc, union pmd *pmdp, gfn_t gfn, 234 union asce asce, bool uses_skeys) 235 { 236 union pgste pgste_init; 237 struct page_table *pt; 238 union pmd new, old; 239 union pte init; 240 int i; 241 242 BUG_ON(!mc); 243 old = READ_ONCE(*pmdp); 244 245 /* Already split, nothing to do. */ 246 if (!old.h.i && !old.h.fc) 247 return 0; 248 249 pt = dat_alloc_pt_noinit(mc); 250 if (!pt) 251 return -ENOMEM; 252 new.val = virt_to_phys(pt); 253 254 while (old.h.i || old.h.fc) { 255 init.val = pmd_origin_large(old); 256 init.h.p = old.h.p; 257 init.h.i = old.h.i; 258 init.s.d = old.s.fc1.d; 259 init.s.w = old.s.fc1.w; 260 init.s.y = old.s.fc1.y; 261 init.s.sd = old.s.fc1.sd; 262 init.s.pr = old.s.fc1.pr; 263 pgste_init.val = 0; 264 if (old.h.fc) { 265 for (i = 0; i < _PAGE_ENTRIES; i++) 266 pt->ptes[i].val = init.val | i * PAGE_SIZE; 267 /* No need to take locks as the page table is not installed yet. */ 268 pgste_init.prefix_notif = old.s.fc1.prefix_notif; 269 pgste_init.vsie_notif = old.s.fc1.vsie_notif; 270 pgste_init.vsie_gmem = old.s.fc1.vsie_notif; 271 pgste_init.pcl = uses_skeys && init.h.i; 272 dat_init_pgstes(pt, pgste_init.val); 273 } else { 274 dat_init_page_table(pt, init.val, 0); 275 } 276 277 if (dat_pmdp_xchg_atomic(pmdp, old, new, gfn, asce)) { 278 if (!pgste_init.pcl) 279 return 0; 280 for (i = 0; i < _PAGE_ENTRIES; i++) { 281 union pgste pgste = pt->pgstes[i]; 282 283 pgste = dat_save_storage_key_into_pgste(pt->ptes[i], pgste); 284 pgste_set_unlock(pt->ptes + i, pgste); 285 } 286 return 0; 287 } 288 old = READ_ONCE(*pmdp); 289 } 290 291 dat_free_pt(pt); 292 return 0; 293 } 294 295 /* 296 * dat_split_crste() - Split a crste into smaller crstes. 297 * 298 * Context: This function is assumed to be called with kvm->mmu_lock held. 299 * 300 * Return: %0 in case of success, %-ENOMEM if running out of memory. 301 */ 302 static int dat_split_crste(struct kvm_s390_mmu_cache *mc, union crste *crstep, 303 gfn_t gfn, union asce asce, bool uses_skeys) 304 { 305 struct crst_table *table; 306 union crste old, new, init; 307 int i; 308 309 old = READ_ONCE(*crstep); 310 if (is_pmd(old)) 311 return dat_split_ste(mc, &crstep->pmd, gfn, asce, uses_skeys); 312 313 BUG_ON(!mc); 314 315 /* Already split, nothing to do. */ 316 if (!old.h.i && !old.h.fc) 317 return 0; 318 319 table = dat_alloc_crst_noinit(mc); 320 if (!table) 321 return -ENOMEM; 322 323 new.val = virt_to_phys(table); 324 new.h.tt = old.h.tt; 325 new.h.fc0.tl = _REGION_ENTRY_LENGTH; 326 327 while (old.h.i || old.h.fc) { 328 init = old; 329 init.h.tt--; 330 if (old.h.fc) { 331 for (i = 0; i < _CRST_ENTRIES; i++) 332 table->crstes[i].val = init.val | i * HPAGE_SIZE; 333 } else { 334 crst_table_init((void *)table, init.val); 335 } 336 if (dat_crstep_xchg_atomic(crstep, old, new, gfn, asce)) 337 return 0; 338 old = READ_ONCE(*crstep); 339 } 340 341 dat_free_crst(table); 342 return 0; 343 } 344 345 /** 346 * dat_entry_walk() - Walk the gmap page tables. 347 * @mc: Cache to use to allocate dat tables, if needed; can be NULL if neither 348 * %DAT_WALK_SPLIT or %DAT_WALK_ALLOC is specified in @flags. 349 * @gfn: Guest frame. 350 * @asce: The ASCE of the address space. 351 * @flags: Flags from WALK_* macros. 352 * @walk_level: Level to walk to, from LEVEL_* macros. 353 * @last: Will be filled the last visited non-pte DAT entry. 354 * @ptepp: Will be filled the last visited pte entry, if any, otherwise NULL. 355 * 356 * Returns a table entry pointer for the given guest address and @walk_level. 357 * 358 * The @flags have the following meanings: 359 * * %DAT_WALK_IGN_HOLES: consider holes as normal table entries 360 * * %DAT_WALK_ALLOC: allocate new tables to reach the requested level, if needed 361 * * %DAT_WALK_SPLIT: split existing large pages to reach the requested level, if needed 362 * * %DAT_WALK_LEAF: return successfully whenever a large page is encountered 363 * * %DAT_WALK_ANY: return successfully even if the requested level could not be reached 364 * * %DAT_WALK_CONTINUE: walk to the requested level with the specified flags, and then try to 365 * continue walking to ptes with only DAT_WALK_ANY 366 * * %DAT_WALK_USES_SKEYS: storage keys are in use 367 * 368 * Context: called with kvm->mmu_lock held. 369 * 370 * Return: 371 * * %PGM_ADDRESSING if the requested address lies outside memory 372 * * a PIC number if the requested address lies in a memory hole of type _DAT_TOKEN_PIC 373 * * %-EFAULT if the requested address lies inside a memory hole of a different type 374 * * %-EINVAL if the given ASCE is not compatible with the requested level 375 * * %-EFBIG if the requested level could not be reached because a larger frame was found 376 * * %-ENOENT if the requested level could not be reached for other reasons 377 * * %-ENOMEM if running out of memory while allocating or splitting a table 378 */ 379 int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags, 380 int walk_level, union crste **last, union pte **ptepp) 381 { 382 union vaddress vaddr = { .addr = gfn_to_gpa(gfn) }; 383 bool continue_anyway = flags & DAT_WALK_CONTINUE; 384 bool uses_skeys = flags & DAT_WALK_USES_SKEYS; 385 bool ign_holes = flags & DAT_WALK_IGN_HOLES; 386 bool allocate = flags & DAT_WALK_ALLOC; 387 bool split = flags & DAT_WALK_SPLIT; 388 bool leaf = flags & DAT_WALK_LEAF; 389 bool any = flags & DAT_WALK_ANY; 390 struct page_table *pgtable; 391 struct crst_table *table; 392 union crste entry; 393 int rc; 394 395 *last = NULL; 396 *ptepp = NULL; 397 if (WARN_ON_ONCE(unlikely(!asce.val))) 398 return -EINVAL; 399 if (WARN_ON_ONCE(unlikely(walk_level > asce.dt))) 400 return -EINVAL; 401 if (!asce_contains_gfn(asce, gfn)) 402 return PGM_ADDRESSING; 403 404 table = dereference_asce(asce); 405 if (asce.dt >= ASCE_TYPE_REGION1) { 406 *last = table->crstes + vaddr.rfx; 407 entry = READ_ONCE(**last); 408 if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION1)) 409 return -EINVAL; 410 if (crste_hole(entry) && !ign_holes) 411 return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; 412 if (walk_level == TABLE_TYPE_REGION1) 413 return 0; 414 if (entry.pgd.h.i) { 415 if (!allocate) 416 return any ? 0 : -ENOENT; 417 rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); 418 if (rc) 419 return rc; 420 entry = READ_ONCE(**last); 421 } 422 table = dereference_crste(entry.pgd); 423 } 424 425 if (asce.dt >= ASCE_TYPE_REGION2) { 426 *last = table->crstes + vaddr.rsx; 427 entry = READ_ONCE(**last); 428 if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION2)) 429 return -EINVAL; 430 if (crste_hole(entry) && !ign_holes) 431 return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; 432 if (walk_level == TABLE_TYPE_REGION2) 433 return 0; 434 if (entry.p4d.h.i) { 435 if (!allocate) 436 return any ? 0 : -ENOENT; 437 rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); 438 if (rc) 439 return rc; 440 entry = READ_ONCE(**last); 441 } 442 table = dereference_crste(entry.p4d); 443 } 444 445 if (asce.dt >= ASCE_TYPE_REGION3) { 446 *last = table->crstes + vaddr.rtx; 447 entry = READ_ONCE(**last); 448 if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_REGION3)) 449 return -EINVAL; 450 if (crste_hole(entry) && !ign_holes) 451 return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; 452 if (walk_level == TABLE_TYPE_REGION3 && 453 continue_anyway && !entry.pud.h.fc && !entry.h.i) { 454 walk_level = TABLE_TYPE_PAGE_TABLE; 455 allocate = false; 456 } 457 if (walk_level == TABLE_TYPE_REGION3 || ((leaf || any) && entry.pud.h.fc)) 458 return 0; 459 if (entry.pud.h.i && !entry.pud.h.fc) { 460 if (!allocate) 461 return any ? 0 : -ENOENT; 462 rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); 463 if (rc) 464 return rc; 465 entry = READ_ONCE(**last); 466 } 467 if (walk_level <= TABLE_TYPE_SEGMENT && entry.pud.h.fc) { 468 if (!split) 469 return -EFBIG; 470 rc = dat_split_crste(mc, *last, gfn, asce, uses_skeys); 471 if (rc) 472 return rc; 473 entry = READ_ONCE(**last); 474 } 475 table = dereference_crste(entry.pud); 476 } 477 478 *last = table->crstes + vaddr.sx; 479 entry = READ_ONCE(**last); 480 if (WARN_ON_ONCE(entry.h.tt != TABLE_TYPE_SEGMENT)) 481 return -EINVAL; 482 if (crste_hole(entry) && !ign_holes) 483 return entry.tok.type == _DAT_TOKEN_PIC ? entry.tok.par : -EFAULT; 484 if (continue_anyway && !entry.pmd.h.fc && !entry.h.i) { 485 walk_level = TABLE_TYPE_PAGE_TABLE; 486 allocate = false; 487 } 488 if (walk_level == TABLE_TYPE_SEGMENT || ((leaf || any) && entry.pmd.h.fc)) 489 return 0; 490 491 if (entry.pmd.h.i && !entry.pmd.h.fc) { 492 if (!allocate) 493 return any ? 0 : -ENOENT; 494 rc = dat_split_ste(mc, &(*last)->pmd, gfn, asce, uses_skeys); 495 if (rc) 496 return rc; 497 entry = READ_ONCE(**last); 498 } 499 if (walk_level <= TABLE_TYPE_PAGE_TABLE && entry.pmd.h.fc) { 500 if (!split) 501 return -EFBIG; 502 rc = dat_split_ste(mc, &(*last)->pmd, gfn, asce, uses_skeys); 503 if (rc) 504 return rc; 505 entry = READ_ONCE(**last); 506 } 507 pgtable = dereference_pmd(entry.pmd); 508 *ptepp = pgtable->ptes + vaddr.px; 509 if (pte_hole(**ptepp) && !ign_holes) 510 return (*ptepp)->tok.type == _DAT_TOKEN_PIC ? (*ptepp)->tok.par : -EFAULT; 511 return 0; 512 } 513 514 static long dat_pte_walk_range(gfn_t gfn, gfn_t end, struct page_table *table, struct dat_walk *w) 515 { 516 unsigned int idx = gfn & (_PAGE_ENTRIES - 1); 517 long rc = 0; 518 519 for ( ; gfn < end; idx++, gfn++) { 520 if (pte_hole(READ_ONCE(table->ptes[idx]))) { 521 if (!(w->flags & DAT_WALK_IGN_HOLES)) 522 return -EFAULT; 523 if (!(w->flags & DAT_WALK_ANY)) 524 continue; 525 } 526 527 rc = w->ops->pte_entry(table->ptes + idx, gfn, gfn + 1, w); 528 if (rc) 529 break; 530 } 531 return rc; 532 } 533 534 static long dat_crste_walk_range(gfn_t start, gfn_t end, struct crst_table *table, 535 struct dat_walk *walk) 536 { 537 unsigned long idx, cur_shift, cur_size; 538 dat_walk_op the_op; 539 union crste crste; 540 gfn_t cur, next; 541 long rc = 0; 542 543 cur_shift = 8 + table->crstes[0].h.tt * 11; 544 idx = (start >> cur_shift) & (_CRST_ENTRIES - 1); 545 cur_size = 1UL << cur_shift; 546 547 for (cur = ALIGN_DOWN(start, cur_size); cur < end; idx++, cur = next) { 548 next = cur + cur_size; 549 walk->last = table->crstes + idx; 550 crste = READ_ONCE(*walk->last); 551 552 if (crste_hole(crste)) { 553 if (!(walk->flags & DAT_WALK_IGN_HOLES)) 554 return -EFAULT; 555 if (!(walk->flags & DAT_WALK_ANY)) 556 continue; 557 } 558 559 the_op = walk->ops->crste_ops[crste.h.tt]; 560 if (the_op) { 561 rc = the_op(walk->last, cur, next, walk); 562 crste = READ_ONCE(*walk->last); 563 } 564 if (rc) 565 break; 566 if (!crste.h.i && !crste.h.fc) { 567 if (!is_pmd(crste)) 568 rc = dat_crste_walk_range(max(start, cur), min(end, next), 569 _dereference_crste(crste), walk); 570 else if (walk->ops->pte_entry) 571 rc = dat_pte_walk_range(max(start, cur), min(end, next), 572 dereference_pmd(crste.pmd), walk); 573 if (rc) 574 break; 575 } 576 } 577 return rc; 578 } 579 580 /** 581 * _dat_walk_gfn_range() - Walk DAT tables. 582 * @start: The first guest page frame to walk. 583 * @end: The guest page frame immediately after the last one to walk. 584 * @asce: The ASCE of the guest mapping. 585 * @ops: The gmap_walk_ops that will be used to perform the walk. 586 * @flags: Flags from WALK_* (currently only WALK_IGN_HOLES is supported). 587 * @priv: Will be passed as-is to the callbacks. 588 * 589 * Any callback returning non-zero causes the walk to stop immediately. 590 * 591 * Return: %-EINVAL in case of error, %-EFAULT if @start is too high for the 592 * given ASCE unless the DAT_WALK_IGN_HOLES flag is specified, 593 * otherwise it returns whatever the callbacks return. 594 */ 595 long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce, 596 const struct dat_walk_ops *ops, int flags, void *priv) 597 { 598 struct crst_table *table = dereference_asce(asce); 599 struct dat_walk walk = { 600 .ops = ops, 601 .asce = asce, 602 .priv = priv, 603 .flags = flags, 604 .start = start, 605 .end = end, 606 }; 607 608 if (WARN_ON_ONCE(unlikely(!asce.val))) 609 return -EINVAL; 610 if (!asce_contains_gfn(asce, start)) 611 return (flags & DAT_WALK_IGN_HOLES) ? 0 : -EFAULT; 612 613 return dat_crste_walk_range(start, min(end, asce_end(asce)), table, &walk); 614 } 615 616 int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey) 617 { 618 union crste *crstep; 619 union pgste pgste; 620 union pte *ptep; 621 int rc; 622 623 skey->skey = 0; 624 rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); 625 if (rc) 626 return rc; 627 628 if (!ptep) { 629 union crste crste; 630 631 crste = READ_ONCE(*crstep); 632 if (!crste.h.fc || !crste.s.fc1.pr) 633 return 0; 634 skey->skey = page_get_storage_key(large_crste_to_phys(crste, gfn)); 635 return 0; 636 } 637 pgste = pgste_get_lock(ptep); 638 if (ptep->h.i) { 639 skey->acc = pgste.acc; 640 skey->fp = pgste.fp; 641 } else { 642 skey->skey = page_get_storage_key(pte_origin(*ptep)); 643 } 644 skey->r |= pgste.gr; 645 skey->c |= pgste.gc; 646 pgste_set_unlock(ptep, pgste); 647 return 0; 648 } 649 650 static void dat_update_ptep_sd(union pgste old, union pgste pgste, union pte *ptep) 651 { 652 if (pgste.acc != old.acc || pgste.fp != old.fp || pgste.gr != old.gr || pgste.gc != old.gc) 653 __atomic64_or(_PAGE_SD, &ptep->val); 654 } 655 656 int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, 657 union skey skey, bool nq) 658 { 659 union pgste pgste, old; 660 union crste *crstep; 661 union pte *ptep; 662 int rc; 663 664 rc = dat_entry_walk(mc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, 665 &crstep, &ptep); 666 if (rc) 667 return rc; 668 669 if (!ptep) { 670 page_set_storage_key(large_crste_to_phys(*crstep, gfn), skey.skey, !nq); 671 return 0; 672 } 673 674 old = pgste_get_lock(ptep); 675 pgste = old; 676 677 pgste.acc = skey.acc; 678 pgste.fp = skey.fp; 679 pgste.gc = skey.c; 680 pgste.gr = skey.r; 681 682 if (!ptep->h.i) { 683 union skey old_skey; 684 685 old_skey.skey = page_get_storage_key(pte_origin(*ptep)); 686 pgste.hc |= old_skey.c; 687 pgste.hr |= old_skey.r; 688 old_skey.c = old.gc; 689 old_skey.r = old.gr; 690 skey.r = 0; 691 skey.c = 0; 692 page_set_storage_key(pte_origin(*ptep), skey.skey, !nq); 693 } 694 695 dat_update_ptep_sd(old, pgste, ptep); 696 pgste_set_unlock(ptep, pgste); 697 return 0; 698 } 699 700 static bool page_cond_set_storage_key(phys_addr_t paddr, union skey skey, union skey *oldkey, 701 bool nq, bool mr, bool mc) 702 { 703 oldkey->skey = page_get_storage_key(paddr); 704 if (oldkey->acc == skey.acc && oldkey->fp == skey.fp && 705 (oldkey->r == skey.r || mr) && (oldkey->c == skey.c || mc)) 706 return false; 707 page_set_storage_key(paddr, skey.skey, !nq); 708 return true; 709 } 710 711 int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn, 712 union skey skey, union skey *oldkey, bool nq, bool mr, bool mc) 713 { 714 union pgste pgste, old; 715 union crste *crstep; 716 union skey prev; 717 union pte *ptep; 718 int rc; 719 720 rc = dat_entry_walk(mmc, gfn, asce, DAT_WALK_LEAF_ALLOC, TABLE_TYPE_PAGE_TABLE, 721 &crstep, &ptep); 722 if (rc) 723 return rc; 724 725 if (!ptep) 726 return page_cond_set_storage_key(large_crste_to_phys(*crstep, gfn), skey, oldkey, 727 nq, mr, mc); 728 729 old = pgste_get_lock(ptep); 730 pgste = old; 731 732 rc = 1; 733 pgste.acc = skey.acc; 734 pgste.fp = skey.fp; 735 pgste.gc = skey.c; 736 pgste.gr = skey.r; 737 738 if (!ptep->h.i) { 739 rc = page_cond_set_storage_key(pte_origin(*ptep), skey, &prev, nq, mr, mc); 740 pgste.hc |= prev.c; 741 pgste.hr |= prev.r; 742 prev.c |= old.gc; 743 prev.r |= old.gr; 744 } else { 745 prev.acc = old.acc; 746 prev.fp = old.fp; 747 prev.c = old.gc; 748 prev.r = old.gr; 749 } 750 if (oldkey) 751 *oldkey = prev; 752 753 dat_update_ptep_sd(old, pgste, ptep); 754 pgste_set_unlock(ptep, pgste); 755 return rc; 756 } 757 758 int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey) 759 { 760 union pgste pgste, old; 761 union crste *crstep; 762 union pte *ptep; 763 int rc; 764 765 skey->skey = 0; 766 767 rc = dat_entry_walk(NULL, gfn, asce, DAT_WALK_ANY, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep); 768 if (rc) 769 return rc; 770 771 if (!ptep) { 772 union crste crste = READ_ONCE(*crstep); 773 774 if (!crste.h.fc || !crste.s.fc1.pr) 775 return 0; 776 skey->skey = page_reset_referenced(large_crste_to_phys(*crstep, gfn)) << 1; 777 return 0; 778 } 779 old = pgste_get_lock(ptep); 780 pgste = old; 781 782 if (!ptep->h.i) { 783 skey->skey = page_reset_referenced(pte_origin(*ptep)) << 1; 784 pgste.hr = skey->r; 785 } 786 skey->r |= pgste.gr; 787 skey->c |= pgste.gc; 788 pgste.gr = 0; 789 790 dat_update_ptep_sd(old, pgste, ptep); 791 pgste_set_unlock(ptep, pgste); 792 return 0; 793 } 794 795 static long dat_reset_skeys_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 796 { 797 union pgste pgste; 798 799 pgste = pgste_get_lock(ptep); 800 pgste.acc = 0; 801 pgste.fp = 0; 802 pgste.gr = 0; 803 pgste.gc = 0; 804 if (ptep->s.pr) 805 page_set_storage_key(pte_origin(*ptep), PAGE_DEFAULT_KEY, 1); 806 pgste_set_unlock(ptep, pgste); 807 808 if (need_resched()) 809 return next; 810 return 0; 811 } 812 813 static long dat_reset_skeys_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 814 { 815 phys_addr_t addr, end, origin = crste_origin_large(*crstep); 816 817 if (!crstep->h.fc || !crstep->s.fc1.pr) 818 return 0; 819 820 addr = ((max(gfn, walk->start) - gfn) << PAGE_SHIFT) + origin; 821 end = ((min(next, walk->end) - gfn) << PAGE_SHIFT) + origin; 822 while (ALIGN(addr + 1, _SEGMENT_SIZE) <= end) 823 addr = sske_frame(addr, PAGE_DEFAULT_KEY); 824 for ( ; addr < end; addr += PAGE_SIZE) 825 page_set_storage_key(addr, PAGE_DEFAULT_KEY, 1); 826 827 if (need_resched()) 828 return next; 829 return 0; 830 } 831 832 long dat_reset_skeys(union asce asce, gfn_t start) 833 { 834 const struct dat_walk_ops ops = { 835 .pte_entry = dat_reset_skeys_pte, 836 .pmd_entry = dat_reset_skeys_crste, 837 .pud_entry = dat_reset_skeys_crste, 838 }; 839 840 return _dat_walk_gfn_range(start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, NULL); 841 } 842 843 struct slot_priv { 844 unsigned long token; 845 struct kvm_s390_mmu_cache *mc; 846 }; 847 848 static long _dat_slot_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 849 { 850 struct slot_priv *p = walk->priv; 851 union crste dummy = { .val = p->token }; 852 union pte new_pte, pte = READ_ONCE(*ptep); 853 union pgste pgste; 854 855 new_pte = _PTE_TOK(dummy.tok.type, dummy.tok.par); 856 857 /* Table entry already in the desired state. */ 858 if (pte.val == new_pte.val) 859 return 0; 860 861 pgste = pgste_get_lock(ptep); 862 pgste = __dat_ptep_xchg(ptep, pgste, new_pte, gfn, walk->asce, false); 863 pgste.cmma_d = 0; 864 pgste_set_unlock(ptep, pgste); 865 866 return 0; 867 } 868 869 static long _dat_slot_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 870 { 871 union crste new_crste, crste = READ_ONCE(*crstep); 872 struct slot_priv *p = walk->priv; 873 874 new_crste.val = p->token; 875 new_crste.h.tt = crste.h.tt; 876 877 /* Table entry already in the desired state. */ 878 if (crste.val == new_crste.val) 879 return 0; 880 881 /* This table entry needs to be updated. */ 882 if (walk->start <= gfn && walk->end >= next) { 883 if (!dat_crstep_xchg_atomic(crstep, crste, new_crste, gfn, walk->asce)) 884 return -EINVAL; 885 /* A lower level table was present, needs to be freed. */ 886 if (!crste.h.fc && !crste.h.i) { 887 if (is_pmd(crste)) 888 dat_free_pt(dereference_pmd(crste.pmd)); 889 else 890 dat_free_level(dereference_crste(crste), true); 891 } 892 return 0; 893 } 894 895 /* A lower level table is present, things will handled there. */ 896 if (!crste.h.fc && !crste.h.i) 897 return 0; 898 /* Split (install a lower level table), and handle things there. */ 899 return dat_split_crste(p->mc, crstep, gfn, walk->asce, false); 900 } 901 902 static const struct dat_walk_ops dat_slot_ops = { 903 .pte_entry = _dat_slot_pte, 904 .crste_ops = { _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, _dat_slot_crste, }, 905 }; 906 907 int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, 908 u16 type, u16 param) 909 { 910 struct slot_priv priv = { 911 .token = _CRSTE_TOK(0, type, param).val, 912 .mc = mc, 913 }; 914 915 return _dat_walk_gfn_range(start, end, asce, &dat_slot_ops, 916 DAT_WALK_IGN_HOLES | DAT_WALK_ANY, &priv); 917 } 918 919 static void pgste_set_unlock_multiple(union pte *first, int n, union pgste *pgstes) 920 { 921 int i; 922 923 for (i = 0; i < n; i++) { 924 if (!pgstes[i].pcl) 925 break; 926 pgste_set_unlock(first + i, pgstes[i]); 927 } 928 } 929 930 static bool pgste_get_trylock_multiple(union pte *first, int n, union pgste *pgstes) 931 { 932 int i; 933 934 for (i = 0; i < n; i++) { 935 if (!pgste_get_trylock(first + i, pgstes + i)) 936 break; 937 } 938 if (i == n) 939 return true; 940 pgste_set_unlock_multiple(first, n, pgstes); 941 return false; 942 } 943 944 unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param) 945 { 946 union pgste pgstes[4] = {}; 947 unsigned long res = 0; 948 int i, n; 949 950 n = param.len + 1; 951 952 while (!pgste_get_trylock_multiple(table->ptes + param.offset, n, pgstes)) 953 cpu_relax(); 954 955 for (i = 0; i < n; i++) 956 res = res << 16 | pgstes[i].val16; 957 958 pgste_set_unlock_multiple(table->ptes + param.offset, n, pgstes); 959 return res; 960 } 961 962 void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val) 963 { 964 union pgste pgstes[4] = {}; 965 int i, n; 966 967 n = param.len + 1; 968 969 while (!pgste_get_trylock_multiple(table->ptes + param.offset, n, pgstes)) 970 cpu_relax(); 971 972 for (i = param.len; i >= 0; i--) { 973 pgstes[i].val16 = val; 974 val = val >> 16; 975 } 976 977 pgste_set_unlock_multiple(table->ptes + param.offset, n, pgstes); 978 } 979 980 static long _dat_test_young_pte(union pte *ptep, gfn_t start, gfn_t end, struct dat_walk *walk) 981 { 982 return ptep->s.y; 983 } 984 985 static long _dat_test_young_crste(union crste *crstep, gfn_t start, gfn_t end, 986 struct dat_walk *walk) 987 { 988 return crstep->h.fc && crstep->s.fc1.y; 989 } 990 991 static const struct dat_walk_ops test_age_ops = { 992 .pte_entry = _dat_test_young_pte, 993 .pmd_entry = _dat_test_young_crste, 994 .pud_entry = _dat_test_young_crste, 995 }; 996 997 /** 998 * dat_test_age_gfn() - Test young. 999 * @asce: The ASCE whose address range is to be tested. 1000 * @start: The first guest frame of the range to check. 1001 * @end: The guest frame after the last in the range. 1002 * 1003 * Context: called by KVM common code with the kvm mmu write lock held. 1004 * 1005 * Return: %true if any page in the given range is young, otherwise %false. 1006 */ 1007 bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end) 1008 { 1009 return _dat_walk_gfn_range(start, end, asce, &test_age_ops, 0, NULL) > 0; 1010 } 1011 1012 static long dat_set_pn_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1013 { 1014 union crste newcrste, oldcrste; 1015 int *n = walk->priv; 1016 1017 do { 1018 oldcrste = READ_ONCE(*crstep); 1019 if (!oldcrste.h.fc || oldcrste.h.i || oldcrste.h.p) 1020 return 0; 1021 if (oldcrste.s.fc1.prefix_notif) 1022 break; 1023 newcrste = oldcrste; 1024 newcrste.s.fc1.prefix_notif = 1; 1025 } while (!dat_crstep_xchg_atomic(crstep, oldcrste, newcrste, gfn, walk->asce)); 1026 *n = 2; 1027 return 0; 1028 } 1029 1030 static long dat_set_pn_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1031 { 1032 int *n = walk->priv; 1033 union pgste pgste; 1034 1035 pgste = pgste_get_lock(ptep); 1036 if (!ptep->h.i && !ptep->h.p) { 1037 pgste.prefix_notif = 1; 1038 *n += 1; 1039 } 1040 pgste_set_unlock(ptep, pgste); 1041 return 0; 1042 } 1043 1044 int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn) 1045 { 1046 static const struct dat_walk_ops ops = { 1047 .pte_entry = dat_set_pn_pte, 1048 .pmd_entry = dat_set_pn_crste, 1049 .pud_entry = dat_set_pn_crste, 1050 }; 1051 1052 int n = 0; 1053 1054 _dat_walk_gfn_range(gfn, gfn + 2, asce, &ops, DAT_WALK_IGN_HOLES, &n); 1055 if (n != 2) 1056 return -EAGAIN; 1057 return 0; 1058 } 1059 1060 /** 1061 * dat_perform_essa() - Perform ESSA actions on the PGSTE. 1062 * @asce: The asce to operate on. 1063 * @gfn: The guest page frame to operate on. 1064 * @orc: The specific action to perform, see the ESSA_SET_* macros. 1065 * @state: The storage attributes to be returned to the guest. 1066 * @dirty: Returns whether the function dirtied a previously clean entry. 1067 * 1068 * Context: Called with kvm->mmu_lock held. 1069 * 1070 * Return: 1071 * * %1 if the page state has been altered and the page is to be added to the CBRL 1072 * * %0 if the page state has been altered, but the page is not to be added to the CBRL 1073 * * %-1 if the page state has not been altered and the page is not to be added to the CBRL 1074 */ 1075 int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty) 1076 { 1077 union crste *crstep; 1078 union pgste pgste; 1079 union pte *ptep; 1080 int res = 0; 1081 1082 if (dat_entry_walk(NULL, gfn, asce, 0, TABLE_TYPE_PAGE_TABLE, &crstep, &ptep)) { 1083 *state = (union essa_state) { .exception = 1 }; 1084 return -1; 1085 } 1086 1087 pgste = pgste_get_lock(ptep); 1088 1089 *state = (union essa_state) { 1090 .content = (ptep->h.i << 1) + (ptep->h.i && pgste.zero), 1091 .nodat = pgste.nodat, 1092 .usage = pgste.usage, 1093 }; 1094 1095 switch (orc) { 1096 case ESSA_GET_STATE: 1097 res = -1; 1098 break; 1099 case ESSA_SET_STABLE: 1100 pgste.usage = PGSTE_GPS_USAGE_STABLE; 1101 pgste.nodat = 0; 1102 break; 1103 case ESSA_SET_UNUSED: 1104 pgste.usage = PGSTE_GPS_USAGE_UNUSED; 1105 if (ptep->h.i) 1106 res = 1; 1107 break; 1108 case ESSA_SET_VOLATILE: 1109 pgste.usage = PGSTE_GPS_USAGE_VOLATILE; 1110 if (ptep->h.i) 1111 res = 1; 1112 break; 1113 case ESSA_SET_POT_VOLATILE: 1114 if (!ptep->h.i) { 1115 pgste.usage = PGSTE_GPS_USAGE_POT_VOLATILE; 1116 } else if (pgste.zero) { 1117 pgste.usage = PGSTE_GPS_USAGE_VOLATILE; 1118 } else if (!pgste.gc) { 1119 pgste.usage = PGSTE_GPS_USAGE_VOLATILE; 1120 res = 1; 1121 } 1122 break; 1123 case ESSA_SET_STABLE_RESIDENT: 1124 pgste.usage = PGSTE_GPS_USAGE_STABLE; 1125 /* 1126 * Since the resident state can go away any time after this 1127 * call, we will not make this page resident. We can revisit 1128 * this decision if a guest will ever start using this. 1129 */ 1130 break; 1131 case ESSA_SET_STABLE_IF_RESIDENT: 1132 if (!ptep->h.i) 1133 pgste.usage = PGSTE_GPS_USAGE_STABLE; 1134 break; 1135 case ESSA_SET_STABLE_NODAT: 1136 pgste.usage = PGSTE_GPS_USAGE_STABLE; 1137 pgste.nodat = 1; 1138 break; 1139 default: 1140 WARN_ONCE(1, "Invalid ORC!"); 1141 res = -1; 1142 break; 1143 } 1144 /* If we are discarding a page, set it to logical zero. */ 1145 pgste.zero = res == 1; 1146 if (orc > 0) { 1147 *dirty = !pgste.cmma_d; 1148 pgste.cmma_d = 1; 1149 } 1150 1151 pgste_set_unlock(ptep, pgste); 1152 1153 return res; 1154 } 1155 1156 static long dat_reset_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1157 { 1158 union pgste pgste; 1159 1160 pgste = pgste_get_lock(ptep); 1161 pgste.usage = 0; 1162 pgste.nodat = 0; 1163 pgste.cmma_d = 0; 1164 pgste_set_unlock(ptep, pgste); 1165 if (need_resched()) 1166 return next; 1167 return 0; 1168 } 1169 1170 long dat_reset_cmma(union asce asce, gfn_t start) 1171 { 1172 const struct dat_walk_ops dat_reset_cmma_ops = { 1173 .pte_entry = dat_reset_cmma_pte, 1174 }; 1175 1176 return _dat_walk_gfn_range(start, asce_end(asce), asce, &dat_reset_cmma_ops, 1177 DAT_WALK_IGN_HOLES, NULL); 1178 } 1179 1180 struct dat_get_cmma_state { 1181 gfn_t start; 1182 gfn_t end; 1183 unsigned int count; 1184 u8 *values; 1185 atomic64_t *remaining; 1186 }; 1187 1188 static long __dat_peek_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1189 { 1190 struct dat_get_cmma_state *state = walk->priv; 1191 union pgste pgste; 1192 1193 pgste = pgste_get_lock(ptep); 1194 state->values[gfn - walk->start] = pgste.usage | (pgste.nodat << 6); 1195 pgste_set_unlock(ptep, pgste); 1196 state->end = next; 1197 1198 return 0; 1199 } 1200 1201 static long __dat_peek_cmma_crste(union crste *crstep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1202 { 1203 struct dat_get_cmma_state *state = walk->priv; 1204 1205 if (crstep->h.i) 1206 state->end = min(walk->end, next); 1207 return 0; 1208 } 1209 1210 int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values) 1211 { 1212 const struct dat_walk_ops ops = { 1213 .pte_entry = __dat_peek_cmma_pte, 1214 .pmd_entry = __dat_peek_cmma_crste, 1215 .pud_entry = __dat_peek_cmma_crste, 1216 .p4d_entry = __dat_peek_cmma_crste, 1217 .pgd_entry = __dat_peek_cmma_crste, 1218 }; 1219 struct dat_get_cmma_state state = { .values = values, }; 1220 int rc; 1221 1222 rc = _dat_walk_gfn_range(start, start + *count, asce, &ops, DAT_WALK_DEFAULT, &state); 1223 *count = state.end >= start ? state.end - start : 0; 1224 /* Return success if at least one value was saved, otherwise an error. */ 1225 return (rc == -EFAULT && *count > 0) ? 0 : rc; 1226 } 1227 1228 static long __dat_get_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1229 { 1230 struct dat_get_cmma_state *state = walk->priv; 1231 union pgste pgste; 1232 1233 if (state->start != -1) { 1234 if ((gfn - state->end) > KVM_S390_MAX_BIT_DISTANCE) 1235 return 1; 1236 if (gfn - state->start >= state->count) 1237 return 1; 1238 } 1239 1240 if (!READ_ONCE(*pgste_of(ptep)).cmma_d) 1241 return 0; 1242 1243 pgste = pgste_get_lock(ptep); 1244 if (pgste.cmma_d) { 1245 if (state->start == -1) 1246 state->start = gfn; 1247 pgste.cmma_d = 0; 1248 atomic64_dec(state->remaining); 1249 state->values[gfn - state->start] = pgste.usage | pgste.nodat << 6; 1250 state->end = next; 1251 } 1252 pgste_set_unlock(ptep, pgste); 1253 return 0; 1254 } 1255 1256 int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem) 1257 { 1258 const struct dat_walk_ops ops = { .pte_entry = __dat_get_cmma_pte, }; 1259 struct dat_get_cmma_state state = { 1260 .remaining = rem, 1261 .values = values, 1262 .count = *count, 1263 .start = -1, 1264 }; 1265 1266 _dat_walk_gfn_range(*start, asce_end(asce), asce, &ops, DAT_WALK_IGN_HOLES, &state); 1267 /* If no dirty pages were found, wrap around and continue searching */ 1268 if (*start && state.start == -1) 1269 _dat_walk_gfn_range(0, *start, asce, &ops, DAT_WALK_IGN_HOLES, &state); 1270 1271 if (state.start == -1) { 1272 *count = 0; 1273 } else { 1274 *count = state.end - state.start; 1275 *start = state.start; 1276 } 1277 1278 return 0; 1279 } 1280 1281 struct dat_set_cmma_state { 1282 unsigned long mask; 1283 const u8 *bits; 1284 }; 1285 1286 static long __dat_set_cmma_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk) 1287 { 1288 struct dat_set_cmma_state *state = walk->priv; 1289 union pgste pgste, tmp; 1290 1291 tmp.val = (state->bits[gfn - walk->start] << 24) & state->mask; 1292 1293 pgste = pgste_get_lock(ptep); 1294 pgste.usage = tmp.usage; 1295 pgste.nodat = tmp.nodat; 1296 pgste_set_unlock(ptep, pgste); 1297 1298 return 0; 1299 } 1300 1301 /** 1302 * dat_set_cmma_bits() - Set CMMA bits for a range of guest pages. 1303 * @mc: Cache used for allocations. 1304 * @asce: The ASCE of the guest. 1305 * @gfn: The guest frame of the fist page whose CMMA bits are to set. 1306 * @count: How many pages need to be processed. 1307 * @mask: Which PGSTE bits should be set. 1308 * @bits: Points to an array with the CMMA attributes. 1309 * 1310 * This function sets the CMMA attributes for the given pages. If the input 1311 * buffer has zero length, no action is taken, otherwise the attributes are 1312 * set and the mm->context.uses_cmm flag is set. 1313 * 1314 * Each byte in @bits contains new values for bits 32-39 of the PGSTE. 1315 * Currently, only the fields NT and US are applied. 1316 * 1317 * Return: %0 in case of success, a negative error value otherwise. 1318 */ 1319 int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, 1320 unsigned long count, unsigned long mask, const uint8_t *bits) 1321 { 1322 const struct dat_walk_ops ops = { .pte_entry = __dat_set_cmma_pte, }; 1323 struct dat_set_cmma_state state = { .mask = mask, .bits = bits, }; 1324 union crste *crstep; 1325 union pte *ptep; 1326 gfn_t cur; 1327 int rc; 1328 1329 for (cur = ALIGN_DOWN(gfn, _PAGE_ENTRIES); cur < gfn + count; cur += _PAGE_ENTRIES) { 1330 rc = dat_entry_walk(mc, cur, asce, DAT_WALK_ALLOC, TABLE_TYPE_PAGE_TABLE, 1331 &crstep, &ptep); 1332 if (rc) 1333 return rc; 1334 } 1335 return _dat_walk_gfn_range(gfn, gfn + count, asce, &ops, DAT_WALK_IGN_HOLES, &state); 1336 } 1337