1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * KVM guest address space mapping code 4 * 5 * Copyright IBM Corp. 2024, 2025 6 * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> 7 */ 8 9 #ifndef ARCH_KVM_GMAP_DAT_H 10 #define ARCH_KVM_GMAP_DAT_H 11 12 #include <linux/kvm_host.h> 13 #include <linux/radix-tree.h> 14 #include <linux/refcount.h> 15 #include <linux/io.h> 16 #include <linux/kvm_types.h> 17 #include <linux/pgalloc.h> 18 #include <asm/tlbflush.h> 19 #include <asm/dat-bits.h> 20 21 /* 22 * Base address and length must be sent at the start of each block, therefore 23 * it's cheaper to send some clean data, as long as it's less than the size of 24 * two longs. 25 */ 26 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *)) 27 /* For consistency */ 28 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX) 29 30 #define _ASCE(x) ((union asce) { .val = (x), }) 31 #define NULL_ASCE _ASCE(0) 32 33 enum { 34 _DAT_TOKEN_NONE = 0, 35 _DAT_TOKEN_PIC, 36 }; 37 38 #define _CRSTE_TOK(l, t, p) ((union crste) { \ 39 .tok.i = 1, \ 40 .tok.tt = (l), \ 41 .tok.type = (t), \ 42 .tok.par = (p) \ 43 }) 44 #define _CRSTE_PIC(l, p) _CRSTE_TOK(l, _DAT_TOKEN_PIC, p) 45 46 #define _CRSTE_HOLE(l) _CRSTE_PIC(l, PGM_ADDRESSING) 47 #define _CRSTE_EMPTY(l) _CRSTE_TOK(l, _DAT_TOKEN_NONE, 0) 48 49 #define _PMD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_SEGMENT) 50 51 #define _PTE_TOK(t, p) ((union pte) { .tok.i = 1, .tok.type = (t), .tok.par = (p) }) 52 #define _PTE_EMPTY _PTE_TOK(_DAT_TOKEN_NONE, 0) 53 54 /* This fake table type is used for page table walks (both for normal page tables and vSIE) */ 55 #define TABLE_TYPE_PAGE_TABLE -1 56 57 enum dat_walk_flags { 58 DAT_WALK_USES_SKEYS = 0x40, 59 DAT_WALK_CONTINUE = 0x20, 60 DAT_WALK_IGN_HOLES = 0x10, 61 DAT_WALK_SPLIT = 0x08, 62 DAT_WALK_ALLOC = 0x04, 63 DAT_WALK_ANY = 0x02, 64 DAT_WALK_LEAF = 0x01, 65 DAT_WALK_DEFAULT = 0 66 }; 67 68 #define DAT_WALK_SPLIT_ALLOC (DAT_WALK_SPLIT | DAT_WALK_ALLOC) 69 #define DAT_WALK_ALLOC_CONTINUE (DAT_WALK_CONTINUE | DAT_WALK_ALLOC) 70 #define DAT_WALK_LEAF_ALLOC (DAT_WALK_LEAF | DAT_WALK_ALLOC) 71 72 union pte { 73 unsigned long val; 74 union page_table_entry h; 75 struct { 76 unsigned long :56; /* Hardware bits */ 77 unsigned long u : 1; /* Page unused */ 78 unsigned long s : 1; /* Special */ 79 unsigned long w : 1; /* Writable */ 80 unsigned long r : 1; /* Readable */ 81 unsigned long d : 1; /* Dirty */ 82 unsigned long y : 1; /* Young */ 83 unsigned long sd: 1; /* Soft dirty */ 84 unsigned long pr: 1; /* Present */ 85 } s; 86 struct { 87 unsigned char hwbytes[7]; 88 unsigned char swbyte; 89 }; 90 union { 91 struct { 92 unsigned long type :16; /* Token type */ 93 unsigned long par :16; /* Token parameter */ 94 unsigned long :20; 95 unsigned long : 1; /* Must be 0 */ 96 unsigned long i : 1; /* Must be 1 */ 97 unsigned long : 2; 98 unsigned long : 7; 99 unsigned long pr : 1; /* Must be 0 */ 100 }; 101 struct { 102 unsigned long token:32; /* Token and parameter */ 103 unsigned long :32; 104 }; 105 } tok; 106 }; 107 108 #define _SEGMENT_FR_MASK (_SEGMENT_MASK >> PAGE_SHIFT) 109 #define _REGION3_FR_MASK (_REGION3_MASK >> PAGE_SHIFT) 110 #define _PAGES_PER_SEGMENT _PAGE_ENTRIES 111 #define _PAGES_PER_REGION3 (_PAGES_PER_SEGMENT * _CRST_ENTRIES) 112 113 /* Soft dirty, needed as macro for atomic operations on ptes */ 114 #define _PAGE_SD 0x002 115 116 /* Needed as macro to perform atomic operations */ 117 #define PGSTE_PCL_BIT 0x0080000000000000UL /* PCL lock, HW bit */ 118 #define PGSTE_CMMA_D_BIT 0x0000000000008000UL /* CMMA dirty soft-bit */ 119 120 enum pgste_gps_usage { 121 PGSTE_GPS_USAGE_STABLE = 0, 122 PGSTE_GPS_USAGE_UNUSED, 123 PGSTE_GPS_USAGE_POT_VOLATILE, 124 PGSTE_GPS_USAGE_VOLATILE, 125 }; 126 127 union pgste { 128 unsigned long val; 129 struct { 130 unsigned long acc : 4; 131 unsigned long fp : 1; 132 unsigned long : 3; 133 unsigned long pcl : 1; 134 unsigned long hr : 1; 135 unsigned long hc : 1; 136 unsigned long : 2; 137 unsigned long gr : 1; 138 unsigned long gc : 1; 139 unsigned long : 1; 140 unsigned long :16; /* val16 */ 141 unsigned long zero : 1; 142 unsigned long nodat : 1; 143 unsigned long : 4; 144 unsigned long usage : 2; 145 unsigned long : 8; 146 unsigned long cmma_d : 1; /* Dirty flag for CMMA bits */ 147 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ 148 unsigned long vsie_notif : 1; /* Referenced in a shadow table */ 149 unsigned long vsie_gmem : 1; /* Contains nested guest memory */ 150 unsigned long : 4; 151 unsigned long : 8; 152 }; 153 struct { 154 unsigned short hwbytes0; 155 unsigned short val16; /* Used to store chunked values, see dat_{s,g}et_ptval() */ 156 unsigned short hwbytes4; 157 unsigned char flags; /* Maps to the software bits */ 158 unsigned char hwbyte7; 159 } __packed; 160 }; 161 162 union pmd { 163 unsigned long val; 164 union segment_table_entry h; 165 struct { 166 struct { 167 unsigned long :44; /* HW */ 168 unsigned long : 3; /* Unused */ 169 unsigned long : 1; /* HW */ 170 unsigned long s : 1; /* Special */ 171 unsigned long w : 1; /* Writable soft-bit */ 172 unsigned long r : 1; /* Readable soft-bit */ 173 unsigned long d : 1; /* Dirty */ 174 unsigned long y : 1; /* Young */ 175 unsigned long : 3; /* HW */ 176 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ 177 unsigned long vsie_notif : 1; /* Referenced in a shadow table */ 178 unsigned long : 4; /* HW */ 179 unsigned long sd : 1; /* Soft-Dirty */ 180 unsigned long pr : 1; /* Present */ 181 } fc1; 182 } s; 183 }; 184 185 union pud { 186 unsigned long val; 187 union region3_table_entry h; 188 struct { 189 struct { 190 unsigned long :33; /* HW */ 191 unsigned long :14; /* Unused */ 192 unsigned long : 1; /* HW */ 193 unsigned long s : 1; /* Special */ 194 unsigned long w : 1; /* Writable soft-bit */ 195 unsigned long r : 1; /* Readable soft-bit */ 196 unsigned long d : 1; /* Dirty */ 197 unsigned long y : 1; /* Young */ 198 unsigned long : 3; /* HW */ 199 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ 200 unsigned long vsie_notif : 1; /* Referenced in a shadow table */ 201 unsigned long : 4; /* HW */ 202 unsigned long sd : 1; /* Soft-Dirty */ 203 unsigned long pr : 1; /* Present */ 204 } fc1; 205 } s; 206 }; 207 208 union p4d { 209 unsigned long val; 210 union region2_table_entry h; 211 }; 212 213 union pgd { 214 unsigned long val; 215 union region1_table_entry h; 216 }; 217 218 union crste { 219 unsigned long val; 220 union { 221 struct { 222 unsigned long :52; 223 unsigned long : 1; 224 unsigned long fc: 1; 225 unsigned long p : 1; 226 unsigned long : 1; 227 unsigned long : 2; 228 unsigned long i : 1; 229 unsigned long : 1; 230 unsigned long tt: 2; 231 unsigned long : 2; 232 }; 233 struct { 234 unsigned long to:52; 235 unsigned long : 1; 236 unsigned long fc: 1; 237 unsigned long p : 1; 238 unsigned long : 1; 239 unsigned long tf: 2; 240 unsigned long i : 1; 241 unsigned long : 1; 242 unsigned long tt: 2; 243 unsigned long tl: 2; 244 } fc0; 245 struct { 246 unsigned long :47; 247 unsigned long av : 1; /* ACCF-Validity Control */ 248 unsigned long acc: 4; /* Access-Control Bits */ 249 unsigned long f : 1; /* Fetch-Protection Bit */ 250 unsigned long fc : 1; /* Format-Control */ 251 unsigned long p : 1; /* DAT-Protection Bit */ 252 unsigned long iep: 1; /* Instruction-Execution-Protection */ 253 unsigned long : 2; 254 unsigned long i : 1; /* Segment-Invalid Bit */ 255 unsigned long cs : 1; /* Common-Segment Bit */ 256 unsigned long tt : 2; /* Table-Type Bits */ 257 unsigned long : 2; 258 } fc1; 259 } h; 260 struct { 261 struct { 262 unsigned long :47; 263 unsigned long : 1; /* HW (should be 0) */ 264 unsigned long s : 1; /* Special */ 265 unsigned long w : 1; /* Writable */ 266 unsigned long r : 1; /* Readable */ 267 unsigned long d : 1; /* Dirty */ 268 unsigned long y : 1; /* Young */ 269 unsigned long : 3; /* HW */ 270 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ 271 unsigned long vsie_notif : 1; /* Referenced in a shadow table */ 272 unsigned long : 4; /* HW */ 273 unsigned long sd : 1; /* Soft-Dirty */ 274 unsigned long pr : 1; /* Present */ 275 } fc1; 276 } s; 277 union { 278 struct { 279 unsigned long type :16; /* Token type */ 280 unsigned long par :16; /* Token parameter */ 281 unsigned long :26; 282 unsigned long i : 1; /* Must be 1 */ 283 unsigned long : 1; 284 unsigned long tt : 2; 285 unsigned long : 1; 286 unsigned long pr : 1; /* Must be 0 */ 287 }; 288 struct { 289 unsigned long token:32; /* Token and parameter */ 290 unsigned long :32; 291 }; 292 } tok; 293 union pmd pmd; 294 union pud pud; 295 union p4d p4d; 296 union pgd pgd; 297 }; 298 299 union skey { 300 unsigned char skey; 301 struct { 302 unsigned char acc :4; 303 unsigned char fp :1; 304 unsigned char r :1; 305 unsigned char c :1; 306 unsigned char zero:1; 307 }; 308 }; 309 310 static_assert(sizeof(union pgste) == sizeof(unsigned long)); 311 static_assert(sizeof(union pte) == sizeof(unsigned long)); 312 static_assert(sizeof(union pmd) == sizeof(unsigned long)); 313 static_assert(sizeof(union pud) == sizeof(unsigned long)); 314 static_assert(sizeof(union p4d) == sizeof(unsigned long)); 315 static_assert(sizeof(union pgd) == sizeof(unsigned long)); 316 static_assert(sizeof(union crste) == sizeof(unsigned long)); 317 static_assert(sizeof(union skey) == sizeof(char)); 318 319 struct segment_table { 320 union pmd pmds[_CRST_ENTRIES]; 321 }; 322 323 struct region3_table { 324 union pud puds[_CRST_ENTRIES]; 325 }; 326 327 struct region2_table { 328 union p4d p4ds[_CRST_ENTRIES]; 329 }; 330 331 struct region1_table { 332 union pgd pgds[_CRST_ENTRIES]; 333 }; 334 335 struct crst_table { 336 union { 337 union crste crstes[_CRST_ENTRIES]; 338 struct segment_table segment; 339 struct region3_table region3; 340 struct region2_table region2; 341 struct region1_table region1; 342 }; 343 }; 344 345 struct page_table { 346 union pte ptes[_PAGE_ENTRIES]; 347 union pgste pgstes[_PAGE_ENTRIES]; 348 }; 349 350 static_assert(sizeof(struct crst_table) == _CRST_TABLE_SIZE); 351 static_assert(sizeof(struct page_table) == PAGE_SIZE); 352 353 struct dat_walk; 354 355 typedef long (*dat_walk_op)(union crste *crste, gfn_t gfn, gfn_t next, struct dat_walk *w); 356 357 struct dat_walk_ops { 358 union { 359 dat_walk_op crste_ops[4]; 360 struct { 361 dat_walk_op pmd_entry; 362 dat_walk_op pud_entry; 363 dat_walk_op p4d_entry; 364 dat_walk_op pgd_entry; 365 }; 366 }; 367 long (*pte_entry)(union pte *pte, gfn_t gfn, gfn_t next, struct dat_walk *w); 368 }; 369 370 struct dat_walk { 371 const struct dat_walk_ops *ops; 372 union crste *last; 373 union pte *last_pte; 374 union asce asce; 375 gfn_t start; 376 gfn_t end; 377 int flags; 378 void *priv; 379 }; 380 381 struct ptval_param { 382 unsigned char offset : 6; 383 unsigned char len : 2; 384 }; 385 386 /** 387 * _pte() - Useful constructor for union pte 388 * @pfn: the pfn this pte should point to. 389 * @writable: whether the pte should be writable. 390 * @dirty: whether the pte should be dirty. 391 * @special: whether the pte should be marked as special 392 * 393 * The pte is also marked as young and present. If the pte is marked as dirty, 394 * it gets marked as soft-dirty too. If the pte is not dirty, the hardware 395 * protect bit is set (independently of the write softbit); this way proper 396 * dirty tracking can be performed. 397 * 398 * Return: a union pte value. 399 */ 400 static inline union pte _pte(kvm_pfn_t pfn, bool writable, bool dirty, bool special) 401 { 402 union pte res = { .val = PFN_PHYS(pfn) }; 403 404 res.h.p = !dirty; 405 res.s.y = 1; 406 res.s.pr = 1; 407 res.s.w = writable; 408 res.s.d = dirty; 409 res.s.sd = dirty; 410 res.s.s = special; 411 return res; 412 } 413 414 static inline union crste _crste_fc0(kvm_pfn_t pfn, int tt) 415 { 416 union crste res = { .val = PFN_PHYS(pfn) }; 417 418 res.h.tt = tt; 419 res.h.fc0.tl = _REGION_ENTRY_LENGTH; 420 res.h.fc0.tf = 0; 421 return res; 422 } 423 424 /** 425 * _crste() - Useful constructor for union crste with FC=1 426 * @pfn: the pfn this pte should point to. 427 * @tt: the table type 428 * @writable: whether the pte should be writable. 429 * @dirty: whether the pte should be dirty. 430 * 431 * The crste is also marked as young and present. If the crste is marked as 432 * dirty, it gets marked as soft-dirty too. If the crste is not dirty, the 433 * hardware protect bit is set (independently of the write softbit); this way 434 * proper dirty tracking can be performed. 435 * 436 * Return: a union crste value. 437 */ 438 static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool writable, bool dirty) 439 { 440 union crste res = { .val = PFN_PHYS(pfn) & _SEGMENT_MASK }; 441 442 res.h.tt = tt; 443 res.h.p = !dirty; 444 res.h.fc = 1; 445 res.s.fc1.y = 1; 446 res.s.fc1.pr = 1; 447 res.s.fc1.w = writable; 448 res.s.fc1.d = dirty; 449 res.s.fc1.sd = dirty; 450 return res; 451 } 452 453 union essa_state { 454 unsigned char val; 455 struct { 456 unsigned char : 2; 457 unsigned char nodat : 1; 458 unsigned char exception : 1; 459 unsigned char usage : 2; 460 unsigned char content : 2; 461 }; 462 }; 463 464 /** 465 * struct vsie_rmap - reverse mapping for shadow page table entries 466 * @next: pointer to next rmap in the list 467 * @r_gfn: virtual rmap address in the shadow guest address space 468 */ 469 struct vsie_rmap { 470 struct vsie_rmap *next; 471 union { 472 unsigned long val; 473 struct { 474 long level: 8; 475 unsigned long : 4; 476 unsigned long r_gfn:52; 477 }; 478 }; 479 }; 480 481 static_assert(sizeof(struct vsie_rmap) == 2 * sizeof(long)); 482 483 #define KVM_S390_MMU_CACHE_N_CRSTS 6 484 #define KVM_S390_MMU_CACHE_N_PTS 2 485 #define KVM_S390_MMU_CACHE_N_RMAPS 16 486 struct kvm_s390_mmu_cache { 487 void *crsts[KVM_S390_MMU_CACHE_N_CRSTS]; 488 void *pts[KVM_S390_MMU_CACHE_N_PTS]; 489 void *rmaps[KVM_S390_MMU_CACHE_N_RMAPS]; 490 short int n_crsts; 491 short int n_pts; 492 short int n_rmaps; 493 }; 494 495 struct guest_fault { 496 gfn_t gfn; /* Guest frame */ 497 kvm_pfn_t pfn; /* Host PFN */ 498 struct page *page; /* Host page */ 499 union pte *ptep; /* Used to resolve the fault, or NULL */ 500 union crste *crstep; /* Used to resolve the fault, or NULL */ 501 bool writable; /* Mapping is writable */ 502 bool write_attempt; /* Write access attempted */ 503 bool attempt_pfault; /* Attempt a pfault first */ 504 bool valid; /* This entry contains valid data */ 505 bool crste_region3; /* Whether crstep refers to a region3 entry */ 506 void (*callback)(struct guest_fault *f); 507 void *priv; 508 }; 509 510 /* 511 * 0 1 2 3 4 5 6 7 512 * +-------+-------+-------+-------+-------+-------+-------+-------+ 513 * 0 | | PGT_ADDR | 514 * 8 | VMADDR | | 515 * 16 | | 516 * 24 | | 517 */ 518 #define MKPTVAL(o, l) ((struct ptval_param) { .offset = (o), .len = ((l) + 1) / 2 - 1}) 519 #define PTVAL_PGT_ADDR MKPTVAL(4, 8) 520 #define PTVAL_VMADDR MKPTVAL(8, 6) 521 522 union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, 523 gfn_t gfn, union asce asce, bool uses_skeys); 524 bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn, 525 union asce asce); 526 void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce); 527 528 long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce, 529 const struct dat_walk_ops *ops, int flags, void *priv); 530 531 int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags, 532 int walk_level, union crste **last, union pte **ptepp); 533 void dat_free_level(struct crst_table *table, bool owns_ptes); 534 struct crst_table *dat_alloc_crst_sleepable(unsigned long init); 535 int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype); 536 537 #if KVM_S390_MANAGES_S390_GUEST 538 int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey); 539 int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, 540 union skey skey, bool nq); 541 int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn, 542 union skey skey, union skey *oldkey, bool nq, bool mr, bool mc); 543 int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey); 544 long dat_reset_skeys(union asce asce, gfn_t start); 545 #endif /* KVM_S390_MANAGES_S390_GUEST */ 546 547 unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param); 548 void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val); 549 550 int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, 551 u16 type, u16 param); 552 553 #if KVM_S390_MANAGES_S390_GUEST 554 int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); 555 #else 556 static inline int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn) 557 { 558 return 0; 559 } 560 #endif /* KVM_S390_MANAGES_S390_GUEST */ 561 562 bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end); 563 564 #if KVM_S390_MANAGES_S390_GUEST 565 int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty); 566 long dat_reset_cmma(union asce asce, gfn_t start_gfn); 567 int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values); 568 int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem); 569 int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, 570 unsigned long count, unsigned long mask, const uint8_t *bits); 571 #endif /* KVM_S390_MANAGES_S390_GUEST */ 572 573 int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc); 574 575 #define GFP_KVM_S390_MMU_CACHE (GFP_ATOMIC | __GFP_ACCOUNT | __GFP_NOWARN) 576 577 static inline struct page_table *kvm_s390_mmu_cache_alloc_pt(struct kvm_s390_mmu_cache *mc) 578 { 579 if (mc->n_pts) 580 return mc->pts[--mc->n_pts]; 581 return (void *)__get_free_page(GFP_KVM_S390_MMU_CACHE); 582 } 583 584 static inline struct crst_table *kvm_s390_mmu_cache_alloc_crst(struct kvm_s390_mmu_cache *mc) 585 { 586 if (mc->n_crsts) 587 return mc->crsts[--mc->n_crsts]; 588 return (void *)__get_free_pages(GFP_KVM_S390_MMU_CACHE | __GFP_COMP, CRST_ALLOC_ORDER); 589 } 590 591 static inline struct vsie_rmap *kvm_s390_mmu_cache_alloc_rmap(struct kvm_s390_mmu_cache *mc) 592 { 593 if (mc->n_rmaps) 594 return mc->rmaps[--mc->n_rmaps]; 595 return kzalloc_obj(struct vsie_rmap, GFP_KVM_S390_MMU_CACHE); 596 } 597 598 static inline struct crst_table *crste_table_start(union crste *crstep) 599 { 600 return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE); 601 } 602 603 static inline struct page_table *pte_table_start(union pte *ptep) 604 { 605 return (struct page_table *)ALIGN_DOWN((unsigned long)ptep, _PAGE_TABLE_SIZE); 606 } 607 608 static inline bool crdte_crste(union crste *crstep, union crste old, union crste new, gfn_t gfn, 609 union asce asce) 610 { 611 unsigned long dtt = 0x10 | new.h.tt << 2; 612 void *table = crste_table_start(crstep); 613 614 return crdte(old.val, new.val, table, dtt, gfn_to_gpa(gfn), asce.val); 615 } 616 617 /** 618 * idte_crste() - invalidate a crste entry using idte 619 * @crstep: pointer to the crste to be invalidated 620 * @gfn: a gfn mapped by the crste 621 * @opt: options for the idte instruction 622 * @asce: the asce 623 * @local: whether the operation is cpu-local 624 */ 625 static __always_inline void idte_crste(union crste *crstep, gfn_t gfn, unsigned long opt, 626 union asce asce, int local) 627 { 628 unsigned long table_origin = __pa(crste_table_start(crstep)); 629 unsigned long gaddr = gfn_to_gpa(gfn) & HPAGE_MASK; 630 631 if (__builtin_constant_p(opt) && opt == 0) { 632 /* flush without guest asce */ 633 asm volatile("idte %[table_origin],0,%[gaddr],%[local]" 634 : "+m" (*crstep) 635 : [table_origin] "a" (table_origin), [gaddr] "a" (gaddr), 636 [local] "i" (local) 637 : "cc"); 638 } else { 639 /* flush with guest asce */ 640 asm volatile("idte %[table_origin],%[asce],%[gaddr_opt],%[local]" 641 : "+m" (*crstep) 642 : [table_origin] "a" (table_origin), [gaddr_opt] "a" (gaddr | opt), 643 [asce] "a" (asce.val), [local] "i" (local) 644 : "cc"); 645 } 646 } 647 648 static inline void dat_init_pgstes(struct page_table *pt, unsigned long val) 649 { 650 memset64((void *)pt->pgstes, val, PTRS_PER_PTE); 651 } 652 653 static inline void dat_init_page_table(struct page_table *pt, unsigned long ptes, 654 unsigned long pgstes) 655 { 656 memset64((void *)pt->ptes, ptes, PTRS_PER_PTE); 657 dat_init_pgstes(pt, pgstes); 658 } 659 660 static inline gfn_t asce_end(union asce asce) 661 { 662 return 1ULL << ((asce.dt + 1) * 11 + _SEGMENT_SHIFT - PAGE_SHIFT); 663 } 664 665 #define _CRSTE(x) ((union crste) { .val = _Generic((x), \ 666 union pgd : (x).val, \ 667 union p4d : (x).val, \ 668 union pud : (x).val, \ 669 union pmd : (x).val, \ 670 union crste : (x).val)}) 671 672 #define _CRSTEP(x) ((union crste *)_Generic((*(x)), \ 673 union pgd : (x), \ 674 union p4d : (x), \ 675 union pud : (x), \ 676 union pmd : (x), \ 677 union crste : (x))) 678 679 #define _CRSTP(x) ((struct crst_table *)_Generic((*(x)), \ 680 struct crst_table : (x), \ 681 struct segment_table : (x), \ 682 struct region3_table : (x), \ 683 struct region2_table : (x), \ 684 struct region1_table : (x))) 685 686 static inline bool asce_contains_gfn(union asce asce, gfn_t gfn) 687 { 688 return gfn < asce_end(asce); 689 } 690 691 static inline bool is_pmd(union crste crste) 692 { 693 return crste.h.tt == TABLE_TYPE_SEGMENT; 694 } 695 696 static inline bool is_pud(union crste crste) 697 { 698 return crste.h.tt == TABLE_TYPE_REGION3; 699 } 700 701 static inline bool is_p4d(union crste crste) 702 { 703 return crste.h.tt == TABLE_TYPE_REGION2; 704 } 705 706 static inline bool is_pgd(union crste crste) 707 { 708 return crste.h.tt == TABLE_TYPE_REGION1; 709 } 710 711 static inline phys_addr_t pmd_origin_large(union pmd pmd) 712 { 713 return pmd.val & _SEGMENT_ENTRY_ORIGIN_LARGE; 714 } 715 716 static inline phys_addr_t pud_origin_large(union pud pud) 717 { 718 return pud.val & _REGION3_ENTRY_ORIGIN_LARGE; 719 } 720 721 /** 722 * crste_origin_large() - Return the large frame origin of a large crste 723 * @crste: The crste whose origin is to be returned. Should be either a 724 * region-3 table entry or a segment table entry, in both cases with 725 * FC set to 1 (large pages). 726 * 727 * Return: The origin of the large frame pointed to by @crste, or -1 if the 728 * crste was not large (wrong table type, or FC==0) 729 */ 730 static inline phys_addr_t crste_origin_large(union crste crste) 731 { 732 if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3)) 733 return -1; 734 if (is_pmd(crste)) 735 return pmd_origin_large(crste.pmd); 736 return pud_origin_large(crste.pud); 737 } 738 739 #define crste_origin(x) (_Generic((x), \ 740 union pmd : (x).val & _SEGMENT_ENTRY_ORIGIN, \ 741 union pud : (x).val & _REGION_ENTRY_ORIGIN, \ 742 union p4d : (x).val & _REGION_ENTRY_ORIGIN, \ 743 union pgd : (x).val & _REGION_ENTRY_ORIGIN)) 744 745 static inline unsigned long pte_origin(union pte pte) 746 { 747 return pte.val & PAGE_MASK; 748 } 749 750 static inline bool pmd_prefix(union pmd pmd) 751 { 752 return pmd.h.fc && pmd.s.fc1.prefix_notif; 753 } 754 755 static inline bool pud_prefix(union pud pud) 756 { 757 return pud.h.fc && pud.s.fc1.prefix_notif; 758 } 759 760 static inline bool crste_leaf(union crste crste) 761 { 762 return (crste.h.tt <= TABLE_TYPE_REGION3) && crste.h.fc; 763 } 764 765 static inline bool crste_prefix(union crste crste) 766 { 767 return crste_leaf(crste) && crste.s.fc1.prefix_notif; 768 } 769 770 static inline bool crste_dirty(union crste crste) 771 { 772 return crste_leaf(crste) && crste.s.fc1.d; 773 } 774 775 static inline union pgste *pgste_of(union pte *pte) 776 { 777 return (union pgste *)(pte + _PAGE_ENTRIES); 778 } 779 780 static inline bool pte_hole(union pte pte) 781 { 782 return pte.h.i && !pte.tok.pr && pte.tok.type != _DAT_TOKEN_NONE; 783 } 784 785 static inline bool _crste_hole(union crste crste) 786 { 787 return crste.h.i && !crste.tok.pr && crste.tok.type != _DAT_TOKEN_NONE; 788 } 789 790 #define crste_hole(x) _crste_hole(_CRSTE(x)) 791 792 static inline bool _crste_none(union crste crste) 793 { 794 return crste.h.i && !crste.tok.pr && crste.tok.type == _DAT_TOKEN_NONE; 795 } 796 797 #define crste_none(x) _crste_none(_CRSTE(x)) 798 799 static inline phys_addr_t large_pud_to_phys(union pud pud, gfn_t gfn) 800 { 801 return pud_origin_large(pud) | (gfn_to_gpa(gfn) & ~_REGION3_MASK); 802 } 803 804 static inline phys_addr_t large_pmd_to_phys(union pmd pmd, gfn_t gfn) 805 { 806 return pmd_origin_large(pmd) | (gfn_to_gpa(gfn) & ~_SEGMENT_MASK); 807 } 808 809 static inline phys_addr_t large_crste_to_phys(union crste crste, gfn_t gfn) 810 { 811 if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3)) 812 return -1; 813 if (is_pmd(crste)) 814 return large_pmd_to_phys(crste.pmd, gfn); 815 return large_pud_to_phys(crste.pud, gfn); 816 } 817 818 static inline bool cspg_crste(union crste *crstep, union crste old, union crste new) 819 { 820 return cspg(&crstep->val, old.val, new.val); 821 } 822 823 static inline struct page_table *dereference_pmd(union pmd pmd) 824 { 825 return phys_to_virt(crste_origin(pmd)); 826 } 827 828 static inline struct segment_table *dereference_pud(union pud pud) 829 { 830 return phys_to_virt(crste_origin(pud)); 831 } 832 833 static inline struct region3_table *dereference_p4d(union p4d p4d) 834 { 835 return phys_to_virt(crste_origin(p4d)); 836 } 837 838 static inline struct region2_table *dereference_pgd(union pgd pgd) 839 { 840 return phys_to_virt(crste_origin(pgd)); 841 } 842 843 static inline struct crst_table *_dereference_crste(union crste crste) 844 { 845 if (unlikely(is_pmd(crste))) 846 return NULL; 847 return phys_to_virt(crste_origin(crste.pud)); 848 } 849 850 #define dereference_crste(x) (_Generic((x), \ 851 union pud : _dereference_crste(_CRSTE(x)), \ 852 union p4d : _dereference_crste(_CRSTE(x)), \ 853 union pgd : _dereference_crste(_CRSTE(x)), \ 854 union crste : _dereference_crste(_CRSTE(x)))) 855 856 static inline struct crst_table *dereference_asce(union asce asce) 857 { 858 return phys_to_virt(asce.val & _ASCE_ORIGIN); 859 } 860 861 static inline void asce_flush_tlb(union asce asce) 862 { 863 __tlb_flush_idte(asce.val); 864 } 865 866 static inline bool pgste_get_trylock(union pte *ptep, union pgste *res) 867 { 868 union pgste *pgstep = pgste_of(ptep); 869 union pgste old_pgste; 870 871 if (READ_ONCE(pgstep->val) & PGSTE_PCL_BIT) 872 return false; 873 old_pgste.val = __atomic64_or_barrier(PGSTE_PCL_BIT, &pgstep->val); 874 if (old_pgste.pcl) 875 return false; 876 old_pgste.pcl = 1; 877 *res = old_pgste; 878 return true; 879 } 880 881 static inline union pgste pgste_get_lock(union pte *ptep) 882 { 883 union pgste res; 884 885 while (!pgste_get_trylock(ptep, &res)) 886 cpu_relax(); 887 return res; 888 } 889 890 static inline void pgste_set_unlock(union pte *ptep, union pgste pgste) 891 { 892 pgste.pcl = 0; 893 barrier(); 894 WRITE_ONCE(*pgste_of(ptep), pgste); 895 } 896 897 static inline void dat_ptep_xchg(union pte *ptep, union pte new, gfn_t gfn, union asce asce, 898 bool has_skeys) 899 { 900 union pgste pgste; 901 902 pgste = pgste_get_lock(ptep); 903 pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, asce, has_skeys); 904 pgste_set_unlock(ptep, pgste); 905 } 906 907 static inline void dat_ptep_clear(union pte *ptep, gfn_t gfn, union asce asce, bool has_skeys) 908 { 909 dat_ptep_xchg(ptep, _PTE_EMPTY, gfn, asce, has_skeys); 910 } 911 912 static inline void dat_free_pt(struct page_table *pt) 913 { 914 free_page((unsigned long)pt); 915 } 916 917 static inline void _dat_free_crst(struct crst_table *table) 918 { 919 free_pages((unsigned long)table, CRST_ALLOC_ORDER); 920 } 921 922 #define dat_free_crst(x) _dat_free_crst(_CRSTP(x)) 923 924 static inline void kvm_s390_free_mmu_cache(struct kvm_s390_mmu_cache *mc) 925 { 926 if (!mc) 927 return; 928 while (mc->n_pts) 929 dat_free_pt(mc->pts[--mc->n_pts]); 930 while (mc->n_crsts) 931 _dat_free_crst(mc->crsts[--mc->n_crsts]); 932 while (mc->n_rmaps) 933 kfree(mc->rmaps[--mc->n_rmaps]); 934 kfree(mc); 935 } 936 937 DEFINE_FREE(kvm_s390_mmu_cache, struct kvm_s390_mmu_cache *, if (_T) kvm_s390_free_mmu_cache(_T)) 938 939 static inline struct kvm_s390_mmu_cache *kvm_s390_new_mmu_cache(void) 940 { 941 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; 942 943 mc = kzalloc_obj(*mc, GFP_KERNEL_ACCOUNT); 944 if (mc && !kvm_s390_mmu_cache_topup(mc)) 945 return_ptr(mc); 946 return NULL; 947 } 948 949 static inline bool dat_pmdp_xchg_atomic(union pmd *pmdp, union pmd old, union pmd new, 950 gfn_t gfn, union asce asce) 951 { 952 return dat_crstep_xchg_atomic(_CRSTEP(pmdp), _CRSTE(old), _CRSTE(new), gfn, asce); 953 } 954 955 static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pud new, 956 gfn_t gfn, union asce asce) 957 { 958 return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce); 959 } 960 961 static inline union crste dat_crstep_clear_atomic(union crste *crstep, gfn_t gfn, union asce asce) 962 { 963 union crste oldcrste, empty = _CRSTE_EMPTY(crstep->h.tt); 964 965 do { 966 oldcrste = READ_ONCE(*crstep); 967 } while (!dat_crstep_xchg_atomic(crstep, oldcrste, empty, gfn, asce)); 968 return oldcrste; 969 } 970 971 static inline int get_level(union crste *crstep, union pte *ptep) 972 { 973 return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt; 974 } 975 976 static inline int dat_delete_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, 977 unsigned long npages) 978 { 979 return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_PIC, PGM_ADDRESSING); 980 } 981 982 static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, 983 unsigned long npages) 984 { 985 return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_NONE, 0); 986 } 987 988 static inline bool crste_is_ucas(union crste crste) 989 { 990 return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0; 991 } 992 993 #endif /* ARCH_KVM_GMAP_DAT_H */ 994