1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * KVM guest address space mapping code 4 * 5 * Copyright IBM Corp. 2024, 2025 6 * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com> 7 */ 8 9 #ifndef __KVM_S390_DAT_H 10 #define __KVM_S390_DAT_H 11 12 #include <linux/radix-tree.h> 13 #include <linux/refcount.h> 14 #include <linux/io.h> 15 #include <linux/kvm_types.h> 16 #include <linux/pgalloc.h> 17 #include <asm/tlbflush.h> 18 #include <asm/dat-bits.h> 19 20 /* 21 * Base address and length must be sent at the start of each block, therefore 22 * it's cheaper to send some clean data, as long as it's less than the size of 23 * two longs. 24 */ 25 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *)) 26 /* For consistency */ 27 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX) 28 29 #define _ASCE(x) ((union asce) { .val = (x), }) 30 #define NULL_ASCE _ASCE(0) 31 32 enum { 33 _DAT_TOKEN_NONE = 0, 34 _DAT_TOKEN_PIC, 35 }; 36 37 #define _CRSTE_TOK(l, t, p) ((union crste) { \ 38 .tok.i = 1, \ 39 .tok.tt = (l), \ 40 .tok.type = (t), \ 41 .tok.par = (p) \ 42 }) 43 #define _CRSTE_PIC(l, p) _CRSTE_TOK(l, _DAT_TOKEN_PIC, p) 44 45 #define _CRSTE_HOLE(l) _CRSTE_PIC(l, PGM_ADDRESSING) 46 #define _CRSTE_EMPTY(l) _CRSTE_TOK(l, _DAT_TOKEN_NONE, 0) 47 48 #define _PMD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_SEGMENT) 49 50 #define _PTE_TOK(t, p) ((union pte) { .tok.i = 1, .tok.type = (t), .tok.par = (p) }) 51 #define _PTE_EMPTY _PTE_TOK(_DAT_TOKEN_NONE, 0) 52 53 /* This fake table type is used for page table walks (both for normal page tables and vSIE) */ 54 #define TABLE_TYPE_PAGE_TABLE -1 55 56 enum dat_walk_flags { 57 DAT_WALK_USES_SKEYS = 0x40, 58 DAT_WALK_CONTINUE = 0x20, 59 DAT_WALK_IGN_HOLES = 0x10, 60 DAT_WALK_SPLIT = 0x08, 61 DAT_WALK_ALLOC = 0x04, 62 DAT_WALK_ANY = 0x02, 63 DAT_WALK_LEAF = 0x01, 64 DAT_WALK_DEFAULT = 0 65 }; 66 67 #define DAT_WALK_SPLIT_ALLOC (DAT_WALK_SPLIT | DAT_WALK_ALLOC) 68 #define DAT_WALK_ALLOC_CONTINUE (DAT_WALK_CONTINUE | DAT_WALK_ALLOC) 69 #define DAT_WALK_LEAF_ALLOC (DAT_WALK_LEAF | DAT_WALK_ALLOC) 70 71 union pte { 72 unsigned long val; 73 union page_table_entry h; 74 struct { 75 unsigned long :56; /* Hardware bits */ 76 unsigned long u : 1; /* Page unused */ 77 unsigned long s : 1; /* Special */ 78 unsigned long w : 1; /* Writable */ 79 unsigned long r : 1; /* Readable */ 80 unsigned long d : 1; /* Dirty */ 81 unsigned long y : 1; /* Young */ 82 unsigned long sd: 1; /* Soft dirty */ 83 unsigned long pr: 1; /* Present */ 84 } s; 85 struct { 86 unsigned char hwbytes[7]; 87 unsigned char swbyte; 88 }; 89 union { 90 struct { 91 unsigned long type :16; /* Token type */ 92 unsigned long par :16; /* Token parameter */ 93 unsigned long :20; 94 unsigned long : 1; /* Must be 0 */ 95 unsigned long i : 1; /* Must be 1 */ 96 unsigned long : 2; 97 unsigned long : 7; 98 unsigned long pr : 1; /* Must be 0 */ 99 }; 100 struct { 101 unsigned long token:32; /* Token and parameter */ 102 unsigned long :32; 103 }; 104 } tok; 105 }; 106 107 /* Soft dirty, needed as macro for atomic operations on ptes */ 108 #define _PAGE_SD 0x002 109 110 /* Needed as macro to perform atomic operations */ 111 #define PGSTE_PCL_BIT 0x0080000000000000UL /* PCL lock, HW bit */ 112 #define PGSTE_CMMA_D_BIT 0x0000000000008000UL /* CMMA dirty soft-bit */ 113 114 enum pgste_gps_usage { 115 PGSTE_GPS_USAGE_STABLE = 0, 116 PGSTE_GPS_USAGE_UNUSED, 117 PGSTE_GPS_USAGE_POT_VOLATILE, 118 PGSTE_GPS_USAGE_VOLATILE, 119 }; 120 121 union pgste { 122 unsigned long val; 123 struct { 124 unsigned long acc : 4; 125 unsigned long fp : 1; 126 unsigned long : 3; 127 unsigned long pcl : 1; 128 unsigned long hr : 1; 129 unsigned long hc : 1; 130 unsigned long : 2; 131 unsigned long gr : 1; 132 unsigned long gc : 1; 133 unsigned long : 1; 134 unsigned long :16; /* val16 */ 135 unsigned long zero : 1; 136 unsigned long nodat : 1; 137 unsigned long : 4; 138 unsigned long usage : 2; 139 unsigned long : 8; 140 unsigned long cmma_d : 1; /* Dirty flag for CMMA bits */ 141 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ 142 unsigned long vsie_notif : 1; /* Referenced in a shadow table */ 143 unsigned long : 5; 144 unsigned long : 8; 145 }; 146 struct { 147 unsigned short hwbytes0; 148 unsigned short val16; /* Used to store chunked values, see dat_{s,g}et_ptval() */ 149 unsigned short hwbytes4; 150 unsigned char flags; /* Maps to the software bits */ 151 unsigned char hwbyte7; 152 } __packed; 153 }; 154 155 union pmd { 156 unsigned long val; 157 union segment_table_entry h; 158 struct { 159 struct { 160 unsigned long :44; /* HW */ 161 unsigned long : 3; /* Unused */ 162 unsigned long : 1; /* HW */ 163 unsigned long w : 1; /* Writable soft-bit */ 164 unsigned long r : 1; /* Readable soft-bit */ 165 unsigned long d : 1; /* Dirty */ 166 unsigned long y : 1; /* Young */ 167 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ 168 unsigned long : 3; /* HW */ 169 unsigned long vsie_notif : 1; /* Referenced in a shadow table */ 170 unsigned long : 1; /* Unused */ 171 unsigned long : 4; /* HW */ 172 unsigned long sd : 1; /* Soft-Dirty */ 173 unsigned long pr : 1; /* Present */ 174 } fc1; 175 } s; 176 }; 177 178 union pud { 179 unsigned long val; 180 union region3_table_entry h; 181 struct { 182 struct { 183 unsigned long :33; /* HW */ 184 unsigned long :14; /* Unused */ 185 unsigned long : 1; /* HW */ 186 unsigned long w : 1; /* Writable soft-bit */ 187 unsigned long r : 1; /* Readable soft-bit */ 188 unsigned long d : 1; /* Dirty */ 189 unsigned long y : 1; /* Young */ 190 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ 191 unsigned long : 3; /* HW */ 192 unsigned long vsie_notif : 1; /* Referenced in a shadow table */ 193 unsigned long : 1; /* Unused */ 194 unsigned long : 4; /* HW */ 195 unsigned long sd : 1; /* Soft-Dirty */ 196 unsigned long pr : 1; /* Present */ 197 } fc1; 198 } s; 199 }; 200 201 union p4d { 202 unsigned long val; 203 union region2_table_entry h; 204 }; 205 206 union pgd { 207 unsigned long val; 208 union region1_table_entry h; 209 }; 210 211 union crste { 212 unsigned long val; 213 union { 214 struct { 215 unsigned long :52; 216 unsigned long : 1; 217 unsigned long fc: 1; 218 unsigned long p : 1; 219 unsigned long : 1; 220 unsigned long : 2; 221 unsigned long i : 1; 222 unsigned long : 1; 223 unsigned long tt: 2; 224 unsigned long : 2; 225 }; 226 struct { 227 unsigned long to:52; 228 unsigned long : 1; 229 unsigned long fc: 1; 230 unsigned long p : 1; 231 unsigned long : 1; 232 unsigned long tf: 2; 233 unsigned long i : 1; 234 unsigned long : 1; 235 unsigned long tt: 2; 236 unsigned long tl: 2; 237 } fc0; 238 struct { 239 unsigned long :47; 240 unsigned long av : 1; /* ACCF-Validity Control */ 241 unsigned long acc: 4; /* Access-Control Bits */ 242 unsigned long f : 1; /* Fetch-Protection Bit */ 243 unsigned long fc : 1; /* Format-Control */ 244 unsigned long p : 1; /* DAT-Protection Bit */ 245 unsigned long iep: 1; /* Instruction-Execution-Protection */ 246 unsigned long : 2; 247 unsigned long i : 1; /* Segment-Invalid Bit */ 248 unsigned long cs : 1; /* Common-Segment Bit */ 249 unsigned long tt : 2; /* Table-Type Bits */ 250 unsigned long : 2; 251 } fc1; 252 } h; 253 struct { 254 struct { 255 unsigned long :47; 256 unsigned long : 1; /* HW (should be 0) */ 257 unsigned long w : 1; /* Writable */ 258 unsigned long r : 1; /* Readable */ 259 unsigned long d : 1; /* Dirty */ 260 unsigned long y : 1; /* Young */ 261 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */ 262 unsigned long : 3; /* HW */ 263 unsigned long vsie_notif : 1; /* Referenced in a shadow table */ 264 unsigned long : 1; 265 unsigned long : 4; /* HW */ 266 unsigned long sd : 1; /* Soft-Dirty */ 267 unsigned long pr : 1; /* Present */ 268 } fc1; 269 } s; 270 union { 271 struct { 272 unsigned long type :16; /* Token type */ 273 unsigned long par :16; /* Token parameter */ 274 unsigned long :26; 275 unsigned long i : 1; /* Must be 1 */ 276 unsigned long : 1; 277 unsigned long tt : 2; 278 unsigned long : 1; 279 unsigned long pr : 1; /* Must be 0 */ 280 }; 281 struct { 282 unsigned long token:32; /* Token and parameter */ 283 unsigned long :32; 284 }; 285 } tok; 286 union pmd pmd; 287 union pud pud; 288 union p4d p4d; 289 union pgd pgd; 290 }; 291 292 union skey { 293 unsigned char skey; 294 struct { 295 unsigned char acc :4; 296 unsigned char fp :1; 297 unsigned char r :1; 298 unsigned char c :1; 299 unsigned char zero:1; 300 }; 301 }; 302 303 static_assert(sizeof(union pgste) == sizeof(unsigned long)); 304 static_assert(sizeof(union pte) == sizeof(unsigned long)); 305 static_assert(sizeof(union pmd) == sizeof(unsigned long)); 306 static_assert(sizeof(union pud) == sizeof(unsigned long)); 307 static_assert(sizeof(union p4d) == sizeof(unsigned long)); 308 static_assert(sizeof(union pgd) == sizeof(unsigned long)); 309 static_assert(sizeof(union crste) == sizeof(unsigned long)); 310 static_assert(sizeof(union skey) == sizeof(char)); 311 312 struct segment_table { 313 union pmd pmds[_CRST_ENTRIES]; 314 }; 315 316 struct region3_table { 317 union pud puds[_CRST_ENTRIES]; 318 }; 319 320 struct region2_table { 321 union p4d p4ds[_CRST_ENTRIES]; 322 }; 323 324 struct region1_table { 325 union pgd pgds[_CRST_ENTRIES]; 326 }; 327 328 struct crst_table { 329 union { 330 union crste crstes[_CRST_ENTRIES]; 331 struct segment_table segment; 332 struct region3_table region3; 333 struct region2_table region2; 334 struct region1_table region1; 335 }; 336 }; 337 338 struct page_table { 339 union pte ptes[_PAGE_ENTRIES]; 340 union pgste pgstes[_PAGE_ENTRIES]; 341 }; 342 343 static_assert(sizeof(struct crst_table) == _CRST_TABLE_SIZE); 344 static_assert(sizeof(struct page_table) == PAGE_SIZE); 345 346 struct dat_walk; 347 348 typedef long (*dat_walk_op)(union crste *crste, gfn_t gfn, gfn_t next, struct dat_walk *w); 349 350 struct dat_walk_ops { 351 union { 352 dat_walk_op crste_ops[4]; 353 struct { 354 dat_walk_op pmd_entry; 355 dat_walk_op pud_entry; 356 dat_walk_op p4d_entry; 357 dat_walk_op pgd_entry; 358 }; 359 }; 360 long (*pte_entry)(union pte *pte, gfn_t gfn, gfn_t next, struct dat_walk *w); 361 }; 362 363 struct dat_walk { 364 const struct dat_walk_ops *ops; 365 union crste *last; 366 union pte *last_pte; 367 union asce asce; 368 gfn_t start; 369 gfn_t end; 370 int flags; 371 void *priv; 372 }; 373 374 struct ptval_param { 375 unsigned char offset : 6; 376 unsigned char len : 2; 377 }; 378 379 /** 380 * _pte() - Useful constructor for union pte 381 * @pfn: the pfn this pte should point to. 382 * @writable: whether the pte should be writable. 383 * @dirty: whether the pte should be dirty. 384 * @special: whether the pte should be marked as special 385 * 386 * The pte is also marked as young and present. If the pte is marked as dirty, 387 * it gets marked as soft-dirty too. If the pte is not dirty, the hardware 388 * protect bit is set (independently of the write softbit); this way proper 389 * dirty tracking can be performed. 390 * 391 * Return: a union pte value. 392 */ 393 static inline union pte _pte(kvm_pfn_t pfn, bool writable, bool dirty, bool special) 394 { 395 union pte res = { .val = PFN_PHYS(pfn) }; 396 397 res.h.p = !dirty; 398 res.s.y = 1; 399 res.s.pr = 1; 400 res.s.w = writable; 401 res.s.d = dirty; 402 res.s.sd = dirty; 403 res.s.s = special; 404 return res; 405 } 406 407 static inline union crste _crste_fc0(kvm_pfn_t pfn, int tt) 408 { 409 union crste res = { .val = PFN_PHYS(pfn) }; 410 411 res.h.tt = tt; 412 res.h.fc0.tl = _REGION_ENTRY_LENGTH; 413 res.h.fc0.tf = 0; 414 return res; 415 } 416 417 /** 418 * _crste() - Useful constructor for union crste with FC=1 419 * @pfn: the pfn this pte should point to. 420 * @tt: the table type 421 * @writable: whether the pte should be writable. 422 * @dirty: whether the pte should be dirty. 423 * 424 * The crste is also marked as young and present. If the crste is marked as 425 * dirty, it gets marked as soft-dirty too. If the crste is not dirty, the 426 * hardware protect bit is set (independently of the write softbit); this way 427 * proper dirty tracking can be performed. 428 * 429 * Return: a union crste value. 430 */ 431 static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool writable, bool dirty) 432 { 433 union crste res = { .val = PFN_PHYS(pfn) & _SEGMENT_MASK }; 434 435 res.h.tt = tt; 436 res.h.p = !dirty; 437 res.h.fc = 1; 438 res.s.fc1.y = 1; 439 res.s.fc1.pr = 1; 440 res.s.fc1.w = writable; 441 res.s.fc1.d = dirty; 442 res.s.fc1.sd = dirty; 443 return res; 444 } 445 446 union essa_state { 447 unsigned char val; 448 struct { 449 unsigned char : 2; 450 unsigned char nodat : 1; 451 unsigned char exception : 1; 452 unsigned char usage : 2; 453 unsigned char content : 2; 454 }; 455 }; 456 457 /** 458 * struct vsie_rmap - reverse mapping for shadow page table entries 459 * @next: pointer to next rmap in the list 460 * @r_gfn: virtual rmap address in the shadow guest address space 461 */ 462 struct vsie_rmap { 463 struct vsie_rmap *next; 464 union { 465 unsigned long val; 466 struct { 467 long level: 8; 468 unsigned long : 4; 469 unsigned long r_gfn:52; 470 }; 471 }; 472 }; 473 474 static_assert(sizeof(struct vsie_rmap) == 2 * sizeof(long)); 475 476 #define KVM_S390_MMU_CACHE_N_CRSTS 6 477 #define KVM_S390_MMU_CACHE_N_PTS 2 478 #define KVM_S390_MMU_CACHE_N_RMAPS 16 479 struct kvm_s390_mmu_cache { 480 void *crsts[KVM_S390_MMU_CACHE_N_CRSTS]; 481 void *pts[KVM_S390_MMU_CACHE_N_PTS]; 482 void *rmaps[KVM_S390_MMU_CACHE_N_RMAPS]; 483 short int n_crsts; 484 short int n_pts; 485 short int n_rmaps; 486 }; 487 488 struct guest_fault { 489 gfn_t gfn; /* Guest frame */ 490 kvm_pfn_t pfn; /* Host PFN */ 491 struct page *page; /* Host page */ 492 union pte *ptep; /* Used to resolve the fault, or NULL */ 493 union crste *crstep; /* Used to resolve the fault, or NULL */ 494 bool writable; /* Mapping is writable */ 495 bool write_attempt; /* Write access attempted */ 496 bool attempt_pfault; /* Attempt a pfault first */ 497 bool valid; /* This entry contains valid data */ 498 void (*callback)(struct guest_fault *f); 499 void *priv; 500 }; 501 502 /* 503 * 0 1 2 3 4 5 6 7 504 * +-------+-------+-------+-------+-------+-------+-------+-------+ 505 * 0 | | PGT_ADDR | 506 * 8 | VMADDR | | 507 * 16 | | 508 * 24 | | 509 */ 510 #define MKPTVAL(o, l) ((struct ptval_param) { .offset = (o), .len = ((l) + 1) / 2 - 1}) 511 #define PTVAL_PGT_ADDR MKPTVAL(4, 8) 512 #define PTVAL_VMADDR MKPTVAL(8, 6) 513 514 union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new, 515 gfn_t gfn, union asce asce, bool uses_skeys); 516 bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn, 517 union asce asce); 518 void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce); 519 520 long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce, 521 const struct dat_walk_ops *ops, int flags, void *priv); 522 523 int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags, 524 int walk_level, union crste **last, union pte **ptepp); 525 void dat_free_level(struct crst_table *table, bool owns_ptes); 526 struct crst_table *dat_alloc_crst_sleepable(unsigned long init); 527 int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype); 528 int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey); 529 int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, 530 union skey skey, bool nq); 531 int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn, 532 union skey skey, union skey *oldkey, bool nq, bool mr, bool mc); 533 int dat_reset_reference_bit(union asce asce, gfn_t gfn); 534 long dat_reset_skeys(union asce asce, gfn_t start); 535 536 unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param); 537 void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val); 538 539 int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end, 540 u16 type, u16 param); 541 int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn); 542 bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end); 543 int dat_link(struct kvm_s390_mmu_cache *mc, union asce asce, int level, 544 bool uses_skeys, struct guest_fault *f); 545 546 int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty); 547 long dat_reset_cmma(union asce asce, gfn_t start_gfn); 548 int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values); 549 int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem); 550 int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn, 551 unsigned long count, unsigned long mask, const uint8_t *bits); 552 553 int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc); 554 555 #define GFP_KVM_S390_MMU_CACHE (GFP_ATOMIC | __GFP_ACCOUNT | __GFP_NOWARN) 556 557 static inline struct page_table *kvm_s390_mmu_cache_alloc_pt(struct kvm_s390_mmu_cache *mc) 558 { 559 if (mc->n_pts) 560 return mc->pts[--mc->n_pts]; 561 return (void *)__get_free_page(GFP_KVM_S390_MMU_CACHE); 562 } 563 564 static inline struct crst_table *kvm_s390_mmu_cache_alloc_crst(struct kvm_s390_mmu_cache *mc) 565 { 566 if (mc->n_crsts) 567 return mc->crsts[--mc->n_crsts]; 568 return (void *)__get_free_pages(GFP_KVM_S390_MMU_CACHE | __GFP_COMP, CRST_ALLOC_ORDER); 569 } 570 571 static inline struct vsie_rmap *kvm_s390_mmu_cache_alloc_rmap(struct kvm_s390_mmu_cache *mc) 572 { 573 if (mc->n_rmaps) 574 return mc->rmaps[--mc->n_rmaps]; 575 return kzalloc(sizeof(struct vsie_rmap), GFP_KVM_S390_MMU_CACHE); 576 } 577 578 static inline struct crst_table *crste_table_start(union crste *crstep) 579 { 580 return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE); 581 } 582 583 static inline struct page_table *pte_table_start(union pte *ptep) 584 { 585 return (struct page_table *)ALIGN_DOWN((unsigned long)ptep, _PAGE_TABLE_SIZE); 586 } 587 588 static inline bool crdte_crste(union crste *crstep, union crste old, union crste new, gfn_t gfn, 589 union asce asce) 590 { 591 unsigned long dtt = 0x10 | new.h.tt << 2; 592 void *table = crste_table_start(crstep); 593 594 return crdte(old.val, new.val, table, dtt, gfn_to_gpa(gfn), asce.val); 595 } 596 597 /** 598 * idte_crste() - invalidate a crste entry using idte 599 * @crstep: pointer to the crste to be invalidated 600 * @gfn: a gfn mapped by the crste 601 * @opt: options for the idte instruction 602 * @asce: the asce 603 * @local: whether the operation is cpu-local 604 */ 605 static __always_inline void idte_crste(union crste *crstep, gfn_t gfn, unsigned long opt, 606 union asce asce, int local) 607 { 608 unsigned long table_origin = __pa(crste_table_start(crstep)); 609 unsigned long gaddr = gfn_to_gpa(gfn) & HPAGE_MASK; 610 611 if (__builtin_constant_p(opt) && opt == 0) { 612 /* flush without guest asce */ 613 asm volatile("idte %[table_origin],0,%[gaddr],%[local]" 614 : "+m" (*crstep) 615 : [table_origin] "a" (table_origin), [gaddr] "a" (gaddr), 616 [local] "i" (local) 617 : "cc"); 618 } else { 619 /* flush with guest asce */ 620 asm volatile("idte %[table_origin],%[asce],%[gaddr_opt],%[local]" 621 : "+m" (*crstep) 622 : [table_origin] "a" (table_origin), [gaddr_opt] "a" (gaddr | opt), 623 [asce] "a" (asce.val), [local] "i" (local) 624 : "cc"); 625 } 626 } 627 628 static inline void dat_init_pgstes(struct page_table *pt, unsigned long val) 629 { 630 memset64((void *)pt->pgstes, val, PTRS_PER_PTE); 631 } 632 633 static inline void dat_init_page_table(struct page_table *pt, unsigned long ptes, 634 unsigned long pgstes) 635 { 636 memset64((void *)pt->ptes, ptes, PTRS_PER_PTE); 637 dat_init_pgstes(pt, pgstes); 638 } 639 640 static inline gfn_t asce_end(union asce asce) 641 { 642 return 1ULL << ((asce.dt + 1) * 11 + _SEGMENT_SHIFT - PAGE_SHIFT); 643 } 644 645 #define _CRSTE(x) ((union crste) { .val = _Generic((x), \ 646 union pgd : (x).val, \ 647 union p4d : (x).val, \ 648 union pud : (x).val, \ 649 union pmd : (x).val, \ 650 union crste : (x).val)}) 651 652 #define _CRSTEP(x) ((union crste *)_Generic((*(x)), \ 653 union pgd : (x), \ 654 union p4d : (x), \ 655 union pud : (x), \ 656 union pmd : (x), \ 657 union crste : (x))) 658 659 #define _CRSTP(x) ((struct crst_table *)_Generic((*(x)), \ 660 struct crst_table : (x), \ 661 struct segment_table : (x), \ 662 struct region3_table : (x), \ 663 struct region2_table : (x), \ 664 struct region1_table : (x))) 665 666 static inline bool asce_contains_gfn(union asce asce, gfn_t gfn) 667 { 668 return gfn < asce_end(asce); 669 } 670 671 static inline bool is_pmd(union crste crste) 672 { 673 return crste.h.tt == TABLE_TYPE_SEGMENT; 674 } 675 676 static inline bool is_pud(union crste crste) 677 { 678 return crste.h.tt == TABLE_TYPE_REGION3; 679 } 680 681 static inline bool is_p4d(union crste crste) 682 { 683 return crste.h.tt == TABLE_TYPE_REGION2; 684 } 685 686 static inline bool is_pgd(union crste crste) 687 { 688 return crste.h.tt == TABLE_TYPE_REGION1; 689 } 690 691 static inline phys_addr_t pmd_origin_large(union pmd pmd) 692 { 693 return pmd.val & _SEGMENT_ENTRY_ORIGIN_LARGE; 694 } 695 696 static inline phys_addr_t pud_origin_large(union pud pud) 697 { 698 return pud.val & _REGION3_ENTRY_ORIGIN_LARGE; 699 } 700 701 /** 702 * crste_origin_large() - Return the large frame origin of a large crste 703 * @crste: The crste whose origin is to be returned. Should be either a 704 * region-3 table entry or a segment table entry, in both cases with 705 * FC set to 1 (large pages). 706 * 707 * Return: The origin of the large frame pointed to by @crste, or -1 if the 708 * crste was not large (wrong table type, or FC==0) 709 */ 710 static inline phys_addr_t crste_origin_large(union crste crste) 711 { 712 if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3)) 713 return -1; 714 if (is_pmd(crste)) 715 return pmd_origin_large(crste.pmd); 716 return pud_origin_large(crste.pud); 717 } 718 719 #define crste_origin(x) (_Generic((x), \ 720 union pmd : (x).val & _SEGMENT_ENTRY_ORIGIN, \ 721 union pud : (x).val & _REGION_ENTRY_ORIGIN, \ 722 union p4d : (x).val & _REGION_ENTRY_ORIGIN, \ 723 union pgd : (x).val & _REGION_ENTRY_ORIGIN)) 724 725 static inline unsigned long pte_origin(union pte pte) 726 { 727 return pte.val & PAGE_MASK; 728 } 729 730 static inline bool pmd_prefix(union pmd pmd) 731 { 732 return pmd.h.fc && pmd.s.fc1.prefix_notif; 733 } 734 735 static inline bool pud_prefix(union pud pud) 736 { 737 return pud.h.fc && pud.s.fc1.prefix_notif; 738 } 739 740 static inline bool crste_leaf(union crste crste) 741 { 742 return (crste.h.tt <= TABLE_TYPE_REGION3) && crste.h.fc; 743 } 744 745 static inline bool crste_prefix(union crste crste) 746 { 747 return crste_leaf(crste) && crste.s.fc1.prefix_notif; 748 } 749 750 static inline bool crste_dirty(union crste crste) 751 { 752 return crste_leaf(crste) && crste.s.fc1.d; 753 } 754 755 static inline union pgste *pgste_of(union pte *pte) 756 { 757 return (union pgste *)(pte + _PAGE_ENTRIES); 758 } 759 760 static inline bool pte_hole(union pte pte) 761 { 762 return pte.h.i && !pte.tok.pr && pte.tok.type != _DAT_TOKEN_NONE; 763 } 764 765 static inline bool _crste_hole(union crste crste) 766 { 767 return crste.h.i && !crste.tok.pr && crste.tok.type != _DAT_TOKEN_NONE; 768 } 769 770 #define crste_hole(x) _crste_hole(_CRSTE(x)) 771 772 static inline bool _crste_none(union crste crste) 773 { 774 return crste.h.i && !crste.tok.pr && crste.tok.type == _DAT_TOKEN_NONE; 775 } 776 777 #define crste_none(x) _crste_none(_CRSTE(x)) 778 779 static inline phys_addr_t large_pud_to_phys(union pud pud, gfn_t gfn) 780 { 781 return pud_origin_large(pud) | (gfn_to_gpa(gfn) & ~_REGION3_MASK); 782 } 783 784 static inline phys_addr_t large_pmd_to_phys(union pmd pmd, gfn_t gfn) 785 { 786 return pmd_origin_large(pmd) | (gfn_to_gpa(gfn) & ~_SEGMENT_MASK); 787 } 788 789 static inline phys_addr_t large_crste_to_phys(union crste crste, gfn_t gfn) 790 { 791 if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3)) 792 return -1; 793 if (is_pmd(crste)) 794 return large_pmd_to_phys(crste.pmd, gfn); 795 return large_pud_to_phys(crste.pud, gfn); 796 } 797 798 static inline bool cspg_crste(union crste *crstep, union crste old, union crste new) 799 { 800 return cspg(&crstep->val, old.val, new.val); 801 } 802 803 static inline struct page_table *dereference_pmd(union pmd pmd) 804 { 805 return phys_to_virt(crste_origin(pmd)); 806 } 807 808 static inline struct segment_table *dereference_pud(union pud pud) 809 { 810 return phys_to_virt(crste_origin(pud)); 811 } 812 813 static inline struct region3_table *dereference_p4d(union p4d p4d) 814 { 815 return phys_to_virt(crste_origin(p4d)); 816 } 817 818 static inline struct region2_table *dereference_pgd(union pgd pgd) 819 { 820 return phys_to_virt(crste_origin(pgd)); 821 } 822 823 static inline struct crst_table *_dereference_crste(union crste crste) 824 { 825 if (unlikely(is_pmd(crste))) 826 return NULL; 827 return phys_to_virt(crste_origin(crste.pud)); 828 } 829 830 #define dereference_crste(x) (_Generic((x), \ 831 union pud : _dereference_crste(_CRSTE(x)), \ 832 union p4d : _dereference_crste(_CRSTE(x)), \ 833 union pgd : _dereference_crste(_CRSTE(x)), \ 834 union crste : _dereference_crste(_CRSTE(x)))) 835 836 static inline struct crst_table *dereference_asce(union asce asce) 837 { 838 return phys_to_virt(asce.val & _ASCE_ORIGIN); 839 } 840 841 static inline void asce_flush_tlb(union asce asce) 842 { 843 __tlb_flush_idte(asce.val); 844 } 845 846 static inline bool pgste_get_trylock(union pte *ptep, union pgste *res) 847 { 848 union pgste *pgstep = pgste_of(ptep); 849 union pgste old_pgste; 850 851 if (READ_ONCE(pgstep->val) & PGSTE_PCL_BIT) 852 return false; 853 old_pgste.val = __atomic64_or_barrier(PGSTE_PCL_BIT, &pgstep->val); 854 if (old_pgste.pcl) 855 return false; 856 old_pgste.pcl = 1; 857 *res = old_pgste; 858 return true; 859 } 860 861 static inline union pgste pgste_get_lock(union pte *ptep) 862 { 863 union pgste res; 864 865 while (!pgste_get_trylock(ptep, &res)) 866 cpu_relax(); 867 return res; 868 } 869 870 static inline void pgste_set_unlock(union pte *ptep, union pgste pgste) 871 { 872 pgste.pcl = 0; 873 barrier(); 874 WRITE_ONCE(*pgste_of(ptep), pgste); 875 } 876 877 static inline void dat_ptep_xchg(union pte *ptep, union pte new, gfn_t gfn, union asce asce, 878 bool has_skeys) 879 { 880 union pgste pgste; 881 882 pgste = pgste_get_lock(ptep); 883 pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, asce, has_skeys); 884 pgste_set_unlock(ptep, pgste); 885 } 886 887 static inline void dat_ptep_clear(union pte *ptep, gfn_t gfn, union asce asce, bool has_skeys) 888 { 889 dat_ptep_xchg(ptep, _PTE_EMPTY, gfn, asce, has_skeys); 890 } 891 892 static inline void dat_free_pt(struct page_table *pt) 893 { 894 free_page((unsigned long)pt); 895 } 896 897 static inline void _dat_free_crst(struct crst_table *table) 898 { 899 free_pages((unsigned long)table, CRST_ALLOC_ORDER); 900 } 901 902 #define dat_free_crst(x) _dat_free_crst(_CRSTP(x)) 903 904 static inline void kvm_s390_free_mmu_cache(struct kvm_s390_mmu_cache *mc) 905 { 906 if (!mc) 907 return; 908 while (mc->n_pts) 909 dat_free_pt(mc->pts[--mc->n_pts]); 910 while (mc->n_crsts) 911 _dat_free_crst(mc->crsts[--mc->n_crsts]); 912 while (mc->n_rmaps) 913 kfree(mc->rmaps[--mc->n_rmaps]); 914 kfree(mc); 915 } 916 917 DEFINE_FREE(kvm_s390_mmu_cache, struct kvm_s390_mmu_cache *, if (_T) kvm_s390_free_mmu_cache(_T)) 918 919 static inline struct kvm_s390_mmu_cache *kvm_s390_new_mmu_cache(void) 920 { 921 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL; 922 923 mc = kzalloc(sizeof(*mc), GFP_KERNEL_ACCOUNT); 924 if (mc && !kvm_s390_mmu_cache_topup(mc)) 925 return_ptr(mc); 926 return NULL; 927 } 928 929 static inline bool dat_pmdp_xchg_atomic(union pmd *pmdp, union pmd old, union pmd new, 930 gfn_t gfn, union asce asce) 931 { 932 return dat_crstep_xchg_atomic(_CRSTEP(pmdp), _CRSTE(old), _CRSTE(new), gfn, asce); 933 } 934 935 static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pud new, 936 gfn_t gfn, union asce asce) 937 { 938 return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce); 939 } 940 941 static inline void dat_crstep_clear(union crste *crstep, gfn_t gfn, union asce asce) 942 { 943 union crste newcrste = _CRSTE_EMPTY(crstep->h.tt); 944 945 dat_crstep_xchg(crstep, newcrste, gfn, asce); 946 } 947 948 static inline int get_level(union crste *crstep, union pte *ptep) 949 { 950 return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt; 951 } 952 953 static inline int dat_delete_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, 954 unsigned long npages) 955 { 956 return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_PIC, PGM_ADDRESSING); 957 } 958 959 static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, 960 unsigned long npages) 961 { 962 return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_NONE, 0); 963 } 964 965 static inline bool crste_is_ucas(union crste crste) 966 { 967 return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0; 968 } 969 970 #endif /* __KVM_S390_DAT_H */ 971