1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * KVM guest address space mapping code
4 *
5 * Copyright IBM Corp. 2024, 2025
6 * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com>
7 */
8
9 #ifndef ARCH_KVM_GMAP_DAT_H
10 #define ARCH_KVM_GMAP_DAT_H
11
12 #include <linux/kvm_host.h>
13 #include <linux/radix-tree.h>
14 #include <linux/refcount.h>
15 #include <linux/io.h>
16 #include <linux/kvm_types.h>
17 #include <linux/pgalloc.h>
18 #include <asm/tlbflush.h>
19 #include <asm/dat-bits.h>
20
21 /*
22 * Base address and length must be sent at the start of each block, therefore
23 * it's cheaper to send some clean data, as long as it's less than the size of
24 * two longs.
25 */
26 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
27 /* For consistency */
28 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
29
30 #define _ASCE(x) ((union asce) { .val = (x), })
31 #define NULL_ASCE _ASCE(0)
32
33 enum {
34 _DAT_TOKEN_NONE = 0,
35 _DAT_TOKEN_PIC,
36 };
37
38 #define _CRSTE_TOK(l, t, p) ((union crste) { \
39 .tok.i = 1, \
40 .tok.tt = (l), \
41 .tok.type = (t), \
42 .tok.par = (p) \
43 })
44 #define _CRSTE_PIC(l, p) _CRSTE_TOK(l, _DAT_TOKEN_PIC, p)
45
46 #define _CRSTE_HOLE(l) _CRSTE_PIC(l, PGM_ADDRESSING)
47 #define _CRSTE_EMPTY(l) _CRSTE_TOK(l, _DAT_TOKEN_NONE, 0)
48
49 #define _PMD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_SEGMENT)
50
51 #define _PTE_TOK(t, p) ((union pte) { .tok.i = 1, .tok.type = (t), .tok.par = (p) })
52 #define _PTE_EMPTY _PTE_TOK(_DAT_TOKEN_NONE, 0)
53
54 /* This fake table type is used for page table walks (both for normal page tables and vSIE) */
55 #define TABLE_TYPE_PAGE_TABLE -1
56
57 enum dat_walk_flags {
58 DAT_WALK_USES_SKEYS = 0x40,
59 DAT_WALK_CONTINUE = 0x20,
60 DAT_WALK_IGN_HOLES = 0x10,
61 DAT_WALK_SPLIT = 0x08,
62 DAT_WALK_ALLOC = 0x04,
63 DAT_WALK_ANY = 0x02,
64 DAT_WALK_LEAF = 0x01,
65 DAT_WALK_DEFAULT = 0
66 };
67
68 #define DAT_WALK_SPLIT_ALLOC (DAT_WALK_SPLIT | DAT_WALK_ALLOC)
69 #define DAT_WALK_ALLOC_CONTINUE (DAT_WALK_CONTINUE | DAT_WALK_ALLOC)
70 #define DAT_WALK_LEAF_ALLOC (DAT_WALK_LEAF | DAT_WALK_ALLOC)
71
72 union pte {
73 unsigned long val;
74 union page_table_entry h;
75 struct {
76 unsigned long :56; /* Hardware bits */
77 unsigned long u : 1; /* Page unused */
78 unsigned long s : 1; /* Special */
79 unsigned long w : 1; /* Writable */
80 unsigned long r : 1; /* Readable */
81 unsigned long d : 1; /* Dirty */
82 unsigned long y : 1; /* Young */
83 unsigned long sd: 1; /* Soft dirty */
84 unsigned long pr: 1; /* Present */
85 } s;
86 struct {
87 unsigned char hwbytes[7];
88 unsigned char swbyte;
89 };
90 union {
91 struct {
92 unsigned long type :16; /* Token type */
93 unsigned long par :16; /* Token parameter */
94 unsigned long :20;
95 unsigned long : 1; /* Must be 0 */
96 unsigned long i : 1; /* Must be 1 */
97 unsigned long : 2;
98 unsigned long : 7;
99 unsigned long pr : 1; /* Must be 0 */
100 };
101 struct {
102 unsigned long token:32; /* Token and parameter */
103 unsigned long :32;
104 };
105 } tok;
106 };
107
108 #define _SEGMENT_FR_MASK (_SEGMENT_MASK >> PAGE_SHIFT)
109 #define _REGION3_FR_MASK (_REGION3_MASK >> PAGE_SHIFT)
110 #define _PAGES_PER_SEGMENT _PAGE_ENTRIES
111 #define _PAGES_PER_REGION3 (_PAGES_PER_SEGMENT * _CRST_ENTRIES)
112
113 /* Soft dirty, needed as macro for atomic operations on ptes */
114 #define _PAGE_SD 0x002
115
116 /* Needed as macro to perform atomic operations */
117 #define PGSTE_PCL_BIT 0x0080000000000000UL /* PCL lock, HW bit */
118 #define PGSTE_CMMA_D_BIT 0x0000000000008000UL /* CMMA dirty soft-bit */
119
120 enum pgste_gps_usage {
121 PGSTE_GPS_USAGE_STABLE = 0,
122 PGSTE_GPS_USAGE_UNUSED,
123 PGSTE_GPS_USAGE_POT_VOLATILE,
124 PGSTE_GPS_USAGE_VOLATILE,
125 };
126
127 union pgste {
128 unsigned long val;
129 struct {
130 unsigned long acc : 4;
131 unsigned long fp : 1;
132 unsigned long : 3;
133 unsigned long pcl : 1;
134 unsigned long hr : 1;
135 unsigned long hc : 1;
136 unsigned long : 2;
137 unsigned long gr : 1;
138 unsigned long gc : 1;
139 unsigned long : 1;
140 unsigned long :16; /* val16 */
141 unsigned long zero : 1;
142 unsigned long nodat : 1;
143 unsigned long : 4;
144 unsigned long usage : 2;
145 unsigned long : 8;
146 unsigned long cmma_d : 1; /* Dirty flag for CMMA bits */
147 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
148 unsigned long vsie_notif : 1; /* Referenced in a shadow table */
149 unsigned long vsie_gmem : 1; /* Contains nested guest memory */
150 unsigned long : 4;
151 unsigned long : 8;
152 };
153 struct {
154 unsigned short hwbytes0;
155 unsigned short val16; /* Used to store chunked values, see dat_{s,g}et_ptval() */
156 unsigned short hwbytes4;
157 unsigned char flags; /* Maps to the software bits */
158 unsigned char hwbyte7;
159 } __packed;
160 };
161
162 union pmd {
163 unsigned long val;
164 union segment_table_entry h;
165 struct {
166 struct {
167 unsigned long :44; /* HW */
168 unsigned long : 3; /* Unused */
169 unsigned long : 1; /* HW */
170 unsigned long s : 1; /* Special */
171 unsigned long w : 1; /* Writable soft-bit */
172 unsigned long r : 1; /* Readable soft-bit */
173 unsigned long d : 1; /* Dirty */
174 unsigned long y : 1; /* Young */
175 unsigned long : 3; /* HW */
176 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
177 unsigned long vsie_notif : 1; /* Referenced in a shadow table */
178 unsigned long : 4; /* HW */
179 unsigned long sd : 1; /* Soft-Dirty */
180 unsigned long pr : 1; /* Present */
181 } fc1;
182 } s;
183 };
184
185 union pud {
186 unsigned long val;
187 union region3_table_entry h;
188 struct {
189 struct {
190 unsigned long :33; /* HW */
191 unsigned long :14; /* Unused */
192 unsigned long : 1; /* HW */
193 unsigned long s : 1; /* Special */
194 unsigned long w : 1; /* Writable soft-bit */
195 unsigned long r : 1; /* Readable soft-bit */
196 unsigned long d : 1; /* Dirty */
197 unsigned long y : 1; /* Young */
198 unsigned long : 3; /* HW */
199 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
200 unsigned long vsie_notif : 1; /* Referenced in a shadow table */
201 unsigned long : 4; /* HW */
202 unsigned long sd : 1; /* Soft-Dirty */
203 unsigned long pr : 1; /* Present */
204 } fc1;
205 } s;
206 };
207
208 union p4d {
209 unsigned long val;
210 union region2_table_entry h;
211 };
212
213 union pgd {
214 unsigned long val;
215 union region1_table_entry h;
216 };
217
218 union crste {
219 unsigned long val;
220 union {
221 struct {
222 unsigned long :52;
223 unsigned long : 1;
224 unsigned long fc: 1;
225 unsigned long p : 1;
226 unsigned long : 1;
227 unsigned long : 2;
228 unsigned long i : 1;
229 unsigned long : 1;
230 unsigned long tt: 2;
231 unsigned long : 2;
232 };
233 struct {
234 unsigned long to:52;
235 unsigned long : 1;
236 unsigned long fc: 1;
237 unsigned long p : 1;
238 unsigned long : 1;
239 unsigned long tf: 2;
240 unsigned long i : 1;
241 unsigned long : 1;
242 unsigned long tt: 2;
243 unsigned long tl: 2;
244 } fc0;
245 struct {
246 unsigned long :47;
247 unsigned long av : 1; /* ACCF-Validity Control */
248 unsigned long acc: 4; /* Access-Control Bits */
249 unsigned long f : 1; /* Fetch-Protection Bit */
250 unsigned long fc : 1; /* Format-Control */
251 unsigned long p : 1; /* DAT-Protection Bit */
252 unsigned long iep: 1; /* Instruction-Execution-Protection */
253 unsigned long : 2;
254 unsigned long i : 1; /* Segment-Invalid Bit */
255 unsigned long cs : 1; /* Common-Segment Bit */
256 unsigned long tt : 2; /* Table-Type Bits */
257 unsigned long : 2;
258 } fc1;
259 } h;
260 struct {
261 struct {
262 unsigned long :47;
263 unsigned long : 1; /* HW (should be 0) */
264 unsigned long s : 1; /* Special */
265 unsigned long w : 1; /* Writable */
266 unsigned long r : 1; /* Readable */
267 unsigned long d : 1; /* Dirty */
268 unsigned long y : 1; /* Young */
269 unsigned long : 3; /* HW */
270 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
271 unsigned long vsie_notif : 1; /* Referenced in a shadow table */
272 unsigned long : 4; /* HW */
273 unsigned long sd : 1; /* Soft-Dirty */
274 unsigned long pr : 1; /* Present */
275 } fc1;
276 } s;
277 union {
278 struct {
279 unsigned long type :16; /* Token type */
280 unsigned long par :16; /* Token parameter */
281 unsigned long :26;
282 unsigned long i : 1; /* Must be 1 */
283 unsigned long : 1;
284 unsigned long tt : 2;
285 unsigned long : 1;
286 unsigned long pr : 1; /* Must be 0 */
287 };
288 struct {
289 unsigned long token:32; /* Token and parameter */
290 unsigned long :32;
291 };
292 } tok;
293 union pmd pmd;
294 union pud pud;
295 union p4d p4d;
296 union pgd pgd;
297 };
298
299 union skey {
300 unsigned char skey;
301 struct {
302 unsigned char acc :4;
303 unsigned char fp :1;
304 unsigned char r :1;
305 unsigned char c :1;
306 unsigned char zero:1;
307 };
308 };
309
310 static_assert(sizeof(union pgste) == sizeof(unsigned long));
311 static_assert(sizeof(union pte) == sizeof(unsigned long));
312 static_assert(sizeof(union pmd) == sizeof(unsigned long));
313 static_assert(sizeof(union pud) == sizeof(unsigned long));
314 static_assert(sizeof(union p4d) == sizeof(unsigned long));
315 static_assert(sizeof(union pgd) == sizeof(unsigned long));
316 static_assert(sizeof(union crste) == sizeof(unsigned long));
317 static_assert(sizeof(union skey) == sizeof(char));
318
319 struct segment_table {
320 union pmd pmds[_CRST_ENTRIES];
321 };
322
323 struct region3_table {
324 union pud puds[_CRST_ENTRIES];
325 };
326
327 struct region2_table {
328 union p4d p4ds[_CRST_ENTRIES];
329 };
330
331 struct region1_table {
332 union pgd pgds[_CRST_ENTRIES];
333 };
334
335 struct crst_table {
336 union {
337 union crste crstes[_CRST_ENTRIES];
338 struct segment_table segment;
339 struct region3_table region3;
340 struct region2_table region2;
341 struct region1_table region1;
342 };
343 };
344
345 struct page_table {
346 union pte ptes[_PAGE_ENTRIES];
347 union pgste pgstes[_PAGE_ENTRIES];
348 };
349
350 static_assert(sizeof(struct crst_table) == _CRST_TABLE_SIZE);
351 static_assert(sizeof(struct page_table) == PAGE_SIZE);
352
353 struct dat_walk;
354
355 typedef long (*dat_walk_op)(union crste *crste, gfn_t gfn, gfn_t next, struct dat_walk *w);
356
357 struct dat_walk_ops {
358 union {
359 dat_walk_op crste_ops[4];
360 struct {
361 dat_walk_op pmd_entry;
362 dat_walk_op pud_entry;
363 dat_walk_op p4d_entry;
364 dat_walk_op pgd_entry;
365 };
366 };
367 long (*pte_entry)(union pte *pte, gfn_t gfn, gfn_t next, struct dat_walk *w);
368 };
369
370 struct dat_walk {
371 const struct dat_walk_ops *ops;
372 union crste *last;
373 union pte *last_pte;
374 union asce asce;
375 gfn_t start;
376 gfn_t end;
377 int flags;
378 void *priv;
379 };
380
381 struct ptval_param {
382 unsigned char offset : 6;
383 unsigned char len : 2;
384 };
385
386 /**
387 * _pte() - Useful constructor for union pte
388 * @pfn: the pfn this pte should point to.
389 * @writable: whether the pte should be writable.
390 * @dirty: whether the pte should be dirty.
391 * @special: whether the pte should be marked as special
392 *
393 * The pte is also marked as young and present. If the pte is marked as dirty,
394 * it gets marked as soft-dirty too. If the pte is not dirty, the hardware
395 * protect bit is set (independently of the write softbit); this way proper
396 * dirty tracking can be performed.
397 *
398 * Return: a union pte value.
399 */
_pte(kvm_pfn_t pfn,bool writable,bool dirty,bool special)400 static inline union pte _pte(kvm_pfn_t pfn, bool writable, bool dirty, bool special)
401 {
402 union pte res = { .val = PFN_PHYS(pfn) };
403
404 res.h.p = !dirty;
405 res.s.y = 1;
406 res.s.pr = 1;
407 res.s.w = writable;
408 res.s.d = dirty;
409 res.s.sd = dirty;
410 res.s.s = special;
411 return res;
412 }
413
_crste_fc0(kvm_pfn_t pfn,int tt)414 static inline union crste _crste_fc0(kvm_pfn_t pfn, int tt)
415 {
416 union crste res = { .val = PFN_PHYS(pfn) };
417
418 res.h.tt = tt;
419 res.h.fc0.tl = _REGION_ENTRY_LENGTH;
420 res.h.fc0.tf = 0;
421 return res;
422 }
423
424 /**
425 * _crste() - Useful constructor for union crste with FC=1
426 * @pfn: the pfn this pte should point to.
427 * @tt: the table type
428 * @writable: whether the pte should be writable.
429 * @dirty: whether the pte should be dirty.
430 *
431 * The crste is also marked as young and present. If the crste is marked as
432 * dirty, it gets marked as soft-dirty too. If the crste is not dirty, the
433 * hardware protect bit is set (independently of the write softbit); this way
434 * proper dirty tracking can be performed.
435 *
436 * Return: a union crste value.
437 */
_crste_fc1(kvm_pfn_t pfn,int tt,bool writable,bool dirty)438 static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool writable, bool dirty)
439 {
440 union crste res = { .val = PFN_PHYS(pfn) & _SEGMENT_MASK };
441
442 res.h.tt = tt;
443 res.h.p = !dirty;
444 res.h.fc = 1;
445 res.s.fc1.y = 1;
446 res.s.fc1.pr = 1;
447 res.s.fc1.w = writable;
448 res.s.fc1.d = dirty;
449 res.s.fc1.sd = dirty;
450 return res;
451 }
452
453 union essa_state {
454 unsigned char val;
455 struct {
456 unsigned char : 2;
457 unsigned char nodat : 1;
458 unsigned char exception : 1;
459 unsigned char usage : 2;
460 unsigned char content : 2;
461 };
462 };
463
464 /**
465 * struct vsie_rmap - reverse mapping for shadow page table entries
466 * @next: pointer to next rmap in the list
467 * @r_gfn: virtual rmap address in the shadow guest address space
468 */
469 struct vsie_rmap {
470 struct vsie_rmap *next;
471 union {
472 unsigned long val;
473 struct {
474 long level: 8;
475 unsigned long : 4;
476 unsigned long r_gfn:52;
477 };
478 };
479 };
480
481 static_assert(sizeof(struct vsie_rmap) == 2 * sizeof(long));
482
483 #define KVM_S390_MMU_CACHE_N_CRSTS 6
484 #define KVM_S390_MMU_CACHE_N_PTS 2
485 #define KVM_S390_MMU_CACHE_N_RMAPS 16
486 struct kvm_s390_mmu_cache {
487 void *crsts[KVM_S390_MMU_CACHE_N_CRSTS];
488 void *pts[KVM_S390_MMU_CACHE_N_PTS];
489 void *rmaps[KVM_S390_MMU_CACHE_N_RMAPS];
490 short int n_crsts;
491 short int n_pts;
492 short int n_rmaps;
493 };
494
495 struct guest_fault {
496 gfn_t gfn; /* Guest frame */
497 kvm_pfn_t pfn; /* Host PFN */
498 struct page *page; /* Host page */
499 union pte *ptep; /* Used to resolve the fault, or NULL */
500 union crste *crstep; /* Used to resolve the fault, or NULL */
501 bool writable; /* Mapping is writable */
502 bool write_attempt; /* Write access attempted */
503 bool attempt_pfault; /* Attempt a pfault first */
504 bool valid; /* This entry contains valid data */
505 bool crste_region3; /* Whether crstep refers to a region3 entry */
506 void (*callback)(struct guest_fault *f);
507 void *priv;
508 };
509
510 /*
511 * 0 1 2 3 4 5 6 7
512 * +-------+-------+-------+-------+-------+-------+-------+-------+
513 * 0 | | PGT_ADDR |
514 * 8 | VMADDR | |
515 * 16 | |
516 * 24 | |
517 */
518 #define MKPTVAL(o, l) ((struct ptval_param) { .offset = (o), .len = ((l) + 1) / 2 - 1})
519 #define PTVAL_PGT_ADDR MKPTVAL(4, 8)
520 #define PTVAL_VMADDR MKPTVAL(8, 6)
521
522 union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new,
523 gfn_t gfn, union asce asce, bool uses_skeys);
524 bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
525 union asce asce);
526 void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce);
527
528 long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce,
529 const struct dat_walk_ops *ops, int flags, void *priv);
530
531 int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags,
532 int walk_level, union crste **last, union pte **ptepp);
533 void dat_free_level(struct crst_table *table, bool owns_ptes);
534 struct crst_table *dat_alloc_crst_sleepable(unsigned long init);
535 int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype);
536
537 #if KVM_S390_MANAGES_S390_GUEST
538 int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey);
539 int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn,
540 union skey skey, bool nq);
541 int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn,
542 union skey skey, union skey *oldkey, bool nq, bool mr, bool mc);
543 int dat_reset_reference_bit(union asce asce, gfn_t gfn, union skey *skey);
544 long dat_reset_skeys(union asce asce, gfn_t start);
545 #endif /* KVM_S390_MANAGES_S390_GUEST */
546
547 unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param);
548 void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val);
549
550 int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end);
551
552 #if KVM_S390_MANAGES_S390_GUEST
553 int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn);
554 #else
dat_set_prefix_notif_bit(union asce asce,gfn_t gfn)555 static inline int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn)
556 {
557 return 0;
558 }
559 #endif /* KVM_S390_MANAGES_S390_GUEST */
560
561 bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end);
562
563 #if KVM_S390_MANAGES_S390_GUEST
564 int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty);
565 long dat_reset_cmma(union asce asce, gfn_t start_gfn);
566 int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values);
567 int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem);
568 int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn,
569 unsigned long count, unsigned long mask, const uint8_t *bits);
570 #endif /* KVM_S390_MANAGES_S390_GUEST */
571
572 int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc);
573
574 #define GFP_KVM_S390_MMU_CACHE (GFP_ATOMIC | __GFP_ACCOUNT | __GFP_NOWARN)
575
kvm_s390_mmu_cache_alloc_pt(struct kvm_s390_mmu_cache * mc)576 static inline struct page_table *kvm_s390_mmu_cache_alloc_pt(struct kvm_s390_mmu_cache *mc)
577 {
578 if (mc->n_pts)
579 return mc->pts[--mc->n_pts];
580 return (void *)__get_free_page(GFP_KVM_S390_MMU_CACHE);
581 }
582
kvm_s390_mmu_cache_alloc_crst(struct kvm_s390_mmu_cache * mc)583 static inline struct crst_table *kvm_s390_mmu_cache_alloc_crst(struct kvm_s390_mmu_cache *mc)
584 {
585 if (mc->n_crsts)
586 return mc->crsts[--mc->n_crsts];
587 return (void *)__get_free_pages(GFP_KVM_S390_MMU_CACHE | __GFP_COMP, CRST_ALLOC_ORDER);
588 }
589
kvm_s390_mmu_cache_alloc_rmap(struct kvm_s390_mmu_cache * mc)590 static inline struct vsie_rmap *kvm_s390_mmu_cache_alloc_rmap(struct kvm_s390_mmu_cache *mc)
591 {
592 if (mc->n_rmaps)
593 return mc->rmaps[--mc->n_rmaps];
594 return kzalloc_obj(struct vsie_rmap, GFP_KVM_S390_MMU_CACHE);
595 }
596
crste_table_start(union crste * crstep)597 static inline struct crst_table *crste_table_start(union crste *crstep)
598 {
599 return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
600 }
601
pte_table_start(union pte * ptep)602 static inline struct page_table *pte_table_start(union pte *ptep)
603 {
604 return (struct page_table *)ALIGN_DOWN((unsigned long)ptep, _PAGE_TABLE_SIZE);
605 }
606
crdte_crste(union crste * crstep,union crste old,union crste new,gfn_t gfn,union asce asce)607 static inline bool crdte_crste(union crste *crstep, union crste old, union crste new, gfn_t gfn,
608 union asce asce)
609 {
610 unsigned long dtt = 0x10 | new.h.tt << 2;
611 void *table = crste_table_start(crstep);
612
613 return crdte(old.val, new.val, table, dtt, gfn_to_gpa(gfn), asce.val);
614 }
615
616 /**
617 * idte_crste() - invalidate a crste entry using idte
618 * @crstep: pointer to the crste to be invalidated
619 * @gfn: a gfn mapped by the crste
620 * @opt: options for the idte instruction
621 * @asce: the asce
622 * @local: whether the operation is cpu-local
623 */
idte_crste(union crste * crstep,gfn_t gfn,unsigned long opt,union asce asce,int local)624 static __always_inline void idte_crste(union crste *crstep, gfn_t gfn, unsigned long opt,
625 union asce asce, int local)
626 {
627 unsigned long table_origin = __pa(crste_table_start(crstep));
628 unsigned long gaddr = gfn_to_gpa(gfn) & HPAGE_MASK;
629
630 if (__builtin_constant_p(opt) && opt == 0) {
631 /* flush without guest asce */
632 asm volatile("idte %[table_origin],0,%[gaddr],%[local]"
633 : "+m" (*crstep)
634 : [table_origin] "a" (table_origin), [gaddr] "a" (gaddr),
635 [local] "i" (local)
636 : "cc");
637 } else {
638 /* flush with guest asce */
639 asm volatile("idte %[table_origin],%[asce],%[gaddr_opt],%[local]"
640 : "+m" (*crstep)
641 : [table_origin] "a" (table_origin), [gaddr_opt] "a" (gaddr | opt),
642 [asce] "a" (asce.val), [local] "i" (local)
643 : "cc");
644 }
645 }
646
dat_init_pgstes(struct page_table * pt,unsigned long val)647 static inline void dat_init_pgstes(struct page_table *pt, unsigned long val)
648 {
649 memset64((void *)pt->pgstes, val, PTRS_PER_PTE);
650 }
651
dat_init_page_table(struct page_table * pt,unsigned long ptes,unsigned long pgstes)652 static inline void dat_init_page_table(struct page_table *pt, unsigned long ptes,
653 unsigned long pgstes)
654 {
655 memset64((void *)pt->ptes, ptes, PTRS_PER_PTE);
656 dat_init_pgstes(pt, pgstes);
657 }
658
asce_end(union asce asce)659 static inline gfn_t asce_end(union asce asce)
660 {
661 return 1ULL << ((asce.dt + 1) * 11 + _SEGMENT_SHIFT - PAGE_SHIFT);
662 }
663
664 #define _CRSTE(x) ((union crste) { .val = _Generic((x), \
665 union pgd : (x).val, \
666 union p4d : (x).val, \
667 union pud : (x).val, \
668 union pmd : (x).val, \
669 union crste : (x).val)})
670
671 #define _CRSTEP(x) ((union crste *)_Generic((*(x)), \
672 union pgd : (x), \
673 union p4d : (x), \
674 union pud : (x), \
675 union pmd : (x), \
676 union crste : (x)))
677
678 #define _CRSTP(x) ((struct crst_table *)_Generic((*(x)), \
679 struct crst_table : (x), \
680 struct segment_table : (x), \
681 struct region3_table : (x), \
682 struct region2_table : (x), \
683 struct region1_table : (x)))
684
asce_contains_gfn(union asce asce,gfn_t gfn)685 static inline bool asce_contains_gfn(union asce asce, gfn_t gfn)
686 {
687 return gfn < asce_end(asce);
688 }
689
is_pmd(union crste crste)690 static inline bool is_pmd(union crste crste)
691 {
692 return crste.h.tt == TABLE_TYPE_SEGMENT;
693 }
694
is_pud(union crste crste)695 static inline bool is_pud(union crste crste)
696 {
697 return crste.h.tt == TABLE_TYPE_REGION3;
698 }
699
is_p4d(union crste crste)700 static inline bool is_p4d(union crste crste)
701 {
702 return crste.h.tt == TABLE_TYPE_REGION2;
703 }
704
is_pgd(union crste crste)705 static inline bool is_pgd(union crste crste)
706 {
707 return crste.h.tt == TABLE_TYPE_REGION1;
708 }
709
pmd_origin_large(union pmd pmd)710 static inline phys_addr_t pmd_origin_large(union pmd pmd)
711 {
712 return pmd.val & _SEGMENT_ENTRY_ORIGIN_LARGE;
713 }
714
pud_origin_large(union pud pud)715 static inline phys_addr_t pud_origin_large(union pud pud)
716 {
717 return pud.val & _REGION3_ENTRY_ORIGIN_LARGE;
718 }
719
720 /**
721 * crste_origin_large() - Return the large frame origin of a large crste
722 * @crste: The crste whose origin is to be returned. Should be either a
723 * region-3 table entry or a segment table entry, in both cases with
724 * FC set to 1 (large pages).
725 *
726 * Return: The origin of the large frame pointed to by @crste, or -1 if the
727 * crste was not large (wrong table type, or FC==0)
728 */
crste_origin_large(union crste crste)729 static inline phys_addr_t crste_origin_large(union crste crste)
730 {
731 if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
732 return -1;
733 if (is_pmd(crste))
734 return pmd_origin_large(crste.pmd);
735 return pud_origin_large(crste.pud);
736 }
737
738 #define crste_origin(x) (_Generic((x), \
739 union pmd : (x).val & _SEGMENT_ENTRY_ORIGIN, \
740 union pud : (x).val & _REGION_ENTRY_ORIGIN, \
741 union p4d : (x).val & _REGION_ENTRY_ORIGIN, \
742 union pgd : (x).val & _REGION_ENTRY_ORIGIN))
743
pte_origin(union pte pte)744 static inline unsigned long pte_origin(union pte pte)
745 {
746 return pte.val & PAGE_MASK;
747 }
748
pmd_prefix(union pmd pmd)749 static inline bool pmd_prefix(union pmd pmd)
750 {
751 return pmd.h.fc && pmd.s.fc1.prefix_notif;
752 }
753
pud_prefix(union pud pud)754 static inline bool pud_prefix(union pud pud)
755 {
756 return pud.h.fc && pud.s.fc1.prefix_notif;
757 }
758
crste_leaf(union crste crste)759 static inline bool crste_leaf(union crste crste)
760 {
761 return (crste.h.tt <= TABLE_TYPE_REGION3) && crste.h.fc;
762 }
763
crste_prefix(union crste crste)764 static inline bool crste_prefix(union crste crste)
765 {
766 return crste_leaf(crste) && crste.s.fc1.prefix_notif;
767 }
768
crste_dirty(union crste crste)769 static inline bool crste_dirty(union crste crste)
770 {
771 return crste_leaf(crste) && crste.s.fc1.d;
772 }
773
pgste_of(union pte * pte)774 static inline union pgste *pgste_of(union pte *pte)
775 {
776 return (union pgste *)(pte + _PAGE_ENTRIES);
777 }
778
pte_hole(union pte pte)779 static inline bool pte_hole(union pte pte)
780 {
781 return pte.h.i && !pte.tok.pr && pte.tok.type != _DAT_TOKEN_NONE;
782 }
783
_crste_hole(union crste crste)784 static inline bool _crste_hole(union crste crste)
785 {
786 return crste.h.i && !crste.tok.pr && crste.tok.type != _DAT_TOKEN_NONE;
787 }
788
789 #define crste_hole(x) _crste_hole(_CRSTE(x))
790
_crste_none(union crste crste)791 static inline bool _crste_none(union crste crste)
792 {
793 return crste.h.i && !crste.tok.pr && crste.tok.type == _DAT_TOKEN_NONE;
794 }
795
796 #define crste_none(x) _crste_none(_CRSTE(x))
797
large_pud_to_phys(union pud pud,gfn_t gfn)798 static inline phys_addr_t large_pud_to_phys(union pud pud, gfn_t gfn)
799 {
800 return pud_origin_large(pud) | (gfn_to_gpa(gfn) & ~_REGION3_MASK);
801 }
802
large_pmd_to_phys(union pmd pmd,gfn_t gfn)803 static inline phys_addr_t large_pmd_to_phys(union pmd pmd, gfn_t gfn)
804 {
805 return pmd_origin_large(pmd) | (gfn_to_gpa(gfn) & ~_SEGMENT_MASK);
806 }
807
large_crste_to_phys(union crste crste,gfn_t gfn)808 static inline phys_addr_t large_crste_to_phys(union crste crste, gfn_t gfn)
809 {
810 if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
811 return -1;
812 if (is_pmd(crste))
813 return large_pmd_to_phys(crste.pmd, gfn);
814 return large_pud_to_phys(crste.pud, gfn);
815 }
816
cspg_crste(union crste * crstep,union crste old,union crste new)817 static inline bool cspg_crste(union crste *crstep, union crste old, union crste new)
818 {
819 return cspg(&crstep->val, old.val, new.val);
820 }
821
dereference_pmd(union pmd pmd)822 static inline struct page_table *dereference_pmd(union pmd pmd)
823 {
824 return phys_to_virt(crste_origin(pmd));
825 }
826
dereference_pud(union pud pud)827 static inline struct segment_table *dereference_pud(union pud pud)
828 {
829 return phys_to_virt(crste_origin(pud));
830 }
831
dereference_p4d(union p4d p4d)832 static inline struct region3_table *dereference_p4d(union p4d p4d)
833 {
834 return phys_to_virt(crste_origin(p4d));
835 }
836
dereference_pgd(union pgd pgd)837 static inline struct region2_table *dereference_pgd(union pgd pgd)
838 {
839 return phys_to_virt(crste_origin(pgd));
840 }
841
_dereference_crste(union crste crste)842 static inline struct crst_table *_dereference_crste(union crste crste)
843 {
844 if (unlikely(is_pmd(crste)))
845 return NULL;
846 return phys_to_virt(crste_origin(crste.pud));
847 }
848
849 #define dereference_crste(x) (_Generic((x), \
850 union pud : _dereference_crste(_CRSTE(x)), \
851 union p4d : _dereference_crste(_CRSTE(x)), \
852 union pgd : _dereference_crste(_CRSTE(x)), \
853 union crste : _dereference_crste(_CRSTE(x))))
854
dereference_asce(union asce asce)855 static inline struct crst_table *dereference_asce(union asce asce)
856 {
857 return phys_to_virt(asce.val & _ASCE_ORIGIN);
858 }
859
asce_flush_tlb(union asce asce)860 static inline void asce_flush_tlb(union asce asce)
861 {
862 __tlb_flush_idte(asce.val);
863 }
864
pgste_get_trylock(union pte * ptep,union pgste * res)865 static inline bool pgste_get_trylock(union pte *ptep, union pgste *res)
866 {
867 union pgste *pgstep = pgste_of(ptep);
868 union pgste old_pgste;
869
870 if (READ_ONCE(pgstep->val) & PGSTE_PCL_BIT)
871 return false;
872 old_pgste.val = __atomic64_or_barrier(PGSTE_PCL_BIT, &pgstep->val);
873 if (old_pgste.pcl)
874 return false;
875 old_pgste.pcl = 1;
876 *res = old_pgste;
877 return true;
878 }
879
pgste_get_lock(union pte * ptep)880 static inline union pgste pgste_get_lock(union pte *ptep)
881 {
882 union pgste res;
883
884 while (!pgste_get_trylock(ptep, &res))
885 cpu_relax();
886 return res;
887 }
888
pgste_set_unlock(union pte * ptep,union pgste pgste)889 static inline void pgste_set_unlock(union pte *ptep, union pgste pgste)
890 {
891 pgste.pcl = 0;
892 barrier();
893 WRITE_ONCE(*pgste_of(ptep), pgste);
894 }
895
dat_ptep_xchg(union pte * ptep,union pte new,gfn_t gfn,union asce asce,bool has_skeys)896 static inline void dat_ptep_xchg(union pte *ptep, union pte new, gfn_t gfn, union asce asce,
897 bool has_skeys)
898 {
899 union pgste pgste;
900
901 pgste = pgste_get_lock(ptep);
902 pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, asce, has_skeys);
903 pgste_set_unlock(ptep, pgste);
904 }
905
dat_ptep_clear(union pte * ptep,gfn_t gfn,union asce asce,bool has_skeys)906 static inline void dat_ptep_clear(union pte *ptep, gfn_t gfn, union asce asce, bool has_skeys)
907 {
908 dat_ptep_xchg(ptep, _PTE_EMPTY, gfn, asce, has_skeys);
909 }
910
dat_free_pt(struct page_table * pt)911 static inline void dat_free_pt(struct page_table *pt)
912 {
913 free_page((unsigned long)pt);
914 }
915
_dat_free_crst(struct crst_table * table)916 static inline void _dat_free_crst(struct crst_table *table)
917 {
918 free_pages((unsigned long)table, CRST_ALLOC_ORDER);
919 }
920
921 #define dat_free_crst(x) _dat_free_crst(_CRSTP(x))
922
kvm_s390_free_mmu_cache(struct kvm_s390_mmu_cache * mc)923 static inline void kvm_s390_free_mmu_cache(struct kvm_s390_mmu_cache *mc)
924 {
925 if (!mc)
926 return;
927 while (mc->n_pts)
928 dat_free_pt(mc->pts[--mc->n_pts]);
929 while (mc->n_crsts)
930 _dat_free_crst(mc->crsts[--mc->n_crsts]);
931 while (mc->n_rmaps)
932 kfree(mc->rmaps[--mc->n_rmaps]);
933 kfree(mc);
934 }
935
DEFINE_FREE(kvm_s390_mmu_cache,struct kvm_s390_mmu_cache *,if (_T)kvm_s390_free_mmu_cache (_T))936 DEFINE_FREE(kvm_s390_mmu_cache, struct kvm_s390_mmu_cache *, if (_T) kvm_s390_free_mmu_cache(_T))
937
938 static inline struct kvm_s390_mmu_cache *kvm_s390_new_mmu_cache(void)
939 {
940 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
941
942 mc = kzalloc_obj(*mc, GFP_KERNEL_ACCOUNT);
943 if (mc && !kvm_s390_mmu_cache_topup(mc))
944 return_ptr(mc);
945 return NULL;
946 }
947
dat_pmdp_xchg_atomic(union pmd * pmdp,union pmd old,union pmd new,gfn_t gfn,union asce asce)948 static inline bool dat_pmdp_xchg_atomic(union pmd *pmdp, union pmd old, union pmd new,
949 gfn_t gfn, union asce asce)
950 {
951 return dat_crstep_xchg_atomic(_CRSTEP(pmdp), _CRSTE(old), _CRSTE(new), gfn, asce);
952 }
953
dat_pudp_xchg_atomic(union pud * pudp,union pud old,union pud new,gfn_t gfn,union asce asce)954 static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pud new,
955 gfn_t gfn, union asce asce)
956 {
957 return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce);
958 }
959
dat_crstep_clear_atomic(union crste * crstep,gfn_t gfn,union asce asce)960 static inline union crste dat_crstep_clear_atomic(union crste *crstep, gfn_t gfn, union asce asce)
961 {
962 union crste oldcrste, empty = _CRSTE_EMPTY(crstep->h.tt);
963
964 do {
965 oldcrste = READ_ONCE(*crstep);
966 } while (!dat_crstep_xchg_atomic(crstep, oldcrste, empty, gfn, asce));
967 return oldcrste;
968 }
969
get_level(union crste * crstep,union pte * ptep)970 static inline int get_level(union crste *crstep, union pte *ptep)
971 {
972 return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt;
973 }
974
dat_delete_slot(union asce asce,gfn_t start,unsigned long npages)975 static inline int dat_delete_slot(union asce asce, gfn_t start, unsigned long npages)
976 {
977 return dat_set_slot(NULL, asce, start, start + npages);
978 }
979
dat_create_slot(struct kvm_s390_mmu_cache * mc,union asce asce,gfn_t start,unsigned long npages)980 static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start,
981 unsigned long npages)
982 {
983 return dat_set_slot(mc, asce, start, start + npages);
984 }
985
crste_is_ucas(union crste crste)986 static inline bool crste_is_ucas(union crste crste)
987 {
988 return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0;
989 }
990
991 #endif /* ARCH_KVM_GMAP_DAT_H */
992