1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * KVM guest address space mapping code
4 *
5 * Copyright IBM Corp. 2024, 2025
6 * Author(s): Claudio Imbrenda <imbrenda@linux.ibm.com>
7 */
8
9 #ifndef __KVM_S390_DAT_H
10 #define __KVM_S390_DAT_H
11
12 #include <linux/radix-tree.h>
13 #include <linux/refcount.h>
14 #include <linux/io.h>
15 #include <linux/kvm_types.h>
16 #include <linux/pgalloc.h>
17 #include <asm/tlbflush.h>
18 #include <asm/dat-bits.h>
19
20 /*
21 * Base address and length must be sent at the start of each block, therefore
22 * it's cheaper to send some clean data, as long as it's less than the size of
23 * two longs.
24 */
25 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
26 /* For consistency */
27 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
28
29 #define _ASCE(x) ((union asce) { .val = (x), })
30 #define NULL_ASCE _ASCE(0)
31
32 enum {
33 _DAT_TOKEN_NONE = 0,
34 _DAT_TOKEN_PIC,
35 };
36
37 #define _CRSTE_TOK(l, t, p) ((union crste) { \
38 .tok.i = 1, \
39 .tok.tt = (l), \
40 .tok.type = (t), \
41 .tok.par = (p) \
42 })
43 #define _CRSTE_PIC(l, p) _CRSTE_TOK(l, _DAT_TOKEN_PIC, p)
44
45 #define _CRSTE_HOLE(l) _CRSTE_PIC(l, PGM_ADDRESSING)
46 #define _CRSTE_EMPTY(l) _CRSTE_TOK(l, _DAT_TOKEN_NONE, 0)
47
48 #define _PMD_EMPTY _CRSTE_EMPTY(TABLE_TYPE_SEGMENT)
49
50 #define _PTE_TOK(t, p) ((union pte) { .tok.i = 1, .tok.type = (t), .tok.par = (p) })
51 #define _PTE_EMPTY _PTE_TOK(_DAT_TOKEN_NONE, 0)
52
53 /* This fake table type is used for page table walks (both for normal page tables and vSIE) */
54 #define TABLE_TYPE_PAGE_TABLE -1
55
56 enum dat_walk_flags {
57 DAT_WALK_USES_SKEYS = 0x40,
58 DAT_WALK_CONTINUE = 0x20,
59 DAT_WALK_IGN_HOLES = 0x10,
60 DAT_WALK_SPLIT = 0x08,
61 DAT_WALK_ALLOC = 0x04,
62 DAT_WALK_ANY = 0x02,
63 DAT_WALK_LEAF = 0x01,
64 DAT_WALK_DEFAULT = 0
65 };
66
67 #define DAT_WALK_SPLIT_ALLOC (DAT_WALK_SPLIT | DAT_WALK_ALLOC)
68 #define DAT_WALK_ALLOC_CONTINUE (DAT_WALK_CONTINUE | DAT_WALK_ALLOC)
69 #define DAT_WALK_LEAF_ALLOC (DAT_WALK_LEAF | DAT_WALK_ALLOC)
70
71 union pte {
72 unsigned long val;
73 union page_table_entry h;
74 struct {
75 unsigned long :56; /* Hardware bits */
76 unsigned long u : 1; /* Page unused */
77 unsigned long s : 1; /* Special */
78 unsigned long w : 1; /* Writable */
79 unsigned long r : 1; /* Readable */
80 unsigned long d : 1; /* Dirty */
81 unsigned long y : 1; /* Young */
82 unsigned long sd: 1; /* Soft dirty */
83 unsigned long pr: 1; /* Present */
84 } s;
85 struct {
86 unsigned char hwbytes[7];
87 unsigned char swbyte;
88 };
89 union {
90 struct {
91 unsigned long type :16; /* Token type */
92 unsigned long par :16; /* Token parameter */
93 unsigned long :20;
94 unsigned long : 1; /* Must be 0 */
95 unsigned long i : 1; /* Must be 1 */
96 unsigned long : 2;
97 unsigned long : 7;
98 unsigned long pr : 1; /* Must be 0 */
99 };
100 struct {
101 unsigned long token:32; /* Token and parameter */
102 unsigned long :32;
103 };
104 } tok;
105 };
106
107 /* Soft dirty, needed as macro for atomic operations on ptes */
108 #define _PAGE_SD 0x002
109
110 /* Needed as macro to perform atomic operations */
111 #define PGSTE_PCL_BIT 0x0080000000000000UL /* PCL lock, HW bit */
112 #define PGSTE_CMMA_D_BIT 0x0000000000008000UL /* CMMA dirty soft-bit */
113
114 enum pgste_gps_usage {
115 PGSTE_GPS_USAGE_STABLE = 0,
116 PGSTE_GPS_USAGE_UNUSED,
117 PGSTE_GPS_USAGE_POT_VOLATILE,
118 PGSTE_GPS_USAGE_VOLATILE,
119 };
120
121 union pgste {
122 unsigned long val;
123 struct {
124 unsigned long acc : 4;
125 unsigned long fp : 1;
126 unsigned long : 3;
127 unsigned long pcl : 1;
128 unsigned long hr : 1;
129 unsigned long hc : 1;
130 unsigned long : 2;
131 unsigned long gr : 1;
132 unsigned long gc : 1;
133 unsigned long : 1;
134 unsigned long :16; /* val16 */
135 unsigned long zero : 1;
136 unsigned long nodat : 1;
137 unsigned long : 4;
138 unsigned long usage : 2;
139 unsigned long : 8;
140 unsigned long cmma_d : 1; /* Dirty flag for CMMA bits */
141 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
142 unsigned long vsie_notif : 1; /* Referenced in a shadow table */
143 unsigned long : 5;
144 unsigned long : 8;
145 };
146 struct {
147 unsigned short hwbytes0;
148 unsigned short val16; /* Used to store chunked values, see dat_{s,g}et_ptval() */
149 unsigned short hwbytes4;
150 unsigned char flags; /* Maps to the software bits */
151 unsigned char hwbyte7;
152 } __packed;
153 };
154
155 union pmd {
156 unsigned long val;
157 union segment_table_entry h;
158 struct {
159 struct {
160 unsigned long :44; /* HW */
161 unsigned long : 3; /* Unused */
162 unsigned long : 1; /* HW */
163 unsigned long w : 1; /* Writable soft-bit */
164 unsigned long r : 1; /* Readable soft-bit */
165 unsigned long d : 1; /* Dirty */
166 unsigned long y : 1; /* Young */
167 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
168 unsigned long : 3; /* HW */
169 unsigned long vsie_notif : 1; /* Referenced in a shadow table */
170 unsigned long : 1; /* Unused */
171 unsigned long : 4; /* HW */
172 unsigned long sd : 1; /* Soft-Dirty */
173 unsigned long pr : 1; /* Present */
174 } fc1;
175 } s;
176 };
177
178 union pud {
179 unsigned long val;
180 union region3_table_entry h;
181 struct {
182 struct {
183 unsigned long :33; /* HW */
184 unsigned long :14; /* Unused */
185 unsigned long : 1; /* HW */
186 unsigned long w : 1; /* Writable soft-bit */
187 unsigned long r : 1; /* Readable soft-bit */
188 unsigned long d : 1; /* Dirty */
189 unsigned long y : 1; /* Young */
190 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
191 unsigned long : 3; /* HW */
192 unsigned long vsie_notif : 1; /* Referenced in a shadow table */
193 unsigned long : 1; /* Unused */
194 unsigned long : 4; /* HW */
195 unsigned long sd : 1; /* Soft-Dirty */
196 unsigned long pr : 1; /* Present */
197 } fc1;
198 } s;
199 };
200
201 union p4d {
202 unsigned long val;
203 union region2_table_entry h;
204 };
205
206 union pgd {
207 unsigned long val;
208 union region1_table_entry h;
209 };
210
211 union crste {
212 unsigned long val;
213 union {
214 struct {
215 unsigned long :52;
216 unsigned long : 1;
217 unsigned long fc: 1;
218 unsigned long p : 1;
219 unsigned long : 1;
220 unsigned long : 2;
221 unsigned long i : 1;
222 unsigned long : 1;
223 unsigned long tt: 2;
224 unsigned long : 2;
225 };
226 struct {
227 unsigned long to:52;
228 unsigned long : 1;
229 unsigned long fc: 1;
230 unsigned long p : 1;
231 unsigned long : 1;
232 unsigned long tf: 2;
233 unsigned long i : 1;
234 unsigned long : 1;
235 unsigned long tt: 2;
236 unsigned long tl: 2;
237 } fc0;
238 struct {
239 unsigned long :47;
240 unsigned long av : 1; /* ACCF-Validity Control */
241 unsigned long acc: 4; /* Access-Control Bits */
242 unsigned long f : 1; /* Fetch-Protection Bit */
243 unsigned long fc : 1; /* Format-Control */
244 unsigned long p : 1; /* DAT-Protection Bit */
245 unsigned long iep: 1; /* Instruction-Execution-Protection */
246 unsigned long : 2;
247 unsigned long i : 1; /* Segment-Invalid Bit */
248 unsigned long cs : 1; /* Common-Segment Bit */
249 unsigned long tt : 2; /* Table-Type Bits */
250 unsigned long : 2;
251 } fc1;
252 } h;
253 struct {
254 struct {
255 unsigned long :47;
256 unsigned long : 1; /* HW (should be 0) */
257 unsigned long w : 1; /* Writable */
258 unsigned long r : 1; /* Readable */
259 unsigned long d : 1; /* Dirty */
260 unsigned long y : 1; /* Young */
261 unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
262 unsigned long : 3; /* HW */
263 unsigned long vsie_notif : 1; /* Referenced in a shadow table */
264 unsigned long : 1;
265 unsigned long : 4; /* HW */
266 unsigned long sd : 1; /* Soft-Dirty */
267 unsigned long pr : 1; /* Present */
268 } fc1;
269 } s;
270 union {
271 struct {
272 unsigned long type :16; /* Token type */
273 unsigned long par :16; /* Token parameter */
274 unsigned long :26;
275 unsigned long i : 1; /* Must be 1 */
276 unsigned long : 1;
277 unsigned long tt : 2;
278 unsigned long : 1;
279 unsigned long pr : 1; /* Must be 0 */
280 };
281 struct {
282 unsigned long token:32; /* Token and parameter */
283 unsigned long :32;
284 };
285 } tok;
286 union pmd pmd;
287 union pud pud;
288 union p4d p4d;
289 union pgd pgd;
290 };
291
292 union skey {
293 unsigned char skey;
294 struct {
295 unsigned char acc :4;
296 unsigned char fp :1;
297 unsigned char r :1;
298 unsigned char c :1;
299 unsigned char zero:1;
300 };
301 };
302
303 static_assert(sizeof(union pgste) == sizeof(unsigned long));
304 static_assert(sizeof(union pte) == sizeof(unsigned long));
305 static_assert(sizeof(union pmd) == sizeof(unsigned long));
306 static_assert(sizeof(union pud) == sizeof(unsigned long));
307 static_assert(sizeof(union p4d) == sizeof(unsigned long));
308 static_assert(sizeof(union pgd) == sizeof(unsigned long));
309 static_assert(sizeof(union crste) == sizeof(unsigned long));
310 static_assert(sizeof(union skey) == sizeof(char));
311
312 struct segment_table {
313 union pmd pmds[_CRST_ENTRIES];
314 };
315
316 struct region3_table {
317 union pud puds[_CRST_ENTRIES];
318 };
319
320 struct region2_table {
321 union p4d p4ds[_CRST_ENTRIES];
322 };
323
324 struct region1_table {
325 union pgd pgds[_CRST_ENTRIES];
326 };
327
328 struct crst_table {
329 union {
330 union crste crstes[_CRST_ENTRIES];
331 struct segment_table segment;
332 struct region3_table region3;
333 struct region2_table region2;
334 struct region1_table region1;
335 };
336 };
337
338 struct page_table {
339 union pte ptes[_PAGE_ENTRIES];
340 union pgste pgstes[_PAGE_ENTRIES];
341 };
342
343 static_assert(sizeof(struct crst_table) == _CRST_TABLE_SIZE);
344 static_assert(sizeof(struct page_table) == PAGE_SIZE);
345
346 struct dat_walk;
347
348 typedef long (*dat_walk_op)(union crste *crste, gfn_t gfn, gfn_t next, struct dat_walk *w);
349
350 struct dat_walk_ops {
351 union {
352 dat_walk_op crste_ops[4];
353 struct {
354 dat_walk_op pmd_entry;
355 dat_walk_op pud_entry;
356 dat_walk_op p4d_entry;
357 dat_walk_op pgd_entry;
358 };
359 };
360 long (*pte_entry)(union pte *pte, gfn_t gfn, gfn_t next, struct dat_walk *w);
361 };
362
363 struct dat_walk {
364 const struct dat_walk_ops *ops;
365 union crste *last;
366 union pte *last_pte;
367 union asce asce;
368 gfn_t start;
369 gfn_t end;
370 int flags;
371 void *priv;
372 };
373
374 struct ptval_param {
375 unsigned char offset : 6;
376 unsigned char len : 2;
377 };
378
379 /**
380 * _pte() - Useful constructor for union pte
381 * @pfn: the pfn this pte should point to.
382 * @writable: whether the pte should be writable.
383 * @dirty: whether the pte should be dirty.
384 * @special: whether the pte should be marked as special
385 *
386 * The pte is also marked as young and present. If the pte is marked as dirty,
387 * it gets marked as soft-dirty too. If the pte is not dirty, the hardware
388 * protect bit is set (independently of the write softbit); this way proper
389 * dirty tracking can be performed.
390 *
391 * Return: a union pte value.
392 */
_pte(kvm_pfn_t pfn,bool writable,bool dirty,bool special)393 static inline union pte _pte(kvm_pfn_t pfn, bool writable, bool dirty, bool special)
394 {
395 union pte res = { .val = PFN_PHYS(pfn) };
396
397 res.h.p = !dirty;
398 res.s.y = 1;
399 res.s.pr = 1;
400 res.s.w = writable;
401 res.s.d = dirty;
402 res.s.sd = dirty;
403 res.s.s = special;
404 return res;
405 }
406
_crste_fc0(kvm_pfn_t pfn,int tt)407 static inline union crste _crste_fc0(kvm_pfn_t pfn, int tt)
408 {
409 union crste res = { .val = PFN_PHYS(pfn) };
410
411 res.h.tt = tt;
412 res.h.fc0.tl = _REGION_ENTRY_LENGTH;
413 res.h.fc0.tf = 0;
414 return res;
415 }
416
417 /**
418 * _crste() - Useful constructor for union crste with FC=1
419 * @pfn: the pfn this pte should point to.
420 * @tt: the table type
421 * @writable: whether the pte should be writable.
422 * @dirty: whether the pte should be dirty.
423 *
424 * The crste is also marked as young and present. If the crste is marked as
425 * dirty, it gets marked as soft-dirty too. If the crste is not dirty, the
426 * hardware protect bit is set (independently of the write softbit); this way
427 * proper dirty tracking can be performed.
428 *
429 * Return: a union crste value.
430 */
_crste_fc1(kvm_pfn_t pfn,int tt,bool writable,bool dirty)431 static inline union crste _crste_fc1(kvm_pfn_t pfn, int tt, bool writable, bool dirty)
432 {
433 union crste res = { .val = PFN_PHYS(pfn) & _SEGMENT_MASK };
434
435 res.h.tt = tt;
436 res.h.p = !dirty;
437 res.h.fc = 1;
438 res.s.fc1.y = 1;
439 res.s.fc1.pr = 1;
440 res.s.fc1.w = writable;
441 res.s.fc1.d = dirty;
442 res.s.fc1.sd = dirty;
443 return res;
444 }
445
446 union essa_state {
447 unsigned char val;
448 struct {
449 unsigned char : 2;
450 unsigned char nodat : 1;
451 unsigned char exception : 1;
452 unsigned char usage : 2;
453 unsigned char content : 2;
454 };
455 };
456
457 /**
458 * struct vsie_rmap - reverse mapping for shadow page table entries
459 * @next: pointer to next rmap in the list
460 * @r_gfn: virtual rmap address in the shadow guest address space
461 */
462 struct vsie_rmap {
463 struct vsie_rmap *next;
464 union {
465 unsigned long val;
466 struct {
467 long level: 8;
468 unsigned long : 4;
469 unsigned long r_gfn:52;
470 };
471 };
472 };
473
474 static_assert(sizeof(struct vsie_rmap) == 2 * sizeof(long));
475
476 #define KVM_S390_MMU_CACHE_N_CRSTS 6
477 #define KVM_S390_MMU_CACHE_N_PTS 2
478 #define KVM_S390_MMU_CACHE_N_RMAPS 16
479 struct kvm_s390_mmu_cache {
480 void *crsts[KVM_S390_MMU_CACHE_N_CRSTS];
481 void *pts[KVM_S390_MMU_CACHE_N_PTS];
482 void *rmaps[KVM_S390_MMU_CACHE_N_RMAPS];
483 short int n_crsts;
484 short int n_pts;
485 short int n_rmaps;
486 };
487
488 struct guest_fault {
489 gfn_t gfn; /* Guest frame */
490 kvm_pfn_t pfn; /* Host PFN */
491 struct page *page; /* Host page */
492 union pte *ptep; /* Used to resolve the fault, or NULL */
493 union crste *crstep; /* Used to resolve the fault, or NULL */
494 bool writable; /* Mapping is writable */
495 bool write_attempt; /* Write access attempted */
496 bool attempt_pfault; /* Attempt a pfault first */
497 bool valid; /* This entry contains valid data */
498 void (*callback)(struct guest_fault *f);
499 void *priv;
500 };
501
502 /*
503 * 0 1 2 3 4 5 6 7
504 * +-------+-------+-------+-------+-------+-------+-------+-------+
505 * 0 | | PGT_ADDR |
506 * 8 | VMADDR | |
507 * 16 | |
508 * 24 | |
509 */
510 #define MKPTVAL(o, l) ((struct ptval_param) { .offset = (o), .len = ((l) + 1) / 2 - 1})
511 #define PTVAL_PGT_ADDR MKPTVAL(4, 8)
512 #define PTVAL_VMADDR MKPTVAL(8, 6)
513
514 union pgste __must_check __dat_ptep_xchg(union pte *ptep, union pgste pgste, union pte new,
515 gfn_t gfn, union asce asce, bool uses_skeys);
516 bool dat_crstep_xchg_atomic(union crste *crstep, union crste old, union crste new, gfn_t gfn,
517 union asce asce);
518 void dat_crstep_xchg(union crste *crstep, union crste new, gfn_t gfn, union asce asce);
519
520 long _dat_walk_gfn_range(gfn_t start, gfn_t end, union asce asce,
521 const struct dat_walk_ops *ops, int flags, void *priv);
522
523 int dat_entry_walk(struct kvm_s390_mmu_cache *mc, gfn_t gfn, union asce asce, int flags,
524 int walk_level, union crste **last, union pte **ptepp);
525 void dat_free_level(struct crst_table *table, bool owns_ptes);
526 struct crst_table *dat_alloc_crst_sleepable(unsigned long init);
527 int dat_set_asce_limit(struct kvm_s390_mmu_cache *mc, union asce *asce, int newtype);
528 int dat_get_storage_key(union asce asce, gfn_t gfn, union skey *skey);
529 int dat_set_storage_key(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn,
530 union skey skey, bool nq);
531 int dat_cond_set_storage_key(struct kvm_s390_mmu_cache *mmc, union asce asce, gfn_t gfn,
532 union skey skey, union skey *oldkey, bool nq, bool mr, bool mc);
533 int dat_reset_reference_bit(union asce asce, gfn_t gfn);
534 long dat_reset_skeys(union asce asce, gfn_t start);
535
536 unsigned long dat_get_ptval(struct page_table *table, struct ptval_param param);
537 void dat_set_ptval(struct page_table *table, struct ptval_param param, unsigned long val);
538
539 int dat_set_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start, gfn_t end,
540 u16 type, u16 param);
541 int dat_set_prefix_notif_bit(union asce asce, gfn_t gfn);
542 bool dat_test_age_gfn(union asce asce, gfn_t start, gfn_t end);
543 int dat_link(struct kvm_s390_mmu_cache *mc, union asce asce, int level,
544 bool uses_skeys, struct guest_fault *f);
545
546 int dat_perform_essa(union asce asce, gfn_t gfn, int orc, union essa_state *state, bool *dirty);
547 long dat_reset_cmma(union asce asce, gfn_t start_gfn);
548 int dat_peek_cmma(gfn_t start, union asce asce, unsigned int *count, u8 *values);
549 int dat_get_cmma(union asce asce, gfn_t *start, unsigned int *count, u8 *values, atomic64_t *rem);
550 int dat_set_cmma_bits(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t gfn,
551 unsigned long count, unsigned long mask, const uint8_t *bits);
552
553 int kvm_s390_mmu_cache_topup(struct kvm_s390_mmu_cache *mc);
554
555 #define GFP_KVM_S390_MMU_CACHE (GFP_ATOMIC | __GFP_ACCOUNT | __GFP_NOWARN)
556
kvm_s390_mmu_cache_alloc_pt(struct kvm_s390_mmu_cache * mc)557 static inline struct page_table *kvm_s390_mmu_cache_alloc_pt(struct kvm_s390_mmu_cache *mc)
558 {
559 if (mc->n_pts)
560 return mc->pts[--mc->n_pts];
561 return (void *)__get_free_page(GFP_KVM_S390_MMU_CACHE);
562 }
563
kvm_s390_mmu_cache_alloc_crst(struct kvm_s390_mmu_cache * mc)564 static inline struct crst_table *kvm_s390_mmu_cache_alloc_crst(struct kvm_s390_mmu_cache *mc)
565 {
566 if (mc->n_crsts)
567 return mc->crsts[--mc->n_crsts];
568 return (void *)__get_free_pages(GFP_KVM_S390_MMU_CACHE | __GFP_COMP, CRST_ALLOC_ORDER);
569 }
570
kvm_s390_mmu_cache_alloc_rmap(struct kvm_s390_mmu_cache * mc)571 static inline struct vsie_rmap *kvm_s390_mmu_cache_alloc_rmap(struct kvm_s390_mmu_cache *mc)
572 {
573 if (mc->n_rmaps)
574 return mc->rmaps[--mc->n_rmaps];
575 return kzalloc_obj(struct vsie_rmap, GFP_KVM_S390_MMU_CACHE);
576 }
577
crste_table_start(union crste * crstep)578 static inline struct crst_table *crste_table_start(union crste *crstep)
579 {
580 return (struct crst_table *)ALIGN_DOWN((unsigned long)crstep, _CRST_TABLE_SIZE);
581 }
582
pte_table_start(union pte * ptep)583 static inline struct page_table *pte_table_start(union pte *ptep)
584 {
585 return (struct page_table *)ALIGN_DOWN((unsigned long)ptep, _PAGE_TABLE_SIZE);
586 }
587
crdte_crste(union crste * crstep,union crste old,union crste new,gfn_t gfn,union asce asce)588 static inline bool crdte_crste(union crste *crstep, union crste old, union crste new, gfn_t gfn,
589 union asce asce)
590 {
591 unsigned long dtt = 0x10 | new.h.tt << 2;
592 void *table = crste_table_start(crstep);
593
594 return crdte(old.val, new.val, table, dtt, gfn_to_gpa(gfn), asce.val);
595 }
596
597 /**
598 * idte_crste() - invalidate a crste entry using idte
599 * @crstep: pointer to the crste to be invalidated
600 * @gfn: a gfn mapped by the crste
601 * @opt: options for the idte instruction
602 * @asce: the asce
603 * @local: whether the operation is cpu-local
604 */
idte_crste(union crste * crstep,gfn_t gfn,unsigned long opt,union asce asce,int local)605 static __always_inline void idte_crste(union crste *crstep, gfn_t gfn, unsigned long opt,
606 union asce asce, int local)
607 {
608 unsigned long table_origin = __pa(crste_table_start(crstep));
609 unsigned long gaddr = gfn_to_gpa(gfn) & HPAGE_MASK;
610
611 if (__builtin_constant_p(opt) && opt == 0) {
612 /* flush without guest asce */
613 asm volatile("idte %[table_origin],0,%[gaddr],%[local]"
614 : "+m" (*crstep)
615 : [table_origin] "a" (table_origin), [gaddr] "a" (gaddr),
616 [local] "i" (local)
617 : "cc");
618 } else {
619 /* flush with guest asce */
620 asm volatile("idte %[table_origin],%[asce],%[gaddr_opt],%[local]"
621 : "+m" (*crstep)
622 : [table_origin] "a" (table_origin), [gaddr_opt] "a" (gaddr | opt),
623 [asce] "a" (asce.val), [local] "i" (local)
624 : "cc");
625 }
626 }
627
dat_init_pgstes(struct page_table * pt,unsigned long val)628 static inline void dat_init_pgstes(struct page_table *pt, unsigned long val)
629 {
630 memset64((void *)pt->pgstes, val, PTRS_PER_PTE);
631 }
632
dat_init_page_table(struct page_table * pt,unsigned long ptes,unsigned long pgstes)633 static inline void dat_init_page_table(struct page_table *pt, unsigned long ptes,
634 unsigned long pgstes)
635 {
636 memset64((void *)pt->ptes, ptes, PTRS_PER_PTE);
637 dat_init_pgstes(pt, pgstes);
638 }
639
asce_end(union asce asce)640 static inline gfn_t asce_end(union asce asce)
641 {
642 return 1ULL << ((asce.dt + 1) * 11 + _SEGMENT_SHIFT - PAGE_SHIFT);
643 }
644
645 #define _CRSTE(x) ((union crste) { .val = _Generic((x), \
646 union pgd : (x).val, \
647 union p4d : (x).val, \
648 union pud : (x).val, \
649 union pmd : (x).val, \
650 union crste : (x).val)})
651
652 #define _CRSTEP(x) ((union crste *)_Generic((*(x)), \
653 union pgd : (x), \
654 union p4d : (x), \
655 union pud : (x), \
656 union pmd : (x), \
657 union crste : (x)))
658
659 #define _CRSTP(x) ((struct crst_table *)_Generic((*(x)), \
660 struct crst_table : (x), \
661 struct segment_table : (x), \
662 struct region3_table : (x), \
663 struct region2_table : (x), \
664 struct region1_table : (x)))
665
asce_contains_gfn(union asce asce,gfn_t gfn)666 static inline bool asce_contains_gfn(union asce asce, gfn_t gfn)
667 {
668 return gfn < asce_end(asce);
669 }
670
is_pmd(union crste crste)671 static inline bool is_pmd(union crste crste)
672 {
673 return crste.h.tt == TABLE_TYPE_SEGMENT;
674 }
675
is_pud(union crste crste)676 static inline bool is_pud(union crste crste)
677 {
678 return crste.h.tt == TABLE_TYPE_REGION3;
679 }
680
is_p4d(union crste crste)681 static inline bool is_p4d(union crste crste)
682 {
683 return crste.h.tt == TABLE_TYPE_REGION2;
684 }
685
is_pgd(union crste crste)686 static inline bool is_pgd(union crste crste)
687 {
688 return crste.h.tt == TABLE_TYPE_REGION1;
689 }
690
pmd_origin_large(union pmd pmd)691 static inline phys_addr_t pmd_origin_large(union pmd pmd)
692 {
693 return pmd.val & _SEGMENT_ENTRY_ORIGIN_LARGE;
694 }
695
pud_origin_large(union pud pud)696 static inline phys_addr_t pud_origin_large(union pud pud)
697 {
698 return pud.val & _REGION3_ENTRY_ORIGIN_LARGE;
699 }
700
701 /**
702 * crste_origin_large() - Return the large frame origin of a large crste
703 * @crste: The crste whose origin is to be returned. Should be either a
704 * region-3 table entry or a segment table entry, in both cases with
705 * FC set to 1 (large pages).
706 *
707 * Return: The origin of the large frame pointed to by @crste, or -1 if the
708 * crste was not large (wrong table type, or FC==0)
709 */
crste_origin_large(union crste crste)710 static inline phys_addr_t crste_origin_large(union crste crste)
711 {
712 if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
713 return -1;
714 if (is_pmd(crste))
715 return pmd_origin_large(crste.pmd);
716 return pud_origin_large(crste.pud);
717 }
718
719 #define crste_origin(x) (_Generic((x), \
720 union pmd : (x).val & _SEGMENT_ENTRY_ORIGIN, \
721 union pud : (x).val & _REGION_ENTRY_ORIGIN, \
722 union p4d : (x).val & _REGION_ENTRY_ORIGIN, \
723 union pgd : (x).val & _REGION_ENTRY_ORIGIN))
724
pte_origin(union pte pte)725 static inline unsigned long pte_origin(union pte pte)
726 {
727 return pte.val & PAGE_MASK;
728 }
729
pmd_prefix(union pmd pmd)730 static inline bool pmd_prefix(union pmd pmd)
731 {
732 return pmd.h.fc && pmd.s.fc1.prefix_notif;
733 }
734
pud_prefix(union pud pud)735 static inline bool pud_prefix(union pud pud)
736 {
737 return pud.h.fc && pud.s.fc1.prefix_notif;
738 }
739
crste_leaf(union crste crste)740 static inline bool crste_leaf(union crste crste)
741 {
742 return (crste.h.tt <= TABLE_TYPE_REGION3) && crste.h.fc;
743 }
744
crste_prefix(union crste crste)745 static inline bool crste_prefix(union crste crste)
746 {
747 return crste_leaf(crste) && crste.s.fc1.prefix_notif;
748 }
749
crste_dirty(union crste crste)750 static inline bool crste_dirty(union crste crste)
751 {
752 return crste_leaf(crste) && crste.s.fc1.d;
753 }
754
pgste_of(union pte * pte)755 static inline union pgste *pgste_of(union pte *pte)
756 {
757 return (union pgste *)(pte + _PAGE_ENTRIES);
758 }
759
pte_hole(union pte pte)760 static inline bool pte_hole(union pte pte)
761 {
762 return pte.h.i && !pte.tok.pr && pte.tok.type != _DAT_TOKEN_NONE;
763 }
764
_crste_hole(union crste crste)765 static inline bool _crste_hole(union crste crste)
766 {
767 return crste.h.i && !crste.tok.pr && crste.tok.type != _DAT_TOKEN_NONE;
768 }
769
770 #define crste_hole(x) _crste_hole(_CRSTE(x))
771
_crste_none(union crste crste)772 static inline bool _crste_none(union crste crste)
773 {
774 return crste.h.i && !crste.tok.pr && crste.tok.type == _DAT_TOKEN_NONE;
775 }
776
777 #define crste_none(x) _crste_none(_CRSTE(x))
778
large_pud_to_phys(union pud pud,gfn_t gfn)779 static inline phys_addr_t large_pud_to_phys(union pud pud, gfn_t gfn)
780 {
781 return pud_origin_large(pud) | (gfn_to_gpa(gfn) & ~_REGION3_MASK);
782 }
783
large_pmd_to_phys(union pmd pmd,gfn_t gfn)784 static inline phys_addr_t large_pmd_to_phys(union pmd pmd, gfn_t gfn)
785 {
786 return pmd_origin_large(pmd) | (gfn_to_gpa(gfn) & ~_SEGMENT_MASK);
787 }
788
large_crste_to_phys(union crste crste,gfn_t gfn)789 static inline phys_addr_t large_crste_to_phys(union crste crste, gfn_t gfn)
790 {
791 if (unlikely(!crste.h.fc || crste.h.tt > TABLE_TYPE_REGION3))
792 return -1;
793 if (is_pmd(crste))
794 return large_pmd_to_phys(crste.pmd, gfn);
795 return large_pud_to_phys(crste.pud, gfn);
796 }
797
cspg_crste(union crste * crstep,union crste old,union crste new)798 static inline bool cspg_crste(union crste *crstep, union crste old, union crste new)
799 {
800 return cspg(&crstep->val, old.val, new.val);
801 }
802
dereference_pmd(union pmd pmd)803 static inline struct page_table *dereference_pmd(union pmd pmd)
804 {
805 return phys_to_virt(crste_origin(pmd));
806 }
807
dereference_pud(union pud pud)808 static inline struct segment_table *dereference_pud(union pud pud)
809 {
810 return phys_to_virt(crste_origin(pud));
811 }
812
dereference_p4d(union p4d p4d)813 static inline struct region3_table *dereference_p4d(union p4d p4d)
814 {
815 return phys_to_virt(crste_origin(p4d));
816 }
817
dereference_pgd(union pgd pgd)818 static inline struct region2_table *dereference_pgd(union pgd pgd)
819 {
820 return phys_to_virt(crste_origin(pgd));
821 }
822
_dereference_crste(union crste crste)823 static inline struct crst_table *_dereference_crste(union crste crste)
824 {
825 if (unlikely(is_pmd(crste)))
826 return NULL;
827 return phys_to_virt(crste_origin(crste.pud));
828 }
829
830 #define dereference_crste(x) (_Generic((x), \
831 union pud : _dereference_crste(_CRSTE(x)), \
832 union p4d : _dereference_crste(_CRSTE(x)), \
833 union pgd : _dereference_crste(_CRSTE(x)), \
834 union crste : _dereference_crste(_CRSTE(x))))
835
dereference_asce(union asce asce)836 static inline struct crst_table *dereference_asce(union asce asce)
837 {
838 return phys_to_virt(asce.val & _ASCE_ORIGIN);
839 }
840
asce_flush_tlb(union asce asce)841 static inline void asce_flush_tlb(union asce asce)
842 {
843 __tlb_flush_idte(asce.val);
844 }
845
pgste_get_trylock(union pte * ptep,union pgste * res)846 static inline bool pgste_get_trylock(union pte *ptep, union pgste *res)
847 {
848 union pgste *pgstep = pgste_of(ptep);
849 union pgste old_pgste;
850
851 if (READ_ONCE(pgstep->val) & PGSTE_PCL_BIT)
852 return false;
853 old_pgste.val = __atomic64_or_barrier(PGSTE_PCL_BIT, &pgstep->val);
854 if (old_pgste.pcl)
855 return false;
856 old_pgste.pcl = 1;
857 *res = old_pgste;
858 return true;
859 }
860
pgste_get_lock(union pte * ptep)861 static inline union pgste pgste_get_lock(union pte *ptep)
862 {
863 union pgste res;
864
865 while (!pgste_get_trylock(ptep, &res))
866 cpu_relax();
867 return res;
868 }
869
pgste_set_unlock(union pte * ptep,union pgste pgste)870 static inline void pgste_set_unlock(union pte *ptep, union pgste pgste)
871 {
872 pgste.pcl = 0;
873 barrier();
874 WRITE_ONCE(*pgste_of(ptep), pgste);
875 }
876
dat_ptep_xchg(union pte * ptep,union pte new,gfn_t gfn,union asce asce,bool has_skeys)877 static inline void dat_ptep_xchg(union pte *ptep, union pte new, gfn_t gfn, union asce asce,
878 bool has_skeys)
879 {
880 union pgste pgste;
881
882 pgste = pgste_get_lock(ptep);
883 pgste = __dat_ptep_xchg(ptep, pgste, new, gfn, asce, has_skeys);
884 pgste_set_unlock(ptep, pgste);
885 }
886
dat_ptep_clear(union pte * ptep,gfn_t gfn,union asce asce,bool has_skeys)887 static inline void dat_ptep_clear(union pte *ptep, gfn_t gfn, union asce asce, bool has_skeys)
888 {
889 dat_ptep_xchg(ptep, _PTE_EMPTY, gfn, asce, has_skeys);
890 }
891
dat_free_pt(struct page_table * pt)892 static inline void dat_free_pt(struct page_table *pt)
893 {
894 free_page((unsigned long)pt);
895 }
896
_dat_free_crst(struct crst_table * table)897 static inline void _dat_free_crst(struct crst_table *table)
898 {
899 free_pages((unsigned long)table, CRST_ALLOC_ORDER);
900 }
901
902 #define dat_free_crst(x) _dat_free_crst(_CRSTP(x))
903
kvm_s390_free_mmu_cache(struct kvm_s390_mmu_cache * mc)904 static inline void kvm_s390_free_mmu_cache(struct kvm_s390_mmu_cache *mc)
905 {
906 if (!mc)
907 return;
908 while (mc->n_pts)
909 dat_free_pt(mc->pts[--mc->n_pts]);
910 while (mc->n_crsts)
911 _dat_free_crst(mc->crsts[--mc->n_crsts]);
912 while (mc->n_rmaps)
913 kfree(mc->rmaps[--mc->n_rmaps]);
914 kfree(mc);
915 }
916
DEFINE_FREE(kvm_s390_mmu_cache,struct kvm_s390_mmu_cache *,if (_T)kvm_s390_free_mmu_cache (_T))917 DEFINE_FREE(kvm_s390_mmu_cache, struct kvm_s390_mmu_cache *, if (_T) kvm_s390_free_mmu_cache(_T))
918
919 static inline struct kvm_s390_mmu_cache *kvm_s390_new_mmu_cache(void)
920 {
921 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
922
923 mc = kzalloc_obj(*mc, GFP_KERNEL_ACCOUNT);
924 if (mc && !kvm_s390_mmu_cache_topup(mc))
925 return_ptr(mc);
926 return NULL;
927 }
928
dat_pmdp_xchg_atomic(union pmd * pmdp,union pmd old,union pmd new,gfn_t gfn,union asce asce)929 static inline bool dat_pmdp_xchg_atomic(union pmd *pmdp, union pmd old, union pmd new,
930 gfn_t gfn, union asce asce)
931 {
932 return dat_crstep_xchg_atomic(_CRSTEP(pmdp), _CRSTE(old), _CRSTE(new), gfn, asce);
933 }
934
dat_pudp_xchg_atomic(union pud * pudp,union pud old,union pud new,gfn_t gfn,union asce asce)935 static inline bool dat_pudp_xchg_atomic(union pud *pudp, union pud old, union pud new,
936 gfn_t gfn, union asce asce)
937 {
938 return dat_crstep_xchg_atomic(_CRSTEP(pudp), _CRSTE(old), _CRSTE(new), gfn, asce);
939 }
940
dat_crstep_clear(union crste * crstep,gfn_t gfn,union asce asce)941 static inline void dat_crstep_clear(union crste *crstep, gfn_t gfn, union asce asce)
942 {
943 union crste newcrste = _CRSTE_EMPTY(crstep->h.tt);
944
945 dat_crstep_xchg(crstep, newcrste, gfn, asce);
946 }
947
get_level(union crste * crstep,union pte * ptep)948 static inline int get_level(union crste *crstep, union pte *ptep)
949 {
950 return ptep ? TABLE_TYPE_PAGE_TABLE : crstep->h.tt;
951 }
952
dat_delete_slot(struct kvm_s390_mmu_cache * mc,union asce asce,gfn_t start,unsigned long npages)953 static inline int dat_delete_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start,
954 unsigned long npages)
955 {
956 return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_PIC, PGM_ADDRESSING);
957 }
958
dat_create_slot(struct kvm_s390_mmu_cache * mc,union asce asce,gfn_t start,unsigned long npages)959 static inline int dat_create_slot(struct kvm_s390_mmu_cache *mc, union asce asce, gfn_t start,
960 unsigned long npages)
961 {
962 return dat_set_slot(mc, asce, start, start + npages, _DAT_TOKEN_NONE, 0);
963 }
964
crste_is_ucas(union crste crste)965 static inline bool crste_is_ucas(union crste crste)
966 {
967 return is_pmd(crste) && crste.h.i && crste.h.fc0.tl == 1 && crste.h.fc == 0;
968 }
969
970 #endif /* __KVM_S390_DAT_H */
971