1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES 4 * 5 * This header is included after the format. It contains definitions 6 * that build on the format definitions to create the basic format API. 7 * 8 * The format API is listed here, with kdocs. The functions without bodies are 9 * implemented in the format using the pattern: 10 * static inline FMTpt_XXX(..) {..} 11 * #define pt_XXX FMTpt_XXX 12 * 13 * If the format doesn't implement a function then pt_fmt_defaults.h can provide 14 * a generic version. 15 * 16 * The routines marked "@pts: Entry to query" operate on the entire contiguous 17 * entry and can be called with a pts->index pointing to any sub item that makes 18 * up that entry. 19 * 20 * The header order is: 21 * pt_defs.h 22 * FMT.h 23 * pt_common.h 24 */ 25 #ifndef __GENERIC_PT_PT_COMMON_H 26 #define __GENERIC_PT_PT_COMMON_H 27 28 #include "pt_defs.h" 29 #include "pt_fmt_defaults.h" 30 31 /** 32 * pt_attr_from_entry() - Convert the permission bits back to attrs 33 * @pts: Entry to convert from 34 * @attrs: Resulting attrs 35 * 36 * Fill in the attrs with the permission bits encoded in the current leaf entry. 37 * The attrs should be usable with pt_install_leaf_entry() to reconstruct the 38 * same entry. 39 */ 40 static inline void pt_attr_from_entry(const struct pt_state *pts, 41 struct pt_write_attrs *attrs); 42 43 /** 44 * pt_can_have_leaf() - True if the current level can have an OA entry 45 * @pts: The current level 46 * 47 * True if the current level can support pt_install_leaf_entry(). A leaf 48 * entry produce an OA. 49 */ 50 static inline bool pt_can_have_leaf(const struct pt_state *pts); 51 52 /** 53 * pt_can_have_table() - True if the current level can have a lower table 54 * @pts: The current level 55 * 56 * Every level except 0 is allowed to have a lower table. 57 */ 58 static inline bool pt_can_have_table(const struct pt_state *pts) 59 { 60 /* No further tables at level 0 */ 61 return pts->level > 0; 62 } 63 64 /** 65 * pt_clear_entries() - Make entries empty (non-present) 66 * @pts: Starting table index 67 * @num_contig_lg2: Number of contiguous items to clear 68 * 69 * Clear a run of entries. A cleared entry will load back as PT_ENTRY_EMPTY 70 * and does not have any effect on table walking. The starting index must be 71 * aligned to num_contig_lg2. 72 */ 73 static inline void pt_clear_entries(struct pt_state *pts, 74 unsigned int num_contig_lg2); 75 76 /** 77 * pt_entry_make_write_dirty() - Make an entry dirty 78 * @pts: Table entry to change 79 * 80 * Make pt_entry_is_write_dirty() return true for this entry. This can be called 81 * asynchronously with any other table manipulation under a RCU lock and must 82 * not corrupt the table. 83 */ 84 static inline bool pt_entry_make_write_dirty(struct pt_state *pts); 85 86 /** 87 * pt_entry_make_write_clean() - Make the entry write clean 88 * @pts: Table entry to change 89 * 90 * Modify the entry so that pt_entry_is_write_dirty() == false. The HW will 91 * eventually be notified of this change via a TLB flush, which is the point 92 * that the HW must become synchronized. Any "write dirty" prior to the TLB 93 * flush can be lost, but once the TLB flush completes all writes must make 94 * their entries write dirty. 95 * 96 * The format should alter the entry in a way that is compatible with any 97 * concurrent update from HW. The entire contiguous entry is changed. 98 */ 99 static inline void pt_entry_make_write_clean(struct pt_state *pts); 100 101 /** 102 * pt_entry_is_write_dirty() - True if the entry has been written to 103 * @pts: Entry to query 104 * 105 * "write dirty" means that the HW has written to the OA translated 106 * by this entry. If the entry is contiguous then the consolidated 107 * "write dirty" for all the items must be returned. 108 */ 109 static inline bool pt_entry_is_write_dirty(const struct pt_state *pts); 110 111 /** 112 * pt_dirty_supported() - True if the page table supports dirty tracking 113 * @common: Page table to query 114 */ 115 static inline bool pt_dirty_supported(struct pt_common *common); 116 117 /** 118 * pt_entry_num_contig_lg2() - Number of contiguous items for this leaf entry 119 * @pts: Entry to query 120 * 121 * Return the number of contiguous items this leaf entry spans. If the entry 122 * is single item it returns ilog2(1). 123 */ 124 static inline unsigned int pt_entry_num_contig_lg2(const struct pt_state *pts); 125 126 /** 127 * pt_entry_oa() - Output Address for this leaf entry 128 * @pts: Entry to query 129 * 130 * Return the output address for the start of the entry. If the entry 131 * is contiguous this returns the same value for each sub-item. I.e.:: 132 * 133 * log2_mod(pt_entry_oa(), pt_entry_oa_lg2sz()) == 0 134 * 135 * See pt_item_oa(). The format should implement one of these two functions 136 * depending on how it stores the OAs in the table. 137 */ 138 static inline pt_oaddr_t pt_entry_oa(const struct pt_state *pts); 139 140 /** 141 * pt_entry_oa_lg2sz() - Return the size of an OA entry 142 * @pts: Entry to query 143 * 144 * If the entry is not contiguous this returns pt_table_item_lg2sz(), otherwise 145 * it returns the total VA/OA size of the entire contiguous entry. 146 */ 147 static inline unsigned int pt_entry_oa_lg2sz(const struct pt_state *pts) 148 { 149 return pt_entry_num_contig_lg2(pts) + pt_table_item_lg2sz(pts); 150 } 151 152 /** 153 * pt_entry_oa_exact() - Return the complete OA for an entry 154 * @pts: Entry to query 155 * 156 * During iteration the first entry could have a VA with an offset from the 157 * natural start of the entry. Return the exact OA including the pts's VA 158 * offset. 159 */ 160 static inline pt_oaddr_t pt_entry_oa_exact(const struct pt_state *pts) 161 { 162 return _pt_entry_oa_fast(pts) | 163 log2_mod(pts->range->va, pt_entry_oa_lg2sz(pts)); 164 } 165 166 /** 167 * pt_full_va_prefix() - The top bits of the VA 168 * @common: Page table to query 169 * 170 * This is usually 0, but some formats have their VA space going downward from 171 * PT_VADDR_MAX, and will return that instead. This value must always be 172 * adjusted by struct pt_common max_vasz_lg2. 173 */ 174 static inline pt_vaddr_t pt_full_va_prefix(const struct pt_common *common); 175 176 /** 177 * pt_has_system_page_size() - True if level 0 can install a PAGE_SHIFT entry 178 * @common: Page table to query 179 * 180 * If true the caller can use, at level 0, pt_install_leaf_entry(PAGE_SHIFT). 181 * This is useful to create optimized paths for common cases of PAGE_SIZE 182 * mappings. 183 */ 184 static inline bool pt_has_system_page_size(const struct pt_common *common); 185 186 /** 187 * pt_install_leaf_entry() - Write a leaf entry to the table 188 * @pts: Table index to change 189 * @oa: Output Address for this leaf 190 * @oasz_lg2: Size in VA/OA for this leaf 191 * @attrs: Attributes to modify the entry 192 * 193 * A leaf OA entry will return PT_ENTRY_OA from pt_load_entry(). It translates 194 * the VA indicated by pts to the given OA. 195 * 196 * For a single item non-contiguous entry oasz_lg2 is pt_table_item_lg2sz(). 197 * For contiguous it is pt_table_item_lg2sz() + num_contig_lg2. 198 * 199 * This must not be called if pt_can_have_leaf() == false. Contiguous sizes 200 * not indicated by pt_possible_sizes() must not be specified. 201 */ 202 static inline void pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa, 203 unsigned int oasz_lg2, 204 const struct pt_write_attrs *attrs); 205 206 /** 207 * pt_install_table() - Write a table entry to the table 208 * @pts: Table index to change 209 * @table_pa: CPU physical address of the lower table's memory 210 * @attrs: Attributes to modify the table index 211 * 212 * A table entry will return PT_ENTRY_TABLE from pt_load_entry(). The table_pa 213 * is the table at pts->level - 1. This is done by cmpxchg so pts must have the 214 * current entry loaded. The pts is updated with the installed entry. 215 * 216 * This must not be called if pt_can_have_table() == false. 217 * 218 * Returns: true if the table was installed successfully. 219 */ 220 static inline bool pt_install_table(struct pt_state *pts, pt_oaddr_t table_pa, 221 const struct pt_write_attrs *attrs); 222 223 /** 224 * pt_item_oa() - Output Address for this leaf item 225 * @pts: Item to query 226 * 227 * Return the output address for this item. If the item is part of a contiguous 228 * entry it returns the value of the OA for this individual sub item. 229 * 230 * See pt_entry_oa(). The format should implement one of these two functions 231 * depending on how it stores the OA's in the table. 232 */ 233 static inline pt_oaddr_t pt_item_oa(const struct pt_state *pts); 234 235 /** 236 * pt_load_entry_raw() - Read from the location pts points at into the pts 237 * @pts: Table index to load 238 * 239 * Return the type of entry that was loaded. pts->entry will be filled in with 240 * the entry's content. See pt_load_entry() 241 */ 242 static inline enum pt_entry_type pt_load_entry_raw(struct pt_state *pts); 243 244 /** 245 * pt_max_oa_lg2() - Return the maximum OA the table format can hold 246 * @common: Page table to query 247 * 248 * The value oalog2_to_max_int(pt_max_oa_lg2()) is the MAX for the 249 * OA. This is the absolute maximum address the table can hold. struct pt_common 250 * max_oasz_lg2 sets a lower dynamic maximum based on HW capability. 251 */ 252 static inline unsigned int 253 pt_max_oa_lg2(const struct pt_common *common); 254 255 /** 256 * pt_num_items_lg2() - Return the number of items in this table level 257 * @pts: The current level 258 * 259 * The number of items in a table level defines the number of bits this level 260 * decodes from the VA. This function is not called for the top level, 261 * so it does not need to compute a special value for the top case. The 262 * result for the top is based on pt_common max_vasz_lg2. 263 * 264 * The value is used as part of determining the table indexes via the 265 * equation:: 266 * 267 * log2_mod(log2_div(VA, pt_table_item_lg2sz()), pt_num_items_lg2()) 268 */ 269 static inline unsigned int pt_num_items_lg2(const struct pt_state *pts); 270 271 /** 272 * pt_pgsz_lg2_to_level - Return the level that maps the page size 273 * @common: Page table to query 274 * @pgsize_lg2: Log2 page size 275 * 276 * Returns the table level that will map the given page size. The page 277 * size must be part of the pt_possible_sizes() for some level. 278 */ 279 static inline unsigned int pt_pgsz_lg2_to_level(struct pt_common *common, 280 unsigned int pgsize_lg2); 281 282 /** 283 * pt_possible_sizes() - Return a bitmap of possible output sizes at this level 284 * @pts: The current level 285 * 286 * Each level has a list of possible output sizes that can be installed as 287 * leaf entries. If pt_can_have_leaf() is false returns zero. 288 * 289 * Otherwise the bit in position pt_table_item_lg2sz() should be set indicating 290 * that a non-contiguous single item leaf entry is supported. The following 291 * pt_num_items_lg2() number of bits can be set indicating contiguous entries 292 * are supported. Bit pt_table_item_lg2sz() + pt_num_items_lg2() must not be 293 * set, contiguous entries cannot span the entire table. 294 * 295 * The OR of pt_possible_sizes() of all levels is the typical bitmask of all 296 * supported sizes in the entire table. 297 */ 298 static inline pt_vaddr_t pt_possible_sizes(const struct pt_state *pts); 299 300 /** 301 * pt_table_item_lg2sz() - Size of a single item entry in this table level 302 * @pts: The current level 303 * 304 * The size of the item specifies how much VA and OA a single item occupies. 305 * 306 * See pt_entry_oa_lg2sz() for the same value including the effect of contiguous 307 * entries. 308 */ 309 static inline unsigned int pt_table_item_lg2sz(const struct pt_state *pts); 310 311 /** 312 * pt_table_oa_lg2sz() - Return the VA/OA size of the entire table 313 * @pts: The current level 314 * 315 * Return the size of VA decoded by the entire table level. 316 */ 317 static inline unsigned int pt_table_oa_lg2sz(const struct pt_state *pts) 318 { 319 if (pts->range->top_level == pts->level) 320 return pts->range->max_vasz_lg2; 321 return min_t(unsigned int, pts->range->common->max_vasz_lg2, 322 pt_num_items_lg2(pts) + pt_table_item_lg2sz(pts)); 323 } 324 325 /** 326 * pt_table_pa() - Return the CPU physical address of the table entry 327 * @pts: Entry to query 328 * 329 * This is only ever called on PT_ENTRY_TABLE entries. Must return the same 330 * value passed to pt_install_table(). 331 */ 332 static inline pt_oaddr_t pt_table_pa(const struct pt_state *pts); 333 334 /** 335 * pt_table_ptr() - Return a CPU pointer for a table item 336 * @pts: Entry to query 337 * 338 * Same as pt_table_pa() but returns a CPU pointer. 339 */ 340 static inline struct pt_table_p *pt_table_ptr(const struct pt_state *pts) 341 { 342 return __va(pt_table_pa(pts)); 343 } 344 345 /** 346 * pt_max_sw_bit() - Return the maximum software bit usable for any level and 347 * entry 348 * @common: Page table 349 * 350 * The swbit can be passed as bitnr to the other sw_bit functions. 351 */ 352 static inline unsigned int pt_max_sw_bit(struct pt_common *common); 353 354 /** 355 * pt_test_sw_bit_acquire() - Read a software bit in an item 356 * @pts: Entry to read 357 * @bitnr: Bit to read 358 * 359 * Software bits are ignored by HW and can be used for any purpose by the 360 * software. This does a test bit and acquire operation. 361 */ 362 static inline bool pt_test_sw_bit_acquire(struct pt_state *pts, 363 unsigned int bitnr); 364 365 /** 366 * pt_set_sw_bit_release() - Set a software bit in an item 367 * @pts: Entry to set 368 * @bitnr: Bit to set 369 * 370 * Software bits are ignored by HW and can be used for any purpose by the 371 * software. This does a set bit and release operation. 372 */ 373 static inline void pt_set_sw_bit_release(struct pt_state *pts, 374 unsigned int bitnr); 375 376 /** 377 * pt_load_entry() - Read from the location pts points at into the pts 378 * @pts: Table index to load 379 * 380 * Set the type of entry that was loaded. pts->entry and pts->table_lower 381 * will be filled in with the entry's content. 382 */ 383 static inline void pt_load_entry(struct pt_state *pts) 384 { 385 pts->type = pt_load_entry_raw(pts); 386 if (pts->type == PT_ENTRY_TABLE) 387 pts->table_lower = pt_table_ptr(pts); 388 } 389 #endif 390