1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES 4 * 5 * Default definitions for formats that don't define these functions. 6 */ 7 #ifndef __GENERIC_PT_PT_FMT_DEFAULTS_H 8 #define __GENERIC_PT_PT_FMT_DEFAULTS_H 9 10 #include "pt_defs.h" 11 #include <linux/log2.h> 12 13 /* Header self-compile default defines */ 14 #ifndef pt_load_entry_raw 15 #include "fmt/amdv1.h" 16 #endif 17 18 /* 19 * The format must provide PT_GRANULE_LG2SZ, PT_TABLEMEM_LG2SZ, and 20 * PT_ITEM_WORD_SIZE. They must be the same at every level excluding the top. 21 */ 22 #ifndef pt_table_item_lg2sz 23 static inline unsigned int pt_table_item_lg2sz(const struct pt_state *pts) 24 { 25 return PT_GRANULE_LG2SZ + 26 (PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE)) * pts->level; 27 } 28 #endif 29 30 #ifndef pt_pgsz_lg2_to_level 31 static inline unsigned int pt_pgsz_lg2_to_level(struct pt_common *common, 32 unsigned int pgsize_lg2) 33 { 34 return ((unsigned int)(pgsize_lg2 - PT_GRANULE_LG2SZ)) / 35 (PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE)); 36 } 37 #endif 38 39 /* 40 * If not supplied by the format then contiguous pages are not supported. 41 * 42 * If contiguous pages are supported then the format must also provide 43 * pt_contig_count_lg2() if it supports a single contiguous size per level, 44 * or pt_possible_sizes() if it supports multiple sizes per level. 45 */ 46 #ifndef pt_entry_num_contig_lg2 47 static inline unsigned int pt_entry_num_contig_lg2(const struct pt_state *pts) 48 { 49 return ilog2(1); 50 } 51 52 /* 53 * Return the number of contiguous OA items forming an entry at this table level 54 */ 55 static inline unsigned short pt_contig_count_lg2(const struct pt_state *pts) 56 { 57 return ilog2(1); 58 } 59 #endif 60 61 /* If not supplied by the format then dirty tracking is not supported */ 62 #ifndef pt_entry_is_write_dirty 63 static inline bool pt_entry_is_write_dirty(const struct pt_state *pts) 64 { 65 return false; 66 } 67 68 static inline void pt_entry_make_write_clean(struct pt_state *pts) 69 { 70 } 71 72 static inline bool pt_dirty_supported(struct pt_common *common) 73 { 74 return false; 75 } 76 #else 77 /* If not supplied then dirty tracking is always enabled */ 78 #ifndef pt_dirty_supported 79 static inline bool pt_dirty_supported(struct pt_common *common) 80 { 81 return true; 82 } 83 #endif 84 #endif 85 86 #ifndef pt_entry_make_write_dirty 87 static inline bool pt_entry_make_write_dirty(struct pt_state *pts) 88 { 89 return false; 90 } 91 #endif 92 93 /* 94 * Format supplies either: 95 * pt_entry_oa - OA is at the start of a contiguous entry 96 * or 97 * pt_item_oa - OA is adjusted for every item in a contiguous entry 98 * 99 * Build the missing one 100 * 101 * The internal helper _pt_entry_oa_fast() allows generating 102 * an efficient pt_entry_oa_exact(), it doesn't care which 103 * option is selected. 104 */ 105 #ifdef pt_entry_oa 106 static inline pt_oaddr_t pt_item_oa(const struct pt_state *pts) 107 { 108 return pt_entry_oa(pts) | 109 log2_mul(pts->index, pt_table_item_lg2sz(pts)); 110 } 111 #define _pt_entry_oa_fast pt_entry_oa 112 #endif 113 114 #ifdef pt_item_oa 115 static inline pt_oaddr_t pt_entry_oa(const struct pt_state *pts) 116 { 117 return log2_set_mod(pt_item_oa(pts), 0, 118 pt_entry_num_contig_lg2(pts) + 119 pt_table_item_lg2sz(pts)); 120 } 121 #define _pt_entry_oa_fast pt_item_oa 122 #endif 123 124 /* 125 * If not supplied by the format then use the constant 126 * PT_MAX_OUTPUT_ADDRESS_LG2. 127 */ 128 #ifndef pt_max_oa_lg2 129 static inline unsigned int 130 pt_max_oa_lg2(const struct pt_common *common) 131 { 132 return PT_MAX_OUTPUT_ADDRESS_LG2; 133 } 134 #endif 135 136 #ifndef pt_has_system_page_size 137 static inline bool pt_has_system_page_size(const struct pt_common *common) 138 { 139 return PT_GRANULE_LG2SZ == PAGE_SHIFT; 140 } 141 #endif 142 143 /* 144 * If not supplied by the format then assume only one contiguous size determined 145 * by pt_contig_count_lg2() 146 */ 147 #ifndef pt_possible_sizes 148 static inline unsigned short pt_contig_count_lg2(const struct pt_state *pts); 149 150 /* Return a bitmap of possible leaf page sizes at this level */ 151 static inline pt_vaddr_t pt_possible_sizes(const struct pt_state *pts) 152 { 153 unsigned int isz_lg2 = pt_table_item_lg2sz(pts); 154 155 if (!pt_can_have_leaf(pts)) 156 return 0; 157 return log2_to_int(isz_lg2) | 158 log2_to_int(pt_contig_count_lg2(pts) + isz_lg2); 159 } 160 #endif 161 162 /* If not supplied by the format then use 0. */ 163 #ifndef pt_full_va_prefix 164 static inline pt_vaddr_t pt_full_va_prefix(const struct pt_common *common) 165 { 166 return 0; 167 } 168 #endif 169 170 /* If not supplied by the format then zero fill using PT_ITEM_WORD_SIZE */ 171 #ifndef pt_clear_entries 172 static inline void pt_clear_entries64(struct pt_state *pts, 173 unsigned int num_contig_lg2) 174 { 175 u64 *tablep = pt_cur_table(pts, u64) + pts->index; 176 u64 *end = tablep + log2_to_int(num_contig_lg2); 177 178 PT_WARN_ON(log2_mod(pts->index, num_contig_lg2)); 179 for (; tablep != end; tablep++) 180 WRITE_ONCE(*tablep, 0); 181 } 182 183 static inline void pt_clear_entries32(struct pt_state *pts, 184 unsigned int num_contig_lg2) 185 { 186 u32 *tablep = pt_cur_table(pts, u32) + pts->index; 187 u32 *end = tablep + log2_to_int(num_contig_lg2); 188 189 PT_WARN_ON(log2_mod(pts->index, num_contig_lg2)); 190 for (; tablep != end; tablep++) 191 WRITE_ONCE(*tablep, 0); 192 } 193 194 static inline void pt_clear_entries(struct pt_state *pts, 195 unsigned int num_contig_lg2) 196 { 197 if (PT_ITEM_WORD_SIZE == sizeof(u32)) 198 pt_clear_entries32(pts, num_contig_lg2); 199 else 200 pt_clear_entries64(pts, num_contig_lg2); 201 } 202 #define pt_clear_entries pt_clear_entries 203 #endif 204 205 /* If not supplied then SW bits are not supported */ 206 #ifdef pt_sw_bit 207 static inline bool pt_test_sw_bit_acquire(struct pt_state *pts, 208 unsigned int bitnr) 209 { 210 /* Acquire, pairs with pt_set_sw_bit_release() */ 211 smp_mb(); 212 /* For a contiguous entry the sw bit is only stored in the first item. */ 213 return pts->entry & pt_sw_bit(bitnr); 214 } 215 #define pt_test_sw_bit_acquire pt_test_sw_bit_acquire 216 217 static inline void pt_set_sw_bit_release(struct pt_state *pts, 218 unsigned int bitnr) 219 { 220 #if !IS_ENABLED(CONFIG_GENERIC_ATOMIC64) 221 if (PT_ITEM_WORD_SIZE == sizeof(u64)) { 222 u64 *entryp = pt_cur_table(pts, u64) + pts->index; 223 u64 old_entry = pts->entry; 224 u64 new_entry; 225 226 do { 227 new_entry = old_entry | pt_sw_bit(bitnr); 228 } while (!try_cmpxchg64_release(entryp, &old_entry, new_entry)); 229 pts->entry = new_entry; 230 return; 231 } 232 #endif 233 if (PT_ITEM_WORD_SIZE == sizeof(u32)) { 234 u32 *entryp = pt_cur_table(pts, u32) + pts->index; 235 u32 old_entry = pts->entry; 236 u32 new_entry; 237 238 do { 239 new_entry = old_entry | pt_sw_bit(bitnr); 240 } while (!try_cmpxchg_release(entryp, &old_entry, new_entry)); 241 pts->entry = new_entry; 242 } else 243 BUILD_BUG(); 244 } 245 #define pt_set_sw_bit_release pt_set_sw_bit_release 246 #else 247 static inline unsigned int pt_max_sw_bit(struct pt_common *common) 248 { 249 return 0; 250 } 251 252 extern void __pt_no_sw_bit(void); 253 static inline bool pt_test_sw_bit_acquire(struct pt_state *pts, 254 unsigned int bitnr) 255 { 256 __pt_no_sw_bit(); 257 return false; 258 } 259 260 static inline void pt_set_sw_bit_release(struct pt_state *pts, 261 unsigned int bitnr) 262 { 263 __pt_no_sw_bit(); 264 } 265 #endif 266 267 /* 268 * Format can call in the pt_install_leaf_entry() to check the arguments are all 269 * aligned correctly. 270 */ 271 static inline bool pt_check_install_leaf_args(struct pt_state *pts, 272 pt_oaddr_t oa, 273 unsigned int oasz_lg2) 274 { 275 unsigned int isz_lg2 = pt_table_item_lg2sz(pts); 276 277 if (PT_WARN_ON(oalog2_mod(oa, oasz_lg2))) 278 return false; 279 280 #ifdef pt_possible_sizes 281 if (PT_WARN_ON(isz_lg2 > oasz_lg2 || 282 oasz_lg2 > isz_lg2 + pt_num_items_lg2(pts))) 283 return false; 284 #else 285 if (PT_WARN_ON(oasz_lg2 != isz_lg2 && 286 oasz_lg2 != isz_lg2 + pt_contig_count_lg2(pts))) 287 return false; 288 #endif 289 290 if (PT_WARN_ON(oalog2_mod(pts->index, oasz_lg2 - isz_lg2))) 291 return false; 292 return true; 293 } 294 295 #endif 296