1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #ifndef _XE_RTP_H_ 7 #define _XE_RTP_H_ 8 9 #include <linux/types.h> 10 #include <linux/xarray.h> 11 12 #define _XE_RTP_INCLUDE_PRIVATE_HELPERS 13 14 #include "xe_rtp_helpers.h" 15 #include "xe_rtp_types.h" 16 17 #undef _XE_RTP_INCLUDE_PRIVATE_HELPERS 18 19 /* 20 * Register table poke infrastructure 21 */ 22 23 struct xe_hw_engine; 24 struct xe_gt; 25 struct xe_reg_sr; 26 27 /* 28 * Macros to encode rules to match against platform, IP version, stepping, etc. 29 * Shouldn't be used directly - see XE_RTP_RULES() 30 */ 31 #define _XE_RTP_RULE_PLATFORM(plat__) \ 32 { .match_type = XE_RTP_MATCH_PLATFORM, .platform = plat__ } 33 34 #define _XE_RTP_RULE_SUBPLATFORM(plat__, sub__) \ 35 { .match_type = XE_RTP_MATCH_SUBPLATFORM, \ 36 .platform = plat__, .subplatform = sub__ } 37 38 #define _XE_RTP_RULE_PLATFORM_STEP(start__, end__) \ 39 { .match_type = XE_RTP_MATCH_PLATFORM_STEP, \ 40 .step_start = start__, .step_end = end__ } 41 42 #define _XE_RTP_RULE_GRAPHICS_STEP(start__, end__) \ 43 { .match_type = XE_RTP_MATCH_GRAPHICS_STEP, \ 44 .step_start = start__, .step_end = end__ } 45 46 #define _XE_RTP_RULE_MEDIA_STEP(start__, end__) \ 47 { .match_type = XE_RTP_MATCH_MEDIA_STEP, \ 48 .step_start = start__, .step_end = end__ } 49 50 #define _XE_RTP_RULE_ENGINE_CLASS(cls__) \ 51 { .match_type = XE_RTP_MATCH_ENGINE_CLASS, \ 52 .engine_class = (cls__) } 53 54 /** 55 * XE_RTP_RULE_PLATFORM - Create rule matching platform 56 * @plat_: platform to match 57 * 58 * Refer to XE_RTP_RULES() for expected usage. 59 */ 60 #define XE_RTP_RULE_PLATFORM(plat_) \ 61 _XE_RTP_RULE_PLATFORM(XE_##plat_) 62 63 /** 64 * XE_RTP_RULE_SUBPLATFORM - Create rule matching platform and sub-platform 65 * @plat_: platform to match 66 * @sub_: sub-platform to match 67 * 68 * Refer to XE_RTP_RULES() for expected usage. 69 */ 70 #define XE_RTP_RULE_SUBPLATFORM(plat_, sub_) \ 71 _XE_RTP_RULE_SUBPLATFORM(XE_##plat_, XE_SUBPLATFORM_##plat_##_##sub_) 72 73 /** 74 * XE_RTP_RULE_PLATFORM_STEP - Create rule matching platform-level stepping 75 * @start_: First stepping matching the rule 76 * @end_: First stepping that does not match the rule 77 * 78 * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive 79 * on the left, exclusive on the right. 80 * 81 * You need to make sure that proper support for reading platform-level stepping 82 * information is present for the target platform before using this rule. 83 * 84 * Refer to XE_RTP_RULES() for expected usage. 85 */ 86 #define XE_RTP_RULE_PLATFORM_STEP(start_, end_) \ 87 _XE_RTP_RULE_PLATFORM_STEP(STEP_##start_, STEP_##end_) 88 89 /** 90 * XE_RTP_RULE_GRAPHICS_STEP - Create rule matching graphics stepping 91 * @start_: First stepping matching the rule 92 * @end_: First stepping that does not match the rule 93 * 94 * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive 95 * on the left, exclusive on the right. 96 * 97 * Refer to XE_RTP_RULES() for expected usage. 98 */ 99 #define XE_RTP_RULE_GRAPHICS_STEP(start_, end_) \ 100 _XE_RTP_RULE_GRAPHICS_STEP(STEP_##start_, STEP_##end_) 101 102 /** 103 * XE_RTP_RULE_MEDIA_STEP - Create rule matching media stepping 104 * @start_: First stepping matching the rule 105 * @end_: First stepping that does not match the rule 106 * 107 * Note that the range matching this rule is [ @start_, @end_ ), i.e. inclusive 108 * on the left, exclusive on the right. 109 * 110 * Refer to XE_RTP_RULES() for expected usage. 111 */ 112 #define XE_RTP_RULE_MEDIA_STEP(start_, end_) \ 113 _XE_RTP_RULE_MEDIA_STEP(STEP_##start_, STEP_##end_) 114 115 /** 116 * XE_RTP_RULE_ENGINE_CLASS - Create rule matching an engine class 117 * @cls_: Engine class to match 118 * 119 * Refer to XE_RTP_RULES() for expected usage. 120 */ 121 #define XE_RTP_RULE_ENGINE_CLASS(cls_) \ 122 _XE_RTP_RULE_ENGINE_CLASS(XE_ENGINE_CLASS_##cls_) 123 124 /** 125 * XE_RTP_RULE_FUNC - Create rule using callback function for match 126 * @func__: Function to call to decide if rule matches 127 * 128 * This allows more complex checks to be performed. The ``XE_RTP`` 129 * infrastructure will simply call the function @func_ passed to decide if this 130 * rule matches the device. 131 * 132 * Refer to XE_RTP_RULES() for expected usage. 133 */ 134 #define XE_RTP_RULE_FUNC(func__) \ 135 { .match_type = XE_RTP_MATCH_FUNC, \ 136 .match_func = (func__) } 137 138 /** 139 * XE_RTP_RULE_GRAPHICS_VERSION - Create rule matching graphics version 140 * @ver__: Graphics IP version to match 141 * 142 * Refer to XE_RTP_RULES() for expected usage. 143 */ 144 #define XE_RTP_RULE_GRAPHICS_VERSION(ver__) \ 145 { .match_type = XE_RTP_MATCH_GRAPHICS_VERSION, \ 146 .ver_start = ver__, } 147 148 /** 149 * XE_RTP_RULE_GRAPHICS_VERSION_RANGE - Create rule matching a range of graphics version 150 * @ver_start__: First graphics IP version to match 151 * @ver_end__: Last graphics IP version to match 152 * 153 * Note that the range matching this rule is [ @ver_start__, @ver_end__ ], i.e. 154 * inclusive on both sides 155 * 156 * Refer to XE_RTP_RULES() for expected usage. 157 */ 158 #define XE_RTP_RULE_GRAPHICS_VERSION_RANGE(ver_start__, ver_end__) \ 159 { .match_type = XE_RTP_MATCH_GRAPHICS_VERSION_RANGE, \ 160 .ver_start = ver_start__, .ver_end = ver_end__, } 161 162 /** 163 * XE_RTP_RULE_GRAPHICS_VERSION_ANY_GT - Create rule matching graphics version on any GT 164 * @ver__: Graphics IP version to match 165 * 166 * Like XE_RTP_RULE_GRAPHICS_VERSION, but it matches even if the current GT 167 * being checked is not of the graphics type. It allows to add RTP entries to 168 * another GT when the device contains a Graphics IP with that version. 169 * 170 * Refer to XE_RTP_RULES() for expected usage. 171 */ 172 #define XE_RTP_RULE_GRAPHICS_VERSION_ANY_GT(ver__) \ 173 { .match_type = XE_RTP_MATCH_GRAPHICS_VERSION_ANY_GT, \ 174 .ver_start = ver__, } 175 176 /** 177 * XE_RTP_RULE_MEDIA_VERSION - Create rule matching media version 178 * @ver__: Media IP version to match 179 * 180 * Refer to XE_RTP_RULES() for expected usage. 181 */ 182 #define XE_RTP_RULE_MEDIA_VERSION(ver__) \ 183 { .match_type = XE_RTP_MATCH_MEDIA_VERSION, \ 184 .ver_start = ver__, } 185 186 /** 187 * XE_RTP_RULE_MEDIA_VERSION_RANGE - Create rule matching a range of media version 188 * @ver_start__: First media IP version to match 189 * @ver_end__: Last media IP version to match 190 * 191 * Note that the range matching this rule is [ @ver_start__, @ver_end__ ], i.e. 192 * inclusive on both sides 193 * 194 * Refer to XE_RTP_RULES() for expected usage. 195 */ 196 #define XE_RTP_RULE_MEDIA_VERSION_RANGE(ver_start__, ver_end__) \ 197 { .match_type = XE_RTP_MATCH_MEDIA_VERSION_RANGE, \ 198 .ver_start = ver_start__, .ver_end = ver_end__, } 199 200 /** 201 * XE_RTP_RULE_MEDIA_VERSION_ANY_GT - Create rule matching media version on any GT 202 * @ver__: Media IP version to match 203 * 204 * Like XE_RTP_RULE_MEDIA_VERSION, but it matches even if the current GT being 205 * checked is not of the media type. It allows to add RTP entries to another 206 * GT when the device contains a Media IP with that version. 207 * 208 * Refer to XE_RTP_RULES() for expected usage. 209 */ 210 #define XE_RTP_RULE_MEDIA_VERSION_ANY_GT(ver__) \ 211 { .match_type = XE_RTP_MATCH_MEDIA_VERSION_ANY_GT, \ 212 .ver_start = ver__, } 213 214 /** 215 * XE_RTP_RULE_IS_INTEGRATED - Create a rule matching integrated graphics devices 216 * 217 * Refer to XE_RTP_RULES() for expected usage. 218 */ 219 #define XE_RTP_RULE_IS_INTEGRATED \ 220 { .match_type = XE_RTP_MATCH_INTEGRATED } 221 222 /** 223 * XE_RTP_RULE_IS_DISCRETE - Create a rule matching discrete graphics devices 224 * 225 * Refer to XE_RTP_RULES() for expected usage. 226 */ 227 #define XE_RTP_RULE_IS_DISCRETE \ 228 { .match_type = XE_RTP_MATCH_DISCRETE } 229 230 /** 231 * XE_RTP_RULE_OR - Create an OR condition for rtp rules 232 * 233 * RTP rules are AND'ed when evaluated and all of them need to match. 234 * XE_RTP_RULE_OR allows to create set of rules where any of them matching is 235 * sufficient for the action to trigger. Example: 236 * 237 * .. code-block:: c 238 * 239 * const struct xe_rtp_entry_sr entries[] = { 240 * ... 241 * { XE_RTP_NAME("test-entry"), 242 * XE_RTP_RULES(PLATFORM(DG2), OR, PLATFORM(TIGERLAKE)), 243 * ... 244 * }, 245 * ... 246 * }; 247 */ 248 #define XE_RTP_RULE_OR \ 249 { .match_type = XE_RTP_MATCH_OR } 250 251 /** 252 * XE_RTP_ACTION_WR - Helper to write a value to the register, overriding all 253 * the bits 254 * @reg_: Register 255 * @val_: Value to set 256 * @...: Additional fields to override in the struct xe_rtp_action entry 257 * 258 * The correspondent notation in bspec is: 259 * 260 * REGNAME = VALUE 261 */ 262 #define XE_RTP_ACTION_WR(reg_, val_, ...) \ 263 { .reg = XE_RTP_DROP_CAST(reg_), \ 264 .clr_bits = ~0u, .set_bits = (val_), \ 265 .read_mask = (~0u), ##__VA_ARGS__ } 266 267 /** 268 * XE_RTP_ACTION_SET - Set bits from @val_ in the register. 269 * @reg_: Register 270 * @val_: Bits to set in the register 271 * @...: Additional fields to override in the struct xe_rtp_action entry 272 * 273 * For masked registers this translates to a single write, while for other 274 * registers it's a RMW. The correspondent bspec notation is (example for bits 2 275 * and 5, but could be any): 276 * 277 * REGNAME[2] = 1 278 * REGNAME[5] = 1 279 */ 280 #define XE_RTP_ACTION_SET(reg_, val_, ...) \ 281 { .reg = XE_RTP_DROP_CAST(reg_), \ 282 .clr_bits = val_, .set_bits = val_, \ 283 .read_mask = val_, ##__VA_ARGS__ } 284 285 /** 286 * XE_RTP_ACTION_CLR: Clear bits from @val_ in the register. 287 * @reg_: Register 288 * @val_: Bits to clear in the register 289 * @...: Additional fields to override in the struct xe_rtp_action entry 290 * 291 * For masked registers this translates to a single write, while for other 292 * registers it's a RMW. The correspondent bspec notation is (example for bits 2 293 * and 5, but could be any): 294 * 295 * REGNAME[2] = 0 296 * REGNAME[5] = 0 297 */ 298 #define XE_RTP_ACTION_CLR(reg_, val_, ...) \ 299 { .reg = XE_RTP_DROP_CAST(reg_), \ 300 .clr_bits = val_, .set_bits = 0, \ 301 .read_mask = val_, ##__VA_ARGS__ } 302 303 /** 304 * XE_RTP_ACTION_FIELD_SET: Set a bit range 305 * @reg_: Register 306 * @mask_bits_: Mask of bits to be changed in the register, forming a field 307 * @val_: Value to set in the field denoted by @mask_bits_ 308 * @...: Additional fields to override in the struct xe_rtp_action entry 309 * 310 * For masked registers this translates to a single write, while for other 311 * registers it's a RMW. The correspondent bspec notation is: 312 * 313 * REGNAME[<end>:<start>] = VALUE 314 */ 315 #define XE_RTP_ACTION_FIELD_SET(reg_, mask_bits_, val_, ...) \ 316 { .reg = XE_RTP_DROP_CAST(reg_), \ 317 .clr_bits = mask_bits_, .set_bits = val_, \ 318 .read_mask = mask_bits_, ##__VA_ARGS__ } 319 320 #define XE_RTP_ACTION_FIELD_SET_NO_READ_MASK(reg_, mask_bits_, val_, ...) \ 321 { .reg = XE_RTP_DROP_CAST(reg_), \ 322 .clr_bits = (mask_bits_), .set_bits = (val_), \ 323 .read_mask = 0, ##__VA_ARGS__ } 324 325 /** 326 * XE_RTP_ACTION_WHITELIST - Add register to userspace whitelist 327 * @reg_: Register 328 * @val_: Whitelist-specific flags to set 329 * @...: Additional fields to override in the struct xe_rtp_action entry 330 * 331 * Add a register to the whitelist, allowing userspace to modify the ster with 332 * regular user privileges. 333 */ 334 #define XE_RTP_ACTION_WHITELIST(reg_, val_, ...) \ 335 /* TODO fail build if ((flags) & ~(RING_FORCE_TO_NONPRIV_MASK_VALID)) */\ 336 { .reg = XE_RTP_DROP_CAST(reg_), \ 337 .set_bits = val_, \ 338 .clr_bits = RING_FORCE_TO_NONPRIV_MASK_VALID, \ 339 ##__VA_ARGS__ } 340 341 /** 342 * XE_RTP_NAME - Helper to set the name in xe_rtp_entry 343 * @s_: Name describing this rule, often a HW-specific number 344 * 345 * TODO: maybe move this behind a debug config? 346 */ 347 #define XE_RTP_NAME(s_) .name = (s_) 348 349 /** 350 * XE_RTP_ENTRY_FLAG - Helper to add multiple flags to a struct xe_rtp_entry_sr 351 * @...: Entry flags, without the ``XE_RTP_ENTRY_FLAG_`` prefix 352 * 353 * Helper to automatically add a ``XE_RTP_ENTRY_FLAG_`` prefix to the flags 354 * when defining struct xe_rtp_entry entries. Example: 355 * 356 * .. code-block:: c 357 * 358 * const struct xe_rtp_entry_sr wa_entries[] = { 359 * ... 360 * { XE_RTP_NAME("test-entry"), 361 * ... 362 * XE_RTP_ENTRY_FLAG(FOREACH_ENGINE), 363 * ... 364 * }, 365 * ... 366 * }; 367 */ 368 #define XE_RTP_ENTRY_FLAG(...) \ 369 .flags = (XE_RTP_PASTE_FOREACH(ENTRY_FLAG_, BITWISE_OR, (__VA_ARGS__))) 370 371 /** 372 * XE_RTP_ACTION_FLAG - Helper to add multiple flags to a struct xe_rtp_action 373 * @...: Action flags, without the ``XE_RTP_ACTION_FLAG_`` prefix 374 * 375 * Helper to automatically add a ``XE_RTP_ACTION_FLAG_`` prefix to the flags 376 * when defining struct xe_rtp_action entries. Example: 377 * 378 * .. code-block:: c 379 * 380 * const struct xe_rtp_entry_sr wa_entries[] = { 381 * ... 382 * { XE_RTP_NAME("test-entry"), 383 * ... 384 * XE_RTP_ACTION_SET(..., XE_RTP_ACTION_FLAG(FOREACH_ENGINE)), 385 * ... 386 * }, 387 * ... 388 * }; 389 */ 390 #define XE_RTP_ACTION_FLAG(...) \ 391 .flags = (XE_RTP_PASTE_FOREACH(ACTION_FLAG_, BITWISE_OR, (__VA_ARGS__))) 392 393 /** 394 * XE_RTP_RULES - Helper to set multiple rules to a struct xe_rtp_entry_sr entry 395 * @...: Rules 396 * 397 * When an RTP table is being processed, the rules of each entry are evaluated 398 * to check if they match the target entity (platform, gt or hwe, depending on 399 * the specific RTP table). 400 * 401 * The sequence of arguments of this macro must follow the following eBNF 402 * grammar:: 403 * 404 * rules = disjunction; 405 * disjunction = conjunction, { "OR", conjunction }; 406 * conjunction = single_rule, { single_rule }; 407 * (* the AND operator is implicit *) 408 * single_rule = ? GRAPHICS_VERSION(...), MEDIA_VERSION(...), 409 * FUNC(...), etc ? 410 * 411 * Examples: 412 * 413 * .. code-block:: c 414 * 415 * const struct xe_rtp_entry_sr wa_entries[] = { 416 * ... 417 * { XE_RTP_NAME("entry-a"), 418 * // Match DG2-G10 with graphics steppings A0 up-to B0 419 * // (exclusive). 420 * XE_RTP_RULES(SUBPLATFORM(DG2, G10), GRAPHICS_STEP(A0, B0)), 421 * ... 422 * }, 423 * { XE_RTP_NAME("entry-b"), 424 * // Match graphics version 20 (all steppings) or graphics 425 * // version 30 steppings A0 up-to B0 (exclusive). 426 * XE_RTP_RULES(GRAPHICS_VERSION(2000), OR, 427 * GRAPHICS_VERSION(3000), GRAPHICS_STEP(A0, B0)) 428 * ... 429 * }, 430 * ... 431 * }; 432 */ 433 #define XE_RTP_RULES(...) \ 434 .n_rules = COUNT_ARGS(__VA_ARGS__), \ 435 .rules = (const struct xe_rtp_rule[]) { \ 436 XE_RTP_PASTE_FOREACH(RULE_, COMMA, (__VA_ARGS__)) \ 437 } 438 439 /** 440 * XE_RTP_ACTIONS - Helper to set multiple actions to a struct xe_rtp_entry_sr 441 * @...: Actions to be taken 442 * 443 * At least one action is needed and up to 12 are supported. See XE_RTP_ACTION_* 444 * for the possible actions. Example: 445 * 446 * .. code-block:: c 447 * 448 * const struct xe_rtp_entry_sr wa_entries[] = { 449 * ... 450 * { XE_RTP_NAME("test-entry"), 451 * XE_RTP_RULES(...), 452 * XE_RTP_ACTIONS(SET(..), SET(...), CLR(...)), 453 * ... 454 * }, 455 * ... 456 * }; 457 */ 458 #define XE_RTP_ACTIONS(...) \ 459 .n_actions = COUNT_ARGS(__VA_ARGS__), \ 460 .actions = (const struct xe_rtp_action[]) { \ 461 XE_RTP_PASTE_FOREACH(ACTION_, COMMA, (__VA_ARGS__)) \ 462 } 463 464 #define XE_RTP_PROCESS_CTX_INITIALIZER(arg__) _Generic((arg__), \ 465 struct xe_hw_engine * : (struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_ENGINE }, \ 466 struct xe_gt * : (struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_GT }, \ 467 struct xe_device * : (struct xe_rtp_process_ctx){ { (void *)(arg__) }, XE_RTP_PROCESS_TYPE_DEVICE }) 468 469 void xe_rtp_process_ctx_enable_active_tracking(struct xe_rtp_process_ctx *ctx, 470 unsigned long *active_entries, 471 size_t n_entries); 472 473 void xe_rtp_process_to_sr(struct xe_rtp_process_ctx *ctx, 474 const struct xe_rtp_entry_sr *entries, 475 size_t n_entries, struct xe_reg_sr *sr, 476 bool process_in_vf); 477 478 void xe_rtp_process(struct xe_rtp_process_ctx *ctx, 479 const struct xe_rtp_entry *entries); 480 481 /* Match functions to be used with XE_RTP_MATCH_FUNC */ 482 483 /** 484 * xe_rtp_match_always - Match RTP entry unconditionally 485 * @xe: Device structure 486 * @gt: GT structure 487 * @hwe: Engine instance 488 * 489 * Returns: true, regardless of inputs 490 */ 491 bool xe_rtp_match_always(const struct xe_device *xe, 492 const struct xe_gt *gt, 493 const struct xe_hw_engine *hwe); 494 495 /** 496 * xe_rtp_match_even_instance - Match if engine instance is even 497 * @xe: Device structure 498 * @gt: GT structure 499 * @hwe: Engine instance 500 * 501 * Returns: true if engine instance is even, false otherwise 502 */ 503 bool xe_rtp_match_even_instance(const struct xe_device *xe, 504 const struct xe_gt *gt, 505 const struct xe_hw_engine *hwe); 506 507 /* 508 * xe_rtp_match_first_render_or_compute - Match if it's first render or compute 509 * engine in the GT 510 * 511 * @xe: Device structure 512 * @gt: GT structure 513 * @hwe: Engine instance 514 * 515 * Registers on the render reset domain need to have their values re-applied 516 * when any of those engines are reset. Since the engines reset together, a 517 * programming can be set to just one of them. For simplicity the first engine 518 * of either render or compute class can be chosen. 519 * 520 * Returns: true if engine id is the first to match the render reset domain, 521 * false otherwise. 522 */ 523 bool xe_rtp_match_first_render_or_compute(const struct xe_device *xe, 524 const struct xe_gt *gt, 525 const struct xe_hw_engine *hwe); 526 527 /* 528 * xe_rtp_match_not_sriov_vf - Match when not on SR-IOV VF device 529 * 530 * @xe: Device structure 531 * @gt: GT structure 532 * @hwe: Engine instance 533 * 534 * Returns: true if device is not VF, false otherwise. 535 */ 536 bool xe_rtp_match_not_sriov_vf(const struct xe_device *xe, 537 const struct xe_gt *gt, 538 const struct xe_hw_engine *hwe); 539 540 bool xe_rtp_match_psmi_enabled(const struct xe_device *xe, 541 const struct xe_gt *gt, 542 const struct xe_hw_engine *hwe); 543 544 bool xe_rtp_match_gt_has_discontiguous_dss_groups(const struct xe_device *xe, 545 const struct xe_gt *gt, 546 const struct xe_hw_engine *hwe); 547 548 /** 549 * xe_rtp_match_has_flat_ccs - Match when platform has FlatCCS compression 550 * @xe: Device structure 551 * @gt: GT structure 552 * @hwe: Engine instance 553 * 554 * Returns: true if platform has FlatCCS compression, false otherwise 555 */ 556 bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe, 557 const struct xe_gt *gt, 558 const struct xe_hw_engine *hwe); 559 560 /** 561 * xe_rtp_match_has_msix - Match when platform has MSI-X 562 * @xe: Device structure 563 * @gt: GT structure 564 * @hwe: Engine instance 565 * 566 * Returns: true if platform has MSI-X interrupt support 567 */ 568 bool xe_rtp_match_has_msix(const struct xe_device *xe, 569 const struct xe_gt *gt, 570 const struct xe_hw_engine *hwe); 571 572 #endif 573