1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * DAMON api 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8 #ifndef _DAMON_H_ 9 #define _DAMON_H_ 10 11 #include <linux/memcontrol.h> 12 #include <linux/mutex.h> 13 #include <linux/time64.h> 14 #include <linux/types.h> 15 #include <linux/random.h> 16 17 /* Minimal region size. Every damon_region is aligned by this. */ 18 #define DAMON_MIN_REGION PAGE_SIZE 19 /* Max priority score for DAMON-based operation schemes */ 20 #define DAMOS_MAX_SCORE (99) 21 22 /* Get a random number in [l, r) */ 23 static inline unsigned long damon_rand(unsigned long l, unsigned long r) 24 { 25 return l + get_random_u32_below(r - l); 26 } 27 28 /** 29 * struct damon_addr_range - Represents an address region of [@start, @end). 30 * @start: Start address of the region (inclusive). 31 * @end: End address of the region (exclusive). 32 */ 33 struct damon_addr_range { 34 unsigned long start; 35 unsigned long end; 36 }; 37 38 /** 39 * struct damon_region - Represents a monitoring target region. 40 * @ar: The address range of the region. 41 * @sampling_addr: Address of the sample for the next access check. 42 * @nr_accesses: Access frequency of this region. 43 * @nr_accesses_bp: @nr_accesses in basis point (0.01%) that updated for 44 * each sampling interval. 45 * @list: List head for siblings. 46 * @age: Age of this region. 47 * 48 * @nr_accesses is reset to zero for every &damon_attrs->aggr_interval and be 49 * increased for every &damon_attrs->sample_interval if an access to the region 50 * during the last sampling interval is found. The update of this field should 51 * not be done with direct access but with the helper function, 52 * damon_update_region_access_rate(). 53 * 54 * @nr_accesses_bp is another representation of @nr_accesses in basis point 55 * (1 in 10,000) that updated for every &damon_attrs->sample_interval in a 56 * manner similar to moving sum. By the algorithm, this value becomes 57 * @nr_accesses * 10000 for every &struct damon_attrs->aggr_interval. This can 58 * be used when the aggregation interval is too huge and therefore cannot wait 59 * for it before getting the access monitoring results. 60 * 61 * @age is initially zero, increased for each aggregation interval, and reset 62 * to zero again if the access frequency is significantly changed. If two 63 * regions are merged into a new region, both @nr_accesses and @age of the new 64 * region are set as region size-weighted average of those of the two regions. 65 */ 66 struct damon_region { 67 struct damon_addr_range ar; 68 unsigned long sampling_addr; 69 unsigned int nr_accesses; 70 unsigned int nr_accesses_bp; 71 struct list_head list; 72 73 unsigned int age; 74 /* private: Internal value for age calculation. */ 75 unsigned int last_nr_accesses; 76 }; 77 78 /** 79 * struct damon_target - Represents a monitoring target. 80 * @pid: The PID of the virtual address space to monitor. 81 * @nr_regions: Number of monitoring target regions of this target. 82 * @regions_list: Head of the monitoring target regions of this target. 83 * @list: List head for siblings. 84 * 85 * Each monitoring context could have multiple targets. For example, a context 86 * for virtual memory address spaces could have multiple target processes. The 87 * @pid should be set for appropriate &struct damon_operations including the 88 * virtual address spaces monitoring operations. 89 */ 90 struct damon_target { 91 struct pid *pid; 92 unsigned int nr_regions; 93 struct list_head regions_list; 94 struct list_head list; 95 }; 96 97 /** 98 * enum damos_action - Represents an action of a Data Access Monitoring-based 99 * Operation Scheme. 100 * 101 * @DAMOS_WILLNEED: Call ``madvise()`` for the region with MADV_WILLNEED. 102 * @DAMOS_COLD: Call ``madvise()`` for the region with MADV_COLD. 103 * @DAMOS_PAGEOUT: Call ``madvise()`` for the region with MADV_PAGEOUT. 104 * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. 105 * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. 106 * @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists. 107 * @DAMOS_LRU_DEPRIO: Deprioritize the region on its LRU lists. 108 * @DAMOS_MIGRATE_HOT: Migrate the regions prioritizing warmer regions. 109 * @DAMOS_MIGRATE_COLD: Migrate the regions prioritizing colder regions. 110 * @DAMOS_STAT: Do nothing but count the stat. 111 * @NR_DAMOS_ACTIONS: Total number of DAMOS actions 112 * 113 * The support of each action is up to running &struct damon_operations. 114 * &enum DAMON_OPS_VADDR and &enum DAMON_OPS_FVADDR supports all actions except 115 * &enum DAMOS_LRU_PRIO and &enum DAMOS_LRU_DEPRIO. &enum DAMON_OPS_PADDR 116 * supports only &enum DAMOS_PAGEOUT, &enum DAMOS_LRU_PRIO, &enum 117 * DAMOS_LRU_DEPRIO, and &DAMOS_STAT. 118 */ 119 enum damos_action { 120 DAMOS_WILLNEED, 121 DAMOS_COLD, 122 DAMOS_PAGEOUT, 123 DAMOS_HUGEPAGE, 124 DAMOS_NOHUGEPAGE, 125 DAMOS_LRU_PRIO, 126 DAMOS_LRU_DEPRIO, 127 DAMOS_MIGRATE_HOT, 128 DAMOS_MIGRATE_COLD, 129 DAMOS_STAT, /* Do nothing but only record the stat */ 130 NR_DAMOS_ACTIONS, 131 }; 132 133 /** 134 * enum damos_quota_goal_metric - Represents the metric to be used as the goal 135 * 136 * @DAMOS_QUOTA_USER_INPUT: User-input value. 137 * @DAMOS_QUOTA_SOME_MEM_PSI_US: System level some memory PSI in us. 138 * @NR_DAMOS_QUOTA_GOAL_METRICS: Number of DAMOS quota goal metrics. 139 * 140 * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported. 141 */ 142 enum damos_quota_goal_metric { 143 DAMOS_QUOTA_USER_INPUT, 144 DAMOS_QUOTA_SOME_MEM_PSI_US, 145 NR_DAMOS_QUOTA_GOAL_METRICS, 146 }; 147 148 /** 149 * struct damos_quota_goal - DAMOS scheme quota auto-tuning goal. 150 * @metric: Metric to be used for representing the goal. 151 * @target_value: Target value of @metric to achieve with the tuning. 152 * @current_value: Current value of @metric. 153 * @last_psi_total: Last measured total PSI 154 * @list: List head for siblings. 155 * 156 * Data structure for getting the current score of the quota tuning goal. The 157 * score is calculated by how close @current_value and @target_value are. Then 158 * the score is entered to DAMON's internal feedback loop mechanism to get the 159 * auto-tuned quota. 160 * 161 * If @metric is DAMOS_QUOTA_USER_INPUT, @current_value should be manually 162 * entered by the user, probably inside the kdamond callbacks. Otherwise, 163 * DAMON sets @current_value with self-measured value of @metric. 164 */ 165 struct damos_quota_goal { 166 enum damos_quota_goal_metric metric; 167 unsigned long target_value; 168 unsigned long current_value; 169 /* metric-dependent fields */ 170 union { 171 u64 last_psi_total; 172 }; 173 struct list_head list; 174 }; 175 176 /** 177 * struct damos_quota - Controls the aggressiveness of the given scheme. 178 * @reset_interval: Charge reset interval in milliseconds. 179 * @ms: Maximum milliseconds that the scheme can use. 180 * @sz: Maximum bytes of memory that the action can be applied. 181 * @goals: Head of quota tuning goals (&damos_quota_goal) list. 182 * @esz: Effective size quota in bytes. 183 * 184 * @weight_sz: Weight of the region's size for prioritization. 185 * @weight_nr_accesses: Weight of the region's nr_accesses for prioritization. 186 * @weight_age: Weight of the region's age for prioritization. 187 * 188 * To avoid consuming too much CPU time or IO resources for applying the 189 * &struct damos->action to large memory, DAMON allows users to set time and/or 190 * size quotas. The quotas can be set by writing non-zero values to &ms and 191 * &sz, respectively. If the time quota is set, DAMON tries to use only up to 192 * &ms milliseconds within &reset_interval for applying the action. If the 193 * size quota is set, DAMON tries to apply the action only up to &sz bytes 194 * within &reset_interval. 195 * 196 * To convince the different types of quotas and goals, DAMON internally 197 * converts those into one single size quota called "effective quota". DAMON 198 * internally uses it as the only one real quota. The conversion is made as 199 * follows. 200 * 201 * The time quota is transformed to a size quota using estimated throughput of 202 * the scheme's action. DAMON then compares it against &sz and uses smaller 203 * one as the effective quota. 204 * 205 * If @goals is not empty, DAMON calculates yet another size quota based on the 206 * goals using its internal feedback loop algorithm, for every @reset_interval. 207 * Then, if the new size quota is smaller than the effective quota, it uses the 208 * new size quota as the effective quota. 209 * 210 * The resulting effective size quota in bytes is set to @esz. 211 * 212 * For selecting regions within the quota, DAMON prioritizes current scheme's 213 * target memory regions using the &struct damon_operations->get_scheme_score. 214 * You could customize the prioritization logic by setting &weight_sz, 215 * &weight_nr_accesses, and &weight_age, because monitoring operations are 216 * encouraged to respect those. 217 */ 218 struct damos_quota { 219 unsigned long reset_interval; 220 unsigned long ms; 221 unsigned long sz; 222 struct list_head goals; 223 unsigned long esz; 224 225 unsigned int weight_sz; 226 unsigned int weight_nr_accesses; 227 unsigned int weight_age; 228 229 /* private: */ 230 /* For throughput estimation */ 231 unsigned long total_charged_sz; 232 unsigned long total_charged_ns; 233 234 /* For charging the quota */ 235 unsigned long charged_sz; 236 unsigned long charged_from; 237 struct damon_target *charge_target_from; 238 unsigned long charge_addr_from; 239 240 /* For prioritization */ 241 unsigned int min_score; 242 243 /* For feedback loop */ 244 unsigned long esz_bp; 245 }; 246 247 /** 248 * enum damos_wmark_metric - Represents the watermark metric. 249 * 250 * @DAMOS_WMARK_NONE: Ignore the watermarks of the given scheme. 251 * @DAMOS_WMARK_FREE_MEM_RATE: Free memory rate of the system in [0,1000]. 252 * @NR_DAMOS_WMARK_METRICS: Total number of DAMOS watermark metrics 253 */ 254 enum damos_wmark_metric { 255 DAMOS_WMARK_NONE, 256 DAMOS_WMARK_FREE_MEM_RATE, 257 NR_DAMOS_WMARK_METRICS, 258 }; 259 260 /** 261 * struct damos_watermarks - Controls when a given scheme should be activated. 262 * @metric: Metric for the watermarks. 263 * @interval: Watermarks check time interval in microseconds. 264 * @high: High watermark. 265 * @mid: Middle watermark. 266 * @low: Low watermark. 267 * 268 * If &metric is &DAMOS_WMARK_NONE, the scheme is always active. Being active 269 * means DAMON does monitoring and applying the action of the scheme to 270 * appropriate memory regions. Else, DAMON checks &metric of the system for at 271 * least every &interval microseconds and works as below. 272 * 273 * If &metric is higher than &high, the scheme is inactivated. If &metric is 274 * between &mid and &low, the scheme is activated. If &metric is lower than 275 * &low, the scheme is inactivated. 276 */ 277 struct damos_watermarks { 278 enum damos_wmark_metric metric; 279 unsigned long interval; 280 unsigned long high; 281 unsigned long mid; 282 unsigned long low; 283 284 /* private: */ 285 bool activated; 286 }; 287 288 /** 289 * struct damos_stat - Statistics on a given scheme. 290 * @nr_tried: Total number of regions that the scheme is tried to be applied. 291 * @sz_tried: Total size of regions that the scheme is tried to be applied. 292 * @nr_applied: Total number of regions that the scheme is applied. 293 * @sz_applied: Total size of regions that the scheme is applied. 294 * @sz_ops_filter_passed: 295 * Total bytes that passed ops layer-handled DAMOS filters. 296 * @qt_exceeds: Total number of times the quota of the scheme has exceeded. 297 * 298 * "Tried an action to a region" in this context means the DAMOS core logic 299 * determined the region as eligible to apply the action. The access pattern 300 * (&struct damos_access_pattern), quotas (&struct damos_quota), watermarks 301 * (&struct damos_watermarks) and filters (&struct damos_filter) that handled 302 * on core logic can affect this. The core logic asks the operation set 303 * (&struct damon_operations) to apply the action to the region. 304 * 305 * "Applied an action to a region" in this context means the operation set 306 * (&struct damon_operations) successfully applied the action to the region, at 307 * least to a part of the region. The filters (&struct damos_filter) that 308 * handled on operation set layer and type of the action and pages of the 309 * region can affect this. For example, if a filter is set to exclude 310 * anonymous pages and the region has only anonymous pages, the region will be 311 * failed at applying the action. If the action is &DAMOS_PAGEOUT and all 312 * pages of the region are already paged out, the region will be failed at 313 * applying the action. 314 */ 315 struct damos_stat { 316 unsigned long nr_tried; 317 unsigned long sz_tried; 318 unsigned long nr_applied; 319 unsigned long sz_applied; 320 unsigned long sz_ops_filter_passed; 321 unsigned long qt_exceeds; 322 }; 323 324 /** 325 * enum damos_filter_type - Type of memory for &struct damos_filter 326 * @DAMOS_FILTER_TYPE_ANON: Anonymous pages. 327 * @DAMOS_FILTER_TYPE_MEMCG: Specific memcg's pages. 328 * @DAMOS_FILTER_TYPE_YOUNG: Recently accessed pages. 329 * @DAMOS_FILTER_TYPE_ADDR: Address range. 330 * @DAMOS_FILTER_TYPE_TARGET: Data Access Monitoring target. 331 * @NR_DAMOS_FILTER_TYPES: Number of filter types. 332 * 333 * The anon pages type and memcg type filters are handled by underlying 334 * &struct damon_operations as a part of scheme action trying, and therefore 335 * accounted as 'tried'. In contrast, other types are handled by core layer 336 * before trying of the action and therefore not accounted as 'tried'. 337 * 338 * The support of the filters that handled by &struct damon_operations depend 339 * on the running &struct damon_operations. 340 * &enum DAMON_OPS_PADDR supports both anon pages type and memcg type filters, 341 * while &enum DAMON_OPS_VADDR and &enum DAMON_OPS_FVADDR don't support any of 342 * the two types. 343 */ 344 enum damos_filter_type { 345 DAMOS_FILTER_TYPE_ANON, 346 DAMOS_FILTER_TYPE_MEMCG, 347 DAMOS_FILTER_TYPE_YOUNG, 348 DAMOS_FILTER_TYPE_ADDR, 349 DAMOS_FILTER_TYPE_TARGET, 350 NR_DAMOS_FILTER_TYPES, 351 }; 352 353 /** 354 * struct damos_filter - DAMOS action target memory filter. 355 * @type: Type of the target memory. 356 * @matching: Whether this is for @type-matching memory. 357 * @allow: Whether to include or exclude the @matching memory. 358 * @memcg_id: Memcg id of the question if @type is DAMOS_FILTER_MEMCG. 359 * @addr_range: Address range if @type is DAMOS_FILTER_TYPE_ADDR. 360 * @target_idx: Index of the &struct damon_target of 361 * &damon_ctx->adaptive_targets if @type is 362 * DAMOS_FILTER_TYPE_TARGET. 363 * @list: List head for siblings. 364 * 365 * Before applying the &damos->action to a memory region, DAMOS checks if each 366 * byte of the region matches to this given condition and avoid applying the 367 * action if so. Support of each filter type depends on the running &struct 368 * damon_operations and the type. Refer to &enum damos_filter_type for more 369 * details. 370 */ 371 struct damos_filter { 372 enum damos_filter_type type; 373 bool matching; 374 bool allow; 375 union { 376 unsigned short memcg_id; 377 struct damon_addr_range addr_range; 378 int target_idx; 379 }; 380 struct list_head list; 381 }; 382 383 struct damon_ctx; 384 struct damos; 385 386 /** 387 * struct damos_walk_control - Control damos_walk(). 388 * 389 * @walk_fn: Function to be called back for each region. 390 * @data: Data that will be passed to walk functions. 391 * 392 * Control damos_walk(), which requests specific kdamond to invoke the given 393 * function to each region that eligible to apply actions of the kdamond's 394 * schemes. Refer to damos_walk() for more details. 395 */ 396 struct damos_walk_control { 397 void (*walk_fn)(void *data, struct damon_ctx *ctx, 398 struct damon_target *t, struct damon_region *r, 399 struct damos *s, unsigned long sz_filter_passed); 400 void *data; 401 /* private: internal use only */ 402 /* informs if the kdamond finished handling of the walk request */ 403 struct completion completion; 404 /* informs if the walk is canceled. */ 405 bool canceled; 406 }; 407 408 /** 409 * struct damos_access_pattern - Target access pattern of the given scheme. 410 * @min_sz_region: Minimum size of target regions. 411 * @max_sz_region: Maximum size of target regions. 412 * @min_nr_accesses: Minimum ``->nr_accesses`` of target regions. 413 * @max_nr_accesses: Maximum ``->nr_accesses`` of target regions. 414 * @min_age_region: Minimum age of target regions. 415 * @max_age_region: Maximum age of target regions. 416 */ 417 struct damos_access_pattern { 418 unsigned long min_sz_region; 419 unsigned long max_sz_region; 420 unsigned int min_nr_accesses; 421 unsigned int max_nr_accesses; 422 unsigned int min_age_region; 423 unsigned int max_age_region; 424 }; 425 426 /** 427 * struct damos - Represents a Data Access Monitoring-based Operation Scheme. 428 * @pattern: Access pattern of target regions. 429 * @action: &damo_action to be applied to the target regions. 430 * @apply_interval_us: The time between applying the @action. 431 * @quota: Control the aggressiveness of this scheme. 432 * @wmarks: Watermarks for automated (in)activation of this scheme. 433 * @target_nid: Destination node if @action is "migrate_{hot,cold}". 434 * @filters: Additional set of &struct damos_filter for &action. 435 * @stat: Statistics of this scheme. 436 * @list: List head for siblings. 437 * 438 * For each @apply_interval_us, DAMON finds regions which fit in the 439 * &pattern and applies &action to those. To avoid consuming too much 440 * CPU time or IO resources for the &action, "a is used. 441 * 442 * If @apply_interval_us is zero, &damon_attrs->aggr_interval is used instead. 443 * 444 * To do the work only when needed, schemes can be activated for specific 445 * system situations using &wmarks. If all schemes that registered to the 446 * monitoring context are inactive, DAMON stops monitoring either, and just 447 * repeatedly checks the watermarks. 448 * 449 * @target_nid is used to set the migration target node for migrate_hot or 450 * migrate_cold actions, which means it's only meaningful when @action is either 451 * "migrate_hot" or "migrate_cold". 452 * 453 * Before applying the &action to a memory region, &struct damon_operations 454 * implementation could check pages of the region and skip &action to respect 455 * &filters 456 * 457 * After applying the &action to each region, &stat_count and &stat_sz is 458 * updated to reflect the number of regions and total size of regions that the 459 * &action is applied. 460 */ 461 struct damos { 462 struct damos_access_pattern pattern; 463 enum damos_action action; 464 unsigned long apply_interval_us; 465 /* private: internal use only */ 466 /* 467 * number of sample intervals that should be passed before applying 468 * @action 469 */ 470 unsigned long next_apply_sis; 471 /* informs if ongoing DAMOS walk for this scheme is finished */ 472 bool walk_completed; 473 /* public: */ 474 struct damos_quota quota; 475 struct damos_watermarks wmarks; 476 union { 477 int target_nid; 478 }; 479 struct list_head filters; 480 struct damos_stat stat; 481 struct list_head list; 482 }; 483 484 /** 485 * enum damon_ops_id - Identifier for each monitoring operations implementation 486 * 487 * @DAMON_OPS_VADDR: Monitoring operations for virtual address spaces 488 * @DAMON_OPS_FVADDR: Monitoring operations for only fixed ranges of virtual 489 * address spaces 490 * @DAMON_OPS_PADDR: Monitoring operations for the physical address space 491 * @NR_DAMON_OPS: Number of monitoring operations implementations 492 */ 493 enum damon_ops_id { 494 DAMON_OPS_VADDR, 495 DAMON_OPS_FVADDR, 496 DAMON_OPS_PADDR, 497 NR_DAMON_OPS, 498 }; 499 500 /** 501 * struct damon_operations - Monitoring operations for given use cases. 502 * 503 * @id: Identifier of this operations set. 504 * @init: Initialize operations-related data structures. 505 * @update: Update operations-related data structures. 506 * @prepare_access_checks: Prepare next access check of target regions. 507 * @check_accesses: Check the accesses to target regions. 508 * @reset_aggregated: Reset aggregated accesses monitoring results. 509 * @get_scheme_score: Get the score of a region for a scheme. 510 * @apply_scheme: Apply a DAMON-based operation scheme. 511 * @target_valid: Determine if the target is valid. 512 * @cleanup: Clean up the context. 513 * 514 * DAMON can be extended for various address spaces and usages. For this, 515 * users should register the low level operations for their target address 516 * space and usecase via the &damon_ctx.ops. Then, the monitoring thread 517 * (&damon_ctx.kdamond) calls @init and @prepare_access_checks before starting 518 * the monitoring, @update after each &damon_attrs.ops_update_interval, and 519 * @check_accesses, @target_valid and @prepare_access_checks after each 520 * &damon_attrs.sample_interval. Finally, @reset_aggregated is called after 521 * each &damon_attrs.aggr_interval. 522 * 523 * Each &struct damon_operations instance having valid @id can be registered 524 * via damon_register_ops() and selected by damon_select_ops() later. 525 * @init should initialize operations-related data structures. For example, 526 * this could be used to construct proper monitoring target regions and link 527 * those to @damon_ctx.adaptive_targets. 528 * @update should update the operations-related data structures. For example, 529 * this could be used to update monitoring target regions for current status. 530 * @prepare_access_checks should manipulate the monitoring regions to be 531 * prepared for the next access check. 532 * @check_accesses should check the accesses to each region that made after the 533 * last preparation and update the number of observed accesses of each region. 534 * It should also return max number of observed accesses that made as a result 535 * of its update. The value will be used for regions adjustment threshold. 536 * @reset_aggregated should reset the access monitoring results that aggregated 537 * by @check_accesses. 538 * @get_scheme_score should return the priority score of a region for a scheme 539 * as an integer in [0, &DAMOS_MAX_SCORE]. 540 * @apply_scheme is called from @kdamond when a region for user provided 541 * DAMON-based operation scheme is found. It should apply the scheme's action 542 * to the region and return bytes of the region that the action is successfully 543 * applied. It should also report how many bytes of the region has passed 544 * filters (&struct damos_filter) that handled by itself. 545 * @target_valid should check whether the target is still valid for the 546 * monitoring. 547 * @cleanup is called from @kdamond just before its termination. 548 */ 549 struct damon_operations { 550 enum damon_ops_id id; 551 void (*init)(struct damon_ctx *context); 552 void (*update)(struct damon_ctx *context); 553 void (*prepare_access_checks)(struct damon_ctx *context); 554 unsigned int (*check_accesses)(struct damon_ctx *context); 555 void (*reset_aggregated)(struct damon_ctx *context); 556 int (*get_scheme_score)(struct damon_ctx *context, 557 struct damon_target *t, struct damon_region *r, 558 struct damos *scheme); 559 unsigned long (*apply_scheme)(struct damon_ctx *context, 560 struct damon_target *t, struct damon_region *r, 561 struct damos *scheme, unsigned long *sz_filter_passed); 562 bool (*target_valid)(struct damon_target *t); 563 void (*cleanup)(struct damon_ctx *context); 564 }; 565 566 /** 567 * struct damon_callback - Monitoring events notification callbacks. 568 * 569 * @before_start: Called before starting the monitoring. 570 * @after_wmarks_check: Called after each schemes' watermarks check. 571 * @after_sampling: Called after each sampling. 572 * @after_aggregation: Called after each aggregation. 573 * @before_damos_apply: Called before applying DAMOS action. 574 * @before_terminate: Called before terminating the monitoring. 575 * @private: User private data. 576 * 577 * The monitoring thread (&damon_ctx.kdamond) calls @before_start and 578 * @before_terminate just before starting and finishing the monitoring, 579 * respectively. Therefore, those are good places for installing and cleaning 580 * @private. 581 * 582 * The monitoring thread calls @after_wmarks_check after each DAMON-based 583 * operation schemes' watermarks check. If users need to make changes to the 584 * attributes of the monitoring context while it's deactivated due to the 585 * watermarks, this is the good place to do. 586 * 587 * The monitoring thread calls @after_sampling and @after_aggregation for each 588 * of the sampling intervals and aggregation intervals, respectively. 589 * Therefore, users can safely access the monitoring results without additional 590 * protection. For the reason, users are recommended to use these callback for 591 * the accesses to the results. 592 * 593 * If any callback returns non-zero, monitoring stops. 594 */ 595 struct damon_callback { 596 void *private; 597 598 int (*before_start)(struct damon_ctx *context); 599 int (*after_wmarks_check)(struct damon_ctx *context); 600 int (*after_sampling)(struct damon_ctx *context); 601 int (*after_aggregation)(struct damon_ctx *context); 602 int (*before_damos_apply)(struct damon_ctx *context, 603 struct damon_target *target, 604 struct damon_region *region, 605 struct damos *scheme); 606 void (*before_terminate)(struct damon_ctx *context); 607 }; 608 609 /* 610 * struct damon_call_control - Control damon_call(). 611 * 612 * @fn: Function to be called back. 613 * @data: Data that will be passed to @fn. 614 * @return_code: Return code from @fn invocation. 615 * 616 * Control damon_call(), which requests specific kdamond to invoke a given 617 * function. Refer to damon_call() for more details. 618 */ 619 struct damon_call_control { 620 int (*fn)(void *data); 621 void *data; 622 int return_code; 623 /* private: internal use only */ 624 /* informs if the kdamond finished handling of the request */ 625 struct completion completion; 626 /* informs if the kdamond canceled @fn infocation */ 627 bool canceled; 628 }; 629 630 /** 631 * struct damon_attrs - Monitoring attributes for accuracy/overhead control. 632 * 633 * @sample_interval: The time between access samplings. 634 * @aggr_interval: The time between monitor results aggregations. 635 * @ops_update_interval: The time between monitoring operations updates. 636 * @min_nr_regions: The minimum number of adaptive monitoring 637 * regions. 638 * @max_nr_regions: The maximum number of adaptive monitoring 639 * regions. 640 * 641 * For each @sample_interval, DAMON checks whether each region is accessed or 642 * not during the last @sample_interval. If such access is found, DAMON 643 * aggregates the information by increasing &damon_region->nr_accesses for 644 * @aggr_interval time. For each @aggr_interval, the count is reset. DAMON 645 * also checks whether the target memory regions need update (e.g., by 646 * ``mmap()`` calls from the application, in case of virtual memory monitoring) 647 * and applies the changes for each @ops_update_interval. All time intervals 648 * are in micro-seconds. Please refer to &struct damon_operations and &struct 649 * damon_callback for more detail. 650 */ 651 struct damon_attrs { 652 unsigned long sample_interval; 653 unsigned long aggr_interval; 654 unsigned long ops_update_interval; 655 unsigned long min_nr_regions; 656 unsigned long max_nr_regions; 657 }; 658 659 /** 660 * struct damon_ctx - Represents a context for each monitoring. This is the 661 * main interface that allows users to set the attributes and get the results 662 * of the monitoring. 663 * 664 * @attrs: Monitoring attributes for accuracy/overhead control. 665 * @kdamond: Kernel thread who does the monitoring. 666 * @kdamond_lock: Mutex for the synchronizations with @kdamond. 667 * 668 * For each monitoring context, one kernel thread for the monitoring is 669 * created. The pointer to the thread is stored in @kdamond. 670 * 671 * Once started, the monitoring thread runs until explicitly required to be 672 * terminated or every monitoring target is invalid. The validity of the 673 * targets is checked via the &damon_operations.target_valid of @ops. The 674 * termination can also be explicitly requested by calling damon_stop(). 675 * The thread sets @kdamond to NULL when it terminates. Therefore, users can 676 * know whether the monitoring is ongoing or terminated by reading @kdamond. 677 * Reads and writes to @kdamond from outside of the monitoring thread must 678 * be protected by @kdamond_lock. 679 * 680 * Note that the monitoring thread protects only @kdamond via @kdamond_lock. 681 * Accesses to other fields must be protected by themselves. 682 * 683 * @ops: Set of monitoring operations for given use cases. 684 * @callback: Set of callbacks for monitoring events notifications. 685 * 686 * @adaptive_targets: Head of monitoring targets (&damon_target) list. 687 * @schemes: Head of schemes (&damos) list. 688 */ 689 struct damon_ctx { 690 struct damon_attrs attrs; 691 692 /* private: internal use only */ 693 /* number of sample intervals that passed since this context started */ 694 unsigned long passed_sample_intervals; 695 /* 696 * number of sample intervals that should be passed before next 697 * aggregation 698 */ 699 unsigned long next_aggregation_sis; 700 /* 701 * number of sample intervals that should be passed before next ops 702 * update 703 */ 704 unsigned long next_ops_update_sis; 705 /* for waiting until the execution of the kdamond_fn is started */ 706 struct completion kdamond_started; 707 /* for scheme quotas prioritization */ 708 unsigned long *regions_score_histogram; 709 710 struct damon_call_control *call_control; 711 struct mutex call_control_lock; 712 713 struct damos_walk_control *walk_control; 714 struct mutex walk_control_lock; 715 716 /* public: */ 717 struct task_struct *kdamond; 718 struct mutex kdamond_lock; 719 720 struct damon_operations ops; 721 struct damon_callback callback; 722 723 struct list_head adaptive_targets; 724 struct list_head schemes; 725 }; 726 727 static inline struct damon_region *damon_next_region(struct damon_region *r) 728 { 729 return container_of(r->list.next, struct damon_region, list); 730 } 731 732 static inline struct damon_region *damon_prev_region(struct damon_region *r) 733 { 734 return container_of(r->list.prev, struct damon_region, list); 735 } 736 737 static inline struct damon_region *damon_last_region(struct damon_target *t) 738 { 739 return list_last_entry(&t->regions_list, struct damon_region, list); 740 } 741 742 static inline struct damon_region *damon_first_region(struct damon_target *t) 743 { 744 return list_first_entry(&t->regions_list, struct damon_region, list); 745 } 746 747 static inline unsigned long damon_sz_region(struct damon_region *r) 748 { 749 return r->ar.end - r->ar.start; 750 } 751 752 753 #define damon_for_each_region(r, t) \ 754 list_for_each_entry(r, &t->regions_list, list) 755 756 #define damon_for_each_region_from(r, t) \ 757 list_for_each_entry_from(r, &t->regions_list, list) 758 759 #define damon_for_each_region_safe(r, next, t) \ 760 list_for_each_entry_safe(r, next, &t->regions_list, list) 761 762 #define damon_for_each_target(t, ctx) \ 763 list_for_each_entry(t, &(ctx)->adaptive_targets, list) 764 765 #define damon_for_each_target_safe(t, next, ctx) \ 766 list_for_each_entry_safe(t, next, &(ctx)->adaptive_targets, list) 767 768 #define damon_for_each_scheme(s, ctx) \ 769 list_for_each_entry(s, &(ctx)->schemes, list) 770 771 #define damon_for_each_scheme_safe(s, next, ctx) \ 772 list_for_each_entry_safe(s, next, &(ctx)->schemes, list) 773 774 #define damos_for_each_quota_goal(goal, quota) \ 775 list_for_each_entry(goal, "a->goals, list) 776 777 #define damos_for_each_quota_goal_safe(goal, next, quota) \ 778 list_for_each_entry_safe(goal, next, &(quota)->goals, list) 779 780 #define damos_for_each_filter(f, scheme) \ 781 list_for_each_entry(f, &(scheme)->filters, list) 782 783 #define damos_for_each_filter_safe(f, next, scheme) \ 784 list_for_each_entry_safe(f, next, &(scheme)->filters, list) 785 786 #ifdef CONFIG_DAMON 787 788 struct damon_region *damon_new_region(unsigned long start, unsigned long end); 789 790 /* 791 * Add a region between two other regions 792 */ 793 static inline void damon_insert_region(struct damon_region *r, 794 struct damon_region *prev, struct damon_region *next, 795 struct damon_target *t) 796 { 797 __list_add(&r->list, &prev->list, &next->list); 798 t->nr_regions++; 799 } 800 801 void damon_add_region(struct damon_region *r, struct damon_target *t); 802 void damon_destroy_region(struct damon_region *r, struct damon_target *t); 803 int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, 804 unsigned int nr_ranges); 805 void damon_update_region_access_rate(struct damon_region *r, bool accessed, 806 struct damon_attrs *attrs); 807 808 struct damos_filter *damos_new_filter(enum damos_filter_type type, 809 bool matching, bool allow); 810 void damos_add_filter(struct damos *s, struct damos_filter *f); 811 void damos_destroy_filter(struct damos_filter *f); 812 813 struct damos_quota_goal *damos_new_quota_goal( 814 enum damos_quota_goal_metric metric, 815 unsigned long target_value); 816 void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g); 817 void damos_destroy_quota_goal(struct damos_quota_goal *goal); 818 819 struct damos *damon_new_scheme(struct damos_access_pattern *pattern, 820 enum damos_action action, 821 unsigned long apply_interval_us, 822 struct damos_quota *quota, 823 struct damos_watermarks *wmarks, 824 int target_nid); 825 void damon_add_scheme(struct damon_ctx *ctx, struct damos *s); 826 void damon_destroy_scheme(struct damos *s); 827 int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src); 828 829 struct damon_target *damon_new_target(void); 830 void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); 831 bool damon_targets_empty(struct damon_ctx *ctx); 832 void damon_free_target(struct damon_target *t); 833 void damon_destroy_target(struct damon_target *t); 834 unsigned int damon_nr_regions(struct damon_target *t); 835 836 struct damon_ctx *damon_new_ctx(void); 837 void damon_destroy_ctx(struct damon_ctx *ctx); 838 int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs); 839 void damon_set_schemes(struct damon_ctx *ctx, 840 struct damos **schemes, ssize_t nr_schemes); 841 int damon_commit_ctx(struct damon_ctx *old_ctx, struct damon_ctx *new_ctx); 842 int damon_nr_running_ctxs(void); 843 bool damon_is_registered_ops(enum damon_ops_id id); 844 int damon_register_ops(struct damon_operations *ops); 845 int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id); 846 847 static inline bool damon_target_has_pid(const struct damon_ctx *ctx) 848 { 849 return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR; 850 } 851 852 static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs) 853 { 854 /* {aggr,sample}_interval are unsigned long, hence could overflow */ 855 return min(attrs->aggr_interval / attrs->sample_interval, 856 (unsigned long)UINT_MAX); 857 } 858 859 860 int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive); 861 int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); 862 863 int damon_call(struct damon_ctx *ctx, struct damon_call_control *control); 864 int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control); 865 866 int damon_set_region_biggest_system_ram_default(struct damon_target *t, 867 unsigned long *start, unsigned long *end); 868 869 #endif /* CONFIG_DAMON */ 870 871 #endif /* _DAMON_H */ 872