1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2014-2018, The Linux Foundation. All rights reserved. 4 * Copyright (C) 2021-2022 Linaro Ltd 5 * Author: Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>, based on 6 * previous work of Thara Gopinath and msm-4.9 downstream sources. 7 */ 8 9 #include <linux/err.h> 10 #include <linux/interconnect.h> 11 #include <linux/interrupt.h> 12 #include <linux/io.h> 13 #include <linux/kernel.h> 14 #include <linux/module.h> 15 #include <linux/of.h> 16 #include <linux/platform_device.h> 17 #include <linux/pm_opp.h> 18 #include <linux/regmap.h> 19 #include <linux/sizes.h> 20 #define CREATE_TRACE_POINTS 21 #include "trace_icc-bwmon.h" 22 23 /* 24 * The BWMON samples data throughput within 'sample_ms' time. With three 25 * configurable thresholds (Low, Medium and High) gives four windows (called 26 * zones) of current bandwidth: 27 * 28 * Zone 0: byte count < THRES_LO 29 * Zone 1: THRES_LO < byte count < THRES_MED 30 * Zone 2: THRES_MED < byte count < THRES_HIGH 31 * Zone 3: THRES_HIGH < byte count 32 * 33 * Zones 0 and 2 are not used by this driver. 34 */ 35 36 /* Internal sampling clock frequency */ 37 #define HW_TIMER_HZ 19200000 38 39 #define BWMON_V4_GLOBAL_IRQ_CLEAR 0x108 40 #define BWMON_V4_GLOBAL_IRQ_ENABLE 0x10c 41 /* 42 * All values here and further are matching regmap fields, so without absolute 43 * register offsets. 44 */ 45 #define BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE BIT(0) 46 47 /* 48 * Starting with SDM845, the BWMON4 register space has changed a bit: 49 * the global registers were jammed into the beginning of the monitor region. 50 * To keep the proper offsets, one would have to map <GLOBAL_BASE 0x200> and 51 * <GLOBAL_BASE+0x100 0x300>, which is straight up wrong. 52 * To facilitate for that, while allowing the older, arguably more proper 53 * implementations to work, offset the global registers by -0x100 to avoid 54 * having to map half of the global registers twice. 55 */ 56 #define BWMON_V4_845_OFFSET 0x100 57 #define BWMON_V4_GLOBAL_IRQ_CLEAR_845 (BWMON_V4_GLOBAL_IRQ_CLEAR - BWMON_V4_845_OFFSET) 58 #define BWMON_V4_GLOBAL_IRQ_ENABLE_845 (BWMON_V4_GLOBAL_IRQ_ENABLE - BWMON_V4_845_OFFSET) 59 60 #define BWMON_V4_IRQ_STATUS 0x100 61 #define BWMON_V4_IRQ_CLEAR 0x108 62 63 #define BWMON_V4_IRQ_ENABLE 0x10c 64 #define BWMON_IRQ_ENABLE_MASK (BIT(1) | BIT(3)) 65 #define BWMON_V5_IRQ_STATUS 0x000 66 #define BWMON_V5_IRQ_CLEAR 0x008 67 #define BWMON_V5_IRQ_ENABLE 0x00c 68 69 #define BWMON_V4_ENABLE 0x2a0 70 #define BWMON_V5_ENABLE 0x010 71 #define BWMON_ENABLE_ENABLE BIT(0) 72 73 #define BWMON_V4_CLEAR 0x2a4 74 #define BWMON_V5_CLEAR 0x014 75 #define BWMON_CLEAR_CLEAR BIT(0) 76 #define BWMON_CLEAR_CLEAR_ALL BIT(1) 77 78 #define BWMON_V4_SAMPLE_WINDOW 0x2a8 79 #define BWMON_V5_SAMPLE_WINDOW 0x020 80 81 #define BWMON_V4_THRESHOLD_HIGH 0x2ac 82 #define BWMON_V4_THRESHOLD_MED 0x2b0 83 #define BWMON_V4_THRESHOLD_LOW 0x2b4 84 #define BWMON_V5_THRESHOLD_HIGH 0x024 85 #define BWMON_V5_THRESHOLD_MED 0x028 86 #define BWMON_V5_THRESHOLD_LOW 0x02c 87 88 #define BWMON_V4_ZONE_ACTIONS 0x2b8 89 #define BWMON_V5_ZONE_ACTIONS 0x030 90 /* 91 * Actions to perform on some zone 'z' when current zone hits the threshold: 92 * Increment counter of zone 'z' 93 */ 94 #define BWMON_ZONE_ACTIONS_INCREMENT(z) (0x2 << ((z) * 2)) 95 /* Clear counter of zone 'z' */ 96 #define BWMON_ZONE_ACTIONS_CLEAR(z) (0x1 << ((z) * 2)) 97 98 /* Zone 0 threshold hit: Clear zone count */ 99 #define BWMON_ZONE_ACTIONS_ZONE0 (BWMON_ZONE_ACTIONS_CLEAR(0)) 100 101 /* Zone 1 threshold hit: Increment zone count & clear lower zones */ 102 #define BWMON_ZONE_ACTIONS_ZONE1 (BWMON_ZONE_ACTIONS_INCREMENT(1) | \ 103 BWMON_ZONE_ACTIONS_CLEAR(0)) 104 105 /* Zone 2 threshold hit: Increment zone count & clear lower zones */ 106 #define BWMON_ZONE_ACTIONS_ZONE2 (BWMON_ZONE_ACTIONS_INCREMENT(2) | \ 107 BWMON_ZONE_ACTIONS_CLEAR(1) | \ 108 BWMON_ZONE_ACTIONS_CLEAR(0)) 109 110 /* Zone 3 threshold hit: Increment zone count & clear lower zones */ 111 #define BWMON_ZONE_ACTIONS_ZONE3 (BWMON_ZONE_ACTIONS_INCREMENT(3) | \ 112 BWMON_ZONE_ACTIONS_CLEAR(2) | \ 113 BWMON_ZONE_ACTIONS_CLEAR(1) | \ 114 BWMON_ZONE_ACTIONS_CLEAR(0)) 115 116 /* 117 * There is no clear documentation/explanation of BWMON_V4_THRESHOLD_COUNT 118 * register. Based on observations, this is number of times one threshold has to 119 * be reached, to trigger interrupt in given zone. 120 * 121 * 0xff are maximum values meant to ignore the zones 0 and 2. 122 */ 123 #define BWMON_V4_THRESHOLD_COUNT 0x2bc 124 #define BWMON_V5_THRESHOLD_COUNT 0x034 125 #define BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT 0xff 126 #define BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT 0xff 127 128 #define BWMON_V4_ZONE_MAX(zone) (0x2e0 + 4 * (zone)) 129 #define BWMON_V5_ZONE_MAX(zone) (0x044 + 4 * (zone)) 130 131 /* Quirks for specific BWMON types */ 132 #define BWMON_HAS_GLOBAL_IRQ BIT(0) 133 #define BWMON_NEEDS_FORCE_CLEAR BIT(1) 134 135 enum bwmon_fields { 136 /* Global region fields, keep them at the top */ 137 F_GLOBAL_IRQ_CLEAR, 138 F_GLOBAL_IRQ_ENABLE, 139 F_NUM_GLOBAL_FIELDS, 140 141 /* Monitor region fields */ 142 F_IRQ_STATUS = F_NUM_GLOBAL_FIELDS, 143 F_IRQ_CLEAR, 144 F_IRQ_ENABLE, 145 F_ENABLE, 146 F_CLEAR, 147 F_SAMPLE_WINDOW, 148 F_THRESHOLD_HIGH, 149 F_THRESHOLD_MED, 150 F_THRESHOLD_LOW, 151 F_ZONE_ACTIONS_ZONE0, 152 F_ZONE_ACTIONS_ZONE1, 153 F_ZONE_ACTIONS_ZONE2, 154 F_ZONE_ACTIONS_ZONE3, 155 F_THRESHOLD_COUNT_ZONE0, 156 F_THRESHOLD_COUNT_ZONE1, 157 F_THRESHOLD_COUNT_ZONE2, 158 F_THRESHOLD_COUNT_ZONE3, 159 F_ZONE0_MAX, 160 F_ZONE1_MAX, 161 F_ZONE2_MAX, 162 F_ZONE3_MAX, 163 164 F_NUM_FIELDS 165 }; 166 167 struct icc_bwmon_data { 168 unsigned int sample_ms; 169 unsigned int count_unit_kb; /* kbytes */ 170 u8 zone1_thres_count; 171 u8 zone3_thres_count; 172 unsigned int quirks; 173 174 const struct regmap_config *regmap_cfg; 175 const struct reg_field *regmap_fields; 176 177 const struct regmap_config *global_regmap_cfg; 178 const struct reg_field *global_regmap_fields; 179 }; 180 181 struct icc_bwmon { 182 struct device *dev; 183 const struct icc_bwmon_data *data; 184 int irq; 185 186 struct regmap_field *regs[F_NUM_FIELDS]; 187 struct regmap_field *global_regs[F_NUM_GLOBAL_FIELDS]; 188 189 unsigned int max_bw_kbps; 190 unsigned int min_bw_kbps; 191 unsigned int target_kbps; 192 unsigned int current_kbps; 193 }; 194 195 /* BWMON v4 */ 196 static const struct reg_field msm8998_bwmon_reg_fields[] = { 197 [F_GLOBAL_IRQ_CLEAR] = {}, 198 [F_GLOBAL_IRQ_ENABLE] = {}, 199 [F_IRQ_STATUS] = REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7), 200 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7), 201 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7), 202 /* F_ENABLE covers entire register to disable other features */ 203 [F_ENABLE] = REG_FIELD(BWMON_V4_ENABLE, 0, 31), 204 [F_CLEAR] = REG_FIELD(BWMON_V4_CLEAR, 0, 1), 205 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23), 206 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11), 207 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11), 208 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11), 209 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7), 210 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15), 211 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23), 212 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31), 213 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7), 214 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15), 215 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23), 216 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31), 217 [F_ZONE0_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11), 218 [F_ZONE1_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11), 219 [F_ZONE2_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11), 220 [F_ZONE3_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11), 221 }; 222 223 static const struct regmap_range msm8998_bwmon_reg_noread_ranges[] = { 224 regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR), 225 regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR), 226 }; 227 228 static const struct regmap_access_table msm8998_bwmon_reg_read_table = { 229 .no_ranges = msm8998_bwmon_reg_noread_ranges, 230 .n_no_ranges = ARRAY_SIZE(msm8998_bwmon_reg_noread_ranges), 231 }; 232 233 static const struct regmap_range msm8998_bwmon_reg_volatile_ranges[] = { 234 regmap_reg_range(BWMON_V4_IRQ_STATUS, BWMON_V4_IRQ_STATUS), 235 regmap_reg_range(BWMON_V4_ZONE_MAX(0), BWMON_V4_ZONE_MAX(3)), 236 }; 237 238 static const struct regmap_access_table msm8998_bwmon_reg_volatile_table = { 239 .yes_ranges = msm8998_bwmon_reg_volatile_ranges, 240 .n_yes_ranges = ARRAY_SIZE(msm8998_bwmon_reg_volatile_ranges), 241 }; 242 243 static const struct reg_field msm8998_bwmon_global_reg_fields[] = { 244 [F_GLOBAL_IRQ_CLEAR] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR, 0, 0), 245 [F_GLOBAL_IRQ_ENABLE] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE, 0, 0), 246 }; 247 248 static const struct regmap_range msm8998_bwmon_global_reg_noread_ranges[] = { 249 regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR, BWMON_V4_GLOBAL_IRQ_CLEAR), 250 }; 251 252 static const struct regmap_access_table msm8998_bwmon_global_reg_read_table = { 253 .no_ranges = msm8998_bwmon_global_reg_noread_ranges, 254 .n_no_ranges = ARRAY_SIZE(msm8998_bwmon_global_reg_noread_ranges), 255 }; 256 257 /* 258 * Fill the cache for non-readable registers only as rest does not really 259 * matter and can be read from the device. 260 */ 261 static const struct reg_default msm8998_bwmon_reg_defaults[] = { 262 { BWMON_V4_IRQ_CLEAR, 0x0 }, 263 { BWMON_V4_CLEAR, 0x0 }, 264 }; 265 266 static const struct reg_default msm8998_bwmon_global_reg_defaults[] = { 267 { BWMON_V4_GLOBAL_IRQ_CLEAR, 0x0 }, 268 }; 269 270 static const struct regmap_config msm8998_bwmon_regmap_cfg = { 271 .reg_bits = 32, 272 .reg_stride = 4, 273 .val_bits = 32, 274 /* 275 * No concurrent access expected - driver has one interrupt handler, 276 * regmap is not shared, no driver or user-space API. 277 */ 278 .disable_locking = true, 279 .rd_table = &msm8998_bwmon_reg_read_table, 280 .volatile_table = &msm8998_bwmon_reg_volatile_table, 281 .reg_defaults = msm8998_bwmon_reg_defaults, 282 .num_reg_defaults = ARRAY_SIZE(msm8998_bwmon_reg_defaults), 283 /* 284 * Cache is necessary for using regmap fields with non-readable 285 * registers. 286 */ 287 .cache_type = REGCACHE_MAPLE, 288 }; 289 290 static const struct regmap_config msm8998_bwmon_global_regmap_cfg = { 291 .reg_bits = 32, 292 .reg_stride = 4, 293 .val_bits = 32, 294 /* 295 * No concurrent access expected - driver has one interrupt handler, 296 * regmap is not shared, no driver or user-space API. 297 */ 298 .disable_locking = true, 299 .rd_table = &msm8998_bwmon_global_reg_read_table, 300 .reg_defaults = msm8998_bwmon_global_reg_defaults, 301 .num_reg_defaults = ARRAY_SIZE(msm8998_bwmon_global_reg_defaults), 302 /* 303 * Cache is necessary for using regmap fields with non-readable 304 * registers. 305 */ 306 .cache_type = REGCACHE_MAPLE, 307 }; 308 309 static const struct reg_field sdm845_cpu_bwmon_reg_fields[] = { 310 [F_GLOBAL_IRQ_CLEAR] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0, 0), 311 [F_GLOBAL_IRQ_ENABLE] = REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE_845, 0, 0), 312 [F_IRQ_STATUS] = REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7), 313 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7), 314 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7), 315 /* F_ENABLE covers entire register to disable other features */ 316 [F_ENABLE] = REG_FIELD(BWMON_V4_ENABLE, 0, 31), 317 [F_CLEAR] = REG_FIELD(BWMON_V4_CLEAR, 0, 1), 318 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23), 319 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11), 320 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11), 321 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11), 322 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7), 323 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15), 324 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23), 325 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31), 326 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7), 327 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15), 328 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23), 329 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31), 330 [F_ZONE0_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11), 331 [F_ZONE1_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11), 332 [F_ZONE2_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11), 333 [F_ZONE3_MAX] = REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11), 334 }; 335 336 static const struct regmap_range sdm845_cpu_bwmon_reg_noread_ranges[] = { 337 regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR_845, BWMON_V4_GLOBAL_IRQ_CLEAR_845), 338 regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR), 339 regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR), 340 }; 341 342 static const struct regmap_access_table sdm845_cpu_bwmon_reg_read_table = { 343 .no_ranges = sdm845_cpu_bwmon_reg_noread_ranges, 344 .n_no_ranges = ARRAY_SIZE(sdm845_cpu_bwmon_reg_noread_ranges), 345 }; 346 347 /* 348 * Fill the cache for non-readable registers only as rest does not really 349 * matter and can be read from the device. 350 */ 351 static const struct reg_default sdm845_cpu_bwmon_reg_defaults[] = { 352 { BWMON_V4_GLOBAL_IRQ_CLEAR_845, 0x0 }, 353 { BWMON_V4_IRQ_CLEAR, 0x0 }, 354 { BWMON_V4_CLEAR, 0x0 }, 355 }; 356 357 static const struct regmap_config sdm845_cpu_bwmon_regmap_cfg = { 358 .reg_bits = 32, 359 .reg_stride = 4, 360 .val_bits = 32, 361 /* 362 * No concurrent access expected - driver has one interrupt handler, 363 * regmap is not shared, no driver or user-space API. 364 */ 365 .disable_locking = true, 366 .rd_table = &sdm845_cpu_bwmon_reg_read_table, 367 .volatile_table = &msm8998_bwmon_reg_volatile_table, 368 .reg_defaults = sdm845_cpu_bwmon_reg_defaults, 369 .num_reg_defaults = ARRAY_SIZE(sdm845_cpu_bwmon_reg_defaults), 370 /* 371 * Cache is necessary for using regmap fields with non-readable 372 * registers. 373 */ 374 .cache_type = REGCACHE_MAPLE, 375 }; 376 377 /* BWMON v5 */ 378 static const struct reg_field sdm845_llcc_bwmon_reg_fields[] = { 379 [F_GLOBAL_IRQ_CLEAR] = {}, 380 [F_GLOBAL_IRQ_ENABLE] = {}, 381 [F_IRQ_STATUS] = REG_FIELD(BWMON_V5_IRQ_STATUS, 0, 3), 382 [F_IRQ_CLEAR] = REG_FIELD(BWMON_V5_IRQ_CLEAR, 0, 3), 383 [F_IRQ_ENABLE] = REG_FIELD(BWMON_V5_IRQ_ENABLE, 0, 3), 384 /* F_ENABLE covers entire register to disable other features */ 385 [F_ENABLE] = REG_FIELD(BWMON_V5_ENABLE, 0, 31), 386 [F_CLEAR] = REG_FIELD(BWMON_V5_CLEAR, 0, 1), 387 [F_SAMPLE_WINDOW] = REG_FIELD(BWMON_V5_SAMPLE_WINDOW, 0, 19), 388 [F_THRESHOLD_HIGH] = REG_FIELD(BWMON_V5_THRESHOLD_HIGH, 0, 11), 389 [F_THRESHOLD_MED] = REG_FIELD(BWMON_V5_THRESHOLD_MED, 0, 11), 390 [F_THRESHOLD_LOW] = REG_FIELD(BWMON_V5_THRESHOLD_LOW, 0, 11), 391 [F_ZONE_ACTIONS_ZONE0] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 0, 7), 392 [F_ZONE_ACTIONS_ZONE1] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 8, 15), 393 [F_ZONE_ACTIONS_ZONE2] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 16, 23), 394 [F_ZONE_ACTIONS_ZONE3] = REG_FIELD(BWMON_V5_ZONE_ACTIONS, 24, 31), 395 [F_THRESHOLD_COUNT_ZONE0] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 0, 7), 396 [F_THRESHOLD_COUNT_ZONE1] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 8, 15), 397 [F_THRESHOLD_COUNT_ZONE2] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 16, 23), 398 [F_THRESHOLD_COUNT_ZONE3] = REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 24, 31), 399 [F_ZONE0_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(0), 0, 11), 400 [F_ZONE1_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(1), 0, 11), 401 [F_ZONE2_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(2), 0, 11), 402 [F_ZONE3_MAX] = REG_FIELD(BWMON_V5_ZONE_MAX(3), 0, 11), 403 }; 404 405 static const struct regmap_range sdm845_llcc_bwmon_reg_noread_ranges[] = { 406 regmap_reg_range(BWMON_V5_IRQ_CLEAR, BWMON_V5_IRQ_CLEAR), 407 regmap_reg_range(BWMON_V5_CLEAR, BWMON_V5_CLEAR), 408 }; 409 410 static const struct regmap_access_table sdm845_llcc_bwmon_reg_read_table = { 411 .no_ranges = sdm845_llcc_bwmon_reg_noread_ranges, 412 .n_no_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_noread_ranges), 413 }; 414 415 static const struct regmap_range sdm845_llcc_bwmon_reg_volatile_ranges[] = { 416 regmap_reg_range(BWMON_V5_IRQ_STATUS, BWMON_V5_IRQ_STATUS), 417 regmap_reg_range(BWMON_V5_ZONE_MAX(0), BWMON_V5_ZONE_MAX(3)), 418 }; 419 420 static const struct regmap_access_table sdm845_llcc_bwmon_reg_volatile_table = { 421 .yes_ranges = sdm845_llcc_bwmon_reg_volatile_ranges, 422 .n_yes_ranges = ARRAY_SIZE(sdm845_llcc_bwmon_reg_volatile_ranges), 423 }; 424 425 /* 426 * Fill the cache for non-readable registers only as rest does not really 427 * matter and can be read from the device. 428 */ 429 static const struct reg_default sdm845_llcc_bwmon_reg_defaults[] = { 430 { BWMON_V5_IRQ_CLEAR, 0x0 }, 431 { BWMON_V5_CLEAR, 0x0 }, 432 }; 433 434 static const struct regmap_config sdm845_llcc_bwmon_regmap_cfg = { 435 .reg_bits = 32, 436 .reg_stride = 4, 437 .val_bits = 32, 438 /* 439 * No concurrent access expected - driver has one interrupt handler, 440 * regmap is not shared, no driver or user-space API. 441 */ 442 .disable_locking = true, 443 .rd_table = &sdm845_llcc_bwmon_reg_read_table, 444 .volatile_table = &sdm845_llcc_bwmon_reg_volatile_table, 445 .reg_defaults = sdm845_llcc_bwmon_reg_defaults, 446 .num_reg_defaults = ARRAY_SIZE(sdm845_llcc_bwmon_reg_defaults), 447 /* 448 * Cache is necessary for using regmap fields with non-readable 449 * registers. 450 */ 451 .cache_type = REGCACHE_MAPLE, 452 }; 453 454 static void bwmon_clear_counters(struct icc_bwmon *bwmon, bool clear_all) 455 { 456 unsigned int val = BWMON_CLEAR_CLEAR; 457 458 if (clear_all) 459 val |= BWMON_CLEAR_CLEAR_ALL; 460 /* 461 * Clear counters. The order and barriers are 462 * important. Quoting downstream Qualcomm msm-4.9 tree: 463 * 464 * The counter clear and IRQ clear bits are not in the same 4KB 465 * region. So, we need to make sure the counter clear is completed 466 * before we try to clear the IRQ or do any other counter operations. 467 */ 468 regmap_field_force_write(bwmon->regs[F_CLEAR], val); 469 if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR) 470 regmap_field_force_write(bwmon->regs[F_CLEAR], 0); 471 } 472 473 static void bwmon_clear_irq(struct icc_bwmon *bwmon) 474 { 475 struct regmap_field *global_irq_clr; 476 477 if (bwmon->data->global_regmap_fields) 478 global_irq_clr = bwmon->global_regs[F_GLOBAL_IRQ_CLEAR]; 479 else 480 global_irq_clr = bwmon->regs[F_GLOBAL_IRQ_CLEAR]; 481 482 /* 483 * Clear zone and global interrupts. The order and barriers are 484 * important. Quoting downstream Qualcomm msm-4.9 tree: 485 * 486 * Synchronize the local interrupt clear in mon_irq_clear() 487 * with the global interrupt clear here. Otherwise, the CPU 488 * may reorder the two writes and clear the global interrupt 489 * before the local interrupt, causing the global interrupt 490 * to be retriggered by the local interrupt still being high. 491 * 492 * Similarly, because the global registers are in a different 493 * region than the local registers, we need to ensure any register 494 * writes to enable the monitor after this call are ordered with the 495 * clearing here so that local writes don't happen before the 496 * interrupt is cleared. 497 */ 498 regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], BWMON_IRQ_ENABLE_MASK); 499 if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR) 500 regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], 0); 501 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ) 502 regmap_field_force_write(global_irq_clr, 503 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE); 504 } 505 506 static void bwmon_disable(struct icc_bwmon *bwmon) 507 { 508 struct regmap_field *global_irq_en; 509 510 if (bwmon->data->global_regmap_fields) 511 global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE]; 512 else 513 global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE]; 514 515 /* Disable interrupts. Strict ordering, see bwmon_clear_irq(). */ 516 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ) 517 regmap_field_write(global_irq_en, 0x0); 518 regmap_field_write(bwmon->regs[F_IRQ_ENABLE], 0x0); 519 520 /* 521 * Disable bwmon. Must happen before bwmon_clear_irq() to avoid spurious 522 * IRQ. 523 */ 524 regmap_field_write(bwmon->regs[F_ENABLE], 0x0); 525 } 526 527 static void bwmon_enable(struct icc_bwmon *bwmon, unsigned int irq_enable) 528 { 529 struct regmap_field *global_irq_en; 530 531 if (bwmon->data->global_regmap_fields) 532 global_irq_en = bwmon->global_regs[F_GLOBAL_IRQ_ENABLE]; 533 else 534 global_irq_en = bwmon->regs[F_GLOBAL_IRQ_ENABLE]; 535 536 /* Enable interrupts */ 537 if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ) 538 regmap_field_write(global_irq_en, 539 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE); 540 541 regmap_field_write(bwmon->regs[F_IRQ_ENABLE], irq_enable); 542 543 /* Enable bwmon */ 544 regmap_field_write(bwmon->regs[F_ENABLE], BWMON_ENABLE_ENABLE); 545 } 546 547 static unsigned int bwmon_kbps_to_count(struct icc_bwmon *bwmon, 548 unsigned int kbps) 549 { 550 return kbps / bwmon->data->count_unit_kb; 551 } 552 553 static void bwmon_set_threshold(struct icc_bwmon *bwmon, 554 struct regmap_field *reg, unsigned int kbps) 555 { 556 unsigned int thres; 557 558 thres = mult_frac(bwmon_kbps_to_count(bwmon, kbps), 559 bwmon->data->sample_ms, MSEC_PER_SEC); 560 regmap_field_write(reg, thres); 561 } 562 563 static void bwmon_start(struct icc_bwmon *bwmon) 564 { 565 const struct icc_bwmon_data *data = bwmon->data; 566 u32 bw_low = 0; 567 int window; 568 569 /* No need to check for errors, as this must have succeeded before. */ 570 dev_pm_opp_put(dev_pm_opp_find_bw_ceil(bwmon->dev, &bw_low, 0)); 571 572 bwmon_clear_counters(bwmon, true); 573 574 window = mult_frac(bwmon->data->sample_ms, HW_TIMER_HZ, MSEC_PER_SEC); 575 /* Maximum sampling window: 0xffffff for v4 and 0xfffff for v5 */ 576 regmap_field_write(bwmon->regs[F_SAMPLE_WINDOW], window); 577 578 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH], bw_low); 579 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED], bw_low); 580 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_LOW], 0); 581 582 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE0], 583 BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT); 584 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE1], 585 data->zone1_thres_count); 586 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE2], 587 BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT); 588 regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE3], 589 data->zone3_thres_count); 590 591 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE0], 592 BWMON_ZONE_ACTIONS_ZONE0); 593 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE1], 594 BWMON_ZONE_ACTIONS_ZONE1); 595 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE2], 596 BWMON_ZONE_ACTIONS_ZONE2); 597 regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE3], 598 BWMON_ZONE_ACTIONS_ZONE3); 599 600 bwmon_clear_irq(bwmon); 601 bwmon_enable(bwmon, BWMON_IRQ_ENABLE_MASK); 602 } 603 604 static irqreturn_t bwmon_intr(int irq, void *dev_id) 605 { 606 struct icc_bwmon *bwmon = dev_id; 607 unsigned int status, max; 608 int zone; 609 610 if (regmap_field_read(bwmon->regs[F_IRQ_STATUS], &status)) 611 return IRQ_NONE; 612 613 status &= BWMON_IRQ_ENABLE_MASK; 614 if (!status) { 615 /* 616 * Only zone 1 and zone 3 interrupts are enabled but zone 2 617 * threshold could be hit and trigger interrupt even if not 618 * enabled. 619 * Such spurious interrupt might come with valuable max count or 620 * not, so solution would be to always check all 621 * BWMON_ZONE_MAX() registers to find the highest value. 622 * Such case is currently ignored. 623 */ 624 return IRQ_NONE; 625 } 626 627 bwmon_disable(bwmon); 628 629 zone = get_bitmask_order(status) - 1; 630 /* 631 * Zone max bytes count register returns count units within sampling 632 * window. Downstream kernel for BWMONv4 (called BWMON type 2 in 633 * downstream) always increments the max bytes count by one. 634 */ 635 if (regmap_field_read(bwmon->regs[F_ZONE0_MAX + zone], &max)) 636 return IRQ_NONE; 637 638 max += 1; 639 max *= bwmon->data->count_unit_kb; 640 bwmon->target_kbps = mult_frac(max, MSEC_PER_SEC, bwmon->data->sample_ms); 641 642 return IRQ_WAKE_THREAD; 643 } 644 645 static irqreturn_t bwmon_intr_thread(int irq, void *dev_id) 646 { 647 struct icc_bwmon *bwmon = dev_id; 648 unsigned int irq_enable = 0; 649 struct dev_pm_opp *opp, *target_opp; 650 unsigned int bw_kbps, up_kbps, down_kbps, meas_kbps; 651 652 bw_kbps = bwmon->target_kbps; 653 meas_kbps = bwmon->target_kbps; 654 655 target_opp = dev_pm_opp_find_bw_ceil(bwmon->dev, &bw_kbps, 0); 656 if (IS_ERR(target_opp) && PTR_ERR(target_opp) == -ERANGE) 657 target_opp = dev_pm_opp_find_bw_floor(bwmon->dev, &bw_kbps, 0); 658 659 bwmon->target_kbps = bw_kbps; 660 661 bw_kbps--; 662 opp = dev_pm_opp_find_bw_floor(bwmon->dev, &bw_kbps, 0); 663 if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE) 664 down_kbps = bwmon->target_kbps; 665 else 666 down_kbps = bw_kbps; 667 668 up_kbps = bwmon->target_kbps + 1; 669 670 if (bwmon->target_kbps >= bwmon->max_bw_kbps) 671 irq_enable = BIT(1); 672 else if (bwmon->target_kbps <= bwmon->min_bw_kbps) 673 irq_enable = BIT(3); 674 else 675 irq_enable = BWMON_IRQ_ENABLE_MASK; 676 677 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH], 678 up_kbps); 679 bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED], 680 down_kbps); 681 bwmon_clear_counters(bwmon, false); 682 bwmon_clear_irq(bwmon); 683 bwmon_enable(bwmon, irq_enable); 684 685 trace_qcom_bwmon_update(dev_name(bwmon->dev), meas_kbps, up_kbps, down_kbps); 686 if (bwmon->target_kbps == bwmon->current_kbps) 687 goto out; 688 689 dev_pm_opp_set_opp(bwmon->dev, target_opp); 690 bwmon->current_kbps = bwmon->target_kbps; 691 692 out: 693 dev_pm_opp_put(target_opp); 694 if (!IS_ERR(opp)) 695 dev_pm_opp_put(opp); 696 697 return IRQ_HANDLED; 698 } 699 700 static int bwmon_init_regmap(struct platform_device *pdev, 701 struct icc_bwmon *bwmon) 702 { 703 struct device *dev = &pdev->dev; 704 void __iomem *base; 705 struct regmap *map; 706 int ret; 707 708 /* Map the monitor base */ 709 base = devm_platform_ioremap_resource(pdev, 0); 710 if (IS_ERR(base)) 711 return dev_err_probe(dev, PTR_ERR(base), 712 "failed to map bwmon registers\n"); 713 714 map = devm_regmap_init_mmio(dev, base, bwmon->data->regmap_cfg); 715 if (IS_ERR(map)) 716 return dev_err_probe(dev, PTR_ERR(map), 717 "failed to initialize regmap\n"); 718 719 BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_global_reg_fields) != F_NUM_GLOBAL_FIELDS); 720 BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_reg_fields) != F_NUM_FIELDS); 721 BUILD_BUG_ON(ARRAY_SIZE(sdm845_cpu_bwmon_reg_fields) != F_NUM_FIELDS); 722 BUILD_BUG_ON(ARRAY_SIZE(sdm845_llcc_bwmon_reg_fields) != F_NUM_FIELDS); 723 724 ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->regs, 725 bwmon->data->regmap_fields, 726 F_NUM_FIELDS); 727 if (ret) 728 return ret; 729 730 if (bwmon->data->global_regmap_cfg) { 731 /* Map the global base, if separate */ 732 base = devm_platform_ioremap_resource(pdev, 1); 733 if (IS_ERR(base)) 734 return dev_err_probe(dev, PTR_ERR(base), 735 "failed to map bwmon global registers\n"); 736 737 map = devm_regmap_init_mmio(dev, base, bwmon->data->global_regmap_cfg); 738 if (IS_ERR(map)) 739 return dev_err_probe(dev, PTR_ERR(map), 740 "failed to initialize global regmap\n"); 741 742 ret = devm_regmap_field_bulk_alloc(dev, map, bwmon->global_regs, 743 bwmon->data->global_regmap_fields, 744 F_NUM_GLOBAL_FIELDS); 745 } 746 747 return ret; 748 } 749 750 static int bwmon_probe(struct platform_device *pdev) 751 { 752 struct device *dev = &pdev->dev; 753 struct dev_pm_opp *opp; 754 struct icc_bwmon *bwmon; 755 int ret; 756 757 bwmon = devm_kzalloc(dev, sizeof(*bwmon), GFP_KERNEL); 758 if (!bwmon) 759 return -ENOMEM; 760 761 bwmon->data = of_device_get_match_data(dev); 762 763 ret = bwmon_init_regmap(pdev, bwmon); 764 if (ret) 765 return ret; 766 767 bwmon->irq = platform_get_irq(pdev, 0); 768 if (bwmon->irq < 0) 769 return bwmon->irq; 770 771 ret = devm_pm_opp_of_add_table(dev); 772 if (ret) 773 return dev_err_probe(dev, ret, "failed to add OPP table\n"); 774 775 bwmon->max_bw_kbps = UINT_MAX; 776 opp = dev_pm_opp_find_bw_floor(dev, &bwmon->max_bw_kbps, 0); 777 if (IS_ERR(opp)) 778 return dev_err_probe(dev, PTR_ERR(opp), "failed to find max peak bandwidth\n"); 779 dev_pm_opp_put(opp); 780 781 bwmon->min_bw_kbps = 0; 782 opp = dev_pm_opp_find_bw_ceil(dev, &bwmon->min_bw_kbps, 0); 783 if (IS_ERR(opp)) 784 return dev_err_probe(dev, PTR_ERR(opp), "failed to find min peak bandwidth\n"); 785 dev_pm_opp_put(opp); 786 787 bwmon->dev = dev; 788 789 bwmon_disable(bwmon); 790 791 /* 792 * SoCs with multiple cpu-bwmon instances can end up using a shared interrupt 793 * line. Using the devm_ variant might result in the IRQ handler being executed 794 * after bwmon_disable in bwmon_remove() 795 */ 796 ret = request_threaded_irq(bwmon->irq, bwmon_intr, bwmon_intr_thread, 797 IRQF_ONESHOT | IRQF_SHARED, dev_name(dev), bwmon); 798 if (ret) 799 return dev_err_probe(dev, ret, "failed to request IRQ\n"); 800 801 platform_set_drvdata(pdev, bwmon); 802 bwmon_start(bwmon); 803 804 return 0; 805 } 806 807 static void bwmon_remove(struct platform_device *pdev) 808 { 809 struct icc_bwmon *bwmon = platform_get_drvdata(pdev); 810 811 bwmon_disable(bwmon); 812 free_irq(bwmon->irq, bwmon); 813 } 814 815 static const struct icc_bwmon_data msm8998_bwmon_data = { 816 .sample_ms = 4, 817 .count_unit_kb = 1024, 818 .zone1_thres_count = 16, 819 .zone3_thres_count = 1, 820 .quirks = BWMON_HAS_GLOBAL_IRQ, 821 .regmap_fields = msm8998_bwmon_reg_fields, 822 .regmap_cfg = &msm8998_bwmon_regmap_cfg, 823 .global_regmap_fields = msm8998_bwmon_global_reg_fields, 824 .global_regmap_cfg = &msm8998_bwmon_global_regmap_cfg, 825 }; 826 827 static const struct icc_bwmon_data sdm845_cpu_bwmon_data = { 828 .sample_ms = 4, 829 .count_unit_kb = 64, 830 .zone1_thres_count = 16, 831 .zone3_thres_count = 1, 832 .quirks = BWMON_HAS_GLOBAL_IRQ, 833 .regmap_fields = sdm845_cpu_bwmon_reg_fields, 834 .regmap_cfg = &sdm845_cpu_bwmon_regmap_cfg, 835 }; 836 837 static const struct icc_bwmon_data sdm845_llcc_bwmon_data = { 838 .sample_ms = 4, 839 .count_unit_kb = 1024, 840 .zone1_thres_count = 16, 841 .zone3_thres_count = 1, 842 .regmap_fields = sdm845_llcc_bwmon_reg_fields, 843 .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg, 844 }; 845 846 static const struct icc_bwmon_data sc7280_llcc_bwmon_data = { 847 .sample_ms = 4, 848 .count_unit_kb = 64, 849 .zone1_thres_count = 16, 850 .zone3_thres_count = 1, 851 .quirks = BWMON_NEEDS_FORCE_CLEAR, 852 .regmap_fields = sdm845_llcc_bwmon_reg_fields, 853 .regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg, 854 }; 855 856 static const struct of_device_id bwmon_of_match[] = { 857 /* BWMONv4, separate monitor and global register spaces */ 858 { .compatible = "qcom,msm8998-bwmon", .data = &msm8998_bwmon_data }, 859 /* BWMONv4, unified register space */ 860 { .compatible = "qcom,sdm845-bwmon", .data = &sdm845_cpu_bwmon_data }, 861 /* BWMONv5 */ 862 { .compatible = "qcom,sdm845-llcc-bwmon", .data = &sdm845_llcc_bwmon_data }, 863 { .compatible = "qcom,sc7280-llcc-bwmon", .data = &sc7280_llcc_bwmon_data }, 864 865 /* Compatibles kept for legacy reasons */ 866 { .compatible = "qcom,sc7280-cpu-bwmon", .data = &sdm845_cpu_bwmon_data }, 867 { .compatible = "qcom,sc8280xp-cpu-bwmon", .data = &sdm845_cpu_bwmon_data }, 868 { .compatible = "qcom,sm8550-cpu-bwmon", .data = &sdm845_cpu_bwmon_data }, 869 {} 870 }; 871 MODULE_DEVICE_TABLE(of, bwmon_of_match); 872 873 static struct platform_driver bwmon_driver = { 874 .probe = bwmon_probe, 875 .remove_new = bwmon_remove, 876 .driver = { 877 .name = "qcom-bwmon", 878 .of_match_table = bwmon_of_match, 879 }, 880 }; 881 module_platform_driver(bwmon_driver); 882 883 MODULE_AUTHOR("Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>"); 884 MODULE_DESCRIPTION("QCOM BWMON driver"); 885 MODULE_LICENSE("GPL"); 886