1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * System Control and Management Interface (SCMI) Message Protocol driver 4 * 5 * SCMI Message Protocol is used between the System Control Processor(SCP) 6 * and the Application Processors(AP). The Message Handling Unit(MHU) 7 * provides a mechanism for inter-processor communication between SCP's 8 * Cortex M3 and AP. 9 * 10 * SCP offers control and management of the core/cluster power states, 11 * various power domain DVFS including the core/cluster, certain system 12 * clocks configuration, thermal sensors and many others. 13 * 14 * Copyright (C) 2018-2025 ARM Ltd. 15 */ 16 17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 18 19 #include <linux/bitmap.h> 20 #include <linux/cleanup.h> 21 #include <linux/debugfs.h> 22 #include <linux/device.h> 23 #include <linux/export.h> 24 #include <linux/idr.h> 25 #include <linux/io.h> 26 #include <linux/io-64-nonatomic-hi-lo.h> 27 #include <linux/kernel.h> 28 #include <linux/kmod.h> 29 #include <linux/ktime.h> 30 #include <linux/hashtable.h> 31 #include <linux/list.h> 32 #include <linux/module.h> 33 #include <linux/of.h> 34 #include <linux/platform_device.h> 35 #include <linux/processor.h> 36 #include <linux/refcount.h> 37 #include <linux/slab.h> 38 #include <linux/xarray.h> 39 40 #include "common.h" 41 #include "notify.h" 42 #include "quirks.h" 43 44 #include "raw_mode.h" 45 46 #define CREATE_TRACE_POINTS 47 #include <trace/events/scmi.h> 48 49 #define SCMI_VENDOR_MODULE_ALIAS_FMT "scmi-protocol-0x%02x-%s" 50 51 static DEFINE_IDA(scmi_id); 52 53 static DEFINE_XARRAY(scmi_protocols); 54 55 /* List of all SCMI devices active in system */ 56 static LIST_HEAD(scmi_list); 57 /* Protection for the entire list */ 58 static DEFINE_MUTEX(scmi_list_mutex); 59 /* Track the unique id for the transfers for debug & profiling purpose */ 60 static atomic_t transfer_last_id; 61 62 static struct dentry *scmi_top_dentry; 63 64 /** 65 * struct scmi_xfers_info - Structure to manage transfer information 66 * 67 * @xfer_alloc_table: Bitmap table for allocated messages. 68 * Index of this bitmap table is also used for message 69 * sequence identifier. 70 * @xfer_lock: Protection for message allocation 71 * @max_msg: Maximum number of messages that can be pending 72 * @free_xfers: A free list for available to use xfers. It is initialized with 73 * a number of xfers equal to the maximum allowed in-flight 74 * messages. 75 * @pending_xfers: An hashtable, indexed by msg_hdr.seq, used to keep all the 76 * currently in-flight messages. 77 */ 78 struct scmi_xfers_info { 79 unsigned long *xfer_alloc_table; 80 spinlock_t xfer_lock; 81 int max_msg; 82 struct hlist_head free_xfers; 83 DECLARE_HASHTABLE(pending_xfers, SCMI_PENDING_XFERS_HT_ORDER_SZ); 84 }; 85 86 /** 87 * struct scmi_protocol_instance - Describe an initialized protocol instance. 88 * @handle: Reference to the SCMI handle associated to this protocol instance. 89 * @proto: A reference to the protocol descriptor. 90 * @gid: A reference for per-protocol devres management. 91 * @users: A refcount to track effective users of this protocol. 92 * @priv: Reference for optional protocol private data. 93 * @version: Protocol version supported by the platform as detected at runtime. 94 * @negotiated_version: When the platform supports a newer protocol version, 95 * the agent will try to negotiate with the platform the 96 * usage of the newest version known to it, since 97 * backward compatibility is NOT automatically assured. 98 * This field is NON-zero when a successful negotiation 99 * has completed. 100 * @ph: An embedded protocol handle that will be passed down to protocol 101 * initialization code to identify this instance. 102 * 103 * Each protocol is initialized independently once for each SCMI platform in 104 * which is defined by DT and implemented by the SCMI server fw. 105 */ 106 struct scmi_protocol_instance { 107 const struct scmi_handle *handle; 108 const struct scmi_protocol *proto; 109 void *gid; 110 refcount_t users; 111 void *priv; 112 unsigned int version; 113 unsigned int negotiated_version; 114 struct scmi_protocol_handle ph; 115 }; 116 117 #define ph_to_pi(h) container_of(h, struct scmi_protocol_instance, ph) 118 119 /** 120 * struct scmi_info - Structure representing a SCMI instance 121 * 122 * @id: A sequence number starting from zero identifying this instance 123 * @dev: Device pointer 124 * @desc: SoC description for this instance 125 * @version: SCMI revision information containing protocol version, 126 * implementation version and (sub-)vendor identification. 127 * @handle: Instance of SCMI handle to send to clients 128 * @tx_minfo: Universal Transmit Message management info 129 * @rx_minfo: Universal Receive Message management info 130 * @tx_idr: IDR object to map protocol id to Tx channel info pointer 131 * @rx_idr: IDR object to map protocol id to Rx channel info pointer 132 * @protocols: IDR for protocols' instance descriptors initialized for 133 * this SCMI instance: populated on protocol's first attempted 134 * usage. 135 * @protocols_mtx: A mutex to protect protocols instances initialization. 136 * @protocols_imp: List of protocols implemented, currently maximum of 137 * scmi_base_info.num_protocols elements allocated by the 138 * base protocol 139 * @active_protocols: IDR storing device_nodes for protocols actually defined 140 * in the DT and confirmed as implemented by fw. 141 * @notify_priv: Pointer to private data structure specific to notifications. 142 * @node: List head 143 * @users: Number of users of this instance 144 * @bus_nb: A notifier to listen for device bind/unbind on the scmi bus 145 * @dev_req_nb: A notifier to listen for device request/unrequest on the scmi 146 * bus 147 * @devreq_mtx: A mutex to serialize device creation for this SCMI instance 148 * @dbg: A pointer to debugfs related data (if any) 149 * @raw: An opaque reference handle used by SCMI Raw mode. 150 */ 151 struct scmi_info { 152 int id; 153 struct device *dev; 154 const struct scmi_desc *desc; 155 struct scmi_base_info version; 156 struct scmi_handle handle; 157 struct scmi_xfers_info tx_minfo; 158 struct scmi_xfers_info rx_minfo; 159 struct idr tx_idr; 160 struct idr rx_idr; 161 struct idr protocols; 162 /* Ensure mutual exclusive access to protocols instance array */ 163 struct mutex protocols_mtx; 164 u8 *protocols_imp; 165 struct idr active_protocols; 166 void *notify_priv; 167 struct list_head node; 168 int users; 169 struct notifier_block bus_nb; 170 struct notifier_block dev_req_nb; 171 /* Serialize device creation process for this instance */ 172 struct mutex devreq_mtx; 173 struct scmi_debug_info *dbg; 174 void *raw; 175 }; 176 177 #define handle_to_scmi_info(h) container_of(h, struct scmi_info, handle) 178 #define tx_minfo_to_scmi_info(h) container_of(h, struct scmi_info, tx_minfo) 179 #define bus_nb_to_scmi_info(nb) container_of(nb, struct scmi_info, bus_nb) 180 #define req_nb_to_scmi_info(nb) container_of(nb, struct scmi_info, dev_req_nb) 181 182 static void scmi_rx_callback(struct scmi_chan_info *cinfo, 183 u32 msg_hdr, void *priv); 184 static void scmi_bad_message_trace(struct scmi_chan_info *cinfo, 185 u32 msg_hdr, enum scmi_bad_msg err); 186 187 static struct scmi_transport_core_operations scmi_trans_core_ops = { 188 .bad_message_trace = scmi_bad_message_trace, 189 .rx_callback = scmi_rx_callback, 190 }; 191 192 static unsigned long 193 scmi_vendor_protocol_signature(unsigned int protocol_id, char *vendor_id, 194 char *sub_vendor_id, u32 impl_ver) 195 { 196 char *signature, *p; 197 unsigned long hash = 0; 198 199 /* vendor_id/sub_vendor_id guaranteed <= SCMI_SHORT_NAME_MAX_SIZE */ 200 signature = kasprintf(GFP_KERNEL, "%02X|%s|%s|0x%08X", protocol_id, 201 vendor_id ?: "", sub_vendor_id ?: "", impl_ver); 202 if (!signature) 203 return 0; 204 205 p = signature; 206 while (*p) 207 hash = partial_name_hash(tolower(*p++), hash); 208 hash = end_name_hash(hash); 209 210 kfree(signature); 211 212 return hash; 213 } 214 215 static unsigned long 216 scmi_protocol_key_calculate(int protocol_id, char *vendor_id, 217 char *sub_vendor_id, u32 impl_ver) 218 { 219 if (protocol_id < SCMI_PROTOCOL_VENDOR_BASE) 220 return protocol_id; 221 else 222 return scmi_vendor_protocol_signature(protocol_id, vendor_id, 223 sub_vendor_id, impl_ver); 224 } 225 226 static const struct scmi_protocol * 227 __scmi_vendor_protocol_lookup(int protocol_id, char *vendor_id, 228 char *sub_vendor_id, u32 impl_ver) 229 { 230 unsigned long key; 231 struct scmi_protocol *proto = NULL; 232 233 key = scmi_protocol_key_calculate(protocol_id, vendor_id, 234 sub_vendor_id, impl_ver); 235 if (key) 236 proto = xa_load(&scmi_protocols, key); 237 238 return proto; 239 } 240 241 static const struct scmi_protocol * 242 scmi_vendor_protocol_lookup(int protocol_id, char *vendor_id, 243 char *sub_vendor_id, u32 impl_ver) 244 { 245 const struct scmi_protocol *proto = NULL; 246 247 /* Searching for closest match ...*/ 248 proto = __scmi_vendor_protocol_lookup(protocol_id, vendor_id, 249 sub_vendor_id, impl_ver); 250 if (proto) 251 return proto; 252 253 /* Any match just on vendor/sub_vendor ? */ 254 if (impl_ver) { 255 proto = __scmi_vendor_protocol_lookup(protocol_id, vendor_id, 256 sub_vendor_id, 0); 257 if (proto) 258 return proto; 259 } 260 261 /* Any match just on the vendor ? */ 262 if (sub_vendor_id) 263 proto = __scmi_vendor_protocol_lookup(protocol_id, vendor_id, 264 NULL, 0); 265 return proto; 266 } 267 268 static const struct scmi_protocol * 269 scmi_vendor_protocol_get(int protocol_id, struct scmi_base_info *version) 270 { 271 const struct scmi_protocol *proto; 272 273 proto = scmi_vendor_protocol_lookup(protocol_id, version->vendor_id, 274 version->sub_vendor_id, 275 version->impl_ver); 276 if (!proto) { 277 int ret; 278 279 pr_debug("Looking for '" SCMI_VENDOR_MODULE_ALIAS_FMT "'\n", 280 protocol_id, version->vendor_id); 281 282 /* Note that vendor_id is mandatory for vendor protocols */ 283 ret = request_module(SCMI_VENDOR_MODULE_ALIAS_FMT, 284 protocol_id, version->vendor_id); 285 if (ret) { 286 pr_warn("Problem loading module for protocol 0x%x\n", 287 protocol_id); 288 return NULL; 289 } 290 291 /* Lookup again, once modules loaded */ 292 proto = scmi_vendor_protocol_lookup(protocol_id, 293 version->vendor_id, 294 version->sub_vendor_id, 295 version->impl_ver); 296 } 297 298 if (proto) 299 pr_info("Loaded SCMI Vendor Protocol 0x%x - %s %s %X\n", 300 protocol_id, proto->vendor_id ?: "", 301 proto->sub_vendor_id ?: "", proto->impl_ver); 302 303 return proto; 304 } 305 306 static const struct scmi_protocol * 307 scmi_protocol_get(int protocol_id, struct scmi_base_info *version) 308 { 309 const struct scmi_protocol *proto = NULL; 310 311 if (protocol_id < SCMI_PROTOCOL_VENDOR_BASE) 312 proto = xa_load(&scmi_protocols, protocol_id); 313 else 314 proto = scmi_vendor_protocol_get(protocol_id, version); 315 316 if (!proto || !try_module_get(proto->owner)) { 317 pr_warn("SCMI Protocol 0x%x not found!\n", protocol_id); 318 return NULL; 319 } 320 321 pr_debug("Found SCMI Protocol 0x%x\n", protocol_id); 322 323 return proto; 324 } 325 326 static void scmi_protocol_put(const struct scmi_protocol *proto) 327 { 328 if (proto) 329 module_put(proto->owner); 330 } 331 332 static int scmi_vendor_protocol_check(const struct scmi_protocol *proto) 333 { 334 if (!proto->vendor_id) { 335 pr_err("missing vendor_id for protocol 0x%x\n", proto->id); 336 return -EINVAL; 337 } 338 339 if (strlen(proto->vendor_id) >= SCMI_SHORT_NAME_MAX_SIZE) { 340 pr_err("malformed vendor_id for protocol 0x%x\n", proto->id); 341 return -EINVAL; 342 } 343 344 if (proto->sub_vendor_id && 345 strlen(proto->sub_vendor_id) >= SCMI_SHORT_NAME_MAX_SIZE) { 346 pr_err("malformed sub_vendor_id for protocol 0x%x\n", 347 proto->id); 348 return -EINVAL; 349 } 350 351 return 0; 352 } 353 354 int scmi_protocol_register(const struct scmi_protocol *proto) 355 { 356 int ret; 357 unsigned long key; 358 359 if (!proto) { 360 pr_err("invalid protocol\n"); 361 return -EINVAL; 362 } 363 364 if (!proto->instance_init) { 365 pr_err("missing init for protocol 0x%x\n", proto->id); 366 return -EINVAL; 367 } 368 369 if (proto->id >= SCMI_PROTOCOL_VENDOR_BASE && 370 scmi_vendor_protocol_check(proto)) 371 return -EINVAL; 372 373 /* 374 * Calculate a protocol key to register this protocol with the core; 375 * key value 0 is considered invalid. 376 */ 377 key = scmi_protocol_key_calculate(proto->id, proto->vendor_id, 378 proto->sub_vendor_id, 379 proto->impl_ver); 380 if (!key) 381 return -EINVAL; 382 383 ret = xa_insert(&scmi_protocols, key, (void *)proto, GFP_KERNEL); 384 if (ret) { 385 pr_err("unable to allocate SCMI protocol slot for 0x%x - err %d\n", 386 proto->id, ret); 387 return ret; 388 } 389 390 pr_debug("Registered SCMI Protocol 0x%x - %s %s 0x%08X\n", 391 proto->id, proto->vendor_id, proto->sub_vendor_id, 392 proto->impl_ver); 393 394 return 0; 395 } 396 EXPORT_SYMBOL_GPL(scmi_protocol_register); 397 398 void scmi_protocol_unregister(const struct scmi_protocol *proto) 399 { 400 unsigned long key; 401 402 key = scmi_protocol_key_calculate(proto->id, proto->vendor_id, 403 proto->sub_vendor_id, 404 proto->impl_ver); 405 if (!key) 406 return; 407 408 xa_erase(&scmi_protocols, key); 409 410 pr_debug("Unregistered SCMI Protocol 0x%x\n", proto->id); 411 } 412 EXPORT_SYMBOL_GPL(scmi_protocol_unregister); 413 414 /** 415 * scmi_create_protocol_devices - Create devices for all pending requests for 416 * this SCMI instance. 417 * 418 * @np: The device node describing the protocol 419 * @info: The SCMI instance descriptor 420 * @prot_id: The protocol ID 421 * @name: The optional name of the device to be created: if not provided this 422 * call will lead to the creation of all the devices currently requested 423 * for the specified protocol. 424 */ 425 static void scmi_create_protocol_devices(struct device_node *np, 426 struct scmi_info *info, 427 int prot_id, const char *name) 428 { 429 mutex_lock(&info->devreq_mtx); 430 scmi_device_create(np, info->dev, prot_id, name); 431 mutex_unlock(&info->devreq_mtx); 432 } 433 434 static void scmi_destroy_protocol_devices(struct scmi_info *info, 435 int prot_id, const char *name) 436 { 437 mutex_lock(&info->devreq_mtx); 438 scmi_device_destroy(info->dev, prot_id, name); 439 mutex_unlock(&info->devreq_mtx); 440 } 441 442 void scmi_notification_instance_data_set(const struct scmi_handle *handle, 443 void *priv) 444 { 445 struct scmi_info *info = handle_to_scmi_info(handle); 446 447 info->notify_priv = priv; 448 /* Ensure updated protocol private date are visible */ 449 smp_wmb(); 450 } 451 452 void *scmi_notification_instance_data_get(const struct scmi_handle *handle) 453 { 454 struct scmi_info *info = handle_to_scmi_info(handle); 455 456 /* Ensure protocols_private_data has been updated */ 457 smp_rmb(); 458 return info->notify_priv; 459 } 460 461 /** 462 * scmi_xfer_token_set - Reserve and set new token for the xfer at hand 463 * 464 * @minfo: Pointer to Tx/Rx Message management info based on channel type 465 * @xfer: The xfer to act upon 466 * 467 * Pick the next unused monotonically increasing token and set it into 468 * xfer->hdr.seq: picking a monotonically increasing value avoids immediate 469 * reuse of freshly completed or timed-out xfers, thus mitigating the risk 470 * of incorrect association of a late and expired xfer with a live in-flight 471 * transaction, both happening to re-use the same token identifier. 472 * 473 * Since platform is NOT required to answer our request in-order we should 474 * account for a few rare but possible scenarios: 475 * 476 * - exactly 'next_token' may be NOT available so pick xfer_id >= next_token 477 * using find_next_zero_bit() starting from candidate next_token bit 478 * 479 * - all tokens ahead upto (MSG_TOKEN_ID_MASK - 1) are used in-flight but we 480 * are plenty of free tokens at start, so try a second pass using 481 * find_next_zero_bit() and starting from 0. 482 * 483 * X = used in-flight 484 * 485 * Normal 486 * ------ 487 * 488 * |- xfer_id picked 489 * -----------+---------------------------------------------------------- 490 * | | |X|X|X| | | | | | ... ... ... ... ... ... ... ... ... ... ...|X|X| 491 * ---------------------------------------------------------------------- 492 * ^ 493 * |- next_token 494 * 495 * Out-of-order pending at start 496 * ----------------------------- 497 * 498 * |- xfer_id picked, last_token fixed 499 * -----+---------------------------------------------------------------- 500 * |X|X| | | | |X|X| ... ... ... ... ... ... ... ... ... ... ... ...|X| | 501 * ---------------------------------------------------------------------- 502 * ^ 503 * |- next_token 504 * 505 * 506 * Out-of-order pending at end 507 * --------------------------- 508 * 509 * |- xfer_id picked, last_token fixed 510 * -----+---------------------------------------------------------------- 511 * |X|X| | | | |X|X| ... ... ... ... ... ... ... ... ... ... |X|X|X||X|X| 512 * ---------------------------------------------------------------------- 513 * ^ 514 * |- next_token 515 * 516 * Context: Assumes to be called with @xfer_lock already acquired. 517 * 518 * Return: 0 on Success or error 519 */ 520 static int scmi_xfer_token_set(struct scmi_xfers_info *minfo, 521 struct scmi_xfer *xfer) 522 { 523 unsigned long xfer_id, next_token; 524 525 /* 526 * Pick a candidate monotonic token in range [0, MSG_TOKEN_MAX - 1] 527 * using the pre-allocated transfer_id as a base. 528 * Note that the global transfer_id is shared across all message types 529 * so there could be holes in the allocated set of monotonic sequence 530 * numbers, but that is going to limit the effectiveness of the 531 * mitigation only in very rare limit conditions. 532 */ 533 next_token = (xfer->transfer_id & (MSG_TOKEN_MAX - 1)); 534 535 /* Pick the next available xfer_id >= next_token */ 536 xfer_id = find_next_zero_bit(minfo->xfer_alloc_table, 537 MSG_TOKEN_MAX, next_token); 538 if (xfer_id == MSG_TOKEN_MAX) { 539 /* 540 * After heavily out-of-order responses, there are no free 541 * tokens ahead, but only at start of xfer_alloc_table so 542 * try again from the beginning. 543 */ 544 xfer_id = find_next_zero_bit(minfo->xfer_alloc_table, 545 MSG_TOKEN_MAX, 0); 546 /* 547 * Something is wrong if we got here since there can be a 548 * maximum number of (MSG_TOKEN_MAX - 1) in-flight messages 549 * but we have not found any free token [0, MSG_TOKEN_MAX - 1]. 550 */ 551 if (WARN_ON_ONCE(xfer_id == MSG_TOKEN_MAX)) 552 return -ENOMEM; 553 } 554 555 /* Update +/- last_token accordingly if we skipped some hole */ 556 if (xfer_id != next_token) 557 atomic_add((int)(xfer_id - next_token), &transfer_last_id); 558 559 xfer->hdr.seq = (u16)xfer_id; 560 561 return 0; 562 } 563 564 /** 565 * scmi_xfer_token_clear - Release the token 566 * 567 * @minfo: Pointer to Tx/Rx Message management info based on channel type 568 * @xfer: The xfer to act upon 569 */ 570 static inline void scmi_xfer_token_clear(struct scmi_xfers_info *minfo, 571 struct scmi_xfer *xfer) 572 { 573 clear_bit(xfer->hdr.seq, minfo->xfer_alloc_table); 574 } 575 576 /** 577 * scmi_xfer_inflight_register_unlocked - Register the xfer as in-flight 578 * 579 * @xfer: The xfer to register 580 * @minfo: Pointer to Tx/Rx Message management info based on channel type 581 * 582 * Note that this helper assumes that the xfer to be registered as in-flight 583 * had been built using an xfer sequence number which still corresponds to a 584 * free slot in the xfer_alloc_table. 585 * 586 * Context: Assumes to be called with @xfer_lock already acquired. 587 */ 588 static inline void 589 scmi_xfer_inflight_register_unlocked(struct scmi_xfer *xfer, 590 struct scmi_xfers_info *minfo) 591 { 592 /* In this context minfo will be tx_minfo due to the xfer pending */ 593 struct scmi_info *info = tx_minfo_to_scmi_info(minfo); 594 595 /* Set in-flight */ 596 set_bit(xfer->hdr.seq, minfo->xfer_alloc_table); 597 hash_add(minfo->pending_xfers, &xfer->node, xfer->hdr.seq); 598 scmi_inc_count(info->dbg, XFERS_INFLIGHT); 599 600 xfer->pending = true; 601 } 602 603 /** 604 * scmi_xfer_inflight_register - Try to register an xfer as in-flight 605 * 606 * @xfer: The xfer to register 607 * @minfo: Pointer to Tx/Rx Message management info based on channel type 608 * 609 * Note that this helper does NOT assume anything about the sequence number 610 * that was baked into the provided xfer, so it checks at first if it can 611 * be mapped to a free slot and fails with an error if another xfer with the 612 * same sequence number is currently still registered as in-flight. 613 * 614 * Return: 0 on Success or -EBUSY if sequence number embedded in the xfer 615 * could not rbe mapped to a free slot in the xfer_alloc_table. 616 */ 617 static int scmi_xfer_inflight_register(struct scmi_xfer *xfer, 618 struct scmi_xfers_info *minfo) 619 { 620 int ret = 0; 621 unsigned long flags; 622 623 spin_lock_irqsave(&minfo->xfer_lock, flags); 624 if (!test_bit(xfer->hdr.seq, minfo->xfer_alloc_table)) 625 scmi_xfer_inflight_register_unlocked(xfer, minfo); 626 else 627 ret = -EBUSY; 628 spin_unlock_irqrestore(&minfo->xfer_lock, flags); 629 630 return ret; 631 } 632 633 /** 634 * scmi_xfer_raw_inflight_register - An helper to register the given xfer as in 635 * flight on the TX channel, if possible. 636 * 637 * @handle: Pointer to SCMI entity handle 638 * @xfer: The xfer to register 639 * 640 * Return: 0 on Success, error otherwise 641 */ 642 int scmi_xfer_raw_inflight_register(const struct scmi_handle *handle, 643 struct scmi_xfer *xfer) 644 { 645 struct scmi_info *info = handle_to_scmi_info(handle); 646 647 return scmi_xfer_inflight_register(xfer, &info->tx_minfo); 648 } 649 650 /** 651 * scmi_xfer_pending_set - Pick a proper sequence number and mark the xfer 652 * as pending in-flight 653 * 654 * @xfer: The xfer to act upon 655 * @minfo: Pointer to Tx/Rx Message management info based on channel type 656 * 657 * Return: 0 on Success or error otherwise 658 */ 659 static inline int scmi_xfer_pending_set(struct scmi_xfer *xfer, 660 struct scmi_xfers_info *minfo) 661 { 662 int ret; 663 unsigned long flags; 664 665 spin_lock_irqsave(&minfo->xfer_lock, flags); 666 /* Set a new monotonic token as the xfer sequence number */ 667 ret = scmi_xfer_token_set(minfo, xfer); 668 if (!ret) 669 scmi_xfer_inflight_register_unlocked(xfer, minfo); 670 spin_unlock_irqrestore(&minfo->xfer_lock, flags); 671 672 return ret; 673 } 674 675 /** 676 * scmi_xfer_get() - Allocate one message 677 * 678 * @handle: Pointer to SCMI entity handle 679 * @minfo: Pointer to Tx/Rx Message management info based on channel type 680 * 681 * Helper function which is used by various message functions that are 682 * exposed to clients of this driver for allocating a message traffic event. 683 * 684 * Picks an xfer from the free list @free_xfers (if any available) and perform 685 * a basic initialization. 686 * 687 * Note that, at this point, still no sequence number is assigned to the 688 * allocated xfer, nor it is registered as a pending transaction. 689 * 690 * The successfully initialized xfer is refcounted. 691 * 692 * Context: Holds @xfer_lock while manipulating @free_xfers. 693 * 694 * Return: An initialized xfer if all went fine, else pointer error. 695 */ 696 static struct scmi_xfer *scmi_xfer_get(const struct scmi_handle *handle, 697 struct scmi_xfers_info *minfo) 698 { 699 unsigned long flags; 700 struct scmi_xfer *xfer; 701 702 spin_lock_irqsave(&minfo->xfer_lock, flags); 703 if (hlist_empty(&minfo->free_xfers)) { 704 spin_unlock_irqrestore(&minfo->xfer_lock, flags); 705 return ERR_PTR(-ENOMEM); 706 } 707 708 /* grab an xfer from the free_list */ 709 xfer = hlist_entry(minfo->free_xfers.first, struct scmi_xfer, node); 710 hlist_del_init(&xfer->node); 711 712 /* 713 * Allocate transfer_id early so that can be used also as base for 714 * monotonic sequence number generation if needed. 715 */ 716 xfer->transfer_id = atomic_inc_return(&transfer_last_id); 717 718 refcount_set(&xfer->users, 1); 719 atomic_set(&xfer->busy, SCMI_XFER_FREE); 720 spin_unlock_irqrestore(&minfo->xfer_lock, flags); 721 722 return xfer; 723 } 724 725 /** 726 * scmi_xfer_raw_get - Helper to get a bare free xfer from the TX channel 727 * 728 * @handle: Pointer to SCMI entity handle 729 * 730 * Note that xfer is taken from the TX channel structures. 731 * 732 * Return: A valid xfer on Success, or an error-pointer otherwise 733 */ 734 struct scmi_xfer *scmi_xfer_raw_get(const struct scmi_handle *handle) 735 { 736 struct scmi_xfer *xfer; 737 struct scmi_info *info = handle_to_scmi_info(handle); 738 739 xfer = scmi_xfer_get(handle, &info->tx_minfo); 740 if (!IS_ERR(xfer)) 741 xfer->flags |= SCMI_XFER_FLAG_IS_RAW; 742 743 return xfer; 744 } 745 746 /** 747 * scmi_xfer_raw_channel_get - Helper to get a reference to the proper channel 748 * to use for a specific protocol_id Raw transaction. 749 * 750 * @handle: Pointer to SCMI entity handle 751 * @protocol_id: Identifier of the protocol 752 * 753 * Note that in a regular SCMI stack, usually, a protocol has to be defined in 754 * the DT to have an associated channel and be usable; but in Raw mode any 755 * protocol in range is allowed, re-using the Base channel, so as to enable 756 * fuzzing on any protocol without the need of a fully compiled DT. 757 * 758 * Return: A reference to the channel to use, or an ERR_PTR 759 */ 760 struct scmi_chan_info * 761 scmi_xfer_raw_channel_get(const struct scmi_handle *handle, u8 protocol_id) 762 { 763 struct scmi_chan_info *cinfo; 764 struct scmi_info *info = handle_to_scmi_info(handle); 765 766 cinfo = idr_find(&info->tx_idr, protocol_id); 767 if (!cinfo) { 768 if (protocol_id == SCMI_PROTOCOL_BASE) 769 return ERR_PTR(-EINVAL); 770 /* Use Base channel for protocols not defined for DT */ 771 cinfo = idr_find(&info->tx_idr, SCMI_PROTOCOL_BASE); 772 if (!cinfo) 773 return ERR_PTR(-EINVAL); 774 dev_warn_once(handle->dev, 775 "Using Base channel for protocol 0x%X\n", 776 protocol_id); 777 } 778 779 return cinfo; 780 } 781 782 /** 783 * __scmi_xfer_put() - Release a message 784 * 785 * @minfo: Pointer to Tx/Rx Message management info based on channel type 786 * @xfer: message that was reserved by scmi_xfer_get 787 * 788 * After refcount check, possibly release an xfer, clearing the token slot, 789 * removing xfer from @pending_xfers and putting it back into free_xfers. 790 * 791 * This holds a spinlock to maintain integrity of internal data structures. 792 */ 793 static void 794 __scmi_xfer_put(struct scmi_xfers_info *minfo, struct scmi_xfer *xfer) 795 { 796 unsigned long flags; 797 798 spin_lock_irqsave(&minfo->xfer_lock, flags); 799 if (refcount_dec_and_test(&xfer->users)) { 800 if (xfer->pending) { 801 struct scmi_info *info = tx_minfo_to_scmi_info(minfo); 802 803 scmi_xfer_token_clear(minfo, xfer); 804 hash_del(&xfer->node); 805 xfer->pending = false; 806 807 scmi_dec_count(info->dbg, XFERS_INFLIGHT); 808 } 809 xfer->flags = 0; 810 hlist_add_head(&xfer->node, &minfo->free_xfers); 811 } 812 spin_unlock_irqrestore(&minfo->xfer_lock, flags); 813 } 814 815 /** 816 * scmi_xfer_raw_put - Release an xfer that was taken by @scmi_xfer_raw_get 817 * 818 * @handle: Pointer to SCMI entity handle 819 * @xfer: A reference to the xfer to put 820 * 821 * Note that as with other xfer_put() handlers the xfer is really effectively 822 * released only if there are no more users on the system. 823 */ 824 void scmi_xfer_raw_put(const struct scmi_handle *handle, struct scmi_xfer *xfer) 825 { 826 struct scmi_info *info = handle_to_scmi_info(handle); 827 828 return __scmi_xfer_put(&info->tx_minfo, xfer); 829 } 830 831 /** 832 * scmi_xfer_lookup_unlocked - Helper to lookup an xfer_id 833 * 834 * @minfo: Pointer to Tx/Rx Message management info based on channel type 835 * @xfer_id: Token ID to lookup in @pending_xfers 836 * 837 * Refcounting is untouched. 838 * 839 * Context: Assumes to be called with @xfer_lock already acquired. 840 * 841 * Return: A valid xfer on Success or error otherwise 842 */ 843 static struct scmi_xfer * 844 scmi_xfer_lookup_unlocked(struct scmi_xfers_info *minfo, u16 xfer_id) 845 { 846 struct scmi_xfer *xfer = NULL; 847 848 if (test_bit(xfer_id, minfo->xfer_alloc_table)) 849 xfer = XFER_FIND(minfo->pending_xfers, xfer_id); 850 851 return xfer ?: ERR_PTR(-EINVAL); 852 } 853 854 /** 855 * scmi_bad_message_trace - A helper to trace weird messages 856 * 857 * @cinfo: A reference to the channel descriptor on which the message was 858 * received 859 * @msg_hdr: Message header to track 860 * @err: A specific error code used as a status value in traces. 861 * 862 * This helper can be used to trace any kind of weird, incomplete, unexpected, 863 * timed-out message that arrives and as such, can be traced only referring to 864 * the header content, since the payload is missing/unreliable. 865 */ 866 static void scmi_bad_message_trace(struct scmi_chan_info *cinfo, u32 msg_hdr, 867 enum scmi_bad_msg err) 868 { 869 char *tag; 870 struct scmi_info *info = handle_to_scmi_info(cinfo->handle); 871 872 switch (MSG_XTRACT_TYPE(msg_hdr)) { 873 case MSG_TYPE_COMMAND: 874 tag = "!RESP"; 875 break; 876 case MSG_TYPE_DELAYED_RESP: 877 tag = "!DLYD"; 878 break; 879 case MSG_TYPE_NOTIFICATION: 880 tag = "!NOTI"; 881 break; 882 default: 883 tag = "!UNKN"; 884 break; 885 } 886 887 trace_scmi_msg_dump(info->id, cinfo->id, 888 MSG_XTRACT_PROT_ID(msg_hdr), 889 MSG_XTRACT_ID(msg_hdr), tag, 890 MSG_XTRACT_TOKEN(msg_hdr), err, NULL, 0); 891 } 892 893 /** 894 * scmi_msg_response_validate - Validate message type against state of related 895 * xfer 896 * 897 * @cinfo: A reference to the channel descriptor. 898 * @msg_type: Message type to check 899 * @xfer: A reference to the xfer to validate against @msg_type 900 * 901 * This function checks if @msg_type is congruent with the current state of 902 * a pending @xfer; if an asynchronous delayed response is received before the 903 * related synchronous response (Out-of-Order Delayed Response) the missing 904 * synchronous response is assumed to be OK and completed, carrying on with the 905 * Delayed Response: this is done to address the case in which the underlying 906 * SCMI transport can deliver such out-of-order responses. 907 * 908 * Context: Assumes to be called with xfer->lock already acquired. 909 * 910 * Return: 0 on Success, error otherwise 911 */ 912 static inline int scmi_msg_response_validate(struct scmi_chan_info *cinfo, 913 u8 msg_type, 914 struct scmi_xfer *xfer) 915 { 916 /* 917 * Even if a response was indeed expected on this slot at this point, 918 * a buggy platform could wrongly reply feeding us an unexpected 919 * delayed response we're not prepared to handle: bail-out safely 920 * blaming firmware. 921 */ 922 if (msg_type == MSG_TYPE_DELAYED_RESP && !xfer->async_done) { 923 dev_err(cinfo->dev, 924 "Delayed Response for %d not expected! Buggy F/W ?\n", 925 xfer->hdr.seq); 926 return -EINVAL; 927 } 928 929 switch (xfer->state) { 930 case SCMI_XFER_SENT_OK: 931 if (msg_type == MSG_TYPE_DELAYED_RESP) { 932 /* 933 * Delayed Response expected but delivered earlier. 934 * Assume message RESPONSE was OK and skip state. 935 */ 936 xfer->hdr.status = SCMI_SUCCESS; 937 xfer->state = SCMI_XFER_RESP_OK; 938 complete(&xfer->done); 939 dev_warn(cinfo->dev, 940 "Received valid OoO Delayed Response for %d\n", 941 xfer->hdr.seq); 942 } 943 break; 944 case SCMI_XFER_RESP_OK: 945 if (msg_type != MSG_TYPE_DELAYED_RESP) 946 return -EINVAL; 947 break; 948 case SCMI_XFER_DRESP_OK: 949 /* No further message expected once in SCMI_XFER_DRESP_OK */ 950 return -EINVAL; 951 } 952 953 return 0; 954 } 955 956 /** 957 * scmi_xfer_state_update - Update xfer state 958 * 959 * @xfer: A reference to the xfer to update 960 * @msg_type: Type of message being processed. 961 * 962 * Note that this message is assumed to have been already successfully validated 963 * by @scmi_msg_response_validate(), so here we just update the state. 964 * 965 * Context: Assumes to be called on an xfer exclusively acquired using the 966 * busy flag. 967 */ 968 static inline void scmi_xfer_state_update(struct scmi_xfer *xfer, u8 msg_type) 969 { 970 xfer->hdr.type = msg_type; 971 972 /* Unknown command types were already discarded earlier */ 973 if (xfer->hdr.type == MSG_TYPE_COMMAND) 974 xfer->state = SCMI_XFER_RESP_OK; 975 else 976 xfer->state = SCMI_XFER_DRESP_OK; 977 } 978 979 static bool scmi_xfer_acquired(struct scmi_xfer *xfer) 980 { 981 int ret; 982 983 ret = atomic_cmpxchg(&xfer->busy, SCMI_XFER_FREE, SCMI_XFER_BUSY); 984 985 return ret == SCMI_XFER_FREE; 986 } 987 988 /** 989 * scmi_xfer_command_acquire - Helper to lookup and acquire a command xfer 990 * 991 * @cinfo: A reference to the channel descriptor. 992 * @msg_hdr: A message header to use as lookup key 993 * 994 * When a valid xfer is found for the sequence number embedded in the provided 995 * msg_hdr, reference counting is properly updated and exclusive access to this 996 * xfer is granted till released with @scmi_xfer_command_release. 997 * 998 * Return: A valid @xfer on Success or error otherwise. 999 */ 1000 static inline struct scmi_xfer * 1001 scmi_xfer_command_acquire(struct scmi_chan_info *cinfo, u32 msg_hdr) 1002 { 1003 int ret; 1004 unsigned long flags; 1005 struct scmi_xfer *xfer; 1006 struct scmi_info *info = handle_to_scmi_info(cinfo->handle); 1007 struct scmi_xfers_info *minfo = &info->tx_minfo; 1008 u8 msg_type = MSG_XTRACT_TYPE(msg_hdr); 1009 u16 xfer_id = MSG_XTRACT_TOKEN(msg_hdr); 1010 1011 /* Are we even expecting this? */ 1012 spin_lock_irqsave(&minfo->xfer_lock, flags); 1013 xfer = scmi_xfer_lookup_unlocked(minfo, xfer_id); 1014 if (IS_ERR(xfer)) { 1015 dev_err(cinfo->dev, 1016 "Message for %d type %d is not expected!\n", 1017 xfer_id, msg_type); 1018 spin_unlock_irqrestore(&minfo->xfer_lock, flags); 1019 1020 scmi_bad_message_trace(cinfo, msg_hdr, MSG_UNEXPECTED); 1021 scmi_inc_count(info->dbg, ERR_MSG_UNEXPECTED); 1022 1023 return xfer; 1024 } 1025 refcount_inc(&xfer->users); 1026 spin_unlock_irqrestore(&minfo->xfer_lock, flags); 1027 1028 spin_lock_irqsave(&xfer->lock, flags); 1029 ret = scmi_msg_response_validate(cinfo, msg_type, xfer); 1030 /* 1031 * If a pending xfer was found which was also in a congruent state with 1032 * the received message, acquire exclusive access to it setting the busy 1033 * flag. 1034 * Spins only on the rare limit condition of concurrent reception of 1035 * RESP and DRESP for the same xfer. 1036 */ 1037 if (!ret) { 1038 spin_until_cond(scmi_xfer_acquired(xfer)); 1039 scmi_xfer_state_update(xfer, msg_type); 1040 } 1041 spin_unlock_irqrestore(&xfer->lock, flags); 1042 1043 if (ret) { 1044 dev_err(cinfo->dev, 1045 "Invalid message type:%d for %d - HDR:0x%X state:%d\n", 1046 msg_type, xfer_id, msg_hdr, xfer->state); 1047 1048 scmi_bad_message_trace(cinfo, msg_hdr, MSG_INVALID); 1049 scmi_inc_count(info->dbg, ERR_MSG_INVALID); 1050 1051 /* On error the refcount incremented above has to be dropped */ 1052 __scmi_xfer_put(minfo, xfer); 1053 xfer = ERR_PTR(-EINVAL); 1054 } 1055 1056 return xfer; 1057 } 1058 1059 static inline void scmi_xfer_command_release(struct scmi_info *info, 1060 struct scmi_xfer *xfer) 1061 { 1062 atomic_set(&xfer->busy, SCMI_XFER_FREE); 1063 __scmi_xfer_put(&info->tx_minfo, xfer); 1064 } 1065 1066 static inline void scmi_clear_channel(struct scmi_info *info, 1067 struct scmi_chan_info *cinfo) 1068 { 1069 if (!cinfo->is_p2a) { 1070 dev_warn(cinfo->dev, "Invalid clear on A2P channel !\n"); 1071 return; 1072 } 1073 1074 if (info->desc->ops->clear_channel) 1075 info->desc->ops->clear_channel(cinfo); 1076 } 1077 1078 static void scmi_handle_notification(struct scmi_chan_info *cinfo, 1079 u32 msg_hdr, void *priv) 1080 { 1081 struct scmi_xfer *xfer; 1082 struct device *dev = cinfo->dev; 1083 struct scmi_info *info = handle_to_scmi_info(cinfo->handle); 1084 struct scmi_xfers_info *minfo = &info->rx_minfo; 1085 ktime_t ts; 1086 1087 ts = ktime_get_boottime(); 1088 xfer = scmi_xfer_get(cinfo->handle, minfo); 1089 if (IS_ERR(xfer)) { 1090 dev_err(dev, "failed to get free message slot (%ld)\n", 1091 PTR_ERR(xfer)); 1092 1093 scmi_bad_message_trace(cinfo, msg_hdr, MSG_NOMEM); 1094 scmi_inc_count(info->dbg, ERR_MSG_NOMEM); 1095 1096 scmi_clear_channel(info, cinfo); 1097 return; 1098 } 1099 1100 unpack_scmi_header(msg_hdr, &xfer->hdr); 1101 if (priv) 1102 /* Ensure order between xfer->priv store and following ops */ 1103 smp_store_mb(xfer->priv, priv); 1104 info->desc->ops->fetch_notification(cinfo, info->desc->max_msg_size, 1105 xfer); 1106 1107 trace_scmi_msg_dump(info->id, cinfo->id, xfer->hdr.protocol_id, 1108 xfer->hdr.id, "NOTI", xfer->hdr.seq, 1109 xfer->hdr.status, xfer->rx.buf, xfer->rx.len); 1110 scmi_inc_count(info->dbg, NOTIFICATION_OK); 1111 1112 scmi_notify(cinfo->handle, xfer->hdr.protocol_id, 1113 xfer->hdr.id, xfer->rx.buf, xfer->rx.len, ts); 1114 1115 trace_scmi_rx_done(xfer->transfer_id, xfer->hdr.id, 1116 xfer->hdr.protocol_id, xfer->hdr.seq, 1117 MSG_TYPE_NOTIFICATION); 1118 1119 if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) { 1120 xfer->hdr.seq = MSG_XTRACT_TOKEN(msg_hdr); 1121 scmi_raw_message_report(info->raw, xfer, SCMI_RAW_NOTIF_QUEUE, 1122 cinfo->id); 1123 } 1124 1125 __scmi_xfer_put(minfo, xfer); 1126 1127 scmi_clear_channel(info, cinfo); 1128 } 1129 1130 static void scmi_handle_response(struct scmi_chan_info *cinfo, 1131 u32 msg_hdr, void *priv) 1132 { 1133 struct scmi_xfer *xfer; 1134 struct scmi_info *info = handle_to_scmi_info(cinfo->handle); 1135 1136 xfer = scmi_xfer_command_acquire(cinfo, msg_hdr); 1137 if (IS_ERR(xfer)) { 1138 if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) 1139 scmi_raw_error_report(info->raw, cinfo, msg_hdr, priv); 1140 1141 if (MSG_XTRACT_TYPE(msg_hdr) == MSG_TYPE_DELAYED_RESP) 1142 scmi_clear_channel(info, cinfo); 1143 return; 1144 } 1145 1146 /* rx.len could be shrunk in the sync do_xfer, so reset to maxsz */ 1147 if (xfer->hdr.type == MSG_TYPE_DELAYED_RESP) 1148 xfer->rx.len = info->desc->max_msg_size; 1149 1150 if (priv) 1151 /* Ensure order between xfer->priv store and following ops */ 1152 smp_store_mb(xfer->priv, priv); 1153 info->desc->ops->fetch_response(cinfo, xfer); 1154 1155 trace_scmi_msg_dump(info->id, cinfo->id, xfer->hdr.protocol_id, 1156 xfer->hdr.id, 1157 xfer->hdr.type == MSG_TYPE_DELAYED_RESP ? 1158 (!SCMI_XFER_IS_RAW(xfer) ? "DLYD" : "dlyd") : 1159 (!SCMI_XFER_IS_RAW(xfer) ? "RESP" : "resp"), 1160 xfer->hdr.seq, xfer->hdr.status, 1161 xfer->rx.buf, xfer->rx.len); 1162 1163 trace_scmi_rx_done(xfer->transfer_id, xfer->hdr.id, 1164 xfer->hdr.protocol_id, xfer->hdr.seq, 1165 xfer->hdr.type); 1166 1167 if (xfer->hdr.type == MSG_TYPE_DELAYED_RESP) { 1168 scmi_clear_channel(info, cinfo); 1169 complete(xfer->async_done); 1170 scmi_inc_count(info->dbg, DELAYED_RESPONSE_OK); 1171 } else { 1172 complete(&xfer->done); 1173 scmi_inc_count(info->dbg, RESPONSE_OK); 1174 } 1175 1176 if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) { 1177 /* 1178 * When in polling mode avoid to queue the Raw xfer on the IRQ 1179 * RX path since it will be already queued at the end of the TX 1180 * poll loop. 1181 */ 1182 if (!xfer->hdr.poll_completion || 1183 xfer->hdr.type == MSG_TYPE_DELAYED_RESP) 1184 scmi_raw_message_report(info->raw, xfer, 1185 SCMI_RAW_REPLY_QUEUE, 1186 cinfo->id); 1187 } 1188 1189 scmi_xfer_command_release(info, xfer); 1190 } 1191 1192 /** 1193 * scmi_rx_callback() - callback for receiving messages 1194 * 1195 * @cinfo: SCMI channel info 1196 * @msg_hdr: Message header 1197 * @priv: Transport specific private data. 1198 * 1199 * Processes one received message to appropriate transfer information and 1200 * signals completion of the transfer. 1201 * 1202 * NOTE: This function will be invoked in IRQ context, hence should be 1203 * as optimal as possible. 1204 */ 1205 static void scmi_rx_callback(struct scmi_chan_info *cinfo, u32 msg_hdr, 1206 void *priv) 1207 { 1208 u8 msg_type = MSG_XTRACT_TYPE(msg_hdr); 1209 1210 switch (msg_type) { 1211 case MSG_TYPE_NOTIFICATION: 1212 scmi_handle_notification(cinfo, msg_hdr, priv); 1213 break; 1214 case MSG_TYPE_COMMAND: 1215 case MSG_TYPE_DELAYED_RESP: 1216 scmi_handle_response(cinfo, msg_hdr, priv); 1217 break; 1218 default: 1219 WARN_ONCE(1, "received unknown msg_type:%d\n", msg_type); 1220 scmi_bad_message_trace(cinfo, msg_hdr, MSG_UNKNOWN); 1221 break; 1222 } 1223 } 1224 1225 /** 1226 * xfer_put() - Release a transmit message 1227 * 1228 * @ph: Pointer to SCMI protocol handle 1229 * @xfer: message that was reserved by xfer_get_init 1230 */ 1231 static void xfer_put(const struct scmi_protocol_handle *ph, 1232 struct scmi_xfer *xfer) 1233 { 1234 const struct scmi_protocol_instance *pi = ph_to_pi(ph); 1235 struct scmi_info *info = handle_to_scmi_info(pi->handle); 1236 1237 __scmi_xfer_put(&info->tx_minfo, xfer); 1238 } 1239 1240 static bool scmi_xfer_done_no_timeout(struct scmi_chan_info *cinfo, 1241 struct scmi_xfer *xfer, ktime_t stop, 1242 bool *ooo) 1243 { 1244 struct scmi_info *info = handle_to_scmi_info(cinfo->handle); 1245 1246 /* 1247 * Poll also on xfer->done so that polling can be forcibly terminated 1248 * in case of out-of-order receptions of delayed responses 1249 */ 1250 return info->desc->ops->poll_done(cinfo, xfer) || 1251 (*ooo = try_wait_for_completion(&xfer->done)) || 1252 ktime_after(ktime_get(), stop); 1253 } 1254 1255 static int scmi_wait_for_reply(struct device *dev, const struct scmi_desc *desc, 1256 struct scmi_chan_info *cinfo, 1257 struct scmi_xfer *xfer, unsigned int timeout_ms) 1258 { 1259 int ret = 0; 1260 struct scmi_info *info = handle_to_scmi_info(cinfo->handle); 1261 1262 if (xfer->hdr.poll_completion) { 1263 /* 1264 * Real polling is needed only if transport has NOT declared 1265 * itself to support synchronous commands replies. 1266 */ 1267 if (!desc->sync_cmds_completed_on_ret) { 1268 bool ooo = false; 1269 1270 /* 1271 * Poll on xfer using transport provided .poll_done(); 1272 * assumes no completion interrupt was available. 1273 */ 1274 ktime_t stop = ktime_add_ms(ktime_get(), timeout_ms); 1275 1276 spin_until_cond(scmi_xfer_done_no_timeout(cinfo, xfer, 1277 stop, &ooo)); 1278 if (!ooo && !info->desc->ops->poll_done(cinfo, xfer)) { 1279 dev_err(dev, 1280 "timed out in resp(caller: %pS) - polling\n", 1281 (void *)_RET_IP_); 1282 ret = -ETIMEDOUT; 1283 scmi_inc_count(info->dbg, XFERS_RESPONSE_POLLED_TIMEOUT); 1284 } 1285 } 1286 1287 if (!ret) { 1288 unsigned long flags; 1289 1290 /* 1291 * Do not fetch_response if an out-of-order delayed 1292 * response is being processed. 1293 */ 1294 spin_lock_irqsave(&xfer->lock, flags); 1295 if (xfer->state == SCMI_XFER_SENT_OK) { 1296 desc->ops->fetch_response(cinfo, xfer); 1297 xfer->state = SCMI_XFER_RESP_OK; 1298 } 1299 spin_unlock_irqrestore(&xfer->lock, flags); 1300 1301 /* Trace polled replies. */ 1302 trace_scmi_msg_dump(info->id, cinfo->id, 1303 xfer->hdr.protocol_id, xfer->hdr.id, 1304 !SCMI_XFER_IS_RAW(xfer) ? 1305 "RESP" : "resp", 1306 xfer->hdr.seq, xfer->hdr.status, 1307 xfer->rx.buf, xfer->rx.len); 1308 scmi_inc_count(info->dbg, RESPONSE_POLLED_OK); 1309 1310 if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) { 1311 scmi_raw_message_report(info->raw, xfer, 1312 SCMI_RAW_REPLY_QUEUE, 1313 cinfo->id); 1314 } 1315 } 1316 } else { 1317 /* And we wait for the response. */ 1318 if (!wait_for_completion_timeout(&xfer->done, 1319 msecs_to_jiffies(timeout_ms))) { 1320 dev_err(dev, "timed out in resp(caller: %pS)\n", 1321 (void *)_RET_IP_); 1322 ret = -ETIMEDOUT; 1323 scmi_inc_count(info->dbg, XFERS_RESPONSE_TIMEOUT); 1324 } 1325 } 1326 1327 return ret; 1328 } 1329 1330 /** 1331 * scmi_wait_for_message_response - An helper to group all the possible ways of 1332 * waiting for a synchronous message response. 1333 * 1334 * @cinfo: SCMI channel info 1335 * @xfer: Reference to the transfer being waited for. 1336 * 1337 * Chooses waiting strategy (sleep-waiting vs busy-waiting) depending on 1338 * configuration flags like xfer->hdr.poll_completion. 1339 * 1340 * Return: 0 on Success, error otherwise. 1341 */ 1342 static int scmi_wait_for_message_response(struct scmi_chan_info *cinfo, 1343 struct scmi_xfer *xfer) 1344 { 1345 struct scmi_info *info = handle_to_scmi_info(cinfo->handle); 1346 struct device *dev = info->dev; 1347 1348 trace_scmi_xfer_response_wait(xfer->transfer_id, xfer->hdr.id, 1349 xfer->hdr.protocol_id, xfer->hdr.seq, 1350 info->desc->max_rx_timeout_ms, 1351 xfer->hdr.poll_completion); 1352 1353 return scmi_wait_for_reply(dev, info->desc, cinfo, xfer, 1354 info->desc->max_rx_timeout_ms); 1355 } 1356 1357 /** 1358 * scmi_xfer_raw_wait_for_message_response - An helper to wait for a message 1359 * reply to an xfer raw request on a specific channel for the required timeout. 1360 * 1361 * @cinfo: SCMI channel info 1362 * @xfer: Reference to the transfer being waited for. 1363 * @timeout_ms: The maximum timeout in milliseconds 1364 * 1365 * Return: 0 on Success, error otherwise. 1366 */ 1367 int scmi_xfer_raw_wait_for_message_response(struct scmi_chan_info *cinfo, 1368 struct scmi_xfer *xfer, 1369 unsigned int timeout_ms) 1370 { 1371 int ret; 1372 struct scmi_info *info = handle_to_scmi_info(cinfo->handle); 1373 struct device *dev = info->dev; 1374 1375 ret = scmi_wait_for_reply(dev, info->desc, cinfo, xfer, timeout_ms); 1376 if (ret) 1377 dev_dbg(dev, "timed out in RAW response - HDR:%08X\n", 1378 pack_scmi_header(&xfer->hdr)); 1379 1380 return ret; 1381 } 1382 1383 /** 1384 * do_xfer() - Do one transfer 1385 * 1386 * @ph: Pointer to SCMI protocol handle 1387 * @xfer: Transfer to initiate and wait for response 1388 * 1389 * Return: -ETIMEDOUT in case of no response, if transmit error, 1390 * return corresponding error, else if all goes well, 1391 * return 0. 1392 */ 1393 static int do_xfer(const struct scmi_protocol_handle *ph, 1394 struct scmi_xfer *xfer) 1395 { 1396 int ret; 1397 const struct scmi_protocol_instance *pi = ph_to_pi(ph); 1398 struct scmi_info *info = handle_to_scmi_info(pi->handle); 1399 struct device *dev = info->dev; 1400 struct scmi_chan_info *cinfo; 1401 1402 /* Check for polling request on custom command xfers at first */ 1403 if (xfer->hdr.poll_completion && 1404 !is_transport_polling_capable(info->desc)) { 1405 dev_warn_once(dev, 1406 "Polling mode is not supported by transport.\n"); 1407 scmi_inc_count(info->dbg, SENT_FAIL_POLLING_UNSUPPORTED); 1408 return -EINVAL; 1409 } 1410 1411 cinfo = idr_find(&info->tx_idr, pi->proto->id); 1412 if (unlikely(!cinfo)) { 1413 scmi_inc_count(info->dbg, SENT_FAIL_CHANNEL_NOT_FOUND); 1414 return -EINVAL; 1415 } 1416 /* True ONLY if also supported by transport. */ 1417 if (is_polling_enabled(cinfo, info->desc)) 1418 xfer->hdr.poll_completion = true; 1419 1420 /* 1421 * Initialise protocol id now from protocol handle to avoid it being 1422 * overridden by mistake (or malice) by the protocol code mangling with 1423 * the scmi_xfer structure prior to this. 1424 */ 1425 xfer->hdr.protocol_id = pi->proto->id; 1426 reinit_completion(&xfer->done); 1427 1428 trace_scmi_xfer_begin(xfer->transfer_id, xfer->hdr.id, 1429 xfer->hdr.protocol_id, xfer->hdr.seq, 1430 xfer->hdr.poll_completion, 1431 scmi_inflight_count(&info->handle)); 1432 1433 /* Clear any stale status */ 1434 xfer->hdr.status = SCMI_SUCCESS; 1435 xfer->state = SCMI_XFER_SENT_OK; 1436 /* 1437 * Even though spinlocking is not needed here since no race is possible 1438 * on xfer->state due to the monotonically increasing tokens allocation, 1439 * we must anyway ensure xfer->state initialization is not re-ordered 1440 * after the .send_message() to be sure that on the RX path an early 1441 * ISR calling scmi_rx_callback() cannot see an old stale xfer->state. 1442 */ 1443 smp_mb(); 1444 1445 ret = info->desc->ops->send_message(cinfo, xfer); 1446 if (ret < 0) { 1447 dev_dbg(dev, "Failed to send message %d\n", ret); 1448 scmi_inc_count(info->dbg, SENT_FAIL); 1449 return ret; 1450 } 1451 1452 trace_scmi_msg_dump(info->id, cinfo->id, xfer->hdr.protocol_id, 1453 xfer->hdr.id, "CMND", xfer->hdr.seq, 1454 xfer->hdr.status, xfer->tx.buf, xfer->tx.len); 1455 scmi_inc_count(info->dbg, SENT_OK); 1456 1457 ret = scmi_wait_for_message_response(cinfo, xfer); 1458 if (!ret && xfer->hdr.status) { 1459 ret = scmi_to_linux_errno(xfer->hdr.status); 1460 scmi_inc_count(info->dbg, ERR_PROTOCOL); 1461 } 1462 1463 if (info->desc->ops->mark_txdone) 1464 info->desc->ops->mark_txdone(cinfo, ret, xfer); 1465 1466 trace_scmi_xfer_end(xfer->transfer_id, xfer->hdr.id, 1467 xfer->hdr.protocol_id, xfer->hdr.seq, ret, 1468 scmi_inflight_count(&info->handle)); 1469 1470 return ret; 1471 } 1472 1473 static void reset_rx_to_maxsz(const struct scmi_protocol_handle *ph, 1474 struct scmi_xfer *xfer) 1475 { 1476 const struct scmi_protocol_instance *pi = ph_to_pi(ph); 1477 struct scmi_info *info = handle_to_scmi_info(pi->handle); 1478 1479 xfer->rx.len = info->desc->max_msg_size; 1480 } 1481 1482 /** 1483 * do_xfer_with_response() - Do one transfer and wait until the delayed 1484 * response is received 1485 * 1486 * @ph: Pointer to SCMI protocol handle 1487 * @xfer: Transfer to initiate and wait for response 1488 * 1489 * Using asynchronous commands in atomic/polling mode should be avoided since 1490 * it could cause long busy-waiting here, so ignore polling for the delayed 1491 * response and WARN if it was requested for this command transaction since 1492 * upper layers should refrain from issuing such kind of requests. 1493 * 1494 * The only other option would have been to refrain from using any asynchronous 1495 * command even if made available, when an atomic transport is detected, and 1496 * instead forcibly use the synchronous version (thing that can be easily 1497 * attained at the protocol layer), but this would also have led to longer 1498 * stalls of the channel for synchronous commands and possibly timeouts. 1499 * (in other words there is usually a good reason if a platform provides an 1500 * asynchronous version of a command and we should prefer to use it...just not 1501 * when using atomic/polling mode) 1502 * 1503 * Return: -ETIMEDOUT in case of no delayed response, if transmit error, 1504 * return corresponding error, else if all goes well, return 0. 1505 */ 1506 static int do_xfer_with_response(const struct scmi_protocol_handle *ph, 1507 struct scmi_xfer *xfer) 1508 { 1509 int ret, timeout = msecs_to_jiffies(SCMI_MAX_RESPONSE_TIMEOUT); 1510 DECLARE_COMPLETION_ONSTACK(async_response); 1511 1512 xfer->async_done = &async_response; 1513 1514 /* 1515 * Delayed responses should not be polled, so an async command should 1516 * not have been used when requiring an atomic/poll context; WARN and 1517 * perform instead a sleeping wait. 1518 * (Note Async + IgnoreDelayedResponses are sent via do_xfer) 1519 */ 1520 WARN_ON_ONCE(xfer->hdr.poll_completion); 1521 1522 ret = do_xfer(ph, xfer); 1523 if (!ret) { 1524 if (!wait_for_completion_timeout(xfer->async_done, timeout)) { 1525 dev_err(ph->dev, 1526 "timed out in delayed resp(caller: %pS)\n", 1527 (void *)_RET_IP_); 1528 ret = -ETIMEDOUT; 1529 } else if (xfer->hdr.status) { 1530 ret = scmi_to_linux_errno(xfer->hdr.status); 1531 } 1532 } 1533 1534 xfer->async_done = NULL; 1535 return ret; 1536 } 1537 1538 /** 1539 * xfer_get_init() - Allocate and initialise one message for transmit 1540 * 1541 * @ph: Pointer to SCMI protocol handle 1542 * @msg_id: Message identifier 1543 * @tx_size: transmit message size 1544 * @rx_size: receive message size 1545 * @p: pointer to the allocated and initialised message 1546 * 1547 * This function allocates the message using @scmi_xfer_get and 1548 * initialise the header. 1549 * 1550 * Return: 0 if all went fine with @p pointing to message, else 1551 * corresponding error. 1552 */ 1553 static int xfer_get_init(const struct scmi_protocol_handle *ph, 1554 u8 msg_id, size_t tx_size, size_t rx_size, 1555 struct scmi_xfer **p) 1556 { 1557 int ret; 1558 struct scmi_xfer *xfer; 1559 const struct scmi_protocol_instance *pi = ph_to_pi(ph); 1560 struct scmi_info *info = handle_to_scmi_info(pi->handle); 1561 struct scmi_xfers_info *minfo = &info->tx_minfo; 1562 struct device *dev = info->dev; 1563 1564 /* Ensure we have sane transfer sizes */ 1565 if (rx_size > info->desc->max_msg_size || 1566 tx_size > info->desc->max_msg_size) 1567 return -ERANGE; 1568 1569 xfer = scmi_xfer_get(pi->handle, minfo); 1570 if (IS_ERR(xfer)) { 1571 ret = PTR_ERR(xfer); 1572 dev_err(dev, "failed to get free message slot(%d)\n", ret); 1573 return ret; 1574 } 1575 1576 /* Pick a sequence number and register this xfer as in-flight */ 1577 ret = scmi_xfer_pending_set(xfer, minfo); 1578 if (ret) { 1579 dev_err(pi->handle->dev, 1580 "Failed to get monotonic token %d\n", ret); 1581 __scmi_xfer_put(minfo, xfer); 1582 return ret; 1583 } 1584 1585 xfer->tx.len = tx_size; 1586 xfer->rx.len = rx_size ? : info->desc->max_msg_size; 1587 xfer->hdr.type = MSG_TYPE_COMMAND; 1588 xfer->hdr.id = msg_id; 1589 xfer->hdr.poll_completion = false; 1590 1591 *p = xfer; 1592 1593 return 0; 1594 } 1595 1596 /** 1597 * version_get() - command to get the revision of the SCMI entity 1598 * 1599 * @ph: Pointer to SCMI protocol handle 1600 * @version: Holds returned version of protocol. 1601 * 1602 * Updates the SCMI information in the internal data structure. 1603 * 1604 * Return: 0 if all went fine, else return appropriate error. 1605 */ 1606 static int version_get(const struct scmi_protocol_handle *ph, u32 *version) 1607 { 1608 int ret; 1609 __le32 *rev_info; 1610 struct scmi_xfer *t; 1611 1612 ret = xfer_get_init(ph, PROTOCOL_VERSION, 0, sizeof(*version), &t); 1613 if (ret) 1614 return ret; 1615 1616 ret = do_xfer(ph, t); 1617 if (!ret) { 1618 rev_info = t->rx.buf; 1619 *version = le32_to_cpu(*rev_info); 1620 } 1621 1622 xfer_put(ph, t); 1623 return ret; 1624 } 1625 1626 /** 1627 * scmi_set_protocol_priv - Set protocol specific data at init time 1628 * 1629 * @ph: A reference to the protocol handle. 1630 * @priv: The private data to set. 1631 * 1632 * Return: 0 on Success 1633 */ 1634 static int scmi_set_protocol_priv(const struct scmi_protocol_handle *ph, 1635 void *priv) 1636 { 1637 struct scmi_protocol_instance *pi = ph_to_pi(ph); 1638 1639 pi->priv = priv; 1640 1641 return 0; 1642 } 1643 1644 /** 1645 * scmi_get_protocol_priv - Set protocol specific data at init time 1646 * 1647 * @ph: A reference to the protocol handle. 1648 * 1649 * Return: Protocol private data if any was set. 1650 */ 1651 static void *scmi_get_protocol_priv(const struct scmi_protocol_handle *ph) 1652 { 1653 const struct scmi_protocol_instance *pi = ph_to_pi(ph); 1654 1655 return pi->priv; 1656 } 1657 1658 static const struct scmi_xfer_ops xfer_ops = { 1659 .xfer_get_init = xfer_get_init, 1660 .reset_rx_to_maxsz = reset_rx_to_maxsz, 1661 .do_xfer = do_xfer, 1662 .do_xfer_with_response = do_xfer_with_response, 1663 .xfer_put = xfer_put, 1664 }; 1665 1666 struct scmi_msg_resp_domain_name_get { 1667 __le32 flags; 1668 u8 name[SCMI_MAX_STR_SIZE]; 1669 }; 1670 1671 /** 1672 * scmi_common_extended_name_get - Common helper to get extended resources name 1673 * @ph: A protocol handle reference. 1674 * @cmd_id: The specific command ID to use. 1675 * @res_id: The specific resource ID to use. 1676 * @flags: A pointer to specific flags to use, if any. 1677 * @name: A pointer to the preallocated area where the retrieved name will be 1678 * stored as a NULL terminated string. 1679 * @len: The len in bytes of the @name char array. 1680 * 1681 * Return: 0 on Succcess 1682 */ 1683 static int scmi_common_extended_name_get(const struct scmi_protocol_handle *ph, 1684 u8 cmd_id, u32 res_id, u32 *flags, 1685 char *name, size_t len) 1686 { 1687 int ret; 1688 size_t txlen; 1689 struct scmi_xfer *t; 1690 struct scmi_msg_resp_domain_name_get *resp; 1691 1692 txlen = !flags ? sizeof(res_id) : sizeof(res_id) + sizeof(*flags); 1693 ret = ph->xops->xfer_get_init(ph, cmd_id, txlen, sizeof(*resp), &t); 1694 if (ret) 1695 goto out; 1696 1697 put_unaligned_le32(res_id, t->tx.buf); 1698 if (flags) 1699 put_unaligned_le32(*flags, t->tx.buf + sizeof(res_id)); 1700 resp = t->rx.buf; 1701 1702 ret = ph->xops->do_xfer(ph, t); 1703 if (!ret) 1704 strscpy(name, resp->name, len); 1705 1706 ph->xops->xfer_put(ph, t); 1707 out: 1708 if (ret) 1709 dev_warn(ph->dev, 1710 "Failed to get extended name - id:%u (ret:%d). Using %s\n", 1711 res_id, ret, name); 1712 return ret; 1713 } 1714 1715 /** 1716 * scmi_common_get_max_msg_size - Get maximum message size 1717 * @ph: A protocol handle reference. 1718 * 1719 * Return: Maximum message size for the current protocol. 1720 */ 1721 static int scmi_common_get_max_msg_size(const struct scmi_protocol_handle *ph) 1722 { 1723 const struct scmi_protocol_instance *pi = ph_to_pi(ph); 1724 struct scmi_info *info = handle_to_scmi_info(pi->handle); 1725 1726 return info->desc->max_msg_size; 1727 } 1728 1729 /** 1730 * scmi_protocol_msg_check - Check protocol message attributes 1731 * 1732 * @ph: A reference to the protocol handle. 1733 * @message_id: The ID of the message to check. 1734 * @attributes: A parameter to optionally return the retrieved message 1735 * attributes, in case of Success. 1736 * 1737 * An helper to check protocol message attributes for a specific protocol 1738 * and message pair. 1739 * 1740 * Return: 0 on SUCCESS 1741 */ 1742 static int scmi_protocol_msg_check(const struct scmi_protocol_handle *ph, 1743 u32 message_id, u32 *attributes) 1744 { 1745 int ret; 1746 struct scmi_xfer *t; 1747 1748 ret = xfer_get_init(ph, PROTOCOL_MESSAGE_ATTRIBUTES, 1749 sizeof(__le32), 0, &t); 1750 if (ret) 1751 return ret; 1752 1753 put_unaligned_le32(message_id, t->tx.buf); 1754 ret = do_xfer(ph, t); 1755 if (!ret && attributes) 1756 *attributes = get_unaligned_le32(t->rx.buf); 1757 xfer_put(ph, t); 1758 1759 return ret; 1760 } 1761 1762 /** 1763 * struct scmi_iterator - Iterator descriptor 1764 * @msg: A reference to the message TX buffer; filled by @prepare_message with 1765 * a proper custom command payload for each multi-part command request. 1766 * @resp: A reference to the response RX buffer; used by @update_state and 1767 * @process_response to parse the multi-part replies. 1768 * @t: A reference to the underlying xfer initialized and used transparently by 1769 * the iterator internal routines. 1770 * @ph: A reference to the associated protocol handle to be used. 1771 * @ops: A reference to the custom provided iterator operations. 1772 * @state: The current iterator state; used and updated in turn by the iterators 1773 * internal routines and by the caller-provided @scmi_iterator_ops. 1774 * @priv: A reference to optional private data as provided by the caller and 1775 * passed back to the @@scmi_iterator_ops. 1776 */ 1777 struct scmi_iterator { 1778 void *msg; 1779 void *resp; 1780 struct scmi_xfer *t; 1781 const struct scmi_protocol_handle *ph; 1782 struct scmi_iterator_ops *ops; 1783 struct scmi_iterator_state state; 1784 void *priv; 1785 }; 1786 1787 static void *scmi_iterator_init(const struct scmi_protocol_handle *ph, 1788 struct scmi_iterator_ops *ops, 1789 unsigned int max_resources, u8 msg_id, 1790 size_t tx_size, void *priv) 1791 { 1792 int ret; 1793 1794 struct scmi_iterator *i __free(kfree) = kzalloc(sizeof(*i), GFP_KERNEL); 1795 if (!i) 1796 return ERR_PTR(-ENOMEM); 1797 1798 if (!ops || !ph) 1799 return ERR_PTR(-EINVAL); 1800 1801 i->ph = ph; 1802 i->ops = ops; 1803 i->priv = priv; 1804 1805 ret = ph->xops->xfer_get_init(ph, msg_id, tx_size, 0, &i->t); 1806 if (ret) 1807 return ERR_PTR(ret); 1808 1809 i->state.max_resources = max_resources; 1810 i->msg = i->t->tx.buf; 1811 i->resp = i->t->rx.buf; 1812 1813 return no_free_ptr(i); 1814 } 1815 1816 static int __scmi_iterator_run(void *iter, unsigned int *start, unsigned int *end) 1817 { 1818 int ret; 1819 struct scmi_iterator_ops *iops; 1820 const struct scmi_protocol_handle *ph; 1821 struct scmi_iterator_state *st; 1822 struct scmi_iterator *i; 1823 unsigned int n; 1824 1825 if (!iter) 1826 return -EINVAL; 1827 1828 i = iter; 1829 iops = i->ops; 1830 ph = i->ph; 1831 st = &i->state; 1832 1833 /* Reinitialize state for next run */ 1834 st->num_returned = 0; 1835 st->num_remaining = 0; 1836 st->desc_index = start ? *start : 0; 1837 1838 do { 1839 iops->prepare_message(i->msg, st->desc_index, i->priv); 1840 ret = ph->xops->do_xfer(ph, i->t); 1841 if (ret) 1842 return ret; 1843 1844 st->rx_len = i->t->rx.len; 1845 ret = iops->update_state(st, i->resp, i->priv); 1846 if (ret) 1847 return ret; 1848 1849 if (st->num_returned > st->max_resources - st->desc_index) { 1850 dev_err(ph->dev, 1851 "No. of resources can't exceed %d\n", 1852 st->max_resources); 1853 return -EINVAL; 1854 } 1855 1856 if (end) 1857 n = min(st->num_returned, *end - st->desc_index + 1); 1858 else 1859 n = st->num_returned; 1860 for (st->loop_idx = 0; st->loop_idx < n; st->loop_idx++) { 1861 ret = iops->process_response(ph, i->resp, st, i->priv); 1862 if (ret) 1863 return ret; 1864 } 1865 1866 st->desc_index += n; 1867 ph->xops->reset_rx_to_maxsz(ph, i->t); 1868 /* 1869 * check for both returned and remaining to avoid infinite 1870 * loop due to buggy firmware 1871 */ 1872 } while (st->num_returned && st->num_remaining && 1873 (!end || (st->desc_index <= min(*end, st->max_resources - 1)))); 1874 1875 return 0; 1876 } 1877 1878 static void scmi_iterator_bound_cleanup(void *iter) 1879 { 1880 struct scmi_iterator *i = iter; 1881 1882 i->ph->xops->xfer_put(i->ph, i->t); 1883 kfree(i); 1884 } 1885 1886 static int scmi_iterator_run(void *iter) 1887 { 1888 int ret; 1889 1890 ret = __scmi_iterator_run(iter, NULL, NULL); 1891 scmi_iterator_bound_cleanup(iter); 1892 1893 return ret; 1894 } 1895 1896 static int scmi_iterator_run_bound(void *iter, unsigned int *start, 1897 unsigned int *end) 1898 { 1899 return __scmi_iterator_run(iter, start, end); 1900 } 1901 1902 struct scmi_msg_get_fc_info { 1903 __le32 domain; 1904 __le32 message_id; 1905 }; 1906 1907 struct scmi_msg_resp_desc_fc { 1908 __le32 attr; 1909 #define SUPPORTS_DOORBELL(x) ((x) & BIT(0)) 1910 #define DOORBELL_REG_WIDTH(x) FIELD_GET(GENMASK(2, 1), (x)) 1911 __le32 rate_limit; 1912 __le32 chan_addr_low; 1913 __le32 chan_addr_high; 1914 __le32 chan_size; 1915 __le32 db_addr_low; 1916 __le32 db_addr_high; 1917 __le32 db_set_lmask; 1918 __le32 db_set_hmask; 1919 __le32 db_preserve_lmask; 1920 __le32 db_preserve_hmask; 1921 }; 1922 1923 #define QUIRK_PERF_FC_FORCE \ 1924 ({ \ 1925 if (pi->proto->id == SCMI_PROTOCOL_PERF && \ 1926 message_id == 0x8 /* PERF_LEVEL_GET */) \ 1927 attributes |= BIT(0); \ 1928 }) 1929 1930 static void 1931 scmi_common_fastchannel_init(const struct scmi_protocol_handle *ph, 1932 u8 describe_id, u32 message_id, u32 valid_size, 1933 u32 domain, void __iomem **p_addr, 1934 struct scmi_fc_db_info **p_db, u32 *rate_limit) 1935 { 1936 int ret; 1937 u32 flags; 1938 u64 phys_addr; 1939 u32 attributes; 1940 u8 size; 1941 void __iomem *addr; 1942 struct scmi_xfer *t; 1943 struct scmi_fc_db_info *db = NULL; 1944 struct scmi_msg_get_fc_info *info; 1945 struct scmi_msg_resp_desc_fc *resp; 1946 const struct scmi_protocol_instance *pi = ph_to_pi(ph); 1947 1948 /* Check if the MSG_ID supports fastchannel */ 1949 ret = scmi_protocol_msg_check(ph, message_id, &attributes); 1950 SCMI_QUIRK(perf_level_get_fc_force, QUIRK_PERF_FC_FORCE); 1951 if (ret || !MSG_SUPPORTS_FASTCHANNEL(attributes)) { 1952 dev_dbg(ph->dev, 1953 "Skip FC init for 0x%02X/%d domain:%d - ret:%d\n", 1954 pi->proto->id, message_id, domain, ret); 1955 return; 1956 } 1957 1958 if (!p_addr) { 1959 ret = -EINVAL; 1960 goto err_out; 1961 } 1962 1963 ret = ph->xops->xfer_get_init(ph, describe_id, 1964 sizeof(*info), sizeof(*resp), &t); 1965 if (ret) 1966 goto err_out; 1967 1968 info = t->tx.buf; 1969 info->domain = cpu_to_le32(domain); 1970 info->message_id = cpu_to_le32(message_id); 1971 1972 /* 1973 * Bail out on error leaving fc_info addresses zeroed; this includes 1974 * the case in which the requested domain/message_id does NOT support 1975 * fastchannels at all. 1976 */ 1977 ret = ph->xops->do_xfer(ph, t); 1978 if (ret) 1979 goto err_xfer; 1980 1981 resp = t->rx.buf; 1982 flags = le32_to_cpu(resp->attr); 1983 size = le32_to_cpu(resp->chan_size); 1984 if (size != valid_size) { 1985 ret = -EINVAL; 1986 goto err_xfer; 1987 } 1988 1989 if (rate_limit) 1990 *rate_limit = le32_to_cpu(resp->rate_limit) & GENMASK(19, 0); 1991 1992 phys_addr = le32_to_cpu(resp->chan_addr_low); 1993 phys_addr |= (u64)le32_to_cpu(resp->chan_addr_high) << 32; 1994 addr = devm_ioremap(ph->dev, phys_addr, size); 1995 if (!addr) { 1996 ret = -EADDRNOTAVAIL; 1997 goto err_xfer; 1998 } 1999 2000 *p_addr = addr; 2001 2002 if (p_db && SUPPORTS_DOORBELL(flags)) { 2003 db = devm_kzalloc(ph->dev, sizeof(*db), GFP_KERNEL); 2004 if (!db) { 2005 ret = -ENOMEM; 2006 goto err_db; 2007 } 2008 2009 size = 1 << DOORBELL_REG_WIDTH(flags); 2010 phys_addr = le32_to_cpu(resp->db_addr_low); 2011 phys_addr |= (u64)le32_to_cpu(resp->db_addr_high) << 32; 2012 addr = devm_ioremap(ph->dev, phys_addr, size); 2013 if (!addr) { 2014 ret = -EADDRNOTAVAIL; 2015 goto err_db_mem; 2016 } 2017 2018 db->addr = addr; 2019 db->width = size; 2020 db->set = le32_to_cpu(resp->db_set_lmask); 2021 db->set |= (u64)le32_to_cpu(resp->db_set_hmask) << 32; 2022 db->mask = le32_to_cpu(resp->db_preserve_lmask); 2023 db->mask |= (u64)le32_to_cpu(resp->db_preserve_hmask) << 32; 2024 2025 *p_db = db; 2026 } 2027 2028 ph->xops->xfer_put(ph, t); 2029 2030 dev_dbg(ph->dev, 2031 "Using valid FC for protocol %X [MSG_ID:%u / RES_ID:%u]\n", 2032 pi->proto->id, message_id, domain); 2033 2034 return; 2035 2036 err_db_mem: 2037 devm_kfree(ph->dev, db); 2038 2039 err_db: 2040 *p_addr = NULL; 2041 2042 err_xfer: 2043 ph->xops->xfer_put(ph, t); 2044 2045 err_out: 2046 dev_warn(ph->dev, 2047 "Failed to get FC for protocol %X [MSG_ID:%u / RES_ID:%u] - ret:%d. Using regular messaging.\n", 2048 pi->proto->id, message_id, domain, ret); 2049 } 2050 2051 #define SCMI_PROTO_FC_RING_DB(w) \ 2052 do { \ 2053 u##w val = 0; \ 2054 \ 2055 if (db->mask) \ 2056 val = ioread##w(db->addr) & db->mask; \ 2057 iowrite##w((u##w)db->set | val, db->addr); \ 2058 } while (0) 2059 2060 static void scmi_common_fastchannel_db_ring(struct scmi_fc_db_info *db) 2061 { 2062 if (!db || !db->addr) 2063 return; 2064 2065 if (db->width == 1) 2066 SCMI_PROTO_FC_RING_DB(8); 2067 else if (db->width == 2) 2068 SCMI_PROTO_FC_RING_DB(16); 2069 else if (db->width == 4) 2070 SCMI_PROTO_FC_RING_DB(32); 2071 else /* db->width == 8 */ 2072 SCMI_PROTO_FC_RING_DB(64); 2073 } 2074 2075 static const struct scmi_proto_helpers_ops helpers_ops = { 2076 .extended_name_get = scmi_common_extended_name_get, 2077 .get_max_msg_size = scmi_common_get_max_msg_size, 2078 .iter_response_init = scmi_iterator_init, 2079 .iter_response_run = scmi_iterator_run, 2080 .iter_response_run_bound = scmi_iterator_run_bound, 2081 .iter_response_bound_cleanup = scmi_iterator_bound_cleanup, 2082 .protocol_msg_check = scmi_protocol_msg_check, 2083 .fastchannel_init = scmi_common_fastchannel_init, 2084 .fastchannel_db_ring = scmi_common_fastchannel_db_ring, 2085 }; 2086 2087 /** 2088 * scmi_revision_area_get - Retrieve version memory area. 2089 * 2090 * @ph: A reference to the protocol handle. 2091 * 2092 * A helper to grab the version memory area reference during SCMI Base protocol 2093 * initialization. 2094 * 2095 * Return: A reference to the version memory area associated to the SCMI 2096 * instance underlying this protocol handle. 2097 */ 2098 struct scmi_base_info * 2099 scmi_revision_area_get(const struct scmi_protocol_handle *ph) 2100 { 2101 const struct scmi_protocol_instance *pi = ph_to_pi(ph); 2102 2103 return pi->handle->version; 2104 } 2105 2106 /** 2107 * scmi_protocol_version_negotiate - Negotiate protocol version 2108 * 2109 * @ph: A reference to the protocol handle. 2110 * 2111 * An helper to negotiate a protocol version different from the latest 2112 * advertised as supported from the platform: on Success backward 2113 * compatibility is assured by the platform. 2114 * 2115 * Return: 0 on Success 2116 */ 2117 static int scmi_protocol_version_negotiate(struct scmi_protocol_handle *ph) 2118 { 2119 int ret; 2120 struct scmi_xfer *t; 2121 struct scmi_protocol_instance *pi = ph_to_pi(ph); 2122 2123 /* At first check if NEGOTIATE_PROTOCOL_VERSION is supported ... */ 2124 ret = scmi_protocol_msg_check(ph, NEGOTIATE_PROTOCOL_VERSION, NULL); 2125 if (ret) 2126 return ret; 2127 2128 /* ... then attempt protocol version negotiation */ 2129 ret = xfer_get_init(ph, NEGOTIATE_PROTOCOL_VERSION, 2130 sizeof(__le32), 0, &t); 2131 if (ret) 2132 return ret; 2133 2134 put_unaligned_le32(pi->proto->supported_version, t->tx.buf); 2135 ret = do_xfer(ph, t); 2136 if (!ret) 2137 pi->negotiated_version = pi->proto->supported_version; 2138 2139 xfer_put(ph, t); 2140 2141 return ret; 2142 } 2143 2144 /** 2145 * scmi_protocol_version_initialize - Initialize protocol version 2146 * @dev: A device reference. 2147 * @pi: A reference to the protocol instance being initialized 2148 * 2149 * At first retrieve the newest protocol version supported by the platform for 2150 * this specific protoocol. 2151 * 2152 * Negotiation is attempted only when the platform advertised a protocol 2153 * version newer than the most recent version known to this agent, since 2154 * backward compatibility is NOT assured in general between versions. 2155 * 2156 * Failing to negotiate a fallback version or to query supported version at 2157 * all will result in an attempt to use the newest version known to this agent 2158 * even though compatibility is NOT assured. 2159 * 2160 * Versions are defined as: 2161 * 2162 * pi->version: the version supported by the platform as returned by the query. 2163 * pi->proto->supported_version: the newest version supported by this agent 2164 * for this protocol. 2165 * pi->negotiated_version: The version successfully negotiated with the platform. 2166 * ph->version: The final version effectively chosen for this session. 2167 */ 2168 static void scmi_protocol_version_initialize(struct device *dev, 2169 struct scmi_protocol_instance *pi) 2170 { 2171 struct scmi_protocol_handle *ph = &pi->ph; 2172 int ret; 2173 2174 /* 2175 * Query and store platform supported protocol version: this is usually 2176 * the newest version the platfom can support. 2177 */ 2178 ret = version_get(ph, &pi->version); 2179 if (ret) { 2180 dev_warn(dev, 2181 "Failed to query supported version for protocol 0x%X.\n", 2182 pi->proto->id); 2183 goto best_effort; 2184 } 2185 2186 /* Need to negotiate at all ? */ 2187 if (pi->version <= pi->proto->supported_version) { 2188 ph->version = pi->version; 2189 return; 2190 } 2191 2192 /* Attempt negotiation */ 2193 ret = scmi_protocol_version_negotiate(ph); 2194 if (!ret) { 2195 ph->version = pi->negotiated_version; 2196 dev_info(dev, 2197 "Protocol 0x%X successfully negotiated version 0x%X\n", 2198 pi->proto->id, ph->version); 2199 return; 2200 } 2201 2202 dev_warn(dev, 2203 "Detected UNSUPPORTED higher version 0x%X for protocol 0x%X.\n", 2204 pi->version, pi->proto->id); 2205 2206 best_effort: 2207 /* Fallback to use newest version known to this agent */ 2208 ph->version = pi->proto->supported_version; 2209 dev_warn(dev, 2210 "Trying version 0x%X. Backward compatibility is NOT assured.\n", 2211 ph->version); 2212 } 2213 2214 /** 2215 * scmi_alloc_init_protocol_instance - Allocate and initialize a protocol 2216 * instance descriptor. 2217 * @info: The reference to the related SCMI instance. 2218 * @proto: The protocol descriptor. 2219 * 2220 * Allocate a new protocol instance descriptor, using the provided @proto 2221 * description, against the specified SCMI instance @info, and initialize it; 2222 * all resources management is handled via a dedicated per-protocol devres 2223 * group. 2224 * 2225 * Context: Assumes to be called with @protocols_mtx already acquired. 2226 * Return: A reference to a freshly allocated and initialized protocol instance 2227 * or ERR_PTR on failure. On failure the @proto reference is at first 2228 * put using @scmi_protocol_put() before releasing all the devres group. 2229 */ 2230 static struct scmi_protocol_instance * 2231 scmi_alloc_init_protocol_instance(struct scmi_info *info, 2232 const struct scmi_protocol *proto) 2233 { 2234 int ret = -ENOMEM; 2235 void *gid; 2236 struct scmi_protocol_instance *pi; 2237 const struct scmi_handle *handle = &info->handle; 2238 2239 /* Protocol specific devres group */ 2240 gid = devres_open_group(handle->dev, NULL, GFP_KERNEL); 2241 if (!gid) { 2242 scmi_protocol_put(proto); 2243 goto out; 2244 } 2245 2246 pi = devm_kzalloc(handle->dev, sizeof(*pi), GFP_KERNEL); 2247 if (!pi) 2248 goto clean; 2249 2250 pi->gid = gid; 2251 pi->proto = proto; 2252 pi->handle = handle; 2253 pi->ph.dev = handle->dev; 2254 pi->ph.xops = &xfer_ops; 2255 pi->ph.hops = &helpers_ops; 2256 pi->ph.set_priv = scmi_set_protocol_priv; 2257 pi->ph.get_priv = scmi_get_protocol_priv; 2258 refcount_set(&pi->users, 1); 2259 2260 /* 2261 * Initialize effectively used protocol version performing any 2262 * possibly needed negotiations. 2263 */ 2264 scmi_protocol_version_initialize(handle->dev, pi); 2265 2266 /* proto->init is assured NON NULL by scmi_protocol_register */ 2267 ret = pi->proto->instance_init(&pi->ph); 2268 if (ret) 2269 goto clean; 2270 2271 ret = idr_alloc(&info->protocols, pi, proto->id, proto->id + 1, 2272 GFP_KERNEL); 2273 if (ret != proto->id) 2274 goto clean; 2275 2276 /* 2277 * Warn but ignore events registration errors since we do not want 2278 * to skip whole protocols if their notifications are messed up. 2279 */ 2280 if (pi->proto->events) { 2281 ret = scmi_register_protocol_events(handle, pi->proto->id, 2282 &pi->ph, 2283 pi->proto->events); 2284 if (ret) 2285 dev_warn(handle->dev, 2286 "Protocol:%X - Events Registration Failed - err:%d\n", 2287 pi->proto->id, ret); 2288 } 2289 2290 devres_close_group(handle->dev, pi->gid); 2291 dev_dbg(handle->dev, "Initialized protocol: 0x%X\n", pi->proto->id); 2292 2293 return pi; 2294 2295 clean: 2296 /* Take care to put the protocol module's owner before releasing all */ 2297 scmi_protocol_put(proto); 2298 devres_release_group(handle->dev, gid); 2299 out: 2300 return ERR_PTR(ret); 2301 } 2302 2303 /** 2304 * scmi_get_protocol_instance - Protocol initialization helper. 2305 * @handle: A reference to the SCMI platform instance. 2306 * @protocol_id: The protocol being requested. 2307 * 2308 * In case the required protocol has never been requested before for this 2309 * instance, allocate and initialize all the needed structures while handling 2310 * resource allocation with a dedicated per-protocol devres subgroup. 2311 * 2312 * Return: A reference to an initialized protocol instance or error on failure: 2313 * in particular returns -EPROBE_DEFER when the desired protocol could 2314 * NOT be found. 2315 */ 2316 static struct scmi_protocol_instance * __must_check 2317 scmi_get_protocol_instance(const struct scmi_handle *handle, u8 protocol_id) 2318 { 2319 struct scmi_protocol_instance *pi; 2320 struct scmi_info *info = handle_to_scmi_info(handle); 2321 2322 mutex_lock(&info->protocols_mtx); 2323 pi = idr_find(&info->protocols, protocol_id); 2324 2325 if (pi) { 2326 refcount_inc(&pi->users); 2327 } else { 2328 const struct scmi_protocol *proto; 2329 2330 /* Fails if protocol not registered on bus */ 2331 proto = scmi_protocol_get(protocol_id, &info->version); 2332 if (proto) 2333 pi = scmi_alloc_init_protocol_instance(info, proto); 2334 else 2335 pi = ERR_PTR(-EPROBE_DEFER); 2336 } 2337 mutex_unlock(&info->protocols_mtx); 2338 2339 return pi; 2340 } 2341 2342 /** 2343 * scmi_protocol_acquire - Protocol acquire 2344 * @handle: A reference to the SCMI platform instance. 2345 * @protocol_id: The protocol being requested. 2346 * 2347 * Register a new user for the requested protocol on the specified SCMI 2348 * platform instance, possibly triggering its initialization on first user. 2349 * 2350 * Return: 0 if protocol was acquired successfully. 2351 */ 2352 int scmi_protocol_acquire(const struct scmi_handle *handle, u8 protocol_id) 2353 { 2354 return PTR_ERR_OR_ZERO(scmi_get_protocol_instance(handle, protocol_id)); 2355 } 2356 2357 /** 2358 * scmi_protocol_release - Protocol de-initialization helper. 2359 * @handle: A reference to the SCMI platform instance. 2360 * @protocol_id: The protocol being requested. 2361 * 2362 * Remove one user for the specified protocol and triggers de-initialization 2363 * and resources de-allocation once the last user has gone. 2364 */ 2365 void scmi_protocol_release(const struct scmi_handle *handle, u8 protocol_id) 2366 { 2367 struct scmi_info *info = handle_to_scmi_info(handle); 2368 struct scmi_protocol_instance *pi; 2369 2370 mutex_lock(&info->protocols_mtx); 2371 pi = idr_find(&info->protocols, protocol_id); 2372 if (WARN_ON(!pi)) 2373 goto out; 2374 2375 if (refcount_dec_and_test(&pi->users)) { 2376 void *gid = pi->gid; 2377 2378 if (pi->proto->events) 2379 scmi_deregister_protocol_events(handle, protocol_id); 2380 2381 if (pi->proto->instance_deinit) 2382 pi->proto->instance_deinit(&pi->ph); 2383 2384 idr_remove(&info->protocols, protocol_id); 2385 2386 scmi_protocol_put(pi->proto); 2387 2388 devres_release_group(handle->dev, gid); 2389 dev_dbg(handle->dev, "De-Initialized protocol: 0x%X\n", 2390 protocol_id); 2391 } 2392 2393 out: 2394 mutex_unlock(&info->protocols_mtx); 2395 } 2396 2397 void scmi_setup_protocol_implemented(const struct scmi_protocol_handle *ph, 2398 u8 *prot_imp) 2399 { 2400 const struct scmi_protocol_instance *pi = ph_to_pi(ph); 2401 struct scmi_info *info = handle_to_scmi_info(pi->handle); 2402 2403 info->protocols_imp = prot_imp; 2404 } 2405 2406 static bool 2407 scmi_is_protocol_implemented(const struct scmi_handle *handle, u8 prot_id) 2408 { 2409 int i; 2410 struct scmi_info *info = handle_to_scmi_info(handle); 2411 struct scmi_base_info *rev = handle->version; 2412 2413 if (!info->protocols_imp) 2414 return false; 2415 2416 for (i = 0; i < rev->num_protocols; i++) 2417 if (info->protocols_imp[i] == prot_id) 2418 return true; 2419 return false; 2420 } 2421 2422 struct scmi_protocol_devres { 2423 const struct scmi_handle *handle; 2424 u8 protocol_id; 2425 }; 2426 2427 static void scmi_devm_release_protocol(struct device *dev, void *res) 2428 { 2429 struct scmi_protocol_devres *dres = res; 2430 2431 scmi_protocol_release(dres->handle, dres->protocol_id); 2432 } 2433 2434 static struct scmi_protocol_instance __must_check * 2435 scmi_devres_protocol_instance_get(struct scmi_device *sdev, u8 protocol_id) 2436 { 2437 struct scmi_protocol_instance *pi; 2438 struct scmi_protocol_devres *dres; 2439 2440 dres = devres_alloc(scmi_devm_release_protocol, 2441 sizeof(*dres), GFP_KERNEL); 2442 if (!dres) 2443 return ERR_PTR(-ENOMEM); 2444 2445 pi = scmi_get_protocol_instance(sdev->handle, protocol_id); 2446 if (IS_ERR(pi)) { 2447 devres_free(dres); 2448 return pi; 2449 } 2450 2451 dres->handle = sdev->handle; 2452 dres->protocol_id = protocol_id; 2453 devres_add(&sdev->dev, dres); 2454 2455 return pi; 2456 } 2457 2458 /** 2459 * scmi_devm_protocol_get - Devres managed get protocol operations and handle 2460 * @sdev: A reference to an scmi_device whose embedded struct device is to 2461 * be used for devres accounting. 2462 * @protocol_id: The protocol being requested. 2463 * @ph: A pointer reference used to pass back the associated protocol handle. 2464 * 2465 * Get hold of a protocol accounting for its usage, eventually triggering its 2466 * initialization, and returning the protocol specific operations and related 2467 * protocol handle which will be used as first argument in most of the 2468 * protocols operations methods. 2469 * Being a devres based managed method, protocol hold will be automatically 2470 * released, and possibly de-initialized on last user, once the SCMI driver 2471 * owning the scmi_device is unbound from it. 2472 * 2473 * Return: A reference to the requested protocol operations or error. 2474 * Must be checked for errors by caller. 2475 */ 2476 static const void __must_check * 2477 scmi_devm_protocol_get(struct scmi_device *sdev, u8 protocol_id, 2478 struct scmi_protocol_handle **ph) 2479 { 2480 struct scmi_protocol_instance *pi; 2481 2482 if (!ph) 2483 return ERR_PTR(-EINVAL); 2484 2485 pi = scmi_devres_protocol_instance_get(sdev, protocol_id); 2486 if (IS_ERR(pi)) 2487 return pi; 2488 2489 *ph = &pi->ph; 2490 2491 return pi->proto->ops; 2492 } 2493 2494 /** 2495 * scmi_devm_protocol_acquire - Devres managed helper to get hold of a protocol 2496 * @sdev: A reference to an scmi_device whose embedded struct device is to 2497 * be used for devres accounting. 2498 * @protocol_id: The protocol being requested. 2499 * 2500 * Get hold of a protocol accounting for its usage, possibly triggering its 2501 * initialization but without getting access to its protocol specific operations 2502 * and handle. 2503 * 2504 * Being a devres based managed method, protocol hold will be automatically 2505 * released, and possibly de-initialized on last user, once the SCMI driver 2506 * owning the scmi_device is unbound from it. 2507 * 2508 * Return: 0 on SUCCESS 2509 */ 2510 static int __must_check scmi_devm_protocol_acquire(struct scmi_device *sdev, 2511 u8 protocol_id) 2512 { 2513 struct scmi_protocol_instance *pi; 2514 2515 pi = scmi_devres_protocol_instance_get(sdev, protocol_id); 2516 if (IS_ERR(pi)) 2517 return PTR_ERR(pi); 2518 2519 return 0; 2520 } 2521 2522 static int scmi_devm_protocol_match(struct device *dev, void *res, void *data) 2523 { 2524 struct scmi_protocol_devres *dres = res; 2525 2526 if (WARN_ON(!dres || !data)) 2527 return 0; 2528 2529 return dres->protocol_id == *((u8 *)data); 2530 } 2531 2532 /** 2533 * scmi_devm_protocol_put - Devres managed put protocol operations and handle 2534 * @sdev: A reference to an scmi_device whose embedded struct device is to 2535 * be used for devres accounting. 2536 * @protocol_id: The protocol being requested. 2537 * 2538 * Explicitly release a protocol hold previously obtained calling the above 2539 * @scmi_devm_protocol_get. 2540 */ 2541 static void scmi_devm_protocol_put(struct scmi_device *sdev, u8 protocol_id) 2542 { 2543 int ret; 2544 2545 ret = devres_release(&sdev->dev, scmi_devm_release_protocol, 2546 scmi_devm_protocol_match, &protocol_id); 2547 WARN_ON(ret); 2548 } 2549 2550 /** 2551 * scmi_is_transport_atomic - Method to check if underlying transport for an 2552 * SCMI instance is configured as atomic. 2553 * 2554 * @handle: A reference to the SCMI platform instance. 2555 * @atomic_threshold: An optional return value for the system wide currently 2556 * configured threshold for atomic operations. 2557 * 2558 * Return: True if transport is configured as atomic 2559 */ 2560 static bool scmi_is_transport_atomic(const struct scmi_handle *handle, 2561 unsigned int *atomic_threshold) 2562 { 2563 bool ret; 2564 struct scmi_info *info = handle_to_scmi_info(handle); 2565 2566 ret = info->desc->atomic_enabled && 2567 is_transport_polling_capable(info->desc); 2568 if (ret && atomic_threshold) 2569 *atomic_threshold = info->desc->atomic_threshold; 2570 2571 return ret; 2572 } 2573 2574 /** 2575 * scmi_handle_get() - Get the SCMI handle for a device 2576 * 2577 * @dev: pointer to device for which we want SCMI handle 2578 * 2579 * NOTE: The function does not track individual clients of the framework 2580 * and is expected to be maintained by caller of SCMI protocol library. 2581 * scmi_handle_put must be balanced with successful scmi_handle_get 2582 * 2583 * Return: pointer to handle if successful, NULL on error 2584 */ 2585 static struct scmi_handle *scmi_handle_get(struct device *dev) 2586 { 2587 struct scmi_info *info; 2588 struct scmi_handle *handle = NULL; 2589 2590 mutex_lock(&scmi_list_mutex); 2591 list_for_each_entry(info, &scmi_list, node) { 2592 if (dev->parent == info->dev) { 2593 info->users++; 2594 handle = &info->handle; 2595 break; 2596 } 2597 } 2598 mutex_unlock(&scmi_list_mutex); 2599 2600 return handle; 2601 } 2602 2603 /** 2604 * scmi_handle_put() - Release the handle acquired by scmi_handle_get 2605 * 2606 * @handle: handle acquired by scmi_handle_get 2607 * 2608 * NOTE: The function does not track individual clients of the framework 2609 * and is expected to be maintained by caller of SCMI protocol library. 2610 * scmi_handle_put must be balanced with successful scmi_handle_get 2611 * 2612 * Return: 0 is successfully released 2613 * if null was passed, it returns -EINVAL; 2614 */ 2615 static int scmi_handle_put(const struct scmi_handle *handle) 2616 { 2617 struct scmi_info *info; 2618 2619 if (!handle) 2620 return -EINVAL; 2621 2622 info = handle_to_scmi_info(handle); 2623 mutex_lock(&scmi_list_mutex); 2624 if (!WARN_ON(!info->users)) 2625 info->users--; 2626 mutex_unlock(&scmi_list_mutex); 2627 2628 return 0; 2629 } 2630 2631 static void scmi_device_link_add(struct device *consumer, 2632 struct device *supplier) 2633 { 2634 struct device_link *link; 2635 2636 link = device_link_add(consumer, supplier, DL_FLAG_AUTOREMOVE_CONSUMER); 2637 2638 WARN_ON(!link); 2639 } 2640 2641 static void scmi_set_handle(struct scmi_device *scmi_dev) 2642 { 2643 scmi_dev->handle = scmi_handle_get(&scmi_dev->dev); 2644 if (scmi_dev->handle) 2645 scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev); 2646 } 2647 2648 static int __scmi_xfer_info_init(struct scmi_info *sinfo, 2649 struct scmi_xfers_info *info) 2650 { 2651 int i; 2652 struct scmi_xfer *xfer; 2653 struct device *dev = sinfo->dev; 2654 const struct scmi_desc *desc = sinfo->desc; 2655 2656 /* Pre-allocated messages, no more than what hdr.seq can support */ 2657 if (WARN_ON(!info->max_msg || info->max_msg > MSG_TOKEN_MAX)) { 2658 dev_err(dev, 2659 "Invalid maximum messages %d, not in range [1 - %lu]\n", 2660 info->max_msg, MSG_TOKEN_MAX); 2661 return -EINVAL; 2662 } 2663 2664 hash_init(info->pending_xfers); 2665 2666 /* Allocate a bitmask sized to hold MSG_TOKEN_MAX tokens */ 2667 info->xfer_alloc_table = devm_bitmap_zalloc(dev, MSG_TOKEN_MAX, 2668 GFP_KERNEL); 2669 if (!info->xfer_alloc_table) 2670 return -ENOMEM; 2671 2672 /* 2673 * Preallocate a number of xfers equal to max inflight messages, 2674 * pre-initialize the buffer pointer to pre-allocated buffers and 2675 * attach all of them to the free list 2676 */ 2677 INIT_HLIST_HEAD(&info->free_xfers); 2678 for (i = 0; i < info->max_msg; i++) { 2679 xfer = devm_kzalloc(dev, sizeof(*xfer), GFP_KERNEL); 2680 if (!xfer) 2681 return -ENOMEM; 2682 2683 xfer->rx.buf = devm_kcalloc(dev, sizeof(u8), desc->max_msg_size, 2684 GFP_KERNEL); 2685 if (!xfer->rx.buf) 2686 return -ENOMEM; 2687 2688 xfer->tx.buf = xfer->rx.buf; 2689 init_completion(&xfer->done); 2690 spin_lock_init(&xfer->lock); 2691 2692 /* Add initialized xfer to the free list */ 2693 hlist_add_head(&xfer->node, &info->free_xfers); 2694 } 2695 2696 spin_lock_init(&info->xfer_lock); 2697 2698 return 0; 2699 } 2700 2701 static int scmi_channels_max_msg_configure(struct scmi_info *sinfo) 2702 { 2703 const struct scmi_desc *desc = sinfo->desc; 2704 2705 if (!desc->ops->get_max_msg) { 2706 sinfo->tx_minfo.max_msg = desc->max_msg; 2707 sinfo->rx_minfo.max_msg = desc->max_msg; 2708 } else { 2709 struct scmi_chan_info *base_cinfo; 2710 2711 base_cinfo = idr_find(&sinfo->tx_idr, SCMI_PROTOCOL_BASE); 2712 if (!base_cinfo) 2713 return -EINVAL; 2714 sinfo->tx_minfo.max_msg = desc->ops->get_max_msg(base_cinfo); 2715 2716 /* RX channel is optional so can be skipped */ 2717 base_cinfo = idr_find(&sinfo->rx_idr, SCMI_PROTOCOL_BASE); 2718 if (base_cinfo) 2719 sinfo->rx_minfo.max_msg = 2720 desc->ops->get_max_msg(base_cinfo); 2721 } 2722 2723 return 0; 2724 } 2725 2726 static int scmi_xfer_info_init(struct scmi_info *sinfo) 2727 { 2728 int ret; 2729 2730 ret = scmi_channels_max_msg_configure(sinfo); 2731 if (ret) 2732 return ret; 2733 2734 ret = __scmi_xfer_info_init(sinfo, &sinfo->tx_minfo); 2735 if (!ret && !idr_is_empty(&sinfo->rx_idr)) 2736 ret = __scmi_xfer_info_init(sinfo, &sinfo->rx_minfo); 2737 2738 return ret; 2739 } 2740 2741 static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, 2742 int prot_id, bool tx) 2743 { 2744 int ret, idx; 2745 char name[32]; 2746 struct scmi_chan_info *cinfo; 2747 struct idr *idr; 2748 struct scmi_device *tdev = NULL; 2749 2750 /* Transmit channel is first entry i.e. index 0 */ 2751 idx = tx ? 0 : 1; 2752 idr = tx ? &info->tx_idr : &info->rx_idr; 2753 2754 if (!info->desc->ops->chan_available(of_node, idx)) { 2755 cinfo = idr_find(idr, SCMI_PROTOCOL_BASE); 2756 if (unlikely(!cinfo)) /* Possible only if platform has no Rx */ 2757 return -EINVAL; 2758 goto idr_alloc; 2759 } 2760 2761 cinfo = devm_kzalloc(info->dev, sizeof(*cinfo), GFP_KERNEL); 2762 if (!cinfo) 2763 return -ENOMEM; 2764 2765 cinfo->is_p2a = !tx; 2766 cinfo->rx_timeout_ms = info->desc->max_rx_timeout_ms; 2767 cinfo->max_msg_size = info->desc->max_msg_size; 2768 cinfo->no_completion_irq = info->desc->no_completion_irq; 2769 2770 /* Create a unique name for this transport device */ 2771 snprintf(name, 32, "__scmi_transport_device_%s_%02X", 2772 idx ? "rx" : "tx", prot_id); 2773 /* Create a uniquely named, dedicated transport device for this chan */ 2774 tdev = scmi_device_create(of_node, info->dev, prot_id, name); 2775 if (!tdev) { 2776 dev_err(info->dev, 2777 "failed to create transport device (%s)\n", name); 2778 devm_kfree(info->dev, cinfo); 2779 return -EINVAL; 2780 } 2781 of_node_get(of_node); 2782 2783 cinfo->id = prot_id; 2784 cinfo->dev = &tdev->dev; 2785 ret = info->desc->ops->chan_setup(cinfo, info->dev, tx); 2786 if (ret) { 2787 of_node_put(of_node); 2788 scmi_device_destroy(info->dev, prot_id, name); 2789 devm_kfree(info->dev, cinfo); 2790 return ret; 2791 } 2792 2793 if (tx && is_polling_required(cinfo, info->desc)) { 2794 if (is_transport_polling_capable(info->desc)) 2795 dev_info(&tdev->dev, 2796 "Enabled polling mode TX channel - prot_id:%d\n", 2797 prot_id); 2798 else 2799 dev_warn(&tdev->dev, 2800 "Polling mode NOT supported by transport.\n"); 2801 } 2802 2803 idr_alloc: 2804 ret = idr_alloc(idr, cinfo, prot_id, prot_id + 1, GFP_KERNEL); 2805 if (ret != prot_id) { 2806 dev_err(info->dev, 2807 "unable to allocate SCMI idr slot err %d\n", ret); 2808 /* Destroy channel and device only if created by this call. */ 2809 if (tdev) { 2810 of_node_put(of_node); 2811 scmi_device_destroy(info->dev, prot_id, name); 2812 devm_kfree(info->dev, cinfo); 2813 } 2814 return ret; 2815 } 2816 2817 cinfo->handle = &info->handle; 2818 return 0; 2819 } 2820 2821 static inline int 2822 scmi_txrx_setup(struct scmi_info *info, struct device_node *of_node, 2823 int prot_id) 2824 { 2825 int ret = scmi_chan_setup(info, of_node, prot_id, true); 2826 2827 if (!ret) { 2828 /* Rx is optional, report only memory errors */ 2829 ret = scmi_chan_setup(info, of_node, prot_id, false); 2830 if (ret && ret != -ENOMEM) 2831 ret = 0; 2832 } 2833 2834 if (ret) 2835 dev_err(info->dev, 2836 "failed to setup channel for protocol:0x%X\n", prot_id); 2837 2838 return ret; 2839 } 2840 2841 /** 2842 * scmi_channels_setup - Helper to initialize all required channels 2843 * 2844 * @info: The SCMI instance descriptor. 2845 * 2846 * Initialize all the channels found described in the DT against the underlying 2847 * configured transport using custom defined dedicated devices instead of 2848 * borrowing devices from the SCMI drivers; this way channels are initialized 2849 * upfront during core SCMI stack probing and are no more coupled with SCMI 2850 * devices used by SCMI drivers. 2851 * 2852 * Note that, even though a pair of TX/RX channels is associated to each 2853 * protocol defined in the DT, a distinct freshly initialized channel is 2854 * created only if the DT node for the protocol at hand describes a dedicated 2855 * channel: in all the other cases the common BASE protocol channel is reused. 2856 * 2857 * Return: 0 on Success 2858 */ 2859 static int scmi_channels_setup(struct scmi_info *info) 2860 { 2861 int ret; 2862 struct device_node *top_np = info->dev->of_node; 2863 2864 /* Initialize a common generic channel at first */ 2865 ret = scmi_txrx_setup(info, top_np, SCMI_PROTOCOL_BASE); 2866 if (ret) 2867 return ret; 2868 2869 for_each_available_child_of_node_scoped(top_np, child) { 2870 u32 prot_id; 2871 2872 if (of_property_read_u32(child, "reg", &prot_id)) 2873 continue; 2874 2875 if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) 2876 dev_err(info->dev, 2877 "Out of range protocol %d\n", prot_id); 2878 2879 ret = scmi_txrx_setup(info, child, prot_id); 2880 if (ret) 2881 return ret; 2882 } 2883 2884 return 0; 2885 } 2886 2887 static int scmi_chan_destroy(int id, void *p, void *idr) 2888 { 2889 struct scmi_chan_info *cinfo = p; 2890 2891 if (cinfo->dev) { 2892 struct scmi_info *info = handle_to_scmi_info(cinfo->handle); 2893 struct scmi_device *sdev = to_scmi_dev(cinfo->dev); 2894 2895 of_node_put(cinfo->dev->of_node); 2896 scmi_device_destroy(info->dev, id, sdev->name); 2897 cinfo->dev = NULL; 2898 } 2899 2900 idr_remove(idr, id); 2901 2902 return 0; 2903 } 2904 2905 static void scmi_cleanup_channels(struct scmi_info *info, struct idr *idr) 2906 { 2907 /* At first free all channels at the transport layer ... */ 2908 idr_for_each(idr, info->desc->ops->chan_free, idr); 2909 2910 /* ...then destroy all underlying devices */ 2911 idr_for_each(idr, scmi_chan_destroy, idr); 2912 2913 idr_destroy(idr); 2914 } 2915 2916 static void scmi_cleanup_txrx_channels(struct scmi_info *info) 2917 { 2918 scmi_cleanup_channels(info, &info->tx_idr); 2919 2920 scmi_cleanup_channels(info, &info->rx_idr); 2921 } 2922 2923 static int scmi_bus_notifier(struct notifier_block *nb, 2924 unsigned long action, void *data) 2925 { 2926 struct scmi_info *info = bus_nb_to_scmi_info(nb); 2927 struct scmi_device *sdev = to_scmi_dev(data); 2928 2929 /* Skip devices of different SCMI instances */ 2930 if (sdev->dev.parent != info->dev) 2931 return NOTIFY_DONE; 2932 2933 switch (action) { 2934 case BUS_NOTIFY_BIND_DRIVER: 2935 /* setup handle now as the transport is ready */ 2936 scmi_set_handle(sdev); 2937 break; 2938 case BUS_NOTIFY_UNBOUND_DRIVER: 2939 scmi_handle_put(sdev->handle); 2940 sdev->handle = NULL; 2941 break; 2942 default: 2943 return NOTIFY_DONE; 2944 } 2945 2946 dev_dbg(info->dev, "Device %s (%s) is now %s\n", dev_name(&sdev->dev), 2947 sdev->name, action == BUS_NOTIFY_BIND_DRIVER ? 2948 "about to be BOUND." : "UNBOUND."); 2949 2950 return NOTIFY_OK; 2951 } 2952 2953 static int scmi_device_request_notifier(struct notifier_block *nb, 2954 unsigned long action, void *data) 2955 { 2956 struct device_node *np; 2957 struct scmi_device_id *id_table = data; 2958 struct scmi_info *info = req_nb_to_scmi_info(nb); 2959 2960 np = idr_find(&info->active_protocols, id_table->protocol_id); 2961 if (!np) 2962 return NOTIFY_DONE; 2963 2964 dev_dbg(info->dev, "%sRequested device (%s) for protocol 0x%x\n", 2965 action == SCMI_BUS_NOTIFY_DEVICE_REQUEST ? "" : "UN-", 2966 id_table->name, id_table->protocol_id); 2967 2968 switch (action) { 2969 case SCMI_BUS_NOTIFY_DEVICE_REQUEST: 2970 scmi_create_protocol_devices(np, info, id_table->protocol_id, 2971 id_table->name); 2972 break; 2973 case SCMI_BUS_NOTIFY_DEVICE_UNREQUEST: 2974 scmi_destroy_protocol_devices(info, id_table->protocol_id, 2975 id_table->name); 2976 break; 2977 default: 2978 return NOTIFY_DONE; 2979 } 2980 2981 return NOTIFY_OK; 2982 } 2983 2984 static const char * const dbg_counter_strs[] = { 2985 "sent_ok", 2986 "sent_fail", 2987 "sent_fail_polling_unsupported", 2988 "sent_fail_channel_not_found", 2989 "response_ok", 2990 "notification_ok", 2991 "delayed_response_ok", 2992 "xfers_response_timeout", 2993 "xfers_response_polled_timeout", 2994 "response_polled_ok", 2995 "err_msg_unexpected", 2996 "err_msg_invalid", 2997 "err_msg_nomem", 2998 "err_protocol", 2999 "xfers_inflight", 3000 }; 3001 3002 static ssize_t reset_all_on_write(struct file *filp, const char __user *buf, 3003 size_t count, loff_t *ppos) 3004 { 3005 struct scmi_debug_info *dbg = filp->private_data; 3006 3007 for (int i = 0; i < SCMI_DEBUG_COUNTERS_LAST; i++) 3008 atomic_set(&dbg->counters[i], 0); 3009 3010 return count; 3011 } 3012 3013 static const struct file_operations fops_reset_counts = { 3014 .owner = THIS_MODULE, 3015 .open = simple_open, 3016 .write = reset_all_on_write, 3017 }; 3018 3019 static void scmi_debugfs_counters_setup(struct scmi_debug_info *dbg, 3020 struct dentry *trans) 3021 { 3022 struct dentry *counters; 3023 int idx; 3024 3025 counters = debugfs_create_dir("counters", trans); 3026 3027 for (idx = 0; idx < SCMI_DEBUG_COUNTERS_LAST; idx++) 3028 debugfs_create_atomic_t(dbg_counter_strs[idx], 0600, counters, 3029 &dbg->counters[idx]); 3030 3031 debugfs_create_file("reset", 0200, counters, dbg, &fops_reset_counts); 3032 } 3033 3034 static void scmi_debugfs_common_cleanup(void *d) 3035 { 3036 struct scmi_debug_info *dbg = d; 3037 3038 if (!dbg) 3039 return; 3040 3041 debugfs_remove_recursive(dbg->top_dentry); 3042 kfree(dbg->name); 3043 kfree(dbg->type); 3044 } 3045 3046 static struct scmi_debug_info *scmi_debugfs_common_setup(struct scmi_info *info) 3047 { 3048 char top_dir[16]; 3049 struct dentry *trans, *top_dentry; 3050 struct scmi_debug_info *dbg; 3051 const char *c_ptr = NULL; 3052 3053 dbg = devm_kzalloc(info->dev, sizeof(*dbg), GFP_KERNEL); 3054 if (!dbg) 3055 return NULL; 3056 3057 dbg->name = kstrdup(of_node_full_name(info->dev->of_node), GFP_KERNEL); 3058 if (!dbg->name) { 3059 devm_kfree(info->dev, dbg); 3060 return NULL; 3061 } 3062 3063 of_property_read_string(info->dev->of_node, "compatible", &c_ptr); 3064 dbg->type = kstrdup(c_ptr, GFP_KERNEL); 3065 if (!dbg->type) { 3066 kfree(dbg->name); 3067 devm_kfree(info->dev, dbg); 3068 return NULL; 3069 } 3070 3071 snprintf(top_dir, 16, "%d", info->id); 3072 top_dentry = debugfs_create_dir(top_dir, scmi_top_dentry); 3073 trans = debugfs_create_dir("transport", top_dentry); 3074 3075 dbg->is_atomic = info->desc->atomic_enabled && 3076 is_transport_polling_capable(info->desc); 3077 3078 debugfs_create_str("instance_name", 0400, top_dentry, 3079 (char **)&dbg->name); 3080 3081 debugfs_create_u32("atomic_threshold_us", 0400, top_dentry, 3082 (u32 *)&info->desc->atomic_threshold); 3083 3084 debugfs_create_str("type", 0400, trans, (char **)&dbg->type); 3085 3086 debugfs_create_bool("is_atomic", 0400, trans, &dbg->is_atomic); 3087 3088 debugfs_create_u32("max_rx_timeout_ms", 0400, trans, 3089 (u32 *)&info->desc->max_rx_timeout_ms); 3090 3091 debugfs_create_u32("max_msg_size", 0400, trans, 3092 (u32 *)&info->desc->max_msg_size); 3093 3094 debugfs_create_u32("tx_max_msg", 0400, trans, 3095 (u32 *)&info->tx_minfo.max_msg); 3096 3097 debugfs_create_u32("rx_max_msg", 0400, trans, 3098 (u32 *)&info->rx_minfo.max_msg); 3099 3100 if (IS_ENABLED(CONFIG_ARM_SCMI_DEBUG_COUNTERS)) 3101 scmi_debugfs_counters_setup(dbg, trans); 3102 3103 dbg->top_dentry = top_dentry; 3104 3105 if (devm_add_action_or_reset(info->dev, 3106 scmi_debugfs_common_cleanup, dbg)) 3107 return NULL; 3108 3109 return dbg; 3110 } 3111 3112 static int scmi_debugfs_raw_mode_setup(struct scmi_info *info) 3113 { 3114 int id, num_chans = 0, ret = 0; 3115 struct scmi_chan_info *cinfo; 3116 u8 channels[SCMI_MAX_CHANNELS] = {}; 3117 DECLARE_BITMAP(protos, SCMI_MAX_CHANNELS) = {}; 3118 3119 /* Enumerate all channels to collect their ids */ 3120 idr_for_each_entry(&info->tx_idr, cinfo, id) { 3121 /* 3122 * Cannot happen, but be defensive. 3123 * Zero as num_chans is ok, warn and carry on. 3124 */ 3125 if (num_chans >= SCMI_MAX_CHANNELS || !cinfo) { 3126 dev_warn(info->dev, 3127 "SCMI RAW - Error enumerating channels\n"); 3128 break; 3129 } 3130 3131 if (!test_bit(cinfo->id, protos)) { 3132 channels[num_chans++] = cinfo->id; 3133 set_bit(cinfo->id, protos); 3134 } 3135 } 3136 3137 info->raw = scmi_raw_mode_init(&info->handle, info->dbg->top_dentry, 3138 info->id, channels, num_chans, 3139 info->desc, info->tx_minfo.max_msg); 3140 if (IS_ERR(info->raw)) { 3141 dev_err(info->dev, "Failed to initialize SCMI RAW Mode !\n"); 3142 ret = PTR_ERR(info->raw); 3143 info->raw = NULL; 3144 } 3145 3146 return ret; 3147 } 3148 3149 static const struct scmi_desc *scmi_transport_setup(struct device *dev) 3150 { 3151 struct scmi_transport *trans; 3152 int ret; 3153 3154 trans = dev_get_platdata(dev); 3155 if (!trans || !trans->supplier || !trans->core_ops) 3156 return NULL; 3157 3158 if (!device_link_add(dev, trans->supplier, DL_FLAG_AUTOREMOVE_CONSUMER)) { 3159 dev_err(dev, 3160 "Adding link to supplier transport device failed\n"); 3161 return NULL; 3162 } 3163 3164 /* Provide core transport ops */ 3165 *trans->core_ops = &scmi_trans_core_ops; 3166 3167 dev_info(dev, "Using %s\n", dev_driver_string(trans->supplier)); 3168 3169 ret = of_property_read_u32(dev->of_node, "arm,max-rx-timeout-ms", 3170 &trans->desc.max_rx_timeout_ms); 3171 if (ret && ret != -EINVAL) 3172 dev_err(dev, "Malformed arm,max-rx-timeout-ms DT property.\n"); 3173 3174 ret = of_property_read_u32(dev->of_node, "arm,max-msg-size", 3175 &trans->desc.max_msg_size); 3176 if (ret && ret != -EINVAL) 3177 dev_err(dev, "Malformed arm,max-msg-size DT property.\n"); 3178 3179 ret = of_property_read_u32(dev->of_node, "arm,max-msg", 3180 &trans->desc.max_msg); 3181 if (ret && ret != -EINVAL) 3182 dev_err(dev, "Malformed arm,max-msg DT property.\n"); 3183 3184 trans->desc.no_completion_irq = of_property_read_bool(dev->of_node, 3185 "arm,no-completion-irq"); 3186 3187 dev_info(dev, 3188 "SCMI max-rx-timeout: %dms / max-msg-size: %dbytes / max-msg: %d\n", 3189 trans->desc.max_rx_timeout_ms, trans->desc.max_msg_size, 3190 trans->desc.max_msg); 3191 3192 /* System wide atomic threshold for atomic ops .. if any */ 3193 if (!of_property_read_u32(dev->of_node, "atomic-threshold-us", 3194 &trans->desc.atomic_threshold)) 3195 dev_info(dev, 3196 "SCMI System wide atomic threshold set to %u us\n", 3197 trans->desc.atomic_threshold); 3198 3199 return &trans->desc; 3200 } 3201 3202 static void scmi_enable_matching_quirks(struct scmi_info *info) 3203 { 3204 struct scmi_base_info *rev = &info->version; 3205 3206 dev_dbg(info->dev, "Looking for quirks matching: %s/%s/0x%08X\n", 3207 rev->vendor_id, rev->sub_vendor_id, rev->impl_ver); 3208 3209 /* Enable applicable quirks */ 3210 scmi_quirks_enable(info->dev, rev->vendor_id, 3211 rev->sub_vendor_id, rev->impl_ver); 3212 } 3213 3214 static int scmi_probe(struct platform_device *pdev) 3215 { 3216 int ret; 3217 char *err_str = "probe failure\n"; 3218 struct scmi_handle *handle; 3219 const struct scmi_desc *desc; 3220 struct scmi_info *info; 3221 bool coex = IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT_COEX); 3222 struct device *dev = &pdev->dev; 3223 struct device_node *child, *np = dev->of_node; 3224 3225 desc = scmi_transport_setup(dev); 3226 if (!desc) { 3227 err_str = "transport invalid\n"; 3228 ret = -EINVAL; 3229 goto out_err; 3230 } 3231 3232 info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); 3233 if (!info) 3234 return -ENOMEM; 3235 3236 info->id = ida_alloc_min(&scmi_id, 0, GFP_KERNEL); 3237 if (info->id < 0) 3238 return info->id; 3239 3240 info->dev = dev; 3241 info->desc = desc; 3242 info->bus_nb.notifier_call = scmi_bus_notifier; 3243 info->dev_req_nb.notifier_call = scmi_device_request_notifier; 3244 INIT_LIST_HEAD(&info->node); 3245 idr_init(&info->protocols); 3246 mutex_init(&info->protocols_mtx); 3247 idr_init(&info->active_protocols); 3248 mutex_init(&info->devreq_mtx); 3249 3250 platform_set_drvdata(pdev, info); 3251 idr_init(&info->tx_idr); 3252 idr_init(&info->rx_idr); 3253 3254 handle = &info->handle; 3255 handle->dev = info->dev; 3256 handle->version = &info->version; 3257 handle->devm_protocol_acquire = scmi_devm_protocol_acquire; 3258 handle->devm_protocol_get = scmi_devm_protocol_get; 3259 handle->devm_protocol_put = scmi_devm_protocol_put; 3260 handle->is_transport_atomic = scmi_is_transport_atomic; 3261 3262 /* Setup all channels described in the DT at first */ 3263 ret = scmi_channels_setup(info); 3264 if (ret) { 3265 err_str = "failed to setup channels\n"; 3266 goto clear_ida; 3267 } 3268 3269 ret = bus_register_notifier(&scmi_bus_type, &info->bus_nb); 3270 if (ret) { 3271 err_str = "failed to register bus notifier\n"; 3272 goto clear_txrx_setup; 3273 } 3274 3275 ret = blocking_notifier_chain_register(&scmi_requested_devices_nh, 3276 &info->dev_req_nb); 3277 if (ret) { 3278 err_str = "failed to register device notifier\n"; 3279 goto clear_bus_notifier; 3280 } 3281 3282 ret = scmi_xfer_info_init(info); 3283 if (ret) { 3284 err_str = "failed to init xfers pool\n"; 3285 goto clear_dev_req_notifier; 3286 } 3287 3288 if (scmi_top_dentry) { 3289 info->dbg = scmi_debugfs_common_setup(info); 3290 if (!info->dbg) 3291 dev_warn(dev, "Failed to setup SCMI debugfs.\n"); 3292 3293 if (info->dbg && IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) { 3294 ret = scmi_debugfs_raw_mode_setup(info); 3295 if (!coex) { 3296 if (ret) 3297 goto clear_dev_req_notifier; 3298 3299 /* Bail out anyway when coex disabled. */ 3300 return 0; 3301 } 3302 3303 /* Coex enabled, carry on in any case. */ 3304 dev_info(dev, "SCMI RAW Mode COEX enabled !\n"); 3305 } 3306 } 3307 3308 if (scmi_notification_init(handle)) 3309 dev_err(dev, "SCMI Notifications NOT available.\n"); 3310 3311 if (info->desc->atomic_enabled && 3312 !is_transport_polling_capable(info->desc)) 3313 dev_err(dev, 3314 "Transport is not polling capable. Atomic mode not supported.\n"); 3315 3316 /* 3317 * Trigger SCMI Base protocol initialization. 3318 * It's mandatory and won't be ever released/deinit until the 3319 * SCMI stack is shutdown/unloaded as a whole. 3320 */ 3321 ret = scmi_protocol_acquire(handle, SCMI_PROTOCOL_BASE); 3322 if (ret) { 3323 err_str = "unable to communicate with SCMI\n"; 3324 if (coex) { 3325 dev_err(dev, "%s", err_str); 3326 return 0; 3327 } 3328 goto notification_exit; 3329 } 3330 3331 mutex_lock(&scmi_list_mutex); 3332 list_add_tail(&info->node, &scmi_list); 3333 mutex_unlock(&scmi_list_mutex); 3334 3335 scmi_enable_matching_quirks(info); 3336 3337 for_each_available_child_of_node(np, child) { 3338 u32 prot_id; 3339 3340 if (of_property_read_u32(child, "reg", &prot_id)) 3341 continue; 3342 3343 if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) 3344 dev_err(dev, "Out of range protocol %d\n", prot_id); 3345 3346 if (!scmi_is_protocol_implemented(handle, prot_id)) { 3347 dev_err(dev, "SCMI protocol %d not implemented\n", 3348 prot_id); 3349 continue; 3350 } 3351 3352 /* 3353 * Save this valid DT protocol descriptor amongst 3354 * @active_protocols for this SCMI instance/ 3355 */ 3356 ret = idr_alloc(&info->active_protocols, child, 3357 prot_id, prot_id + 1, GFP_KERNEL); 3358 if (ret != prot_id) { 3359 dev_err(dev, "SCMI protocol %d already activated. Skip\n", 3360 prot_id); 3361 continue; 3362 } 3363 3364 of_node_get(child); 3365 scmi_create_protocol_devices(child, info, prot_id, NULL); 3366 } 3367 3368 return 0; 3369 3370 notification_exit: 3371 if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) 3372 scmi_raw_mode_cleanup(info->raw); 3373 scmi_notification_exit(&info->handle); 3374 clear_dev_req_notifier: 3375 blocking_notifier_chain_unregister(&scmi_requested_devices_nh, 3376 &info->dev_req_nb); 3377 clear_bus_notifier: 3378 bus_unregister_notifier(&scmi_bus_type, &info->bus_nb); 3379 clear_txrx_setup: 3380 scmi_cleanup_txrx_channels(info); 3381 clear_ida: 3382 ida_free(&scmi_id, info->id); 3383 3384 out_err: 3385 return dev_err_probe(dev, ret, "%s", err_str); 3386 } 3387 3388 static void scmi_remove(struct platform_device *pdev) 3389 { 3390 int id; 3391 struct scmi_info *info = platform_get_drvdata(pdev); 3392 struct device_node *child; 3393 3394 if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) 3395 scmi_raw_mode_cleanup(info->raw); 3396 3397 mutex_lock(&scmi_list_mutex); 3398 if (info->users) 3399 dev_warn(&pdev->dev, 3400 "Still active SCMI users will be forcibly unbound.\n"); 3401 list_del(&info->node); 3402 mutex_unlock(&scmi_list_mutex); 3403 3404 scmi_notification_exit(&info->handle); 3405 3406 mutex_lock(&info->protocols_mtx); 3407 idr_destroy(&info->protocols); 3408 mutex_unlock(&info->protocols_mtx); 3409 3410 idr_for_each_entry(&info->active_protocols, child, id) 3411 of_node_put(child); 3412 idr_destroy(&info->active_protocols); 3413 3414 blocking_notifier_chain_unregister(&scmi_requested_devices_nh, 3415 &info->dev_req_nb); 3416 bus_unregister_notifier(&scmi_bus_type, &info->bus_nb); 3417 3418 /* Safe to free channels since no more users */ 3419 scmi_cleanup_txrx_channels(info); 3420 3421 ida_free(&scmi_id, info->id); 3422 } 3423 3424 static ssize_t protocol_version_show(struct device *dev, 3425 struct device_attribute *attr, char *buf) 3426 { 3427 struct scmi_info *info = dev_get_drvdata(dev); 3428 3429 return sprintf(buf, "%u.%u\n", info->version.major_ver, 3430 info->version.minor_ver); 3431 } 3432 static DEVICE_ATTR_RO(protocol_version); 3433 3434 static ssize_t firmware_version_show(struct device *dev, 3435 struct device_attribute *attr, char *buf) 3436 { 3437 struct scmi_info *info = dev_get_drvdata(dev); 3438 3439 return sprintf(buf, "0x%x\n", info->version.impl_ver); 3440 } 3441 static DEVICE_ATTR_RO(firmware_version); 3442 3443 static ssize_t vendor_id_show(struct device *dev, 3444 struct device_attribute *attr, char *buf) 3445 { 3446 struct scmi_info *info = dev_get_drvdata(dev); 3447 3448 return sprintf(buf, "%s\n", info->version.vendor_id); 3449 } 3450 static DEVICE_ATTR_RO(vendor_id); 3451 3452 static ssize_t sub_vendor_id_show(struct device *dev, 3453 struct device_attribute *attr, char *buf) 3454 { 3455 struct scmi_info *info = dev_get_drvdata(dev); 3456 3457 return sprintf(buf, "%s\n", info->version.sub_vendor_id); 3458 } 3459 static DEVICE_ATTR_RO(sub_vendor_id); 3460 3461 static struct attribute *versions_attrs[] = { 3462 &dev_attr_firmware_version.attr, 3463 &dev_attr_protocol_version.attr, 3464 &dev_attr_vendor_id.attr, 3465 &dev_attr_sub_vendor_id.attr, 3466 NULL, 3467 }; 3468 ATTRIBUTE_GROUPS(versions); 3469 3470 static struct platform_driver scmi_driver = { 3471 .driver = { 3472 .name = "arm-scmi", 3473 .suppress_bind_attrs = true, 3474 .dev_groups = versions_groups, 3475 }, 3476 .probe = scmi_probe, 3477 .remove = scmi_remove, 3478 }; 3479 3480 static struct dentry *scmi_debugfs_init(void) 3481 { 3482 struct dentry *d; 3483 3484 d = debugfs_create_dir("scmi", NULL); 3485 if (IS_ERR(d)) { 3486 pr_err("Could NOT create SCMI top dentry.\n"); 3487 return NULL; 3488 } 3489 3490 return d; 3491 } 3492 3493 int scmi_inflight_count(const struct scmi_handle *handle) 3494 { 3495 if (IS_ENABLED(CONFIG_ARM_SCMI_DEBUG_COUNTERS)) { 3496 struct scmi_info *info = handle_to_scmi_info(handle); 3497 3498 if (!info->dbg) 3499 return 0; 3500 3501 return atomic_read(&info->dbg->counters[XFERS_INFLIGHT]); 3502 } else { 3503 return 0; 3504 } 3505 } 3506 3507 static int __init scmi_driver_init(void) 3508 { 3509 scmi_quirks_initialize(); 3510 3511 /* Bail out if no SCMI transport was configured */ 3512 if (WARN_ON(!IS_ENABLED(CONFIG_ARM_SCMI_HAVE_TRANSPORT))) 3513 return -EINVAL; 3514 3515 if (IS_ENABLED(CONFIG_ARM_SCMI_HAVE_SHMEM)) 3516 scmi_trans_core_ops.shmem = scmi_shared_mem_operations_get(); 3517 3518 if (IS_ENABLED(CONFIG_ARM_SCMI_HAVE_MSG)) 3519 scmi_trans_core_ops.msg = scmi_message_operations_get(); 3520 3521 if (IS_ENABLED(CONFIG_ARM_SCMI_NEED_DEBUGFS)) 3522 scmi_top_dentry = scmi_debugfs_init(); 3523 3524 scmi_base_register(); 3525 3526 scmi_clock_register(); 3527 scmi_perf_register(); 3528 scmi_power_register(); 3529 scmi_reset_register(); 3530 scmi_sensors_register(); 3531 scmi_voltage_register(); 3532 scmi_system_register(); 3533 scmi_powercap_register(); 3534 scmi_pinctrl_register(); 3535 3536 return platform_driver_register(&scmi_driver); 3537 } 3538 module_init(scmi_driver_init); 3539 3540 static void __exit scmi_driver_exit(void) 3541 { 3542 scmi_base_unregister(); 3543 3544 scmi_clock_unregister(); 3545 scmi_perf_unregister(); 3546 scmi_power_unregister(); 3547 scmi_reset_unregister(); 3548 scmi_sensors_unregister(); 3549 scmi_voltage_unregister(); 3550 scmi_system_unregister(); 3551 scmi_powercap_unregister(); 3552 scmi_pinctrl_unregister(); 3553 3554 platform_driver_unregister(&scmi_driver); 3555 3556 debugfs_remove_recursive(scmi_top_dentry); 3557 } 3558 module_exit(scmi_driver_exit); 3559 3560 MODULE_ALIAS("platform:arm-scmi"); 3561 MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>"); 3562 MODULE_DESCRIPTION("ARM SCMI protocol driver"); 3563 MODULE_LICENSE("GPL v2"); 3564