1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
4  * Copyright (c) 2023-2024, Qualcomm Innovation Center, Inc. All rights reserved.
31 #include <soc/qcom/cmd-db.h>
32 #include <soc/qcom/tcs.h>
33 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
35 #include "rpmh-internal.h"
38 #include "trace-rpmh.h"
71 /* DRV TCS Configuration Information Register */
77 /* Offsets for CONTROL TCS Registers */
88 /* TCS CMD register bit mask */
99  * - The main rpmh-rsc address is the base of a register space that can
101  *   (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
102  *   space are all the TCS blocks. The offset of the TCS blocks is
103  *   specified in the device tree by "qcom,tcs-offset" and used to
105  * - TCS blocks come one after another. Type, count, and order are
106  *   specified by the device tree as "qcom,tcs-config".
107  * - Each TCS block has some registers, then space for up to 16 commands.
109  *   might be present. See ncpt (num cmds per TCS).
113  *  +---------------------------------------------------+
118  *  | +-----------------------------------------------+ |
120  *  | | ctrl/config                                   | |
124  *  | | +------------------------------------------+  | |
128  *  | | +------------------------------------------+  | |
129  *  | | +------------------------------------------+  | |
133  *  | | +------------------------------------------+  | |
134  *  | | +------------------------------------------+  | |
138  *  | | +------------------------------------------+  | |
140  *  | +-----------------------------------------------+ |
141  *  | +-----------------------------------------------+ |
144  *  | +-----------------------------------------------+ |
146  *  +---------------------------------------------------+
198 	return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg;  in tcs_reg_addr()
204 	return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id;  in tcs_cmd_addr()
246 	pr_err("%s: error writing %#x to %d:%#x\n", drv->name,  in write_tcs_reg_sync()
251  * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
265 	struct tcs_group *tcs = &drv->tcs[type];  in tcs_invalidate()  local
268 	if (bitmap_empty(tcs->slots, MAX_TCS_SLOTS))  in tcs_invalidate()
271 	for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++)  in tcs_invalidate()
272 		write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0);  in tcs_invalidate()
274 	bitmap_zero(tcs->slots, MAX_TCS_SLOTS);  in tcs_invalidate()
278  * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
292  * get_tcs_for_msg() - Get the tcs_group used to send the given message.
305 	struct tcs_group *tcs;  in get_tcs_for_msg()  local
307 	switch (msg->state) {  in get_tcs_for_msg()
318 		return ERR_PTR(-EINVAL);  in get_tcs_for_msg()
323 	 * dedicated TCS for active state use, then re-purpose a wake TCS to  in get_tcs_for_msg()
324 	 * send active votes. This is safe because we ensure any active-only  in get_tcs_for_msg()
328 	tcs = &drv->tcs[type];  in get_tcs_for_msg()
329 	if (msg->state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)  in get_tcs_for_msg()
330 		tcs = &drv->tcs[WAKE_TCS];  in get_tcs_for_msg()
332 	return tcs;  in get_tcs_for_msg()
336  * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
338  * @tcs_id: The global ID of this TCS.
354 	struct tcs_group *tcs;  in get_req_from_tcs()  local
358 		tcs = &drv->tcs[i];  in get_req_from_tcs()
359 		if (tcs->mask & BIT(tcs_id))  in get_req_from_tcs()
360 			return tcs->req[tcs_id - tcs->offset];  in get_req_from_tcs()
367  * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
369  * @tcs_id:  The global ID of this TCS.
373  * transfer. That will un-trigger/disable the TCS from the last transfer
376  * If we borrowed a wake TCS for an active-only transfer we'll also call
377  * this function with "trigger=false" to just do the un-trigger/disable
378  * before using the TCS for wake purposes again.
380  * Note that the AP is only in charge of triggering active-only transfers.
386 	u32 reg = drv->regs[RSC_DRV_CONTROL];  in __tcs_set_trigger()
389 	 * HW req: Clear the DRV_CONTROL and enable TCS again  in __tcs_set_trigger()
400 		/* Enable the AMC mode on the TCS and then trigger the TCS */  in __tcs_set_trigger()
409  * enable_tcs_irq() - Enable or disable interrupts on the given TCS.
411  * @tcs_id:  The global ID of this TCS.
414  * We only ever call this when we borrow a wake TCS for an active-only
415  * transfer. For active-only TCSes interrupts are always left enabled.
420 	u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE];  in enable_tcs_irq()
422 	data = readl_relaxed(drv->tcs_base + reg);  in enable_tcs_irq()
427 	writel_relaxed(data, drv->tcs_base + reg);  in enable_tcs_irq()
431  * tcs_tx_done() - TX Done interrupt handler.
447 	irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]);  in tcs_tx_done()
457 		 * If wake tcs was re-purposed for sending active  in tcs_tx_done()
459 		 * disable interrupt for this TCS  in tcs_tx_done()
461 		if (!drv->tcs[ACTIVE_TCS].num_tcs)  in tcs_tx_done()
464 		/* Reclaim the TCS */  in tcs_tx_done()
465 		write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0);  in tcs_tx_done()
466 		writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]);  in tcs_tx_done()
467 		spin_lock(&drv->lock);  in tcs_tx_done()
468 		clear_bit(i, drv->tcs_in_use);  in tcs_tx_done()
470 		 * Disable interrupt for WAKE TCS to avoid being  in tcs_tx_done()
474 		if (!drv->tcs[ACTIVE_TCS].num_tcs)  in tcs_tx_done()
476 		spin_unlock(&drv->lock);  in tcs_tx_done()
477 		wake_up(&drv->tcs_wait);  in tcs_tx_done()
486  * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
488  * @tcs_id: The global ID of this TCS.
489  * @cmd_id: The index within the TCS to start writing.
491  *          pairs to program (but few enough that they all fit in one TCS).
505 	cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;  in __tcs_buffer_write()
507 	for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {  in __tcs_buffer_write()
508 		cmd = &msg->cmds[i];  in __tcs_buffer_write()
512 		 * Additionally, if the cmd->wait is set, make the command  in __tcs_buffer_write()
513 		 * response reqd even if the overall request was fire-n-forget.  in __tcs_buffer_write()
515 		msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0;  in __tcs_buffer_write()
517 		write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid);  in __tcs_buffer_write()
518 		write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr);  in __tcs_buffer_write()
519 		write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data);  in __tcs_buffer_write()
520 		trace_rpmh_send_msg(drv, tcs_id, msg->state, j, msgid, cmd);  in __tcs_buffer_write()
523 	cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id);  in __tcs_buffer_write()
524 	write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable);  in __tcs_buffer_write()
528  * check_for_req_inflight() - Look to see if conflicting cmds are in flight.
530  * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers.
532  *       pairs to program (but few enough that they all fit in one TCS).
536  * one it'll return -EBUSY.
538  * Only for use for active-only transfers.
540  * Must be called with the drv->lock held since that protects tcs_in_use.
542  * Return: 0 if nothing in flight or -EBUSY if we should try again later.
543  *         The caller must re-enable interrupts between tries since that's
547 static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,  in check_for_req_inflight()  argument
553 	int i = tcs->offset;  in check_for_req_inflight()
555 	for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) {  in check_for_req_inflight()
556 		curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i);  in check_for_req_inflight()
559 			addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j);  in check_for_req_inflight()
560 			for (k = 0; k < msg->num_cmds; k++) {  in check_for_req_inflight()
561 				if (cmd_db_match_resource_addr(msg->cmds[k].addr, addr))  in check_for_req_inflight()
562 					return -EBUSY;  in check_for_req_inflight()
571  * find_free_tcs() - Find free tcs in the given tcs_group; only for active.
572  * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
573  *       we borrowed it because there are zero active-only ones).
575  * Must be called with the drv->lock held since that protects tcs_in_use.
577  * Return: The first tcs that's free or -EBUSY if all in use.
579 static int find_free_tcs(struct tcs_group *tcs)  in find_free_tcs()  argument
581 	const struct rsc_drv *drv = tcs->drv;  in find_free_tcs()
583 	unsigned long max = tcs->offset + tcs->num_tcs;  in find_free_tcs()
585 	i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset);  in find_free_tcs()
587 		return -EBUSY;  in find_free_tcs()
593  * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
595  * @tcs: The tcs_group used for ACTIVE_ONLY transfers.
598  * Claims a tcs in the given tcs_group while making sure that no existing cmd
601  * Context: Must be called with the drv->lock held since that protects
604  * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
607 static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,  in claim_tcs_for_req()  argument
614 	 * when one is already in-flight or being processed.  in claim_tcs_for_req()
616 	ret = check_for_req_inflight(drv, tcs, msg);  in claim_tcs_for_req()
620 	return find_free_tcs(tcs);  in claim_tcs_for_req()
624  * rpmh_rsc_send_data() - Write / trigger active-only message.
629  * - This is only used for "ACTIVE_ONLY" since the limitations of this
631  * - To do the transfer, we will grab a whole TCS for ourselves--we don't
634  * - This function will not wait for the commands to be finished, only for
637  * - This function must be called with interrupts enabled. If the hardware
643  * Return: 0 on success, -EINVAL on error.
647 	struct tcs_group *tcs;  in rpmh_rsc_send_data()  local
652 	tcs = get_tcs_for_msg(drv, msg);  in rpmh_rsc_send_data()
653 	if (IS_ERR(tcs))  in rpmh_rsc_send_data()
654 		return PTR_ERR(tcs);  in rpmh_rsc_send_data()
656 	spin_lock_irq(&drv->lock);  in rpmh_rsc_send_data()
658 	/* Wait forever for a free tcs. It better be there eventually! */  in rpmh_rsc_send_data()
659 	wait_event_lock_irq(drv->tcs_wait,  in rpmh_rsc_send_data()
660 			    (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,  in rpmh_rsc_send_data()
661 			    drv->lock);  in rpmh_rsc_send_data()
663 	tcs->req[tcs_id - tcs->offset] = msg;  in rpmh_rsc_send_data()
664 	set_bit(tcs_id, drv->tcs_in_use);  in rpmh_rsc_send_data()
665 	if (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS) {  in rpmh_rsc_send_data()
668 		 * repurposed TCS to avoid triggering them. tcs->slots will be  in rpmh_rsc_send_data()
671 		write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0);  in rpmh_rsc_send_data()
674 	spin_unlock_irq(&drv->lock);  in rpmh_rsc_send_data()
678 	 * - We marked "tcs_in_use" under lock.  in rpmh_rsc_send_data()
679 	 * - Once "tcs_in_use" has been marked nobody else could be writing  in rpmh_rsc_send_data()
681 	 * - The interrupt can't go off until we trigger w/ the last line  in rpmh_rsc_send_data()
691  * find_slots() - Find a place to write the given message.
692  * @tcs:    The tcs group to search.
695  *          TCS to write to here.
697  *          the command array of the returned TCS where the client should
701  * tcs->slots for.
703  * Return: -ENOMEM if there was no room, else 0.
705 static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,  in find_slots()  argument
711 	/* Do over, until we can fit the full payload in a single TCS */  in find_slots()
713 		slot = bitmap_find_next_zero_area(tcs->slots, MAX_TCS_SLOTS,  in find_slots()
714 						  i, msg->num_cmds, 0);  in find_slots()
715 		if (slot >= tcs->num_tcs * tcs->ncpt)  in find_slots()
716 			return -ENOMEM;  in find_slots()
717 		i += tcs->ncpt;  in find_slots()
718 	} while (slot + msg->num_cmds - 1 >= i);  in find_slots()
720 	bitmap_set(tcs->slots, slot, msg->num_cmds);  in find_slots()
722 	offset = slot / tcs->ncpt;  in find_slots()
723 	*tcs_id = offset + tcs->offset;  in find_slots()
724 	*cmd_id = slot % tcs->ncpt;  in find_slots()
730  * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
734  * This should only be called for sleep/wake state, never active-only
740  * Return: 0 if no error; else -error.
744 	struct tcs_group *tcs;  in rpmh_rsc_write_ctrl_data()  local
748 	tcs = get_tcs_for_msg(drv, msg);  in rpmh_rsc_write_ctrl_data()
749 	if (IS_ERR(tcs))  in rpmh_rsc_write_ctrl_data()
750 		return PTR_ERR(tcs);  in rpmh_rsc_write_ctrl_data()
752 	/* find the TCS id and the command in the TCS to write to */  in rpmh_rsc_write_ctrl_data()
753 	ret = find_slots(tcs, msg, &tcs_id, &cmd_id);  in rpmh_rsc_write_ctrl_data()
761  * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
769  * Context: Must be called with the drv->lock held.
772  * * False		- AMCs are idle
773  * * True		- AMCs are busy
778 	const struct tcs_group *tcs = &drv->tcs[ACTIVE_TCS];  in rpmh_rsc_ctrlr_is_busy()  local
783 	 * dedicated TCS for active state use, then re-purposed wake TCSes  in rpmh_rsc_ctrlr_is_busy()
787 	if (!tcs->num_tcs)  in rpmh_rsc_ctrlr_is_busy()
788 		tcs = &drv->tcs[WAKE_TCS];  in rpmh_rsc_ctrlr_is_busy()
790 	max = tcs->offset + tcs->num_tcs;  in rpmh_rsc_ctrlr_is_busy()
791 	set = find_next_bit(drv->tcs_in_use, max, tcs->offset);  in rpmh_rsc_ctrlr_is_busy()
797  * rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS.
809 	if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call)  in rpmh_rsc_write_next_wakeup()
817 	wakeup = dev_pm_genpd_get_next_hrtimer(drv->dev);  in rpmh_rsc_write_next_wakeup()
834 	writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO);  in rpmh_rsc_write_next_wakeup()
835 	writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI);  in rpmh_rsc_write_next_wakeup()
839  * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
864 		cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
871 		 * AND that CPU was not idle AND that CPU was the last non-idle  in rpmh_rsc_cpu_pm_callback()
880 		atomic_dec(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
887 	 * It's likely we're on the last CPU. Grab the drv->lock and write  in rpmh_rsc_cpu_pm_callback()
898 	if (spin_trylock(&drv->lock)) {  in rpmh_rsc_cpu_pm_callback()
899 		if (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client))  in rpmh_rsc_cpu_pm_callback()
901 		spin_unlock(&drv->lock);  in rpmh_rsc_cpu_pm_callback()
908 		/* Double-check if we're here because someone else is up */  in rpmh_rsc_cpu_pm_callback()
913 			atomic_dec(&drv->cpus_in_pm);  in rpmh_rsc_cpu_pm_callback()
920  * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy.
926  * about when cluster-pd is going down. When cluster go down we know no more active
934  * * NOTIFY_OK          - success
935  * * NOTIFY_BAD         - failure
944 	    (rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client)))  in rpmh_rsc_pd_callback()
955 	drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback;  in rpmh_rsc_pd_attach()
956 	ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb);  in rpmh_rsc_pd_attach()
969 	struct device_node *dn = pdev->dev.of_node;  in rpmh_probe_tcs_config()
970 	u32 config, max_tcs, ncpt, offset;  in rpmh_probe_tcs_config()  local
972 	struct tcs_group *tcs;  in rpmh_probe_tcs_config()  local
974 	ret = of_property_read_u32(dn, "qcom,tcs-offset", &offset);  in rpmh_probe_tcs_config()
977 	drv->tcs_base = drv->base + offset;  in rpmh_probe_tcs_config()
979 	config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]);  in rpmh_probe_tcs_config()
981 	max_tcs = config;  in rpmh_probe_tcs_config()
982 	max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);  in rpmh_probe_tcs_config()
983 	max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);  in rpmh_probe_tcs_config()
985 	ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT);  in rpmh_probe_tcs_config()
988 	n = of_property_count_u32_elems(dn, "qcom,tcs-config");  in rpmh_probe_tcs_config()
990 		return -EINVAL;  in rpmh_probe_tcs_config()
993 		ret = of_property_read_u32_index(dn, "qcom,tcs-config",  in rpmh_probe_tcs_config()
998 			return -EINVAL;  in rpmh_probe_tcs_config()
1000 		ret = of_property_read_u32_index(dn, "qcom,tcs-config",  in rpmh_probe_tcs_config()
1005 			return -EINVAL;  in rpmh_probe_tcs_config()
1009 		tcs = &drv->tcs[tcs_cfg[i].type];  in rpmh_probe_tcs_config()
1010 		if (tcs->drv)  in rpmh_probe_tcs_config()
1011 			return -EINVAL;  in rpmh_probe_tcs_config()
1012 		tcs->drv = drv;  in rpmh_probe_tcs_config()
1013 		tcs->type = tcs_cfg[i].type;  in rpmh_probe_tcs_config()
1014 		tcs->num_tcs = tcs_cfg[i].n;  in rpmh_probe_tcs_config()
1015 		tcs->ncpt = ncpt;  in rpmh_probe_tcs_config()
1017 		if (!tcs->num_tcs || tcs->type == CONTROL_TCS)  in rpmh_probe_tcs_config()
1020 		if (st + tcs->num_tcs > max_tcs ||  in rpmh_probe_tcs_config()
1021 		    st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask))  in rpmh_probe_tcs_config()
1022 			return -EINVAL;  in rpmh_probe_tcs_config()
1024 		tcs->mask = ((1 << tcs->num_tcs) - 1) << st;  in rpmh_probe_tcs_config()
1025 		tcs->offset = st;  in rpmh_probe_tcs_config()
1026 		st += tcs->num_tcs;  in rpmh_probe_tcs_config()
1029 	drv->num_tcs = st;  in rpmh_probe_tcs_config()
1036 	struct device_node *dn = pdev->dev.of_node;  in rpmh_rsc_probe()
1044 	 * Even though RPMh doesn't directly use cmd-db, all of its children  in rpmh_rsc_probe()
1049 		if (ret != -EPROBE_DEFER)  in rpmh_rsc_probe()
1050 			dev_err(&pdev->dev, "Command DB not available (%d)\n",  in rpmh_rsc_probe()
1055 	drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);  in rpmh_rsc_probe()
1057 		return -ENOMEM;  in rpmh_rsc_probe()
1059 	ret = of_property_read_u32(dn, "qcom,drv-id", &drv->id);  in rpmh_rsc_probe()
1063 	drv->name = of_get_property(dn, "label", NULL);  in rpmh_rsc_probe()
1064 	if (!drv->name)  in rpmh_rsc_probe()
1065 		drv->name = dev_name(&pdev->dev);  in rpmh_rsc_probe()
1067 	snprintf(drv_id, ARRAY_SIZE(drv_id), "drv-%d", drv->id);  in rpmh_rsc_probe()
1068 	drv->base = devm_platform_ioremap_resource_byname(pdev, drv_id);  in rpmh_rsc_probe()
1069 	if (IS_ERR(drv->base))  in rpmh_rsc_probe()
1070 		return PTR_ERR(drv->base);  in rpmh_rsc_probe()
1072 	rsc_id = readl_relaxed(drv->base + RSC_DRV_ID);  in rpmh_rsc_probe()
1073 	drv->ver.major = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT);  in rpmh_rsc_probe()
1074 	drv->ver.major >>= MAJOR_VER_SHIFT;  in rpmh_rsc_probe()
1075 	drv->ver.minor = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT);  in rpmh_rsc_probe()
1076 	drv->ver.minor >>= MINOR_VER_SHIFT;  in rpmh_rsc_probe()
1078 	if (drv->ver.major == 3)  in rpmh_rsc_probe()
1079 		drv->regs = rpmh_rsc_reg_offset_ver_3_0;  in rpmh_rsc_probe()
1081 		drv->regs = rpmh_rsc_reg_offset_ver_2_7;  in rpmh_rsc_probe()
1087 	spin_lock_init(&drv->lock);  in rpmh_rsc_probe()
1088 	init_waitqueue_head(&drv->tcs_wait);  in rpmh_rsc_probe()
1089 	bitmap_zero(drv->tcs_in_use, MAX_TCS_NR);  in rpmh_rsc_probe()
1091 	irq = platform_get_irq(pdev, drv->id);  in rpmh_rsc_probe()
1095 	ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done,  in rpmh_rsc_probe()
1097 			       drv->name, drv);  in rpmh_rsc_probe()
1106 	solver_config = readl_relaxed(drv->base + drv->regs[DRV_SOLVER_CONFIG]);  in rpmh_rsc_probe()
1110 		if (pdev->dev.pm_domain) {  in rpmh_rsc_probe()
1111 			ret = rpmh_rsc_pd_attach(drv, &pdev->dev);  in rpmh_rsc_probe()
1115 			drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;  in rpmh_rsc_probe()
1116 			cpu_pm_register_notifier(&drv->rsc_pm);  in rpmh_rsc_probe()
1120 	/* Enable the active TCS to send requests immediately */  in rpmh_rsc_probe()
1121 	writel_relaxed(drv->tcs[ACTIVE_TCS].mask,  in rpmh_rsc_probe()
1122 		       drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]);  in rpmh_rsc_probe()
1124 	spin_lock_init(&drv->client.cache_lock);  in rpmh_rsc_probe()
1125 	INIT_LIST_HEAD(&drv->client.cache);  in rpmh_rsc_probe()
1126 	INIT_LIST_HEAD(&drv->client.batch_cache);  in rpmh_rsc_probe()
1128 	dev_set_drvdata(&pdev->dev, drv);  in rpmh_rsc_probe()
1129 	drv->dev = &pdev->dev;  in rpmh_rsc_probe()
1131 	ret = devm_of_platform_populate(&pdev->dev);  in rpmh_rsc_probe()
1132 	if (ret && pdev->dev.pm_domain) {  in rpmh_rsc_probe()
1133 		dev_pm_genpd_remove_notifier(&pdev->dev);  in rpmh_rsc_probe()
1134 		pm_runtime_disable(&pdev->dev);  in rpmh_rsc_probe()
1141 	{ .compatible = "qcom,rpmh-rsc", },