1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ 3 4 #include "mlx5_core.h" 5 #include "mlx5_irq.h" 6 #include "pci_irq.h" 7 8 static void cpu_put(struct mlx5_irq_pool *pool, int cpu) 9 { 10 pool->irqs_per_cpu[cpu]--; 11 } 12 13 static void cpu_get(struct mlx5_irq_pool *pool, int cpu) 14 { 15 pool->irqs_per_cpu[cpu]++; 16 } 17 18 /* Gets the least loaded CPU. e.g.: the CPU with least IRQs bound to it */ 19 static int cpu_get_least_loaded(struct mlx5_irq_pool *pool, 20 const struct cpumask *req_mask) 21 { 22 int best_cpu = -1; 23 int cpu; 24 25 for_each_cpu_and(cpu, req_mask, cpu_online_mask) { 26 /* CPU has zero IRQs on it. No need to search any more CPUs. */ 27 if (!pool->irqs_per_cpu[cpu]) { 28 best_cpu = cpu; 29 break; 30 } 31 if (best_cpu < 0) 32 best_cpu = cpu; 33 if (pool->irqs_per_cpu[cpu] < pool->irqs_per_cpu[best_cpu]) 34 best_cpu = cpu; 35 } 36 if (best_cpu == -1) { 37 /* There isn't online CPUs in req_mask */ 38 mlx5_core_err(pool->dev, "NO online CPUs in req_mask (%*pbl)\n", 39 cpumask_pr_args(req_mask)); 40 best_cpu = cpumask_first(cpu_online_mask); 41 } 42 pool->irqs_per_cpu[best_cpu]++; 43 return best_cpu; 44 } 45 46 /* Creating an IRQ from irq_pool */ 47 static struct mlx5_irq * 48 irq_pool_request_irq(struct mlx5_irq_pool *pool, struct irq_affinity_desc *af_desc) 49 { 50 struct irq_affinity_desc *auto_desc; 51 struct mlx5_irq *irq; 52 u32 irq_index; 53 int err; 54 55 auto_desc = kvzalloc_obj(*auto_desc); 56 if (!auto_desc) 57 return ERR_PTR(-ENOMEM); 58 59 err = xa_alloc(&pool->irqs, &irq_index, NULL, pool->xa_num_irqs, GFP_KERNEL); 60 if (err) { 61 kvfree(auto_desc); 62 return ERR_PTR(err); 63 } 64 65 if (pool->irqs_per_cpu) { 66 if (cpumask_weight(&af_desc->mask) > 1) 67 /* if req_mask contain more then one CPU, set the least loadad CPU 68 * of req_mask 69 */ 70 cpumask_set_cpu(cpu_get_least_loaded(pool, &af_desc->mask), 71 &auto_desc->mask); 72 else 73 cpu_get(pool, cpumask_first(&af_desc->mask)); 74 } 75 76 irq = mlx5_irq_alloc(pool, irq_index, 77 cpumask_empty(&auto_desc->mask) ? af_desc : auto_desc, 78 NULL); 79 if (IS_ERR(irq)) 80 xa_erase(&pool->irqs, irq_index); 81 82 kvfree(auto_desc); 83 84 return irq; 85 } 86 87 /* Looking for the IRQ with the smallest refcount that fits req_mask. 88 * If pool is sf_comp_pool, then we are looking for an IRQ with any of the 89 * requested CPUs in req_mask. 90 * for example: req_mask = 0xf, irq0_mask = 0x10, irq1_mask = 0x1. irq0_mask 91 * isn't subset of req_mask, so we will skip it. irq1_mask is subset of req_mask, 92 * we don't skip it. 93 * If pool is sf_ctrl_pool, then all IRQs have the same mask, so any IRQ will 94 * fit. And since mask is subset of itself, we will pass the first if bellow. 95 */ 96 static struct mlx5_irq * 97 irq_pool_find_least_loaded(struct mlx5_irq_pool *pool, const struct cpumask *req_mask) 98 { 99 int start = pool->xa_num_irqs.min; 100 int end = pool->xa_num_irqs.max; 101 struct mlx5_irq *irq = NULL; 102 struct mlx5_irq *iter; 103 int irq_refcount = 0; 104 unsigned long index; 105 106 lockdep_assert_held(&pool->lock); 107 xa_for_each_range(&pool->irqs, index, iter, start, end) { 108 int iter_refcount = mlx5_irq_read_locked(iter); 109 const struct cpumask *iter_mask; 110 111 iter_mask = irq_get_effective_affinity_mask(mlx5_irq_get_irq(iter)); 112 if (!iter_mask) 113 continue; 114 if (!cpumask_subset(iter_mask, req_mask)) 115 /* skip IRQs with a mask which is not subset of req_mask */ 116 continue; 117 if (iter_refcount < pool->min_threshold) 118 /* If we found an IRQ with less than min_thres, return it */ 119 return iter; 120 if (!irq || iter_refcount < irq_refcount) { 121 /* In case we won't find an IRQ with less than min_thres, 122 * keep a pointer to the least used IRQ 123 */ 124 irq_refcount = iter_refcount; 125 irq = iter; 126 } 127 } 128 return irq; 129 } 130 131 /** 132 * mlx5_irq_affinity_request - request an IRQ according to the given mask. 133 * @dev: mlx5 core device which is requesting the IRQ. 134 * @pool: IRQ pool to request from. 135 * @af_desc: affinity descriptor for this IRQ. 136 * 137 * This function returns a pointer to IRQ, or ERR_PTR in case of error. 138 */ 139 struct mlx5_irq * 140 mlx5_irq_affinity_request(struct mlx5_core_dev *dev, struct mlx5_irq_pool *pool, 141 struct irq_affinity_desc *af_desc) 142 { 143 struct mlx5_irq *least_loaded_irq, *new_irq; 144 int ret; 145 146 mutex_lock(&pool->lock); 147 least_loaded_irq = irq_pool_find_least_loaded(pool, &af_desc->mask); 148 if (least_loaded_irq && 149 mlx5_irq_read_locked(least_loaded_irq) < pool->min_threshold) 150 goto out; 151 /* We didn't find an IRQ with less than min_thres, try to allocate a new IRQ */ 152 new_irq = irq_pool_request_irq(pool, af_desc); 153 if (IS_ERR(new_irq)) { 154 if (!least_loaded_irq) { 155 /* We failed to create an IRQ and we didn't find an IRQ */ 156 mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %pe\n", 157 new_irq); 158 mutex_unlock(&pool->lock); 159 return new_irq; 160 } 161 /* We failed to create a new IRQ for the requested affinity, 162 * sharing existing IRQ. 163 */ 164 goto out; 165 } 166 least_loaded_irq = new_irq; 167 goto unlock; 168 out: 169 mlx5_irq_get_locked(least_loaded_irq); 170 if (mlx5_irq_read_locked(least_loaded_irq) > pool->max_threshold) 171 mlx5_core_dbg(pool->dev, "IRQ %u overloaded, pool_name: %s, %u EQs on this irq\n", 172 pci_irq_vector(pool->dev->pdev, 173 mlx5_irq_get_index(least_loaded_irq)), pool->name, 174 mlx5_irq_read_locked(least_loaded_irq) / MLX5_EQ_REFS_PER_IRQ); 175 unlock: 176 mutex_unlock(&pool->lock); 177 if (mlx5_irq_pool_is_sf_pool(pool)) { 178 ret = auxiliary_device_sysfs_irq_add(mlx5_sf_coredev_to_adev(dev), 179 mlx5_irq_get_irq(least_loaded_irq)); 180 if (ret) { 181 mlx5_core_err(dev, "Failed to create sysfs entry for irq %d, ret = %d\n", 182 mlx5_irq_get_irq(least_loaded_irq), ret); 183 mlx5_irq_put(least_loaded_irq); 184 least_loaded_irq = ERR_PTR(ret); 185 } 186 } 187 return least_loaded_irq; 188 } 189 190 void mlx5_irq_affinity_irq_release(struct mlx5_core_dev *dev, struct mlx5_irq *irq) 191 { 192 struct mlx5_irq_pool *pool = mlx5_irq_get_pool(irq); 193 int cpu; 194 195 cpu = cpumask_first(mlx5_irq_get_affinity_mask(irq)); 196 synchronize_irq(pci_irq_vector(pool->dev->pdev, 197 mlx5_irq_get_index(irq))); 198 if (mlx5_irq_pool_is_sf_pool(pool)) 199 auxiliary_device_sysfs_irq_remove(mlx5_sf_coredev_to_adev(dev), 200 mlx5_irq_get_irq(irq)); 201 if (mlx5_irq_put(irq)) 202 if (pool->irqs_per_cpu) 203 cpu_put(pool, cpu); 204 } 205