1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2018 Mellanox Technologies */ 3 4 #include <linux/mlx5/vport.h> 5 #include <linux/list.h> 6 #include <linux/lockdep.h> 7 #include "lib/devcom.h" 8 #include "lib/mlx5.h" 9 #include "mlx5_core.h" 10 11 static LIST_HEAD(devcom_dev_list); 12 static LIST_HEAD(devcom_comp_list); 13 /* protect device list */ 14 static DEFINE_MUTEX(dev_list_lock); 15 /* protect component list */ 16 static DEFINE_MUTEX(comp_list_lock); 17 18 #define devcom_for_each_component(iter) \ 19 list_for_each_entry(iter, &devcom_comp_list, comp_list) 20 21 struct mlx5_devcom_dev { 22 struct list_head list; 23 struct mlx5_core_dev *dev; 24 struct kref ref; 25 }; 26 27 struct mlx5_devcom_key { 28 u32 flags; 29 union mlx5_devcom_match_key key; 30 possible_net_t net; 31 }; 32 33 struct mlx5_devcom_comp { 34 struct list_head comp_list; 35 enum mlx5_devcom_component id; 36 struct list_head comp_dev_list_head; 37 struct mlx5_devcom_key key; 38 mlx5_devcom_event_handler_t handler; 39 struct kref ref; 40 bool ready; 41 struct rw_semaphore sem; 42 struct lock_class_key lock_key; 43 }; 44 45 struct mlx5_devcom_comp_dev { 46 struct list_head list; 47 struct mlx5_devcom_comp *comp; 48 struct mlx5_devcom_dev *devc; 49 void __rcu *data; 50 }; 51 52 static bool devcom_dev_exists(struct mlx5_core_dev *dev) 53 { 54 struct mlx5_devcom_dev *iter; 55 56 list_for_each_entry(iter, &devcom_dev_list, list) 57 if (iter->dev == dev) 58 return true; 59 60 return false; 61 } 62 63 static struct mlx5_devcom_dev * 64 mlx5_devcom_dev_alloc(struct mlx5_core_dev *dev) 65 { 66 struct mlx5_devcom_dev *devc; 67 68 devc = kzalloc_obj(*devc); 69 if (!devc) 70 return NULL; 71 72 devc->dev = dev; 73 kref_init(&devc->ref); 74 return devc; 75 } 76 77 struct mlx5_devcom_dev * 78 mlx5_devcom_register_device(struct mlx5_core_dev *dev) 79 { 80 struct mlx5_devcom_dev *devc = NULL; 81 82 mutex_lock(&dev_list_lock); 83 84 if (devcom_dev_exists(dev)) { 85 mlx5_core_err(dev, "devcom device already exists"); 86 goto out; 87 } 88 89 devc = mlx5_devcom_dev_alloc(dev); 90 if (!devc) 91 goto out; 92 93 list_add_tail(&devc->list, &devcom_dev_list); 94 out: 95 mutex_unlock(&dev_list_lock); 96 return devc; 97 } 98 99 static void 100 mlx5_devcom_dev_release(struct kref *ref) 101 { 102 struct mlx5_devcom_dev *devc = container_of(ref, struct mlx5_devcom_dev, ref); 103 104 mutex_lock(&dev_list_lock); 105 list_del(&devc->list); 106 mutex_unlock(&dev_list_lock); 107 kfree(devc); 108 } 109 110 void mlx5_devcom_unregister_device(struct mlx5_devcom_dev *devc) 111 { 112 if (!devc) 113 return; 114 115 kref_put(&devc->ref, mlx5_devcom_dev_release); 116 } 117 118 static struct mlx5_devcom_comp * 119 mlx5_devcom_comp_alloc(u64 id, const struct mlx5_devcom_match_attr *attr, 120 mlx5_devcom_event_handler_t handler) 121 { 122 struct mlx5_devcom_comp *comp; 123 124 comp = kzalloc_obj(*comp); 125 if (!comp) 126 return NULL; 127 128 comp->id = id; 129 comp->key.key = attr->key; 130 comp->key.flags = attr->flags; 131 if (attr->flags & MLX5_DEVCOM_MATCH_FLAGS_NS) 132 write_pnet(&comp->key.net, attr->net); 133 comp->handler = handler; 134 init_rwsem(&comp->sem); 135 lockdep_register_key(&comp->lock_key); 136 lockdep_set_class(&comp->sem, &comp->lock_key); 137 kref_init(&comp->ref); 138 INIT_LIST_HEAD(&comp->comp_dev_list_head); 139 140 return comp; 141 } 142 143 static void 144 mlx5_devcom_comp_release(struct kref *ref) 145 { 146 struct mlx5_devcom_comp *comp = container_of(ref, struct mlx5_devcom_comp, ref); 147 148 mutex_lock(&comp_list_lock); 149 list_del(&comp->comp_list); 150 mutex_unlock(&comp_list_lock); 151 lockdep_unregister_key(&comp->lock_key); 152 kfree(comp); 153 } 154 155 static struct mlx5_devcom_comp_dev * 156 devcom_alloc_comp_dev(struct mlx5_devcom_dev *devc, 157 struct mlx5_devcom_comp *comp, 158 void *data) 159 { 160 struct mlx5_devcom_comp_dev *devcom; 161 162 devcom = kzalloc_obj(*devcom); 163 if (!devcom) 164 return NULL; 165 166 kref_get(&devc->ref); 167 devcom->devc = devc; 168 devcom->comp = comp; 169 rcu_assign_pointer(devcom->data, data); 170 171 down_write(&comp->sem); 172 list_add_tail(&devcom->list, &comp->comp_dev_list_head); 173 up_write(&comp->sem); 174 175 return devcom; 176 } 177 178 static void 179 devcom_free_comp_dev(struct mlx5_devcom_comp_dev *devcom) 180 { 181 struct mlx5_devcom_comp *comp = devcom->comp; 182 183 down_write(&comp->sem); 184 list_del(&devcom->list); 185 up_write(&comp->sem); 186 187 kref_put(&devcom->devc->ref, mlx5_devcom_dev_release); 188 kfree(devcom); 189 kref_put(&comp->ref, mlx5_devcom_comp_release); 190 } 191 192 static bool 193 devcom_component_equal(struct mlx5_devcom_comp *devcom, 194 enum mlx5_devcom_component id, 195 const struct mlx5_devcom_match_attr *attr) 196 { 197 if (devcom->id != id) 198 return false; 199 200 if (devcom->key.flags != attr->flags) 201 return false; 202 203 if (memcmp(&devcom->key.key, &attr->key, sizeof(devcom->key.key))) 204 return false; 205 206 if (devcom->key.flags & MLX5_DEVCOM_MATCH_FLAGS_NS && 207 !net_eq(read_pnet(&devcom->key.net), attr->net)) 208 return false; 209 210 return true; 211 } 212 213 static struct mlx5_devcom_comp * 214 devcom_component_get(struct mlx5_devcom_dev *devc, 215 enum mlx5_devcom_component id, 216 const struct mlx5_devcom_match_attr *attr, 217 mlx5_devcom_event_handler_t handler) 218 { 219 struct mlx5_devcom_comp *comp; 220 221 devcom_for_each_component(comp) { 222 if (devcom_component_equal(comp, id, attr)) { 223 if (handler == comp->handler) { 224 kref_get(&comp->ref); 225 return comp; 226 } 227 228 mlx5_core_err(devc->dev, 229 "Cannot register existing devcom component with different handler\n"); 230 return ERR_PTR(-EINVAL); 231 } 232 } 233 234 return NULL; 235 } 236 237 struct mlx5_devcom_comp_dev * 238 mlx5_devcom_register_component(struct mlx5_devcom_dev *devc, 239 enum mlx5_devcom_component id, 240 const struct mlx5_devcom_match_attr *attr, 241 mlx5_devcom_event_handler_t handler, 242 void *data) 243 { 244 struct mlx5_devcom_comp_dev *devcom = NULL; 245 struct mlx5_devcom_comp *comp; 246 247 if (!devc) 248 return NULL; 249 250 mutex_lock(&comp_list_lock); 251 comp = devcom_component_get(devc, id, attr, handler); 252 if (IS_ERR(comp)) 253 goto out_unlock; 254 255 if (!comp) { 256 comp = mlx5_devcom_comp_alloc(id, attr, handler); 257 if (!comp) 258 goto out_unlock; 259 260 list_add_tail(&comp->comp_list, &devcom_comp_list); 261 } 262 mutex_unlock(&comp_list_lock); 263 264 devcom = devcom_alloc_comp_dev(devc, comp, data); 265 if (!devcom) 266 kref_put(&comp->ref, mlx5_devcom_comp_release); 267 268 return devcom; 269 270 out_unlock: 271 mutex_unlock(&comp_list_lock); 272 return devcom; 273 } 274 275 void mlx5_devcom_unregister_component(struct mlx5_devcom_comp_dev *devcom) 276 { 277 if (!devcom) 278 return; 279 280 devcom_free_comp_dev(devcom); 281 } 282 283 int mlx5_devcom_comp_get_size(struct mlx5_devcom_comp_dev *devcom) 284 { 285 struct mlx5_devcom_comp *comp = devcom->comp; 286 287 return kref_read(&comp->ref); 288 } 289 290 int mlx5_devcom_locked_send_event(struct mlx5_devcom_comp_dev *devcom, 291 int event, int rollback_event, 292 void *event_data) 293 { 294 struct mlx5_devcom_comp_dev *pos; 295 struct mlx5_devcom_comp *comp; 296 int err = 0; 297 void *data; 298 299 if (!devcom) 300 return -ENODEV; 301 302 lockdep_assert_held_write(&devcom->comp->sem); 303 comp = devcom->comp; 304 list_for_each_entry(pos, &comp->comp_dev_list_head, list) { 305 data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem)); 306 307 if (pos != devcom && data) { 308 err = comp->handler(event, data, event_data); 309 if (err && rollback_event != DEVCOM_CANT_FAIL) { 310 goto rollback; 311 } else if (err && rollback_event == DEVCOM_CANT_FAIL) { 312 WARN_ONCE(1, "devcom component %d event %d failed: %d\n", 313 comp->id, event, err); 314 return err; 315 } 316 } 317 } 318 319 return 0; 320 321 rollback: 322 if (list_entry_is_head(pos, &comp->comp_dev_list_head, list)) 323 return err; 324 pos = list_prev_entry(pos, list); 325 list_for_each_entry_from_reverse(pos, &comp->comp_dev_list_head, list) { 326 data = rcu_dereference_protected(pos->data, lockdep_is_held(&comp->sem)); 327 328 if (pos != devcom && data) 329 comp->handler(rollback_event, data, event_data); 330 } 331 return err; 332 } 333 334 int mlx5_devcom_send_event(struct mlx5_devcom_comp_dev *devcom, 335 int event, int rollback_event, 336 void *event_data) 337 { 338 struct mlx5_devcom_comp *comp; 339 int err; 340 341 if (!devcom) 342 return -ENODEV; 343 344 comp = devcom->comp; 345 down_write(&comp->sem); 346 err = mlx5_devcom_locked_send_event(devcom, event, rollback_event, 347 event_data); 348 up_write(&comp->sem); 349 return err; 350 } 351 352 void mlx5_devcom_comp_set_ready(struct mlx5_devcom_comp_dev *devcom, bool ready) 353 { 354 WARN_ON(!rwsem_is_locked(&devcom->comp->sem)); 355 356 WRITE_ONCE(devcom->comp->ready, ready); 357 } 358 359 bool mlx5_devcom_comp_is_ready(struct mlx5_devcom_comp_dev *devcom) 360 { 361 if (!devcom) 362 return false; 363 364 return READ_ONCE(devcom->comp->ready); 365 } 366 367 bool mlx5_devcom_for_each_peer_begin(struct mlx5_devcom_comp_dev *devcom) 368 { 369 struct mlx5_devcom_comp *comp; 370 371 if (!devcom) 372 return false; 373 374 comp = devcom->comp; 375 down_read(&comp->sem); 376 if (!READ_ONCE(comp->ready)) { 377 up_read(&comp->sem); 378 return false; 379 } 380 381 return true; 382 } 383 384 void mlx5_devcom_for_each_peer_end(struct mlx5_devcom_comp_dev *devcom) 385 { 386 up_read(&devcom->comp->sem); 387 } 388 389 void *mlx5_devcom_get_next_peer_data(struct mlx5_devcom_comp_dev *devcom, 390 struct mlx5_devcom_comp_dev **pos) 391 { 392 struct mlx5_devcom_comp *comp = devcom->comp; 393 struct mlx5_devcom_comp_dev *tmp; 394 void *data; 395 396 tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list); 397 398 list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) { 399 if (tmp != devcom) { 400 data = rcu_dereference_protected(tmp->data, lockdep_is_held(&comp->sem)); 401 if (data) 402 break; 403 } 404 } 405 406 if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list)) 407 return NULL; 408 409 *pos = tmp; 410 return data; 411 } 412 413 void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom, 414 struct mlx5_devcom_comp_dev **pos) 415 { 416 struct mlx5_devcom_comp *comp = devcom->comp; 417 struct mlx5_devcom_comp_dev *tmp; 418 void *data; 419 420 tmp = list_prepare_entry(*pos, &comp->comp_dev_list_head, list); 421 422 list_for_each_entry_continue(tmp, &comp->comp_dev_list_head, list) { 423 if (tmp != devcom) { 424 /* This can change concurrently, however 'data' pointer will remain 425 * valid for the duration of RCU read section. 426 */ 427 if (!READ_ONCE(comp->ready)) 428 return NULL; 429 data = rcu_dereference(tmp->data); 430 if (data) 431 break; 432 } 433 } 434 435 if (list_entry_is_head(tmp, &comp->comp_dev_list_head, list)) 436 return NULL; 437 438 *pos = tmp; 439 return data; 440 } 441 442 void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom) 443 { 444 if (!devcom) 445 return; 446 down_write(&devcom->comp->sem); 447 } 448 449 void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom) 450 { 451 if (!devcom) 452 return; 453 up_write(&devcom->comp->sem); 454 } 455 456 int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom) 457 { 458 if (!devcom) 459 return 0; 460 return down_write_trylock(&devcom->comp->sem); 461 } 462 463 void mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev *devcom) 464 { 465 if (!devcom) 466 return; 467 lockdep_assert_held_write(&devcom->comp->sem); 468 } 469