7 #include "dm.h"
12 static int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr,  in mlx5_cmd_alloc_memic()  argument
15 	struct mlx5_core_dev *dev = dm->dev;  in mlx5_cmd_alloc_memic()
45 		spin_lock(&dm->lock);  in mlx5_cmd_alloc_memic()
46 		page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages,  in mlx5_cmd_alloc_memic()
52 			bitmap_set(dm->memic_alloc_pages,  in mlx5_cmd_alloc_memic()
55 		spin_unlock(&dm->lock);  in mlx5_cmd_alloc_memic()
65 			spin_lock(&dm->lock);  in mlx5_cmd_alloc_memic()
66 			bitmap_clear(dm->memic_alloc_pages,  in mlx5_cmd_alloc_memic()
68 			spin_unlock(&dm->lock);  in mlx5_cmd_alloc_memic()
87 void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr,  in mlx5_cmd_dealloc_memic()  argument
90 	struct mlx5_core_dev *dev = dm->dev;  in mlx5_cmd_dealloc_memic()
108 	spin_lock(&dm->lock);  in mlx5_cmd_dealloc_memic()
109 	bitmap_clear(dm->memic_alloc_pages,  in mlx5_cmd_dealloc_memic()
111 	spin_unlock(&dm->lock);  in mlx5_cmd_dealloc_memic()
114 void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,  in mlx5_cmd_dealloc_memic_op()  argument
118 	struct mlx5_core_dev *dev = dm->dev;  in mlx5_cmd_dealloc_memic_op()
128 static int mlx5_cmd_alloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr,  in mlx5_cmd_alloc_memic_op()  argument
133 	struct mlx5_core_dev *dev = dm->dev;  in mlx5_cmd_alloc_memic_op()
165 	struct mlx5_ib_dm_memic *dm =  in mlx5_ib_dm_memic_free()  local
167 	struct mlx5_ib_dev *dev = to_mdev(dm->base.ibdm.device);  in mlx5_ib_dm_memic_free()
169 	mlx5_cmd_dealloc_memic(&dev->dm, dm->base.dev_addr, dm->base.size);  in mlx5_ib_dm_memic_free()
170 	kfree(dm);  in mlx5_ib_dm_memic_free()
192 static int map_existing_op(struct mlx5_ib_dm_memic *dm, u8 op,  in map_existing_op()  argument
197 	op_entry = xa_load(&dm->ops, op);  in map_existing_op()
211 	struct mlx5_ib_dm_memic *dm = to_memic(ibdm);  in UVERBS_HANDLER()  local
226 	mutex_lock(&dm->ops_xa_lock);  in UVERBS_HANDLER()
227 	err = map_existing_op(dm, op, attrs);  in UVERBS_HANDLER()
235 	err = mlx5_cmd_alloc_memic_op(&dev->dm, dm->base.dev_addr, op,  in UVERBS_HANDLER()
242 	op_entry->dm = dm;  in UVERBS_HANDLER()
245 				MLX5_IB_MMAP_TYPE_MEMIC_OP, dm->base.size,  in UVERBS_HANDLER()
248 		mlx5_cmd_dealloc_memic_op(&dev->dm, dm->base.dev_addr, op);  in UVERBS_HANDLER()
253 	kref_get(&dm->ref);  in UVERBS_HANDLER()
259 	err = xa_insert(&dm->ops, op, op_entry, GFP_KERNEL);  in UVERBS_HANDLER()
262 	mutex_unlock(&dm->ops_xa_lock);  in UVERBS_HANDLER()
269 	mutex_unlock(&dm->ops_xa_lock);  in UVERBS_HANDLER()
278 	struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm;  in handle_alloc_dm_memic()
279 	struct mlx5_ib_dm_memic *dm;  in handle_alloc_dm_memic()  local
288 	dm = kzalloc(sizeof(*dm), GFP_KERNEL);  in handle_alloc_dm_memic()
289 	if (!dm)  in handle_alloc_dm_memic()
292 	dm->base.type = MLX5_IB_UAPI_DM_TYPE_MEMIC;  in handle_alloc_dm_memic()
293 	dm->base.size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE);  in handle_alloc_dm_memic()
294 	dm->base.ibdm.device = ctx->device;  in handle_alloc_dm_memic()
296 	kref_init(&dm->ref);  in handle_alloc_dm_memic()
297 	xa_init(&dm->ops);  in handle_alloc_dm_memic()
298 	mutex_init(&dm->ops_xa_lock);  in handle_alloc_dm_memic()
299 	dm->req_length = attr->length;  in handle_alloc_dm_memic()
301 	err = mlx5_cmd_alloc_memic(dm_db, &dm->base.dev_addr,  in handle_alloc_dm_memic()
302 				   dm->base.size, attr->alignment);  in handle_alloc_dm_memic()
304 		kfree(dm);  in handle_alloc_dm_memic()
308 	address = dm->base.dev_addr & PAGE_MASK;  in handle_alloc_dm_memic()
309 	err = add_dm_mmap_entry(ctx, &dm->mentry, MLX5_IB_MMAP_TYPE_MEMIC,  in handle_alloc_dm_memic()
310 				dm->base.size, address);  in handle_alloc_dm_memic()
312 		mlx5_cmd_dealloc_memic(dm_db, dm->base.dev_addr, dm->base.size);  in handle_alloc_dm_memic()
313 		kfree(dm);  in handle_alloc_dm_memic()
317 	page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF;  in handle_alloc_dm_memic()
323 	start_offset = dm->base.dev_addr & ~PAGE_MASK;  in handle_alloc_dm_memic()
330 	return &dm->base.ibdm;  in handle_alloc_dm_memic()
333 	rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);  in handle_alloc_dm_memic()
359 	struct mlx5_ib_dm_icm *dm;  in handle_alloc_dm_sw_icm()  local
385 	dm = kzalloc(sizeof(*dm), GFP_KERNEL);  in handle_alloc_dm_sw_icm()
386 	if (!dm)  in handle_alloc_dm_sw_icm()
389 	dm->base.type = type;  in handle_alloc_dm_sw_icm()
390 	dm->base.ibdm.device = ctx->device;  in handle_alloc_dm_sw_icm()
398 	dm->base.size = act_size;  in handle_alloc_dm_sw_icm()
403 				   &dm->base.dev_addr, &dm->obj_id);  in handle_alloc_dm_sw_icm()
408 			     &dm->base.dev_addr, sizeof(dm->base.dev_addr));  in handle_alloc_dm_sw_icm()
410 		mlx5_dm_sw_icm_dealloc(dev, icm_type, dm->base.size,  in handle_alloc_dm_sw_icm()
412 				       dm->base.dev_addr, dm->obj_id);  in handle_alloc_dm_sw_icm()
415 	return &dm->base.ibdm;  in handle_alloc_dm_sw_icm()
417 	kfree(dm);  in handle_alloc_dm_sw_icm()
451 static void dm_memic_remove_ops(struct mlx5_ib_dm_memic *dm)  in dm_memic_remove_ops()  argument
456 	mutex_lock(&dm->ops_xa_lock);  in dm_memic_remove_ops()
457 	xa_for_each(&dm->ops, idx, entry) {  in dm_memic_remove_ops()
458 		xa_erase(&dm->ops, idx);  in dm_memic_remove_ops()
461 	mutex_unlock(&dm->ops_xa_lock);  in dm_memic_remove_ops()
464 static void mlx5_dm_memic_dealloc(struct mlx5_ib_dm_memic *dm)  in mlx5_dm_memic_dealloc()  argument
466 	dm_memic_remove_ops(dm);  in mlx5_dm_memic_dealloc()
467 	rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry);  in mlx5_dm_memic_dealloc()
471 			       struct mlx5_ib_dm_icm *dm)  in mlx5_dm_icm_dealloc()  argument
473 	enum mlx5_sw_icm_type type = get_icm_type(dm->base.type);  in mlx5_dm_icm_dealloc()
474 	struct mlx5_core_dev *dev = to_mdev(dm->base.ibdm.device)->mdev;  in mlx5_dm_icm_dealloc()
477 	err = mlx5_dm_sw_icm_dealloc(dev, type, dm->base.size, ctx->devx_uid,  in mlx5_dm_icm_dealloc()
478 				     dm->base.dev_addr, dm->obj_id);  in mlx5_dm_icm_dealloc()
480 		kfree(dm);  in mlx5_dm_icm_dealloc()
489 	struct mlx5_ib_dm *dm = to_mdm(ibdm);  in mlx5_ib_dealloc_dm()  local
491 	switch (dm->type) {  in mlx5_ib_dealloc_dm()
510 	struct mlx5_ib_dm *dm = to_mdm(ibdm);  in UVERBS_HANDLER()  local
516 	if (dm->type != MLX5_IB_UAPI_DM_TYPE_MEMIC)  in UVERBS_HANDLER()
551 		mdm = op_entry->dm;  in mlx5_ib_dm_mmap_free()
552 		mlx5_cmd_dealloc_memic_op(&dev->dm, mdm->base.dev_addr,  in mlx5_ib_dm_mmap_free()