247 void cma_dev_get(struct cma_device *cma_dev)  in cma_dev_get()  argument
249 	refcount_inc(&cma_dev->refcount);  in cma_dev_get()
252 void cma_dev_put(struct cma_device *cma_dev)  in cma_dev_put()  argument
254 	if (refcount_dec_and_test(&cma_dev->refcount))  in cma_dev_put()
255 		complete(&cma_dev->comp);  in cma_dev_put()
261 	struct cma_device *cma_dev;  in cma_enum_devices_by_ibdev()  local
266 	list_for_each_entry(cma_dev, &dev_list, list)  in cma_enum_devices_by_ibdev()
267 		if (filter(cma_dev->device, cookie)) {  in cma_enum_devices_by_ibdev()
268 			found_cma_dev = cma_dev;  in cma_enum_devices_by_ibdev()
278 int cma_get_default_gid_type(struct cma_device *cma_dev,  in cma_get_default_gid_type()  argument
281 	if (!rdma_is_port_valid(cma_dev->device, port))  in cma_get_default_gid_type()
284 	return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)];  in cma_get_default_gid_type()
287 int cma_set_default_gid_type(struct cma_device *cma_dev,  in cma_set_default_gid_type()  argument
293 	if (!rdma_is_port_valid(cma_dev->device, port))  in cma_set_default_gid_type()
297 	    rdma_protocol_roce_eth_encap(cma_dev->device, port))  in cma_set_default_gid_type()
300 	supported_gids = roce_gid_type_mask_support(cma_dev->device, port);  in cma_set_default_gid_type()
305 	cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] =  in cma_set_default_gid_type()
311 int cma_get_default_roce_tos(struct cma_device *cma_dev, u32 port)  in cma_get_default_roce_tos()  argument
313 	if (!rdma_is_port_valid(cma_dev->device, port))  in cma_get_default_roce_tos()
316 	return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)];  in cma_get_default_roce_tos()
319 int cma_set_default_roce_tos(struct cma_device *cma_dev, u32 port,  in cma_set_default_roce_tos()  argument
322 	if (!rdma_is_port_valid(cma_dev->device, port))  in cma_set_default_roce_tos()
325 	cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] =  in cma_set_default_roce_tos()
330 struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)  in cma_get_ib_dev()  argument
332 	return cma_dev->device;  in cma_get_ib_dev()
575 			       struct cma_device *cma_dev)  in _cma_attach_to_dev()  argument
577 	cma_dev_get(cma_dev);  in _cma_attach_to_dev()
578 	id_priv->cma_dev = cma_dev;  in _cma_attach_to_dev()
579 	id_priv->id.device = cma_dev->device;  in _cma_attach_to_dev()
581 		rdma_node_get_transport(cma_dev->device->node_type);  in _cma_attach_to_dev()
582 	list_add_tail(&id_priv->device_item, &cma_dev->id_list);  in _cma_attach_to_dev()
584 	trace_cm_id_attach(id_priv, cma_dev->device);  in _cma_attach_to_dev()
588 			      struct cma_device *cma_dev)  in cma_attach_to_dev()  argument
590 	_cma_attach_to_dev(id_priv, cma_dev);  in cma_attach_to_dev()
592 		cma_dev->default_gid_type[id_priv->id.port_num -  in cma_attach_to_dev()
593 					  rdma_start_port(cma_dev->device)];  in cma_attach_to_dev()
600 	cma_dev_put(id_priv->cma_dev);  in cma_release_dev()
601 	id_priv->cma_dev = NULL;  in cma_release_dev()
779 	struct cma_device *cma_dev;  in cma_acquire_dev_by_src_ip()  local
795 	list_for_each_entry(cma_dev, &dev_list, list) {  in cma_acquire_dev_by_src_ip()
797 		    id_priv->restricted_node_type != cma_dev->device->node_type)  in cma_acquire_dev_by_src_ip()
799 		rdma_for_each_port (cma_dev->device, port) {  in cma_acquire_dev_by_src_ip()
800 			gidp = rdma_protocol_roce(cma_dev->device, port) ?  in cma_acquire_dev_by_src_ip()
802 			gid_type = cma_dev->default_gid_type[port - 1];  in cma_acquire_dev_by_src_ip()
803 			sgid_attr = cma_validate_port(cma_dev->device, port,  in cma_acquire_dev_by_src_ip()
808 				cma_attach_to_dev(id_priv, cma_dev);  in cma_acquire_dev_by_src_ip()
850 	gid_type = listen_id_priv->cma_dev->default_gid_type[req->port - 1];  in cma_ib_acquire_dev()
863 	cma_attach_to_dev(id_priv, listen_id_priv->cma_dev);  in cma_ib_acquire_dev()
874 	struct cma_device *cma_dev;  in cma_iw_acquire_dev()  local
889 	cma_dev = listen_id_priv->cma_dev;  in cma_iw_acquire_dev()
892 	sgid_attr = cma_validate_port(cma_dev->device, port,  in cma_iw_acquire_dev()
901 	list_for_each_entry(cma_dev, &dev_list, list) {  in cma_iw_acquire_dev()
902 		rdma_for_each_port (cma_dev->device, port) {  in cma_iw_acquire_dev()
903 			if (listen_id_priv->cma_dev == cma_dev &&  in cma_iw_acquire_dev()
907 			gid_type = cma_dev->default_gid_type[port - 1];  in cma_iw_acquire_dev()
908 			sgid_attr = cma_validate_port(cma_dev->device, port,  in cma_iw_acquire_dev()
921 		cma_attach_to_dev(id_priv, cma_dev);  in cma_iw_acquire_dev()
934 	struct cma_device *cma_dev, *cur_dev;  in cma_resolve_ib_dev()  local
943 	cma_dev = NULL;  in cma_resolve_ib_dev()
968 					cma_dev = cur_dev;  in cma_resolve_ib_dev()
974 				if (!cma_dev && (gid.global.subnet_prefix ==  in cma_resolve_ib_dev()
977 					cma_dev = cur_dev;  in cma_resolve_ib_dev()
989 	cma_attach_to_dev(id_priv, cma_dev);  in cma_resolve_ib_dev()
1203 	BUG_ON(id_priv->cma_dev->device != id_priv->id.device);  in cma_modify_qp_rtr()
1982 		if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)  in cma_cancel_operation()
2028 			gid_type = id_priv->cma_dev->default_gid_type  in destroy_mc()
2031 						    id_priv->cma_dev->device)];  in destroy_mc()
2064 	if (id_priv->cma_dev) {  in _destroy_id()
2720 			     struct cma_device *cma_dev,  in cma_listen_on_dev()  argument
2730 	if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1))  in cma_listen_on_dev()
2733 	    id_priv->restricted_node_type != cma_dev->device->node_type)  in cma_listen_on_dev()
2747 	_cma_attach_to_dev(dev_id_priv, cma_dev);  in cma_listen_on_dev()
2765 	dev_warn(&cma_dev->device->dev, "RDMA CMA: %s, error %d\n", __func__, ret);  in cma_listen_on_dev()
2772 	struct cma_device *cma_dev;  in cma_listen_on_all()  local
2777 	list_for_each_entry(cma_dev, &dev_list, list) {  in cma_listen_on_all()
2778 		ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);  in cma_listen_on_all()
3307 	u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num -  in cma_resolve_iboe_route()
3308 					rdma_start_port(id_priv->cma_dev->device)];  in cma_resolve_iboe_route()
3451 	struct cma_device *cma_dev, *cur_dev;  in cma_bind_loopback()  local
3458 	cma_dev = NULL;  in cma_bind_loopback()
3465 		if (!cma_dev)  in cma_bind_loopback()
3466 			cma_dev = cur_dev;  in cma_bind_loopback()
3471 				cma_dev = cur_dev;  in cma_bind_loopback()
3477 	if (!cma_dev) {  in cma_bind_loopback()
3485 	ret = rdma_query_gid(cma_dev->device, p, 0, &gid);  in cma_bind_loopback()
3489 	ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);  in cma_bind_loopback()
3494 		(rdma_protocol_ib(cma_dev->device, p)) ?  in cma_bind_loopback()
3500 	cma_attach_to_dev(id_priv, cma_dev);  in cma_bind_loopback()
3529 	if (!status && !id_priv->cma_dev) {  in addr_handler()
3568 	if (!id_priv->cma_dev) {  in cma_resolve_loopback()
3593 	if (!id_priv->cma_dev) {  in cma_resolve_ib_addr()
3980 	if (id_priv->cma_dev) {  in rdma_listen()
4063 	if (id_priv->cma_dev)  in rdma_bind_addr_dst()
4878 		id_priv->cma_dev  in cma_make_mc_event()
4881 						   id_priv->cma_dev->device)];  in cma_make_mc_event()
5043 	gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -  in cma_iboe_join_multicast()
5044 		   rdma_start_port(id_priv->cma_dev->device)];  in cma_iboe_join_multicast()
5150 		WARN_ON(id_priv->cma_dev->device != id->device);  in rdma_leave_multicast()
5188 	struct cma_device *cma_dev;  in cma_netdev_callback()  local
5199 	list_for_each_entry(cma_dev, &dev_list, list)  in cma_netdev_callback()
5200 		list_for_each_entry(id_priv, &cma_dev->id_list, device_item) {  in cma_netdev_callback()
5332 static void cma_process_remove(struct cma_device *cma_dev)  in cma_process_remove()  argument
5335 	while (!list_empty(&cma_dev->id_list)) {  in cma_process_remove()
5337 			&cma_dev->id_list, struct rdma_id_private, device_item);  in cma_process_remove()
5350 	cma_dev_put(cma_dev);  in cma_process_remove()
5351 	wait_for_completion(&cma_dev->comp);  in cma_process_remove()
5368 	struct cma_device *cma_dev;  in cma_add_one()  local
5377 	cma_dev = kmalloc_obj(*cma_dev);  in cma_add_one()
5378 	if (!cma_dev)  in cma_add_one()
5381 	cma_dev->device = device;  in cma_add_one()
5382 	cma_dev->default_gid_type = kzalloc_objs(*cma_dev->default_gid_type,  in cma_add_one()
5384 	if (!cma_dev->default_gid_type) {  in cma_add_one()
5389 	cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,  in cma_add_one()
5390 					    sizeof(*cma_dev->default_roce_tos),  in cma_add_one()
5392 	if (!cma_dev->default_roce_tos) {  in cma_add_one()
5401 			cma_dev->default_gid_type[i - rdma_start_port(device)] =  in cma_add_one()
5404 			cma_dev->default_gid_type[i - rdma_start_port(device)] =  in cma_add_one()
5406 		cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;  in cma_add_one()
5409 	init_completion(&cma_dev->comp);  in cma_add_one()
5410 	refcount_set(&cma_dev->refcount, 1);  in cma_add_one()
5411 	INIT_LIST_HEAD(&cma_dev->id_list);  in cma_add_one()
5412 	ib_set_client_data(device, &cma_client, cma_dev);  in cma_add_one()
5415 	list_add_tail(&cma_dev->list, &dev_list);  in cma_add_one()
5417 		ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);  in cma_add_one()
5427 	list_del(&cma_dev->list);  in cma_add_one()
5431 	cma_process_remove(cma_dev);  in cma_add_one()
5432 	kfree(cma_dev->default_roce_tos);  in cma_add_one()
5434 	kfree(cma_dev->default_gid_type);  in cma_add_one()
5437 	kfree(cma_dev);  in cma_add_one()
5443 	struct cma_device *cma_dev = client_data;  in cma_remove_one()  local
5448 	list_del(&cma_dev->list);  in cma_remove_one()
5451 	cma_process_remove(cma_dev);  in cma_remove_one()
5452 	kfree(cma_dev->default_roce_tos);  in cma_remove_one()
5453 	kfree(cma_dev->default_gid_type);  in cma_remove_one()
5454 	kfree(cma_dev);  in cma_remove_one()
5658 	if (!id_priv->cma_dev ||  in rdma_resolve_ib_service()