4  * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
16  *	- Redistributions of source code must retain the above
20  *	- Redistributions in binary form must reproduce the above
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
43 		 ib_event_msg(cause->event), cause->event);  in iser_qp_event_callback()
50 		 ib_event_msg(event->event), event->event,  in iser_event_handler()
51 		dev_name(&event->device->dev), event->element.port_num);  in iser_event_handler()
55  * iser_create_device_ib_res - creates Protection Domain (PD), Completion
56  * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with
59  * Return: 0 on success, -1 on failure
63 	struct ib_device *ib_dev = device->ib_device;  in iser_create_device_ib_res()
65 	if (!(ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) {  in iser_create_device_ib_res()
66 		iser_err("IB device does not support memory registrations\n");  in iser_create_device_ib_res()
67 		return -1;  in iser_create_device_ib_res()
70 	device->pd = ib_alloc_pd(ib_dev,  in iser_create_device_ib_res()
72 	if (IS_ERR(device->pd))  in iser_create_device_ib_res()
75 	INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev,  in iser_create_device_ib_res()
77 	ib_register_event_handler(&device->event_handler);  in iser_create_device_ib_res()
82 	return -1;  in iser_create_device_ib_res()
86  * iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR,
91 	ib_unregister_event_handler(&device->event_handler);  in iser_free_device_ib_res()
92 	ib_dealloc_pd(device->pd);  in iser_free_device_ib_res()
94 	device->pd = NULL;  in iser_free_device_ib_res()
104 	struct ib_device *ib_dev = device->ib_device;  in iser_create_fastreg_desc()
110 		return ERR_PTR(-ENOMEM);  in iser_create_fastreg_desc()
112 	if (ib_dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG)  in iser_create_fastreg_desc()
117 	desc->rsc.mr = ib_alloc_mr(pd, mr_type, size);  in iser_create_fastreg_desc()
118 	if (IS_ERR(desc->rsc.mr)) {  in iser_create_fastreg_desc()
119 		ret = PTR_ERR(desc->rsc.mr);  in iser_create_fastreg_desc()
125 		desc->rsc.sig_mr = ib_alloc_mr_integrity(pd, size, size);  in iser_create_fastreg_desc()
126 		if (IS_ERR(desc->rsc.sig_mr)) {  in iser_create_fastreg_desc()
127 			ret = PTR_ERR(desc->rsc.sig_mr);  in iser_create_fastreg_desc()
136 	ib_dereg_mr(desc->rsc.mr);  in iser_create_fastreg_desc()
145 	struct iser_reg_resources *res = &desc->rsc;  in iser_destroy_fastreg_desc()
147 	ib_dereg_mr(res->mr);  in iser_destroy_fastreg_desc()
148 	if (res->sig_mr) {  in iser_destroy_fastreg_desc()
149 		ib_dereg_mr(res->sig_mr);  in iser_destroy_fastreg_desc()
150 		res->sig_mr = NULL;  in iser_destroy_fastreg_desc()
156  * iser_alloc_fastreg_pool - Creates pool of fast_reg descriptors
168 	struct iser_device *device = ib_conn->device;  in iser_alloc_fastreg_pool()
169 	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;  in iser_alloc_fastreg_pool()
173 	INIT_LIST_HEAD(&fr_pool->list);  in iser_alloc_fastreg_pool()
174 	INIT_LIST_HEAD(&fr_pool->all_list);  in iser_alloc_fastreg_pool()
175 	spin_lock_init(&fr_pool->lock);  in iser_alloc_fastreg_pool()
176 	fr_pool->size = 0;  in iser_alloc_fastreg_pool()
178 		desc = iser_create_fastreg_desc(device, device->pd,  in iser_alloc_fastreg_pool()
179 						ib_conn->pi_support, size);  in iser_alloc_fastreg_pool()
185 		list_add_tail(&desc->list, &fr_pool->list);  in iser_alloc_fastreg_pool()
186 		list_add_tail(&desc->all_list, &fr_pool->all_list);  in iser_alloc_fastreg_pool()
187 		fr_pool->size++;  in iser_alloc_fastreg_pool()
198  * iser_free_fastreg_pool - releases the pool of fast_reg descriptors
203 	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;  in iser_free_fastreg_pool()
207 	if (list_empty(&fr_pool->all_list))  in iser_free_fastreg_pool()
212 	list_for_each_entry_safe(desc, tmp, &fr_pool->all_list, all_list) {  in iser_free_fastreg_pool()
213 		list_del(&desc->all_list);  in iser_free_fastreg_pool()
218 	if (i < fr_pool->size)  in iser_free_fastreg_pool()
220 			  fr_pool->size - i);  in iser_free_fastreg_pool()
224  * iser_create_ib_conn_res - Queue-Pair (QP)
226  * Return: 0 on success, -1 on failure
234 	int			ret = -ENOMEM;  in iser_create_ib_conn_res()
237 	BUG_ON(ib_conn->device == NULL);  in iser_create_ib_conn_res()
239 	device = ib_conn->device;  in iser_create_ib_conn_res()
240 	ib_dev = device->ib_device;  in iser_create_ib_conn_res()
243 	if (ib_conn->pi_support)  in iser_create_ib_conn_res()
248 			    (unsigned int)ib_dev->attrs.max_qp_wr);  in iser_create_ib_conn_res()
251 	ib_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_SOFTIRQ);  in iser_create_ib_conn_res()
252 	if (IS_ERR(ib_conn->cq)) {  in iser_create_ib_conn_res()
253 		ret = PTR_ERR(ib_conn->cq);  in iser_create_ib_conn_res()
256 	ib_conn->cq_size = cq_size;  in iser_create_ib_conn_res()
262 	init_attr.send_cq = ib_conn->cq;  in iser_create_ib_conn_res()
263 	init_attr.recv_cq = ib_conn->cq;  in iser_create_ib_conn_res()
271 	if (ib_conn->pi_support)  in iser_create_ib_conn_res()
273 	iser_conn->max_cmds = ISER_GET_MAX_XMIT_CMDS(max_send_wr - 1);  in iser_create_ib_conn_res()
275 	ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);  in iser_create_ib_conn_res()
279 	ib_conn->qp = ib_conn->cma_id->qp;  in iser_create_ib_conn_res()
281 		  ib_conn->cma_id, ib_conn->cma_id->qp, max_send_wr);  in iser_create_ib_conn_res()
285 	ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size);  in iser_create_ib_conn_res()
294  * device for this device. If there's no such, create one.
305 		if (device->ib_device->node_guid == cma_id->device->node_guid)  in iser_device_find_by_ib_device()
313 	device->ib_device = cma_id->device;  in iser_device_find_by_ib_device()
320 	list_add(&device->ig_list, &ig.device_list);  in iser_device_find_by_ib_device()
323 	device->refcount++;  in iser_device_find_by_ib_device()
329 /* if there's no demand for this device, release it */
333 	device->refcount--;  in iser_device_try_release()
334 	iser_info("device %p refcount %d\n", device, device->refcount);  in iser_device_try_release()
335 	if (!device->refcount) {  in iser_device_try_release()
337 		list_del(&device->ig_list);  in iser_device_try_release()
350 	wait_for_completion(&iser_conn->stop_completion);  in iser_release_work()
352 	wait_for_completion(&iser_conn->ib_completion);  in iser_release_work()
354 	mutex_lock(&iser_conn->state_mutex);  in iser_release_work()
355 	iser_conn->state = ISER_CONN_DOWN;  in iser_release_work()
356 	mutex_unlock(&iser_conn->state_mutex);  in iser_release_work()
362  * iser_free_ib_conn_res - release IB related resources
365  *     iser device and memory regoins pool (only iscsi
374 	struct ib_conn *ib_conn = &iser_conn->ib_conn;  in iser_free_ib_conn_res()
375 	struct iser_device *device = ib_conn->device;  in iser_free_ib_conn_res()
378 		  iser_conn, ib_conn->cma_id, ib_conn->qp);  in iser_free_ib_conn_res()
380 	if (ib_conn->qp) {  in iser_free_ib_conn_res()
381 		rdma_destroy_qp(ib_conn->cma_id);  in iser_free_ib_conn_res()
382 		ib_cq_pool_put(ib_conn->cq, ib_conn->cq_size);  in iser_free_ib_conn_res()
383 		ib_conn->qp = NULL;  in iser_free_ib_conn_res()
387 		if (iser_conn->rx_descs)  in iser_free_ib_conn_res()
392 			ib_conn->device = NULL;  in iser_free_ib_conn_res()
398  * iser_conn_release - Frees all conn objects and deallocs conn descriptor
403 	struct ib_conn *ib_conn = &iser_conn->ib_conn;  in iser_conn_release()
406 	list_del(&iser_conn->conn_list);  in iser_conn_release()
409 	mutex_lock(&iser_conn->state_mutex);  in iser_conn_release()
411 	if (iser_conn->state != ISER_CONN_DOWN) {  in iser_conn_release()
413 			  iser_conn, iser_conn->state);  in iser_conn_release()
414 		iscsi_destroy_endpoint(iser_conn->ep);  in iser_conn_release()
415 		iser_conn->state = ISER_CONN_DOWN;  in iser_conn_release()
422 	mutex_unlock(&iser_conn->state_mutex);  in iser_conn_release()
424 	if (ib_conn->cma_id) {  in iser_conn_release()
425 		rdma_destroy_id(ib_conn->cma_id);  in iser_conn_release()
426 		ib_conn->cma_id = NULL;  in iser_conn_release()
433  * iser_conn_terminate - triggers start of the disconnect procedures and
441 	struct ib_conn *ib_conn = &iser_conn->ib_conn;  in iser_conn_terminate()
444 	lockdep_assert_held(&iser_conn->state_mutex);  in iser_conn_terminate()
447 	if (iser_conn->state != ISER_CONN_UP)  in iser_conn_terminate()
450 	iser_conn->state = ISER_CONN_TERMINATING;  in iser_conn_terminate()
451 	iser_info("iser_conn %p state %d\n", iser_conn, iser_conn->state);  in iser_conn_terminate()
454 	if (iser_conn->iscsi_conn)  in iser_conn_terminate()
455 		iscsi_suspend_queue(iser_conn->iscsi_conn);  in iser_conn_terminate()
462 	if (ib_conn->cma_id) {  in iser_conn_terminate()
463 		err = rdma_disconnect(ib_conn->cma_id);  in iser_conn_terminate()
469 		ib_drain_qp(ib_conn->qp);  in iser_conn_terminate()
480 	struct iser_conn *iser_conn = cma_id->context;  in iser_connect_error()
482 	lockdep_assert_held(&iser_conn->state_mutex);  in iser_connect_error()
484 	iser_conn->state = ISER_CONN_TERMINATING;  in iser_connect_error()
490 	struct iser_device *device = iser_conn->ib_conn.device;  in iser_calc_scsi_params()
491 	struct ib_device_attr *attr = &device->ib_device->attrs;  in iser_calc_scsi_params()
502 	if (attr->kernel_cap_flags & IBK_SG_GAPS_REG)  in iser_calc_scsi_params()
507 	if (iser_conn->ib_conn.pi_support)  in iser_calc_scsi_params()
508 		max_num_sg = attr->max_pi_fast_reg_page_list_len;  in iser_calc_scsi_params()
510 		max_num_sg = attr->max_fast_reg_page_list_len;  in iser_calc_scsi_params()
514 				 max_num_sg - reserved_mr_pages);  in iser_calc_scsi_params()
515 	iser_conn->scsi_sg_tablesize = min(sg_tablesize, sup_sg_tablesize);  in iser_calc_scsi_params()
516 	iser_conn->pages_per_mr =  in iser_calc_scsi_params()
517 		iser_conn->scsi_sg_tablesize + reserved_mr_pages;  in iser_calc_scsi_params()
525 	struct iser_conn *iser_conn = cma_id->context;  in iser_addr_handler()
530 	lockdep_assert_held(&iser_conn->state_mutex);  in iser_addr_handler()
532 	if (iser_conn->state != ISER_CONN_PENDING)  in iser_addr_handler()
536 	ib_conn = &iser_conn->ib_conn;  in iser_addr_handler()
544 	ib_conn->device = device;  in iser_addr_handler()
546 	/* connection T10-PI support */  in iser_addr_handler()
548 		if (!(device->ib_device->attrs.kernel_cap_flags &  in iser_addr_handler()
550 			iser_warn("T10-PI requested but not supported on %s, "  in iser_addr_handler()
551 				  "continue without T10-PI\n",  in iser_addr_handler()
552 				  dev_name(&ib_conn->device->ib_device->dev));  in iser_addr_handler()
553 			ib_conn->pi_support = false;  in iser_addr_handler()
555 			ib_conn->pi_support = true;  in iser_addr_handler()
577 	struct iser_conn *iser_conn = cma_id->context;  in iser_route_handler()
578 	struct ib_conn *ib_conn = &iser_conn->ib_conn;  in iser_route_handler()
579 	struct ib_device *ib_dev = ib_conn->device->ib_device;  in iser_route_handler()
581 	lockdep_assert_held(&iser_conn->state_mutex);  in iser_route_handler()
583 	if (iser_conn->state != ISER_CONN_PENDING)  in iser_route_handler()
592 	conn_param.responder_resources = ib_dev->attrs.max_qp_rd_atom;  in iser_route_handler()
621 	struct iser_conn *iser_conn = cma_id->context;  in iser_connected_handler()
625 	lockdep_assert_held(&iser_conn->state_mutex);  in iser_connected_handler()
627 	if (iser_conn->state != ISER_CONN_PENDING)  in iser_connected_handler()
631 	(void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr);  in iser_connected_handler()
632 	iser_info("remote qpn:%x my qpn:%x\n", attr.dest_qp_num, cma_id->qp->qp_num);  in iser_connected_handler()
637 		iser_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP);  in iser_connected_handler()
641 		  iser_conn, iser_conn->snd_w_inv ? "remote" : "local");  in iser_connected_handler()
643 	iser_conn->state = ISER_CONN_UP;  in iser_connected_handler()
644 	complete(&iser_conn->up_completion);  in iser_connected_handler()
653 	struct iser_conn *iser_conn = cma_id->context;  in iser_cleanup_handler()
655 	lockdep_assert_held(&iser_conn->state_mutex);  in iser_cleanup_handler()
662 		if (iser_conn->iscsi_conn)  in iser_cleanup_handler()
663 			iscsi_conn_failure(iser_conn->iscsi_conn,  in iser_cleanup_handler()
669 	complete(&iser_conn->ib_completion);  in iser_cleanup_handler()
678 	iser_conn = cma_id->context;  in iser_cma_handler()
680 		  rdma_event_msg(event->event), event->event,  in iser_cma_handler()
681 		  event->status, cma_id->context, cma_id);  in iser_cma_handler()
683 	mutex_lock(&iser_conn->state_mutex);  in iser_cma_handler()
684 	switch (event->event) {  in iser_cma_handler()
692 		iser_connected_handler(cma_id, event->param.conn.private_data);  in iser_cma_handler()
696 			 rdma_reject_msg(cma_id, event->status));  in iser_cma_handler()
717 		if (iser_conn->state != ISER_CONN_DOWN) {  in iser_cma_handler()
718 			iser_conn->ib_conn.cma_id = NULL;  in iser_cma_handler()
724 			 rdma_event_msg(event->event), event->event);  in iser_cma_handler()
727 	mutex_unlock(&iser_conn->state_mutex);  in iser_cma_handler()
734 	struct ib_conn *ib_conn = &iser_conn->ib_conn;  in iser_conn_init()
736 	iser_conn->state = ISER_CONN_INIT;  in iser_conn_init()
737 	init_completion(&iser_conn->stop_completion);  in iser_conn_init()
738 	init_completion(&iser_conn->ib_completion);  in iser_conn_init()
739 	init_completion(&iser_conn->up_completion);  in iser_conn_init()
740 	INIT_LIST_HEAD(&iser_conn->conn_list);  in iser_conn_init()
741 	mutex_init(&iser_conn->state_mutex);  in iser_conn_init()
743 	ib_conn->reg_cqe.done = iser_reg_comp;  in iser_conn_init()
753 	struct ib_conn *ib_conn = &iser_conn->ib_conn;  in iser_connect()
756 	mutex_lock(&iser_conn->state_mutex);  in iser_connect()
758 	sprintf(iser_conn->name, "%pISp", dst_addr);  in iser_connect()
760 	iser_info("connecting to: %s\n", iser_conn->name);  in iser_connect()
762 	/* the device is known only --after-- address resolution */  in iser_connect()
763 	ib_conn->device = NULL;  in iser_connect()
765 	iser_conn->state = ISER_CONN_PENDING;  in iser_connect()
767 	ib_conn->cma_id = rdma_create_id(&init_net, iser_cma_handler,  in iser_connect()
769 	if (IS_ERR(ib_conn->cma_id)) {  in iser_connect()
770 		err = PTR_ERR(ib_conn->cma_id);  in iser_connect()
775 	err = rdma_resolve_addr(ib_conn->cma_id, src_addr, dst_addr, 1000);  in iser_connect()
782 		wait_for_completion_interruptible(&iser_conn->up_completion);  in iser_connect()
784 		if (iser_conn->state != ISER_CONN_UP) {  in iser_connect()
785 			err =  -EIO;  in iser_connect()
789 	mutex_unlock(&iser_conn->state_mutex);  in iser_connect()
792 	list_add(&iser_conn->conn_list, &ig.connlist);  in iser_connect()
797 	ib_conn->cma_id = NULL;  in iser_connect()
799 	iser_conn->state = ISER_CONN_DOWN;  in iser_connect()
801 	mutex_unlock(&iser_conn->state_mutex);  in iser_connect()
808 	struct ib_conn *ib_conn = &iser_conn->ib_conn;  in iser_post_recvl()
809 	struct iser_login_desc *desc = &iser_conn->login_desc;  in iser_post_recvl()
813 	desc->sge.addr = desc->rsp_dma;  in iser_post_recvl()
814 	desc->sge.length = ISER_RX_LOGIN_SIZE;  in iser_post_recvl()
815 	desc->sge.lkey = ib_conn->device->pd->local_dma_lkey;  in iser_post_recvl()
817 	desc->cqe.done = iser_login_rsp;  in iser_post_recvl()
818 	wr.wr_cqe = &desc->cqe;  in iser_post_recvl()
819 	wr.sg_list = &desc->sge;  in iser_post_recvl()
823 	ret = ib_post_recv(ib_conn->qp, &wr, NULL);  in iser_post_recvl()
832 	struct ib_conn *ib_conn = &iser_conn->ib_conn;  in iser_post_recvm()
836 	rx_desc->cqe.done = iser_task_rsp;  in iser_post_recvm()
837 	wr.wr_cqe = &rx_desc->cqe;  in iser_post_recvm()
838 	wr.sg_list = &rx_desc->rx_sg;  in iser_post_recvm()
842 	ret = ib_post_recv(ib_conn->qp, &wr, NULL);  in iser_post_recvm()
851  * iser_post_send - Initiate a Send DTO operation
855  * Return: 0 on success, -1 on failure
859 	struct ib_send_wr *wr = &tx_desc->send_wr;  in iser_post_send()
863 	ib_dma_sync_single_for_device(ib_conn->device->ib_device,  in iser_post_send()
864 				      tx_desc->dma_addr, ISER_HEADERS_LEN,  in iser_post_send()
867 	wr->next = NULL;  in iser_post_send()
868 	wr->wr_cqe = &tx_desc->cqe;  in iser_post_send()
869 	wr->sg_list = tx_desc->tx_sg;  in iser_post_send()
870 	wr->num_sge = tx_desc->num_sge;  in iser_post_send()
871 	wr->opcode = IB_WR_SEND;  in iser_post_send()
872 	wr->send_flags = IB_SEND_SIGNALED;  in iser_post_send()
874 	if (tx_desc->inv_wr.next)  in iser_post_send()
875 		first_wr = &tx_desc->inv_wr;  in iser_post_send()
876 	else if (tx_desc->reg_wr.wr.next)  in iser_post_send()
877 		first_wr = &tx_desc->reg_wr.wr;  in iser_post_send()
881 	ret = ib_post_send(ib_conn->qp, first_wr, NULL);  in iser_post_send()
884 			 ret, wr->opcode);  in iser_post_send()
892 	struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];  in iser_check_task_pi_status()
893 	struct iser_fr_desc *desc = reg->desc;  in iser_check_task_pi_status()
894 	unsigned long sector_size = iser_task->sc->device->sector_size;  in iser_check_task_pi_status()
898 	if (desc && desc->sig_protected) {  in iser_check_task_pi_status()
899 		desc->sig_protected = false;  in iser_check_task_pi_status()
900 		ret = ib_check_mr_status(desc->rsc.sig_mr,  in iser_check_task_pi_status()
913 			*sector = scsi_get_sector(iser_task->sc) + sector_off;  in iser_check_task_pi_status()
936 void iser_err_comp(struct ib_wc *wc, const char *type)  in iser_err_comp()  argument
938 	if (wc->status != IB_WC_WR_FLUSH_ERR) {  in iser_err_comp()
939 		struct iser_conn *iser_conn = to_iser_conn(wc->qp->qp_context);  in iser_err_comp()
942 			 ib_wc_status_msg(wc->status), wc->status,  in iser_err_comp()
943 			 wc->vendor_err);  in iser_err_comp()
945 		if (iser_conn->iscsi_conn)  in iser_err_comp()
946 			iscsi_conn_failure(iser_conn->iscsi_conn,  in iser_err_comp()
950 			 ib_wc_status_msg(wc->status), wc->status);  in iser_err_comp()