51 void rds_message_addref(struct rds_message *rm)  in rds_message_addref()  argument
53 	rdsdebug("addref rm %p ref %d\n", rm, refcount_read(&rm->m_refcount));  in rds_message_addref()
54 	refcount_inc(&rm->m_refcount);  in rds_message_addref()
130 static void rds_message_purge(struct rds_message *rm)  in rds_message_purge()  argument
135 	if (unlikely(test_bit(RDS_MSG_PAGEVEC, &rm->m_flags)))  in rds_message_purge()
138 	spin_lock_irqsave(&rm->m_rs_lock, flags);  in rds_message_purge()
139 	if (rm->m_rs) {  in rds_message_purge()
140 		struct rds_sock *rs = rm->m_rs;  in rds_message_purge()
142 		if (rm->data.op_mmp_znotifier) {  in rds_message_purge()
144 			rds_rm_zerocopy_callback(rs, rm->data.op_mmp_znotifier);  in rds_message_purge()
146 			rm->data.op_mmp_znotifier = NULL;  in rds_message_purge()
149 		rm->m_rs = NULL;  in rds_message_purge()
151 	spin_unlock_irqrestore(&rm->m_rs_lock, flags);  in rds_message_purge()
153 	for (i = 0; i < rm->data.op_nents; i++) {  in rds_message_purge()
156 			__free_page(sg_page(&rm->data.op_sg[i]));  in rds_message_purge()
158 			put_page(sg_page(&rm->data.op_sg[i]));  in rds_message_purge()
160 	rm->data.op_nents = 0;  in rds_message_purge()
162 	if (rm->rdma.op_active)  in rds_message_purge()
163 		rds_rdma_free_op(&rm->rdma);  in rds_message_purge()
164 	if (rm->rdma.op_rdma_mr)  in rds_message_purge()
165 		kref_put(&rm->rdma.op_rdma_mr->r_kref, __rds_put_mr_final);  in rds_message_purge()
167 	if (rm->atomic.op_active)  in rds_message_purge()
168 		rds_atomic_free_op(&rm->atomic);  in rds_message_purge()
169 	if (rm->atomic.op_rdma_mr)  in rds_message_purge()
170 		kref_put(&rm->atomic.op_rdma_mr->r_kref, __rds_put_mr_final);  in rds_message_purge()
173 void rds_message_put(struct rds_message *rm)  in rds_message_put()  argument
175 	rdsdebug("put rm %p ref %d\n", rm, refcount_read(&rm->m_refcount));  in rds_message_put()
176 	WARN(!refcount_read(&rm->m_refcount), "danger refcount zero on %p\n", rm);  in rds_message_put()
177 	if (refcount_dec_and_test(&rm->m_refcount)) {  in rds_message_put()
178 		BUG_ON(!list_empty(&rm->m_sock_item));  in rds_message_put()
179 		BUG_ON(!list_empty(&rm->m_conn_item));  in rds_message_put()
180 		rds_message_purge(rm);  in rds_message_put()
182 		kfree(rm);  in rds_message_put()
286 	struct rds_message *rm;  in rds_message_alloc()  local
291 	rm = kzalloc(sizeof(struct rds_message) + extra_len, gfp);  in rds_message_alloc()
292 	if (!rm)  in rds_message_alloc()
295 	rm->m_used_sgs = 0;  in rds_message_alloc()
296 	rm->m_total_sgs = extra_len / sizeof(struct scatterlist);  in rds_message_alloc()
298 	refcount_set(&rm->m_refcount, 1);  in rds_message_alloc()
299 	INIT_LIST_HEAD(&rm->m_sock_item);  in rds_message_alloc()
300 	INIT_LIST_HEAD(&rm->m_conn_item);  in rds_message_alloc()
301 	spin_lock_init(&rm->m_rs_lock);  in rds_message_alloc()
302 	init_waitqueue_head(&rm->m_flush_wait);  in rds_message_alloc()
305 	return rm;  in rds_message_alloc()
309  * RDS ops use this to grab SG entries from the rm's sg pool.
311 struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents)  in rds_message_alloc_sgs()  argument
313 	struct scatterlist *sg_first = (struct scatterlist *) &rm[1];  in rds_message_alloc_sgs()
321 	if (rm->m_used_sgs + nents > rm->m_total_sgs) {  in rds_message_alloc_sgs()
323 			rm->m_total_sgs, rm->m_used_sgs, nents);  in rds_message_alloc_sgs()
327 	sg_ret = &sg_first[rm->m_used_sgs];  in rds_message_alloc_sgs()
329 	rm->m_used_sgs += nents;  in rds_message_alloc_sgs()
336 	struct rds_message *rm;  in rds_message_map_pages()  local
341 	rm = rds_message_alloc(extra_bytes, GFP_NOWAIT);  in rds_message_map_pages()
342 	if (!rm)  in rds_message_map_pages()
345 	set_bit(RDS_MSG_PAGEVEC, &rm->m_flags);  in rds_message_map_pages()
346 	rm->m_inc.i_hdr.h_len = cpu_to_be32(total_len);  in rds_message_map_pages()
347 	rm->data.op_nents = DIV_ROUND_UP(total_len, PAGE_SIZE);  in rds_message_map_pages()
348 	rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs);  in rds_message_map_pages()
349 	if (IS_ERR(rm->data.op_sg)) {  in rds_message_map_pages()
350 		void *err = ERR_CAST(rm->data.op_sg);  in rds_message_map_pages()
351 		rds_message_put(rm);  in rds_message_map_pages()
355 	for (i = 0; i < rm->data.op_nents; ++i) {  in rds_message_map_pages()
356 		sg_set_page(&rm->data.op_sg[i],  in rds_message_map_pages()
361 	return rm;  in rds_message_map_pages()
364 static int rds_message_zcopy_from_user(struct rds_message *rm, struct iov_iter *from)  in rds_message_zcopy_from_user()  argument
371 	rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));  in rds_message_zcopy_from_user()
376 	sg = rm->data.op_sg;  in rds_message_zcopy_from_user()
382 	rm->data.op_mmp_znotifier = &info->znotif;  in rds_message_zcopy_from_user()
383 	if (mm_account_pinned_pages(&rm->data.op_mmp_znotifier->z_mmp,  in rds_message_zcopy_from_user()
399 			for (i = 0; i < rm->data.op_nents; i++)  in rds_message_zcopy_from_user()
400 				put_page(sg_page(&rm->data.op_sg[i]));  in rds_message_zcopy_from_user()
401 			mmp = &rm->data.op_mmp_znotifier->z_mmp;  in rds_message_zcopy_from_user()
408 		rm->data.op_nents++;  in rds_message_zcopy_from_user()
415 	rm->data.op_mmp_znotifier = NULL;  in rds_message_zcopy_from_user()
419 int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,  in rds_message_copy_from_user()  argument
427 	rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from));  in rds_message_copy_from_user()
430 	sg = rm->data.op_sg;  in rds_message_copy_from_user()
434 		return rds_message_zcopy_from_user(rm, from);  in rds_message_copy_from_user()
442 			rm->data.op_nents++;  in rds_message_copy_from_user()
466 	struct rds_message *rm;  in rds_message_inc_copy_to_user()  local
474 	rm = container_of(inc, struct rds_message, m_inc);  in rds_message_inc_copy_to_user()
475 	len = be32_to_cpu(rm->m_inc.i_hdr.h_len);  in rds_message_inc_copy_to_user()
477 	sg = rm->data.op_sg;  in rds_message_inc_copy_to_user()
508 void rds_message_wait(struct rds_message *rm)  in rds_message_wait()  argument
510 	wait_event_interruptible(rm->m_flush_wait,  in rds_message_wait()
511 			!test_bit(RDS_MSG_MAPPED, &rm->m_flags));  in rds_message_wait()
514 void rds_message_unmapped(struct rds_message *rm)  in rds_message_unmapped()  argument
516 	clear_bit(RDS_MSG_MAPPED, &rm->m_flags);  in rds_message_unmapped()
517 	wake_up_interruptible(&rm->m_flush_wait);  in rds_message_unmapped()