Lines Matching refs:rm
53 void rds_message_addref(struct rds_message *rm) in rds_message_addref() argument
55 rdsdebug("addref rm %p ref %d\n", rm, refcount_read(&rm->m_refcount)); in rds_message_addref()
56 refcount_inc(&rm->m_refcount); in rds_message_addref()
132 static void rds_message_purge(struct rds_message *rm) in rds_message_purge() argument
138 if (unlikely(test_bit(RDS_MSG_PAGEVEC, &rm->m_flags))) in rds_message_purge()
141 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_message_purge()
142 znotifier = rm->data.op_mmp_znotifier; in rds_message_purge()
143 rm->data.op_mmp_znotifier = NULL; in rds_message_purge()
146 if (rm->m_rs) { in rds_message_purge()
147 struct rds_sock *rs = rm->m_rs; in rds_message_purge()
154 rm->m_rs = NULL; in rds_message_purge()
163 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_message_purge()
165 for (i = 0; i < rm->data.op_nents; i++) { in rds_message_purge()
168 __free_page(sg_page(&rm->data.op_sg[i])); in rds_message_purge()
170 put_page(sg_page(&rm->data.op_sg[i])); in rds_message_purge()
172 rm->data.op_nents = 0; in rds_message_purge()
174 if (rm->rdma.op_active) in rds_message_purge()
175 rds_rdma_free_op(&rm->rdma); in rds_message_purge()
176 if (rm->rdma.op_rdma_mr) in rds_message_purge()
177 kref_put(&rm->rdma.op_rdma_mr->r_kref, __rds_put_mr_final); in rds_message_purge()
179 if (rm->atomic.op_active) in rds_message_purge()
180 rds_atomic_free_op(&rm->atomic); in rds_message_purge()
181 if (rm->atomic.op_rdma_mr) in rds_message_purge()
182 kref_put(&rm->atomic.op_rdma_mr->r_kref, __rds_put_mr_final); in rds_message_purge()
185 void rds_message_put(struct rds_message *rm) in rds_message_put() argument
187 rdsdebug("put rm %p ref %d\n", rm, refcount_read(&rm->m_refcount)); in rds_message_put()
188 WARN(!refcount_read(&rm->m_refcount), "danger refcount zero on %p\n", rm); in rds_message_put()
189 if (refcount_dec_and_test(&rm->m_refcount)) { in rds_message_put()
190 BUG_ON(!list_empty(&rm->m_sock_item)); in rds_message_put()
191 BUG_ON(!list_empty(&rm->m_conn_item)); in rds_message_put()
192 rds_message_purge(rm); in rds_message_put()
194 kfree(rm); in rds_message_put()
336 struct rds_message *rm; in rds_message_alloc() local
341 rm = kzalloc(sizeof(struct rds_message) + extra_len, gfp); in rds_message_alloc()
342 if (!rm) in rds_message_alloc()
345 rm->m_used_sgs = 0; in rds_message_alloc()
346 rm->m_total_sgs = extra_len / sizeof(struct scatterlist); in rds_message_alloc()
348 refcount_set(&rm->m_refcount, 1); in rds_message_alloc()
349 INIT_LIST_HEAD(&rm->m_sock_item); in rds_message_alloc()
350 INIT_LIST_HEAD(&rm->m_conn_item); in rds_message_alloc()
351 spin_lock_init(&rm->m_rs_lock); in rds_message_alloc()
352 init_waitqueue_head(&rm->m_flush_wait); in rds_message_alloc()
355 return rm; in rds_message_alloc()
361 struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents) in rds_message_alloc_sgs() argument
363 struct scatterlist *sg_first = (struct scatterlist *) &rm[1]; in rds_message_alloc_sgs()
371 if (rm->m_used_sgs + nents > rm->m_total_sgs) { in rds_message_alloc_sgs()
373 rm->m_total_sgs, rm->m_used_sgs, nents); in rds_message_alloc_sgs()
377 sg_ret = &sg_first[rm->m_used_sgs]; in rds_message_alloc_sgs()
379 rm->m_used_sgs += nents; in rds_message_alloc_sgs()
386 struct rds_message *rm; in rds_message_map_pages() local
391 rm = rds_message_alloc(extra_bytes, GFP_NOWAIT); in rds_message_map_pages()
392 if (!rm) in rds_message_map_pages()
395 set_bit(RDS_MSG_PAGEVEC, &rm->m_flags); in rds_message_map_pages()
396 rm->m_inc.i_hdr.h_len = cpu_to_be32(total_len); in rds_message_map_pages()
397 rm->data.op_nents = DIV_ROUND_UP(total_len, PAGE_SIZE); in rds_message_map_pages()
398 rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs); in rds_message_map_pages()
399 if (IS_ERR(rm->data.op_sg)) { in rds_message_map_pages()
400 void *err = ERR_CAST(rm->data.op_sg); in rds_message_map_pages()
401 rds_message_put(rm); in rds_message_map_pages()
405 for (i = 0; i < rm->data.op_nents; ++i) { in rds_message_map_pages()
406 sg_set_page(&rm->data.op_sg[i], in rds_message_map_pages()
411 return rm; in rds_message_map_pages()
414 static int rds_message_zcopy_from_user(struct rds_message *rm, struct iov_iter *from) in rds_message_zcopy_from_user() argument
421 rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from)); in rds_message_zcopy_from_user()
426 sg = rm->data.op_sg; in rds_message_zcopy_from_user()
432 rm->data.op_mmp_znotifier = &info->znotif; in rds_message_zcopy_from_user()
433 if (mm_account_pinned_pages(&rm->data.op_mmp_znotifier->z_mmp, in rds_message_zcopy_from_user()
449 for (i = 0; i < rm->data.op_nents; i++) in rds_message_zcopy_from_user()
450 put_page(sg_page(&rm->data.op_sg[i])); in rds_message_zcopy_from_user()
451 mmp = &rm->data.op_mmp_znotifier->z_mmp; in rds_message_zcopy_from_user()
458 rm->data.op_nents++; in rds_message_zcopy_from_user()
465 rm->data.op_mmp_znotifier = NULL; in rds_message_zcopy_from_user()
469 int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from, in rds_message_copy_from_user() argument
477 rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from)); in rds_message_copy_from_user()
480 sg = rm->data.op_sg; in rds_message_copy_from_user()
484 return rds_message_zcopy_from_user(rm, from); in rds_message_copy_from_user()
492 rm->data.op_nents++; in rds_message_copy_from_user()
516 struct rds_message *rm; in rds_message_inc_copy_to_user() local
524 rm = container_of(inc, struct rds_message, m_inc); in rds_message_inc_copy_to_user()
525 len = be32_to_cpu(rm->m_inc.i_hdr.h_len); in rds_message_inc_copy_to_user()
527 sg = rm->data.op_sg; in rds_message_inc_copy_to_user()
558 void rds_message_wait(struct rds_message *rm) in rds_message_wait() argument
560 wait_event_interruptible(rm->m_flush_wait, in rds_message_wait()
561 !test_bit(RDS_MSG_MAPPED, &rm->m_flags)); in rds_message_wait()
564 void rds_message_unmapped(struct rds_message *rm) in rds_message_unmapped() argument
566 clear_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_message_unmapped()
567 wake_up_interruptible(&rm->m_flush_wait); in rds_message_unmapped()