xref: /linux/include/trace/events/rpcrdma.h (revision 42b16d3ac371a2fac9b6f08fd75f23f34ba3955a)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Copyright (c) 2017, 2018 Oracle.  All rights reserved.
4  *
5  * Trace point definitions for the "rpcrdma" subsystem.
6  */
7 #undef TRACE_SYSTEM
8 #define TRACE_SYSTEM rpcrdma
9 
10 #if !defined(_TRACE_RPCRDMA_H) || defined(TRACE_HEADER_MULTI_READ)
11 #define _TRACE_RPCRDMA_H
12 
13 #include <linux/scatterlist.h>
14 #include <linux/sunrpc/rpc_rdma_cid.h>
15 #include <linux/tracepoint.h>
16 #include <rdma/ib_cm.h>
17 
18 #include <trace/misc/rdma.h>
19 #include <trace/misc/sunrpc.h>
20 
21 /**
22  ** Event classes
23  **/
24 
25 DECLARE_EVENT_CLASS(rpcrdma_simple_cid_class,
26 	TP_PROTO(
27 		const struct rpc_rdma_cid *cid
28 	),
29 
30 	TP_ARGS(cid),
31 
32 	TP_STRUCT__entry(
33 		__field(u32, cq_id)
34 		__field(int, completion_id)
35 	),
36 
37 	TP_fast_assign(
38 		__entry->cq_id = cid->ci_queue_id;
39 		__entry->completion_id = cid->ci_completion_id;
40 	),
41 
42 	TP_printk("cq.id=%d cid=%d",
43 		__entry->cq_id, __entry->completion_id
44 	)
45 );
46 
47 #define DEFINE_SIMPLE_CID_EVENT(name)					\
48 		DEFINE_EVENT(rpcrdma_simple_cid_class, name,		\
49 				TP_PROTO(				\
50 					const struct rpc_rdma_cid *cid	\
51 				),					\
52 				TP_ARGS(cid)				\
53 		)
54 
55 DECLARE_EVENT_CLASS(rpcrdma_completion_class,
56 	TP_PROTO(
57 		const struct ib_wc *wc,
58 		const struct rpc_rdma_cid *cid
59 	),
60 
61 	TP_ARGS(wc, cid),
62 
63 	TP_STRUCT__entry(
64 		__field(u32, cq_id)
65 		__field(int, completion_id)
66 		__field(unsigned long, status)
67 		__field(unsigned int, vendor_err)
68 	),
69 
70 	TP_fast_assign(
71 		__entry->cq_id = cid->ci_queue_id;
72 		__entry->completion_id = cid->ci_completion_id;
73 		__entry->status = wc->status;
74 		if (wc->status)
75 			__entry->vendor_err = wc->vendor_err;
76 		else
77 			__entry->vendor_err = 0;
78 	),
79 
80 	TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x)",
81 		__entry->cq_id, __entry->completion_id,
82 		rdma_show_wc_status(__entry->status),
83 		__entry->status, __entry->vendor_err
84 	)
85 );
86 
87 #define DEFINE_COMPLETION_EVENT(name)					\
88 		DEFINE_EVENT(rpcrdma_completion_class, name,		\
89 				TP_PROTO(				\
90 					const struct ib_wc *wc,		\
91 					const struct rpc_rdma_cid *cid	\
92 				),					\
93 				TP_ARGS(wc, cid))
94 
95 DECLARE_EVENT_CLASS(rpcrdma_send_flush_class,
96 	TP_PROTO(
97 		const struct ib_wc *wc,
98 		const struct rpc_rdma_cid *cid
99 	),
100 
101 	TP_ARGS(wc, cid),
102 
103 	TP_STRUCT__entry(
104 		__field(u32, cq_id)
105 		__field(int, completion_id)
106 		__field(unsigned long, status)
107 		__field(unsigned int, vendor_err)
108 	),
109 
110 	TP_fast_assign(
111 		__entry->cq_id = cid->ci_queue_id;
112 		__entry->completion_id = cid->ci_completion_id;
113 		__entry->status = wc->status;
114 		__entry->vendor_err = wc->vendor_err;
115 	),
116 
117 	TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x)",
118 		__entry->cq_id, __entry->completion_id,
119 		rdma_show_wc_status(__entry->status),
120 		__entry->status, __entry->vendor_err
121 	)
122 );
123 
124 #define DEFINE_SEND_FLUSH_EVENT(name)					\
125 		DEFINE_EVENT(rpcrdma_send_flush_class, name,		\
126 				TP_PROTO(				\
127 					const struct ib_wc *wc,		\
128 					const struct rpc_rdma_cid *cid	\
129 				),					\
130 				TP_ARGS(wc, cid))
131 
132 DECLARE_EVENT_CLASS(rpcrdma_mr_completion_class,
133 	TP_PROTO(
134 		const struct ib_wc *wc,
135 		const struct rpc_rdma_cid *cid
136 	),
137 
138 	TP_ARGS(wc, cid),
139 
140 	TP_STRUCT__entry(
141 		__field(u32, cq_id)
142 		__field(int, completion_id)
143 		__field(unsigned long, status)
144 		__field(unsigned int, vendor_err)
145 	),
146 
147 	TP_fast_assign(
148 		__entry->cq_id = cid->ci_queue_id;
149 		__entry->completion_id = cid->ci_completion_id;
150 		__entry->status = wc->status;
151 		if (wc->status)
152 			__entry->vendor_err = wc->vendor_err;
153 		else
154 			__entry->vendor_err = 0;
155 	),
156 
157 	TP_printk("cq.id=%u mr.id=%d status=%s (%lu/0x%x)",
158 		__entry->cq_id, __entry->completion_id,
159 		rdma_show_wc_status(__entry->status),
160 		__entry->status, __entry->vendor_err
161 	)
162 );
163 
164 #define DEFINE_MR_COMPLETION_EVENT(name)				\
165 		DEFINE_EVENT(rpcrdma_mr_completion_class, name,		\
166 				TP_PROTO(				\
167 					const struct ib_wc *wc,		\
168 					const struct rpc_rdma_cid *cid	\
169 				),					\
170 				TP_ARGS(wc, cid))
171 
172 DECLARE_EVENT_CLASS(rpcrdma_receive_completion_class,
173 	TP_PROTO(
174 		const struct ib_wc *wc,
175 		const struct rpc_rdma_cid *cid
176 	),
177 
178 	TP_ARGS(wc, cid),
179 
180 	TP_STRUCT__entry(
181 		__field(u32, cq_id)
182 		__field(int, completion_id)
183 		__field(u32, received)
184 		__field(unsigned long, status)
185 		__field(unsigned int, vendor_err)
186 	),
187 
188 	TP_fast_assign(
189 		__entry->cq_id = cid->ci_queue_id;
190 		__entry->completion_id = cid->ci_completion_id;
191 		__entry->status = wc->status;
192 		if (wc->status) {
193 			__entry->received = 0;
194 			__entry->vendor_err = wc->vendor_err;
195 		} else {
196 			__entry->received = wc->byte_len;
197 			__entry->vendor_err = 0;
198 		}
199 	),
200 
201 	TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x) received=%u",
202 		__entry->cq_id, __entry->completion_id,
203 		rdma_show_wc_status(__entry->status),
204 		__entry->status, __entry->vendor_err,
205 		__entry->received
206 	)
207 );
208 
209 #define DEFINE_RECEIVE_COMPLETION_EVENT(name)				\
210 		DEFINE_EVENT(rpcrdma_receive_completion_class, name,	\
211 				TP_PROTO(				\
212 					const struct ib_wc *wc,		\
213 					const struct rpc_rdma_cid *cid	\
214 				),					\
215 				TP_ARGS(wc, cid))
216 
217 DECLARE_EVENT_CLASS(rpcrdma_receive_success_class,
218 	TP_PROTO(
219 		const struct ib_wc *wc,
220 		const struct rpc_rdma_cid *cid
221 	),
222 
223 	TP_ARGS(wc, cid),
224 
225 	TP_STRUCT__entry(
226 		__field(u32, cq_id)
227 		__field(int, completion_id)
228 		__field(u32, received)
229 	),
230 
231 	TP_fast_assign(
232 		__entry->cq_id = cid->ci_queue_id;
233 		__entry->completion_id = cid->ci_completion_id;
234 		__entry->received = wc->byte_len;
235 	),
236 
237 	TP_printk("cq.id=%u cid=%d received=%u",
238 		__entry->cq_id, __entry->completion_id,
239 		__entry->received
240 	)
241 );
242 
243 #define DEFINE_RECEIVE_SUCCESS_EVENT(name)				\
244 		DEFINE_EVENT(rpcrdma_receive_success_class, name,	\
245 				TP_PROTO(				\
246 					const struct ib_wc *wc,		\
247 					const struct rpc_rdma_cid *cid	\
248 				),					\
249 				TP_ARGS(wc, cid))
250 
251 DECLARE_EVENT_CLASS(rpcrdma_receive_flush_class,
252 	TP_PROTO(
253 		const struct ib_wc *wc,
254 		const struct rpc_rdma_cid *cid
255 	),
256 
257 	TP_ARGS(wc, cid),
258 
259 	TP_STRUCT__entry(
260 		__field(u32, cq_id)
261 		__field(int, completion_id)
262 		__field(unsigned long, status)
263 		__field(unsigned int, vendor_err)
264 	),
265 
266 	TP_fast_assign(
267 		__entry->cq_id = cid->ci_queue_id;
268 		__entry->completion_id = cid->ci_completion_id;
269 		__entry->status = wc->status;
270 		__entry->vendor_err = wc->vendor_err;
271 	),
272 
273 	TP_printk("cq.id=%u cid=%d status=%s (%lu/0x%x)",
274 		__entry->cq_id, __entry->completion_id,
275 		rdma_show_wc_status(__entry->status),
276 		__entry->status, __entry->vendor_err
277 	)
278 );
279 
280 #define DEFINE_RECEIVE_FLUSH_EVENT(name)				\
281 		DEFINE_EVENT(rpcrdma_receive_flush_class, name,		\
282 				TP_PROTO(				\
283 					const struct ib_wc *wc,		\
284 					const struct rpc_rdma_cid *cid	\
285 				),					\
286 				TP_ARGS(wc, cid))
287 
288 DECLARE_EVENT_CLASS(xprtrdma_reply_class,
289 	TP_PROTO(
290 		const struct rpcrdma_rep *rep
291 	),
292 
293 	TP_ARGS(rep),
294 
295 	TP_STRUCT__entry(
296 		__field(u32, xid)
297 		__field(u32, version)
298 		__field(u32, proc)
299 		__string(addr, rpcrdma_addrstr(rep->rr_rxprt))
300 		__string(port, rpcrdma_portstr(rep->rr_rxprt))
301 	),
302 
303 	TP_fast_assign(
304 		__entry->xid = be32_to_cpu(rep->rr_xid);
305 		__entry->version = be32_to_cpu(rep->rr_vers);
306 		__entry->proc = be32_to_cpu(rep->rr_proc);
307 		__assign_str(addr);
308 		__assign_str(port);
309 	),
310 
311 	TP_printk("peer=[%s]:%s xid=0x%08x version=%u proc=%u",
312 		__get_str(addr), __get_str(port),
313 		__entry->xid, __entry->version, __entry->proc
314 	)
315 );
316 
317 #define DEFINE_REPLY_EVENT(name)					\
318 		DEFINE_EVENT(xprtrdma_reply_class,			\
319 				xprtrdma_reply_##name##_err,		\
320 				TP_PROTO(				\
321 					const struct rpcrdma_rep *rep	\
322 				),					\
323 				TP_ARGS(rep))
324 
325 DECLARE_EVENT_CLASS(xprtrdma_rxprt,
326 	TP_PROTO(
327 		const struct rpcrdma_xprt *r_xprt
328 	),
329 
330 	TP_ARGS(r_xprt),
331 
332 	TP_STRUCT__entry(
333 		__string(addr, rpcrdma_addrstr(r_xprt))
334 		__string(port, rpcrdma_portstr(r_xprt))
335 	),
336 
337 	TP_fast_assign(
338 		__assign_str(addr);
339 		__assign_str(port);
340 	),
341 
342 	TP_printk("peer=[%s]:%s",
343 		__get_str(addr), __get_str(port)
344 	)
345 );
346 
347 #define DEFINE_RXPRT_EVENT(name)					\
348 		DEFINE_EVENT(xprtrdma_rxprt, name,			\
349 				TP_PROTO(				\
350 					const struct rpcrdma_xprt *r_xprt \
351 				),					\
352 				TP_ARGS(r_xprt))
353 
354 DECLARE_EVENT_CLASS(xprtrdma_connect_class,
355 	TP_PROTO(
356 		const struct rpcrdma_xprt *r_xprt,
357 		int rc
358 	),
359 
360 	TP_ARGS(r_xprt, rc),
361 
362 	TP_STRUCT__entry(
363 		__field(int, rc)
364 		__field(int, connect_status)
365 		__string(addr, rpcrdma_addrstr(r_xprt))
366 		__string(port, rpcrdma_portstr(r_xprt))
367 	),
368 
369 	TP_fast_assign(
370 		__entry->rc = rc;
371 		__entry->connect_status = r_xprt->rx_ep->re_connect_status;
372 		__assign_str(addr);
373 		__assign_str(port);
374 	),
375 
376 	TP_printk("peer=[%s]:%s rc=%d connection status=%d",
377 		__get_str(addr), __get_str(port),
378 		__entry->rc, __entry->connect_status
379 	)
380 );
381 
382 #define DEFINE_CONN_EVENT(name)						\
383 		DEFINE_EVENT(xprtrdma_connect_class, xprtrdma_##name,	\
384 				TP_PROTO(				\
385 					const struct rpcrdma_xprt *r_xprt, \
386 					int rc				\
387 				),					\
388 				TP_ARGS(r_xprt, rc))
389 
390 DECLARE_EVENT_CLASS(xprtrdma_rdch_event,
391 	TP_PROTO(
392 		const struct rpc_task *task,
393 		unsigned int pos,
394 		struct rpcrdma_mr *mr,
395 		int nsegs
396 	),
397 
398 	TP_ARGS(task, pos, mr, nsegs),
399 
400 	TP_STRUCT__entry(
401 		__field(unsigned int, task_id)
402 		__field(unsigned int, client_id)
403 		__field(unsigned int, pos)
404 		__field(int, nents)
405 		__field(u32, handle)
406 		__field(u32, length)
407 		__field(u64, offset)
408 		__field(int, nsegs)
409 	),
410 
411 	TP_fast_assign(
412 		__entry->task_id = task->tk_pid;
413 		__entry->client_id = task->tk_client->cl_clid;
414 		__entry->pos = pos;
415 		__entry->nents = mr->mr_nents;
416 		__entry->handle = mr->mr_handle;
417 		__entry->length = mr->mr_length;
418 		__entry->offset = mr->mr_offset;
419 		__entry->nsegs = nsegs;
420 	),
421 
422 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
423 		  " pos=%u %u@0x%016llx:0x%08x (%s)",
424 		__entry->task_id, __entry->client_id,
425 		__entry->pos, __entry->length,
426 		(unsigned long long)__entry->offset, __entry->handle,
427 		__entry->nents < __entry->nsegs ? "more" : "last"
428 	)
429 );
430 
431 #define DEFINE_RDCH_EVENT(name)						\
432 		DEFINE_EVENT(xprtrdma_rdch_event, xprtrdma_chunk_##name,\
433 				TP_PROTO(				\
434 					const struct rpc_task *task,	\
435 					unsigned int pos,		\
436 					struct rpcrdma_mr *mr,		\
437 					int nsegs			\
438 				),					\
439 				TP_ARGS(task, pos, mr, nsegs))
440 
441 DECLARE_EVENT_CLASS(xprtrdma_wrch_event,
442 	TP_PROTO(
443 		const struct rpc_task *task,
444 		struct rpcrdma_mr *mr,
445 		int nsegs
446 	),
447 
448 	TP_ARGS(task, mr, nsegs),
449 
450 	TP_STRUCT__entry(
451 		__field(unsigned int, task_id)
452 		__field(unsigned int, client_id)
453 		__field(int, nents)
454 		__field(u32, handle)
455 		__field(u32, length)
456 		__field(u64, offset)
457 		__field(int, nsegs)
458 	),
459 
460 	TP_fast_assign(
461 		__entry->task_id = task->tk_pid;
462 		__entry->client_id = task->tk_client->cl_clid;
463 		__entry->nents = mr->mr_nents;
464 		__entry->handle = mr->mr_handle;
465 		__entry->length = mr->mr_length;
466 		__entry->offset = mr->mr_offset;
467 		__entry->nsegs = nsegs;
468 	),
469 
470 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
471 		  " %u@0x%016llx:0x%08x (%s)",
472 		__entry->task_id, __entry->client_id,
473 		__entry->length, (unsigned long long)__entry->offset,
474 		__entry->handle,
475 		__entry->nents < __entry->nsegs ? "more" : "last"
476 	)
477 );
478 
479 #define DEFINE_WRCH_EVENT(name)						\
480 		DEFINE_EVENT(xprtrdma_wrch_event, xprtrdma_chunk_##name,\
481 				TP_PROTO(				\
482 					const struct rpc_task *task,	\
483 					struct rpcrdma_mr *mr,		\
484 					int nsegs			\
485 				),					\
486 				TP_ARGS(task, mr, nsegs))
487 
488 TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
489 TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
490 TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
491 TRACE_DEFINE_ENUM(DMA_NONE);
492 
493 #define xprtrdma_show_direction(x)					\
494 		__print_symbolic(x,					\
495 				{ DMA_BIDIRECTIONAL, "BIDIR" },		\
496 				{ DMA_TO_DEVICE, "TO_DEVICE" },		\
497 				{ DMA_FROM_DEVICE, "FROM_DEVICE" },	\
498 				{ DMA_NONE, "NONE" })
499 
500 DECLARE_EVENT_CLASS(xprtrdma_mr_class,
501 	TP_PROTO(
502 		const struct rpcrdma_mr *mr
503 	),
504 
505 	TP_ARGS(mr),
506 
507 	TP_STRUCT__entry(
508 		__field(unsigned int, task_id)
509 		__field(unsigned int, client_id)
510 		__field(u32, mr_id)
511 		__field(int, nents)
512 		__field(u32, handle)
513 		__field(u32, length)
514 		__field(u64, offset)
515 		__field(u32, dir)
516 	),
517 
518 	TP_fast_assign(
519 		const struct rpcrdma_req *req = mr->mr_req;
520 
521 		if (req) {
522 			const struct rpc_task *task = req->rl_slot.rq_task;
523 
524 			__entry->task_id = task->tk_pid;
525 			__entry->client_id = task->tk_client->cl_clid;
526 		} else {
527 			__entry->task_id = 0;
528 			__entry->client_id = -1;
529 		}
530 		__entry->mr_id  = mr->mr_ibmr->res.id;
531 		__entry->nents  = mr->mr_nents;
532 		__entry->handle = mr->mr_handle;
533 		__entry->length = mr->mr_length;
534 		__entry->offset = mr->mr_offset;
535 		__entry->dir    = mr->mr_dir;
536 	),
537 
538 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
539 		  " mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
540 		__entry->task_id, __entry->client_id,
541 		__entry->mr_id, __entry->nents, __entry->length,
542 		(unsigned long long)__entry->offset, __entry->handle,
543 		xprtrdma_show_direction(__entry->dir)
544 	)
545 );
546 
547 #define DEFINE_MR_EVENT(name)						\
548 		DEFINE_EVENT(xprtrdma_mr_class,				\
549 				xprtrdma_mr_##name,			\
550 				TP_PROTO(				\
551 					const struct rpcrdma_mr *mr	\
552 				),					\
553 				TP_ARGS(mr))
554 
555 DECLARE_EVENT_CLASS(xprtrdma_anonymous_mr_class,
556 	TP_PROTO(
557 		const struct rpcrdma_mr *mr
558 	),
559 
560 	TP_ARGS(mr),
561 
562 	TP_STRUCT__entry(
563 		__field(u32, mr_id)
564 		__field(int, nents)
565 		__field(u32, handle)
566 		__field(u32, length)
567 		__field(u64, offset)
568 		__field(u32, dir)
569 	),
570 
571 	TP_fast_assign(
572 		__entry->mr_id  = mr->mr_ibmr->res.id;
573 		__entry->nents  = mr->mr_nents;
574 		__entry->handle = mr->mr_handle;
575 		__entry->length = mr->mr_length;
576 		__entry->offset = mr->mr_offset;
577 		__entry->dir    = mr->mr_dir;
578 	),
579 
580 	TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
581 		__entry->mr_id, __entry->nents, __entry->length,
582 		(unsigned long long)__entry->offset, __entry->handle,
583 		xprtrdma_show_direction(__entry->dir)
584 	)
585 );
586 
587 #define DEFINE_ANON_MR_EVENT(name)					\
588 		DEFINE_EVENT(xprtrdma_anonymous_mr_class,		\
589 				xprtrdma_mr_##name,			\
590 				TP_PROTO(				\
591 					const struct rpcrdma_mr *mr	\
592 				),					\
593 				TP_ARGS(mr))
594 
595 DECLARE_EVENT_CLASS(xprtrdma_callback_class,
596 	TP_PROTO(
597 		const struct rpcrdma_xprt *r_xprt,
598 		const struct rpc_rqst *rqst
599 	),
600 
601 	TP_ARGS(r_xprt, rqst),
602 
603 	TP_STRUCT__entry(
604 		__field(u32, xid)
605 		__string(addr, rpcrdma_addrstr(r_xprt))
606 		__string(port, rpcrdma_portstr(r_xprt))
607 	),
608 
609 	TP_fast_assign(
610 		__entry->xid = be32_to_cpu(rqst->rq_xid);
611 		__assign_str(addr);
612 		__assign_str(port);
613 	),
614 
615 	TP_printk("peer=[%s]:%s xid=0x%08x",
616 		__get_str(addr), __get_str(port), __entry->xid
617 	)
618 );
619 
620 #define DEFINE_CALLBACK_EVENT(name)					\
621 		DEFINE_EVENT(xprtrdma_callback_class,			\
622 				xprtrdma_cb_##name,			\
623 				TP_PROTO(				\
624 					const struct rpcrdma_xprt *r_xprt, \
625 					const struct rpc_rqst *rqst	\
626 				),					\
627 				TP_ARGS(r_xprt, rqst))
628 
629 /**
630  ** Connection events
631  **/
632 
633 TRACE_EVENT(xprtrdma_inline_thresh,
634 	TP_PROTO(
635 		const struct rpcrdma_ep *ep
636 	),
637 
638 	TP_ARGS(ep),
639 
640 	TP_STRUCT__entry(
641 		__field(unsigned int, inline_send)
642 		__field(unsigned int, inline_recv)
643 		__field(unsigned int, max_send)
644 		__field(unsigned int, max_recv)
645 		__array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
646 		__array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
647 	),
648 
649 	TP_fast_assign(
650 		const struct rdma_cm_id *id = ep->re_id;
651 
652 		__entry->inline_send = ep->re_inline_send;
653 		__entry->inline_recv = ep->re_inline_recv;
654 		__entry->max_send = ep->re_max_inline_send;
655 		__entry->max_recv = ep->re_max_inline_recv;
656 		memcpy(__entry->srcaddr, &id->route.addr.src_addr,
657 		       sizeof(struct sockaddr_in6));
658 		memcpy(__entry->dstaddr, &id->route.addr.dst_addr,
659 		       sizeof(struct sockaddr_in6));
660 	),
661 
662 	TP_printk("%pISpc -> %pISpc neg send/recv=%u/%u, calc send/recv=%u/%u",
663 		__entry->srcaddr, __entry->dstaddr,
664 		__entry->inline_send, __entry->inline_recv,
665 		__entry->max_send, __entry->max_recv
666 	)
667 );
668 
669 DEFINE_CONN_EVENT(connect);
670 DEFINE_CONN_EVENT(disconnect);
671 
672 TRACE_EVENT(xprtrdma_device_removal,
673 	TP_PROTO(
674 		const struct rdma_cm_id *id
675 	),
676 
677 	TP_ARGS(id),
678 
679 	TP_STRUCT__entry(
680 		__string(name, id->device->name)
681 		__array(unsigned char, addr, sizeof(struct sockaddr_in6))
682 	),
683 
684 	TP_fast_assign(
685 		__assign_str(name);
686 		memcpy(__entry->addr, &id->route.addr.dst_addr,
687 		       sizeof(struct sockaddr_in6));
688 	),
689 
690 	TP_printk("device %s to be removed, disconnecting %pISpc\n",
691 		__get_str(name), __entry->addr
692 	)
693 );
694 
695 DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
696 
697 TRACE_EVENT(xprtrdma_op_connect,
698 	TP_PROTO(
699 		const struct rpcrdma_xprt *r_xprt,
700 		unsigned long delay
701 	),
702 
703 	TP_ARGS(r_xprt, delay),
704 
705 	TP_STRUCT__entry(
706 		__field(unsigned long, delay)
707 		__string(addr, rpcrdma_addrstr(r_xprt))
708 		__string(port, rpcrdma_portstr(r_xprt))
709 	),
710 
711 	TP_fast_assign(
712 		__entry->delay = delay;
713 		__assign_str(addr);
714 		__assign_str(port);
715 	),
716 
717 	TP_printk("peer=[%s]:%s delay=%lu",
718 		__get_str(addr), __get_str(port), __entry->delay
719 	)
720 );
721 
722 
723 TRACE_EVENT(xprtrdma_op_set_cto,
724 	TP_PROTO(
725 		const struct rpcrdma_xprt *r_xprt,
726 		unsigned long connect,
727 		unsigned long reconnect
728 	),
729 
730 	TP_ARGS(r_xprt, connect, reconnect),
731 
732 	TP_STRUCT__entry(
733 		__field(unsigned long, connect)
734 		__field(unsigned long, reconnect)
735 		__string(addr, rpcrdma_addrstr(r_xprt))
736 		__string(port, rpcrdma_portstr(r_xprt))
737 	),
738 
739 	TP_fast_assign(
740 		__entry->connect = connect;
741 		__entry->reconnect = reconnect;
742 		__assign_str(addr);
743 		__assign_str(port);
744 	),
745 
746 	TP_printk("peer=[%s]:%s connect=%lu reconnect=%lu",
747 		__get_str(addr), __get_str(port),
748 		__entry->connect / HZ, __entry->reconnect / HZ
749 	)
750 );
751 
752 /**
753  ** Call events
754  **/
755 
756 TRACE_EVENT(xprtrdma_createmrs,
757 	TP_PROTO(
758 		const struct rpcrdma_xprt *r_xprt,
759 		unsigned int count
760 	),
761 
762 	TP_ARGS(r_xprt, count),
763 
764 	TP_STRUCT__entry(
765 		__string(addr, rpcrdma_addrstr(r_xprt))
766 		__string(port, rpcrdma_portstr(r_xprt))
767 		__field(unsigned int, count)
768 	),
769 
770 	TP_fast_assign(
771 		__entry->count = count;
772 		__assign_str(addr);
773 		__assign_str(port);
774 	),
775 
776 	TP_printk("peer=[%s]:%s created %u MRs",
777 		__get_str(addr), __get_str(port), __entry->count
778 	)
779 );
780 
781 TRACE_EVENT(xprtrdma_nomrs_err,
782 	TP_PROTO(
783 		const struct rpcrdma_xprt *r_xprt,
784 		const struct rpcrdma_req *req
785 	),
786 
787 	TP_ARGS(r_xprt, req),
788 
789 	TP_STRUCT__entry(
790 		__field(unsigned int, task_id)
791 		__field(unsigned int, client_id)
792 		__string(addr, rpcrdma_addrstr(r_xprt))
793 		__string(port, rpcrdma_portstr(r_xprt))
794 	),
795 
796 	TP_fast_assign(
797 		const struct rpc_rqst *rqst = &req->rl_slot;
798 
799 		__entry->task_id = rqst->rq_task->tk_pid;
800 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
801 		__assign_str(addr);
802 		__assign_str(port);
803 	),
804 
805 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " peer=[%s]:%s",
806 		__entry->task_id, __entry->client_id,
807 		__get_str(addr), __get_str(port)
808 	)
809 );
810 
811 DEFINE_RDCH_EVENT(read);
812 DEFINE_WRCH_EVENT(write);
813 DEFINE_WRCH_EVENT(reply);
814 DEFINE_WRCH_EVENT(wp);
815 
816 TRACE_DEFINE_ENUM(rpcrdma_noch);
817 TRACE_DEFINE_ENUM(rpcrdma_noch_pullup);
818 TRACE_DEFINE_ENUM(rpcrdma_noch_mapped);
819 TRACE_DEFINE_ENUM(rpcrdma_readch);
820 TRACE_DEFINE_ENUM(rpcrdma_areadch);
821 TRACE_DEFINE_ENUM(rpcrdma_writech);
822 TRACE_DEFINE_ENUM(rpcrdma_replych);
823 
824 #define xprtrdma_show_chunktype(x)					\
825 		__print_symbolic(x,					\
826 				{ rpcrdma_noch, "inline" },		\
827 				{ rpcrdma_noch_pullup, "pullup" },	\
828 				{ rpcrdma_noch_mapped, "mapped" },	\
829 				{ rpcrdma_readch, "read list" },	\
830 				{ rpcrdma_areadch, "*read list" },	\
831 				{ rpcrdma_writech, "write list" },	\
832 				{ rpcrdma_replych, "reply chunk" })
833 
834 TRACE_EVENT(xprtrdma_marshal,
835 	TP_PROTO(
836 		const struct rpcrdma_req *req,
837 		unsigned int rtype,
838 		unsigned int wtype
839 	),
840 
841 	TP_ARGS(req, rtype, wtype),
842 
843 	TP_STRUCT__entry(
844 		__field(unsigned int, task_id)
845 		__field(unsigned int, client_id)
846 		__field(u32, xid)
847 		__field(unsigned int, hdrlen)
848 		__field(unsigned int, headlen)
849 		__field(unsigned int, pagelen)
850 		__field(unsigned int, taillen)
851 		__field(unsigned int, rtype)
852 		__field(unsigned int, wtype)
853 	),
854 
855 	TP_fast_assign(
856 		const struct rpc_rqst *rqst = &req->rl_slot;
857 
858 		__entry->task_id = rqst->rq_task->tk_pid;
859 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
860 		__entry->xid = be32_to_cpu(rqst->rq_xid);
861 		__entry->hdrlen = req->rl_hdrbuf.len;
862 		__entry->headlen = rqst->rq_snd_buf.head[0].iov_len;
863 		__entry->pagelen = rqst->rq_snd_buf.page_len;
864 		__entry->taillen = rqst->rq_snd_buf.tail[0].iov_len;
865 		__entry->rtype = rtype;
866 		__entry->wtype = wtype;
867 	),
868 
869 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER
870 		  " xid=0x%08x hdr=%u xdr=%u/%u/%u %s/%s",
871 		__entry->task_id, __entry->client_id, __entry->xid,
872 		__entry->hdrlen,
873 		__entry->headlen, __entry->pagelen, __entry->taillen,
874 		xprtrdma_show_chunktype(__entry->rtype),
875 		xprtrdma_show_chunktype(__entry->wtype)
876 	)
877 );
878 
879 TRACE_EVENT(xprtrdma_marshal_failed,
880 	TP_PROTO(const struct rpc_rqst *rqst,
881 		 int ret
882 	),
883 
884 	TP_ARGS(rqst, ret),
885 
886 	TP_STRUCT__entry(
887 		__field(unsigned int, task_id)
888 		__field(unsigned int, client_id)
889 		__field(u32, xid)
890 		__field(int, ret)
891 	),
892 
893 	TP_fast_assign(
894 		__entry->task_id = rqst->rq_task->tk_pid;
895 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
896 		__entry->xid = be32_to_cpu(rqst->rq_xid);
897 		__entry->ret = ret;
898 	),
899 
900 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x ret=%d",
901 		__entry->task_id, __entry->client_id, __entry->xid,
902 		__entry->ret
903 	)
904 );
905 
906 TRACE_EVENT(xprtrdma_prepsend_failed,
907 	TP_PROTO(const struct rpc_rqst *rqst,
908 		 int ret
909 	),
910 
911 	TP_ARGS(rqst, ret),
912 
913 	TP_STRUCT__entry(
914 		__field(unsigned int, task_id)
915 		__field(unsigned int, client_id)
916 		__field(u32, xid)
917 		__field(int, ret)
918 	),
919 
920 	TP_fast_assign(
921 		__entry->task_id = rqst->rq_task->tk_pid;
922 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
923 		__entry->xid = be32_to_cpu(rqst->rq_xid);
924 		__entry->ret = ret;
925 	),
926 
927 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x ret=%d",
928 		__entry->task_id, __entry->client_id, __entry->xid,
929 		__entry->ret
930 	)
931 );
932 
933 TRACE_EVENT(xprtrdma_post_send,
934 	TP_PROTO(
935 		const struct rpcrdma_req *req
936 	),
937 
938 	TP_ARGS(req),
939 
940 	TP_STRUCT__entry(
941 		__field(u32, cq_id)
942 		__field(int, completion_id)
943 		__field(unsigned int, task_id)
944 		__field(unsigned int, client_id)
945 		__field(int, num_sge)
946 		__field(int, signaled)
947 	),
948 
949 	TP_fast_assign(
950 		const struct rpc_rqst *rqst = &req->rl_slot;
951 		const struct rpcrdma_sendctx *sc = req->rl_sendctx;
952 
953 		__entry->cq_id = sc->sc_cid.ci_queue_id;
954 		__entry->completion_id = sc->sc_cid.ci_completion_id;
955 		__entry->task_id = rqst->rq_task->tk_pid;
956 		__entry->client_id = rqst->rq_task->tk_client ?
957 				     rqst->rq_task->tk_client->cl_clid : -1;
958 		__entry->num_sge = req->rl_wr.num_sge;
959 		__entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;
960 	),
961 
962 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " cq.id=%u cid=%d (%d SGE%s) %s",
963 		__entry->task_id, __entry->client_id,
964 		__entry->cq_id, __entry->completion_id,
965 		__entry->num_sge, (__entry->num_sge == 1 ? "" : "s"),
966 		(__entry->signaled ? "signaled" : "")
967 	)
968 );
969 
970 TRACE_EVENT(xprtrdma_post_send_err,
971 	TP_PROTO(
972 		const struct rpcrdma_xprt *r_xprt,
973 		const struct rpcrdma_req *req,
974 		int rc
975 	),
976 
977 	TP_ARGS(r_xprt, req, rc),
978 
979 	TP_STRUCT__entry(
980 		__field(u32, cq_id)
981 		__field(unsigned int, task_id)
982 		__field(unsigned int, client_id)
983 		__field(int, rc)
984 	),
985 
986 	TP_fast_assign(
987 		const struct rpc_rqst *rqst = &req->rl_slot;
988 		const struct rpcrdma_ep *ep = r_xprt->rx_ep;
989 
990 		__entry->cq_id = ep ? ep->re_attr.recv_cq->res.id : 0;
991 		__entry->task_id = rqst->rq_task->tk_pid;
992 		__entry->client_id = rqst->rq_task->tk_client ?
993 				     rqst->rq_task->tk_client->cl_clid : -1;
994 		__entry->rc = rc;
995 	),
996 
997 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " cq.id=%u rc=%d",
998 		__entry->task_id, __entry->client_id,
999 		__entry->cq_id, __entry->rc
1000 	)
1001 );
1002 
1003 DEFINE_SIMPLE_CID_EVENT(xprtrdma_post_recv);
1004 
1005 TRACE_EVENT(xprtrdma_post_recvs,
1006 	TP_PROTO(
1007 		const struct rpcrdma_xprt *r_xprt,
1008 		unsigned int count
1009 	),
1010 
1011 	TP_ARGS(r_xprt, count),
1012 
1013 	TP_STRUCT__entry(
1014 		__field(u32, cq_id)
1015 		__field(unsigned int, count)
1016 		__field(int, posted)
1017 		__string(addr, rpcrdma_addrstr(r_xprt))
1018 		__string(port, rpcrdma_portstr(r_xprt))
1019 	),
1020 
1021 	TP_fast_assign(
1022 		const struct rpcrdma_ep *ep = r_xprt->rx_ep;
1023 
1024 		__entry->cq_id = ep->re_attr.recv_cq->res.id;
1025 		__entry->count = count;
1026 		__entry->posted = ep->re_receive_count;
1027 		__assign_str(addr);
1028 		__assign_str(port);
1029 	),
1030 
1031 	TP_printk("peer=[%s]:%s cq.id=%d %u new recvs, %d active",
1032 		__get_str(addr), __get_str(port), __entry->cq_id,
1033 		__entry->count, __entry->posted
1034 	)
1035 );
1036 
1037 TRACE_EVENT(xprtrdma_post_recvs_err,
1038 	TP_PROTO(
1039 		const struct rpcrdma_xprt *r_xprt,
1040 		int status
1041 	),
1042 
1043 	TP_ARGS(r_xprt, status),
1044 
1045 	TP_STRUCT__entry(
1046 		__field(u32, cq_id)
1047 		__field(int, status)
1048 		__string(addr, rpcrdma_addrstr(r_xprt))
1049 		__string(port, rpcrdma_portstr(r_xprt))
1050 	),
1051 
1052 	TP_fast_assign(
1053 		const struct rpcrdma_ep *ep = r_xprt->rx_ep;
1054 
1055 		__entry->cq_id = ep->re_attr.recv_cq->res.id;
1056 		__entry->status = status;
1057 		__assign_str(addr);
1058 		__assign_str(port);
1059 	),
1060 
1061 	TP_printk("peer=[%s]:%s cq.id=%d rc=%d",
1062 		__get_str(addr), __get_str(port), __entry->cq_id,
1063 		__entry->status
1064 	)
1065 );
1066 
1067 TRACE_EVENT(xprtrdma_post_linv_err,
1068 	TP_PROTO(
1069 		const struct rpcrdma_req *req,
1070 		int status
1071 	),
1072 
1073 	TP_ARGS(req, status),
1074 
1075 	TP_STRUCT__entry(
1076 		__field(unsigned int, task_id)
1077 		__field(unsigned int, client_id)
1078 		__field(int, status)
1079 	),
1080 
1081 	TP_fast_assign(
1082 		const struct rpc_task *task = req->rl_slot.rq_task;
1083 
1084 		__entry->task_id = task->tk_pid;
1085 		__entry->client_id = task->tk_client->cl_clid;
1086 		__entry->status = status;
1087 	),
1088 
1089 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " status=%d",
1090 		__entry->task_id, __entry->client_id, __entry->status
1091 	)
1092 );
1093 
1094 /**
1095  ** Completion events
1096  **/
1097 
1098 DEFINE_RECEIVE_COMPLETION_EVENT(xprtrdma_wc_receive);
1099 
1100 DEFINE_COMPLETION_EVENT(xprtrdma_wc_send);
1101 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_fastreg);
1102 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_li);
1103 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_li_wake);
1104 DEFINE_MR_COMPLETION_EVENT(xprtrdma_wc_li_done);
1105 
1106 TRACE_EVENT(xprtrdma_frwr_alloc,
1107 	TP_PROTO(
1108 		const struct rpcrdma_mr *mr,
1109 		int rc
1110 	),
1111 
1112 	TP_ARGS(mr, rc),
1113 
1114 	TP_STRUCT__entry(
1115 		__field(u32, mr_id)
1116 		__field(int, rc)
1117 	),
1118 
1119 	TP_fast_assign(
1120 		__entry->mr_id = mr->mr_ibmr->res.id;
1121 		__entry->rc = rc;
1122 	),
1123 
1124 	TP_printk("mr.id=%u: rc=%d",
1125 		__entry->mr_id, __entry->rc
1126 	)
1127 );
1128 
1129 TRACE_EVENT(xprtrdma_frwr_dereg,
1130 	TP_PROTO(
1131 		const struct rpcrdma_mr *mr,
1132 		int rc
1133 	),
1134 
1135 	TP_ARGS(mr, rc),
1136 
1137 	TP_STRUCT__entry(
1138 		__field(u32, mr_id)
1139 		__field(int, nents)
1140 		__field(u32, handle)
1141 		__field(u32, length)
1142 		__field(u64, offset)
1143 		__field(u32, dir)
1144 		__field(int, rc)
1145 	),
1146 
1147 	TP_fast_assign(
1148 		__entry->mr_id  = mr->mr_ibmr->res.id;
1149 		__entry->nents  = mr->mr_nents;
1150 		__entry->handle = mr->mr_handle;
1151 		__entry->length = mr->mr_length;
1152 		__entry->offset = mr->mr_offset;
1153 		__entry->dir    = mr->mr_dir;
1154 		__entry->rc	= rc;
1155 	),
1156 
1157 	TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s): rc=%d",
1158 		__entry->mr_id, __entry->nents, __entry->length,
1159 		(unsigned long long)__entry->offset, __entry->handle,
1160 		xprtrdma_show_direction(__entry->dir),
1161 		__entry->rc
1162 	)
1163 );
1164 
1165 TRACE_EVENT(xprtrdma_frwr_sgerr,
1166 	TP_PROTO(
1167 		const struct rpcrdma_mr *mr,
1168 		int sg_nents
1169 	),
1170 
1171 	TP_ARGS(mr, sg_nents),
1172 
1173 	TP_STRUCT__entry(
1174 		__field(u32, mr_id)
1175 		__field(u64, addr)
1176 		__field(u32, dir)
1177 		__field(int, nents)
1178 	),
1179 
1180 	TP_fast_assign(
1181 		__entry->mr_id = mr->mr_ibmr->res.id;
1182 		__entry->addr = mr->mr_sg->dma_address;
1183 		__entry->dir = mr->mr_dir;
1184 		__entry->nents = sg_nents;
1185 	),
1186 
1187 	TP_printk("mr.id=%u DMA addr=0x%llx (%s) sg_nents=%d",
1188 		__entry->mr_id, __entry->addr,
1189 		xprtrdma_show_direction(__entry->dir),
1190 		__entry->nents
1191 	)
1192 );
1193 
1194 TRACE_EVENT(xprtrdma_frwr_maperr,
1195 	TP_PROTO(
1196 		const struct rpcrdma_mr *mr,
1197 		int num_mapped
1198 	),
1199 
1200 	TP_ARGS(mr, num_mapped),
1201 
1202 	TP_STRUCT__entry(
1203 		__field(u32, mr_id)
1204 		__field(u64, addr)
1205 		__field(u32, dir)
1206 		__field(int, num_mapped)
1207 		__field(int, nents)
1208 	),
1209 
1210 	TP_fast_assign(
1211 		__entry->mr_id = mr->mr_ibmr->res.id;
1212 		__entry->addr = mr->mr_sg->dma_address;
1213 		__entry->dir = mr->mr_dir;
1214 		__entry->num_mapped = num_mapped;
1215 		__entry->nents = mr->mr_nents;
1216 	),
1217 
1218 	TP_printk("mr.id=%u DMA addr=0x%llx (%s) nents=%d of %d",
1219 		__entry->mr_id, __entry->addr,
1220 		xprtrdma_show_direction(__entry->dir),
1221 		__entry->num_mapped, __entry->nents
1222 	)
1223 );
1224 
1225 DEFINE_MR_EVENT(fastreg);
1226 DEFINE_MR_EVENT(localinv);
1227 DEFINE_MR_EVENT(reminv);
1228 DEFINE_MR_EVENT(map);
1229 
1230 DEFINE_ANON_MR_EVENT(unmap);
1231 
1232 TRACE_EVENT(xprtrdma_dma_maperr,
1233 	TP_PROTO(
1234 		u64 addr
1235 	),
1236 
1237 	TP_ARGS(addr),
1238 
1239 	TP_STRUCT__entry(
1240 		__field(u64, addr)
1241 	),
1242 
1243 	TP_fast_assign(
1244 		__entry->addr = addr;
1245 	),
1246 
1247 	TP_printk("dma addr=0x%llx\n", __entry->addr)
1248 );
1249 
1250 /**
1251  ** Reply events
1252  **/
1253 
1254 TRACE_EVENT(xprtrdma_reply,
1255 	TP_PROTO(
1256 		const struct rpc_task *task,
1257 		const struct rpcrdma_rep *rep,
1258 		unsigned int credits
1259 	),
1260 
1261 	TP_ARGS(task, rep, credits),
1262 
1263 	TP_STRUCT__entry(
1264 		__field(unsigned int, task_id)
1265 		__field(unsigned int, client_id)
1266 		__field(u32, xid)
1267 		__field(unsigned int, credits)
1268 	),
1269 
1270 	TP_fast_assign(
1271 		__entry->task_id = task->tk_pid;
1272 		__entry->client_id = task->tk_client->cl_clid;
1273 		__entry->xid = be32_to_cpu(rep->rr_xid);
1274 		__entry->credits = credits;
1275 	),
1276 
1277 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x credits=%u",
1278 		__entry->task_id, __entry->client_id, __entry->xid,
1279 		__entry->credits
1280 	)
1281 );
1282 
1283 DEFINE_REPLY_EVENT(vers);
1284 DEFINE_REPLY_EVENT(rqst);
1285 DEFINE_REPLY_EVENT(short);
1286 DEFINE_REPLY_EVENT(hdr);
1287 
1288 TRACE_EVENT(xprtrdma_err_vers,
1289 	TP_PROTO(
1290 		const struct rpc_rqst *rqst,
1291 		__be32 *min,
1292 		__be32 *max
1293 	),
1294 
1295 	TP_ARGS(rqst, min, max),
1296 
1297 	TP_STRUCT__entry(
1298 		__field(unsigned int, task_id)
1299 		__field(unsigned int, client_id)
1300 		__field(u32, xid)
1301 		__field(u32, min)
1302 		__field(u32, max)
1303 	),
1304 
1305 	TP_fast_assign(
1306 		__entry->task_id = rqst->rq_task->tk_pid;
1307 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
1308 		__entry->xid = be32_to_cpu(rqst->rq_xid);
1309 		__entry->min = be32_to_cpup(min);
1310 		__entry->max = be32_to_cpup(max);
1311 	),
1312 
1313 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x versions=[%u, %u]",
1314 		__entry->task_id, __entry->client_id, __entry->xid,
1315 		__entry->min, __entry->max
1316 	)
1317 );
1318 
1319 TRACE_EVENT(xprtrdma_err_chunk,
1320 	TP_PROTO(
1321 		const struct rpc_rqst *rqst
1322 	),
1323 
1324 	TP_ARGS(rqst),
1325 
1326 	TP_STRUCT__entry(
1327 		__field(unsigned int, task_id)
1328 		__field(unsigned int, client_id)
1329 		__field(u32, xid)
1330 	),
1331 
1332 	TP_fast_assign(
1333 		__entry->task_id = rqst->rq_task->tk_pid;
1334 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
1335 		__entry->xid = be32_to_cpu(rqst->rq_xid);
1336 	),
1337 
1338 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x",
1339 		__entry->task_id, __entry->client_id, __entry->xid
1340 	)
1341 );
1342 
1343 TRACE_EVENT(xprtrdma_err_unrecognized,
1344 	TP_PROTO(
1345 		const struct rpc_rqst *rqst,
1346 		__be32 *procedure
1347 	),
1348 
1349 	TP_ARGS(rqst, procedure),
1350 
1351 	TP_STRUCT__entry(
1352 		__field(unsigned int, task_id)
1353 		__field(unsigned int, client_id)
1354 		__field(u32, xid)
1355 		__field(u32, procedure)
1356 	),
1357 
1358 	TP_fast_assign(
1359 		__entry->task_id = rqst->rq_task->tk_pid;
1360 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
1361 		__entry->procedure = be32_to_cpup(procedure);
1362 	),
1363 
1364 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " xid=0x%08x procedure=%u",
1365 		__entry->task_id, __entry->client_id, __entry->xid,
1366 		__entry->procedure
1367 	)
1368 );
1369 
1370 TRACE_EVENT(xprtrdma_fixup,
1371 	TP_PROTO(
1372 		const struct rpc_rqst *rqst,
1373 		unsigned long fixup
1374 	),
1375 
1376 	TP_ARGS(rqst, fixup),
1377 
1378 	TP_STRUCT__entry(
1379 		__field(unsigned int, task_id)
1380 		__field(unsigned int, client_id)
1381 		__field(unsigned long, fixup)
1382 		__field(size_t, headlen)
1383 		__field(unsigned int, pagelen)
1384 		__field(size_t, taillen)
1385 	),
1386 
1387 	TP_fast_assign(
1388 		__entry->task_id = rqst->rq_task->tk_pid;
1389 		__entry->client_id = rqst->rq_task->tk_client->cl_clid;
1390 		__entry->fixup = fixup;
1391 		__entry->headlen = rqst->rq_rcv_buf.head[0].iov_len;
1392 		__entry->pagelen = rqst->rq_rcv_buf.page_len;
1393 		__entry->taillen = rqst->rq_rcv_buf.tail[0].iov_len;
1394 	),
1395 
1396 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER " fixup=%lu xdr=%zu/%u/%zu",
1397 		__entry->task_id, __entry->client_id, __entry->fixup,
1398 		__entry->headlen, __entry->pagelen, __entry->taillen
1399 	)
1400 );
1401 
1402 TRACE_EVENT(xprtrdma_decode_seg,
1403 	TP_PROTO(
1404 		u32 handle,
1405 		u32 length,
1406 		u64 offset
1407 	),
1408 
1409 	TP_ARGS(handle, length, offset),
1410 
1411 	TP_STRUCT__entry(
1412 		__field(u32, handle)
1413 		__field(u32, length)
1414 		__field(u64, offset)
1415 	),
1416 
1417 	TP_fast_assign(
1418 		__entry->handle = handle;
1419 		__entry->length = length;
1420 		__entry->offset = offset;
1421 	),
1422 
1423 	TP_printk("%u@0x%016llx:0x%08x",
1424 		__entry->length, (unsigned long long)__entry->offset,
1425 		__entry->handle
1426 	)
1427 );
1428 
1429 TRACE_EVENT(xprtrdma_mrs_zap,
1430 	TP_PROTO(
1431 		const struct rpc_task *task
1432 	),
1433 
1434 	TP_ARGS(task),
1435 
1436 	TP_STRUCT__entry(
1437 		__field(unsigned int, task_id)
1438 		__field(unsigned int, client_id)
1439 	),
1440 
1441 	TP_fast_assign(
1442 		__entry->task_id = task->tk_pid;
1443 		__entry->client_id = task->tk_client->cl_clid;
1444 	),
1445 
1446 	TP_printk(SUNRPC_TRACE_TASK_SPECIFIER,
1447 		__entry->task_id, __entry->client_id
1448 	)
1449 );
1450 
1451 /**
1452  ** Callback events
1453  **/
1454 
1455 TRACE_EVENT(xprtrdma_cb_setup,
1456 	TP_PROTO(
1457 		const struct rpcrdma_xprt *r_xprt,
1458 		unsigned int reqs
1459 	),
1460 
1461 	TP_ARGS(r_xprt, reqs),
1462 
1463 	TP_STRUCT__entry(
1464 		__field(unsigned int, reqs)
1465 		__string(addr, rpcrdma_addrstr(r_xprt))
1466 		__string(port, rpcrdma_portstr(r_xprt))
1467 	),
1468 
1469 	TP_fast_assign(
1470 		__entry->reqs = reqs;
1471 		__assign_str(addr);
1472 		__assign_str(port);
1473 	),
1474 
1475 	TP_printk("peer=[%s]:%s %u reqs",
1476 		__get_str(addr), __get_str(port), __entry->reqs
1477 	)
1478 );
1479 
1480 DEFINE_CALLBACK_EVENT(call);
1481 DEFINE_CALLBACK_EVENT(reply);
1482 
1483 /**
1484  ** Server-side RPC/RDMA events
1485  **/
1486 
1487 DECLARE_EVENT_CLASS(svcrdma_accept_class,
1488 	TP_PROTO(
1489 		const struct svcxprt_rdma *rdma,
1490 		long status
1491 	),
1492 
1493 	TP_ARGS(rdma, status),
1494 
1495 	TP_STRUCT__entry(
1496 		__field(long, status)
1497 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
1498 	),
1499 
1500 	TP_fast_assign(
1501 		__entry->status = status;
1502 		__assign_str(addr);
1503 	),
1504 
1505 	TP_printk("addr=%s status=%ld",
1506 		__get_str(addr), __entry->status
1507 	)
1508 );
1509 
1510 #define DEFINE_ACCEPT_EVENT(name) \
1511 		DEFINE_EVENT(svcrdma_accept_class, svcrdma_##name##_err, \
1512 				TP_PROTO( \
1513 					const struct svcxprt_rdma *rdma, \
1514 					long status \
1515 				), \
1516 				TP_ARGS(rdma, status))
1517 
1518 DEFINE_ACCEPT_EVENT(pd);
1519 DEFINE_ACCEPT_EVENT(qp);
1520 DEFINE_ACCEPT_EVENT(fabric);
1521 DEFINE_ACCEPT_EVENT(initdepth);
1522 DEFINE_ACCEPT_EVENT(accept);
1523 
1524 TRACE_DEFINE_ENUM(RDMA_MSG);
1525 TRACE_DEFINE_ENUM(RDMA_NOMSG);
1526 TRACE_DEFINE_ENUM(RDMA_MSGP);
1527 TRACE_DEFINE_ENUM(RDMA_DONE);
1528 TRACE_DEFINE_ENUM(RDMA_ERROR);
1529 
1530 #define show_rpcrdma_proc(x)						\
1531 		__print_symbolic(x,					\
1532 				{ RDMA_MSG, "RDMA_MSG" },		\
1533 				{ RDMA_NOMSG, "RDMA_NOMSG" },		\
1534 				{ RDMA_MSGP, "RDMA_MSGP" },		\
1535 				{ RDMA_DONE, "RDMA_DONE" },		\
1536 				{ RDMA_ERROR, "RDMA_ERROR" })
1537 
1538 TRACE_EVENT(svcrdma_decode_rqst,
1539 	TP_PROTO(
1540 		const struct svc_rdma_recv_ctxt *ctxt,
1541 		__be32 *p,
1542 		unsigned int hdrlen
1543 	),
1544 
1545 	TP_ARGS(ctxt, p, hdrlen),
1546 
1547 	TP_STRUCT__entry(
1548 		__field(u32, cq_id)
1549 		__field(int, completion_id)
1550 		__field(u32, xid)
1551 		__field(u32, vers)
1552 		__field(u32, proc)
1553 		__field(u32, credits)
1554 		__field(unsigned int, hdrlen)
1555 	),
1556 
1557 	TP_fast_assign(
1558 		__entry->cq_id = ctxt->rc_cid.ci_queue_id;
1559 		__entry->completion_id = ctxt->rc_cid.ci_completion_id;
1560 		__entry->xid = be32_to_cpup(p++);
1561 		__entry->vers = be32_to_cpup(p++);
1562 		__entry->credits = be32_to_cpup(p++);
1563 		__entry->proc = be32_to_cpup(p);
1564 		__entry->hdrlen = hdrlen;
1565 	),
1566 
1567 	TP_printk("cq.id=%u cid=%d xid=0x%08x vers=%u credits=%u proc=%s hdrlen=%u",
1568 		__entry->cq_id, __entry->completion_id,
1569 		__entry->xid, __entry->vers, __entry->credits,
1570 		show_rpcrdma_proc(__entry->proc), __entry->hdrlen)
1571 );
1572 
1573 TRACE_EVENT(svcrdma_decode_short_err,
1574 	TP_PROTO(
1575 		const struct svc_rdma_recv_ctxt *ctxt,
1576 		unsigned int hdrlen
1577 	),
1578 
1579 	TP_ARGS(ctxt, hdrlen),
1580 
1581 	TP_STRUCT__entry(
1582 		__field(u32, cq_id)
1583 		__field(int, completion_id)
1584 		__field(unsigned int, hdrlen)
1585 	),
1586 
1587 	TP_fast_assign(
1588 		__entry->cq_id = ctxt->rc_cid.ci_queue_id;
1589 		__entry->completion_id = ctxt->rc_cid.ci_completion_id;
1590 		__entry->hdrlen = hdrlen;
1591 	),
1592 
1593 	TP_printk("cq.id=%u cid=%d hdrlen=%u",
1594 		__entry->cq_id, __entry->completion_id,
1595 		__entry->hdrlen)
1596 );
1597 
1598 DECLARE_EVENT_CLASS(svcrdma_badreq_event,
1599 	TP_PROTO(
1600 		const struct svc_rdma_recv_ctxt *ctxt,
1601 		__be32 *p
1602 	),
1603 
1604 	TP_ARGS(ctxt, p),
1605 
1606 	TP_STRUCT__entry(
1607 		__field(u32, cq_id)
1608 		__field(int, completion_id)
1609 		__field(u32, xid)
1610 		__field(u32, vers)
1611 		__field(u32, proc)
1612 		__field(u32, credits)
1613 	),
1614 
1615 	TP_fast_assign(
1616 		__entry->cq_id = ctxt->rc_cid.ci_queue_id;
1617 		__entry->completion_id = ctxt->rc_cid.ci_completion_id;
1618 		__entry->xid = be32_to_cpup(p++);
1619 		__entry->vers = be32_to_cpup(p++);
1620 		__entry->credits = be32_to_cpup(p++);
1621 		__entry->proc = be32_to_cpup(p);
1622 	),
1623 
1624 	TP_printk("cq.id=%u cid=%d xid=0x%08x vers=%u credits=%u proc=%u",
1625 		__entry->cq_id, __entry->completion_id,
1626 		__entry->xid, __entry->vers, __entry->credits, __entry->proc)
1627 );
1628 
1629 #define DEFINE_BADREQ_EVENT(name)					\
1630 		DEFINE_EVENT(svcrdma_badreq_event,			\
1631 			     svcrdma_decode_##name##_err,		\
1632 				TP_PROTO(				\
1633 					const struct svc_rdma_recv_ctxt *ctxt,	\
1634 					__be32 *p			\
1635 				),					\
1636 				TP_ARGS(ctxt, p))
1637 
1638 DEFINE_BADREQ_EVENT(badvers);
1639 DEFINE_BADREQ_EVENT(drop);
1640 DEFINE_BADREQ_EVENT(badproc);
1641 DEFINE_BADREQ_EVENT(parse);
1642 
1643 TRACE_EVENT(svcrdma_encode_wseg,
1644 	TP_PROTO(
1645 		const struct svc_rdma_send_ctxt *ctxt,
1646 		u32 segno,
1647 		u32 handle,
1648 		u32 length,
1649 		u64 offset
1650 	),
1651 
1652 	TP_ARGS(ctxt, segno, handle, length, offset),
1653 
1654 	TP_STRUCT__entry(
1655 		__field(u32, cq_id)
1656 		__field(int, completion_id)
1657 		__field(u32, segno)
1658 		__field(u32, handle)
1659 		__field(u32, length)
1660 		__field(u64, offset)
1661 	),
1662 
1663 	TP_fast_assign(
1664 		__entry->cq_id = ctxt->sc_cid.ci_queue_id;
1665 		__entry->completion_id = ctxt->sc_cid.ci_completion_id;
1666 		__entry->segno = segno;
1667 		__entry->handle = handle;
1668 		__entry->length = length;
1669 		__entry->offset = offset;
1670 	),
1671 
1672 	TP_printk("cq.id=%u cid=%d segno=%u %u@0x%016llx:0x%08x",
1673 		__entry->cq_id, __entry->completion_id,
1674 		__entry->segno, __entry->length,
1675 		(unsigned long long)__entry->offset, __entry->handle
1676 	)
1677 );
1678 
1679 TRACE_EVENT(svcrdma_decode_rseg,
1680 	TP_PROTO(
1681 		const struct rpc_rdma_cid *cid,
1682 		const struct svc_rdma_chunk *chunk,
1683 		const struct svc_rdma_segment *segment
1684 	),
1685 
1686 	TP_ARGS(cid, chunk, segment),
1687 
1688 	TP_STRUCT__entry(
1689 		__field(u32, cq_id)
1690 		__field(int, completion_id)
1691 		__field(u32, segno)
1692 		__field(u32, position)
1693 		__field(u32, handle)
1694 		__field(u32, length)
1695 		__field(u64, offset)
1696 	),
1697 
1698 	TP_fast_assign(
1699 		__entry->cq_id = cid->ci_queue_id;
1700 		__entry->completion_id = cid->ci_completion_id;
1701 		__entry->segno = chunk->ch_segcount;
1702 		__entry->position = chunk->ch_position;
1703 		__entry->handle = segment->rs_handle;
1704 		__entry->length = segment->rs_length;
1705 		__entry->offset = segment->rs_offset;
1706 	),
1707 
1708 	TP_printk("cq.id=%u cid=%d segno=%u position=%u %u@0x%016llx:0x%08x",
1709 		__entry->cq_id, __entry->completion_id,
1710 		__entry->segno, __entry->position, __entry->length,
1711 		(unsigned long long)__entry->offset, __entry->handle
1712 	)
1713 );
1714 
1715 TRACE_EVENT(svcrdma_decode_wseg,
1716 	TP_PROTO(
1717 		const struct rpc_rdma_cid *cid,
1718 		const struct svc_rdma_chunk *chunk,
1719 		u32 segno
1720 	),
1721 
1722 	TP_ARGS(cid, chunk, segno),
1723 
1724 	TP_STRUCT__entry(
1725 		__field(u32, cq_id)
1726 		__field(int, completion_id)
1727 		__field(u32, segno)
1728 		__field(u32, handle)
1729 		__field(u32, length)
1730 		__field(u64, offset)
1731 	),
1732 
1733 	TP_fast_assign(
1734 		const struct svc_rdma_segment *segment =
1735 			&chunk->ch_segments[segno];
1736 
1737 		__entry->cq_id = cid->ci_queue_id;
1738 		__entry->completion_id = cid->ci_completion_id;
1739 		__entry->segno = segno;
1740 		__entry->handle = segment->rs_handle;
1741 		__entry->length = segment->rs_length;
1742 		__entry->offset = segment->rs_offset;
1743 	),
1744 
1745 	TP_printk("cq.id=%u cid=%d segno=%u %u@0x%016llx:0x%08x",
1746 		__entry->cq_id, __entry->completion_id,
1747 		__entry->segno, __entry->length,
1748 		(unsigned long long)__entry->offset, __entry->handle
1749 	)
1750 );
1751 
1752 DECLARE_EVENT_CLASS(svcrdma_error_event,
1753 	TP_PROTO(
1754 		__be32 xid
1755 	),
1756 
1757 	TP_ARGS(xid),
1758 
1759 	TP_STRUCT__entry(
1760 		__field(u32, xid)
1761 	),
1762 
1763 	TP_fast_assign(
1764 		__entry->xid = be32_to_cpu(xid);
1765 	),
1766 
1767 	TP_printk("xid=0x%08x",
1768 		__entry->xid
1769 	)
1770 );
1771 
1772 #define DEFINE_ERROR_EVENT(name)					\
1773 		DEFINE_EVENT(svcrdma_error_event, svcrdma_err_##name,	\
1774 				TP_PROTO(				\
1775 					__be32 xid			\
1776 				),					\
1777 				TP_ARGS(xid))
1778 
1779 DEFINE_ERROR_EVENT(vers);
1780 DEFINE_ERROR_EVENT(chunk);
1781 
1782 /**
1783  ** Server-side RDMA API events
1784  **/
1785 
1786 DECLARE_EVENT_CLASS(svcrdma_dma_map_class,
1787 	TP_PROTO(
1788 		const struct rpc_rdma_cid *cid,
1789 		u64 dma_addr,
1790 		u32 length
1791 	),
1792 
1793 	TP_ARGS(cid, dma_addr, length),
1794 
1795 	TP_STRUCT__entry(
1796 		__field(u32, cq_id)
1797 		__field(int, completion_id)
1798 		__field(u64, dma_addr)
1799 		__field(u32, length)
1800 	),
1801 
1802 	TP_fast_assign(
1803 		__entry->cq_id = cid->ci_queue_id;
1804 		__entry->completion_id = cid->ci_completion_id;
1805 		__entry->dma_addr = dma_addr;
1806 		__entry->length = length;
1807 	),
1808 
1809 	TP_printk("cq.id=%u cid=%d dma_addr=%llu length=%u",
1810 		__entry->cq_id, __entry->completion_id,
1811 		__entry->dma_addr, __entry->length
1812 	)
1813 );
1814 
1815 #define DEFINE_SVC_DMA_EVENT(name)					\
1816 		DEFINE_EVENT(svcrdma_dma_map_class, svcrdma_##name,	\
1817 				TP_PROTO(				\
1818 					const struct rpc_rdma_cid *cid, \
1819 					u64 dma_addr,			\
1820 					u32 length			\
1821 				),					\
1822 				TP_ARGS(cid, dma_addr, length)		\
1823 		)
1824 
1825 DEFINE_SVC_DMA_EVENT(dma_map_page);
1826 DEFINE_SVC_DMA_EVENT(dma_map_err);
1827 DEFINE_SVC_DMA_EVENT(dma_unmap_page);
1828 
1829 TRACE_EVENT(svcrdma_dma_map_rw_err,
1830 	TP_PROTO(
1831 		const struct svcxprt_rdma *rdma,
1832 		u64 offset,
1833 		u32 handle,
1834 		unsigned int nents,
1835 		int status
1836 	),
1837 
1838 	TP_ARGS(rdma, offset, handle, nents, status),
1839 
1840 	TP_STRUCT__entry(
1841 		__field(u32, cq_id)
1842 		__field(u32, handle)
1843 		__field(u64, offset)
1844 		__field(unsigned int, nents)
1845 		__field(int, status)
1846 	),
1847 
1848 	TP_fast_assign(
1849 		__entry->cq_id = rdma->sc_sq_cq->res.id;
1850 		__entry->handle = handle;
1851 		__entry->offset = offset;
1852 		__entry->nents = nents;
1853 		__entry->status = status;
1854 	),
1855 
1856 	TP_printk("cq.id=%u 0x%016llx:0x%08x nents=%u status=%d",
1857 		__entry->cq_id, (unsigned long long)__entry->offset,
1858 		__entry->handle, __entry->nents, __entry->status
1859 	)
1860 );
1861 
1862 TRACE_EVENT(svcrdma_rwctx_empty,
1863 	TP_PROTO(
1864 		const struct svcxprt_rdma *rdma,
1865 		unsigned int num_sges
1866 	),
1867 
1868 	TP_ARGS(rdma, num_sges),
1869 
1870 	TP_STRUCT__entry(
1871 		__field(u32, cq_id)
1872 		__field(unsigned int, num_sges)
1873 	),
1874 
1875 	TP_fast_assign(
1876 		__entry->cq_id = rdma->sc_sq_cq->res.id;
1877 		__entry->num_sges = num_sges;
1878 	),
1879 
1880 	TP_printk("cq.id=%u num_sges=%d",
1881 		__entry->cq_id, __entry->num_sges
1882 	)
1883 );
1884 
1885 TRACE_EVENT(svcrdma_page_overrun_err,
1886 	TP_PROTO(
1887 		const struct rpc_rdma_cid *cid,
1888 		unsigned int pageno
1889 	),
1890 
1891 	TP_ARGS(cid, pageno),
1892 
1893 	TP_STRUCT__entry(
1894 		__field(u32, cq_id)
1895 		__field(int, completion_id)
1896 		__field(unsigned int, pageno)
1897 	),
1898 
1899 	TP_fast_assign(
1900 		__entry->cq_id = cid->ci_queue_id;
1901 		__entry->completion_id = cid->ci_completion_id;
1902 		__entry->pageno = pageno;
1903 	),
1904 
1905 	TP_printk("cq.id=%u cid=%d pageno=%u",
1906 		__entry->cq_id, __entry->completion_id,
1907 		__entry->pageno
1908 	)
1909 );
1910 
1911 TRACE_EVENT(svcrdma_small_wrch_err,
1912 	TP_PROTO(
1913 		const struct rpc_rdma_cid *cid,
1914 		unsigned int remaining,
1915 		unsigned int seg_no,
1916 		unsigned int num_segs
1917 	),
1918 
1919 	TP_ARGS(cid, remaining, seg_no, num_segs),
1920 
1921 	TP_STRUCT__entry(
1922 		__field(u32, cq_id)
1923 		__field(int, completion_id)
1924 		__field(unsigned int, remaining)
1925 		__field(unsigned int, seg_no)
1926 		__field(unsigned int, num_segs)
1927 	),
1928 
1929 	TP_fast_assign(
1930 		__entry->cq_id = cid->ci_queue_id;
1931 		__entry->completion_id = cid->ci_completion_id;
1932 		__entry->remaining = remaining;
1933 		__entry->seg_no = seg_no;
1934 		__entry->num_segs = num_segs;
1935 	),
1936 
1937 	TP_printk("cq.id=%u cid=%d remaining=%u seg_no=%u num_segs=%u",
1938 		__entry->cq_id, __entry->completion_id,
1939 		__entry->remaining, __entry->seg_no, __entry->num_segs
1940 	)
1941 );
1942 
1943 TRACE_EVENT(svcrdma_send_pullup,
1944 	TP_PROTO(
1945 		const struct svc_rdma_send_ctxt *ctxt,
1946 		unsigned int msglen
1947 	),
1948 
1949 	TP_ARGS(ctxt, msglen),
1950 
1951 	TP_STRUCT__entry(
1952 		__field(u32, cq_id)
1953 		__field(int, completion_id)
1954 		__field(unsigned int, hdrlen)
1955 		__field(unsigned int, msglen)
1956 	),
1957 
1958 	TP_fast_assign(
1959 		__entry->cq_id = ctxt->sc_cid.ci_queue_id;
1960 		__entry->completion_id = ctxt->sc_cid.ci_completion_id;
1961 		__entry->hdrlen = ctxt->sc_hdrbuf.len,
1962 		__entry->msglen = msglen;
1963 	),
1964 
1965 	TP_printk("cq.id=%u cid=%d hdr=%u msg=%u (total %u)",
1966 		__entry->cq_id, __entry->completion_id,
1967 		__entry->hdrlen, __entry->msglen,
1968 		__entry->hdrlen + __entry->msglen)
1969 );
1970 
1971 TRACE_EVENT(svcrdma_send_err,
1972 	TP_PROTO(
1973 		const struct svc_rqst *rqst,
1974 		int status
1975 	),
1976 
1977 	TP_ARGS(rqst, status),
1978 
1979 	TP_STRUCT__entry(
1980 		__field(int, status)
1981 		__field(u32, xid)
1982 		__string(addr, rqst->rq_xprt->xpt_remotebuf)
1983 	),
1984 
1985 	TP_fast_assign(
1986 		__entry->status = status;
1987 		__entry->xid = __be32_to_cpu(rqst->rq_xid);
1988 		__assign_str(addr);
1989 	),
1990 
1991 	TP_printk("addr=%s xid=0x%08x status=%d", __get_str(addr),
1992 		__entry->xid, __entry->status
1993 	)
1994 );
1995 
1996 TRACE_EVENT(svcrdma_post_send,
1997 	TP_PROTO(
1998 		const struct svc_rdma_send_ctxt *ctxt
1999 	),
2000 
2001 	TP_ARGS(ctxt),
2002 
2003 	TP_STRUCT__entry(
2004 		__field(u32, cq_id)
2005 		__field(int, completion_id)
2006 		__field(unsigned int, num_sge)
2007 		__field(u32, inv_rkey)
2008 	),
2009 
2010 	TP_fast_assign(
2011 		const struct ib_send_wr *wr = &ctxt->sc_send_wr;
2012 
2013 		__entry->cq_id = ctxt->sc_cid.ci_queue_id;
2014 		__entry->completion_id = ctxt->sc_cid.ci_completion_id;
2015 		__entry->num_sge = wr->num_sge;
2016 		__entry->inv_rkey = (wr->opcode == IB_WR_SEND_WITH_INV) ?
2017 					wr->ex.invalidate_rkey : 0;
2018 	),
2019 
2020 	TP_printk("cq.id=%u cid=%d num_sge=%u inv_rkey=0x%08x",
2021 		__entry->cq_id, __entry->completion_id,
2022 		__entry->num_sge, __entry->inv_rkey
2023 	)
2024 );
2025 
2026 DEFINE_SIMPLE_CID_EVENT(svcrdma_wc_send);
2027 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_send_flush);
2028 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_send_err);
2029 
2030 DEFINE_SIMPLE_CID_EVENT(svcrdma_post_recv);
2031 
2032 DEFINE_RECEIVE_SUCCESS_EVENT(svcrdma_wc_recv);
2033 DEFINE_RECEIVE_FLUSH_EVENT(svcrdma_wc_recv_flush);
2034 DEFINE_RECEIVE_FLUSH_EVENT(svcrdma_wc_recv_err);
2035 
2036 TRACE_EVENT(svcrdma_rq_post_err,
2037 	TP_PROTO(
2038 		const struct svcxprt_rdma *rdma,
2039 		int status
2040 	),
2041 
2042 	TP_ARGS(rdma, status),
2043 
2044 	TP_STRUCT__entry(
2045 		__field(int, status)
2046 		__string(addr, rdma->sc_xprt.xpt_remotebuf)
2047 	),
2048 
2049 	TP_fast_assign(
2050 		__entry->status = status;
2051 		__assign_str(addr);
2052 	),
2053 
2054 	TP_printk("addr=%s status=%d",
2055 		__get_str(addr), __entry->status
2056 	)
2057 );
2058 
2059 DECLARE_EVENT_CLASS(svcrdma_post_chunk_class,
2060 	TP_PROTO(
2061 		const struct rpc_rdma_cid *cid,
2062 		int sqecount
2063 	),
2064 
2065 	TP_ARGS(cid, sqecount),
2066 
2067 	TP_STRUCT__entry(
2068 		__field(u32, cq_id)
2069 		__field(int, completion_id)
2070 		__field(int, sqecount)
2071 	),
2072 
2073 	TP_fast_assign(
2074 		__entry->cq_id = cid->ci_queue_id;
2075 		__entry->completion_id = cid->ci_completion_id;
2076 		__entry->sqecount = sqecount;
2077 	),
2078 
2079 	TP_printk("cq.id=%u cid=%d sqecount=%d",
2080 		__entry->cq_id, __entry->completion_id,
2081 		__entry->sqecount
2082 	)
2083 );
2084 
2085 #define DEFINE_POST_CHUNK_EVENT(name)					\
2086 		DEFINE_EVENT(svcrdma_post_chunk_class,			\
2087 				svcrdma_post_##name##_chunk,		\
2088 				TP_PROTO(				\
2089 					const struct rpc_rdma_cid *cid,	\
2090 					int sqecount			\
2091 				),					\
2092 				TP_ARGS(cid, sqecount))
2093 
2094 DEFINE_POST_CHUNK_EVENT(read);
2095 DEFINE_POST_CHUNK_EVENT(write);
2096 DEFINE_POST_CHUNK_EVENT(reply);
2097 
2098 DEFINE_EVENT(svcrdma_post_chunk_class, svcrdma_cc_release,
2099 	TP_PROTO(
2100 		const struct rpc_rdma_cid *cid,
2101 		int sqecount
2102 	),
2103 	TP_ARGS(cid, sqecount)
2104 );
2105 
2106 TRACE_EVENT(svcrdma_wc_read,
2107 	TP_PROTO(
2108 		const struct ib_wc *wc,
2109 		const struct rpc_rdma_cid *cid,
2110 		unsigned int totalbytes,
2111 		const ktime_t posttime
2112 	),
2113 
2114 	TP_ARGS(wc, cid, totalbytes, posttime),
2115 
2116 	TP_STRUCT__entry(
2117 		__field(u32, cq_id)
2118 		__field(int, completion_id)
2119 		__field(s64, read_latency)
2120 		__field(unsigned int, totalbytes)
2121 	),
2122 
2123 	TP_fast_assign(
2124 		__entry->cq_id = cid->ci_queue_id;
2125 		__entry->completion_id = cid->ci_completion_id;
2126 		__entry->totalbytes = totalbytes;
2127 		__entry->read_latency = ktime_us_delta(ktime_get(), posttime);
2128 	),
2129 
2130 	TP_printk("cq.id=%u cid=%d totalbytes=%u latency-us=%lld",
2131 		__entry->cq_id, __entry->completion_id,
2132 		__entry->totalbytes, __entry->read_latency
2133 	)
2134 );
2135 
2136 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_read_flush);
2137 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_read_err);
2138 DEFINE_SIMPLE_CID_EVENT(svcrdma_read_finished);
2139 
2140 DEFINE_SIMPLE_CID_EVENT(svcrdma_wc_write);
2141 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_write_flush);
2142 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_write_err);
2143 
2144 DEFINE_SIMPLE_CID_EVENT(svcrdma_wc_reply);
2145 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_reply_flush);
2146 DEFINE_SEND_FLUSH_EVENT(svcrdma_wc_reply_err);
2147 
2148 TRACE_EVENT(svcrdma_qp_error,
2149 	TP_PROTO(
2150 		const struct ib_event *event,
2151 		const struct sockaddr *sap
2152 	),
2153 
2154 	TP_ARGS(event, sap),
2155 
2156 	TP_STRUCT__entry(
2157 		__field(unsigned int, event)
2158 		__string(device, event->device->name)
2159 		__array(__u8, addr, INET6_ADDRSTRLEN + 10)
2160 	),
2161 
2162 	TP_fast_assign(
2163 		__entry->event = event->event;
2164 		__assign_str(device);
2165 		snprintf(__entry->addr, sizeof(__entry->addr) - 1,
2166 			 "%pISpc", sap);
2167 	),
2168 
2169 	TP_printk("addr=%s dev=%s event=%s (%u)",
2170 		__entry->addr, __get_str(device),
2171 		rdma_show_ib_event(__entry->event), __entry->event
2172 	)
2173 );
2174 
2175 TRACE_EVENT(svcrdma_device_removal,
2176 	TP_PROTO(
2177 		const struct rdma_cm_id *id
2178 	),
2179 
2180 	TP_ARGS(id),
2181 
2182 	TP_STRUCT__entry(
2183 		__string(name, id->device->name)
2184 		__array(unsigned char, addr, sizeof(struct sockaddr_in6))
2185 	),
2186 
2187 	TP_fast_assign(
2188 		__assign_str(name);
2189 		memcpy(__entry->addr, &id->route.addr.dst_addr,
2190 		       sizeof(struct sockaddr_in6));
2191 	),
2192 
2193 	TP_printk("device %s to be removed, disconnecting %pISpc\n",
2194 		__get_str(name), __entry->addr
2195 	)
2196 );
2197 
2198 DECLARE_EVENT_CLASS(svcrdma_sendqueue_class,
2199 	TP_PROTO(
2200 		const struct svcxprt_rdma *rdma,
2201 		const struct rpc_rdma_cid *cid
2202 	),
2203 
2204 	TP_ARGS(rdma, cid),
2205 
2206 	TP_STRUCT__entry(
2207 		__field(u32, cq_id)
2208 		__field(int, completion_id)
2209 		__field(int, avail)
2210 		__field(int, depth)
2211 	),
2212 
2213 	TP_fast_assign(
2214 		__entry->cq_id = cid->ci_queue_id;
2215 		__entry->completion_id = cid->ci_completion_id;
2216 		__entry->avail = atomic_read(&rdma->sc_sq_avail);
2217 		__entry->depth = rdma->sc_sq_depth;
2218 	),
2219 
2220 	TP_printk("cq.id=%u cid=%d sc_sq_avail=%d/%d",
2221 		__entry->cq_id, __entry->completion_id,
2222 		__entry->avail, __entry->depth
2223 	)
2224 );
2225 
2226 #define DEFINE_SQ_EVENT(name)						\
2227 		DEFINE_EVENT(svcrdma_sendqueue_class, name,		\
2228 			TP_PROTO(					\
2229 				const struct svcxprt_rdma *rdma,	\
2230 				const struct rpc_rdma_cid *cid		\
2231 			),						\
2232 			TP_ARGS(rdma, cid)				\
2233 		)
2234 
2235 DEFINE_SQ_EVENT(svcrdma_sq_full);
2236 DEFINE_SQ_EVENT(svcrdma_sq_retry);
2237 
2238 TRACE_EVENT(svcrdma_sq_post_err,
2239 	TP_PROTO(
2240 		const struct svcxprt_rdma *rdma,
2241 		const struct rpc_rdma_cid *cid,
2242 		int status
2243 	),
2244 
2245 	TP_ARGS(rdma, cid, status),
2246 
2247 	TP_STRUCT__entry(
2248 		__field(u32, cq_id)
2249 		__field(int, completion_id)
2250 		__field(int, avail)
2251 		__field(int, depth)
2252 		__field(int, status)
2253 	),
2254 
2255 	TP_fast_assign(
2256 		__entry->cq_id = cid->ci_queue_id;
2257 		__entry->completion_id = cid->ci_completion_id;
2258 		__entry->avail = atomic_read(&rdma->sc_sq_avail);
2259 		__entry->depth = rdma->sc_sq_depth;
2260 		__entry->status = status;
2261 	),
2262 
2263 	TP_printk("cq.id=%u cid=%d sc_sq_avail=%d/%d status=%d",
2264 		__entry->cq_id, __entry->completion_id,
2265 		__entry->avail, __entry->depth, __entry->status
2266 	)
2267 );
2268 
2269 DECLARE_EVENT_CLASS(rpcrdma_client_device_class,
2270 	TP_PROTO(
2271 		const struct ib_device *device
2272 	),
2273 
2274 	TP_ARGS(device),
2275 
2276 	TP_STRUCT__entry(
2277 		__string(name, device->name)
2278 	),
2279 
2280 	TP_fast_assign(
2281 		__assign_str(name);
2282 	),
2283 
2284 	TP_printk("device=%s",
2285 		__get_str(name)
2286 	)
2287 );
2288 
2289 #define DEFINE_CLIENT_DEVICE_EVENT(name)				\
2290 	DEFINE_EVENT(rpcrdma_client_device_class, name,			\
2291 		TP_PROTO(						\
2292 			const struct ib_device *device			\
2293 		),							\
2294 		TP_ARGS(device)						\
2295 	)
2296 
2297 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_completion);
2298 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_add_one);
2299 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_remove_one);
2300 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_wait_on);
2301 DEFINE_CLIENT_DEVICE_EVENT(rpcrdma_client_remove_one_done);
2302 
2303 DECLARE_EVENT_CLASS(rpcrdma_client_register_class,
2304 	TP_PROTO(
2305 		const struct ib_device *device,
2306 		const struct rpcrdma_notification *rn
2307 	),
2308 
2309 	TP_ARGS(device, rn),
2310 
2311 	TP_STRUCT__entry(
2312 		__string(name, device->name)
2313 		__field(void *, callback)
2314 		__field(u32, index)
2315 	),
2316 
2317 	TP_fast_assign(
2318 		__assign_str(name);
2319 		__entry->callback = rn->rn_done;
2320 		__entry->index = rn->rn_index;
2321 	),
2322 
2323 	TP_printk("device=%s index=%u done callback=%pS\n",
2324 		__get_str(name), __entry->index, __entry->callback
2325 	)
2326 );
2327 
2328 #define DEFINE_CLIENT_REGISTER_EVENT(name)				\
2329 	DEFINE_EVENT(rpcrdma_client_register_class, name,		\
2330 	TP_PROTO(							\
2331 		const struct ib_device *device,				\
2332 		const struct rpcrdma_notification *rn			\
2333 	),								\
2334 	TP_ARGS(device, rn))
2335 
2336 DEFINE_CLIENT_REGISTER_EVENT(rpcrdma_client_register);
2337 DEFINE_CLIENT_REGISTER_EVENT(rpcrdma_client_unregister);
2338 
2339 #endif /* _TRACE_RPCRDMA_H */
2340 
2341 #include <trace/define_trace.h>
2342