1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * rio_cm - RapidIO Channelized Messaging Driver
4 *
5 * Copyright 2013-2016 Integrated Device Technology, Inc.
6 * Copyright (c) 2015, Prodrive Technologies
7 * Copyright (c) 2015, RapidIO Trade Association
8 */
9
10 #include <linux/module.h>
11 #include <linux/kernel.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/delay.h>
14 #include <linux/sched.h>
15 #include <linux/rio.h>
16 #include <linux/rio_drv.h>
17 #include <linux/slab.h>
18 #include <linux/idr.h>
19 #include <linux/interrupt.h>
20 #include <linux/cdev.h>
21 #include <linux/fs.h>
22 #include <linux/poll.h>
23 #include <linux/reboot.h>
24 #include <linux/bitops.h>
25 #include <linux/printk.h>
26 #include <linux/rio_cm_cdev.h>
27
28 #define DRV_NAME "rio_cm"
29 #define DRV_VERSION "1.0.0"
30 #define DRV_AUTHOR "Alexandre Bounine <alexandre.bounine@idt.com>"
31 #define DRV_DESC "RapidIO Channelized Messaging Driver"
32 #define DEV_NAME "rio_cm"
33
34 /* Debug output filtering masks */
35 enum {
36 DBG_NONE = 0,
37 DBG_INIT = BIT(0), /* driver init */
38 DBG_EXIT = BIT(1), /* driver exit */
39 DBG_MPORT = BIT(2), /* mport add/remove */
40 DBG_RDEV = BIT(3), /* RapidIO device add/remove */
41 DBG_CHOP = BIT(4), /* channel operations */
42 DBG_WAIT = BIT(5), /* waiting for events */
43 DBG_TX = BIT(6), /* message TX */
44 DBG_TX_EVENT = BIT(7), /* message TX event */
45 DBG_RX_DATA = BIT(8), /* inbound data messages */
46 DBG_RX_CMD = BIT(9), /* inbound REQ/ACK/NACK messages */
47 DBG_ALL = ~0,
48 };
49
50 #ifdef DEBUG
51 #define riocm_debug(level, fmt, arg...) \
52 do { \
53 if (DBG_##level & dbg_level) \
54 pr_debug(DRV_NAME ": %s " fmt "\n", \
55 __func__, ##arg); \
56 } while (0)
57 #else
58 #define riocm_debug(level, fmt, arg...) \
59 no_printk(KERN_DEBUG pr_fmt(DRV_NAME fmt "\n"), ##arg)
60 #endif
61
62 #define riocm_warn(fmt, arg...) \
63 pr_warn(DRV_NAME ": %s WARNING " fmt "\n", __func__, ##arg)
64
65 #define riocm_error(fmt, arg...) \
66 pr_err(DRV_NAME ": %s ERROR " fmt "\n", __func__, ##arg)
67
68
69 static int cmbox = 1;
70 module_param(cmbox, int, S_IRUGO);
71 MODULE_PARM_DESC(cmbox, "RapidIO Mailbox number (default 1)");
72
73 static int chstart = 256;
74 module_param(chstart, int, S_IRUGO);
75 MODULE_PARM_DESC(chstart,
76 "Start channel number for dynamic allocation (default 256)");
77
78 #ifdef DEBUG
79 static u32 dbg_level = DBG_NONE;
80 module_param(dbg_level, uint, S_IWUSR | S_IRUGO);
81 MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)");
82 #endif
83
84 MODULE_AUTHOR(DRV_AUTHOR);
85 MODULE_DESCRIPTION(DRV_DESC);
86 MODULE_LICENSE("GPL");
87 MODULE_VERSION(DRV_VERSION);
88
89 #define RIOCM_TX_RING_SIZE 128
90 #define RIOCM_RX_RING_SIZE 128
91 #define RIOCM_CONNECT_TO 3 /* connect response TO (in sec) */
92
93 #define RIOCM_MAX_CHNUM 0xffff /* Use full range of u16 field */
94 #define RIOCM_CHNUM_AUTO 0
95 #define RIOCM_MAX_EP_COUNT 0x10000 /* Max number of endpoints */
96
97 enum rio_cm_state {
98 RIO_CM_IDLE,
99 RIO_CM_CONNECT,
100 RIO_CM_CONNECTED,
101 RIO_CM_DISCONNECT,
102 RIO_CM_CHAN_BOUND,
103 RIO_CM_LISTEN,
104 RIO_CM_DESTROYING,
105 };
106
107 enum rio_cm_pkt_type {
108 RIO_CM_SYS = 0xaa,
109 RIO_CM_CHAN = 0x55,
110 };
111
112 enum rio_cm_chop {
113 CM_CONN_REQ,
114 CM_CONN_ACK,
115 CM_CONN_CLOSE,
116 CM_DATA_MSG,
117 };
118
119 struct rio_ch_base_bhdr {
120 u32 src_id;
121 u32 dst_id;
122 #define RIO_HDR_LETTER_MASK 0xffff0000
123 #define RIO_HDR_MBOX_MASK 0x0000ffff
124 u8 src_mbox;
125 u8 dst_mbox;
126 u8 type;
127 } __attribute__((__packed__));
128
129 struct rio_ch_chan_hdr {
130 struct rio_ch_base_bhdr bhdr;
131 u8 ch_op;
132 u16 dst_ch;
133 u16 src_ch;
134 u16 msg_len;
135 u16 rsrvd;
136 } __attribute__((__packed__));
137
138 struct tx_req {
139 struct list_head node;
140 struct rio_dev *rdev;
141 void *buffer;
142 size_t len;
143 };
144
145 struct cm_dev {
146 struct list_head list;
147 struct rio_mport *mport;
148 void *rx_buf[RIOCM_RX_RING_SIZE];
149 int rx_slots;
150 struct mutex rx_lock;
151
152 void *tx_buf[RIOCM_TX_RING_SIZE];
153 int tx_slot;
154 int tx_cnt;
155 int tx_ack_slot;
156 struct list_head tx_reqs;
157 spinlock_t tx_lock;
158
159 struct list_head peers;
160 u32 npeers;
161 struct workqueue_struct *rx_wq;
162 struct work_struct rx_work;
163 };
164
165 struct chan_rx_ring {
166 void *buf[RIOCM_RX_RING_SIZE];
167 int head;
168 int tail;
169 int count;
170
171 /* Tracking RX buffers reported to upper level */
172 void *inuse[RIOCM_RX_RING_SIZE];
173 int inuse_cnt;
174 };
175
176 struct rio_channel {
177 u16 id; /* local channel ID */
178 struct kref ref; /* channel refcount */
179 struct file *filp;
180 struct cm_dev *cmdev; /* associated CM device object */
181 struct rio_dev *rdev; /* remote RapidIO device */
182 enum rio_cm_state state;
183 int error;
184 spinlock_t lock;
185 void *context;
186 u32 loc_destid; /* local destID */
187 u32 rem_destid; /* remote destID */
188 u16 rem_channel; /* remote channel ID */
189 struct list_head accept_queue;
190 struct list_head ch_node;
191 struct completion comp;
192 struct completion comp_close;
193 struct chan_rx_ring rx_ring;
194 };
195
196 struct cm_peer {
197 struct list_head node;
198 struct rio_dev *rdev;
199 };
200
201 struct rio_cm_work {
202 struct work_struct work;
203 struct cm_dev *cm;
204 void *data;
205 };
206
207 struct conn_req {
208 struct list_head node;
209 u32 destid; /* requester destID */
210 u16 chan; /* requester channel ID */
211 struct cm_dev *cmdev;
212 };
213
214 /*
215 * A channel_dev structure represents a CM_CDEV
216 * @cdev Character device
217 * @dev Associated device object
218 */
219 struct channel_dev {
220 struct cdev cdev;
221 struct device *dev;
222 };
223
224 static struct rio_channel *riocm_ch_alloc(u16 ch_num);
225 static void riocm_ch_free(struct kref *ref);
226 static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
227 void *buffer, size_t len);
228 static int riocm_ch_close(struct rio_channel *ch);
229
230 static DEFINE_SPINLOCK(idr_lock);
231 static DEFINE_IDR(ch_idr);
232
233 static LIST_HEAD(cm_dev_list);
234 static DECLARE_RWSEM(rdev_sem);
235
236 static const struct class dev_class = {
237 .name = DRV_NAME,
238 };
239 static unsigned int dev_major;
240 static unsigned int dev_minor_base;
241 static dev_t dev_number;
242 static struct channel_dev riocm_cdev;
243
244 #define is_msg_capable(src_ops, dst_ops) \
245 ((src_ops & RIO_SRC_OPS_DATA_MSG) && \
246 (dst_ops & RIO_DST_OPS_DATA_MSG))
247 #define dev_cm_capable(dev) \
248 is_msg_capable(dev->src_ops, dev->dst_ops)
249
riocm_cmp(struct rio_channel * ch,enum rio_cm_state cmp)250 static int riocm_cmp(struct rio_channel *ch, enum rio_cm_state cmp)
251 {
252 int ret;
253
254 spin_lock_bh(&ch->lock);
255 ret = (ch->state == cmp);
256 spin_unlock_bh(&ch->lock);
257 return ret;
258 }
259
riocm_cmp_exch(struct rio_channel * ch,enum rio_cm_state cmp,enum rio_cm_state exch)260 static int riocm_cmp_exch(struct rio_channel *ch,
261 enum rio_cm_state cmp, enum rio_cm_state exch)
262 {
263 int ret;
264
265 spin_lock_bh(&ch->lock);
266 ret = (ch->state == cmp);
267 if (ret)
268 ch->state = exch;
269 spin_unlock_bh(&ch->lock);
270 return ret;
271 }
272
riocm_exch(struct rio_channel * ch,enum rio_cm_state exch)273 static enum rio_cm_state riocm_exch(struct rio_channel *ch,
274 enum rio_cm_state exch)
275 {
276 enum rio_cm_state old;
277
278 spin_lock_bh(&ch->lock);
279 old = ch->state;
280 ch->state = exch;
281 spin_unlock_bh(&ch->lock);
282 return old;
283 }
284
riocm_get_channel(u16 nr)285 static struct rio_channel *riocm_get_channel(u16 nr)
286 {
287 struct rio_channel *ch;
288
289 spin_lock_bh(&idr_lock);
290 ch = idr_find(&ch_idr, nr);
291 if (ch)
292 kref_get(&ch->ref);
293 spin_unlock_bh(&idr_lock);
294 return ch;
295 }
296
riocm_put_channel(struct rio_channel * ch)297 static void riocm_put_channel(struct rio_channel *ch)
298 {
299 kref_put(&ch->ref, riocm_ch_free);
300 }
301
riocm_rx_get_msg(struct cm_dev * cm)302 static void *riocm_rx_get_msg(struct cm_dev *cm)
303 {
304 void *msg;
305 int i;
306
307 msg = rio_get_inb_message(cm->mport, cmbox);
308 if (msg) {
309 for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
310 if (cm->rx_buf[i] == msg) {
311 cm->rx_buf[i] = NULL;
312 cm->rx_slots++;
313 break;
314 }
315 }
316
317 if (i == RIOCM_RX_RING_SIZE)
318 riocm_warn("no record for buffer 0x%p", msg);
319 }
320
321 return msg;
322 }
323
324 /*
325 * riocm_rx_fill - fills a ring of receive buffers for given cm device
326 * @cm: cm_dev object
327 * @nent: max number of entries to fill
328 *
329 * Returns: none
330 */
riocm_rx_fill(struct cm_dev * cm,int nent)331 static void riocm_rx_fill(struct cm_dev *cm, int nent)
332 {
333 int i;
334
335 if (cm->rx_slots == 0)
336 return;
337
338 for (i = 0; i < RIOCM_RX_RING_SIZE && cm->rx_slots && nent; i++) {
339 if (cm->rx_buf[i] == NULL) {
340 cm->rx_buf[i] = kmalloc(RIO_MAX_MSG_SIZE, GFP_KERNEL);
341 if (cm->rx_buf[i] == NULL)
342 break;
343 rio_add_inb_buffer(cm->mport, cmbox, cm->rx_buf[i]);
344 cm->rx_slots--;
345 nent--;
346 }
347 }
348 }
349
350 /*
351 * riocm_rx_free - frees all receive buffers associated with given cm device
352 * @cm: cm_dev object
353 *
354 * Returns: none
355 */
riocm_rx_free(struct cm_dev * cm)356 static void riocm_rx_free(struct cm_dev *cm)
357 {
358 int i;
359
360 for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
361 if (cm->rx_buf[i] != NULL) {
362 kfree(cm->rx_buf[i]);
363 cm->rx_buf[i] = NULL;
364 }
365 }
366 }
367
368 /*
369 * riocm_req_handler - connection request handler
370 * @cm: cm_dev object
371 * @req_data: pointer to the request packet
372 *
373 * Returns: 0 if success, or
374 * -EINVAL if channel is not in correct state,
375 * -ENODEV if cannot find a channel with specified ID,
376 * -ENOMEM if unable to allocate memory to store the request
377 */
riocm_req_handler(struct cm_dev * cm,void * req_data)378 static int riocm_req_handler(struct cm_dev *cm, void *req_data)
379 {
380 struct rio_channel *ch;
381 struct conn_req *req;
382 struct rio_ch_chan_hdr *hh = req_data;
383 u16 chnum;
384
385 chnum = ntohs(hh->dst_ch);
386
387 ch = riocm_get_channel(chnum);
388
389 if (!ch)
390 return -ENODEV;
391
392 if (ch->state != RIO_CM_LISTEN) {
393 riocm_debug(RX_CMD, "channel %d is not in listen state", chnum);
394 riocm_put_channel(ch);
395 return -EINVAL;
396 }
397
398 req = kzalloc(sizeof(*req), GFP_KERNEL);
399 if (!req) {
400 riocm_put_channel(ch);
401 return -ENOMEM;
402 }
403
404 req->destid = ntohl(hh->bhdr.src_id);
405 req->chan = ntohs(hh->src_ch);
406 req->cmdev = cm;
407
408 spin_lock_bh(&ch->lock);
409 list_add_tail(&req->node, &ch->accept_queue);
410 spin_unlock_bh(&ch->lock);
411 complete(&ch->comp);
412 riocm_put_channel(ch);
413
414 return 0;
415 }
416
417 /*
418 * riocm_resp_handler - response to connection request handler
419 * @resp_data: pointer to the response packet
420 *
421 * Returns: 0 if success, or
422 * -EINVAL if channel is not in correct state,
423 * -ENODEV if cannot find a channel with specified ID,
424 */
riocm_resp_handler(void * resp_data)425 static int riocm_resp_handler(void *resp_data)
426 {
427 struct rio_channel *ch;
428 struct rio_ch_chan_hdr *hh = resp_data;
429 u16 chnum;
430
431 chnum = ntohs(hh->dst_ch);
432 ch = riocm_get_channel(chnum);
433 if (!ch)
434 return -ENODEV;
435
436 if (ch->state != RIO_CM_CONNECT) {
437 riocm_put_channel(ch);
438 return -EINVAL;
439 }
440
441 riocm_exch(ch, RIO_CM_CONNECTED);
442 ch->rem_channel = ntohs(hh->src_ch);
443 complete(&ch->comp);
444 riocm_put_channel(ch);
445
446 return 0;
447 }
448
449 /*
450 * riocm_close_handler - channel close request handler
451 * @req_data: pointer to the request packet
452 *
453 * Returns: 0 if success, or
454 * -ENODEV if cannot find a channel with specified ID,
455 * + error codes returned by riocm_ch_close.
456 */
riocm_close_handler(void * data)457 static int riocm_close_handler(void *data)
458 {
459 struct rio_channel *ch;
460 struct rio_ch_chan_hdr *hh = data;
461 int ret;
462
463 riocm_debug(RX_CMD, "for ch=%d", ntohs(hh->dst_ch));
464
465 spin_lock_bh(&idr_lock);
466 ch = idr_find(&ch_idr, ntohs(hh->dst_ch));
467 if (!ch) {
468 spin_unlock_bh(&idr_lock);
469 return -ENODEV;
470 }
471 idr_remove(&ch_idr, ch->id);
472 spin_unlock_bh(&idr_lock);
473
474 riocm_exch(ch, RIO_CM_DISCONNECT);
475
476 ret = riocm_ch_close(ch);
477 if (ret)
478 riocm_debug(RX_CMD, "riocm_ch_close() returned %d", ret);
479
480 return 0;
481 }
482
483 /*
484 * rio_cm_handler - function that services request (non-data) packets
485 * @cm: cm_dev object
486 * @data: pointer to the packet
487 */
rio_cm_handler(struct cm_dev * cm,void * data)488 static void rio_cm_handler(struct cm_dev *cm, void *data)
489 {
490 struct rio_ch_chan_hdr *hdr;
491
492 if (!rio_mport_is_running(cm->mport))
493 goto out;
494
495 hdr = data;
496
497 riocm_debug(RX_CMD, "OP=%x for ch=%d from %d",
498 hdr->ch_op, ntohs(hdr->dst_ch), ntohs(hdr->src_ch));
499
500 switch (hdr->ch_op) {
501 case CM_CONN_REQ:
502 riocm_req_handler(cm, data);
503 break;
504 case CM_CONN_ACK:
505 riocm_resp_handler(data);
506 break;
507 case CM_CONN_CLOSE:
508 riocm_close_handler(data);
509 break;
510 default:
511 riocm_error("Invalid packet header");
512 break;
513 }
514 out:
515 kfree(data);
516 }
517
518 /*
519 * rio_rx_data_handler - received data packet handler
520 * @cm: cm_dev object
521 * @buf: data packet
522 *
523 * Returns: 0 if success, or
524 * -ENODEV if cannot find a channel with specified ID,
525 * -EIO if channel is not in CONNECTED state,
526 * -ENOMEM if channel RX queue is full (packet discarded)
527 */
rio_rx_data_handler(struct cm_dev * cm,void * buf)528 static int rio_rx_data_handler(struct cm_dev *cm, void *buf)
529 {
530 struct rio_ch_chan_hdr *hdr;
531 struct rio_channel *ch;
532
533 hdr = buf;
534
535 riocm_debug(RX_DATA, "for ch=%d", ntohs(hdr->dst_ch));
536
537 ch = riocm_get_channel(ntohs(hdr->dst_ch));
538 if (!ch) {
539 /* Discard data message for non-existing channel */
540 kfree(buf);
541 return -ENODEV;
542 }
543
544 /* Place pointer to the buffer into channel's RX queue */
545 spin_lock(&ch->lock);
546
547 if (ch->state != RIO_CM_CONNECTED) {
548 /* Channel is not ready to receive data, discard a packet */
549 riocm_debug(RX_DATA, "ch=%d is in wrong state=%d",
550 ch->id, ch->state);
551 spin_unlock(&ch->lock);
552 kfree(buf);
553 riocm_put_channel(ch);
554 return -EIO;
555 }
556
557 if (ch->rx_ring.count == RIOCM_RX_RING_SIZE) {
558 /* If RX ring is full, discard a packet */
559 riocm_debug(RX_DATA, "ch=%d is full", ch->id);
560 spin_unlock(&ch->lock);
561 kfree(buf);
562 riocm_put_channel(ch);
563 return -ENOMEM;
564 }
565
566 ch->rx_ring.buf[ch->rx_ring.head] = buf;
567 ch->rx_ring.head++;
568 ch->rx_ring.count++;
569 ch->rx_ring.head %= RIOCM_RX_RING_SIZE;
570
571 complete(&ch->comp);
572
573 spin_unlock(&ch->lock);
574 riocm_put_channel(ch);
575
576 return 0;
577 }
578
579 /*
580 * rio_ibmsg_handler - inbound message packet handler
581 */
rio_ibmsg_handler(struct work_struct * work)582 static void rio_ibmsg_handler(struct work_struct *work)
583 {
584 struct cm_dev *cm = container_of(work, struct cm_dev, rx_work);
585 void *data;
586 struct rio_ch_chan_hdr *hdr;
587
588 if (!rio_mport_is_running(cm->mport))
589 return;
590
591 while (1) {
592 mutex_lock(&cm->rx_lock);
593 data = riocm_rx_get_msg(cm);
594 if (data)
595 riocm_rx_fill(cm, 1);
596 mutex_unlock(&cm->rx_lock);
597
598 if (data == NULL)
599 break;
600
601 hdr = data;
602
603 if (hdr->bhdr.type != RIO_CM_CHAN) {
604 /* For now simply discard packets other than channel */
605 riocm_error("Unsupported TYPE code (0x%x). Msg dropped",
606 hdr->bhdr.type);
607 kfree(data);
608 continue;
609 }
610
611 /* Process a channel message */
612 if (hdr->ch_op == CM_DATA_MSG)
613 rio_rx_data_handler(cm, data);
614 else
615 rio_cm_handler(cm, data);
616 }
617 }
618
riocm_inb_msg_event(struct rio_mport * mport,void * dev_id,int mbox,int slot)619 static void riocm_inb_msg_event(struct rio_mport *mport, void *dev_id,
620 int mbox, int slot)
621 {
622 struct cm_dev *cm = dev_id;
623
624 if (rio_mport_is_running(cm->mport) && !work_pending(&cm->rx_work))
625 queue_work(cm->rx_wq, &cm->rx_work);
626 }
627
628 /*
629 * rio_txcq_handler - TX completion handler
630 * @cm: cm_dev object
631 * @slot: TX queue slot
632 *
633 * TX completion handler also ensures that pending request packets are placed
634 * into transmit queue as soon as a free slot becomes available. This is done
635 * to give higher priority to request packets during high intensity data flow.
636 */
rio_txcq_handler(struct cm_dev * cm,int slot)637 static void rio_txcq_handler(struct cm_dev *cm, int slot)
638 {
639 int ack_slot;
640
641 /* ATTN: Add TX completion notification if/when direct buffer
642 * transfer is implemented. At this moment only correct tracking
643 * of tx_count is important.
644 */
645 riocm_debug(TX_EVENT, "for mport_%d slot %d tx_cnt %d",
646 cm->mport->id, slot, cm->tx_cnt);
647
648 spin_lock(&cm->tx_lock);
649 ack_slot = cm->tx_ack_slot;
650
651 if (ack_slot == slot)
652 riocm_debug(TX_EVENT, "slot == ack_slot");
653
654 while (cm->tx_cnt && ((ack_slot != slot) ||
655 (cm->tx_cnt == RIOCM_TX_RING_SIZE))) {
656
657 cm->tx_buf[ack_slot] = NULL;
658 ++ack_slot;
659 ack_slot &= (RIOCM_TX_RING_SIZE - 1);
660 cm->tx_cnt--;
661 }
662
663 if (cm->tx_cnt < 0 || cm->tx_cnt > RIOCM_TX_RING_SIZE)
664 riocm_error("tx_cnt %d out of sync", cm->tx_cnt);
665
666 WARN_ON((cm->tx_cnt < 0) || (cm->tx_cnt > RIOCM_TX_RING_SIZE));
667
668 cm->tx_ack_slot = ack_slot;
669
670 /*
671 * If there are pending requests, insert them into transmit queue
672 */
673 if (!list_empty(&cm->tx_reqs) && (cm->tx_cnt < RIOCM_TX_RING_SIZE)) {
674 struct tx_req *req, *_req;
675 int rc;
676
677 list_for_each_entry_safe(req, _req, &cm->tx_reqs, node) {
678 list_del(&req->node);
679 cm->tx_buf[cm->tx_slot] = req->buffer;
680 rc = rio_add_outb_message(cm->mport, req->rdev, cmbox,
681 req->buffer, req->len);
682 kfree(req->buffer);
683 kfree(req);
684
685 ++cm->tx_cnt;
686 ++cm->tx_slot;
687 cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
688 if (cm->tx_cnt == RIOCM_TX_RING_SIZE)
689 break;
690 }
691 }
692
693 spin_unlock(&cm->tx_lock);
694 }
695
riocm_outb_msg_event(struct rio_mport * mport,void * dev_id,int mbox,int slot)696 static void riocm_outb_msg_event(struct rio_mport *mport, void *dev_id,
697 int mbox, int slot)
698 {
699 struct cm_dev *cm = dev_id;
700
701 if (cm && rio_mport_is_running(cm->mport))
702 rio_txcq_handler(cm, slot);
703 }
704
riocm_queue_req(struct cm_dev * cm,struct rio_dev * rdev,void * buffer,size_t len)705 static int riocm_queue_req(struct cm_dev *cm, struct rio_dev *rdev,
706 void *buffer, size_t len)
707 {
708 unsigned long flags;
709 struct tx_req *treq;
710
711 treq = kzalloc(sizeof(*treq), GFP_KERNEL);
712 if (treq == NULL)
713 return -ENOMEM;
714
715 treq->rdev = rdev;
716 treq->buffer = buffer;
717 treq->len = len;
718
719 spin_lock_irqsave(&cm->tx_lock, flags);
720 list_add_tail(&treq->node, &cm->tx_reqs);
721 spin_unlock_irqrestore(&cm->tx_lock, flags);
722 return 0;
723 }
724
725 /*
726 * riocm_post_send - helper function that places packet into msg TX queue
727 * @cm: cm_dev object
728 * @rdev: target RapidIO device object (required by outbound msg interface)
729 * @buffer: pointer to a packet buffer to send
730 * @len: length of data to transfer
731 * @req: request priority flag
732 *
733 * Returns: 0 if success, or error code otherwise.
734 */
riocm_post_send(struct cm_dev * cm,struct rio_dev * rdev,void * buffer,size_t len)735 static int riocm_post_send(struct cm_dev *cm, struct rio_dev *rdev,
736 void *buffer, size_t len)
737 {
738 int rc;
739 unsigned long flags;
740
741 spin_lock_irqsave(&cm->tx_lock, flags);
742
743 if (cm->mport == NULL) {
744 rc = -ENODEV;
745 goto err_out;
746 }
747
748 if (cm->tx_cnt == RIOCM_TX_RING_SIZE) {
749 riocm_debug(TX, "Tx Queue is full");
750 rc = -EBUSY;
751 goto err_out;
752 }
753
754 cm->tx_buf[cm->tx_slot] = buffer;
755 rc = rio_add_outb_message(cm->mport, rdev, cmbox, buffer, len);
756
757 riocm_debug(TX, "Add buf@%p destid=%x tx_slot=%d tx_cnt=%d",
758 buffer, rdev->destid, cm->tx_slot, cm->tx_cnt);
759
760 ++cm->tx_cnt;
761 ++cm->tx_slot;
762 cm->tx_slot &= (RIOCM_TX_RING_SIZE - 1);
763
764 err_out:
765 spin_unlock_irqrestore(&cm->tx_lock, flags);
766 return rc;
767 }
768
769 /*
770 * riocm_ch_send - sends a data packet to a remote device
771 * @ch_id: local channel ID
772 * @buf: pointer to a data buffer to send (including CM header)
773 * @len: length of data to transfer (including CM header)
774 *
775 * ATTN: ASSUMES THAT THE HEADER SPACE IS RESERVED PART OF THE DATA PACKET
776 *
777 * Returns: 0 if success, or
778 * -EINVAL if one or more input parameters is/are not valid,
779 * -ENODEV if cannot find a channel with specified ID,
780 * -EAGAIN if a channel is not in CONNECTED state,
781 * + error codes returned by HW send routine.
782 */
riocm_ch_send(u16 ch_id,void * buf,int len)783 static int riocm_ch_send(u16 ch_id, void *buf, int len)
784 {
785 struct rio_channel *ch;
786 struct rio_ch_chan_hdr *hdr;
787 int ret;
788
789 if (buf == NULL || ch_id == 0 || len == 0 || len > RIO_MAX_MSG_SIZE)
790 return -EINVAL;
791
792 ch = riocm_get_channel(ch_id);
793 if (!ch) {
794 riocm_error("%s(%d) ch_%d not found", current->comm,
795 task_pid_nr(current), ch_id);
796 return -ENODEV;
797 }
798
799 if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
800 ret = -EAGAIN;
801 goto err_out;
802 }
803
804 /*
805 * Fill buffer header section with corresponding channel data
806 */
807 hdr = buf;
808
809 hdr->bhdr.src_id = htonl(ch->loc_destid);
810 hdr->bhdr.dst_id = htonl(ch->rem_destid);
811 hdr->bhdr.src_mbox = cmbox;
812 hdr->bhdr.dst_mbox = cmbox;
813 hdr->bhdr.type = RIO_CM_CHAN;
814 hdr->ch_op = CM_DATA_MSG;
815 hdr->dst_ch = htons(ch->rem_channel);
816 hdr->src_ch = htons(ch->id);
817 hdr->msg_len = htons((u16)len);
818
819 /* ATTN: the function call below relies on the fact that underlying
820 * HW-specific add_outb_message() routine copies TX data into its own
821 * internal transfer buffer (true for all RIONET compatible mport
822 * drivers). Must be reviewed if mport driver uses the buffer directly.
823 */
824
825 ret = riocm_post_send(ch->cmdev, ch->rdev, buf, len);
826 if (ret)
827 riocm_debug(TX, "ch %d send_err=%d", ch->id, ret);
828 err_out:
829 riocm_put_channel(ch);
830 return ret;
831 }
832
riocm_ch_free_rxbuf(struct rio_channel * ch,void * buf)833 static int riocm_ch_free_rxbuf(struct rio_channel *ch, void *buf)
834 {
835 int i, ret = -EINVAL;
836
837 spin_lock_bh(&ch->lock);
838
839 for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
840 if (ch->rx_ring.inuse[i] == buf) {
841 ch->rx_ring.inuse[i] = NULL;
842 ch->rx_ring.inuse_cnt--;
843 ret = 0;
844 break;
845 }
846 }
847
848 spin_unlock_bh(&ch->lock);
849
850 if (!ret)
851 kfree(buf);
852
853 return ret;
854 }
855
856 /*
857 * riocm_ch_receive - fetch a data packet received for the specified channel
858 * @ch: local channel ID
859 * @buf: pointer to a packet buffer
860 * @timeout: timeout to wait for incoming packet (in jiffies)
861 *
862 * Returns: 0 and valid buffer pointer if success, or NULL pointer and one of:
863 * -EAGAIN if a channel is not in CONNECTED state,
864 * -ENOMEM if in-use tracking queue is full,
865 * -ETIME if wait timeout expired,
866 * -EINTR if wait was interrupted.
867 */
riocm_ch_receive(struct rio_channel * ch,void ** buf,long timeout)868 static int riocm_ch_receive(struct rio_channel *ch, void **buf, long timeout)
869 {
870 void *rxmsg = NULL;
871 int i, ret = 0;
872 long wret;
873
874 if (!riocm_cmp(ch, RIO_CM_CONNECTED)) {
875 ret = -EAGAIN;
876 goto out;
877 }
878
879 if (ch->rx_ring.inuse_cnt == RIOCM_RX_RING_SIZE) {
880 /* If we do not have entries to track buffers given to upper
881 * layer, reject request.
882 */
883 ret = -ENOMEM;
884 goto out;
885 }
886
887 wret = wait_for_completion_interruptible_timeout(&ch->comp, timeout);
888
889 riocm_debug(WAIT, "wait on %d returned %ld", ch->id, wret);
890
891 if (!wret)
892 ret = -ETIME;
893 else if (wret == -ERESTARTSYS)
894 ret = -EINTR;
895 else
896 ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -ECONNRESET;
897
898 if (ret)
899 goto out;
900
901 spin_lock_bh(&ch->lock);
902
903 rxmsg = ch->rx_ring.buf[ch->rx_ring.tail];
904 ch->rx_ring.buf[ch->rx_ring.tail] = NULL;
905 ch->rx_ring.count--;
906 ch->rx_ring.tail++;
907 ch->rx_ring.tail %= RIOCM_RX_RING_SIZE;
908 ret = -ENOMEM;
909
910 for (i = 0; i < RIOCM_RX_RING_SIZE; i++) {
911 if (ch->rx_ring.inuse[i] == NULL) {
912 ch->rx_ring.inuse[i] = rxmsg;
913 ch->rx_ring.inuse_cnt++;
914 ret = 0;
915 break;
916 }
917 }
918
919 if (ret) {
920 /* We have no entry to store pending message: drop it */
921 kfree(rxmsg);
922 rxmsg = NULL;
923 }
924
925 spin_unlock_bh(&ch->lock);
926 out:
927 *buf = rxmsg;
928 return ret;
929 }
930
931 /*
932 * riocm_ch_connect - sends a connect request to a remote device
933 * @loc_ch: local channel ID
934 * @cm: CM device to send connect request
935 * @peer: target RapidIO device
936 * @rem_ch: remote channel ID
937 *
938 * Returns: 0 if success, or
939 * -EINVAL if the channel is not in IDLE state,
940 * -EAGAIN if no connection request available immediately,
941 * -ETIME if ACK response timeout expired,
942 * -EINTR if wait for response was interrupted.
943 */
riocm_ch_connect(u16 loc_ch,struct cm_dev * cm,struct cm_peer * peer,u16 rem_ch)944 static int riocm_ch_connect(u16 loc_ch, struct cm_dev *cm,
945 struct cm_peer *peer, u16 rem_ch)
946 {
947 struct rio_channel *ch = NULL;
948 struct rio_ch_chan_hdr *hdr;
949 int ret;
950 long wret;
951
952 ch = riocm_get_channel(loc_ch);
953 if (!ch)
954 return -ENODEV;
955
956 if (!riocm_cmp_exch(ch, RIO_CM_IDLE, RIO_CM_CONNECT)) {
957 ret = -EINVAL;
958 goto conn_done;
959 }
960
961 ch->cmdev = cm;
962 ch->rdev = peer->rdev;
963 ch->context = NULL;
964 ch->loc_destid = cm->mport->host_deviceid;
965 ch->rem_channel = rem_ch;
966
967 /*
968 * Send connect request to the remote RapidIO device
969 */
970
971 hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
972 if (hdr == NULL) {
973 ret = -ENOMEM;
974 goto conn_done;
975 }
976
977 hdr->bhdr.src_id = htonl(ch->loc_destid);
978 hdr->bhdr.dst_id = htonl(peer->rdev->destid);
979 hdr->bhdr.src_mbox = cmbox;
980 hdr->bhdr.dst_mbox = cmbox;
981 hdr->bhdr.type = RIO_CM_CHAN;
982 hdr->ch_op = CM_CONN_REQ;
983 hdr->dst_ch = htons(rem_ch);
984 hdr->src_ch = htons(loc_ch);
985
986 /* ATTN: the function call below relies on the fact that underlying
987 * HW-specific add_outb_message() routine copies TX data into its
988 * internal transfer buffer. Must be reviewed if mport driver uses
989 * this buffer directly.
990 */
991 ret = riocm_post_send(cm, peer->rdev, hdr, sizeof(*hdr));
992
993 if (ret != -EBUSY) {
994 kfree(hdr);
995 } else {
996 ret = riocm_queue_req(cm, peer->rdev, hdr, sizeof(*hdr));
997 if (ret)
998 kfree(hdr);
999 }
1000
1001 if (ret) {
1002 riocm_cmp_exch(ch, RIO_CM_CONNECT, RIO_CM_IDLE);
1003 goto conn_done;
1004 }
1005
1006 /* Wait for connect response from the remote device */
1007 wret = wait_for_completion_interruptible_timeout(&ch->comp,
1008 RIOCM_CONNECT_TO * HZ);
1009 riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1010
1011 if (!wret)
1012 ret = -ETIME;
1013 else if (wret == -ERESTARTSYS)
1014 ret = -EINTR;
1015 else
1016 ret = riocm_cmp(ch, RIO_CM_CONNECTED) ? 0 : -1;
1017
1018 conn_done:
1019 riocm_put_channel(ch);
1020 return ret;
1021 }
1022
riocm_send_ack(struct rio_channel * ch)1023 static int riocm_send_ack(struct rio_channel *ch)
1024 {
1025 struct rio_ch_chan_hdr *hdr;
1026 int ret;
1027
1028 hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1029 if (hdr == NULL)
1030 return -ENOMEM;
1031
1032 hdr->bhdr.src_id = htonl(ch->loc_destid);
1033 hdr->bhdr.dst_id = htonl(ch->rem_destid);
1034 hdr->dst_ch = htons(ch->rem_channel);
1035 hdr->src_ch = htons(ch->id);
1036 hdr->bhdr.src_mbox = cmbox;
1037 hdr->bhdr.dst_mbox = cmbox;
1038 hdr->bhdr.type = RIO_CM_CHAN;
1039 hdr->ch_op = CM_CONN_ACK;
1040
1041 /* ATTN: the function call below relies on the fact that underlying
1042 * add_outb_message() routine copies TX data into its internal transfer
1043 * buffer. Review if switching to direct buffer version.
1044 */
1045 ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1046
1047 if (ret == -EBUSY && !riocm_queue_req(ch->cmdev,
1048 ch->rdev, hdr, sizeof(*hdr)))
1049 return 0;
1050 kfree(hdr);
1051
1052 if (ret)
1053 riocm_error("send ACK to ch_%d on %s failed (ret=%d)",
1054 ch->id, rio_name(ch->rdev), ret);
1055 return ret;
1056 }
1057
1058 /*
1059 * riocm_ch_accept - accept incoming connection request
1060 * @ch_id: channel ID
1061 * @new_ch_id: local mport device
1062 * @timeout: wait timeout (if 0 non-blocking call, do not wait if connection
1063 * request is not available).
1064 *
1065 * Returns: pointer to new channel struct if success, or error-valued pointer:
1066 * -ENODEV - cannot find specified channel or mport,
1067 * -EINVAL - the channel is not in IDLE state,
1068 * -EAGAIN - no connection request available immediately (timeout=0),
1069 * -ENOMEM - unable to allocate new channel,
1070 * -ETIME - wait timeout expired,
1071 * -EINTR - wait was interrupted.
1072 */
riocm_ch_accept(u16 ch_id,u16 * new_ch_id,long timeout)1073 static struct rio_channel *riocm_ch_accept(u16 ch_id, u16 *new_ch_id,
1074 long timeout)
1075 {
1076 struct rio_channel *ch;
1077 struct rio_channel *new_ch;
1078 struct conn_req *req;
1079 struct cm_peer *peer;
1080 int found = 0;
1081 int err = 0;
1082 long wret;
1083
1084 ch = riocm_get_channel(ch_id);
1085 if (!ch)
1086 return ERR_PTR(-EINVAL);
1087
1088 if (!riocm_cmp(ch, RIO_CM_LISTEN)) {
1089 err = -EINVAL;
1090 goto err_put;
1091 }
1092
1093 /* Don't sleep if this is a non blocking call */
1094 if (!timeout) {
1095 if (!try_wait_for_completion(&ch->comp)) {
1096 err = -EAGAIN;
1097 goto err_put;
1098 }
1099 } else {
1100 riocm_debug(WAIT, "on %d", ch->id);
1101
1102 wret = wait_for_completion_interruptible_timeout(&ch->comp,
1103 timeout);
1104 if (!wret) {
1105 err = -ETIME;
1106 goto err_put;
1107 } else if (wret == -ERESTARTSYS) {
1108 err = -EINTR;
1109 goto err_put;
1110 }
1111 }
1112
1113 spin_lock_bh(&ch->lock);
1114
1115 if (ch->state != RIO_CM_LISTEN) {
1116 err = -ECANCELED;
1117 } else if (list_empty(&ch->accept_queue)) {
1118 riocm_debug(WAIT, "on %d accept_queue is empty on completion",
1119 ch->id);
1120 err = -EIO;
1121 }
1122
1123 spin_unlock_bh(&ch->lock);
1124
1125 if (err) {
1126 riocm_debug(WAIT, "on %d returns %d", ch->id, err);
1127 goto err_put;
1128 }
1129
1130 /* Create new channel for this connection */
1131 new_ch = riocm_ch_alloc(RIOCM_CHNUM_AUTO);
1132
1133 if (IS_ERR(new_ch)) {
1134 riocm_error("failed to get channel for new req (%ld)",
1135 PTR_ERR(new_ch));
1136 err = -ENOMEM;
1137 goto err_put;
1138 }
1139
1140 spin_lock_bh(&ch->lock);
1141
1142 req = list_first_entry(&ch->accept_queue, struct conn_req, node);
1143 list_del(&req->node);
1144 new_ch->cmdev = ch->cmdev;
1145 new_ch->loc_destid = ch->loc_destid;
1146 new_ch->rem_destid = req->destid;
1147 new_ch->rem_channel = req->chan;
1148
1149 spin_unlock_bh(&ch->lock);
1150 riocm_put_channel(ch);
1151 ch = NULL;
1152 kfree(req);
1153
1154 down_read(&rdev_sem);
1155 /* Find requester's device object */
1156 list_for_each_entry(peer, &new_ch->cmdev->peers, node) {
1157 if (peer->rdev->destid == new_ch->rem_destid) {
1158 riocm_debug(RX_CMD, "found matching device(%s)",
1159 rio_name(peer->rdev));
1160 found = 1;
1161 break;
1162 }
1163 }
1164 up_read(&rdev_sem);
1165
1166 if (!found) {
1167 /* If peer device object not found, simply ignore the request */
1168 err = -ENODEV;
1169 goto err_put_new_ch;
1170 }
1171
1172 new_ch->rdev = peer->rdev;
1173 new_ch->state = RIO_CM_CONNECTED;
1174 spin_lock_init(&new_ch->lock);
1175
1176 /* Acknowledge the connection request. */
1177 riocm_send_ack(new_ch);
1178
1179 *new_ch_id = new_ch->id;
1180 return new_ch;
1181
1182 err_put_new_ch:
1183 spin_lock_bh(&idr_lock);
1184 idr_remove(&ch_idr, new_ch->id);
1185 spin_unlock_bh(&idr_lock);
1186 riocm_put_channel(new_ch);
1187
1188 err_put:
1189 if (ch)
1190 riocm_put_channel(ch);
1191 *new_ch_id = 0;
1192 return ERR_PTR(err);
1193 }
1194
1195 /*
1196 * riocm_ch_listen - puts a channel into LISTEN state
1197 * @ch_id: channel ID
1198 *
1199 * Returns: 0 if success, or
1200 * -EINVAL if the specified channel does not exists or
1201 * is not in CHAN_BOUND state.
1202 */
riocm_ch_listen(u16 ch_id)1203 static int riocm_ch_listen(u16 ch_id)
1204 {
1205 struct rio_channel *ch = NULL;
1206 int ret = 0;
1207
1208 riocm_debug(CHOP, "(ch_%d)", ch_id);
1209
1210 ch = riocm_get_channel(ch_id);
1211 if (!ch)
1212 return -EINVAL;
1213 if (!riocm_cmp_exch(ch, RIO_CM_CHAN_BOUND, RIO_CM_LISTEN))
1214 ret = -EINVAL;
1215 riocm_put_channel(ch);
1216 return ret;
1217 }
1218
1219 /*
1220 * riocm_ch_bind - associate a channel object and an mport device
1221 * @ch_id: channel ID
1222 * @mport_id: local mport device ID
1223 * @context: pointer to the additional caller's context
1224 *
1225 * Returns: 0 if success, or
1226 * -ENODEV if cannot find specified mport,
1227 * -EINVAL if the specified channel does not exist or
1228 * is not in IDLE state.
1229 */
riocm_ch_bind(u16 ch_id,u8 mport_id,void * context)1230 static int riocm_ch_bind(u16 ch_id, u8 mport_id, void *context)
1231 {
1232 struct rio_channel *ch = NULL;
1233 struct cm_dev *cm;
1234 int rc = -ENODEV;
1235
1236 riocm_debug(CHOP, "ch_%d to mport_%d", ch_id, mport_id);
1237
1238 /* Find matching cm_dev object */
1239 down_read(&rdev_sem);
1240 list_for_each_entry(cm, &cm_dev_list, list) {
1241 if ((cm->mport->id == mport_id) &&
1242 rio_mport_is_running(cm->mport)) {
1243 rc = 0;
1244 break;
1245 }
1246 }
1247
1248 if (rc)
1249 goto exit;
1250
1251 ch = riocm_get_channel(ch_id);
1252 if (!ch) {
1253 rc = -EINVAL;
1254 goto exit;
1255 }
1256
1257 spin_lock_bh(&ch->lock);
1258 if (ch->state != RIO_CM_IDLE) {
1259 spin_unlock_bh(&ch->lock);
1260 rc = -EINVAL;
1261 goto err_put;
1262 }
1263
1264 ch->cmdev = cm;
1265 ch->loc_destid = cm->mport->host_deviceid;
1266 ch->context = context;
1267 ch->state = RIO_CM_CHAN_BOUND;
1268 spin_unlock_bh(&ch->lock);
1269 err_put:
1270 riocm_put_channel(ch);
1271 exit:
1272 up_read(&rdev_sem);
1273 return rc;
1274 }
1275
1276 /*
1277 * riocm_ch_alloc - channel object allocation helper routine
1278 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1279 *
1280 * Return value: pointer to newly created channel object,
1281 * or error-valued pointer
1282 */
riocm_ch_alloc(u16 ch_num)1283 static struct rio_channel *riocm_ch_alloc(u16 ch_num)
1284 {
1285 int id;
1286 int start, end;
1287 struct rio_channel *ch;
1288
1289 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
1290 if (!ch)
1291 return ERR_PTR(-ENOMEM);
1292
1293 if (ch_num) {
1294 /* If requested, try to obtain the specified channel ID */
1295 start = ch_num;
1296 end = ch_num + 1;
1297 } else {
1298 /* Obtain channel ID from the dynamic allocation range */
1299 start = chstart;
1300 end = RIOCM_MAX_CHNUM + 1;
1301 }
1302
1303 idr_preload(GFP_KERNEL);
1304 spin_lock_bh(&idr_lock);
1305 id = idr_alloc_cyclic(&ch_idr, ch, start, end, GFP_NOWAIT);
1306 spin_unlock_bh(&idr_lock);
1307 idr_preload_end();
1308
1309 if (id < 0) {
1310 kfree(ch);
1311 return ERR_PTR(id == -ENOSPC ? -EBUSY : id);
1312 }
1313
1314 ch->id = (u16)id;
1315 ch->state = RIO_CM_IDLE;
1316 spin_lock_init(&ch->lock);
1317 INIT_LIST_HEAD(&ch->accept_queue);
1318 INIT_LIST_HEAD(&ch->ch_node);
1319 init_completion(&ch->comp);
1320 init_completion(&ch->comp_close);
1321 kref_init(&ch->ref);
1322 ch->rx_ring.head = 0;
1323 ch->rx_ring.tail = 0;
1324 ch->rx_ring.count = 0;
1325 ch->rx_ring.inuse_cnt = 0;
1326
1327 return ch;
1328 }
1329
1330 /*
1331 * riocm_ch_create - creates a new channel object and allocates ID for it
1332 * @ch_num: channel ID (1 ... RIOCM_MAX_CHNUM, 0 = automatic)
1333 *
1334 * Allocates and initializes a new channel object. If the parameter ch_num > 0
1335 * and is within the valid range, riocm_ch_create tries to allocate the
1336 * specified ID for the new channel. If ch_num = 0, channel ID will be assigned
1337 * automatically from the range (chstart ... RIOCM_MAX_CHNUM).
1338 * Module parameter 'chstart' defines start of an ID range available for dynamic
1339 * allocation. Range below 'chstart' is reserved for pre-defined ID numbers.
1340 * Available channel numbers are limited by 16-bit size of channel numbers used
1341 * in the packet header.
1342 *
1343 * Return value: PTR to rio_channel structure if successful (with channel number
1344 * updated via pointer) or error-valued pointer if error.
1345 */
riocm_ch_create(u16 * ch_num)1346 static struct rio_channel *riocm_ch_create(u16 *ch_num)
1347 {
1348 struct rio_channel *ch = NULL;
1349
1350 ch = riocm_ch_alloc(*ch_num);
1351
1352 if (IS_ERR(ch))
1353 riocm_debug(CHOP, "Failed to allocate channel %d (err=%ld)",
1354 *ch_num, PTR_ERR(ch));
1355 else
1356 *ch_num = ch->id;
1357
1358 return ch;
1359 }
1360
1361 /*
1362 * riocm_ch_free - channel object release routine
1363 * @ref: pointer to a channel's kref structure
1364 */
riocm_ch_free(struct kref * ref)1365 static void riocm_ch_free(struct kref *ref)
1366 {
1367 struct rio_channel *ch = container_of(ref, struct rio_channel, ref);
1368 int i;
1369
1370 riocm_debug(CHOP, "(ch_%d)", ch->id);
1371
1372 if (ch->rx_ring.inuse_cnt) {
1373 for (i = 0;
1374 i < RIOCM_RX_RING_SIZE && ch->rx_ring.inuse_cnt; i++) {
1375 if (ch->rx_ring.inuse[i] != NULL) {
1376 kfree(ch->rx_ring.inuse[i]);
1377 ch->rx_ring.inuse_cnt--;
1378 }
1379 }
1380 }
1381
1382 if (ch->rx_ring.count)
1383 for (i = 0; i < RIOCM_RX_RING_SIZE && ch->rx_ring.count; i++) {
1384 if (ch->rx_ring.buf[i] != NULL) {
1385 kfree(ch->rx_ring.buf[i]);
1386 ch->rx_ring.count--;
1387 }
1388 }
1389
1390 complete(&ch->comp_close);
1391 }
1392
riocm_send_close(struct rio_channel * ch)1393 static int riocm_send_close(struct rio_channel *ch)
1394 {
1395 struct rio_ch_chan_hdr *hdr;
1396 int ret;
1397
1398 /*
1399 * Send CH_CLOSE notification to the remote RapidIO device
1400 */
1401
1402 hdr = kzalloc(sizeof(*hdr), GFP_KERNEL);
1403 if (hdr == NULL)
1404 return -ENOMEM;
1405
1406 hdr->bhdr.src_id = htonl(ch->loc_destid);
1407 hdr->bhdr.dst_id = htonl(ch->rem_destid);
1408 hdr->bhdr.src_mbox = cmbox;
1409 hdr->bhdr.dst_mbox = cmbox;
1410 hdr->bhdr.type = RIO_CM_CHAN;
1411 hdr->ch_op = CM_CONN_CLOSE;
1412 hdr->dst_ch = htons(ch->rem_channel);
1413 hdr->src_ch = htons(ch->id);
1414
1415 /* ATTN: the function call below relies on the fact that underlying
1416 * add_outb_message() routine copies TX data into its internal transfer
1417 * buffer. Needs to be reviewed if switched to direct buffer mode.
1418 */
1419 ret = riocm_post_send(ch->cmdev, ch->rdev, hdr, sizeof(*hdr));
1420
1421 if (ret == -EBUSY && !riocm_queue_req(ch->cmdev, ch->rdev,
1422 hdr, sizeof(*hdr)))
1423 return 0;
1424 kfree(hdr);
1425
1426 if (ret)
1427 riocm_error("ch(%d) send CLOSE failed (ret=%d)", ch->id, ret);
1428
1429 return ret;
1430 }
1431
1432 /*
1433 * riocm_ch_close - closes a channel object with specified ID (by local request)
1434 * @ch: channel to be closed
1435 */
riocm_ch_close(struct rio_channel * ch)1436 static int riocm_ch_close(struct rio_channel *ch)
1437 {
1438 unsigned long tmo = msecs_to_jiffies(3000);
1439 enum rio_cm_state state;
1440 long wret;
1441 int ret = 0;
1442
1443 riocm_debug(CHOP, "ch_%d by %s(%d)",
1444 ch->id, current->comm, task_pid_nr(current));
1445
1446 state = riocm_exch(ch, RIO_CM_DESTROYING);
1447 if (state == RIO_CM_CONNECTED)
1448 riocm_send_close(ch);
1449
1450 complete_all(&ch->comp);
1451
1452 riocm_put_channel(ch);
1453 wret = wait_for_completion_interruptible_timeout(&ch->comp_close, tmo);
1454
1455 riocm_debug(WAIT, "wait on %d returns %ld", ch->id, wret);
1456
1457 if (wret == 0) {
1458 /* Timeout on wait occurred */
1459 riocm_debug(CHOP, "%s(%d) timed out waiting for ch %d",
1460 current->comm, task_pid_nr(current), ch->id);
1461 ret = -ETIMEDOUT;
1462 } else if (wret == -ERESTARTSYS) {
1463 /* Wait_for_completion was interrupted by a signal */
1464 riocm_debug(CHOP, "%s(%d) wait for ch %d was interrupted",
1465 current->comm, task_pid_nr(current), ch->id);
1466 ret = -EINTR;
1467 }
1468
1469 if (!ret) {
1470 riocm_debug(CHOP, "ch_%d resources released", ch->id);
1471 kfree(ch);
1472 } else {
1473 riocm_debug(CHOP, "failed to release ch_%d resources", ch->id);
1474 }
1475
1476 return ret;
1477 }
1478
1479 /*
1480 * riocm_cdev_open() - Open character device
1481 */
riocm_cdev_open(struct inode * inode,struct file * filp)1482 static int riocm_cdev_open(struct inode *inode, struct file *filp)
1483 {
1484 riocm_debug(INIT, "by %s(%d) filp=%p ",
1485 current->comm, task_pid_nr(current), filp);
1486
1487 if (list_empty(&cm_dev_list))
1488 return -ENODEV;
1489
1490 return 0;
1491 }
1492
1493 /*
1494 * riocm_cdev_release() - Release character device
1495 */
riocm_cdev_release(struct inode * inode,struct file * filp)1496 static int riocm_cdev_release(struct inode *inode, struct file *filp)
1497 {
1498 struct rio_channel *ch, *_c;
1499 unsigned int i;
1500 LIST_HEAD(list);
1501
1502 riocm_debug(EXIT, "by %s(%d) filp=%p",
1503 current->comm, task_pid_nr(current), filp);
1504
1505 /* Check if there are channels associated with this file descriptor */
1506 spin_lock_bh(&idr_lock);
1507 idr_for_each_entry(&ch_idr, ch, i) {
1508 if (ch && ch->filp == filp) {
1509 riocm_debug(EXIT, "ch_%d not released by %s(%d)",
1510 ch->id, current->comm,
1511 task_pid_nr(current));
1512 idr_remove(&ch_idr, ch->id);
1513 list_add(&ch->ch_node, &list);
1514 }
1515 }
1516 spin_unlock_bh(&idr_lock);
1517
1518 if (!list_empty(&list)) {
1519 list_for_each_entry_safe(ch, _c, &list, ch_node) {
1520 list_del(&ch->ch_node);
1521 riocm_ch_close(ch);
1522 }
1523 }
1524
1525 return 0;
1526 }
1527
1528 /*
1529 * cm_ep_get_list_size() - Reports number of endpoints in the network
1530 */
cm_ep_get_list_size(void __user * arg)1531 static int cm_ep_get_list_size(void __user *arg)
1532 {
1533 u32 __user *p = arg;
1534 u32 mport_id;
1535 u32 count = 0;
1536 struct cm_dev *cm;
1537
1538 if (get_user(mport_id, p))
1539 return -EFAULT;
1540 if (mport_id >= RIO_MAX_MPORTS)
1541 return -EINVAL;
1542
1543 /* Find a matching cm_dev object */
1544 down_read(&rdev_sem);
1545 list_for_each_entry(cm, &cm_dev_list, list) {
1546 if (cm->mport->id == mport_id) {
1547 count = cm->npeers;
1548 up_read(&rdev_sem);
1549 if (copy_to_user(arg, &count, sizeof(u32)))
1550 return -EFAULT;
1551 return 0;
1552 }
1553 }
1554 up_read(&rdev_sem);
1555
1556 return -ENODEV;
1557 }
1558
1559 /*
1560 * cm_ep_get_list() - Returns list of attached endpoints
1561 */
cm_ep_get_list(void __user * arg)1562 static int cm_ep_get_list(void __user *arg)
1563 {
1564 struct cm_dev *cm;
1565 struct cm_peer *peer;
1566 u32 info[2];
1567 void *buf;
1568 u32 nent;
1569 u32 *entry_ptr;
1570 u32 i = 0;
1571 int ret = 0;
1572
1573 if (copy_from_user(&info, arg, sizeof(info)))
1574 return -EFAULT;
1575
1576 if (info[1] >= RIO_MAX_MPORTS || info[0] > RIOCM_MAX_EP_COUNT)
1577 return -EINVAL;
1578
1579 /* Find a matching cm_dev object */
1580 down_read(&rdev_sem);
1581 list_for_each_entry(cm, &cm_dev_list, list)
1582 if (cm->mport->id == (u8)info[1])
1583 goto found;
1584
1585 up_read(&rdev_sem);
1586 return -ENODEV;
1587
1588 found:
1589 nent = min(info[0], cm->npeers);
1590 buf = kcalloc(nent + 2, sizeof(u32), GFP_KERNEL);
1591 if (!buf) {
1592 up_read(&rdev_sem);
1593 return -ENOMEM;
1594 }
1595
1596 entry_ptr = (u32 *)((uintptr_t)buf + 2*sizeof(u32));
1597
1598 list_for_each_entry(peer, &cm->peers, node) {
1599 *entry_ptr = (u32)peer->rdev->destid;
1600 entry_ptr++;
1601 if (++i == nent)
1602 break;
1603 }
1604 up_read(&rdev_sem);
1605
1606 ((u32 *)buf)[0] = i; /* report an updated number of entries */
1607 ((u32 *)buf)[1] = info[1]; /* put back an mport ID */
1608 if (copy_to_user(arg, buf, sizeof(u32) * (info[0] + 2)))
1609 ret = -EFAULT;
1610
1611 kfree(buf);
1612 return ret;
1613 }
1614
1615 /*
1616 * cm_mport_get_list() - Returns list of available local mport devices
1617 */
cm_mport_get_list(void __user * arg)1618 static int cm_mport_get_list(void __user *arg)
1619 {
1620 int ret = 0;
1621 u32 entries;
1622 void *buf;
1623 struct cm_dev *cm;
1624 u32 *entry_ptr;
1625 int count = 0;
1626
1627 if (copy_from_user(&entries, arg, sizeof(entries)))
1628 return -EFAULT;
1629 if (entries == 0 || entries > RIO_MAX_MPORTS)
1630 return -EINVAL;
1631 buf = kcalloc(entries + 1, sizeof(u32), GFP_KERNEL);
1632 if (!buf)
1633 return -ENOMEM;
1634
1635 /* Scan all registered cm_dev objects */
1636 entry_ptr = (u32 *)((uintptr_t)buf + sizeof(u32));
1637 down_read(&rdev_sem);
1638 list_for_each_entry(cm, &cm_dev_list, list) {
1639 if (count++ < entries) {
1640 *entry_ptr = (cm->mport->id << 16) |
1641 cm->mport->host_deviceid;
1642 entry_ptr++;
1643 }
1644 }
1645 up_read(&rdev_sem);
1646
1647 *((u32 *)buf) = count; /* report a real number of entries */
1648 if (copy_to_user(arg, buf, sizeof(u32) * (count + 1)))
1649 ret = -EFAULT;
1650
1651 kfree(buf);
1652 return ret;
1653 }
1654
1655 /*
1656 * cm_chan_create() - Create a message exchange channel
1657 */
cm_chan_create(struct file * filp,void __user * arg)1658 static int cm_chan_create(struct file *filp, void __user *arg)
1659 {
1660 u16 __user *p = arg;
1661 u16 ch_num;
1662 struct rio_channel *ch;
1663
1664 if (get_user(ch_num, p))
1665 return -EFAULT;
1666
1667 riocm_debug(CHOP, "ch_%d requested by %s(%d)",
1668 ch_num, current->comm, task_pid_nr(current));
1669 ch = riocm_ch_create(&ch_num);
1670 if (IS_ERR(ch))
1671 return PTR_ERR(ch);
1672
1673 ch->filp = filp;
1674 riocm_debug(CHOP, "ch_%d created by %s(%d)",
1675 ch_num, current->comm, task_pid_nr(current));
1676 return put_user(ch_num, p);
1677 }
1678
1679 /*
1680 * cm_chan_close() - Close channel
1681 * @filp: Pointer to file object
1682 * @arg: Channel to close
1683 */
cm_chan_close(struct file * filp,void __user * arg)1684 static int cm_chan_close(struct file *filp, void __user *arg)
1685 {
1686 u16 __user *p = arg;
1687 u16 ch_num;
1688 struct rio_channel *ch;
1689
1690 if (get_user(ch_num, p))
1691 return -EFAULT;
1692
1693 riocm_debug(CHOP, "ch_%d by %s(%d)",
1694 ch_num, current->comm, task_pid_nr(current));
1695
1696 spin_lock_bh(&idr_lock);
1697 ch = idr_find(&ch_idr, ch_num);
1698 if (!ch) {
1699 spin_unlock_bh(&idr_lock);
1700 return 0;
1701 }
1702 if (ch->filp != filp) {
1703 spin_unlock_bh(&idr_lock);
1704 return -EINVAL;
1705 }
1706 idr_remove(&ch_idr, ch->id);
1707 spin_unlock_bh(&idr_lock);
1708
1709 return riocm_ch_close(ch);
1710 }
1711
1712 /*
1713 * cm_chan_bind() - Bind channel
1714 * @arg: Channel number
1715 */
cm_chan_bind(void __user * arg)1716 static int cm_chan_bind(void __user *arg)
1717 {
1718 struct rio_cm_channel chan;
1719
1720 if (copy_from_user(&chan, arg, sizeof(chan)))
1721 return -EFAULT;
1722 if (chan.mport_id >= RIO_MAX_MPORTS)
1723 return -EINVAL;
1724
1725 return riocm_ch_bind(chan.id, chan.mport_id, NULL);
1726 }
1727
1728 /*
1729 * cm_chan_listen() - Listen on channel
1730 * @arg: Channel number
1731 */
cm_chan_listen(void __user * arg)1732 static int cm_chan_listen(void __user *arg)
1733 {
1734 u16 __user *p = arg;
1735 u16 ch_num;
1736
1737 if (get_user(ch_num, p))
1738 return -EFAULT;
1739
1740 return riocm_ch_listen(ch_num);
1741 }
1742
1743 /*
1744 * cm_chan_accept() - Accept incoming connection
1745 * @filp: Pointer to file object
1746 * @arg: Channel number
1747 */
cm_chan_accept(struct file * filp,void __user * arg)1748 static int cm_chan_accept(struct file *filp, void __user *arg)
1749 {
1750 struct rio_cm_accept param;
1751 long accept_to;
1752 struct rio_channel *ch;
1753
1754 if (copy_from_user(¶m, arg, sizeof(param)))
1755 return -EFAULT;
1756
1757 riocm_debug(CHOP, "on ch_%d by %s(%d)",
1758 param.ch_num, current->comm, task_pid_nr(current));
1759
1760 accept_to = param.wait_to ?
1761 msecs_to_jiffies(param.wait_to) : 0;
1762
1763 ch = riocm_ch_accept(param.ch_num, ¶m.ch_num, accept_to);
1764 if (IS_ERR(ch))
1765 return PTR_ERR(ch);
1766 ch->filp = filp;
1767
1768 riocm_debug(CHOP, "new ch_%d for %s(%d)",
1769 ch->id, current->comm, task_pid_nr(current));
1770
1771 if (copy_to_user(arg, ¶m, sizeof(param)))
1772 return -EFAULT;
1773 return 0;
1774 }
1775
1776 /*
1777 * cm_chan_connect() - Connect on channel
1778 * @arg: Channel information
1779 */
cm_chan_connect(void __user * arg)1780 static int cm_chan_connect(void __user *arg)
1781 {
1782 struct rio_cm_channel chan;
1783 struct cm_dev *cm;
1784 struct cm_peer *peer;
1785 int ret = -ENODEV;
1786
1787 if (copy_from_user(&chan, arg, sizeof(chan)))
1788 return -EFAULT;
1789 if (chan.mport_id >= RIO_MAX_MPORTS)
1790 return -EINVAL;
1791
1792 down_read(&rdev_sem);
1793
1794 /* Find matching cm_dev object */
1795 list_for_each_entry(cm, &cm_dev_list, list) {
1796 if (cm->mport->id == chan.mport_id) {
1797 ret = 0;
1798 break;
1799 }
1800 }
1801
1802 if (ret)
1803 goto err_out;
1804
1805 if (chan.remote_destid >= RIO_ANY_DESTID(cm->mport->sys_size)) {
1806 ret = -EINVAL;
1807 goto err_out;
1808 }
1809
1810 /* Find corresponding RapidIO endpoint device object */
1811 ret = -ENODEV;
1812
1813 list_for_each_entry(peer, &cm->peers, node) {
1814 if (peer->rdev->destid == chan.remote_destid) {
1815 ret = 0;
1816 break;
1817 }
1818 }
1819
1820 if (ret)
1821 goto err_out;
1822
1823 up_read(&rdev_sem);
1824
1825 return riocm_ch_connect(chan.id, cm, peer, chan.remote_channel);
1826 err_out:
1827 up_read(&rdev_sem);
1828 return ret;
1829 }
1830
1831 /*
1832 * cm_chan_msg_send() - Send a message through channel
1833 * @arg: Outbound message information
1834 */
cm_chan_msg_send(void __user * arg)1835 static int cm_chan_msg_send(void __user *arg)
1836 {
1837 struct rio_cm_msg msg;
1838 void *buf;
1839 int ret;
1840
1841 if (copy_from_user(&msg, arg, sizeof(msg)))
1842 return -EFAULT;
1843 if (msg.size > RIO_MAX_MSG_SIZE)
1844 return -EINVAL;
1845
1846 buf = memdup_user((void __user *)(uintptr_t)msg.msg, msg.size);
1847 if (IS_ERR(buf))
1848 return PTR_ERR(buf);
1849
1850 ret = riocm_ch_send(msg.ch_num, buf, msg.size);
1851
1852 kfree(buf);
1853 return ret;
1854 }
1855
1856 /*
1857 * cm_chan_msg_rcv() - Receive a message through channel
1858 * @arg: Inbound message information
1859 */
cm_chan_msg_rcv(void __user * arg)1860 static int cm_chan_msg_rcv(void __user *arg)
1861 {
1862 struct rio_cm_msg msg;
1863 struct rio_channel *ch;
1864 void *buf;
1865 long rxto;
1866 int ret = 0, msg_size;
1867
1868 if (copy_from_user(&msg, arg, sizeof(msg)))
1869 return -EFAULT;
1870
1871 if (msg.ch_num == 0 || msg.size == 0)
1872 return -EINVAL;
1873
1874 ch = riocm_get_channel(msg.ch_num);
1875 if (!ch)
1876 return -ENODEV;
1877
1878 rxto = msg.rxto ? msecs_to_jiffies(msg.rxto) : MAX_SCHEDULE_TIMEOUT;
1879
1880 ret = riocm_ch_receive(ch, &buf, rxto);
1881 if (ret)
1882 goto out;
1883
1884 msg_size = min(msg.size, (u16)(RIO_MAX_MSG_SIZE));
1885
1886 if (copy_to_user((void __user *)(uintptr_t)msg.msg, buf, msg_size))
1887 ret = -EFAULT;
1888
1889 riocm_ch_free_rxbuf(ch, buf);
1890 out:
1891 riocm_put_channel(ch);
1892 return ret;
1893 }
1894
1895 /*
1896 * riocm_cdev_ioctl() - IOCTL requests handler
1897 */
1898 static long
riocm_cdev_ioctl(struct file * filp,unsigned int cmd,unsigned long arg)1899 riocm_cdev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1900 {
1901 switch (cmd) {
1902 case RIO_CM_EP_GET_LIST_SIZE:
1903 return cm_ep_get_list_size((void __user *)arg);
1904 case RIO_CM_EP_GET_LIST:
1905 return cm_ep_get_list((void __user *)arg);
1906 case RIO_CM_CHAN_CREATE:
1907 return cm_chan_create(filp, (void __user *)arg);
1908 case RIO_CM_CHAN_CLOSE:
1909 return cm_chan_close(filp, (void __user *)arg);
1910 case RIO_CM_CHAN_BIND:
1911 return cm_chan_bind((void __user *)arg);
1912 case RIO_CM_CHAN_LISTEN:
1913 return cm_chan_listen((void __user *)arg);
1914 case RIO_CM_CHAN_ACCEPT:
1915 return cm_chan_accept(filp, (void __user *)arg);
1916 case RIO_CM_CHAN_CONNECT:
1917 return cm_chan_connect((void __user *)arg);
1918 case RIO_CM_CHAN_SEND:
1919 return cm_chan_msg_send((void __user *)arg);
1920 case RIO_CM_CHAN_RECEIVE:
1921 return cm_chan_msg_rcv((void __user *)arg);
1922 case RIO_CM_MPORT_GET_LIST:
1923 return cm_mport_get_list((void __user *)arg);
1924 default:
1925 break;
1926 }
1927
1928 return -EINVAL;
1929 }
1930
1931 static const struct file_operations riocm_cdev_fops = {
1932 .owner = THIS_MODULE,
1933 .open = riocm_cdev_open,
1934 .release = riocm_cdev_release,
1935 .unlocked_ioctl = riocm_cdev_ioctl,
1936 };
1937
1938 /*
1939 * riocm_add_dev - add new remote RapidIO device into channel management core
1940 * @dev: device object associated with RapidIO device
1941 * @sif: subsystem interface
1942 *
1943 * Adds the specified RapidIO device (if applicable) into peers list of
1944 * the corresponding channel management device (cm_dev).
1945 */
riocm_add_dev(struct device * dev,struct subsys_interface * sif)1946 static int riocm_add_dev(struct device *dev, struct subsys_interface *sif)
1947 {
1948 struct cm_peer *peer;
1949 struct rio_dev *rdev = to_rio_dev(dev);
1950 struct cm_dev *cm;
1951
1952 /* Check if the remote device has capabilities required to support CM */
1953 if (!dev_cm_capable(rdev))
1954 return 0;
1955
1956 riocm_debug(RDEV, "(%s)", rio_name(rdev));
1957
1958 peer = kmalloc(sizeof(*peer), GFP_KERNEL);
1959 if (!peer)
1960 return -ENOMEM;
1961
1962 /* Find a corresponding cm_dev object */
1963 down_write(&rdev_sem);
1964 list_for_each_entry(cm, &cm_dev_list, list) {
1965 if (cm->mport == rdev->net->hport)
1966 goto found;
1967 }
1968
1969 up_write(&rdev_sem);
1970 kfree(peer);
1971 return -ENODEV;
1972
1973 found:
1974 peer->rdev = rdev;
1975 list_add_tail(&peer->node, &cm->peers);
1976 cm->npeers++;
1977
1978 up_write(&rdev_sem);
1979 return 0;
1980 }
1981
1982 /*
1983 * riocm_remove_dev - remove remote RapidIO device from channel management core
1984 * @dev: device object associated with RapidIO device
1985 * @sif: subsystem interface
1986 *
1987 * Removes the specified RapidIO device (if applicable) from peers list of
1988 * the corresponding channel management device (cm_dev).
1989 */
riocm_remove_dev(struct device * dev,struct subsys_interface * sif)1990 static void riocm_remove_dev(struct device *dev, struct subsys_interface *sif)
1991 {
1992 struct rio_dev *rdev = to_rio_dev(dev);
1993 struct cm_dev *cm;
1994 struct cm_peer *peer;
1995 struct rio_channel *ch, *_c;
1996 unsigned int i;
1997 bool found = false;
1998 LIST_HEAD(list);
1999
2000 /* Check if the remote device has capabilities required to support CM */
2001 if (!dev_cm_capable(rdev))
2002 return;
2003
2004 riocm_debug(RDEV, "(%s)", rio_name(rdev));
2005
2006 /* Find matching cm_dev object */
2007 down_write(&rdev_sem);
2008 list_for_each_entry(cm, &cm_dev_list, list) {
2009 if (cm->mport == rdev->net->hport) {
2010 found = true;
2011 break;
2012 }
2013 }
2014
2015 if (!found) {
2016 up_write(&rdev_sem);
2017 return;
2018 }
2019
2020 /* Remove remote device from the list of peers */
2021 found = false;
2022 list_for_each_entry(peer, &cm->peers, node) {
2023 if (peer->rdev == rdev) {
2024 riocm_debug(RDEV, "removing peer %s", rio_name(rdev));
2025 found = true;
2026 list_del(&peer->node);
2027 cm->npeers--;
2028 kfree(peer);
2029 break;
2030 }
2031 }
2032
2033 up_write(&rdev_sem);
2034
2035 if (!found)
2036 return;
2037
2038 /*
2039 * Release channels associated with this peer
2040 */
2041
2042 spin_lock_bh(&idr_lock);
2043 idr_for_each_entry(&ch_idr, ch, i) {
2044 if (ch && ch->rdev == rdev) {
2045 if (atomic_read(&rdev->state) != RIO_DEVICE_SHUTDOWN)
2046 riocm_exch(ch, RIO_CM_DISCONNECT);
2047 idr_remove(&ch_idr, ch->id);
2048 list_add(&ch->ch_node, &list);
2049 }
2050 }
2051 spin_unlock_bh(&idr_lock);
2052
2053 if (!list_empty(&list)) {
2054 list_for_each_entry_safe(ch, _c, &list, ch_node) {
2055 list_del(&ch->ch_node);
2056 riocm_ch_close(ch);
2057 }
2058 }
2059 }
2060
2061 /*
2062 * riocm_cdev_add() - Create rio_cm char device
2063 * @devno: device number assigned to device (MAJ + MIN)
2064 */
riocm_cdev_add(dev_t devno)2065 static int riocm_cdev_add(dev_t devno)
2066 {
2067 int ret;
2068
2069 cdev_init(&riocm_cdev.cdev, &riocm_cdev_fops);
2070 riocm_cdev.cdev.owner = THIS_MODULE;
2071 ret = cdev_add(&riocm_cdev.cdev, devno, 1);
2072 if (ret < 0) {
2073 riocm_error("Cannot register a device with error %d", ret);
2074 return ret;
2075 }
2076
2077 riocm_cdev.dev = device_create(&dev_class, NULL, devno, NULL, DEV_NAME);
2078 if (IS_ERR(riocm_cdev.dev)) {
2079 cdev_del(&riocm_cdev.cdev);
2080 return PTR_ERR(riocm_cdev.dev);
2081 }
2082
2083 riocm_debug(MPORT, "Added %s cdev(%d:%d)",
2084 DEV_NAME, MAJOR(devno), MINOR(devno));
2085
2086 return 0;
2087 }
2088
2089 /*
2090 * riocm_add_mport - add new local mport device into channel management core
2091 * @dev: device object associated with mport
2092 *
2093 * When a new mport device is added, CM immediately reserves inbound and
2094 * outbound RapidIO mailboxes that will be used.
2095 */
riocm_add_mport(struct device * dev)2096 static int riocm_add_mport(struct device *dev)
2097 {
2098 int rc;
2099 int i;
2100 struct cm_dev *cm;
2101 struct rio_mport *mport = to_rio_mport(dev);
2102
2103 riocm_debug(MPORT, "add mport %s", mport->name);
2104
2105 cm = kzalloc(sizeof(*cm), GFP_KERNEL);
2106 if (!cm)
2107 return -ENOMEM;
2108
2109 cm->mport = mport;
2110
2111 rc = rio_request_outb_mbox(mport, cm, cmbox,
2112 RIOCM_TX_RING_SIZE, riocm_outb_msg_event);
2113 if (rc) {
2114 riocm_error("failed to allocate OBMBOX_%d on %s",
2115 cmbox, mport->name);
2116 kfree(cm);
2117 return -ENODEV;
2118 }
2119
2120 rc = rio_request_inb_mbox(mport, cm, cmbox,
2121 RIOCM_RX_RING_SIZE, riocm_inb_msg_event);
2122 if (rc) {
2123 riocm_error("failed to allocate IBMBOX_%d on %s",
2124 cmbox, mport->name);
2125 rio_release_outb_mbox(mport, cmbox);
2126 kfree(cm);
2127 return -ENODEV;
2128 }
2129
2130 cm->rx_wq = create_workqueue(DRV_NAME "/rxq");
2131 if (!cm->rx_wq) {
2132 rio_release_inb_mbox(mport, cmbox);
2133 rio_release_outb_mbox(mport, cmbox);
2134 kfree(cm);
2135 return -ENOMEM;
2136 }
2137
2138 /*
2139 * Allocate and register inbound messaging buffers to be ready
2140 * to receive channel and system management requests
2141 */
2142 for (i = 0; i < RIOCM_RX_RING_SIZE; i++)
2143 cm->rx_buf[i] = NULL;
2144
2145 cm->rx_slots = RIOCM_RX_RING_SIZE;
2146 mutex_init(&cm->rx_lock);
2147 riocm_rx_fill(cm, RIOCM_RX_RING_SIZE);
2148 INIT_WORK(&cm->rx_work, rio_ibmsg_handler);
2149
2150 cm->tx_slot = 0;
2151 cm->tx_cnt = 0;
2152 cm->tx_ack_slot = 0;
2153 spin_lock_init(&cm->tx_lock);
2154
2155 INIT_LIST_HEAD(&cm->peers);
2156 cm->npeers = 0;
2157 INIT_LIST_HEAD(&cm->tx_reqs);
2158
2159 down_write(&rdev_sem);
2160 list_add_tail(&cm->list, &cm_dev_list);
2161 up_write(&rdev_sem);
2162
2163 return 0;
2164 }
2165
2166 /*
2167 * riocm_remove_mport - remove local mport device from channel management core
2168 * @dev: device object associated with mport
2169 *
2170 * Removes a local mport device from the list of registered devices that provide
2171 * channel management services. Returns an error if the specified mport is not
2172 * registered with the CM core.
2173 */
riocm_remove_mport(struct device * dev)2174 static void riocm_remove_mport(struct device *dev)
2175 {
2176 struct rio_mport *mport = to_rio_mport(dev);
2177 struct cm_dev *cm;
2178 struct cm_peer *peer, *temp;
2179 struct rio_channel *ch, *_c;
2180 unsigned int i;
2181 bool found = false;
2182 LIST_HEAD(list);
2183
2184 riocm_debug(MPORT, "%s", mport->name);
2185
2186 /* Find a matching cm_dev object */
2187 down_write(&rdev_sem);
2188 list_for_each_entry(cm, &cm_dev_list, list) {
2189 if (cm->mport == mport) {
2190 list_del(&cm->list);
2191 found = true;
2192 break;
2193 }
2194 }
2195 up_write(&rdev_sem);
2196 if (!found)
2197 return;
2198
2199 flush_workqueue(cm->rx_wq);
2200 destroy_workqueue(cm->rx_wq);
2201
2202 /* Release channels bound to this mport */
2203 spin_lock_bh(&idr_lock);
2204 idr_for_each_entry(&ch_idr, ch, i) {
2205 if (ch->cmdev == cm) {
2206 riocm_debug(RDEV, "%s drop ch_%d",
2207 mport->name, ch->id);
2208 idr_remove(&ch_idr, ch->id);
2209 list_add(&ch->ch_node, &list);
2210 }
2211 }
2212 spin_unlock_bh(&idr_lock);
2213
2214 if (!list_empty(&list)) {
2215 list_for_each_entry_safe(ch, _c, &list, ch_node) {
2216 list_del(&ch->ch_node);
2217 riocm_ch_close(ch);
2218 }
2219 }
2220
2221 rio_release_inb_mbox(mport, cmbox);
2222 rio_release_outb_mbox(mport, cmbox);
2223
2224 /* Remove and free peer entries */
2225 if (!list_empty(&cm->peers))
2226 riocm_debug(RDEV, "ATTN: peer list not empty");
2227 list_for_each_entry_safe(peer, temp, &cm->peers, node) {
2228 riocm_debug(RDEV, "removing peer %s", rio_name(peer->rdev));
2229 list_del(&peer->node);
2230 kfree(peer);
2231 }
2232
2233 riocm_rx_free(cm);
2234 kfree(cm);
2235 riocm_debug(MPORT, "%s done", mport->name);
2236 }
2237
rio_cm_shutdown(struct notifier_block * nb,unsigned long code,void * unused)2238 static int rio_cm_shutdown(struct notifier_block *nb, unsigned long code,
2239 void *unused)
2240 {
2241 struct rio_channel *ch;
2242 unsigned int i;
2243 LIST_HEAD(list);
2244
2245 riocm_debug(EXIT, ".");
2246
2247 /*
2248 * If there are any channels left in connected state send
2249 * close notification to the connection partner.
2250 * First build a list of channels that require a closing
2251 * notification because function riocm_send_close() should
2252 * be called outside of spinlock protected code.
2253 */
2254 spin_lock_bh(&idr_lock);
2255 idr_for_each_entry(&ch_idr, ch, i) {
2256 if (ch->state == RIO_CM_CONNECTED) {
2257 riocm_debug(EXIT, "close ch %d", ch->id);
2258 idr_remove(&ch_idr, ch->id);
2259 list_add(&ch->ch_node, &list);
2260 }
2261 }
2262 spin_unlock_bh(&idr_lock);
2263
2264 list_for_each_entry(ch, &list, ch_node)
2265 riocm_send_close(ch);
2266
2267 return NOTIFY_DONE;
2268 }
2269
2270 /*
2271 * riocm_interface handles addition/removal of remote RapidIO devices
2272 */
2273 static struct subsys_interface riocm_interface = {
2274 .name = "rio_cm",
2275 .subsys = &rio_bus_type,
2276 .add_dev = riocm_add_dev,
2277 .remove_dev = riocm_remove_dev,
2278 };
2279
2280 /*
2281 * rio_mport_interface handles addition/removal local mport devices
2282 */
2283 static struct class_interface rio_mport_interface __refdata = {
2284 .class = &rio_mport_class,
2285 .add_dev = riocm_add_mport,
2286 .remove_dev = riocm_remove_mport,
2287 };
2288
2289 static struct notifier_block rio_cm_notifier = {
2290 .notifier_call = rio_cm_shutdown,
2291 };
2292
riocm_init(void)2293 static int __init riocm_init(void)
2294 {
2295 int ret;
2296
2297 /* Create device class needed by udev */
2298 ret = class_register(&dev_class);
2299 if (ret) {
2300 riocm_error("Cannot create " DRV_NAME " class");
2301 return ret;
2302 }
2303
2304 ret = alloc_chrdev_region(&dev_number, 0, 1, DRV_NAME);
2305 if (ret) {
2306 class_unregister(&dev_class);
2307 return ret;
2308 }
2309
2310 dev_major = MAJOR(dev_number);
2311 dev_minor_base = MINOR(dev_number);
2312 riocm_debug(INIT, "Registered class with %d major", dev_major);
2313
2314 /*
2315 * Register as rapidio_port class interface to get notifications about
2316 * mport additions and removals.
2317 */
2318 ret = class_interface_register(&rio_mport_interface);
2319 if (ret) {
2320 riocm_error("class_interface_register error: %d", ret);
2321 goto err_reg;
2322 }
2323
2324 /*
2325 * Register as RapidIO bus interface to get notifications about
2326 * addition/removal of remote RapidIO devices.
2327 */
2328 ret = subsys_interface_register(&riocm_interface);
2329 if (ret) {
2330 riocm_error("subsys_interface_register error: %d", ret);
2331 goto err_cl;
2332 }
2333
2334 ret = register_reboot_notifier(&rio_cm_notifier);
2335 if (ret) {
2336 riocm_error("failed to register reboot notifier (err=%d)", ret);
2337 goto err_sif;
2338 }
2339
2340 ret = riocm_cdev_add(dev_number);
2341 if (ret) {
2342 unregister_reboot_notifier(&rio_cm_notifier);
2343 ret = -ENODEV;
2344 goto err_sif;
2345 }
2346
2347 return 0;
2348 err_sif:
2349 subsys_interface_unregister(&riocm_interface);
2350 err_cl:
2351 class_interface_unregister(&rio_mport_interface);
2352 err_reg:
2353 unregister_chrdev_region(dev_number, 1);
2354 class_unregister(&dev_class);
2355 return ret;
2356 }
2357
riocm_exit(void)2358 static void __exit riocm_exit(void)
2359 {
2360 riocm_debug(EXIT, "enter");
2361 unregister_reboot_notifier(&rio_cm_notifier);
2362 subsys_interface_unregister(&riocm_interface);
2363 class_interface_unregister(&rio_mport_interface);
2364 idr_destroy(&ch_idr);
2365
2366 device_unregister(riocm_cdev.dev);
2367 cdev_del(&(riocm_cdev.cdev));
2368
2369 class_unregister(&dev_class);
2370 unregister_chrdev_region(dev_number, 1);
2371 }
2372
2373 late_initcall(riocm_init);
2374 module_exit(riocm_exit);
2375