Lines Matching refs:sdev
94 * @sdev: Pointer to the stream device
101 struct tbstream_dev *sdev;
281 static inline struct tb_xdomain *tbstream_dev_xdomain(struct tbstream_dev *sdev)
283 if (sdev->stream)
284 return tb_service_parent(sdev->stream->svc);
290 struct tbstream_dev *sdev = container_of(kref, struct tbstream_dev, kref);
292 if (sdev->stream) {
293 struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
295 if (sdev->out_hopid > 0)
296 tb_xdomain_release_out_hopid(xd, sdev->out_hopid);
297 if (sdev->in_hopid > 0)
298 tb_xdomain_release_in_hopid(xd, sdev->in_hopid);
300 tbstream_put(sdev->stream);
302 ida_free(&tbstream_indices, sdev->index);
303 kfree(sdev->misc.name);
304 kfree(sdev);
307 static inline void tbstream_dev_put(struct tbstream_dev *sdev)
310 kref_put(&sdev->kref, tbstream_dev_release);
313 static inline struct tbstream_dev *tbstream_dev_get(struct tbstream_dev *sdev)
316 kref_get(&sdev->kref);
317 return sdev;
325 static inline int tbstream_dev_valid(const struct tbstream_dev *sdev)
327 const struct tbstream *stream = sdev->stream;
331 if (sdev->in_hopid <= 0 || sdev->out_hopid <= 0)
336 static inline bool tbstream_dev_removed(const struct tbstream_dev *sdev)
338 return sdev->removed;
341 static inline bool tbstream_dev_closed(const struct tbstream_dev *sdev)
343 return sdev->closed;
351 struct tbstream_dev *sdev = sf->sdev;
357 sdev->rx_ring.prod++;
364 wake_up_interruptible_poll(&sdev->wait, EPOLLIN | EPOLLRDNORM);
368 tbstream_dev_completed_rx(struct tbstream_dev *sdev)
370 struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring);
374 index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring);
375 sf = &sdev->rx_ring.frames[index];
385 static int tbstream_dev_consume_rx(struct tbstream_dev *sdev)
387 struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring);
391 index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring);
392 sdev->rx_ring.cons++;
394 sf = &sdev->rx_ring.frames[index];
403 return tb_ring_rx(sdev->rx_ring.ring, &sf->frame);
406 static int tbstream_dev_alloc_rx_buffers(struct tbstream_dev *sdev)
408 size_t ring_size = tb_ring_size(sdev->rx_ring.ring);
411 sdev->rx_ring.frames = kzalloc_objs(struct tbstream_frame, ring_size);
412 if (!sdev->rx_ring.frames)
416 struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring);
417 struct tbstream_frame *sf = &sdev->rx_ring.frames[i];
432 sf->sdev = sdev;
436 tb_ring_rx(sdev->rx_ring.ring, &sf->frame);
439 sdev->rx_ring.cons = 0;
440 sdev->rx_ring.prod = 0;
449 struct tbstream_dev *sdev = sf->sdev;
454 sdev->tx_ring.prod++;
456 wake_up_interruptible_poll(&sdev->wait, EPOLLOUT | EPOLLWRNORM);
459 static int tbstream_dev_alloc_tx_buffers(struct tbstream_dev *sdev)
461 struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring);
462 size_t ring_size = tb_ring_size(sdev->tx_ring.ring);
465 sdev->tx_ring.frames = kzalloc_objs(struct tbstream_frame, ring_size);
466 if (!sdev->tx_ring.frames)
470 struct tbstream_frame *sf = &sdev->tx_ring.frames[i];
485 sf->sdev = sdev;
491 sdev->tx_ring.cons = 0;
492 sdev->tx_ring.prod = ring_size - 1;
497 tbstream_dev_alloc_tx(struct tbstream_dev *sdev, enum tbstream_frame_pdf pdf,
500 struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring);
504 if (!tbstream_ring_available(&sdev->tx_ring))
507 index = sdev->tx_ring.cons % tb_ring_size(sdev->tx_ring.ring);
508 sdev->tx_ring.cons++;
510 sf = &sdev->tx_ring.frames[index];
518 sdev->tx_ring.cons--;
530 tbstream_dev_send_data(struct tbstream_dev *sdev, struct iov_iter *from,
535 sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_DATA, from, size);
538 return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
542 tbstream_dev_poll_ring(struct tbstream_dev *sdev, struct tbstream_ring *ring)
546 if (!sdev->busy_poll)
553 static int tbstream_dev_send_close(struct tbstream_dev *sdev)
557 if (sdev->busy_poll) {
567 if (tbstream_ring_available(&sdev->tx_ring))
569 tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
574 sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_CLOSE, NULL, SZ_256);
577 return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
580 static int tbstream_dev_start(struct tbstream_dev *sdev)
582 struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
588 if (sdev->busy_poll)
591 ring = tb_ring_alloc_tx(xd->tb->nhi, -1, sdev->ring_size, flags);
594 sdev->tx_ring.ring = ring;
596 ret = tbstream_dev_alloc_tx_buffers(sdev);
604 ring = tb_ring_alloc_rx(xd->tb->nhi, -1, sdev->ring_size, flags,
610 sdev->rx_ring.ring = ring;
612 ret = tb_xdomain_enable_paths(xd, sdev->out_hopid,
613 sdev->tx_ring.ring->hop,
614 sdev->in_hopid,
615 sdev->rx_ring.ring->hop);
619 tb_ring_throttling(sdev->tx_ring.ring, sdev->throttling);
620 tb_ring_throttling(sdev->rx_ring.ring, sdev->throttling);
622 tb_ring_start(sdev->tx_ring.ring);
623 tb_ring_start(sdev->rx_ring.ring);
625 ret = tbstream_dev_alloc_rx_buffers(sdev);
631 tb_ring_stop(sdev->rx_ring.ring);
632 tb_ring_stop(sdev->tx_ring.ring);
634 tb_ring_free(sdev->rx_ring.ring);
636 tbstream_ring_free(&sdev->tx_ring);
638 tb_ring_free(sdev->tx_ring.ring);
643 static bool tbstream_dev_tx_drained(const struct tbstream_dev *sdev)
645 const struct tbstream_ring *ring = &sdev->tx_ring;
654 static void tbstream_dev_stop(struct tbstream_dev *sdev)
658 if (sdev->busy_poll) {
666 if (tbstream_dev_tx_drained(sdev))
668 tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
672 tb_ring_stop(sdev->tx_ring.ring);
673 tb_ring_stop(sdev->rx_ring.ring);
675 tb_ring_flush(sdev->tx_ring.ring, 500);
676 tb_ring_stop(sdev->tx_ring.ring);
677 tb_ring_flush(sdev->rx_ring.ring, 500);
678 tb_ring_stop(sdev->rx_ring.ring);
681 xd = tbstream_dev_xdomain(sdev);
683 tb_xdomain_disable_paths(xd, sdev->out_hopid,
684 sdev->tx_ring.ring->hop,
685 sdev->in_hopid,
686 sdev->rx_ring.ring->hop);
689 tbstream_ring_free(&sdev->rx_ring);
690 tb_ring_free(sdev->rx_ring.ring);
691 sdev->rx_ring.ring = NULL;
692 tbstream_ring_free(&sdev->tx_ring);
693 tb_ring_free(sdev->tx_ring.ring);
694 sdev->tx_ring.ring = NULL;
698 static int tbstream_dev_lock(struct tbstream_dev *sdev, bool nowait)
701 if (!mutex_trylock(&sdev->lock))
704 if (mutex_lock_interruptible(&sdev->lock))
715 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
719 ret = tbstream_dev_valid(sdev);
723 ret = tbstream_dev_lock(sdev, nowait);
729 tbstream_dev_poll_ring(sdev, &sdev->rx_ring);
731 ret = tbstream_dev_valid(sdev);
733 mutex_unlock(&sdev->lock);
737 if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) {
738 mutex_unlock(&sdev->lock);
742 if (tbstream_ring_available(&sdev->rx_ring))
745 mutex_unlock(&sdev->lock);
750 if (sdev->busy_poll) {
755 ret = wait_event_interruptible(sdev->wait,
756 tbstream_ring_available(&sdev->rx_ring) ||
757 tbstream_dev_valid(sdev) != 0 ||
758 tbstream_dev_closed(sdev) ||
759 tbstream_dev_removed(sdev));
764 ret = tbstream_dev_lock(sdev, nowait);
774 sf = tbstream_dev_completed_rx(sdev);
785 tbstream_dev_consume_rx(sdev);
786 sdev->closed = true;
808 ret = tbstream_dev_consume_rx(sdev);
816 mutex_unlock(&sdev->lock);
827 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
831 ret = tbstream_dev_valid(sdev);
835 ret = tbstream_dev_lock(sdev, nowait);
840 tbstream_dev_poll_ring(sdev, &sdev->tx_ring);
842 ret = tbstream_dev_valid(sdev);
844 mutex_unlock(&sdev->lock);
848 if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev)) {
849 mutex_unlock(&sdev->lock);
853 if (tbstream_ring_available(&sdev->tx_ring))
856 mutex_unlock(&sdev->lock);
861 if (sdev->busy_poll) {
866 ret = wait_event_interruptible(sdev->wait,
867 tbstream_ring_available(&sdev->tx_ring) ||
868 tbstream_dev_valid(sdev) != 0 ||
869 tbstream_dev_closed(sdev) ||
870 tbstream_dev_removed(sdev));
875 ret = tbstream_dev_lock(sdev, nowait);
885 ret = tbstream_dev_send_data(sdev, from, size);
899 mutex_unlock(&sdev->lock);
908 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
915 if (sdev->busy_poll)
918 poll_wait(file, &sdev->wait, wait);
919 guard(mutex)(&sdev->lock);
920 if (tbstream_dev_valid(sdev) != 0) {
923 if (tbstream_ring_available(&sdev->tx_ring))
925 if (tbstream_ring_available(&sdev->rx_ring))
933 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
936 tbstream_dev_get(sdev);
938 if (mutex_lock_interruptible(&sdev->lock)) {
939 tbstream_dev_put(sdev);
947 while ((ret = tbstream_dev_valid(sdev))) {
948 mutex_unlock(&sdev->lock);
953 ret = wait_event_interruptible(sdev->wait,
954 tbstream_dev_valid(sdev) == 0 ||
955 tbstream_dev_removed(sdev));
959 if (tbstream_dev_removed(sdev)) {
964 if (mutex_lock_interruptible(&sdev->lock)) {
971 if (!sdev->users++) {
972 ret = tbstream_dev_start(sdev);
974 sdev->users--;
977 sdev->closed = false;
980 mutex_unlock(&sdev->lock);
987 mutex_unlock(&sdev->lock);
989 tbstream_dev_put(sdev);
996 struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
998 mutex_lock(&sdev->lock);
999 if (--sdev->users == 0) {
1005 tbstream_dev_send_close(sdev);
1006 tbstream_dev_stop(sdev);
1008 mutex_unlock(&sdev->lock);
1010 tbstream_dev_put(sdev);
1033 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1035 return sysfs_emit(buf, "%u\n", sdev->busy_poll);
1043 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1051 guard(mutex)(&sdev->lock);
1052 if (sdev->users)
1054 sdev->busy_poll = busy_poll;
1062 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1064 return sysfs_emit(buf, "%d\n", sdev->index);
1071 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1073 return sysfs_emit(buf, "%d\n", sdev->in_hopid);
1170 static int tbstream_dev_update_properties(struct tbstream_dev *sdev)
1175 stream = tbstream_get(sdev->stream);
1180 config_item_name(&sdev->group.cg_item),
1181 sdev->in_hopid, sdev->out_hopid);
1189 static int tbstream_dev_alloc_in_hopid(struct tbstream_dev *sdev, int hopid)
1191 struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
1194 if (sdev->in_hopid > 0 && sdev->in_hopid != hopid)
1195 tb_xdomain_release_in_hopid(xd, sdev->in_hopid);
1197 sdev->in_hopid = hopid;
1211 sdev->in_hopid = ret;
1215 static int tbstream_dev_alloc_out_hopid(struct tbstream_dev *sdev, int hopid)
1217 struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
1220 if (sdev->out_hopid > 0 && sdev->out_hopid != hopid)
1221 tb_xdomain_release_out_hopid(xd, sdev->out_hopid);
1223 sdev->out_hopid = hopid;
1233 sdev->out_hopid = ret;
1242 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1249 guard(mutex)(&sdev->lock);
1250 if (sdev->users)
1252 if (sdev->stream) {
1253 ret = tbstream_dev_alloc_in_hopid(sdev, in_hopid);
1256 ret = tbstream_dev_update_properties(sdev);
1258 sdev->in_hopid = in_hopid;
1267 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1269 return sysfs_emit(buf, "%d\n", sdev->out_hopid);
1277 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1284 guard(mutex)(&sdev->lock);
1285 if (sdev->users)
1287 if (sdev->stream) {
1288 ret = tbstream_dev_alloc_out_hopid(sdev, out_hopid);
1291 ret = tbstream_dev_update_properties(sdev);
1293 sdev->out_hopid = out_hopid;
1302 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1304 return sysfs_emit(buf, "%u\n", sdev->ring_size);
1312 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1324 guard(mutex)(&sdev->lock);
1325 if (sdev->users)
1327 sdev->ring_size = ring_size;
1335 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1337 return sysfs_emit(buf, "%u\n", sdev->throttling);
1345 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1356 guard(mutex)(&sdev->lock);
1357 if (sdev->users)
1359 sdev->throttling = throttling;
1377 struct tbstream_dev *sdev = tbstream_dev_from_group(group);
1379 misc_deregister(&sdev->misc);
1380 tbstream_dev_put(sdev);
1422 tbstream_dev_attach_stream(struct tbstream_dev *sdev, struct tbstream_group *sg)
1424 const char *name = config_item_name(&sdev->group.cg_item);
1431 scoped_guard(mutex, &sdev->lock) {
1432 sdev->stream = stream;
1438 if (sdev->in_hopid <= 0 && sdev->out_hopid <= 0)
1439 service_get_hopids(stream->svc, name, &sdev->in_hopid,
1440 &sdev->out_hopid);
1441 if (sdev->in_hopid)
1442 tbstream_dev_alloc_in_hopid(sdev, sdev->in_hopid);
1443 if (sdev->out_hopid)
1444 tbstream_dev_alloc_out_hopid(sdev, sdev->out_hopid);
1447 service_update_properties(stream->svc, name, sdev->in_hopid,
1448 sdev->out_hopid);
1452 wake_up_interruptible(&sdev->wait);
1455 static void tbstream_dev_detach_stream(struct tbstream_dev *sdev)
1457 const char *name = config_item_name(&sdev->group.cg_item);
1461 scoped_guard(mutex, &sdev->lock) {
1462 stream = sdev->stream;
1465 sdev->stream = NULL;
1467 if (sdev->out_hopid > 0)
1468 tb_xdomain_release_out_hopid(xd, sdev->out_hopid);
1469 if (sdev->in_hopid > 0)
1470 tb_xdomain_release_in_hopid(xd, sdev->in_hopid);
1479 wake_up_interruptible_poll(&sdev->wait, EPOLLHUP | EPOLLERR);
1492 struct tbstream_dev *sdev;
1502 sdev = kzalloc_obj(*sdev);
1503 if (!sdev)
1508 kfree(sdev);
1512 sdev->index = index;
1513 sdev->ring_size = TBSTREAM_DEV_RING_SIZE;
1514 sdev->throttling = TBSTREAM_DEV_THROTTLING;
1515 mutex_init(&sdev->lock);
1516 init_waitqueue_head(&sdev->wait);
1517 INIT_LIST_HEAD(&sdev->list);
1518 /* This point forward tbstream_dev_put() must be used to release sdev */
1519 kref_init(&sdev->kref);
1521 config_group_init_type_name(&sdev->group, name, &tbstream_dev_type);
1524 list_add_tail(&sdev->list, &sg->dev_list);
1526 tbstream_dev_attach_stream(sdev, sg);
1528 sdev->misc.name = kasprintf(GFP_KERNEL, "tbstream%d", index);
1529 sdev->misc.minor = MISC_DYNAMIC_MINOR;
1530 sdev->misc.fops = &tbstream_dev_fops;
1532 ret = misc_register(&sdev->misc);
1534 tbstream_dev_detach_stream(sdev);
1536 list_del(&sdev->list);
1538 config_group_put(&sdev->group);
1542 return &sdev->group;
1549 struct tbstream_dev *sdev = tbstream_dev_from_group(sdev_group);
1553 list_del(&sdev->list);
1555 sdev->removed = true;
1556 wake_up_interruptible_poll(&sdev->wait, EPOLLHUP | EPOLLERR);
1649 struct tbstream_dev *sdev;
1665 list_for_each_entry(sdev, &sg->dev_list, list) {
1666 tbstream_dev_get(sdev);
1667 tbstream_dev_attach_stream(sdev, sg);
1668 tbstream_dev_put(sdev);
1677 struct tbstream_dev *sdev;
1686 list_for_each_entry_reverse(sdev, &sg->dev_list, list) {
1687 tbstream_dev_get(sdev);
1688 tbstream_dev_detach_stream(sdev);
1689 tbstream_dev_put(sdev);
1734 struct tbstream_dev *sdev;
1740 list_for_each_entry_reverse(sdev, &sg->dev_list, list) {
1741 tbstream_dev_get(sdev);
1743 if (sdev->users)
1744 tbstream_dev_stop(sdev);
1745 tbstream_dev_put(sdev);
1757 struct tbstream_dev *sdev;
1763 list_for_each_entry(sdev, &sg->dev_list, list) {
1764 tbstream_dev_get(sdev);
1765 if (sdev->users) {
1768 ret = tbstream_dev_start(sdev);
1770 tbstream_dev_put(sdev);
1775 tbstream_dev_put(sdev);