60 	struct kcm_mux *mux = psock->mux;  in kcm_abort_tx_psock()  local
64 	spin_lock_bh(&mux->lock);  in kcm_abort_tx_psock()
67 		spin_unlock_bh(&mux->lock);  in kcm_abort_tx_psock()
88 	spin_unlock_bh(&mux->lock);  in kcm_abort_tx_psock()
94 /* RX mux lock held. */
95 static void kcm_update_rx_mux_stats(struct kcm_mux *mux,  in kcm_update_rx_mux_stats()  argument
98 	STRP_STATS_ADD(mux->stats.rx_bytes,  in kcm_update_rx_mux_stats()
101 	mux->stats.rx_msgs +=  in kcm_update_rx_mux_stats()
107 static void kcm_update_tx_mux_stats(struct kcm_mux *mux,  in kcm_update_tx_mux_stats()  argument
110 	KCM_STATS_ADD(mux->stats.tx_bytes,  in kcm_update_tx_mux_stats()
112 	mux->stats.tx_msgs +=  in kcm_update_tx_mux_stats()
122  * pending ready messages on a psock. RX mux lock held.
126 	struct kcm_mux *mux = kcm->mux;  in kcm_rcv_ready()  local
133 	while (unlikely((skb = __skb_dequeue(&mux->rx_hold_queue)))) {  in kcm_rcv_ready()
136 			skb_queue_head(&mux->rx_hold_queue, skb);  in kcm_rcv_ready()
142 	while (!list_empty(&mux->psocks_ready)) {  in kcm_rcv_ready()
143 		psock = list_first_entry(&mux->psocks_ready, struct kcm_psock,  in kcm_rcv_ready()
166 		      &kcm->mux->kcm_rx_waiters);  in kcm_rcv_ready()
175 	struct kcm_mux *mux = kcm->mux;  in kcm_rfree()  local
186 		spin_lock_bh(&mux->rx_lock);  in kcm_rfree()
188 		spin_unlock_bh(&mux->rx_lock);  in kcm_rfree()
220  * RX mux lock held.
222 static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head)  in requeue_rx_msgs()  argument
232 		if (list_empty(&mux->kcm_rx_waiters)) {  in requeue_rx_msgs()
233 			skb_queue_tail(&mux->rx_hold_queue, skb);  in requeue_rx_msgs()
237 		kcm = list_first_entry(&mux->kcm_rx_waiters,  in requeue_rx_msgs()
258 	struct kcm_mux *mux = psock->mux;  in reserve_rx_kcm()  local
266 	spin_lock_bh(&mux->rx_lock);  in reserve_rx_kcm()
269 		spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
273 	kcm_update_rx_mux_stats(mux, psock);  in reserve_rx_kcm()
275 	if (list_empty(&mux->kcm_rx_waiters)) {  in reserve_rx_kcm()
279 			      &mux->psocks_ready);  in reserve_rx_kcm()
280 		spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
284 	kcm = list_first_entry(&mux->kcm_rx_waiters,  in reserve_rx_kcm()
294 	spin_unlock_bh(&mux->rx_lock);  in reserve_rx_kcm()
311 	struct kcm_mux *mux = psock->mux;  in unreserve_rx_kcm()  local
316 	spin_lock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
328 		spin_unlock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
339 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in unreserve_rx_kcm()
346 	spin_unlock_bh(&mux->rx_lock);  in unreserve_rx_kcm()
419 	struct kcm_mux *mux;  in psock_write_space()  local
427 	mux = psock->mux;  in psock_write_space()
429 	spin_lock_bh(&mux->lock);  in psock_write_space()
436 	spin_unlock_bh(&mux->lock);  in psock_write_space()
446 	struct kcm_mux *mux = kcm->mux;  in reserve_psock()  local
461 	spin_lock_bh(&mux->lock);  in reserve_psock()
469 		spin_unlock_bh(&mux->lock);  in reserve_psock()
473 	if (!list_empty(&mux->psocks_avail)) {  in reserve_psock()
474 		psock = list_first_entry(&mux->psocks_avail,  in reserve_psock()
487 			      &mux->kcm_tx_waiters);  in reserve_psock()
491 	spin_unlock_bh(&mux->lock);  in reserve_psock()
496 /* mux lock held */
499 	struct kcm_mux *mux = psock->mux;  in psock_now_avail()  local
502 	if (list_empty(&mux->kcm_tx_waiters)) {  in psock_now_avail()
504 			      &mux->psocks_avail);  in psock_now_avail()
506 		kcm = list_first_entry(&mux->kcm_tx_waiters,  in psock_now_avail()
528 	struct kcm_mux *mux = kcm->mux;  in unreserve_psock()  local
530 	spin_lock_bh(&mux->lock);  in unreserve_psock()
535 		spin_unlock_bh(&mux->lock);  in unreserve_psock()
541 	kcm_update_tx_mux_stats(mux, psock);  in unreserve_psock()
553 			mux->psocks_cnt--;  in unreserve_psock()
561 		spin_unlock_bh(&mux->lock);  in unreserve_psock()
568 	spin_unlock_bh(&mux->lock);  in unreserve_psock()
573 	struct kcm_mux *mux = kcm->mux;  in kcm_report_tx_retry()  local
575 	spin_lock_bh(&mux->lock);  in kcm_report_tx_retry()
576 	KCM_STATS_INCR(mux->stats.tx_retries);  in kcm_report_tx_retry()
577 	spin_unlock_bh(&mux->lock);  in kcm_report_tx_retry()
1079 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_disable()  local
1084 	spin_lock_bh(&mux->rx_lock);  in kcm_recv_disable()
1096 		requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue);  in kcm_recv_disable()
1099 	spin_unlock_bh(&mux->rx_lock);  in kcm_recv_disable()
1105 	struct kcm_mux *mux = kcm->mux;  in kcm_recv_enable()  local
1110 	spin_lock_bh(&mux->rx_lock);  in kcm_recv_enable()
1115 	spin_unlock_bh(&mux->rx_lock);  in kcm_recv_enable()
1184 static void init_kcm_sock(struct kcm_sock *kcm, struct kcm_mux *mux)  in init_kcm_sock()  argument
1196 	/* Add to mux's kcm sockets list */  in init_kcm_sock()
1197 	kcm->mux = mux;  in init_kcm_sock()
1198 	spin_lock_bh(&mux->lock);  in init_kcm_sock()
1200 	head = &mux->kcm_socks;  in init_kcm_sock()
1201 	list_for_each_entry(tkcm, &mux->kcm_socks, kcm_sock_list) {  in init_kcm_sock()
1211 	mux->kcm_socks_cnt++;  in init_kcm_sock()
1212 	spin_unlock_bh(&mux->lock);  in init_kcm_sock()
1217 	spin_lock_bh(&mux->rx_lock);  in init_kcm_sock()
1219 	spin_unlock_bh(&mux->rx_lock);  in init_kcm_sock()
1226 	struct kcm_mux *mux = kcm->mux;  in kcm_attach()  local
1263 	psock->mux = mux;  in kcm_attach()
1298 	/* Finished initialization, now add the psock to the MUX. */  in kcm_attach()
1299 	spin_lock_bh(&mux->lock);  in kcm_attach()
1300 	head = &mux->psocks;  in kcm_attach()
1301 	list_for_each_entry(tpsock, &mux->psocks, psock_list) {  in kcm_attach()
1311 	KCM_STATS_INCR(mux->stats.psock_attach);  in kcm_attach()
1312 	mux->psocks_cnt++;  in kcm_attach()
1314 	spin_unlock_bh(&mux->lock);  in kcm_attach()
1358 	struct kcm_mux *mux = psock->mux;  in kcm_unattach()  local
1378 	spin_lock_bh(&mux->rx_lock);  in kcm_unattach()
1387 		KCM_STATS_INCR(mux->stats.rx_ready_drops);  in kcm_unattach()
1390 	spin_unlock_bh(&mux->rx_lock);  in kcm_unattach()
1401 	spin_lock_bh(&mux->lock);  in kcm_unattach()
1403 	aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats);  in kcm_unattach()
1404 	save_strp_stats(&psock->strp, &mux->aggregate_strp_stats);  in kcm_unattach()
1406 	KCM_STATS_INCR(mux->stats.psock_unattach);  in kcm_unattach()
1413 		KCM_STATS_INCR(mux->stats.psock_unattach_rsvd);  in kcm_unattach()
1414 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1418 		 * to do this without the mux lock.  in kcm_unattach()
1422 		spin_lock_bh(&mux->lock);  in kcm_unattach()
1424 			/* psock now unreserved in window mux was unlocked */  in kcm_unattach()
1434 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1440 		mux->psocks_cnt--;  in kcm_unattach()
1441 		spin_unlock_bh(&mux->lock);  in kcm_unattach()
1454 	struct kcm_mux *mux = kcm->mux;  in kcm_unattach_ioctl()  local
1472 	spin_lock_bh(&mux->lock);  in kcm_unattach_ioctl()
1474 	list_for_each_entry(psock, &mux->psocks, psock_list) {  in kcm_unattach_ioctl()
1487 		spin_unlock_bh(&mux->lock);  in kcm_unattach_ioctl()
1496 	spin_unlock_bh(&mux->lock);  in kcm_unattach_ioctl()
1531 	init_kcm_sock(kcm_sk(newsk), kcm_sk(osock->sk)->mux);  in kcm_clone()
1592 static void release_mux(struct kcm_mux *mux)  in release_mux()  argument
1594 	struct kcm_net *knet = mux->knet;  in release_mux()
1599 				 &mux->psocks, psock_list) {  in release_mux()
1604 	if (WARN_ON(mux->psocks_cnt))  in release_mux()
1607 	__skb_queue_purge(&mux->rx_hold_queue);  in release_mux()
1610 	aggregate_mux_stats(&mux->stats, &knet->aggregate_mux_stats);  in release_mux()
1611 	aggregate_psock_stats(&mux->aggregate_psock_stats,  in release_mux()
1613 	aggregate_strp_stats(&mux->aggregate_strp_stats,  in release_mux()
1615 	list_del_rcu(&mux->kcm_mux_list);  in release_mux()
1619 	kfree_rcu(mux, rcu);  in release_mux()
1624 	struct kcm_mux *mux = kcm->mux;  in kcm_done()  local
1628 	spin_lock_bh(&mux->rx_lock);  in kcm_done()
1634 		spin_unlock_bh(&mux->rx_lock);  in kcm_done()
1644 	requeue_rx_msgs(mux, &sk->sk_receive_queue);  in kcm_done()
1646 	spin_unlock_bh(&mux->rx_lock);  in kcm_done()
1651 	/* Detach from MUX */  in kcm_done()
1652 	spin_lock_bh(&mux->lock);  in kcm_done()
1655 	mux->kcm_socks_cnt--;  in kcm_done()
1656 	socks_cnt = mux->kcm_socks_cnt;  in kcm_done()
1658 	spin_unlock_bh(&mux->lock);  in kcm_done()
1661 		/* We are done with the mux now. */  in kcm_done()
1662 		release_mux(mux);  in kcm_done()
1671  * If this is the last KCM socket on the MUX, destroy the MUX.
1677 	struct kcm_mux *mux;  in kcm_release()  local
1684 	mux = kcm->mux;  in kcm_release()
1698 	spin_lock_bh(&mux->lock);  in kcm_release()
1706 	spin_unlock_bh(&mux->lock);  in kcm_release()
1784 	struct kcm_mux *mux;  in kcm_create()  local
1804 	/* Allocate a kcm mux, shared between KCM sockets */  in kcm_create()
1805 	mux = kmem_cache_zalloc(kcm_muxp, GFP_KERNEL);  in kcm_create()
1806 	if (!mux) {  in kcm_create()
1811 	spin_lock_init(&mux->lock);  in kcm_create()
1812 	spin_lock_init(&mux->rx_lock);  in kcm_create()
1813 	INIT_LIST_HEAD(&mux->kcm_socks);  in kcm_create()
1814 	INIT_LIST_HEAD(&mux->kcm_rx_waiters);  in kcm_create()
1815 	INIT_LIST_HEAD(&mux->kcm_tx_waiters);  in kcm_create()
1817 	INIT_LIST_HEAD(&mux->psocks);  in kcm_create()
1818 	INIT_LIST_HEAD(&mux->psocks_ready);  in kcm_create()
1819 	INIT_LIST_HEAD(&mux->psocks_avail);  in kcm_create()
1821 	mux->knet = knet;  in kcm_create()
1823 	/* Add new MUX to list */  in kcm_create()
1825 	list_add_rcu(&mux->kcm_mux_list, &knet->mux_list);  in kcm_create()
1829 	skb_queue_head_init(&mux->rx_hold_queue);  in kcm_create()
1833 	init_kcm_sock(kcm_sk(sk), mux);  in kcm_create()