1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP 3 * 4 * Copyright (c) 2022, SUSE. 5 */ 6 7 #define pr_fmt(fmt) "MPTCP: " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/list.h> 12 #include <linux/rculist.h> 13 #include <linux/spinlock.h> 14 #include "protocol.h" 15 16 static DEFINE_SPINLOCK(mptcp_sched_list_lock); 17 static LIST_HEAD(mptcp_sched_list); 18 19 static int mptcp_sched_default_get_send(struct mptcp_sock *msk, 20 struct mptcp_sched_data *data) 21 { 22 struct sock *ssk; 23 24 ssk = mptcp_subflow_get_send(msk); 25 if (!ssk) 26 return -EINVAL; 27 28 mptcp_subflow_set_scheduled(mptcp_subflow_ctx(ssk), true); 29 return 0; 30 } 31 32 static int mptcp_sched_default_get_retrans(struct mptcp_sock *msk, 33 struct mptcp_sched_data *data) 34 { 35 struct sock *ssk; 36 37 ssk = mptcp_subflow_get_retrans(msk); 38 if (!ssk) 39 return -EINVAL; 40 41 mptcp_subflow_set_scheduled(mptcp_subflow_ctx(ssk), true); 42 return 0; 43 } 44 45 static struct mptcp_sched_ops mptcp_sched_default = { 46 .get_send = mptcp_sched_default_get_send, 47 .get_retrans = mptcp_sched_default_get_retrans, 48 .name = "default", 49 .owner = THIS_MODULE, 50 }; 51 52 /* Must be called with rcu read lock held */ 53 struct mptcp_sched_ops *mptcp_sched_find(const char *name) 54 { 55 struct mptcp_sched_ops *sched, *ret = NULL; 56 57 list_for_each_entry_rcu(sched, &mptcp_sched_list, list) { 58 if (!strcmp(sched->name, name)) { 59 ret = sched; 60 break; 61 } 62 } 63 64 return ret; 65 } 66 67 /* Build string with list of available scheduler values. 68 * Similar to tcp_get_available_congestion_control() 69 */ 70 void mptcp_get_available_schedulers(char *buf, size_t maxlen) 71 { 72 struct mptcp_sched_ops *sched; 73 size_t offs = 0; 74 75 rcu_read_lock(); 76 list_for_each_entry_rcu(sched, &mptcp_sched_list, list) { 77 offs += snprintf(buf + offs, maxlen - offs, 78 "%s%s", 79 offs == 0 ? "" : " ", sched->name); 80 81 if (WARN_ON_ONCE(offs >= maxlen)) 82 break; 83 } 84 rcu_read_unlock(); 85 } 86 87 int mptcp_register_scheduler(struct mptcp_sched_ops *sched) 88 { 89 if (!sched->get_send) 90 return -EINVAL; 91 92 spin_lock(&mptcp_sched_list_lock); 93 if (mptcp_sched_find(sched->name)) { 94 spin_unlock(&mptcp_sched_list_lock); 95 return -EEXIST; 96 } 97 list_add_tail_rcu(&sched->list, &mptcp_sched_list); 98 spin_unlock(&mptcp_sched_list_lock); 99 100 pr_debug("%s registered\n", sched->name); 101 return 0; 102 } 103 104 void mptcp_unregister_scheduler(struct mptcp_sched_ops *sched) 105 { 106 if (sched == &mptcp_sched_default) 107 return; 108 109 spin_lock(&mptcp_sched_list_lock); 110 list_del_rcu(&sched->list); 111 spin_unlock(&mptcp_sched_list_lock); 112 } 113 114 void mptcp_sched_init(void) 115 { 116 mptcp_register_scheduler(&mptcp_sched_default); 117 } 118 119 int mptcp_init_sched(struct mptcp_sock *msk, 120 struct mptcp_sched_ops *sched) 121 { 122 if (!sched) 123 sched = &mptcp_sched_default; 124 125 if (!bpf_try_module_get(sched, sched->owner)) 126 return -EBUSY; 127 128 msk->sched = sched; 129 if (msk->sched->init) 130 msk->sched->init(msk); 131 132 pr_debug("sched=%s\n", msk->sched->name); 133 134 return 0; 135 } 136 137 void mptcp_release_sched(struct mptcp_sock *msk) 138 { 139 struct mptcp_sched_ops *sched = msk->sched; 140 141 if (!sched) 142 return; 143 144 msk->sched = NULL; 145 if (sched->release) 146 sched->release(msk); 147 148 bpf_module_put(sched, sched->owner); 149 } 150 151 void mptcp_subflow_set_scheduled(struct mptcp_subflow_context *subflow, 152 bool scheduled) 153 { 154 WRITE_ONCE(subflow->scheduled, scheduled); 155 } 156 157 int mptcp_sched_get_send(struct mptcp_sock *msk) 158 { 159 struct mptcp_subflow_context *subflow; 160 struct mptcp_sched_data *data = NULL; 161 162 msk_owned_by_me(msk); 163 164 /* the following check is moved out of mptcp_subflow_get_send */ 165 if (__mptcp_check_fallback(msk)) { 166 if (msk->first && 167 __tcp_can_send(msk->first) && 168 sk_stream_memory_free(msk->first)) { 169 mptcp_subflow_set_scheduled(mptcp_subflow_ctx(msk->first), true); 170 return 0; 171 } 172 return -EINVAL; 173 } 174 175 mptcp_for_each_subflow(msk, subflow) { 176 if (READ_ONCE(subflow->scheduled)) 177 return 0; 178 } 179 180 if (msk->sched == &mptcp_sched_default || !msk->sched) 181 return mptcp_sched_default_get_send(msk, data); 182 return msk->sched->get_send(msk, data); 183 } 184 185 int mptcp_sched_get_retrans(struct mptcp_sock *msk) 186 { 187 struct mptcp_subflow_context *subflow; 188 struct mptcp_sched_data *data = NULL; 189 190 msk_owned_by_me(msk); 191 192 /* the following check is moved out of mptcp_subflow_get_retrans */ 193 if (__mptcp_check_fallback(msk)) 194 return -EINVAL; 195 196 mptcp_for_each_subflow(msk, subflow) { 197 if (READ_ONCE(subflow->scheduled)) 198 return 0; 199 } 200 201 if (msk->sched == &mptcp_sched_default || !msk->sched) 202 return mptcp_sched_default_get_retrans(msk, data); 203 if (msk->sched->get_retrans) 204 return msk->sched->get_retrans(msk, data); 205 return msk->sched->get_send(msk, data); 206 } 207