1 // SPDX-License-Identifier: GPL-2.0 2 /* Multipath TCP token management 3 * Copyright (c) 2017 - 2019, Intel Corporation. 4 * 5 * Note: This code is based on mptcp_ctrl.c from multipath-tcp.org, 6 * authored by: 7 * 8 * Sébastien Barré <sebastien.barre@uclouvain.be> 9 * Christoph Paasch <christoph.paasch@uclouvain.be> 10 * Jaakko Korkeaniemi <jaakko.korkeaniemi@aalto.fi> 11 * Gregory Detal <gregory.detal@uclouvain.be> 12 * Fabien Duchêne <fabien.duchene@uclouvain.be> 13 * Andreas Seelinger <Andreas.Seelinger@rwth-aachen.de> 14 * Lavkesh Lahngir <lavkesh51@gmail.com> 15 * Andreas Ripke <ripke@neclab.eu> 16 * Vlad Dogaru <vlad.dogaru@intel.com> 17 * Octavian Purdila <octavian.purdila@intel.com> 18 * John Ronan <jronan@tssg.org> 19 * Catalin Nicutar <catalin.nicutar@gmail.com> 20 * Brandon Heller <brandonh@stanford.edu> 21 */ 22 23 #define pr_fmt(fmt) "MPTCP: " fmt 24 25 #include <linux/kernel.h> 26 #include <linux/module.h> 27 #include <linux/memblock.h> 28 #include <linux/ip.h> 29 #include <linux/tcp.h> 30 #include <net/sock.h> 31 #include <net/inet_common.h> 32 #include <net/protocol.h> 33 #include <net/mptcp.h> 34 #include "protocol.h" 35 36 #define TOKEN_MAX_CHAIN_LEN 4 37 38 struct token_bucket { 39 spinlock_t lock; 40 int chain_len; 41 struct hlist_nulls_head req_chain; 42 struct hlist_nulls_head msk_chain; 43 }; 44 45 static struct token_bucket *token_hash __read_mostly; 46 static unsigned int token_mask __read_mostly; 47 48 static struct token_bucket *token_bucket(u32 token) 49 { 50 return &token_hash[token & token_mask]; 51 } 52 53 /* called with bucket lock held */ 54 static struct mptcp_subflow_request_sock * 55 __token_lookup_req(struct token_bucket *t, u32 token) 56 { 57 struct mptcp_subflow_request_sock *req; 58 struct hlist_nulls_node *pos; 59 60 hlist_nulls_for_each_entry_rcu(req, pos, &t->req_chain, token_node) 61 if (req->token == token) 62 return req; 63 return NULL; 64 } 65 66 /* called with bucket lock held */ 67 static struct mptcp_sock * 68 __token_lookup_msk(struct token_bucket *t, u32 token) 69 { 70 struct hlist_nulls_node *pos; 71 struct sock *sk; 72 73 sk_nulls_for_each_rcu(sk, pos, &t->msk_chain) 74 if (mptcp_sk(sk)->token == token) 75 return mptcp_sk(sk); 76 return NULL; 77 } 78 79 static bool __token_bucket_busy(struct token_bucket *t, u32 token) 80 { 81 return !token || t->chain_len >= TOKEN_MAX_CHAIN_LEN || 82 __token_lookup_req(t, token) || __token_lookup_msk(t, token); 83 } 84 85 static void mptcp_crypto_key_gen_sha(u64 *key, u32 *token, u64 *idsn) 86 { 87 /* we might consider a faster version that computes the key as a 88 * hash of some information available in the MPTCP socket. Use 89 * random data at the moment, as it's probably the safest option 90 * in case multiple sockets are opened in different namespaces at 91 * the same time. 92 */ 93 get_random_bytes(key, sizeof(u64)); 94 mptcp_crypto_key_sha(*key, token, idsn); 95 } 96 97 /** 98 * mptcp_token_new_request - create new key/idsn/token for subflow_request 99 * @req: the request socket 100 * 101 * This function is called when a new mptcp connection is coming in. 102 * 103 * It creates a unique token to identify the new mptcp connection, 104 * a secret local key and the initial data sequence number (idsn). 105 * 106 * Returns 0 on success. 107 */ 108 int mptcp_token_new_request(struct request_sock *req) 109 { 110 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 111 struct token_bucket *bucket; 112 u32 token; 113 114 mptcp_crypto_key_sha(subflow_req->local_key, 115 &subflow_req->token, 116 &subflow_req->idsn); 117 pr_debug("req=%p local_key=%llu, token=%u, idsn=%llu\n", 118 req, subflow_req->local_key, subflow_req->token, 119 subflow_req->idsn); 120 121 token = subflow_req->token; 122 bucket = token_bucket(token); 123 spin_lock_bh(&bucket->lock); 124 if (__token_bucket_busy(bucket, token)) { 125 spin_unlock_bh(&bucket->lock); 126 return -EBUSY; 127 } 128 129 hlist_nulls_add_head_rcu(&subflow_req->token_node, &bucket->req_chain); 130 bucket->chain_len++; 131 spin_unlock_bh(&bucket->lock); 132 return 0; 133 } 134 135 /** 136 * mptcp_token_new_connect - create new key/idsn/token for subflow 137 * @sk: the socket that will initiate a connection 138 * 139 * This function is called when a new outgoing mptcp connection is 140 * initiated. 141 * 142 * It creates a unique token to identify the new mptcp connection, 143 * a secret local key and the initial data sequence number (idsn). 144 * 145 * On success, the mptcp connection can be found again using 146 * the computed token at a later time, this is needed to process 147 * join requests. 148 * 149 * returns 0 on success. 150 */ 151 int mptcp_token_new_connect(struct sock *sk) 152 { 153 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 154 struct mptcp_sock *msk = mptcp_sk(subflow->conn); 155 int retries = MPTCP_TOKEN_MAX_RETRIES; 156 struct token_bucket *bucket; 157 158 again: 159 mptcp_crypto_key_gen_sha(&subflow->local_key, &subflow->token, 160 &subflow->idsn); 161 162 bucket = token_bucket(subflow->token); 163 spin_lock_bh(&bucket->lock); 164 if (__token_bucket_busy(bucket, subflow->token)) { 165 spin_unlock_bh(&bucket->lock); 166 if (!--retries) 167 return -EBUSY; 168 goto again; 169 } 170 171 pr_debug("ssk=%p, local_key=%llu, token=%u, idsn=%llu\n", 172 sk, subflow->local_key, subflow->token, subflow->idsn); 173 174 WRITE_ONCE(msk->token, subflow->token); 175 __sk_nulls_add_node_rcu((struct sock *)msk, &bucket->msk_chain); 176 bucket->chain_len++; 177 spin_unlock_bh(&bucket->lock); 178 return 0; 179 } 180 181 /** 182 * mptcp_token_accept - replace a req sk with full sock in token hash 183 * @req: the request socket to be removed 184 * @msk: the just cloned socket linked to the new connection 185 * 186 * Called when a SYN packet creates a new logical connection, i.e. 187 * is not a join request. 188 */ 189 void mptcp_token_accept(struct mptcp_subflow_request_sock *req, 190 struct mptcp_sock *msk) 191 { 192 struct mptcp_subflow_request_sock *pos; 193 struct token_bucket *bucket; 194 195 bucket = token_bucket(req->token); 196 spin_lock_bh(&bucket->lock); 197 198 /* pedantic lookup check for the moved token */ 199 pos = __token_lookup_req(bucket, req->token); 200 if (!WARN_ON_ONCE(pos != req)) 201 hlist_nulls_del_init_rcu(&req->token_node); 202 __sk_nulls_add_node_rcu((struct sock *)msk, &bucket->msk_chain); 203 spin_unlock_bh(&bucket->lock); 204 } 205 206 bool mptcp_token_exists(u32 token) 207 { 208 struct hlist_nulls_node *pos; 209 struct token_bucket *bucket; 210 struct mptcp_sock *msk; 211 struct sock *sk; 212 213 rcu_read_lock(); 214 bucket = token_bucket(token); 215 216 again: 217 sk_nulls_for_each_rcu(sk, pos, &bucket->msk_chain) { 218 msk = mptcp_sk(sk); 219 if (READ_ONCE(msk->token) == token) 220 goto found; 221 } 222 if (get_nulls_value(pos) != (token & token_mask)) 223 goto again; 224 225 rcu_read_unlock(); 226 return false; 227 found: 228 rcu_read_unlock(); 229 return true; 230 } 231 232 /** 233 * mptcp_token_get_sock - retrieve mptcp connection sock using its token 234 * @net: restrict to this namespace 235 * @token: token of the mptcp connection to retrieve 236 * 237 * This function returns the mptcp connection structure with the given token. 238 * A reference count on the mptcp socket returned is taken. 239 * 240 * returns NULL if no connection with the given token value exists. 241 */ 242 struct mptcp_sock *mptcp_token_get_sock(struct net *net, u32 token) 243 { 244 struct hlist_nulls_node *pos; 245 struct token_bucket *bucket; 246 struct mptcp_sock *msk; 247 struct sock *sk; 248 249 rcu_read_lock(); 250 bucket = token_bucket(token); 251 252 again: 253 sk_nulls_for_each_rcu(sk, pos, &bucket->msk_chain) { 254 msk = mptcp_sk(sk); 255 if (READ_ONCE(msk->token) != token || 256 !net_eq(sock_net(sk), net)) 257 continue; 258 259 if (!refcount_inc_not_zero(&sk->sk_refcnt)) 260 goto not_found; 261 262 if (READ_ONCE(msk->token) != token || 263 !net_eq(sock_net(sk), net)) { 264 sock_put(sk); 265 goto again; 266 } 267 goto found; 268 } 269 if (get_nulls_value(pos) != (token & token_mask)) 270 goto again; 271 272 not_found: 273 msk = NULL; 274 275 found: 276 rcu_read_unlock(); 277 return msk; 278 } 279 EXPORT_SYMBOL_GPL(mptcp_token_get_sock); 280 281 /** 282 * mptcp_token_iter_next - iterate over the token container from given pos 283 * @net: namespace to be iterated 284 * @s_slot: start slot number 285 * @s_num: start number inside the given lock 286 * 287 * This function returns the first mptcp connection structure found inside the 288 * token container starting from the specified position, or NULL. 289 * 290 * On successful iteration, the iterator is move to the next position and the 291 * the acquires a reference to the returned socket. 292 */ 293 struct mptcp_sock *mptcp_token_iter_next(const struct net *net, long *s_slot, 294 long *s_num) 295 { 296 struct mptcp_sock *ret = NULL; 297 struct hlist_nulls_node *pos; 298 int slot, num = 0; 299 300 for (slot = *s_slot; slot <= token_mask; *s_num = 0, slot++) { 301 struct token_bucket *bucket = &token_hash[slot]; 302 struct sock *sk; 303 304 num = 0; 305 306 if (hlist_nulls_empty(&bucket->msk_chain)) 307 continue; 308 309 rcu_read_lock(); 310 sk_nulls_for_each_rcu(sk, pos, &bucket->msk_chain) { 311 ++num; 312 if (!net_eq(sock_net(sk), net)) 313 continue; 314 315 if (num <= *s_num) 316 continue; 317 318 if (!refcount_inc_not_zero(&sk->sk_refcnt)) 319 continue; 320 321 if (!net_eq(sock_net(sk), net)) { 322 sock_put(sk); 323 continue; 324 } 325 326 ret = mptcp_sk(sk); 327 rcu_read_unlock(); 328 goto out; 329 } 330 rcu_read_unlock(); 331 } 332 333 out: 334 *s_slot = slot; 335 *s_num = num; 336 return ret; 337 } 338 EXPORT_SYMBOL_GPL(mptcp_token_iter_next); 339 340 /** 341 * mptcp_token_destroy_request - remove mptcp connection/token 342 * @req: mptcp request socket dropping the token 343 * 344 * Remove the token associated to @req. 345 */ 346 void mptcp_token_destroy_request(struct request_sock *req) 347 { 348 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 349 struct mptcp_subflow_request_sock *pos; 350 struct token_bucket *bucket; 351 352 if (hlist_nulls_unhashed(&subflow_req->token_node)) 353 return; 354 355 bucket = token_bucket(subflow_req->token); 356 spin_lock_bh(&bucket->lock); 357 pos = __token_lookup_req(bucket, subflow_req->token); 358 if (!WARN_ON_ONCE(pos != subflow_req)) { 359 hlist_nulls_del_init_rcu(&pos->token_node); 360 bucket->chain_len--; 361 } 362 spin_unlock_bh(&bucket->lock); 363 } 364 365 /** 366 * mptcp_token_destroy - remove mptcp connection/token 367 * @msk: mptcp connection dropping the token 368 * 369 * Remove the token associated to @msk 370 */ 371 void mptcp_token_destroy(struct mptcp_sock *msk) 372 { 373 struct token_bucket *bucket; 374 struct mptcp_sock *pos; 375 376 if (sk_unhashed((struct sock *)msk)) 377 return; 378 379 bucket = token_bucket(msk->token); 380 spin_lock_bh(&bucket->lock); 381 pos = __token_lookup_msk(bucket, msk->token); 382 if (!WARN_ON_ONCE(pos != msk)) { 383 __sk_nulls_del_node_init_rcu((struct sock *)pos); 384 bucket->chain_len--; 385 } 386 spin_unlock_bh(&bucket->lock); 387 WRITE_ONCE(msk->token, 0); 388 } 389 390 void __init mptcp_token_init(void) 391 { 392 int i; 393 394 token_hash = alloc_large_system_hash("MPTCP token", 395 sizeof(struct token_bucket), 396 0, 397 20,/* one slot per 1MB of memory */ 398 HASH_ZERO, 399 NULL, 400 &token_mask, 401 0, 402 64 * 1024); 403 for (i = 0; i < token_mask + 1; ++i) { 404 INIT_HLIST_NULLS_HEAD(&token_hash[i].req_chain, i); 405 INIT_HLIST_NULLS_HEAD(&token_hash[i].msk_chain, i); 406 spin_lock_init(&token_hash[i].lock); 407 } 408 } 409 410 #if IS_MODULE(CONFIG_MPTCP_KUNIT_TEST) 411 EXPORT_SYMBOL_GPL(mptcp_token_new_request); 412 EXPORT_SYMBOL_GPL(mptcp_token_new_connect); 413 EXPORT_SYMBOL_GPL(mptcp_token_accept); 414 EXPORT_SYMBOL_GPL(mptcp_token_destroy_request); 415 EXPORT_SYMBOL_GPL(mptcp_token_destroy); 416 #endif 417