1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Codel/FQ_Codel and PIE/FQ-PIE Code: 5 * Copyright (C) 2016 Centre for Advanced Internet Architectures, 6 * Swinburne University of Technology, Melbourne, Australia. 7 * Portions of this code were made possible in part by a gift from 8 * The Comcast Innovation Fund. 9 * Implemented by Rasool Al-Saadi <ralsaadi@swin.edu.au> 10 * 11 * Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa 12 * Portions Copyright (c) 2000 Akamba Corp. 13 * All rights reserved 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 */ 36 37 #include <sys/cdefs.h> 38 __FBSDID("$FreeBSD$"); 39 40 /* 41 * Configuration and internal object management for dummynet. 42 */ 43 44 #include "opt_inet6.h" 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/malloc.h> 49 #include <sys/mbuf.h> 50 #include <sys/kernel.h> 51 #include <sys/lock.h> 52 #include <sys/module.h> 53 #include <sys/mutex.h> 54 #include <sys/priv.h> 55 #include <sys/proc.h> 56 #include <sys/rwlock.h> 57 #include <sys/socket.h> 58 #include <sys/socketvar.h> 59 #include <sys/time.h> 60 #include <sys/taskqueue.h> 61 #include <net/if.h> /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ 62 #include <netinet/in.h> 63 #include <netinet/ip_var.h> /* ip_output(), IP_FORWARDING */ 64 #include <netinet/ip_fw.h> 65 #include <netinet/ip_dummynet.h> 66 67 #include <netpfil/ipfw/ip_fw_private.h> 68 #include <netpfil/ipfw/dn_heap.h> 69 #include <netpfil/ipfw/ip_dn_private.h> 70 #ifdef NEW_AQM 71 #include <netpfil/ipfw/dn_aqm.h> 72 #endif 73 #include <netpfil/ipfw/dn_sched.h> 74 75 /* which objects to copy */ 76 #define DN_C_LINK 0x01 77 #define DN_C_SCH 0x02 78 #define DN_C_FLOW 0x04 79 #define DN_C_FS 0x08 80 #define DN_C_QUEUE 0x10 81 82 /* we use this argument in case of a schk_new */ 83 struct schk_new_arg { 84 struct dn_alg *fp; 85 struct dn_sch *sch; 86 }; 87 88 /*---- callout hooks. ----*/ 89 static struct callout dn_timeout; 90 static int dn_gone; 91 static struct task dn_task; 92 static struct taskqueue *dn_tq = NULL; 93 94 static void 95 dummynet(void *arg) 96 { 97 98 (void)arg; /* UNUSED */ 99 taskqueue_enqueue(dn_tq, &dn_task); 100 } 101 102 void 103 dn_reschedule(void) 104 { 105 106 if (dn_gone != 0) 107 return; 108 callout_reset_sbt(&dn_timeout, tick_sbt, 0, dummynet, NULL, 109 C_HARDCLOCK | C_DIRECT_EXEC); 110 } 111 /*----- end of callout hooks -----*/ 112 113 #ifdef NEW_AQM 114 /* Return AQM descriptor for given type or name. */ 115 static struct dn_aqm * 116 find_aqm_type(int type, char *name) 117 { 118 struct dn_aqm *d; 119 120 SLIST_FOREACH(d, &dn_cfg.aqmlist, next) { 121 if (d->type == type || (name && !strcasecmp(d->name, name))) 122 return d; 123 } 124 return NULL; /* not found */ 125 } 126 #endif 127 128 /* Return a scheduler descriptor given the type or name. */ 129 static struct dn_alg * 130 find_sched_type(int type, char *name) 131 { 132 struct dn_alg *d; 133 134 SLIST_FOREACH(d, &dn_cfg.schedlist, next) { 135 if (d->type == type || (name && !strcasecmp(d->name, name))) 136 return d; 137 } 138 return NULL; /* not found */ 139 } 140 141 int 142 ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg) 143 { 144 int oldv = *v; 145 const char *op = NULL; 146 if (dflt < lo) 147 dflt = lo; 148 if (dflt > hi) 149 dflt = hi; 150 if (oldv < lo) { 151 *v = dflt; 152 op = "Bump"; 153 } else if (oldv > hi) { 154 *v = hi; 155 op = "Clamp"; 156 } else 157 return *v; 158 if (op && msg) 159 printf("%s %s to %d (was %d)\n", op, msg, *v, oldv); 160 return *v; 161 } 162 163 /*---- flow_id mask, hash and compare functions ---*/ 164 /* 165 * The flow_id includes the 5-tuple, the queue/pipe number 166 * which we store in the extra area in host order, 167 * and for ipv6 also the flow_id6. 168 * XXX see if we want the tos byte (can store in 'flags') 169 */ 170 static struct ipfw_flow_id * 171 flow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id) 172 { 173 int is_v6 = IS_IP6_FLOW_ID(id); 174 175 id->dst_port &= mask->dst_port; 176 id->src_port &= mask->src_port; 177 id->proto &= mask->proto; 178 id->extra &= mask->extra; 179 if (is_v6) { 180 APPLY_MASK(&id->dst_ip6, &mask->dst_ip6); 181 APPLY_MASK(&id->src_ip6, &mask->src_ip6); 182 id->flow_id6 &= mask->flow_id6; 183 } else { 184 id->dst_ip &= mask->dst_ip; 185 id->src_ip &= mask->src_ip; 186 } 187 return id; 188 } 189 190 /* computes an OR of two masks, result in dst and also returned */ 191 static struct ipfw_flow_id * 192 flow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst) 193 { 194 int is_v6 = IS_IP6_FLOW_ID(dst); 195 196 dst->dst_port |= src->dst_port; 197 dst->src_port |= src->src_port; 198 dst->proto |= src->proto; 199 dst->extra |= src->extra; 200 if (is_v6) { 201 #define OR_MASK(_d, _s) \ 202 (_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \ 203 (_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \ 204 (_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \ 205 (_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3]; 206 OR_MASK(&dst->dst_ip6, &src->dst_ip6); 207 OR_MASK(&dst->src_ip6, &src->src_ip6); 208 #undef OR_MASK 209 dst->flow_id6 |= src->flow_id6; 210 } else { 211 dst->dst_ip |= src->dst_ip; 212 dst->src_ip |= src->src_ip; 213 } 214 return dst; 215 } 216 217 static int 218 nonzero_mask(struct ipfw_flow_id *m) 219 { 220 if (m->dst_port || m->src_port || m->proto || m->extra) 221 return 1; 222 if (IS_IP6_FLOW_ID(m)) { 223 return 224 m->dst_ip6.__u6_addr.__u6_addr32[0] || 225 m->dst_ip6.__u6_addr.__u6_addr32[1] || 226 m->dst_ip6.__u6_addr.__u6_addr32[2] || 227 m->dst_ip6.__u6_addr.__u6_addr32[3] || 228 m->src_ip6.__u6_addr.__u6_addr32[0] || 229 m->src_ip6.__u6_addr.__u6_addr32[1] || 230 m->src_ip6.__u6_addr.__u6_addr32[2] || 231 m->src_ip6.__u6_addr.__u6_addr32[3] || 232 m->flow_id6; 233 } else { 234 return m->dst_ip || m->src_ip; 235 } 236 } 237 238 /* XXX we may want a better hash function */ 239 static uint32_t 240 flow_id_hash(struct ipfw_flow_id *id) 241 { 242 uint32_t i; 243 244 if (IS_IP6_FLOW_ID(id)) { 245 uint32_t *d = (uint32_t *)&id->dst_ip6; 246 uint32_t *s = (uint32_t *)&id->src_ip6; 247 i = (d[0] ) ^ (d[1]) ^ 248 (d[2] ) ^ (d[3]) ^ 249 (d[0] >> 15) ^ (d[1] >> 15) ^ 250 (d[2] >> 15) ^ (d[3] >> 15) ^ 251 (s[0] << 1) ^ (s[1] << 1) ^ 252 (s[2] << 1) ^ (s[3] << 1) ^ 253 (s[0] << 16) ^ (s[1] << 16) ^ 254 (s[2] << 16) ^ (s[3] << 16) ^ 255 (id->dst_port << 1) ^ (id->src_port) ^ 256 (id->extra) ^ 257 (id->proto ) ^ (id->flow_id6); 258 } else { 259 i = (id->dst_ip) ^ (id->dst_ip >> 15) ^ 260 (id->src_ip << 1) ^ (id->src_ip >> 16) ^ 261 (id->extra) ^ 262 (id->dst_port << 1) ^ (id->src_port) ^ (id->proto); 263 } 264 return i; 265 } 266 267 /* Like bcmp, returns 0 if ids match, 1 otherwise. */ 268 static int 269 flow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2) 270 { 271 int is_v6 = IS_IP6_FLOW_ID(id1); 272 273 if (!is_v6) { 274 if (IS_IP6_FLOW_ID(id2)) 275 return 1; /* different address families */ 276 277 return (id1->dst_ip == id2->dst_ip && 278 id1->src_ip == id2->src_ip && 279 id1->dst_port == id2->dst_port && 280 id1->src_port == id2->src_port && 281 id1->proto == id2->proto && 282 id1->extra == id2->extra) ? 0 : 1; 283 } 284 /* the ipv6 case */ 285 return ( 286 !bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) && 287 !bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) && 288 id1->dst_port == id2->dst_port && 289 id1->src_port == id2->src_port && 290 id1->proto == id2->proto && 291 id1->extra == id2->extra && 292 id1->flow_id6 == id2->flow_id6) ? 0 : 1; 293 } 294 /*--------- end of flow-id mask, hash and compare ---------*/ 295 296 /*--- support functions for the qht hashtable ---- 297 * Entries are hashed by flow-id 298 */ 299 static uint32_t 300 q_hash(uintptr_t key, int flags, void *arg) 301 { 302 /* compute the hash slot from the flow id */ 303 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 304 &((struct dn_queue *)key)->ni.fid : 305 (struct ipfw_flow_id *)key; 306 307 return flow_id_hash(id); 308 } 309 310 static int 311 q_match(void *obj, uintptr_t key, int flags, void *arg) 312 { 313 struct dn_queue *o = (struct dn_queue *)obj; 314 struct ipfw_flow_id *id2; 315 316 if (flags & DNHT_KEY_IS_OBJ) { 317 /* compare pointers */ 318 id2 = &((struct dn_queue *)key)->ni.fid; 319 } else { 320 id2 = (struct ipfw_flow_id *)key; 321 } 322 return (0 == flow_id_cmp(&o->ni.fid, id2)); 323 } 324 325 /* 326 * create a new queue instance for the given 'key'. 327 */ 328 static void * 329 q_new(uintptr_t key, int flags, void *arg) 330 { 331 struct dn_queue *q, *template = arg; 332 struct dn_fsk *fs = template->fs; 333 int size = sizeof(*q) + fs->sched->fp->q_datalen; 334 335 q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO); 336 if (q == NULL) { 337 D("no memory for new queue"); 338 return NULL; 339 } 340 341 set_oid(&q->ni.oid, DN_QUEUE, size); 342 if (fs->fs.flags & DN_QHT_HASH) 343 q->ni.fid = *(struct ipfw_flow_id *)key; 344 q->fs = fs; 345 q->_si = template->_si; 346 q->_si->q_count++; 347 348 if (fs->sched->fp->new_queue) 349 fs->sched->fp->new_queue(q); 350 351 #ifdef NEW_AQM 352 /* call AQM init function after creating a queue*/ 353 if (fs->aqmfp && fs->aqmfp->init) 354 if(fs->aqmfp->init(q)) 355 D("unable to init AQM for fs %d", fs->fs.fs_nr); 356 #endif 357 dn_cfg.queue_count++; 358 359 return q; 360 } 361 362 /* 363 * Notify schedulers that a queue is going away. 364 * If (flags & DN_DESTROY), also free the packets. 365 * The version for callbacks is called q_delete_cb(). 366 */ 367 static void 368 dn_delete_queue(struct dn_queue *q, int flags) 369 { 370 struct dn_fsk *fs = q->fs; 371 372 #ifdef NEW_AQM 373 /* clean up AQM status for queue 'q' 374 * cleanup here is called just with MULTIQUEUE 375 */ 376 if (fs && fs->aqmfp && fs->aqmfp->cleanup) 377 fs->aqmfp->cleanup(q); 378 #endif 379 // D("fs %p si %p\n", fs, q->_si); 380 /* notify the parent scheduler that the queue is going away */ 381 if (fs && fs->sched->fp->free_queue) 382 fs->sched->fp->free_queue(q); 383 q->_si->q_count--; 384 q->_si = NULL; 385 if (flags & DN_DESTROY) { 386 if (q->mq.head) 387 dn_free_pkts(q->mq.head); 388 bzero(q, sizeof(*q)); // safety 389 free(q, M_DUMMYNET); 390 dn_cfg.queue_count--; 391 } 392 } 393 394 static int 395 q_delete_cb(void *q, void *arg) 396 { 397 int flags = (int)(uintptr_t)arg; 398 dn_delete_queue(q, flags); 399 return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0; 400 } 401 402 /* 403 * calls dn_delete_queue/q_delete_cb on all queues, 404 * which notifies the parent scheduler and possibly drains packets. 405 * flags & DN_DESTROY: drains queues and destroy qht; 406 */ 407 static void 408 qht_delete(struct dn_fsk *fs, int flags) 409 { 410 ND("fs %d start flags %d qht %p", 411 fs->fs.fs_nr, flags, fs->qht); 412 if (!fs->qht) 413 return; 414 if (fs->fs.flags & DN_QHT_HASH) { 415 dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags); 416 if (flags & DN_DESTROY) { 417 dn_ht_free(fs->qht, 0); 418 fs->qht = NULL; 419 } 420 } else { 421 dn_delete_queue((struct dn_queue *)(fs->qht), flags); 422 if (flags & DN_DESTROY) 423 fs->qht = NULL; 424 } 425 } 426 427 /* 428 * Find and possibly create the queue for a MULTIQUEUE scheduler. 429 * We never call it for !MULTIQUEUE (the queue is in the sch_inst). 430 */ 431 struct dn_queue * 432 ipdn_q_find(struct dn_fsk *fs, struct dn_sch_inst *si, 433 struct ipfw_flow_id *id) 434 { 435 struct dn_queue template; 436 437 template._si = si; 438 template.fs = fs; 439 440 if (fs->fs.flags & DN_QHT_HASH) { 441 struct ipfw_flow_id masked_id; 442 if (fs->qht == NULL) { 443 fs->qht = dn_ht_init(NULL, fs->fs.buckets, 444 offsetof(struct dn_queue, q_next), 445 q_hash, q_match, q_new); 446 if (fs->qht == NULL) 447 return NULL; 448 } 449 masked_id = *id; 450 flow_id_mask(&fs->fsk_mask, &masked_id); 451 return dn_ht_find(fs->qht, (uintptr_t)&masked_id, 452 DNHT_INSERT, &template); 453 } else { 454 if (fs->qht == NULL) 455 fs->qht = q_new(0, 0, &template); 456 return (struct dn_queue *)fs->qht; 457 } 458 } 459 /*--- end of queue hash table ---*/ 460 461 /*--- support functions for the sch_inst hashtable ---- 462 * 463 * These are hashed by flow-id 464 */ 465 static uint32_t 466 si_hash(uintptr_t key, int flags, void *arg) 467 { 468 /* compute the hash slot from the flow id */ 469 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 470 &((struct dn_sch_inst *)key)->ni.fid : 471 (struct ipfw_flow_id *)key; 472 473 return flow_id_hash(id); 474 } 475 476 static int 477 si_match(void *obj, uintptr_t key, int flags, void *arg) 478 { 479 struct dn_sch_inst *o = obj; 480 struct ipfw_flow_id *id2; 481 482 id2 = (flags & DNHT_KEY_IS_OBJ) ? 483 &((struct dn_sch_inst *)key)->ni.fid : 484 (struct ipfw_flow_id *)key; 485 return flow_id_cmp(&o->ni.fid, id2) == 0; 486 } 487 488 /* 489 * create a new instance for the given 'key' 490 * Allocate memory for instance, delay line and scheduler private data. 491 */ 492 static void * 493 si_new(uintptr_t key, int flags, void *arg) 494 { 495 struct dn_schk *s = arg; 496 struct dn_sch_inst *si; 497 int l = sizeof(*si) + s->fp->si_datalen; 498 499 si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 500 if (si == NULL) 501 goto error; 502 503 /* Set length only for the part passed up to userland. */ 504 set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow)); 505 set_oid(&(si->dline.oid), DN_DELAY_LINE, 506 sizeof(struct delay_line)); 507 /* mark si and dline as outside the event queue */ 508 si->ni.oid.id = si->dline.oid.id = -1; 509 510 si->sched = s; 511 si->dline.si = si; 512 513 if (s->fp->new_sched && s->fp->new_sched(si)) { 514 D("new_sched error"); 515 goto error; 516 } 517 if (s->sch.flags & DN_HAVE_MASK) 518 si->ni.fid = *(struct ipfw_flow_id *)key; 519 520 #ifdef NEW_AQM 521 /* init AQM status for !DN_MULTIQUEUE sched*/ 522 if (!(s->fp->flags & DN_MULTIQUEUE)) 523 if (s->fs->aqmfp && s->fs->aqmfp->init) 524 if(s->fs->aqmfp->init((struct dn_queue *)(si + 1))) { 525 D("unable to init AQM for fs %d", s->fs->fs.fs_nr); 526 goto error; 527 } 528 #endif 529 530 dn_cfg.si_count++; 531 return si; 532 533 error: 534 if (si) { 535 bzero(si, sizeof(*si)); // safety 536 free(si, M_DUMMYNET); 537 } 538 return NULL; 539 } 540 541 /* 542 * Callback from siht to delete all scheduler instances. Remove 543 * si and delay line from the system heap, destroy all queues. 544 * We assume that all flowset have been notified and do not 545 * point to us anymore. 546 */ 547 static int 548 si_destroy(void *_si, void *arg) 549 { 550 struct dn_sch_inst *si = _si; 551 struct dn_schk *s = si->sched; 552 struct delay_line *dl = &si->dline; 553 554 if (dl->oid.subtype) /* remove delay line from event heap */ 555 heap_extract(&dn_cfg.evheap, dl); 556 dn_free_pkts(dl->mq.head); /* drain delay line */ 557 if (si->kflags & DN_ACTIVE) /* remove si from event heap */ 558 heap_extract(&dn_cfg.evheap, si); 559 560 #ifdef NEW_AQM 561 /* clean up AQM status for !DN_MULTIQUEUE sched 562 * Note that all queues belong to fs were cleaned up in fsk_detach. 563 * When drain_scheduler is called s->fs and q->fs are pointing 564 * to a correct fs, so we can use fs in this case. 565 */ 566 if (!(s->fp->flags & DN_MULTIQUEUE)) { 567 struct dn_queue *q = (struct dn_queue *)(si + 1); 568 if (q->aqm_status && q->fs->aqmfp) 569 if (q->fs->aqmfp->cleanup) 570 q->fs->aqmfp->cleanup(q); 571 } 572 #endif 573 if (s->fp->free_sched) 574 s->fp->free_sched(si); 575 bzero(si, sizeof(*si)); /* safety */ 576 free(si, M_DUMMYNET); 577 dn_cfg.si_count--; 578 return DNHT_SCAN_DEL; 579 } 580 581 /* 582 * Find the scheduler instance for this packet. If we need to apply 583 * a mask, do on a local copy of the flow_id to preserve the original. 584 * Assume siht is always initialized if we have a mask. 585 */ 586 struct dn_sch_inst * 587 ipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id) 588 { 589 590 if (s->sch.flags & DN_HAVE_MASK) { 591 struct ipfw_flow_id id_t = *id; 592 flow_id_mask(&s->sch.sched_mask, &id_t); 593 return dn_ht_find(s->siht, (uintptr_t)&id_t, 594 DNHT_INSERT, s); 595 } 596 if (!s->siht) 597 s->siht = si_new(0, 0, s); 598 return (struct dn_sch_inst *)s->siht; 599 } 600 601 /* callback to flush credit for the scheduler instance */ 602 static int 603 si_reset_credit(void *_si, void *arg) 604 { 605 struct dn_sch_inst *si = _si; 606 struct dn_link *p = &si->sched->link; 607 608 si->credit = p->burst + (dn_cfg.io_fast ? p->bandwidth : 0); 609 return 0; 610 } 611 612 static void 613 schk_reset_credit(struct dn_schk *s) 614 { 615 if (s->sch.flags & DN_HAVE_MASK) 616 dn_ht_scan(s->siht, si_reset_credit, NULL); 617 else if (s->siht) 618 si_reset_credit(s->siht, NULL); 619 } 620 /*---- end of sch_inst hashtable ---------------------*/ 621 622 /*------------------------------------------------------- 623 * flowset hash (fshash) support. Entries are hashed by fs_nr. 624 * New allocations are put in the fsunlinked list, from which 625 * they are removed when they point to a specific scheduler. 626 */ 627 static uint32_t 628 fsk_hash(uintptr_t key, int flags, void *arg) 629 { 630 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 631 ((struct dn_fsk *)key)->fs.fs_nr; 632 633 return ( (i>>8)^(i>>4)^i ); 634 } 635 636 static int 637 fsk_match(void *obj, uintptr_t key, int flags, void *arg) 638 { 639 struct dn_fsk *fs = obj; 640 int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 641 ((struct dn_fsk *)key)->fs.fs_nr; 642 643 return (fs->fs.fs_nr == i); 644 } 645 646 static void * 647 fsk_new(uintptr_t key, int flags, void *arg) 648 { 649 struct dn_fsk *fs; 650 651 fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO); 652 if (fs) { 653 set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs)); 654 dn_cfg.fsk_count++; 655 fs->drain_bucket = 0; 656 SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); 657 } 658 return fs; 659 } 660 661 #ifdef NEW_AQM 662 /* callback function for cleaning up AQM queue status belongs to a flowset 663 * connected to scheduler instance '_si' (for !DN_MULTIQUEUE only). 664 */ 665 static int 666 si_cleanup_q(void *_si, void *arg) 667 { 668 struct dn_sch_inst *si = _si; 669 670 if (!(si->sched->fp->flags & DN_MULTIQUEUE)) { 671 if (si->sched->fs->aqmfp && si->sched->fs->aqmfp->cleanup) 672 si->sched->fs->aqmfp->cleanup((struct dn_queue *) (si+1)); 673 } 674 return 0; 675 } 676 677 /* callback to clean up queue AQM status.*/ 678 static int 679 q_cleanup_q(void *_q, void *arg) 680 { 681 struct dn_queue *q = _q; 682 q->fs->aqmfp->cleanup(q); 683 return 0; 684 } 685 686 /* Clean up all AQM queues status belongs to flowset 'fs' and then 687 * deconfig AQM for flowset 'fs' 688 */ 689 static void 690 aqm_cleanup_deconfig_fs(struct dn_fsk *fs) 691 { 692 struct dn_sch_inst *si; 693 694 /* clean up AQM status for all queues for !DN_MULTIQUEUE sched*/ 695 if (fs->fs.fs_nr > DN_MAX_ID) { 696 if (fs->sched && !(fs->sched->fp->flags & DN_MULTIQUEUE)) { 697 if (fs->sched->sch.flags & DN_HAVE_MASK) 698 dn_ht_scan(fs->sched->siht, si_cleanup_q, NULL); 699 else { 700 /* single si i.e. no sched mask */ 701 si = (struct dn_sch_inst *) fs->sched->siht; 702 if (si && fs->aqmfp && fs->aqmfp->cleanup) 703 fs->aqmfp->cleanup((struct dn_queue *) (si+1)); 704 } 705 } 706 } 707 708 /* clean up AQM status for all queues for DN_MULTIQUEUE sched*/ 709 if (fs->sched && fs->sched->fp->flags & DN_MULTIQUEUE && fs->qht) { 710 if (fs->fs.flags & DN_QHT_HASH) 711 dn_ht_scan(fs->qht, q_cleanup_q, NULL); 712 else 713 fs->aqmfp->cleanup((struct dn_queue *)(fs->qht)); 714 } 715 716 /* deconfig AQM */ 717 if(fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) 718 fs->aqmfp->deconfig(fs); 719 } 720 #endif 721 722 /* 723 * detach flowset from its current scheduler. Flags as follows: 724 * DN_DETACH removes from the fsk_list 725 * DN_DESTROY deletes individual queues 726 * DN_DELETE_FS destroys the flowset (otherwise goes in unlinked). 727 */ 728 static void 729 fsk_detach(struct dn_fsk *fs, int flags) 730 { 731 if (flags & DN_DELETE_FS) 732 flags |= DN_DESTROY; 733 ND("fs %d from sched %d flags %s %s %s", 734 fs->fs.fs_nr, fs->fs.sched_nr, 735 (flags & DN_DELETE_FS) ? "DEL_FS":"", 736 (flags & DN_DESTROY) ? "DEL":"", 737 (flags & DN_DETACH) ? "DET":""); 738 if (flags & DN_DETACH) { /* detach from the list */ 739 struct dn_fsk_head *h; 740 h = fs->sched ? &fs->sched->fsk_list : &dn_cfg.fsu; 741 SLIST_REMOVE(h, fs, dn_fsk, sch_chain); 742 } 743 /* Free the RED parameters, they will be recomputed on 744 * subsequent attach if needed. 745 */ 746 if (fs->w_q_lookup) 747 free(fs->w_q_lookup, M_DUMMYNET); 748 fs->w_q_lookup = NULL; 749 qht_delete(fs, flags); 750 #ifdef NEW_AQM 751 aqm_cleanup_deconfig_fs(fs); 752 #endif 753 754 if (fs->sched && fs->sched->fp->free_fsk) 755 fs->sched->fp->free_fsk(fs); 756 fs->sched = NULL; 757 if (flags & DN_DELETE_FS) { 758 bzero(fs, sizeof(*fs)); /* safety */ 759 free(fs, M_DUMMYNET); 760 dn_cfg.fsk_count--; 761 } else { 762 SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); 763 } 764 } 765 766 /* 767 * Detach or destroy all flowsets in a list. 768 * flags specifies what to do: 769 * DN_DESTROY: flush all queues 770 * DN_DELETE_FS: DN_DESTROY + destroy flowset 771 * DN_DELETE_FS implies DN_DESTROY 772 */ 773 static void 774 fsk_detach_list(struct dn_fsk_head *h, int flags) 775 { 776 struct dn_fsk *fs; 777 int n = 0; /* only for stats */ 778 779 ND("head %p flags %x", h, flags); 780 while ((fs = SLIST_FIRST(h))) { 781 SLIST_REMOVE_HEAD(h, sch_chain); 782 n++; 783 fsk_detach(fs, flags); 784 } 785 ND("done %d flowsets", n); 786 } 787 788 /* 789 * called on 'queue X delete' -- removes the flowset from fshash, 790 * deletes all queues for the flowset, and removes the flowset. 791 */ 792 static int 793 delete_fs(int i, int locked) 794 { 795 struct dn_fsk *fs; 796 int err = 0; 797 798 if (!locked) 799 DN_BH_WLOCK(); 800 fs = dn_ht_find(dn_cfg.fshash, i, DNHT_REMOVE, NULL); 801 ND("fs %d found %p", i, fs); 802 if (fs) { 803 fsk_detach(fs, DN_DETACH | DN_DELETE_FS); 804 err = 0; 805 } else 806 err = EINVAL; 807 if (!locked) 808 DN_BH_WUNLOCK(); 809 return err; 810 } 811 812 /*----- end of flowset hashtable support -------------*/ 813 814 /*------------------------------------------------------------ 815 * Scheduler hash. When searching by index we pass sched_nr, 816 * otherwise we pass struct dn_sch * which is the first field in 817 * struct dn_schk so we can cast between the two. We use this trick 818 * because in the create phase (but it should be fixed). 819 */ 820 static uint32_t 821 schk_hash(uintptr_t key, int flags, void *_arg) 822 { 823 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 824 ((struct dn_schk *)key)->sch.sched_nr; 825 return ( (i>>8)^(i>>4)^i ); 826 } 827 828 static int 829 schk_match(void *obj, uintptr_t key, int flags, void *_arg) 830 { 831 struct dn_schk *s = (struct dn_schk *)obj; 832 int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 833 ((struct dn_schk *)key)->sch.sched_nr; 834 return (s->sch.sched_nr == i); 835 } 836 837 /* 838 * Create the entry and intialize with the sched hash if needed. 839 * Leave s->fp unset so we can tell whether a dn_ht_find() returns 840 * a new object or a previously existing one. 841 */ 842 static void * 843 schk_new(uintptr_t key, int flags, void *arg) 844 { 845 struct schk_new_arg *a = arg; 846 struct dn_schk *s; 847 int l = sizeof(*s) +a->fp->schk_datalen; 848 849 s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 850 if (s == NULL) 851 return NULL; 852 set_oid(&s->link.oid, DN_LINK, sizeof(s->link)); 853 s->sch = *a->sch; // copy initial values 854 s->link.link_nr = s->sch.sched_nr; 855 SLIST_INIT(&s->fsk_list); 856 /* initialize the hash table or create the single instance */ 857 s->fp = a->fp; /* si_new needs this */ 858 s->drain_bucket = 0; 859 if (s->sch.flags & DN_HAVE_MASK) { 860 s->siht = dn_ht_init(NULL, s->sch.buckets, 861 offsetof(struct dn_sch_inst, si_next), 862 si_hash, si_match, si_new); 863 if (s->siht == NULL) { 864 free(s, M_DUMMYNET); 865 return NULL; 866 } 867 } 868 s->fp = NULL; /* mark as a new scheduler */ 869 dn_cfg.schk_count++; 870 return s; 871 } 872 873 /* 874 * Callback for sched delete. Notify all attached flowsets to 875 * detach from the scheduler, destroy the internal flowset, and 876 * all instances. The scheduler goes away too. 877 * arg is 0 (only detach flowsets and destroy instances) 878 * DN_DESTROY (detach & delete queues, delete schk) 879 * or DN_DELETE_FS (delete queues and flowsets, delete schk) 880 */ 881 static int 882 schk_delete_cb(void *obj, void *arg) 883 { 884 struct dn_schk *s = obj; 885 #if 0 886 int a = (int)arg; 887 ND("sched %d arg %s%s", 888 s->sch.sched_nr, 889 a&DN_DESTROY ? "DEL ":"", 890 a&DN_DELETE_FS ? "DEL_FS":""); 891 #endif 892 fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0); 893 /* no more flowset pointing to us now */ 894 if (s->sch.flags & DN_HAVE_MASK) { 895 dn_ht_scan(s->siht, si_destroy, NULL); 896 dn_ht_free(s->siht, 0); 897 } else if (s->siht) 898 si_destroy(s->siht, NULL); 899 if (s->profile) { 900 free(s->profile, M_DUMMYNET); 901 s->profile = NULL; 902 } 903 s->siht = NULL; 904 if (s->fp->destroy) 905 s->fp->destroy(s); 906 bzero(s, sizeof(*s)); // safety 907 free(obj, M_DUMMYNET); 908 dn_cfg.schk_count--; 909 return DNHT_SCAN_DEL; 910 } 911 912 /* 913 * called on a 'sched X delete' command. Deletes a single scheduler. 914 * This is done by removing from the schedhash, unlinking all 915 * flowsets and deleting their traffic. 916 */ 917 static int 918 delete_schk(int i) 919 { 920 struct dn_schk *s; 921 922 s = dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 923 ND("%d %p", i, s); 924 if (!s) 925 return EINVAL; 926 delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */ 927 /* then detach flowsets, delete traffic */ 928 schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY); 929 return 0; 930 } 931 /*--- end of schk hashtable support ---*/ 932 933 static int 934 copy_obj(char **start, char *end, void *_o, const char *msg, int i) 935 { 936 struct dn_id o; 937 union { 938 struct dn_link l; 939 struct dn_schk s; 940 } dn; 941 int have = end - *start; 942 943 memcpy(&o, _o, sizeof(o)); 944 if (have < o.len || o.len == 0 || o.type == 0) { 945 D("(WARN) type %d %s %d have %d need %d", 946 o.type, msg, i, have, o.len); 947 return 1; 948 } 949 ND("type %d %s %d len %d", o.type, msg, i, o.len); 950 if (o.type == DN_LINK) { 951 memcpy(&dn.l, _o, sizeof(dn.l)); 952 /* Adjust burst parameter for link */ 953 dn.l.burst = div64(dn.l.burst, 8 * hz); 954 dn.l.delay = dn.l.delay * 1000 / hz; 955 memcpy(*start, &dn.l, sizeof(dn.l)); 956 } else if (o.type == DN_SCH) { 957 /* Set dn.s.sch.oid.id to the number of instances */ 958 memcpy(&dn.s, _o, sizeof(dn.s)); 959 dn.s.sch.oid.id = (dn.s.sch.flags & DN_HAVE_MASK) ? 960 dn_ht_entries(dn.s.siht) : (dn.s.siht ? 1 : 0); 961 memcpy(*start, &dn.s, sizeof(dn.s)); 962 } else 963 memcpy(*start, _o, o.len); 964 *start += o.len; 965 return 0; 966 } 967 968 /* Specific function to copy a queue. 969 * Copies only the user-visible part of a queue (which is in 970 * a struct dn_flow), and sets len accordingly. 971 */ 972 static int 973 copy_obj_q(char **start, char *end, void *_o, const char *msg, int i) 974 { 975 struct dn_id *o = _o; 976 int have = end - *start; 977 int len = sizeof(struct dn_flow); /* see above comment */ 978 979 if (have < len || o->len == 0 || o->type != DN_QUEUE) { 980 D("ERROR type %d %s %d have %d need %d", 981 o->type, msg, i, have, len); 982 return 1; 983 } 984 ND("type %d %s %d len %d", o->type, msg, i, len); 985 memcpy(*start, _o, len); 986 ((struct dn_id*)(*start))->len = len; 987 *start += len; 988 return 0; 989 } 990 991 static int 992 copy_q_cb(void *obj, void *arg) 993 { 994 struct dn_queue *q = obj; 995 struct copy_args *a = arg; 996 struct dn_flow *ni = (struct dn_flow *)(*a->start); 997 if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1)) 998 return DNHT_SCAN_END; 999 ni->oid.type = DN_FLOW; /* override the DN_QUEUE */ 1000 ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL); 1001 return 0; 1002 } 1003 1004 static int 1005 copy_q(struct copy_args *a, struct dn_fsk *fs, int flags) 1006 { 1007 if (!fs->qht) 1008 return 0; 1009 if (fs->fs.flags & DN_QHT_HASH) 1010 dn_ht_scan(fs->qht, copy_q_cb, a); 1011 else 1012 copy_q_cb(fs->qht, a); 1013 return 0; 1014 } 1015 1016 /* 1017 * This routine only copies the initial part of a profile ? XXX 1018 */ 1019 static int 1020 copy_profile(struct copy_args *a, struct dn_profile *p) 1021 { 1022 int have = a->end - *a->start; 1023 /* XXX here we check for max length */ 1024 int profile_len = sizeof(struct dn_profile) - 1025 ED_MAX_SAMPLES_NO*sizeof(int); 1026 1027 if (p == NULL) 1028 return 0; 1029 if (have < profile_len) { 1030 D("error have %d need %d", have, profile_len); 1031 return 1; 1032 } 1033 memcpy(*a->start, p, profile_len); 1034 ((struct dn_id *)(*a->start))->len = profile_len; 1035 *a->start += profile_len; 1036 return 0; 1037 } 1038 1039 static int 1040 copy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags) 1041 { 1042 struct dn_fs *ufs = (struct dn_fs *)(*a->start); 1043 if (!fs) 1044 return 0; 1045 ND("flowset %d", fs->fs.fs_nr); 1046 if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr)) 1047 return DNHT_SCAN_END; 1048 ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ? 1049 dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0); 1050 if (flags) { /* copy queues */ 1051 copy_q(a, fs, 0); 1052 } 1053 return 0; 1054 } 1055 1056 static int 1057 copy_si_cb(void *obj, void *arg) 1058 { 1059 struct dn_sch_inst *si = obj; 1060 struct copy_args *a = arg; 1061 struct dn_flow *ni = (struct dn_flow *)(*a->start); 1062 if (copy_obj(a->start, a->end, &si->ni, "inst", 1063 si->sched->sch.sched_nr)) 1064 return DNHT_SCAN_END; 1065 ni->oid.type = DN_FLOW; /* override the DN_SCH_I */ 1066 ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL); 1067 return 0; 1068 } 1069 1070 static int 1071 copy_si(struct copy_args *a, struct dn_schk *s, int flags) 1072 { 1073 if (s->sch.flags & DN_HAVE_MASK) 1074 dn_ht_scan(s->siht, copy_si_cb, a); 1075 else if (s->siht) 1076 copy_si_cb(s->siht, a); 1077 return 0; 1078 } 1079 1080 /* 1081 * compute a list of children of a scheduler and copy up 1082 */ 1083 static int 1084 copy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags) 1085 { 1086 struct dn_fsk *fs; 1087 struct dn_id *o; 1088 uint32_t *p; 1089 1090 int n = 0, space = sizeof(*o); 1091 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 1092 if (fs->fs.fs_nr < DN_MAX_ID) 1093 n++; 1094 } 1095 space += n * sizeof(uint32_t); 1096 DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n); 1097 if (a->end - *(a->start) < space) 1098 return DNHT_SCAN_END; 1099 o = (struct dn_id *)(*(a->start)); 1100 o->len = space; 1101 *a->start += o->len; 1102 o->type = DN_TEXT; 1103 p = (uint32_t *)(o+1); 1104 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) 1105 if (fs->fs.fs_nr < DN_MAX_ID) 1106 *p++ = fs->fs.fs_nr; 1107 return 0; 1108 } 1109 1110 static int 1111 copy_data_helper(void *_o, void *_arg) 1112 { 1113 struct copy_args *a = _arg; 1114 uint32_t *r = a->extra->r; /* start of first range */ 1115 uint32_t *lim; /* first invalid pointer */ 1116 int n; 1117 1118 lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len); 1119 1120 if (a->type == DN_LINK || a->type == DN_SCH) { 1121 /* pipe|sched show, we receive a dn_schk */ 1122 struct dn_schk *s = _o; 1123 1124 n = s->sch.sched_nr; 1125 if (a->type == DN_SCH && n >= DN_MAX_ID) 1126 return 0; /* not a scheduler */ 1127 if (a->type == DN_LINK && n <= DN_MAX_ID) 1128 return 0; /* not a pipe */ 1129 1130 /* see if the object is within one of our ranges */ 1131 for (;r < lim; r += 2) { 1132 if (n < r[0] || n > r[1]) 1133 continue; 1134 /* Found a valid entry, copy and we are done */ 1135 if (a->flags & DN_C_LINK) { 1136 if (copy_obj(a->start, a->end, 1137 &s->link, "link", n)) 1138 return DNHT_SCAN_END; 1139 if (copy_profile(a, s->profile)) 1140 return DNHT_SCAN_END; 1141 if (copy_flowset(a, s->fs, 0)) 1142 return DNHT_SCAN_END; 1143 } 1144 if (a->flags & DN_C_SCH) { 1145 if (copy_obj(a->start, a->end, 1146 &s->sch, "sched", n)) 1147 return DNHT_SCAN_END; 1148 /* list all attached flowsets */ 1149 if (copy_fsk_list(a, s, 0)) 1150 return DNHT_SCAN_END; 1151 } 1152 if (a->flags & DN_C_FLOW) 1153 copy_si(a, s, 0); 1154 break; 1155 } 1156 } else if (a->type == DN_FS) { 1157 /* queue show, skip internal flowsets */ 1158 struct dn_fsk *fs = _o; 1159 1160 n = fs->fs.fs_nr; 1161 if (n >= DN_MAX_ID) 1162 return 0; 1163 /* see if the object is within one of our ranges */ 1164 for (;r < lim; r += 2) { 1165 if (n < r[0] || n > r[1]) 1166 continue; 1167 if (copy_flowset(a, fs, 0)) 1168 return DNHT_SCAN_END; 1169 copy_q(a, fs, 0); 1170 break; /* we are done */ 1171 } 1172 } 1173 return 0; 1174 } 1175 1176 static inline struct dn_schk * 1177 locate_scheduler(int i) 1178 { 1179 return dn_ht_find(dn_cfg.schedhash, i, 0, NULL); 1180 } 1181 1182 /* 1183 * red parameters are in fixed point arithmetic. 1184 */ 1185 static int 1186 config_red(struct dn_fsk *fs) 1187 { 1188 int64_t s, idle, weight, w0; 1189 int t, i; 1190 1191 fs->w_q = fs->fs.w_q; 1192 fs->max_p = fs->fs.max_p; 1193 ND("called"); 1194 /* Doing stuff that was in userland */ 1195 i = fs->sched->link.bandwidth; 1196 s = (i <= 0) ? 0 : 1197 hz * dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i; 1198 1199 idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */ 1200 fs->lookup_step = div64(idle , dn_cfg.red_lookup_depth); 1201 /* fs->lookup_step not scaled, */ 1202 if (!fs->lookup_step) 1203 fs->lookup_step = 1; 1204 w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled 1205 1206 for (t = fs->lookup_step; t > 1; --t) 1207 weight = SCALE_MUL(weight, w0); 1208 fs->lookup_weight = (int)(weight); // scaled 1209 1210 /* Now doing stuff that was in kerneland */ 1211 fs->min_th = SCALE(fs->fs.min_th); 1212 fs->max_th = SCALE(fs->fs.max_th); 1213 1214 if (fs->fs.max_th == fs->fs.min_th) 1215 fs->c_1 = fs->max_p; 1216 else 1217 fs->c_1 = SCALE((int64_t)(fs->max_p)) / (fs->fs.max_th - fs->fs.min_th); 1218 fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th)); 1219 1220 if (fs->fs.flags & DN_IS_GENTLE_RED) { 1221 fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th; 1222 fs->c_4 = SCALE(1) - 2 * fs->max_p; 1223 } 1224 1225 /* If the lookup table already exist, free and create it again. */ 1226 if (fs->w_q_lookup) { 1227 free(fs->w_q_lookup, M_DUMMYNET); 1228 fs->w_q_lookup = NULL; 1229 } 1230 if (dn_cfg.red_lookup_depth == 0) { 1231 printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth" 1232 "must be > 0\n"); 1233 fs->fs.flags &= ~DN_IS_RED; 1234 fs->fs.flags &= ~DN_IS_GENTLE_RED; 1235 return (EINVAL); 1236 } 1237 fs->lookup_depth = dn_cfg.red_lookup_depth; 1238 fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int), 1239 M_DUMMYNET, M_NOWAIT); 1240 if (fs->w_q_lookup == NULL) { 1241 printf("dummynet: sorry, cannot allocate red lookup table\n"); 1242 fs->fs.flags &= ~DN_IS_RED; 1243 fs->fs.flags &= ~DN_IS_GENTLE_RED; 1244 return(ENOSPC); 1245 } 1246 1247 /* Fill the lookup table with (1 - w_q)^x */ 1248 fs->w_q_lookup[0] = SCALE(1) - fs->w_q; 1249 1250 for (i = 1; i < fs->lookup_depth; i++) 1251 fs->w_q_lookup[i] = 1252 SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight); 1253 1254 if (dn_cfg.red_avg_pkt_size < 1) 1255 dn_cfg.red_avg_pkt_size = 512; 1256 fs->avg_pkt_size = dn_cfg.red_avg_pkt_size; 1257 if (dn_cfg.red_max_pkt_size < 1) 1258 dn_cfg.red_max_pkt_size = 1500; 1259 fs->max_pkt_size = dn_cfg.red_max_pkt_size; 1260 ND("exit"); 1261 return 0; 1262 } 1263 1264 /* Scan all flowset attached to this scheduler and update red */ 1265 static void 1266 update_red(struct dn_schk *s) 1267 { 1268 struct dn_fsk *fs; 1269 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 1270 if (fs && (fs->fs.flags & DN_IS_RED)) 1271 config_red(fs); 1272 } 1273 } 1274 1275 /* attach flowset to scheduler s, possibly requeue */ 1276 static void 1277 fsk_attach(struct dn_fsk *fs, struct dn_schk *s) 1278 { 1279 ND("remove fs %d from fsunlinked, link to sched %d", 1280 fs->fs.fs_nr, s->sch.sched_nr); 1281 SLIST_REMOVE(&dn_cfg.fsu, fs, dn_fsk, sch_chain); 1282 fs->sched = s; 1283 SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain); 1284 if (s->fp->new_fsk) 1285 s->fp->new_fsk(fs); 1286 /* XXX compute fsk_mask */ 1287 fs->fsk_mask = fs->fs.flow_mask; 1288 if (fs->sched->sch.flags & DN_HAVE_MASK) 1289 flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask); 1290 if (fs->qht) { 1291 /* 1292 * we must drain qht according to the old 1293 * type, and reinsert according to the new one. 1294 * The requeue is complex -- in general we need to 1295 * reclassify every single packet. 1296 * For the time being, let's hope qht is never set 1297 * when we reach this point. 1298 */ 1299 D("XXX TODO requeue from fs %d to sch %d", 1300 fs->fs.fs_nr, s->sch.sched_nr); 1301 fs->qht = NULL; 1302 } 1303 /* set the new type for qht */ 1304 if (nonzero_mask(&fs->fsk_mask)) 1305 fs->fs.flags |= DN_QHT_HASH; 1306 else 1307 fs->fs.flags &= ~DN_QHT_HASH; 1308 1309 /* XXX config_red() can fail... */ 1310 if (fs->fs.flags & DN_IS_RED) 1311 config_red(fs); 1312 } 1313 1314 /* update all flowsets which may refer to this scheduler */ 1315 static void 1316 update_fs(struct dn_schk *s) 1317 { 1318 struct dn_fsk *fs, *tmp; 1319 1320 SLIST_FOREACH_SAFE(fs, &dn_cfg.fsu, sch_chain, tmp) { 1321 if (s->sch.sched_nr != fs->fs.sched_nr) { 1322 D("fs %d for sch %d not %d still unlinked", 1323 fs->fs.fs_nr, fs->fs.sched_nr, 1324 s->sch.sched_nr); 1325 continue; 1326 } 1327 fsk_attach(fs, s); 1328 } 1329 } 1330 1331 #ifdef NEW_AQM 1332 /* Retrieve AQM configurations to ipfw userland 1333 */ 1334 static int 1335 get_aqm_parms(struct sockopt *sopt) 1336 { 1337 struct dn_extra_parms *ep; 1338 struct dn_fsk *fs; 1339 size_t sopt_valsize; 1340 int l, err = 0; 1341 1342 sopt_valsize = sopt->sopt_valsize; 1343 l = sizeof(*ep); 1344 if (sopt->sopt_valsize < l) { 1345 D("bad len sopt->sopt_valsize %d len %d", 1346 (int) sopt->sopt_valsize , l); 1347 err = EINVAL; 1348 return err; 1349 } 1350 ep = malloc(l, M_DUMMYNET, M_WAITOK); 1351 if(!ep) { 1352 err = ENOMEM ; 1353 return err; 1354 } 1355 do { 1356 err = sooptcopyin(sopt, ep, l, l); 1357 if(err) 1358 break; 1359 sopt->sopt_valsize = sopt_valsize; 1360 if (ep->oid.len < l) { 1361 err = EINVAL; 1362 break; 1363 } 1364 1365 fs = dn_ht_find(dn_cfg.fshash, ep->nr, 0, NULL); 1366 if (!fs) { 1367 D("fs %d not found", ep->nr); 1368 err = EINVAL; 1369 break; 1370 } 1371 1372 if (fs->aqmfp && fs->aqmfp->getconfig) { 1373 if(fs->aqmfp->getconfig(fs, ep)) { 1374 D("Error while trying to get AQM params"); 1375 err = EINVAL; 1376 break; 1377 } 1378 ep->oid.len = l; 1379 err = sooptcopyout(sopt, ep, l); 1380 } 1381 }while(0); 1382 1383 free(ep, M_DUMMYNET); 1384 return err; 1385 } 1386 1387 /* Retrieve AQM configurations to ipfw userland 1388 */ 1389 static int 1390 get_sched_parms(struct sockopt *sopt) 1391 { 1392 struct dn_extra_parms *ep; 1393 struct dn_schk *schk; 1394 size_t sopt_valsize; 1395 int l, err = 0; 1396 1397 sopt_valsize = sopt->sopt_valsize; 1398 l = sizeof(*ep); 1399 if (sopt->sopt_valsize < l) { 1400 D("bad len sopt->sopt_valsize %d len %d", 1401 (int) sopt->sopt_valsize , l); 1402 err = EINVAL; 1403 return err; 1404 } 1405 ep = malloc(l, M_DUMMYNET, M_WAITOK); 1406 if(!ep) { 1407 err = ENOMEM ; 1408 return err; 1409 } 1410 do { 1411 err = sooptcopyin(sopt, ep, l, l); 1412 if(err) 1413 break; 1414 sopt->sopt_valsize = sopt_valsize; 1415 if (ep->oid.len < l) { 1416 err = EINVAL; 1417 break; 1418 } 1419 1420 schk = locate_scheduler(ep->nr); 1421 if (!schk) { 1422 D("sched %d not found", ep->nr); 1423 err = EINVAL; 1424 break; 1425 } 1426 1427 if (schk->fp && schk->fp->getconfig) { 1428 if(schk->fp->getconfig(schk, ep)) { 1429 D("Error while trying to get sched params"); 1430 err = EINVAL; 1431 break; 1432 } 1433 ep->oid.len = l; 1434 err = sooptcopyout(sopt, ep, l); 1435 } 1436 }while(0); 1437 free(ep, M_DUMMYNET); 1438 1439 return err; 1440 } 1441 1442 /* Configure AQM for flowset 'fs'. 1443 * extra parameters are passed from userland. 1444 */ 1445 static int 1446 config_aqm(struct dn_fsk *fs, struct dn_extra_parms *ep, int busy) 1447 { 1448 int err = 0; 1449 1450 do { 1451 /* no configurations */ 1452 if (!ep) { 1453 err = 0; 1454 break; 1455 } 1456 1457 /* no AQM for this flowset*/ 1458 if (!strcmp(ep->name,"")) { 1459 err = 0; 1460 break; 1461 } 1462 if (ep->oid.len < sizeof(*ep)) { 1463 D("short aqm len %d", ep->oid.len); 1464 err = EINVAL; 1465 break; 1466 } 1467 1468 if (busy) { 1469 D("Unable to configure flowset, flowset busy!"); 1470 err = EINVAL; 1471 break; 1472 } 1473 1474 /* deconfigure old aqm if exist */ 1475 if (fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) { 1476 aqm_cleanup_deconfig_fs(fs); 1477 } 1478 1479 if (!(fs->aqmfp = find_aqm_type(0, ep->name))) { 1480 D("AQM functions not found for type %s!", ep->name); 1481 fs->fs.flags &= ~DN_IS_AQM; 1482 err = EINVAL; 1483 break; 1484 } else 1485 fs->fs.flags |= DN_IS_AQM; 1486 1487 if (ep->oid.subtype != DN_AQM_PARAMS) { 1488 D("Wrong subtype"); 1489 err = EINVAL; 1490 break; 1491 } 1492 1493 if (fs->aqmfp->config) { 1494 err = fs->aqmfp->config(fs, ep, ep->oid.len); 1495 if (err) { 1496 D("Unable to configure AQM for FS %d", fs->fs.fs_nr ); 1497 fs->fs.flags &= ~DN_IS_AQM; 1498 fs->aqmfp = NULL; 1499 break; 1500 } 1501 } 1502 } while(0); 1503 1504 return err; 1505 } 1506 #endif 1507 1508 /* 1509 * Configuration -- to preserve backward compatibility we use 1510 * the following scheme (N is 65536) 1511 * NUMBER SCHED LINK FLOWSET 1512 * 1 .. N-1 (1)WFQ (2)WFQ (3)queue 1513 * N+1 .. 2N-1 (4)FIFO (5)FIFO (6)FIFO for sched 1..N-1 1514 * 2N+1 .. 3N-1 -- -- (7)FIFO for sched N+1..2N-1 1515 * 1516 * "pipe i config" configures #1, #2 and #3 1517 * "sched i config" configures #1 and possibly #6 1518 * "queue i config" configures #3 1519 * #1 is configured with 'pipe i config' or 'sched i config' 1520 * #2 is configured with 'pipe i config', and created if not 1521 * existing with 'sched i config' 1522 * #3 is configured with 'queue i config' 1523 * #4 is automatically configured after #1, can only be FIFO 1524 * #5 is automatically configured after #2 1525 * #6 is automatically created when #1 is !MULTIQUEUE, 1526 * and can be updated. 1527 * #7 is automatically configured after #2 1528 */ 1529 1530 /* 1531 * configure a link (and its FIFO instance) 1532 */ 1533 static int 1534 config_link(struct dn_link *p, struct dn_id *arg) 1535 { 1536 int i; 1537 1538 if (p->oid.len != sizeof(*p)) { 1539 D("invalid pipe len %d", p->oid.len); 1540 return EINVAL; 1541 } 1542 i = p->link_nr; 1543 if (i <= 0 || i >= DN_MAX_ID) 1544 return EINVAL; 1545 /* 1546 * The config program passes parameters as follows: 1547 * bw = bits/second (0 means no limits), 1548 * delay = ms, must be translated into ticks. 1549 * qsize = slots/bytes 1550 * burst ??? 1551 */ 1552 p->delay = (p->delay * hz) / 1000; 1553 /* Scale burst size: bytes -> bits * hz */ 1554 p->burst *= 8 * hz; 1555 1556 DN_BH_WLOCK(); 1557 /* do it twice, base link and FIFO link */ 1558 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1559 struct dn_schk *s = locate_scheduler(i); 1560 if (s == NULL) { 1561 DN_BH_WUNLOCK(); 1562 D("sched %d not found", i); 1563 return EINVAL; 1564 } 1565 /* remove profile if exists */ 1566 if (s->profile) { 1567 free(s->profile, M_DUMMYNET); 1568 s->profile = NULL; 1569 } 1570 /* copy all parameters */ 1571 s->link.oid = p->oid; 1572 s->link.link_nr = i; 1573 s->link.delay = p->delay; 1574 if (s->link.bandwidth != p->bandwidth) { 1575 /* XXX bandwidth changes, need to update red params */ 1576 s->link.bandwidth = p->bandwidth; 1577 update_red(s); 1578 } 1579 s->link.burst = p->burst; 1580 schk_reset_credit(s); 1581 } 1582 dn_cfg.id++; 1583 DN_BH_WUNLOCK(); 1584 return 0; 1585 } 1586 1587 /* 1588 * configure a flowset. Can be called from inside with locked=1, 1589 */ 1590 static struct dn_fsk * 1591 config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) 1592 { 1593 int i; 1594 struct dn_fsk *fs; 1595 #ifdef NEW_AQM 1596 struct dn_extra_parms *ep; 1597 #endif 1598 1599 if (nfs->oid.len != sizeof(*nfs)) { 1600 D("invalid flowset len %d", nfs->oid.len); 1601 return NULL; 1602 } 1603 i = nfs->fs_nr; 1604 if (i <= 0 || i >= 3*DN_MAX_ID) 1605 return NULL; 1606 #ifdef NEW_AQM 1607 ep = NULL; 1608 if (arg != NULL) { 1609 ep = malloc(sizeof(*ep), M_TEMP, locked ? M_NOWAIT : M_WAITOK); 1610 if (ep == NULL) 1611 return (NULL); 1612 memcpy(ep, arg, sizeof(*ep)); 1613 } 1614 #endif 1615 ND("flowset %d", i); 1616 /* XXX other sanity checks */ 1617 if (nfs->flags & DN_QSIZE_BYTES) { 1618 ipdn_bound_var(&nfs->qsize, 16384, 1619 1500, dn_cfg.byte_limit, NULL); // "queue byte size"); 1620 } else { 1621 ipdn_bound_var(&nfs->qsize, 50, 1622 1, dn_cfg.slot_limit, NULL); // "queue slot size"); 1623 } 1624 if (nfs->flags & DN_HAVE_MASK) { 1625 /* make sure we have some buckets */ 1626 ipdn_bound_var((int *)&nfs->buckets, dn_cfg.hash_size, 1627 1, dn_cfg.max_hash_size, "flowset buckets"); 1628 } else { 1629 nfs->buckets = 1; /* we only need 1 */ 1630 } 1631 if (!locked) 1632 DN_BH_WLOCK(); 1633 do { /* exit with break when done */ 1634 struct dn_schk *s; 1635 int flags = nfs->sched_nr ? DNHT_INSERT : 0; 1636 int j; 1637 int oldc = dn_cfg.fsk_count; 1638 fs = dn_ht_find(dn_cfg.fshash, i, flags, NULL); 1639 if (fs == NULL) { 1640 D("missing sched for flowset %d", i); 1641 break; 1642 } 1643 /* grab some defaults from the existing one */ 1644 if (nfs->sched_nr == 0) /* reuse */ 1645 nfs->sched_nr = fs->fs.sched_nr; 1646 for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) { 1647 if (nfs->par[j] == -1) /* reuse */ 1648 nfs->par[j] = fs->fs.par[j]; 1649 } 1650 if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) { 1651 ND("flowset %d unchanged", i); 1652 #ifdef NEW_AQM 1653 if (ep != NULL) { 1654 /* 1655 * Reconfigure AQM as the parameters can be changed. 1656 * We consider the flowset as busy if it has scheduler 1657 * instance(s). 1658 */ 1659 s = locate_scheduler(nfs->sched_nr); 1660 config_aqm(fs, ep, s != NULL && s->siht != NULL); 1661 } 1662 #endif 1663 break; /* no change, nothing to do */ 1664 } 1665 if (oldc != dn_cfg.fsk_count) /* new item */ 1666 dn_cfg.id++; 1667 s = locate_scheduler(nfs->sched_nr); 1668 /* detach from old scheduler if needed, preserving 1669 * queues if we need to reattach. Then update the 1670 * configuration, and possibly attach to the new sched. 1671 */ 1672 DX(2, "fs %d changed sched %d@%p to %d@%p", 1673 fs->fs.fs_nr, 1674 fs->fs.sched_nr, fs->sched, nfs->sched_nr, s); 1675 if (fs->sched) { 1676 int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY); 1677 flags |= DN_DESTROY; /* XXX temporary */ 1678 fsk_detach(fs, flags); 1679 } 1680 fs->fs = *nfs; /* copy configuration */ 1681 #ifdef NEW_AQM 1682 fs->aqmfp = NULL; 1683 if (ep != NULL) 1684 config_aqm(fs, ep, s != NULL && 1685 s->siht != NULL); 1686 #endif 1687 if (s != NULL) 1688 fsk_attach(fs, s); 1689 } while (0); 1690 if (!locked) 1691 DN_BH_WUNLOCK(); 1692 #ifdef NEW_AQM 1693 if (ep != NULL) 1694 free(ep, M_TEMP); 1695 #endif 1696 return fs; 1697 } 1698 1699 /* 1700 * config/reconfig a scheduler and its FIFO variant. 1701 * For !MULTIQUEUE schedulers, also set up the flowset. 1702 * 1703 * On reconfigurations (detected because s->fp is set), 1704 * detach existing flowsets preserving traffic, preserve link, 1705 * and delete the old scheduler creating a new one. 1706 */ 1707 static int 1708 config_sched(struct dn_sch *_nsch, struct dn_id *arg) 1709 { 1710 struct dn_schk *s; 1711 struct schk_new_arg a; /* argument for schk_new */ 1712 int i; 1713 struct dn_link p; /* copy of oldlink */ 1714 struct dn_profile *pf = NULL; /* copy of old link profile */ 1715 /* Used to preserv mask parameter */ 1716 struct ipfw_flow_id new_mask; 1717 int new_buckets = 0; 1718 int new_flags = 0; 1719 int pipe_cmd; 1720 int err = ENOMEM; 1721 1722 a.sch = _nsch; 1723 if (a.sch->oid.len != sizeof(*a.sch)) { 1724 D("bad sched len %d", a.sch->oid.len); 1725 return EINVAL; 1726 } 1727 i = a.sch->sched_nr; 1728 if (i <= 0 || i >= DN_MAX_ID) 1729 return EINVAL; 1730 /* make sure we have some buckets */ 1731 if (a.sch->flags & DN_HAVE_MASK) 1732 ipdn_bound_var((int *)&a.sch->buckets, dn_cfg.hash_size, 1733 1, dn_cfg.max_hash_size, "sched buckets"); 1734 /* XXX other sanity checks */ 1735 bzero(&p, sizeof(p)); 1736 1737 pipe_cmd = a.sch->flags & DN_PIPE_CMD; 1738 a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set? 1739 if (pipe_cmd) { 1740 /* Copy mask parameter */ 1741 new_mask = a.sch->sched_mask; 1742 new_buckets = a.sch->buckets; 1743 new_flags = a.sch->flags; 1744 } 1745 DN_BH_WLOCK(); 1746 again: /* run twice, for wfq and fifo */ 1747 /* 1748 * lookup the type. If not supplied, use the previous one 1749 * or default to WF2Q+. Otherwise, return an error. 1750 */ 1751 dn_cfg.id++; 1752 a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name); 1753 if (a.fp != NULL) { 1754 /* found. Lookup or create entry */ 1755 s = dn_ht_find(dn_cfg.schedhash, i, DNHT_INSERT, &a); 1756 } else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) { 1757 /* No type. search existing s* or retry with WF2Q+ */ 1758 s = dn_ht_find(dn_cfg.schedhash, i, 0, &a); 1759 if (s != NULL) { 1760 a.fp = s->fp; 1761 /* Scheduler exists, skip to FIFO scheduler 1762 * if command was pipe config... 1763 */ 1764 if (pipe_cmd) 1765 goto next; 1766 } else { 1767 /* New scheduler, create a wf2q+ with no mask 1768 * if command was pipe config... 1769 */ 1770 if (pipe_cmd) { 1771 /* clear mask parameter */ 1772 bzero(&a.sch->sched_mask, sizeof(new_mask)); 1773 a.sch->buckets = 0; 1774 a.sch->flags &= ~DN_HAVE_MASK; 1775 } 1776 a.sch->oid.subtype = DN_SCHED_WF2QP; 1777 goto again; 1778 } 1779 } else { 1780 D("invalid scheduler type %d %s", 1781 a.sch->oid.subtype, a.sch->name); 1782 err = EINVAL; 1783 goto error; 1784 } 1785 /* normalize name and subtype */ 1786 a.sch->oid.subtype = a.fp->type; 1787 bzero(a.sch->name, sizeof(a.sch->name)); 1788 strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name)); 1789 if (s == NULL) { 1790 D("cannot allocate scheduler %d", i); 1791 goto error; 1792 } 1793 /* restore existing link if any */ 1794 if (p.link_nr) { 1795 s->link = p; 1796 if (!pf || pf->link_nr != p.link_nr) { /* no saved value */ 1797 s->profile = NULL; /* XXX maybe not needed */ 1798 } else { 1799 s->profile = malloc(sizeof(struct dn_profile), 1800 M_DUMMYNET, M_NOWAIT | M_ZERO); 1801 if (s->profile == NULL) { 1802 D("cannot allocate profile"); 1803 goto error; //XXX 1804 } 1805 memcpy(s->profile, pf, sizeof(*pf)); 1806 } 1807 } 1808 p.link_nr = 0; 1809 if (s->fp == NULL) { 1810 DX(2, "sched %d new type %s", i, a.fp->name); 1811 } else if (s->fp != a.fp || 1812 bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) { 1813 /* already existing. */ 1814 DX(2, "sched %d type changed from %s to %s", 1815 i, s->fp->name, a.fp->name); 1816 DX(4, " type/sub %d/%d -> %d/%d", 1817 s->sch.oid.type, s->sch.oid.subtype, 1818 a.sch->oid.type, a.sch->oid.subtype); 1819 if (s->link.link_nr == 0) 1820 D("XXX WARNING link 0 for sched %d", i); 1821 p = s->link; /* preserve link */ 1822 if (s->profile) {/* preserve profile */ 1823 if (!pf) 1824 pf = malloc(sizeof(*pf), 1825 M_DUMMYNET, M_NOWAIT | M_ZERO); 1826 if (pf) /* XXX should issue a warning otherwise */ 1827 memcpy(pf, s->profile, sizeof(*pf)); 1828 } 1829 /* remove from the hash */ 1830 dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 1831 /* Detach flowsets, preserve queues. */ 1832 // schk_delete_cb(s, NULL); 1833 // XXX temporarily, kill queues 1834 schk_delete_cb(s, (void *)DN_DESTROY); 1835 goto again; 1836 } else { 1837 DX(4, "sched %d unchanged type %s", i, a.fp->name); 1838 } 1839 /* complete initialization */ 1840 s->sch = *a.sch; 1841 s->fp = a.fp; 1842 s->cfg = arg; 1843 // XXX schk_reset_credit(s); 1844 /* create the internal flowset if needed, 1845 * trying to reuse existing ones if available 1846 */ 1847 if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) { 1848 s->fs = dn_ht_find(dn_cfg.fshash, i, 0, NULL); 1849 if (!s->fs) { 1850 struct dn_fs fs; 1851 bzero(&fs, sizeof(fs)); 1852 set_oid(&fs.oid, DN_FS, sizeof(fs)); 1853 fs.fs_nr = i + DN_MAX_ID; 1854 fs.sched_nr = i; 1855 s->fs = config_fs(&fs, NULL, 1 /* locked */); 1856 } 1857 if (!s->fs) { 1858 schk_delete_cb(s, (void *)DN_DESTROY); 1859 D("error creating internal fs for %d", i); 1860 goto error; 1861 } 1862 } 1863 /* call init function after the flowset is created */ 1864 if (s->fp->config) 1865 s->fp->config(s); 1866 update_fs(s); 1867 next: 1868 if (i < DN_MAX_ID) { /* now configure the FIFO instance */ 1869 i += DN_MAX_ID; 1870 if (pipe_cmd) { 1871 /* Restore mask parameter for FIFO */ 1872 a.sch->sched_mask = new_mask; 1873 a.sch->buckets = new_buckets; 1874 a.sch->flags = new_flags; 1875 } else { 1876 /* sched config shouldn't modify the FIFO scheduler */ 1877 if (dn_ht_find(dn_cfg.schedhash, i, 0, &a) != NULL) { 1878 /* FIFO already exist, don't touch it */ 1879 err = 0; /* and this is not an error */ 1880 goto error; 1881 } 1882 } 1883 a.sch->sched_nr = i; 1884 a.sch->oid.subtype = DN_SCHED_FIFO; 1885 bzero(a.sch->name, sizeof(a.sch->name)); 1886 goto again; 1887 } 1888 err = 0; 1889 error: 1890 DN_BH_WUNLOCK(); 1891 if (pf) 1892 free(pf, M_DUMMYNET); 1893 return err; 1894 } 1895 1896 /* 1897 * attach a profile to a link 1898 */ 1899 static int 1900 config_profile(struct dn_profile *pf, struct dn_id *arg) 1901 { 1902 struct dn_schk *s; 1903 int i, olen, err = 0; 1904 1905 if (pf->oid.len < sizeof(*pf)) { 1906 D("short profile len %d", pf->oid.len); 1907 return EINVAL; 1908 } 1909 i = pf->link_nr; 1910 if (i <= 0 || i >= DN_MAX_ID) 1911 return EINVAL; 1912 /* XXX other sanity checks */ 1913 DN_BH_WLOCK(); 1914 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1915 s = locate_scheduler(i); 1916 1917 if (s == NULL) { 1918 err = EINVAL; 1919 break; 1920 } 1921 dn_cfg.id++; 1922 /* 1923 * If we had a profile and the new one does not fit, 1924 * or it is deleted, then we need to free memory. 1925 */ 1926 if (s->profile && (pf->samples_no == 0 || 1927 s->profile->oid.len < pf->oid.len)) { 1928 free(s->profile, M_DUMMYNET); 1929 s->profile = NULL; 1930 } 1931 if (pf->samples_no == 0) 1932 continue; 1933 /* 1934 * new profile, possibly allocate memory 1935 * and copy data. 1936 */ 1937 if (s->profile == NULL) 1938 s->profile = malloc(pf->oid.len, 1939 M_DUMMYNET, M_NOWAIT | M_ZERO); 1940 if (s->profile == NULL) { 1941 D("no memory for profile %d", i); 1942 err = ENOMEM; 1943 break; 1944 } 1945 /* preserve larger length XXX double check */ 1946 olen = s->profile->oid.len; 1947 if (olen < pf->oid.len) 1948 olen = pf->oid.len; 1949 memcpy(s->profile, pf, pf->oid.len); 1950 s->profile->oid.len = olen; 1951 } 1952 DN_BH_WUNLOCK(); 1953 return err; 1954 } 1955 1956 /* 1957 * Delete all objects: 1958 */ 1959 static void 1960 dummynet_flush(void) 1961 { 1962 1963 /* delete all schedulers and related links/queues/flowsets */ 1964 dn_ht_scan(dn_cfg.schedhash, schk_delete_cb, 1965 (void *)(uintptr_t)DN_DELETE_FS); 1966 /* delete all remaining (unlinked) flowsets */ 1967 DX(4, "still %d unlinked fs", dn_cfg.fsk_count); 1968 dn_ht_free(dn_cfg.fshash, DNHT_REMOVE); 1969 fsk_detach_list(&dn_cfg.fsu, DN_DELETE_FS); 1970 /* Reinitialize system heap... */ 1971 heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 1972 } 1973 1974 /* 1975 * Main handler for configuration. We are guaranteed to be called 1976 * with an oid which is at least a dn_id. 1977 * - the first object is the command (config, delete, flush, ...) 1978 * - config_link must be issued after the corresponding config_sched 1979 * - parameters (DN_TXT) for an object must precede the object 1980 * processed on a config_sched. 1981 */ 1982 int 1983 do_config(void *p, int l) 1984 { 1985 struct dn_id o; 1986 union { 1987 struct dn_profile profile; 1988 struct dn_fs fs; 1989 struct dn_link link; 1990 struct dn_sch sched; 1991 } *dn; 1992 struct dn_id *arg; 1993 uintptr_t a; 1994 int err, err2, off; 1995 1996 memcpy(&o, p, sizeof(o)); 1997 if (o.id != DN_API_VERSION) { 1998 D("invalid api version got %d need %d", o.id, DN_API_VERSION); 1999 return EINVAL; 2000 } 2001 arg = NULL; 2002 dn = NULL; 2003 for (off = 0; l >= sizeof(o); memcpy(&o, (char *)p + off, sizeof(o))) { 2004 if (o.len < sizeof(o) || l < o.len) { 2005 D("bad len o.len %d len %d", o.len, l); 2006 err = EINVAL; 2007 break; 2008 } 2009 l -= o.len; 2010 err = 0; 2011 switch (o.type) { 2012 default: 2013 D("cmd %d not implemented", o.type); 2014 break; 2015 2016 #ifdef EMULATE_SYSCTL 2017 /* sysctl emulation. 2018 * if we recognize the command, jump to the correct 2019 * handler and return 2020 */ 2021 case DN_SYSCTL_SET: 2022 err = kesysctl_emu_set(p, l); 2023 return err; 2024 #endif 2025 2026 case DN_CMD_CONFIG: /* simply a header */ 2027 break; 2028 2029 case DN_CMD_DELETE: 2030 /* the argument is in the first uintptr_t after o */ 2031 if (o.len < sizeof(o) + sizeof(a)) { 2032 err = EINVAL; 2033 break; 2034 } 2035 memcpy(&a, (char *)p + off + sizeof(o), sizeof(a)); 2036 switch (o.subtype) { 2037 case DN_LINK: 2038 /* delete base and derived schedulers */ 2039 DN_BH_WLOCK(); 2040 err = delete_schk(a); 2041 err2 = delete_schk(a + DN_MAX_ID); 2042 DN_BH_WUNLOCK(); 2043 if (!err) 2044 err = err2; 2045 break; 2046 2047 default: 2048 D("invalid delete type %d", o.subtype); 2049 err = EINVAL; 2050 break; 2051 2052 case DN_FS: 2053 err = (a < 1 || a >= DN_MAX_ID) ? 2054 EINVAL : delete_fs(a, 0) ; 2055 break; 2056 } 2057 break; 2058 2059 case DN_CMD_FLUSH: 2060 DN_BH_WLOCK(); 2061 dummynet_flush(); 2062 DN_BH_WUNLOCK(); 2063 break; 2064 case DN_TEXT: /* store argument of next block */ 2065 if (arg != NULL) 2066 free(arg, M_TEMP); 2067 arg = malloc(o.len, M_TEMP, M_WAITOK); 2068 memcpy(arg, (char *)p + off, o.len); 2069 break; 2070 case DN_LINK: 2071 if (dn == NULL) 2072 dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); 2073 memcpy(&dn->link, (char *)p + off, sizeof(dn->link)); 2074 err = config_link(&dn->link, arg); 2075 break; 2076 case DN_PROFILE: 2077 if (dn == NULL) 2078 dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); 2079 memcpy(&dn->profile, (char *)p + off, 2080 sizeof(dn->profile)); 2081 err = config_profile(&dn->profile, arg); 2082 break; 2083 case DN_SCH: 2084 if (dn == NULL) 2085 dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); 2086 memcpy(&dn->sched, (char *)p + off, 2087 sizeof(dn->sched)); 2088 err = config_sched(&dn->sched, arg); 2089 break; 2090 case DN_FS: 2091 if (dn == NULL) 2092 dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK); 2093 memcpy(&dn->fs, (char *)p + off, sizeof(dn->fs)); 2094 err = (NULL == config_fs(&dn->fs, arg, 0)); 2095 break; 2096 } 2097 if (err != 0) 2098 break; 2099 off += o.len; 2100 } 2101 if (arg != NULL) 2102 free(arg, M_TEMP); 2103 if (dn != NULL) 2104 free(dn, M_TEMP); 2105 return err; 2106 } 2107 2108 static int 2109 compute_space(struct dn_id *cmd, struct copy_args *a) 2110 { 2111 int x = 0, need = 0; 2112 int profile_size = sizeof(struct dn_profile) - 2113 ED_MAX_SAMPLES_NO*sizeof(int); 2114 2115 /* NOTE about compute space: 2116 * NP = dn_cfg.schk_count 2117 * NSI = dn_cfg.si_count 2118 * NF = dn_cfg.fsk_count 2119 * NQ = dn_cfg.queue_count 2120 * - ipfw pipe show 2121 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 2122 * link, scheduler template, flowset 2123 * integrated in scheduler and header 2124 * for flowset list 2125 * (NSI)*(dn_flow) all scheduler instance (includes 2126 * the queue instance) 2127 * - ipfw sched show 2128 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 2129 * link, scheduler template, flowset 2130 * integrated in scheduler and header 2131 * for flowset list 2132 * (NSI * dn_flow) all scheduler instances 2133 * (NF * sizeof(uint_32)) space for flowset list linked to scheduler 2134 * (NQ * dn_queue) all queue [XXXfor now not listed] 2135 * - ipfw queue show 2136 * (NF * dn_fs) all flowset 2137 * (NQ * dn_queue) all queues 2138 */ 2139 switch (cmd->subtype) { 2140 default: 2141 return -1; 2142 /* XXX where do LINK and SCH differ ? */ 2143 /* 'ipfw sched show' could list all queues associated to 2144 * a scheduler. This feature for now is disabled 2145 */ 2146 case DN_LINK: /* pipe show */ 2147 x = DN_C_LINK | DN_C_SCH | DN_C_FLOW; 2148 need += dn_cfg.schk_count * 2149 (sizeof(struct dn_fs) + profile_size) / 2; 2150 need += dn_cfg.fsk_count * sizeof(uint32_t); 2151 break; 2152 case DN_SCH: /* sched show */ 2153 need += dn_cfg.schk_count * 2154 (sizeof(struct dn_fs) + profile_size) / 2; 2155 need += dn_cfg.fsk_count * sizeof(uint32_t); 2156 x = DN_C_SCH | DN_C_LINK | DN_C_FLOW; 2157 break; 2158 case DN_FS: /* queue show */ 2159 x = DN_C_FS | DN_C_QUEUE; 2160 break; 2161 case DN_GET_COMPAT: /* compatibility mode */ 2162 need = dn_compat_calc_size(); 2163 break; 2164 } 2165 a->flags = x; 2166 if (x & DN_C_SCH) { 2167 need += dn_cfg.schk_count * sizeof(struct dn_sch) / 2; 2168 /* NOT also, each fs might be attached to a sched */ 2169 need += dn_cfg.schk_count * sizeof(struct dn_id) / 2; 2170 } 2171 if (x & DN_C_FS) 2172 need += dn_cfg.fsk_count * sizeof(struct dn_fs); 2173 if (x & DN_C_LINK) { 2174 need += dn_cfg.schk_count * sizeof(struct dn_link) / 2; 2175 } 2176 /* 2177 * When exporting a queue to userland, only pass up the 2178 * struct dn_flow, which is the only visible part. 2179 */ 2180 2181 if (x & DN_C_QUEUE) 2182 need += dn_cfg.queue_count * sizeof(struct dn_flow); 2183 if (x & DN_C_FLOW) 2184 need += dn_cfg.si_count * (sizeof(struct dn_flow)); 2185 return need; 2186 } 2187 2188 /* 2189 * If compat != NULL dummynet_get is called in compatibility mode. 2190 * *compat will be the pointer to the buffer to pass to ipfw 2191 */ 2192 int 2193 dummynet_get(struct sockopt *sopt, void **compat) 2194 { 2195 int have, i, need, error; 2196 char *start = NULL, *buf; 2197 size_t sopt_valsize; 2198 struct dn_id *cmd; 2199 struct copy_args a; 2200 struct copy_range r; 2201 int l = sizeof(struct dn_id); 2202 2203 bzero(&a, sizeof(a)); 2204 bzero(&r, sizeof(r)); 2205 2206 /* save and restore original sopt_valsize around copyin */ 2207 sopt_valsize = sopt->sopt_valsize; 2208 2209 cmd = &r.o; 2210 2211 if (!compat) { 2212 /* copy at least an oid, and possibly a full object */ 2213 error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd)); 2214 sopt->sopt_valsize = sopt_valsize; 2215 if (error) 2216 goto done; 2217 l = cmd->len; 2218 #ifdef EMULATE_SYSCTL 2219 /* sysctl emulation. */ 2220 if (cmd->type == DN_SYSCTL_GET) 2221 return kesysctl_emu_get(sopt); 2222 #endif 2223 if (l > sizeof(r)) { 2224 /* request larger than default, allocate buffer */ 2225 cmd = malloc(l, M_DUMMYNET, M_WAITOK); 2226 error = sooptcopyin(sopt, cmd, l, l); 2227 sopt->sopt_valsize = sopt_valsize; 2228 if (error) 2229 goto done; 2230 } 2231 } else { /* compatibility */ 2232 error = 0; 2233 cmd->type = DN_CMD_GET; 2234 cmd->len = sizeof(struct dn_id); 2235 cmd->subtype = DN_GET_COMPAT; 2236 // cmd->id = sopt_valsize; 2237 D("compatibility mode"); 2238 } 2239 2240 #ifdef NEW_AQM 2241 /* get AQM params */ 2242 if(cmd->subtype == DN_AQM_PARAMS) { 2243 error = get_aqm_parms(sopt); 2244 goto done; 2245 /* get Scheduler params */ 2246 } else if (cmd->subtype == DN_SCH_PARAMS) { 2247 error = get_sched_parms(sopt); 2248 goto done; 2249 } 2250 #endif 2251 2252 a.extra = (struct copy_range *)cmd; 2253 if (cmd->len == sizeof(*cmd)) { /* no range, create a default */ 2254 uint32_t *rp = (uint32_t *)(cmd + 1); 2255 cmd->len += 2* sizeof(uint32_t); 2256 rp[0] = 1; 2257 rp[1] = DN_MAX_ID - 1; 2258 if (cmd->subtype == DN_LINK) { 2259 rp[0] += DN_MAX_ID; 2260 rp[1] += DN_MAX_ID; 2261 } 2262 } 2263 /* Count space (under lock) and allocate (outside lock). 2264 * Exit with lock held if we manage to get enough buffer. 2265 * Try a few times then give up. 2266 */ 2267 for (have = 0, i = 0; i < 10; i++) { 2268 DN_BH_WLOCK(); 2269 need = compute_space(cmd, &a); 2270 2271 /* if there is a range, ignore value from compute_space() */ 2272 if (l > sizeof(*cmd)) 2273 need = sopt_valsize - sizeof(*cmd); 2274 2275 if (need < 0) { 2276 DN_BH_WUNLOCK(); 2277 error = EINVAL; 2278 goto done; 2279 } 2280 need += sizeof(*cmd); 2281 cmd->id = need; 2282 if (have >= need) 2283 break; 2284 2285 DN_BH_WUNLOCK(); 2286 if (start) 2287 free(start, M_DUMMYNET); 2288 start = NULL; 2289 if (need > sopt_valsize) 2290 break; 2291 2292 have = need; 2293 start = malloc(have, M_DUMMYNET, M_WAITOK | M_ZERO); 2294 } 2295 2296 if (start == NULL) { 2297 if (compat) { 2298 *compat = NULL; 2299 error = 1; // XXX 2300 } else { 2301 error = sooptcopyout(sopt, cmd, sizeof(*cmd)); 2302 } 2303 goto done; 2304 } 2305 ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, " 2306 "%d:%d si %d, %d:%d queues %d", 2307 dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH, 2308 dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK, 2309 dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS, 2310 dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I, 2311 dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE); 2312 sopt->sopt_valsize = sopt_valsize; 2313 a.type = cmd->subtype; 2314 2315 if (compat == NULL) { 2316 memcpy(start, cmd, sizeof(*cmd)); 2317 ((struct dn_id*)(start))->len = sizeof(struct dn_id); 2318 buf = start + sizeof(*cmd); 2319 } else 2320 buf = start; 2321 a.start = &buf; 2322 a.end = start + have; 2323 /* start copying other objects */ 2324 if (compat) { 2325 a.type = DN_COMPAT_PIPE; 2326 dn_ht_scan(dn_cfg.schedhash, copy_data_helper_compat, &a); 2327 a.type = DN_COMPAT_QUEUE; 2328 dn_ht_scan(dn_cfg.fshash, copy_data_helper_compat, &a); 2329 } else if (a.type == DN_FS) { 2330 dn_ht_scan(dn_cfg.fshash, copy_data_helper, &a); 2331 } else { 2332 dn_ht_scan(dn_cfg.schedhash, copy_data_helper, &a); 2333 } 2334 DN_BH_WUNLOCK(); 2335 2336 if (compat) { 2337 *compat = start; 2338 sopt->sopt_valsize = buf - start; 2339 /* free() is done by ip_dummynet_compat() */ 2340 start = NULL; //XXX hack 2341 } else { 2342 error = sooptcopyout(sopt, start, buf - start); 2343 } 2344 done: 2345 if (cmd && cmd != &r.o) 2346 free(cmd, M_DUMMYNET); 2347 if (start) 2348 free(start, M_DUMMYNET); 2349 return error; 2350 } 2351 2352 /* Callback called on scheduler instance to delete it if idle */ 2353 static int 2354 drain_scheduler_cb(void *_si, void *arg) 2355 { 2356 struct dn_sch_inst *si = _si; 2357 2358 if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL) 2359 return 0; 2360 2361 if (si->sched->fp->flags & DN_MULTIQUEUE) { 2362 if (si->q_count == 0) 2363 return si_destroy(si, NULL); 2364 else 2365 return 0; 2366 } else { /* !DN_MULTIQUEUE */ 2367 if ((si+1)->ni.length == 0) 2368 return si_destroy(si, NULL); 2369 else 2370 return 0; 2371 } 2372 return 0; /* unreachable */ 2373 } 2374 2375 /* Callback called on scheduler to check if it has instances */ 2376 static int 2377 drain_scheduler_sch_cb(void *_s, void *arg) 2378 { 2379 struct dn_schk *s = _s; 2380 2381 if (s->sch.flags & DN_HAVE_MASK) { 2382 dn_ht_scan_bucket(s->siht, &s->drain_bucket, 2383 drain_scheduler_cb, NULL); 2384 s->drain_bucket++; 2385 } else { 2386 if (s->siht) { 2387 if (drain_scheduler_cb(s->siht, NULL) == DNHT_SCAN_DEL) 2388 s->siht = NULL; 2389 } 2390 } 2391 return 0; 2392 } 2393 2394 /* Called every tick, try to delete a 'bucket' of scheduler */ 2395 void 2396 dn_drain_scheduler(void) 2397 { 2398 dn_ht_scan_bucket(dn_cfg.schedhash, &dn_cfg.drain_sch, 2399 drain_scheduler_sch_cb, NULL); 2400 dn_cfg.drain_sch++; 2401 } 2402 2403 /* Callback called on queue to delete if it is idle */ 2404 static int 2405 drain_queue_cb(void *_q, void *arg) 2406 { 2407 struct dn_queue *q = _q; 2408 2409 if (q->ni.length == 0) { 2410 dn_delete_queue(q, DN_DESTROY); 2411 return DNHT_SCAN_DEL; /* queue is deleted */ 2412 } 2413 2414 return 0; /* queue isn't deleted */ 2415 } 2416 2417 /* Callback called on flowset used to check if it has queues */ 2418 static int 2419 drain_queue_fs_cb(void *_fs, void *arg) 2420 { 2421 struct dn_fsk *fs = _fs; 2422 2423 if (fs->fs.flags & DN_QHT_HASH) { 2424 /* Flowset has a hash table for queues */ 2425 dn_ht_scan_bucket(fs->qht, &fs->drain_bucket, 2426 drain_queue_cb, NULL); 2427 fs->drain_bucket++; 2428 } else { 2429 /* No hash table for this flowset, null the pointer 2430 * if the queue is deleted 2431 */ 2432 if (fs->qht) { 2433 if (drain_queue_cb(fs->qht, NULL) == DNHT_SCAN_DEL) 2434 fs->qht = NULL; 2435 } 2436 } 2437 return 0; 2438 } 2439 2440 /* Called every tick, try to delete a 'bucket' of queue */ 2441 void 2442 dn_drain_queue(void) 2443 { 2444 /* scan a bucket of flowset */ 2445 dn_ht_scan_bucket(dn_cfg.fshash, &dn_cfg.drain_fs, 2446 drain_queue_fs_cb, NULL); 2447 dn_cfg.drain_fs++; 2448 } 2449 2450 /* 2451 * Handler for the various dummynet socket options 2452 */ 2453 static int 2454 ip_dn_ctl(struct sockopt *sopt) 2455 { 2456 void *p = NULL; 2457 int error, l; 2458 2459 error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET); 2460 if (error) 2461 return (error); 2462 2463 /* Disallow sets in really-really secure mode. */ 2464 if (sopt->sopt_dir == SOPT_SET) { 2465 error = securelevel_ge(sopt->sopt_td->td_ucred, 3); 2466 if (error) 2467 return (error); 2468 } 2469 2470 switch (sopt->sopt_name) { 2471 default : 2472 D("dummynet: unknown option %d", sopt->sopt_name); 2473 error = EINVAL; 2474 break; 2475 2476 case IP_DUMMYNET_FLUSH: 2477 case IP_DUMMYNET_CONFIGURE: 2478 case IP_DUMMYNET_DEL: /* remove a pipe or queue */ 2479 case IP_DUMMYNET_GET: 2480 D("dummynet: compat option %d", sopt->sopt_name); 2481 error = ip_dummynet_compat(sopt); 2482 break; 2483 2484 case IP_DUMMYNET3 : 2485 if (sopt->sopt_dir == SOPT_GET) { 2486 error = dummynet_get(sopt, NULL); 2487 break; 2488 } 2489 l = sopt->sopt_valsize; 2490 if (l < sizeof(struct dn_id) || l > 12000) { 2491 D("argument len %d invalid", l); 2492 break; 2493 } 2494 p = malloc(l, M_TEMP, M_WAITOK); // XXX can it fail ? 2495 error = sooptcopyin(sopt, p, l, l); 2496 if (error) 2497 break ; 2498 error = do_config(p, l); 2499 break; 2500 } 2501 2502 if (p != NULL) 2503 free(p, M_TEMP); 2504 2505 return error ; 2506 } 2507 2508 2509 static void 2510 ip_dn_init(void) 2511 { 2512 if (dn_cfg.init_done) 2513 return; 2514 printf("DUMMYNET %p with IPv6 initialized (100409)\n", curvnet); 2515 dn_cfg.init_done = 1; 2516 /* Set defaults here. MSVC does not accept initializers, 2517 * and this is also useful for vimages 2518 */ 2519 /* queue limits */ 2520 dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */ 2521 dn_cfg.byte_limit = 1024 * 1024; 2522 dn_cfg.expire = 1; 2523 2524 /* RED parameters */ 2525 dn_cfg.red_lookup_depth = 256; /* default lookup table depth */ 2526 dn_cfg.red_avg_pkt_size = 512; /* default medium packet size */ 2527 dn_cfg.red_max_pkt_size = 1500; /* default max packet size */ 2528 2529 /* hash tables */ 2530 dn_cfg.max_hash_size = 65536; /* max in the hash tables */ 2531 dn_cfg.hash_size = 64; /* default hash size */ 2532 2533 /* create hash tables for schedulers and flowsets. 2534 * In both we search by key and by pointer. 2535 */ 2536 dn_cfg.schedhash = dn_ht_init(NULL, dn_cfg.hash_size, 2537 offsetof(struct dn_schk, schk_next), 2538 schk_hash, schk_match, schk_new); 2539 dn_cfg.fshash = dn_ht_init(NULL, dn_cfg.hash_size, 2540 offsetof(struct dn_fsk, fsk_next), 2541 fsk_hash, fsk_match, fsk_new); 2542 2543 /* bucket index to drain object */ 2544 dn_cfg.drain_fs = 0; 2545 dn_cfg.drain_sch = 0; 2546 2547 heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 2548 SLIST_INIT(&dn_cfg.fsu); 2549 SLIST_INIT(&dn_cfg.schedlist); 2550 2551 DN_LOCK_INIT(); 2552 2553 NET_TASK_INIT(&dn_task, 0, dummynet_task, curvnet); 2554 dn_tq = taskqueue_create_fast("dummynet", M_WAITOK, 2555 taskqueue_thread_enqueue, &dn_tq); 2556 taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet"); 2557 2558 callout_init(&dn_timeout, 1); 2559 dn_reschedule(); 2560 2561 /* Initialize curr_time adjustment mechanics. */ 2562 getmicrouptime(&dn_cfg.prev_t); 2563 } 2564 2565 static void 2566 ip_dn_destroy(int last) 2567 { 2568 DN_BH_WLOCK(); 2569 /* ensure no more callouts are started */ 2570 dn_gone = 1; 2571 2572 /* check for last */ 2573 if (last) { 2574 ND("removing last instance\n"); 2575 ip_dn_ctl_ptr = NULL; 2576 ip_dn_io_ptr = NULL; 2577 } 2578 2579 dummynet_flush(); 2580 DN_BH_WUNLOCK(); 2581 2582 callout_drain(&dn_timeout); 2583 taskqueue_drain(dn_tq, &dn_task); 2584 taskqueue_free(dn_tq); 2585 2586 dn_ht_free(dn_cfg.schedhash, 0); 2587 dn_ht_free(dn_cfg.fshash, 0); 2588 heap_free(&dn_cfg.evheap); 2589 2590 DN_LOCK_DESTROY(); 2591 } 2592 2593 static int 2594 dummynet_modevent(module_t mod, int type, void *data) 2595 { 2596 2597 if (type == MOD_LOAD) { 2598 if (ip_dn_io_ptr) { 2599 printf("DUMMYNET already loaded\n"); 2600 return EEXIST ; 2601 } 2602 ip_dn_init(); 2603 ip_dn_ctl_ptr = ip_dn_ctl; 2604 ip_dn_io_ptr = dummynet_io; 2605 return 0; 2606 } else if (type == MOD_UNLOAD) { 2607 ip_dn_destroy(1 /* last */); 2608 return 0; 2609 } else 2610 return EOPNOTSUPP; 2611 } 2612 2613 /* modevent helpers for the modules */ 2614 static int 2615 load_dn_sched(struct dn_alg *d) 2616 { 2617 struct dn_alg *s; 2618 2619 if (d == NULL) 2620 return 1; /* error */ 2621 ip_dn_init(); /* just in case, we need the lock */ 2622 2623 /* Check that mandatory funcs exists */ 2624 if (d->enqueue == NULL || d->dequeue == NULL) { 2625 D("missing enqueue or dequeue for %s", d->name); 2626 return 1; 2627 } 2628 2629 /* Search if scheduler already exists */ 2630 DN_BH_WLOCK(); 2631 SLIST_FOREACH(s, &dn_cfg.schedlist, next) { 2632 if (strcmp(s->name, d->name) == 0) { 2633 D("%s already loaded", d->name); 2634 break; /* scheduler already exists */ 2635 } 2636 } 2637 if (s == NULL) 2638 SLIST_INSERT_HEAD(&dn_cfg.schedlist, d, next); 2639 DN_BH_WUNLOCK(); 2640 D("dn_sched %s %sloaded", d->name, s ? "not ":""); 2641 return s ? 1 : 0; 2642 } 2643 2644 static int 2645 unload_dn_sched(struct dn_alg *s) 2646 { 2647 struct dn_alg *tmp, *r; 2648 int err = EINVAL; 2649 2650 ND("called for %s", s->name); 2651 2652 DN_BH_WLOCK(); 2653 SLIST_FOREACH_SAFE(r, &dn_cfg.schedlist, next, tmp) { 2654 if (strcmp(s->name, r->name) != 0) 2655 continue; 2656 ND("ref_count = %d", r->ref_count); 2657 err = (r->ref_count != 0) ? EBUSY : 0; 2658 if (err == 0) 2659 SLIST_REMOVE(&dn_cfg.schedlist, r, dn_alg, next); 2660 break; 2661 } 2662 DN_BH_WUNLOCK(); 2663 D("dn_sched %s %sunloaded", s->name, err ? "not ":""); 2664 return err; 2665 } 2666 2667 int 2668 dn_sched_modevent(module_t mod, int cmd, void *arg) 2669 { 2670 struct dn_alg *sch = arg; 2671 2672 if (cmd == MOD_LOAD) 2673 return load_dn_sched(sch); 2674 else if (cmd == MOD_UNLOAD) 2675 return unload_dn_sched(sch); 2676 else 2677 return EINVAL; 2678 } 2679 2680 static moduledata_t dummynet_mod = { 2681 "dummynet", dummynet_modevent, NULL 2682 }; 2683 2684 #define DN_SI_SUB SI_SUB_PROTO_FIREWALL 2685 #define DN_MODEV_ORD (SI_ORDER_ANY - 128) /* after ipfw */ 2686 DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD); 2687 MODULE_DEPEND(dummynet, ipfw, 3, 3, 3); 2688 MODULE_VERSION(dummynet, 3); 2689 2690 /* 2691 * Starting up. Done in order after dummynet_modevent() has been called. 2692 * VNET_SYSINIT is also called for each existing vnet and each new vnet. 2693 */ 2694 //VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_init, NULL); 2695 2696 /* 2697 * Shutdown handlers up shop. These are done in REVERSE ORDER, but still 2698 * after dummynet_modevent() has been called. Not called on reboot. 2699 * VNET_SYSUNINIT is also called for each exiting vnet as it exits. 2700 * or when the module is unloaded. 2701 */ 2702 //VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_destroy, NULL); 2703 2704 #ifdef NEW_AQM 2705 2706 /* modevent helpers for the AQM modules */ 2707 static int 2708 load_dn_aqm(struct dn_aqm *d) 2709 { 2710 struct dn_aqm *aqm=NULL; 2711 2712 if (d == NULL) 2713 return 1; /* error */ 2714 ip_dn_init(); /* just in case, we need the lock */ 2715 2716 /* Check that mandatory funcs exists */ 2717 if (d->enqueue == NULL || d->dequeue == NULL) { 2718 D("missing enqueue or dequeue for %s", d->name); 2719 return 1; 2720 } 2721 2722 /* Search if AQM already exists */ 2723 DN_BH_WLOCK(); 2724 SLIST_FOREACH(aqm, &dn_cfg.aqmlist, next) { 2725 if (strcmp(aqm->name, d->name) == 0) { 2726 D("%s already loaded", d->name); 2727 break; /* AQM already exists */ 2728 } 2729 } 2730 if (aqm == NULL) 2731 SLIST_INSERT_HEAD(&dn_cfg.aqmlist, d, next); 2732 DN_BH_WUNLOCK(); 2733 D("dn_aqm %s %sloaded", d->name, aqm ? "not ":""); 2734 return aqm ? 1 : 0; 2735 } 2736 2737 2738 /* Callback to clean up AQM status for queues connected to a flowset 2739 * and then deconfigure the flowset. 2740 * This function is called before an AQM module is unloaded 2741 */ 2742 static int 2743 fs_cleanup(void *_fs, void *arg) 2744 { 2745 struct dn_fsk *fs = _fs; 2746 uint32_t type = *(uint32_t *)arg; 2747 2748 if (fs->aqmfp && fs->aqmfp->type == type) 2749 aqm_cleanup_deconfig_fs(fs); 2750 2751 return 0; 2752 } 2753 2754 static int 2755 unload_dn_aqm(struct dn_aqm *aqm) 2756 { 2757 struct dn_aqm *tmp, *r; 2758 int err = EINVAL; 2759 err = 0; 2760 ND("called for %s", aqm->name); 2761 2762 DN_BH_WLOCK(); 2763 2764 /* clean up AQM status and deconfig flowset */ 2765 dn_ht_scan(dn_cfg.fshash, fs_cleanup, &aqm->type); 2766 2767 SLIST_FOREACH_SAFE(r, &dn_cfg.aqmlist, next, tmp) { 2768 if (strcmp(aqm->name, r->name) != 0) 2769 continue; 2770 ND("ref_count = %d", r->ref_count); 2771 err = (r->ref_count != 0 || r->cfg_ref_count != 0) ? EBUSY : 0; 2772 if (err == 0) 2773 SLIST_REMOVE(&dn_cfg.aqmlist, r, dn_aqm, next); 2774 break; 2775 } 2776 DN_BH_WUNLOCK(); 2777 D("%s %sunloaded", aqm->name, err ? "not ":""); 2778 if (err) 2779 D("ref_count=%d, cfg_ref_count=%d", r->ref_count, r->cfg_ref_count); 2780 return err; 2781 } 2782 2783 int 2784 dn_aqm_modevent(module_t mod, int cmd, void *arg) 2785 { 2786 struct dn_aqm *aqm = arg; 2787 2788 if (cmd == MOD_LOAD) 2789 return load_dn_aqm(aqm); 2790 else if (cmd == MOD_UNLOAD) 2791 return unload_dn_aqm(aqm); 2792 else 2793 return EINVAL; 2794 } 2795 #endif 2796 2797 /* end of file */ 2798 2799