1 /*- 2 * Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa 3 * Portions Copyright (c) 2000 Akamba Corp. 4 * All rights reserved 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 /* 32 * Configuration and internal object management for dummynet. 33 */ 34 35 #include "opt_inet6.h" 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/kernel.h> 42 #include <sys/lock.h> 43 #include <sys/module.h> 44 #include <sys/mutex.h> 45 #include <sys/priv.h> 46 #include <sys/proc.h> 47 #include <sys/rwlock.h> 48 #include <sys/socket.h> 49 #include <sys/socketvar.h> 50 #include <sys/time.h> 51 #include <sys/taskqueue.h> 52 #include <net/if.h> /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ 53 #include <netinet/in.h> 54 #include <netinet/ip_var.h> /* ip_output(), IP_FORWARDING */ 55 #include <netinet/ip_fw.h> 56 #include <netinet/ip_dummynet.h> 57 58 #include <netpfil/ipfw/ip_fw_private.h> 59 #include <netpfil/ipfw/dn_heap.h> 60 #include <netpfil/ipfw/ip_dn_private.h> 61 #include <netpfil/ipfw/dn_sched.h> 62 63 /* which objects to copy */ 64 #define DN_C_LINK 0x01 65 #define DN_C_SCH 0x02 66 #define DN_C_FLOW 0x04 67 #define DN_C_FS 0x08 68 #define DN_C_QUEUE 0x10 69 70 /* we use this argument in case of a schk_new */ 71 struct schk_new_arg { 72 struct dn_alg *fp; 73 struct dn_sch *sch; 74 }; 75 76 /*---- callout hooks. ----*/ 77 static struct callout dn_timeout; 78 static struct task dn_task; 79 static struct taskqueue *dn_tq = NULL; 80 81 static void 82 dummynet(void *arg) 83 { 84 85 (void)arg; /* UNUSED */ 86 taskqueue_enqueue_fast(dn_tq, &dn_task); 87 } 88 89 void 90 dn_reschedule(void) 91 { 92 93 callout_reset_sbt(&dn_timeout, tick_sbt, 0, dummynet, NULL, 94 C_HARDCLOCK | C_DIRECT_EXEC); 95 } 96 /*----- end of callout hooks -----*/ 97 98 /* Return a scheduler descriptor given the type or name. */ 99 static struct dn_alg * 100 find_sched_type(int type, char *name) 101 { 102 struct dn_alg *d; 103 104 SLIST_FOREACH(d, &dn_cfg.schedlist, next) { 105 if (d->type == type || (name && !strcasecmp(d->name, name))) 106 return d; 107 } 108 return NULL; /* not found */ 109 } 110 111 int 112 ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg) 113 { 114 int oldv = *v; 115 const char *op = NULL; 116 if (dflt < lo) 117 dflt = lo; 118 if (dflt > hi) 119 dflt = hi; 120 if (oldv < lo) { 121 *v = dflt; 122 op = "Bump"; 123 } else if (oldv > hi) { 124 *v = hi; 125 op = "Clamp"; 126 } else 127 return *v; 128 if (op && msg) 129 printf("%s %s to %d (was %d)\n", op, msg, *v, oldv); 130 return *v; 131 } 132 133 /*---- flow_id mask, hash and compare functions ---*/ 134 /* 135 * The flow_id includes the 5-tuple, the queue/pipe number 136 * which we store in the extra area in host order, 137 * and for ipv6 also the flow_id6. 138 * XXX see if we want the tos byte (can store in 'flags') 139 */ 140 static struct ipfw_flow_id * 141 flow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id) 142 { 143 int is_v6 = IS_IP6_FLOW_ID(id); 144 145 id->dst_port &= mask->dst_port; 146 id->src_port &= mask->src_port; 147 id->proto &= mask->proto; 148 id->extra &= mask->extra; 149 if (is_v6) { 150 APPLY_MASK(&id->dst_ip6, &mask->dst_ip6); 151 APPLY_MASK(&id->src_ip6, &mask->src_ip6); 152 id->flow_id6 &= mask->flow_id6; 153 } else { 154 id->dst_ip &= mask->dst_ip; 155 id->src_ip &= mask->src_ip; 156 } 157 return id; 158 } 159 160 /* computes an OR of two masks, result in dst and also returned */ 161 static struct ipfw_flow_id * 162 flow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst) 163 { 164 int is_v6 = IS_IP6_FLOW_ID(dst); 165 166 dst->dst_port |= src->dst_port; 167 dst->src_port |= src->src_port; 168 dst->proto |= src->proto; 169 dst->extra |= src->extra; 170 if (is_v6) { 171 #define OR_MASK(_d, _s) \ 172 (_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \ 173 (_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \ 174 (_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \ 175 (_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3]; 176 OR_MASK(&dst->dst_ip6, &src->dst_ip6); 177 OR_MASK(&dst->src_ip6, &src->src_ip6); 178 #undef OR_MASK 179 dst->flow_id6 |= src->flow_id6; 180 } else { 181 dst->dst_ip |= src->dst_ip; 182 dst->src_ip |= src->src_ip; 183 } 184 return dst; 185 } 186 187 static int 188 nonzero_mask(struct ipfw_flow_id *m) 189 { 190 if (m->dst_port || m->src_port || m->proto || m->extra) 191 return 1; 192 if (IS_IP6_FLOW_ID(m)) { 193 return 194 m->dst_ip6.__u6_addr.__u6_addr32[0] || 195 m->dst_ip6.__u6_addr.__u6_addr32[1] || 196 m->dst_ip6.__u6_addr.__u6_addr32[2] || 197 m->dst_ip6.__u6_addr.__u6_addr32[3] || 198 m->src_ip6.__u6_addr.__u6_addr32[0] || 199 m->src_ip6.__u6_addr.__u6_addr32[1] || 200 m->src_ip6.__u6_addr.__u6_addr32[2] || 201 m->src_ip6.__u6_addr.__u6_addr32[3] || 202 m->flow_id6; 203 } else { 204 return m->dst_ip || m->src_ip; 205 } 206 } 207 208 /* XXX we may want a better hash function */ 209 static uint32_t 210 flow_id_hash(struct ipfw_flow_id *id) 211 { 212 uint32_t i; 213 214 if (IS_IP6_FLOW_ID(id)) { 215 uint32_t *d = (uint32_t *)&id->dst_ip6; 216 uint32_t *s = (uint32_t *)&id->src_ip6; 217 i = (d[0] ) ^ (d[1]) ^ 218 (d[2] ) ^ (d[3]) ^ 219 (d[0] >> 15) ^ (d[1] >> 15) ^ 220 (d[2] >> 15) ^ (d[3] >> 15) ^ 221 (s[0] << 1) ^ (s[1] << 1) ^ 222 (s[2] << 1) ^ (s[3] << 1) ^ 223 (s[0] << 16) ^ (s[1] << 16) ^ 224 (s[2] << 16) ^ (s[3] << 16) ^ 225 (id->dst_port << 1) ^ (id->src_port) ^ 226 (id->extra) ^ 227 (id->proto ) ^ (id->flow_id6); 228 } else { 229 i = (id->dst_ip) ^ (id->dst_ip >> 15) ^ 230 (id->src_ip << 1) ^ (id->src_ip >> 16) ^ 231 (id->extra) ^ 232 (id->dst_port << 1) ^ (id->src_port) ^ (id->proto); 233 } 234 return i; 235 } 236 237 /* Like bcmp, returns 0 if ids match, 1 otherwise. */ 238 static int 239 flow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2) 240 { 241 int is_v6 = IS_IP6_FLOW_ID(id1); 242 243 if (!is_v6) { 244 if (IS_IP6_FLOW_ID(id2)) 245 return 1; /* different address families */ 246 247 return (id1->dst_ip == id2->dst_ip && 248 id1->src_ip == id2->src_ip && 249 id1->dst_port == id2->dst_port && 250 id1->src_port == id2->src_port && 251 id1->proto == id2->proto && 252 id1->extra == id2->extra) ? 0 : 1; 253 } 254 /* the ipv6 case */ 255 return ( 256 !bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) && 257 !bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) && 258 id1->dst_port == id2->dst_port && 259 id1->src_port == id2->src_port && 260 id1->proto == id2->proto && 261 id1->extra == id2->extra && 262 id1->flow_id6 == id2->flow_id6) ? 0 : 1; 263 } 264 /*--------- end of flow-id mask, hash and compare ---------*/ 265 266 /*--- support functions for the qht hashtable ---- 267 * Entries are hashed by flow-id 268 */ 269 static uint32_t 270 q_hash(uintptr_t key, int flags, void *arg) 271 { 272 /* compute the hash slot from the flow id */ 273 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 274 &((struct dn_queue *)key)->ni.fid : 275 (struct ipfw_flow_id *)key; 276 277 return flow_id_hash(id); 278 } 279 280 static int 281 q_match(void *obj, uintptr_t key, int flags, void *arg) 282 { 283 struct dn_queue *o = (struct dn_queue *)obj; 284 struct ipfw_flow_id *id2; 285 286 if (flags & DNHT_KEY_IS_OBJ) { 287 /* compare pointers */ 288 id2 = &((struct dn_queue *)key)->ni.fid; 289 } else { 290 id2 = (struct ipfw_flow_id *)key; 291 } 292 return (0 == flow_id_cmp(&o->ni.fid, id2)); 293 } 294 295 /* 296 * create a new queue instance for the given 'key'. 297 */ 298 static void * 299 q_new(uintptr_t key, int flags, void *arg) 300 { 301 struct dn_queue *q, *template = arg; 302 struct dn_fsk *fs = template->fs; 303 int size = sizeof(*q) + fs->sched->fp->q_datalen; 304 305 q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO); 306 if (q == NULL) { 307 D("no memory for new queue"); 308 return NULL; 309 } 310 311 set_oid(&q->ni.oid, DN_QUEUE, size); 312 if (fs->fs.flags & DN_QHT_HASH) 313 q->ni.fid = *(struct ipfw_flow_id *)key; 314 q->fs = fs; 315 q->_si = template->_si; 316 q->_si->q_count++; 317 318 if (fs->sched->fp->new_queue) 319 fs->sched->fp->new_queue(q); 320 dn_cfg.queue_count++; 321 return q; 322 } 323 324 /* 325 * Notify schedulers that a queue is going away. 326 * If (flags & DN_DESTROY), also free the packets. 327 * The version for callbacks is called q_delete_cb(). 328 */ 329 static void 330 dn_delete_queue(struct dn_queue *q, int flags) 331 { 332 struct dn_fsk *fs = q->fs; 333 334 // D("fs %p si %p\n", fs, q->_si); 335 /* notify the parent scheduler that the queue is going away */ 336 if (fs && fs->sched->fp->free_queue) 337 fs->sched->fp->free_queue(q); 338 q->_si->q_count--; 339 q->_si = NULL; 340 if (flags & DN_DESTROY) { 341 if (q->mq.head) 342 dn_free_pkts(q->mq.head); 343 bzero(q, sizeof(*q)); // safety 344 free(q, M_DUMMYNET); 345 dn_cfg.queue_count--; 346 } 347 } 348 349 static int 350 q_delete_cb(void *q, void *arg) 351 { 352 int flags = (int)(uintptr_t)arg; 353 dn_delete_queue(q, flags); 354 return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0; 355 } 356 357 /* 358 * calls dn_delete_queue/q_delete_cb on all queues, 359 * which notifies the parent scheduler and possibly drains packets. 360 * flags & DN_DESTROY: drains queues and destroy qht; 361 */ 362 static void 363 qht_delete(struct dn_fsk *fs, int flags) 364 { 365 ND("fs %d start flags %d qht %p", 366 fs->fs.fs_nr, flags, fs->qht); 367 if (!fs->qht) 368 return; 369 if (fs->fs.flags & DN_QHT_HASH) { 370 dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags); 371 if (flags & DN_DESTROY) { 372 dn_ht_free(fs->qht, 0); 373 fs->qht = NULL; 374 } 375 } else { 376 dn_delete_queue((struct dn_queue *)(fs->qht), flags); 377 if (flags & DN_DESTROY) 378 fs->qht = NULL; 379 } 380 } 381 382 /* 383 * Find and possibly create the queue for a MULTIQUEUE scheduler. 384 * We never call it for !MULTIQUEUE (the queue is in the sch_inst). 385 */ 386 struct dn_queue * 387 ipdn_q_find(struct dn_fsk *fs, struct dn_sch_inst *si, 388 struct ipfw_flow_id *id) 389 { 390 struct dn_queue template; 391 392 template._si = si; 393 template.fs = fs; 394 395 if (fs->fs.flags & DN_QHT_HASH) { 396 struct ipfw_flow_id masked_id; 397 if (fs->qht == NULL) { 398 fs->qht = dn_ht_init(NULL, fs->fs.buckets, 399 offsetof(struct dn_queue, q_next), 400 q_hash, q_match, q_new); 401 if (fs->qht == NULL) 402 return NULL; 403 } 404 masked_id = *id; 405 flow_id_mask(&fs->fsk_mask, &masked_id); 406 return dn_ht_find(fs->qht, (uintptr_t)&masked_id, 407 DNHT_INSERT, &template); 408 } else { 409 if (fs->qht == NULL) 410 fs->qht = q_new(0, 0, &template); 411 return (struct dn_queue *)fs->qht; 412 } 413 } 414 /*--- end of queue hash table ---*/ 415 416 /*--- support functions for the sch_inst hashtable ---- 417 * 418 * These are hashed by flow-id 419 */ 420 static uint32_t 421 si_hash(uintptr_t key, int flags, void *arg) 422 { 423 /* compute the hash slot from the flow id */ 424 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 425 &((struct dn_sch_inst *)key)->ni.fid : 426 (struct ipfw_flow_id *)key; 427 428 return flow_id_hash(id); 429 } 430 431 static int 432 si_match(void *obj, uintptr_t key, int flags, void *arg) 433 { 434 struct dn_sch_inst *o = obj; 435 struct ipfw_flow_id *id2; 436 437 id2 = (flags & DNHT_KEY_IS_OBJ) ? 438 &((struct dn_sch_inst *)key)->ni.fid : 439 (struct ipfw_flow_id *)key; 440 return flow_id_cmp(&o->ni.fid, id2) == 0; 441 } 442 443 /* 444 * create a new instance for the given 'key' 445 * Allocate memory for instance, delay line and scheduler private data. 446 */ 447 static void * 448 si_new(uintptr_t key, int flags, void *arg) 449 { 450 struct dn_schk *s = arg; 451 struct dn_sch_inst *si; 452 int l = sizeof(*si) + s->fp->si_datalen; 453 454 si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 455 if (si == NULL) 456 goto error; 457 458 /* Set length only for the part passed up to userland. */ 459 set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow)); 460 set_oid(&(si->dline.oid), DN_DELAY_LINE, 461 sizeof(struct delay_line)); 462 /* mark si and dline as outside the event queue */ 463 si->ni.oid.id = si->dline.oid.id = -1; 464 465 si->sched = s; 466 si->dline.si = si; 467 468 if (s->fp->new_sched && s->fp->new_sched(si)) { 469 D("new_sched error"); 470 goto error; 471 } 472 if (s->sch.flags & DN_HAVE_MASK) 473 si->ni.fid = *(struct ipfw_flow_id *)key; 474 475 dn_cfg.si_count++; 476 return si; 477 478 error: 479 if (si) { 480 bzero(si, sizeof(*si)); // safety 481 free(si, M_DUMMYNET); 482 } 483 return NULL; 484 } 485 486 /* 487 * Callback from siht to delete all scheduler instances. Remove 488 * si and delay line from the system heap, destroy all queues. 489 * We assume that all flowset have been notified and do not 490 * point to us anymore. 491 */ 492 static int 493 si_destroy(void *_si, void *arg) 494 { 495 struct dn_sch_inst *si = _si; 496 struct dn_schk *s = si->sched; 497 struct delay_line *dl = &si->dline; 498 499 if (dl->oid.subtype) /* remove delay line from event heap */ 500 heap_extract(&dn_cfg.evheap, dl); 501 dn_free_pkts(dl->mq.head); /* drain delay line */ 502 if (si->kflags & DN_ACTIVE) /* remove si from event heap */ 503 heap_extract(&dn_cfg.evheap, si); 504 if (s->fp->free_sched) 505 s->fp->free_sched(si); 506 bzero(si, sizeof(*si)); /* safety */ 507 free(si, M_DUMMYNET); 508 dn_cfg.si_count--; 509 return DNHT_SCAN_DEL; 510 } 511 512 /* 513 * Find the scheduler instance for this packet. If we need to apply 514 * a mask, do on a local copy of the flow_id to preserve the original. 515 * Assume siht is always initialized if we have a mask. 516 */ 517 struct dn_sch_inst * 518 ipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id) 519 { 520 521 if (s->sch.flags & DN_HAVE_MASK) { 522 struct ipfw_flow_id id_t = *id; 523 flow_id_mask(&s->sch.sched_mask, &id_t); 524 return dn_ht_find(s->siht, (uintptr_t)&id_t, 525 DNHT_INSERT, s); 526 } 527 if (!s->siht) 528 s->siht = si_new(0, 0, s); 529 return (struct dn_sch_inst *)s->siht; 530 } 531 532 /* callback to flush credit for the scheduler instance */ 533 static int 534 si_reset_credit(void *_si, void *arg) 535 { 536 struct dn_sch_inst *si = _si; 537 struct dn_link *p = &si->sched->link; 538 539 si->credit = p->burst + (dn_cfg.io_fast ? p->bandwidth : 0); 540 return 0; 541 } 542 543 static void 544 schk_reset_credit(struct dn_schk *s) 545 { 546 if (s->sch.flags & DN_HAVE_MASK) 547 dn_ht_scan(s->siht, si_reset_credit, NULL); 548 else if (s->siht) 549 si_reset_credit(s->siht, NULL); 550 } 551 /*---- end of sch_inst hashtable ---------------------*/ 552 553 /*------------------------------------------------------- 554 * flowset hash (fshash) support. Entries are hashed by fs_nr. 555 * New allocations are put in the fsunlinked list, from which 556 * they are removed when they point to a specific scheduler. 557 */ 558 static uint32_t 559 fsk_hash(uintptr_t key, int flags, void *arg) 560 { 561 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 562 ((struct dn_fsk *)key)->fs.fs_nr; 563 564 return ( (i>>8)^(i>>4)^i ); 565 } 566 567 static int 568 fsk_match(void *obj, uintptr_t key, int flags, void *arg) 569 { 570 struct dn_fsk *fs = obj; 571 int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 572 ((struct dn_fsk *)key)->fs.fs_nr; 573 574 return (fs->fs.fs_nr == i); 575 } 576 577 static void * 578 fsk_new(uintptr_t key, int flags, void *arg) 579 { 580 struct dn_fsk *fs; 581 582 fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO); 583 if (fs) { 584 set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs)); 585 dn_cfg.fsk_count++; 586 fs->drain_bucket = 0; 587 SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); 588 } 589 return fs; 590 } 591 592 /* 593 * detach flowset from its current scheduler. Flags as follows: 594 * DN_DETACH removes from the fsk_list 595 * DN_DESTROY deletes individual queues 596 * DN_DELETE_FS destroys the flowset (otherwise goes in unlinked). 597 */ 598 static void 599 fsk_detach(struct dn_fsk *fs, int flags) 600 { 601 if (flags & DN_DELETE_FS) 602 flags |= DN_DESTROY; 603 ND("fs %d from sched %d flags %s %s %s", 604 fs->fs.fs_nr, fs->fs.sched_nr, 605 (flags & DN_DELETE_FS) ? "DEL_FS":"", 606 (flags & DN_DESTROY) ? "DEL":"", 607 (flags & DN_DETACH) ? "DET":""); 608 if (flags & DN_DETACH) { /* detach from the list */ 609 struct dn_fsk_head *h; 610 h = fs->sched ? &fs->sched->fsk_list : &dn_cfg.fsu; 611 SLIST_REMOVE(h, fs, dn_fsk, sch_chain); 612 } 613 /* Free the RED parameters, they will be recomputed on 614 * subsequent attach if needed. 615 */ 616 if (fs->w_q_lookup) 617 free(fs->w_q_lookup, M_DUMMYNET); 618 fs->w_q_lookup = NULL; 619 qht_delete(fs, flags); 620 if (fs->sched && fs->sched->fp->free_fsk) 621 fs->sched->fp->free_fsk(fs); 622 fs->sched = NULL; 623 if (flags & DN_DELETE_FS) { 624 bzero(fs, sizeof(*fs)); /* safety */ 625 free(fs, M_DUMMYNET); 626 dn_cfg.fsk_count--; 627 } else { 628 SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); 629 } 630 } 631 632 /* 633 * Detach or destroy all flowsets in a list. 634 * flags specifies what to do: 635 * DN_DESTROY: flush all queues 636 * DN_DELETE_FS: DN_DESTROY + destroy flowset 637 * DN_DELETE_FS implies DN_DESTROY 638 */ 639 static void 640 fsk_detach_list(struct dn_fsk_head *h, int flags) 641 { 642 struct dn_fsk *fs; 643 int n = 0; /* only for stats */ 644 645 ND("head %p flags %x", h, flags); 646 while ((fs = SLIST_FIRST(h))) { 647 SLIST_REMOVE_HEAD(h, sch_chain); 648 n++; 649 fsk_detach(fs, flags); 650 } 651 ND("done %d flowsets", n); 652 } 653 654 /* 655 * called on 'queue X delete' -- removes the flowset from fshash, 656 * deletes all queues for the flowset, and removes the flowset. 657 */ 658 static int 659 delete_fs(int i, int locked) 660 { 661 struct dn_fsk *fs; 662 int err = 0; 663 664 if (!locked) 665 DN_BH_WLOCK(); 666 fs = dn_ht_find(dn_cfg.fshash, i, DNHT_REMOVE, NULL); 667 ND("fs %d found %p", i, fs); 668 if (fs) { 669 fsk_detach(fs, DN_DETACH | DN_DELETE_FS); 670 err = 0; 671 } else 672 err = EINVAL; 673 if (!locked) 674 DN_BH_WUNLOCK(); 675 return err; 676 } 677 678 /*----- end of flowset hashtable support -------------*/ 679 680 /*------------------------------------------------------------ 681 * Scheduler hash. When searching by index we pass sched_nr, 682 * otherwise we pass struct dn_sch * which is the first field in 683 * struct dn_schk so we can cast between the two. We use this trick 684 * because in the create phase (but it should be fixed). 685 */ 686 static uint32_t 687 schk_hash(uintptr_t key, int flags, void *_arg) 688 { 689 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 690 ((struct dn_schk *)key)->sch.sched_nr; 691 return ( (i>>8)^(i>>4)^i ); 692 } 693 694 static int 695 schk_match(void *obj, uintptr_t key, int flags, void *_arg) 696 { 697 struct dn_schk *s = (struct dn_schk *)obj; 698 int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 699 ((struct dn_schk *)key)->sch.sched_nr; 700 return (s->sch.sched_nr == i); 701 } 702 703 /* 704 * Create the entry and intialize with the sched hash if needed. 705 * Leave s->fp unset so we can tell whether a dn_ht_find() returns 706 * a new object or a previously existing one. 707 */ 708 static void * 709 schk_new(uintptr_t key, int flags, void *arg) 710 { 711 struct schk_new_arg *a = arg; 712 struct dn_schk *s; 713 int l = sizeof(*s) +a->fp->schk_datalen; 714 715 s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 716 if (s == NULL) 717 return NULL; 718 set_oid(&s->link.oid, DN_LINK, sizeof(s->link)); 719 s->sch = *a->sch; // copy initial values 720 s->link.link_nr = s->sch.sched_nr; 721 SLIST_INIT(&s->fsk_list); 722 /* initialize the hash table or create the single instance */ 723 s->fp = a->fp; /* si_new needs this */ 724 s->drain_bucket = 0; 725 if (s->sch.flags & DN_HAVE_MASK) { 726 s->siht = dn_ht_init(NULL, s->sch.buckets, 727 offsetof(struct dn_sch_inst, si_next), 728 si_hash, si_match, si_new); 729 if (s->siht == NULL) { 730 free(s, M_DUMMYNET); 731 return NULL; 732 } 733 } 734 s->fp = NULL; /* mark as a new scheduler */ 735 dn_cfg.schk_count++; 736 return s; 737 } 738 739 /* 740 * Callback for sched delete. Notify all attached flowsets to 741 * detach from the scheduler, destroy the internal flowset, and 742 * all instances. The scheduler goes away too. 743 * arg is 0 (only detach flowsets and destroy instances) 744 * DN_DESTROY (detach & delete queues, delete schk) 745 * or DN_DELETE_FS (delete queues and flowsets, delete schk) 746 */ 747 static int 748 schk_delete_cb(void *obj, void *arg) 749 { 750 struct dn_schk *s = obj; 751 #if 0 752 int a = (int)arg; 753 ND("sched %d arg %s%s", 754 s->sch.sched_nr, 755 a&DN_DESTROY ? "DEL ":"", 756 a&DN_DELETE_FS ? "DEL_FS":""); 757 #endif 758 fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0); 759 /* no more flowset pointing to us now */ 760 if (s->sch.flags & DN_HAVE_MASK) { 761 dn_ht_scan(s->siht, si_destroy, NULL); 762 dn_ht_free(s->siht, 0); 763 } else if (s->siht) 764 si_destroy(s->siht, NULL); 765 if (s->profile) { 766 free(s->profile, M_DUMMYNET); 767 s->profile = NULL; 768 } 769 s->siht = NULL; 770 if (s->fp->destroy) 771 s->fp->destroy(s); 772 bzero(s, sizeof(*s)); // safety 773 free(obj, M_DUMMYNET); 774 dn_cfg.schk_count--; 775 return DNHT_SCAN_DEL; 776 } 777 778 /* 779 * called on a 'sched X delete' command. Deletes a single scheduler. 780 * This is done by removing from the schedhash, unlinking all 781 * flowsets and deleting their traffic. 782 */ 783 static int 784 delete_schk(int i) 785 { 786 struct dn_schk *s; 787 788 s = dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 789 ND("%d %p", i, s); 790 if (!s) 791 return EINVAL; 792 delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */ 793 /* then detach flowsets, delete traffic */ 794 schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY); 795 return 0; 796 } 797 /*--- end of schk hashtable support ---*/ 798 799 static int 800 copy_obj(char **start, char *end, void *_o, const char *msg, int i) 801 { 802 struct dn_id *o = _o; 803 int have = end - *start; 804 805 if (have < o->len || o->len == 0 || o->type == 0) { 806 D("(WARN) type %d %s %d have %d need %d", 807 o->type, msg, i, have, o->len); 808 return 1; 809 } 810 ND("type %d %s %d len %d", o->type, msg, i, o->len); 811 bcopy(_o, *start, o->len); 812 if (o->type == DN_LINK) { 813 /* Adjust burst parameter for link */ 814 struct dn_link *l = (struct dn_link *)*start; 815 l->burst = div64(l->burst, 8 * hz); 816 l->delay = l->delay * 1000 / hz; 817 } else if (o->type == DN_SCH) { 818 /* Set id->id to the number of instances */ 819 struct dn_schk *s = _o; 820 struct dn_id *id = (struct dn_id *)(*start); 821 id->id = (s->sch.flags & DN_HAVE_MASK) ? 822 dn_ht_entries(s->siht) : (s->siht ? 1 : 0); 823 } 824 *start += o->len; 825 return 0; 826 } 827 828 /* Specific function to copy a queue. 829 * Copies only the user-visible part of a queue (which is in 830 * a struct dn_flow), and sets len accordingly. 831 */ 832 static int 833 copy_obj_q(char **start, char *end, void *_o, const char *msg, int i) 834 { 835 struct dn_id *o = _o; 836 int have = end - *start; 837 int len = sizeof(struct dn_flow); /* see above comment */ 838 839 if (have < len || o->len == 0 || o->type != DN_QUEUE) { 840 D("ERROR type %d %s %d have %d need %d", 841 o->type, msg, i, have, len); 842 return 1; 843 } 844 ND("type %d %s %d len %d", o->type, msg, i, len); 845 bcopy(_o, *start, len); 846 ((struct dn_id*)(*start))->len = len; 847 *start += len; 848 return 0; 849 } 850 851 static int 852 copy_q_cb(void *obj, void *arg) 853 { 854 struct dn_queue *q = obj; 855 struct copy_args *a = arg; 856 struct dn_flow *ni = (struct dn_flow *)(*a->start); 857 if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1)) 858 return DNHT_SCAN_END; 859 ni->oid.type = DN_FLOW; /* override the DN_QUEUE */ 860 ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL); 861 return 0; 862 } 863 864 static int 865 copy_q(struct copy_args *a, struct dn_fsk *fs, int flags) 866 { 867 if (!fs->qht) 868 return 0; 869 if (fs->fs.flags & DN_QHT_HASH) 870 dn_ht_scan(fs->qht, copy_q_cb, a); 871 else 872 copy_q_cb(fs->qht, a); 873 return 0; 874 } 875 876 /* 877 * This routine only copies the initial part of a profile ? XXX 878 */ 879 static int 880 copy_profile(struct copy_args *a, struct dn_profile *p) 881 { 882 int have = a->end - *a->start; 883 /* XXX here we check for max length */ 884 int profile_len = sizeof(struct dn_profile) - 885 ED_MAX_SAMPLES_NO*sizeof(int); 886 887 if (p == NULL) 888 return 0; 889 if (have < profile_len) { 890 D("error have %d need %d", have, profile_len); 891 return 1; 892 } 893 bcopy(p, *a->start, profile_len); 894 ((struct dn_id *)(*a->start))->len = profile_len; 895 *a->start += profile_len; 896 return 0; 897 } 898 899 static int 900 copy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags) 901 { 902 struct dn_fs *ufs = (struct dn_fs *)(*a->start); 903 if (!fs) 904 return 0; 905 ND("flowset %d", fs->fs.fs_nr); 906 if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr)) 907 return DNHT_SCAN_END; 908 ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ? 909 dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0); 910 if (flags) { /* copy queues */ 911 copy_q(a, fs, 0); 912 } 913 return 0; 914 } 915 916 static int 917 copy_si_cb(void *obj, void *arg) 918 { 919 struct dn_sch_inst *si = obj; 920 struct copy_args *a = arg; 921 struct dn_flow *ni = (struct dn_flow *)(*a->start); 922 if (copy_obj(a->start, a->end, &si->ni, "inst", 923 si->sched->sch.sched_nr)) 924 return DNHT_SCAN_END; 925 ni->oid.type = DN_FLOW; /* override the DN_SCH_I */ 926 ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL); 927 return 0; 928 } 929 930 static int 931 copy_si(struct copy_args *a, struct dn_schk *s, int flags) 932 { 933 if (s->sch.flags & DN_HAVE_MASK) 934 dn_ht_scan(s->siht, copy_si_cb, a); 935 else if (s->siht) 936 copy_si_cb(s->siht, a); 937 return 0; 938 } 939 940 /* 941 * compute a list of children of a scheduler and copy up 942 */ 943 static int 944 copy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags) 945 { 946 struct dn_fsk *fs; 947 struct dn_id *o; 948 uint32_t *p; 949 950 int n = 0, space = sizeof(*o); 951 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 952 if (fs->fs.fs_nr < DN_MAX_ID) 953 n++; 954 } 955 space += n * sizeof(uint32_t); 956 DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n); 957 if (a->end - *(a->start) < space) 958 return DNHT_SCAN_END; 959 o = (struct dn_id *)(*(a->start)); 960 o->len = space; 961 *a->start += o->len; 962 o->type = DN_TEXT; 963 p = (uint32_t *)(o+1); 964 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) 965 if (fs->fs.fs_nr < DN_MAX_ID) 966 *p++ = fs->fs.fs_nr; 967 return 0; 968 } 969 970 static int 971 copy_data_helper(void *_o, void *_arg) 972 { 973 struct copy_args *a = _arg; 974 uint32_t *r = a->extra->r; /* start of first range */ 975 uint32_t *lim; /* first invalid pointer */ 976 int n; 977 978 lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len); 979 980 if (a->type == DN_LINK || a->type == DN_SCH) { 981 /* pipe|sched show, we receive a dn_schk */ 982 struct dn_schk *s = _o; 983 984 n = s->sch.sched_nr; 985 if (a->type == DN_SCH && n >= DN_MAX_ID) 986 return 0; /* not a scheduler */ 987 if (a->type == DN_LINK && n <= DN_MAX_ID) 988 return 0; /* not a pipe */ 989 990 /* see if the object is within one of our ranges */ 991 for (;r < lim; r += 2) { 992 if (n < r[0] || n > r[1]) 993 continue; 994 /* Found a valid entry, copy and we are done */ 995 if (a->flags & DN_C_LINK) { 996 if (copy_obj(a->start, a->end, 997 &s->link, "link", n)) 998 return DNHT_SCAN_END; 999 if (copy_profile(a, s->profile)) 1000 return DNHT_SCAN_END; 1001 if (copy_flowset(a, s->fs, 0)) 1002 return DNHT_SCAN_END; 1003 } 1004 if (a->flags & DN_C_SCH) { 1005 if (copy_obj(a->start, a->end, 1006 &s->sch, "sched", n)) 1007 return DNHT_SCAN_END; 1008 /* list all attached flowsets */ 1009 if (copy_fsk_list(a, s, 0)) 1010 return DNHT_SCAN_END; 1011 } 1012 if (a->flags & DN_C_FLOW) 1013 copy_si(a, s, 0); 1014 break; 1015 } 1016 } else if (a->type == DN_FS) { 1017 /* queue show, skip internal flowsets */ 1018 struct dn_fsk *fs = _o; 1019 1020 n = fs->fs.fs_nr; 1021 if (n >= DN_MAX_ID) 1022 return 0; 1023 /* see if the object is within one of our ranges */ 1024 for (;r < lim; r += 2) { 1025 if (n < r[0] || n > r[1]) 1026 continue; 1027 if (copy_flowset(a, fs, 0)) 1028 return DNHT_SCAN_END; 1029 copy_q(a, fs, 0); 1030 break; /* we are done */ 1031 } 1032 } 1033 return 0; 1034 } 1035 1036 static inline struct dn_schk * 1037 locate_scheduler(int i) 1038 { 1039 return dn_ht_find(dn_cfg.schedhash, i, 0, NULL); 1040 } 1041 1042 /* 1043 * red parameters are in fixed point arithmetic. 1044 */ 1045 static int 1046 config_red(struct dn_fsk *fs) 1047 { 1048 int64_t s, idle, weight, w0; 1049 int t, i; 1050 1051 fs->w_q = fs->fs.w_q; 1052 fs->max_p = fs->fs.max_p; 1053 ND("called"); 1054 /* Doing stuff that was in userland */ 1055 i = fs->sched->link.bandwidth; 1056 s = (i <= 0) ? 0 : 1057 hz * dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i; 1058 1059 idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */ 1060 fs->lookup_step = div64(idle , dn_cfg.red_lookup_depth); 1061 /* fs->lookup_step not scaled, */ 1062 if (!fs->lookup_step) 1063 fs->lookup_step = 1; 1064 w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled 1065 1066 for (t = fs->lookup_step; t > 1; --t) 1067 weight = SCALE_MUL(weight, w0); 1068 fs->lookup_weight = (int)(weight); // scaled 1069 1070 /* Now doing stuff that was in kerneland */ 1071 fs->min_th = SCALE(fs->fs.min_th); 1072 fs->max_th = SCALE(fs->fs.max_th); 1073 1074 if (fs->fs.max_th == fs->fs.min_th) 1075 fs->c_1 = fs->max_p; 1076 else 1077 fs->c_1 = SCALE((int64_t)(fs->max_p)) / (fs->fs.max_th - fs->fs.min_th); 1078 fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th)); 1079 1080 if (fs->fs.flags & DN_IS_GENTLE_RED) { 1081 fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th; 1082 fs->c_4 = SCALE(1) - 2 * fs->max_p; 1083 } 1084 1085 /* If the lookup table already exist, free and create it again. */ 1086 if (fs->w_q_lookup) { 1087 free(fs->w_q_lookup, M_DUMMYNET); 1088 fs->w_q_lookup = NULL; 1089 } 1090 if (dn_cfg.red_lookup_depth == 0) { 1091 printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth" 1092 "must be > 0\n"); 1093 fs->fs.flags &= ~DN_IS_RED; 1094 fs->fs.flags &= ~DN_IS_GENTLE_RED; 1095 return (EINVAL); 1096 } 1097 fs->lookup_depth = dn_cfg.red_lookup_depth; 1098 fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int), 1099 M_DUMMYNET, M_NOWAIT); 1100 if (fs->w_q_lookup == NULL) { 1101 printf("dummynet: sorry, cannot allocate red lookup table\n"); 1102 fs->fs.flags &= ~DN_IS_RED; 1103 fs->fs.flags &= ~DN_IS_GENTLE_RED; 1104 return(ENOSPC); 1105 } 1106 1107 /* Fill the lookup table with (1 - w_q)^x */ 1108 fs->w_q_lookup[0] = SCALE(1) - fs->w_q; 1109 1110 for (i = 1; i < fs->lookup_depth; i++) 1111 fs->w_q_lookup[i] = 1112 SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight); 1113 1114 if (dn_cfg.red_avg_pkt_size < 1) 1115 dn_cfg.red_avg_pkt_size = 512; 1116 fs->avg_pkt_size = dn_cfg.red_avg_pkt_size; 1117 if (dn_cfg.red_max_pkt_size < 1) 1118 dn_cfg.red_max_pkt_size = 1500; 1119 fs->max_pkt_size = dn_cfg.red_max_pkt_size; 1120 ND("exit"); 1121 return 0; 1122 } 1123 1124 /* Scan all flowset attached to this scheduler and update red */ 1125 static void 1126 update_red(struct dn_schk *s) 1127 { 1128 struct dn_fsk *fs; 1129 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 1130 if (fs && (fs->fs.flags & DN_IS_RED)) 1131 config_red(fs); 1132 } 1133 } 1134 1135 /* attach flowset to scheduler s, possibly requeue */ 1136 static void 1137 fsk_attach(struct dn_fsk *fs, struct dn_schk *s) 1138 { 1139 ND("remove fs %d from fsunlinked, link to sched %d", 1140 fs->fs.fs_nr, s->sch.sched_nr); 1141 SLIST_REMOVE(&dn_cfg.fsu, fs, dn_fsk, sch_chain); 1142 fs->sched = s; 1143 SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain); 1144 if (s->fp->new_fsk) 1145 s->fp->new_fsk(fs); 1146 /* XXX compute fsk_mask */ 1147 fs->fsk_mask = fs->fs.flow_mask; 1148 if (fs->sched->sch.flags & DN_HAVE_MASK) 1149 flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask); 1150 if (fs->qht) { 1151 /* 1152 * we must drain qht according to the old 1153 * type, and reinsert according to the new one. 1154 * The requeue is complex -- in general we need to 1155 * reclassify every single packet. 1156 * For the time being, let's hope qht is never set 1157 * when we reach this point. 1158 */ 1159 D("XXX TODO requeue from fs %d to sch %d", 1160 fs->fs.fs_nr, s->sch.sched_nr); 1161 fs->qht = NULL; 1162 } 1163 /* set the new type for qht */ 1164 if (nonzero_mask(&fs->fsk_mask)) 1165 fs->fs.flags |= DN_QHT_HASH; 1166 else 1167 fs->fs.flags &= ~DN_QHT_HASH; 1168 1169 /* XXX config_red() can fail... */ 1170 if (fs->fs.flags & DN_IS_RED) 1171 config_red(fs); 1172 } 1173 1174 /* update all flowsets which may refer to this scheduler */ 1175 static void 1176 update_fs(struct dn_schk *s) 1177 { 1178 struct dn_fsk *fs, *tmp; 1179 1180 SLIST_FOREACH_SAFE(fs, &dn_cfg.fsu, sch_chain, tmp) { 1181 if (s->sch.sched_nr != fs->fs.sched_nr) { 1182 D("fs %d for sch %d not %d still unlinked", 1183 fs->fs.fs_nr, fs->fs.sched_nr, 1184 s->sch.sched_nr); 1185 continue; 1186 } 1187 fsk_attach(fs, s); 1188 } 1189 } 1190 1191 /* 1192 * Configuration -- to preserve backward compatibility we use 1193 * the following scheme (N is 65536) 1194 * NUMBER SCHED LINK FLOWSET 1195 * 1 .. N-1 (1)WFQ (2)WFQ (3)queue 1196 * N+1 .. 2N-1 (4)FIFO (5)FIFO (6)FIFO for sched 1..N-1 1197 * 2N+1 .. 3N-1 -- -- (7)FIFO for sched N+1..2N-1 1198 * 1199 * "pipe i config" configures #1, #2 and #3 1200 * "sched i config" configures #1 and possibly #6 1201 * "queue i config" configures #3 1202 * #1 is configured with 'pipe i config' or 'sched i config' 1203 * #2 is configured with 'pipe i config', and created if not 1204 * existing with 'sched i config' 1205 * #3 is configured with 'queue i config' 1206 * #4 is automatically configured after #1, can only be FIFO 1207 * #5 is automatically configured after #2 1208 * #6 is automatically created when #1 is !MULTIQUEUE, 1209 * and can be updated. 1210 * #7 is automatically configured after #2 1211 */ 1212 1213 /* 1214 * configure a link (and its FIFO instance) 1215 */ 1216 static int 1217 config_link(struct dn_link *p, struct dn_id *arg) 1218 { 1219 int i; 1220 1221 if (p->oid.len != sizeof(*p)) { 1222 D("invalid pipe len %d", p->oid.len); 1223 return EINVAL; 1224 } 1225 i = p->link_nr; 1226 if (i <= 0 || i >= DN_MAX_ID) 1227 return EINVAL; 1228 /* 1229 * The config program passes parameters as follows: 1230 * bw = bits/second (0 means no limits), 1231 * delay = ms, must be translated into ticks. 1232 * qsize = slots/bytes 1233 * burst ??? 1234 */ 1235 p->delay = (p->delay * hz) / 1000; 1236 /* Scale burst size: bytes -> bits * hz */ 1237 p->burst *= 8 * hz; 1238 1239 DN_BH_WLOCK(); 1240 /* do it twice, base link and FIFO link */ 1241 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1242 struct dn_schk *s = locate_scheduler(i); 1243 if (s == NULL) { 1244 DN_BH_WUNLOCK(); 1245 D("sched %d not found", i); 1246 return EINVAL; 1247 } 1248 /* remove profile if exists */ 1249 if (s->profile) { 1250 free(s->profile, M_DUMMYNET); 1251 s->profile = NULL; 1252 } 1253 /* copy all parameters */ 1254 s->link.oid = p->oid; 1255 s->link.link_nr = i; 1256 s->link.delay = p->delay; 1257 if (s->link.bandwidth != p->bandwidth) { 1258 /* XXX bandwidth changes, need to update red params */ 1259 s->link.bandwidth = p->bandwidth; 1260 update_red(s); 1261 } 1262 s->link.burst = p->burst; 1263 schk_reset_credit(s); 1264 } 1265 dn_cfg.id++; 1266 DN_BH_WUNLOCK(); 1267 return 0; 1268 } 1269 1270 /* 1271 * configure a flowset. Can be called from inside with locked=1, 1272 */ 1273 static struct dn_fsk * 1274 config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) 1275 { 1276 int i; 1277 struct dn_fsk *fs; 1278 1279 if (nfs->oid.len != sizeof(*nfs)) { 1280 D("invalid flowset len %d", nfs->oid.len); 1281 return NULL; 1282 } 1283 i = nfs->fs_nr; 1284 if (i <= 0 || i >= 3*DN_MAX_ID) 1285 return NULL; 1286 ND("flowset %d", i); 1287 /* XXX other sanity checks */ 1288 if (nfs->flags & DN_QSIZE_BYTES) { 1289 ipdn_bound_var(&nfs->qsize, 16384, 1290 1500, dn_cfg.byte_limit, NULL); // "queue byte size"); 1291 } else { 1292 ipdn_bound_var(&nfs->qsize, 50, 1293 1, dn_cfg.slot_limit, NULL); // "queue slot size"); 1294 } 1295 if (nfs->flags & DN_HAVE_MASK) { 1296 /* make sure we have some buckets */ 1297 ipdn_bound_var((int *)&nfs->buckets, dn_cfg.hash_size, 1298 1, dn_cfg.max_hash_size, "flowset buckets"); 1299 } else { 1300 nfs->buckets = 1; /* we only need 1 */ 1301 } 1302 if (!locked) 1303 DN_BH_WLOCK(); 1304 do { /* exit with break when done */ 1305 struct dn_schk *s; 1306 int flags = nfs->sched_nr ? DNHT_INSERT : 0; 1307 int j; 1308 int oldc = dn_cfg.fsk_count; 1309 fs = dn_ht_find(dn_cfg.fshash, i, flags, NULL); 1310 if (fs == NULL) { 1311 D("missing sched for flowset %d", i); 1312 break; 1313 } 1314 /* grab some defaults from the existing one */ 1315 if (nfs->sched_nr == 0) /* reuse */ 1316 nfs->sched_nr = fs->fs.sched_nr; 1317 for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) { 1318 if (nfs->par[j] == -1) /* reuse */ 1319 nfs->par[j] = fs->fs.par[j]; 1320 } 1321 if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) { 1322 ND("flowset %d unchanged", i); 1323 break; /* no change, nothing to do */ 1324 } 1325 if (oldc != dn_cfg.fsk_count) /* new item */ 1326 dn_cfg.id++; 1327 s = locate_scheduler(nfs->sched_nr); 1328 /* detach from old scheduler if needed, preserving 1329 * queues if we need to reattach. Then update the 1330 * configuration, and possibly attach to the new sched. 1331 */ 1332 DX(2, "fs %d changed sched %d@%p to %d@%p", 1333 fs->fs.fs_nr, 1334 fs->fs.sched_nr, fs->sched, nfs->sched_nr, s); 1335 if (fs->sched) { 1336 int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY); 1337 flags |= DN_DESTROY; /* XXX temporary */ 1338 fsk_detach(fs, flags); 1339 } 1340 fs->fs = *nfs; /* copy configuration */ 1341 if (s != NULL) 1342 fsk_attach(fs, s); 1343 } while (0); 1344 if (!locked) 1345 DN_BH_WUNLOCK(); 1346 return fs; 1347 } 1348 1349 /* 1350 * config/reconfig a scheduler and its FIFO variant. 1351 * For !MULTIQUEUE schedulers, also set up the flowset. 1352 * 1353 * On reconfigurations (detected because s->fp is set), 1354 * detach existing flowsets preserving traffic, preserve link, 1355 * and delete the old scheduler creating a new one. 1356 */ 1357 static int 1358 config_sched(struct dn_sch *_nsch, struct dn_id *arg) 1359 { 1360 struct dn_schk *s; 1361 struct schk_new_arg a; /* argument for schk_new */ 1362 int i; 1363 struct dn_link p; /* copy of oldlink */ 1364 struct dn_profile *pf = NULL; /* copy of old link profile */ 1365 /* Used to preserv mask parameter */ 1366 struct ipfw_flow_id new_mask; 1367 int new_buckets = 0; 1368 int new_flags = 0; 1369 int pipe_cmd; 1370 int err = ENOMEM; 1371 1372 a.sch = _nsch; 1373 if (a.sch->oid.len != sizeof(*a.sch)) { 1374 D("bad sched len %d", a.sch->oid.len); 1375 return EINVAL; 1376 } 1377 i = a.sch->sched_nr; 1378 if (i <= 0 || i >= DN_MAX_ID) 1379 return EINVAL; 1380 /* make sure we have some buckets */ 1381 if (a.sch->flags & DN_HAVE_MASK) 1382 ipdn_bound_var((int *)&a.sch->buckets, dn_cfg.hash_size, 1383 1, dn_cfg.max_hash_size, "sched buckets"); 1384 /* XXX other sanity checks */ 1385 bzero(&p, sizeof(p)); 1386 1387 pipe_cmd = a.sch->flags & DN_PIPE_CMD; 1388 a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set? 1389 if (pipe_cmd) { 1390 /* Copy mask parameter */ 1391 new_mask = a.sch->sched_mask; 1392 new_buckets = a.sch->buckets; 1393 new_flags = a.sch->flags; 1394 } 1395 DN_BH_WLOCK(); 1396 again: /* run twice, for wfq and fifo */ 1397 /* 1398 * lookup the type. If not supplied, use the previous one 1399 * or default to WF2Q+. Otherwise, return an error. 1400 */ 1401 dn_cfg.id++; 1402 a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name); 1403 if (a.fp != NULL) { 1404 /* found. Lookup or create entry */ 1405 s = dn_ht_find(dn_cfg.schedhash, i, DNHT_INSERT, &a); 1406 } else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) { 1407 /* No type. search existing s* or retry with WF2Q+ */ 1408 s = dn_ht_find(dn_cfg.schedhash, i, 0, &a); 1409 if (s != NULL) { 1410 a.fp = s->fp; 1411 /* Scheduler exists, skip to FIFO scheduler 1412 * if command was pipe config... 1413 */ 1414 if (pipe_cmd) 1415 goto next; 1416 } else { 1417 /* New scheduler, create a wf2q+ with no mask 1418 * if command was pipe config... 1419 */ 1420 if (pipe_cmd) { 1421 /* clear mask parameter */ 1422 bzero(&a.sch->sched_mask, sizeof(new_mask)); 1423 a.sch->buckets = 0; 1424 a.sch->flags &= ~DN_HAVE_MASK; 1425 } 1426 a.sch->oid.subtype = DN_SCHED_WF2QP; 1427 goto again; 1428 } 1429 } else { 1430 D("invalid scheduler type %d %s", 1431 a.sch->oid.subtype, a.sch->name); 1432 err = EINVAL; 1433 goto error; 1434 } 1435 /* normalize name and subtype */ 1436 a.sch->oid.subtype = a.fp->type; 1437 bzero(a.sch->name, sizeof(a.sch->name)); 1438 strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name)); 1439 if (s == NULL) { 1440 D("cannot allocate scheduler %d", i); 1441 goto error; 1442 } 1443 /* restore existing link if any */ 1444 if (p.link_nr) { 1445 s->link = p; 1446 if (!pf || pf->link_nr != p.link_nr) { /* no saved value */ 1447 s->profile = NULL; /* XXX maybe not needed */ 1448 } else { 1449 s->profile = malloc(sizeof(struct dn_profile), 1450 M_DUMMYNET, M_NOWAIT | M_ZERO); 1451 if (s->profile == NULL) { 1452 D("cannot allocate profile"); 1453 goto error; //XXX 1454 } 1455 bcopy(pf, s->profile, sizeof(*pf)); 1456 } 1457 } 1458 p.link_nr = 0; 1459 if (s->fp == NULL) { 1460 DX(2, "sched %d new type %s", i, a.fp->name); 1461 } else if (s->fp != a.fp || 1462 bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) { 1463 /* already existing. */ 1464 DX(2, "sched %d type changed from %s to %s", 1465 i, s->fp->name, a.fp->name); 1466 DX(4, " type/sub %d/%d -> %d/%d", 1467 s->sch.oid.type, s->sch.oid.subtype, 1468 a.sch->oid.type, a.sch->oid.subtype); 1469 if (s->link.link_nr == 0) 1470 D("XXX WARNING link 0 for sched %d", i); 1471 p = s->link; /* preserve link */ 1472 if (s->profile) {/* preserve profile */ 1473 if (!pf) 1474 pf = malloc(sizeof(*pf), 1475 M_DUMMYNET, M_NOWAIT | M_ZERO); 1476 if (pf) /* XXX should issue a warning otherwise */ 1477 bcopy(s->profile, pf, sizeof(*pf)); 1478 } 1479 /* remove from the hash */ 1480 dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 1481 /* Detach flowsets, preserve queues. */ 1482 // schk_delete_cb(s, NULL); 1483 // XXX temporarily, kill queues 1484 schk_delete_cb(s, (void *)DN_DESTROY); 1485 goto again; 1486 } else { 1487 DX(4, "sched %d unchanged type %s", i, a.fp->name); 1488 } 1489 /* complete initialization */ 1490 s->sch = *a.sch; 1491 s->fp = a.fp; 1492 s->cfg = arg; 1493 // XXX schk_reset_credit(s); 1494 /* create the internal flowset if needed, 1495 * trying to reuse existing ones if available 1496 */ 1497 if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) { 1498 s->fs = dn_ht_find(dn_cfg.fshash, i, 0, NULL); 1499 if (!s->fs) { 1500 struct dn_fs fs; 1501 bzero(&fs, sizeof(fs)); 1502 set_oid(&fs.oid, DN_FS, sizeof(fs)); 1503 fs.fs_nr = i + DN_MAX_ID; 1504 fs.sched_nr = i; 1505 s->fs = config_fs(&fs, NULL, 1 /* locked */); 1506 } 1507 if (!s->fs) { 1508 schk_delete_cb(s, (void *)DN_DESTROY); 1509 D("error creating internal fs for %d", i); 1510 goto error; 1511 } 1512 } 1513 /* call init function after the flowset is created */ 1514 if (s->fp->config) 1515 s->fp->config(s); 1516 update_fs(s); 1517 next: 1518 if (i < DN_MAX_ID) { /* now configure the FIFO instance */ 1519 i += DN_MAX_ID; 1520 if (pipe_cmd) { 1521 /* Restore mask parameter for FIFO */ 1522 a.sch->sched_mask = new_mask; 1523 a.sch->buckets = new_buckets; 1524 a.sch->flags = new_flags; 1525 } else { 1526 /* sched config shouldn't modify the FIFO scheduler */ 1527 if (dn_ht_find(dn_cfg.schedhash, i, 0, &a) != NULL) { 1528 /* FIFO already exist, don't touch it */ 1529 err = 0; /* and this is not an error */ 1530 goto error; 1531 } 1532 } 1533 a.sch->sched_nr = i; 1534 a.sch->oid.subtype = DN_SCHED_FIFO; 1535 bzero(a.sch->name, sizeof(a.sch->name)); 1536 goto again; 1537 } 1538 err = 0; 1539 error: 1540 DN_BH_WUNLOCK(); 1541 if (pf) 1542 free(pf, M_DUMMYNET); 1543 return err; 1544 } 1545 1546 /* 1547 * attach a profile to a link 1548 */ 1549 static int 1550 config_profile(struct dn_profile *pf, struct dn_id *arg) 1551 { 1552 struct dn_schk *s; 1553 int i, olen, err = 0; 1554 1555 if (pf->oid.len < sizeof(*pf)) { 1556 D("short profile len %d", pf->oid.len); 1557 return EINVAL; 1558 } 1559 i = pf->link_nr; 1560 if (i <= 0 || i >= DN_MAX_ID) 1561 return EINVAL; 1562 /* XXX other sanity checks */ 1563 DN_BH_WLOCK(); 1564 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1565 s = locate_scheduler(i); 1566 1567 if (s == NULL) { 1568 err = EINVAL; 1569 break; 1570 } 1571 dn_cfg.id++; 1572 /* 1573 * If we had a profile and the new one does not fit, 1574 * or it is deleted, then we need to free memory. 1575 */ 1576 if (s->profile && (pf->samples_no == 0 || 1577 s->profile->oid.len < pf->oid.len)) { 1578 free(s->profile, M_DUMMYNET); 1579 s->profile = NULL; 1580 } 1581 if (pf->samples_no == 0) 1582 continue; 1583 /* 1584 * new profile, possibly allocate memory 1585 * and copy data. 1586 */ 1587 if (s->profile == NULL) 1588 s->profile = malloc(pf->oid.len, 1589 M_DUMMYNET, M_NOWAIT | M_ZERO); 1590 if (s->profile == NULL) { 1591 D("no memory for profile %d", i); 1592 err = ENOMEM; 1593 break; 1594 } 1595 /* preserve larger length XXX double check */ 1596 olen = s->profile->oid.len; 1597 if (olen < pf->oid.len) 1598 olen = pf->oid.len; 1599 bcopy(pf, s->profile, pf->oid.len); 1600 s->profile->oid.len = olen; 1601 } 1602 DN_BH_WUNLOCK(); 1603 return err; 1604 } 1605 1606 /* 1607 * Delete all objects: 1608 */ 1609 static void 1610 dummynet_flush(void) 1611 { 1612 1613 /* delete all schedulers and related links/queues/flowsets */ 1614 dn_ht_scan(dn_cfg.schedhash, schk_delete_cb, 1615 (void *)(uintptr_t)DN_DELETE_FS); 1616 /* delete all remaining (unlinked) flowsets */ 1617 DX(4, "still %d unlinked fs", dn_cfg.fsk_count); 1618 dn_ht_free(dn_cfg.fshash, DNHT_REMOVE); 1619 fsk_detach_list(&dn_cfg.fsu, DN_DELETE_FS); 1620 /* Reinitialize system heap... */ 1621 heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 1622 } 1623 1624 /* 1625 * Main handler for configuration. We are guaranteed to be called 1626 * with an oid which is at least a dn_id. 1627 * - the first object is the command (config, delete, flush, ...) 1628 * - config_link must be issued after the corresponding config_sched 1629 * - parameters (DN_TXT) for an object must preceed the object 1630 * processed on a config_sched. 1631 */ 1632 int 1633 do_config(void *p, int l) 1634 { 1635 struct dn_id *next, *o; 1636 int err = 0, err2 = 0; 1637 struct dn_id *arg = NULL; 1638 uintptr_t *a; 1639 1640 o = p; 1641 if (o->id != DN_API_VERSION) { 1642 D("invalid api version got %d need %d", 1643 o->id, DN_API_VERSION); 1644 return EINVAL; 1645 } 1646 for (; l >= sizeof(*o); o = next) { 1647 struct dn_id *prev = arg; 1648 if (o->len < sizeof(*o) || l < o->len) { 1649 D("bad len o->len %d len %d", o->len, l); 1650 err = EINVAL; 1651 break; 1652 } 1653 l -= o->len; 1654 next = (struct dn_id *)((char *)o + o->len); 1655 err = 0; 1656 switch (o->type) { 1657 default: 1658 D("cmd %d not implemented", o->type); 1659 break; 1660 1661 #ifdef EMULATE_SYSCTL 1662 /* sysctl emulation. 1663 * if we recognize the command, jump to the correct 1664 * handler and return 1665 */ 1666 case DN_SYSCTL_SET: 1667 err = kesysctl_emu_set(p, l); 1668 return err; 1669 #endif 1670 1671 case DN_CMD_CONFIG: /* simply a header */ 1672 break; 1673 1674 case DN_CMD_DELETE: 1675 /* the argument is in the first uintptr_t after o */ 1676 a = (uintptr_t *)(o+1); 1677 if (o->len < sizeof(*o) + sizeof(*a)) { 1678 err = EINVAL; 1679 break; 1680 } 1681 switch (o->subtype) { 1682 case DN_LINK: 1683 /* delete base and derived schedulers */ 1684 DN_BH_WLOCK(); 1685 err = delete_schk(*a); 1686 err2 = delete_schk(*a + DN_MAX_ID); 1687 DN_BH_WUNLOCK(); 1688 if (!err) 1689 err = err2; 1690 break; 1691 1692 default: 1693 D("invalid delete type %d", 1694 o->subtype); 1695 err = EINVAL; 1696 break; 1697 1698 case DN_FS: 1699 err = (*a <1 || *a >= DN_MAX_ID) ? 1700 EINVAL : delete_fs(*a, 0) ; 1701 break; 1702 } 1703 break; 1704 1705 case DN_CMD_FLUSH: 1706 DN_BH_WLOCK(); 1707 dummynet_flush(); 1708 DN_BH_WUNLOCK(); 1709 break; 1710 case DN_TEXT: /* store argument the next block */ 1711 prev = NULL; 1712 arg = o; 1713 break; 1714 case DN_LINK: 1715 err = config_link((struct dn_link *)o, arg); 1716 break; 1717 case DN_PROFILE: 1718 err = config_profile((struct dn_profile *)o, arg); 1719 break; 1720 case DN_SCH: 1721 err = config_sched((struct dn_sch *)o, arg); 1722 break; 1723 case DN_FS: 1724 err = (NULL==config_fs((struct dn_fs *)o, arg, 0)); 1725 break; 1726 } 1727 if (prev) 1728 arg = NULL; 1729 if (err != 0) 1730 break; 1731 } 1732 return err; 1733 } 1734 1735 static int 1736 compute_space(struct dn_id *cmd, struct copy_args *a) 1737 { 1738 int x = 0, need = 0; 1739 int profile_size = sizeof(struct dn_profile) - 1740 ED_MAX_SAMPLES_NO*sizeof(int); 1741 1742 /* NOTE about compute space: 1743 * NP = dn_cfg.schk_count 1744 * NSI = dn_cfg.si_count 1745 * NF = dn_cfg.fsk_count 1746 * NQ = dn_cfg.queue_count 1747 * - ipfw pipe show 1748 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 1749 * link, scheduler template, flowset 1750 * integrated in scheduler and header 1751 * for flowset list 1752 * (NSI)*(dn_flow) all scheduler instance (includes 1753 * the queue instance) 1754 * - ipfw sched show 1755 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 1756 * link, scheduler template, flowset 1757 * integrated in scheduler and header 1758 * for flowset list 1759 * (NSI * dn_flow) all scheduler instances 1760 * (NF * sizeof(uint_32)) space for flowset list linked to scheduler 1761 * (NQ * dn_queue) all queue [XXXfor now not listed] 1762 * - ipfw queue show 1763 * (NF * dn_fs) all flowset 1764 * (NQ * dn_queue) all queues 1765 */ 1766 switch (cmd->subtype) { 1767 default: 1768 return -1; 1769 /* XXX where do LINK and SCH differ ? */ 1770 /* 'ipfw sched show' could list all queues associated to 1771 * a scheduler. This feature for now is disabled 1772 */ 1773 case DN_LINK: /* pipe show */ 1774 x = DN_C_LINK | DN_C_SCH | DN_C_FLOW; 1775 need += dn_cfg.schk_count * 1776 (sizeof(struct dn_fs) + profile_size) / 2; 1777 need += dn_cfg.fsk_count * sizeof(uint32_t); 1778 break; 1779 case DN_SCH: /* sched show */ 1780 need += dn_cfg.schk_count * 1781 (sizeof(struct dn_fs) + profile_size) / 2; 1782 need += dn_cfg.fsk_count * sizeof(uint32_t); 1783 x = DN_C_SCH | DN_C_LINK | DN_C_FLOW; 1784 break; 1785 case DN_FS: /* queue show */ 1786 x = DN_C_FS | DN_C_QUEUE; 1787 break; 1788 case DN_GET_COMPAT: /* compatibility mode */ 1789 need = dn_compat_calc_size(); 1790 break; 1791 } 1792 a->flags = x; 1793 if (x & DN_C_SCH) { 1794 need += dn_cfg.schk_count * sizeof(struct dn_sch) / 2; 1795 /* NOT also, each fs might be attached to a sched */ 1796 need += dn_cfg.schk_count * sizeof(struct dn_id) / 2; 1797 } 1798 if (x & DN_C_FS) 1799 need += dn_cfg.fsk_count * sizeof(struct dn_fs); 1800 if (x & DN_C_LINK) { 1801 need += dn_cfg.schk_count * sizeof(struct dn_link) / 2; 1802 } 1803 /* 1804 * When exporting a queue to userland, only pass up the 1805 * struct dn_flow, which is the only visible part. 1806 */ 1807 1808 if (x & DN_C_QUEUE) 1809 need += dn_cfg.queue_count * sizeof(struct dn_flow); 1810 if (x & DN_C_FLOW) 1811 need += dn_cfg.si_count * (sizeof(struct dn_flow)); 1812 return need; 1813 } 1814 1815 /* 1816 * If compat != NULL dummynet_get is called in compatibility mode. 1817 * *compat will be the pointer to the buffer to pass to ipfw 1818 */ 1819 int 1820 dummynet_get(struct sockopt *sopt, void **compat) 1821 { 1822 int have, i, need, error; 1823 char *start = NULL, *buf; 1824 size_t sopt_valsize; 1825 struct dn_id *cmd; 1826 struct copy_args a; 1827 struct copy_range r; 1828 int l = sizeof(struct dn_id); 1829 1830 bzero(&a, sizeof(a)); 1831 bzero(&r, sizeof(r)); 1832 1833 /* save and restore original sopt_valsize around copyin */ 1834 sopt_valsize = sopt->sopt_valsize; 1835 1836 cmd = &r.o; 1837 1838 if (!compat) { 1839 /* copy at least an oid, and possibly a full object */ 1840 error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd)); 1841 sopt->sopt_valsize = sopt_valsize; 1842 if (error) 1843 goto done; 1844 l = cmd->len; 1845 #ifdef EMULATE_SYSCTL 1846 /* sysctl emulation. */ 1847 if (cmd->type == DN_SYSCTL_GET) 1848 return kesysctl_emu_get(sopt); 1849 #endif 1850 if (l > sizeof(r)) { 1851 /* request larger than default, allocate buffer */ 1852 cmd = malloc(l, M_DUMMYNET, M_WAITOK); 1853 error = sooptcopyin(sopt, cmd, l, l); 1854 sopt->sopt_valsize = sopt_valsize; 1855 if (error) 1856 goto done; 1857 } 1858 } else { /* compatibility */ 1859 error = 0; 1860 cmd->type = DN_CMD_GET; 1861 cmd->len = sizeof(struct dn_id); 1862 cmd->subtype = DN_GET_COMPAT; 1863 // cmd->id = sopt_valsize; 1864 D("compatibility mode"); 1865 } 1866 a.extra = (struct copy_range *)cmd; 1867 if (cmd->len == sizeof(*cmd)) { /* no range, create a default */ 1868 uint32_t *rp = (uint32_t *)(cmd + 1); 1869 cmd->len += 2* sizeof(uint32_t); 1870 rp[0] = 1; 1871 rp[1] = DN_MAX_ID - 1; 1872 if (cmd->subtype == DN_LINK) { 1873 rp[0] += DN_MAX_ID; 1874 rp[1] += DN_MAX_ID; 1875 } 1876 } 1877 /* Count space (under lock) and allocate (outside lock). 1878 * Exit with lock held if we manage to get enough buffer. 1879 * Try a few times then give up. 1880 */ 1881 for (have = 0, i = 0; i < 10; i++) { 1882 DN_BH_WLOCK(); 1883 need = compute_space(cmd, &a); 1884 1885 /* if there is a range, ignore value from compute_space() */ 1886 if (l > sizeof(*cmd)) 1887 need = sopt_valsize - sizeof(*cmd); 1888 1889 if (need < 0) { 1890 DN_BH_WUNLOCK(); 1891 error = EINVAL; 1892 goto done; 1893 } 1894 need += sizeof(*cmd); 1895 cmd->id = need; 1896 if (have >= need) 1897 break; 1898 1899 DN_BH_WUNLOCK(); 1900 if (start) 1901 free(start, M_DUMMYNET); 1902 start = NULL; 1903 if (need > sopt_valsize) 1904 break; 1905 1906 have = need; 1907 start = malloc(have, M_DUMMYNET, M_WAITOK | M_ZERO); 1908 } 1909 1910 if (start == NULL) { 1911 if (compat) { 1912 *compat = NULL; 1913 error = 1; // XXX 1914 } else { 1915 error = sooptcopyout(sopt, cmd, sizeof(*cmd)); 1916 } 1917 goto done; 1918 } 1919 ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, " 1920 "%d:%d si %d, %d:%d queues %d", 1921 dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH, 1922 dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK, 1923 dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS, 1924 dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I, 1925 dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE); 1926 sopt->sopt_valsize = sopt_valsize; 1927 a.type = cmd->subtype; 1928 1929 if (compat == NULL) { 1930 bcopy(cmd, start, sizeof(*cmd)); 1931 ((struct dn_id*)(start))->len = sizeof(struct dn_id); 1932 buf = start + sizeof(*cmd); 1933 } else 1934 buf = start; 1935 a.start = &buf; 1936 a.end = start + have; 1937 /* start copying other objects */ 1938 if (compat) { 1939 a.type = DN_COMPAT_PIPE; 1940 dn_ht_scan(dn_cfg.schedhash, copy_data_helper_compat, &a); 1941 a.type = DN_COMPAT_QUEUE; 1942 dn_ht_scan(dn_cfg.fshash, copy_data_helper_compat, &a); 1943 } else if (a.type == DN_FS) { 1944 dn_ht_scan(dn_cfg.fshash, copy_data_helper, &a); 1945 } else { 1946 dn_ht_scan(dn_cfg.schedhash, copy_data_helper, &a); 1947 } 1948 DN_BH_WUNLOCK(); 1949 1950 if (compat) { 1951 *compat = start; 1952 sopt->sopt_valsize = buf - start; 1953 /* free() is done by ip_dummynet_compat() */ 1954 start = NULL; //XXX hack 1955 } else { 1956 error = sooptcopyout(sopt, start, buf - start); 1957 } 1958 done: 1959 if (cmd && cmd != &r.o) 1960 free(cmd, M_DUMMYNET); 1961 if (start) 1962 free(start, M_DUMMYNET); 1963 return error; 1964 } 1965 1966 /* Callback called on scheduler instance to delete it if idle */ 1967 static int 1968 drain_scheduler_cb(void *_si, void *arg) 1969 { 1970 struct dn_sch_inst *si = _si; 1971 1972 if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL) 1973 return 0; 1974 1975 if (si->sched->fp->flags & DN_MULTIQUEUE) { 1976 if (si->q_count == 0) 1977 return si_destroy(si, NULL); 1978 else 1979 return 0; 1980 } else { /* !DN_MULTIQUEUE */ 1981 if ((si+1)->ni.length == 0) 1982 return si_destroy(si, NULL); 1983 else 1984 return 0; 1985 } 1986 return 0; /* unreachable */ 1987 } 1988 1989 /* Callback called on scheduler to check if it has instances */ 1990 static int 1991 drain_scheduler_sch_cb(void *_s, void *arg) 1992 { 1993 struct dn_schk *s = _s; 1994 1995 if (s->sch.flags & DN_HAVE_MASK) { 1996 dn_ht_scan_bucket(s->siht, &s->drain_bucket, 1997 drain_scheduler_cb, NULL); 1998 s->drain_bucket++; 1999 } else { 2000 if (s->siht) { 2001 if (drain_scheduler_cb(s->siht, NULL) == DNHT_SCAN_DEL) 2002 s->siht = NULL; 2003 } 2004 } 2005 return 0; 2006 } 2007 2008 /* Called every tick, try to delete a 'bucket' of scheduler */ 2009 void 2010 dn_drain_scheduler(void) 2011 { 2012 dn_ht_scan_bucket(dn_cfg.schedhash, &dn_cfg.drain_sch, 2013 drain_scheduler_sch_cb, NULL); 2014 dn_cfg.drain_sch++; 2015 } 2016 2017 /* Callback called on queue to delete if it is idle */ 2018 static int 2019 drain_queue_cb(void *_q, void *arg) 2020 { 2021 struct dn_queue *q = _q; 2022 2023 if (q->ni.length == 0) { 2024 dn_delete_queue(q, DN_DESTROY); 2025 return DNHT_SCAN_DEL; /* queue is deleted */ 2026 } 2027 2028 return 0; /* queue isn't deleted */ 2029 } 2030 2031 /* Callback called on flowset used to check if it has queues */ 2032 static int 2033 drain_queue_fs_cb(void *_fs, void *arg) 2034 { 2035 struct dn_fsk *fs = _fs; 2036 2037 if (fs->fs.flags & DN_QHT_HASH) { 2038 /* Flowset has a hash table for queues */ 2039 dn_ht_scan_bucket(fs->qht, &fs->drain_bucket, 2040 drain_queue_cb, NULL); 2041 fs->drain_bucket++; 2042 } else { 2043 /* No hash table for this flowset, null the pointer 2044 * if the queue is deleted 2045 */ 2046 if (fs->qht) { 2047 if (drain_queue_cb(fs->qht, NULL) == DNHT_SCAN_DEL) 2048 fs->qht = NULL; 2049 } 2050 } 2051 return 0; 2052 } 2053 2054 /* Called every tick, try to delete a 'bucket' of queue */ 2055 void 2056 dn_drain_queue(void) 2057 { 2058 /* scan a bucket of flowset */ 2059 dn_ht_scan_bucket(dn_cfg.fshash, &dn_cfg.drain_fs, 2060 drain_queue_fs_cb, NULL); 2061 dn_cfg.drain_fs++; 2062 } 2063 2064 /* 2065 * Handler for the various dummynet socket options 2066 */ 2067 static int 2068 ip_dn_ctl(struct sockopt *sopt) 2069 { 2070 void *p = NULL; 2071 int error, l; 2072 2073 error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET); 2074 if (error) 2075 return (error); 2076 2077 /* Disallow sets in really-really secure mode. */ 2078 if (sopt->sopt_dir == SOPT_SET) { 2079 error = securelevel_ge(sopt->sopt_td->td_ucred, 3); 2080 if (error) 2081 return (error); 2082 } 2083 2084 switch (sopt->sopt_name) { 2085 default : 2086 D("dummynet: unknown option %d", sopt->sopt_name); 2087 error = EINVAL; 2088 break; 2089 2090 case IP_DUMMYNET_FLUSH: 2091 case IP_DUMMYNET_CONFIGURE: 2092 case IP_DUMMYNET_DEL: /* remove a pipe or queue */ 2093 case IP_DUMMYNET_GET: 2094 D("dummynet: compat option %d", sopt->sopt_name); 2095 error = ip_dummynet_compat(sopt); 2096 break; 2097 2098 case IP_DUMMYNET3 : 2099 if (sopt->sopt_dir == SOPT_GET) { 2100 error = dummynet_get(sopt, NULL); 2101 break; 2102 } 2103 l = sopt->sopt_valsize; 2104 if (l < sizeof(struct dn_id) || l > 12000) { 2105 D("argument len %d invalid", l); 2106 break; 2107 } 2108 p = malloc(l, M_TEMP, M_WAITOK); // XXX can it fail ? 2109 error = sooptcopyin(sopt, p, l, l); 2110 if (error) 2111 break ; 2112 error = do_config(p, l); 2113 break; 2114 } 2115 2116 if (p != NULL) 2117 free(p, M_TEMP); 2118 2119 return error ; 2120 } 2121 2122 2123 static void 2124 ip_dn_init(void) 2125 { 2126 if (dn_cfg.init_done) 2127 return; 2128 printf("DUMMYNET %p with IPv6 initialized (100409)\n", curvnet); 2129 dn_cfg.init_done = 1; 2130 /* Set defaults here. MSVC does not accept initializers, 2131 * and this is also useful for vimages 2132 */ 2133 /* queue limits */ 2134 dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */ 2135 dn_cfg.byte_limit = 1024 * 1024; 2136 dn_cfg.expire = 1; 2137 2138 /* RED parameters */ 2139 dn_cfg.red_lookup_depth = 256; /* default lookup table depth */ 2140 dn_cfg.red_avg_pkt_size = 512; /* default medium packet size */ 2141 dn_cfg.red_max_pkt_size = 1500; /* default max packet size */ 2142 2143 /* hash tables */ 2144 dn_cfg.max_hash_size = 65536; /* max in the hash tables */ 2145 dn_cfg.hash_size = 64; /* default hash size */ 2146 2147 /* create hash tables for schedulers and flowsets. 2148 * In both we search by key and by pointer. 2149 */ 2150 dn_cfg.schedhash = dn_ht_init(NULL, dn_cfg.hash_size, 2151 offsetof(struct dn_schk, schk_next), 2152 schk_hash, schk_match, schk_new); 2153 dn_cfg.fshash = dn_ht_init(NULL, dn_cfg.hash_size, 2154 offsetof(struct dn_fsk, fsk_next), 2155 fsk_hash, fsk_match, fsk_new); 2156 2157 /* bucket index to drain object */ 2158 dn_cfg.drain_fs = 0; 2159 dn_cfg.drain_sch = 0; 2160 2161 heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 2162 SLIST_INIT(&dn_cfg.fsu); 2163 SLIST_INIT(&dn_cfg.schedlist); 2164 2165 DN_LOCK_INIT(); 2166 2167 TASK_INIT(&dn_task, 0, dummynet_task, curvnet); 2168 dn_tq = taskqueue_create_fast("dummynet", M_WAITOK, 2169 taskqueue_thread_enqueue, &dn_tq); 2170 taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet"); 2171 2172 callout_init(&dn_timeout, CALLOUT_MPSAFE); 2173 dn_reschedule(); 2174 2175 /* Initialize curr_time adjustment mechanics. */ 2176 getmicrouptime(&dn_cfg.prev_t); 2177 } 2178 2179 static void 2180 ip_dn_destroy(int last) 2181 { 2182 callout_drain(&dn_timeout); 2183 2184 DN_BH_WLOCK(); 2185 if (last) { 2186 ND("removing last instance\n"); 2187 ip_dn_ctl_ptr = NULL; 2188 ip_dn_io_ptr = NULL; 2189 } 2190 2191 dummynet_flush(); 2192 DN_BH_WUNLOCK(); 2193 taskqueue_drain(dn_tq, &dn_task); 2194 taskqueue_free(dn_tq); 2195 2196 dn_ht_free(dn_cfg.schedhash, 0); 2197 dn_ht_free(dn_cfg.fshash, 0); 2198 heap_free(&dn_cfg.evheap); 2199 2200 DN_LOCK_DESTROY(); 2201 } 2202 2203 static int 2204 dummynet_modevent(module_t mod, int type, void *data) 2205 { 2206 2207 if (type == MOD_LOAD) { 2208 if (ip_dn_io_ptr) { 2209 printf("DUMMYNET already loaded\n"); 2210 return EEXIST ; 2211 } 2212 ip_dn_init(); 2213 ip_dn_ctl_ptr = ip_dn_ctl; 2214 ip_dn_io_ptr = dummynet_io; 2215 return 0; 2216 } else if (type == MOD_UNLOAD) { 2217 ip_dn_destroy(1 /* last */); 2218 return 0; 2219 } else 2220 return EOPNOTSUPP; 2221 } 2222 2223 /* modevent helpers for the modules */ 2224 static int 2225 load_dn_sched(struct dn_alg *d) 2226 { 2227 struct dn_alg *s; 2228 2229 if (d == NULL) 2230 return 1; /* error */ 2231 ip_dn_init(); /* just in case, we need the lock */ 2232 2233 /* Check that mandatory funcs exists */ 2234 if (d->enqueue == NULL || d->dequeue == NULL) { 2235 D("missing enqueue or dequeue for %s", d->name); 2236 return 1; 2237 } 2238 2239 /* Search if scheduler already exists */ 2240 DN_BH_WLOCK(); 2241 SLIST_FOREACH(s, &dn_cfg.schedlist, next) { 2242 if (strcmp(s->name, d->name) == 0) { 2243 D("%s already loaded", d->name); 2244 break; /* scheduler already exists */ 2245 } 2246 } 2247 if (s == NULL) 2248 SLIST_INSERT_HEAD(&dn_cfg.schedlist, d, next); 2249 DN_BH_WUNLOCK(); 2250 D("dn_sched %s %sloaded", d->name, s ? "not ":""); 2251 return s ? 1 : 0; 2252 } 2253 2254 static int 2255 unload_dn_sched(struct dn_alg *s) 2256 { 2257 struct dn_alg *tmp, *r; 2258 int err = EINVAL; 2259 2260 ND("called for %s", s->name); 2261 2262 DN_BH_WLOCK(); 2263 SLIST_FOREACH_SAFE(r, &dn_cfg.schedlist, next, tmp) { 2264 if (strcmp(s->name, r->name) != 0) 2265 continue; 2266 ND("ref_count = %d", r->ref_count); 2267 err = (r->ref_count != 0) ? EBUSY : 0; 2268 if (err == 0) 2269 SLIST_REMOVE(&dn_cfg.schedlist, r, dn_alg, next); 2270 break; 2271 } 2272 DN_BH_WUNLOCK(); 2273 D("dn_sched %s %sunloaded", s->name, err ? "not ":""); 2274 return err; 2275 } 2276 2277 int 2278 dn_sched_modevent(module_t mod, int cmd, void *arg) 2279 { 2280 struct dn_alg *sch = arg; 2281 2282 if (cmd == MOD_LOAD) 2283 return load_dn_sched(sch); 2284 else if (cmd == MOD_UNLOAD) 2285 return unload_dn_sched(sch); 2286 else 2287 return EINVAL; 2288 } 2289 2290 static moduledata_t dummynet_mod = { 2291 "dummynet", dummynet_modevent, NULL 2292 }; 2293 2294 #define DN_SI_SUB SI_SUB_PROTO_IFATTACHDOMAIN 2295 #define DN_MODEV_ORD (SI_ORDER_ANY - 128) /* after ipfw */ 2296 DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD); 2297 MODULE_DEPEND(dummynet, ipfw, 3, 3, 3); 2298 MODULE_VERSION(dummynet, 3); 2299 2300 /* 2301 * Starting up. Done in order after dummynet_modevent() has been called. 2302 * VNET_SYSINIT is also called for each existing vnet and each new vnet. 2303 */ 2304 //VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_init, NULL); 2305 2306 /* 2307 * Shutdown handlers up shop. These are done in REVERSE ORDER, but still 2308 * after dummynet_modevent() has been called. Not called on reboot. 2309 * VNET_SYSUNINIT is also called for each exiting vnet as it exits. 2310 * or when the module is unloaded. 2311 */ 2312 //VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_destroy, NULL); 2313 2314 /* end of file */ 2315