1 /*- 2 * Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa 3 * Portions Copyright (c) 2000 Akamba Corp. 4 * All rights reserved 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 /* 32 * Configuration and internal object management for dummynet. 33 */ 34 35 #include "opt_inet6.h" 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/mbuf.h> 41 #include <sys/kernel.h> 42 #include <sys/lock.h> 43 #include <sys/module.h> 44 #include <sys/mutex.h> 45 #include <sys/priv.h> 46 #include <sys/proc.h> 47 #include <sys/rwlock.h> 48 #include <sys/socket.h> 49 #include <sys/socketvar.h> 50 #include <sys/time.h> 51 #include <sys/taskqueue.h> 52 #include <net/if.h> /* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */ 53 #include <netinet/in.h> 54 #include <netinet/ip_var.h> /* ip_output(), IP_FORWARDING */ 55 #include <netinet/ip_fw.h> 56 #include <netinet/ip_dummynet.h> 57 58 #include <netpfil/ipfw/ip_fw_private.h> 59 #include <netpfil/ipfw/dn_heap.h> 60 #include <netpfil/ipfw/ip_dn_private.h> 61 #include <netpfil/ipfw/dn_sched.h> 62 63 /* which objects to copy */ 64 #define DN_C_LINK 0x01 65 #define DN_C_SCH 0x02 66 #define DN_C_FLOW 0x04 67 #define DN_C_FS 0x08 68 #define DN_C_QUEUE 0x10 69 70 /* we use this argument in case of a schk_new */ 71 struct schk_new_arg { 72 struct dn_alg *fp; 73 struct dn_sch *sch; 74 }; 75 76 /*---- callout hooks. ----*/ 77 static struct callout dn_timeout; 78 static int dn_gone; 79 static struct task dn_task; 80 static struct taskqueue *dn_tq = NULL; 81 82 static void 83 dummynet(void *arg) 84 { 85 86 (void)arg; /* UNUSED */ 87 taskqueue_enqueue_fast(dn_tq, &dn_task); 88 } 89 90 void 91 dn_reschedule(void) 92 { 93 94 if (dn_gone != 0) 95 return; 96 callout_reset_sbt(&dn_timeout, tick_sbt, 0, dummynet, NULL, 97 C_HARDCLOCK | C_DIRECT_EXEC); 98 } 99 /*----- end of callout hooks -----*/ 100 101 /* Return a scheduler descriptor given the type or name. */ 102 static struct dn_alg * 103 find_sched_type(int type, char *name) 104 { 105 struct dn_alg *d; 106 107 SLIST_FOREACH(d, &dn_cfg.schedlist, next) { 108 if (d->type == type || (name && !strcasecmp(d->name, name))) 109 return d; 110 } 111 return NULL; /* not found */ 112 } 113 114 int 115 ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg) 116 { 117 int oldv = *v; 118 const char *op = NULL; 119 if (dflt < lo) 120 dflt = lo; 121 if (dflt > hi) 122 dflt = hi; 123 if (oldv < lo) { 124 *v = dflt; 125 op = "Bump"; 126 } else if (oldv > hi) { 127 *v = hi; 128 op = "Clamp"; 129 } else 130 return *v; 131 if (op && msg) 132 printf("%s %s to %d (was %d)\n", op, msg, *v, oldv); 133 return *v; 134 } 135 136 /*---- flow_id mask, hash and compare functions ---*/ 137 /* 138 * The flow_id includes the 5-tuple, the queue/pipe number 139 * which we store in the extra area in host order, 140 * and for ipv6 also the flow_id6. 141 * XXX see if we want the tos byte (can store in 'flags') 142 */ 143 static struct ipfw_flow_id * 144 flow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id) 145 { 146 int is_v6 = IS_IP6_FLOW_ID(id); 147 148 id->dst_port &= mask->dst_port; 149 id->src_port &= mask->src_port; 150 id->proto &= mask->proto; 151 id->extra &= mask->extra; 152 if (is_v6) { 153 APPLY_MASK(&id->dst_ip6, &mask->dst_ip6); 154 APPLY_MASK(&id->src_ip6, &mask->src_ip6); 155 id->flow_id6 &= mask->flow_id6; 156 } else { 157 id->dst_ip &= mask->dst_ip; 158 id->src_ip &= mask->src_ip; 159 } 160 return id; 161 } 162 163 /* computes an OR of two masks, result in dst and also returned */ 164 static struct ipfw_flow_id * 165 flow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst) 166 { 167 int is_v6 = IS_IP6_FLOW_ID(dst); 168 169 dst->dst_port |= src->dst_port; 170 dst->src_port |= src->src_port; 171 dst->proto |= src->proto; 172 dst->extra |= src->extra; 173 if (is_v6) { 174 #define OR_MASK(_d, _s) \ 175 (_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \ 176 (_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \ 177 (_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \ 178 (_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3]; 179 OR_MASK(&dst->dst_ip6, &src->dst_ip6); 180 OR_MASK(&dst->src_ip6, &src->src_ip6); 181 #undef OR_MASK 182 dst->flow_id6 |= src->flow_id6; 183 } else { 184 dst->dst_ip |= src->dst_ip; 185 dst->src_ip |= src->src_ip; 186 } 187 return dst; 188 } 189 190 static int 191 nonzero_mask(struct ipfw_flow_id *m) 192 { 193 if (m->dst_port || m->src_port || m->proto || m->extra) 194 return 1; 195 if (IS_IP6_FLOW_ID(m)) { 196 return 197 m->dst_ip6.__u6_addr.__u6_addr32[0] || 198 m->dst_ip6.__u6_addr.__u6_addr32[1] || 199 m->dst_ip6.__u6_addr.__u6_addr32[2] || 200 m->dst_ip6.__u6_addr.__u6_addr32[3] || 201 m->src_ip6.__u6_addr.__u6_addr32[0] || 202 m->src_ip6.__u6_addr.__u6_addr32[1] || 203 m->src_ip6.__u6_addr.__u6_addr32[2] || 204 m->src_ip6.__u6_addr.__u6_addr32[3] || 205 m->flow_id6; 206 } else { 207 return m->dst_ip || m->src_ip; 208 } 209 } 210 211 /* XXX we may want a better hash function */ 212 static uint32_t 213 flow_id_hash(struct ipfw_flow_id *id) 214 { 215 uint32_t i; 216 217 if (IS_IP6_FLOW_ID(id)) { 218 uint32_t *d = (uint32_t *)&id->dst_ip6; 219 uint32_t *s = (uint32_t *)&id->src_ip6; 220 i = (d[0] ) ^ (d[1]) ^ 221 (d[2] ) ^ (d[3]) ^ 222 (d[0] >> 15) ^ (d[1] >> 15) ^ 223 (d[2] >> 15) ^ (d[3] >> 15) ^ 224 (s[0] << 1) ^ (s[1] << 1) ^ 225 (s[2] << 1) ^ (s[3] << 1) ^ 226 (s[0] << 16) ^ (s[1] << 16) ^ 227 (s[2] << 16) ^ (s[3] << 16) ^ 228 (id->dst_port << 1) ^ (id->src_port) ^ 229 (id->extra) ^ 230 (id->proto ) ^ (id->flow_id6); 231 } else { 232 i = (id->dst_ip) ^ (id->dst_ip >> 15) ^ 233 (id->src_ip << 1) ^ (id->src_ip >> 16) ^ 234 (id->extra) ^ 235 (id->dst_port << 1) ^ (id->src_port) ^ (id->proto); 236 } 237 return i; 238 } 239 240 /* Like bcmp, returns 0 if ids match, 1 otherwise. */ 241 static int 242 flow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2) 243 { 244 int is_v6 = IS_IP6_FLOW_ID(id1); 245 246 if (!is_v6) { 247 if (IS_IP6_FLOW_ID(id2)) 248 return 1; /* different address families */ 249 250 return (id1->dst_ip == id2->dst_ip && 251 id1->src_ip == id2->src_ip && 252 id1->dst_port == id2->dst_port && 253 id1->src_port == id2->src_port && 254 id1->proto == id2->proto && 255 id1->extra == id2->extra) ? 0 : 1; 256 } 257 /* the ipv6 case */ 258 return ( 259 !bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) && 260 !bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) && 261 id1->dst_port == id2->dst_port && 262 id1->src_port == id2->src_port && 263 id1->proto == id2->proto && 264 id1->extra == id2->extra && 265 id1->flow_id6 == id2->flow_id6) ? 0 : 1; 266 } 267 /*--------- end of flow-id mask, hash and compare ---------*/ 268 269 /*--- support functions for the qht hashtable ---- 270 * Entries are hashed by flow-id 271 */ 272 static uint32_t 273 q_hash(uintptr_t key, int flags, void *arg) 274 { 275 /* compute the hash slot from the flow id */ 276 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 277 &((struct dn_queue *)key)->ni.fid : 278 (struct ipfw_flow_id *)key; 279 280 return flow_id_hash(id); 281 } 282 283 static int 284 q_match(void *obj, uintptr_t key, int flags, void *arg) 285 { 286 struct dn_queue *o = (struct dn_queue *)obj; 287 struct ipfw_flow_id *id2; 288 289 if (flags & DNHT_KEY_IS_OBJ) { 290 /* compare pointers */ 291 id2 = &((struct dn_queue *)key)->ni.fid; 292 } else { 293 id2 = (struct ipfw_flow_id *)key; 294 } 295 return (0 == flow_id_cmp(&o->ni.fid, id2)); 296 } 297 298 /* 299 * create a new queue instance for the given 'key'. 300 */ 301 static void * 302 q_new(uintptr_t key, int flags, void *arg) 303 { 304 struct dn_queue *q, *template = arg; 305 struct dn_fsk *fs = template->fs; 306 int size = sizeof(*q) + fs->sched->fp->q_datalen; 307 308 q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO); 309 if (q == NULL) { 310 D("no memory for new queue"); 311 return NULL; 312 } 313 314 set_oid(&q->ni.oid, DN_QUEUE, size); 315 if (fs->fs.flags & DN_QHT_HASH) 316 q->ni.fid = *(struct ipfw_flow_id *)key; 317 q->fs = fs; 318 q->_si = template->_si; 319 q->_si->q_count++; 320 321 if (fs->sched->fp->new_queue) 322 fs->sched->fp->new_queue(q); 323 dn_cfg.queue_count++; 324 return q; 325 } 326 327 /* 328 * Notify schedulers that a queue is going away. 329 * If (flags & DN_DESTROY), also free the packets. 330 * The version for callbacks is called q_delete_cb(). 331 */ 332 static void 333 dn_delete_queue(struct dn_queue *q, int flags) 334 { 335 struct dn_fsk *fs = q->fs; 336 337 // D("fs %p si %p\n", fs, q->_si); 338 /* notify the parent scheduler that the queue is going away */ 339 if (fs && fs->sched->fp->free_queue) 340 fs->sched->fp->free_queue(q); 341 q->_si->q_count--; 342 q->_si = NULL; 343 if (flags & DN_DESTROY) { 344 if (q->mq.head) 345 dn_free_pkts(q->mq.head); 346 bzero(q, sizeof(*q)); // safety 347 free(q, M_DUMMYNET); 348 dn_cfg.queue_count--; 349 } 350 } 351 352 static int 353 q_delete_cb(void *q, void *arg) 354 { 355 int flags = (int)(uintptr_t)arg; 356 dn_delete_queue(q, flags); 357 return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0; 358 } 359 360 /* 361 * calls dn_delete_queue/q_delete_cb on all queues, 362 * which notifies the parent scheduler and possibly drains packets. 363 * flags & DN_DESTROY: drains queues and destroy qht; 364 */ 365 static void 366 qht_delete(struct dn_fsk *fs, int flags) 367 { 368 ND("fs %d start flags %d qht %p", 369 fs->fs.fs_nr, flags, fs->qht); 370 if (!fs->qht) 371 return; 372 if (fs->fs.flags & DN_QHT_HASH) { 373 dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags); 374 if (flags & DN_DESTROY) { 375 dn_ht_free(fs->qht, 0); 376 fs->qht = NULL; 377 } 378 } else { 379 dn_delete_queue((struct dn_queue *)(fs->qht), flags); 380 if (flags & DN_DESTROY) 381 fs->qht = NULL; 382 } 383 } 384 385 /* 386 * Find and possibly create the queue for a MULTIQUEUE scheduler. 387 * We never call it for !MULTIQUEUE (the queue is in the sch_inst). 388 */ 389 struct dn_queue * 390 ipdn_q_find(struct dn_fsk *fs, struct dn_sch_inst *si, 391 struct ipfw_flow_id *id) 392 { 393 struct dn_queue template; 394 395 template._si = si; 396 template.fs = fs; 397 398 if (fs->fs.flags & DN_QHT_HASH) { 399 struct ipfw_flow_id masked_id; 400 if (fs->qht == NULL) { 401 fs->qht = dn_ht_init(NULL, fs->fs.buckets, 402 offsetof(struct dn_queue, q_next), 403 q_hash, q_match, q_new); 404 if (fs->qht == NULL) 405 return NULL; 406 } 407 masked_id = *id; 408 flow_id_mask(&fs->fsk_mask, &masked_id); 409 return dn_ht_find(fs->qht, (uintptr_t)&masked_id, 410 DNHT_INSERT, &template); 411 } else { 412 if (fs->qht == NULL) 413 fs->qht = q_new(0, 0, &template); 414 return (struct dn_queue *)fs->qht; 415 } 416 } 417 /*--- end of queue hash table ---*/ 418 419 /*--- support functions for the sch_inst hashtable ---- 420 * 421 * These are hashed by flow-id 422 */ 423 static uint32_t 424 si_hash(uintptr_t key, int flags, void *arg) 425 { 426 /* compute the hash slot from the flow id */ 427 struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ? 428 &((struct dn_sch_inst *)key)->ni.fid : 429 (struct ipfw_flow_id *)key; 430 431 return flow_id_hash(id); 432 } 433 434 static int 435 si_match(void *obj, uintptr_t key, int flags, void *arg) 436 { 437 struct dn_sch_inst *o = obj; 438 struct ipfw_flow_id *id2; 439 440 id2 = (flags & DNHT_KEY_IS_OBJ) ? 441 &((struct dn_sch_inst *)key)->ni.fid : 442 (struct ipfw_flow_id *)key; 443 return flow_id_cmp(&o->ni.fid, id2) == 0; 444 } 445 446 /* 447 * create a new instance for the given 'key' 448 * Allocate memory for instance, delay line and scheduler private data. 449 */ 450 static void * 451 si_new(uintptr_t key, int flags, void *arg) 452 { 453 struct dn_schk *s = arg; 454 struct dn_sch_inst *si; 455 int l = sizeof(*si) + s->fp->si_datalen; 456 457 si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 458 if (si == NULL) 459 goto error; 460 461 /* Set length only for the part passed up to userland. */ 462 set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow)); 463 set_oid(&(si->dline.oid), DN_DELAY_LINE, 464 sizeof(struct delay_line)); 465 /* mark si and dline as outside the event queue */ 466 si->ni.oid.id = si->dline.oid.id = -1; 467 468 si->sched = s; 469 si->dline.si = si; 470 471 if (s->fp->new_sched && s->fp->new_sched(si)) { 472 D("new_sched error"); 473 goto error; 474 } 475 if (s->sch.flags & DN_HAVE_MASK) 476 si->ni.fid = *(struct ipfw_flow_id *)key; 477 478 dn_cfg.si_count++; 479 return si; 480 481 error: 482 if (si) { 483 bzero(si, sizeof(*si)); // safety 484 free(si, M_DUMMYNET); 485 } 486 return NULL; 487 } 488 489 /* 490 * Callback from siht to delete all scheduler instances. Remove 491 * si and delay line from the system heap, destroy all queues. 492 * We assume that all flowset have been notified and do not 493 * point to us anymore. 494 */ 495 static int 496 si_destroy(void *_si, void *arg) 497 { 498 struct dn_sch_inst *si = _si; 499 struct dn_schk *s = si->sched; 500 struct delay_line *dl = &si->dline; 501 502 if (dl->oid.subtype) /* remove delay line from event heap */ 503 heap_extract(&dn_cfg.evheap, dl); 504 dn_free_pkts(dl->mq.head); /* drain delay line */ 505 if (si->kflags & DN_ACTIVE) /* remove si from event heap */ 506 heap_extract(&dn_cfg.evheap, si); 507 if (s->fp->free_sched) 508 s->fp->free_sched(si); 509 bzero(si, sizeof(*si)); /* safety */ 510 free(si, M_DUMMYNET); 511 dn_cfg.si_count--; 512 return DNHT_SCAN_DEL; 513 } 514 515 /* 516 * Find the scheduler instance for this packet. If we need to apply 517 * a mask, do on a local copy of the flow_id to preserve the original. 518 * Assume siht is always initialized if we have a mask. 519 */ 520 struct dn_sch_inst * 521 ipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id) 522 { 523 524 if (s->sch.flags & DN_HAVE_MASK) { 525 struct ipfw_flow_id id_t = *id; 526 flow_id_mask(&s->sch.sched_mask, &id_t); 527 return dn_ht_find(s->siht, (uintptr_t)&id_t, 528 DNHT_INSERT, s); 529 } 530 if (!s->siht) 531 s->siht = si_new(0, 0, s); 532 return (struct dn_sch_inst *)s->siht; 533 } 534 535 /* callback to flush credit for the scheduler instance */ 536 static int 537 si_reset_credit(void *_si, void *arg) 538 { 539 struct dn_sch_inst *si = _si; 540 struct dn_link *p = &si->sched->link; 541 542 si->credit = p->burst + (dn_cfg.io_fast ? p->bandwidth : 0); 543 return 0; 544 } 545 546 static void 547 schk_reset_credit(struct dn_schk *s) 548 { 549 if (s->sch.flags & DN_HAVE_MASK) 550 dn_ht_scan(s->siht, si_reset_credit, NULL); 551 else if (s->siht) 552 si_reset_credit(s->siht, NULL); 553 } 554 /*---- end of sch_inst hashtable ---------------------*/ 555 556 /*------------------------------------------------------- 557 * flowset hash (fshash) support. Entries are hashed by fs_nr. 558 * New allocations are put in the fsunlinked list, from which 559 * they are removed when they point to a specific scheduler. 560 */ 561 static uint32_t 562 fsk_hash(uintptr_t key, int flags, void *arg) 563 { 564 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 565 ((struct dn_fsk *)key)->fs.fs_nr; 566 567 return ( (i>>8)^(i>>4)^i ); 568 } 569 570 static int 571 fsk_match(void *obj, uintptr_t key, int flags, void *arg) 572 { 573 struct dn_fsk *fs = obj; 574 int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 575 ((struct dn_fsk *)key)->fs.fs_nr; 576 577 return (fs->fs.fs_nr == i); 578 } 579 580 static void * 581 fsk_new(uintptr_t key, int flags, void *arg) 582 { 583 struct dn_fsk *fs; 584 585 fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO); 586 if (fs) { 587 set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs)); 588 dn_cfg.fsk_count++; 589 fs->drain_bucket = 0; 590 SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); 591 } 592 return fs; 593 } 594 595 /* 596 * detach flowset from its current scheduler. Flags as follows: 597 * DN_DETACH removes from the fsk_list 598 * DN_DESTROY deletes individual queues 599 * DN_DELETE_FS destroys the flowset (otherwise goes in unlinked). 600 */ 601 static void 602 fsk_detach(struct dn_fsk *fs, int flags) 603 { 604 if (flags & DN_DELETE_FS) 605 flags |= DN_DESTROY; 606 ND("fs %d from sched %d flags %s %s %s", 607 fs->fs.fs_nr, fs->fs.sched_nr, 608 (flags & DN_DELETE_FS) ? "DEL_FS":"", 609 (flags & DN_DESTROY) ? "DEL":"", 610 (flags & DN_DETACH) ? "DET":""); 611 if (flags & DN_DETACH) { /* detach from the list */ 612 struct dn_fsk_head *h; 613 h = fs->sched ? &fs->sched->fsk_list : &dn_cfg.fsu; 614 SLIST_REMOVE(h, fs, dn_fsk, sch_chain); 615 } 616 /* Free the RED parameters, they will be recomputed on 617 * subsequent attach if needed. 618 */ 619 if (fs->w_q_lookup) 620 free(fs->w_q_lookup, M_DUMMYNET); 621 fs->w_q_lookup = NULL; 622 qht_delete(fs, flags); 623 if (fs->sched && fs->sched->fp->free_fsk) 624 fs->sched->fp->free_fsk(fs); 625 fs->sched = NULL; 626 if (flags & DN_DELETE_FS) { 627 bzero(fs, sizeof(*fs)); /* safety */ 628 free(fs, M_DUMMYNET); 629 dn_cfg.fsk_count--; 630 } else { 631 SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain); 632 } 633 } 634 635 /* 636 * Detach or destroy all flowsets in a list. 637 * flags specifies what to do: 638 * DN_DESTROY: flush all queues 639 * DN_DELETE_FS: DN_DESTROY + destroy flowset 640 * DN_DELETE_FS implies DN_DESTROY 641 */ 642 static void 643 fsk_detach_list(struct dn_fsk_head *h, int flags) 644 { 645 struct dn_fsk *fs; 646 int n = 0; /* only for stats */ 647 648 ND("head %p flags %x", h, flags); 649 while ((fs = SLIST_FIRST(h))) { 650 SLIST_REMOVE_HEAD(h, sch_chain); 651 n++; 652 fsk_detach(fs, flags); 653 } 654 ND("done %d flowsets", n); 655 } 656 657 /* 658 * called on 'queue X delete' -- removes the flowset from fshash, 659 * deletes all queues for the flowset, and removes the flowset. 660 */ 661 static int 662 delete_fs(int i, int locked) 663 { 664 struct dn_fsk *fs; 665 int err = 0; 666 667 if (!locked) 668 DN_BH_WLOCK(); 669 fs = dn_ht_find(dn_cfg.fshash, i, DNHT_REMOVE, NULL); 670 ND("fs %d found %p", i, fs); 671 if (fs) { 672 fsk_detach(fs, DN_DETACH | DN_DELETE_FS); 673 err = 0; 674 } else 675 err = EINVAL; 676 if (!locked) 677 DN_BH_WUNLOCK(); 678 return err; 679 } 680 681 /*----- end of flowset hashtable support -------------*/ 682 683 /*------------------------------------------------------------ 684 * Scheduler hash. When searching by index we pass sched_nr, 685 * otherwise we pass struct dn_sch * which is the first field in 686 * struct dn_schk so we can cast between the two. We use this trick 687 * because in the create phase (but it should be fixed). 688 */ 689 static uint32_t 690 schk_hash(uintptr_t key, int flags, void *_arg) 691 { 692 uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key : 693 ((struct dn_schk *)key)->sch.sched_nr; 694 return ( (i>>8)^(i>>4)^i ); 695 } 696 697 static int 698 schk_match(void *obj, uintptr_t key, int flags, void *_arg) 699 { 700 struct dn_schk *s = (struct dn_schk *)obj; 701 int i = !(flags & DNHT_KEY_IS_OBJ) ? key : 702 ((struct dn_schk *)key)->sch.sched_nr; 703 return (s->sch.sched_nr == i); 704 } 705 706 /* 707 * Create the entry and intialize with the sched hash if needed. 708 * Leave s->fp unset so we can tell whether a dn_ht_find() returns 709 * a new object or a previously existing one. 710 */ 711 static void * 712 schk_new(uintptr_t key, int flags, void *arg) 713 { 714 struct schk_new_arg *a = arg; 715 struct dn_schk *s; 716 int l = sizeof(*s) +a->fp->schk_datalen; 717 718 s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO); 719 if (s == NULL) 720 return NULL; 721 set_oid(&s->link.oid, DN_LINK, sizeof(s->link)); 722 s->sch = *a->sch; // copy initial values 723 s->link.link_nr = s->sch.sched_nr; 724 SLIST_INIT(&s->fsk_list); 725 /* initialize the hash table or create the single instance */ 726 s->fp = a->fp; /* si_new needs this */ 727 s->drain_bucket = 0; 728 if (s->sch.flags & DN_HAVE_MASK) { 729 s->siht = dn_ht_init(NULL, s->sch.buckets, 730 offsetof(struct dn_sch_inst, si_next), 731 si_hash, si_match, si_new); 732 if (s->siht == NULL) { 733 free(s, M_DUMMYNET); 734 return NULL; 735 } 736 } 737 s->fp = NULL; /* mark as a new scheduler */ 738 dn_cfg.schk_count++; 739 return s; 740 } 741 742 /* 743 * Callback for sched delete. Notify all attached flowsets to 744 * detach from the scheduler, destroy the internal flowset, and 745 * all instances. The scheduler goes away too. 746 * arg is 0 (only detach flowsets and destroy instances) 747 * DN_DESTROY (detach & delete queues, delete schk) 748 * or DN_DELETE_FS (delete queues and flowsets, delete schk) 749 */ 750 static int 751 schk_delete_cb(void *obj, void *arg) 752 { 753 struct dn_schk *s = obj; 754 #if 0 755 int a = (int)arg; 756 ND("sched %d arg %s%s", 757 s->sch.sched_nr, 758 a&DN_DESTROY ? "DEL ":"", 759 a&DN_DELETE_FS ? "DEL_FS":""); 760 #endif 761 fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0); 762 /* no more flowset pointing to us now */ 763 if (s->sch.flags & DN_HAVE_MASK) { 764 dn_ht_scan(s->siht, si_destroy, NULL); 765 dn_ht_free(s->siht, 0); 766 } else if (s->siht) 767 si_destroy(s->siht, NULL); 768 if (s->profile) { 769 free(s->profile, M_DUMMYNET); 770 s->profile = NULL; 771 } 772 s->siht = NULL; 773 if (s->fp->destroy) 774 s->fp->destroy(s); 775 bzero(s, sizeof(*s)); // safety 776 free(obj, M_DUMMYNET); 777 dn_cfg.schk_count--; 778 return DNHT_SCAN_DEL; 779 } 780 781 /* 782 * called on a 'sched X delete' command. Deletes a single scheduler. 783 * This is done by removing from the schedhash, unlinking all 784 * flowsets and deleting their traffic. 785 */ 786 static int 787 delete_schk(int i) 788 { 789 struct dn_schk *s; 790 791 s = dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 792 ND("%d %p", i, s); 793 if (!s) 794 return EINVAL; 795 delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */ 796 /* then detach flowsets, delete traffic */ 797 schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY); 798 return 0; 799 } 800 /*--- end of schk hashtable support ---*/ 801 802 static int 803 copy_obj(char **start, char *end, void *_o, const char *msg, int i) 804 { 805 struct dn_id *o = _o; 806 int have = end - *start; 807 808 if (have < o->len || o->len == 0 || o->type == 0) { 809 D("(WARN) type %d %s %d have %d need %d", 810 o->type, msg, i, have, o->len); 811 return 1; 812 } 813 ND("type %d %s %d len %d", o->type, msg, i, o->len); 814 bcopy(_o, *start, o->len); 815 if (o->type == DN_LINK) { 816 /* Adjust burst parameter for link */ 817 struct dn_link *l = (struct dn_link *)*start; 818 l->burst = div64(l->burst, 8 * hz); 819 l->delay = l->delay * 1000 / hz; 820 } else if (o->type == DN_SCH) { 821 /* Set id->id to the number of instances */ 822 struct dn_schk *s = _o; 823 struct dn_id *id = (struct dn_id *)(*start); 824 id->id = (s->sch.flags & DN_HAVE_MASK) ? 825 dn_ht_entries(s->siht) : (s->siht ? 1 : 0); 826 } 827 *start += o->len; 828 return 0; 829 } 830 831 /* Specific function to copy a queue. 832 * Copies only the user-visible part of a queue (which is in 833 * a struct dn_flow), and sets len accordingly. 834 */ 835 static int 836 copy_obj_q(char **start, char *end, void *_o, const char *msg, int i) 837 { 838 struct dn_id *o = _o; 839 int have = end - *start; 840 int len = sizeof(struct dn_flow); /* see above comment */ 841 842 if (have < len || o->len == 0 || o->type != DN_QUEUE) { 843 D("ERROR type %d %s %d have %d need %d", 844 o->type, msg, i, have, len); 845 return 1; 846 } 847 ND("type %d %s %d len %d", o->type, msg, i, len); 848 bcopy(_o, *start, len); 849 ((struct dn_id*)(*start))->len = len; 850 *start += len; 851 return 0; 852 } 853 854 static int 855 copy_q_cb(void *obj, void *arg) 856 { 857 struct dn_queue *q = obj; 858 struct copy_args *a = arg; 859 struct dn_flow *ni = (struct dn_flow *)(*a->start); 860 if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1)) 861 return DNHT_SCAN_END; 862 ni->oid.type = DN_FLOW; /* override the DN_QUEUE */ 863 ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL); 864 return 0; 865 } 866 867 static int 868 copy_q(struct copy_args *a, struct dn_fsk *fs, int flags) 869 { 870 if (!fs->qht) 871 return 0; 872 if (fs->fs.flags & DN_QHT_HASH) 873 dn_ht_scan(fs->qht, copy_q_cb, a); 874 else 875 copy_q_cb(fs->qht, a); 876 return 0; 877 } 878 879 /* 880 * This routine only copies the initial part of a profile ? XXX 881 */ 882 static int 883 copy_profile(struct copy_args *a, struct dn_profile *p) 884 { 885 int have = a->end - *a->start; 886 /* XXX here we check for max length */ 887 int profile_len = sizeof(struct dn_profile) - 888 ED_MAX_SAMPLES_NO*sizeof(int); 889 890 if (p == NULL) 891 return 0; 892 if (have < profile_len) { 893 D("error have %d need %d", have, profile_len); 894 return 1; 895 } 896 bcopy(p, *a->start, profile_len); 897 ((struct dn_id *)(*a->start))->len = profile_len; 898 *a->start += profile_len; 899 return 0; 900 } 901 902 static int 903 copy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags) 904 { 905 struct dn_fs *ufs = (struct dn_fs *)(*a->start); 906 if (!fs) 907 return 0; 908 ND("flowset %d", fs->fs.fs_nr); 909 if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr)) 910 return DNHT_SCAN_END; 911 ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ? 912 dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0); 913 if (flags) { /* copy queues */ 914 copy_q(a, fs, 0); 915 } 916 return 0; 917 } 918 919 static int 920 copy_si_cb(void *obj, void *arg) 921 { 922 struct dn_sch_inst *si = obj; 923 struct copy_args *a = arg; 924 struct dn_flow *ni = (struct dn_flow *)(*a->start); 925 if (copy_obj(a->start, a->end, &si->ni, "inst", 926 si->sched->sch.sched_nr)) 927 return DNHT_SCAN_END; 928 ni->oid.type = DN_FLOW; /* override the DN_SCH_I */ 929 ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL); 930 return 0; 931 } 932 933 static int 934 copy_si(struct copy_args *a, struct dn_schk *s, int flags) 935 { 936 if (s->sch.flags & DN_HAVE_MASK) 937 dn_ht_scan(s->siht, copy_si_cb, a); 938 else if (s->siht) 939 copy_si_cb(s->siht, a); 940 return 0; 941 } 942 943 /* 944 * compute a list of children of a scheduler and copy up 945 */ 946 static int 947 copy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags) 948 { 949 struct dn_fsk *fs; 950 struct dn_id *o; 951 uint32_t *p; 952 953 int n = 0, space = sizeof(*o); 954 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 955 if (fs->fs.fs_nr < DN_MAX_ID) 956 n++; 957 } 958 space += n * sizeof(uint32_t); 959 DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n); 960 if (a->end - *(a->start) < space) 961 return DNHT_SCAN_END; 962 o = (struct dn_id *)(*(a->start)); 963 o->len = space; 964 *a->start += o->len; 965 o->type = DN_TEXT; 966 p = (uint32_t *)(o+1); 967 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) 968 if (fs->fs.fs_nr < DN_MAX_ID) 969 *p++ = fs->fs.fs_nr; 970 return 0; 971 } 972 973 static int 974 copy_data_helper(void *_o, void *_arg) 975 { 976 struct copy_args *a = _arg; 977 uint32_t *r = a->extra->r; /* start of first range */ 978 uint32_t *lim; /* first invalid pointer */ 979 int n; 980 981 lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len); 982 983 if (a->type == DN_LINK || a->type == DN_SCH) { 984 /* pipe|sched show, we receive a dn_schk */ 985 struct dn_schk *s = _o; 986 987 n = s->sch.sched_nr; 988 if (a->type == DN_SCH && n >= DN_MAX_ID) 989 return 0; /* not a scheduler */ 990 if (a->type == DN_LINK && n <= DN_MAX_ID) 991 return 0; /* not a pipe */ 992 993 /* see if the object is within one of our ranges */ 994 for (;r < lim; r += 2) { 995 if (n < r[0] || n > r[1]) 996 continue; 997 /* Found a valid entry, copy and we are done */ 998 if (a->flags & DN_C_LINK) { 999 if (copy_obj(a->start, a->end, 1000 &s->link, "link", n)) 1001 return DNHT_SCAN_END; 1002 if (copy_profile(a, s->profile)) 1003 return DNHT_SCAN_END; 1004 if (copy_flowset(a, s->fs, 0)) 1005 return DNHT_SCAN_END; 1006 } 1007 if (a->flags & DN_C_SCH) { 1008 if (copy_obj(a->start, a->end, 1009 &s->sch, "sched", n)) 1010 return DNHT_SCAN_END; 1011 /* list all attached flowsets */ 1012 if (copy_fsk_list(a, s, 0)) 1013 return DNHT_SCAN_END; 1014 } 1015 if (a->flags & DN_C_FLOW) 1016 copy_si(a, s, 0); 1017 break; 1018 } 1019 } else if (a->type == DN_FS) { 1020 /* queue show, skip internal flowsets */ 1021 struct dn_fsk *fs = _o; 1022 1023 n = fs->fs.fs_nr; 1024 if (n >= DN_MAX_ID) 1025 return 0; 1026 /* see if the object is within one of our ranges */ 1027 for (;r < lim; r += 2) { 1028 if (n < r[0] || n > r[1]) 1029 continue; 1030 if (copy_flowset(a, fs, 0)) 1031 return DNHT_SCAN_END; 1032 copy_q(a, fs, 0); 1033 break; /* we are done */ 1034 } 1035 } 1036 return 0; 1037 } 1038 1039 static inline struct dn_schk * 1040 locate_scheduler(int i) 1041 { 1042 return dn_ht_find(dn_cfg.schedhash, i, 0, NULL); 1043 } 1044 1045 /* 1046 * red parameters are in fixed point arithmetic. 1047 */ 1048 static int 1049 config_red(struct dn_fsk *fs) 1050 { 1051 int64_t s, idle, weight, w0; 1052 int t, i; 1053 1054 fs->w_q = fs->fs.w_q; 1055 fs->max_p = fs->fs.max_p; 1056 ND("called"); 1057 /* Doing stuff that was in userland */ 1058 i = fs->sched->link.bandwidth; 1059 s = (i <= 0) ? 0 : 1060 hz * dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i; 1061 1062 idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */ 1063 fs->lookup_step = div64(idle , dn_cfg.red_lookup_depth); 1064 /* fs->lookup_step not scaled, */ 1065 if (!fs->lookup_step) 1066 fs->lookup_step = 1; 1067 w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled 1068 1069 for (t = fs->lookup_step; t > 1; --t) 1070 weight = SCALE_MUL(weight, w0); 1071 fs->lookup_weight = (int)(weight); // scaled 1072 1073 /* Now doing stuff that was in kerneland */ 1074 fs->min_th = SCALE(fs->fs.min_th); 1075 fs->max_th = SCALE(fs->fs.max_th); 1076 1077 if (fs->fs.max_th == fs->fs.min_th) 1078 fs->c_1 = fs->max_p; 1079 else 1080 fs->c_1 = SCALE((int64_t)(fs->max_p)) / (fs->fs.max_th - fs->fs.min_th); 1081 fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th)); 1082 1083 if (fs->fs.flags & DN_IS_GENTLE_RED) { 1084 fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th; 1085 fs->c_4 = SCALE(1) - 2 * fs->max_p; 1086 } 1087 1088 /* If the lookup table already exist, free and create it again. */ 1089 if (fs->w_q_lookup) { 1090 free(fs->w_q_lookup, M_DUMMYNET); 1091 fs->w_q_lookup = NULL; 1092 } 1093 if (dn_cfg.red_lookup_depth == 0) { 1094 printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth" 1095 "must be > 0\n"); 1096 fs->fs.flags &= ~DN_IS_RED; 1097 fs->fs.flags &= ~DN_IS_GENTLE_RED; 1098 return (EINVAL); 1099 } 1100 fs->lookup_depth = dn_cfg.red_lookup_depth; 1101 fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int), 1102 M_DUMMYNET, M_NOWAIT); 1103 if (fs->w_q_lookup == NULL) { 1104 printf("dummynet: sorry, cannot allocate red lookup table\n"); 1105 fs->fs.flags &= ~DN_IS_RED; 1106 fs->fs.flags &= ~DN_IS_GENTLE_RED; 1107 return(ENOSPC); 1108 } 1109 1110 /* Fill the lookup table with (1 - w_q)^x */ 1111 fs->w_q_lookup[0] = SCALE(1) - fs->w_q; 1112 1113 for (i = 1; i < fs->lookup_depth; i++) 1114 fs->w_q_lookup[i] = 1115 SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight); 1116 1117 if (dn_cfg.red_avg_pkt_size < 1) 1118 dn_cfg.red_avg_pkt_size = 512; 1119 fs->avg_pkt_size = dn_cfg.red_avg_pkt_size; 1120 if (dn_cfg.red_max_pkt_size < 1) 1121 dn_cfg.red_max_pkt_size = 1500; 1122 fs->max_pkt_size = dn_cfg.red_max_pkt_size; 1123 ND("exit"); 1124 return 0; 1125 } 1126 1127 /* Scan all flowset attached to this scheduler and update red */ 1128 static void 1129 update_red(struct dn_schk *s) 1130 { 1131 struct dn_fsk *fs; 1132 SLIST_FOREACH(fs, &s->fsk_list, sch_chain) { 1133 if (fs && (fs->fs.flags & DN_IS_RED)) 1134 config_red(fs); 1135 } 1136 } 1137 1138 /* attach flowset to scheduler s, possibly requeue */ 1139 static void 1140 fsk_attach(struct dn_fsk *fs, struct dn_schk *s) 1141 { 1142 ND("remove fs %d from fsunlinked, link to sched %d", 1143 fs->fs.fs_nr, s->sch.sched_nr); 1144 SLIST_REMOVE(&dn_cfg.fsu, fs, dn_fsk, sch_chain); 1145 fs->sched = s; 1146 SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain); 1147 if (s->fp->new_fsk) 1148 s->fp->new_fsk(fs); 1149 /* XXX compute fsk_mask */ 1150 fs->fsk_mask = fs->fs.flow_mask; 1151 if (fs->sched->sch.flags & DN_HAVE_MASK) 1152 flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask); 1153 if (fs->qht) { 1154 /* 1155 * we must drain qht according to the old 1156 * type, and reinsert according to the new one. 1157 * The requeue is complex -- in general we need to 1158 * reclassify every single packet. 1159 * For the time being, let's hope qht is never set 1160 * when we reach this point. 1161 */ 1162 D("XXX TODO requeue from fs %d to sch %d", 1163 fs->fs.fs_nr, s->sch.sched_nr); 1164 fs->qht = NULL; 1165 } 1166 /* set the new type for qht */ 1167 if (nonzero_mask(&fs->fsk_mask)) 1168 fs->fs.flags |= DN_QHT_HASH; 1169 else 1170 fs->fs.flags &= ~DN_QHT_HASH; 1171 1172 /* XXX config_red() can fail... */ 1173 if (fs->fs.flags & DN_IS_RED) 1174 config_red(fs); 1175 } 1176 1177 /* update all flowsets which may refer to this scheduler */ 1178 static void 1179 update_fs(struct dn_schk *s) 1180 { 1181 struct dn_fsk *fs, *tmp; 1182 1183 SLIST_FOREACH_SAFE(fs, &dn_cfg.fsu, sch_chain, tmp) { 1184 if (s->sch.sched_nr != fs->fs.sched_nr) { 1185 D("fs %d for sch %d not %d still unlinked", 1186 fs->fs.fs_nr, fs->fs.sched_nr, 1187 s->sch.sched_nr); 1188 continue; 1189 } 1190 fsk_attach(fs, s); 1191 } 1192 } 1193 1194 /* 1195 * Configuration -- to preserve backward compatibility we use 1196 * the following scheme (N is 65536) 1197 * NUMBER SCHED LINK FLOWSET 1198 * 1 .. N-1 (1)WFQ (2)WFQ (3)queue 1199 * N+1 .. 2N-1 (4)FIFO (5)FIFO (6)FIFO for sched 1..N-1 1200 * 2N+1 .. 3N-1 -- -- (7)FIFO for sched N+1..2N-1 1201 * 1202 * "pipe i config" configures #1, #2 and #3 1203 * "sched i config" configures #1 and possibly #6 1204 * "queue i config" configures #3 1205 * #1 is configured with 'pipe i config' or 'sched i config' 1206 * #2 is configured with 'pipe i config', and created if not 1207 * existing with 'sched i config' 1208 * #3 is configured with 'queue i config' 1209 * #4 is automatically configured after #1, can only be FIFO 1210 * #5 is automatically configured after #2 1211 * #6 is automatically created when #1 is !MULTIQUEUE, 1212 * and can be updated. 1213 * #7 is automatically configured after #2 1214 */ 1215 1216 /* 1217 * configure a link (and its FIFO instance) 1218 */ 1219 static int 1220 config_link(struct dn_link *p, struct dn_id *arg) 1221 { 1222 int i; 1223 1224 if (p->oid.len != sizeof(*p)) { 1225 D("invalid pipe len %d", p->oid.len); 1226 return EINVAL; 1227 } 1228 i = p->link_nr; 1229 if (i <= 0 || i >= DN_MAX_ID) 1230 return EINVAL; 1231 /* 1232 * The config program passes parameters as follows: 1233 * bw = bits/second (0 means no limits), 1234 * delay = ms, must be translated into ticks. 1235 * qsize = slots/bytes 1236 * burst ??? 1237 */ 1238 p->delay = (p->delay * hz) / 1000; 1239 /* Scale burst size: bytes -> bits * hz */ 1240 p->burst *= 8 * hz; 1241 1242 DN_BH_WLOCK(); 1243 /* do it twice, base link and FIFO link */ 1244 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1245 struct dn_schk *s = locate_scheduler(i); 1246 if (s == NULL) { 1247 DN_BH_WUNLOCK(); 1248 D("sched %d not found", i); 1249 return EINVAL; 1250 } 1251 /* remove profile if exists */ 1252 if (s->profile) { 1253 free(s->profile, M_DUMMYNET); 1254 s->profile = NULL; 1255 } 1256 /* copy all parameters */ 1257 s->link.oid = p->oid; 1258 s->link.link_nr = i; 1259 s->link.delay = p->delay; 1260 if (s->link.bandwidth != p->bandwidth) { 1261 /* XXX bandwidth changes, need to update red params */ 1262 s->link.bandwidth = p->bandwidth; 1263 update_red(s); 1264 } 1265 s->link.burst = p->burst; 1266 schk_reset_credit(s); 1267 } 1268 dn_cfg.id++; 1269 DN_BH_WUNLOCK(); 1270 return 0; 1271 } 1272 1273 /* 1274 * configure a flowset. Can be called from inside with locked=1, 1275 */ 1276 static struct dn_fsk * 1277 config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked) 1278 { 1279 int i; 1280 struct dn_fsk *fs; 1281 1282 if (nfs->oid.len != sizeof(*nfs)) { 1283 D("invalid flowset len %d", nfs->oid.len); 1284 return NULL; 1285 } 1286 i = nfs->fs_nr; 1287 if (i <= 0 || i >= 3*DN_MAX_ID) 1288 return NULL; 1289 ND("flowset %d", i); 1290 /* XXX other sanity checks */ 1291 if (nfs->flags & DN_QSIZE_BYTES) { 1292 ipdn_bound_var(&nfs->qsize, 16384, 1293 1500, dn_cfg.byte_limit, NULL); // "queue byte size"); 1294 } else { 1295 ipdn_bound_var(&nfs->qsize, 50, 1296 1, dn_cfg.slot_limit, NULL); // "queue slot size"); 1297 } 1298 if (nfs->flags & DN_HAVE_MASK) { 1299 /* make sure we have some buckets */ 1300 ipdn_bound_var((int *)&nfs->buckets, dn_cfg.hash_size, 1301 1, dn_cfg.max_hash_size, "flowset buckets"); 1302 } else { 1303 nfs->buckets = 1; /* we only need 1 */ 1304 } 1305 if (!locked) 1306 DN_BH_WLOCK(); 1307 do { /* exit with break when done */ 1308 struct dn_schk *s; 1309 int flags = nfs->sched_nr ? DNHT_INSERT : 0; 1310 int j; 1311 int oldc = dn_cfg.fsk_count; 1312 fs = dn_ht_find(dn_cfg.fshash, i, flags, NULL); 1313 if (fs == NULL) { 1314 D("missing sched for flowset %d", i); 1315 break; 1316 } 1317 /* grab some defaults from the existing one */ 1318 if (nfs->sched_nr == 0) /* reuse */ 1319 nfs->sched_nr = fs->fs.sched_nr; 1320 for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) { 1321 if (nfs->par[j] == -1) /* reuse */ 1322 nfs->par[j] = fs->fs.par[j]; 1323 } 1324 if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) { 1325 ND("flowset %d unchanged", i); 1326 break; /* no change, nothing to do */ 1327 } 1328 if (oldc != dn_cfg.fsk_count) /* new item */ 1329 dn_cfg.id++; 1330 s = locate_scheduler(nfs->sched_nr); 1331 /* detach from old scheduler if needed, preserving 1332 * queues if we need to reattach. Then update the 1333 * configuration, and possibly attach to the new sched. 1334 */ 1335 DX(2, "fs %d changed sched %d@%p to %d@%p", 1336 fs->fs.fs_nr, 1337 fs->fs.sched_nr, fs->sched, nfs->sched_nr, s); 1338 if (fs->sched) { 1339 int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY); 1340 flags |= DN_DESTROY; /* XXX temporary */ 1341 fsk_detach(fs, flags); 1342 } 1343 fs->fs = *nfs; /* copy configuration */ 1344 if (s != NULL) 1345 fsk_attach(fs, s); 1346 } while (0); 1347 if (!locked) 1348 DN_BH_WUNLOCK(); 1349 return fs; 1350 } 1351 1352 /* 1353 * config/reconfig a scheduler and its FIFO variant. 1354 * For !MULTIQUEUE schedulers, also set up the flowset. 1355 * 1356 * On reconfigurations (detected because s->fp is set), 1357 * detach existing flowsets preserving traffic, preserve link, 1358 * and delete the old scheduler creating a new one. 1359 */ 1360 static int 1361 config_sched(struct dn_sch *_nsch, struct dn_id *arg) 1362 { 1363 struct dn_schk *s; 1364 struct schk_new_arg a; /* argument for schk_new */ 1365 int i; 1366 struct dn_link p; /* copy of oldlink */ 1367 struct dn_profile *pf = NULL; /* copy of old link profile */ 1368 /* Used to preserv mask parameter */ 1369 struct ipfw_flow_id new_mask; 1370 int new_buckets = 0; 1371 int new_flags = 0; 1372 int pipe_cmd; 1373 int err = ENOMEM; 1374 1375 a.sch = _nsch; 1376 if (a.sch->oid.len != sizeof(*a.sch)) { 1377 D("bad sched len %d", a.sch->oid.len); 1378 return EINVAL; 1379 } 1380 i = a.sch->sched_nr; 1381 if (i <= 0 || i >= DN_MAX_ID) 1382 return EINVAL; 1383 /* make sure we have some buckets */ 1384 if (a.sch->flags & DN_HAVE_MASK) 1385 ipdn_bound_var((int *)&a.sch->buckets, dn_cfg.hash_size, 1386 1, dn_cfg.max_hash_size, "sched buckets"); 1387 /* XXX other sanity checks */ 1388 bzero(&p, sizeof(p)); 1389 1390 pipe_cmd = a.sch->flags & DN_PIPE_CMD; 1391 a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set? 1392 if (pipe_cmd) { 1393 /* Copy mask parameter */ 1394 new_mask = a.sch->sched_mask; 1395 new_buckets = a.sch->buckets; 1396 new_flags = a.sch->flags; 1397 } 1398 DN_BH_WLOCK(); 1399 again: /* run twice, for wfq and fifo */ 1400 /* 1401 * lookup the type. If not supplied, use the previous one 1402 * or default to WF2Q+. Otherwise, return an error. 1403 */ 1404 dn_cfg.id++; 1405 a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name); 1406 if (a.fp != NULL) { 1407 /* found. Lookup or create entry */ 1408 s = dn_ht_find(dn_cfg.schedhash, i, DNHT_INSERT, &a); 1409 } else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) { 1410 /* No type. search existing s* or retry with WF2Q+ */ 1411 s = dn_ht_find(dn_cfg.schedhash, i, 0, &a); 1412 if (s != NULL) { 1413 a.fp = s->fp; 1414 /* Scheduler exists, skip to FIFO scheduler 1415 * if command was pipe config... 1416 */ 1417 if (pipe_cmd) 1418 goto next; 1419 } else { 1420 /* New scheduler, create a wf2q+ with no mask 1421 * if command was pipe config... 1422 */ 1423 if (pipe_cmd) { 1424 /* clear mask parameter */ 1425 bzero(&a.sch->sched_mask, sizeof(new_mask)); 1426 a.sch->buckets = 0; 1427 a.sch->flags &= ~DN_HAVE_MASK; 1428 } 1429 a.sch->oid.subtype = DN_SCHED_WF2QP; 1430 goto again; 1431 } 1432 } else { 1433 D("invalid scheduler type %d %s", 1434 a.sch->oid.subtype, a.sch->name); 1435 err = EINVAL; 1436 goto error; 1437 } 1438 /* normalize name and subtype */ 1439 a.sch->oid.subtype = a.fp->type; 1440 bzero(a.sch->name, sizeof(a.sch->name)); 1441 strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name)); 1442 if (s == NULL) { 1443 D("cannot allocate scheduler %d", i); 1444 goto error; 1445 } 1446 /* restore existing link if any */ 1447 if (p.link_nr) { 1448 s->link = p; 1449 if (!pf || pf->link_nr != p.link_nr) { /* no saved value */ 1450 s->profile = NULL; /* XXX maybe not needed */ 1451 } else { 1452 s->profile = malloc(sizeof(struct dn_profile), 1453 M_DUMMYNET, M_NOWAIT | M_ZERO); 1454 if (s->profile == NULL) { 1455 D("cannot allocate profile"); 1456 goto error; //XXX 1457 } 1458 bcopy(pf, s->profile, sizeof(*pf)); 1459 } 1460 } 1461 p.link_nr = 0; 1462 if (s->fp == NULL) { 1463 DX(2, "sched %d new type %s", i, a.fp->name); 1464 } else if (s->fp != a.fp || 1465 bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) { 1466 /* already existing. */ 1467 DX(2, "sched %d type changed from %s to %s", 1468 i, s->fp->name, a.fp->name); 1469 DX(4, " type/sub %d/%d -> %d/%d", 1470 s->sch.oid.type, s->sch.oid.subtype, 1471 a.sch->oid.type, a.sch->oid.subtype); 1472 if (s->link.link_nr == 0) 1473 D("XXX WARNING link 0 for sched %d", i); 1474 p = s->link; /* preserve link */ 1475 if (s->profile) {/* preserve profile */ 1476 if (!pf) 1477 pf = malloc(sizeof(*pf), 1478 M_DUMMYNET, M_NOWAIT | M_ZERO); 1479 if (pf) /* XXX should issue a warning otherwise */ 1480 bcopy(s->profile, pf, sizeof(*pf)); 1481 } 1482 /* remove from the hash */ 1483 dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL); 1484 /* Detach flowsets, preserve queues. */ 1485 // schk_delete_cb(s, NULL); 1486 // XXX temporarily, kill queues 1487 schk_delete_cb(s, (void *)DN_DESTROY); 1488 goto again; 1489 } else { 1490 DX(4, "sched %d unchanged type %s", i, a.fp->name); 1491 } 1492 /* complete initialization */ 1493 s->sch = *a.sch; 1494 s->fp = a.fp; 1495 s->cfg = arg; 1496 // XXX schk_reset_credit(s); 1497 /* create the internal flowset if needed, 1498 * trying to reuse existing ones if available 1499 */ 1500 if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) { 1501 s->fs = dn_ht_find(dn_cfg.fshash, i, 0, NULL); 1502 if (!s->fs) { 1503 struct dn_fs fs; 1504 bzero(&fs, sizeof(fs)); 1505 set_oid(&fs.oid, DN_FS, sizeof(fs)); 1506 fs.fs_nr = i + DN_MAX_ID; 1507 fs.sched_nr = i; 1508 s->fs = config_fs(&fs, NULL, 1 /* locked */); 1509 } 1510 if (!s->fs) { 1511 schk_delete_cb(s, (void *)DN_DESTROY); 1512 D("error creating internal fs for %d", i); 1513 goto error; 1514 } 1515 } 1516 /* call init function after the flowset is created */ 1517 if (s->fp->config) 1518 s->fp->config(s); 1519 update_fs(s); 1520 next: 1521 if (i < DN_MAX_ID) { /* now configure the FIFO instance */ 1522 i += DN_MAX_ID; 1523 if (pipe_cmd) { 1524 /* Restore mask parameter for FIFO */ 1525 a.sch->sched_mask = new_mask; 1526 a.sch->buckets = new_buckets; 1527 a.sch->flags = new_flags; 1528 } else { 1529 /* sched config shouldn't modify the FIFO scheduler */ 1530 if (dn_ht_find(dn_cfg.schedhash, i, 0, &a) != NULL) { 1531 /* FIFO already exist, don't touch it */ 1532 err = 0; /* and this is not an error */ 1533 goto error; 1534 } 1535 } 1536 a.sch->sched_nr = i; 1537 a.sch->oid.subtype = DN_SCHED_FIFO; 1538 bzero(a.sch->name, sizeof(a.sch->name)); 1539 goto again; 1540 } 1541 err = 0; 1542 error: 1543 DN_BH_WUNLOCK(); 1544 if (pf) 1545 free(pf, M_DUMMYNET); 1546 return err; 1547 } 1548 1549 /* 1550 * attach a profile to a link 1551 */ 1552 static int 1553 config_profile(struct dn_profile *pf, struct dn_id *arg) 1554 { 1555 struct dn_schk *s; 1556 int i, olen, err = 0; 1557 1558 if (pf->oid.len < sizeof(*pf)) { 1559 D("short profile len %d", pf->oid.len); 1560 return EINVAL; 1561 } 1562 i = pf->link_nr; 1563 if (i <= 0 || i >= DN_MAX_ID) 1564 return EINVAL; 1565 /* XXX other sanity checks */ 1566 DN_BH_WLOCK(); 1567 for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) { 1568 s = locate_scheduler(i); 1569 1570 if (s == NULL) { 1571 err = EINVAL; 1572 break; 1573 } 1574 dn_cfg.id++; 1575 /* 1576 * If we had a profile and the new one does not fit, 1577 * or it is deleted, then we need to free memory. 1578 */ 1579 if (s->profile && (pf->samples_no == 0 || 1580 s->profile->oid.len < pf->oid.len)) { 1581 free(s->profile, M_DUMMYNET); 1582 s->profile = NULL; 1583 } 1584 if (pf->samples_no == 0) 1585 continue; 1586 /* 1587 * new profile, possibly allocate memory 1588 * and copy data. 1589 */ 1590 if (s->profile == NULL) 1591 s->profile = malloc(pf->oid.len, 1592 M_DUMMYNET, M_NOWAIT | M_ZERO); 1593 if (s->profile == NULL) { 1594 D("no memory for profile %d", i); 1595 err = ENOMEM; 1596 break; 1597 } 1598 /* preserve larger length XXX double check */ 1599 olen = s->profile->oid.len; 1600 if (olen < pf->oid.len) 1601 olen = pf->oid.len; 1602 bcopy(pf, s->profile, pf->oid.len); 1603 s->profile->oid.len = olen; 1604 } 1605 DN_BH_WUNLOCK(); 1606 return err; 1607 } 1608 1609 /* 1610 * Delete all objects: 1611 */ 1612 static void 1613 dummynet_flush(void) 1614 { 1615 1616 /* delete all schedulers and related links/queues/flowsets */ 1617 dn_ht_scan(dn_cfg.schedhash, schk_delete_cb, 1618 (void *)(uintptr_t)DN_DELETE_FS); 1619 /* delete all remaining (unlinked) flowsets */ 1620 DX(4, "still %d unlinked fs", dn_cfg.fsk_count); 1621 dn_ht_free(dn_cfg.fshash, DNHT_REMOVE); 1622 fsk_detach_list(&dn_cfg.fsu, DN_DELETE_FS); 1623 /* Reinitialize system heap... */ 1624 heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 1625 } 1626 1627 /* 1628 * Main handler for configuration. We are guaranteed to be called 1629 * with an oid which is at least a dn_id. 1630 * - the first object is the command (config, delete, flush, ...) 1631 * - config_link must be issued after the corresponding config_sched 1632 * - parameters (DN_TXT) for an object must preceed the object 1633 * processed on a config_sched. 1634 */ 1635 int 1636 do_config(void *p, int l) 1637 { 1638 struct dn_id *next, *o; 1639 int err = 0, err2 = 0; 1640 struct dn_id *arg = NULL; 1641 uintptr_t *a; 1642 1643 o = p; 1644 if (o->id != DN_API_VERSION) { 1645 D("invalid api version got %d need %d", 1646 o->id, DN_API_VERSION); 1647 return EINVAL; 1648 } 1649 for (; l >= sizeof(*o); o = next) { 1650 struct dn_id *prev = arg; 1651 if (o->len < sizeof(*o) || l < o->len) { 1652 D("bad len o->len %d len %d", o->len, l); 1653 err = EINVAL; 1654 break; 1655 } 1656 l -= o->len; 1657 next = (struct dn_id *)((char *)o + o->len); 1658 err = 0; 1659 switch (o->type) { 1660 default: 1661 D("cmd %d not implemented", o->type); 1662 break; 1663 1664 #ifdef EMULATE_SYSCTL 1665 /* sysctl emulation. 1666 * if we recognize the command, jump to the correct 1667 * handler and return 1668 */ 1669 case DN_SYSCTL_SET: 1670 err = kesysctl_emu_set(p, l); 1671 return err; 1672 #endif 1673 1674 case DN_CMD_CONFIG: /* simply a header */ 1675 break; 1676 1677 case DN_CMD_DELETE: 1678 /* the argument is in the first uintptr_t after o */ 1679 a = (uintptr_t *)(o+1); 1680 if (o->len < sizeof(*o) + sizeof(*a)) { 1681 err = EINVAL; 1682 break; 1683 } 1684 switch (o->subtype) { 1685 case DN_LINK: 1686 /* delete base and derived schedulers */ 1687 DN_BH_WLOCK(); 1688 err = delete_schk(*a); 1689 err2 = delete_schk(*a + DN_MAX_ID); 1690 DN_BH_WUNLOCK(); 1691 if (!err) 1692 err = err2; 1693 break; 1694 1695 default: 1696 D("invalid delete type %d", 1697 o->subtype); 1698 err = EINVAL; 1699 break; 1700 1701 case DN_FS: 1702 err = (*a <1 || *a >= DN_MAX_ID) ? 1703 EINVAL : delete_fs(*a, 0) ; 1704 break; 1705 } 1706 break; 1707 1708 case DN_CMD_FLUSH: 1709 DN_BH_WLOCK(); 1710 dummynet_flush(); 1711 DN_BH_WUNLOCK(); 1712 break; 1713 case DN_TEXT: /* store argument the next block */ 1714 prev = NULL; 1715 arg = o; 1716 break; 1717 case DN_LINK: 1718 err = config_link((struct dn_link *)o, arg); 1719 break; 1720 case DN_PROFILE: 1721 err = config_profile((struct dn_profile *)o, arg); 1722 break; 1723 case DN_SCH: 1724 err = config_sched((struct dn_sch *)o, arg); 1725 break; 1726 case DN_FS: 1727 err = (NULL==config_fs((struct dn_fs *)o, arg, 0)); 1728 break; 1729 } 1730 if (prev) 1731 arg = NULL; 1732 if (err != 0) 1733 break; 1734 } 1735 return err; 1736 } 1737 1738 static int 1739 compute_space(struct dn_id *cmd, struct copy_args *a) 1740 { 1741 int x = 0, need = 0; 1742 int profile_size = sizeof(struct dn_profile) - 1743 ED_MAX_SAMPLES_NO*sizeof(int); 1744 1745 /* NOTE about compute space: 1746 * NP = dn_cfg.schk_count 1747 * NSI = dn_cfg.si_count 1748 * NF = dn_cfg.fsk_count 1749 * NQ = dn_cfg.queue_count 1750 * - ipfw pipe show 1751 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 1752 * link, scheduler template, flowset 1753 * integrated in scheduler and header 1754 * for flowset list 1755 * (NSI)*(dn_flow) all scheduler instance (includes 1756 * the queue instance) 1757 * - ipfw sched show 1758 * (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler 1759 * link, scheduler template, flowset 1760 * integrated in scheduler and header 1761 * for flowset list 1762 * (NSI * dn_flow) all scheduler instances 1763 * (NF * sizeof(uint_32)) space for flowset list linked to scheduler 1764 * (NQ * dn_queue) all queue [XXXfor now not listed] 1765 * - ipfw queue show 1766 * (NF * dn_fs) all flowset 1767 * (NQ * dn_queue) all queues 1768 */ 1769 switch (cmd->subtype) { 1770 default: 1771 return -1; 1772 /* XXX where do LINK and SCH differ ? */ 1773 /* 'ipfw sched show' could list all queues associated to 1774 * a scheduler. This feature for now is disabled 1775 */ 1776 case DN_LINK: /* pipe show */ 1777 x = DN_C_LINK | DN_C_SCH | DN_C_FLOW; 1778 need += dn_cfg.schk_count * 1779 (sizeof(struct dn_fs) + profile_size) / 2; 1780 need += dn_cfg.fsk_count * sizeof(uint32_t); 1781 break; 1782 case DN_SCH: /* sched show */ 1783 need += dn_cfg.schk_count * 1784 (sizeof(struct dn_fs) + profile_size) / 2; 1785 need += dn_cfg.fsk_count * sizeof(uint32_t); 1786 x = DN_C_SCH | DN_C_LINK | DN_C_FLOW; 1787 break; 1788 case DN_FS: /* queue show */ 1789 x = DN_C_FS | DN_C_QUEUE; 1790 break; 1791 case DN_GET_COMPAT: /* compatibility mode */ 1792 need = dn_compat_calc_size(); 1793 break; 1794 } 1795 a->flags = x; 1796 if (x & DN_C_SCH) { 1797 need += dn_cfg.schk_count * sizeof(struct dn_sch) / 2; 1798 /* NOT also, each fs might be attached to a sched */ 1799 need += dn_cfg.schk_count * sizeof(struct dn_id) / 2; 1800 } 1801 if (x & DN_C_FS) 1802 need += dn_cfg.fsk_count * sizeof(struct dn_fs); 1803 if (x & DN_C_LINK) { 1804 need += dn_cfg.schk_count * sizeof(struct dn_link) / 2; 1805 } 1806 /* 1807 * When exporting a queue to userland, only pass up the 1808 * struct dn_flow, which is the only visible part. 1809 */ 1810 1811 if (x & DN_C_QUEUE) 1812 need += dn_cfg.queue_count * sizeof(struct dn_flow); 1813 if (x & DN_C_FLOW) 1814 need += dn_cfg.si_count * (sizeof(struct dn_flow)); 1815 return need; 1816 } 1817 1818 /* 1819 * If compat != NULL dummynet_get is called in compatibility mode. 1820 * *compat will be the pointer to the buffer to pass to ipfw 1821 */ 1822 int 1823 dummynet_get(struct sockopt *sopt, void **compat) 1824 { 1825 int have, i, need, error; 1826 char *start = NULL, *buf; 1827 size_t sopt_valsize; 1828 struct dn_id *cmd; 1829 struct copy_args a; 1830 struct copy_range r; 1831 int l = sizeof(struct dn_id); 1832 1833 bzero(&a, sizeof(a)); 1834 bzero(&r, sizeof(r)); 1835 1836 /* save and restore original sopt_valsize around copyin */ 1837 sopt_valsize = sopt->sopt_valsize; 1838 1839 cmd = &r.o; 1840 1841 if (!compat) { 1842 /* copy at least an oid, and possibly a full object */ 1843 error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd)); 1844 sopt->sopt_valsize = sopt_valsize; 1845 if (error) 1846 goto done; 1847 l = cmd->len; 1848 #ifdef EMULATE_SYSCTL 1849 /* sysctl emulation. */ 1850 if (cmd->type == DN_SYSCTL_GET) 1851 return kesysctl_emu_get(sopt); 1852 #endif 1853 if (l > sizeof(r)) { 1854 /* request larger than default, allocate buffer */ 1855 cmd = malloc(l, M_DUMMYNET, M_WAITOK); 1856 error = sooptcopyin(sopt, cmd, l, l); 1857 sopt->sopt_valsize = sopt_valsize; 1858 if (error) 1859 goto done; 1860 } 1861 } else { /* compatibility */ 1862 error = 0; 1863 cmd->type = DN_CMD_GET; 1864 cmd->len = sizeof(struct dn_id); 1865 cmd->subtype = DN_GET_COMPAT; 1866 // cmd->id = sopt_valsize; 1867 D("compatibility mode"); 1868 } 1869 a.extra = (struct copy_range *)cmd; 1870 if (cmd->len == sizeof(*cmd)) { /* no range, create a default */ 1871 uint32_t *rp = (uint32_t *)(cmd + 1); 1872 cmd->len += 2* sizeof(uint32_t); 1873 rp[0] = 1; 1874 rp[1] = DN_MAX_ID - 1; 1875 if (cmd->subtype == DN_LINK) { 1876 rp[0] += DN_MAX_ID; 1877 rp[1] += DN_MAX_ID; 1878 } 1879 } 1880 /* Count space (under lock) and allocate (outside lock). 1881 * Exit with lock held if we manage to get enough buffer. 1882 * Try a few times then give up. 1883 */ 1884 for (have = 0, i = 0; i < 10; i++) { 1885 DN_BH_WLOCK(); 1886 need = compute_space(cmd, &a); 1887 1888 /* if there is a range, ignore value from compute_space() */ 1889 if (l > sizeof(*cmd)) 1890 need = sopt_valsize - sizeof(*cmd); 1891 1892 if (need < 0) { 1893 DN_BH_WUNLOCK(); 1894 error = EINVAL; 1895 goto done; 1896 } 1897 need += sizeof(*cmd); 1898 cmd->id = need; 1899 if (have >= need) 1900 break; 1901 1902 DN_BH_WUNLOCK(); 1903 if (start) 1904 free(start, M_DUMMYNET); 1905 start = NULL; 1906 if (need > sopt_valsize) 1907 break; 1908 1909 have = need; 1910 start = malloc(have, M_DUMMYNET, M_WAITOK | M_ZERO); 1911 } 1912 1913 if (start == NULL) { 1914 if (compat) { 1915 *compat = NULL; 1916 error = 1; // XXX 1917 } else { 1918 error = sooptcopyout(sopt, cmd, sizeof(*cmd)); 1919 } 1920 goto done; 1921 } 1922 ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, " 1923 "%d:%d si %d, %d:%d queues %d", 1924 dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH, 1925 dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK, 1926 dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS, 1927 dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I, 1928 dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE); 1929 sopt->sopt_valsize = sopt_valsize; 1930 a.type = cmd->subtype; 1931 1932 if (compat == NULL) { 1933 bcopy(cmd, start, sizeof(*cmd)); 1934 ((struct dn_id*)(start))->len = sizeof(struct dn_id); 1935 buf = start + sizeof(*cmd); 1936 } else 1937 buf = start; 1938 a.start = &buf; 1939 a.end = start + have; 1940 /* start copying other objects */ 1941 if (compat) { 1942 a.type = DN_COMPAT_PIPE; 1943 dn_ht_scan(dn_cfg.schedhash, copy_data_helper_compat, &a); 1944 a.type = DN_COMPAT_QUEUE; 1945 dn_ht_scan(dn_cfg.fshash, copy_data_helper_compat, &a); 1946 } else if (a.type == DN_FS) { 1947 dn_ht_scan(dn_cfg.fshash, copy_data_helper, &a); 1948 } else { 1949 dn_ht_scan(dn_cfg.schedhash, copy_data_helper, &a); 1950 } 1951 DN_BH_WUNLOCK(); 1952 1953 if (compat) { 1954 *compat = start; 1955 sopt->sopt_valsize = buf - start; 1956 /* free() is done by ip_dummynet_compat() */ 1957 start = NULL; //XXX hack 1958 } else { 1959 error = sooptcopyout(sopt, start, buf - start); 1960 } 1961 done: 1962 if (cmd && cmd != &r.o) 1963 free(cmd, M_DUMMYNET); 1964 if (start) 1965 free(start, M_DUMMYNET); 1966 return error; 1967 } 1968 1969 /* Callback called on scheduler instance to delete it if idle */ 1970 static int 1971 drain_scheduler_cb(void *_si, void *arg) 1972 { 1973 struct dn_sch_inst *si = _si; 1974 1975 if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL) 1976 return 0; 1977 1978 if (si->sched->fp->flags & DN_MULTIQUEUE) { 1979 if (si->q_count == 0) 1980 return si_destroy(si, NULL); 1981 else 1982 return 0; 1983 } else { /* !DN_MULTIQUEUE */ 1984 if ((si+1)->ni.length == 0) 1985 return si_destroy(si, NULL); 1986 else 1987 return 0; 1988 } 1989 return 0; /* unreachable */ 1990 } 1991 1992 /* Callback called on scheduler to check if it has instances */ 1993 static int 1994 drain_scheduler_sch_cb(void *_s, void *arg) 1995 { 1996 struct dn_schk *s = _s; 1997 1998 if (s->sch.flags & DN_HAVE_MASK) { 1999 dn_ht_scan_bucket(s->siht, &s->drain_bucket, 2000 drain_scheduler_cb, NULL); 2001 s->drain_bucket++; 2002 } else { 2003 if (s->siht) { 2004 if (drain_scheduler_cb(s->siht, NULL) == DNHT_SCAN_DEL) 2005 s->siht = NULL; 2006 } 2007 } 2008 return 0; 2009 } 2010 2011 /* Called every tick, try to delete a 'bucket' of scheduler */ 2012 void 2013 dn_drain_scheduler(void) 2014 { 2015 dn_ht_scan_bucket(dn_cfg.schedhash, &dn_cfg.drain_sch, 2016 drain_scheduler_sch_cb, NULL); 2017 dn_cfg.drain_sch++; 2018 } 2019 2020 /* Callback called on queue to delete if it is idle */ 2021 static int 2022 drain_queue_cb(void *_q, void *arg) 2023 { 2024 struct dn_queue *q = _q; 2025 2026 if (q->ni.length == 0) { 2027 dn_delete_queue(q, DN_DESTROY); 2028 return DNHT_SCAN_DEL; /* queue is deleted */ 2029 } 2030 2031 return 0; /* queue isn't deleted */ 2032 } 2033 2034 /* Callback called on flowset used to check if it has queues */ 2035 static int 2036 drain_queue_fs_cb(void *_fs, void *arg) 2037 { 2038 struct dn_fsk *fs = _fs; 2039 2040 if (fs->fs.flags & DN_QHT_HASH) { 2041 /* Flowset has a hash table for queues */ 2042 dn_ht_scan_bucket(fs->qht, &fs->drain_bucket, 2043 drain_queue_cb, NULL); 2044 fs->drain_bucket++; 2045 } else { 2046 /* No hash table for this flowset, null the pointer 2047 * if the queue is deleted 2048 */ 2049 if (fs->qht) { 2050 if (drain_queue_cb(fs->qht, NULL) == DNHT_SCAN_DEL) 2051 fs->qht = NULL; 2052 } 2053 } 2054 return 0; 2055 } 2056 2057 /* Called every tick, try to delete a 'bucket' of queue */ 2058 void 2059 dn_drain_queue(void) 2060 { 2061 /* scan a bucket of flowset */ 2062 dn_ht_scan_bucket(dn_cfg.fshash, &dn_cfg.drain_fs, 2063 drain_queue_fs_cb, NULL); 2064 dn_cfg.drain_fs++; 2065 } 2066 2067 /* 2068 * Handler for the various dummynet socket options 2069 */ 2070 static int 2071 ip_dn_ctl(struct sockopt *sopt) 2072 { 2073 void *p = NULL; 2074 int error, l; 2075 2076 error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET); 2077 if (error) 2078 return (error); 2079 2080 /* Disallow sets in really-really secure mode. */ 2081 if (sopt->sopt_dir == SOPT_SET) { 2082 error = securelevel_ge(sopt->sopt_td->td_ucred, 3); 2083 if (error) 2084 return (error); 2085 } 2086 2087 switch (sopt->sopt_name) { 2088 default : 2089 D("dummynet: unknown option %d", sopt->sopt_name); 2090 error = EINVAL; 2091 break; 2092 2093 case IP_DUMMYNET_FLUSH: 2094 case IP_DUMMYNET_CONFIGURE: 2095 case IP_DUMMYNET_DEL: /* remove a pipe or queue */ 2096 case IP_DUMMYNET_GET: 2097 D("dummynet: compat option %d", sopt->sopt_name); 2098 error = ip_dummynet_compat(sopt); 2099 break; 2100 2101 case IP_DUMMYNET3 : 2102 if (sopt->sopt_dir == SOPT_GET) { 2103 error = dummynet_get(sopt, NULL); 2104 break; 2105 } 2106 l = sopt->sopt_valsize; 2107 if (l < sizeof(struct dn_id) || l > 12000) { 2108 D("argument len %d invalid", l); 2109 break; 2110 } 2111 p = malloc(l, M_TEMP, M_WAITOK); // XXX can it fail ? 2112 error = sooptcopyin(sopt, p, l, l); 2113 if (error) 2114 break ; 2115 error = do_config(p, l); 2116 break; 2117 } 2118 2119 if (p != NULL) 2120 free(p, M_TEMP); 2121 2122 return error ; 2123 } 2124 2125 2126 static void 2127 ip_dn_init(void) 2128 { 2129 if (dn_cfg.init_done) 2130 return; 2131 printf("DUMMYNET %p with IPv6 initialized (100409)\n", curvnet); 2132 dn_cfg.init_done = 1; 2133 /* Set defaults here. MSVC does not accept initializers, 2134 * and this is also useful for vimages 2135 */ 2136 /* queue limits */ 2137 dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */ 2138 dn_cfg.byte_limit = 1024 * 1024; 2139 dn_cfg.expire = 1; 2140 2141 /* RED parameters */ 2142 dn_cfg.red_lookup_depth = 256; /* default lookup table depth */ 2143 dn_cfg.red_avg_pkt_size = 512; /* default medium packet size */ 2144 dn_cfg.red_max_pkt_size = 1500; /* default max packet size */ 2145 2146 /* hash tables */ 2147 dn_cfg.max_hash_size = 65536; /* max in the hash tables */ 2148 dn_cfg.hash_size = 64; /* default hash size */ 2149 2150 /* create hash tables for schedulers and flowsets. 2151 * In both we search by key and by pointer. 2152 */ 2153 dn_cfg.schedhash = dn_ht_init(NULL, dn_cfg.hash_size, 2154 offsetof(struct dn_schk, schk_next), 2155 schk_hash, schk_match, schk_new); 2156 dn_cfg.fshash = dn_ht_init(NULL, dn_cfg.hash_size, 2157 offsetof(struct dn_fsk, fsk_next), 2158 fsk_hash, fsk_match, fsk_new); 2159 2160 /* bucket index to drain object */ 2161 dn_cfg.drain_fs = 0; 2162 dn_cfg.drain_sch = 0; 2163 2164 heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id)); 2165 SLIST_INIT(&dn_cfg.fsu); 2166 SLIST_INIT(&dn_cfg.schedlist); 2167 2168 DN_LOCK_INIT(); 2169 2170 TASK_INIT(&dn_task, 0, dummynet_task, curvnet); 2171 dn_tq = taskqueue_create_fast("dummynet", M_WAITOK, 2172 taskqueue_thread_enqueue, &dn_tq); 2173 taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet"); 2174 2175 callout_init(&dn_timeout, 1); 2176 dn_reschedule(); 2177 2178 /* Initialize curr_time adjustment mechanics. */ 2179 getmicrouptime(&dn_cfg.prev_t); 2180 } 2181 2182 static void 2183 ip_dn_destroy(int last) 2184 { 2185 DN_BH_WLOCK(); 2186 /* ensure no more callouts are started */ 2187 dn_gone = 1; 2188 2189 /* check for last */ 2190 if (last) { 2191 ND("removing last instance\n"); 2192 ip_dn_ctl_ptr = NULL; 2193 ip_dn_io_ptr = NULL; 2194 } 2195 2196 dummynet_flush(); 2197 DN_BH_WUNLOCK(); 2198 2199 callout_drain(&dn_timeout); 2200 taskqueue_drain(dn_tq, &dn_task); 2201 taskqueue_free(dn_tq); 2202 2203 dn_ht_free(dn_cfg.schedhash, 0); 2204 dn_ht_free(dn_cfg.fshash, 0); 2205 heap_free(&dn_cfg.evheap); 2206 2207 DN_LOCK_DESTROY(); 2208 } 2209 2210 static int 2211 dummynet_modevent(module_t mod, int type, void *data) 2212 { 2213 2214 if (type == MOD_LOAD) { 2215 if (ip_dn_io_ptr) { 2216 printf("DUMMYNET already loaded\n"); 2217 return EEXIST ; 2218 } 2219 ip_dn_init(); 2220 ip_dn_ctl_ptr = ip_dn_ctl; 2221 ip_dn_io_ptr = dummynet_io; 2222 return 0; 2223 } else if (type == MOD_UNLOAD) { 2224 ip_dn_destroy(1 /* last */); 2225 return 0; 2226 } else 2227 return EOPNOTSUPP; 2228 } 2229 2230 /* modevent helpers for the modules */ 2231 static int 2232 load_dn_sched(struct dn_alg *d) 2233 { 2234 struct dn_alg *s; 2235 2236 if (d == NULL) 2237 return 1; /* error */ 2238 ip_dn_init(); /* just in case, we need the lock */ 2239 2240 /* Check that mandatory funcs exists */ 2241 if (d->enqueue == NULL || d->dequeue == NULL) { 2242 D("missing enqueue or dequeue for %s", d->name); 2243 return 1; 2244 } 2245 2246 /* Search if scheduler already exists */ 2247 DN_BH_WLOCK(); 2248 SLIST_FOREACH(s, &dn_cfg.schedlist, next) { 2249 if (strcmp(s->name, d->name) == 0) { 2250 D("%s already loaded", d->name); 2251 break; /* scheduler already exists */ 2252 } 2253 } 2254 if (s == NULL) 2255 SLIST_INSERT_HEAD(&dn_cfg.schedlist, d, next); 2256 DN_BH_WUNLOCK(); 2257 D("dn_sched %s %sloaded", d->name, s ? "not ":""); 2258 return s ? 1 : 0; 2259 } 2260 2261 static int 2262 unload_dn_sched(struct dn_alg *s) 2263 { 2264 struct dn_alg *tmp, *r; 2265 int err = EINVAL; 2266 2267 ND("called for %s", s->name); 2268 2269 DN_BH_WLOCK(); 2270 SLIST_FOREACH_SAFE(r, &dn_cfg.schedlist, next, tmp) { 2271 if (strcmp(s->name, r->name) != 0) 2272 continue; 2273 ND("ref_count = %d", r->ref_count); 2274 err = (r->ref_count != 0) ? EBUSY : 0; 2275 if (err == 0) 2276 SLIST_REMOVE(&dn_cfg.schedlist, r, dn_alg, next); 2277 break; 2278 } 2279 DN_BH_WUNLOCK(); 2280 D("dn_sched %s %sunloaded", s->name, err ? "not ":""); 2281 return err; 2282 } 2283 2284 int 2285 dn_sched_modevent(module_t mod, int cmd, void *arg) 2286 { 2287 struct dn_alg *sch = arg; 2288 2289 if (cmd == MOD_LOAD) 2290 return load_dn_sched(sch); 2291 else if (cmd == MOD_UNLOAD) 2292 return unload_dn_sched(sch); 2293 else 2294 return EINVAL; 2295 } 2296 2297 static moduledata_t dummynet_mod = { 2298 "dummynet", dummynet_modevent, NULL 2299 }; 2300 2301 #define DN_SI_SUB SI_SUB_PROTO_IFATTACHDOMAIN 2302 #define DN_MODEV_ORD (SI_ORDER_ANY - 128) /* after ipfw */ 2303 DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD); 2304 MODULE_DEPEND(dummynet, ipfw, 3, 3, 3); 2305 MODULE_VERSION(dummynet, 3); 2306 2307 /* 2308 * Starting up. Done in order after dummynet_modevent() has been called. 2309 * VNET_SYSINIT is also called for each existing vnet and each new vnet. 2310 */ 2311 //VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_init, NULL); 2312 2313 /* 2314 * Shutdown handlers up shop. These are done in REVERSE ORDER, but still 2315 * after dummynet_modevent() has been called. Not called on reboot. 2316 * VNET_SYSUNINIT is also called for each exiting vnet as it exits. 2317 * or when the module is unloaded. 2318 */ 2319 //VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_destroy, NULL); 2320 2321 /* end of file */ 2322