1 /* 2 * Copyright (c) 2010 Riccardo Panicucci, Universita` di Pisa 3 * Copyright (c) 2000-2002 Luigi Rizzo, Universita` di Pisa 4 * All rights reserved 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* 29 * $FreeBSD$ 30 */ 31 32 #ifdef _KERNEL 33 #include <sys/malloc.h> 34 #include <sys/socket.h> 35 #include <sys/socketvar.h> 36 #include <sys/kernel.h> 37 #include <sys/lock.h> 38 #include <sys/mbuf.h> 39 #include <sys/module.h> 40 #include <sys/rwlock.h> 41 #include <net/if.h> /* IFNAMSIZ */ 42 #include <netinet/in.h> 43 #include <netinet/ip_var.h> /* ipfw_rule_ref */ 44 #include <netinet/ip_fw.h> /* flow_id */ 45 #include <netinet/ip_dummynet.h> 46 #include <netpfil/ipfw/ip_fw_private.h> 47 #include <netpfil/ipfw/dn_heap.h> 48 #include <netpfil/ipfw/ip_dn_private.h> 49 #ifdef NEW_AQM 50 #include <netpfil/ipfw/dn_aqm.h> 51 #endif 52 #include <netpfil/ipfw/dn_sched.h> 53 #else 54 #include <dn_test.h> 55 #endif 56 57 #ifndef MAX64 58 #define MAX64(x,y) (( (int64_t) ( (y)-(x) )) > 0 ) ? (y) : (x) 59 #endif 60 61 /* 62 * timestamps are computed on 64 bit using fixed point arithmetic. 63 * LMAX_BITS, WMAX_BITS are the max number of bits for the packet len 64 * and sum of weights, respectively. FRAC_BITS is the number of 65 * fractional bits. We want FRAC_BITS >> WMAX_BITS to avoid too large 66 * errors when computing the inverse, FRAC_BITS < 32 so we can do 1/w 67 * using an unsigned 32-bit division, and to avoid wraparounds we need 68 * LMAX_BITS + WMAX_BITS + FRAC_BITS << 64 69 * As an example 70 * FRAC_BITS = 26, LMAX_BITS=14, WMAX_BITS = 19 71 */ 72 #ifndef FRAC_BITS 73 #define FRAC_BITS 28 /* shift for fixed point arithmetic */ 74 #define ONE_FP (1UL << FRAC_BITS) 75 #endif 76 77 /* 78 * Private information for the scheduler instance: 79 * sch_heap (key is Finish time) returns the next queue to serve 80 * ne_heap (key is Start time) stores not-eligible queues 81 * idle_heap (key=start/finish time) stores idle flows. It must 82 * support extract-from-middle. 83 * A flow is only in 1 of the three heaps. 84 * XXX todo: use a more efficient data structure, e.g. a tree sorted 85 * by F with min_subtree(S) in each node 86 */ 87 struct wf2qp_si { 88 struct dn_heap sch_heap; /* top extract - key Finish time */ 89 struct dn_heap ne_heap; /* top extract - key Start time */ 90 struct dn_heap idle_heap; /* random extract - key Start=Finish time */ 91 uint64_t V; /* virtual time */ 92 uint32_t inv_wsum; /* inverse of sum of weights */ 93 uint32_t wsum; /* sum of weights */ 94 }; 95 96 struct wf2qp_queue { 97 struct dn_queue _q; 98 uint64_t S, F; /* start time, finish time */ 99 uint32_t inv_w; /* ONE_FP / weight */ 100 int32_t heap_pos; /* position (index) of struct in heap */ 101 }; 102 103 /* 104 * This file implements a WF2Q+ scheduler as it has been in dummynet 105 * since 2000. 106 * The scheduler supports per-flow queues and has O(log N) complexity. 107 * 108 * WF2Q+ needs to drain entries from the idle heap so that we 109 * can keep the sum of weights up to date. We can do it whenever 110 * we get a chance, or periodically, or following some other 111 * strategy. The function idle_check() drains at most N elements 112 * from the idle heap. 113 */ 114 static void 115 idle_check(struct wf2qp_si *si, int n, int force) 116 { 117 struct dn_heap *h = &si->idle_heap; 118 while (n-- > 0 && h->elements > 0 && 119 (force || DN_KEY_LT(HEAP_TOP(h)->key, si->V))) { 120 struct dn_queue *q = HEAP_TOP(h)->object; 121 struct wf2qp_queue *alg_fq = (struct wf2qp_queue *)q; 122 123 heap_extract(h, NULL); 124 /* XXX to let the flowset delete the queue we should 125 * mark it as 'unused' by the scheduler. 126 */ 127 alg_fq->S = alg_fq->F + 1; /* Mark timestamp as invalid. */ 128 si->wsum -= q->fs->fs.par[0]; /* adjust sum of weights */ 129 if (si->wsum > 0) 130 si->inv_wsum = ONE_FP/si->wsum; 131 } 132 } 133 134 static int 135 wf2qp_enqueue(struct dn_sch_inst *_si, struct dn_queue *q, struct mbuf *m) 136 { 137 struct dn_fsk *fs = q->fs; 138 struct wf2qp_si *si = (struct wf2qp_si *)(_si + 1); 139 struct wf2qp_queue *alg_fq; 140 uint64_t len = m->m_pkthdr.len; 141 142 if (m != q->mq.head) { 143 if (dn_enqueue(q, m, 0)) /* packet was dropped */ 144 return 1; 145 if (m != q->mq.head) /* queue was already busy */ 146 return 0; 147 } 148 149 /* If reach this point, queue q was idle */ 150 alg_fq = (struct wf2qp_queue *)q; 151 152 if (DN_KEY_LT(alg_fq->F, alg_fq->S)) { 153 /* F<S means timestamps are invalid ->brand new queue. */ 154 alg_fq->S = si->V; /* init start time */ 155 si->wsum += fs->fs.par[0]; /* add weight of new queue. */ 156 si->inv_wsum = ONE_FP/si->wsum; 157 } else { /* if it was idle then it was in the idle heap */ 158 heap_extract(&si->idle_heap, q); 159 alg_fq->S = MAX64(alg_fq->F, si->V); /* compute new S */ 160 } 161 alg_fq->F = alg_fq->S + len * alg_fq->inv_w; 162 163 /* if nothing is backlogged, make sure this flow is eligible */ 164 if (si->ne_heap.elements == 0 && si->sch_heap.elements == 0) 165 si->V = MAX64(alg_fq->S, si->V); 166 167 /* 168 * Look at eligibility. A flow is not eligibile if S>V (when 169 * this happens, it means that there is some other flow already 170 * scheduled for the same pipe, so the sch_heap cannot be 171 * empty). If the flow is not eligible we just store it in the 172 * ne_heap. Otherwise, we store in the sch_heap. 173 * Note that for all flows in sch_heap (SCH), S_i <= V, 174 * and for all flows in ne_heap (NEH), S_i > V. 175 * So when we need to compute max(V, min(S_i)) forall i in 176 * SCH+NEH, we only need to look into NEH. 177 */ 178 if (DN_KEY_LT(si->V, alg_fq->S)) { 179 /* S>V means flow Not eligible. */ 180 if (si->sch_heap.elements == 0) 181 D("++ ouch! not eligible but empty scheduler!"); 182 heap_insert(&si->ne_heap, alg_fq->S, q); 183 } else { 184 heap_insert(&si->sch_heap, alg_fq->F, q); 185 } 186 return 0; 187 } 188 189 /* XXX invariant: sch > 0 || V >= min(S in neh) */ 190 static struct mbuf * 191 wf2qp_dequeue(struct dn_sch_inst *_si) 192 { 193 /* Access scheduler instance private data */ 194 struct wf2qp_si *si = (struct wf2qp_si *)(_si + 1); 195 struct mbuf *m; 196 struct dn_queue *q; 197 struct dn_heap *sch = &si->sch_heap; 198 struct dn_heap *neh = &si->ne_heap; 199 struct wf2qp_queue *alg_fq; 200 201 if (sch->elements == 0 && neh->elements == 0) { 202 /* we have nothing to do. We could kill the idle heap 203 * altogether and reset V 204 */ 205 idle_check(si, 0x7fffffff, 1); 206 si->V = 0; 207 si->wsum = 0; /* should be set already */ 208 return NULL; /* quick return if nothing to do */ 209 } 210 idle_check(si, 1, 0); /* drain something from the idle heap */ 211 212 /* make sure at least one element is eligible, bumping V 213 * and moving entries that have become eligible. 214 * We need to repeat the first part twice, before and 215 * after extracting the candidate, or enqueue() will 216 * find the data structure in a wrong state. 217 */ 218 m = NULL; 219 for(;;) { 220 /* 221 * Compute V = max(V, min(S_i)). Remember that all elements 222 * in sch have by definition S_i <= V so if sch is not empty, 223 * V is surely the max and we must not update it. Conversely, 224 * if sch is empty we only need to look at neh. 225 * We don't need to move the queues, as it will be done at the 226 * next enqueue 227 */ 228 if (sch->elements == 0 && neh->elements > 0) { 229 si->V = MAX64(si->V, HEAP_TOP(neh)->key); 230 } 231 while (neh->elements > 0 && 232 DN_KEY_LEQ(HEAP_TOP(neh)->key, si->V)) { 233 q = HEAP_TOP(neh)->object; 234 alg_fq = (struct wf2qp_queue *)q; 235 heap_extract(neh, NULL); 236 heap_insert(sch, alg_fq->F, q); 237 } 238 if (m) /* pkt found in previous iteration */ 239 break; 240 /* ok we have at least one eligible pkt */ 241 q = HEAP_TOP(sch)->object; 242 alg_fq = (struct wf2qp_queue *)q; 243 m = dn_dequeue(q); 244 heap_extract(sch, NULL); /* Remove queue from heap. */ 245 si->V += (uint64_t)(m->m_pkthdr.len) * si->inv_wsum; 246 alg_fq->S = alg_fq->F; /* Update start time. */ 247 if (q->mq.head == 0) { /* not backlogged any more. */ 248 heap_insert(&si->idle_heap, alg_fq->F, q); 249 } else { /* Still backlogged. */ 250 /* Update F, store in neh or sch */ 251 uint64_t len = q->mq.head->m_pkthdr.len; 252 alg_fq->F += len * alg_fq->inv_w; 253 if (DN_KEY_LEQ(alg_fq->S, si->V)) { 254 heap_insert(sch, alg_fq->F, q); 255 } else { 256 heap_insert(neh, alg_fq->S, q); 257 } 258 } 259 } 260 return m; 261 } 262 263 static int 264 wf2qp_new_sched(struct dn_sch_inst *_si) 265 { 266 struct wf2qp_si *si = (struct wf2qp_si *)(_si + 1); 267 int ofs = offsetof(struct wf2qp_queue, heap_pos); 268 269 /* all heaps support extract from middle */ 270 if (heap_init(&si->idle_heap, 16, ofs) || 271 heap_init(&si->sch_heap, 16, ofs) || 272 heap_init(&si->ne_heap, 16, ofs)) { 273 heap_free(&si->ne_heap); 274 heap_free(&si->sch_heap); 275 heap_free(&si->idle_heap); 276 return ENOMEM; 277 } 278 return 0; 279 } 280 281 static int 282 wf2qp_free_sched(struct dn_sch_inst *_si) 283 { 284 struct wf2qp_si *si = (struct wf2qp_si *)(_si + 1); 285 286 heap_free(&si->sch_heap); 287 heap_free(&si->ne_heap); 288 heap_free(&si->idle_heap); 289 290 return 0; 291 } 292 293 static int 294 wf2qp_new_fsk(struct dn_fsk *fs) 295 { 296 ipdn_bound_var(&fs->fs.par[0], 1, 297 1, 100, "WF2Q+ weight"); 298 return 0; 299 } 300 301 static int 302 wf2qp_new_queue(struct dn_queue *_q) 303 { 304 struct wf2qp_queue *q = (struct wf2qp_queue *)_q; 305 306 _q->ni.oid.subtype = DN_SCHED_WF2QP; 307 q->F = 0; /* not strictly necessary */ 308 q->S = q->F + 1; /* mark timestamp as invalid. */ 309 q->inv_w = ONE_FP / _q->fs->fs.par[0]; 310 if (_q->mq.head != NULL) { 311 wf2qp_enqueue(_q->_si, _q, _q->mq.head); 312 } 313 return 0; 314 } 315 316 /* 317 * Called when the infrastructure removes a queue (e.g. flowset 318 * is reconfigured). Nothing to do if we did not 'own' the queue, 319 * otherwise remove it from the right heap and adjust the sum 320 * of weights. 321 */ 322 static int 323 wf2qp_free_queue(struct dn_queue *q) 324 { 325 struct wf2qp_queue *alg_fq = (struct wf2qp_queue *)q; 326 struct wf2qp_si *si = (struct wf2qp_si *)(q->_si + 1); 327 328 if (alg_fq->S >= alg_fq->F + 1) 329 return 0; /* nothing to do, not in any heap */ 330 si->wsum -= q->fs->fs.par[0]; 331 if (si->wsum > 0) 332 si->inv_wsum = ONE_FP/si->wsum; 333 334 /* extract from the heap. XXX TODO we may need to adjust V 335 * to make sure the invariants hold. 336 */ 337 if (q->mq.head == NULL) { 338 heap_extract(&si->idle_heap, q); 339 } else if (DN_KEY_LT(si->V, alg_fq->S)) { 340 heap_extract(&si->ne_heap, q); 341 } else { 342 heap_extract(&si->sch_heap, q); 343 } 344 return 0; 345 } 346 347 /* 348 * WF2Q+ scheduler descriptor 349 * contains the type of the scheduler, the name, the size of the 350 * structures and function pointers. 351 */ 352 static struct dn_alg wf2qp_desc = { 353 _SI( .type = ) DN_SCHED_WF2QP, 354 _SI( .name = ) "WF2Q+", 355 _SI( .flags = ) DN_MULTIQUEUE, 356 357 /* we need extra space in the si and the queue */ 358 _SI( .schk_datalen = ) 0, 359 _SI( .si_datalen = ) sizeof(struct wf2qp_si), 360 _SI( .q_datalen = ) sizeof(struct wf2qp_queue) - 361 sizeof(struct dn_queue), 362 363 _SI( .enqueue = ) wf2qp_enqueue, 364 _SI( .dequeue = ) wf2qp_dequeue, 365 366 _SI( .config = ) NULL, 367 _SI( .destroy = ) NULL, 368 _SI( .new_sched = ) wf2qp_new_sched, 369 _SI( .free_sched = ) wf2qp_free_sched, 370 371 _SI( .new_fsk = ) wf2qp_new_fsk, 372 _SI( .free_fsk = ) NULL, 373 374 _SI( .new_queue = ) wf2qp_new_queue, 375 _SI( .free_queue = ) wf2qp_free_queue, 376 #ifdef NEW_AQM 377 _SI( .getconfig = ) NULL, 378 #endif 379 380 }; 381 382 383 DECLARE_DNSCHED_MODULE(dn_wf2qp, &wf2qp_desc); 384