1 /* 2 * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 /* $FreeBSD$ */ 27 28 #include <sys/types.h> 29 #include <sys/module.h> 30 #include <sys/errno.h> 31 #include <sys/param.h> /* defines used in kernel.h */ 32 #include <sys/kernel.h> /* types used in module initialization */ 33 #include <sys/conf.h> /* DEV_MODULE */ 34 35 #include <sys/rwlock.h> 36 37 #include <vm/vm.h> /* vtophys */ 38 #include <vm/pmap.h> /* vtophys */ 39 #include <vm/vm_param.h> 40 #include <vm/vm_object.h> 41 #include <vm/vm_page.h> 42 #include <vm/vm_pager.h> 43 #include <vm/uma.h> 44 45 46 #include <sys/malloc.h> 47 #include <sys/socket.h> /* sockaddrs */ 48 #include <sys/selinfo.h> 49 #include <net/if.h> 50 #include <net/if_var.h> 51 #include <machine/bus.h> /* bus_dmamap_* */ 52 53 #include <net/netmap.h> 54 #include <dev/netmap/netmap_kern.h> 55 #include <dev/netmap/netmap_mem2.h> 56 57 58 /* ======================== FREEBSD-SPECIFIC ROUTINES ================== */ 59 60 /* 61 * Intercept the rx routine in the standard device driver. 62 * Second argument is non-zero to intercept, 0 to restore 63 */ 64 int 65 netmap_catch_rx(struct netmap_adapter *na, int intercept) 66 { 67 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 68 struct ifnet *ifp = na->ifp; 69 70 if (intercept) { 71 if (gna->save_if_input) { 72 D("cannot intercept again"); 73 return EINVAL; /* already set */ 74 } 75 gna->save_if_input = ifp->if_input; 76 ifp->if_input = generic_rx_handler; 77 } else { 78 if (!gna->save_if_input){ 79 D("cannot restore"); 80 return EINVAL; /* not saved */ 81 } 82 ifp->if_input = gna->save_if_input; 83 gna->save_if_input = NULL; 84 } 85 86 return 0; 87 } 88 89 90 /* 91 * Intercept the packet steering routine in the tx path, 92 * so that we can decide which queue is used for an mbuf. 93 * Second argument is non-zero to intercept, 0 to restore. 94 * 95 * actually we also need to redirect the if_transmit ? 96 * 97 * XXX see if FreeBSD has such a mechanism 98 */ 99 void 100 netmap_catch_tx(struct netmap_generic_adapter *gna, int enable) 101 { 102 struct netmap_adapter *na = &gna->up.up; 103 struct ifnet *ifp = na->ifp; 104 105 if (enable) { 106 na->if_transmit = ifp->if_transmit; 107 ifp->if_transmit = netmap_transmit; 108 } else { 109 ifp->if_transmit = na->if_transmit; 110 } 111 } 112 113 114 /* Transmit routine used by generic_netmap_txsync(). Returns 0 on success 115 * and non-zero on error (which may be packet drops or other errors). 116 * addr and len identify the netmap buffer, m is the (preallocated) 117 * mbuf to use for transmissions. 118 * 119 * We should add a reference to the mbuf so the m_freem() at the end 120 * of the transmission does not consume resources. 121 * 122 * On FreeBSD, and on multiqueue cards, we can force the queue using 123 * if ((m->m_flags & M_FLOWID) != 0) 124 * i = m->m_pkthdr.flowid % adapter->num_queues; 125 * else 126 * i = curcpu % adapter->num_queues; 127 * 128 */ 129 int 130 generic_xmit_frame(struct ifnet *ifp, struct mbuf *m, 131 void *addr, u_int len, u_int ring_nr) 132 { 133 int ret; 134 135 m->m_len = m->m_pkthdr.len = 0; 136 137 // copy data to the mbuf 138 m_copyback(m, 0, len, addr); 139 // inc refcount. We are alone, so we can skip the atomic 140 atomic_fetchadd_int(m->m_ext.ref_cnt, 1); 141 m->m_flags |= M_FLOWID; 142 m->m_pkthdr.flowid = ring_nr; 143 m->m_pkthdr.rcvif = ifp; /* used for tx notification */ 144 ret = NA(ifp)->if_transmit(ifp, m); 145 return ret; 146 } 147 148 149 /* 150 * The following two functions are empty until we have a generic 151 * way to extract the info from the ifp 152 */ 153 int 154 generic_find_num_desc(struct ifnet *ifp, unsigned int *tx, unsigned int *rx) 155 { 156 D("called"); 157 return 0; 158 } 159 160 161 void 162 generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq) 163 { 164 D("called"); 165 *txq = 1; 166 *rxq = 1; 167 } 168 169 170 void netmap_mitigation_init(struct netmap_generic_adapter *na) 171 { 172 ND("called"); 173 na->mit_pending = 0; 174 } 175 176 177 void netmap_mitigation_start(struct netmap_generic_adapter *na) 178 { 179 ND("called"); 180 } 181 182 183 void netmap_mitigation_restart(struct netmap_generic_adapter *na) 184 { 185 ND("called"); 186 } 187 188 189 int netmap_mitigation_active(struct netmap_generic_adapter *na) 190 { 191 ND("called"); 192 return 0; 193 } 194 195 196 void netmap_mitigation_cleanup(struct netmap_generic_adapter *na) 197 { 198 ND("called"); 199 } 200 201 202 /* 203 * In order to track whether pages are still mapped, we hook into 204 * the standard cdev_pager and intercept the constructor and 205 * destructor. 206 */ 207 208 struct netmap_vm_handle_t { 209 struct cdev *dev; 210 struct netmap_priv_d *priv; 211 }; 212 213 214 static int 215 netmap_dev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 216 vm_ooffset_t foff, struct ucred *cred, u_short *color) 217 { 218 struct netmap_vm_handle_t *vmh = handle; 219 D("handle %p size %jd prot %d foff %jd", 220 handle, (intmax_t)size, prot, (intmax_t)foff); 221 dev_ref(vmh->dev); 222 return 0; 223 } 224 225 226 static void 227 netmap_dev_pager_dtor(void *handle) 228 { 229 struct netmap_vm_handle_t *vmh = handle; 230 struct cdev *dev = vmh->dev; 231 struct netmap_priv_d *priv = vmh->priv; 232 D("handle %p", handle); 233 netmap_dtor(priv); 234 free(vmh, M_DEVBUF); 235 dev_rel(dev); 236 } 237 238 239 static int 240 netmap_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, 241 int prot, vm_page_t *mres) 242 { 243 struct netmap_vm_handle_t *vmh = object->handle; 244 struct netmap_priv_d *priv = vmh->priv; 245 vm_paddr_t paddr; 246 vm_page_t page; 247 vm_memattr_t memattr; 248 vm_pindex_t pidx; 249 250 ND("object %p offset %jd prot %d mres %p", 251 object, (intmax_t)offset, prot, mres); 252 memattr = object->memattr; 253 pidx = OFF_TO_IDX(offset); 254 paddr = netmap_mem_ofstophys(priv->np_mref, offset); 255 if (paddr == 0) 256 return VM_PAGER_FAIL; 257 258 if (((*mres)->flags & PG_FICTITIOUS) != 0) { 259 /* 260 * If the passed in result page is a fake page, update it with 261 * the new physical address. 262 */ 263 page = *mres; 264 vm_page_updatefake(page, paddr, memattr); 265 } else { 266 /* 267 * Replace the passed in reqpage page with our own fake page and 268 * free up the all of the original pages. 269 */ 270 #ifndef VM_OBJECT_WUNLOCK /* FreeBSD < 10.x */ 271 #define VM_OBJECT_WUNLOCK VM_OBJECT_UNLOCK 272 #define VM_OBJECT_WLOCK VM_OBJECT_LOCK 273 #endif /* VM_OBJECT_WUNLOCK */ 274 275 VM_OBJECT_WUNLOCK(object); 276 page = vm_page_getfake(paddr, memattr); 277 VM_OBJECT_WLOCK(object); 278 vm_page_lock(*mres); 279 vm_page_free(*mres); 280 vm_page_unlock(*mres); 281 *mres = page; 282 vm_page_insert(page, object, pidx); 283 } 284 page->valid = VM_PAGE_BITS_ALL; 285 return (VM_PAGER_OK); 286 } 287 288 289 static struct cdev_pager_ops netmap_cdev_pager_ops = { 290 .cdev_pg_ctor = netmap_dev_pager_ctor, 291 .cdev_pg_dtor = netmap_dev_pager_dtor, 292 .cdev_pg_fault = netmap_dev_pager_fault, 293 }; 294 295 296 static int 297 netmap_mmap_single(struct cdev *cdev, vm_ooffset_t *foff, 298 vm_size_t objsize, vm_object_t *objp, int prot) 299 { 300 int error; 301 struct netmap_vm_handle_t *vmh; 302 struct netmap_priv_d *priv; 303 vm_object_t obj; 304 305 D("cdev %p foff %jd size %jd objp %p prot %d", cdev, 306 (intmax_t )*foff, (intmax_t )objsize, objp, prot); 307 308 vmh = malloc(sizeof(struct netmap_vm_handle_t), M_DEVBUF, 309 M_NOWAIT | M_ZERO); 310 if (vmh == NULL) 311 return ENOMEM; 312 vmh->dev = cdev; 313 314 NMG_LOCK(); 315 error = devfs_get_cdevpriv((void**)&priv); 316 if (error) 317 goto err_unlock; 318 vmh->priv = priv; 319 priv->np_refcount++; 320 NMG_UNLOCK(); 321 322 error = netmap_get_memory(priv); 323 if (error) 324 goto err_deref; 325 326 obj = cdev_pager_allocate(vmh, OBJT_DEVICE, 327 &netmap_cdev_pager_ops, objsize, prot, 328 *foff, NULL); 329 if (obj == NULL) { 330 D("cdev_pager_allocate failed"); 331 error = EINVAL; 332 goto err_deref; 333 } 334 335 *objp = obj; 336 return 0; 337 338 err_deref: 339 NMG_LOCK(); 340 priv->np_refcount--; 341 err_unlock: 342 NMG_UNLOCK(); 343 // err: 344 free(vmh, M_DEVBUF); 345 return error; 346 } 347 348 349 // XXX can we remove this ? 350 static int 351 netmap_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 352 { 353 if (netmap_verbose) 354 D("dev %p fflag 0x%x devtype %d td %p", 355 dev, fflag, devtype, td); 356 return 0; 357 } 358 359 360 static int 361 netmap_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 362 { 363 struct netmap_priv_d *priv; 364 int error; 365 366 (void)dev; 367 (void)oflags; 368 (void)devtype; 369 (void)td; 370 371 // XXX wait or nowait ? 372 priv = malloc(sizeof(struct netmap_priv_d), M_DEVBUF, 373 M_NOWAIT | M_ZERO); 374 if (priv == NULL) 375 return ENOMEM; 376 377 error = devfs_set_cdevpriv(priv, netmap_dtor); 378 if (error) 379 return error; 380 381 priv->np_refcount = 1; 382 383 return 0; 384 } 385 386 387 struct cdevsw netmap_cdevsw = { 388 .d_version = D_VERSION, 389 .d_name = "netmap", 390 .d_open = netmap_open, 391 .d_mmap_single = netmap_mmap_single, 392 .d_ioctl = netmap_ioctl, 393 .d_poll = netmap_poll, 394 .d_close = netmap_close, 395 }; 396 397 398 /* 399 * Kernel entry point. 400 * 401 * Initialize/finalize the module and return. 402 * 403 * Return 0 on success, errno on failure. 404 */ 405 static int 406 netmap_loader(__unused struct module *module, int event, __unused void *arg) 407 { 408 int error = 0; 409 410 switch (event) { 411 case MOD_LOAD: 412 error = netmap_init(); 413 break; 414 415 case MOD_UNLOAD: 416 netmap_fini(); 417 break; 418 419 default: 420 error = EOPNOTSUPP; 421 break; 422 } 423 424 return (error); 425 } 426 427 428 DEV_MODULE(netmap, netmap_loader, NULL); 429