1 /* 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Cavium, Inc. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 /*$FreeBSD$*/ 34 35 /* \file lio_droq.h 36 * \brief Implementation of Octeon Output queues. "Output" is with 37 * respect to the Octeon device on the NIC. From this driver's point of 38 * view they are ingress queues. 39 */ 40 41 #ifndef __LIO_DROQ_H__ 42 #define __LIO_DROQ_H__ 43 44 /* 45 * Octeon descriptor format. 46 * The descriptor ring is made of descriptors which have 2 64-bit values: 47 * -# Physical (bus) address of the data buffer. 48 * -# Physical (bus) address of a lio_droq_info structure. 49 * The Octeon device DMA's incoming packets and its information at the address 50 * given by these descriptor fields. 51 */ 52 struct lio_droq_desc { 53 /* The buffer pointer */ 54 uint64_t buffer_ptr; 55 56 /* The Info pointer */ 57 uint64_t info_ptr; 58 }; 59 60 #define LIO_DROQ_DESC_SIZE (sizeof(struct lio_droq_desc)) 61 62 /* 63 * Information about packet DMA'ed by Octeon. 64 * The format of the information available at Info Pointer after Octeon 65 * has posted a packet. Not all descriptors have valid information. Only 66 * the Info field of the first descriptor for a packet has information 67 * about the packet. 68 */ 69 struct lio_droq_info { 70 /* The Length of the packet. */ 71 uint64_t length; 72 73 /* The Output Receive Header. */ 74 union octeon_rh rh; 75 76 }; 77 78 #define LIO_DROQ_INFO_SIZE (sizeof(struct lio_droq_info)) 79 80 /* 81 * Pointer to data buffer. 82 * Driver keeps a pointer to the data buffer that it made available to 83 * the Octeon device. Since the descriptor ring keeps physical (bus) 84 * addresses, this field is required for the driver to keep track of 85 * the virtual address pointers. 86 */ 87 struct lio_recv_buffer { 88 /* Packet buffer, including metadata. */ 89 void *buffer; 90 91 /* Data in the packet buffer. */ 92 uint8_t *data; 93 }; 94 95 #define LIO_DROQ_RECVBUF_SIZE (sizeof(struct lio_recv_buffer)) 96 97 /* Output Queue statistics. Each output queue has four stats fields. */ 98 struct lio_droq_stats { 99 /* Number of packets received in this queue. */ 100 uint64_t pkts_received; 101 102 /* Bytes received by this queue. */ 103 uint64_t bytes_received; 104 105 /* Packets dropped due to no dispatch function. */ 106 uint64_t dropped_nodispatch; 107 108 /* Packets dropped due to no memory available. */ 109 uint64_t dropped_nomem; 110 111 /* Packets dropped due to large number of pkts to process. */ 112 uint64_t dropped_toomany; 113 114 /* Number of packets sent to stack from this queue. */ 115 uint64_t rx_pkts_received; 116 117 /* Number of Bytes sent to stack from this queue. */ 118 uint64_t rx_bytes_received; 119 120 /* Num of Packets dropped due to receive path failures. */ 121 uint64_t rx_dropped; 122 123 uint64_t rx_vxlan; 124 125 /* Num of failures of lio_recv_buffer_alloc() */ 126 uint64_t rx_alloc_failure; 127 128 }; 129 130 /* 131 * The maximum number of buffers that can be dispatched from the 132 * output/dma queue. Set to 64 assuming 1K buffers in DROQ and the fact that 133 * max packet size from DROQ is 64K. 134 */ 135 #define LIO_MAX_RECV_BUFS 64 136 137 /* 138 * Receive Packet format used when dispatching output queue packets 139 * with non-raw opcodes. 140 * The received packet will be sent to the upper layers using this 141 * structure which is passed as a parameter to the dispatch function 142 */ 143 struct lio_recv_pkt { 144 /* Number of buffers in this received packet */ 145 uint16_t buffer_count; 146 147 /* Id of the device that is sending the packet up */ 148 uint16_t octeon_id; 149 150 /* Length of data in the packet buffer */ 151 uint32_t length; 152 153 /* The receive header */ 154 union octeon_rh rh; 155 156 /* Pointer to the OS-specific packet buffer */ 157 struct mbuf *buffer_ptr[LIO_MAX_RECV_BUFS]; 158 159 /* Size of the buffers pointed to by ptr's in buffer_ptr */ 160 uint32_t buffer_size[LIO_MAX_RECV_BUFS]; 161 }; 162 163 #define LIO_RECV_PKT_SIZE (sizeof(struct lio_recv_pkt)) 164 165 /* 166 * The first parameter of a dispatch function. 167 * For a raw mode opcode, the driver dispatches with the device 168 * pointer in this structure. 169 * For non-raw mode opcode, the driver dispatches the recv_pkt 170 * created to contain the buffers with data received from Octeon. 171 * --------------------- 172 * | *recv_pkt ----|--- 173 * |-------------------| | 174 * | 0 or more bytes | | 175 * | reserved by driver| | 176 * |-------------------|<-/ 177 * | lio_recv_pkt | 178 * | | 179 * |___________________| 180 */ 181 struct lio_recv_info { 182 void *rsvd; 183 struct lio_recv_pkt *recv_pkt; 184 }; 185 186 #define LIO_RECV_INFO_SIZE (sizeof(struct lio_recv_info)) 187 188 /* 189 * Allocate a recv_info structure. The recv_pkt pointer in the recv_info 190 * structure is filled in before this call returns. 191 * @param extra_bytes - extra bytes to be allocated at the end of the recv info 192 * structure. 193 * @return - pointer to a newly allocated recv_info structure. 194 */ 195 static inline struct lio_recv_info * 196 lio_alloc_recv_info(int extra_bytes) 197 { 198 struct lio_recv_info *recv_info; 199 uint8_t *buf; 200 201 buf = malloc(LIO_RECV_PKT_SIZE + LIO_RECV_INFO_SIZE + 202 extra_bytes, M_DEVBUF, M_NOWAIT | M_ZERO); 203 if (buf == NULL) 204 return (NULL); 205 206 recv_info = (struct lio_recv_info *)buf; 207 recv_info->recv_pkt = (struct lio_recv_pkt *)(buf + LIO_RECV_INFO_SIZE); 208 recv_info->rsvd = NULL; 209 if (extra_bytes) 210 recv_info->rsvd = buf + LIO_RECV_INFO_SIZE + LIO_RECV_PKT_SIZE; 211 212 return (recv_info); 213 } 214 215 /* 216 * Free a recv_info structure. 217 * @param recv_info - Pointer to receive_info to be freed 218 */ 219 static inline void 220 lio_free_recv_info(struct lio_recv_info *recv_info) 221 { 222 223 free(recv_info, M_DEVBUF); 224 } 225 226 typedef int (*lio_dispatch_fn_t)(struct lio_recv_info *, void *); 227 228 /* 229 * Used by NIC module to register packet handler and to get device 230 * information for each octeon device. 231 */ 232 struct lio_droq_ops { 233 /* 234 * This registered function will be called by the driver with 235 * the pointer to buffer from droq and length of 236 * data in the buffer. The receive header gives the port 237 * number to the caller. Function pointer is set by caller. 238 */ 239 void (*fptr) (void *, uint32_t, union octeon_rh *, void *, 240 void *); 241 void *farg; 242 243 /* 244 * Flag indicating if the DROQ handler should drop packets that 245 * it cannot handle in one iteration. Set by caller. 246 */ 247 uint32_t drop_on_max; 248 }; 249 250 /* 251 * The Descriptor Ring Output Queue structure. 252 * This structure has all the information required to implement a 253 * Octeon DROQ. 254 */ 255 struct lio_droq { 256 /* A lock to protect access to this ring. */ 257 struct mtx lock; 258 259 uint32_t q_no; 260 261 uint32_t pkt_count; 262 263 struct lio_droq_ops ops; 264 265 struct octeon_device *oct_dev; 266 267 /* The 8B aligned descriptor ring starts at this address. */ 268 struct lio_droq_desc *desc_ring; 269 270 /* Index in the ring where the driver should read the next packet */ 271 uint32_t read_idx; 272 273 /* 274 * Index in the ring where the driver will refill the descriptor's 275 * buffer 276 */ 277 uint32_t refill_idx; 278 279 /* Packets pending to be processed */ 280 volatile int pkts_pending; 281 282 /* Number of descriptors in this ring. */ 283 uint32_t max_count; 284 285 /* The number of descriptors pending refill. */ 286 uint32_t refill_count; 287 288 uint32_t pkts_per_intr; 289 uint32_t refill_threshold; 290 291 /* 292 * The max number of descriptors in DROQ without a buffer. 293 * This field is used to keep track of empty space threshold. If the 294 * refill_count reaches this value, the DROQ cannot accept a max-sized 295 * (64K) packet. 296 */ 297 uint32_t max_empty_descs; 298 299 /* 300 * The receive buffer list. This list has the virtual addresses of 301 * the buffers. 302 */ 303 struct lio_recv_buffer *recv_buf_list; 304 305 /* The size of each buffer pointed by the buffer pointer. */ 306 uint32_t buffer_size; 307 308 /* 309 * Offset to packet credit register. 310 * Host writes number of info/buffer ptrs available to this register 311 */ 312 uint32_t pkts_credit_reg; 313 314 /* 315 * Offset packet sent register. 316 * Octeon writes the number of packets DMA'ed to host memory 317 * in this register. 318 */ 319 uint32_t pkts_sent_reg; 320 321 struct lio_stailq_head dispatch_stq_head; 322 323 /* Statistics for this DROQ. */ 324 struct lio_droq_stats stats; 325 326 /* DMA mapped address of the DROQ descriptor ring. */ 327 vm_paddr_t desc_ring_dma; 328 329 /* application context */ 330 void *app_ctx; 331 332 uint32_t cpu_id; 333 334 struct task droq_task; 335 struct taskqueue *droq_taskqueue; 336 337 struct lro_ctrl lro; 338 }; 339 340 #define LIO_DROQ_SIZE (sizeof(struct lio_droq)) 341 342 /* 343 * Allocates space for the descriptor ring for the droq and sets the 344 * base addr, num desc etc in Octeon registers. 345 * 346 * @param oct_dev - pointer to the octeon device structure 347 * @param q_no - droq no. 348 * @param app_ctx - pointer to application context 349 * @return Success: 0 Failure: 1 350 */ 351 int lio_init_droq(struct octeon_device *oct_dev, 352 uint32_t q_no, uint32_t num_descs, uint32_t desc_size, 353 void *app_ctx); 354 355 /* 356 * Frees the space for descriptor ring for the droq. 357 * 358 * @param oct_dev - pointer to the octeon device structure 359 * @param q_no - droq no. 360 * @return: Success: 0 Failure: 1 361 */ 362 int lio_delete_droq(struct octeon_device *oct_dev, uint32_t q_no); 363 364 /* 365 * Register a change in droq operations. The ops field has a pointer to a 366 * function which will called by the DROQ handler for all packets arriving 367 * on output queues given by q_no irrespective of the type of packet. 368 * The ops field also has a flag which if set tells the DROQ handler to 369 * drop packets if it receives more than what it can process in one 370 * invocation of the handler. 371 * @param oct - octeon device 372 * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 373 * @param ops - the droq_ops settings for this queue 374 * @return - 0 on success, -ENODEV or -EINVAL on error. 375 */ 376 int lio_register_droq_ops(struct octeon_device *oct, uint32_t q_no, 377 struct lio_droq_ops *ops); 378 379 /* 380 * Resets the function pointer and flag settings made by 381 * lio_register_droq_ops(). After this routine is called, the DROQ handler 382 * will lookup dispatch function for each arriving packet on the output queue 383 * given by q_no. 384 * @param oct - octeon device 385 * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 386 * @return - 0 on success, -ENODEV or -EINVAL on error. 387 */ 388 int lio_unregister_droq_ops(struct octeon_device *oct, uint32_t q_no); 389 390 /* 391 * Register a dispatch function for a opcode/subcode. The driver will call 392 * this dispatch function when it receives a packet with the given 393 * opcode/subcode in its output queues along with the user specified 394 * argument. 395 * @param oct - the octeon device to register with. 396 * @param opcode - the opcode for which the dispatch will be registered. 397 * @param subcode - the subcode for which the dispatch will be registered 398 * @param fn - the dispatch function. 399 * @param fn_arg - user specified that will be passed along with the 400 * dispatch function by the driver. 401 * @return Success: 0; Failure: 1 402 */ 403 int lio_register_dispatch_fn(struct octeon_device *oct, uint16_t opcode, 404 uint16_t subcode, lio_dispatch_fn_t fn, 405 void *fn_arg); 406 407 /* 408 * Remove registration for an opcode/subcode. This will delete the mapping for 409 * an opcode/subcode. The dispatch function will be unregistered and will no 410 * longer be called if a packet with the opcode/subcode arrives in the driver 411 * output queues. 412 * @param oct - the octeon device to unregister from. 413 * @param opcode - the opcode to be unregistered. 414 * @param subcode - the subcode to be unregistered. 415 * 416 * @return Success: 0; Failure: 1 417 */ 418 int lio_unregister_dispatch_fn(struct octeon_device *oct, uint16_t opcode, 419 uint16_t subcode); 420 421 uint32_t lio_droq_check_hw_for_pkts(struct lio_droq *droq); 422 423 int lio_create_droq(struct octeon_device *oct, uint32_t q_no, 424 uint32_t num_descs, uint32_t desc_size, void *app_ctx); 425 426 int lio_droq_process_packets(struct octeon_device *oct, 427 struct lio_droq *droq, uint32_t budget); 428 429 uint32_t lio_droq_refill(struct octeon_device *octeon_dev, 430 struct lio_droq *droq); 431 void lio_droq_bh(void *ptr, int pending __unused); 432 #endif /* __LIO_DROQ_H__ */ 433