1 /* 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Cavium, Inc. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* \file lio_droq.h 35 * \brief Implementation of Octeon Output queues. "Output" is with 36 * respect to the Octeon device on the NIC. From this driver's point of 37 * view they are ingress queues. 38 */ 39 40 #ifndef __LIO_DROQ_H__ 41 #define __LIO_DROQ_H__ 42 43 /* 44 * Octeon descriptor format. 45 * The descriptor ring is made of descriptors which have 2 64-bit values: 46 * -# Physical (bus) address of the data buffer. 47 * -# Physical (bus) address of a lio_droq_info structure. 48 * The Octeon device DMA's incoming packets and its information at the address 49 * given by these descriptor fields. 50 */ 51 struct lio_droq_desc { 52 /* The buffer pointer */ 53 uint64_t buffer_ptr; 54 55 /* The Info pointer */ 56 uint64_t info_ptr; 57 }; 58 59 #define LIO_DROQ_DESC_SIZE (sizeof(struct lio_droq_desc)) 60 61 /* 62 * Information about packet DMA'ed by Octeon. 63 * The format of the information available at Info Pointer after Octeon 64 * has posted a packet. Not all descriptors have valid information. Only 65 * the Info field of the first descriptor for a packet has information 66 * about the packet. 67 */ 68 struct lio_droq_info { 69 /* The Length of the packet. */ 70 uint64_t length; 71 72 /* The Output Receive Header. */ 73 union octeon_rh rh; 74 75 }; 76 77 #define LIO_DROQ_INFO_SIZE (sizeof(struct lio_droq_info)) 78 79 /* 80 * Pointer to data buffer. 81 * Driver keeps a pointer to the data buffer that it made available to 82 * the Octeon device. Since the descriptor ring keeps physical (bus) 83 * addresses, this field is required for the driver to keep track of 84 * the virtual address pointers. 85 */ 86 struct lio_recv_buffer { 87 /* Packet buffer, including metadata. */ 88 void *buffer; 89 90 /* Data in the packet buffer. */ 91 uint8_t *data; 92 }; 93 94 #define LIO_DROQ_RECVBUF_SIZE (sizeof(struct lio_recv_buffer)) 95 96 /* Output Queue statistics. Each output queue has four stats fields. */ 97 struct lio_droq_stats { 98 /* Number of packets received in this queue. */ 99 uint64_t pkts_received; 100 101 /* Bytes received by this queue. */ 102 uint64_t bytes_received; 103 104 /* Packets dropped due to no dispatch function. */ 105 uint64_t dropped_nodispatch; 106 107 /* Packets dropped due to no memory available. */ 108 uint64_t dropped_nomem; 109 110 /* Packets dropped due to large number of pkts to process. */ 111 uint64_t dropped_toomany; 112 113 /* Number of packets sent to stack from this queue. */ 114 uint64_t rx_pkts_received; 115 116 /* Number of Bytes sent to stack from this queue. */ 117 uint64_t rx_bytes_received; 118 119 /* Num of Packets dropped due to receive path failures. */ 120 uint64_t rx_dropped; 121 122 uint64_t rx_vxlan; 123 124 /* Num of failures of lio_recv_buffer_alloc() */ 125 uint64_t rx_alloc_failure; 126 127 }; 128 129 /* 130 * The maximum number of buffers that can be dispatched from the 131 * output/dma queue. Set to 64 assuming 1K buffers in DROQ and the fact that 132 * max packet size from DROQ is 64K. 133 */ 134 #define LIO_MAX_RECV_BUFS 64 135 136 /* 137 * Receive Packet format used when dispatching output queue packets 138 * with non-raw opcodes. 139 * The received packet will be sent to the upper layers using this 140 * structure which is passed as a parameter to the dispatch function 141 */ 142 struct lio_recv_pkt { 143 /* Number of buffers in this received packet */ 144 uint16_t buffer_count; 145 146 /* Id of the device that is sending the packet up */ 147 uint16_t octeon_id; 148 149 /* Length of data in the packet buffer */ 150 uint32_t length; 151 152 /* The receive header */ 153 union octeon_rh rh; 154 155 /* Pointer to the OS-specific packet buffer */ 156 struct mbuf *buffer_ptr[LIO_MAX_RECV_BUFS]; 157 158 /* Size of the buffers pointed to by ptr's in buffer_ptr */ 159 uint32_t buffer_size[LIO_MAX_RECV_BUFS]; 160 }; 161 162 #define LIO_RECV_PKT_SIZE (sizeof(struct lio_recv_pkt)) 163 164 /* 165 * The first parameter of a dispatch function. 166 * For a raw mode opcode, the driver dispatches with the device 167 * pointer in this structure. 168 * For non-raw mode opcode, the driver dispatches the recv_pkt 169 * created to contain the buffers with data received from Octeon. 170 * --------------------- 171 * | *recv_pkt ----|--- 172 * |-------------------| | 173 * | 0 or more bytes | | 174 * | reserved by driver| | 175 * |-------------------|<-/ 176 * | lio_recv_pkt | 177 * | | 178 * |___________________| 179 */ 180 struct lio_recv_info { 181 void *rsvd; 182 struct lio_recv_pkt *recv_pkt; 183 }; 184 185 #define LIO_RECV_INFO_SIZE (sizeof(struct lio_recv_info)) 186 187 /* 188 * Allocate a recv_info structure. The recv_pkt pointer in the recv_info 189 * structure is filled in before this call returns. 190 * @param extra_bytes - extra bytes to be allocated at the end of the recv info 191 * structure. 192 * @return - pointer to a newly allocated recv_info structure. 193 */ 194 static inline struct lio_recv_info * 195 lio_alloc_recv_info(int extra_bytes) 196 { 197 struct lio_recv_info *recv_info; 198 uint8_t *buf; 199 200 buf = malloc(LIO_RECV_PKT_SIZE + LIO_RECV_INFO_SIZE + 201 extra_bytes, M_DEVBUF, M_NOWAIT | M_ZERO); 202 if (buf == NULL) 203 return (NULL); 204 205 recv_info = (struct lio_recv_info *)buf; 206 recv_info->recv_pkt = (struct lio_recv_pkt *)(buf + LIO_RECV_INFO_SIZE); 207 recv_info->rsvd = NULL; 208 if (extra_bytes) 209 recv_info->rsvd = buf + LIO_RECV_INFO_SIZE + LIO_RECV_PKT_SIZE; 210 211 return (recv_info); 212 } 213 214 /* 215 * Free a recv_info structure. 216 * @param recv_info - Pointer to receive_info to be freed 217 */ 218 static inline void 219 lio_free_recv_info(struct lio_recv_info *recv_info) 220 { 221 222 free(recv_info, M_DEVBUF); 223 } 224 225 typedef int (*lio_dispatch_fn_t)(struct lio_recv_info *, void *); 226 227 /* 228 * Used by NIC module to register packet handler and to get device 229 * information for each octeon device. 230 */ 231 struct lio_droq_ops { 232 /* 233 * This registered function will be called by the driver with 234 * the pointer to buffer from droq and length of 235 * data in the buffer. The receive header gives the port 236 * number to the caller. Function pointer is set by caller. 237 */ 238 void (*fptr) (void *, uint32_t, union octeon_rh *, void *, 239 void *); 240 void *farg; 241 242 /* 243 * Flag indicating if the DROQ handler should drop packets that 244 * it cannot handle in one iteration. Set by caller. 245 */ 246 uint32_t drop_on_max; 247 }; 248 249 /* 250 * The Descriptor Ring Output Queue structure. 251 * This structure has all the information required to implement a 252 * Octeon DROQ. 253 */ 254 struct lio_droq { 255 /* A lock to protect access to this ring. */ 256 struct mtx lock; 257 258 uint32_t q_no; 259 260 uint32_t pkt_count; 261 262 struct lio_droq_ops ops; 263 264 struct octeon_device *oct_dev; 265 266 /* The 8B aligned descriptor ring starts at this address. */ 267 struct lio_droq_desc *desc_ring; 268 269 /* Index in the ring where the driver should read the next packet */ 270 uint32_t read_idx; 271 272 /* 273 * Index in the ring where the driver will refill the descriptor's 274 * buffer 275 */ 276 uint32_t refill_idx; 277 278 /* Packets pending to be processed */ 279 volatile int pkts_pending; 280 281 /* Number of descriptors in this ring. */ 282 uint32_t max_count; 283 284 /* The number of descriptors pending refill. */ 285 uint32_t refill_count; 286 287 uint32_t pkts_per_intr; 288 uint32_t refill_threshold; 289 290 /* 291 * The max number of descriptors in DROQ without a buffer. 292 * This field is used to keep track of empty space threshold. If the 293 * refill_count reaches this value, the DROQ cannot accept a max-sized 294 * (64K) packet. 295 */ 296 uint32_t max_empty_descs; 297 298 /* 299 * The receive buffer list. This list has the virtual addresses of 300 * the buffers. 301 */ 302 struct lio_recv_buffer *recv_buf_list; 303 304 /* The size of each buffer pointed by the buffer pointer. */ 305 uint32_t buffer_size; 306 307 /* 308 * Offset to packet credit register. 309 * Host writes number of info/buffer ptrs available to this register 310 */ 311 uint32_t pkts_credit_reg; 312 313 /* 314 * Offset packet sent register. 315 * Octeon writes the number of packets DMA'ed to host memory 316 * in this register. 317 */ 318 uint32_t pkts_sent_reg; 319 320 struct lio_stailq_head dispatch_stq_head; 321 322 /* Statistics for this DROQ. */ 323 struct lio_droq_stats stats; 324 325 /* DMA mapped address of the DROQ descriptor ring. */ 326 vm_paddr_t desc_ring_dma; 327 328 /* application context */ 329 void *app_ctx; 330 331 uint32_t cpu_id; 332 333 struct task droq_task; 334 struct taskqueue *droq_taskqueue; 335 336 struct lro_ctrl lro; 337 }; 338 339 #define LIO_DROQ_SIZE (sizeof(struct lio_droq)) 340 341 /* 342 * Allocates space for the descriptor ring for the droq and sets the 343 * base addr, num desc etc in Octeon registers. 344 * 345 * @param oct_dev - pointer to the octeon device structure 346 * @param q_no - droq no. 347 * @param app_ctx - pointer to application context 348 * @return Success: 0 Failure: 1 349 */ 350 int lio_init_droq(struct octeon_device *oct_dev, 351 uint32_t q_no, uint32_t num_descs, uint32_t desc_size, 352 void *app_ctx); 353 354 /* 355 * Frees the space for descriptor ring for the droq. 356 * 357 * @param oct_dev - pointer to the octeon device structure 358 * @param q_no - droq no. 359 * @return: Success: 0 Failure: 1 360 */ 361 int lio_delete_droq(struct octeon_device *oct_dev, uint32_t q_no); 362 363 /* 364 * Register a change in droq operations. The ops field has a pointer to a 365 * function which will called by the DROQ handler for all packets arriving 366 * on output queues given by q_no irrespective of the type of packet. 367 * The ops field also has a flag which if set tells the DROQ handler to 368 * drop packets if it receives more than what it can process in one 369 * invocation of the handler. 370 * @param oct - octeon device 371 * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 372 * @param ops - the droq_ops settings for this queue 373 * @return - 0 on success, -ENODEV or -EINVAL on error. 374 */ 375 int lio_register_droq_ops(struct octeon_device *oct, uint32_t q_no, 376 struct lio_droq_ops *ops); 377 378 /* 379 * Resets the function pointer and flag settings made by 380 * lio_register_droq_ops(). After this routine is called, the DROQ handler 381 * will lookup dispatch function for each arriving packet on the output queue 382 * given by q_no. 383 * @param oct - octeon device 384 * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 385 * @return - 0 on success, -ENODEV or -EINVAL on error. 386 */ 387 int lio_unregister_droq_ops(struct octeon_device *oct, uint32_t q_no); 388 389 /* 390 * Register a dispatch function for a opcode/subcode. The driver will call 391 * this dispatch function when it receives a packet with the given 392 * opcode/subcode in its output queues along with the user specified 393 * argument. 394 * @param oct - the octeon device to register with. 395 * @param opcode - the opcode for which the dispatch will be registered. 396 * @param subcode - the subcode for which the dispatch will be registered 397 * @param fn - the dispatch function. 398 * @param fn_arg - user specified that will be passed along with the 399 * dispatch function by the driver. 400 * @return Success: 0; Failure: 1 401 */ 402 int lio_register_dispatch_fn(struct octeon_device *oct, uint16_t opcode, 403 uint16_t subcode, lio_dispatch_fn_t fn, 404 void *fn_arg); 405 406 /* 407 * Remove registration for an opcode/subcode. This will delete the mapping for 408 * an opcode/subcode. The dispatch function will be unregistered and will no 409 * longer be called if a packet with the opcode/subcode arrives in the driver 410 * output queues. 411 * @param oct - the octeon device to unregister from. 412 * @param opcode - the opcode to be unregistered. 413 * @param subcode - the subcode to be unregistered. 414 * 415 * @return Success: 0; Failure: 1 416 */ 417 int lio_unregister_dispatch_fn(struct octeon_device *oct, uint16_t opcode, 418 uint16_t subcode); 419 420 uint32_t lio_droq_check_hw_for_pkts(struct lio_droq *droq); 421 422 int lio_create_droq(struct octeon_device *oct, uint32_t q_no, 423 uint32_t num_descs, uint32_t desc_size, void *app_ctx); 424 425 int lio_droq_process_packets(struct octeon_device *oct, 426 struct lio_droq *droq, uint32_t budget); 427 428 uint32_t lio_droq_refill(struct octeon_device *octeon_dev, 429 struct lio_droq *droq); 430 void lio_droq_bh(void *ptr, int pending __unused); 431 #endif /* __LIO_DROQ_H__ */ 432