1 /* 2 * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 * 5 * This program is free software; you may redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; version 2 of the License. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 10 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 11 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 12 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 13 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 14 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 15 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 16 * SOFTWARE. 17 * 18 */ 19 20 #ifndef _VNIC_RQ_H_ 21 #define _VNIC_RQ_H_ 22 23 #include <linux/pci.h> 24 #include <linux/netdevice.h> 25 26 #include "vnic_dev.h" 27 #include "vnic_cq.h" 28 29 /* Receive queue control */ 30 struct vnic_rq_ctrl { 31 u64 ring_base; /* 0x00 */ 32 u32 ring_size; /* 0x08 */ 33 u32 pad0; 34 u32 posted_index; /* 0x10 */ 35 u32 pad1; 36 u32 cq_index; /* 0x18 */ 37 u32 pad2; 38 u32 enable; /* 0x20 */ 39 u32 pad3; 40 u32 running; /* 0x28 */ 41 u32 pad4; 42 u32 fetch_index; /* 0x30 */ 43 u32 pad5; 44 u32 error_interrupt_enable; /* 0x38 */ 45 u32 pad6; 46 u32 error_interrupt_offset; /* 0x40 */ 47 u32 pad7; 48 u32 error_status; /* 0x48 */ 49 u32 pad8; 50 u32 dropped_packet_count; /* 0x50 */ 51 u32 pad9; 52 u32 dropped_packet_count_rc; /* 0x58 */ 53 u32 pad10; 54 }; 55 56 /* Break the vnic_rq_buf allocations into blocks of 32/64 entries */ 57 #define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32 58 #define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64 59 #define VNIC_RQ_BUF_BLK_ENTRIES(entries) \ 60 ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \ 61 VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES)) 62 #define VNIC_RQ_BUF_BLK_SZ(entries) \ 63 (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf)) 64 #define VNIC_RQ_BUF_BLKS_NEEDED(entries) \ 65 DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries)) 66 #define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096) 67 68 struct vnic_rq_buf { 69 struct vnic_rq_buf *next; 70 dma_addr_t dma_addr; 71 void *os_buf; 72 unsigned int os_buf_index; 73 unsigned int len; 74 unsigned int index; 75 void *desc; 76 uint64_t wr_id; 77 }; 78 79 enum enic_poll_state { 80 ENIC_POLL_STATE_IDLE, 81 ENIC_POLL_STATE_NAPI, 82 ENIC_POLL_STATE_POLL 83 }; 84 85 struct vnic_rq { 86 unsigned int index; 87 struct vnic_dev *vdev; 88 struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ 89 struct vnic_dev_ring ring; 90 struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX]; 91 struct vnic_rq_buf *to_use; 92 struct vnic_rq_buf *to_clean; 93 void *os_buf_head; 94 unsigned int pkts_outstanding; 95 #ifdef CONFIG_NET_RX_BUSY_POLL 96 atomic_t bpoll_state; 97 #endif /* CONFIG_NET_RX_BUSY_POLL */ 98 }; 99 100 static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) 101 { 102 /* how many does SW own? */ 103 return rq->ring.desc_avail; 104 } 105 106 static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq) 107 { 108 /* how many does HW own? */ 109 return rq->ring.desc_count - rq->ring.desc_avail - 1; 110 } 111 112 static inline void *vnic_rq_next_desc(struct vnic_rq *rq) 113 { 114 return rq->to_use->desc; 115 } 116 117 static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq) 118 { 119 return rq->to_use->index; 120 } 121 122 static inline void vnic_rq_post(struct vnic_rq *rq, 123 void *os_buf, unsigned int os_buf_index, 124 dma_addr_t dma_addr, unsigned int len, 125 uint64_t wrid) 126 { 127 struct vnic_rq_buf *buf = rq->to_use; 128 129 buf->os_buf = os_buf; 130 buf->os_buf_index = os_buf_index; 131 buf->dma_addr = dma_addr; 132 buf->len = len; 133 buf->wr_id = wrid; 134 135 buf = buf->next; 136 rq->to_use = buf; 137 rq->ring.desc_avail--; 138 139 /* Move the posted_index every nth descriptor 140 */ 141 142 #ifndef VNIC_RQ_RETURN_RATE 143 #define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */ 144 #endif 145 146 if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) { 147 /* Adding write memory barrier prevents compiler and/or CPU 148 * reordering, thus avoiding descriptor posting before 149 * descriptor is initialized. Otherwise, hardware can read 150 * stale descriptor fields. 151 */ 152 wmb(); 153 iowrite32(buf->index, &rq->ctrl->posted_index); 154 } 155 } 156 157 static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count) 158 { 159 rq->ring.desc_avail += count; 160 } 161 162 enum desc_return_options { 163 VNIC_RQ_RETURN_DESC, 164 VNIC_RQ_DEFER_RETURN_DESC, 165 }; 166 167 static inline void vnic_rq_service(struct vnic_rq *rq, 168 struct cq_desc *cq_desc, u16 completed_index, 169 int desc_return, void (*buf_service)(struct vnic_rq *rq, 170 struct cq_desc *cq_desc, struct vnic_rq_buf *buf, 171 int skipped, void *opaque), void *opaque) 172 { 173 struct vnic_rq_buf *buf; 174 int skipped; 175 176 buf = rq->to_clean; 177 while (1) { 178 179 skipped = (buf->index != completed_index); 180 181 (*buf_service)(rq, cq_desc, buf, skipped, opaque); 182 183 if (desc_return == VNIC_RQ_RETURN_DESC) 184 rq->ring.desc_avail++; 185 186 rq->to_clean = buf->next; 187 188 if (!skipped) 189 break; 190 191 buf = rq->to_clean; 192 } 193 } 194 195 static inline int vnic_rq_fill(struct vnic_rq *rq, 196 int (*buf_fill)(struct vnic_rq *rq)) 197 { 198 int err; 199 200 while (vnic_rq_desc_avail(rq) > 0) { 201 202 err = (*buf_fill)(rq); 203 if (err) 204 return err; 205 } 206 207 return 0; 208 } 209 210 #ifdef CONFIG_NET_RX_BUSY_POLL 211 static inline void enic_busy_poll_init_lock(struct vnic_rq *rq) 212 { 213 atomic_set(&rq->bpoll_state, ENIC_POLL_STATE_IDLE); 214 } 215 216 static inline bool enic_poll_lock_napi(struct vnic_rq *rq) 217 { 218 int rc = atomic_cmpxchg(&rq->bpoll_state, ENIC_POLL_STATE_IDLE, 219 ENIC_POLL_STATE_NAPI); 220 221 return (rc == ENIC_POLL_STATE_IDLE); 222 } 223 224 static inline void enic_poll_unlock_napi(struct vnic_rq *rq, 225 struct napi_struct *napi) 226 { 227 WARN_ON(atomic_read(&rq->bpoll_state) != ENIC_POLL_STATE_NAPI); 228 napi_gro_flush(napi, false); 229 atomic_set(&rq->bpoll_state, ENIC_POLL_STATE_IDLE); 230 } 231 232 static inline bool enic_poll_lock_poll(struct vnic_rq *rq) 233 { 234 int rc = atomic_cmpxchg(&rq->bpoll_state, ENIC_POLL_STATE_IDLE, 235 ENIC_POLL_STATE_POLL); 236 237 return (rc == ENIC_POLL_STATE_IDLE); 238 } 239 240 241 static inline void enic_poll_unlock_poll(struct vnic_rq *rq) 242 { 243 WARN_ON(atomic_read(&rq->bpoll_state) != ENIC_POLL_STATE_POLL); 244 atomic_set(&rq->bpoll_state, ENIC_POLL_STATE_IDLE); 245 } 246 247 static inline bool enic_poll_busy_polling(struct vnic_rq *rq) 248 { 249 return atomic_read(&rq->bpoll_state) & ENIC_POLL_STATE_POLL; 250 } 251 252 #else 253 254 static inline void enic_busy_poll_init_lock(struct vnic_rq *rq) 255 { 256 } 257 258 static inline bool enic_poll_lock_napi(struct vnic_rq *rq) 259 { 260 return true; 261 } 262 263 static inline bool enic_poll_unlock_napi(struct vnic_rq *rq, 264 struct napi_struct *napi) 265 { 266 return false; 267 } 268 269 static inline bool enic_poll_lock_poll(struct vnic_rq *rq) 270 { 271 return false; 272 } 273 274 static inline bool enic_poll_unlock_poll(struct vnic_rq *rq) 275 { 276 return false; 277 } 278 279 static inline bool enic_poll_ll_polling(struct vnic_rq *rq) 280 { 281 return false; 282 } 283 #endif /* CONFIG_NET_RX_BUSY_POLL */ 284 285 void vnic_rq_free(struct vnic_rq *rq); 286 int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, 287 unsigned int desc_count, unsigned int desc_size); 288 void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, 289 unsigned int error_interrupt_enable, 290 unsigned int error_interrupt_offset); 291 unsigned int vnic_rq_error_status(struct vnic_rq *rq); 292 void vnic_rq_enable(struct vnic_rq *rq); 293 int vnic_rq_disable(struct vnic_rq *rq); 294 void vnic_rq_clean(struct vnic_rq *rq, 295 void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)); 296 297 #endif /* _VNIC_RQ_H_ */ 298