1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved. 4 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/errno.h> 9 #include <linux/types.h> 10 #include <linux/pci.h> 11 #include <linux/delay.h> 12 #include <linux/slab.h> 13 14 #include "vnic_dev.h" 15 #include "vnic_rq.h" 16 #include "enic.h" 17 18 static int vnic_rq_alloc_bufs(struct vnic_rq *rq) 19 { 20 struct vnic_rq_buf *buf; 21 unsigned int i, j, count = rq->ring.desc_count; 22 unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count); 23 24 for (i = 0; i < blks; i++) { 25 rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_KERNEL); 26 if (!rq->bufs[i]) 27 return -ENOMEM; 28 } 29 30 for (i = 0; i < blks; i++) { 31 buf = rq->bufs[i]; 32 for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) { 33 buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j; 34 buf->desc = (u8 *)rq->ring.descs + 35 rq->ring.desc_size * buf->index; 36 if (buf->index + 1 == count) { 37 buf->next = rq->bufs[0]; 38 break; 39 } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) { 40 buf->next = rq->bufs[i + 1]; 41 } else { 42 buf->next = buf + 1; 43 buf++; 44 } 45 } 46 } 47 48 rq->to_use = rq->to_clean = rq->bufs[0]; 49 50 return 0; 51 } 52 53 void vnic_rq_free(struct vnic_rq *rq) 54 { 55 struct vnic_dev *vdev; 56 unsigned int i; 57 58 vdev = rq->vdev; 59 60 vnic_dev_free_desc_ring(vdev, &rq->ring); 61 62 for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) { 63 if (rq->bufs[i]) { 64 kfree(rq->bufs[i]); 65 rq->bufs[i] = NULL; 66 } 67 } 68 69 rq->ctrl = NULL; 70 } 71 72 int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, 73 unsigned int desc_count, unsigned int desc_size) 74 { 75 int err; 76 77 rq->index = index; 78 rq->vdev = vdev; 79 80 rq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_RQ, index); 81 if (!rq->ctrl) { 82 vdev_err(vdev, "Failed to hook RQ[%d] resource\n", index); 83 return -EINVAL; 84 } 85 86 vnic_rq_disable(rq); 87 88 err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size); 89 if (err) 90 return err; 91 92 err = vnic_rq_alloc_bufs(rq); 93 if (err) { 94 vnic_rq_free(rq); 95 return err; 96 } 97 98 return 0; 99 } 100 101 static void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, 102 unsigned int fetch_index, unsigned int posted_index, 103 unsigned int error_interrupt_enable, 104 unsigned int error_interrupt_offset) 105 { 106 u64 paddr; 107 unsigned int count = rq->ring.desc_count; 108 109 paddr = (u64)rq->ring.base_addr | VNIC_PADDR_TARGET; 110 writeq(paddr, &rq->ctrl->ring_base); 111 iowrite32(count, &rq->ctrl->ring_size); 112 iowrite32(cq_index, &rq->ctrl->cq_index); 113 iowrite32(error_interrupt_enable, &rq->ctrl->error_interrupt_enable); 114 iowrite32(error_interrupt_offset, &rq->ctrl->error_interrupt_offset); 115 iowrite32(0, &rq->ctrl->dropped_packet_count); 116 iowrite32(0, &rq->ctrl->error_status); 117 iowrite32(fetch_index, &rq->ctrl->fetch_index); 118 iowrite32(posted_index, &rq->ctrl->posted_index); 119 120 rq->to_use = rq->to_clean = 121 &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] 122 [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; 123 } 124 125 void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, 126 unsigned int error_interrupt_enable, 127 unsigned int error_interrupt_offset) 128 { 129 vnic_rq_init_start(rq, cq_index, 0, 0, error_interrupt_enable, 130 error_interrupt_offset); 131 } 132 133 unsigned int vnic_rq_error_status(struct vnic_rq *rq) 134 { 135 return ioread32(&rq->ctrl->error_status); 136 } 137 138 void vnic_rq_enable(struct vnic_rq *rq) 139 { 140 iowrite32(1, &rq->ctrl->enable); 141 } 142 143 int vnic_rq_disable(struct vnic_rq *rq) 144 { 145 unsigned int wait; 146 struct vnic_dev *vdev = rq->vdev; 147 int i; 148 149 /* Due to a race condition with clearing RQ "mini-cache" in hw, we need 150 * to disable the RQ twice to guarantee that stale descriptors are not 151 * used when this RQ is re-enabled. 152 */ 153 for (i = 0; i < 2; i++) { 154 iowrite32(0, &rq->ctrl->enable); 155 156 /* Wait for HW to ACK disable request */ 157 for (wait = 20000; wait > 0; wait--) 158 if (!ioread32(&rq->ctrl->running)) 159 break; 160 if (!wait) { 161 vdev_neterr(vdev, "Failed to disable RQ[%d]\n", 162 rq->index); 163 164 return -ETIMEDOUT; 165 } 166 } 167 168 return 0; 169 } 170 171 void vnic_rq_clean(struct vnic_rq *rq, 172 void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)) 173 { 174 struct vnic_rq_buf *buf; 175 u32 fetch_index; 176 unsigned int count = rq->ring.desc_count; 177 int i; 178 179 buf = rq->to_clean; 180 181 for (i = 0; i < rq->ring.desc_count; i++) { 182 (*buf_clean)(rq, buf); 183 buf = buf->next; 184 } 185 rq->ring.desc_avail = rq->ring.desc_count - 1; 186 187 /* Use current fetch_index as the ring starting point */ 188 fetch_index = ioread32(&rq->ctrl->fetch_index); 189 190 if (fetch_index == 0xFFFFFFFF) { /* check for hardware gone */ 191 /* Hardware surprise removal: reset fetch_index */ 192 fetch_index = 0; 193 } 194 rq->to_use = rq->to_clean = 195 &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] 196 [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; 197 iowrite32(fetch_index, &rq->ctrl->posted_index); 198 199 /* Anytime we write fetch_index, we need to re-write 0 to rq->enable 200 * to re-sync internal VIC state. 201 */ 202 iowrite32(0, &rq->ctrl->enable); 203 204 vnic_dev_clear_desc_ring(&rq->ring); 205 } 206