1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SolidRun DPU driver for control plane 4 * 5 * Copyright (C) 2022-2023 SolidRun 6 * 7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com> 8 * 9 */ 10 11 #include <linux/iopoll.h> 12 13 #include "snet_vdpa.h" 14 15 enum snet_ctrl_opcodes { 16 SNET_CTRL_OP_DESTROY = 1, 17 SNET_CTRL_OP_READ_VQ_STATE, 18 SNET_CTRL_OP_SUSPEND, 19 }; 20 21 #define SNET_CTRL_TIMEOUT 2000000 22 23 #define SNET_CTRL_DATA_SIZE_MASK 0x0000FFFF 24 #define SNET_CTRL_IN_PROCESS_MASK 0x00010000 25 #define SNET_CTRL_CHUNK_RDY_MASK 0x00020000 26 #define SNET_CTRL_ERROR_MASK 0x0FFC0000 27 28 #define SNET_VAL_TO_ERR(val) (-(((val) & SNET_CTRL_ERROR_MASK) >> 18)) 29 #define SNET_EMPTY_CTRL(val) (((val) & SNET_CTRL_ERROR_MASK) || \ 30 !((val) & SNET_CTRL_IN_PROCESS_MASK)) 31 #define SNET_DATA_READY(val) ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK)) 32 33 /* Control register used to read data from the DPU */ 34 struct snet_ctrl_reg_ctrl { 35 /* Chunk size in 4B words */ 36 u16 data_size; 37 /* We are in the middle of a command */ 38 u16 in_process:1; 39 /* A data chunk is ready and can be consumed */ 40 u16 chunk_ready:1; 41 /* Error code */ 42 u16 error:10; 43 /* Saved for future usage */ 44 u16 rsvd:4; 45 }; 46 47 /* Opcode register */ 48 struct snet_ctrl_reg_op { 49 u16 opcode; 50 /* Only if VQ index is relevant for the command */ 51 u16 vq_idx; 52 }; 53 54 struct snet_ctrl_regs { 55 struct snet_ctrl_reg_op op; 56 struct snet_ctrl_reg_ctrl ctrl; 57 u32 rsvd; 58 u32 data[]; 59 }; 60 61 static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet) 62 { 63 return snet->bar + snet->psnet->cfg.ctrl_off; 64 } 65 66 static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs) 67 { 68 u32 val; 69 70 return readx_poll_timeout(ioread32, ®s->ctrl, val, SNET_EMPTY_CTRL(val), 10, 71 SNET_CTRL_TIMEOUT); 72 } 73 74 static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs) 75 { 76 u32 val; 77 78 return readx_poll_timeout(ioread32, ®s->op, val, !val, 10, SNET_CTRL_TIMEOUT); 79 } 80 81 static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs) 82 { 83 u32 val; 84 85 return readx_poll_timeout(ioread32, ®s->ctrl, val, SNET_DATA_READY(val), 10, 86 SNET_CTRL_TIMEOUT); 87 } 88 89 static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx) 90 { 91 return ioread32(&ctrl_regs->data[word_idx]); 92 } 93 94 static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs) 95 { 96 return ioread32(&ctrl_regs->ctrl); 97 } 98 99 static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val) 100 { 101 iowrite32(val, &ctrl_regs->ctrl); 102 } 103 104 static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val) 105 { 106 iowrite32(val, &ctrl_regs->op); 107 } 108 109 static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs) 110 { 111 /* Wait until the DPU finishes completely. 112 * It will clear the opcode register. 113 */ 114 return snet_wait_for_empty_op(ctrl_regs); 115 } 116 117 /* Reading ctrl from the DPU: 118 * buf_size must be 4B aligned 119 * 120 * Steps: 121 * 122 * (1) Verify that the DPU is not in the middle of another operation by 123 * reading the in_process and error bits in the control register. 124 * (2) Write the request opcode and the VQ idx in the opcode register 125 * and write the buffer size in the control register. 126 * (3) Start readind chunks of data, chunk_ready bit indicates that a 127 * data chunk is available, we signal that we read the data by clearing the bit. 128 * (4) Detect that the transfer is completed when the in_process bit 129 * in the control register is cleared or when the an error appears. 130 */ 131 static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer, 132 u32 buf_size) 133 { 134 struct pci_dev *pdev = snet->pdev; 135 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); 136 u32 *bfr_ptr = (u32 *)buffer; 137 u32 val; 138 u16 buf_words; 139 int ret; 140 u16 words, i, tot_words = 0; 141 142 /* Supported for config 2+ */ 143 if (!SNET_CFG_VER(snet, 2)) 144 return -EOPNOTSUPP; 145 146 if (!IS_ALIGNED(buf_size, 4)) 147 return -EINVAL; 148 149 mutex_lock(&snet->ctrl_lock); 150 151 buf_words = buf_size / 4; 152 153 /* Make sure control register is empty */ 154 ret = snet_wait_for_empty_ctrl(regs); 155 if (ret) { 156 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n"); 157 goto exit; 158 } 159 160 /* We need to write the buffer size in the control register, and the opcode + vq index in 161 * the opcode register. 162 * We use a spinlock to serialize the writes. 163 */ 164 spin_lock(&snet->ctrl_spinlock); 165 166 snet_write_ctrl(regs, buf_words); 167 snet_write_op(regs, opcode | (vq_idx << 16)); 168 169 spin_unlock(&snet->ctrl_spinlock); 170 171 while (buf_words != tot_words) { 172 ret = snet_wait_for_data(regs); 173 if (ret) { 174 SNET_WARN(pdev, "Timeout waiting for control data\n"); 175 goto exit; 176 } 177 178 val = snet_read_ctrl(regs); 179 180 /* Error? */ 181 if (val & SNET_CTRL_ERROR_MASK) { 182 ret = SNET_VAL_TO_ERR(val); 183 SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret); 184 goto exit; 185 } 186 187 words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words); 188 189 for (i = 0; i < words; i++) { 190 *bfr_ptr = snet_read32_word(regs, i); 191 bfr_ptr++; 192 } 193 194 tot_words += words; 195 196 /* Is the job completed? */ 197 if (!(val & SNET_CTRL_IN_PROCESS_MASK)) 198 break; 199 200 /* Clear the chunk ready bit and continue */ 201 val &= ~SNET_CTRL_CHUNK_RDY_MASK; 202 snet_write_ctrl(regs, val); 203 } 204 205 ret = snet_wait_for_dpu_completion(regs); 206 if (ret) 207 SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n"); 208 209 exit: 210 mutex_unlock(&snet->ctrl_lock); 211 return ret; 212 } 213 214 /* Send a control message to the DPU using the old mechanism 215 * used with config version 1. 216 */ 217 static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode) 218 { 219 struct pci_dev *pdev = snet->pdev; 220 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); 221 int ret; 222 223 mutex_lock(&snet->ctrl_lock); 224 225 /* Old mechanism uses just 1 register, the opcode register. 226 * Make sure that the opcode register is empty, and that the DPU isn't 227 * processing an old message. 228 */ 229 ret = snet_wait_for_empty_op(regs); 230 if (ret) { 231 SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n"); 232 goto exit; 233 } 234 235 /* Write the message */ 236 snet_write_op(regs, opcode); 237 238 /* DPU ACKs the message by clearing the opcode register */ 239 ret = snet_wait_for_empty_op(regs); 240 if (ret) 241 SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n"); 242 243 exit: 244 mutex_unlock(&snet->ctrl_lock); 245 return ret; 246 } 247 248 /* Send a control message to the DPU. 249 * A control message is a message without payload. 250 */ 251 static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx) 252 { 253 struct pci_dev *pdev = snet->pdev; 254 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); 255 u32 val; 256 int ret; 257 258 /* If config version is not 2+, use the old mechanism */ 259 if (!SNET_CFG_VER(snet, 2)) 260 return snet_send_ctrl_msg_old(snet, opcode); 261 262 mutex_lock(&snet->ctrl_lock); 263 264 /* Make sure control register is empty */ 265 ret = snet_wait_for_empty_ctrl(regs); 266 if (ret) { 267 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n"); 268 goto exit; 269 } 270 271 /* We need to clear the control register and write the opcode + vq index in the opcode 272 * register. 273 * We use a spinlock to serialize the writes. 274 */ 275 spin_lock(&snet->ctrl_spinlock); 276 277 snet_write_ctrl(regs, 0); 278 snet_write_op(regs, opcode | (vq_idx << 16)); 279 280 spin_unlock(&snet->ctrl_spinlock); 281 282 /* The DPU ACKs control messages by setting the chunk ready bit 283 * without data. 284 */ 285 ret = snet_wait_for_data(regs); 286 if (ret) { 287 SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n"); 288 goto exit; 289 } 290 291 /* Check for errors */ 292 val = snet_read_ctrl(regs); 293 ret = SNET_VAL_TO_ERR(val); 294 295 /* Clear the chunk ready bit */ 296 val &= ~SNET_CTRL_CHUNK_RDY_MASK; 297 snet_write_ctrl(regs, val); 298 299 ret = snet_wait_for_dpu_completion(regs); 300 if (ret) 301 SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n", 302 ret); 303 304 exit: 305 mutex_unlock(&snet->ctrl_lock); 306 return ret; 307 } 308 309 void snet_ctrl_clear(struct snet *snet) 310 { 311 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); 312 313 snet_write_op(regs, 0); 314 } 315 316 int snet_destroy_dev(struct snet *snet) 317 { 318 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0); 319 } 320 321 int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state) 322 { 323 return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state, 324 sizeof(*state)); 325 } 326 327 int snet_suspend_dev(struct snet *snet) 328 { 329 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0); 330 } 331