1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SolidRun DPU driver for control plane 4 * 5 * Copyright (C) 2022-2023 SolidRun 6 * 7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com> 8 * 9 */ 10 11 #include <linux/iopoll.h> 12 13 #include "snet_vdpa.h" 14 15 enum snet_ctrl_opcodes { 16 SNET_CTRL_OP_DESTROY = 1, 17 SNET_CTRL_OP_READ_VQ_STATE, 18 SNET_CTRL_OP_SUSPEND, 19 SNET_CTRL_OP_RESUME, 20 }; 21 22 #define SNET_CTRL_TIMEOUT 2000000 23 24 #define SNET_CTRL_DATA_SIZE_MASK 0x0000FFFF 25 #define SNET_CTRL_IN_PROCESS_MASK 0x00010000 26 #define SNET_CTRL_CHUNK_RDY_MASK 0x00020000 27 #define SNET_CTRL_ERROR_MASK 0x0FFC0000 28 29 #define SNET_VAL_TO_ERR(val) (-(((val) & SNET_CTRL_ERROR_MASK) >> 18)) 30 #define SNET_EMPTY_CTRL(val) (((val) & SNET_CTRL_ERROR_MASK) || \ 31 !((val) & SNET_CTRL_IN_PROCESS_MASK)) 32 #define SNET_DATA_READY(val) ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK)) 33 34 /* Control register used to read data from the DPU */ 35 struct snet_ctrl_reg_ctrl { 36 /* Chunk size in 4B words */ 37 u16 data_size; 38 /* We are in the middle of a command */ 39 u16 in_process:1; 40 /* A data chunk is ready and can be consumed */ 41 u16 chunk_ready:1; 42 /* Error code */ 43 u16 error:10; 44 /* Saved for future usage */ 45 u16 rsvd:4; 46 }; 47 48 /* Opcode register */ 49 struct snet_ctrl_reg_op { 50 u16 opcode; 51 /* Only if VQ index is relevant for the command */ 52 u16 vq_idx; 53 }; 54 55 struct snet_ctrl_regs { 56 struct snet_ctrl_reg_op op; 57 struct snet_ctrl_reg_ctrl ctrl; 58 u32 rsvd; 59 u32 data[]; 60 }; 61 62 static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet) 63 { 64 return snet->bar + snet->psnet->cfg.ctrl_off; 65 } 66 67 static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs) 68 { 69 u32 val; 70 71 return readx_poll_timeout(ioread32, ®s->ctrl, val, SNET_EMPTY_CTRL(val), 10, 72 SNET_CTRL_TIMEOUT); 73 } 74 75 static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs) 76 { 77 u32 val; 78 79 return readx_poll_timeout(ioread32, ®s->op, val, !val, 10, SNET_CTRL_TIMEOUT); 80 } 81 82 static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs) 83 { 84 u32 val; 85 86 return readx_poll_timeout(ioread32, ®s->ctrl, val, SNET_DATA_READY(val), 10, 87 SNET_CTRL_TIMEOUT); 88 } 89 90 static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx) 91 { 92 return ioread32(&ctrl_regs->data[word_idx]); 93 } 94 95 static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs) 96 { 97 return ioread32(&ctrl_regs->ctrl); 98 } 99 100 static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val) 101 { 102 iowrite32(val, &ctrl_regs->ctrl); 103 } 104 105 static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val) 106 { 107 iowrite32(val, &ctrl_regs->op); 108 } 109 110 static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs) 111 { 112 /* Wait until the DPU finishes completely. 113 * It will clear the opcode register. 114 */ 115 return snet_wait_for_empty_op(ctrl_regs); 116 } 117 118 /* Reading ctrl from the DPU: 119 * buf_size must be 4B aligned 120 * 121 * Steps: 122 * 123 * (1) Verify that the DPU is not in the middle of another operation by 124 * reading the in_process and error bits in the control register. 125 * (2) Write the request opcode and the VQ idx in the opcode register 126 * and write the buffer size in the control register. 127 * (3) Start readind chunks of data, chunk_ready bit indicates that a 128 * data chunk is available, we signal that we read the data by clearing the bit. 129 * (4) Detect that the transfer is completed when the in_process bit 130 * in the control register is cleared or when the an error appears. 131 */ 132 static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer, 133 u32 buf_size) 134 { 135 struct pci_dev *pdev = snet->pdev; 136 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); 137 u32 *bfr_ptr = (u32 *)buffer; 138 u32 val; 139 u16 buf_words; 140 int ret; 141 u16 words, i, tot_words = 0; 142 143 /* Supported for config 2+ */ 144 if (!SNET_CFG_VER(snet, 2)) 145 return -EOPNOTSUPP; 146 147 if (!IS_ALIGNED(buf_size, 4)) 148 return -EINVAL; 149 150 mutex_lock(&snet->ctrl_lock); 151 152 buf_words = buf_size / 4; 153 154 /* Make sure control register is empty */ 155 ret = snet_wait_for_empty_ctrl(regs); 156 if (ret) { 157 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n"); 158 goto exit; 159 } 160 161 /* We need to write the buffer size in the control register, and the opcode + vq index in 162 * the opcode register. 163 * We use a spinlock to serialize the writes. 164 */ 165 spin_lock(&snet->ctrl_spinlock); 166 167 snet_write_ctrl(regs, buf_words); 168 snet_write_op(regs, opcode | (vq_idx << 16)); 169 170 spin_unlock(&snet->ctrl_spinlock); 171 172 while (buf_words != tot_words) { 173 ret = snet_wait_for_data(regs); 174 if (ret) { 175 SNET_WARN(pdev, "Timeout waiting for control data\n"); 176 goto exit; 177 } 178 179 val = snet_read_ctrl(regs); 180 181 /* Error? */ 182 if (val & SNET_CTRL_ERROR_MASK) { 183 ret = SNET_VAL_TO_ERR(val); 184 SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret); 185 goto exit; 186 } 187 188 words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words); 189 190 for (i = 0; i < words; i++) { 191 *bfr_ptr = snet_read32_word(regs, i); 192 bfr_ptr++; 193 } 194 195 tot_words += words; 196 197 /* Is the job completed? */ 198 if (!(val & SNET_CTRL_IN_PROCESS_MASK)) 199 break; 200 201 /* Clear the chunk ready bit and continue */ 202 val &= ~SNET_CTRL_CHUNK_RDY_MASK; 203 snet_write_ctrl(regs, val); 204 } 205 206 ret = snet_wait_for_dpu_completion(regs); 207 if (ret) 208 SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n"); 209 210 exit: 211 mutex_unlock(&snet->ctrl_lock); 212 return ret; 213 } 214 215 /* Send a control message to the DPU using the old mechanism 216 * used with config version 1. 217 */ 218 static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode) 219 { 220 struct pci_dev *pdev = snet->pdev; 221 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); 222 int ret; 223 224 mutex_lock(&snet->ctrl_lock); 225 226 /* Old mechanism uses just 1 register, the opcode register. 227 * Make sure that the opcode register is empty, and that the DPU isn't 228 * processing an old message. 229 */ 230 ret = snet_wait_for_empty_op(regs); 231 if (ret) { 232 SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n"); 233 goto exit; 234 } 235 236 /* Write the message */ 237 snet_write_op(regs, opcode); 238 239 /* DPU ACKs the message by clearing the opcode register */ 240 ret = snet_wait_for_empty_op(regs); 241 if (ret) 242 SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n"); 243 244 exit: 245 mutex_unlock(&snet->ctrl_lock); 246 return ret; 247 } 248 249 /* Send a control message to the DPU. 250 * A control message is a message without payload. 251 */ 252 static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx) 253 { 254 struct pci_dev *pdev = snet->pdev; 255 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); 256 u32 val; 257 int ret; 258 259 /* If config version is not 2+, use the old mechanism */ 260 if (!SNET_CFG_VER(snet, 2)) 261 return snet_send_ctrl_msg_old(snet, opcode); 262 263 mutex_lock(&snet->ctrl_lock); 264 265 /* Make sure control register is empty */ 266 ret = snet_wait_for_empty_ctrl(regs); 267 if (ret) { 268 SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n"); 269 goto exit; 270 } 271 272 /* We need to clear the control register and write the opcode + vq index in the opcode 273 * register. 274 * We use a spinlock to serialize the writes. 275 */ 276 spin_lock(&snet->ctrl_spinlock); 277 278 snet_write_ctrl(regs, 0); 279 snet_write_op(regs, opcode | (vq_idx << 16)); 280 281 spin_unlock(&snet->ctrl_spinlock); 282 283 /* The DPU ACKs control messages by setting the chunk ready bit 284 * without data. 285 */ 286 ret = snet_wait_for_data(regs); 287 if (ret) { 288 SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n"); 289 goto exit; 290 } 291 292 /* Check for errors */ 293 val = snet_read_ctrl(regs); 294 ret = SNET_VAL_TO_ERR(val); 295 296 /* Clear the chunk ready bit */ 297 val &= ~SNET_CTRL_CHUNK_RDY_MASK; 298 snet_write_ctrl(regs, val); 299 300 ret = snet_wait_for_dpu_completion(regs); 301 if (ret) 302 SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n", 303 ret); 304 305 exit: 306 mutex_unlock(&snet->ctrl_lock); 307 return ret; 308 } 309 310 void snet_ctrl_clear(struct snet *snet) 311 { 312 struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet); 313 314 snet_write_op(regs, 0); 315 } 316 317 int snet_destroy_dev(struct snet *snet) 318 { 319 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0); 320 } 321 322 int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state) 323 { 324 return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state, 325 sizeof(*state)); 326 } 327 328 int snet_suspend_dev(struct snet *snet) 329 { 330 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0); 331 } 332 333 int snet_resume_dev(struct snet *snet) 334 { 335 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_RESUME, 0); 336 } 337