1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-21 Intel Corporation. 4 */ 5 6 #include "iosm_ipc_protocol.h" 7 #include "iosm_ipc_protocol_ops.h" 8 9 /* Get the next free message element.*/ 10 static union ipc_mem_msg_entry * 11 ipc_protocol_free_msg_get(struct iosm_protocol *ipc_protocol, int *index) 12 { 13 u32 head = le32_to_cpu(ipc_protocol->p_ap_shm->msg_head); 14 u32 new_head = (head + 1) % IPC_MEM_MSG_ENTRIES; 15 union ipc_mem_msg_entry *msg; 16 17 if (head >= IPC_MEM_MSG_ENTRIES) { 18 dev_err(ipc_protocol->dev, "invalid message head %u", head); 19 return NULL; 20 } 21 22 if (new_head == le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail)) { 23 dev_err(ipc_protocol->dev, "message ring is full"); 24 return NULL; 25 } 26 27 /* Get the pointer to the next free message element, 28 * reset the fields and mark is as invalid. 29 */ 30 msg = &ipc_protocol->p_ap_shm->msg_ring[head]; 31 memset(msg, 0, sizeof(*msg)); 32 33 /* return index in message ring */ 34 *index = head; 35 36 return msg; 37 } 38 39 /* Updates the message ring Head pointer */ 40 void ipc_protocol_msg_hp_update(struct iosm_imem *ipc_imem) 41 { 42 struct iosm_protocol *ipc_protocol = ipc_imem->ipc_protocol; 43 u32 head = le32_to_cpu(ipc_protocol->p_ap_shm->msg_head); 44 u32 new_head = (head + 1) % IPC_MEM_MSG_ENTRIES; 45 46 /* Update head pointer and fire doorbell. */ 47 ipc_protocol->p_ap_shm->msg_head = cpu_to_le32(new_head); 48 ipc_protocol->old_msg_tail = 49 le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail); 50 51 ipc_pm_signal_hpda_doorbell(&ipc_protocol->pm, IPC_HP_MR, false); 52 } 53 54 /* Allocate and prepare a OPEN_PIPE message. 55 * This also allocates the memory for the new TDR structure and 56 * updates the pipe structure referenced in the preparation arguments. 57 */ 58 static int ipc_protocol_msg_prepipe_open(struct iosm_protocol *ipc_protocol, 59 union ipc_msg_prep_args *args) 60 { 61 int index; 62 union ipc_mem_msg_entry *msg = 63 ipc_protocol_free_msg_get(ipc_protocol, &index); 64 struct ipc_pipe *pipe = args->pipe_open.pipe; 65 struct ipc_protocol_td *tdr; 66 struct sk_buff **skbr; 67 68 if (!msg) { 69 dev_err(ipc_protocol->dev, "failed to get free message"); 70 return -EIO; 71 } 72 73 /* Allocate the skbuf elements for the skbuf which are on the way. 74 * SKB ring is internal memory allocation for driver. No need to 75 * re-calculate the start and end addresses. 76 */ 77 skbr = kzalloc_objs(*skbr, pipe->nr_of_entries, GFP_ATOMIC); 78 if (!skbr) 79 return -ENOMEM; 80 81 /* Allocate the transfer descriptors for the pipe. */ 82 tdr = dma_alloc_coherent(&ipc_protocol->pcie->pci->dev, 83 pipe->nr_of_entries * sizeof(*tdr), 84 &pipe->phy_tdr_start, GFP_ATOMIC); 85 if (!tdr) { 86 kfree(skbr); 87 dev_err(ipc_protocol->dev, "tdr alloc error"); 88 return -ENOMEM; 89 } 90 91 pipe->max_nr_of_queued_entries = pipe->nr_of_entries - 1; 92 pipe->nr_of_queued_entries = 0; 93 pipe->tdr_start = tdr; 94 pipe->skbr_start = skbr; 95 pipe->old_tail = 0; 96 97 ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr] = 0; 98 99 msg->open_pipe.type_of_message = IPC_MEM_MSG_OPEN_PIPE; 100 msg->open_pipe.pipe_nr = pipe->pipe_nr; 101 msg->open_pipe.tdr_addr = cpu_to_le64(pipe->phy_tdr_start); 102 msg->open_pipe.tdr_entries = cpu_to_le16(pipe->nr_of_entries); 103 msg->open_pipe.accumulation_backoff = 104 cpu_to_le32(pipe->accumulation_backoff); 105 msg->open_pipe.irq_vector = cpu_to_le32(pipe->irq); 106 107 return index; 108 } 109 110 static int ipc_protocol_msg_prepipe_close(struct iosm_protocol *ipc_protocol, 111 union ipc_msg_prep_args *args) 112 { 113 int index = -1; 114 union ipc_mem_msg_entry *msg = 115 ipc_protocol_free_msg_get(ipc_protocol, &index); 116 struct ipc_pipe *pipe = args->pipe_close.pipe; 117 118 if (!msg) 119 return -EIO; 120 121 msg->close_pipe.type_of_message = IPC_MEM_MSG_CLOSE_PIPE; 122 msg->close_pipe.pipe_nr = pipe->pipe_nr; 123 124 dev_dbg(ipc_protocol->dev, "IPC_MEM_MSG_CLOSE_PIPE(pipe_nr=%d)", 125 msg->close_pipe.pipe_nr); 126 127 return index; 128 } 129 130 static int ipc_protocol_msg_prep_sleep(struct iosm_protocol *ipc_protocol, 131 union ipc_msg_prep_args *args) 132 { 133 int index = -1; 134 union ipc_mem_msg_entry *msg = 135 ipc_protocol_free_msg_get(ipc_protocol, &index); 136 137 if (!msg) { 138 dev_err(ipc_protocol->dev, "failed to get free message"); 139 return -EIO; 140 } 141 142 /* Prepare and send the host sleep message to CP to enter or exit D3. */ 143 msg->host_sleep.type_of_message = IPC_MEM_MSG_SLEEP; 144 msg->host_sleep.target = args->sleep.target; /* 0=host, 1=device */ 145 146 /* state; 0=enter, 1=exit 2=enter w/o protocol */ 147 msg->host_sleep.state = args->sleep.state; 148 149 dev_dbg(ipc_protocol->dev, "IPC_MEM_MSG_SLEEP(target=%d; state=%d)", 150 msg->host_sleep.target, msg->host_sleep.state); 151 152 return index; 153 } 154 155 static int ipc_protocol_msg_prep_feature_set(struct iosm_protocol *ipc_protocol, 156 union ipc_msg_prep_args *args) 157 { 158 int index = -1; 159 union ipc_mem_msg_entry *msg = 160 ipc_protocol_free_msg_get(ipc_protocol, &index); 161 162 if (!msg) { 163 dev_err(ipc_protocol->dev, "failed to get free message"); 164 return -EIO; 165 } 166 167 msg->feature_set.type_of_message = IPC_MEM_MSG_FEATURE_SET; 168 msg->feature_set.reset_enable = args->feature_set.reset_enable << 169 RESET_BIT; 170 171 dev_dbg(ipc_protocol->dev, "IPC_MEM_MSG_FEATURE_SET(reset_enable=%d)", 172 msg->feature_set.reset_enable >> RESET_BIT); 173 174 return index; 175 } 176 177 /* Processes the message consumed by CP. */ 178 bool ipc_protocol_msg_process(struct iosm_imem *ipc_imem, int irq) 179 { 180 struct iosm_protocol *ipc_protocol = ipc_imem->ipc_protocol; 181 struct ipc_rsp **rsp_ring = ipc_protocol->rsp_ring; 182 bool msg_processed = false; 183 u32 i; 184 185 if (le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail) >= 186 IPC_MEM_MSG_ENTRIES) { 187 dev_err(ipc_protocol->dev, "msg_tail out of range: %d", 188 le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail)); 189 return msg_processed; 190 } 191 192 if (irq != IMEM_IRQ_DONT_CARE && 193 irq != ipc_protocol->p_ap_shm->ci.msg_irq_vector) 194 return msg_processed; 195 196 for (i = ipc_protocol->old_msg_tail; 197 i != le32_to_cpu(ipc_protocol->p_ap_shm->msg_tail); 198 i = (i + 1) % IPC_MEM_MSG_ENTRIES) { 199 union ipc_mem_msg_entry *msg = 200 &ipc_protocol->p_ap_shm->msg_ring[i]; 201 202 dev_dbg(ipc_protocol->dev, "msg[%d]: type=%u status=%d", i, 203 msg->common.type_of_message, 204 msg->common.completion_status); 205 206 /* Update response with status and wake up waiting requestor */ 207 if (rsp_ring[i]) { 208 rsp_ring[i]->status = 209 le32_to_cpu(msg->common.completion_status); 210 complete(&rsp_ring[i]->completion); 211 rsp_ring[i] = NULL; 212 } 213 msg_processed = true; 214 } 215 216 ipc_protocol->old_msg_tail = i; 217 return msg_processed; 218 } 219 220 /* Sends data from UL list to CP for the provided pipe by updating the Head 221 * pointer of given pipe. 222 */ 223 bool ipc_protocol_ul_td_send(struct iosm_protocol *ipc_protocol, 224 struct ipc_pipe *pipe, 225 struct sk_buff_head *p_ul_list) 226 { 227 struct ipc_protocol_td *td; 228 bool hpda_pending = false; 229 struct sk_buff *skb; 230 s32 free_elements; 231 u32 head; 232 u32 tail; 233 234 if (!ipc_protocol->p_ap_shm) { 235 dev_err(ipc_protocol->dev, "driver is not initialized"); 236 return false; 237 } 238 239 /* Get head and tail of the td list and calculate 240 * the number of free elements. 241 */ 242 head = le32_to_cpu(ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr]); 243 tail = pipe->old_tail; 244 245 while (!skb_queue_empty(p_ul_list)) { 246 if (head < tail) 247 free_elements = tail - head - 1; 248 else 249 free_elements = 250 pipe->nr_of_entries - head + ((s32)tail - 1); 251 252 if (free_elements <= 0) { 253 dev_dbg(ipc_protocol->dev, 254 "no free td elements for UL pipe %d", 255 pipe->pipe_nr); 256 break; 257 } 258 259 /* Get the td address. */ 260 td = &pipe->tdr_start[head]; 261 262 /* Take the first element of the uplink list and add it 263 * to the td list. 264 */ 265 skb = skb_dequeue(p_ul_list); 266 if (WARN_ON(!skb)) 267 break; 268 269 /* Save the reference to the uplink skbuf. */ 270 pipe->skbr_start[head] = skb; 271 272 td->buffer.address = IPC_CB(skb)->mapping; 273 td->scs = cpu_to_le32(skb->len) & cpu_to_le32(SIZE_MASK); 274 td->next = 0; 275 276 pipe->nr_of_queued_entries++; 277 278 /* Calculate the new head and save it. */ 279 head++; 280 if (head >= pipe->nr_of_entries) 281 head = 0; 282 283 ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr] = 284 cpu_to_le32(head); 285 } 286 287 if (pipe->old_head != head) { 288 dev_dbg(ipc_protocol->dev, "New UL TDs Pipe:%d", pipe->pipe_nr); 289 290 pipe->old_head = head; 291 /* Trigger doorbell because of pending UL packets. */ 292 hpda_pending = true; 293 } 294 295 return hpda_pending; 296 } 297 298 /* Checks for Tail pointer update from CP and returns the data as SKB. */ 299 struct sk_buff *ipc_protocol_ul_td_process(struct iosm_protocol *ipc_protocol, 300 struct ipc_pipe *pipe) 301 { 302 struct ipc_protocol_td *p_td = &pipe->tdr_start[pipe->old_tail]; 303 struct sk_buff *skb = pipe->skbr_start[pipe->old_tail]; 304 305 pipe->nr_of_queued_entries--; 306 pipe->old_tail++; 307 if (pipe->old_tail >= pipe->nr_of_entries) 308 pipe->old_tail = 0; 309 310 if (!p_td->buffer.address) { 311 dev_err(ipc_protocol->dev, "Td buffer address is NULL"); 312 return NULL; 313 } 314 315 if (p_td->buffer.address != IPC_CB(skb)->mapping) { 316 dev_err(ipc_protocol->dev, 317 "pipe %d: invalid buf_addr or skb_data", 318 pipe->pipe_nr); 319 return NULL; 320 } 321 322 return skb; 323 } 324 325 /* Allocates an SKB for CP to send data and updates the Head Pointer 326 * of the given Pipe#. 327 */ 328 bool ipc_protocol_dl_td_prepare(struct iosm_protocol *ipc_protocol, 329 struct ipc_pipe *pipe) 330 { 331 struct ipc_protocol_td *td; 332 dma_addr_t mapping = 0; 333 u32 head, new_head; 334 struct sk_buff *skb; 335 u32 tail; 336 337 /* Get head and tail of the td list and calculate 338 * the number of free elements. 339 */ 340 head = le32_to_cpu(ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr]); 341 tail = le32_to_cpu(ipc_protocol->p_ap_shm->tail_array[pipe->pipe_nr]); 342 343 new_head = head + 1; 344 if (new_head >= pipe->nr_of_entries) 345 new_head = 0; 346 347 if (new_head == tail) 348 return false; 349 350 /* Get the td address. */ 351 td = &pipe->tdr_start[head]; 352 353 /* Allocate the skbuf for the descriptor. */ 354 skb = ipc_pcie_alloc_skb(ipc_protocol->pcie, pipe->buf_size, GFP_ATOMIC, 355 &mapping, DMA_FROM_DEVICE, 356 IPC_MEM_DL_ETH_OFFSET); 357 if (!skb) 358 return false; 359 360 td->buffer.address = mapping; 361 td->scs = cpu_to_le32(pipe->buf_size) & cpu_to_le32(SIZE_MASK); 362 td->next = 0; 363 364 /* store the new head value. */ 365 ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr] = 366 cpu_to_le32(new_head); 367 368 /* Save the reference to the skbuf. */ 369 pipe->skbr_start[head] = skb; 370 371 pipe->nr_of_queued_entries++; 372 373 return true; 374 } 375 376 /* Processes DL TD's */ 377 struct sk_buff *ipc_protocol_dl_td_process(struct iosm_protocol *ipc_protocol, 378 struct ipc_pipe *pipe) 379 { 380 struct ipc_protocol_td *p_td; 381 struct sk_buff *skb; 382 383 if (!pipe->tdr_start) 384 return NULL; 385 386 /* Copy the reference to the downlink buffer. */ 387 p_td = &pipe->tdr_start[pipe->old_tail]; 388 skb = pipe->skbr_start[pipe->old_tail]; 389 390 /* Reset the ring elements. */ 391 pipe->skbr_start[pipe->old_tail] = NULL; 392 393 pipe->nr_of_queued_entries--; 394 395 pipe->old_tail++; 396 if (pipe->old_tail >= pipe->nr_of_entries) 397 pipe->old_tail = 0; 398 399 if (!skb) { 400 dev_err(ipc_protocol->dev, "skb is null"); 401 goto ret; 402 } else if (!p_td->buffer.address) { 403 dev_err(ipc_protocol->dev, "td/buffer address is null"); 404 ipc_pcie_kfree_skb(ipc_protocol->pcie, skb); 405 skb = NULL; 406 goto ret; 407 } 408 409 if (p_td->buffer.address != IPC_CB(skb)->mapping) { 410 dev_err(ipc_protocol->dev, "invalid buf=%llx or skb=%p", 411 (unsigned long long)p_td->buffer.address, skb->data); 412 ipc_pcie_kfree_skb(ipc_protocol->pcie, skb); 413 skb = NULL; 414 goto ret; 415 } else if ((le32_to_cpu(p_td->scs) & SIZE_MASK) > pipe->buf_size) { 416 dev_err(ipc_protocol->dev, "invalid buffer size %d > %d", 417 le32_to_cpu(p_td->scs) & SIZE_MASK, 418 pipe->buf_size); 419 ipc_pcie_kfree_skb(ipc_protocol->pcie, skb); 420 skb = NULL; 421 goto ret; 422 } else if (le32_to_cpu(p_td->scs) >> COMPLETION_STATUS == 423 IPC_MEM_TD_CS_ABORT) { 424 /* Discard aborted buffers. */ 425 dev_dbg(ipc_protocol->dev, "discard 'aborted' buffers"); 426 ipc_pcie_kfree_skb(ipc_protocol->pcie, skb); 427 skb = NULL; 428 goto ret; 429 } 430 431 /* Set the length field in skbuf. */ 432 skb_put(skb, le32_to_cpu(p_td->scs) & SIZE_MASK); 433 434 ret: 435 return skb; 436 } 437 438 void ipc_protocol_get_head_tail_index(struct iosm_protocol *ipc_protocol, 439 struct ipc_pipe *pipe, u32 *head, 440 u32 *tail) 441 { 442 struct ipc_protocol_ap_shm *ipc_ap_shm = ipc_protocol->p_ap_shm; 443 444 if (head) 445 *head = le32_to_cpu(ipc_ap_shm->head_array[pipe->pipe_nr]); 446 447 if (tail) 448 *tail = le32_to_cpu(ipc_ap_shm->tail_array[pipe->pipe_nr]); 449 } 450 451 /* Frees the TDs given to CP. */ 452 void ipc_protocol_pipe_cleanup(struct iosm_protocol *ipc_protocol, 453 struct ipc_pipe *pipe) 454 { 455 struct sk_buff *skb; 456 u32 head; 457 u32 tail; 458 459 /* Get the start and the end of the buffer list. */ 460 head = le32_to_cpu(ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr]); 461 tail = pipe->old_tail; 462 463 /* Reset tail and head to 0. */ 464 ipc_protocol->p_ap_shm->tail_array[pipe->pipe_nr] = 0; 465 ipc_protocol->p_ap_shm->head_array[pipe->pipe_nr] = 0; 466 467 /* Free pending uplink and downlink buffers. */ 468 if (pipe->skbr_start) { 469 while (head != tail) { 470 /* Get the reference to the skbuf, 471 * which is on the way and free it. 472 */ 473 skb = pipe->skbr_start[tail]; 474 if (skb) 475 ipc_pcie_kfree_skb(ipc_protocol->pcie, skb); 476 477 tail++; 478 if (tail >= pipe->nr_of_entries) 479 tail = 0; 480 } 481 482 kfree(pipe->skbr_start); 483 pipe->skbr_start = NULL; 484 } 485 486 pipe->old_tail = 0; 487 488 /* Free and reset the td and skbuf circular buffers. kfree is save! */ 489 if (pipe->tdr_start) { 490 dma_free_coherent(&ipc_protocol->pcie->pci->dev, 491 sizeof(*pipe->tdr_start) * pipe->nr_of_entries, 492 pipe->tdr_start, pipe->phy_tdr_start); 493 494 pipe->tdr_start = NULL; 495 } 496 } 497 498 enum ipc_mem_device_ipc_state ipc_protocol_get_ipc_status(struct iosm_protocol 499 *ipc_protocol) 500 { 501 return (enum ipc_mem_device_ipc_state) 502 le32_to_cpu(ipc_protocol->p_ap_shm->device_info.ipc_status); 503 } 504 505 enum ipc_mem_exec_stage 506 ipc_protocol_get_ap_exec_stage(struct iosm_protocol *ipc_protocol) 507 { 508 return le32_to_cpu(ipc_protocol->p_ap_shm->device_info.execution_stage); 509 } 510 511 int ipc_protocol_msg_prep(struct iosm_imem *ipc_imem, 512 enum ipc_msg_prep_type msg_type, 513 union ipc_msg_prep_args *args) 514 { 515 struct iosm_protocol *ipc_protocol = ipc_imem->ipc_protocol; 516 517 switch (msg_type) { 518 case IPC_MSG_PREP_SLEEP: 519 return ipc_protocol_msg_prep_sleep(ipc_protocol, args); 520 521 case IPC_MSG_PREP_PIPE_OPEN: 522 return ipc_protocol_msg_prepipe_open(ipc_protocol, args); 523 524 case IPC_MSG_PREP_PIPE_CLOSE: 525 return ipc_protocol_msg_prepipe_close(ipc_protocol, args); 526 527 case IPC_MSG_PREP_FEATURE_SET: 528 return ipc_protocol_msg_prep_feature_set(ipc_protocol, args); 529 530 /* Unsupported messages in protocol */ 531 case IPC_MSG_PREP_MAP: 532 case IPC_MSG_PREP_UNMAP: 533 default: 534 dev_err(ipc_protocol->dev, 535 "unsupported message type: %d in protocol", msg_type); 536 return -EINVAL; 537 } 538 } 539 540 u32 541 ipc_protocol_pm_dev_get_sleep_notification(struct iosm_protocol *ipc_protocol) 542 { 543 struct ipc_protocol_ap_shm *ipc_ap_shm = ipc_protocol->p_ap_shm; 544 545 return le32_to_cpu(ipc_ap_shm->device_info.device_sleep_notification); 546 } 547