1 // SPDX-License-Identifier: GPL-2.0
2
3 /*
4 * Copyright 2016-2022 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8 #include "habanalabs.h"
9
10 #include <linux/slab.h>
11
12 /**
13 * struct hl_eqe_work - This structure is used to schedule work of EQ
14 * entry and cpucp_reset event
15 *
16 * @eq_work: workqueue object to run when EQ entry is received
17 * @hdev: pointer to device structure
18 * @eq_entry: copy of the EQ entry
19 */
20 struct hl_eqe_work {
21 struct work_struct eq_work;
22 struct hl_device *hdev;
23 struct hl_eq_entry eq_entry;
24 };
25
26 /**
27 * hl_cq_inc_ptr - increment ci or pi of cq
28 *
29 * @ptr: the current ci or pi value of the completion queue
30 *
31 * Increment ptr by 1. If it reaches the number of completion queue
32 * entries, set it to 0
33 */
hl_cq_inc_ptr(u32 ptr)34 inline u32 hl_cq_inc_ptr(u32 ptr)
35 {
36 ptr++;
37 if (unlikely(ptr == HL_CQ_LENGTH))
38 ptr = 0;
39 return ptr;
40 }
41
42 /**
43 * hl_eq_inc_ptr - increment ci of eq
44 *
45 * @ptr: the current ci value of the event queue
46 *
47 * Increment ptr by 1. If it reaches the number of event queue
48 * entries, set it to 0
49 */
hl_eq_inc_ptr(u32 ptr)50 static inline u32 hl_eq_inc_ptr(u32 ptr)
51 {
52 ptr++;
53 if (unlikely(ptr == HL_EQ_LENGTH))
54 ptr = 0;
55 return ptr;
56 }
57
irq_handle_eqe(struct work_struct * work)58 static void irq_handle_eqe(struct work_struct *work)
59 {
60 struct hl_eqe_work *eqe_work = container_of(work, struct hl_eqe_work,
61 eq_work);
62 struct hl_device *hdev = eqe_work->hdev;
63
64 hdev->asic_funcs->handle_eqe(hdev, &eqe_work->eq_entry);
65
66 kfree(eqe_work);
67 }
68
69 /**
70 * job_finish - queue job finish work
71 *
72 * @hdev: pointer to device structure
73 * @cs_seq: command submission sequence
74 * @cq: completion queue
75 * @timestamp: interrupt timestamp
76 *
77 */
job_finish(struct hl_device * hdev,u32 cs_seq,struct hl_cq * cq,ktime_t timestamp)78 static void job_finish(struct hl_device *hdev, u32 cs_seq, struct hl_cq *cq, ktime_t timestamp)
79 {
80 struct hl_hw_queue *queue;
81 struct hl_cs_job *job;
82
83 queue = &hdev->kernel_queues[cq->hw_queue_id];
84 job = queue->shadow_queue[hl_pi_2_offset(cs_seq)];
85 job->timestamp = timestamp;
86 queue_work(hdev->cq_wq[cq->cq_idx], &job->finish_work);
87
88 atomic_inc(&queue->ci);
89 }
90
91 /**
92 * cs_finish - queue all cs jobs finish work
93 *
94 * @hdev: pointer to device structure
95 * @cs_seq: command submission sequence
96 * @timestamp: interrupt timestamp
97 *
98 */
cs_finish(struct hl_device * hdev,u16 cs_seq,ktime_t timestamp)99 static void cs_finish(struct hl_device *hdev, u16 cs_seq, ktime_t timestamp)
100 {
101 struct asic_fixed_properties *prop = &hdev->asic_prop;
102 struct hl_hw_queue *queue;
103 struct hl_cs *cs;
104 struct hl_cs_job *job;
105
106 cs = hdev->shadow_cs_queue[cs_seq & (prop->max_pending_cs - 1)];
107 if (!cs) {
108 dev_warn(hdev->dev,
109 "No pointer to CS in shadow array at index %d\n",
110 cs_seq);
111 return;
112 }
113
114 list_for_each_entry(job, &cs->job_list, cs_node) {
115 queue = &hdev->kernel_queues[job->hw_queue_id];
116 atomic_inc(&queue->ci);
117 }
118
119 cs->completion_timestamp = timestamp;
120 queue_work(hdev->cs_cmplt_wq, &cs->finish_work);
121 }
122
123 /**
124 * hl_irq_handler_cq - irq handler for completion queue
125 *
126 * @irq: irq number
127 * @arg: pointer to completion queue structure
128 *
129 */
hl_irq_handler_cq(int irq,void * arg)130 irqreturn_t hl_irq_handler_cq(int irq, void *arg)
131 {
132 struct hl_cq *cq = arg;
133 struct hl_device *hdev = cq->hdev;
134 bool shadow_index_valid, entry_ready;
135 u16 shadow_index;
136 struct hl_cq_entry *cq_entry, *cq_base;
137 ktime_t timestamp = ktime_get();
138
139 if (hdev->disabled) {
140 dev_dbg(hdev->dev,
141 "Device disabled but received IRQ %d for CQ %d\n",
142 irq, cq->hw_queue_id);
143 return IRQ_HANDLED;
144 }
145
146 cq_base = cq->kernel_address;
147
148 while (1) {
149 cq_entry = (struct hl_cq_entry *) &cq_base[cq->ci];
150
151 entry_ready = !!FIELD_GET(CQ_ENTRY_READY_MASK,
152 le32_to_cpu(cq_entry->data));
153 if (!entry_ready)
154 break;
155
156 /* Make sure we read CQ entry contents after we've
157 * checked the ownership bit.
158 */
159 dma_rmb();
160
161 shadow_index_valid =
162 !!FIELD_GET(CQ_ENTRY_SHADOW_INDEX_VALID_MASK,
163 le32_to_cpu(cq_entry->data));
164
165 shadow_index = FIELD_GET(CQ_ENTRY_SHADOW_INDEX_MASK,
166 le32_to_cpu(cq_entry->data));
167
168 /*
169 * CQ interrupt handler has 2 modes of operation:
170 * 1. Interrupt per CS completion: (Single CQ for all queues)
171 * CQ entry represents a completed CS
172 *
173 * 2. Interrupt per CS job completion in queue: (CQ per queue)
174 * CQ entry represents a completed job in a certain queue
175 */
176 if (shadow_index_valid && !hdev->disabled) {
177 if (hdev->asic_prop.completion_mode ==
178 HL_COMPLETION_MODE_CS)
179 cs_finish(hdev, shadow_index, timestamp);
180 else
181 job_finish(hdev, shadow_index, cq, timestamp);
182 }
183
184 /* Clear CQ entry ready bit */
185 cq_entry->data = cpu_to_le32(le32_to_cpu(cq_entry->data) &
186 ~CQ_ENTRY_READY_MASK);
187
188 cq->ci = hl_cq_inc_ptr(cq->ci);
189
190 /* Increment free slots */
191 atomic_inc(&cq->free_slots_cnt);
192 }
193
194 return IRQ_HANDLED;
195 }
196
197 /*
198 * hl_ts_free_objects - handler of the free objects workqueue.
199 * This function should put refcount to objects that the registration node
200 * took refcount to them.
201 * @work: workqueue object pointer
202 */
hl_ts_free_objects(struct work_struct * work)203 static void hl_ts_free_objects(struct work_struct *work)
204 {
205 struct timestamp_reg_work_obj *job =
206 container_of(work, struct timestamp_reg_work_obj, free_obj);
207 struct list_head *dynamic_alloc_free_list_head = job->dynamic_alloc_free_obj_head;
208 struct timestamp_reg_free_node *free_obj, *temp_free_obj;
209 struct list_head *free_list_head = job->free_obj_head;
210
211 struct hl_device *hdev = job->hdev;
212
213 list_for_each_entry_safe(free_obj, temp_free_obj, free_list_head, free_objects_node) {
214 dev_dbg(hdev->dev, "About to put refcount to buf (%p) cq_cb(%p)\n",
215 free_obj->buf,
216 free_obj->cq_cb);
217
218 hl_mmap_mem_buf_put(free_obj->buf);
219 hl_cb_put(free_obj->cq_cb);
220 atomic_set(&free_obj->in_use, 0);
221 }
222
223 kfree(free_list_head);
224
225 if (dynamic_alloc_free_list_head) {
226 list_for_each_entry_safe(free_obj, temp_free_obj, dynamic_alloc_free_list_head,
227 free_objects_node) {
228 dev_dbg(hdev->dev,
229 "Dynamic_Alloc list: About to put refcount to buf (%p) cq_cb(%p)\n",
230 free_obj->buf,
231 free_obj->cq_cb);
232
233 hl_mmap_mem_buf_put(free_obj->buf);
234 hl_cb_put(free_obj->cq_cb);
235 list_del(&free_obj->free_objects_node);
236 kfree(free_obj);
237 }
238
239 kfree(dynamic_alloc_free_list_head);
240 }
241
242 kfree(job);
243 }
244
245 /*
246 * This function called with spin_lock of wait_list_lock taken
247 * This function will set timestamp and delete the registration node from the
248 * wait_list_lock.
249 * and since we're protected with spin_lock here, so we cannot just put the refcount
250 * for the objects here, since the release function may be called and it's also a long
251 * logic (which might sleep also) that cannot be handled in irq context.
252 * so here we'll be filling a list with nodes of "put" jobs and then will send this
253 * list to a dedicated workqueue to do the actual put.
254 */
handle_registration_node(struct hl_device * hdev,struct hl_user_pending_interrupt * pend,struct list_head ** free_list,struct list_head ** dynamic_alloc_list,struct hl_user_interrupt * intr)255 static int handle_registration_node(struct hl_device *hdev, struct hl_user_pending_interrupt *pend,
256 struct list_head **free_list,
257 struct list_head **dynamic_alloc_list,
258 struct hl_user_interrupt *intr)
259 {
260 struct hl_ts_free_jobs *ts_free_jobs_data;
261 struct timestamp_reg_free_node *free_node;
262 u32 free_node_index;
263 u64 timestamp;
264
265 ts_free_jobs_data = &intr->ts_free_jobs_data;
266 free_node_index = ts_free_jobs_data->next_avail_free_node_idx;
267
268 if (!(*free_list)) {
269 /* Alloc/Init the timestamp registration free objects list */
270 *free_list = kmalloc_obj(struct list_head, GFP_ATOMIC);
271 if (!(*free_list))
272 return -ENOMEM;
273
274 INIT_LIST_HEAD(*free_list);
275 }
276
277 free_node = &ts_free_jobs_data->free_nodes_pool[free_node_index];
278 if (atomic_cmpxchg(&free_node->in_use, 0, 1)) {
279 dev_dbg(hdev->dev,
280 "Timestamp free node pool is full, buff: %p, record: %p, irq: %u\n",
281 pend->ts_reg_info.buf,
282 pend,
283 intr->interrupt_id);
284
285 if (!(*dynamic_alloc_list)) {
286 *dynamic_alloc_list = kmalloc_obj(struct list_head,
287 GFP_ATOMIC);
288 if (!(*dynamic_alloc_list))
289 return -ENOMEM;
290
291 INIT_LIST_HEAD(*dynamic_alloc_list);
292 }
293
294 free_node = kmalloc_obj(struct timestamp_reg_free_node,
295 GFP_ATOMIC);
296 if (!free_node)
297 return -ENOMEM;
298
299 free_node->dynamic_alloc = 1;
300 }
301
302 timestamp = ktime_to_ns(intr->timestamp);
303
304 *pend->ts_reg_info.timestamp_kernel_addr = timestamp;
305
306 dev_dbg(hdev->dev, "Irq handle: Timestamp record (%p) ts cb address (%p), interrupt_id: %u\n",
307 pend, pend->ts_reg_info.timestamp_kernel_addr, intr->interrupt_id);
308
309 list_del(&pend->list_node);
310
311 /* Putting the refcount for ts_buff and cq_cb objects will be handled
312 * in workqueue context, just add job to free_list.
313 */
314 free_node->buf = pend->ts_reg_info.buf;
315 free_node->cq_cb = pend->ts_reg_info.cq_cb;
316
317 if (free_node->dynamic_alloc) {
318 list_add(&free_node->free_objects_node, *dynamic_alloc_list);
319 } else {
320 ts_free_jobs_data->next_avail_free_node_idx =
321 (++free_node_index) % ts_free_jobs_data->free_nodes_length;
322 list_add(&free_node->free_objects_node, *free_list);
323 }
324
325 /* Mark TS record as free */
326 pend->ts_reg_info.in_use = false;
327
328 return 0;
329 }
330
handle_user_interrupt_ts_list(struct hl_device * hdev,struct hl_user_interrupt * intr)331 static void handle_user_interrupt_ts_list(struct hl_device *hdev, struct hl_user_interrupt *intr)
332 {
333 struct list_head *ts_reg_free_list_head = NULL, *dynamic_alloc_list_head = NULL;
334 struct hl_user_pending_interrupt *pend, *temp_pend;
335 struct timestamp_reg_work_obj *job;
336 bool reg_node_handle_fail = false;
337 unsigned long flags;
338 int rc;
339
340 /* For registration nodes:
341 * As part of handling the registration nodes, we should put refcount to
342 * some objects. the problem is that we cannot do that under spinlock
343 * or in irq handler context at all (since release functions are long and
344 * might sleep), so we will need to handle that part in workqueue context.
345 * To avoid handling kmalloc failure which compels us rolling back actions
346 * and move nodes hanged on the free list back to the interrupt ts list
347 * we always alloc the job of the WQ at the beginning.
348 */
349 job = kmalloc_obj(*job, GFP_ATOMIC);
350 if (!job)
351 return;
352
353 spin_lock_irqsave(&intr->ts_list_lock, flags);
354 list_for_each_entry_safe(pend, temp_pend, &intr->ts_list_head, list_node) {
355 if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
356 !pend->cq_kernel_addr) {
357 if (!reg_node_handle_fail) {
358 rc = handle_registration_node(hdev, pend,
359 &ts_reg_free_list_head,
360 &dynamic_alloc_list_head, intr);
361 if (rc)
362 reg_node_handle_fail = true;
363 }
364 }
365 }
366 spin_unlock_irqrestore(&intr->ts_list_lock, flags);
367
368 if (ts_reg_free_list_head) {
369 INIT_WORK(&job->free_obj, hl_ts_free_objects);
370 job->free_obj_head = ts_reg_free_list_head;
371 job->dynamic_alloc_free_obj_head = dynamic_alloc_list_head;
372 job->hdev = hdev;
373 queue_work(hdev->ts_free_obj_wq, &job->free_obj);
374 } else {
375 kfree(job);
376 }
377 }
378
handle_user_interrupt_wait_list(struct hl_device * hdev,struct hl_user_interrupt * intr)379 static void handle_user_interrupt_wait_list(struct hl_device *hdev, struct hl_user_interrupt *intr)
380 {
381 struct hl_user_pending_interrupt *pend, *temp_pend;
382 unsigned long flags;
383
384 spin_lock_irqsave(&intr->wait_list_lock, flags);
385 list_for_each_entry_safe(pend, temp_pend, &intr->wait_list_head, list_node) {
386 if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) ||
387 !pend->cq_kernel_addr) {
388 /* Handle wait target value node */
389 pend->fence.timestamp = intr->timestamp;
390 complete_all(&pend->fence.completion);
391 }
392 }
393 spin_unlock_irqrestore(&intr->wait_list_lock, flags);
394 }
395
handle_tpc_interrupt(struct hl_device * hdev)396 static void handle_tpc_interrupt(struct hl_device *hdev)
397 {
398 u64 event_mask;
399 u32 flags;
400
401 event_mask = HL_NOTIFIER_EVENT_TPC_ASSERT |
402 HL_NOTIFIER_EVENT_USER_ENGINE_ERR |
403 HL_NOTIFIER_EVENT_DEVICE_RESET;
404
405 flags = HL_DRV_RESET_DELAY;
406
407 dev_err_ratelimited(hdev->dev, "Received TPC assert\n");
408 hl_device_cond_reset(hdev, flags, event_mask);
409 }
410
handle_unexpected_user_interrupt(struct hl_device * hdev)411 static void handle_unexpected_user_interrupt(struct hl_device *hdev)
412 {
413 dev_err_ratelimited(hdev->dev, "Received unexpected user error interrupt\n");
414 }
415
416 /**
417 * hl_irq_user_interrupt_handler - irq handler for user interrupts.
418 *
419 * @irq: irq number
420 * @arg: pointer to user interrupt structure
421 */
hl_irq_user_interrupt_handler(int irq,void * arg)422 irqreturn_t hl_irq_user_interrupt_handler(int irq, void *arg)
423 {
424 struct hl_user_interrupt *user_int = arg;
425 struct hl_device *hdev = user_int->hdev;
426
427 user_int->timestamp = ktime_get();
428 switch (user_int->type) {
429 case HL_USR_INTERRUPT_CQ:
430 /* First handle user waiters threads */
431 handle_user_interrupt_wait_list(hdev, &hdev->common_user_cq_interrupt);
432 handle_user_interrupt_wait_list(hdev, user_int);
433
434 /* Second handle user timestamp registrations */
435 handle_user_interrupt_ts_list(hdev, &hdev->common_user_cq_interrupt);
436 handle_user_interrupt_ts_list(hdev, user_int);
437 break;
438 case HL_USR_INTERRUPT_DECODER:
439 handle_user_interrupt_wait_list(hdev, &hdev->common_decoder_interrupt);
440
441 /* Handle decoder interrupt registered on this specific irq */
442 handle_user_interrupt_wait_list(hdev, user_int);
443 break;
444 default:
445 break;
446 }
447
448 return IRQ_HANDLED;
449 }
450
451 /**
452 * hl_irq_user_interrupt_thread_handler - irq thread handler for user interrupts.
453 * This function is invoked by threaded irq mechanism
454 *
455 * @irq: irq number
456 * @arg: pointer to user interrupt structure
457 *
458 */
hl_irq_user_interrupt_thread_handler(int irq,void * arg)459 irqreturn_t hl_irq_user_interrupt_thread_handler(int irq, void *arg)
460 {
461 struct hl_user_interrupt *user_int = arg;
462 struct hl_device *hdev = user_int->hdev;
463
464 user_int->timestamp = ktime_get();
465 switch (user_int->type) {
466 case HL_USR_INTERRUPT_TPC:
467 handle_tpc_interrupt(hdev);
468 break;
469 case HL_USR_INTERRUPT_UNEXPECTED:
470 handle_unexpected_user_interrupt(hdev);
471 break;
472 default:
473 break;
474 }
475
476 return IRQ_HANDLED;
477 }
478
hl_irq_eq_error_interrupt_thread_handler(int irq,void * arg)479 irqreturn_t hl_irq_eq_error_interrupt_thread_handler(int irq, void *arg)
480 {
481 u64 event_mask = HL_NOTIFIER_EVENT_DEVICE_RESET | HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE;
482 struct hl_device *hdev = arg;
483
484 dev_err(hdev->dev, "EQ error interrupt received\n");
485
486 hl_device_cond_reset(hdev, HL_DRV_RESET_HARD, event_mask);
487
488 return IRQ_HANDLED;
489 }
490
491 /**
492 * hl_irq_handler_eq - irq handler for event queue
493 *
494 * @irq: irq number
495 * @arg: pointer to event queue structure
496 *
497 */
hl_irq_handler_eq(int irq,void * arg)498 irqreturn_t hl_irq_handler_eq(int irq, void *arg)
499 {
500 struct hl_eq *eq = arg;
501 struct hl_device *hdev = eq->hdev;
502 struct hl_eq_entry *eq_entry;
503 struct hl_eq_entry *eq_base;
504 struct hl_eqe_work *handle_eqe_work;
505 bool entry_ready;
506 u32 cur_eqe, ctl;
507 u16 cur_eqe_index, event_type;
508
509 eq_base = eq->kernel_address;
510
511 while (1) {
512 cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
513 entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);
514
515 if (!entry_ready)
516 break;
517
518 cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
519 if ((hdev->event_queue.check_eqe_index) &&
520 (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK) != cur_eqe_index)) {
521 dev_err(hdev->dev,
522 "EQE %#x in queue is ready but index does not match %d!=%d",
523 cur_eqe,
524 ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
525 cur_eqe_index);
526 break;
527 }
528
529 eq->prev_eqe_index++;
530
531 eq_entry = &eq_base[eq->ci];
532
533 /*
534 * Make sure we read EQ entry contents after we've
535 * checked the ownership bit.
536 */
537 dma_rmb();
538
539 if (hdev->disabled && !hdev->reset_info.in_compute_reset) {
540 ctl = le32_to_cpu(eq_entry->hdr.ctl);
541 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK) >> EQ_CTL_EVENT_TYPE_SHIFT);
542 dev_warn(hdev->dev,
543 "Device disabled but received an EQ event (%u)\n", event_type);
544 goto skip_irq;
545 }
546
547 handle_eqe_work = kmalloc_obj(*handle_eqe_work, GFP_ATOMIC);
548 if (handle_eqe_work) {
549 INIT_WORK(&handle_eqe_work->eq_work, irq_handle_eqe);
550 handle_eqe_work->hdev = hdev;
551
552 memcpy(&handle_eqe_work->eq_entry, eq_entry,
553 sizeof(*eq_entry));
554
555 queue_work(hdev->eq_wq, &handle_eqe_work->eq_work);
556 }
557 skip_irq:
558 /* Clear EQ entry ready bit */
559 eq_entry->hdr.ctl =
560 cpu_to_le32(le32_to_cpu(eq_entry->hdr.ctl) &
561 ~EQ_CTL_READY_MASK);
562
563 eq->ci = hl_eq_inc_ptr(eq->ci);
564
565 hdev->asic_funcs->update_eq_ci(hdev, eq->ci);
566 }
567
568 return IRQ_HANDLED;
569 }
570
571 /**
572 * hl_irq_handler_dec_abnrm - Decoder error interrupt handler
573 * @irq: IRQ number
574 * @arg: pointer to decoder structure.
575 */
hl_irq_handler_dec_abnrm(int irq,void * arg)576 irqreturn_t hl_irq_handler_dec_abnrm(int irq, void *arg)
577 {
578 struct hl_dec *dec = arg;
579
580 schedule_work(&dec->abnrm_intr_work);
581
582 return IRQ_HANDLED;
583 }
584
585 /**
586 * hl_cq_init - main initialization function for an cq object
587 *
588 * @hdev: pointer to device structure
589 * @q: pointer to cq structure
590 * @hw_queue_id: The H/W queue ID this completion queue belongs to
591 * HL_INVALID_QUEUE if cq is not attached to any specific queue
592 *
593 * Allocate dma-able memory for the completion queue and initialize fields
594 * Returns 0 on success
595 */
hl_cq_init(struct hl_device * hdev,struct hl_cq * q,u32 hw_queue_id)596 int hl_cq_init(struct hl_device *hdev, struct hl_cq *q, u32 hw_queue_id)
597 {
598 void *p;
599
600 p = hl_asic_dma_alloc_coherent(hdev, HL_CQ_SIZE_IN_BYTES, &q->bus_address,
601 GFP_KERNEL | __GFP_ZERO);
602 if (!p)
603 return -ENOMEM;
604
605 q->hdev = hdev;
606 q->kernel_address = p;
607 q->hw_queue_id = hw_queue_id;
608 q->ci = 0;
609 q->pi = 0;
610
611 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
612
613 return 0;
614 }
615
616 /**
617 * hl_cq_fini - destroy completion queue
618 *
619 * @hdev: pointer to device structure
620 * @q: pointer to cq structure
621 *
622 * Free the completion queue memory
623 */
hl_cq_fini(struct hl_device * hdev,struct hl_cq * q)624 void hl_cq_fini(struct hl_device *hdev, struct hl_cq *q)
625 {
626 hl_asic_dma_free_coherent(hdev, HL_CQ_SIZE_IN_BYTES, q->kernel_address, q->bus_address);
627 }
628
hl_cq_reset(struct hl_device * hdev,struct hl_cq * q)629 void hl_cq_reset(struct hl_device *hdev, struct hl_cq *q)
630 {
631 q->ci = 0;
632 q->pi = 0;
633
634 atomic_set(&q->free_slots_cnt, HL_CQ_LENGTH);
635
636 /*
637 * It's not enough to just reset the PI/CI because the H/W may have
638 * written valid completion entries before it was halted and therefore
639 * we need to clean the actual queues so we won't process old entries
640 * when the device is operational again
641 */
642
643 memset(q->kernel_address, 0, HL_CQ_SIZE_IN_BYTES);
644 }
645
646 /**
647 * hl_eq_init - main initialization function for an event queue object
648 *
649 * @hdev: pointer to device structure
650 * @q: pointer to eq structure
651 *
652 * Allocate dma-able memory for the event queue and initialize fields
653 * Returns 0 on success
654 */
hl_eq_init(struct hl_device * hdev,struct hl_eq * q)655 int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
656 {
657 u32 size = hdev->asic_prop.fw_event_queue_size ? : HL_EQ_SIZE_IN_BYTES;
658 void *p;
659
660 p = hl_cpu_accessible_dma_pool_alloc(hdev, size, &q->bus_address);
661 if (!p)
662 return -ENOMEM;
663
664 q->hdev = hdev;
665 q->kernel_address = p;
666 q->size = size;
667 q->ci = 0;
668 q->prev_eqe_index = 0;
669
670 return 0;
671 }
672
673 /**
674 * hl_eq_fini - destroy event queue
675 *
676 * @hdev: pointer to device structure
677 * @q: pointer to eq structure
678 *
679 * Free the event queue memory
680 */
hl_eq_fini(struct hl_device * hdev,struct hl_eq * q)681 void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
682 {
683 flush_workqueue(hdev->eq_wq);
684
685 hl_cpu_accessible_dma_pool_free(hdev, q->size, q->kernel_address);
686 }
687
hl_eq_reset(struct hl_device * hdev,struct hl_eq * q)688 void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
689 {
690 q->ci = 0;
691 q->prev_eqe_index = 0;
692
693 /*
694 * It's not enough to just reset the PI/CI because the H/W may have
695 * written valid completion entries before it was halted and therefore
696 * we need to clean the actual queues so we won't process old entries
697 * when the device is operational again
698 */
699
700 memset(q->kernel_address, 0, q->size);
701 }
702
hl_eq_dump(struct hl_device * hdev,struct hl_eq * q)703 void hl_eq_dump(struct hl_device *hdev, struct hl_eq *q)
704 {
705 u32 eq_length, eqe_size, ctl, ready, mode, type, index;
706 struct hl_eq_header *hdr;
707 u8 *ptr;
708 int i;
709
710 eq_length = HL_EQ_LENGTH;
711 eqe_size = q->size / HL_EQ_LENGTH;
712
713 dev_info(hdev->dev, "Contents of EQ entries headers:\n");
714
715 for (i = 0, ptr = q->kernel_address ; i < eq_length ; ++i, ptr += eqe_size) {
716 hdr = (struct hl_eq_header *) ptr;
717 ctl = le32_to_cpu(hdr->ctl);
718 ready = FIELD_GET(EQ_CTL_READY_MASK, ctl);
719 mode = FIELD_GET(EQ_CTL_EVENT_MODE_MASK, ctl);
720 type = FIELD_GET(EQ_CTL_EVENT_TYPE_MASK, ctl);
721 index = FIELD_GET(EQ_CTL_INDEX_MASK, ctl);
722
723 dev_info(hdev->dev, "%02u: %#010x [ready: %u, mode %u, type %04u, index %05u]\n",
724 i, ctl, ready, mode, type, index);
725 }
726 }
727