1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2018 Intel Corporation. 4 * 5 */ 6 #include "iowait.h" 7 #include "trace_iowait.h" 8 9 /* 1 priority == 16 starve_cnt */ 10 #define IOWAIT_PRIORITY_STARVE_SHIFT 4 11 12 void iowait_set_flag(struct iowait *wait, u32 flag) 13 { 14 trace_hfi1_iowait_set(wait, flag); 15 set_bit(flag, &wait->flags); 16 } 17 18 bool iowait_flag_set(struct iowait *wait, u32 flag) 19 { 20 return test_bit(flag, &wait->flags); 21 } 22 23 inline void iowait_clear_flag(struct iowait *wait, u32 flag) 24 { 25 trace_hfi1_iowait_clear(wait, flag); 26 clear_bit(flag, &wait->flags); 27 } 28 29 /* 30 * iowait_init() - initialize wait structure 31 * @wait: wait struct to initialize 32 * @tx_limit: limit for overflow queuing 33 * @func: restart function for workqueue 34 * @sleep: sleep function for no space 35 * @resume: wakeup function for no space 36 * 37 * This function initializes the iowait 38 * structure embedded in the QP or PQ. 39 * 40 */ 41 void iowait_init(struct iowait *wait, u32 tx_limit, 42 void (*func)(struct work_struct *work), 43 void (*tidfunc)(struct work_struct *work), 44 int (*sleep)(struct sdma_engine *sde, 45 struct iowait_work *wait, 46 struct sdma_txreq *tx, 47 uint seq, 48 bool pkts_sent), 49 void (*wakeup)(struct iowait *wait, int reason), 50 void (*sdma_drained)(struct iowait *wait), 51 void (*init_priority)(struct iowait *wait)) 52 { 53 int i; 54 55 wait->count = 0; 56 INIT_LIST_HEAD(&wait->list); 57 init_waitqueue_head(&wait->wait_dma); 58 init_waitqueue_head(&wait->wait_pio); 59 atomic_set(&wait->sdma_busy, 0); 60 atomic_set(&wait->pio_busy, 0); 61 wait->tx_limit = tx_limit; 62 wait->sleep = sleep; 63 wait->wakeup = wakeup; 64 wait->sdma_drained = sdma_drained; 65 wait->init_priority = init_priority; 66 wait->flags = 0; 67 for (i = 0; i < IOWAIT_SES; i++) { 68 wait->wait[i].iow = wait; 69 INIT_LIST_HEAD(&wait->wait[i].tx_head); 70 if (i == IOWAIT_IB_SE) 71 INIT_WORK(&wait->wait[i].iowork, func); 72 else 73 INIT_WORK(&wait->wait[i].iowork, tidfunc); 74 } 75 } 76 77 /** 78 * iowait_cancel_work - cancel all work in iowait 79 * @w: the iowait struct 80 */ 81 void iowait_cancel_work(struct iowait *w) 82 { 83 cancel_work_sync(&iowait_get_ib_work(w)->iowork); 84 /* Make sure that the iowork for TID RDMA is used */ 85 if (iowait_get_tid_work(w)->iowork.func) 86 cancel_work_sync(&iowait_get_tid_work(w)->iowork); 87 } 88 89 /** 90 * iowait_set_work_flag - set work flag based on leg 91 * @w: the iowait work struct 92 */ 93 int iowait_set_work_flag(struct iowait_work *w) 94 { 95 if (w == &w->iow->wait[IOWAIT_IB_SE]) { 96 iowait_set_flag(w->iow, IOWAIT_PENDING_IB); 97 return IOWAIT_IB_SE; 98 } 99 iowait_set_flag(w->iow, IOWAIT_PENDING_TID); 100 return IOWAIT_TID_SE; 101 } 102 103 /** 104 * iowait_priority_update_top - update the top priority entry 105 * @w: the iowait struct 106 * @top: a pointer to the top priority entry 107 * @idx: the index of the current iowait in an array 108 * @top_idx: the array index for the iowait entry that has the top priority 109 * 110 * This function is called to compare the priority of a given 111 * iowait with the given top priority entry. The top index will 112 * be returned. 113 */ 114 uint iowait_priority_update_top(struct iowait *w, 115 struct iowait *top, 116 uint idx, uint top_idx) 117 { 118 u8 cnt, tcnt; 119 120 /* Convert priority into starve_cnt and compare the total.*/ 121 cnt = (w->priority << IOWAIT_PRIORITY_STARVE_SHIFT) + w->starved_cnt; 122 tcnt = (top->priority << IOWAIT_PRIORITY_STARVE_SHIFT) + 123 top->starved_cnt; 124 if (cnt > tcnt) 125 return idx; 126 else 127 return top_idx; 128 } 129