1 /* SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause */ 2 /* 3 * Copyright(c) 2015 - 2018 Intel Corporation. 4 */ 5 6 #ifndef _HFI1_IOWAIT_H 7 #define _HFI1_IOWAIT_H 8 9 #include <linux/list.h> 10 #include <linux/workqueue.h> 11 #include <linux/wait.h> 12 #include <linux/sched.h> 13 14 #include "sdma_txreq.h" 15 16 /* 17 * typedef (*restart_t)() - restart callback 18 * @work: pointer to work structure 19 */ 20 typedef void (*restart_t)(struct work_struct *work); 21 22 #define IOWAIT_PENDING_IB 0x0 23 #define IOWAIT_PENDING_TID 0x1 24 25 /* 26 * A QP can have multiple Send Engines (SEs). 27 * 28 * The current use case is for supporting a TID RDMA 29 * packet build/xmit mechanism independent from verbs. 30 */ 31 #define IOWAIT_SES 2 32 #define IOWAIT_IB_SE 0 33 #define IOWAIT_TID_SE 1 34 35 struct sdma_txreq; 36 struct sdma_engine; 37 /** 38 * @iowork: the work struct 39 * @tx_head: list of prebuilt packets 40 * @iow: the parent iowait structure 41 * 42 * This structure is the work item (process) specific 43 * details associated with the each of the two SEs of the 44 * QP. 45 * 46 * The workstruct and the queued TXs are unique to each 47 * SE. 48 */ 49 struct iowait; 50 struct iowait_work { 51 struct work_struct iowork; 52 struct list_head tx_head; 53 struct iowait *iow; 54 }; 55 56 /** 57 * @list: used to add/insert into QP/PQ wait lists 58 * @tx_head: overflow list of sdma_txreq's 59 * @sleep: no space callback 60 * @wakeup: space callback wakeup 61 * @sdma_drained: sdma count drained 62 * @init_priority: callback to manipulate priority 63 * @lock: lock protected head of wait queue 64 * @iowork: workqueue overhead 65 * @wait_dma: wait for sdma_busy == 0 66 * @wait_pio: wait for pio_busy == 0 67 * @sdma_busy: # of packets in flight 68 * @count: total number of descriptors in tx_head'ed list 69 * @tx_limit: limit for overflow queuing 70 * @tx_count: number of tx entry's in tx_head'ed list 71 * @flags: wait flags (one per QP) 72 * @wait: SE array for multiple legs 73 * 74 * This is to be embedded in user's state structure 75 * (QP or PQ). 76 * 77 * The sleep and wakeup members are a 78 * bit misnamed. They do not strictly 79 * speaking sleep or wake up, but they 80 * are callbacks for the ULP to implement 81 * what ever queuing/dequeuing of 82 * the embedded iowait and its containing struct 83 * when a resource shortage like SDMA ring space 84 * or PIO credit space is seen. 85 * 86 * Both potentially have locks help 87 * so sleeping is not allowed and it is not 88 * supported to submit txreqs from the wakeup 89 * call directly because of lock conflicts. 90 * 91 * The wait_dma member along with the iow 92 * 93 * The lock field is used by waiters to record 94 * the seqlock_t that guards the list head. 95 * Waiters explicity know that, but the destroy 96 * code that unwaits QPs does not. 97 */ 98 struct iowait { 99 struct list_head list; 100 int (*sleep)( 101 struct sdma_engine *sde, 102 struct iowait_work *wait, 103 struct sdma_txreq *tx, 104 uint seq, 105 bool pkts_sent 106 ); 107 void (*wakeup)(struct iowait *wait, int reason); 108 void (*sdma_drained)(struct iowait *wait); 109 void (*init_priority)(struct iowait *wait); 110 seqlock_t *lock; 111 wait_queue_head_t wait_dma; 112 wait_queue_head_t wait_pio; 113 atomic_t sdma_busy; 114 atomic_t pio_busy; 115 u32 count; 116 u32 tx_limit; 117 u32 tx_count; 118 u8 starved_cnt; 119 u8 priority; 120 unsigned long flags; 121 struct iowait_work wait[IOWAIT_SES]; 122 }; 123 124 #define SDMA_AVAIL_REASON 0 125 126 void iowait_set_flag(struct iowait *wait, u32 flag); 127 bool iowait_flag_set(struct iowait *wait, u32 flag); 128 void iowait_clear_flag(struct iowait *wait, u32 flag); 129 130 void iowait_init(struct iowait *wait, u32 tx_limit, 131 void (*func)(struct work_struct *work), 132 void (*tidfunc)(struct work_struct *work), 133 int (*sleep)(struct sdma_engine *sde, 134 struct iowait_work *wait, 135 struct sdma_txreq *tx, 136 uint seq, 137 bool pkts_sent), 138 void (*wakeup)(struct iowait *wait, int reason), 139 void (*sdma_drained)(struct iowait *wait), 140 void (*init_priority)(struct iowait *wait)); 141 142 /** 143 * iowait_schedule() - schedule the default send engine work 144 * @wait: wait struct to schedule 145 * @wq: workqueue for schedule 146 * @cpu: cpu 147 */ 148 static inline bool iowait_schedule(struct iowait *wait, 149 struct workqueue_struct *wq, int cpu) 150 { 151 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork); 152 } 153 154 /** 155 * iowait_tid_schedule - schedule the tid SE 156 * @wait: the iowait structure 157 * @wq: the work queue 158 * @cpu: the cpu 159 */ 160 static inline bool iowait_tid_schedule(struct iowait *wait, 161 struct workqueue_struct *wq, int cpu) 162 { 163 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork); 164 } 165 166 /** 167 * iowait_sdma_drain() - wait for DMAs to drain 168 * 169 * @wait: iowait structure 170 * 171 * This will delay until the iowait sdmas have 172 * completed. 173 */ 174 static inline void iowait_sdma_drain(struct iowait *wait) 175 { 176 wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy)); 177 } 178 179 /** 180 * iowait_sdma_pending() - return sdma pending count 181 * 182 * @wait: iowait structure 183 * 184 */ 185 static inline int iowait_sdma_pending(struct iowait *wait) 186 { 187 return atomic_read(&wait->sdma_busy); 188 } 189 190 /** 191 * iowait_sdma_inc - note sdma io pending 192 * @wait: iowait structure 193 */ 194 static inline void iowait_sdma_inc(struct iowait *wait) 195 { 196 atomic_inc(&wait->sdma_busy); 197 } 198 199 /** 200 * iowait_sdma_add - add count to pending 201 * @wait: iowait structure 202 */ 203 static inline void iowait_sdma_add(struct iowait *wait, int count) 204 { 205 atomic_add(count, &wait->sdma_busy); 206 } 207 208 /** 209 * iowait_sdma_dec - note sdma complete 210 * @wait: iowait structure 211 */ 212 static inline int iowait_sdma_dec(struct iowait *wait) 213 { 214 if (!wait) 215 return 0; 216 return atomic_dec_and_test(&wait->sdma_busy); 217 } 218 219 /** 220 * iowait_pio_drain() - wait for pios to drain 221 * 222 * @wait: iowait structure 223 * 224 * This will delay until the iowait pios have 225 * completed. 226 */ 227 static inline void iowait_pio_drain(struct iowait *wait) 228 { 229 wait_event_timeout(wait->wait_pio, 230 !atomic_read(&wait->pio_busy), 231 HZ); 232 } 233 234 /** 235 * iowait_pio_pending() - return pio pending count 236 * 237 * @wait: iowait structure 238 * 239 */ 240 static inline int iowait_pio_pending(struct iowait *wait) 241 { 242 return atomic_read(&wait->pio_busy); 243 } 244 245 /** 246 * iowait_pio_inc - note pio pending 247 * @wait: iowait structure 248 */ 249 static inline void iowait_pio_inc(struct iowait *wait) 250 { 251 atomic_inc(&wait->pio_busy); 252 } 253 254 /** 255 * iowait_pio_dec - note pio complete 256 * @wait: iowait structure 257 */ 258 static inline int iowait_pio_dec(struct iowait *wait) 259 { 260 if (!wait) 261 return 0; 262 return atomic_dec_and_test(&wait->pio_busy); 263 } 264 265 /** 266 * iowait_drain_wakeup() - trigger iowait_drain() waiter 267 * 268 * @wait: iowait structure 269 * 270 * This will trigger any waiters. 271 */ 272 static inline void iowait_drain_wakeup(struct iowait *wait) 273 { 274 wake_up(&wait->wait_dma); 275 wake_up(&wait->wait_pio); 276 if (wait->sdma_drained) 277 wait->sdma_drained(wait); 278 } 279 280 /** 281 * iowait_get_txhead() - get packet off of iowait list 282 * 283 * @wait: iowait_work structure 284 */ 285 static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait) 286 { 287 struct sdma_txreq *tx = NULL; 288 289 if (!list_empty(&wait->tx_head)) { 290 tx = list_first_entry( 291 &wait->tx_head, 292 struct sdma_txreq, 293 list); 294 list_del_init(&tx->list); 295 } 296 return tx; 297 } 298 299 static inline u16 iowait_get_desc(struct iowait_work *w) 300 { 301 u16 num_desc = 0; 302 struct sdma_txreq *tx = NULL; 303 304 if (!list_empty(&w->tx_head)) { 305 tx = list_first_entry(&w->tx_head, struct sdma_txreq, 306 list); 307 num_desc = tx->num_desc; 308 if (tx->flags & SDMA_TXREQ_F_VIP) 309 w->iow->priority++; 310 } 311 return num_desc; 312 } 313 314 static inline u32 iowait_get_all_desc(struct iowait *w) 315 { 316 u32 num_desc = 0; 317 318 num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]); 319 num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]); 320 return num_desc; 321 } 322 323 static inline void iowait_update_priority(struct iowait_work *w) 324 { 325 struct sdma_txreq *tx = NULL; 326 327 if (!list_empty(&w->tx_head)) { 328 tx = list_first_entry(&w->tx_head, struct sdma_txreq, 329 list); 330 if (tx->flags & SDMA_TXREQ_F_VIP) 331 w->iow->priority++; 332 } 333 } 334 335 static inline void iowait_update_all_priority(struct iowait *w) 336 { 337 iowait_update_priority(&w->wait[IOWAIT_IB_SE]); 338 iowait_update_priority(&w->wait[IOWAIT_TID_SE]); 339 } 340 341 static inline void iowait_init_priority(struct iowait *w) 342 { 343 w->priority = 0; 344 if (w->init_priority) 345 w->init_priority(w); 346 } 347 348 static inline void iowait_get_priority(struct iowait *w) 349 { 350 iowait_init_priority(w); 351 iowait_update_all_priority(w); 352 } 353 354 /** 355 * iowait_queue - Put the iowait on a wait queue 356 * @pkts_sent: have some packets been sent before queuing? 357 * @w: the iowait struct 358 * @wait_head: the wait queue 359 * 360 * This function is called to insert an iowait struct into a 361 * wait queue after a resource (eg, sdma descriptor or pio 362 * buffer) is run out. 363 */ 364 static inline void iowait_queue(bool pkts_sent, struct iowait *w, 365 struct list_head *wait_head) 366 { 367 /* 368 * To play fair, insert the iowait at the tail of the wait queue if it 369 * has already sent some packets; Otherwise, put it at the head. 370 * However, if it has priority packets to send, also put it at the 371 * head. 372 */ 373 if (pkts_sent) 374 w->starved_cnt = 0; 375 else 376 w->starved_cnt++; 377 378 if (w->priority > 0 || !pkts_sent) 379 list_add(&w->list, wait_head); 380 else 381 list_add_tail(&w->list, wait_head); 382 } 383 384 /** 385 * iowait_starve_clear - clear the wait queue's starve count 386 * @pkts_sent: have some packets been sent? 387 * @w: the iowait struct 388 * 389 * This function is called to clear the starve count. If no 390 * packets have been sent, the starve count will not be cleared. 391 */ 392 static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w) 393 { 394 if (pkts_sent) 395 w->starved_cnt = 0; 396 } 397 398 /* Update the top priority index */ 399 uint iowait_priority_update_top(struct iowait *w, 400 struct iowait *top, 401 uint idx, uint top_idx); 402 403 /** 404 * iowait_packet_queued() - determine if a packet is queued 405 * @wait: the iowait_work structure 406 */ 407 static inline bool iowait_packet_queued(struct iowait_work *wait) 408 { 409 return !list_empty(&wait->tx_head); 410 } 411 412 /** 413 * inc_wait_count - increment wait counts 414 * @w: the log work struct 415 * @n: the count 416 */ 417 static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n) 418 { 419 if (!w) 420 return; 421 w->iow->tx_count++; 422 w->iow->count += n; 423 } 424 425 /** 426 * iowait_get_tid_work - return iowait_work for tid SE 427 * @w: the iowait struct 428 */ 429 static inline struct iowait_work *iowait_get_tid_work(struct iowait *w) 430 { 431 return &w->wait[IOWAIT_TID_SE]; 432 } 433 434 /** 435 * iowait_get_ib_work - return iowait_work for ib SE 436 * @w: the iowait struct 437 */ 438 static inline struct iowait_work *iowait_get_ib_work(struct iowait *w) 439 { 440 return &w->wait[IOWAIT_IB_SE]; 441 } 442 443 /** 444 * iowait_ioww_to_iow - return iowait given iowait_work 445 * @w: the iowait_work struct 446 */ 447 static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w) 448 { 449 if (likely(w)) 450 return w->iow; 451 return NULL; 452 } 453 454 void iowait_cancel_work(struct iowait *w); 455 int iowait_set_work_flag(struct iowait_work *w); 456 457 #endif 458