1 #ifndef _HFI1_IOWAIT_H 2 #define _HFI1_IOWAIT_H 3 /* 4 * Copyright(c) 2015 - 2018 Intel Corporation. 5 * 6 * This file is provided under a dual BSD/GPLv2 license. When using or 7 * redistributing this file, you may do so under either license. 8 * 9 * GPL LICENSE SUMMARY 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of version 2 of the GNU General Public License as 13 * published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, but 16 * WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18 * General Public License for more details. 19 * 20 * BSD LICENSE 21 * 22 * Redistribution and use in source and binary forms, with or without 23 * modification, are permitted provided that the following conditions 24 * are met: 25 * 26 * - Redistributions of source code must retain the above copyright 27 * notice, this list of conditions and the following disclaimer. 28 * - Redistributions in binary form must reproduce the above copyright 29 * notice, this list of conditions and the following disclaimer in 30 * the documentation and/or other materials provided with the 31 * distribution. 32 * - Neither the name of Intel Corporation nor the names of its 33 * contributors may be used to endorse or promote products derived 34 * from this software without specific prior written permission. 35 * 36 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 39 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 40 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 41 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 42 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 43 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 44 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 45 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 46 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 47 * 48 */ 49 50 #include <linux/list.h> 51 #include <linux/workqueue.h> 52 #include <linux/wait.h> 53 #include <linux/sched.h> 54 55 #include "sdma_txreq.h" 56 57 /* 58 * typedef (*restart_t)() - restart callback 59 * @work: pointer to work structure 60 */ 61 typedef void (*restart_t)(struct work_struct *work); 62 63 #define IOWAIT_PENDING_IB 0x0 64 #define IOWAIT_PENDING_TID 0x1 65 66 /* 67 * A QP can have multiple Send Engines (SEs). 68 * 69 * The current use case is for supporting a TID RDMA 70 * packet build/xmit mechanism independent from verbs. 71 */ 72 #define IOWAIT_SES 2 73 #define IOWAIT_IB_SE 0 74 #define IOWAIT_TID_SE 1 75 76 struct sdma_txreq; 77 struct sdma_engine; 78 /** 79 * @iowork: the work struct 80 * @tx_head: list of prebuilt packets 81 * @iow: the parent iowait structure 82 * 83 * This structure is the work item (process) specific 84 * details associated with the each of the two SEs of the 85 * QP. 86 * 87 * The workstruct and the queued TXs are unique to each 88 * SE. 89 */ 90 struct iowait; 91 struct iowait_work { 92 struct work_struct iowork; 93 struct list_head tx_head; 94 struct iowait *iow; 95 }; 96 97 /** 98 * @list: used to add/insert into QP/PQ wait lists 99 * @tx_head: overflow list of sdma_txreq's 100 * @sleep: no space callback 101 * @wakeup: space callback wakeup 102 * @sdma_drained: sdma count drained 103 * @init_priority: callback to manipulate priority 104 * @lock: lock protected head of wait queue 105 * @iowork: workqueue overhead 106 * @wait_dma: wait for sdma_busy == 0 107 * @wait_pio: wait for pio_busy == 0 108 * @sdma_busy: # of packets in flight 109 * @count: total number of descriptors in tx_head'ed list 110 * @tx_limit: limit for overflow queuing 111 * @tx_count: number of tx entry's in tx_head'ed list 112 * @flags: wait flags (one per QP) 113 * @wait: SE array for multiple legs 114 * 115 * This is to be embedded in user's state structure 116 * (QP or PQ). 117 * 118 * The sleep and wakeup members are a 119 * bit misnamed. They do not strictly 120 * speaking sleep or wake up, but they 121 * are callbacks for the ULP to implement 122 * what ever queuing/dequeuing of 123 * the embedded iowait and its containing struct 124 * when a resource shortage like SDMA ring space 125 * or PIO credit space is seen. 126 * 127 * Both potentially have locks help 128 * so sleeping is not allowed and it is not 129 * supported to submit txreqs from the wakeup 130 * call directly because of lock conflicts. 131 * 132 * The wait_dma member along with the iow 133 * 134 * The lock field is used by waiters to record 135 * the seqlock_t that guards the list head. 136 * Waiters explicity know that, but the destroy 137 * code that unwaits QPs does not. 138 */ 139 struct iowait { 140 struct list_head list; 141 int (*sleep)( 142 struct sdma_engine *sde, 143 struct iowait_work *wait, 144 struct sdma_txreq *tx, 145 uint seq, 146 bool pkts_sent 147 ); 148 void (*wakeup)(struct iowait *wait, int reason); 149 void (*sdma_drained)(struct iowait *wait); 150 void (*init_priority)(struct iowait *wait); 151 seqlock_t *lock; 152 wait_queue_head_t wait_dma; 153 wait_queue_head_t wait_pio; 154 atomic_t sdma_busy; 155 atomic_t pio_busy; 156 u32 count; 157 u32 tx_limit; 158 u32 tx_count; 159 u8 starved_cnt; 160 u8 priority; 161 unsigned long flags; 162 struct iowait_work wait[IOWAIT_SES]; 163 }; 164 165 #define SDMA_AVAIL_REASON 0 166 167 void iowait_set_flag(struct iowait *wait, u32 flag); 168 bool iowait_flag_set(struct iowait *wait, u32 flag); 169 void iowait_clear_flag(struct iowait *wait, u32 flag); 170 171 void iowait_init(struct iowait *wait, u32 tx_limit, 172 void (*func)(struct work_struct *work), 173 void (*tidfunc)(struct work_struct *work), 174 int (*sleep)(struct sdma_engine *sde, 175 struct iowait_work *wait, 176 struct sdma_txreq *tx, 177 uint seq, 178 bool pkts_sent), 179 void (*wakeup)(struct iowait *wait, int reason), 180 void (*sdma_drained)(struct iowait *wait), 181 void (*init_priority)(struct iowait *wait)); 182 183 /** 184 * iowait_schedule() - schedule the default send engine work 185 * @wait: wait struct to schedule 186 * @wq: workqueue for schedule 187 * @cpu: cpu 188 */ 189 static inline bool iowait_schedule(struct iowait *wait, 190 struct workqueue_struct *wq, int cpu) 191 { 192 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork); 193 } 194 195 /** 196 * iowait_tid_schedule - schedule the tid SE 197 * @wait: the iowait structure 198 * @wq: the work queue 199 * @cpu: the cpu 200 */ 201 static inline bool iowait_tid_schedule(struct iowait *wait, 202 struct workqueue_struct *wq, int cpu) 203 { 204 return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork); 205 } 206 207 /** 208 * iowait_sdma_drain() - wait for DMAs to drain 209 * 210 * @wait: iowait structure 211 * 212 * This will delay until the iowait sdmas have 213 * completed. 214 */ 215 static inline void iowait_sdma_drain(struct iowait *wait) 216 { 217 wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy)); 218 } 219 220 /** 221 * iowait_sdma_pending() - return sdma pending count 222 * 223 * @wait: iowait structure 224 * 225 */ 226 static inline int iowait_sdma_pending(struct iowait *wait) 227 { 228 return atomic_read(&wait->sdma_busy); 229 } 230 231 /** 232 * iowait_sdma_inc - note sdma io pending 233 * @wait: iowait structure 234 */ 235 static inline void iowait_sdma_inc(struct iowait *wait) 236 { 237 atomic_inc(&wait->sdma_busy); 238 } 239 240 /** 241 * iowait_sdma_add - add count to pending 242 * @wait: iowait structure 243 */ 244 static inline void iowait_sdma_add(struct iowait *wait, int count) 245 { 246 atomic_add(count, &wait->sdma_busy); 247 } 248 249 /** 250 * iowait_sdma_dec - note sdma complete 251 * @wait: iowait structure 252 */ 253 static inline int iowait_sdma_dec(struct iowait *wait) 254 { 255 if (!wait) 256 return 0; 257 return atomic_dec_and_test(&wait->sdma_busy); 258 } 259 260 /** 261 * iowait_pio_drain() - wait for pios to drain 262 * 263 * @wait: iowait structure 264 * 265 * This will delay until the iowait pios have 266 * completed. 267 */ 268 static inline void iowait_pio_drain(struct iowait *wait) 269 { 270 wait_event_timeout(wait->wait_pio, 271 !atomic_read(&wait->pio_busy), 272 HZ); 273 } 274 275 /** 276 * iowait_pio_pending() - return pio pending count 277 * 278 * @wait: iowait structure 279 * 280 */ 281 static inline int iowait_pio_pending(struct iowait *wait) 282 { 283 return atomic_read(&wait->pio_busy); 284 } 285 286 /** 287 * iowait_pio_inc - note pio pending 288 * @wait: iowait structure 289 */ 290 static inline void iowait_pio_inc(struct iowait *wait) 291 { 292 atomic_inc(&wait->pio_busy); 293 } 294 295 /** 296 * iowait_pio_dec - note pio complete 297 * @wait: iowait structure 298 */ 299 static inline int iowait_pio_dec(struct iowait *wait) 300 { 301 if (!wait) 302 return 0; 303 return atomic_dec_and_test(&wait->pio_busy); 304 } 305 306 /** 307 * iowait_drain_wakeup() - trigger iowait_drain() waiter 308 * 309 * @wait: iowait structure 310 * 311 * This will trigger any waiters. 312 */ 313 static inline void iowait_drain_wakeup(struct iowait *wait) 314 { 315 wake_up(&wait->wait_dma); 316 wake_up(&wait->wait_pio); 317 if (wait->sdma_drained) 318 wait->sdma_drained(wait); 319 } 320 321 /** 322 * iowait_get_txhead() - get packet off of iowait list 323 * 324 * @wait iowait_work struture 325 */ 326 static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait) 327 { 328 struct sdma_txreq *tx = NULL; 329 330 if (!list_empty(&wait->tx_head)) { 331 tx = list_first_entry( 332 &wait->tx_head, 333 struct sdma_txreq, 334 list); 335 list_del_init(&tx->list); 336 } 337 return tx; 338 } 339 340 static inline u16 iowait_get_desc(struct iowait_work *w) 341 { 342 u16 num_desc = 0; 343 struct sdma_txreq *tx = NULL; 344 345 if (!list_empty(&w->tx_head)) { 346 tx = list_first_entry(&w->tx_head, struct sdma_txreq, 347 list); 348 num_desc = tx->num_desc; 349 if (tx->flags & SDMA_TXREQ_F_VIP) 350 w->iow->priority++; 351 } 352 return num_desc; 353 } 354 355 static inline u32 iowait_get_all_desc(struct iowait *w) 356 { 357 u32 num_desc = 0; 358 359 num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]); 360 num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]); 361 return num_desc; 362 } 363 364 static inline void iowait_update_priority(struct iowait_work *w) 365 { 366 struct sdma_txreq *tx = NULL; 367 368 if (!list_empty(&w->tx_head)) { 369 tx = list_first_entry(&w->tx_head, struct sdma_txreq, 370 list); 371 if (tx->flags & SDMA_TXREQ_F_VIP) 372 w->iow->priority++; 373 } 374 } 375 376 static inline void iowait_update_all_priority(struct iowait *w) 377 { 378 iowait_update_priority(&w->wait[IOWAIT_IB_SE]); 379 iowait_update_priority(&w->wait[IOWAIT_TID_SE]); 380 } 381 382 static inline void iowait_init_priority(struct iowait *w) 383 { 384 w->priority = 0; 385 if (w->init_priority) 386 w->init_priority(w); 387 } 388 389 static inline void iowait_get_priority(struct iowait *w) 390 { 391 iowait_init_priority(w); 392 iowait_update_all_priority(w); 393 } 394 395 /** 396 * iowait_queue - Put the iowait on a wait queue 397 * @pkts_sent: have some packets been sent before queuing? 398 * @w: the iowait struct 399 * @wait_head: the wait queue 400 * 401 * This function is called to insert an iowait struct into a 402 * wait queue after a resource (eg, sdma descriptor or pio 403 * buffer) is run out. 404 */ 405 static inline void iowait_queue(bool pkts_sent, struct iowait *w, 406 struct list_head *wait_head) 407 { 408 /* 409 * To play fair, insert the iowait at the tail of the wait queue if it 410 * has already sent some packets; Otherwise, put it at the head. 411 * However, if it has priority packets to send, also put it at the 412 * head. 413 */ 414 if (pkts_sent) 415 w->starved_cnt = 0; 416 else 417 w->starved_cnt++; 418 419 if (w->priority > 0 || !pkts_sent) 420 list_add(&w->list, wait_head); 421 else 422 list_add_tail(&w->list, wait_head); 423 } 424 425 /** 426 * iowait_starve_clear - clear the wait queue's starve count 427 * @pkts_sent: have some packets been sent? 428 * @w: the iowait struct 429 * 430 * This function is called to clear the starve count. If no 431 * packets have been sent, the starve count will not be cleared. 432 */ 433 static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w) 434 { 435 if (pkts_sent) 436 w->starved_cnt = 0; 437 } 438 439 /* Update the top priority index */ 440 uint iowait_priority_update_top(struct iowait *w, 441 struct iowait *top, 442 uint idx, uint top_idx); 443 444 /** 445 * iowait_packet_queued() - determine if a packet is queued 446 * @wait: the iowait_work structure 447 */ 448 static inline bool iowait_packet_queued(struct iowait_work *wait) 449 { 450 return !list_empty(&wait->tx_head); 451 } 452 453 /** 454 * inc_wait_count - increment wait counts 455 * @w: the log work struct 456 * @n: the count 457 */ 458 static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n) 459 { 460 if (!w) 461 return; 462 w->iow->tx_count++; 463 w->iow->count += n; 464 } 465 466 /** 467 * iowait_get_tid_work - return iowait_work for tid SE 468 * @w: the iowait struct 469 */ 470 static inline struct iowait_work *iowait_get_tid_work(struct iowait *w) 471 { 472 return &w->wait[IOWAIT_TID_SE]; 473 } 474 475 /** 476 * iowait_get_ib_work - return iowait_work for ib SE 477 * @w: the iowait struct 478 */ 479 static inline struct iowait_work *iowait_get_ib_work(struct iowait *w) 480 { 481 return &w->wait[IOWAIT_IB_SE]; 482 } 483 484 /** 485 * iowait_ioww_to_iow - return iowait given iowait_work 486 * @w: the iowait_work struct 487 */ 488 static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w) 489 { 490 if (likely(w)) 491 return w->iow; 492 return NULL; 493 } 494 495 void iowait_cancel_work(struct iowait *w); 496 int iowait_set_work_flag(struct iowait_work *w); 497 498 #endif 499