1 /* 2 * Copyright(c) 2015-2017 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 #ifndef _ASPM_H 48 #define _ASPM_H 49 50 #include "hfi.h" 51 52 extern uint aspm_mode; 53 54 enum aspm_mode { 55 ASPM_MODE_DISABLED = 0, /* ASPM always disabled, performance mode */ 56 ASPM_MODE_ENABLED = 1, /* ASPM always enabled, power saving mode */ 57 ASPM_MODE_DYNAMIC = 2, /* ASPM enabled/disabled dynamically */ 58 }; 59 60 /* Time after which the timer interrupt will re-enable ASPM */ 61 #define ASPM_TIMER_MS 1000 62 /* Time for which interrupts are ignored after a timer has been scheduled */ 63 #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2) 64 /* Two interrupts within this time trigger ASPM disable */ 65 #define ASPM_TRIGGER_MS 1 66 #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull) 67 #define ASPM_L1_SUPPORTED(reg) \ 68 (((reg & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2) 69 70 static inline bool aspm_hw_l1_supported(struct hfi1_devdata *dd) 71 { 72 struct pci_dev *parent = dd->pcidev->bus->self; 73 u32 up, dn; 74 75 /* 76 * If the driver does not have access to the upstream component, 77 * it cannot support ASPM L1 at all. 78 */ 79 if (!parent) 80 return false; 81 82 pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn); 83 dn = ASPM_L1_SUPPORTED(dn); 84 85 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up); 86 up = ASPM_L1_SUPPORTED(up); 87 88 /* ASPM works on A-step but is reported as not supported */ 89 return (!!dn || is_ax(dd)) && !!up; 90 } 91 92 /* Set L1 entrance latency for slower entry to L1 */ 93 static inline void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd) 94 { 95 u32 l1_ent_lat = 0x4u; 96 u32 reg32; 97 98 pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32); 99 reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK; 100 reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT; 101 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32); 102 } 103 104 static inline void aspm_hw_enable_l1(struct hfi1_devdata *dd) 105 { 106 struct pci_dev *parent = dd->pcidev->bus->self; 107 108 /* 109 * If the driver does not have access to the upstream component, 110 * it cannot support ASPM L1 at all. 111 */ 112 if (!parent) 113 return; 114 115 /* Enable ASPM L1 first in upstream component and then downstream */ 116 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 117 PCI_EXP_LNKCTL_ASPMC, 118 PCI_EXP_LNKCTL_ASPM_L1); 119 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, 120 PCI_EXP_LNKCTL_ASPMC, 121 PCI_EXP_LNKCTL_ASPM_L1); 122 } 123 124 static inline void aspm_hw_disable_l1(struct hfi1_devdata *dd) 125 { 126 struct pci_dev *parent = dd->pcidev->bus->self; 127 128 /* Disable ASPM L1 first in downstream component and then upstream */ 129 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, 130 PCI_EXP_LNKCTL_ASPMC, 0x0); 131 if (parent) 132 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 133 PCI_EXP_LNKCTL_ASPMC, 0x0); 134 } 135 136 static inline void aspm_enable(struct hfi1_devdata *dd) 137 { 138 if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED || 139 !dd->aspm_supported) 140 return; 141 142 aspm_hw_enable_l1(dd); 143 dd->aspm_enabled = true; 144 } 145 146 static inline void aspm_disable(struct hfi1_devdata *dd) 147 { 148 if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED) 149 return; 150 151 aspm_hw_disable_l1(dd); 152 dd->aspm_enabled = false; 153 } 154 155 static inline void aspm_disable_inc(struct hfi1_devdata *dd) 156 { 157 unsigned long flags; 158 159 spin_lock_irqsave(&dd->aspm_lock, flags); 160 aspm_disable(dd); 161 atomic_inc(&dd->aspm_disabled_cnt); 162 spin_unlock_irqrestore(&dd->aspm_lock, flags); 163 } 164 165 static inline void aspm_enable_dec(struct hfi1_devdata *dd) 166 { 167 unsigned long flags; 168 169 spin_lock_irqsave(&dd->aspm_lock, flags); 170 if (atomic_dec_and_test(&dd->aspm_disabled_cnt)) 171 aspm_enable(dd); 172 spin_unlock_irqrestore(&dd->aspm_lock, flags); 173 } 174 175 /* ASPM processing for each receive context interrupt */ 176 static inline void aspm_ctx_disable(struct hfi1_ctxtdata *rcd) 177 { 178 bool restart_timer; 179 bool close_interrupts; 180 unsigned long flags; 181 ktime_t now, prev; 182 183 /* Quickest exit for minimum impact */ 184 if (!rcd->aspm_intr_supported) 185 return; 186 187 spin_lock_irqsave(&rcd->aspm_lock, flags); 188 /* PSM contexts are open */ 189 if (!rcd->aspm_intr_enable) 190 goto unlock; 191 192 prev = rcd->aspm_ts_last_intr; 193 now = ktime_get(); 194 rcd->aspm_ts_last_intr = now; 195 196 /* An interrupt pair close together in time */ 197 close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS; 198 199 /* Don't push out our timer till this much time has elapsed */ 200 restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) > 201 ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC; 202 restart_timer = restart_timer && close_interrupts; 203 204 /* Disable ASPM and schedule timer */ 205 if (rcd->aspm_enabled && close_interrupts) { 206 aspm_disable_inc(rcd->dd); 207 rcd->aspm_enabled = false; 208 restart_timer = true; 209 } 210 211 if (restart_timer) { 212 mod_timer(&rcd->aspm_timer, 213 jiffies + msecs_to_jiffies(ASPM_TIMER_MS)); 214 rcd->aspm_ts_timer_sched = now; 215 } 216 unlock: 217 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 218 } 219 220 /* Timer function for re-enabling ASPM in the absence of interrupt activity */ 221 static inline void aspm_ctx_timer_function(struct timer_list *t) 222 { 223 struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer); 224 unsigned long flags; 225 226 spin_lock_irqsave(&rcd->aspm_lock, flags); 227 aspm_enable_dec(rcd->dd); 228 rcd->aspm_enabled = true; 229 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 230 } 231 232 /* 233 * Disable interrupt processing for verbs contexts when PSM or VNIC contexts 234 * are open. 235 */ 236 static inline void aspm_disable_all(struct hfi1_devdata *dd) 237 { 238 struct hfi1_ctxtdata *rcd; 239 unsigned long flags; 240 u16 i; 241 242 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { 243 rcd = hfi1_rcd_get_by_index(dd, i); 244 if (rcd) { 245 del_timer_sync(&rcd->aspm_timer); 246 spin_lock_irqsave(&rcd->aspm_lock, flags); 247 rcd->aspm_intr_enable = false; 248 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 249 hfi1_rcd_put(rcd); 250 } 251 } 252 253 aspm_disable(dd); 254 atomic_set(&dd->aspm_disabled_cnt, 0); 255 } 256 257 /* Re-enable interrupt processing for verbs contexts */ 258 static inline void aspm_enable_all(struct hfi1_devdata *dd) 259 { 260 struct hfi1_ctxtdata *rcd; 261 unsigned long flags; 262 u16 i; 263 264 aspm_enable(dd); 265 266 if (aspm_mode != ASPM_MODE_DYNAMIC) 267 return; 268 269 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { 270 rcd = hfi1_rcd_get_by_index(dd, i); 271 if (rcd) { 272 spin_lock_irqsave(&rcd->aspm_lock, flags); 273 rcd->aspm_intr_enable = true; 274 rcd->aspm_enabled = true; 275 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 276 hfi1_rcd_put(rcd); 277 } 278 } 279 } 280 281 static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd) 282 { 283 spin_lock_init(&rcd->aspm_lock); 284 timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0); 285 rcd->aspm_intr_supported = rcd->dd->aspm_supported && 286 aspm_mode == ASPM_MODE_DYNAMIC && 287 rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt; 288 } 289 290 static inline void aspm_init(struct hfi1_devdata *dd) 291 { 292 struct hfi1_ctxtdata *rcd; 293 u16 i; 294 295 spin_lock_init(&dd->aspm_lock); 296 dd->aspm_supported = aspm_hw_l1_supported(dd); 297 298 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { 299 rcd = hfi1_rcd_get_by_index(dd, i); 300 if (rcd) 301 aspm_ctx_init(rcd); 302 hfi1_rcd_put(rcd); 303 } 304 305 /* Start with ASPM disabled */ 306 aspm_hw_set_l1_ent_latency(dd); 307 dd->aspm_enabled = false; 308 aspm_hw_disable_l1(dd); 309 310 /* Now turn on ASPM if configured */ 311 aspm_enable_all(dd); 312 } 313 314 static inline void aspm_exit(struct hfi1_devdata *dd) 315 { 316 aspm_disable_all(dd); 317 318 /* Turn on ASPM on exit to conserve power */ 319 aspm_enable(dd); 320 } 321 322 #endif /* _ASPM_H */ 323