1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _X86_POSTED_INTR_H 3 #define _X86_POSTED_INTR_H 4 5 #include <asm/cmpxchg.h> 6 #include <asm/rwonce.h> 7 #include <asm/irq_vectors.h> 8 9 #include <linux/bitmap.h> 10 11 #define POSTED_INTR_ON 0 12 #define POSTED_INTR_SN 1 13 14 #define PID_TABLE_ENTRY_VALID 1 15 16 #define NR_PIR_VECTORS 256 17 #define NR_PIR_WORDS (NR_PIR_VECTORS / BITS_PER_LONG) 18 19 /* Posted-Interrupt Descriptor */ 20 struct pi_desc { 21 unsigned long pir[NR_PIR_WORDS]; /* Posted interrupt requested */ 22 union { 23 struct { 24 u16 notifications; /* Suppress and outstanding bits */ 25 u8 nv; 26 u8 rsvd_2; 27 u32 ndst; 28 }; 29 u64 control; 30 }; 31 u32 rsvd[6]; 32 } __aligned(64); 33 34 /* 35 * De-multiplexing posted interrupts is on the performance path, the code 36 * below is written to optimize the cache performance based on the following 37 * considerations: 38 * 1.Posted interrupt descriptor (PID) fits in a cache line that is frequently 39 * accessed by both CPU and IOMMU. 40 * 2.During software processing of posted interrupts, the CPU needs to do 41 * natural width read and xchg for checking and clearing posted interrupt 42 * request (PIR), a 256 bit field within the PID. 43 * 3.On the other side, the IOMMU does atomic swaps of the entire PID cache 44 * line when posting interrupts and setting control bits. 45 * 4.The CPU can access the cache line a magnitude faster than the IOMMU. 46 * 5.Each time the IOMMU does interrupt posting to the PIR will evict the PID 47 * cache line. The cache line states after each operation are as follows, 48 * assuming a 64-bit kernel: 49 * CPU IOMMU PID Cache line state 50 * --------------------------------------------------------------- 51 *...read64 exclusive 52 *...lock xchg64 modified 53 *... post/atomic swap invalid 54 *...------------------------------------------------------------- 55 * 56 * To reduce L1 data cache miss, it is important to avoid contention with 57 * IOMMU's interrupt posting/atomic swap. Therefore, a copy of PIR is used 58 * when processing posted interrupts in software, e.g. to dispatch interrupt 59 * handlers for posted MSIs, or to move interrupts from the PIR to the vIRR 60 * in KVM. 61 * 62 * In addition, the code is trying to keep the cache line state consistent 63 * as much as possible. e.g. when making a copy and clearing the PIR 64 * (assuming non-zero PIR bits are present in the entire PIR), it does: 65 * read, read, read, read, xchg, xchg, xchg, xchg 66 * instead of: 67 * read, xchg, read, xchg, read, xchg, read, xchg 68 */ 69 static __always_inline bool pi_harvest_pir(unsigned long *pir, 70 unsigned long *pir_vals) 71 { 72 unsigned long pending = 0; 73 int i; 74 75 for (i = 0; i < NR_PIR_WORDS; i++) { 76 pir_vals[i] = READ_ONCE(pir[i]); 77 pending |= pir_vals[i]; 78 } 79 80 if (!pending) 81 return false; 82 83 for (i = 0; i < NR_PIR_WORDS; i++) { 84 if (!pir_vals[i]) 85 continue; 86 87 pir_vals[i] = arch_xchg(&pir[i], 0); 88 } 89 90 return true; 91 } 92 93 static inline bool pi_test_and_set_on(struct pi_desc *pi_desc) 94 { 95 return test_and_set_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control); 96 } 97 98 static inline bool pi_test_and_clear_on(struct pi_desc *pi_desc) 99 { 100 return test_and_clear_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control); 101 } 102 103 static inline bool pi_test_and_clear_sn(struct pi_desc *pi_desc) 104 { 105 return test_and_clear_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control); 106 } 107 108 static inline bool pi_test_and_set_pir(int vector, struct pi_desc *pi_desc) 109 { 110 return test_and_set_bit(vector, pi_desc->pir); 111 } 112 113 static inline bool pi_is_pir_empty(struct pi_desc *pi_desc) 114 { 115 return bitmap_empty(pi_desc->pir, NR_VECTORS); 116 } 117 118 static inline void pi_set_sn(struct pi_desc *pi_desc) 119 { 120 set_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control); 121 } 122 123 static inline void pi_set_on(struct pi_desc *pi_desc) 124 { 125 set_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control); 126 } 127 128 static inline void pi_clear_on(struct pi_desc *pi_desc) 129 { 130 clear_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control); 131 } 132 133 static inline void pi_clear_sn(struct pi_desc *pi_desc) 134 { 135 clear_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control); 136 } 137 138 static inline bool pi_test_on(struct pi_desc *pi_desc) 139 { 140 return test_bit(POSTED_INTR_ON, (unsigned long *)&pi_desc->control); 141 } 142 143 static inline bool pi_test_sn(struct pi_desc *pi_desc) 144 { 145 return test_bit(POSTED_INTR_SN, (unsigned long *)&pi_desc->control); 146 } 147 148 static inline bool pi_test_pir(int vector, struct pi_desc *pi_desc) 149 { 150 return test_bit(vector, (unsigned long *)pi_desc->pir); 151 } 152 153 /* Non-atomic helpers */ 154 static inline void __pi_set_sn(struct pi_desc *pi_desc) 155 { 156 pi_desc->notifications |= BIT(POSTED_INTR_SN); 157 } 158 159 static inline void __pi_clear_sn(struct pi_desc *pi_desc) 160 { 161 pi_desc->notifications &= ~BIT(POSTED_INTR_SN); 162 } 163 164 #ifdef CONFIG_X86_POSTED_MSI 165 /* 166 * Not all external vectors are subject to interrupt remapping, e.g. IOMMU's 167 * own interrupts. Here we do not distinguish them since those vector bits in 168 * PIR will always be zero. 169 */ 170 static inline bool pi_pending_this_cpu(unsigned int vector) 171 { 172 struct pi_desc *pid = this_cpu_ptr(&posted_msi_pi_desc); 173 174 if (WARN_ON_ONCE(vector > NR_VECTORS || vector < FIRST_EXTERNAL_VECTOR)) 175 return false; 176 177 return test_bit(vector, pid->pir); 178 } 179 180 extern void intel_posted_msi_init(void); 181 #else 182 static inline bool pi_pending_this_cpu(unsigned int vector) { return false; } 183 184 static inline void intel_posted_msi_init(void) {}; 185 #endif /* X86_POSTED_MSI */ 186 187 #endif /* _X86_POSTED_INTR_H */ 188