1 /******************************************************************************
2 SPDX-License-Identifier: BSD-3-Clause
3
4 Copyright (c) 2001-2020, Intel Corporation
5 All rights reserved.
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
12
13 2. Redistributions in binary form must reproduce the above copyright
14 notice, this list of conditions and the following disclaimer in the
15 documentation and/or other materials provided with the distribution.
16
17 3. Neither the name of the Intel Corporation nor the names of its
18 contributors may be used to endorse or promote products derived from
19 this software without specific prior written permission.
20
21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
25 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 POSSIBILITY OF SUCH DAMAGE.
32
33 ******************************************************************************/
34
35
36 #ifndef _FREEBSD_OS_H_
37 #define _FREEBSD_OS_H_
38
39 #include <sys/types.h>
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/proc.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/mbuf.h>
46 #include <sys/protosw.h>
47 #include <sys/socket.h>
48 #include <sys/malloc.h>
49 #include <sys/kernel.h>
50 #include <sys/bus.h>
51
52 #include <net/ethernet.h>
53 #include <net/if.h>
54 #include <net/if_var.h>
55 #include <net/iflib.h>
56
57
58
59 #include <machine/bus.h>
60 #include <sys/rman.h>
61 #include <machine/resource.h>
62 #include <vm/vm.h>
63 #include <vm/pmap.h>
64 #include <machine/clock.h>
65 #include <dev/pci/pcivar.h>
66 #include <dev/pci/pcireg.h>
67
68
69 #define ASSERT(x) if(!(x)) panic("EM: x")
70 #define us_scale(x) max(1, (x/(1000000/hz)))
71 static inline int
ms_scale(int x)72 ms_scale(int x) {
73 if (hz == 1000) {
74 return (x);
75 } else if (hz > 1000) {
76 return (x*(hz/1000));
77 } else {
78 return (max(1, x/(1000/hz)));
79 }
80 }
81
82 extern int e1000_use_pause_delay;
83
84 static inline void
safe_pause_us(int x)85 safe_pause_us(int x) {
86 if (!e1000_use_pause_delay) {
87 DELAY(x);
88 } else {
89 pause("e1000_delay", max(1, x/(1000000/hz)));
90 }
91 }
92
93 static inline void
safe_pause_ms(int x)94 safe_pause_ms(int x) {
95 if (!e1000_use_pause_delay) {
96 DELAY(x*1000);
97 } else {
98 pause("e1000_delay", ms_scale(x));
99 }
100 }
101
102 #define usec_delay(x) safe_pause_us(x)
103 #define usec_delay_irq(x) usec_delay(x)
104 #define msec_delay(x) safe_pause_ms(x)
105 #define msec_delay_irq(x) msec_delay(x)
106
107 /* Enable/disable debugging statements in shared code */
108 #define DBG 0
109
110 #define DEBUGOUT(...) \
111 do { if (DBG) printf(__VA_ARGS__); } while (0)
112 #define DEBUGOUT1(...) DEBUGOUT(__VA_ARGS__)
113 #define DEBUGOUT2(...) DEBUGOUT(__VA_ARGS__)
114 #define DEBUGOUT3(...) DEBUGOUT(__VA_ARGS__)
115 #define DEBUGOUT7(...) DEBUGOUT(__VA_ARGS__)
116 #define DEBUGFUNC(F) DEBUGOUT(F "\n")
117
118 #define STATIC static
119 #define CMD_MEM_WRT_INVALIDATE 0x0010 /* BIT_4 */
120 #define PCI_COMMAND_REGISTER PCIR_COMMAND
121
122 typedef uint64_t u64;
123 typedef uint32_t u32;
124 typedef uint16_t u16;
125 typedef uint8_t u8;
126 typedef int64_t s64;
127 typedef int32_t s32;
128 typedef int16_t s16;
129 typedef int8_t s8;
130
131 #define __le16 u16
132 #define __le32 u32
133 #define __le64 u64
134
135 #ifdef INVARIANTS
136 #define ASSERT_CTX_LOCK_HELD(hw) (sx_assert(iflib_ctx_lock_get(((struct e1000_osdep *)hw->back)->ctx), SX_XLOCKED))
137 #else
138 #define ASSERT_CTX_LOCK_HELD(hw)
139 #endif
140
141 #if defined(__i386__) || defined(__amd64__)
142 static __inline
prefetch(void * x)143 void prefetch(void *x)
144 {
145 __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
146 }
147 #else
148 #define prefetch(x)
149 #endif
150
151 struct e1000_osdep
152 {
153 bus_space_tag_t mem_bus_space_tag;
154 bus_space_handle_t mem_bus_space_handle;
155 #ifdef INVARIANTS
156 bus_size_t mem_bus_space_size;
157 #endif
158 bus_space_tag_t io_bus_space_tag;
159 bus_space_handle_t io_bus_space_handle;
160 bus_space_tag_t flash_bus_space_tag;
161 bus_space_handle_t flash_bus_space_handle;
162 device_t dev;
163 if_ctx_t ctx;
164 bool pcim2pci_arbiter_wa;
165 bool vf;
166 bool vf_82576;
167 };
168
169 void e1000_pcim2pci_write(struct e1000_osdep *, uint32_t, uint32_t);
170
171 #ifdef INVARIANTS
172 /*
173 * 82576 and I350 VFs expose a sparse register file. Keep this list local to
174 * the OS accessors so a PF-only register that leaks into the shared VF path
175 * fails at its first access instead of returning reserved-register garbage.
176 *
177 * The driver intentionally uses only queue pair zero. It also clears the
178 * retained configuration of the unused second 82576 queue pair after VFLR.
179 * Expanding VF data-path queue support must extend this predicate from the
180 * applicable device CSR map.
181 */
182 static __inline bool
e1000_vf_reg_valid(uint32_t reg,bool write,bool vf_82576)183 e1000_vf_reg_valid(uint32_t reg, bool write, bool vf_82576)
184 {
185 /* VF mailbox memory: 16 dwords beginning at 0x800. */
186 if (reg >= 0x00800 && reg <= 0x0083c && (reg & 3) == 0)
187 return (true);
188
189 /* The three VF MSI-X interrupt-throttling registers. */
190 if (reg >= 0x01680 && reg <= 0x01688 && (reg & 3) == 0)
191 return (true);
192
193 /* Receive queue zero. */
194 switch (reg) {
195 case 0x02800: /* RDBAL */
196 case 0x02804: /* RDBAH */
197 case 0x02808: /* RDLEN */
198 case 0x0280c: /* SRRCTL */
199 case 0x02810: /* RDH */
200 case 0x02814: /* RXCTL */
201 case 0x02818: /* RDT */
202 case 0x02828: /* RXDCTL */
203 return (true);
204 }
205
206 /* Transmit queue zero. */
207 switch (reg) {
208 case 0x03800: /* TDBAL */
209 case 0x03804: /* TDBAH */
210 case 0x03808: /* TDLEN */
211 case 0x03810: /* TDH */
212 case 0x03814: /* TXCTL */
213 case 0x03818: /* TDT */
214 case 0x03828: /* TXDCTL */
215 case 0x03838: /* TDWBAL */
216 case 0x0383c: /* TDWBAH */
217 return (true);
218 }
219
220 /* Retained configuration from 82576 virtual queue one. */
221 if (vf_82576) {
222 switch (reg) {
223 case 0x0290c: /* SRRCTL(1) */
224 case 0x02914: /* RXCTL(1) */
225 case 0x02928: /* RXDCTL(1) */
226 case 0x03914: /* TXCTL(1) */
227 case 0x03928: /* TXDCTL(1) */
228 case 0x03938: /* TDWBAL(1) */
229 case 0x0393c: /* TDWBAH(1) */
230 return (true);
231 }
232 }
233
234 /*
235 * 82576 exposes VFMPRC at 0xf3c. I350 erratum 31 makes
236 * its corrected 0xf38 address inaccessible to a VF.
237 */
238 if (vf_82576 && reg == 0x00f3c)
239 return (!write);
240
241 switch (reg) {
242 case 0x00000: /* CTRL */
243 case 0x000c4: /* Legacy ITR, listed but unused by igb VFs */
244 case 0x00c40: /* V2PMAILBOX(0) */
245 case 0x00f0c: /* VFPSRTYPE */
246 case 0x01524: /* EIMS */
247 case 0x0152c: /* EIAC */
248 case 0x01530: /* EIAM */
249 case 0x01700: /* IVAR0 */
250 case 0x01740: /* IVAR_MISC */
251 return (true);
252 case 0x01520: /* EICS */
253 case 0x01528: /* EIMC */
254 return (write);
255 case 0x00008: /* STATUS */
256 case 0x00f10: /* VFGPRC */
257 case 0x00f14: /* VFGPTC */
258 case 0x00f18: /* VFGORC */
259 case 0x00f34: /* VFGOTC */
260 case 0x00f40: /* VFGPRLBC */
261 case 0x00f44: /* VFGPTLBC */
262 case 0x00f48: /* VFGORLBC */
263 case 0x00f50: /* VFGOTLBC */
264 case 0x01580: /* EICR */
265 return (!write);
266 default:
267 return (false);
268 }
269 }
270 #endif
271
272 #define E1000_REGISTER(hw, reg) (((hw)->mac.type >= e1000_82543) \
273 ? reg : e1000_translate_register_82542(reg))
274
275 #define E1000_WRITE_FLUSH(a) E1000_READ_REG(a, E1000_STATUS)
276
277 /* Read from an absolute offset in the adapter's memory space */
278 #define E1000_READ_OFFSET(hw, offset) \
279 bus_space_read_4(((struct e1000_osdep *)(hw)->back)->mem_bus_space_tag, \
280 ((struct e1000_osdep *)(hw)->back)->mem_bus_space_handle, offset)
281
282 static __inline uint32_t
e1000_rd32(struct e1000_osdep * osdep,uint32_t reg)283 e1000_rd32(struct e1000_osdep *osdep, uint32_t reg)
284 {
285
286 KASSERT(reg < osdep->mem_bus_space_size,
287 ("e1000: register offset %#jx too large (max is %#jx)",
288 (uintmax_t)reg, (uintmax_t)osdep->mem_bus_space_size));
289 #ifdef INVARIANTS
290 KASSERT(!osdep->vf ||
291 e1000_vf_reg_valid(reg, false, osdep->vf_82576),
292 ("e1000: invalid VF register read at %#x", reg));
293 #endif
294
295 return (bus_space_read_4(osdep->mem_bus_space_tag,
296 osdep->mem_bus_space_handle, reg));
297 }
298
299
300 static __inline void
e1000_wr32(struct e1000_osdep * osdep,uint32_t reg,uint32_t value)301 e1000_wr32(struct e1000_osdep *osdep, uint32_t reg, uint32_t value)
302 {
303
304 KASSERT(reg < osdep->mem_bus_space_size,
305 ("e1000: register offset %#jx too large (max is %#jx)",
306 (uintmax_t)reg, (uintmax_t)osdep->mem_bus_space_size));
307 #ifdef INVARIANTS
308 KASSERT(!osdep->vf ||
309 e1000_vf_reg_valid(reg, true, osdep->vf_82576),
310 ("e1000: invalid VF register write at %#x", reg));
311 #endif
312
313 if (__predict_true(!osdep->pcim2pci_arbiter_wa)) {
314 bus_space_write_4(osdep->mem_bus_space_tag,
315 osdep->mem_bus_space_handle, reg, value);
316 return;
317 }
318 e1000_pcim2pci_write(osdep, reg, value);
319 }
320
321 /* Write to an absolute offset in the adapter's memory space. */
322 #define E1000_WRITE_OFFSET(hw, offset, value) \
323 e1000_wr32((hw)->back, (offset), (value))
324
325 /* Register READ/WRITE macros */
326
327 #define E1000_READ_REG(hw, reg) \
328 e1000_rd32((hw)->back, E1000_REGISTER(hw, reg))
329
330 #define E1000_WRITE_REG(hw, reg, value) \
331 e1000_wr32((hw)->back, E1000_REGISTER(hw, reg), value)
332
333 #define E1000_READ_REG_ARRAY(hw, reg, index) \
334 e1000_rd32((hw)->back, E1000_REGISTER(hw, reg) + ((index) << 2))
335
336 #define E1000_WRITE_REG_ARRAY(hw, reg, index, value) \
337 e1000_wr32((hw)->back, E1000_REGISTER(hw, reg) + ((index) << 2), value)
338
339 #define E1000_READ_REG_ARRAY_DWORD E1000_READ_REG_ARRAY
340 #define E1000_WRITE_REG_ARRAY_DWORD E1000_WRITE_REG_ARRAY
341
342 #define E1000_READ_REG_ARRAY_BYTE(hw, reg, index) \
343 bus_space_read_1(((struct e1000_osdep *)(hw)->back)->mem_bus_space_tag, \
344 ((struct e1000_osdep *)(hw)->back)->mem_bus_space_handle, \
345 E1000_REGISTER(hw, reg) + index)
346
347 #define E1000_WRITE_REG_ARRAY_BYTE(hw, reg, index, value) \
348 bus_space_write_1(((struct e1000_osdep *)(hw)->back)->mem_bus_space_tag, \
349 ((struct e1000_osdep *)(hw)->back)->mem_bus_space_handle, \
350 E1000_REGISTER(hw, reg) + index, value)
351
352 #define E1000_WRITE_REG_ARRAY_WORD(hw, reg, index, value) \
353 bus_space_write_2(((struct e1000_osdep *)(hw)->back)->mem_bus_space_tag, \
354 ((struct e1000_osdep *)(hw)->back)->mem_bus_space_handle, \
355 E1000_REGISTER(hw, reg) + (index << 1), value)
356
357 #define E1000_WRITE_REG_IO(hw, reg, value) do {\
358 bus_space_write_4(((struct e1000_osdep *)(hw)->back)->io_bus_space_tag, \
359 ((struct e1000_osdep *)(hw)->back)->io_bus_space_handle, \
360 (hw)->io_base, reg); \
361 bus_space_write_4(((struct e1000_osdep *)(hw)->back)->io_bus_space_tag, \
362 ((struct e1000_osdep *)(hw)->back)->io_bus_space_handle, \
363 (hw)->io_base + 4, value); } while (0)
364
365 #define E1000_READ_FLASH_REG(hw, reg) \
366 bus_space_read_4(((struct e1000_osdep *)(hw)->back)->flash_bus_space_tag, \
367 ((struct e1000_osdep *)(hw)->back)->flash_bus_space_handle, reg)
368
369 #define E1000_READ_FLASH_REG16(hw, reg) \
370 bus_space_read_2(((struct e1000_osdep *)(hw)->back)->flash_bus_space_tag, \
371 ((struct e1000_osdep *)(hw)->back)->flash_bus_space_handle, reg)
372
373 #define E1000_WRITE_FLASH_REG(hw, reg, value) \
374 bus_space_write_4(((struct e1000_osdep *)(hw)->back)->flash_bus_space_tag, \
375 ((struct e1000_osdep *)(hw)->back)->flash_bus_space_handle, reg, value)
376
377 #define E1000_WRITE_FLASH_REG16(hw, reg, value) \
378 bus_space_write_2(((struct e1000_osdep *)(hw)->back)->flash_bus_space_tag, \
379 ((struct e1000_osdep *)(hw)->back)->flash_bus_space_handle, reg, value)
380
381
382 #if defined(INVARIANTS)
383 #include <sys/proc.h>
384
385 #define ASSERT_NO_LOCKS() \
386 do { \
387 int unknown_locks = curthread->td_locks - mtx_owned(&Giant); \
388 if (unknown_locks > 0) { \
389 WITNESS_WARN(WARN_GIANTOK|WARN_SLEEPOK|WARN_PANIC, NULL, "unexpected non-sleepable lock"); \
390 } \
391 MPASS(curthread->td_rw_rlocks == 0); \
392 MPASS(curthread->td_lk_slocks == 0); \
393 } while (0)
394 #else
395 #define ASSERT_NO_LOCKS()
396 #endif
397
398 #endif /* _FREEBSD_OS_H_ */
399