1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /*
3 * Copyright (C) 2003-2014, 2018-2022, 2024-2025 Intel Corporation
4 * Copyright (C) 2015-2016 Intel Deutschland GmbH
5 */
6 #if defined(__FreeBSD__)
7 #include <linux/delay.h>
8 #endif
9 #include <linux/device.h>
10 #include <linux/export.h>
11
12 #include "iwl-drv.h"
13 #include "iwl-io.h"
14 #include "iwl-csr.h"
15 #include "iwl-debug.h"
16 #include "iwl-prph.h"
17 #include "iwl-fh.h"
18 #include "pcie/gen1_2/internal.h"
19
iwl_write8(struct iwl_trans * trans,u32 ofs,u8 val)20 void iwl_write8(struct iwl_trans *trans, u32 ofs, u8 val)
21 {
22 trace_iwlwifi_dev_iowrite8(trans->dev, ofs, val);
23 iwl_trans_write8(trans, ofs, val);
24 }
25 IWL_EXPORT_SYMBOL(iwl_write8);
26
iwl_write32(struct iwl_trans * trans,u32 ofs,u32 val)27 void iwl_write32(struct iwl_trans *trans, u32 ofs, u32 val)
28 {
29 trace_iwlwifi_dev_iowrite32(trans->dev, ofs, val);
30 iwl_trans_write32(trans, ofs, val);
31 }
32 IWL_EXPORT_SYMBOL(iwl_write32);
33
iwl_write64(struct iwl_trans * trans,u64 ofs,u64 val)34 void iwl_write64(struct iwl_trans *trans, u64 ofs, u64 val)
35 {
36 trace_iwlwifi_dev_iowrite64(trans->dev, ofs, val);
37 iwl_trans_write32(trans, ofs, lower_32_bits(val));
38 iwl_trans_write32(trans, ofs + 4, upper_32_bits(val));
39 }
40 IWL_EXPORT_SYMBOL(iwl_write64);
41
iwl_read32(struct iwl_trans * trans,u32 ofs)42 u32 iwl_read32(struct iwl_trans *trans, u32 ofs)
43 {
44 u32 val = iwl_trans_read32(trans, ofs);
45
46 trace_iwlwifi_dev_ioread32(trans->dev, ofs, val);
47 return val;
48 }
49 IWL_EXPORT_SYMBOL(iwl_read32);
50
51 #define IWL_POLL_INTERVAL 10 /* microseconds */
52
iwl_poll_bits_mask(struct iwl_trans * trans,u32 addr,u32 bits,u32 mask,int timeout)53 int iwl_poll_bits_mask(struct iwl_trans *trans, u32 addr,
54 u32 bits, u32 mask, int timeout)
55 {
56 int t = 0;
57
58 do {
59 if ((iwl_read32(trans, addr) & mask) == (bits & mask))
60 return 0;
61 udelay(IWL_POLL_INTERVAL);
62 t += IWL_POLL_INTERVAL;
63 } while (t < timeout);
64
65 return -ETIMEDOUT;
66 }
67 IWL_EXPORT_SYMBOL(iwl_poll_bits_mask);
68
iwl_read_direct32(struct iwl_trans * trans,u32 reg)69 u32 iwl_read_direct32(struct iwl_trans *trans, u32 reg)
70 {
71 if (iwl_trans_grab_nic_access(trans)) {
72 u32 value = iwl_read32(trans, reg);
73
74 iwl_trans_release_nic_access(trans);
75 return value;
76 }
77
78 /* return as if we have a HW timeout/failure */
79 return 0x5a5a5a5a;
80 }
81
iwl_write_direct32(struct iwl_trans * trans,u32 reg,u32 value)82 void iwl_write_direct32(struct iwl_trans *trans, u32 reg, u32 value)
83 {
84 if (iwl_trans_grab_nic_access(trans)) {
85 iwl_write32(trans, reg, value);
86 iwl_trans_release_nic_access(trans);
87 }
88 }
89 IWL_EXPORT_SYMBOL(iwl_write_direct32);
90
iwl_write_direct64(struct iwl_trans * trans,u64 reg,u64 value)91 void iwl_write_direct64(struct iwl_trans *trans, u64 reg, u64 value)
92 {
93 if (iwl_trans_grab_nic_access(trans)) {
94 iwl_write64(trans, reg, value);
95 iwl_trans_release_nic_access(trans);
96 }
97 }
98
iwl_poll_direct_bit(struct iwl_trans * trans,u32 addr,u32 mask,int timeout)99 int iwl_poll_direct_bit(struct iwl_trans *trans, u32 addr, u32 mask,
100 int timeout)
101 {
102 int t = 0;
103
104 do {
105 if ((iwl_read_direct32(trans, addr) & mask) == mask)
106 return t;
107 udelay(IWL_POLL_INTERVAL);
108 t += IWL_POLL_INTERVAL;
109 } while (t < timeout);
110
111 return -ETIMEDOUT;
112 }
113
iwl_read_prph_no_grab(struct iwl_trans * trans,u32 ofs)114 u32 iwl_read_prph_no_grab(struct iwl_trans *trans, u32 ofs)
115 {
116 u32 val = iwl_trans_read_prph(trans, ofs);
117 trace_iwlwifi_dev_ioread_prph32(trans->dev, ofs, val);
118 return val;
119 }
120
iwl_write_prph_no_grab(struct iwl_trans * trans,u32 ofs,u32 val)121 void iwl_write_prph_no_grab(struct iwl_trans *trans, u32 ofs, u32 val)
122 {
123 trace_iwlwifi_dev_iowrite_prph32(trans->dev, ofs, val);
124 iwl_trans_write_prph(trans, ofs, val);
125 }
126
iwl_write_prph64_no_grab(struct iwl_trans * trans,u64 ofs,u64 val)127 void iwl_write_prph64_no_grab(struct iwl_trans *trans, u64 ofs, u64 val)
128 {
129 trace_iwlwifi_dev_iowrite_prph64(trans->dev, ofs, val);
130 iwl_write_prph_no_grab(trans, ofs, val & 0xffffffff);
131 iwl_write_prph_no_grab(trans, ofs + 4, val >> 32);
132 }
133
iwl_read_prph(struct iwl_trans * trans,u32 ofs)134 u32 iwl_read_prph(struct iwl_trans *trans, u32 ofs)
135 {
136 if (iwl_trans_grab_nic_access(trans)) {
137 u32 val = iwl_read_prph_no_grab(trans, ofs);
138
139 iwl_trans_release_nic_access(trans);
140
141 return val;
142 }
143
144 /* return as if we have a HW timeout/failure */
145 return 0x5a5a5a5a;
146 }
147 IWL_EXPORT_SYMBOL(iwl_read_prph);
148
iwl_write_prph_delay(struct iwl_trans * trans,u32 ofs,u32 val,u32 delay_ms)149 void iwl_write_prph_delay(struct iwl_trans *trans, u32 ofs, u32 val, u32 delay_ms)
150 {
151 if (iwl_trans_grab_nic_access(trans)) {
152 mdelay(delay_ms);
153 iwl_write_prph_no_grab(trans, ofs, val);
154 iwl_trans_release_nic_access(trans);
155 }
156 }
157 IWL_EXPORT_SYMBOL(iwl_write_prph_delay);
158
iwl_poll_prph_bit(struct iwl_trans * trans,u32 addr,u32 bits,u32 mask,int timeout)159 int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr,
160 u32 bits, u32 mask, int timeout)
161 {
162 int t = 0;
163
164 do {
165 if ((iwl_read_prph(trans, addr) & mask) == (bits & mask))
166 return 0;
167 udelay(IWL_POLL_INTERVAL);
168 t += IWL_POLL_INTERVAL;
169 } while (t < timeout);
170
171 return -ETIMEDOUT;
172 }
173
iwl_set_bits_prph(struct iwl_trans * trans,u32 ofs,u32 mask)174 void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
175 {
176 if (iwl_trans_grab_nic_access(trans)) {
177 iwl_write_prph_no_grab(trans, ofs,
178 iwl_read_prph_no_grab(trans, ofs) |
179 mask);
180 iwl_trans_release_nic_access(trans);
181 }
182 }
183 IWL_EXPORT_SYMBOL(iwl_set_bits_prph);
184
iwl_set_bits_mask_prph(struct iwl_trans * trans,u32 ofs,u32 bits,u32 mask)185 void iwl_set_bits_mask_prph(struct iwl_trans *trans, u32 ofs,
186 u32 bits, u32 mask)
187 {
188 if (iwl_trans_grab_nic_access(trans)) {
189 iwl_write_prph_no_grab(trans, ofs,
190 (iwl_read_prph_no_grab(trans, ofs) &
191 mask) | bits);
192 iwl_trans_release_nic_access(trans);
193 }
194 }
195 IWL_EXPORT_SYMBOL(iwl_set_bits_mask_prph);
196
iwl_clear_bits_prph(struct iwl_trans * trans,u32 ofs,u32 mask)197 void iwl_clear_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
198 {
199 u32 val;
200
201 if (iwl_trans_grab_nic_access(trans)) {
202 val = iwl_read_prph_no_grab(trans, ofs);
203 iwl_write_prph_no_grab(trans, ofs, (val & ~mask));
204 iwl_trans_release_nic_access(trans);
205 }
206 }
207 IWL_EXPORT_SYMBOL(iwl_clear_bits_prph);
208
iwl_force_nmi(struct iwl_trans * trans)209 void iwl_force_nmi(struct iwl_trans *trans)
210 {
211 if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_9000)
212 iwl_write_prph_delay(trans, DEVICE_SET_NMI_REG,
213 DEVICE_SET_NMI_VAL_DRV, 1);
214 else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
215 iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER,
216 UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER);
217 else if (trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
218 iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
219 UREG_DOORBELL_TO_ISR6_NMI_BIT);
220 else
221 iwl_write32(trans, CSR_DOORBELL_VECTOR,
222 UREG_DOORBELL_TO_ISR6_NMI_BIT);
223 }
224 IWL_EXPORT_SYMBOL(iwl_force_nmi);
225
get_rfh_string(int cmd)226 static const char *get_rfh_string(int cmd)
227 {
228 #define IWL_CMD(x) case x: return #x
229 #define IWL_CMD_MQ(arg, reg, q) { if (arg == reg(q)) return #reg; }
230
231 int i;
232
233 for (i = 0; i < IWL_MAX_RX_HW_QUEUES; i++) {
234 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_BA_LSB, i);
235 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_WIDX, i);
236 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_RIDX, i);
237 IWL_CMD_MQ(cmd, RFH_Q_URBD_STTS_WPTR_LSB, i);
238 }
239
240 switch (cmd) {
241 IWL_CMD(RFH_RXF_DMA_CFG);
242 IWL_CMD(RFH_GEN_CFG);
243 IWL_CMD(RFH_GEN_STATUS);
244 IWL_CMD(FH_TSSR_TX_STATUS_REG);
245 IWL_CMD(FH_TSSR_TX_ERROR_REG);
246 default:
247 return "UNKNOWN";
248 }
249 #undef IWL_CMD_MQ
250 }
251
252 struct reg {
253 u32 addr;
254 bool is64;
255 };
256
iwl_dump_rfh(struct iwl_trans * trans,char ** buf)257 static int iwl_dump_rfh(struct iwl_trans *trans, char **buf)
258 {
259 int i, q;
260 int num_q = trans->info.num_rxqs;
261 static const u32 rfh_tbl[] = {
262 RFH_RXF_DMA_CFG,
263 RFH_GEN_CFG,
264 RFH_GEN_STATUS,
265 FH_TSSR_TX_STATUS_REG,
266 FH_TSSR_TX_ERROR_REG,
267 };
268 static const struct reg rfh_mq_tbl[] = {
269 { RFH_Q0_FRBDCB_BA_LSB, true },
270 { RFH_Q0_FRBDCB_WIDX, false },
271 { RFH_Q0_FRBDCB_RIDX, false },
272 { RFH_Q0_URBD_STTS_WPTR_LSB, true },
273 };
274
275 #ifdef CONFIG_IWLWIFI_DEBUGFS
276 if (buf) {
277 int pos = 0;
278 /*
279 * Register (up to 34 for name + 8 blank/q for MQ): 40 chars
280 * Colon + space: 2 characters
281 * 0X%08x: 10 characters
282 * New line: 1 character
283 * Total of 53 characters
284 */
285 size_t bufsz = ARRAY_SIZE(rfh_tbl) * 53 +
286 ARRAY_SIZE(rfh_mq_tbl) * 53 * num_q + 40;
287
288 *buf = kmalloc(bufsz, GFP_KERNEL);
289 if (!*buf)
290 return -ENOMEM;
291
292 pos += scnprintf(*buf + pos, bufsz - pos,
293 "RFH register values:\n");
294
295 for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
296 pos += scnprintf(*buf + pos, bufsz - pos,
297 "%40s: 0X%08x\n",
298 get_rfh_string(rfh_tbl[i]),
299 iwl_read_prph(trans, rfh_tbl[i]));
300
301 for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
302 for (q = 0; q < num_q; q++) {
303 u32 addr = rfh_mq_tbl[i].addr;
304
305 addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
306 pos += scnprintf(*buf + pos, bufsz - pos,
307 "%34s(q %2d): 0X%08x\n",
308 get_rfh_string(addr), q,
309 iwl_read_prph(trans, addr));
310 }
311
312 return pos;
313 }
314 #endif
315
316 IWL_ERR(trans, "RFH register values:\n");
317 for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
318 IWL_ERR(trans, " %34s: 0X%08x\n",
319 get_rfh_string(rfh_tbl[i]),
320 iwl_read_prph(trans, rfh_tbl[i]));
321
322 for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
323 for (q = 0; q < num_q; q++) {
324 u32 addr = rfh_mq_tbl[i].addr;
325
326 addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
327 IWL_ERR(trans, " %34s(q %d): 0X%08x\n",
328 get_rfh_string(addr), q,
329 iwl_read_prph(trans, addr));
330 }
331
332 return 0;
333 }
334
get_fh_string(int cmd)335 static const char *get_fh_string(int cmd)
336 {
337 switch (cmd) {
338 IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG);
339 IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG);
340 IWL_CMD(FH_RSCSR_CHNL0_WPTR);
341 IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG);
342 IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG);
343 IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG);
344 IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV);
345 IWL_CMD(FH_TSSR_TX_STATUS_REG);
346 IWL_CMD(FH_TSSR_TX_ERROR_REG);
347 default:
348 return "UNKNOWN";
349 }
350 #undef IWL_CMD
351 }
352
iwl_dump_fh(struct iwl_trans * trans,char ** buf)353 int iwl_dump_fh(struct iwl_trans *trans, char **buf)
354 {
355 int i;
356 static const u32 fh_tbl[] = {
357 FH_RSCSR_CHNL0_STTS_WPTR_REG,
358 FH_RSCSR_CHNL0_RBDCB_BASE_REG,
359 FH_RSCSR_CHNL0_WPTR,
360 FH_MEM_RCSR_CHNL0_CONFIG_REG,
361 FH_MEM_RSSR_SHARED_CTRL_REG,
362 FH_MEM_RSSR_RX_STATUS_REG,
363 FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV,
364 FH_TSSR_TX_STATUS_REG,
365 FH_TSSR_TX_ERROR_REG
366 };
367
368 if (trans->mac_cfg->mq_rx_supported)
369 return iwl_dump_rfh(trans, buf);
370
371 #ifdef CONFIG_IWLWIFI_DEBUGFS
372 if (buf) {
373 int pos = 0;
374 size_t bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40;
375
376 *buf = kmalloc(bufsz, GFP_KERNEL);
377 if (!*buf)
378 return -ENOMEM;
379
380 pos += scnprintf(*buf + pos, bufsz - pos,
381 "FH register values:\n");
382
383 for (i = 0; i < ARRAY_SIZE(fh_tbl); i++)
384 pos += scnprintf(*buf + pos, bufsz - pos,
385 " %34s: 0X%08x\n",
386 get_fh_string(fh_tbl[i]),
387 iwl_read_direct32(trans, fh_tbl[i]));
388
389 return pos;
390 }
391 #endif
392
393 IWL_ERR(trans, "FH register values:\n");
394 for (i = 0; i < ARRAY_SIZE(fh_tbl); i++)
395 IWL_ERR(trans, " %34s: 0X%08x\n",
396 get_fh_string(fh_tbl[i]),
397 iwl_read_direct32(trans, fh_tbl[i]));
398
399 return 0;
400 }
401
iwl_trans_activate_nic(struct iwl_trans * trans)402 int iwl_trans_activate_nic(struct iwl_trans *trans)
403 {
404 return iwl_pcie_gen1_2_activate_nic(trans);
405 }
406 IWL_EXPORT_SYMBOL(iwl_trans_activate_nic);
407
iwl_trans_sync_nmi_with_addr(struct iwl_trans * trans,u32 inta_addr,u32 sw_err_bit)408 void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,
409 u32 sw_err_bit)
410 {
411 unsigned long timeout = jiffies + IWL_TRANS_NMI_TIMEOUT;
412 bool interrupts_enabled = test_bit(STATUS_INT_ENABLED, &trans->status);
413
414 /* if the interrupts were already disabled, there is no point in
415 * calling iwl_disable_interrupts
416 */
417 if (interrupts_enabled)
418 iwl_trans_interrupts(trans, false);
419
420 iwl_force_nmi(trans);
421 while (time_after(timeout, jiffies)) {
422 u32 inta_hw = iwl_read32(trans, inta_addr);
423
424 /* Error detected by uCode */
425 if (inta_hw & sw_err_bit) {
426 /* Clear causes register */
427 iwl_write32(trans, inta_addr, inta_hw & sw_err_bit);
428 break;
429 }
430
431 mdelay(1);
432 }
433
434 /* enable interrupts only if there were already enabled before this
435 * function to avoid a case were the driver enable interrupts before
436 * proper configurations were made
437 */
438 if (interrupts_enabled)
439 iwl_trans_interrupts(trans, true);
440
441 iwl_trans_fw_error(trans, IWL_ERR_TYPE_NMI_FORCED);
442 }
443