11f4d4ed6SAlexander Lobakin // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2fe56b9e6SYuval Mintz /* QLogic qed NIC Driver
3e8f1cb50SMintz, Yuval * Copyright (c) 2015-2017 QLogic Corporation
4663eacd8SAlexander Lobakin * Copyright (c) 2019-2020 Marvell International Ltd.
5fe56b9e6SYuval Mintz */
6fe56b9e6SYuval Mintz
7fe56b9e6SYuval Mintz #include <linux/types.h>
8fe56b9e6SYuval Mintz #include <linux/io.h>
9fe56b9e6SYuval Mintz #include <linux/delay.h>
10fe56b9e6SYuval Mintz #include <linux/dma-mapping.h>
11fe56b9e6SYuval Mintz #include <linux/errno.h>
12fe56b9e6SYuval Mintz #include <linux/kernel.h>
13fe56b9e6SYuval Mintz #include <linux/list.h>
14fe56b9e6SYuval Mintz #include <linux/mutex.h>
15fe56b9e6SYuval Mintz #include <linux/pci.h>
16fe56b9e6SYuval Mintz #include <linux/slab.h>
17fe56b9e6SYuval Mintz #include <linux/spinlock.h>
18fe56b9e6SYuval Mintz #include <linux/string.h>
19fe56b9e6SYuval Mintz #include <linux/qed/qed_chain.h>
20fe56b9e6SYuval Mintz #include "qed.h"
21fe56b9e6SYuval Mintz #include "qed_hsi.h"
22fe56b9e6SYuval Mintz #include "qed_hw.h"
23fe56b9e6SYuval Mintz #include "qed_reg_addr.h"
241408cc1fSYuval Mintz #include "qed_sriov.h"
25fe56b9e6SYuval Mintz
26*e346e231SKonstantin Khorenko #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT 1000
27*e346e231SKonstantin Khorenko #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP 1000
28*e346e231SKonstantin Khorenko #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT 100000
29*e346e231SKonstantin Khorenko #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY 10
30fe56b9e6SYuval Mintz
31fe56b9e6SYuval Mintz /* Invalid values */
32fe56b9e6SYuval Mintz #define QED_BAR_INVALID_OFFSET (cpu_to_le32(-1))
33fe56b9e6SYuval Mintz
34fe56b9e6SYuval Mintz struct qed_ptt {
35fe56b9e6SYuval Mintz struct list_head list_entry;
36fe56b9e6SYuval Mintz unsigned int idx;
37fe56b9e6SYuval Mintz struct pxp_ptt_entry pxp;
383a50d351SMintz, Yuval u8 hwfn_id;
39fe56b9e6SYuval Mintz };
40fe56b9e6SYuval Mintz
41fe56b9e6SYuval Mintz struct qed_ptt_pool {
42fe56b9e6SYuval Mintz struct list_head free_list;
43fe56b9e6SYuval Mintz spinlock_t lock; /* ptt synchronized access */
44fe56b9e6SYuval Mintz struct qed_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM];
45fe56b9e6SYuval Mintz };
46fe56b9e6SYuval Mintz
qed_ptt_pool_alloc(struct qed_hwfn * p_hwfn)47fe56b9e6SYuval Mintz int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn)
48fe56b9e6SYuval Mintz {
491a635e48SYuval Mintz struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL);
50fe56b9e6SYuval Mintz int i;
51fe56b9e6SYuval Mintz
52fe56b9e6SYuval Mintz if (!p_pool)
53fe56b9e6SYuval Mintz return -ENOMEM;
54fe56b9e6SYuval Mintz
55fe56b9e6SYuval Mintz INIT_LIST_HEAD(&p_pool->free_list);
56fe56b9e6SYuval Mintz for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
57fe56b9e6SYuval Mintz p_pool->ptts[i].idx = i;
58fe56b9e6SYuval Mintz p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET;
59fe56b9e6SYuval Mintz p_pool->ptts[i].pxp.pretend.control = 0;
603a50d351SMintz, Yuval p_pool->ptts[i].hwfn_id = p_hwfn->my_id;
61fe56b9e6SYuval Mintz if (i >= RESERVED_PTT_MAX)
62fe56b9e6SYuval Mintz list_add(&p_pool->ptts[i].list_entry,
63fe56b9e6SYuval Mintz &p_pool->free_list);
64fe56b9e6SYuval Mintz }
65fe56b9e6SYuval Mintz
66fe56b9e6SYuval Mintz p_hwfn->p_ptt_pool = p_pool;
67fe56b9e6SYuval Mintz spin_lock_init(&p_pool->lock);
68fe56b9e6SYuval Mintz
69fe56b9e6SYuval Mintz return 0;
70fe56b9e6SYuval Mintz }
71fe56b9e6SYuval Mintz
qed_ptt_invalidate(struct qed_hwfn * p_hwfn)72fe56b9e6SYuval Mintz void qed_ptt_invalidate(struct qed_hwfn *p_hwfn)
73fe56b9e6SYuval Mintz {
74fe56b9e6SYuval Mintz struct qed_ptt *p_ptt;
75fe56b9e6SYuval Mintz int i;
76fe56b9e6SYuval Mintz
77fe56b9e6SYuval Mintz for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
78fe56b9e6SYuval Mintz p_ptt = &p_hwfn->p_ptt_pool->ptts[i];
79fe56b9e6SYuval Mintz p_ptt->pxp.offset = QED_BAR_INVALID_OFFSET;
80fe56b9e6SYuval Mintz }
81fe56b9e6SYuval Mintz }
82fe56b9e6SYuval Mintz
qed_ptt_pool_free(struct qed_hwfn * p_hwfn)83fe56b9e6SYuval Mintz void qed_ptt_pool_free(struct qed_hwfn *p_hwfn)
84fe56b9e6SYuval Mintz {
85fe56b9e6SYuval Mintz kfree(p_hwfn->p_ptt_pool);
86fe56b9e6SYuval Mintz p_hwfn->p_ptt_pool = NULL;
87fe56b9e6SYuval Mintz }
88fe56b9e6SYuval Mintz
qed_ptt_acquire(struct qed_hwfn * p_hwfn)89fe56b9e6SYuval Mintz struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn)
90fe56b9e6SYuval Mintz {
91*e346e231SKonstantin Khorenko return qed_ptt_acquire_context(p_hwfn, false);
92*e346e231SKonstantin Khorenko }
93*e346e231SKonstantin Khorenko
qed_ptt_acquire_context(struct qed_hwfn * p_hwfn,bool is_atomic)94*e346e231SKonstantin Khorenko struct qed_ptt *qed_ptt_acquire_context(struct qed_hwfn *p_hwfn, bool is_atomic)
95*e346e231SKonstantin Khorenko {
96fe56b9e6SYuval Mintz struct qed_ptt *p_ptt;
97*e346e231SKonstantin Khorenko unsigned int i, count;
98*e346e231SKonstantin Khorenko
99*e346e231SKonstantin Khorenko if (is_atomic)
100*e346e231SKonstantin Khorenko count = QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT;
101*e346e231SKonstantin Khorenko else
102*e346e231SKonstantin Khorenko count = QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT;
103fe56b9e6SYuval Mintz
104fe56b9e6SYuval Mintz /* Take the free PTT from the list */
105*e346e231SKonstantin Khorenko for (i = 0; i < count; i++) {
106fe56b9e6SYuval Mintz spin_lock_bh(&p_hwfn->p_ptt_pool->lock);
107fe56b9e6SYuval Mintz
108fe56b9e6SYuval Mintz if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) {
109fe56b9e6SYuval Mintz p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list,
110fe56b9e6SYuval Mintz struct qed_ptt, list_entry);
111fe56b9e6SYuval Mintz list_del(&p_ptt->list_entry);
112fe56b9e6SYuval Mintz
113fe56b9e6SYuval Mintz spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
114fe56b9e6SYuval Mintz
115fe56b9e6SYuval Mintz DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
116fe56b9e6SYuval Mintz "allocated ptt %d\n", p_ptt->idx);
117fe56b9e6SYuval Mintz return p_ptt;
118fe56b9e6SYuval Mintz }
119fe56b9e6SYuval Mintz
120fe56b9e6SYuval Mintz spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
121*e346e231SKonstantin Khorenko
122*e346e231SKonstantin Khorenko if (is_atomic)
123*e346e231SKonstantin Khorenko udelay(QED_BAR_ACQUIRE_TIMEOUT_UDELAY);
124*e346e231SKonstantin Khorenko else
125*e346e231SKonstantin Khorenko usleep_range(QED_BAR_ACQUIRE_TIMEOUT_USLEEP,
126*e346e231SKonstantin Khorenko QED_BAR_ACQUIRE_TIMEOUT_USLEEP * 2);
127fe56b9e6SYuval Mintz }
128fe56b9e6SYuval Mintz
129fe56b9e6SYuval Mintz DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n");
130fe56b9e6SYuval Mintz return NULL;
131fe56b9e6SYuval Mintz }
132fe56b9e6SYuval Mintz
qed_ptt_release(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt)1331a635e48SYuval Mintz void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
134fe56b9e6SYuval Mintz {
135fe56b9e6SYuval Mintz spin_lock_bh(&p_hwfn->p_ptt_pool->lock);
136fe56b9e6SYuval Mintz list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list);
137fe56b9e6SYuval Mintz spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
138fe56b9e6SYuval Mintz }
139fe56b9e6SYuval Mintz
qed_ptt_get_hw_addr(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt)1401a635e48SYuval Mintz u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
141fe56b9e6SYuval Mintz {
142fe56b9e6SYuval Mintz /* The HW is using DWORDS and we need to translate it to Bytes */
143fe56b9e6SYuval Mintz return le32_to_cpu(p_ptt->pxp.offset) << 2;
144fe56b9e6SYuval Mintz }
145fe56b9e6SYuval Mintz
qed_ptt_config_addr(struct qed_ptt * p_ptt)146fe56b9e6SYuval Mintz static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt)
147fe56b9e6SYuval Mintz {
148fe56b9e6SYuval Mintz return PXP_PF_WINDOW_ADMIN_PER_PF_START +
149fe56b9e6SYuval Mintz p_ptt->idx * sizeof(struct pxp_ptt_entry);
150fe56b9e6SYuval Mintz }
151fe56b9e6SYuval Mintz
qed_ptt_get_bar_addr(struct qed_ptt * p_ptt)152fe56b9e6SYuval Mintz u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt)
153fe56b9e6SYuval Mintz {
154fe56b9e6SYuval Mintz return PXP_EXTERNAL_BAR_PF_WINDOW_START +
155fe56b9e6SYuval Mintz p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE;
156fe56b9e6SYuval Mintz }
157fe56b9e6SYuval Mintz
qed_ptt_set_win(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 new_hw_addr)158fe56b9e6SYuval Mintz void qed_ptt_set_win(struct qed_hwfn *p_hwfn,
1591a635e48SYuval Mintz struct qed_ptt *p_ptt, u32 new_hw_addr)
160fe56b9e6SYuval Mintz {
161fe56b9e6SYuval Mintz u32 prev_hw_addr;
162fe56b9e6SYuval Mintz
163fe56b9e6SYuval Mintz prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt);
164fe56b9e6SYuval Mintz
165fe56b9e6SYuval Mintz if (new_hw_addr == prev_hw_addr)
166fe56b9e6SYuval Mintz return;
167fe56b9e6SYuval Mintz
168fe56b9e6SYuval Mintz /* Update PTT entery in admin window */
169fe56b9e6SYuval Mintz DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
170fe56b9e6SYuval Mintz "Updating PTT entry %d to offset 0x%x\n",
171fe56b9e6SYuval Mintz p_ptt->idx, new_hw_addr);
172fe56b9e6SYuval Mintz
173fe56b9e6SYuval Mintz /* The HW is using DWORDS and the address is in Bytes */
174fe56b9e6SYuval Mintz p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2);
175fe56b9e6SYuval Mintz
176fe56b9e6SYuval Mintz REG_WR(p_hwfn,
177fe56b9e6SYuval Mintz qed_ptt_config_addr(p_ptt) +
178fe56b9e6SYuval Mintz offsetof(struct pxp_ptt_entry, offset),
179fe56b9e6SYuval Mintz le32_to_cpu(p_ptt->pxp.offset));
180fe56b9e6SYuval Mintz }
181fe56b9e6SYuval Mintz
qed_set_ptt(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 hw_addr)182fe56b9e6SYuval Mintz static u32 qed_set_ptt(struct qed_hwfn *p_hwfn,
1831a635e48SYuval Mintz struct qed_ptt *p_ptt, u32 hw_addr)
184fe56b9e6SYuval Mintz {
185fe56b9e6SYuval Mintz u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt);
186fe56b9e6SYuval Mintz u32 offset;
187fe56b9e6SYuval Mintz
188fe56b9e6SYuval Mintz offset = hw_addr - win_hw_addr;
189fe56b9e6SYuval Mintz
1903a50d351SMintz, Yuval if (p_ptt->hwfn_id != p_hwfn->my_id)
1913a50d351SMintz, Yuval DP_NOTICE(p_hwfn,
1923a50d351SMintz, Yuval "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n",
1933a50d351SMintz, Yuval p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id);
1943a50d351SMintz, Yuval
195fe56b9e6SYuval Mintz /* Verify the address is within the window */
196fe56b9e6SYuval Mintz if (hw_addr < win_hw_addr ||
197fe56b9e6SYuval Mintz offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) {
198fe56b9e6SYuval Mintz qed_ptt_set_win(p_hwfn, p_ptt, hw_addr);
199fe56b9e6SYuval Mintz offset = 0;
200fe56b9e6SYuval Mintz }
201fe56b9e6SYuval Mintz
202fe56b9e6SYuval Mintz return qed_ptt_get_bar_addr(p_ptt) + offset;
203fe56b9e6SYuval Mintz }
204fe56b9e6SYuval Mintz
qed_get_reserved_ptt(struct qed_hwfn * p_hwfn,enum reserved_ptts ptt_idx)205fe56b9e6SYuval Mintz struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn,
206fe56b9e6SYuval Mintz enum reserved_ptts ptt_idx)
207fe56b9e6SYuval Mintz {
208fe56b9e6SYuval Mintz if (ptt_idx >= RESERVED_PTT_MAX) {
209fe56b9e6SYuval Mintz DP_NOTICE(p_hwfn,
210fe56b9e6SYuval Mintz "Requested PTT %d is out of range\n", ptt_idx);
211fe56b9e6SYuval Mintz return NULL;
212fe56b9e6SYuval Mintz }
213fe56b9e6SYuval Mintz
214fe56b9e6SYuval Mintz return &p_hwfn->p_ptt_pool->ptts[ptt_idx];
215fe56b9e6SYuval Mintz }
216fe56b9e6SYuval Mintz
qed_wr(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 hw_addr,u32 val)217fe56b9e6SYuval Mintz void qed_wr(struct qed_hwfn *p_hwfn,
218fe56b9e6SYuval Mintz struct qed_ptt *p_ptt,
219fe56b9e6SYuval Mintz u32 hw_addr, u32 val)
220fe56b9e6SYuval Mintz {
221fe56b9e6SYuval Mintz u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr);
222fe56b9e6SYuval Mintz
223fe56b9e6SYuval Mintz REG_WR(p_hwfn, bar_addr, val);
224fe56b9e6SYuval Mintz DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
225fe56b9e6SYuval Mintz "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
226fe56b9e6SYuval Mintz bar_addr, hw_addr, val);
227fe56b9e6SYuval Mintz }
228fe56b9e6SYuval Mintz
qed_rd(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 hw_addr)229fe56b9e6SYuval Mintz u32 qed_rd(struct qed_hwfn *p_hwfn,
230fe56b9e6SYuval Mintz struct qed_ptt *p_ptt,
231fe56b9e6SYuval Mintz u32 hw_addr)
232fe56b9e6SYuval Mintz {
233fe56b9e6SYuval Mintz u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr);
234fe56b9e6SYuval Mintz u32 val = REG_RD(p_hwfn, bar_addr);
235fe56b9e6SYuval Mintz
236fe56b9e6SYuval Mintz DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
237fe56b9e6SYuval Mintz "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
238fe56b9e6SYuval Mintz bar_addr, hw_addr, val);
239fe56b9e6SYuval Mintz
240fe56b9e6SYuval Mintz return val;
241fe56b9e6SYuval Mintz }
242fe56b9e6SYuval Mintz
qed_memcpy_hw(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,void * addr,u32 hw_addr,size_t n,bool to_device)243fe56b9e6SYuval Mintz static void qed_memcpy_hw(struct qed_hwfn *p_hwfn,
244fe56b9e6SYuval Mintz struct qed_ptt *p_ptt,
2451a635e48SYuval Mintz void *addr, u32 hw_addr, size_t n, bool to_device)
246fe56b9e6SYuval Mintz {
247fe56b9e6SYuval Mintz u32 dw_count, *host_addr, hw_offset;
248fe56b9e6SYuval Mintz size_t quota, done = 0;
249fe56b9e6SYuval Mintz u32 __iomem *reg_addr;
250fe56b9e6SYuval Mintz
251fe56b9e6SYuval Mintz while (done < n) {
252fe56b9e6SYuval Mintz quota = min_t(size_t, n - done,
253fe56b9e6SYuval Mintz PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE);
254fe56b9e6SYuval Mintz
2551408cc1fSYuval Mintz if (IS_PF(p_hwfn->cdev)) {
256fe56b9e6SYuval Mintz qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done);
257fe56b9e6SYuval Mintz hw_offset = qed_ptt_get_bar_addr(p_ptt);
2581408cc1fSYuval Mintz } else {
2591408cc1fSYuval Mintz hw_offset = hw_addr + done;
2601408cc1fSYuval Mintz }
261fe56b9e6SYuval Mintz
262fe56b9e6SYuval Mintz dw_count = quota / 4;
263fe56b9e6SYuval Mintz host_addr = (u32 *)((u8 *)addr + done);
264fe56b9e6SYuval Mintz reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset);
265fe56b9e6SYuval Mintz if (to_device)
266fe56b9e6SYuval Mintz while (dw_count--)
267fe56b9e6SYuval Mintz DIRECT_REG_WR(reg_addr++, *host_addr++);
268fe56b9e6SYuval Mintz else
269fe56b9e6SYuval Mintz while (dw_count--)
270fe56b9e6SYuval Mintz *host_addr++ = DIRECT_REG_RD(reg_addr++);
271fe56b9e6SYuval Mintz
272fe56b9e6SYuval Mintz done += quota;
273fe56b9e6SYuval Mintz }
274fe56b9e6SYuval Mintz }
275fe56b9e6SYuval Mintz
qed_memcpy_from(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,void * dest,u32 hw_addr,size_t n)276fe56b9e6SYuval Mintz void qed_memcpy_from(struct qed_hwfn *p_hwfn,
2771a635e48SYuval Mintz struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n)
278fe56b9e6SYuval Mintz {
279fe56b9e6SYuval Mintz DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
280fe56b9e6SYuval Mintz "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n",
281fe56b9e6SYuval Mintz hw_addr, dest, hw_addr, (unsigned long)n);
282fe56b9e6SYuval Mintz
283fe56b9e6SYuval Mintz qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false);
284fe56b9e6SYuval Mintz }
285fe56b9e6SYuval Mintz
qed_memcpy_to(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 hw_addr,void * src,size_t n)286fe56b9e6SYuval Mintz void qed_memcpy_to(struct qed_hwfn *p_hwfn,
2871a635e48SYuval Mintz struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n)
288fe56b9e6SYuval Mintz {
289fe56b9e6SYuval Mintz DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
290fe56b9e6SYuval Mintz "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n",
291fe56b9e6SYuval Mintz hw_addr, hw_addr, src, (unsigned long)n);
292fe56b9e6SYuval Mintz
293fe56b9e6SYuval Mintz qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true);
294fe56b9e6SYuval Mintz }
295fe56b9e6SYuval Mintz
qed_fid_pretend(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u16 fid)2961a635e48SYuval Mintz void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid)
297fe56b9e6SYuval Mintz {
298fe56b9e6SYuval Mintz u16 control = 0;
299fe56b9e6SYuval Mintz
300fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
301fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
302fe56b9e6SYuval Mintz
303fe56b9e6SYuval Mintz /* Every pretend undos previous pretends, including
304fe56b9e6SYuval Mintz * previous port pretend.
305fe56b9e6SYuval Mintz */
306fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
307fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
308fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
309fe56b9e6SYuval Mintz
310fe56b9e6SYuval Mintz if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
311fe56b9e6SYuval Mintz fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
312fe56b9e6SYuval Mintz
313fe56b9e6SYuval Mintz p_ptt->pxp.pretend.control = cpu_to_le16(control);
314fe56b9e6SYuval Mintz p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid);
315fe56b9e6SYuval Mintz
316fe56b9e6SYuval Mintz REG_WR(p_hwfn,
317fe56b9e6SYuval Mintz qed_ptt_config_addr(p_ptt) +
318fe56b9e6SYuval Mintz offsetof(struct pxp_ptt_entry, pretend),
319fe56b9e6SYuval Mintz *(u32 *)&p_ptt->pxp.pretend);
320fe56b9e6SYuval Mintz }
321fe56b9e6SYuval Mintz
qed_port_pretend(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u8 port_id)322fe56b9e6SYuval Mintz void qed_port_pretend(struct qed_hwfn *p_hwfn,
3231a635e48SYuval Mintz struct qed_ptt *p_ptt, u8 port_id)
324fe56b9e6SYuval Mintz {
325fe56b9e6SYuval Mintz u16 control = 0;
326fe56b9e6SYuval Mintz
327fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
328fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
329fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
330fe56b9e6SYuval Mintz
331fe56b9e6SYuval Mintz p_ptt->pxp.pretend.control = cpu_to_le16(control);
332fe56b9e6SYuval Mintz
333fe56b9e6SYuval Mintz REG_WR(p_hwfn,
334fe56b9e6SYuval Mintz qed_ptt_config_addr(p_ptt) +
335fe56b9e6SYuval Mintz offsetof(struct pxp_ptt_entry, pretend),
336fe56b9e6SYuval Mintz *(u32 *)&p_ptt->pxp.pretend);
337fe56b9e6SYuval Mintz }
338fe56b9e6SYuval Mintz
qed_port_unpretend(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt)3391a635e48SYuval Mintz void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
340fe56b9e6SYuval Mintz {
341fe56b9e6SYuval Mintz u16 control = 0;
342fe56b9e6SYuval Mintz
343fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
344fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
345fe56b9e6SYuval Mintz SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
346fe56b9e6SYuval Mintz
347fe56b9e6SYuval Mintz p_ptt->pxp.pretend.control = cpu_to_le16(control);
348fe56b9e6SYuval Mintz
349fe56b9e6SYuval Mintz REG_WR(p_hwfn,
350fe56b9e6SYuval Mintz qed_ptt_config_addr(p_ptt) +
351fe56b9e6SYuval Mintz offsetof(struct pxp_ptt_entry, pretend),
352fe56b9e6SYuval Mintz *(u32 *)&p_ptt->pxp.pretend);
353fe56b9e6SYuval Mintz }
354fe56b9e6SYuval Mintz
qed_port_fid_pretend(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u8 port_id,u16 fid)355d52c89f1SMichal Kalderon void qed_port_fid_pretend(struct qed_hwfn *p_hwfn,
356d52c89f1SMichal Kalderon struct qed_ptt *p_ptt, u8 port_id, u16 fid)
357d52c89f1SMichal Kalderon {
358d52c89f1SMichal Kalderon u16 control = 0;
359d52c89f1SMichal Kalderon
360d52c89f1SMichal Kalderon SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
361d52c89f1SMichal Kalderon SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
362d52c89f1SMichal Kalderon SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
363d52c89f1SMichal Kalderon SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
364d52c89f1SMichal Kalderon SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
365d52c89f1SMichal Kalderon if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
366d52c89f1SMichal Kalderon fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
367d52c89f1SMichal Kalderon p_ptt->pxp.pretend.control = cpu_to_le16(control);
368d52c89f1SMichal Kalderon p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid);
369d52c89f1SMichal Kalderon REG_WR(p_hwfn,
370d52c89f1SMichal Kalderon qed_ptt_config_addr(p_ptt) +
371d52c89f1SMichal Kalderon offsetof(struct pxp_ptt_entry, pretend),
372d52c89f1SMichal Kalderon *(u32 *)&p_ptt->pxp.pretend);
373d52c89f1SMichal Kalderon }
374d52c89f1SMichal Kalderon
qed_vfid_to_concrete(struct qed_hwfn * p_hwfn,u8 vfid)37532a47e72SYuval Mintz u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid)
37632a47e72SYuval Mintz {
37732a47e72SYuval Mintz u32 concrete_fid = 0;
37832a47e72SYuval Mintz
37932a47e72SYuval Mintz SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id);
38032a47e72SYuval Mintz SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid);
38132a47e72SYuval Mintz SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1);
38232a47e72SYuval Mintz
38332a47e72SYuval Mintz return concrete_fid;
38432a47e72SYuval Mintz }
38532a47e72SYuval Mintz
386fe56b9e6SYuval Mintz /* DMAE */
38783bf76e3SMichal Kalderon #define QED_DMAE_FLAGS_IS_SET(params, flag) \
388804c5702SMichal Kalderon ((params) != NULL && GET_FIELD((params)->flags, QED_DMAE_PARAMS_##flag))
38983bf76e3SMichal Kalderon
qed_dmae_opcode(struct qed_hwfn * p_hwfn,const u8 is_src_type_grc,const u8 is_dst_type_grc,struct qed_dmae_params * p_params)390fe56b9e6SYuval Mintz static void qed_dmae_opcode(struct qed_hwfn *p_hwfn,
391fe56b9e6SYuval Mintz const u8 is_src_type_grc,
392fe56b9e6SYuval Mintz const u8 is_dst_type_grc,
393fe56b9e6SYuval Mintz struct qed_dmae_params *p_params)
394fe56b9e6SYuval Mintz {
39583bf76e3SMichal Kalderon u8 src_pfid, dst_pfid, port_id;
39637bff2b9SYuval Mintz u16 opcode_b = 0;
397fe56b9e6SYuval Mintz u32 opcode = 0;
398fe56b9e6SYuval Mintz
399fe56b9e6SYuval Mintz /* Whether the source is the PCIe or the GRC.
400fe56b9e6SYuval Mintz * 0- The source is the PCIe
401fe56b9e6SYuval Mintz * 1- The source is the GRC.
402fe56b9e6SYuval Mintz */
403804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_SRC,
404804c5702SMichal Kalderon (is_src_type_grc ? dmae_cmd_src_grc : dmae_cmd_src_pcie));
405804c5702SMichal Kalderon src_pfid = QED_DMAE_FLAGS_IS_SET(p_params, SRC_PF_VALID) ?
40683bf76e3SMichal Kalderon p_params->src_pfid : p_hwfn->rel_pf_id;
407804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_SRC_PF_ID, src_pfid);
408fe56b9e6SYuval Mintz
409fe56b9e6SYuval Mintz /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */
410804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_DST,
411804c5702SMichal Kalderon (is_dst_type_grc ? dmae_cmd_dst_grc : dmae_cmd_dst_pcie));
412804c5702SMichal Kalderon dst_pfid = QED_DMAE_FLAGS_IS_SET(p_params, DST_PF_VALID) ?
41383bf76e3SMichal Kalderon p_params->dst_pfid : p_hwfn->rel_pf_id;
414804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_DST_PF_ID, dst_pfid);
415804c5702SMichal Kalderon
416fe56b9e6SYuval Mintz
417fe56b9e6SYuval Mintz /* Whether to write a completion word to the completion destination:
418fe56b9e6SYuval Mintz * 0-Do not write a completion word
419fe56b9e6SYuval Mintz * 1-Write the completion word
420fe56b9e6SYuval Mintz */
421804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_COMP_WORD_EN, 1);
422804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1);
423fe56b9e6SYuval Mintz
42483bf76e3SMichal Kalderon if (QED_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST))
425804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_COMP_FUNC, 1);
426fe56b9e6SYuval Mintz
427804c5702SMichal Kalderon /* swapping mode 3 - big endian */
428804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_ENDIANITY_MODE, DMAE_CMD_ENDIANITY);
429fe56b9e6SYuval Mintz
430804c5702SMichal Kalderon port_id = (QED_DMAE_FLAGS_IS_SET(p_params, PORT_VALID)) ?
43183bf76e3SMichal Kalderon p_params->port_id : p_hwfn->port_id;
432804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_PORT_ID, port_id);
433fe56b9e6SYuval Mintz
434fe56b9e6SYuval Mintz /* reset source address in next go */
435804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1);
436fe56b9e6SYuval Mintz
437fe56b9e6SYuval Mintz /* reset dest address in next go */
438804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_DST_ADDR_RESET, 1);
439fe56b9e6SYuval Mintz
44037bff2b9SYuval Mintz /* SRC/DST VFID: all 1's - pf, otherwise VF id */
441804c5702SMichal Kalderon if (QED_DMAE_FLAGS_IS_SET(p_params, SRC_VF_VALID)) {
442804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_SRC_VF_ID_VALID, 1);
443804c5702SMichal Kalderon SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, p_params->src_vfid);
44437bff2b9SYuval Mintz } else {
445804c5702SMichal Kalderon SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, 0xFF);
44637bff2b9SYuval Mintz }
447804c5702SMichal Kalderon if (QED_DMAE_FLAGS_IS_SET(p_params, DST_VF_VALID)) {
448804c5702SMichal Kalderon SET_FIELD(opcode, DMAE_CMD_DST_VF_ID_VALID, 1);
449804c5702SMichal Kalderon SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, p_params->dst_vfid);
45037bff2b9SYuval Mintz } else {
451804c5702SMichal Kalderon SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, 0xFF);
45237bff2b9SYuval Mintz }
453fe56b9e6SYuval Mintz
454fe56b9e6SYuval Mintz p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode);
45537bff2b9SYuval Mintz p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b);
456fe56b9e6SYuval Mintz }
457fe56b9e6SYuval Mintz
qed_dmae_idx_to_go_cmd(u8 idx)458fe56b9e6SYuval Mintz u32 qed_dmae_idx_to_go_cmd(u8 idx)
459fe56b9e6SYuval Mintz {
460fe56b9e6SYuval Mintz /* All the DMAE 'go' registers form an array in internal memory */
461fe56b9e6SYuval Mintz return DMAE_REG_GO_C0 + (idx << 2);
462fe56b9e6SYuval Mintz }
463fe56b9e6SYuval Mintz
qed_dmae_post_command(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt)4641a635e48SYuval Mintz static int qed_dmae_post_command(struct qed_hwfn *p_hwfn,
465fe56b9e6SYuval Mintz struct qed_ptt *p_ptt)
466fe56b9e6SYuval Mintz {
4671a635e48SYuval Mintz struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd;
468fe56b9e6SYuval Mintz u8 idx_cmd = p_hwfn->dmae_info.channel, i;
469fe56b9e6SYuval Mintz int qed_status = 0;
470fe56b9e6SYuval Mintz
471fe56b9e6SYuval Mintz /* verify address is not NULL */
4721a635e48SYuval Mintz if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) ||
4731a635e48SYuval Mintz ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) {
474fe56b9e6SYuval Mintz DP_NOTICE(p_hwfn,
475fe56b9e6SYuval Mintz "source or destination address 0 idx_cmd=%d\n"
476fe56b9e6SYuval Mintz "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
477fe56b9e6SYuval Mintz idx_cmd,
4781a635e48SYuval Mintz le32_to_cpu(p_command->opcode),
4791a635e48SYuval Mintz le16_to_cpu(p_command->opcode_b),
4801a635e48SYuval Mintz le16_to_cpu(p_command->length_dw),
4811a635e48SYuval Mintz le32_to_cpu(p_command->src_addr_hi),
4821a635e48SYuval Mintz le32_to_cpu(p_command->src_addr_lo),
4831a635e48SYuval Mintz le32_to_cpu(p_command->dst_addr_hi),
4841a635e48SYuval Mintz le32_to_cpu(p_command->dst_addr_lo));
485fe56b9e6SYuval Mintz
486fe56b9e6SYuval Mintz return -EINVAL;
487fe56b9e6SYuval Mintz }
488fe56b9e6SYuval Mintz
489fe56b9e6SYuval Mintz DP_VERBOSE(p_hwfn,
490fe56b9e6SYuval Mintz NETIF_MSG_HW,
491fe56b9e6SYuval Mintz "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
492fe56b9e6SYuval Mintz idx_cmd,
4931a635e48SYuval Mintz le32_to_cpu(p_command->opcode),
4941a635e48SYuval Mintz le16_to_cpu(p_command->opcode_b),
4951a635e48SYuval Mintz le16_to_cpu(p_command->length_dw),
4961a635e48SYuval Mintz le32_to_cpu(p_command->src_addr_hi),
4971a635e48SYuval Mintz le32_to_cpu(p_command->src_addr_lo),
4981a635e48SYuval Mintz le32_to_cpu(p_command->dst_addr_hi),
4991a635e48SYuval Mintz le32_to_cpu(p_command->dst_addr_lo));
500fe56b9e6SYuval Mintz
501fe56b9e6SYuval Mintz /* Copy the command to DMAE - need to do it before every call
502fe56b9e6SYuval Mintz * for source/dest address no reset.
503fe56b9e6SYuval Mintz * The first 9 DWs are the command registers, the 10 DW is the
504fe56b9e6SYuval Mintz * GO register, and the rest are result registers
505fe56b9e6SYuval Mintz * (which are read only by the client).
506fe56b9e6SYuval Mintz */
507fe56b9e6SYuval Mintz for (i = 0; i < DMAE_CMD_SIZE; i++) {
508fe56b9e6SYuval Mintz u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ?
5091a635e48SYuval Mintz *(((u32 *)p_command) + i) : 0;
510fe56b9e6SYuval Mintz
511fe56b9e6SYuval Mintz qed_wr(p_hwfn, p_ptt,
512fe56b9e6SYuval Mintz DMAE_REG_CMD_MEM +
513fe56b9e6SYuval Mintz (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) +
514fe56b9e6SYuval Mintz (i * sizeof(u32)), data);
515fe56b9e6SYuval Mintz }
516fe56b9e6SYuval Mintz
5171a635e48SYuval Mintz qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE);
518fe56b9e6SYuval Mintz
519fe56b9e6SYuval Mintz return qed_status;
520fe56b9e6SYuval Mintz }
521fe56b9e6SYuval Mintz
qed_dmae_info_alloc(struct qed_hwfn * p_hwfn)522fe56b9e6SYuval Mintz int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn)
523fe56b9e6SYuval Mintz {
524fe56b9e6SYuval Mintz dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr;
525fe56b9e6SYuval Mintz struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd;
526fe56b9e6SYuval Mintz u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer;
527fe56b9e6SYuval Mintz u32 **p_comp = &p_hwfn->dmae_info.p_completion_word;
528fe56b9e6SYuval Mintz
529fe56b9e6SYuval Mintz *p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
5301a635e48SYuval Mintz sizeof(u32), p_addr, GFP_KERNEL);
5312591c280SJoe Perches if (!*p_comp)
532fe56b9e6SYuval Mintz goto err;
533fe56b9e6SYuval Mintz
534fe56b9e6SYuval Mintz p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr;
535fe56b9e6SYuval Mintz *p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
536fe56b9e6SYuval Mintz sizeof(struct dmae_cmd),
537fe56b9e6SYuval Mintz p_addr, GFP_KERNEL);
5382591c280SJoe Perches if (!*p_cmd)
539fe56b9e6SYuval Mintz goto err;
540fe56b9e6SYuval Mintz
541fe56b9e6SYuval Mintz p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr;
542fe56b9e6SYuval Mintz *p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
543fe56b9e6SYuval Mintz sizeof(u32) * DMAE_MAX_RW_SIZE,
544fe56b9e6SYuval Mintz p_addr, GFP_KERNEL);
5452591c280SJoe Perches if (!*p_buff)
546fe56b9e6SYuval Mintz goto err;
547fe56b9e6SYuval Mintz
548fe56b9e6SYuval Mintz p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id;
549fe56b9e6SYuval Mintz
550fe56b9e6SYuval Mintz return 0;
551fe56b9e6SYuval Mintz err:
552fe56b9e6SYuval Mintz qed_dmae_info_free(p_hwfn);
553fe56b9e6SYuval Mintz return -ENOMEM;
554fe56b9e6SYuval Mintz }
555fe56b9e6SYuval Mintz
qed_dmae_info_free(struct qed_hwfn * p_hwfn)556fe56b9e6SYuval Mintz void qed_dmae_info_free(struct qed_hwfn *p_hwfn)
557fe56b9e6SYuval Mintz {
558fe56b9e6SYuval Mintz dma_addr_t p_phys;
559fe56b9e6SYuval Mintz
560fe56b9e6SYuval Mintz /* Just make sure no one is in the middle */
561fe56b9e6SYuval Mintz mutex_lock(&p_hwfn->dmae_info.mutex);
562fe56b9e6SYuval Mintz
563fe56b9e6SYuval Mintz if (p_hwfn->dmae_info.p_completion_word) {
564fe56b9e6SYuval Mintz p_phys = p_hwfn->dmae_info.completion_word_phys_addr;
565fe56b9e6SYuval Mintz dma_free_coherent(&p_hwfn->cdev->pdev->dev,
566fe56b9e6SYuval Mintz sizeof(u32),
5671a635e48SYuval Mintz p_hwfn->dmae_info.p_completion_word, p_phys);
568fe56b9e6SYuval Mintz p_hwfn->dmae_info.p_completion_word = NULL;
569fe56b9e6SYuval Mintz }
570fe56b9e6SYuval Mintz
571fe56b9e6SYuval Mintz if (p_hwfn->dmae_info.p_dmae_cmd) {
572fe56b9e6SYuval Mintz p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr;
573fe56b9e6SYuval Mintz dma_free_coherent(&p_hwfn->cdev->pdev->dev,
574fe56b9e6SYuval Mintz sizeof(struct dmae_cmd),
5751a635e48SYuval Mintz p_hwfn->dmae_info.p_dmae_cmd, p_phys);
576fe56b9e6SYuval Mintz p_hwfn->dmae_info.p_dmae_cmd = NULL;
577fe56b9e6SYuval Mintz }
578fe56b9e6SYuval Mintz
579fe56b9e6SYuval Mintz if (p_hwfn->dmae_info.p_intermediate_buffer) {
580fe56b9e6SYuval Mintz p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
581fe56b9e6SYuval Mintz dma_free_coherent(&p_hwfn->cdev->pdev->dev,
582fe56b9e6SYuval Mintz sizeof(u32) * DMAE_MAX_RW_SIZE,
583fe56b9e6SYuval Mintz p_hwfn->dmae_info.p_intermediate_buffer,
584fe56b9e6SYuval Mintz p_phys);
585fe56b9e6SYuval Mintz p_hwfn->dmae_info.p_intermediate_buffer = NULL;
586fe56b9e6SYuval Mintz }
587fe56b9e6SYuval Mintz
588fe56b9e6SYuval Mintz mutex_unlock(&p_hwfn->dmae_info.mutex);
589fe56b9e6SYuval Mintz }
590fe56b9e6SYuval Mintz
qed_dmae_operation_wait(struct qed_hwfn * p_hwfn)591fe56b9e6SYuval Mintz static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn)
592fe56b9e6SYuval Mintz {
5931a635e48SYuval Mintz u32 wait_cnt_limit = 10000, wait_cnt = 0;
594fe56b9e6SYuval Mintz int qed_status = 0;
595fe56b9e6SYuval Mintz
596fe56b9e6SYuval Mintz barrier();
597fe56b9e6SYuval Mintz while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) {
598fe56b9e6SYuval Mintz udelay(DMAE_MIN_WAIT_TIME);
599fe56b9e6SYuval Mintz if (++wait_cnt > wait_cnt_limit) {
600fe56b9e6SYuval Mintz DP_NOTICE(p_hwfn->cdev,
601fe56b9e6SYuval Mintz "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n",
602fe56b9e6SYuval Mintz *p_hwfn->dmae_info.p_completion_word,
603fe56b9e6SYuval Mintz DMAE_COMPLETION_VAL);
604fe56b9e6SYuval Mintz qed_status = -EBUSY;
605fe56b9e6SYuval Mintz break;
606fe56b9e6SYuval Mintz }
607fe56b9e6SYuval Mintz
608fe56b9e6SYuval Mintz /* to sync the completion_word since we are not
609fe56b9e6SYuval Mintz * using the volatile keyword for p_completion_word
610fe56b9e6SYuval Mintz */
611fe56b9e6SYuval Mintz barrier();
612fe56b9e6SYuval Mintz }
613fe56b9e6SYuval Mintz
614fe56b9e6SYuval Mintz if (qed_status == 0)
615fe56b9e6SYuval Mintz *p_hwfn->dmae_info.p_completion_word = 0;
616fe56b9e6SYuval Mintz
617fe56b9e6SYuval Mintz return qed_status;
618fe56b9e6SYuval Mintz }
619fe56b9e6SYuval Mintz
qed_dmae_execute_sub_operation(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u64 src_addr,u64 dst_addr,u8 src_type,u8 dst_type,u32 length_dw)620fe56b9e6SYuval Mintz static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn,
621fe56b9e6SYuval Mintz struct qed_ptt *p_ptt,
622fe56b9e6SYuval Mintz u64 src_addr,
623fe56b9e6SYuval Mintz u64 dst_addr,
624fe56b9e6SYuval Mintz u8 src_type,
625fe56b9e6SYuval Mintz u8 dst_type,
6261a635e48SYuval Mintz u32 length_dw)
627fe56b9e6SYuval Mintz {
628fe56b9e6SYuval Mintz dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
629fe56b9e6SYuval Mintz struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
630fe56b9e6SYuval Mintz int qed_status = 0;
631fe56b9e6SYuval Mintz
632fe56b9e6SYuval Mintz switch (src_type) {
633fe56b9e6SYuval Mintz case QED_DMAE_ADDRESS_GRC:
634fe56b9e6SYuval Mintz case QED_DMAE_ADDRESS_HOST_PHYS:
635fe56b9e6SYuval Mintz cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr));
636fe56b9e6SYuval Mintz cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr));
637fe56b9e6SYuval Mintz break;
638fe56b9e6SYuval Mintz /* for virtual source addresses we use the intermediate buffer. */
639fe56b9e6SYuval Mintz case QED_DMAE_ADDRESS_HOST_VIRT:
640fe56b9e6SYuval Mintz cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys));
641fe56b9e6SYuval Mintz cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys));
642fe56b9e6SYuval Mintz memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0],
643fe56b9e6SYuval Mintz (void *)(uintptr_t)src_addr,
6441a635e48SYuval Mintz length_dw * sizeof(u32));
645fe56b9e6SYuval Mintz break;
646fe56b9e6SYuval Mintz default:
647fe56b9e6SYuval Mintz return -EINVAL;
648fe56b9e6SYuval Mintz }
649fe56b9e6SYuval Mintz
650fe56b9e6SYuval Mintz switch (dst_type) {
651fe56b9e6SYuval Mintz case QED_DMAE_ADDRESS_GRC:
652fe56b9e6SYuval Mintz case QED_DMAE_ADDRESS_HOST_PHYS:
653fe56b9e6SYuval Mintz cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr));
654fe56b9e6SYuval Mintz cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr));
655fe56b9e6SYuval Mintz break;
656fe56b9e6SYuval Mintz /* for virtual source addresses we use the intermediate buffer. */
657fe56b9e6SYuval Mintz case QED_DMAE_ADDRESS_HOST_VIRT:
658fe56b9e6SYuval Mintz cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys));
659fe56b9e6SYuval Mintz cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys));
660fe56b9e6SYuval Mintz break;
661fe56b9e6SYuval Mintz default:
662fe56b9e6SYuval Mintz return -EINVAL;
663fe56b9e6SYuval Mintz }
664fe56b9e6SYuval Mintz
6651a635e48SYuval Mintz cmd->length_dw = cpu_to_le16((u16)length_dw);
666fe56b9e6SYuval Mintz
667fe56b9e6SYuval Mintz qed_dmae_post_command(p_hwfn, p_ptt);
668fe56b9e6SYuval Mintz
669fe56b9e6SYuval Mintz qed_status = qed_dmae_operation_wait(p_hwfn);
670fe56b9e6SYuval Mintz
671fe56b9e6SYuval Mintz if (qed_status) {
672fe56b9e6SYuval Mintz DP_NOTICE(p_hwfn,
673fe56b9e6SYuval Mintz "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n",
6741a635e48SYuval Mintz src_addr, dst_addr, length_dw);
675fe56b9e6SYuval Mintz return qed_status;
676fe56b9e6SYuval Mintz }
677fe56b9e6SYuval Mintz
678fe56b9e6SYuval Mintz if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT)
679fe56b9e6SYuval Mintz memcpy((void *)(uintptr_t)(dst_addr),
680fe56b9e6SYuval Mintz &p_hwfn->dmae_info.p_intermediate_buffer[0],
6811a635e48SYuval Mintz length_dw * sizeof(u32));
682fe56b9e6SYuval Mintz
683fe56b9e6SYuval Mintz return 0;
684fe56b9e6SYuval Mintz }
685fe56b9e6SYuval Mintz
qed_dmae_execute_command(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u64 src_addr,u64 dst_addr,u8 src_type,u8 dst_type,u32 size_in_dwords,struct qed_dmae_params * p_params)686fe56b9e6SYuval Mintz static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn,
687fe56b9e6SYuval Mintz struct qed_ptt *p_ptt,
688fe56b9e6SYuval Mintz u64 src_addr, u64 dst_addr,
689fe56b9e6SYuval Mintz u8 src_type, u8 dst_type,
690fe56b9e6SYuval Mintz u32 size_in_dwords,
691fe56b9e6SYuval Mintz struct qed_dmae_params *p_params)
692fe56b9e6SYuval Mintz {
693fe56b9e6SYuval Mintz dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr;
694fe56b9e6SYuval Mintz u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0;
695fe56b9e6SYuval Mintz struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
696fe56b9e6SYuval Mintz u64 src_addr_split = 0, dst_addr_split = 0;
697fe56b9e6SYuval Mintz u16 length_limit = DMAE_MAX_RW_SIZE;
698fe56b9e6SYuval Mintz int qed_status = 0;
699fe56b9e6SYuval Mintz u32 offset = 0;
700fe56b9e6SYuval Mintz
70164515dc8STomer Tayar if (p_hwfn->cdev->recov_in_prog) {
70264515dc8STomer Tayar DP_VERBOSE(p_hwfn,
70364515dc8STomer Tayar NETIF_MSG_HW,
70464515dc8STomer Tayar "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n",
70564515dc8STomer Tayar src_addr, src_type, dst_addr, dst_type,
70664515dc8STomer Tayar size_in_dwords);
70764515dc8STomer Tayar
70864515dc8STomer Tayar /* Let the flow complete w/o any error handling */
70964515dc8STomer Tayar return 0;
71064515dc8STomer Tayar }
71164515dc8STomer Tayar
712fe56b9e6SYuval Mintz qed_dmae_opcode(p_hwfn,
713fe56b9e6SYuval Mintz (src_type == QED_DMAE_ADDRESS_GRC),
714fe56b9e6SYuval Mintz (dst_type == QED_DMAE_ADDRESS_GRC),
715fe56b9e6SYuval Mintz p_params);
716fe56b9e6SYuval Mintz
717fe56b9e6SYuval Mintz cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys));
718fe56b9e6SYuval Mintz cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys));
719fe56b9e6SYuval Mintz cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL);
720fe56b9e6SYuval Mintz
721fe56b9e6SYuval Mintz /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */
722fe56b9e6SYuval Mintz cnt_split = size_in_dwords / length_limit;
723fe56b9e6SYuval Mintz length_mod = size_in_dwords % length_limit;
724fe56b9e6SYuval Mintz
725fe56b9e6SYuval Mintz src_addr_split = src_addr;
726fe56b9e6SYuval Mintz dst_addr_split = dst_addr;
727fe56b9e6SYuval Mintz
728fe56b9e6SYuval Mintz for (i = 0; i <= cnt_split; i++) {
729fe56b9e6SYuval Mintz offset = length_limit * i;
730fe56b9e6SYuval Mintz
73183bf76e3SMichal Kalderon if (!QED_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) {
732fe56b9e6SYuval Mintz if (src_type == QED_DMAE_ADDRESS_GRC)
733fe56b9e6SYuval Mintz src_addr_split = src_addr + offset;
734fe56b9e6SYuval Mintz else
735fe56b9e6SYuval Mintz src_addr_split = src_addr + (offset * 4);
736fe56b9e6SYuval Mintz }
737fe56b9e6SYuval Mintz
738fe56b9e6SYuval Mintz if (dst_type == QED_DMAE_ADDRESS_GRC)
739fe56b9e6SYuval Mintz dst_addr_split = dst_addr + offset;
740fe56b9e6SYuval Mintz else
741fe56b9e6SYuval Mintz dst_addr_split = dst_addr + (offset * 4);
742fe56b9e6SYuval Mintz
743fe56b9e6SYuval Mintz length_cur = (cnt_split == i) ? length_mod : length_limit;
744fe56b9e6SYuval Mintz
745fe56b9e6SYuval Mintz /* might be zero on last iteration */
746fe56b9e6SYuval Mintz if (!length_cur)
747fe56b9e6SYuval Mintz continue;
748fe56b9e6SYuval Mintz
749fe56b9e6SYuval Mintz qed_status = qed_dmae_execute_sub_operation(p_hwfn,
750fe56b9e6SYuval Mintz p_ptt,
751fe56b9e6SYuval Mintz src_addr_split,
752fe56b9e6SYuval Mintz dst_addr_split,
753fe56b9e6SYuval Mintz src_type,
754fe56b9e6SYuval Mintz dst_type,
755fe56b9e6SYuval Mintz length_cur);
756fe56b9e6SYuval Mintz if (qed_status) {
7572ec276d5SIgor Russkikh qed_hw_err_notify(p_hwfn, p_ptt, QED_HW_ERR_DMAE_FAIL,
758fe56b9e6SYuval Mintz "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n",
7592ec276d5SIgor Russkikh qed_status, src_addr,
7602ec276d5SIgor Russkikh dst_addr, length_cur);
761fe56b9e6SYuval Mintz break;
762fe56b9e6SYuval Mintz }
763fe56b9e6SYuval Mintz }
764fe56b9e6SYuval Mintz
765fe56b9e6SYuval Mintz return qed_status;
766fe56b9e6SYuval Mintz }
767fe56b9e6SYuval Mintz
qed_dmae_host2grc(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u64 source_addr,u32 grc_addr,u32 size_in_dwords,struct qed_dmae_params * p_params)768fe56b9e6SYuval Mintz int qed_dmae_host2grc(struct qed_hwfn *p_hwfn,
769fe56b9e6SYuval Mintz struct qed_ptt *p_ptt,
77083bf76e3SMichal Kalderon u64 source_addr, u32 grc_addr, u32 size_in_dwords,
77183bf76e3SMichal Kalderon struct qed_dmae_params *p_params)
772fe56b9e6SYuval Mintz {
773fe56b9e6SYuval Mintz u32 grc_addr_in_dw = grc_addr / sizeof(u32);
774fe56b9e6SYuval Mintz int rc;
775fe56b9e6SYuval Mintz
776fe56b9e6SYuval Mintz
777fe56b9e6SYuval Mintz mutex_lock(&p_hwfn->dmae_info.mutex);
778fe56b9e6SYuval Mintz
779fe56b9e6SYuval Mintz rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr,
780fe56b9e6SYuval Mintz grc_addr_in_dw,
781fe56b9e6SYuval Mintz QED_DMAE_ADDRESS_HOST_VIRT,
782fe56b9e6SYuval Mintz QED_DMAE_ADDRESS_GRC,
78383bf76e3SMichal Kalderon size_in_dwords, p_params);
784fe56b9e6SYuval Mintz
785fe56b9e6SYuval Mintz mutex_unlock(&p_hwfn->dmae_info.mutex);
786fe56b9e6SYuval Mintz
787fe56b9e6SYuval Mintz return rc;
788fe56b9e6SYuval Mintz }
789fe56b9e6SYuval Mintz
qed_dmae_grc2host(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 grc_addr,dma_addr_t dest_addr,u32 size_in_dwords,struct qed_dmae_params * p_params)7901a635e48SYuval Mintz int qed_dmae_grc2host(struct qed_hwfn *p_hwfn,
7911a635e48SYuval Mintz struct qed_ptt *p_ptt,
7921a635e48SYuval Mintz u32 grc_addr,
79383bf76e3SMichal Kalderon dma_addr_t dest_addr, u32 size_in_dwords,
79483bf76e3SMichal Kalderon struct qed_dmae_params *p_params)
795722003acSSudarsana Reddy Kalluru {
796722003acSSudarsana Reddy Kalluru u32 grc_addr_in_dw = grc_addr / sizeof(u32);
797722003acSSudarsana Reddy Kalluru int rc;
798722003acSSudarsana Reddy Kalluru
799722003acSSudarsana Reddy Kalluru
800722003acSSudarsana Reddy Kalluru mutex_lock(&p_hwfn->dmae_info.mutex);
801722003acSSudarsana Reddy Kalluru
802722003acSSudarsana Reddy Kalluru rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw,
803722003acSSudarsana Reddy Kalluru dest_addr, QED_DMAE_ADDRESS_GRC,
804722003acSSudarsana Reddy Kalluru QED_DMAE_ADDRESS_HOST_VIRT,
80583bf76e3SMichal Kalderon size_in_dwords, p_params);
806722003acSSudarsana Reddy Kalluru
807722003acSSudarsana Reddy Kalluru mutex_unlock(&p_hwfn->dmae_info.mutex);
808722003acSSudarsana Reddy Kalluru
809722003acSSudarsana Reddy Kalluru return rc;
810722003acSSudarsana Reddy Kalluru }
811722003acSSudarsana Reddy Kalluru
qed_dmae_host2host(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,dma_addr_t source_addr,dma_addr_t dest_addr,u32 size_in_dwords,struct qed_dmae_params * p_params)8121a635e48SYuval Mintz int qed_dmae_host2host(struct qed_hwfn *p_hwfn,
81337bff2b9SYuval Mintz struct qed_ptt *p_ptt,
81437bff2b9SYuval Mintz dma_addr_t source_addr,
81537bff2b9SYuval Mintz dma_addr_t dest_addr,
81637bff2b9SYuval Mintz u32 size_in_dwords, struct qed_dmae_params *p_params)
81737bff2b9SYuval Mintz {
81837bff2b9SYuval Mintz int rc;
81937bff2b9SYuval Mintz
82037bff2b9SYuval Mintz mutex_lock(&(p_hwfn->dmae_info.mutex));
82137bff2b9SYuval Mintz
82237bff2b9SYuval Mintz rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr,
82337bff2b9SYuval Mintz dest_addr,
82437bff2b9SYuval Mintz QED_DMAE_ADDRESS_HOST_PHYS,
82537bff2b9SYuval Mintz QED_DMAE_ADDRESS_HOST_PHYS,
82637bff2b9SYuval Mintz size_in_dwords, p_params);
82737bff2b9SYuval Mintz
82837bff2b9SYuval Mintz mutex_unlock(&(p_hwfn->dmae_info.mutex));
82937bff2b9SYuval Mintz
83037bff2b9SYuval Mintz return rc;
83137bff2b9SYuval Mintz }
83237bff2b9SYuval Mintz
qed_hw_err_notify(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,enum qed_hw_err_type err_type,const char * fmt,...)833365cd2ceSAlexander Lobakin void qed_hw_err_notify(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
834365cd2ceSAlexander Lobakin enum qed_hw_err_type err_type, const char *fmt, ...)
835d639836aSIgor Russkikh {
836d639836aSIgor Russkikh char buf[QED_HW_ERR_MAX_STR_SIZE];
837d639836aSIgor Russkikh va_list vl;
838d639836aSIgor Russkikh int len;
839d639836aSIgor Russkikh
840d639836aSIgor Russkikh if (fmt) {
841d639836aSIgor Russkikh va_start(vl, fmt);
842d639836aSIgor Russkikh len = vsnprintf(buf, QED_HW_ERR_MAX_STR_SIZE, fmt, vl);
843d639836aSIgor Russkikh va_end(vl);
844d639836aSIgor Russkikh
845d639836aSIgor Russkikh if (len > QED_HW_ERR_MAX_STR_SIZE - 1)
846d639836aSIgor Russkikh len = QED_HW_ERR_MAX_STR_SIZE - 1;
847d639836aSIgor Russkikh
848d639836aSIgor Russkikh DP_NOTICE(p_hwfn, "%s", buf);
849d639836aSIgor Russkikh }
850d639836aSIgor Russkikh
851d639836aSIgor Russkikh /* Fan failure cannot be masked by handling of another HW error */
852d639836aSIgor Russkikh if (p_hwfn->cdev->recov_in_prog &&
853d639836aSIgor Russkikh err_type != QED_HW_ERR_FAN_FAIL) {
854d639836aSIgor Russkikh DP_VERBOSE(p_hwfn,
855d639836aSIgor Russkikh NETIF_MSG_DRV,
856d639836aSIgor Russkikh "Recovery is in progress. Avoid notifying about HW error %d.\n",
857d639836aSIgor Russkikh err_type);
858d639836aSIgor Russkikh return;
859d639836aSIgor Russkikh }
860d639836aSIgor Russkikh
861d639836aSIgor Russkikh qed_hw_error_occurred(p_hwfn, err_type);
862d8d6c5a7SIgor Russkikh
863d8d6c5a7SIgor Russkikh if (fmt)
864d8d6c5a7SIgor Russkikh qed_mcp_send_raw_debug_data(p_hwfn, p_ptt, buf, len);
865d639836aSIgor Russkikh }
866d639836aSIgor Russkikh
qed_dmae_sanity(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,const char * phase)867da090917STomer Tayar int qed_dmae_sanity(struct qed_hwfn *p_hwfn,
868da090917STomer Tayar struct qed_ptt *p_ptt, const char *phase)
869da090917STomer Tayar {
870da090917STomer Tayar u32 size = PAGE_SIZE / 2, val;
871da090917STomer Tayar int rc = 0;
872da090917STomer Tayar dma_addr_t p_phys;
873da090917STomer Tayar void *p_virt;
874da090917STomer Tayar u32 *p_tmp;
875da090917STomer Tayar
876da090917STomer Tayar p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
877da090917STomer Tayar 2 * size, &p_phys, GFP_KERNEL);
878da090917STomer Tayar if (!p_virt) {
879da090917STomer Tayar DP_NOTICE(p_hwfn,
880da090917STomer Tayar "DMAE sanity [%s]: failed to allocate memory\n",
881da090917STomer Tayar phase);
882da090917STomer Tayar return -ENOMEM;
883da090917STomer Tayar }
884da090917STomer Tayar
885da090917STomer Tayar /* Fill the bottom half of the allocated memory with a known pattern */
886da090917STomer Tayar for (p_tmp = (u32 *)p_virt;
887da090917STomer Tayar p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) {
888da090917STomer Tayar /* Save the address itself as the value */
889da090917STomer Tayar val = (u32)(uintptr_t)p_tmp;
890da090917STomer Tayar *p_tmp = val;
891da090917STomer Tayar }
892da090917STomer Tayar
893da090917STomer Tayar /* Zero the top half of the allocated memory */
894da090917STomer Tayar memset((u8 *)p_virt + size, 0, size);
895da090917STomer Tayar
896da090917STomer Tayar DP_VERBOSE(p_hwfn,
897da090917STomer Tayar QED_MSG_SP,
898da090917STomer Tayar "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n",
899da090917STomer Tayar phase,
900da090917STomer Tayar (u64)p_phys,
901da090917STomer Tayar p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size);
902da090917STomer Tayar
903da090917STomer Tayar rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size,
90483bf76e3SMichal Kalderon size / 4, NULL);
905da090917STomer Tayar if (rc) {
906da090917STomer Tayar DP_NOTICE(p_hwfn,
907da090917STomer Tayar "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n",
908da090917STomer Tayar phase, rc);
909da090917STomer Tayar goto out;
910da090917STomer Tayar }
911da090917STomer Tayar
912da090917STomer Tayar /* Verify that the top half of the allocated memory has the pattern */
913da090917STomer Tayar for (p_tmp = (u32 *)((u8 *)p_virt + size);
914da090917STomer Tayar p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) {
915da090917STomer Tayar /* The corresponding address in the bottom half */
916da090917STomer Tayar val = (u32)(uintptr_t)p_tmp - size;
917da090917STomer Tayar
918da090917STomer Tayar if (*p_tmp != val) {
919da090917STomer Tayar DP_NOTICE(p_hwfn,
920da090917STomer Tayar "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n",
921da090917STomer Tayar phase,
922da090917STomer Tayar (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt),
923da090917STomer Tayar p_tmp, *p_tmp, val);
924da090917STomer Tayar rc = -EINVAL;
925da090917STomer Tayar goto out;
926da090917STomer Tayar }
927da090917STomer Tayar }
928da090917STomer Tayar
929da090917STomer Tayar out:
930da090917STomer Tayar dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys);
931da090917STomer Tayar return rc;
932da090917STomer Tayar }
933