1d3e4dfe0SJakub Kicinski // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2d3e4dfe0SJakub Kicinski /* Copyright (C) 2019 Netronome Systems, Inc. */
3d3e4dfe0SJakub Kicinski
4d3e4dfe0SJakub Kicinski #include <linux/bitfield.h>
5d3e4dfe0SJakub Kicinski #include <linux/io.h>
6d3e4dfe0SJakub Kicinski #include <linux/skbuff.h>
7d3e4dfe0SJakub Kicinski
8d3e4dfe0SJakub Kicinski #include "ccm.h"
9d3e4dfe0SJakub Kicinski #include "nfp_net.h"
10d3e4dfe0SJakub Kicinski
11d3e4dfe0SJakub Kicinski /* CCM messages via the mailbox. CMSGs get wrapped into simple TLVs
12d3e4dfe0SJakub Kicinski * and copied into the mailbox. Multiple messages can be copied to
13d3e4dfe0SJakub Kicinski * form a batch. Threads come in with CMSG formed in an skb, then
14d3e4dfe0SJakub Kicinski * enqueue that skb onto the request queue. If threads skb is first
15d3e4dfe0SJakub Kicinski * in queue this thread will handle the mailbox operation. It copies
16f6dfa315SDirk van der Merwe * up to 64 messages into the mailbox (making sure that both requests
17d3e4dfe0SJakub Kicinski * and replies will fit. After FW is done processing the batch it
18d3e4dfe0SJakub Kicinski * copies the data out and wakes waiting threads.
19d3e4dfe0SJakub Kicinski * If a thread is waiting it either gets its the message completed
20d3e4dfe0SJakub Kicinski * (response is copied into the same skb as the request, overwriting
21d3e4dfe0SJakub Kicinski * it), or becomes the first in queue.
22d3e4dfe0SJakub Kicinski * Completions and next-to-run are signaled via the control buffer
23d3e4dfe0SJakub Kicinski * to limit potential cache line bounces.
24d3e4dfe0SJakub Kicinski */
25d3e4dfe0SJakub Kicinski
26f6dfa315SDirk van der Merwe #define NFP_CCM_MBOX_BATCH_LIMIT 64
27d3e4dfe0SJakub Kicinski #define NFP_CCM_TIMEOUT (NFP_NET_POLL_TIMEOUT * 1000)
28f6dfa315SDirk van der Merwe #define NFP_CCM_MAX_QLEN 1024
29d3e4dfe0SJakub Kicinski
30d3e4dfe0SJakub Kicinski enum nfp_net_mbox_cmsg_state {
31d3e4dfe0SJakub Kicinski NFP_NET_MBOX_CMSG_STATE_QUEUED,
32d3e4dfe0SJakub Kicinski NFP_NET_MBOX_CMSG_STATE_NEXT,
33d3e4dfe0SJakub Kicinski NFP_NET_MBOX_CMSG_STATE_BUSY,
34d3e4dfe0SJakub Kicinski NFP_NET_MBOX_CMSG_STATE_REPLY_FOUND,
35d3e4dfe0SJakub Kicinski NFP_NET_MBOX_CMSG_STATE_DONE,
36d3e4dfe0SJakub Kicinski };
37d3e4dfe0SJakub Kicinski
38d3e4dfe0SJakub Kicinski /**
39a507b164SYang Shen * struct nfp_ccm_mbox_cmsg_cb - CCM mailbox specific info
40d3e4dfe0SJakub Kicinski * @state: processing state (/stage) of the message
41d3e4dfe0SJakub Kicinski * @err: error encountered during processing if any
42d3e4dfe0SJakub Kicinski * @max_len: max(request_len, reply_len)
43d3e4dfe0SJakub Kicinski * @exp_reply: expected reply length (0 means don't validate)
44e2c7114aSJakub Kicinski * @posted: the message was posted and nobody waits for the reply
45d3e4dfe0SJakub Kicinski */
46d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb {
47d3e4dfe0SJakub Kicinski enum nfp_net_mbox_cmsg_state state;
48d3e4dfe0SJakub Kicinski int err;
49d3e4dfe0SJakub Kicinski unsigned int max_len;
50d3e4dfe0SJakub Kicinski unsigned int exp_reply;
51e2c7114aSJakub Kicinski bool posted;
52d3e4dfe0SJakub Kicinski };
53d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_max_msg(struct nfp_net * nn)54d3e4dfe0SJakub Kicinski static u32 nfp_ccm_mbox_max_msg(struct nfp_net *nn)
55d3e4dfe0SJakub Kicinski {
56d3e4dfe0SJakub Kicinski return round_down(nn->tlv_caps.mbox_len, 4) -
57d3e4dfe0SJakub Kicinski NFP_NET_CFG_MBOX_SIMPLE_VAL - /* common mbox command header */
58d3e4dfe0SJakub Kicinski 4 * 2; /* Msg TLV plus End TLV headers */
59d3e4dfe0SJakub Kicinski }
60d3e4dfe0SJakub Kicinski
61d3e4dfe0SJakub Kicinski static void
nfp_ccm_mbox_msg_init(struct sk_buff * skb,unsigned int exp_reply,int max_len)62d3e4dfe0SJakub Kicinski nfp_ccm_mbox_msg_init(struct sk_buff *skb, unsigned int exp_reply, int max_len)
63d3e4dfe0SJakub Kicinski {
64d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb;
65d3e4dfe0SJakub Kicinski
66d3e4dfe0SJakub Kicinski cb->state = NFP_NET_MBOX_CMSG_STATE_QUEUED;
67d3e4dfe0SJakub Kicinski cb->err = 0;
68d3e4dfe0SJakub Kicinski cb->max_len = max_len;
69d3e4dfe0SJakub Kicinski cb->exp_reply = exp_reply;
70e2c7114aSJakub Kicinski cb->posted = false;
71d3e4dfe0SJakub Kicinski }
72d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_maxlen(const struct sk_buff * skb)73d3e4dfe0SJakub Kicinski static int nfp_ccm_mbox_maxlen(const struct sk_buff *skb)
74d3e4dfe0SJakub Kicinski {
75d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb;
76d3e4dfe0SJakub Kicinski
77d3e4dfe0SJakub Kicinski return cb->max_len;
78d3e4dfe0SJakub Kicinski }
79d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_done(struct sk_buff * skb)80d3e4dfe0SJakub Kicinski static bool nfp_ccm_mbox_done(struct sk_buff *skb)
81d3e4dfe0SJakub Kicinski {
82d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb;
83d3e4dfe0SJakub Kicinski
84d3e4dfe0SJakub Kicinski return cb->state == NFP_NET_MBOX_CMSG_STATE_DONE;
85d3e4dfe0SJakub Kicinski }
86d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_in_progress(struct sk_buff * skb)87d3e4dfe0SJakub Kicinski static bool nfp_ccm_mbox_in_progress(struct sk_buff *skb)
88d3e4dfe0SJakub Kicinski {
89d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb;
90d3e4dfe0SJakub Kicinski
91d3e4dfe0SJakub Kicinski return cb->state != NFP_NET_MBOX_CMSG_STATE_QUEUED &&
92d3e4dfe0SJakub Kicinski cb->state != NFP_NET_MBOX_CMSG_STATE_NEXT;
93d3e4dfe0SJakub Kicinski }
94d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_set_busy(struct sk_buff * skb)95d3e4dfe0SJakub Kicinski static void nfp_ccm_mbox_set_busy(struct sk_buff *skb)
96d3e4dfe0SJakub Kicinski {
97d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb;
98d3e4dfe0SJakub Kicinski
99d3e4dfe0SJakub Kicinski cb->state = NFP_NET_MBOX_CMSG_STATE_BUSY;
100d3e4dfe0SJakub Kicinski }
101d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_is_posted(struct sk_buff * skb)102e2c7114aSJakub Kicinski static bool nfp_ccm_mbox_is_posted(struct sk_buff *skb)
103e2c7114aSJakub Kicinski {
104e2c7114aSJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb;
105e2c7114aSJakub Kicinski
106e2c7114aSJakub Kicinski return cb->posted;
107e2c7114aSJakub Kicinski }
108e2c7114aSJakub Kicinski
nfp_ccm_mbox_mark_posted(struct sk_buff * skb)109e2c7114aSJakub Kicinski static void nfp_ccm_mbox_mark_posted(struct sk_buff *skb)
110e2c7114aSJakub Kicinski {
111e2c7114aSJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb;
112e2c7114aSJakub Kicinski
113e2c7114aSJakub Kicinski cb->posted = true;
114e2c7114aSJakub Kicinski }
115e2c7114aSJakub Kicinski
nfp_ccm_mbox_is_first(struct nfp_net * nn,struct sk_buff * skb)116d3e4dfe0SJakub Kicinski static bool nfp_ccm_mbox_is_first(struct nfp_net *nn, struct sk_buff *skb)
117d3e4dfe0SJakub Kicinski {
118d3e4dfe0SJakub Kicinski return skb_queue_is_first(&nn->mbox_cmsg.queue, skb);
119d3e4dfe0SJakub Kicinski }
120d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_should_run(struct nfp_net * nn,struct sk_buff * skb)121d3e4dfe0SJakub Kicinski static bool nfp_ccm_mbox_should_run(struct nfp_net *nn, struct sk_buff *skb)
122d3e4dfe0SJakub Kicinski {
123d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb;
124d3e4dfe0SJakub Kicinski
125d3e4dfe0SJakub Kicinski return cb->state == NFP_NET_MBOX_CMSG_STATE_NEXT;
126d3e4dfe0SJakub Kicinski }
127d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_mark_next_runner(struct nfp_net * nn)128d3e4dfe0SJakub Kicinski static void nfp_ccm_mbox_mark_next_runner(struct nfp_net *nn)
129d3e4dfe0SJakub Kicinski {
130d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb;
131d3e4dfe0SJakub Kicinski struct sk_buff *skb;
132d3e4dfe0SJakub Kicinski
133d3e4dfe0SJakub Kicinski skb = skb_peek(&nn->mbox_cmsg.queue);
134d3e4dfe0SJakub Kicinski if (!skb)
135d3e4dfe0SJakub Kicinski return;
136d3e4dfe0SJakub Kicinski
137d3e4dfe0SJakub Kicinski cb = (void *)skb->cb;
138d3e4dfe0SJakub Kicinski cb->state = NFP_NET_MBOX_CMSG_STATE_NEXT;
139e2c7114aSJakub Kicinski if (cb->posted)
140e2c7114aSJakub Kicinski queue_work(nn->mbox_cmsg.workq, &nn->mbox_cmsg.runq_work);
141d3e4dfe0SJakub Kicinski }
142d3e4dfe0SJakub Kicinski
143d3e4dfe0SJakub Kicinski static void
nfp_ccm_mbox_write_tlv(struct nfp_net * nn,u32 off,u32 type,u32 len)144d3e4dfe0SJakub Kicinski nfp_ccm_mbox_write_tlv(struct nfp_net *nn, u32 off, u32 type, u32 len)
145d3e4dfe0SJakub Kicinski {
146d3e4dfe0SJakub Kicinski nn_writel(nn, off,
147d3e4dfe0SJakub Kicinski FIELD_PREP(NFP_NET_MBOX_TLV_TYPE, type) |
148d3e4dfe0SJakub Kicinski FIELD_PREP(NFP_NET_MBOX_TLV_LEN, len));
149d3e4dfe0SJakub Kicinski }
150d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_copy_in(struct nfp_net * nn,struct sk_buff * last)151d3e4dfe0SJakub Kicinski static void nfp_ccm_mbox_copy_in(struct nfp_net *nn, struct sk_buff *last)
152d3e4dfe0SJakub Kicinski {
153d3e4dfe0SJakub Kicinski struct sk_buff *skb;
154d3e4dfe0SJakub Kicinski int reserve, i, cnt;
155d3e4dfe0SJakub Kicinski __be32 *data;
156d3e4dfe0SJakub Kicinski u32 off, len;
157d3e4dfe0SJakub Kicinski
158d3e4dfe0SJakub Kicinski off = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL;
159d3e4dfe0SJakub Kicinski skb = __skb_peek(&nn->mbox_cmsg.queue);
160d3e4dfe0SJakub Kicinski while (true) {
161d3e4dfe0SJakub Kicinski nfp_ccm_mbox_write_tlv(nn, off, NFP_NET_MBOX_TLV_TYPE_MSG,
162d3e4dfe0SJakub Kicinski skb->len);
163d3e4dfe0SJakub Kicinski off += 4;
164d3e4dfe0SJakub Kicinski
165d3e4dfe0SJakub Kicinski /* Write data word by word, skb->data should be aligned */
166d3e4dfe0SJakub Kicinski data = (__be32 *)skb->data;
167d3e4dfe0SJakub Kicinski cnt = skb->len / 4;
168d3e4dfe0SJakub Kicinski for (i = 0 ; i < cnt; i++) {
169d3e4dfe0SJakub Kicinski nn_writel(nn, off, be32_to_cpu(data[i]));
170d3e4dfe0SJakub Kicinski off += 4;
171d3e4dfe0SJakub Kicinski }
172d3e4dfe0SJakub Kicinski if (skb->len & 3) {
173d3e4dfe0SJakub Kicinski __be32 tmp = 0;
174d3e4dfe0SJakub Kicinski
175d3e4dfe0SJakub Kicinski memcpy(&tmp, &data[i], skb->len & 3);
176d3e4dfe0SJakub Kicinski nn_writel(nn, off, be32_to_cpu(tmp));
177d3e4dfe0SJakub Kicinski off += 4;
178d3e4dfe0SJakub Kicinski }
179d3e4dfe0SJakub Kicinski
180d3e4dfe0SJakub Kicinski /* Reserve space if reply is bigger */
181d3e4dfe0SJakub Kicinski len = round_up(skb->len, 4);
182d3e4dfe0SJakub Kicinski reserve = nfp_ccm_mbox_maxlen(skb) - len;
183d3e4dfe0SJakub Kicinski if (reserve > 0) {
184d3e4dfe0SJakub Kicinski nfp_ccm_mbox_write_tlv(nn, off,
185d3e4dfe0SJakub Kicinski NFP_NET_MBOX_TLV_TYPE_RESV,
186d3e4dfe0SJakub Kicinski reserve);
187d3e4dfe0SJakub Kicinski off += 4 + reserve;
188d3e4dfe0SJakub Kicinski }
189d3e4dfe0SJakub Kicinski
190d3e4dfe0SJakub Kicinski if (skb == last)
191d3e4dfe0SJakub Kicinski break;
192d3e4dfe0SJakub Kicinski skb = skb_queue_next(&nn->mbox_cmsg.queue, skb);
193d3e4dfe0SJakub Kicinski }
194d3e4dfe0SJakub Kicinski
195d3e4dfe0SJakub Kicinski nfp_ccm_mbox_write_tlv(nn, off, NFP_NET_MBOX_TLV_TYPE_END, 0);
196d3e4dfe0SJakub Kicinski }
197d3e4dfe0SJakub Kicinski
198d3e4dfe0SJakub Kicinski static struct sk_buff *
nfp_ccm_mbox_find_req(struct nfp_net * nn,__be16 tag,struct sk_buff * last)199d3e4dfe0SJakub Kicinski nfp_ccm_mbox_find_req(struct nfp_net *nn, __be16 tag, struct sk_buff *last)
200d3e4dfe0SJakub Kicinski {
201d3e4dfe0SJakub Kicinski struct sk_buff *skb;
202d3e4dfe0SJakub Kicinski
203d3e4dfe0SJakub Kicinski skb = __skb_peek(&nn->mbox_cmsg.queue);
204d3e4dfe0SJakub Kicinski while (true) {
205d3e4dfe0SJakub Kicinski if (__nfp_ccm_get_tag(skb) == tag)
206d3e4dfe0SJakub Kicinski return skb;
207d3e4dfe0SJakub Kicinski
208d3e4dfe0SJakub Kicinski if (skb == last)
209d3e4dfe0SJakub Kicinski return NULL;
210d3e4dfe0SJakub Kicinski skb = skb_queue_next(&nn->mbox_cmsg.queue, skb);
211d3e4dfe0SJakub Kicinski }
212d3e4dfe0SJakub Kicinski }
213d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_copy_out(struct nfp_net * nn,struct sk_buff * last)214d3e4dfe0SJakub Kicinski static void nfp_ccm_mbox_copy_out(struct nfp_net *nn, struct sk_buff *last)
215d3e4dfe0SJakub Kicinski {
216d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb;
217d3e4dfe0SJakub Kicinski u8 __iomem *data, *end;
218d3e4dfe0SJakub Kicinski struct sk_buff *skb;
219d3e4dfe0SJakub Kicinski
220d3e4dfe0SJakub Kicinski data = nn->dp.ctrl_bar + nn->tlv_caps.mbox_off +
221d3e4dfe0SJakub Kicinski NFP_NET_CFG_MBOX_SIMPLE_VAL;
222d3e4dfe0SJakub Kicinski end = data + nn->tlv_caps.mbox_len;
223d3e4dfe0SJakub Kicinski
224d3e4dfe0SJakub Kicinski while (true) {
225d3e4dfe0SJakub Kicinski unsigned int length, offset, type;
226d3e4dfe0SJakub Kicinski struct nfp_ccm_hdr hdr;
227d3e4dfe0SJakub Kicinski u32 tlv_hdr;
228d3e4dfe0SJakub Kicinski
229d3e4dfe0SJakub Kicinski tlv_hdr = readl(data);
230d3e4dfe0SJakub Kicinski type = FIELD_GET(NFP_NET_MBOX_TLV_TYPE, tlv_hdr);
231d3e4dfe0SJakub Kicinski length = FIELD_GET(NFP_NET_MBOX_TLV_LEN, tlv_hdr);
232d3e4dfe0SJakub Kicinski offset = data - nn->dp.ctrl_bar;
233d3e4dfe0SJakub Kicinski
234d3e4dfe0SJakub Kicinski /* Advance past the header */
235d3e4dfe0SJakub Kicinski data += 4;
236d3e4dfe0SJakub Kicinski
237d3e4dfe0SJakub Kicinski if (data + length > end) {
238d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp, "mailbox oversized TLV type:%d offset:%u len:%u\n",
239d3e4dfe0SJakub Kicinski type, offset, length);
240d3e4dfe0SJakub Kicinski break;
241d3e4dfe0SJakub Kicinski }
242d3e4dfe0SJakub Kicinski
243d3e4dfe0SJakub Kicinski if (type == NFP_NET_MBOX_TLV_TYPE_END)
244d3e4dfe0SJakub Kicinski break;
245d3e4dfe0SJakub Kicinski if (type == NFP_NET_MBOX_TLV_TYPE_RESV)
246d3e4dfe0SJakub Kicinski goto next_tlv;
247d3e4dfe0SJakub Kicinski if (type != NFP_NET_MBOX_TLV_TYPE_MSG &&
248d3e4dfe0SJakub Kicinski type != NFP_NET_MBOX_TLV_TYPE_MSG_NOSUP) {
249d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp, "mailbox unknown TLV type:%d offset:%u len:%u\n",
250d3e4dfe0SJakub Kicinski type, offset, length);
251d3e4dfe0SJakub Kicinski break;
252d3e4dfe0SJakub Kicinski }
253d3e4dfe0SJakub Kicinski
254d3e4dfe0SJakub Kicinski if (length < 4) {
255d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp, "mailbox msg too short to contain header TLV type:%d offset:%u len:%u\n",
256d3e4dfe0SJakub Kicinski type, offset, length);
257d3e4dfe0SJakub Kicinski break;
258d3e4dfe0SJakub Kicinski }
259d3e4dfe0SJakub Kicinski
260d3e4dfe0SJakub Kicinski hdr.raw = cpu_to_be32(readl(data));
261d3e4dfe0SJakub Kicinski
262d3e4dfe0SJakub Kicinski skb = nfp_ccm_mbox_find_req(nn, hdr.tag, last);
263d3e4dfe0SJakub Kicinski if (!skb) {
264d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp, "mailbox request not found:%u\n",
265d3e4dfe0SJakub Kicinski be16_to_cpu(hdr.tag));
266d3e4dfe0SJakub Kicinski break;
267d3e4dfe0SJakub Kicinski }
268d3e4dfe0SJakub Kicinski cb = (void *)skb->cb;
269d3e4dfe0SJakub Kicinski
270d3e4dfe0SJakub Kicinski if (type == NFP_NET_MBOX_TLV_TYPE_MSG_NOSUP) {
271d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp,
272d3e4dfe0SJakub Kicinski "mailbox msg not supported type:%d\n",
273d3e4dfe0SJakub Kicinski nfp_ccm_get_type(skb));
274d3e4dfe0SJakub Kicinski cb->err = -EIO;
275d3e4dfe0SJakub Kicinski goto next_tlv;
276d3e4dfe0SJakub Kicinski }
277d3e4dfe0SJakub Kicinski
278d3e4dfe0SJakub Kicinski if (hdr.type != __NFP_CCM_REPLY(nfp_ccm_get_type(skb))) {
279d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp, "mailbox msg reply wrong type:%u expected:%lu\n",
280d3e4dfe0SJakub Kicinski hdr.type,
281d3e4dfe0SJakub Kicinski __NFP_CCM_REPLY(nfp_ccm_get_type(skb)));
282d3e4dfe0SJakub Kicinski cb->err = -EIO;
283d3e4dfe0SJakub Kicinski goto next_tlv;
284d3e4dfe0SJakub Kicinski }
285d3e4dfe0SJakub Kicinski if (cb->exp_reply && length != cb->exp_reply) {
286d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp, "mailbox msg reply wrong size type:%u expected:%u have:%u\n",
287d3e4dfe0SJakub Kicinski hdr.type, length, cb->exp_reply);
288d3e4dfe0SJakub Kicinski cb->err = -EIO;
289d3e4dfe0SJakub Kicinski goto next_tlv;
290d3e4dfe0SJakub Kicinski }
291d3e4dfe0SJakub Kicinski if (length > cb->max_len) {
292d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp, "mailbox msg oversized reply type:%u max:%u have:%u\n",
293d3e4dfe0SJakub Kicinski hdr.type, cb->max_len, length);
294d3e4dfe0SJakub Kicinski cb->err = -EIO;
295d3e4dfe0SJakub Kicinski goto next_tlv;
296d3e4dfe0SJakub Kicinski }
297d3e4dfe0SJakub Kicinski
298e2c7114aSJakub Kicinski if (!cb->posted) {
299e2c7114aSJakub Kicinski __be32 *skb_data;
300e2c7114aSJakub Kicinski int i, cnt;
301e2c7114aSJakub Kicinski
302d3e4dfe0SJakub Kicinski if (length <= skb->len)
303d3e4dfe0SJakub Kicinski __skb_trim(skb, length);
304d3e4dfe0SJakub Kicinski else
305d3e4dfe0SJakub Kicinski skb_put(skb, length - skb->len);
306d3e4dfe0SJakub Kicinski
307e2c7114aSJakub Kicinski /* We overcopy here slightly, but that's okay,
308e2c7114aSJakub Kicinski * the skb is large enough, and the garbage will
309e2c7114aSJakub Kicinski * be ignored (beyond skb->len).
310d3e4dfe0SJakub Kicinski */
311d3e4dfe0SJakub Kicinski skb_data = (__be32 *)skb->data;
312d3e4dfe0SJakub Kicinski memcpy(skb_data, &hdr, 4);
313d3e4dfe0SJakub Kicinski
314d3e4dfe0SJakub Kicinski cnt = DIV_ROUND_UP(length, 4);
315d3e4dfe0SJakub Kicinski for (i = 1 ; i < cnt; i++)
316d3e4dfe0SJakub Kicinski skb_data[i] = cpu_to_be32(readl(data + i * 4));
317e2c7114aSJakub Kicinski }
318d3e4dfe0SJakub Kicinski
319d3e4dfe0SJakub Kicinski cb->state = NFP_NET_MBOX_CMSG_STATE_REPLY_FOUND;
320d3e4dfe0SJakub Kicinski next_tlv:
321d3e4dfe0SJakub Kicinski data += round_up(length, 4);
322d3e4dfe0SJakub Kicinski if (data + 4 > end) {
323d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp,
324d3e4dfe0SJakub Kicinski "reached end of MBOX without END TLV\n");
325d3e4dfe0SJakub Kicinski break;
326d3e4dfe0SJakub Kicinski }
327d3e4dfe0SJakub Kicinski }
328d3e4dfe0SJakub Kicinski
329d3e4dfe0SJakub Kicinski smp_wmb(); /* order the skb->data vs. cb->state */
330d3e4dfe0SJakub Kicinski spin_lock_bh(&nn->mbox_cmsg.queue.lock);
331d3e4dfe0SJakub Kicinski do {
332d3e4dfe0SJakub Kicinski skb = __skb_dequeue(&nn->mbox_cmsg.queue);
333d3e4dfe0SJakub Kicinski cb = (void *)skb->cb;
334d3e4dfe0SJakub Kicinski
335d3e4dfe0SJakub Kicinski if (cb->state != NFP_NET_MBOX_CMSG_STATE_REPLY_FOUND) {
336d3e4dfe0SJakub Kicinski cb->err = -ENOENT;
337d3e4dfe0SJakub Kicinski smp_wmb(); /* order the cb->err vs. cb->state */
338d3e4dfe0SJakub Kicinski }
339d3e4dfe0SJakub Kicinski cb->state = NFP_NET_MBOX_CMSG_STATE_DONE;
340e2c7114aSJakub Kicinski
341e2c7114aSJakub Kicinski if (cb->posted) {
342e2c7114aSJakub Kicinski if (cb->err)
343e2c7114aSJakub Kicinski nn_dp_warn(&nn->dp,
344e2c7114aSJakub Kicinski "mailbox posted msg failed type:%u err:%d\n",
345e2c7114aSJakub Kicinski nfp_ccm_get_type(skb), cb->err);
346e2c7114aSJakub Kicinski dev_consume_skb_any(skb);
347e2c7114aSJakub Kicinski }
348d3e4dfe0SJakub Kicinski } while (skb != last);
349d3e4dfe0SJakub Kicinski
350d3e4dfe0SJakub Kicinski nfp_ccm_mbox_mark_next_runner(nn);
351d3e4dfe0SJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
352d3e4dfe0SJakub Kicinski }
353d3e4dfe0SJakub Kicinski
354d3e4dfe0SJakub Kicinski static void
nfp_ccm_mbox_mark_all_err(struct nfp_net * nn,struct sk_buff * last,int err)355d3e4dfe0SJakub Kicinski nfp_ccm_mbox_mark_all_err(struct nfp_net *nn, struct sk_buff *last, int err)
356d3e4dfe0SJakub Kicinski {
357d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb;
358d3e4dfe0SJakub Kicinski struct sk_buff *skb;
359d3e4dfe0SJakub Kicinski
360d3e4dfe0SJakub Kicinski spin_lock_bh(&nn->mbox_cmsg.queue.lock);
361d3e4dfe0SJakub Kicinski do {
362d3e4dfe0SJakub Kicinski skb = __skb_dequeue(&nn->mbox_cmsg.queue);
363d3e4dfe0SJakub Kicinski cb = (void *)skb->cb;
364d3e4dfe0SJakub Kicinski
365d3e4dfe0SJakub Kicinski cb->err = err;
366d3e4dfe0SJakub Kicinski smp_wmb(); /* order the cb->err vs. cb->state */
367d3e4dfe0SJakub Kicinski cb->state = NFP_NET_MBOX_CMSG_STATE_DONE;
368d3e4dfe0SJakub Kicinski } while (skb != last);
369d3e4dfe0SJakub Kicinski
370d3e4dfe0SJakub Kicinski nfp_ccm_mbox_mark_next_runner(nn);
371d3e4dfe0SJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
372d3e4dfe0SJakub Kicinski }
373d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_run_queue_unlock(struct nfp_net * nn)374d3e4dfe0SJakub Kicinski static void nfp_ccm_mbox_run_queue_unlock(struct nfp_net *nn)
375d3e4dfe0SJakub Kicinski __releases(&nn->mbox_cmsg.queue.lock)
376d3e4dfe0SJakub Kicinski {
377d3e4dfe0SJakub Kicinski int space = nn->tlv_caps.mbox_len - NFP_NET_CFG_MBOX_SIMPLE_VAL;
378d3e4dfe0SJakub Kicinski struct sk_buff *skb, *last;
379d3e4dfe0SJakub Kicinski int cnt, err;
380d3e4dfe0SJakub Kicinski
381d3e4dfe0SJakub Kicinski space -= 4; /* for End TLV */
382d3e4dfe0SJakub Kicinski
383d3e4dfe0SJakub Kicinski /* First skb must fit, because it's ours and we checked it fits */
384d3e4dfe0SJakub Kicinski cnt = 1;
385d3e4dfe0SJakub Kicinski last = skb = __skb_peek(&nn->mbox_cmsg.queue);
386d3e4dfe0SJakub Kicinski space -= 4 + nfp_ccm_mbox_maxlen(skb);
387d3e4dfe0SJakub Kicinski
388d3e4dfe0SJakub Kicinski while (!skb_queue_is_last(&nn->mbox_cmsg.queue, last)) {
389d3e4dfe0SJakub Kicinski skb = skb_queue_next(&nn->mbox_cmsg.queue, last);
390d3e4dfe0SJakub Kicinski space -= 4 + nfp_ccm_mbox_maxlen(skb);
391d3e4dfe0SJakub Kicinski if (space < 0)
392d3e4dfe0SJakub Kicinski break;
393d3e4dfe0SJakub Kicinski last = skb;
394d3e4dfe0SJakub Kicinski nfp_ccm_mbox_set_busy(skb);
395d3e4dfe0SJakub Kicinski cnt++;
396d3e4dfe0SJakub Kicinski if (cnt == NFP_CCM_MBOX_BATCH_LIMIT)
397d3e4dfe0SJakub Kicinski break;
398d3e4dfe0SJakub Kicinski }
399d3e4dfe0SJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
400d3e4dfe0SJakub Kicinski
401d3e4dfe0SJakub Kicinski /* Now we own all skb's marked in progress, new requests may arrive
402d3e4dfe0SJakub Kicinski * at the end of the queue.
403d3e4dfe0SJakub Kicinski */
404d3e4dfe0SJakub Kicinski
405d3e4dfe0SJakub Kicinski nn_ctrl_bar_lock(nn);
406d3e4dfe0SJakub Kicinski
407d3e4dfe0SJakub Kicinski nfp_ccm_mbox_copy_in(nn, last);
408d3e4dfe0SJakub Kicinski
409d3e4dfe0SJakub Kicinski err = nfp_net_mbox_reconfig(nn, NFP_NET_CFG_MBOX_CMD_TLV_CMSG);
410d3e4dfe0SJakub Kicinski if (!err)
411d3e4dfe0SJakub Kicinski nfp_ccm_mbox_copy_out(nn, last);
412d3e4dfe0SJakub Kicinski else
413d3e4dfe0SJakub Kicinski nfp_ccm_mbox_mark_all_err(nn, last, -EIO);
414d3e4dfe0SJakub Kicinski
415d3e4dfe0SJakub Kicinski nn_ctrl_bar_unlock(nn);
416d3e4dfe0SJakub Kicinski
417d3e4dfe0SJakub Kicinski wake_up_all(&nn->mbox_cmsg.wq);
418d3e4dfe0SJakub Kicinski }
419d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_skb_return(struct sk_buff * skb)420d3e4dfe0SJakub Kicinski static int nfp_ccm_mbox_skb_return(struct sk_buff *skb)
421d3e4dfe0SJakub Kicinski {
422d3e4dfe0SJakub Kicinski struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb;
423d3e4dfe0SJakub Kicinski
424d3e4dfe0SJakub Kicinski if (cb->err)
425d3e4dfe0SJakub Kicinski dev_kfree_skb_any(skb);
426d3e4dfe0SJakub Kicinski return cb->err;
427d3e4dfe0SJakub Kicinski }
428d3e4dfe0SJakub Kicinski
429d3e4dfe0SJakub Kicinski /* If wait timed out but the command is already in progress we have
430d3e4dfe0SJakub Kicinski * to wait until it finishes. Runners has ownership of the skbs marked
431d3e4dfe0SJakub Kicinski * as busy.
432d3e4dfe0SJakub Kicinski */
433d3e4dfe0SJakub Kicinski static int
nfp_ccm_mbox_unlink_unlock(struct nfp_net * nn,struct sk_buff * skb,enum nfp_ccm_type type)434d3e4dfe0SJakub Kicinski nfp_ccm_mbox_unlink_unlock(struct nfp_net *nn, struct sk_buff *skb,
435d3e4dfe0SJakub Kicinski enum nfp_ccm_type type)
436d3e4dfe0SJakub Kicinski __releases(&nn->mbox_cmsg.queue.lock)
437d3e4dfe0SJakub Kicinski {
438d3e4dfe0SJakub Kicinski bool was_first;
439d3e4dfe0SJakub Kicinski
440d3e4dfe0SJakub Kicinski if (nfp_ccm_mbox_in_progress(skb)) {
441d3e4dfe0SJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
442d3e4dfe0SJakub Kicinski
443d3e4dfe0SJakub Kicinski wait_event(nn->mbox_cmsg.wq, nfp_ccm_mbox_done(skb));
444d3e4dfe0SJakub Kicinski smp_rmb(); /* pairs with smp_wmb() after data is written */
445d3e4dfe0SJakub Kicinski return nfp_ccm_mbox_skb_return(skb);
446d3e4dfe0SJakub Kicinski }
447d3e4dfe0SJakub Kicinski
448d3e4dfe0SJakub Kicinski was_first = nfp_ccm_mbox_should_run(nn, skb);
449d3e4dfe0SJakub Kicinski __skb_unlink(skb, &nn->mbox_cmsg.queue);
450d3e4dfe0SJakub Kicinski if (was_first)
451d3e4dfe0SJakub Kicinski nfp_ccm_mbox_mark_next_runner(nn);
452d3e4dfe0SJakub Kicinski
453d3e4dfe0SJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
454d3e4dfe0SJakub Kicinski
455d3e4dfe0SJakub Kicinski if (was_first)
456d3e4dfe0SJakub Kicinski wake_up_all(&nn->mbox_cmsg.wq);
457d3e4dfe0SJakub Kicinski
458d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp, "time out waiting for mbox response to 0x%02x\n",
459d3e4dfe0SJakub Kicinski type);
460d3e4dfe0SJakub Kicinski return -ETIMEDOUT;
461d3e4dfe0SJakub Kicinski }
462d3e4dfe0SJakub Kicinski
463d3e4dfe0SJakub Kicinski static int
nfp_ccm_mbox_msg_prepare(struct nfp_net * nn,struct sk_buff * skb,enum nfp_ccm_type type,unsigned int reply_size,unsigned int max_reply_size,gfp_t flags)464d3e4dfe0SJakub Kicinski nfp_ccm_mbox_msg_prepare(struct nfp_net *nn, struct sk_buff *skb,
465d3e4dfe0SJakub Kicinski enum nfp_ccm_type type,
466d3e4dfe0SJakub Kicinski unsigned int reply_size, unsigned int max_reply_size,
467d3e4dfe0SJakub Kicinski gfp_t flags)
468d3e4dfe0SJakub Kicinski {
469d3e4dfe0SJakub Kicinski const unsigned int mbox_max = nfp_ccm_mbox_max_msg(nn);
470d3e4dfe0SJakub Kicinski unsigned int max_len;
471d3e4dfe0SJakub Kicinski ssize_t undersize;
472d3e4dfe0SJakub Kicinski int err;
473d3e4dfe0SJakub Kicinski
474d3e4dfe0SJakub Kicinski if (unlikely(!(nn->tlv_caps.mbox_cmsg_types & BIT(type)))) {
475d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp,
476d3e4dfe0SJakub Kicinski "message type %d not supported by mailbox\n", type);
477d3e4dfe0SJakub Kicinski return -EINVAL;
478d3e4dfe0SJakub Kicinski }
479d3e4dfe0SJakub Kicinski
480d3e4dfe0SJakub Kicinski /* If the reply size is unknown assume it will take the entire
481d3e4dfe0SJakub Kicinski * mailbox, the callers should do their best for this to never
482d3e4dfe0SJakub Kicinski * happen.
483d3e4dfe0SJakub Kicinski */
484d3e4dfe0SJakub Kicinski if (!max_reply_size)
485d3e4dfe0SJakub Kicinski max_reply_size = mbox_max;
486d3e4dfe0SJakub Kicinski max_reply_size = round_up(max_reply_size, 4);
487d3e4dfe0SJakub Kicinski
488d3e4dfe0SJakub Kicinski /* Make sure we can fit the entire reply into the skb,
489d3e4dfe0SJakub Kicinski * and that we don't have to slow down the mbox handler
490d3e4dfe0SJakub Kicinski * with allocations.
491d3e4dfe0SJakub Kicinski */
492d3e4dfe0SJakub Kicinski undersize = max_reply_size - (skb_end_pointer(skb) - skb->data);
493d3e4dfe0SJakub Kicinski if (undersize > 0) {
494d3e4dfe0SJakub Kicinski err = pskb_expand_head(skb, 0, undersize, flags);
495d3e4dfe0SJakub Kicinski if (err) {
496d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp,
497d3e4dfe0SJakub Kicinski "can't allocate reply buffer for mailbox\n");
498d3e4dfe0SJakub Kicinski return err;
499d3e4dfe0SJakub Kicinski }
500d3e4dfe0SJakub Kicinski }
501d3e4dfe0SJakub Kicinski
502d3e4dfe0SJakub Kicinski /* Make sure that request and response both fit into the mailbox */
503d3e4dfe0SJakub Kicinski max_len = max(max_reply_size, round_up(skb->len, 4));
504d3e4dfe0SJakub Kicinski if (max_len > mbox_max) {
505d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp,
506*3df96774SColin Ian King "message too big for the mailbox: %u/%u vs %u\n",
507d3e4dfe0SJakub Kicinski skb->len, max_reply_size, mbox_max);
508d3e4dfe0SJakub Kicinski return -EMSGSIZE;
509d3e4dfe0SJakub Kicinski }
510d3e4dfe0SJakub Kicinski
511d3e4dfe0SJakub Kicinski nfp_ccm_mbox_msg_init(skb, reply_size, max_len);
512d3e4dfe0SJakub Kicinski
513d3e4dfe0SJakub Kicinski return 0;
514d3e4dfe0SJakub Kicinski }
515d3e4dfe0SJakub Kicinski
516d3e4dfe0SJakub Kicinski static int
nfp_ccm_mbox_msg_enqueue(struct nfp_net * nn,struct sk_buff * skb,enum nfp_ccm_type type,bool critical)517d3e4dfe0SJakub Kicinski nfp_ccm_mbox_msg_enqueue(struct nfp_net *nn, struct sk_buff *skb,
5180f93242dSJakub Kicinski enum nfp_ccm_type type, bool critical)
519d3e4dfe0SJakub Kicinski {
520d3e4dfe0SJakub Kicinski struct nfp_ccm_hdr *hdr;
521d3e4dfe0SJakub Kicinski
522d3e4dfe0SJakub Kicinski assert_spin_locked(&nn->mbox_cmsg.queue.lock);
523d3e4dfe0SJakub Kicinski
5240f93242dSJakub Kicinski if (!critical && nn->mbox_cmsg.queue.qlen >= NFP_CCM_MAX_QLEN) {
525d3e4dfe0SJakub Kicinski nn_dp_warn(&nn->dp, "mailbox request queue too long\n");
526d3e4dfe0SJakub Kicinski return -EBUSY;
527d3e4dfe0SJakub Kicinski }
528d3e4dfe0SJakub Kicinski
529d3e4dfe0SJakub Kicinski hdr = (void *)skb->data;
530d3e4dfe0SJakub Kicinski hdr->ver = NFP_CCM_ABI_VERSION;
531d3e4dfe0SJakub Kicinski hdr->type = type;
532d3e4dfe0SJakub Kicinski hdr->tag = cpu_to_be16(nn->mbox_cmsg.tag++);
533d3e4dfe0SJakub Kicinski
534d3e4dfe0SJakub Kicinski __skb_queue_tail(&nn->mbox_cmsg.queue, skb);
535d3e4dfe0SJakub Kicinski
536d3e4dfe0SJakub Kicinski return 0;
537d3e4dfe0SJakub Kicinski }
538d3e4dfe0SJakub Kicinski
__nfp_ccm_mbox_communicate(struct nfp_net * nn,struct sk_buff * skb,enum nfp_ccm_type type,unsigned int reply_size,unsigned int max_reply_size,bool critical)5390f93242dSJakub Kicinski int __nfp_ccm_mbox_communicate(struct nfp_net *nn, struct sk_buff *skb,
540d3e4dfe0SJakub Kicinski enum nfp_ccm_type type,
541d3e4dfe0SJakub Kicinski unsigned int reply_size,
5420f93242dSJakub Kicinski unsigned int max_reply_size, bool critical)
543d3e4dfe0SJakub Kicinski {
544d3e4dfe0SJakub Kicinski int err;
545d3e4dfe0SJakub Kicinski
546d3e4dfe0SJakub Kicinski err = nfp_ccm_mbox_msg_prepare(nn, skb, type, reply_size,
547d3e4dfe0SJakub Kicinski max_reply_size, GFP_KERNEL);
548d3e4dfe0SJakub Kicinski if (err)
549d3e4dfe0SJakub Kicinski goto err_free_skb;
550d3e4dfe0SJakub Kicinski
551d3e4dfe0SJakub Kicinski spin_lock_bh(&nn->mbox_cmsg.queue.lock);
552d3e4dfe0SJakub Kicinski
5530f93242dSJakub Kicinski err = nfp_ccm_mbox_msg_enqueue(nn, skb, type, critical);
554d3e4dfe0SJakub Kicinski if (err)
555d3e4dfe0SJakub Kicinski goto err_unlock;
556d3e4dfe0SJakub Kicinski
557d3e4dfe0SJakub Kicinski /* First in queue takes the mailbox lock and processes the batch */
558d3e4dfe0SJakub Kicinski if (!nfp_ccm_mbox_is_first(nn, skb)) {
559d3e4dfe0SJakub Kicinski bool to;
560d3e4dfe0SJakub Kicinski
561d3e4dfe0SJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
562d3e4dfe0SJakub Kicinski
563d3e4dfe0SJakub Kicinski to = !wait_event_timeout(nn->mbox_cmsg.wq,
564d3e4dfe0SJakub Kicinski nfp_ccm_mbox_done(skb) ||
565d3e4dfe0SJakub Kicinski nfp_ccm_mbox_should_run(nn, skb),
566d3e4dfe0SJakub Kicinski msecs_to_jiffies(NFP_CCM_TIMEOUT));
567d3e4dfe0SJakub Kicinski
568d3e4dfe0SJakub Kicinski /* fast path for those completed by another thread */
569d3e4dfe0SJakub Kicinski if (nfp_ccm_mbox_done(skb)) {
570d3e4dfe0SJakub Kicinski smp_rmb(); /* pairs with wmb after data is written */
571d3e4dfe0SJakub Kicinski return nfp_ccm_mbox_skb_return(skb);
572d3e4dfe0SJakub Kicinski }
573d3e4dfe0SJakub Kicinski
574d3e4dfe0SJakub Kicinski spin_lock_bh(&nn->mbox_cmsg.queue.lock);
575d3e4dfe0SJakub Kicinski
576d3e4dfe0SJakub Kicinski if (!nfp_ccm_mbox_is_first(nn, skb)) {
577d3e4dfe0SJakub Kicinski WARN_ON(!to);
578d3e4dfe0SJakub Kicinski
579d3e4dfe0SJakub Kicinski err = nfp_ccm_mbox_unlink_unlock(nn, skb, type);
580d3e4dfe0SJakub Kicinski if (err)
581d3e4dfe0SJakub Kicinski goto err_free_skb;
582d3e4dfe0SJakub Kicinski return 0;
583d3e4dfe0SJakub Kicinski }
584d3e4dfe0SJakub Kicinski }
585d3e4dfe0SJakub Kicinski
586d3e4dfe0SJakub Kicinski /* run queue expects the lock held */
587d3e4dfe0SJakub Kicinski nfp_ccm_mbox_run_queue_unlock(nn);
588d3e4dfe0SJakub Kicinski return nfp_ccm_mbox_skb_return(skb);
589d3e4dfe0SJakub Kicinski
590d3e4dfe0SJakub Kicinski err_unlock:
591d3e4dfe0SJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
592d3e4dfe0SJakub Kicinski err_free_skb:
593d3e4dfe0SJakub Kicinski dev_kfree_skb_any(skb);
594d3e4dfe0SJakub Kicinski return err;
595d3e4dfe0SJakub Kicinski }
596d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_communicate(struct nfp_net * nn,struct sk_buff * skb,enum nfp_ccm_type type,unsigned int reply_size,unsigned int max_reply_size)5970f93242dSJakub Kicinski int nfp_ccm_mbox_communicate(struct nfp_net *nn, struct sk_buff *skb,
5980f93242dSJakub Kicinski enum nfp_ccm_type type,
5990f93242dSJakub Kicinski unsigned int reply_size,
6000f93242dSJakub Kicinski unsigned int max_reply_size)
6010f93242dSJakub Kicinski {
6020f93242dSJakub Kicinski return __nfp_ccm_mbox_communicate(nn, skb, type, reply_size,
6030f93242dSJakub Kicinski max_reply_size, false);
6040f93242dSJakub Kicinski }
6050f93242dSJakub Kicinski
nfp_ccm_mbox_post_runq_work(struct work_struct * work)606e2c7114aSJakub Kicinski static void nfp_ccm_mbox_post_runq_work(struct work_struct *work)
607e2c7114aSJakub Kicinski {
608e2c7114aSJakub Kicinski struct sk_buff *skb;
609e2c7114aSJakub Kicinski struct nfp_net *nn;
610e2c7114aSJakub Kicinski
611e2c7114aSJakub Kicinski nn = container_of(work, struct nfp_net, mbox_cmsg.runq_work);
612e2c7114aSJakub Kicinski
613e2c7114aSJakub Kicinski spin_lock_bh(&nn->mbox_cmsg.queue.lock);
614e2c7114aSJakub Kicinski
615e2c7114aSJakub Kicinski skb = __skb_peek(&nn->mbox_cmsg.queue);
616e2c7114aSJakub Kicinski if (WARN_ON(!skb || !nfp_ccm_mbox_is_posted(skb) ||
617e2c7114aSJakub Kicinski !nfp_ccm_mbox_should_run(nn, skb))) {
618e2c7114aSJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
619e2c7114aSJakub Kicinski return;
620e2c7114aSJakub Kicinski }
621e2c7114aSJakub Kicinski
622e2c7114aSJakub Kicinski nfp_ccm_mbox_run_queue_unlock(nn);
623e2c7114aSJakub Kicinski }
624e2c7114aSJakub Kicinski
nfp_ccm_mbox_post_wait_work(struct work_struct * work)625e2c7114aSJakub Kicinski static void nfp_ccm_mbox_post_wait_work(struct work_struct *work)
626e2c7114aSJakub Kicinski {
627e2c7114aSJakub Kicinski struct sk_buff *skb;
628e2c7114aSJakub Kicinski struct nfp_net *nn;
629e2c7114aSJakub Kicinski int err;
630e2c7114aSJakub Kicinski
631e2c7114aSJakub Kicinski nn = container_of(work, struct nfp_net, mbox_cmsg.wait_work);
632e2c7114aSJakub Kicinski
633e2c7114aSJakub Kicinski skb = skb_peek(&nn->mbox_cmsg.queue);
634e2c7114aSJakub Kicinski if (WARN_ON(!skb || !nfp_ccm_mbox_is_posted(skb)))
635e2c7114aSJakub Kicinski /* Should never happen so it's unclear what to do here.. */
636e2c7114aSJakub Kicinski goto exit_unlock_wake;
637e2c7114aSJakub Kicinski
638e2c7114aSJakub Kicinski err = nfp_net_mbox_reconfig_wait_posted(nn);
639e2c7114aSJakub Kicinski if (!err)
640e2c7114aSJakub Kicinski nfp_ccm_mbox_copy_out(nn, skb);
641e2c7114aSJakub Kicinski else
642e2c7114aSJakub Kicinski nfp_ccm_mbox_mark_all_err(nn, skb, -EIO);
643e2c7114aSJakub Kicinski exit_unlock_wake:
644e2c7114aSJakub Kicinski nn_ctrl_bar_unlock(nn);
645e2c7114aSJakub Kicinski wake_up_all(&nn->mbox_cmsg.wq);
646e2c7114aSJakub Kicinski }
647e2c7114aSJakub Kicinski
nfp_ccm_mbox_post(struct nfp_net * nn,struct sk_buff * skb,enum nfp_ccm_type type,unsigned int max_reply_size)648e2c7114aSJakub Kicinski int nfp_ccm_mbox_post(struct nfp_net *nn, struct sk_buff *skb,
649e2c7114aSJakub Kicinski enum nfp_ccm_type type, unsigned int max_reply_size)
650e2c7114aSJakub Kicinski {
651e2c7114aSJakub Kicinski int err;
652e2c7114aSJakub Kicinski
653e2c7114aSJakub Kicinski err = nfp_ccm_mbox_msg_prepare(nn, skb, type, 0, max_reply_size,
654e2c7114aSJakub Kicinski GFP_ATOMIC);
655e2c7114aSJakub Kicinski if (err)
656e2c7114aSJakub Kicinski goto err_free_skb;
657e2c7114aSJakub Kicinski
658e2c7114aSJakub Kicinski nfp_ccm_mbox_mark_posted(skb);
659e2c7114aSJakub Kicinski
660e2c7114aSJakub Kicinski spin_lock_bh(&nn->mbox_cmsg.queue.lock);
661e2c7114aSJakub Kicinski
6620f93242dSJakub Kicinski err = nfp_ccm_mbox_msg_enqueue(nn, skb, type, false);
663e2c7114aSJakub Kicinski if (err)
664e2c7114aSJakub Kicinski goto err_unlock;
665e2c7114aSJakub Kicinski
666e2c7114aSJakub Kicinski if (nfp_ccm_mbox_is_first(nn, skb)) {
667e2c7114aSJakub Kicinski if (nn_ctrl_bar_trylock(nn)) {
668e2c7114aSJakub Kicinski nfp_ccm_mbox_copy_in(nn, skb);
669e2c7114aSJakub Kicinski nfp_net_mbox_reconfig_post(nn,
670e2c7114aSJakub Kicinski NFP_NET_CFG_MBOX_CMD_TLV_CMSG);
671e2c7114aSJakub Kicinski queue_work(nn->mbox_cmsg.workq,
672e2c7114aSJakub Kicinski &nn->mbox_cmsg.wait_work);
673e2c7114aSJakub Kicinski } else {
674e2c7114aSJakub Kicinski nfp_ccm_mbox_mark_next_runner(nn);
675e2c7114aSJakub Kicinski }
676e2c7114aSJakub Kicinski }
677e2c7114aSJakub Kicinski
678e2c7114aSJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
679e2c7114aSJakub Kicinski
680e2c7114aSJakub Kicinski return 0;
681e2c7114aSJakub Kicinski
682e2c7114aSJakub Kicinski err_unlock:
683e2c7114aSJakub Kicinski spin_unlock_bh(&nn->mbox_cmsg.queue.lock);
684e2c7114aSJakub Kicinski err_free_skb:
685e2c7114aSJakub Kicinski dev_kfree_skb_any(skb);
686e2c7114aSJakub Kicinski return err;
687e2c7114aSJakub Kicinski }
688e2c7114aSJakub Kicinski
689d3e4dfe0SJakub Kicinski struct sk_buff *
nfp_ccm_mbox_msg_alloc(struct nfp_net * nn,unsigned int req_size,unsigned int reply_size,gfp_t flags)690d7053e04SJakub Kicinski nfp_ccm_mbox_msg_alloc(struct nfp_net *nn, unsigned int req_size,
691d3e4dfe0SJakub Kicinski unsigned int reply_size, gfp_t flags)
692d3e4dfe0SJakub Kicinski {
693d3e4dfe0SJakub Kicinski unsigned int max_size;
694d3e4dfe0SJakub Kicinski struct sk_buff *skb;
695d3e4dfe0SJakub Kicinski
696d3e4dfe0SJakub Kicinski if (!reply_size)
697d3e4dfe0SJakub Kicinski max_size = nfp_ccm_mbox_max_msg(nn);
698d3e4dfe0SJakub Kicinski else
699d3e4dfe0SJakub Kicinski max_size = max(req_size, reply_size);
700d3e4dfe0SJakub Kicinski max_size = round_up(max_size, 4);
701d3e4dfe0SJakub Kicinski
702d3e4dfe0SJakub Kicinski skb = alloc_skb(max_size, flags);
703d3e4dfe0SJakub Kicinski if (!skb)
704d3e4dfe0SJakub Kicinski return NULL;
705d3e4dfe0SJakub Kicinski
706d3e4dfe0SJakub Kicinski skb_put(skb, req_size);
707d3e4dfe0SJakub Kicinski
708d3e4dfe0SJakub Kicinski return skb;
709d3e4dfe0SJakub Kicinski }
710d3e4dfe0SJakub Kicinski
nfp_ccm_mbox_fits(struct nfp_net * nn,unsigned int size)711d3e4dfe0SJakub Kicinski bool nfp_ccm_mbox_fits(struct nfp_net *nn, unsigned int size)
712d3e4dfe0SJakub Kicinski {
713d3e4dfe0SJakub Kicinski return nfp_ccm_mbox_max_msg(nn) >= size;
714d3e4dfe0SJakub Kicinski }
715e2c7114aSJakub Kicinski
nfp_ccm_mbox_init(struct nfp_net * nn)716e2c7114aSJakub Kicinski int nfp_ccm_mbox_init(struct nfp_net *nn)
717e2c7114aSJakub Kicinski {
718e2c7114aSJakub Kicinski return 0;
719e2c7114aSJakub Kicinski }
720e2c7114aSJakub Kicinski
nfp_ccm_mbox_clean(struct nfp_net * nn)721e2c7114aSJakub Kicinski void nfp_ccm_mbox_clean(struct nfp_net *nn)
722e2c7114aSJakub Kicinski {
723e2c7114aSJakub Kicinski drain_workqueue(nn->mbox_cmsg.workq);
724e2c7114aSJakub Kicinski }
725e2c7114aSJakub Kicinski
nfp_ccm_mbox_alloc(struct nfp_net * nn)726e2c7114aSJakub Kicinski int nfp_ccm_mbox_alloc(struct nfp_net *nn)
727e2c7114aSJakub Kicinski {
728e2c7114aSJakub Kicinski skb_queue_head_init(&nn->mbox_cmsg.queue);
729e2c7114aSJakub Kicinski init_waitqueue_head(&nn->mbox_cmsg.wq);
730e2c7114aSJakub Kicinski INIT_WORK(&nn->mbox_cmsg.wait_work, nfp_ccm_mbox_post_wait_work);
731e2c7114aSJakub Kicinski INIT_WORK(&nn->mbox_cmsg.runq_work, nfp_ccm_mbox_post_runq_work);
732e2c7114aSJakub Kicinski
733e2c7114aSJakub Kicinski nn->mbox_cmsg.workq = alloc_workqueue("nfp-ccm-mbox", WQ_UNBOUND, 0);
734e2c7114aSJakub Kicinski if (!nn->mbox_cmsg.workq)
735e2c7114aSJakub Kicinski return -ENOMEM;
736e2c7114aSJakub Kicinski return 0;
737e2c7114aSJakub Kicinski }
738e2c7114aSJakub Kicinski
nfp_ccm_mbox_free(struct nfp_net * nn)739e2c7114aSJakub Kicinski void nfp_ccm_mbox_free(struct nfp_net *nn)
740e2c7114aSJakub Kicinski {
741e2c7114aSJakub Kicinski destroy_workqueue(nn->mbox_cmsg.workq);
742e2c7114aSJakub Kicinski WARN_ON(!skb_queue_empty(&nn->mbox_cmsg.queue));
743e2c7114aSJakub Kicinski }
744