116 static int pp_find_next_peer(struct pp_ctx *pp)  in pp_find_next_peer()  argument
121 	link = ntb_link_is_up(pp->ntb, NULL, NULL);  in pp_find_next_peer()
124 	if (link & pp->nmask)  in pp_find_next_peer()
125 		pidx = __ffs64(link & pp->nmask);  in pp_find_next_peer()
126 	else if (link & pp->pmask)  in pp_find_next_peer()
127 		pidx = __ffs64(link & pp->pmask);  in pp_find_next_peer()
131 	out_db = BIT_ULL(ntb_peer_port_number(pp->ntb, pidx));  in pp_find_next_peer()
133 	spin_lock(&pp->lock);  in pp_find_next_peer()
134 	pp->out_pidx = pidx;  in pp_find_next_peer()
135 	pp->out_db = out_db;  in pp_find_next_peer()
136 	spin_unlock(&pp->lock);  in pp_find_next_peer()
141 static void pp_setup(struct pp_ctx *pp)  in pp_setup()  argument
145 	ntb_db_set_mask(pp->ntb, pp->in_db);  in pp_setup()
147 	hrtimer_cancel(&pp->timer);  in pp_setup()
149 	ret = pp_find_next_peer(pp);  in pp_setup()
151 		dev_dbg(&pp->ntb->dev, "Got no peers, so cancel\n");  in pp_setup()
155 	dev_dbg(&pp->ntb->dev, "Ping-pong started with port %d, db %#llx\n",  in pp_setup()
156 		ntb_peer_port_number(pp->ntb, pp->out_pidx), pp->out_db);  in pp_setup()
158 	hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);  in pp_setup()
161 static void pp_clear(struct pp_ctx *pp)  in pp_clear()  argument
163 	hrtimer_cancel(&pp->timer);  in pp_clear()
165 	ntb_db_set_mask(pp->ntb, pp->in_db);  in pp_clear()
167 	dev_dbg(&pp->ntb->dev, "Ping-pong cancelled\n");  in pp_clear()
170 static void pp_ping(struct pp_ctx *pp)  in pp_ping()  argument
174 	count = atomic_read(&pp->count);  in pp_ping()
176 	spin_lock(&pp->lock);  in pp_ping()
177 	ntb_peer_spad_write(pp->ntb, pp->out_pidx, 0, count);  in pp_ping()
178 	ntb_peer_msg_write(pp->ntb, pp->out_pidx, 0, count);  in pp_ping()
180 	dev_dbg(&pp->ntb->dev, "Ping port %d spad %#x, msg %#x\n",  in pp_ping()
181 		ntb_peer_port_number(pp->ntb, pp->out_pidx), count, count);  in pp_ping()
183 	ntb_peer_db_set(pp->ntb, pp->out_db);  in pp_ping()
184 	ntb_db_clear_mask(pp->ntb, pp->in_db);  in pp_ping()
185 	spin_unlock(&pp->lock);  in pp_ping()
188 static void pp_pong(struct pp_ctx *pp)  in pp_pong()  argument
194 	spad_data = ntb_spad_read(pp->ntb, 0);  in pp_pong()
195 	msg_data = ntb_msg_read(pp->ntb, &pidx, 0);  in pp_pong()
196 	ntb_msg_clear_sts(pp->ntb, -1);  in pp_pong()
203 	dev_dbg(&pp->ntb->dev, "Pong spad %#x, msg %#x (port %d)\n",  in pp_pong()
204 		spad_data, msg_data, ntb_peer_port_number(pp->ntb, pidx));  in pp_pong()
206 	atomic_inc(&pp->count);  in pp_pong()
208 	ntb_db_set_mask(pp->ntb, pp->in_db);  in pp_pong()
209 	ntb_db_clear(pp->ntb, pp->in_db);  in pp_pong()
211 	hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);  in pp_pong()
216 	struct pp_ctx *pp = to_pp_timer(t);  in pp_timer_func()  local
218 	pp_ping(pp);  in pp_timer_func()
225 	struct pp_ctx *pp = ctx;  in pp_link_event()  local
227 	pp_setup(pp);  in pp_link_event()
232 	struct pp_ctx *pp = ctx;  in pp_db_event()  local
234 	pp_pong(pp);  in pp_db_event()
278 	struct pp_ctx *pp;  in pp_create_data()  local
280 	pp = devm_kzalloc(&ntb->dev, sizeof(*pp), GFP_KERNEL);  in pp_create_data()
281 	if (!pp)  in pp_create_data()
284 	pp->ntb = ntb;  in pp_create_data()
285 	atomic_set(&pp->count, 0);  in pp_create_data()
286 	spin_lock_init(&pp->lock);  in pp_create_data()
287 	hrtimer_init(&pp->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in pp_create_data()
288 	pp->timer.function = pp_timer_func;  in pp_create_data()
290 	return pp;  in pp_create_data()
293 static void pp_init_flds(struct pp_ctx *pp)  in pp_init_flds()  argument
298 	lport = ntb_port_number(pp->ntb);  in pp_init_flds()
299 	pcnt = ntb_peer_port_count(pp->ntb);  in pp_init_flds()
301 		if (lport < ntb_peer_port_number(pp->ntb, pidx))  in pp_init_flds()
305 	pp->in_db = BIT_ULL(lport);  in pp_init_flds()
306 	pp->pmask = GENMASK_ULL(pidx, 0) >> 1;  in pp_init_flds()
307 	pp->nmask = GENMASK_ULL(pcnt - 1, pidx);  in pp_init_flds()
309 	dev_dbg(&pp->ntb->dev, "Inbound db %#llx, prev %#llx, next %#llx\n",  in pp_init_flds()
310 		pp->in_db, pp->pmask, pp->nmask);  in pp_init_flds()
313 static int pp_mask_events(struct pp_ctx *pp)  in pp_mask_events()  argument
318 	db_mask = ntb_db_valid_mask(pp->ntb);  in pp_mask_events()
319 	ret = ntb_db_set_mask(pp->ntb, db_mask);  in pp_mask_events()
324 	if (ntb_msg_count(pp->ntb) < 1)  in pp_mask_events()
327 	msg_mask = ntb_msg_outbits(pp->ntb) | ntb_msg_inbits(pp->ntb);  in pp_mask_events()
328 	return ntb_msg_set_mask(pp->ntb, msg_mask);  in pp_mask_events()
331 static int pp_setup_ctx(struct pp_ctx *pp)  in pp_setup_ctx()  argument
335 	ret = ntb_set_ctx(pp->ntb, pp, &pp_ops);  in pp_setup_ctx()
339 	ntb_link_enable(pp->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);  in pp_setup_ctx()
341 	ntb_link_event(pp->ntb);  in pp_setup_ctx()
346 static void pp_clear_ctx(struct pp_ctx *pp)  in pp_clear_ctx()  argument
348 	ntb_link_disable(pp->ntb);  in pp_clear_ctx()
350 	ntb_clear_ctx(pp->ntb);  in pp_clear_ctx()
353 static void pp_setup_dbgfs(struct pp_ctx *pp)  in pp_setup_dbgfs()  argument
355 	struct pci_dev *pdev = pp->ntb->pdev;  in pp_setup_dbgfs()
357 	pp->dbgfs_dir = debugfs_create_dir(pci_name(pdev), pp_dbgfs_topdir);  in pp_setup_dbgfs()
359 	debugfs_create_atomic_t("count", 0600, pp->dbgfs_dir, &pp->count);  in pp_setup_dbgfs()
362 static void pp_clear_dbgfs(struct pp_ctx *pp)  in pp_clear_dbgfs()  argument
364 	debugfs_remove_recursive(pp->dbgfs_dir);  in pp_clear_dbgfs()
369 	struct pp_ctx *pp;  in pp_probe()  local
376 	pp = pp_create_data(ntb);  in pp_probe()
377 	if (IS_ERR(pp))  in pp_probe()
378 		return PTR_ERR(pp);  in pp_probe()
380 	pp_init_flds(pp);  in pp_probe()
382 	ret = pp_mask_events(pp);  in pp_probe()
386 	ret = pp_setup_ctx(pp);  in pp_probe()
390 	pp_setup_dbgfs(pp);  in pp_probe()
397 	struct pp_ctx *pp = ntb->ctx;  in pp_remove()  local
399 	pp_clear_dbgfs(pp);  in pp_remove()
401 	pp_clear_ctx(pp);  in pp_remove()
403 	pp_clear(pp);  in pp_remove()