1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 3 #include <linux/module.h> 4 #include <linux/debugfs.h> 5 #include <linux/ntb.h> 6 #include <linux/pci.h> 7 #include <linux/radix-tree.h> 8 #include <linux/workqueue.h> 9 10 MODULE_LICENSE("Dual BSD/GPL"); 11 MODULE_VERSION("0.1"); 12 MODULE_AUTHOR("Logan Gunthorpe <logang@deltatee.com>"); 13 MODULE_DESCRIPTION("Test for sending MSI interrupts over an NTB memory window"); 14 15 static int num_irqs = 4; 16 module_param(num_irqs, int, 0644); 17 MODULE_PARM_DESC(num_irqs, "number of irqs to use"); 18 19 struct ntb_msit_ctx { 20 struct ntb_dev *ntb; 21 struct dentry *dbgfs_dir; 22 struct work_struct setup_work; 23 24 struct ntb_msit_isr_ctx { 25 int irq_idx; 26 int irq_num; 27 int occurrences; 28 struct ntb_msit_ctx *nm; 29 struct ntb_msi_desc desc; 30 } *isr_ctx; 31 32 struct ntb_msit_peer { 33 struct ntb_msit_ctx *nm; 34 int pidx; 35 int num_irqs; 36 struct completion init_comp; 37 struct ntb_msi_desc *msi_desc; 38 } peers[]; 39 }; 40 41 static struct dentry *ntb_msit_dbgfs_topdir; 42 43 static irqreturn_t ntb_msit_isr(int irq, void *dev) 44 { 45 struct ntb_msit_isr_ctx *isr_ctx = dev; 46 struct ntb_msit_ctx *nm = isr_ctx->nm; 47 48 dev_dbg(&nm->ntb->dev, "Interrupt Occurred: %d", 49 isr_ctx->irq_idx); 50 51 isr_ctx->occurrences++; 52 53 return IRQ_HANDLED; 54 } 55 56 static void ntb_msit_setup_work(struct work_struct *work) 57 { 58 struct ntb_msit_ctx *nm = container_of(work, struct ntb_msit_ctx, 59 setup_work); 60 int irq_count = 0; 61 int irq; 62 int ret; 63 uintptr_t i; 64 65 ret = ntb_msi_setup_mws(nm->ntb); 66 if (ret) { 67 dev_err(&nm->ntb->dev, "Unable to setup MSI windows: %d\n", 68 ret); 69 return; 70 } 71 72 for (i = 0; i < num_irqs; i++) { 73 nm->isr_ctx[i].irq_idx = i; 74 nm->isr_ctx[i].nm = nm; 75 76 if (!nm->isr_ctx[i].irq_num) { 77 irq = ntbm_msi_request_irq(nm->ntb, ntb_msit_isr, 78 KBUILD_MODNAME, 79 &nm->isr_ctx[i], 80 &nm->isr_ctx[i].desc); 81 if (irq < 0) 82 break; 83 84 nm->isr_ctx[i].irq_num = irq; 85 } 86 87 ret = ntb_spad_write(nm->ntb, 2 * i + 1, 88 nm->isr_ctx[i].desc.addr_offset); 89 if (ret) 90 break; 91 92 ret = ntb_spad_write(nm->ntb, 2 * i + 2, 93 nm->isr_ctx[i].desc.data); 94 if (ret) 95 break; 96 97 irq_count++; 98 } 99 100 ntb_spad_write(nm->ntb, 0, irq_count); 101 ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb))); 102 } 103 104 static void ntb_msit_desc_changed(void *ctx) 105 { 106 struct ntb_msit_ctx *nm = ctx; 107 int i; 108 109 dev_dbg(&nm->ntb->dev, "MSI Descriptors Changed\n"); 110 111 for (i = 0; i < num_irqs; i++) { 112 ntb_spad_write(nm->ntb, 2 * i + 1, 113 nm->isr_ctx[i].desc.addr_offset); 114 ntb_spad_write(nm->ntb, 2 * i + 2, 115 nm->isr_ctx[i].desc.data); 116 } 117 118 ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb))); 119 } 120 121 static void ntb_msit_link_event(void *ctx) 122 { 123 struct ntb_msit_ctx *nm = ctx; 124 125 if (!ntb_link_is_up(nm->ntb, NULL, NULL)) 126 return; 127 128 schedule_work(&nm->setup_work); 129 } 130 131 static void ntb_msit_copy_peer_desc(struct ntb_msit_ctx *nm, int peer) 132 { 133 int i; 134 struct ntb_msi_desc *desc = nm->peers[peer].msi_desc; 135 int irq_count = nm->peers[peer].num_irqs; 136 137 for (i = 0; i < irq_count; i++) { 138 desc[i].addr_offset = ntb_peer_spad_read(nm->ntb, peer, 139 2 * i + 1); 140 desc[i].data = ntb_peer_spad_read(nm->ntb, peer, 2 * i + 2); 141 } 142 143 dev_info(&nm->ntb->dev, "Found %d interrupts on peer %d\n", 144 irq_count, peer); 145 146 complete_all(&nm->peers[peer].init_comp); 147 } 148 149 static void ntb_msit_db_event(void *ctx, int vec) 150 { 151 struct ntb_msit_ctx *nm = ctx; 152 struct ntb_msi_desc *desc; 153 u64 peer_mask = ntb_db_read(nm->ntb); 154 u32 irq_count; 155 int peer; 156 157 ntb_db_clear(nm->ntb, peer_mask); 158 159 for (peer = 0; peer < sizeof(peer_mask) * 8; peer++) { 160 if (!(peer_mask & BIT(peer))) 161 continue; 162 163 irq_count = ntb_peer_spad_read(nm->ntb, peer, 0); 164 if (irq_count == -1) 165 continue; 166 167 desc = kcalloc(irq_count, sizeof(*desc), GFP_ATOMIC); 168 if (!desc) 169 continue; 170 171 kfree(nm->peers[peer].msi_desc); 172 nm->peers[peer].msi_desc = desc; 173 nm->peers[peer].num_irqs = irq_count; 174 175 ntb_msit_copy_peer_desc(nm, peer); 176 } 177 } 178 179 static const struct ntb_ctx_ops ntb_msit_ops = { 180 .link_event = ntb_msit_link_event, 181 .db_event = ntb_msit_db_event, 182 }; 183 184 static int ntb_msit_dbgfs_trigger(void *data, u64 idx) 185 { 186 struct ntb_msit_peer *peer = data; 187 188 if (idx >= peer->num_irqs) 189 return -EINVAL; 190 191 dev_dbg(&peer->nm->ntb->dev, "trigger irq %llu on peer %u\n", 192 idx, peer->pidx); 193 194 return ntb_msi_peer_trigger(peer->nm->ntb, peer->pidx, 195 &peer->msi_desc[idx]); 196 } 197 198 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_trigger_fops, NULL, 199 ntb_msit_dbgfs_trigger, "%llu\n"); 200 201 static int ntb_msit_dbgfs_port_get(void *data, u64 *port) 202 { 203 struct ntb_msit_peer *peer = data; 204 205 *port = ntb_peer_port_number(peer->nm->ntb, peer->pidx); 206 207 return 0; 208 } 209 210 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_port_fops, ntb_msit_dbgfs_port_get, 211 NULL, "%llu\n"); 212 213 static int ntb_msit_dbgfs_count_get(void *data, u64 *count) 214 { 215 struct ntb_msit_peer *peer = data; 216 217 *count = peer->num_irqs; 218 219 return 0; 220 } 221 222 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_count_fops, ntb_msit_dbgfs_count_get, 223 NULL, "%llu\n"); 224 225 static int ntb_msit_dbgfs_ready_get(void *data, u64 *ready) 226 { 227 struct ntb_msit_peer *peer = data; 228 229 *ready = try_wait_for_completion(&peer->init_comp); 230 231 return 0; 232 } 233 234 static int ntb_msit_dbgfs_ready_set(void *data, u64 ready) 235 { 236 struct ntb_msit_peer *peer = data; 237 238 return wait_for_completion_interruptible(&peer->init_comp); 239 } 240 241 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_ready_fops, ntb_msit_dbgfs_ready_get, 242 ntb_msit_dbgfs_ready_set, "%llu\n"); 243 244 static int ntb_msit_dbgfs_occurrences_get(void *data, u64 *occurrences) 245 { 246 struct ntb_msit_isr_ctx *isr_ctx = data; 247 248 *occurrences = isr_ctx->occurrences; 249 250 return 0; 251 } 252 253 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_occurrences_fops, 254 ntb_msit_dbgfs_occurrences_get, 255 NULL, "%llu\n"); 256 257 static int ntb_msit_dbgfs_local_port_get(void *data, u64 *port) 258 { 259 struct ntb_msit_ctx *nm = data; 260 261 *port = ntb_port_number(nm->ntb); 262 263 return 0; 264 } 265 266 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_local_port_fops, 267 ntb_msit_dbgfs_local_port_get, 268 NULL, "%llu\n"); 269 270 static void ntb_msit_create_dbgfs(struct ntb_msit_ctx *nm) 271 { 272 struct pci_dev *pdev = nm->ntb->pdev; 273 char buf[32]; 274 int i; 275 struct dentry *peer_dir; 276 277 nm->dbgfs_dir = debugfs_create_dir(pci_name(pdev), 278 ntb_msit_dbgfs_topdir); 279 debugfs_create_file("port", 0400, nm->dbgfs_dir, nm, 280 &ntb_msit_local_port_fops); 281 282 for (i = 0; i < ntb_peer_port_count(nm->ntb); i++) { 283 nm->peers[i].pidx = i; 284 nm->peers[i].nm = nm; 285 init_completion(&nm->peers[i].init_comp); 286 287 snprintf(buf, sizeof(buf), "peer%d", i); 288 peer_dir = debugfs_create_dir(buf, nm->dbgfs_dir); 289 290 debugfs_create_file_unsafe("trigger", 0200, peer_dir, 291 &nm->peers[i], 292 &ntb_msit_trigger_fops); 293 294 debugfs_create_file_unsafe("port", 0400, peer_dir, 295 &nm->peers[i], &ntb_msit_port_fops); 296 297 debugfs_create_file_unsafe("count", 0400, peer_dir, 298 &nm->peers[i], 299 &ntb_msit_count_fops); 300 301 debugfs_create_file_unsafe("ready", 0600, peer_dir, 302 &nm->peers[i], 303 &ntb_msit_ready_fops); 304 } 305 306 for (i = 0; i < num_irqs; i++) { 307 snprintf(buf, sizeof(buf), "irq%d_occurrences", i); 308 debugfs_create_file_unsafe(buf, 0400, nm->dbgfs_dir, 309 &nm->isr_ctx[i], 310 &ntb_msit_occurrences_fops); 311 } 312 } 313 314 static void ntb_msit_remove_dbgfs(struct ntb_msit_ctx *nm) 315 { 316 debugfs_remove_recursive(nm->dbgfs_dir); 317 } 318 319 static int ntb_msit_probe(struct ntb_client *client, struct ntb_dev *ntb) 320 { 321 struct ntb_msit_ctx *nm; 322 size_t struct_size; 323 int peers; 324 int ret; 325 326 peers = ntb_peer_port_count(ntb); 327 if (peers <= 0) 328 return -EINVAL; 329 330 if (ntb_spad_is_unsafe(ntb) || ntb_spad_count(ntb) < 2 * num_irqs + 1) { 331 dev_err(&ntb->dev, "NTB MSI test requires at least %d spads for %d irqs\n", 332 2 * num_irqs + 1, num_irqs); 333 return -EFAULT; 334 } 335 336 ret = ntb_spad_write(ntb, 0, -1); 337 if (ret) { 338 dev_err(&ntb->dev, "Unable to write spads: %d\n", ret); 339 return ret; 340 } 341 342 ret = ntb_db_clear_mask(ntb, GENMASK(peers - 1, 0)); 343 if (ret) { 344 dev_err(&ntb->dev, "Unable to clear doorbell mask: %d\n", ret); 345 return ret; 346 } 347 348 ret = ntb_msi_init(ntb, ntb_msit_desc_changed); 349 if (ret) { 350 dev_err(&ntb->dev, "Unable to initialize MSI library: %d\n", 351 ret); 352 return ret; 353 } 354 355 struct_size = sizeof(*nm) + sizeof(*nm->peers) * peers; 356 357 nm = devm_kzalloc(&ntb->dev, struct_size, GFP_KERNEL); 358 if (!nm) 359 return -ENOMEM; 360 361 nm->isr_ctx = devm_kcalloc(&ntb->dev, num_irqs, sizeof(*nm->isr_ctx), 362 GFP_KERNEL); 363 if (!nm->isr_ctx) 364 return -ENOMEM; 365 366 INIT_WORK(&nm->setup_work, ntb_msit_setup_work); 367 nm->ntb = ntb; 368 369 ntb_msit_create_dbgfs(nm); 370 371 ret = ntb_set_ctx(ntb, nm, &ntb_msit_ops); 372 if (ret) 373 goto remove_dbgfs; 374 375 if (!nm->isr_ctx) 376 goto remove_dbgfs; 377 378 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 379 380 return 0; 381 382 remove_dbgfs: 383 ntb_msit_remove_dbgfs(nm); 384 devm_kfree(&ntb->dev, nm->isr_ctx); 385 devm_kfree(&ntb->dev, nm); 386 return ret; 387 } 388 389 static void ntb_msit_remove(struct ntb_client *client, struct ntb_dev *ntb) 390 { 391 struct ntb_msit_ctx *nm = ntb->ctx; 392 int i; 393 394 ntb_link_disable(ntb); 395 ntb_db_set_mask(ntb, ntb_db_valid_mask(ntb)); 396 ntb_msi_clear_mws(ntb); 397 398 for (i = 0; i < ntb_peer_port_count(ntb); i++) 399 kfree(nm->peers[i].msi_desc); 400 401 ntb_clear_ctx(ntb); 402 ntb_msit_remove_dbgfs(nm); 403 } 404 405 static struct ntb_client ntb_msit_client = { 406 .ops = { 407 .probe = ntb_msit_probe, 408 .remove = ntb_msit_remove 409 } 410 }; 411 412 static int __init ntb_msit_init(void) 413 { 414 int ret; 415 416 if (debugfs_initialized()) 417 ntb_msit_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, 418 NULL); 419 420 ret = ntb_register_client(&ntb_msit_client); 421 if (ret) 422 debugfs_remove_recursive(ntb_msit_dbgfs_topdir); 423 424 return ret; 425 } 426 module_init(ntb_msit_init); 427 428 static void __exit ntb_msit_exit(void) 429 { 430 ntb_unregister_client(&ntb_msit_client); 431 debugfs_remove_recursive(ntb_msit_dbgfs_topdir); 432 } 433 module_exit(ntb_msit_exit); 434