Lines Matching +full:num +full:- +full:vectors
1 // SPDX-License-Identifier: GPL-2.0
7 #define pr_fmt(fmt) "riscv-imsic: " fmt
22 #include "irq-riscv-imsic-state.h"
63 return imsic ? &imsic->global : NULL; in imsic_get_global_config()
74 imask = BIT(id & (__riscv_xlen - 1)); in __imsic_eix_read_clear()
102 * are XLEN-wide and we must not touch IDs which in __imsic_eix_update()
106 for (i = id & (__riscv_xlen - 1); id < last_id && i < __riscv_xlen; i++) { in __imsic_eix_update()
133 lockdep_assert_held(&lpriv->lock); in __imsic_local_sync()
135 for_each_set_bit(i, lpriv->dirty_bitmap, imsic->global.nr_ids + 1) { in __imsic_local_sync()
138 vec = &lpriv->vectors[i]; in __imsic_local_sync()
140 if (READ_ONCE(vec->enable)) in __imsic_local_sync()
148 * ID pending bit and re-trigger the new ID on other CPU in __imsic_local_sync()
151 mvec = READ_ONCE(vec->move); in __imsic_local_sync()
152 WRITE_ONCE(vec->move, NULL); in __imsic_local_sync()
155 mlocal = per_cpu_ptr(imsic->global.local, mvec->cpu); in __imsic_local_sync()
156 writel_relaxed(mvec->local_id, mlocal->msi_va); in __imsic_local_sync()
159 imsic_vector_free(&lpriv->vectors[i]); in __imsic_local_sync()
163 bitmap_clear(lpriv->dirty_bitmap, i, 1); in __imsic_local_sync()
169 struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); in imsic_local_sync_all()
172 raw_spin_lock_irqsave(&lpriv->lock, flags); in imsic_local_sync_all()
173 bitmap_fill(lpriv->dirty_bitmap, imsic->global.nr_ids + 1); in imsic_local_sync_all()
175 raw_spin_unlock_irqrestore(&lpriv->lock, flags); in imsic_local_sync_all()
193 struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); in imsic_local_timer_callback()
196 raw_spin_lock_irqsave(&lpriv->lock, flags); in imsic_local_timer_callback()
198 raw_spin_unlock_irqrestore(&lpriv->lock, flags); in imsic_local_timer_callback()
203 lockdep_assert_held(&lpriv->lock); in __imsic_remote_sync()
223 if (!timer_pending(&lpriv->timer)) { in __imsic_remote_sync()
224 lpriv->timer.expires = jiffies + 1; in __imsic_remote_sync()
225 add_timer_on(&lpriv->timer, cpu); in __imsic_remote_sync()
232 lockdep_assert_held(&lpriv->lock); in __imsic_remote_sync()
241 lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); in imsic_vector_mask()
242 if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec)) in imsic_vector_mask()
250 raw_spin_lock(&lpriv->lock); in imsic_vector_mask()
252 WRITE_ONCE(vec->enable, false); in imsic_vector_mask()
253 bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); in imsic_vector_mask()
254 __imsic_remote_sync(lpriv, vec->cpu); in imsic_vector_mask()
256 raw_spin_unlock(&lpriv->lock); in imsic_vector_mask()
263 lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); in imsic_vector_unmask()
264 if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec)) in imsic_vector_unmask()
272 raw_spin_lock(&lpriv->lock); in imsic_vector_unmask()
274 WRITE_ONCE(vec->enable, true); in imsic_vector_unmask()
275 bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); in imsic_vector_unmask()
276 __imsic_remote_sync(lpriv, vec->cpu); in imsic_vector_unmask()
278 raw_spin_unlock(&lpriv->lock); in imsic_vector_unmask()
287 raw_spin_lock_irqsave(&lpriv->lock, flags); in imsic_vector_move_update()
290 enabled = READ_ONCE(vec->enable); in imsic_vector_move_update()
291 WRITE_ONCE(vec->enable, new_enable); in imsic_vector_move_update()
292 WRITE_ONCE(vec->move, new_move); in imsic_vector_move_update()
295 bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); in imsic_vector_move_update()
296 __imsic_remote_sync(lpriv, vec->cpu); in imsic_vector_move_update()
298 raw_spin_unlock_irqrestore(&lpriv->lock, flags); in imsic_vector_move_update()
308 if (WARN_ON_ONCE(old_vec->cpu == new_vec->cpu)) in imsic_vector_move()
311 old_lpriv = per_cpu_ptr(imsic->lpriv, old_vec->cpu); in imsic_vector_move()
312 if (WARN_ON_ONCE(&old_lpriv->vectors[old_vec->local_id] != old_vec)) in imsic_vector_move()
315 new_lpriv = per_cpu_ptr(imsic->lpriv, new_vec->cpu); in imsic_vector_move()
316 if (WARN_ON_ONCE(&new_lpriv->vectors[new_vec->local_id] != new_vec)) in imsic_vector_move()
320 * Move and re-trigger the new vector based on the pending in imsic_vector_move()
336 lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); in imsic_vector_debug_show()
337 if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec)) in imsic_vector_debug_show()
343 seq_printf(m, "%*starget_cpu : %5u\n", ind, "", vec->cpu); in imsic_vector_debug_show()
344 seq_printf(m, "%*starget_local_id : %5u\n", ind, "", vec->local_id); in imsic_vector_debug_show()
346 (vec->local_id <= IMSIC_IPI_ID) ? 1 : 0); in imsic_vector_debug_show()
350 seq_printf(m, "%*smove_cpu : %5u\n", ind, "", mvec->cpu); in imsic_vector_debug_show()
351 seq_printf(m, "%*smove_local_id : %5u\n", ind, "", mvec->local_id); in imsic_vector_debug_show()
357 irq_matrix_debug_show(m, imsic->matrix, ind); in imsic_vector_debug_show_summary()
363 struct imsic_local_priv *lpriv = per_cpu_ptr(imsic->lpriv, cpu); in imsic_vector_from_local_id()
365 if (!lpriv || imsic->global.nr_ids < local_id) in imsic_vector_from_local_id()
368 return &lpriv->vectors[local_id]; in imsic_vector_from_local_id()
379 raw_spin_lock_irqsave(&imsic->matrix_lock, flags); in imsic_vector_alloc()
380 local_id = irq_matrix_alloc(imsic->matrix, mask, false, &cpu); in imsic_vector_alloc()
381 raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); in imsic_vector_alloc()
385 lpriv = per_cpu_ptr(imsic->lpriv, cpu); in imsic_vector_alloc()
386 vec = &lpriv->vectors[local_id]; in imsic_vector_alloc()
387 vec->hwirq = hwirq; in imsic_vector_alloc()
388 vec->enable = false; in imsic_vector_alloc()
389 vec->move = NULL; in imsic_vector_alloc()
398 raw_spin_lock_irqsave(&imsic->matrix_lock, flags); in imsic_vector_free()
399 vec->hwirq = UINT_MAX; in imsic_vector_free()
400 irq_matrix_free(imsic->matrix, vec->cpu, vec->local_id, false); in imsic_vector_free()
401 raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); in imsic_vector_free()
410 lpriv = per_cpu_ptr(imsic->lpriv, cpu); in imsic_local_cleanup()
412 bitmap_free(lpriv->dirty_bitmap); in imsic_local_cleanup()
413 kfree(lpriv->vectors); in imsic_local_cleanup()
416 free_percpu(imsic->lpriv); in imsic_local_cleanup()
421 struct imsic_global_config *global = &imsic->global; in imsic_local_init()
426 /* Allocate per-CPU private state */ in imsic_local_init()
427 imsic->lpriv = alloc_percpu(typeof(*imsic->lpriv)); in imsic_local_init()
428 if (!imsic->lpriv) in imsic_local_init()
429 return -ENOMEM; in imsic_local_init()
431 /* Setup per-CPU private state */ in imsic_local_init()
433 lpriv = per_cpu_ptr(imsic->lpriv, cpu); in imsic_local_init()
435 raw_spin_lock_init(&lpriv->lock); in imsic_local_init()
438 lpriv->dirty_bitmap = bitmap_zalloc(global->nr_ids + 1, GFP_KERNEL); in imsic_local_init()
439 if (!lpriv->dirty_bitmap) in imsic_local_init()
444 timer_setup(&lpriv->timer, imsic_local_timer_callback, TIMER_PINNED); in imsic_local_init()
448 lpriv->vectors = kcalloc(global->nr_ids + 1, sizeof(*lpriv->vectors), in imsic_local_init()
450 if (!lpriv->vectors) in imsic_local_init()
454 for (i = 0; i <= global->nr_ids; i++) { in imsic_local_init()
455 vec = &lpriv->vectors[i]; in imsic_local_init()
456 vec->cpu = cpu; in imsic_local_init()
457 vec->local_id = i; in imsic_local_init()
458 vec->hwirq = UINT_MAX; in imsic_local_init()
466 return -ENOMEM; in imsic_local_init()
473 raw_spin_lock_irqsave(&imsic->matrix_lock, flags); in imsic_state_online()
474 irq_matrix_online(imsic->matrix); in imsic_state_online()
475 raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); in imsic_state_online()
482 raw_spin_lock_irqsave(&imsic->matrix_lock, flags); in imsic_state_offline()
483 irq_matrix_offline(imsic->matrix); in imsic_state_offline()
484 raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); in imsic_state_offline()
487 struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); in imsic_state_offline()
489 raw_spin_lock_irqsave(&lpriv->lock, flags); in imsic_state_offline()
490 WARN_ON_ONCE(try_to_del_timer_sync(&lpriv->timer) < 0); in imsic_state_offline()
491 raw_spin_unlock_irqrestore(&lpriv->lock, flags); in imsic_state_offline()
497 struct imsic_global_config *global = &imsic->global; in imsic_matrix_init()
499 raw_spin_lock_init(&imsic->matrix_lock); in imsic_matrix_init()
500 imsic->matrix = irq_alloc_matrix(global->nr_ids + 1, in imsic_matrix_init()
501 0, global->nr_ids + 1); in imsic_matrix_init()
502 if (!imsic->matrix) in imsic_matrix_init()
503 return -ENOMEM; in imsic_matrix_init()
506 irq_matrix_assign_system(imsic->matrix, 0, false); in imsic_matrix_init()
509 irq_matrix_assign_system(imsic->matrix, IMSIC_IPI_ID, false); in imsic_matrix_init()
521 rc = of_property_read_u32(to_of_node(fwnode), "riscv,guest-index-bits", in imsic_populate_global_dt()
522 &global->guest_index_bits); in imsic_populate_global_dt()
524 global->guest_index_bits = 0; in imsic_populate_global_dt()
527 rc = of_property_read_u32(to_of_node(fwnode), "riscv,hart-index-bits", in imsic_populate_global_dt()
528 &global->hart_index_bits); in imsic_populate_global_dt()
531 global->hart_index_bits = __fls(*nr_parent_irqs); in imsic_populate_global_dt()
532 if (BIT(global->hart_index_bits) < *nr_parent_irqs) in imsic_populate_global_dt()
533 global->hart_index_bits++; in imsic_populate_global_dt()
537 rc = of_property_read_u32(to_of_node(fwnode), "riscv,group-index-bits", in imsic_populate_global_dt()
538 &global->group_index_bits); in imsic_populate_global_dt()
540 global->group_index_bits = 0; in imsic_populate_global_dt()
544 * If not specified assumed the default APLIC-IMSIC configuration. in imsic_populate_global_dt()
546 rc = of_property_read_u32(to_of_node(fwnode), "riscv,group-index-shift", in imsic_populate_global_dt()
547 &global->group_index_shift); in imsic_populate_global_dt()
549 global->group_index_shift = IMSIC_MMIO_PAGE_SHIFT * 2; in imsic_populate_global_dt()
552 rc = of_property_read_u32(to_of_node(fwnode), "riscv,num-ids", in imsic_populate_global_dt()
553 &global->nr_ids); in imsic_populate_global_dt()
560 rc = of_property_read_u32(to_of_node(fwnode), "riscv,num-guest-ids", in imsic_populate_global_dt()
561 &global->nr_guest_ids); in imsic_populate_global_dt()
563 global->nr_guest_ids = global->nr_ids; in imsic_populate_global_dt()
574 global->guest_index_bits = imsic->guest_index_bits; in imsic_populate_global_acpi()
575 global->hart_index_bits = imsic->hart_index_bits; in imsic_populate_global_acpi()
576 global->group_index_bits = imsic->group_index_bits; in imsic_populate_global_acpi()
577 global->group_index_shift = imsic->group_index_shift; in imsic_populate_global_acpi()
578 global->nr_ids = imsic->num_ids; in imsic_populate_global_acpi()
579 global->nr_guest_ids = imsic->num_guest_ids; in imsic_populate_global_acpi()
594 return -EINVAL; in imsic_get_parent_hartid()
608 return -EINVAL; in imsic_get_parent_hartid()
641 return -EINVAL; in imsic_parse_fwnode()
653 i = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT; in imsic_parse_fwnode()
654 if (i < global->guest_index_bits) { in imsic_parse_fwnode()
656 return -EINVAL; in imsic_parse_fwnode()
660 i = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT - global->guest_index_bits; in imsic_parse_fwnode()
661 if (i < global->hart_index_bits) { in imsic_parse_fwnode()
663 return -EINVAL; in imsic_parse_fwnode()
667 i = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT - in imsic_parse_fwnode()
668 global->guest_index_bits - global->hart_index_bits; in imsic_parse_fwnode()
669 if (i < global->group_index_bits) { in imsic_parse_fwnode()
671 return -EINVAL; in imsic_parse_fwnode()
675 i = global->group_index_bits + global->group_index_shift - 1; in imsic_parse_fwnode()
678 return -EINVAL; in imsic_parse_fwnode()
682 if (global->nr_ids < IMSIC_MIN_ID || in imsic_parse_fwnode()
683 global->nr_ids >= IMSIC_MAX_ID || in imsic_parse_fwnode()
684 (global->nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) { in imsic_parse_fwnode()
686 return -EINVAL; in imsic_parse_fwnode()
690 if (global->nr_guest_ids < IMSIC_MIN_ID || in imsic_parse_fwnode()
691 global->nr_guest_ids >= IMSIC_MAX_ID || in imsic_parse_fwnode()
692 (global->nr_guest_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) { in imsic_parse_fwnode()
694 return -EINVAL; in imsic_parse_fwnode()
701 return -EINVAL; in imsic_parse_fwnode()
703 global->base_addr = res.start; in imsic_parse_fwnode()
704 global->base_addr &= ~(BIT(global->guest_index_bits + in imsic_parse_fwnode()
705 global->hart_index_bits + in imsic_parse_fwnode()
706 IMSIC_MMIO_PAGE_SHIFT) - 1); in imsic_parse_fwnode()
707 global->base_addr &= ~((BIT(global->group_index_bits) - 1) << in imsic_parse_fwnode()
708 global->group_index_shift); in imsic_parse_fwnode()
730 * implementation of SMP IRQ affinity and per-CPU IPIs. in imsic_setup_state()
732 * This means on a multi-socket (or multi-die) platform we in imsic_setup_state()
737 return -EALREADY; in imsic_setup_state()
742 return -ENODEV; in imsic_setup_state()
747 return -ENOMEM; in imsic_setup_state()
748 imsic->fwnode = fwnode; in imsic_setup_state()
749 global = &imsic->global; in imsic_setup_state()
751 global->local = alloc_percpu(typeof(*global->local)); in imsic_setup_state()
752 if (!global->local) { in imsic_setup_state()
753 rc = -ENOMEM; in imsic_setup_state()
765 rc = -ENOMEM; in imsic_setup_state()
772 rc = -ENOMEM; in imsic_setup_state()
785 base_addr &= ~(BIT(global->guest_index_bits + in imsic_setup_state()
786 global->hart_index_bits + in imsic_setup_state()
787 IMSIC_MMIO_PAGE_SHIFT) - 1); in imsic_setup_state()
788 base_addr &= ~((BIT(global->group_index_bits) - 1) << in imsic_setup_state()
789 global->group_index_shift); in imsic_setup_state()
790 if (base_addr != global->base_addr) { in imsic_setup_state()
791 rc = -EINVAL; in imsic_setup_state()
798 rc = -EIO; in imsic_setup_state()
804 /* Initialize local (or per-CPU )state */ in imsic_setup_state()
828 reloff = i * BIT(global->guest_index_bits) * in imsic_setup_state()
838 * BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ in imsic_setup_state()
841 reloff -= ALIGN(resource_size(&mmios[j]), in imsic_setup_state()
842 BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ); in imsic_setup_state()
849 local = per_cpu_ptr(global->local, cpu); in imsic_setup_state()
850 local->msi_pa = mmios[index].start + reloff; in imsic_setup_state()
851 local->msi_va = mmios_va[index] + reloff; in imsic_setup_state()
859 rc = -ENODEV; in imsic_setup_state()
870 /* We don't need MMIO arrays anymore so let's free-up */ in imsic_setup_state()
886 free_percpu(imsic->global.local); in imsic_setup_state()