127 static bool __imsic_local_sync(struct imsic_local_priv *lpriv)  in __imsic_local_sync()  argument
134 	lockdep_assert_held(&lpriv->lock);  in __imsic_local_sync()
136 	for_each_set_bit(i, lpriv->dirty_bitmap, imsic->global.nr_ids + 1) {  in __imsic_local_sync()
139 		vec = &lpriv->vectors[i];  in __imsic_local_sync()
180 				NULL : &lpriv->vectors[mvec->local_id];  in __imsic_local_sync()
204 		bitmap_clear(lpriv->dirty_bitmap, i, 1);  in __imsic_local_sync()
211 static void __imsic_local_timer_start(struct imsic_local_priv *lpriv, unsigned int cpu)  in __imsic_local_timer_start()  argument
213 	lockdep_assert_held(&lpriv->lock);  in __imsic_local_timer_start()
215 	if (!timer_pending(&lpriv->timer)) {  in __imsic_local_timer_start()
216 		lpriv->timer.expires = jiffies + 1;  in __imsic_local_timer_start()
217 		add_timer_on(&lpriv->timer, cpu);  in __imsic_local_timer_start()
221 static inline void __imsic_local_timer_start(struct imsic_local_priv *lpriv, unsigned int cpu)  in __imsic_local_timer_start()  argument
228 	struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv);  in imsic_local_sync_all()  local
231 	raw_spin_lock_irqsave(&lpriv->lock, flags);  in imsic_local_sync_all()
234 		bitmap_fill(lpriv->dirty_bitmap, imsic->global.nr_ids + 1);  in imsic_local_sync_all()
235 	if (!__imsic_local_sync(lpriv))  in imsic_local_sync_all()
236 		__imsic_local_timer_start(lpriv, smp_processor_id());  in imsic_local_sync_all()
238 	raw_spin_unlock_irqrestore(&lpriv->lock, flags);  in imsic_local_sync_all()
259 static void __imsic_remote_sync(struct imsic_local_priv *lpriv, unsigned int cpu)  in __imsic_remote_sync()  argument
261 	lockdep_assert_held(&lpriv->lock);  in __imsic_remote_sync()
277 			if (__imsic_local_sync(lpriv))  in __imsic_remote_sync()
281 		__imsic_local_timer_start(lpriv, cpu);  in __imsic_remote_sync()
285 static void __imsic_remote_sync(struct imsic_local_priv *lpriv, unsigned int cpu)  in __imsic_remote_sync()  argument
287 	lockdep_assert_held(&lpriv->lock);  in __imsic_remote_sync()
288 	__imsic_local_sync(lpriv);  in __imsic_remote_sync()
294 	struct imsic_local_priv *lpriv;  in imsic_vector_mask()  local
296 	lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu);  in imsic_vector_mask()
297 	if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec))  in imsic_vector_mask()
305 	raw_spin_lock(&lpriv->lock);  in imsic_vector_mask()
308 	bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1);  in imsic_vector_mask()
309 	__imsic_remote_sync(lpriv, vec->cpu);  in imsic_vector_mask()
311 	raw_spin_unlock(&lpriv->lock);  in imsic_vector_mask()
316 	struct imsic_local_priv *lpriv;  in imsic_vector_unmask()  local
318 	lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu);  in imsic_vector_unmask()
319 	if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec))  in imsic_vector_unmask()
327 	raw_spin_lock(&lpriv->lock);  in imsic_vector_unmask()
330 	bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1);  in imsic_vector_unmask()
331 	__imsic_remote_sync(lpriv, vec->cpu);  in imsic_vector_unmask()
333 	raw_spin_unlock(&lpriv->lock);  in imsic_vector_unmask()
338 	struct imsic_local_priv *lpriv;  in imsic_vector_force_move_cleanup()  local
342 	lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu);  in imsic_vector_force_move_cleanup()
343 	raw_spin_lock_irqsave(&lpriv->lock, flags);  in imsic_vector_force_move_cleanup()
350 	raw_spin_unlock_irqrestore(&lpriv->lock, flags);  in imsic_vector_force_move_cleanup()
353 static bool imsic_vector_move_update(struct imsic_local_priv *lpriv,  in imsic_vector_move_update()  argument
360 	raw_spin_lock_irqsave(&lpriv->lock, flags);  in imsic_vector_move_update()
371 	bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1);  in imsic_vector_move_update()
372 	__imsic_remote_sync(lpriv, vec->cpu);  in imsic_vector_move_update()
374 	raw_spin_unlock_irqrestore(&lpriv->lock, flags);  in imsic_vector_move_update()
387 	old_lpriv = per_cpu_ptr(imsic->lpriv, old_vec->cpu);  in imsic_vector_move()
391 	new_lpriv = per_cpu_ptr(imsic->lpriv, new_vec->cpu);  in imsic_vector_move()
408 	struct imsic_local_priv *lpriv;  in imsic_vector_debug_show()  local
412 	lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu);  in imsic_vector_debug_show()
413 	if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec))  in imsic_vector_debug_show()
439 	struct imsic_local_priv *lpriv = per_cpu_ptr(imsic->lpriv, cpu);  in imsic_vector_from_local_id()  local
441 	if (!lpriv || imsic->global.nr_ids < local_id)  in imsic_vector_from_local_id()
444 	return &lpriv->vectors[local_id];  in imsic_vector_from_local_id()
450 	struct imsic_local_priv *lpriv;  in imsic_vector_alloc()  local
461 	lpriv = per_cpu_ptr(imsic->lpriv, cpu);  in imsic_vector_alloc()
462 	vec = &lpriv->vectors[local_id];  in imsic_vector_alloc()
483 	struct imsic_local_priv *lpriv;  in imsic_local_cleanup()  local
487 		lpriv = per_cpu_ptr(imsic->lpriv, cpu);  in imsic_local_cleanup()
489 		bitmap_free(lpriv->dirty_bitmap);  in imsic_local_cleanup()
490 		kfree(lpriv->vectors);  in imsic_local_cleanup()
493 	free_percpu(imsic->lpriv);  in imsic_local_cleanup()
499 	struct imsic_local_priv *lpriv;  in imsic_local_init()  local
504 	imsic->lpriv = alloc_percpu(typeof(*imsic->lpriv));  in imsic_local_init()
505 	if (!imsic->lpriv)  in imsic_local_init()
510 		lpriv = per_cpu_ptr(imsic->lpriv, cpu);  in imsic_local_init()
512 		raw_spin_lock_init(&lpriv->lock);  in imsic_local_init()
515 		lpriv->dirty_bitmap = bitmap_zalloc(global->nr_ids + 1, GFP_KERNEL);  in imsic_local_init()
516 		if (!lpriv->dirty_bitmap)  in imsic_local_init()
521 		timer_setup(&lpriv->timer, imsic_local_timer_callback, TIMER_PINNED);  in imsic_local_init()
525 		lpriv->vectors = kcalloc(global->nr_ids + 1, sizeof(*lpriv->vectors),  in imsic_local_init()
527 		if (!lpriv->vectors)  in imsic_local_init()
532 			vec = &lpriv->vectors[i];  in imsic_local_init()
564 	struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv);  in imsic_state_offline()  local
566 	raw_spin_lock_irqsave(&lpriv->lock, flags);  in imsic_state_offline()
567 	WARN_ON_ONCE(timer_delete_sync_try(&lpriv->timer) < 0);  in imsic_state_offline()
568 	raw_spin_unlock_irqrestore(&lpriv->lock, flags);  in imsic_state_offline()