Lines Matching refs:group
437 static bool tmigr_check_migrator(struct tmigr_group *group, u8 childmask) in tmigr_check_migrator() argument
441 s.state = atomic_read(&group->migr_state); in tmigr_check_migrator()
449 static bool tmigr_check_migrator_and_lonely(struct tmigr_group *group, u8 childmask) in tmigr_check_migrator_and_lonely() argument
455 s.state = atomic_read(&group->migr_state); in tmigr_check_migrator_and_lonely()
466 static bool tmigr_check_lonely(struct tmigr_group *group) in tmigr_check_lonely() argument
471 s.state = atomic_read(&group->migr_state); in tmigr_check_lonely()
528 struct tmigr_group *child = NULL, *group = tmc->tmgroup; in __walk_groups() local
531 WARN_ON_ONCE(group->level >= tmigr_hierarchy_levels); in __walk_groups()
533 if (up(group, child, data)) in __walk_groups()
536 child = group; in __walk_groups()
537 group = group->parent; in __walk_groups()
539 } while (group); in __walk_groups()
555 static struct tmigr_event *tmigr_next_groupevt(struct tmigr_group *group) in tmigr_next_groupevt() argument
560 lockdep_assert_held(&group->lock); in tmigr_next_groupevt()
562 WRITE_ONCE(group->next_expiry, KTIME_MAX); in tmigr_next_groupevt()
564 while ((node = timerqueue_getnext(&group->events))) { in tmigr_next_groupevt()
568 WRITE_ONCE(group->next_expiry, evt->nextevt.expires); in tmigr_next_groupevt()
576 if (!timerqueue_del(&group->events, node)) in tmigr_next_groupevt()
588 static struct tmigr_event *tmigr_next_expired_groupevt(struct tmigr_group *group, in tmigr_next_expired_groupevt() argument
591 struct tmigr_event *evt = tmigr_next_groupevt(group); in tmigr_next_expired_groupevt()
599 timerqueue_del(&group->events, &evt->nextevt); in tmigr_next_expired_groupevt()
600 tmigr_next_groupevt(group); in tmigr_next_expired_groupevt()
605 static u64 tmigr_next_groupevt_expires(struct tmigr_group *group) in tmigr_next_groupevt_expires() argument
609 evt = tmigr_next_groupevt(group); in tmigr_next_groupevt_expires()
617 static bool tmigr_active_up(struct tmigr_group *group, in tmigr_active_up() argument
631 curstate.state = atomic_read(&group->migr_state); in tmigr_active_up()
647 } while (!atomic_try_cmpxchg(&group->migr_state, &curstate.state, newstate.state)); in tmigr_active_up()
649 trace_tmigr_group_set_cpu_active(group, newstate, childmask); in tmigr_active_up()
663 group->groupevt.ignore = true; in tmigr_active_up()
717 bool tmigr_update_events(struct tmigr_group *group, struct tmigr_group *child, in tmigr_update_events() argument
728 raw_spin_lock_nested(&group->lock, SINGLE_DEPTH_NESTING); in tmigr_update_events()
731 groupstate.state = atomic_read(&group->migr_state); in tmigr_update_events()
772 if (evt->ignore && !remote && group->parent) in tmigr_update_events()
775 raw_spin_lock(&group->lock); in tmigr_update_events()
778 groupstate.state = atomic_read(&group->migr_state); in tmigr_update_events()
792 if (!timerqueue_del(&group->events, &evt->nextevt)) in tmigr_update_events()
793 WRITE_ONCE(group->next_expiry, KTIME_MAX); in tmigr_update_events()
816 if (timerqueue_add(&group->events, &evt->nextevt)) in tmigr_update_events()
817 WRITE_ONCE(group->next_expiry, nextexp); in tmigr_update_events()
821 if (!group->parent && (groupstate.migrator == TMIGR_NONE)) { in tmigr_update_events()
840 data->firstexp = tmigr_next_groupevt_expires(group); in tmigr_update_events()
843 trace_tmigr_update_events(child, group, childstate, groupstate, in tmigr_update_events()
847 raw_spin_unlock(&group->lock); in tmigr_update_events()
855 static bool tmigr_new_timer_up(struct tmigr_group *group, in tmigr_new_timer_up() argument
859 return tmigr_update_events(group, child, data); in tmigr_new_timer_up()
988 static bool tmigr_handle_remote_up(struct tmigr_group *group, in tmigr_handle_remote_up() argument
1002 trace_tmigr_handle_remote(group); in tmigr_handle_remote_up()
1009 if (!tmigr_check_migrator(group, childmask)) in tmigr_handle_remote_up()
1012 raw_spin_lock_irq(&group->lock); in tmigr_handle_remote_up()
1014 evt = tmigr_next_expired_groupevt(group, now); in tmigr_handle_remote_up()
1019 raw_spin_unlock_irq(&group->lock); in tmigr_handle_remote_up()
1032 data->firstexp = group->next_expiry; in tmigr_handle_remote_up()
1034 raw_spin_unlock_irq(&group->lock); in tmigr_handle_remote_up()
1087 static bool tmigr_requires_handle_remote_up(struct tmigr_group *group, in tmigr_requires_handle_remote_up() argument
1100 if (!tmigr_check_migrator(group, childmask)) in tmigr_requires_handle_remote_up()
1108 if (group->parent && !data->tmc_active) in tmigr_requires_handle_remote_up()
1118 data->firstexp = READ_ONCE(group->next_expiry); in tmigr_requires_handle_remote_up()
1124 raw_spin_lock(&group->lock); in tmigr_requires_handle_remote_up()
1125 data->firstexp = group->next_expiry; in tmigr_requires_handle_remote_up()
1126 if (data->now >= group->next_expiry) { in tmigr_requires_handle_remote_up()
1128 raw_spin_unlock(&group->lock); in tmigr_requires_handle_remote_up()
1131 raw_spin_unlock(&group->lock); in tmigr_requires_handle_remote_up()
1230 static bool tmigr_inactive_up(struct tmigr_group *group, in tmigr_inactive_up() argument
1247 curstate.state = atomic_read_acquire(&group->migr_state); in tmigr_inactive_up()
1285 if (atomic_try_cmpxchg(&group->migr_state, &curstate.state, newstate.state)) { in tmigr_inactive_up()
1286 trace_tmigr_group_set_cpu_inactive(group, newstate, childmask); in tmigr_inactive_up()
1302 tmigr_update_events(group, child, data); in tmigr_inactive_up()
1382 struct tmigr_group *group = tmc->tmgroup; in tmigr_quick_check() local
1394 if (!tmigr_check_lonely(group)) { in tmigr_quick_check()
1403 nextevt = min_t(u64, nextevt, READ_ONCE(group->next_expiry)); in tmigr_quick_check()
1404 if (!group->parent) in tmigr_quick_check()
1407 group = group->parent; in tmigr_quick_check()
1408 } while (group); in tmigr_quick_check()
1473 static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl, in tmigr_init_group() argument
1478 raw_spin_lock_init(&group->lock); in tmigr_init_group()
1480 group->level = lvl; in tmigr_init_group()
1481 group->numa_node = lvl < tmigr_crossnode_level ? node : NUMA_NO_NODE; in tmigr_init_group()
1483 group->num_children = 0; in tmigr_init_group()
1488 atomic_set(&group->migr_state, s.state); in tmigr_init_group()
1490 timerqueue_init_head(&group->events); in tmigr_init_group()
1491 timerqueue_init(&group->groupevt.nextevt); in tmigr_init_group()
1492 group->groupevt.nextevt.expires = KTIME_MAX; in tmigr_init_group()
1493 WRITE_ONCE(group->next_expiry, KTIME_MAX); in tmigr_init_group()
1494 group->groupevt.ignore = true; in tmigr_init_group()
1500 struct tmigr_group *tmp, *group = NULL; in tmigr_get_group() local
1524 group = tmp; in tmigr_get_group()
1528 if (group) in tmigr_get_group()
1529 return group; in tmigr_get_group()
1532 group = kzalloc_node(sizeof(*group), GFP_KERNEL, node); in tmigr_get_group()
1533 if (!group) in tmigr_get_group()
1536 tmigr_init_group(group, lvl, node); in tmigr_get_group()
1539 list_add(&group->list, &tmigr_level_list[lvl]); in tmigr_get_group()
1540 trace_tmigr_group_set(group); in tmigr_get_group()
1541 return group; in tmigr_get_group()
1601 struct tmigr_group *group, *child, **stack; in tmigr_setup_groups() local
1610 group = tmigr_get_group(cpu, node, i); in tmigr_setup_groups()
1611 if (IS_ERR(group)) { in tmigr_setup_groups()
1612 err = PTR_ERR(group); in tmigr_setup_groups()
1617 stack[i++] = group; in tmigr_setup_groups()
1627 if (group->parent || i == tmigr_hierarchy_levels || in tmigr_setup_groups()
1635 group = stack[--i]; in tmigr_setup_groups()
1638 list_del(&group->list); in tmigr_setup_groups()
1639 kfree(group); in tmigr_setup_groups()
1643 WARN_ON_ONCE(i != group->level); in tmigr_setup_groups()
1651 raw_spin_lock_irq(&group->lock); in tmigr_setup_groups()
1653 tmc->tmgroup = group; in tmigr_setup_groups()
1654 tmc->groupmask = BIT(group->num_children++); in tmigr_setup_groups()
1656 raw_spin_unlock_irq(&group->lock); in tmigr_setup_groups()
1665 tmigr_connect_child_parent(child, group, false); in tmigr_setup_groups()
1675 if (group->num_children == 1 && list_is_singular(lvllist)) { in tmigr_setup_groups()
1690 tmigr_connect_child_parent(child, group, true); in tmigr_setup_groups()