core.c (ef5b28372c565128bdce7a59bc78402a8ce68e1b) core.c (571d91dcadfa3cef499010b4eddb9b58b0da4d24)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Performance events core code:
4 *
5 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>

--- 1800 unchanged lines hidden (view full) ---

1809 * Initialize event state based on the perf_event_attr::disabled.
1810 */
1811static inline void perf_event__state_init(struct perf_event *event)
1812{
1813 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1814 PERF_EVENT_STATE_INACTIVE;
1815}
1816
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Performance events core code:
4 *
5 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
8 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>

--- 1800 unchanged lines hidden (view full) ---

1809 * Initialize event state based on the perf_event_attr::disabled.
1810 */
1811static inline void perf_event__state_init(struct perf_event *event)
1812{
1813 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1814 PERF_EVENT_STATE_INACTIVE;
1815}
1816
1817static int __perf_event_read_size(u64 read_format, int nr_siblings)
1817static void __perf_event_read_size(struct perf_event *event, int nr_siblings)
1818{
1819 int entry = sizeof(u64); /* value */
1820 int size = 0;
1821 int nr = 1;
1822
1818{
1819 int entry = sizeof(u64); /* value */
1820 int size = 0;
1821 int nr = 1;
1822
1823 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1823 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1824 size += sizeof(u64);
1825
1824 size += sizeof(u64);
1825
1826 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1826 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1827 size += sizeof(u64);
1828
1827 size += sizeof(u64);
1828
1829 if (read_format & PERF_FORMAT_ID)
1829 if (event->attr.read_format & PERF_FORMAT_ID)
1830 entry += sizeof(u64);
1831
1830 entry += sizeof(u64);
1831
1832 if (read_format & PERF_FORMAT_LOST)
1832 if (event->attr.read_format & PERF_FORMAT_LOST)
1833 entry += sizeof(u64);
1834
1833 entry += sizeof(u64);
1834
1835 if (read_format & PERF_FORMAT_GROUP) {
1835 if (event->attr.read_format & PERF_FORMAT_GROUP) {
1836 nr += nr_siblings;
1837 size += sizeof(u64);
1838 }
1839
1836 nr += nr_siblings;
1837 size += sizeof(u64);
1838 }
1839
1840 /*
1841 * Since perf_event_validate_size() limits this to 16k and inhibits
1842 * adding more siblings, this will never overflow.
1843 */
1844 return size + nr * entry;
1840 size += entry * nr;
1841 event->read_size = size;
1845}
1846
1847static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
1848{
1849 struct perf_sample_data *data;
1850 u16 size = 0;
1851
1852 if (sample_type & PERF_SAMPLE_IP)

--- 33 unchanged lines hidden (view full) ---

1886}
1887
1888/*
1889 * Called at perf_event creation and when events are attached/detached from a
1890 * group.
1891 */
1892static void perf_event__header_size(struct perf_event *event)
1893{
1842}
1843
1844static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
1845{
1846 struct perf_sample_data *data;
1847 u16 size = 0;
1848
1849 if (sample_type & PERF_SAMPLE_IP)

--- 33 unchanged lines hidden (view full) ---

1883}
1884
1885/*
1886 * Called at perf_event creation and when events are attached/detached from a
1887 * group.
1888 */
1889static void perf_event__header_size(struct perf_event *event)
1890{
1894 event->read_size =
1895 __perf_event_read_size(event->attr.read_format,
1896 event->group_leader->nr_siblings);
1891 __perf_event_read_size(event,
1892 event->group_leader->nr_siblings);
1897 __perf_event_header_size(event, event->attr.sample_type);
1898}
1899
1900static void perf_event__id_header_size(struct perf_event *event)
1901{
1902 struct perf_sample_data *data;
1903 u64 sample_type = event->attr.sample_type;
1904 u16 size = 0;

--- 14 unchanged lines hidden (view full) ---

1919 size += sizeof(data->stream_id);
1920
1921 if (sample_type & PERF_SAMPLE_CPU)
1922 size += sizeof(data->cpu_entry);
1923
1924 event->id_header_size = size;
1925}
1926
1893 __perf_event_header_size(event, event->attr.sample_type);
1894}
1895
1896static void perf_event__id_header_size(struct perf_event *event)
1897{
1898 struct perf_sample_data *data;
1899 u64 sample_type = event->attr.sample_type;
1900 u16 size = 0;

--- 14 unchanged lines hidden (view full) ---

1915 size += sizeof(data->stream_id);
1916
1917 if (sample_type & PERF_SAMPLE_CPU)
1918 size += sizeof(data->cpu_entry);
1919
1920 event->id_header_size = size;
1921}
1922
1927/*
1928 * Check that adding an event to the group does not result in anybody
1929 * overflowing the 64k event limit imposed by the output buffer.
1930 *
1931 * Specifically, check that the read_size for the event does not exceed 16k,
1932 * read_size being the one term that grows with groups size. Since read_size
1933 * depends on per-event read_format, also (re)check the existing events.
1934 *
1935 * This leaves 48k for the constant size fields and things like callchains,
1936 * branch stacks and register sets.
1937 */
1938static bool perf_event_validate_size(struct perf_event *event)
1939{
1923static bool perf_event_validate_size(struct perf_event *event)
1924{
1940 struct perf_event *sibling, *group_leader = event->group_leader;
1925 /*
1926 * The values computed here will be over-written when we actually
1927 * attach the event.
1928 */
1929 __perf_event_read_size(event, event->group_leader->nr_siblings + 1);
1930 __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ);
1931 perf_event__id_header_size(event);
1941
1932
1942 if (__perf_event_read_size(event->attr.read_format,
1943 group_leader->nr_siblings + 1) > 16*1024)
1933 /*
1934 * Sum the lot; should not exceed the 64k limit we have on records.
1935 * Conservative limit to allow for callchains and other variable fields.
1936 */
1937 if (event->read_size + event->header_size +
1938 event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)
1944 return false;
1945
1939 return false;
1940
1946 if (__perf_event_read_size(group_leader->attr.read_format,
1947 group_leader->nr_siblings + 1) > 16*1024)
1948 return false;
1949
1950 for_each_sibling_event(sibling, group_leader) {
1951 if (__perf_event_read_size(sibling->attr.read_format,
1952 group_leader->nr_siblings + 1) > 16*1024)
1953 return false;
1954 }
1955
1956 return true;
1957}
1958
1959static void perf_group_attach(struct perf_event *event)
1960{
1961 struct perf_event *group_leader = event->group_leader, *pos;
1962
1963 lockdep_assert_held(&event->ctx->lock);

--- 11 unchanged lines hidden (view full) ---

1975 return;
1976
1977 WARN_ON_ONCE(group_leader->ctx != event->ctx);
1978
1979 group_leader->group_caps &= event->event_caps;
1980
1981 list_add_tail(&event->sibling_list, &group_leader->sibling_list);
1982 group_leader->nr_siblings++;
1941 return true;
1942}
1943
1944static void perf_group_attach(struct perf_event *event)
1945{
1946 struct perf_event *group_leader = event->group_leader, *pos;
1947
1948 lockdep_assert_held(&event->ctx->lock);

--- 11 unchanged lines hidden (view full) ---

1960 return;
1961
1962 WARN_ON_ONCE(group_leader->ctx != event->ctx);
1963
1964 group_leader->group_caps &= event->event_caps;
1965
1966 list_add_tail(&event->sibling_list, &group_leader->sibling_list);
1967 group_leader->nr_siblings++;
1983 group_leader->group_generation++;
1984
1985 perf_event__header_size(group_leader);
1986
1987 for_each_sibling_event(pos, group_leader)
1988 perf_event__header_size(pos);
1989}
1990
1991/*

--- 174 unchanged lines hidden (view full) ---

2166 perf_put_aux_event(event);
2167
2168 /*
2169 * If this is a sibling, remove it from its group.
2170 */
2171 if (leader != event) {
2172 list_del_init(&event->sibling_list);
2173 event->group_leader->nr_siblings--;
1968
1969 perf_event__header_size(group_leader);
1970
1971 for_each_sibling_event(pos, group_leader)
1972 perf_event__header_size(pos);
1973}
1974
1975/*

--- 174 unchanged lines hidden (view full) ---

2150 perf_put_aux_event(event);
2151
2152 /*
2153 * If this is a sibling, remove it from its group.
2154 */
2155 if (leader != event) {
2156 list_del_init(&event->sibling_list);
2157 event->group_leader->nr_siblings--;
2174 event->group_leader->group_generation++;
2175 goto out;
2176 }
2177
2178 /*
2179 * If this was a group event with sibling events then
2180 * upgrade the siblings to singleton events by adding them
2181 * to whatever list we are on.
2182 */

--- 2655 unchanged lines hidden (view full) ---

4838static struct perf_event_pmu_context *
4839find_get_pmu_context(struct pmu *pmu, struct perf_event_context *ctx,
4840 struct perf_event *event)
4841{
4842 struct perf_event_pmu_context *new = NULL, *epc;
4843 void *task_ctx_data = NULL;
4844
4845 if (!ctx->task) {
2158 goto out;
2159 }
2160
2161 /*
2162 * If this was a group event with sibling events then
2163 * upgrade the siblings to singleton events by adding them
2164 * to whatever list we are on.
2165 */

--- 2655 unchanged lines hidden (view full) ---

4821static struct perf_event_pmu_context *
4822find_get_pmu_context(struct pmu *pmu, struct perf_event_context *ctx,
4823 struct perf_event *event)
4824{
4825 struct perf_event_pmu_context *new = NULL, *epc;
4826 void *task_ctx_data = NULL;
4827
4828 if (!ctx->task) {
4846 /*
4847 * perf_pmu_migrate_context() / __perf_pmu_install_event()
4848 * relies on the fact that find_get_pmu_context() cannot fail
4849 * for CPU contexts.
4850 */
4851 struct perf_cpu_pmu_context *cpc;
4852
4853 cpc = per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);
4854 epc = &cpc->epc;
4855 raw_spin_lock_irq(&ctx->lock);
4856 if (!epc->ctx) {
4857 atomic_set(&epc->refcount, 1);
4858 epc->embedded = 1;

--- 615 unchanged lines hidden (view full) ---

5474 return count;
5475}
5476EXPORT_SYMBOL_GPL(perf_event_read_value);
5477
5478static int __perf_read_group_add(struct perf_event *leader,
5479 u64 read_format, u64 *values)
5480{
5481 struct perf_event_context *ctx = leader->ctx;
4829 struct perf_cpu_pmu_context *cpc;
4830
4831 cpc = per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);
4832 epc = &cpc->epc;
4833 raw_spin_lock_irq(&ctx->lock);
4834 if (!epc->ctx) {
4835 atomic_set(&epc->refcount, 1);
4836 epc->embedded = 1;

--- 615 unchanged lines hidden (view full) ---

5452 return count;
5453}
5454EXPORT_SYMBOL_GPL(perf_event_read_value);
5455
5456static int __perf_read_group_add(struct perf_event *leader,
5457 u64 read_format, u64 *values)
5458{
5459 struct perf_event_context *ctx = leader->ctx;
5482 struct perf_event *sub, *parent;
5460 struct perf_event *sub;
5483 unsigned long flags;
5484 int n = 1; /* skip @nr */
5485 int ret;
5486
5487 ret = perf_event_read(leader, true);
5488 if (ret)
5489 return ret;
5490
5491 raw_spin_lock_irqsave(&ctx->lock, flags);
5461 unsigned long flags;
5462 int n = 1; /* skip @nr */
5463 int ret;
5464
5465 ret = perf_event_read(leader, true);
5466 if (ret)
5467 return ret;
5468
5469 raw_spin_lock_irqsave(&ctx->lock, flags);
5492 /*
5493 * Verify the grouping between the parent and child (inherited)
5494 * events is still in tact.
5495 *
5496 * Specifically:
5497 * - leader->ctx->lock pins leader->sibling_list
5498 * - parent->child_mutex pins parent->child_list
5499 * - parent->ctx->mutex pins parent->sibling_list
5500 *
5501 * Because parent->ctx != leader->ctx (and child_list nests inside
5502 * ctx->mutex), group destruction is not atomic between children, also
5503 * see perf_event_release_kernel(). Additionally, parent can grow the
5504 * group.
5505 *
5506 * Therefore it is possible to have parent and child groups in a
5507 * different configuration and summing over such a beast makes no sense
5508 * what so ever.
5509 *
5510 * Reject this.
5511 */
5512 parent = leader->parent;
5513 if (parent &&
5514 (parent->group_generation != leader->group_generation ||
5515 parent->nr_siblings != leader->nr_siblings)) {
5516 ret = -ECHILD;
5517 goto unlock;
5518 }
5519
5520 /*
5521 * Since we co-schedule groups, {enabled,running} times of siblings
5522 * will be identical to those of the leader, so we only publish one
5523 * set.
5524 */
5525 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
5526 values[n++] += leader->total_time_enabled +

--- 17 unchanged lines hidden (view full) ---

5544 for_each_sibling_event(sub, leader) {
5545 values[n++] += perf_event_count(sub);
5546 if (read_format & PERF_FORMAT_ID)
5547 values[n++] = primary_event_id(sub);
5548 if (read_format & PERF_FORMAT_LOST)
5549 values[n++] = atomic64_read(&sub->lost_samples);
5550 }
5551
5470
5471 /*
5472 * Since we co-schedule groups, {enabled,running} times of siblings
5473 * will be identical to those of the leader, so we only publish one
5474 * set.
5475 */
5476 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
5477 values[n++] += leader->total_time_enabled +

--- 17 unchanged lines hidden (view full) ---

5495 for_each_sibling_event(sub, leader) {
5496 values[n++] += perf_event_count(sub);
5497 if (read_format & PERF_FORMAT_ID)
5498 values[n++] = primary_event_id(sub);
5499 if (read_format & PERF_FORMAT_LOST)
5500 values[n++] = atomic64_read(&sub->lost_samples);
5501 }
5502
5552unlock:
5553 raw_spin_unlock_irqrestore(&ctx->lock, flags);
5503 raw_spin_unlock_irqrestore(&ctx->lock, flags);
5554 return ret;
5504 return 0;
5555}
5556
5557static int perf_read_group(struct perf_event *event,
5558 u64 read_format, char __user *buf)
5559{
5560 struct perf_event *leader = event->group_leader, *child;
5561 struct perf_event_context *ctx = leader->ctx;
5562 int ret;
5563 u64 *values;
5564
5565 lockdep_assert_held(&ctx->mutex);
5566
5567 values = kzalloc(event->read_size, GFP_KERNEL);
5568 if (!values)
5569 return -ENOMEM;
5570
5571 values[0] = 1 + leader->nr_siblings;
5572
5505}
5506
5507static int perf_read_group(struct perf_event *event,
5508 u64 read_format, char __user *buf)
5509{
5510 struct perf_event *leader = event->group_leader, *child;
5511 struct perf_event_context *ctx = leader->ctx;
5512 int ret;
5513 u64 *values;
5514
5515 lockdep_assert_held(&ctx->mutex);
5516
5517 values = kzalloc(event->read_size, GFP_KERNEL);
5518 if (!values)
5519 return -ENOMEM;
5520
5521 values[0] = 1 + leader->nr_siblings;
5522
5523 /*
5524 * By locking the child_mutex of the leader we effectively
5525 * lock the child list of all siblings.. XXX explain how.
5526 */
5573 mutex_lock(&leader->child_mutex);
5574
5575 ret = __perf_read_group_add(leader, read_format, values);
5576 if (ret)
5577 goto unlock;
5578
5579 list_for_each_entry(child, &leader->child_list, child_list) {
5580 ret = __perf_read_group_add(child, read_format, values);

--- 1801 unchanged lines hidden (view full) ---

7382
7383 size = data->br_stack->nr
7384 * sizeof(struct perf_branch_entry);
7385
7386 perf_output_put(handle, data->br_stack->nr);
7387 if (branch_sample_hw_index(event))
7388 perf_output_put(handle, data->br_stack->hw_idx);
7389 perf_output_copy(handle, data->br_stack->entries, size);
5527 mutex_lock(&leader->child_mutex);
5528
5529 ret = __perf_read_group_add(leader, read_format, values);
5530 if (ret)
5531 goto unlock;
5532
5533 list_for_each_entry(child, &leader->child_list, child_list) {
5534 ret = __perf_read_group_add(child, read_format, values);

--- 1801 unchanged lines hidden (view full) ---

7336
7337 size = data->br_stack->nr
7338 * sizeof(struct perf_branch_entry);
7339
7340 perf_output_put(handle, data->br_stack->nr);
7341 if (branch_sample_hw_index(event))
7342 perf_output_put(handle, data->br_stack->hw_idx);
7343 perf_output_copy(handle, data->br_stack->entries, size);
7344 /*
7345 * Add the extension space which is appended
7346 * right after the struct perf_branch_stack.
7347 */
7348 if (data->br_stack_cntr) {
7349 size = data->br_stack->nr * sizeof(u64);
7350 perf_output_copy(handle, data->br_stack_cntr, size);
7351 }
7390 } else {
7391 /*
7392 * we always store at least the value of nr
7393 */
7394 u64 nr = 0;
7395 perf_output_put(handle, nr);
7396 }
7397 }

--- 5506 unchanged lines hidden (view full) ---

12904 }
12905}
12906
12907static void __perf_pmu_install_event(struct pmu *pmu,
12908 struct perf_event_context *ctx,
12909 int cpu, struct perf_event *event)
12910{
12911 struct perf_event_pmu_context *epc;
7352 } else {
7353 /*
7354 * we always store at least the value of nr
7355 */
7356 u64 nr = 0;
7357 perf_output_put(handle, nr);
7358 }
7359 }

--- 5506 unchanged lines hidden (view full) ---

12866 }
12867}
12868
12869static void __perf_pmu_install_event(struct pmu *pmu,
12870 struct perf_event_context *ctx,
12871 int cpu, struct perf_event *event)
12872{
12873 struct perf_event_pmu_context *epc;
12912 struct perf_event_context *old_ctx = event->ctx;
12913
12874
12914 get_ctx(ctx); /* normally find_get_context() */
12915
12916 event->cpu = cpu;
12917 epc = find_get_pmu_context(pmu, ctx, event);
12918 event->pmu_ctx = epc;
12919
12920 if (event->state >= PERF_EVENT_STATE_OFF)
12921 event->state = PERF_EVENT_STATE_INACTIVE;
12922 perf_install_in_context(ctx, event, cpu);
12875 event->cpu = cpu;
12876 epc = find_get_pmu_context(pmu, ctx, event);
12877 event->pmu_ctx = epc;
12878
12879 if (event->state >= PERF_EVENT_STATE_OFF)
12880 event->state = PERF_EVENT_STATE_INACTIVE;
12881 perf_install_in_context(ctx, event, cpu);
12923
12924 /*
12925 * Now that event->ctx is updated and visible, put the old ctx.
12926 */
12927 put_ctx(old_ctx);
12928}
12929
12930static void __perf_pmu_install(struct perf_event_context *ctx,
12931 int cpu, struct pmu *pmu, struct list_head *events)
12932{
12933 struct perf_event *event, *tmp;
12934
12935 /*

--- 22 unchanged lines hidden (view full) ---

12958 }
12959}
12960
12961void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
12962{
12963 struct perf_event_context *src_ctx, *dst_ctx;
12964 LIST_HEAD(events);
12965
12882}
12883
12884static void __perf_pmu_install(struct perf_event_context *ctx,
12885 int cpu, struct pmu *pmu, struct list_head *events)
12886{
12887 struct perf_event *event, *tmp;
12888
12889 /*

--- 22 unchanged lines hidden (view full) ---

12912 }
12913}
12914
12915void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
12916{
12917 struct perf_event_context *src_ctx, *dst_ctx;
12918 LIST_HEAD(events);
12919
12966 /*
12967 * Since per-cpu context is persistent, no need to grab an extra
12968 * reference.
12969 */
12970 src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;
12971 dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;
12972
12973 /*
12974 * See perf_event_ctx_lock() for comments on the details
12975 * of swizzling perf_event::ctx.
12976 */
12977 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);

--- 438 unchanged lines hidden (view full) ---

13416 child, leader, child_ctx);
13417 if (IS_ERR(child_ctr))
13418 return PTR_ERR(child_ctr);
13419
13420 if (sub->aux_event == parent_event && child_ctr &&
13421 !perf_get_aux_event(child_ctr, leader))
13422 return -EINVAL;
13423 }
12920 src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx;
12921 dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx;
12922
12923 /*
12924 * See perf_event_ctx_lock() for comments on the details
12925 * of swizzling perf_event::ctx.
12926 */
12927 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex);

--- 438 unchanged lines hidden (view full) ---

13366 child, leader, child_ctx);
13367 if (IS_ERR(child_ctr))
13368 return PTR_ERR(child_ctr);
13369
13370 if (sub->aux_event == parent_event && child_ctr &&
13371 !perf_get_aux_event(child_ctr, leader))
13372 return -EINVAL;
13373 }
13424 if (leader)
13425 leader->group_generation = parent_event->group_generation;
13426 return 0;
13427}
13428
13429/*
13430 * Creates the child task context and tries to inherit the event-group.
13431 *
13432 * Clears @inherited_all on !attr.inherited or error. Note that we'll leave
13433 * inherited_all set when we 'fail' to inherit an orphaned event; this is

--- 426 unchanged lines hidden --- 		13374 return 0;
13375}
13376
13377/*
13378 * Creates the child task context and tries to inherit the event-group.
13379 *
13380 * Clears @inherited_all on !attr.inherited or error. Note that we'll leave
13381 * inherited_all set when we 'fail' to inherit an orphaned event; this is

--- 426 unchanged lines hidden ---