1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013 Advanced Micro Devices, Inc.
4 *
5 * Author: Jacob Shin <jacob.shin@amd.com>
6 */
7
8 #include <linux/perf_event.h>
9 #include <linux/percpu.h>
10 #include <linux/types.h>
11 #include <linux/slab.h>
12 #include <linux/init.h>
13 #include <linux/cpu.h>
14 #include <linux/cpumask.h>
15 #include <linux/cpufeature.h>
16 #include <linux/smp.h>
17
18 #include <asm/perf_event.h>
19 #include <asm/cpuid/api.h>
20 #include <asm/msr.h>
21
22 #define NUM_COUNTERS_NB 4
23 #define NUM_COUNTERS_L2 4
24 #define NUM_COUNTERS_L3 6
25 #define NUM_COUNTERS_MAX 64
26
27 #define RDPMC_BASE_NB 6
28 #define RDPMC_BASE_LLC 10
29
30 #define COUNTER_SHIFT 16
31 #define UNCORE_NAME_LEN 16
32 #define UNCORE_GROUP_MAX 256
33
34 #undef pr_fmt
35 #define pr_fmt(fmt) "amd_uncore: " fmt
36
37 static int pmu_version;
38
39 struct amd_uncore_ctx {
40 int refcnt;
41 int cpu;
42 struct perf_event **events;
43 unsigned long active_mask[BITS_TO_LONGS(NUM_COUNTERS_MAX)];
44 int nr_active;
45 struct hrtimer hrtimer;
46 u64 hrtimer_duration;
47 };
48
49 struct amd_uncore_pmu {
50 char name[UNCORE_NAME_LEN];
51 int num_counters;
52 int rdpmc_base;
53 u32 msr_base;
54 int group;
55 cpumask_t active_mask;
56 struct pmu pmu;
57 struct amd_uncore_ctx * __percpu *ctx;
58 };
59
60 enum {
61 UNCORE_TYPE_DF,
62 UNCORE_TYPE_L3,
63 UNCORE_TYPE_UMC,
64
65 UNCORE_TYPE_MAX
66 };
67
68 union amd_uncore_info {
69 struct {
70 u64 aux_data:32; /* auxiliary data */
71 u64 num_pmcs:8; /* number of counters */
72 u64 gid:8; /* group id */
73 u64 cid:8; /* context id */
74 } split;
75 u64 full;
76 };
77
78 struct amd_uncore {
79 union amd_uncore_info __percpu *info;
80 struct amd_uncore_pmu *pmus;
81 unsigned int num_pmus;
82 bool init_done;
83 void (*scan)(struct amd_uncore *uncore, unsigned int cpu);
84 int (*init)(struct amd_uncore *uncore, unsigned int cpu);
85 void (*move)(struct amd_uncore *uncore, unsigned int cpu);
86 void (*free)(struct amd_uncore *uncore, unsigned int cpu);
87 };
88
89 static struct amd_uncore uncores[UNCORE_TYPE_MAX];
90
91 /* Interval for hrtimer, defaults to 60000 milliseconds */
92 static unsigned int update_interval = 60 * MSEC_PER_SEC;
93 module_param(update_interval, uint, 0444);
94
event_to_amd_uncore_pmu(struct perf_event * event)95 static struct amd_uncore_pmu *event_to_amd_uncore_pmu(struct perf_event *event)
96 {
97 return container_of(event->pmu, struct amd_uncore_pmu, pmu);
98 }
99
amd_uncore_hrtimer(struct hrtimer * hrtimer)100 static enum hrtimer_restart amd_uncore_hrtimer(struct hrtimer *hrtimer)
101 {
102 struct amd_uncore_ctx *ctx;
103 struct perf_event *event;
104 int bit;
105
106 ctx = container_of(hrtimer, struct amd_uncore_ctx, hrtimer);
107
108 if (!ctx->nr_active || ctx->cpu != smp_processor_id())
109 return HRTIMER_NORESTART;
110
111 for_each_set_bit(bit, ctx->active_mask, NUM_COUNTERS_MAX) {
112 event = ctx->events[bit];
113 event->pmu->read(event);
114 }
115
116 hrtimer_forward_now(hrtimer, ns_to_ktime(ctx->hrtimer_duration));
117 return HRTIMER_RESTART;
118 }
119
amd_uncore_start_hrtimer(struct amd_uncore_ctx * ctx)120 static void amd_uncore_start_hrtimer(struct amd_uncore_ctx *ctx)
121 {
122 hrtimer_start(&ctx->hrtimer, ns_to_ktime(ctx->hrtimer_duration),
123 HRTIMER_MODE_REL_PINNED_HARD);
124 }
125
amd_uncore_cancel_hrtimer(struct amd_uncore_ctx * ctx)126 static void amd_uncore_cancel_hrtimer(struct amd_uncore_ctx *ctx)
127 {
128 hrtimer_cancel(&ctx->hrtimer);
129 }
130
amd_uncore_init_hrtimer(struct amd_uncore_ctx * ctx)131 static void amd_uncore_init_hrtimer(struct amd_uncore_ctx *ctx)
132 {
133 hrtimer_setup(&ctx->hrtimer, amd_uncore_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
134 }
135
amd_uncore_read(struct perf_event * event)136 static void amd_uncore_read(struct perf_event *event)
137 {
138 struct hw_perf_event *hwc = &event->hw;
139 u64 prev, new;
140 s64 delta;
141
142 /*
143 * since we do not enable counter overflow interrupts,
144 * we do not have to worry about prev_count changing on us
145 */
146
147 prev = local64_read(&hwc->prev_count);
148
149 /*
150 * Some uncore PMUs do not have RDPMC assignments. In such cases,
151 * read counts directly from the corresponding PERF_CTR.
152 */
153 if (hwc->event_base_rdpmc < 0)
154 rdmsrq(hwc->event_base, new);
155 else
156 new = rdpmc(hwc->event_base_rdpmc);
157
158 local64_set(&hwc->prev_count, new);
159 delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
160 delta >>= COUNTER_SHIFT;
161 local64_add(delta, &event->count);
162 }
163
amd_uncore_start(struct perf_event * event,int flags)164 static void amd_uncore_start(struct perf_event *event, int flags)
165 {
166 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
167 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
168 struct hw_perf_event *hwc = &event->hw;
169
170 if (!ctx->nr_active++)
171 amd_uncore_start_hrtimer(ctx);
172
173 if (flags & PERF_EF_RELOAD)
174 wrmsrq(hwc->event_base, (u64)local64_read(&hwc->prev_count));
175
176 hwc->state = 0;
177 __set_bit(hwc->idx, ctx->active_mask);
178 wrmsrq(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
179 perf_event_update_userpage(event);
180 }
181
amd_uncore_stop(struct perf_event * event,int flags)182 static void amd_uncore_stop(struct perf_event *event, int flags)
183 {
184 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
185 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
186 struct hw_perf_event *hwc = &event->hw;
187
188 wrmsrq(hwc->config_base, hwc->config);
189 hwc->state |= PERF_HES_STOPPED;
190
191 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
192 event->pmu->read(event);
193 hwc->state |= PERF_HES_UPTODATE;
194 }
195
196 if (!--ctx->nr_active)
197 amd_uncore_cancel_hrtimer(ctx);
198
199 __clear_bit(hwc->idx, ctx->active_mask);
200 }
201
amd_uncore_add(struct perf_event * event,int flags)202 static int amd_uncore_add(struct perf_event *event, int flags)
203 {
204 int i;
205 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
206 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
207 struct hw_perf_event *hwc = &event->hw;
208
209 /* are we already assigned? */
210 if (hwc->idx != -1 && ctx->events[hwc->idx] == event)
211 goto out;
212
213 for (i = 0; i < pmu->num_counters; i++) {
214 if (ctx->events[i] == event) {
215 hwc->idx = i;
216 goto out;
217 }
218 }
219
220 /* if not, take the first available counter */
221 hwc->idx = -1;
222 for (i = 0; i < pmu->num_counters; i++) {
223 struct perf_event *tmp = NULL;
224
225 if (try_cmpxchg(&ctx->events[i], &tmp, event)) {
226 hwc->idx = i;
227 break;
228 }
229 }
230
231 out:
232 if (hwc->idx == -1)
233 return -EBUSY;
234
235 hwc->config_base = pmu->msr_base + (2 * hwc->idx);
236 hwc->event_base = pmu->msr_base + 1 + (2 * hwc->idx);
237 hwc->event_base_rdpmc = pmu->rdpmc_base + hwc->idx;
238 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
239
240 if (pmu->rdpmc_base < 0)
241 hwc->event_base_rdpmc = -1;
242
243 if (flags & PERF_EF_START)
244 event->pmu->start(event, PERF_EF_RELOAD);
245
246 return 0;
247 }
248
amd_uncore_del(struct perf_event * event,int flags)249 static void amd_uncore_del(struct perf_event *event, int flags)
250 {
251 int i;
252 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
253 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
254 struct hw_perf_event *hwc = &event->hw;
255
256 event->pmu->stop(event, PERF_EF_UPDATE);
257
258 for (i = 0; i < pmu->num_counters; i++) {
259 struct perf_event *tmp = event;
260
261 if (try_cmpxchg(&ctx->events[i], &tmp, NULL))
262 break;
263 }
264
265 hwc->idx = -1;
266 }
267
amd_uncore_group_valid(struct perf_event * event)268 static bool amd_uncore_group_valid(struct perf_event *event)
269 {
270 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
271 struct perf_event *leader = event->group_leader;
272 struct perf_event *sibling;
273 int counters = 0;
274
275 if (leader->pmu == event->pmu)
276 counters++;
277
278 for_each_sibling_event(sibling, leader) {
279 if (sibling->pmu == event->pmu &&
280 sibling->state > PERF_EVENT_STATE_OFF)
281 counters++;
282 }
283
284 /*
285 * When pmu->event_init() is called, the event is yet to be linked to
286 * its leader's sibling list, so it is counted separately
287 */
288 return (counters + 1) <= pmu->num_counters;
289 }
290
amd_uncore_event_init(struct perf_event * event)291 static int amd_uncore_event_init(struct perf_event *event)
292 {
293 struct amd_uncore_pmu *pmu;
294 struct amd_uncore_ctx *ctx;
295 struct hw_perf_event *hwc = &event->hw;
296
297 if (event->attr.type != event->pmu->type)
298 return -ENOENT;
299
300 if (event->cpu < 0)
301 return -EINVAL;
302
303 pmu = event_to_amd_uncore_pmu(event);
304 ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
305 if (!ctx)
306 return -ENODEV;
307
308 /*
309 * Ensure that all events in a group can be scheduled together so that
310 * a failure can be reported at perf_event_open() time rather than
311 * silently at pmu->add() time when no free counter is found
312 */
313 if (event->group_leader != event && !amd_uncore_group_valid(event))
314 return -EINVAL;
315
316 /*
317 * NB and Last level cache counters (MSRs) are shared across all cores
318 * that share the same NB / Last level cache. On family 16h and below,
319 * Interrupts can be directed to a single target core, however, event
320 * counts generated by processes running on other cores cannot be masked
321 * out. So we do not support sampling and per-thread events via
322 * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts:
323 */
324 hwc->config = event->attr.config;
325 hwc->idx = -1;
326
327 /*
328 * since request can come in to any of the shared cores, we will remap
329 * to a single common cpu.
330 */
331 event->cpu = ctx->cpu;
332
333 return 0;
334 }
335
336 static umode_t
amd_f17h_uncore_is_visible(struct kobject * kobj,struct attribute * attr,int i)337 amd_f17h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i)
338 {
339 return boot_cpu_data.x86 >= 0x17 && boot_cpu_data.x86 < 0x19 ?
340 attr->mode : 0;
341 }
342
343 static umode_t
amd_f19h_uncore_is_visible(struct kobject * kobj,struct attribute * attr,int i)344 amd_f19h_uncore_is_visible(struct kobject *kobj, struct attribute *attr, int i)
345 {
346 return boot_cpu_data.x86 >= 0x19 ? attr->mode : 0;
347 }
348
amd_uncore_attr_show_cpumask(struct device * dev,struct device_attribute * attr,char * buf)349 static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
350 struct device_attribute *attr,
351 char *buf)
352 {
353 struct pmu *ptr = dev_get_drvdata(dev);
354 struct amd_uncore_pmu *pmu = container_of(ptr, struct amd_uncore_pmu, pmu);
355
356 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&pmu->active_mask));
357 }
358 static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
359
360 static struct attribute *amd_uncore_attrs[] = {
361 &dev_attr_cpumask.attr,
362 NULL,
363 };
364
365 static struct attribute_group amd_uncore_attr_group = {
366 .attrs = amd_uncore_attrs,
367 };
368
369 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \
370 static ssize_t __uncore_##_var##_show(struct device *dev, \
371 struct device_attribute *attr, \
372 char *page) \
373 { \
374 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \
375 return sprintf(page, _format "\n"); \
376 } \
377 static struct device_attribute format_attr_##_var = \
378 __ATTR(_name, 0444, __uncore_##_var##_show, NULL)
379
380 DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35");
381 DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */
382 DEFINE_UNCORE_FORMAT_ATTR(event14v2, event, "config:0-7,32-37"); /* PerfMonV2 DF */
383 DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3, PerfMonV2 UMC */
384 DEFINE_UNCORE_FORMAT_ATTR(umask8, umask, "config:8-15");
385 DEFINE_UNCORE_FORMAT_ATTR(umask12, umask, "config:8-15,24-27"); /* PerfMonV2 DF */
386 DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */
387 DEFINE_UNCORE_FORMAT_ATTR(slicemask, slicemask, "config:48-51"); /* F17h L3 */
388 DEFINE_UNCORE_FORMAT_ATTR(threadmask8, threadmask, "config:56-63"); /* F17h L3 */
389 DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L3 */
390 DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */
391 DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */
392 DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */
393 DEFINE_UNCORE_FORMAT_ATTR(rdwrmask, rdwrmask, "config:8-9"); /* PerfMonV2 UMC */
394
395 /* Common DF and NB attributes */
396 static struct attribute *amd_uncore_df_format_attr[] = {
397 &format_attr_event12.attr, /* event */
398 &format_attr_umask8.attr, /* umask */
399 NULL,
400 };
401
402 /* Common L2 and L3 attributes */
403 static struct attribute *amd_uncore_l3_format_attr[] = {
404 &format_attr_event12.attr, /* event */
405 &format_attr_umask8.attr, /* umask */
406 NULL, /* threadmask */
407 NULL,
408 };
409
410 /* Common UMC attributes */
411 static struct attribute *amd_uncore_umc_format_attr[] = {
412 &format_attr_event8.attr, /* event */
413 &format_attr_rdwrmask.attr, /* rdwrmask */
414 NULL,
415 };
416
417 /* F17h unique L3 attributes */
418 static struct attribute *amd_f17h_uncore_l3_format_attr[] = {
419 &format_attr_slicemask.attr, /* slicemask */
420 NULL,
421 };
422
423 /* F19h unique L3 attributes */
424 static struct attribute *amd_f19h_uncore_l3_format_attr[] = {
425 &format_attr_coreid.attr, /* coreid */
426 &format_attr_enallslices.attr, /* enallslices */
427 &format_attr_enallcores.attr, /* enallcores */
428 &format_attr_sliceid.attr, /* sliceid */
429 NULL,
430 };
431
432 static struct attribute_group amd_uncore_df_format_group = {
433 .name = "format",
434 .attrs = amd_uncore_df_format_attr,
435 };
436
437 static struct attribute_group amd_uncore_l3_format_group = {
438 .name = "format",
439 .attrs = amd_uncore_l3_format_attr,
440 };
441
442 static struct attribute_group amd_f17h_uncore_l3_format_group = {
443 .name = "format",
444 .attrs = amd_f17h_uncore_l3_format_attr,
445 .is_visible = amd_f17h_uncore_is_visible,
446 };
447
448 static struct attribute_group amd_f19h_uncore_l3_format_group = {
449 .name = "format",
450 .attrs = amd_f19h_uncore_l3_format_attr,
451 .is_visible = amd_f19h_uncore_is_visible,
452 };
453
454 static struct attribute_group amd_uncore_umc_format_group = {
455 .name = "format",
456 .attrs = amd_uncore_umc_format_attr,
457 };
458
459 static const struct attribute_group *amd_uncore_df_attr_groups[] = {
460 &amd_uncore_attr_group,
461 &amd_uncore_df_format_group,
462 NULL,
463 };
464
465 static const struct attribute_group *amd_uncore_l3_attr_groups[] = {
466 &amd_uncore_attr_group,
467 &amd_uncore_l3_format_group,
468 NULL,
469 };
470
471 static const struct attribute_group *amd_uncore_l3_attr_update[] = {
472 &amd_f17h_uncore_l3_format_group,
473 &amd_f19h_uncore_l3_format_group,
474 NULL,
475 };
476
477 static const struct attribute_group *amd_uncore_umc_attr_groups[] = {
478 &amd_uncore_attr_group,
479 &amd_uncore_umc_format_group,
480 NULL,
481 };
482
483 static __always_inline
amd_uncore_ctx_cid(struct amd_uncore * uncore,unsigned int cpu)484 int amd_uncore_ctx_cid(struct amd_uncore *uncore, unsigned int cpu)
485 {
486 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
487 return info->split.cid;
488 }
489
490 static __always_inline
amd_uncore_ctx_gid(struct amd_uncore * uncore,unsigned int cpu)491 int amd_uncore_ctx_gid(struct amd_uncore *uncore, unsigned int cpu)
492 {
493 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
494 return info->split.gid;
495 }
496
497 static __always_inline
amd_uncore_ctx_num_pmcs(struct amd_uncore * uncore,unsigned int cpu)498 int amd_uncore_ctx_num_pmcs(struct amd_uncore *uncore, unsigned int cpu)
499 {
500 union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
501 return info->split.num_pmcs;
502 }
503
amd_uncore_ctx_free(struct amd_uncore * uncore,unsigned int cpu)504 static void amd_uncore_ctx_free(struct amd_uncore *uncore, unsigned int cpu)
505 {
506 struct amd_uncore_pmu *pmu;
507 struct amd_uncore_ctx *ctx;
508 int i;
509
510 if (!uncore->init_done)
511 return;
512
513 for (i = 0; i < uncore->num_pmus; i++) {
514 pmu = &uncore->pmus[i];
515 ctx = *per_cpu_ptr(pmu->ctx, cpu);
516 if (!ctx)
517 continue;
518
519 if (cpu == ctx->cpu)
520 cpumask_clear_cpu(cpu, &pmu->active_mask);
521
522 if (!--ctx->refcnt) {
523 kfree(ctx->events);
524 kfree(ctx);
525 }
526
527 *per_cpu_ptr(pmu->ctx, cpu) = NULL;
528 }
529 }
530
amd_uncore_ctx_init(struct amd_uncore * uncore,unsigned int cpu)531 static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
532 {
533 struct amd_uncore_ctx *curr, *prev;
534 struct amd_uncore_pmu *pmu;
535 int node, cid, gid, i, j;
536
537 if (!uncore->init_done || !uncore->num_pmus)
538 return 0;
539
540 cid = amd_uncore_ctx_cid(uncore, cpu);
541 gid = amd_uncore_ctx_gid(uncore, cpu);
542
543 for (i = 0; i < uncore->num_pmus; i++) {
544 pmu = &uncore->pmus[i];
545 *per_cpu_ptr(pmu->ctx, cpu) = NULL;
546 curr = NULL;
547
548 /* Check for group exclusivity */
549 if (gid != pmu->group)
550 continue;
551
552 /* Find a sibling context */
553 for_each_online_cpu(j) {
554 if (cpu == j)
555 continue;
556
557 prev = *per_cpu_ptr(pmu->ctx, j);
558 if (!prev)
559 continue;
560
561 if (cid == amd_uncore_ctx_cid(uncore, j)) {
562 curr = prev;
563 break;
564 }
565 }
566
567 /* Allocate context if sibling does not exist */
568 if (!curr) {
569 node = cpu_to_node(cpu);
570 curr = kzalloc_node(sizeof(*curr), GFP_KERNEL, node);
571 if (!curr)
572 goto fail;
573
574 curr->cpu = cpu;
575 curr->events = kzalloc_node(sizeof(*curr->events) *
576 pmu->num_counters,
577 GFP_KERNEL, node);
578 if (!curr->events) {
579 kfree(curr);
580 goto fail;
581 }
582
583 amd_uncore_init_hrtimer(curr);
584 curr->hrtimer_duration = (u64)update_interval * NSEC_PER_MSEC;
585
586 cpumask_set_cpu(cpu, &pmu->active_mask);
587 }
588
589 curr->refcnt++;
590 *per_cpu_ptr(pmu->ctx, cpu) = curr;
591 }
592
593 return 0;
594
595 fail:
596 amd_uncore_ctx_free(uncore, cpu);
597
598 return -ENOMEM;
599 }
600
amd_uncore_ctx_move(struct amd_uncore * uncore,unsigned int cpu)601 static void amd_uncore_ctx_move(struct amd_uncore *uncore, unsigned int cpu)
602 {
603 struct amd_uncore_ctx *curr, *next;
604 struct amd_uncore_pmu *pmu;
605 int i, j;
606
607 if (!uncore->init_done)
608 return;
609
610 for (i = 0; i < uncore->num_pmus; i++) {
611 pmu = &uncore->pmus[i];
612 curr = *per_cpu_ptr(pmu->ctx, cpu);
613 if (!curr)
614 continue;
615
616 /* Migrate to a shared sibling if possible */
617 for_each_online_cpu(j) {
618 next = *per_cpu_ptr(pmu->ctx, j);
619 if (!next || cpu == j)
620 continue;
621
622 if (curr == next) {
623 perf_pmu_migrate_context(&pmu->pmu, cpu, j);
624 cpumask_clear_cpu(cpu, &pmu->active_mask);
625 cpumask_set_cpu(j, &pmu->active_mask);
626 next->cpu = j;
627 break;
628 }
629 }
630 }
631 }
632
amd_uncore_cpu_starting(unsigned int cpu)633 static int amd_uncore_cpu_starting(unsigned int cpu)
634 {
635 struct amd_uncore *uncore;
636 int i;
637
638 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
639 uncore = &uncores[i];
640 uncore->scan(uncore, cpu);
641 }
642
643 return 0;
644 }
645
amd_uncore_cpu_online(unsigned int cpu)646 static int amd_uncore_cpu_online(unsigned int cpu)
647 {
648 struct amd_uncore *uncore;
649 int i;
650
651 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
652 uncore = &uncores[i];
653 if (uncore->init(uncore, cpu))
654 break;
655 }
656
657 return 0;
658 }
659
amd_uncore_cpu_down_prepare(unsigned int cpu)660 static int amd_uncore_cpu_down_prepare(unsigned int cpu)
661 {
662 struct amd_uncore *uncore;
663 int i;
664
665 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
666 uncore = &uncores[i];
667 uncore->move(uncore, cpu);
668 }
669
670 return 0;
671 }
672
amd_uncore_cpu_dead(unsigned int cpu)673 static int amd_uncore_cpu_dead(unsigned int cpu)
674 {
675 struct amd_uncore *uncore;
676 int i;
677
678 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
679 uncore = &uncores[i];
680 uncore->free(uncore, cpu);
681 }
682
683 return 0;
684 }
685
amd_uncore_df_event_init(struct perf_event * event)686 static int amd_uncore_df_event_init(struct perf_event *event)
687 {
688 struct hw_perf_event *hwc = &event->hw;
689 int ret = amd_uncore_event_init(event);
690
691 hwc->config = event->attr.config &
692 (pmu_version >= 2 ? AMD64_PERFMON_V2_RAW_EVENT_MASK_NB :
693 AMD64_RAW_EVENT_MASK_NB);
694
695 return ret;
696 }
697
amd_uncore_df_add(struct perf_event * event,int flags)698 static int amd_uncore_df_add(struct perf_event *event, int flags)
699 {
700 int ret = amd_uncore_add(event, flags & ~PERF_EF_START);
701 struct hw_perf_event *hwc = &event->hw;
702
703 if (ret)
704 return ret;
705
706 /*
707 * The first four DF counters are accessible via RDPMC index 6 to 9
708 * followed by the L3 counters from index 10 to 15. For processors
709 * with more than four DF counters, the DF RDPMC assignments become
710 * discontiguous as the additional counters are accessible starting
711 * from index 16.
712 */
713 if (hwc->idx >= NUM_COUNTERS_NB)
714 hwc->event_base_rdpmc += NUM_COUNTERS_L3;
715
716 /* Delayed start after rdpmc base update */
717 if (flags & PERF_EF_START)
718 amd_uncore_start(event, PERF_EF_RELOAD);
719
720 return 0;
721 }
722
723 static
amd_uncore_df_ctx_scan(struct amd_uncore * uncore,unsigned int cpu)724 void amd_uncore_df_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
725 {
726 union cpuid_0x80000022_ebx ebx;
727 union amd_uncore_info info;
728
729 if (!boot_cpu_has(X86_FEATURE_PERFCTR_NB))
730 return;
731
732 info.split.aux_data = 0;
733 info.split.num_pmcs = NUM_COUNTERS_NB;
734 info.split.gid = 0;
735 info.split.cid = topology_amd_node_id(cpu);
736
737 if (pmu_version >= 2) {
738 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
739 info.split.num_pmcs = ebx.split.num_df_pmc;
740 }
741
742 *per_cpu_ptr(uncore->info, cpu) = info;
743 }
744
745 static
amd_uncore_df_ctx_init(struct amd_uncore * uncore,unsigned int cpu)746 int amd_uncore_df_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
747 {
748 struct attribute **df_attr = amd_uncore_df_format_attr;
749 struct amd_uncore_pmu *pmu;
750 int num_counters;
751
752 /* Run just once */
753 if (uncore->init_done)
754 return amd_uncore_ctx_init(uncore, cpu);
755
756 num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);
757 if (!num_counters)
758 goto done;
759
760 /* No grouping, single instance for a system */
761 uncore->pmus = kzalloc_obj(*uncore->pmus);
762 if (!uncore->pmus)
763 goto done;
764
765 /*
766 * For Family 17h and above, the Northbridge counters are repurposed
767 * as Data Fabric counters. The PMUs are exported based on family as
768 * either NB or DF.
769 */
770 pmu = &uncore->pmus[0];
771 strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_df" : "amd_nb",
772 sizeof(pmu->name));
773 pmu->num_counters = num_counters;
774 pmu->msr_base = MSR_F15H_NB_PERF_CTL;
775 pmu->rdpmc_base = RDPMC_BASE_NB;
776 pmu->group = amd_uncore_ctx_gid(uncore, cpu);
777
778 if (pmu_version >= 2) {
779 *df_attr++ = &format_attr_event14v2.attr;
780 *df_attr++ = &format_attr_umask12.attr;
781 } else if (boot_cpu_data.x86 >= 0x17) {
782 *df_attr = &format_attr_event14.attr;
783 }
784
785 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
786 if (!pmu->ctx)
787 goto done;
788
789 pmu->pmu = (struct pmu) {
790 .task_ctx_nr = perf_invalid_context,
791 .attr_groups = amd_uncore_df_attr_groups,
792 .name = pmu->name,
793 .event_init = amd_uncore_df_event_init,
794 .add = amd_uncore_df_add,
795 .del = amd_uncore_del,
796 .start = amd_uncore_start,
797 .stop = amd_uncore_stop,
798 .read = amd_uncore_read,
799 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
800 .module = THIS_MODULE,
801 };
802
803 if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) {
804 free_percpu(pmu->ctx);
805 pmu->ctx = NULL;
806 goto done;
807 }
808
809 pr_info("%d %s%s counters detected\n", pmu->num_counters,
810 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "",
811 pmu->pmu.name);
812
813 uncore->num_pmus = 1;
814
815 done:
816 uncore->init_done = true;
817
818 return amd_uncore_ctx_init(uncore, cpu);
819 }
820
amd_uncore_l3_event_init(struct perf_event * event)821 static int amd_uncore_l3_event_init(struct perf_event *event)
822 {
823 int ret = amd_uncore_event_init(event);
824 struct hw_perf_event *hwc = &event->hw;
825 u64 config = event->attr.config;
826 u64 mask;
827
828 hwc->config = config & AMD64_RAW_EVENT_MASK_NB;
829
830 /*
831 * SliceMask and ThreadMask need to be set for certain L3 events.
832 * For other events, the two fields do not affect the count.
833 */
834 if (ret || boot_cpu_data.x86 < 0x17)
835 return ret;
836
837 mask = config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK |
838 AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES |
839 AMD64_L3_COREID_MASK);
840
841 if (boot_cpu_data.x86 <= 0x18)
842 mask = ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) |
843 ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK);
844
845 /*
846 * If the user doesn't specify a ThreadMask, they're not trying to
847 * count core 0, so we enable all cores & threads.
848 * We'll also assume that they want to count slice 0 if they specify
849 * a ThreadMask and leave SliceId and EnAllSlices unpopulated.
850 */
851 else if (!(config & AMD64_L3_F19H_THREAD_MASK))
852 mask = AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES |
853 AMD64_L3_EN_ALL_CORES;
854
855 hwc->config |= mask;
856
857 return 0;
858 }
859
860 static
amd_uncore_l3_ctx_scan(struct amd_uncore * uncore,unsigned int cpu)861 void amd_uncore_l3_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
862 {
863 union amd_uncore_info info;
864
865 if (!boot_cpu_has(X86_FEATURE_PERFCTR_LLC))
866 return;
867
868 info.split.aux_data = 0;
869 info.split.num_pmcs = NUM_COUNTERS_L2;
870 info.split.gid = 0;
871 info.split.cid = per_cpu_llc_id(cpu);
872
873 if (boot_cpu_data.x86 >= 0x17)
874 info.split.num_pmcs = NUM_COUNTERS_L3;
875
876 *per_cpu_ptr(uncore->info, cpu) = info;
877 }
878
879 static
amd_uncore_l3_ctx_init(struct amd_uncore * uncore,unsigned int cpu)880 int amd_uncore_l3_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
881 {
882 struct attribute **l3_attr = amd_uncore_l3_format_attr;
883 struct amd_uncore_pmu *pmu;
884 int num_counters;
885
886 /* Run just once */
887 if (uncore->init_done)
888 return amd_uncore_ctx_init(uncore, cpu);
889
890 num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);
891 if (!num_counters)
892 goto done;
893
894 /* No grouping, single instance for a system */
895 uncore->pmus = kzalloc_obj(*uncore->pmus);
896 if (!uncore->pmus)
897 goto done;
898
899 /*
900 * For Family 17h and above, L3 cache counters are available instead
901 * of L2 cache counters. The PMUs are exported based on family as
902 * either L2 or L3.
903 */
904 pmu = &uncore->pmus[0];
905 strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_l3" : "amd_l2",
906 sizeof(pmu->name));
907 pmu->num_counters = num_counters;
908 pmu->msr_base = MSR_F16H_L2I_PERF_CTL;
909 pmu->rdpmc_base = RDPMC_BASE_LLC;
910 pmu->group = amd_uncore_ctx_gid(uncore, cpu);
911
912 if (boot_cpu_data.x86 >= 0x17) {
913 *l3_attr++ = &format_attr_event8.attr;
914 *l3_attr++ = &format_attr_umask8.attr;
915 *l3_attr++ = boot_cpu_data.x86 >= 0x19 ?
916 &format_attr_threadmask2.attr :
917 &format_attr_threadmask8.attr;
918 }
919
920 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
921 if (!pmu->ctx)
922 goto done;
923
924 pmu->pmu = (struct pmu) {
925 .task_ctx_nr = perf_invalid_context,
926 .attr_groups = amd_uncore_l3_attr_groups,
927 .attr_update = amd_uncore_l3_attr_update,
928 .name = pmu->name,
929 .event_init = amd_uncore_l3_event_init,
930 .add = amd_uncore_add,
931 .del = amd_uncore_del,
932 .start = amd_uncore_start,
933 .stop = amd_uncore_stop,
934 .read = amd_uncore_read,
935 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
936 .module = THIS_MODULE,
937 };
938
939 if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) {
940 free_percpu(pmu->ctx);
941 pmu->ctx = NULL;
942 goto done;
943 }
944
945 pr_info("%d %s%s counters detected\n", pmu->num_counters,
946 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "",
947 pmu->pmu.name);
948
949 uncore->num_pmus = 1;
950
951 done:
952 uncore->init_done = true;
953
954 return amd_uncore_ctx_init(uncore, cpu);
955 }
956
amd_uncore_umc_event_init(struct perf_event * event)957 static int amd_uncore_umc_event_init(struct perf_event *event)
958 {
959 struct hw_perf_event *hwc = &event->hw;
960 int ret = amd_uncore_event_init(event);
961
962 if (ret)
963 return ret;
964
965 hwc->config = event->attr.config & AMD64_PERFMON_V2_RAW_EVENT_MASK_UMC;
966
967 return 0;
968 }
969
amd_uncore_umc_start(struct perf_event * event,int flags)970 static void amd_uncore_umc_start(struct perf_event *event, int flags)
971 {
972 struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
973 struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
974 struct hw_perf_event *hwc = &event->hw;
975
976 if (!ctx->nr_active++)
977 amd_uncore_start_hrtimer(ctx);
978
979 if (flags & PERF_EF_RELOAD)
980 wrmsrq(hwc->event_base, (u64)local64_read(&hwc->prev_count));
981
982 hwc->state = 0;
983 __set_bit(hwc->idx, ctx->active_mask);
984 wrmsrq(hwc->config_base, (hwc->config | AMD64_PERFMON_V2_ENABLE_UMC));
985 perf_event_update_userpage(event);
986 }
987
amd_uncore_umc_read(struct perf_event * event)988 static void amd_uncore_umc_read(struct perf_event *event)
989 {
990 struct hw_perf_event *hwc = &event->hw;
991 u64 prev, new, shift;
992 s64 delta;
993
994 shift = COUNTER_SHIFT + 1;
995 prev = local64_read(&hwc->prev_count);
996
997 /*
998 * UMC counters do not have RDPMC assignments. Read counts directly
999 * from the corresponding PERF_CTR.
1000 */
1001 rdmsrq(hwc->event_base, new);
1002
1003 /*
1004 * Unlike the other uncore counters, UMC counters saturate and set the
1005 * Overflow bit (bit 48) on overflow. Since they do not roll over,
1006 * proactively reset the corresponding PERF_CTR when bit 47 is set so
1007 * that the counter never gets a chance to saturate.
1008 */
1009 if (new & BIT_ULL(63 - COUNTER_SHIFT)) {
1010 wrmsrq(hwc->event_base, 0);
1011 local64_set(&hwc->prev_count, 0);
1012 } else {
1013 local64_set(&hwc->prev_count, new);
1014 }
1015
1016 delta = (new << shift) - (prev << shift);
1017 delta >>= shift;
1018 local64_add(delta, &event->count);
1019 }
1020
1021 static
amd_uncore_umc_ctx_scan(struct amd_uncore * uncore,unsigned int cpu)1022 void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
1023 {
1024 union cpuid_0x80000022_ebx ebx;
1025 union amd_uncore_info info;
1026 unsigned int eax, ecx, edx;
1027
1028 if (pmu_version < 2)
1029 return;
1030
1031 cpuid(EXT_PERFMON_DEBUG_FEATURES, &eax, &ebx.full, &ecx, &edx);
1032 info.split.aux_data = ecx; /* stash active mask */
1033 info.split.num_pmcs = ebx.split.num_umc_pmc;
1034 info.split.gid = topology_amd_node_id(cpu);
1035 info.split.cid = topology_amd_node_id(cpu);
1036 *per_cpu_ptr(uncore->info, cpu) = info;
1037 }
1038
1039 static
amd_uncore_umc_ctx_init(struct amd_uncore * uncore,unsigned int cpu)1040 int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
1041 {
1042 DECLARE_BITMAP(gmask, UNCORE_GROUP_MAX) = { 0 };
1043 u8 group_num_pmus[UNCORE_GROUP_MAX] = { 0 };
1044 u8 group_num_pmcs[UNCORE_GROUP_MAX] = { 0 };
1045 union amd_uncore_info info;
1046 struct amd_uncore_pmu *pmu;
1047 int gid, i;
1048 u16 index = 0;
1049
1050 if (pmu_version < 2)
1051 return 0;
1052
1053 /* Run just once */
1054 if (uncore->init_done)
1055 return amd_uncore_ctx_init(uncore, cpu);
1056
1057 /* Find unique groups */
1058 for_each_online_cpu(i) {
1059 info = *per_cpu_ptr(uncore->info, i);
1060 gid = info.split.gid;
1061 if (test_bit(gid, gmask))
1062 continue;
1063
1064 __set_bit(gid, gmask);
1065 group_num_pmus[gid] = hweight32(info.split.aux_data);
1066 group_num_pmcs[gid] = info.split.num_pmcs;
1067 uncore->num_pmus += group_num_pmus[gid];
1068 }
1069
1070 uncore->pmus = kzalloc(sizeof(*uncore->pmus) * uncore->num_pmus,
1071 GFP_KERNEL);
1072 if (!uncore->pmus) {
1073 uncore->num_pmus = 0;
1074 goto done;
1075 }
1076
1077 for_each_set_bit(gid, gmask, UNCORE_GROUP_MAX) {
1078 for (i = 0; i < group_num_pmus[gid]; i++) {
1079 pmu = &uncore->pmus[index];
1080 snprintf(pmu->name, sizeof(pmu->name), "amd_umc_%hu", index);
1081 pmu->num_counters = group_num_pmcs[gid] / group_num_pmus[gid];
1082 pmu->msr_base = MSR_F19H_UMC_PERF_CTL + i * pmu->num_counters * 2;
1083 pmu->rdpmc_base = -1;
1084 pmu->group = gid;
1085
1086 pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
1087 if (!pmu->ctx)
1088 goto done;
1089
1090 pmu->pmu = (struct pmu) {
1091 .task_ctx_nr = perf_invalid_context,
1092 .attr_groups = amd_uncore_umc_attr_groups,
1093 .name = pmu->name,
1094 .event_init = amd_uncore_umc_event_init,
1095 .add = amd_uncore_add,
1096 .del = amd_uncore_del,
1097 .start = amd_uncore_umc_start,
1098 .stop = amd_uncore_stop,
1099 .read = amd_uncore_umc_read,
1100 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
1101 .module = THIS_MODULE,
1102 };
1103
1104 if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) {
1105 free_percpu(pmu->ctx);
1106 pmu->ctx = NULL;
1107 goto done;
1108 }
1109
1110 pr_info("%d %s counters detected\n", pmu->num_counters,
1111 pmu->pmu.name);
1112
1113 index++;
1114 }
1115 }
1116
1117 done:
1118 uncore->num_pmus = index;
1119 uncore->init_done = true;
1120
1121 return amd_uncore_ctx_init(uncore, cpu);
1122 }
1123
1124 static struct amd_uncore uncores[UNCORE_TYPE_MAX] = {
1125 /* UNCORE_TYPE_DF */
1126 {
1127 .scan = amd_uncore_df_ctx_scan,
1128 .init = amd_uncore_df_ctx_init,
1129 .move = amd_uncore_ctx_move,
1130 .free = amd_uncore_ctx_free,
1131 },
1132 /* UNCORE_TYPE_L3 */
1133 {
1134 .scan = amd_uncore_l3_ctx_scan,
1135 .init = amd_uncore_l3_ctx_init,
1136 .move = amd_uncore_ctx_move,
1137 .free = amd_uncore_ctx_free,
1138 },
1139 /* UNCORE_TYPE_UMC */
1140 {
1141 .scan = amd_uncore_umc_ctx_scan,
1142 .init = amd_uncore_umc_ctx_init,
1143 .move = amd_uncore_ctx_move,
1144 .free = amd_uncore_ctx_free,
1145 },
1146 };
1147
amd_uncore_init(void)1148 static int __init amd_uncore_init(void)
1149 {
1150 struct amd_uncore *uncore;
1151 int ret = -ENODEV;
1152 int i;
1153
1154 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
1155 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
1156 return -ENODEV;
1157
1158 if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
1159 return -ENODEV;
1160
1161 if (boot_cpu_has(X86_FEATURE_PERFMON_V2))
1162 pmu_version = 2;
1163
1164 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
1165 uncore = &uncores[i];
1166
1167 BUG_ON(!uncore->scan);
1168 BUG_ON(!uncore->init);
1169 BUG_ON(!uncore->move);
1170 BUG_ON(!uncore->free);
1171
1172 uncore->info = alloc_percpu(union amd_uncore_info);
1173 if (!uncore->info) {
1174 ret = -ENOMEM;
1175 goto fail;
1176 }
1177 };
1178
1179 /*
1180 * Install callbacks. Core will call them for each online cpu.
1181 */
1182 ret = cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
1183 "perf/x86/amd/uncore:prepare",
1184 NULL, amd_uncore_cpu_dead);
1185 if (ret)
1186 goto fail;
1187
1188 ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
1189 "perf/x86/amd/uncore:starting",
1190 amd_uncore_cpu_starting, NULL);
1191 if (ret)
1192 goto fail_prep;
1193
1194 ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
1195 "perf/x86/amd/uncore:online",
1196 amd_uncore_cpu_online,
1197 amd_uncore_cpu_down_prepare);
1198 if (ret)
1199 goto fail_start;
1200
1201 return 0;
1202
1203 fail_start:
1204 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
1205 fail_prep:
1206 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
1207 fail:
1208 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
1209 uncore = &uncores[i];
1210 if (uncore->info) {
1211 free_percpu(uncore->info);
1212 uncore->info = NULL;
1213 }
1214 }
1215
1216 return ret;
1217 }
1218
amd_uncore_exit(void)1219 static void __exit amd_uncore_exit(void)
1220 {
1221 struct amd_uncore *uncore;
1222 struct amd_uncore_pmu *pmu;
1223 int i, j;
1224
1225 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE);
1226 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
1227 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
1228
1229 for (i = 0; i < UNCORE_TYPE_MAX; i++) {
1230 uncore = &uncores[i];
1231 if (!uncore->info)
1232 continue;
1233
1234 free_percpu(uncore->info);
1235 uncore->info = NULL;
1236
1237 for (j = 0; j < uncore->num_pmus; j++) {
1238 pmu = &uncore->pmus[j];
1239 if (!pmu->ctx)
1240 continue;
1241
1242 perf_pmu_unregister(&pmu->pmu);
1243 free_percpu(pmu->ctx);
1244 pmu->ctx = NULL;
1245 }
1246
1247 kfree(uncore->pmus);
1248 uncore->pmus = NULL;
1249 }
1250 }
1251
1252 module_init(amd_uncore_init);
1253 module_exit(amd_uncore_exit);
1254
1255 MODULE_DESCRIPTION("AMD Uncore Driver");
1256 MODULE_LICENSE("GPL v2");
1257