xref: /freebsd/sys/dev/hwpmc/hwpmc_core.c (revision 4d293dd8dcde59fc9842a0ce1125fef8fcf83a8c)
1 /*-
2  * Copyright (c) 2008 Joseph Koshy
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 /*
28  * Intel Core PMCs.
29  */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/bus.h>
36 #include <sys/pmc.h>
37 #include <sys/pmckern.h>
38 #include <sys/systm.h>
39 
40 #include <machine/intr_machdep.h>
41 #if (__FreeBSD_version >= 1100000)
42 #include <x86/apicvar.h>
43 #else
44 #include <machine/apicvar.h>
45 #endif
46 #include <machine/cpu.h>
47 #include <machine/cpufunc.h>
48 #include <machine/md_var.h>
49 #include <machine/specialreg.h>
50 
51 #define	CORE_CPUID_REQUEST		0xA
52 #define	CORE_CPUID_REQUEST_SIZE		0x4
53 #define	CORE_CPUID_EAX			0x0
54 #define	CORE_CPUID_EBX			0x1
55 #define	CORE_CPUID_ECX			0x2
56 #define	CORE_CPUID_EDX			0x3
57 
58 #define	IAF_PMC_CAPS			\
59 	(PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
60 	 PMC_CAP_USER | PMC_CAP_SYSTEM)
61 #define	IAF_RI_TO_MSR(RI)		((RI) + (1 << 30))
62 
63 #define	IAP_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \
64     PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE |	 \
65     PMC_CAP_INVERT | PMC_CAP_QUALIFIER | PMC_CAP_PRECISE)
66 
67 #define	EV_IS_NOTARCH		0
68 #define	EV_IS_ARCH_SUPP		1
69 #define	EV_IS_ARCH_NOTSUPP	-1
70 
71 /*
72  * "Architectural" events defined by Intel.  The values of these
73  * symbols correspond to positions in the bitmask returned by
74  * the CPUID.0AH instruction.
75  */
76 enum core_arch_events {
77 	CORE_AE_BRANCH_INSTRUCTION_RETIRED	= 5,
78 	CORE_AE_BRANCH_MISSES_RETIRED		= 6,
79 	CORE_AE_INSTRUCTION_RETIRED		= 1,
80 	CORE_AE_LLC_MISSES			= 4,
81 	CORE_AE_LLC_REFERENCE			= 3,
82 	CORE_AE_UNHALTED_REFERENCE_CYCLES	= 2,
83 	CORE_AE_UNHALTED_CORE_CYCLES		= 0
84 };
85 
86 static enum pmc_cputype	core_cputype;
87 
88 struct core_cpu {
89 	volatile uint32_t	pc_resync;
90 	volatile uint32_t	pc_iafctrl;	/* Fixed function control. */
91 	volatile uint64_t	pc_globalctrl;	/* Global control register. */
92 	struct pmc_hw		pc_corepmcs[];
93 };
94 
95 static struct core_cpu **core_pcpu;
96 
97 static uint32_t core_architectural_events;
98 static uint64_t core_pmcmask;
99 
100 static int core_iaf_ri;		/* relative index of fixed counters */
101 static int core_iaf_width;
102 static int core_iaf_npmc;
103 
104 static int core_iap_width;
105 static int core_iap_npmc;
106 
107 static int
108 core_pcpu_noop(struct pmc_mdep *md, int cpu)
109 {
110 	(void) md;
111 	(void) cpu;
112 	return (0);
113 }
114 
115 static int
116 core_pcpu_init(struct pmc_mdep *md, int cpu)
117 {
118 	struct pmc_cpu *pc;
119 	struct core_cpu *cc;
120 	struct pmc_hw *phw;
121 	int core_ri, n, npmc;
122 
123 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
124 	    ("[iaf,%d] insane cpu number %d", __LINE__, cpu));
125 
126 	PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu);
127 
128 	core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
129 	npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
130 
131 	if (core_cputype != PMC_CPU_INTEL_CORE)
132 		npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
133 
134 	cc = malloc(sizeof(struct core_cpu) + npmc * sizeof(struct pmc_hw),
135 	    M_PMC, M_WAITOK | M_ZERO);
136 
137 	core_pcpu[cpu] = cc;
138 	pc = pmc_pcpu[cpu];
139 
140 	KASSERT(pc != NULL && cc != NULL,
141 	    ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
142 
143 	for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) {
144 		phw->phw_state 	  = PMC_PHW_FLAG_IS_ENABLED |
145 		    PMC_PHW_CPU_TO_STATE(cpu) |
146 		    PMC_PHW_INDEX_TO_STATE(n + core_ri);
147 		phw->phw_pmc	  = NULL;
148 		pc->pc_hwpmcs[n + core_ri]  = phw;
149 	}
150 
151 	return (0);
152 }
153 
154 static int
155 core_pcpu_fini(struct pmc_mdep *md, int cpu)
156 {
157 	int core_ri, n, npmc;
158 	struct pmc_cpu *pc;
159 	struct core_cpu *cc;
160 	uint64_t msr = 0;
161 
162 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
163 	    ("[core,%d] insane cpu number (%d)", __LINE__, cpu));
164 
165 	PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu);
166 
167 	if ((cc = core_pcpu[cpu]) == NULL)
168 		return (0);
169 
170 	core_pcpu[cpu] = NULL;
171 
172 	pc = pmc_pcpu[cpu];
173 
174 	KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
175 		cpu));
176 
177 	npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
178 	core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
179 
180 	for (n = 0; n < npmc; n++) {
181 		msr = rdmsr(IAP_EVSEL0 + n) & ~IAP_EVSEL_MASK;
182 		wrmsr(IAP_EVSEL0 + n, msr);
183 	}
184 
185 	if (core_cputype != PMC_CPU_INTEL_CORE) {
186 		msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
187 		wrmsr(IAF_CTRL, msr);
188 		npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
189 	}
190 
191 	for (n = 0; n < npmc; n++)
192 		pc->pc_hwpmcs[n + core_ri] = NULL;
193 
194 	free(cc, M_PMC);
195 
196 	return (0);
197 }
198 
199 /*
200  * Fixed function counters.
201  */
202 
203 static pmc_value_t
204 iaf_perfctr_value_to_reload_count(pmc_value_t v)
205 {
206 
207 	/* If the PMC has overflowed, return a reload count of zero. */
208 	if ((v & (1ULL << (core_iaf_width - 1))) == 0)
209 		return (0);
210 	v &= (1ULL << core_iaf_width) - 1;
211 	return (1ULL << core_iaf_width) - v;
212 }
213 
214 static pmc_value_t
215 iaf_reload_count_to_perfctr_value(pmc_value_t rlc)
216 {
217 	return (1ULL << core_iaf_width) - rlc;
218 }
219 
220 static int
221 iaf_allocate_pmc(int cpu, int ri, struct pmc *pm,
222     const struct pmc_op_pmcallocate *a)
223 {
224 	enum pmc_event ev;
225 	uint32_t caps, flags, validflags;
226 
227 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
228 	    ("[core,%d] illegal CPU %d", __LINE__, cpu));
229 
230 	PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
231 
232 	if (ri < 0 || ri > core_iaf_npmc)
233 		return (EINVAL);
234 
235 	caps = a->pm_caps;
236 
237 	if (a->pm_class != PMC_CLASS_IAF ||
238 	    (caps & IAF_PMC_CAPS) != caps)
239 		return (EINVAL);
240 
241 	ev = pm->pm_event;
242 	if (ev < PMC_EV_IAF_FIRST || ev > PMC_EV_IAF_LAST)
243 		return (EINVAL);
244 
245 	if (ev == PMC_EV_IAF_INSTR_RETIRED_ANY && ri != 0)
246 		return (EINVAL);
247 	if (ev == PMC_EV_IAF_CPU_CLK_UNHALTED_CORE && ri != 1)
248 		return (EINVAL);
249 	if (ev == PMC_EV_IAF_CPU_CLK_UNHALTED_REF && ri != 2)
250 		return (EINVAL);
251 
252 	flags = a->pm_md.pm_iaf.pm_iaf_flags;
253 
254 	validflags = IAF_MASK;
255 
256 	if (core_cputype != PMC_CPU_INTEL_ATOM &&
257 		core_cputype != PMC_CPU_INTEL_ATOM_SILVERMONT)
258 		validflags &= ~IAF_ANY;
259 
260 	if ((flags & ~validflags) != 0)
261 		return (EINVAL);
262 
263 	if (caps & PMC_CAP_INTERRUPT)
264 		flags |= IAF_PMI;
265 	if (caps & PMC_CAP_SYSTEM)
266 		flags |= IAF_OS;
267 	if (caps & PMC_CAP_USER)
268 		flags |= IAF_USR;
269 	if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
270 		flags |= (IAF_OS | IAF_USR);
271 
272 	pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4));
273 
274 	PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx",
275 	    (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl);
276 
277 	return (0);
278 }
279 
280 static int
281 iaf_config_pmc(int cpu, int ri, struct pmc *pm)
282 {
283 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
284 	    ("[core,%d] illegal CPU %d", __LINE__, cpu));
285 
286 	KASSERT(ri >= 0 && ri < core_iaf_npmc,
287 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
288 
289 	PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
290 
291 	KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
292 	    cpu));
293 
294 	core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm;
295 
296 	return (0);
297 }
298 
299 static int
300 iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
301 {
302 	int error;
303 	struct pmc_hw *phw;
304 	char iaf_name[PMC_NAME_MAX];
305 
306 	phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri];
307 
308 	(void) snprintf(iaf_name, sizeof(iaf_name), "IAF-%d", ri);
309 	if ((error = copystr(iaf_name, pi->pm_name, PMC_NAME_MAX,
310 	    NULL)) != 0)
311 		return (error);
312 
313 	pi->pm_class = PMC_CLASS_IAF;
314 
315 	if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
316 		pi->pm_enabled = TRUE;
317 		*ppmc          = phw->phw_pmc;
318 	} else {
319 		pi->pm_enabled = FALSE;
320 		*ppmc          = NULL;
321 	}
322 
323 	return (0);
324 }
325 
326 static int
327 iaf_get_config(int cpu, int ri, struct pmc **ppm)
328 {
329 	*ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
330 
331 	return (0);
332 }
333 
334 static int
335 iaf_get_msr(int ri, uint32_t *msr)
336 {
337 	KASSERT(ri >= 0 && ri < core_iaf_npmc,
338 	    ("[iaf,%d] ri %d out of range", __LINE__, ri));
339 
340 	*msr = IAF_RI_TO_MSR(ri);
341 
342 	return (0);
343 }
344 
345 static int
346 iaf_read_pmc(int cpu, int ri, pmc_value_t *v)
347 {
348 	struct pmc *pm;
349 	pmc_value_t tmp;
350 
351 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
352 	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
353 	KASSERT(ri >= 0 && ri < core_iaf_npmc,
354 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
355 
356 	pm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
357 
358 	KASSERT(pm,
359 	    ("[core,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu,
360 		ri, ri + core_iaf_ri));
361 
362 	tmp = rdpmc(IAF_RI_TO_MSR(ri));
363 
364 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
365 		*v = iaf_perfctr_value_to_reload_count(tmp);
366 	else
367 		*v = tmp;
368 
369 	PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
370 	    IAF_RI_TO_MSR(ri), *v);
371 
372 	return (0);
373 }
374 
375 static int
376 iaf_release_pmc(int cpu, int ri, struct pmc *pmc)
377 {
378 	PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
379 
380 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
381 	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
382 	KASSERT(ri >= 0 && ri < core_iaf_npmc,
383 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
384 
385 	KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL,
386 	    ("[core,%d] PHW pmc non-NULL", __LINE__));
387 
388 	return (0);
389 }
390 
391 static int
392 iaf_start_pmc(int cpu, int ri)
393 {
394 	struct pmc *pm;
395 	struct core_cpu *iafc;
396 	uint64_t msr = 0;
397 
398 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
399 	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
400 	KASSERT(ri >= 0 && ri < core_iaf_npmc,
401 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
402 
403 	PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
404 
405 	iafc = core_pcpu[cpu];
406 	pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
407 
408 	iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
409 
410  	msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
411  	wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
412 
413 	do {
414 		iafc->pc_resync = 0;
415 		iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
416  		msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
417  		wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
418  					     IAF_GLOBAL_CTRL_MASK));
419 	} while (iafc->pc_resync != 0);
420 
421 	PMCDBG4(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
422 	    iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
423 	    iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
424 
425 	return (0);
426 }
427 
428 static int
429 iaf_stop_pmc(int cpu, int ri)
430 {
431 	uint32_t fc;
432 	struct core_cpu *iafc;
433 	uint64_t msr = 0;
434 
435 	PMCDBG2(MDP,STO,1,"iaf-stop cpu=%d ri=%d", cpu, ri);
436 
437 	iafc = core_pcpu[cpu];
438 
439 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
440 	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
441 	KASSERT(ri >= 0 && ri < core_iaf_npmc,
442 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
443 
444 	fc = (IAF_MASK << (ri * 4));
445 
446 	if (core_cputype != PMC_CPU_INTEL_ATOM &&
447 		core_cputype != PMC_CPU_INTEL_ATOM_SILVERMONT)
448 		fc &= ~IAF_ANY;
449 
450 	iafc->pc_iafctrl &= ~fc;
451 
452 	PMCDBG1(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl);
453  	msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
454  	wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
455 
456 	do {
457 		iafc->pc_resync = 0;
458 		iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET));
459  		msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
460  		wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
461  					     IAF_GLOBAL_CTRL_MASK));
462 	} while (iafc->pc_resync != 0);
463 
464 	PMCDBG4(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
465 	    iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
466 	    iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
467 
468 	return (0);
469 }
470 
471 static int
472 iaf_write_pmc(int cpu, int ri, pmc_value_t v)
473 {
474 	struct core_cpu *cc;
475 	struct pmc *pm;
476 	uint64_t msr;
477 
478 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
479 	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
480 	KASSERT(ri >= 0 && ri < core_iaf_npmc,
481 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
482 
483 	cc = core_pcpu[cpu];
484 	pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
485 
486 	KASSERT(pm,
487 	    ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
488 
489 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
490 		v = iaf_reload_count_to_perfctr_value(v);
491 
492 	/* Turn off fixed counters */
493 	msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
494 	wrmsr(IAF_CTRL, msr);
495 
496 	wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));
497 
498 	/* Turn on fixed counters */
499 	msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
500 	wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK));
501 
502 	PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
503 	    "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
504 	    (uintmax_t) rdmsr(IAF_CTRL),
505 	    (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));
506 
507 	return (0);
508 }
509 
510 
511 static void
512 iaf_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth)
513 {
514 	struct pmc_classdep *pcd;
515 
516 	KASSERT(md != NULL, ("[iaf,%d] md is NULL", __LINE__));
517 
518 	PMCDBG0(MDP,INI,1, "iaf-initialize");
519 
520 	pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF];
521 
522 	pcd->pcd_caps	= IAF_PMC_CAPS;
523 	pcd->pcd_class	= PMC_CLASS_IAF;
524 	pcd->pcd_num	= npmc;
525 	pcd->pcd_ri	= md->pmd_npmc;
526 	pcd->pcd_width	= pmcwidth;
527 
528 	pcd->pcd_allocate_pmc	= iaf_allocate_pmc;
529 	pcd->pcd_config_pmc	= iaf_config_pmc;
530 	pcd->pcd_describe	= iaf_describe;
531 	pcd->pcd_get_config	= iaf_get_config;
532 	pcd->pcd_get_msr	= iaf_get_msr;
533 	pcd->pcd_pcpu_fini	= core_pcpu_noop;
534 	pcd->pcd_pcpu_init	= core_pcpu_noop;
535 	pcd->pcd_read_pmc	= iaf_read_pmc;
536 	pcd->pcd_release_pmc	= iaf_release_pmc;
537 	pcd->pcd_start_pmc	= iaf_start_pmc;
538 	pcd->pcd_stop_pmc	= iaf_stop_pmc;
539 	pcd->pcd_write_pmc	= iaf_write_pmc;
540 
541 	md->pmd_npmc	       += npmc;
542 }
543 
544 /*
545  * Intel programmable PMCs.
546  */
547 
548 /*
549  * Event descriptor tables.
550  *
551  * For each event id, we track:
552  *
553  * 1. The CPUs that the event is valid for.
554  *
555  * 2. If the event uses a fixed UMASK, the value of the umask field.
556  *    If the event doesn't use a fixed UMASK, a mask of legal bits
557  *    to check against.
558  */
559 
560 struct iap_event_descr {
561 	enum pmc_event	iap_ev;
562 	unsigned char	iap_evcode;
563 	unsigned char	iap_umask;
564 	unsigned int	iap_flags;
565 };
566 
567 #define	IAP_F_CC	(1 << 0)	/* CPU: Core */
568 #define	IAP_F_CC2	(1 << 1)	/* CPU: Core2 family */
569 #define	IAP_F_CC2E	(1 << 2)	/* CPU: Core2 Extreme only */
570 #define	IAP_F_CA	(1 << 3)	/* CPU: Atom */
571 #define	IAP_F_I7	(1 << 4)	/* CPU: Core i7 */
572 #define	IAP_F_I7O	(1 << 4)	/* CPU: Core i7 (old) */
573 #define	IAP_F_WM	(1 << 5)	/* CPU: Westmere */
574 #define	IAP_F_SB	(1 << 6)	/* CPU: Sandy Bridge */
575 #define	IAP_F_IB	(1 << 7)	/* CPU: Ivy Bridge */
576 #define	IAP_F_SBX	(1 << 8)	/* CPU: Sandy Bridge Xeon */
577 #define	IAP_F_IBX	(1 << 9)	/* CPU: Ivy Bridge Xeon */
578 #define	IAP_F_HW	(1 << 10)	/* CPU: Haswell */
579 #define	IAP_F_CAS	(1 << 11)	/* CPU: Atom Silvermont */
580 #define	IAP_F_HWX	(1 << 12)	/* CPU: Haswell Xeon */
581 #define	IAP_F_BW	(1 << 13)	/* CPU: Broadwell */
582 #define	IAP_F_BWX	(1 << 14)	/* CPU: Broadwell Xeon */
583 #define	IAP_F_SL	(1 << 15)	/* CPU: Skylake */
584 #define	IAP_F_FM	(1 << 18)	/* Fixed mask */
585 
586 #define	IAP_F_ALLCPUSCORE2					\
587     (IAP_F_CC | IAP_F_CC2 | IAP_F_CC2E | IAP_F_CA)
588 
589 /* Sub fields of UMASK that this event supports. */
590 #define	IAP_M_CORE		(1 << 0) /* Core specificity */
591 #define	IAP_M_AGENT		(1 << 1) /* Agent specificity */
592 #define	IAP_M_PREFETCH		(1 << 2) /* Prefetch */
593 #define	IAP_M_MESI		(1 << 3) /* MESI */
594 #define	IAP_M_SNOOPRESPONSE	(1 << 4) /* Snoop response */
595 #define	IAP_M_SNOOPTYPE		(1 << 5) /* Snoop type */
596 #define	IAP_M_TRANSITION	(1 << 6) /* Transition */
597 
598 #define	IAP_F_CORE		(0x3 << 14) /* Core specificity */
599 #define	IAP_F_AGENT		(0x1 << 13) /* Agent specificity */
600 #define	IAP_F_PREFETCH		(0x3 << 12) /* Prefetch */
601 #define	IAP_F_MESI		(0xF <<  8) /* MESI */
602 #define	IAP_F_SNOOPRESPONSE	(0xB <<  8) /* Snoop response */
603 #define	IAP_F_SNOOPTYPE		(0x3 <<  8) /* Snoop type */
604 #define	IAP_F_TRANSITION	(0x1 << 12) /* Transition */
605 
606 #define	IAP_PREFETCH_RESERVED	(0x2 << 12)
607 #define	IAP_CORE_THIS		(0x1 << 14)
608 #define	IAP_CORE_ALL		(0x3 << 14)
609 #define	IAP_F_CMASK		0xFF000000
610 
611 static struct iap_event_descr iap_events[] = {
612 #undef IAPDESCR
613 #define	IAPDESCR(N,EV,UM,FLAGS) {					\
614 	.iap_ev = PMC_EV_IAP_EVENT_##N,					\
615 	.iap_evcode = (EV),						\
616 	.iap_umask = (UM),						\
617 	.iap_flags = (FLAGS)						\
618 	}
619 
620     IAPDESCR(02H_01H, 0x02, 0x01, IAP_F_FM | IAP_F_I7O),
621     IAPDESCR(02H_81H, 0x02, 0x81, IAP_F_FM | IAP_F_CA),
622 
623     IAPDESCR(03H_00H, 0x03, 0x00, IAP_F_FM | IAP_F_CC),
624     IAPDESCR(03H_01H, 0x03, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SB |
625 	IAP_F_SBX | IAP_F_CAS),
626     IAPDESCR(03H_02H, 0x03, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
627 	IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
628 	IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
629     IAPDESCR(03H_04H, 0x03, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O |
630 	IAP_F_CAS),
631     IAPDESCR(03H_08H, 0x03, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB |
632 	IAP_F_SBX | IAP_F_CAS | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
633 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
634     IAPDESCR(03H_10H, 0x03, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SB |
635 	IAP_F_SBX | IAP_F_CAS),
636     IAPDESCR(03H_20H, 0x03, 0x20, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS),
637     IAPDESCR(03H_40H, 0x03, 0x40, IAP_F_CAS),
638     IAPDESCR(03H_80H, 0x03, 0x80, IAP_F_CAS),
639 
640     IAPDESCR(04H_00H, 0x04, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CAS),
641     IAPDESCR(04H_01H, 0x04, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O |
642 	IAP_F_CAS),
643     IAPDESCR(04H_02H, 0x04, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS),
644     IAPDESCR(04H_04H, 0x04, 0x04, IAP_F_CAS),
645     IAPDESCR(04H_07H, 0x04, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
646     IAPDESCR(04H_08H, 0x04, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS),
647     IAPDESCR(04H_10H, 0x04, 0x10, IAP_F_CAS),
648     IAPDESCR(04H_20H, 0x04, 0x20, IAP_F_CAS),
649     IAPDESCR(04H_40H, 0x04, 0x40, IAP_F_CAS),
650     IAPDESCR(04H_80H, 0x04, 0x80, IAP_F_CAS),
651 
652     IAPDESCR(05H_00H, 0x05, 0x00, IAP_F_FM | IAP_F_CC),
653     IAPDESCR(05H_01H, 0x05, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_IB |
654 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX |  IAP_F_BW |
655 	IAP_F_BWX),
656     IAPDESCR(05H_02H, 0x05, 0x02, IAP_F_FM | IAP_F_I7O | IAP_F_WM | IAP_F_SB |
657 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX |
658 	IAP_F_BW | IAP_F_BWX),
659     IAPDESCR(05H_03H, 0x05, 0x03, IAP_F_FM | IAP_F_I7O | IAP_F_CAS),
660 
661     IAPDESCR(06H_00H, 0x06, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2 |
662 	IAP_F_CC2E | IAP_F_CA),
663     IAPDESCR(06H_01H, 0x06, 0x01, IAP_F_FM | IAP_F_I7O),
664     IAPDESCR(06H_02H, 0x06, 0x02, IAP_F_FM | IAP_F_I7O),
665     IAPDESCR(06H_04H, 0x06, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
666     IAPDESCR(06H_08H, 0x06, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
667     IAPDESCR(06H_0FH, 0x06, 0x0F, IAP_F_FM | IAP_F_I7O),
668 
669     IAPDESCR(07H_00H, 0x07, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
670     IAPDESCR(07H_01H, 0x07, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
671 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
672 	IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
673     IAPDESCR(07H_02H, 0x07, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2),
674     IAPDESCR(07H_03H, 0x07, 0x03, IAP_F_FM | IAP_F_ALLCPUSCORE2),
675     IAPDESCR(07H_06H, 0x07, 0x06, IAP_F_FM | IAP_F_CA),
676     IAPDESCR(07H_08H, 0x07, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_SB |
677 	IAP_F_SBX),
678 
679     IAPDESCR(08H_01H, 0x08, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
680 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX |
681 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
682     IAPDESCR(08H_02H, 0x08, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
683 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX |
684         IAP_F_BW | IAP_F_BWX),
685     IAPDESCR(08H_04H, 0x08, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
686 	IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_HW | IAP_F_HWX),
687     IAPDESCR(08H_05H, 0x08, 0x05, IAP_F_FM | IAP_F_CA),
688     IAPDESCR(08H_06H, 0x08, 0x06, IAP_F_FM | IAP_F_CA),
689     IAPDESCR(08H_07H, 0x08, 0x07, IAP_F_FM | IAP_F_CA),
690     IAPDESCR(08H_08H, 0x08, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
691     IAPDESCR(08H_09H, 0x08, 0x09, IAP_F_FM | IAP_F_CA),
692     IAPDESCR(08H_0EH, 0x08, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
693     IAPDESCR(08H_10H, 0x08, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
694 	IAP_F_SBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
695     IAPDESCR(08H_20H, 0x08, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW |
696         IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
697     IAPDESCR(08H_40H, 0x08, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX),
698     IAPDESCR(08H_60H, 0x08, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
699     IAPDESCR(08H_80H, 0x08, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX),
700     IAPDESCR(08H_81H, 0x08, 0x81, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
701     IAPDESCR(08H_82H, 0x08, 0x82, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
702     IAPDESCR(08H_84H, 0x08, 0x84, IAP_F_FM | IAP_F_IB | IAP_F_IBX),
703     IAPDESCR(08H_88H, 0x08, 0x88, IAP_F_IB | IAP_F_IBX),
704 
705     IAPDESCR(09H_01H, 0x09, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O),
706     IAPDESCR(09H_02H, 0x09, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_I7O),
707     IAPDESCR(09H_04H, 0x09, 0x04, IAP_F_FM | IAP_F_I7O),
708     IAPDESCR(09H_08H, 0x09, 0x08, IAP_F_FM | IAP_F_I7O),
709 
710     IAPDESCR(0BH_01H, 0x0B, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
711     IAPDESCR(0BH_02H, 0x0B, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
712     IAPDESCR(0BH_10H, 0x0B, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
713 
714     IAPDESCR(0CH_01H, 0x0C, 0x01, IAP_F_FM | IAP_F_CC2 | IAP_F_I7 |
715 	IAP_F_WM | IAP_F_SL),
716     IAPDESCR(0CH_02H, 0x0C, 0x02, IAP_F_FM | IAP_F_CC2),
717     IAPDESCR(0CH_03H, 0x0C, 0x03, IAP_F_FM | IAP_F_CA),
718 
719     IAPDESCR(0DH_03H, 0x0D, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_HW |
720        IAP_F_IB | IAP_F_IBX | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
721     IAPDESCR(0DH_40H, 0x0D, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
722     IAPDESCR(0DH_80H, 0x0D, 0x00, IAP_F_FM | IAP_F_SL),
723 
724     IAPDESCR(0EH_01H, 0x0E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
725 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
726 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
727     IAPDESCR(0EH_02H, 0x0E, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SL),
728     IAPDESCR(0EH_10H, 0x0E, 0x10, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
729         IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
730     IAPDESCR(0EH_20H, 0x0E, 0x20, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
731         IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
732     IAPDESCR(0EH_40H, 0x0E, 0x40, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
733         IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
734 
735     IAPDESCR(0FH_01H, 0x0F, 0x01, IAP_F_FM | IAP_F_I7),
736     IAPDESCR(0FH_02H, 0x0F, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
737     IAPDESCR(0FH_08H, 0x0F, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
738     IAPDESCR(0FH_10H, 0x0F, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
739     IAPDESCR(0FH_20H, 0x0F, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
740     IAPDESCR(0FH_80H, 0x0F, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
741 
742     IAPDESCR(10H_00H, 0x10, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
743     IAPDESCR(10H_01H, 0x10, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
744 	IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX ),
745     IAPDESCR(10H_02H, 0x10, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
746     IAPDESCR(10H_04H, 0x10, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
747     IAPDESCR(10H_08H, 0x10, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
748     IAPDESCR(10H_10H, 0x10, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
749 	IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
750     IAPDESCR(10H_20H, 0x10, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
751 	IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
752     IAPDESCR(10H_40H, 0x10, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
753 	IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
754     IAPDESCR(10H_80H, 0x10, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
755 	IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
756     IAPDESCR(10H_81H, 0x10, 0x81, IAP_F_FM | IAP_F_CA),
757 
758     IAPDESCR(11H_00H, 0x11, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
759     IAPDESCR(11H_01H, 0x11, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_SB |
760 	IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
761     IAPDESCR(11H_02H, 0x11, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB | IAP_F_IBX),
762     IAPDESCR(11H_81H, 0x11, 0x81, IAP_F_FM | IAP_F_CA),
763 
764     IAPDESCR(12H_00H, 0x12, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
765     IAPDESCR(12H_01H, 0x12, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM),
766     IAPDESCR(12H_02H, 0x12, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
767     IAPDESCR(12H_04H, 0x12, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
768     IAPDESCR(12H_08H, 0x12, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
769     IAPDESCR(12H_10H, 0x12, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
770     IAPDESCR(12H_20H, 0x12, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
771     IAPDESCR(12H_40H, 0x12, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
772     IAPDESCR(12H_81H, 0x12, 0x81, IAP_F_FM | IAP_F_CA),
773 
774     IAPDESCR(13H_00H, 0x13, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
775     IAPDESCR(13H_01H, 0x13, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 | IAP_F_WM),
776     IAPDESCR(13H_02H, 0x13, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
777     IAPDESCR(13H_04H, 0x13, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
778     IAPDESCR(13H_07H, 0x13, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
779     IAPDESCR(13H_81H, 0x13, 0x81, IAP_F_FM | IAP_F_CA),
780 
781     IAPDESCR(14H_00H, 0x14, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
782     IAPDESCR(14H_01H, 0x14, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
783 	 IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
784 	 IAP_F_BW | IAP_F_BWX | IAP_F_SL),
785     IAPDESCR(14H_02H, 0x14, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
786 
787     IAPDESCR(17H_01H, 0x17, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
788 	IAP_F_SBX),
789 
790     IAPDESCR(18H_00H, 0x18, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
791     IAPDESCR(18H_01H, 0x18, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
792 
793     IAPDESCR(19H_00H, 0x19, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
794     IAPDESCR(19H_01H, 0x19, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
795 	IAP_F_I7 | IAP_F_WM),
796     IAPDESCR(19H_02H, 0x19, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
797 
798     IAPDESCR(1DH_01H, 0x1D, 0x01, IAP_F_FM | IAP_F_I7O),
799     IAPDESCR(1DH_02H, 0x1D, 0x02, IAP_F_FM | IAP_F_I7O),
800     IAPDESCR(1DH_04H, 0x1D, 0x04, IAP_F_FM | IAP_F_I7O),
801 
802     IAPDESCR(1EH_01H, 0x1E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
803 
804     IAPDESCR(20H_01H, 0x20, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
805     IAPDESCR(21H, 0x21, IAP_M_CORE, IAP_F_ALLCPUSCORE2),
806     IAPDESCR(22H, 0x22, IAP_M_CORE, IAP_F_CC2),
807     IAPDESCR(23H, 0x23, IAP_M_CORE, IAP_F_ALLCPUSCORE2),
808 
809     IAPDESCR(24H, 0x24, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2),
810     IAPDESCR(24H_01H, 0x24, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
811 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX ),
812     IAPDESCR(24H_02H, 0x24, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
813     IAPDESCR(24H_03H, 0x24, 0x03, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
814 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
815     IAPDESCR(24H_04H, 0x24, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
816 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
817     IAPDESCR(24H_08H, 0x24, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
818 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
819     IAPDESCR(24H_0CH, 0x24, 0x0C, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
820 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
821     IAPDESCR(24H_10H, 0x24, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
822 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
823     IAPDESCR(24H_20H, 0x24, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
824 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
825     IAPDESCR(24H_21H, 0x24, 0x21, IAP_F_FM | IAP_F_HW | IAP_F_HWX |
826 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
827     IAPDESCR(24H_22H, 0x24, 0x22, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
828     IAPDESCR(24H_24H, 0x24, 0x24, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
829     IAPDESCR(24H_27H, 0x24, 0x27, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
830     IAPDESCR(24H_30H, 0x24, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
831 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
832 	IAP_F_BW | IAP_F_BWX),
833     IAPDESCR(24H_38H, 0x24, 0x00, IAP_F_FM | IAP_F_SL),
834     IAPDESCR(24H_3FH, 0x24, 0x00, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
835     IAPDESCR(24H_40H, 0x24, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
836 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
837     IAPDESCR(24H_41H, 0x24, 0x41, IAP_F_FM | IAP_F_HW | IAP_F_HWX |
838 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
839     IAPDESCR(24H_42H, 0x24, 0x42, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
840     IAPDESCR(24H_44H, 0x24, 0x44, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
841     IAPDESCR(24H_50H, 0x24, 0x50, IAP_F_FM | IAP_F_HW | IAP_F_HWX |
842 	IAP_F_BW | IAP_F_BWX),
843     IAPDESCR(24H_80H, 0x24, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
844 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
845     IAPDESCR(24H_AAH, 0x24, 0xAA, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
846     IAPDESCR(24H_C0H, 0x24, 0xC0, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
847 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
848     IAPDESCR(24H_D8H, 0x24, 0x00, IAP_F_FM | IAP_F_SL),
849     IAPDESCR(24H_E1H, 0x24, 0xE1, IAP_F_FM | IAP_F_HW | IAP_F_HWX |
850 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
851     IAPDESCR(24H_E2H, 0x24, 0xE2, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX |
852 	IAP_F_SL),
853     IAPDESCR(24H_E4H, 0x24, 0xE4, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX |
854 	IAP_F_SL),
855     IAPDESCR(24H_E7H, 0x24, 0xE7, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
856     IAPDESCR(24H_EFH, 0x24, 0x00, IAP_F_FM | IAP_F_SL),
857     IAPDESCR(24H_F8H, 0x24, 0xF8, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX |
858 	IAP_F_SL),
859     IAPDESCR(24H_FFH, 0x24, 0xFF, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW |
860         IAP_F_HWX),
861 
862     IAPDESCR(25H, 0x25, IAP_M_CORE, IAP_F_ALLCPUSCORE2),
863 
864     IAPDESCR(26H, 0x26, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2),
865     IAPDESCR(26H_01H, 0x26, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
866     IAPDESCR(26H_02H, 0x26, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
867     IAPDESCR(26H_04H, 0x26, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
868     IAPDESCR(26H_08H, 0x26, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
869     IAPDESCR(26H_0FH, 0x26, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
870     IAPDESCR(26H_10H, 0x26, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
871     IAPDESCR(26H_20H, 0x26, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
872     IAPDESCR(26H_40H, 0x26, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
873     IAPDESCR(26H_80H, 0x26, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
874     IAPDESCR(26H_F0H, 0x26, 0xF0, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
875     IAPDESCR(26H_FFH, 0x26, 0xFF, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
876 
877     IAPDESCR(27H, 0x27, IAP_M_CORE | IAP_M_PREFETCH, IAP_F_ALLCPUSCORE2),
878     IAPDESCR(27H_01H, 0x27, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
879 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
880     IAPDESCR(27H_02H, 0x27, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
881     IAPDESCR(27H_04H, 0x27, 0x04, IAP_F_FM | IAP_F_I7O | IAP_F_SB |
882 	IAP_F_SBX),
883     IAPDESCR(27H_08H, 0x27, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
884 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
885     IAPDESCR(27H_0EH, 0x27, 0x0E, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
886     IAPDESCR(27H_0FH, 0x27, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
887 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
888     IAPDESCR(27H_10H, 0x27, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
889     IAPDESCR(27H_20H, 0x27, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
890     IAPDESCR(27H_40H, 0x27, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
891     IAPDESCR(27H_50H, 0x27, 0x50, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
892     IAPDESCR(27H_80H, 0x27, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
893     IAPDESCR(27H_E0H, 0x27, 0xE0, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
894     IAPDESCR(27H_F0H, 0x27, 0xF0, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
895 
896     IAPDESCR(28H, 0x28, IAP_M_CORE | IAP_M_MESI, IAP_F_ALLCPUSCORE2),
897     IAPDESCR(28H_01H, 0x28, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB |
898 	IAP_F_SBX | IAP_F_IBX),
899     IAPDESCR(28H_02H, 0x28, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SBX),
900     IAPDESCR(28H_04H, 0x28, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
901 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
902     IAPDESCR(28H_08H, 0x28, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
903 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
904     IAPDESCR(28H_0FH, 0x28, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB |
905 	IAP_F_SBX | IAP_F_IBX),
906 
907     IAPDESCR(29H, 0x29, IAP_M_CORE | IAP_M_MESI, IAP_F_CC),
908     IAPDESCR(29H, 0x29, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH,
909 	IAP_F_CA | IAP_F_CC2),
910     IAPDESCR(2AH, 0x2A, IAP_M_CORE | IAP_M_MESI, IAP_F_ALLCPUSCORE2),
911     IAPDESCR(2BH, 0x2B, IAP_M_CORE | IAP_M_MESI, IAP_F_CA | IAP_F_CC2),
912 
913     IAPDESCR(2EH, 0x2E, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH,
914 	IAP_F_ALLCPUSCORE2),
915     IAPDESCR(2EH_01H, 0x2E, 0x01, IAP_F_FM | IAP_F_WM),
916     IAPDESCR(2EH_02H, 0x2E, 0x02, IAP_F_FM | IAP_F_WM),
917     IAPDESCR(2EH_41H, 0x2E, 0x41, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 |
918 	IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
919 	IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
920     IAPDESCR(2EH_4FH, 0x2E, 0x4F, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7 |
921 	IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
922 	IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
923 
924     IAPDESCR(30H, 0x30, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH,
925 	IAP_F_ALLCPUSCORE2),
926     IAPDESCR(30H_00H, 0x30, 0x00, IAP_F_CAS),
927     IAPDESCR(31H_00H, 0x31, 0x00, IAP_F_CAS),
928     IAPDESCR(32H, 0x32, IAP_M_CORE | IAP_M_MESI | IAP_M_PREFETCH, IAP_F_CC),
929     IAPDESCR(32H, 0x32, IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
930 
931     IAPDESCR(3AH, 0x3A, IAP_M_TRANSITION, IAP_F_CC),
932     IAPDESCR(3AH_00H, 0x3A, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
933 
934     IAPDESCR(3BH_C0H, 0x3B, 0xC0, IAP_F_FM | IAP_F_ALLCPUSCORE2),
935 
936     IAPDESCR(3CH_00H, 0x3C, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
937 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
938 	IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
939     IAPDESCR(3CH_01H, 0x3C, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
940 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
941 	IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
942     IAPDESCR(3CH_02H, 0x3C, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_SL),
943 
944     IAPDESCR(3DH_01H, 0x3D, 0x01, IAP_F_FM | IAP_F_I7O),
945 
946     IAPDESCR(40H, 0x40, IAP_M_MESI, IAP_F_CC | IAP_F_CC2),
947     IAPDESCR(40H_01H, 0x40, 0x01, IAP_F_FM | IAP_F_I7),
948     IAPDESCR(40H_02H, 0x40, 0x02, IAP_F_FM | IAP_F_I7),
949     IAPDESCR(40H_04H, 0x40, 0x04, IAP_F_FM | IAP_F_I7),
950     IAPDESCR(40H_08H, 0x40, 0x08, IAP_F_FM | IAP_F_I7),
951     IAPDESCR(40H_0FH, 0x40, 0x0F, IAP_F_FM | IAP_F_I7),
952     IAPDESCR(40H_21H, 0x40, 0x21, IAP_F_FM | IAP_F_CA),
953 
954     IAPDESCR(41H, 0x41, IAP_M_MESI, IAP_F_CC | IAP_F_CC2),
955     IAPDESCR(41H_01H, 0x41, 0x01, IAP_F_FM | IAP_F_I7O),
956     IAPDESCR(41H_02H, 0x41, 0x02, IAP_F_FM | IAP_F_I7),
957     IAPDESCR(41H_04H, 0x41, 0x04, IAP_F_FM | IAP_F_I7),
958     IAPDESCR(41H_08H, 0x41, 0x08, IAP_F_FM | IAP_F_I7),
959     IAPDESCR(41H_0FH, 0x41, 0x0F, IAP_F_FM | IAP_F_I7O),
960     IAPDESCR(41H_22H, 0x41, 0x22, IAP_F_FM | IAP_F_CA),
961 
962     IAPDESCR(42H, 0x42, IAP_M_MESI, IAP_F_ALLCPUSCORE2),
963     IAPDESCR(42H_01H, 0x42, 0x01, IAP_F_FM | IAP_F_I7),
964     IAPDESCR(42H_02H, 0x42, 0x02, IAP_F_FM | IAP_F_I7),
965     IAPDESCR(42H_04H, 0x42, 0x04, IAP_F_FM | IAP_F_I7),
966     IAPDESCR(42H_08H, 0x42, 0x08, IAP_F_FM | IAP_F_I7),
967     IAPDESCR(42H_10H, 0x42, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
968 
969     IAPDESCR(43H_01H, 0x43, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
970 	IAP_F_I7),
971     IAPDESCR(43H_02H, 0x43, 0x02, IAP_F_FM | IAP_F_CA |
972 	IAP_F_CC2 | IAP_F_I7),
973 
974     IAPDESCR(44H_02H, 0x44, 0x02, IAP_F_FM | IAP_F_CC),
975 
976     IAPDESCR(45H_0FH, 0x45, 0x0F, IAP_F_FM | IAP_F_ALLCPUSCORE2),
977 
978     IAPDESCR(46H_00H, 0x46, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
979     IAPDESCR(47H_00H, 0x47, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
980 
981     IAPDESCR(48H_00H, 0x48, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
982     IAPDESCR(48H_01H, 0x48, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
983 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
984     IAPDESCR(48H_02H, 0x48, 0x02, IAP_F_FM | IAP_F_I7O | IAP_F_SL),
985 
986     IAPDESCR(49H_00H, 0x49, 0x00, IAP_F_FM | IAP_F_CC),
987     IAPDESCR(49H_01H, 0x49, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
988 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX  | IAP_F_IBX |
989 	IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
990     IAPDESCR(49H_02H, 0x49, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
991 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX |
992 	IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
993     IAPDESCR(49H_04H, 0x49, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_SB | IAP_F_IB |
994 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
995     IAPDESCR(49H_0EH, 0x49, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
996     IAPDESCR(49H_10H, 0x49, 0x1,  IAP_F_FM | IAP_F_I7 | IAP_F_WM |
997 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
998 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
999     IAPDESCR(49H_20H, 0x49, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX |
1000 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1001     IAPDESCR(49H_40H, 0x49, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX),
1002     IAPDESCR(49H_60H, 0x49, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
1003     IAPDESCR(49H_80H, 0x49, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7 | IAP_F_HW |
1004         IAP_F_HWX),
1005 
1006     IAPDESCR(4BH_00H, 0x4B, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1007     IAPDESCR(4BH_01H, 0x4B, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7O),
1008     IAPDESCR(4BH_02H, 0x4B, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1009     IAPDESCR(4BH_03H, 0x4B, 0x03, IAP_F_FM | IAP_F_CC),
1010     IAPDESCR(4BH_08H, 0x4B, 0x08, IAP_F_FM | IAP_F_I7O),
1011 
1012     IAPDESCR(4CH_00H, 0x4C, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1013     IAPDESCR(4CH_01H, 0x4C, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1014 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
1015     IAPDESCR(4CH_02H, 0x4C, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1016 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1017 
1018     IAPDESCR(4DH_01H, 0x4D, 0x01, IAP_F_FM | IAP_F_I7O),
1019 
1020     IAPDESCR(4EH_01H, 0x4E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1021     IAPDESCR(4EH_02H, 0x4E, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1022 	IAP_F_SB | IAP_F_SBX),
1023     IAPDESCR(4EH_04H, 0x4E, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1024     IAPDESCR(4EH_10H, 0x4E, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1025 
1026     IAPDESCR(4FH_00H, 0x4F, 0x00, IAP_F_FM | IAP_F_CC),
1027     IAPDESCR(4FH_02H, 0x4F, 0x02, IAP_F_FM | IAP_F_I7O),
1028     IAPDESCR(4FH_04H, 0x4F, 0x04, IAP_F_FM | IAP_F_I7O),
1029     IAPDESCR(4FH_08H, 0x4F, 0x08, IAP_F_FM | IAP_F_I7O),
1030     IAPDESCR(4FH_10H, 0x4F, 0x10, IAP_F_FM | IAP_F_WM | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1031 
1032     IAPDESCR(51H_01H, 0x51, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1033 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
1034 	IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1035     IAPDESCR(51H_02H, 0x51, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1036 	IAP_F_SB | IAP_F_SBX),
1037     IAPDESCR(51H_04H, 0x51, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1038 	IAP_F_SB | IAP_F_SBX),
1039     IAPDESCR(51H_08H, 0x51, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1040 	IAP_F_SB | IAP_F_SBX),
1041 
1042     IAPDESCR(52H_01H, 0x52, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1043 
1044     IAPDESCR(53H_01H, 0x53, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1045 
1046     IAPDESCR(58H_01H, 0x58, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
1047         IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1048     IAPDESCR(58H_02H, 0x58, 0x02, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
1049         IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1050     IAPDESCR(58H_04H, 0x58, 0x04, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
1051         IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1052     IAPDESCR(58H_08H, 0x58, 0x08, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
1053         IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1054 
1055     IAPDESCR(59H_20H, 0x59, 0x1,  IAP_F_SB | IAP_F_SBX),
1056     IAPDESCR(59H_40H, 0x59, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1057     IAPDESCR(59H_80H, 0x59, 0x80, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1058 
1059     IAPDESCR(5BH_0CH, 0x5B, 0x0C, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1060     IAPDESCR(5BH_0FH, 0x5B, 0x0F, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1061     IAPDESCR(5BH_40H, 0x5B, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1062     IAPDESCR(5BH_4FH, 0x5B, 0x4F, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1063 
1064     IAPDESCR(5CH_01H, 0x5C, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1065 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1066     IAPDESCR(5CH_02H, 0x5C, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1067 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1068 
1069     IAPDESCR(5EH_01H, 0x5E, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1070 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1071 
1072     IAPDESCR(5FH_01H, 0x5F, 0x01, IAP_F_FM | IAP_F_IB ), 		/* IB not in manual */
1073     IAPDESCR(5FH_04H, 0x5F, 0x04, IAP_F_IBX | IAP_F_IB),
1074 
1075     IAPDESCR(60H, 0x60, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
1076     IAPDESCR(60H_01H, 0x60, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
1077 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1078 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1079     IAPDESCR(60H_02H, 0x60, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB |
1080 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1081     IAPDESCR(60H_04H, 0x60, 0x01, IAP_F_FM |IAP_F_I7O |
1082 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1083 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1084     IAPDESCR(60H_08H, 0x60, 0x01, IAP_F_FM |IAP_F_I7O |
1085 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1086         IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1087     IAPDESCR(60H_10H, 0x7, 0x00, IAP_F_SL),
1088 
1089     IAPDESCR(61H, 0x61, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2),
1090 
1091     IAPDESCR(61H_00H, 0x61, 0x00, IAP_F_FM | IAP_F_CC),
1092 
1093     IAPDESCR(62H, 0x62, IAP_M_AGENT, IAP_F_ALLCPUSCORE2),
1094     IAPDESCR(62H_00H, 0x62, 0x00, IAP_F_FM | IAP_F_CC),
1095 
1096     IAPDESCR(63H, 0x63, IAP_M_AGENT | IAP_M_CORE,
1097 	IAP_F_CA | IAP_F_CC2),
1098     IAPDESCR(63H, 0x63, IAP_M_CORE, IAP_F_CC),
1099     IAPDESCR(63H_01H, 0x63, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1100 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1101 	IAP_F_BW | IAP_F_BWX ),
1102     IAPDESCR(63H_02H, 0x63, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1103 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1104 	IAP_F_BW | IAP_F_BWX),
1105 
1106     IAPDESCR(64H, 0x64, IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
1107     IAPDESCR(64H_40H, 0x64, 0x40, IAP_F_FM | IAP_F_CC),
1108 
1109     IAPDESCR(65H, 0x65, IAP_M_AGENT | IAP_M_CORE,
1110 	IAP_F_CA | IAP_F_CC2),
1111     IAPDESCR(65H, 0x65, IAP_M_CORE, IAP_F_CC),
1112 
1113     IAPDESCR(66H, 0x66, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
1114 
1115     IAPDESCR(67H, 0x67, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
1116     IAPDESCR(67H, 0x67, IAP_M_AGENT, IAP_F_CC),
1117 
1118     IAPDESCR(68H, 0x68, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
1119     IAPDESCR(69H, 0x69, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
1120     IAPDESCR(6AH, 0x6A, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
1121     IAPDESCR(6BH, 0x6B, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
1122 
1123     IAPDESCR(6CH, 0x6C, IAP_M_AGENT | IAP_M_CORE, IAP_F_ALLCPUSCORE2),
1124     IAPDESCR(6CH_01H, 0x6C, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1125 
1126     IAPDESCR(6DH, 0x6D, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
1127     IAPDESCR(6DH, 0x6D, IAP_M_CORE, IAP_F_CC),
1128 
1129     IAPDESCR(6EH, 0x6E, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
1130     IAPDESCR(6EH, 0x6E, IAP_M_CORE, IAP_F_CC),
1131 
1132     IAPDESCR(6FH, 0x6F, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
1133     IAPDESCR(6FH, 0x6F, IAP_M_CORE, IAP_F_CC),
1134 
1135     IAPDESCR(70H, 0x70, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
1136     IAPDESCR(70H, 0x70, IAP_M_CORE, IAP_F_CC),
1137 
1138     IAPDESCR(77H, 0x77, IAP_M_AGENT | IAP_M_SNOOPRESPONSE,
1139 	IAP_F_CA | IAP_F_CC2),
1140     IAPDESCR(77H, 0x77, IAP_M_AGENT | IAP_M_MESI, IAP_F_CC),
1141 
1142     IAPDESCR(78H, 0x78, IAP_M_CORE, IAP_F_CC),
1143     IAPDESCR(78H, 0x78, IAP_M_CORE | IAP_M_SNOOPTYPE, IAP_F_CA | IAP_F_CC2),
1144 
1145     IAPDESCR(79H_02H, 0x79, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1146 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1147     IAPDESCR(79H_04H, 0x79, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1148 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1149     IAPDESCR(79H_08H, 0x79, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1150 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_SL | IAP_F_BW | IAP_F_BWX),
1151     IAPDESCR(79H_10H, 0x79, 0x1, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1152 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1153 
1154     IAPDESCR(79H_18H, 0x79, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1155 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1156     IAPDESCR(79H_20H, 0x79, 0x1, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1157 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1158 
1159     IAPDESCR(79H_24H, 0x79, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1160 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1161 
1162     IAPDESCR(79H_30H, 0x79, 0x1,  IAP_F_FM | IAP_F_SB | IAP_F_IB |
1163 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1164 
1165     IAPDESCR(79H_3CH, 0x79, 0x3C, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
1166         IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1167 
1168     IAPDESCR(7AH, 0x7A, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2),
1169 
1170     IAPDESCR(7BH, 0x7B, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2),
1171 
1172     IAPDESCR(7DH, 0x7D, IAP_M_CORE, IAP_F_ALLCPUSCORE2),
1173 
1174     IAPDESCR(7EH, 0x7E, IAP_M_AGENT | IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
1175     IAPDESCR(7EH_00H, 0x7E, 0x00, IAP_F_FM | IAP_F_CC),
1176 
1177     IAPDESCR(7FH, 0x7F, IAP_M_CORE, IAP_F_CA | IAP_F_CC2),
1178 
1179     IAPDESCR(80H_00H, 0x80, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1180     IAPDESCR(80H_01H, 0x80, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_CAS),
1181     IAPDESCR(80H_02H, 0x80, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
1182 	IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
1183 	IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1184     IAPDESCR(80H_03H, 0x80, 0x03, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
1185 	IAP_F_WM | IAP_F_CAS),
1186     IAPDESCR(80H_04H, 0x80, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB |
1187 	IAP_F_IBX | IAP_F_SL), /* SL may have a spec bug two with same entry no cmask */
1188 
1189     IAPDESCR(81H_00H, 0x81, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1190     IAPDESCR(81H_01H, 0x81, 0x01, IAP_F_FM | IAP_F_I7O),
1191     IAPDESCR(81H_02H, 0x81, 0x02, IAP_F_FM | IAP_F_I7O),
1192 
1193     IAPDESCR(82H_01H, 0x82, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1194     IAPDESCR(82H_02H, 0x82, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1195     IAPDESCR(82H_04H, 0x82, 0x04, IAP_F_FM | IAP_F_CA),
1196     IAPDESCR(82H_10H, 0x82, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1197     IAPDESCR(82H_12H, 0x82, 0x12, IAP_F_FM | IAP_F_CC2),
1198     IAPDESCR(82H_40H, 0x82, 0x40, IAP_F_FM | IAP_F_CC2),
1199 
1200     IAPDESCR(83H_01H, 0x83, 0x01, IAP_F_FM | IAP_F_I7O | IAP_F_SL),
1201     IAPDESCR(83H_02H, 0x83, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SL),
1202 
1203     IAPDESCR(85H_00H, 0x85, 0x00, IAP_F_FM | IAP_F_CC),
1204     IAPDESCR(85H_01H, 0x85, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1205 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1206 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1207     IAPDESCR(85H_02H, 0x85, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1208 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1209 	IAP_F_BW | IAP_F_BWX),
1210     IAPDESCR(85H_04H, 0x85, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
1211 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1212     IAPDESCR(85H_0EH, 0x85, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
1213     IAPDESCR(85H_10H, 0x85, 0x10, IAP_F_FM | IAP_F_I7O | IAP_F_SB | IAP_F_IB |
1214 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1215     IAPDESCR(85H_20H, 0x85, 0x20, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX |
1216 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1217     IAPDESCR(85H_40H, 0x85, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX),
1218     IAPDESCR(85H_60H, 0x85, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
1219     IAPDESCR(85H_80H, 0x85, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7O),
1220 
1221     IAPDESCR(86H_00H, 0x86, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1222 
1223     IAPDESCR(87H_00H, 0x87, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1224     IAPDESCR(87H_01H, 0x87, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1225 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1226 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1227     IAPDESCR(87H_02H, 0x87, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1228     IAPDESCR(87H_04H, 0x87, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1229 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1230     IAPDESCR(87H_08H, 0x87, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1231     IAPDESCR(87H_0FH, 0x87, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1232 
1233     IAPDESCR(88H_00H, 0x88, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1234     IAPDESCR(88H_01H, 0x88, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1235     IAPDESCR(88H_02H, 0x88, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1236     IAPDESCR(88H_04H, 0x88, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1237     IAPDESCR(88H_07H, 0x88, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1238     IAPDESCR(88H_08H, 0x88, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1239     IAPDESCR(88H_10H, 0x88, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1240     IAPDESCR(88H_20H, 0x88, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1241     IAPDESCR(88H_30H, 0x88, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1242     IAPDESCR(88H_40H, 0x88, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1243     IAPDESCR(88H_41H, 0x88, 0x41, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1244 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1245     IAPDESCR(88H_7FH, 0x88, 0x7F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1246     IAPDESCR(88H_80H, 0x88, 0x0, IAP_F_FM | IAP_F_BW | IAP_F_BWX),
1247     IAPDESCR(88H_81H, 0x88, 0x81, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1248 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1249     IAPDESCR(88H_82H, 0x88, 0x82, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1250 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1251     IAPDESCR(88H_84H, 0x88, 0x84, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1252 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1253     IAPDESCR(88H_88H, 0x88, 0x88, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1254 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1255     IAPDESCR(88H_90H, 0x88, 0x90, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1256 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1257     IAPDESCR(88H_A0H, 0x88, 0xA0, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1258 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1259     IAPDESCR(88H_FFH, 0x88, 0xFF, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1260 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1261 
1262     IAPDESCR(89H_00H, 0x89, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1263     IAPDESCR(89H_01H, 0x89, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1264     IAPDESCR(89H_02H, 0x89, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1265     IAPDESCR(89H_04H, 0x89, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1266     IAPDESCR(89H_07H, 0x89, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1267     IAPDESCR(89H_08H, 0x89, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1268     IAPDESCR(89H_10H, 0x89, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1269     IAPDESCR(89H_20H, 0x89, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1270     IAPDESCR(89H_30H, 0x89, 0x30, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1271     IAPDESCR(89H_40H, 0x89, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_BW | IAP_F_BWX),
1272     IAPDESCR(89H_41H, 0x89, 0x41, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1273 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1274     IAPDESCR(89H_7FH, 0x89, 0x7F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1275     IAPDESCR(89H_80H, 0x89, 0x0, IAP_F_FM | IAP_F_BW | IAP_F_BWX),
1276     IAPDESCR(89H_81H, 0x89, 0x81, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1277 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1278     IAPDESCR(89H_82H, 0x89, 0x82, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1279 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1280     IAPDESCR(89H_84H, 0x89, 0x84, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1281 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1282     IAPDESCR(89H_88H, 0x89, 0x88, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1283 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1284     IAPDESCR(89H_90H, 0x89, 0x90, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1285 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1286     IAPDESCR(89H_A0H, 0x89, 0xA0, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1287 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1288     IAPDESCR(89H_FFH, 0x89, 0xFF, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1289 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1290 
1291     IAPDESCR(8AH_00H, 0x8A, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1292     IAPDESCR(8BH_00H, 0x8B, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1293     IAPDESCR(8CH_00H, 0x8C, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1294     IAPDESCR(8DH_00H, 0x8D, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1295     IAPDESCR(8EH_00H, 0x8E, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1296     IAPDESCR(8FH_00H, 0x8F, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1297 
1298     IAPDESCR(90H_00H, 0x90, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1299     IAPDESCR(91H_00H, 0x91, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1300     IAPDESCR(92H_00H, 0x92, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1301     IAPDESCR(93H_00H, 0x93, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1302     IAPDESCR(94H_00H, 0x94, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1303 
1304     IAPDESCR(97H_00H, 0x97, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1305     IAPDESCR(98H_00H, 0x98, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1306 
1307     IAPDESCR(9CH_01H, 0x9C, 0x01,  IAP_F_FM | IAP_F_SB | IAP_F_IB |
1308 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1309 
1310     IAPDESCR(A0H_00H, 0xA0, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1311 
1312     IAPDESCR(A1H_01H, 0xA1, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1313 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1314 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1315     IAPDESCR(A1H_02H, 0xA1, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1316 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1317 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1318     IAPDESCR(A1H_04H, 0xA1, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |   /* No desc in IB for this*/
1319 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1320 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1321     IAPDESCR(A1H_08H, 0xA1, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |   /* No desc in IB for this*/
1322 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1323 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1324     IAPDESCR(A1H_0CH, 0xA1, 0x0C, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1325 	IAP_F_SBX | IAP_F_IBX),
1326     IAPDESCR(A1H_10H, 0xA1, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |   /* No desc in IB for this*/
1327 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1328 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1329     IAPDESCR(A1H_20H, 0xA1, 0x20, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |   /* No desc in IB for this*/
1330 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1331 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1332     IAPDESCR(A1H_30H, 0xA1, 0x30, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1333 	IAP_F_SBX | IAP_F_IBX),
1334     IAPDESCR(A1H_40H, 0xA1, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1335 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1336     IAPDESCR(A1H_80H, 0xA1, 0x80, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1337 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1338 
1339     IAPDESCR(A2H_00H, 0xA2, 0x00, IAP_F_FM | IAP_F_CC),
1340     IAPDESCR(A2H_01H, 0xA2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1341 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1342 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1343     IAPDESCR(A2H_02H, 0xA2, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1344 	IAP_F_SB | IAP_F_SBX),
1345     IAPDESCR(A2H_04H, 0xA2, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1346 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1347 	IAP_F_BW | IAP_F_BWX),
1348     IAPDESCR(A2H_08H, 0xA2, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1349 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1350 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1351     IAPDESCR(A2H_10H, 0xA2, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1352 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1353 	IAP_F_BW | IAP_F_BWX),
1354     IAPDESCR(A2H_20H, 0xA2, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1355 	IAP_F_SB | IAP_F_SBX),
1356     IAPDESCR(A2H_40H, 0xA2, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1357 	IAP_F_SB | IAP_F_SBX),
1358     IAPDESCR(A2H_80H, 0xA2, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1359 	IAP_F_SB | IAP_F_SBX),
1360 
1361     IAPDESCR(A3H_01H, 0xA3, 0x02, IAP_F_SBX | IAP_F_IBX | IAP_F_IB | IAP_F_HW |
1362 	IAP_F_HWX | IAP_F_SL),
1363     IAPDESCR(A3H_02H, 0xA3, 0x02, IAP_F_SBX | IAP_F_IBX | IAP_F_IB | IAP_F_HW |
1364         IAP_F_HWX | IAP_F_SL),
1365     IAPDESCR(A3H_04H, 0xA3, 0x04, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB | IAP_F_SL),
1366     IAPDESCR(A3H_05H, 0xA3, 0x05, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
1367     IAPDESCR(A3H_06H, 0xA3, 0x06, IAP_F_FM | IAP_F_SL),
1368     IAPDESCR(A3H_08H, 0xA3, 0x08, IAP_F_IBX | IAP_F_HW | IAP_F_IB | IAP_F_HWX |
1369         IAP_F_SL),
1370     IAPDESCR(A3H_0CH, 0xA3, 0x0C, IAP_F_FM | IAP_F_HW | IAP_F_HW | IAP_F_SL),
1371     IAPDESCR(A3H_10H, 0xA3, 0x10, IAP_F_FM | IAP_F_SL),
1372     IAPDESCR(A3H_14H, 0xA3, 0x14, IAP_F_FM | IAP_F_SL),
1373 
1374     IAPDESCR(A6H_01H, 0xA6, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SL),
1375     IAPDESCR(A6H_02H, 0xA3, 0x0, IAP_F_FM | IAP_F_SL),
1376     IAPDESCR(A6H_04H, 0xA3, 0x0, IAP_F_FM | IAP_F_SL),
1377     IAPDESCR(A6H_08H, 0xA3, 0x0, IAP_F_FM | IAP_F_SL),
1378     IAPDESCR(A6H_10H, 0xA3, 0x0, IAP_F_FM | IAP_F_SL),
1379     IAPDESCR(A6H_40H, 0xA3, 0x0, IAP_F_FM | IAP_F_SL),
1380 
1381     IAPDESCR(A7H_01H, 0xA7, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM ),
1382     IAPDESCR(A8H_01H, 0xA8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IBX |
1383 	IAP_F_IB |IAP_F_SB |  IAP_F_SBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX |
1384 	IAP_F_SL),
1385 
1386     IAPDESCR(AAH_01H, 0xAA, 0x01, IAP_F_FM | IAP_F_CC2),
1387     IAPDESCR(AAH_02H, 0xAA, 0x02, IAP_F_FM | IAP_F_CA),
1388     IAPDESCR(AAH_03H, 0xAA, 0x03, IAP_F_FM | IAP_F_CA),
1389     IAPDESCR(AAH_08H, 0xAA, 0x08, IAP_F_FM | IAP_F_CC2),
1390 
1391     IAPDESCR(ABH_01H, 0xAB, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1392 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
1393     IAPDESCR(ABH_02H, 0xAB, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1394 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_BW | IAP_F_BWX),
1395 
1396     IAPDESCR(ACH_02H, 0xAC, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_SL),
1397     IAPDESCR(ACH_08H, 0xAC, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1398 	IAP_F_SBX | IAP_F_IBX),
1399     IAPDESCR(ACH_0AH, 0xAC, 0x0A, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1400 
1401     IAPDESCR(AEH_01H, 0xAE, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1402 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1403 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1404 
1405     IAPDESCR(B0H_00H, 0xB0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1406     IAPDESCR(B0H_01H, 0xB0, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
1407 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1408 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1409     IAPDESCR(B0H_02H, 0xB0, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_IB |
1410 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1411     IAPDESCR(B0H_04H, 0xB0, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
1412 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1413 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1414     IAPDESCR(B0H_08H, 0xB0, 0x08, IAP_F_FM | IAP_F_WM | IAP_F_I7O |
1415 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1416 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1417     IAPDESCR(B0H_10H, 0xB0, 0x10, IAP_F_FM | IAP_F_WM | IAP_F_I7O | IAP_F_SL),
1418     IAPDESCR(B0H_20H, 0xB0, 0x20, IAP_F_FM | IAP_F_I7O),
1419     IAPDESCR(B0H_40H, 0xB0, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1420     IAPDESCR(B0H_80H, 0xB0, 0x80, IAP_F_FM | IAP_F_CA | IAP_F_WM | IAP_F_I7O | IAP_F_SL),
1421 
1422     IAPDESCR(B1H_00H, 0xB1, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1423     IAPDESCR(B1H_01H, 0xB1, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1424 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1425     IAPDESCR(B1H_02H, 0xB1, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1426 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1427 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1428     IAPDESCR(B1H_04H, 0xB1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1429     IAPDESCR(B1H_08H, 0xB1, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1430     IAPDESCR(B1H_10H, 0xB1, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1431     IAPDESCR(B1H_1FH, 0xB1, 0x1F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1432     IAPDESCR(B1H_20H, 0xB1, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1433     IAPDESCR(B1H_3FH, 0xB1, 0x3F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1434     IAPDESCR(B1H_40H, 0xB1, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1435     IAPDESCR(B1H_80H, 0xB1, 0x80, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
1436 	IAP_F_WM),
1437 
1438     IAPDESCR(B2H_01H, 0xB2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1439 	IAP_F_SB | IAP_F_SBX),
1440 
1441     IAPDESCR(B3H_01H, 0xB3, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
1442 	IAP_F_WM | IAP_F_I7O),
1443     IAPDESCR(B3H_02H, 0xB3, 0x02, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
1444 	IAP_F_WM | IAP_F_I7O),
1445     IAPDESCR(B3H_04H, 0xB3, 0x04, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
1446 	IAP_F_WM | IAP_F_I7O),
1447     IAPDESCR(B3H_08H, 0xB3, 0x08, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1448     IAPDESCR(B3H_10H, 0xB3, 0x10, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1449     IAPDESCR(B3H_20H, 0xB3, 0x20, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1450     IAPDESCR(B3H_81H, 0xB3, 0x81, IAP_F_FM | IAP_F_CA),
1451     IAPDESCR(B3H_82H, 0xB3, 0x82, IAP_F_FM | IAP_F_CA),
1452     IAPDESCR(B3H_84H, 0xB3, 0x84, IAP_F_FM | IAP_F_CA),
1453     IAPDESCR(B3H_88H, 0xB3, 0x88, IAP_F_FM | IAP_F_CA),
1454     IAPDESCR(B3H_90H, 0xB3, 0x90, IAP_F_FM | IAP_F_CA),
1455     IAPDESCR(B3H_A0H, 0xB3, 0xA0, IAP_F_FM | IAP_F_CA),
1456 
1457     IAPDESCR(B4H_01H, 0xB4, 0x01, IAP_F_FM | IAP_F_WM),
1458     IAPDESCR(B4H_02H, 0xB4, 0x02, IAP_F_FM | IAP_F_WM),
1459     IAPDESCR(B4H_04H, 0xB4, 0x04, IAP_F_FM | IAP_F_WM),
1460 
1461     IAPDESCR(B6H_01H, 0xB6, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1462     IAPDESCR(B6H_04H, 0xB6, 0x04, IAP_F_CAS),
1463 
1464     IAPDESCR(B7H_01H, 0xB7, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1465 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS |
1466 	IAP_F_HWX |IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1467     IAPDESCR(B7H_02H, 0xB7, 0x02, IAP_F_CAS),
1468 
1469     IAPDESCR(B8H_01H, 0xB8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1470     IAPDESCR(B8H_02H, 0xB8, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1471     IAPDESCR(B8H_04H, 0xB8, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1472 
1473     IAPDESCR(BAH_01H, 0xBA, 0x01, IAP_F_FM | IAP_F_I7O),
1474     IAPDESCR(BAH_02H, 0xBA, 0x02, IAP_F_FM | IAP_F_I7O),
1475 
1476     IAPDESCR(BBH_01H, 0xBB, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1477 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1478 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1479 
1480     IAPDESCR(BCH_11H, 0xBC, 0x11, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1481     IAPDESCR(BCH_12H, 0xBC, 0x12, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1482     IAPDESCR(BCH_14H, 0xBC, 0x14, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1483     IAPDESCR(BCH_18H, 0xBC, 0x18, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1484     IAPDESCR(BCH_21H, 0xBC, 0x21, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1485     IAPDESCR(BCH_22H, 0xBC, 0x22, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1486     IAPDESCR(BCH_24H, 0xBC, 0x24, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1487     IAPDESCR(BCH_28H, 0xBC, 0x28, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
1488 
1489     IAPDESCR(BDH_01H, 0xBD, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1490 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_SL), /* spec bug SL? */
1491     IAPDESCR(BDH_20H, 0xBD, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1492 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1493 
1494     IAPDESCR(BFH_05H, 0xBF, 0x05, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1495 
1496     IAPDESCR(C0H_00H, 0xC0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
1497 	IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
1498 	IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1499     IAPDESCR(C0H_01H, 0xC0, 0x0a,  IAP_F_CA | IAP_F_CC2 |
1500 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
1501 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1502     IAPDESCR(C0H_02H, 0xC0, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1503 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_BW | IAP_F_BWX),
1504     IAPDESCR(C0H_04H, 0xC0, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1505 	IAP_F_I7 | IAP_F_WM),
1506     IAPDESCR(C0H_08H, 0xC0, 0x08, IAP_F_FM | IAP_F_CC2E),
1507 
1508     IAPDESCR(C1H_00H, 0xC1, 0x00, IAP_F_FM | IAP_F_CC),
1509     IAPDESCR(C1H_01H, 0xC1, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1510     IAPDESCR(C1H_02H, 0xC1, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
1511     IAPDESCR(C1H_08H, 0xC1, 0x08, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1512 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1513     IAPDESCR(C1H_10H, 0xC1, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1514 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1515     IAPDESCR(C1H_20H, 0xC1, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1516 	IAP_F_SBX | IAP_F_IBX),
1517     IAPDESCR(C1H_3FH, 0xC1, 0x00, IAP_F_FM | IAP_F_SL),
1518     IAPDESCR(C1H_40H, 0xC1, 0x40, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1519     IAPDESCR(C1H_80H, 0xC1, 0x80, IAP_F_IB | IAP_F_IBX),
1520     IAPDESCR(C1H_FEH, 0xC1, 0xFE, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1521 
1522     IAPDESCR(C2H_00H, 0xC2, 0x00, IAP_F_FM | IAP_F_CC),
1523     IAPDESCR(C2H_01H, 0xC2, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1524 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
1525 	IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1526     IAPDESCR(C2H_02H, 0xC2, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1527 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
1528 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1529     IAPDESCR(C2H_04H, 0xC2, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1530 	IAP_F_I7 | IAP_F_WM),
1531     IAPDESCR(C2H_07H, 0xC2, 0x07, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1532     IAPDESCR(C2H_08H, 0xC2, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1533     IAPDESCR(C2H_0FH, 0xC2, 0x0F, IAP_F_FM | IAP_F_CC2),
1534     IAPDESCR(C2H_10H, 0xC2, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CAS),
1535 
1536     IAPDESCR(C3H_00H, 0xC3, 0x00, IAP_F_FM | IAP_F_CC),
1537     IAPDESCR(C3H_01H, 0xC3, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1538 	IAP_F_I7 | IAP_F_WM | IAP_F_CAS | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1539     IAPDESCR(C3H_02H, 0xC3, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1540 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
1541 	IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1542     IAPDESCR(C3H_04H, 0xC3, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1543 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
1544 	IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1545     IAPDESCR(C3H_08H, 0xC3, 0x08, IAP_F_CAS),
1546     IAPDESCR(C3H_10H, 0xC3, 0x10, IAP_F_FM | IAP_F_I7O),
1547     IAPDESCR(C3H_20H, 0xC3, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1548 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1549 
1550     IAPDESCR(C4H_00H, 0xC4, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
1551 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
1552 	IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1553     IAPDESCR(C4H_01H, 0xC4, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1554 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
1555 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1556     IAPDESCR(C4H_02H, 0xC4, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1557 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
1558 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1559     IAPDESCR(C4H_04H, 0xC4, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1560 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
1561 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1562     IAPDESCR(C4H_08H, 0xC4, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1563 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW |
1564         IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1565     IAPDESCR(C4H_0CH, 0xC4, 0x0C, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1566     IAPDESCR(C4H_0FH, 0xC4, 0x0F, IAP_F_FM | IAP_F_CA),
1567     IAPDESCR(C4H_10H, 0xC4, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1568 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1569     IAPDESCR(C4H_20H, 0xC4, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1570 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1571     IAPDESCR(C4H_40H, 0xC4, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1572 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1573     IAPDESCR(C4H_7EH, 0xC4, 0x7E, IAP_F_CAS),
1574     IAPDESCR(C4H_BFH, 0xC4, 0xBF, IAP_F_CAS),
1575     IAPDESCR(C4H_EBH, 0xC4, 0xEB, IAP_F_CAS),
1576     IAPDESCR(C4H_F7H, 0xC4, 0xF7, IAP_F_CAS),
1577     IAPDESCR(C4H_F9H, 0xC4, 0xF9, IAP_F_CAS),
1578     IAPDESCR(C4H_FBH, 0xC4, 0xFB, IAP_F_CAS),
1579     IAPDESCR(C4H_FDH, 0xC4, 0xFD, IAP_F_CAS),
1580     IAPDESCR(C4H_FEH, 0xC4, 0xFE, IAP_F_CAS),
1581 
1582     IAPDESCR(C5H_00H, 0xC5, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
1583 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_IB | IAP_F_SBX |
1584 	IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1585     IAPDESCR(C5H_01H, 0xC5, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_SB |
1586 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1587     IAPDESCR(C5H_02H, 0xC5, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1588 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_SL),
1589     IAPDESCR(C5H_04H, 0xC5, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_SB |
1590 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX |
1591 	IAP_F_SL),
1592     IAPDESCR(C5H_10H, 0xC5, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1593 	IAP_F_SBX | IAP_F_IBX),
1594     IAPDESCR(C5H_20H, 0xC5, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1595 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_SL),
1596     IAPDESCR(C5H_7EH, 0xC5, 0x7E, IAP_F_CAS),
1597     IAPDESCR(C5H_BFH, 0xC5, 0xBF, IAP_F_CAS),
1598     IAPDESCR(C5H_EBH, 0xC5, 0xEB, IAP_F_CAS),
1599     IAPDESCR(C5H_F7H, 0xC5, 0xF7, IAP_F_CAS),
1600     IAPDESCR(C5H_F9H, 0xC5, 0xF9, IAP_F_CAS),
1601     IAPDESCR(C5H_FBH, 0xC5, 0xFB, IAP_F_CAS),
1602     IAPDESCR(C5H_FDH, 0xC5, 0xFD, IAP_F_CAS),
1603     IAPDESCR(C5H_FEH, 0xC5, 0xFE, IAP_F_CAS),
1604 
1605     IAPDESCR(C6H_00H, 0xC6, 0x00, IAP_F_FM | IAP_F_CC),
1606 	     /* For SL C6_01 needs EV_SEL? 0x11, 0x12, 0x13, 0x14, 0x15? */
1607     IAPDESCR(C6H_01H, 0xC6, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_SL),
1608     IAPDESCR(C6H_02H, 0xC6, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1609 
1610     IAPDESCR(C7H_00H, 0xC7, 0x00, IAP_F_FM | IAP_F_CC),
1611     IAPDESCR(C7H_01H, 0xC7, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1612 	IAP_F_I7 | IAP_F_WM | IAP_F_SL),
1613     IAPDESCR(C7H_02H, 0xC7, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1614 	IAP_F_I7 | IAP_F_WM | IAP_F_SL),
1615     IAPDESCR(C7H_04H, 0xC7, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1616 	IAP_F_I7 | IAP_F_WM | IAP_F_SL),
1617     IAPDESCR(C7H_08H, 0xC7, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1618 	IAP_F_I7 | IAP_F_WM | IAP_F_SL),
1619     IAPDESCR(C7H_10H, 0xC7, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1620 	IAP_F_I7 | IAP_F_WM | IAP_F_SL),
1621     IAPDESCR(C7H_1FH, 0xC7, 0x1F, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1622     IAPDESCR(C7H_20H, 0xC7, 0x0, IAP_F_FM | IAP_F_SL),
1623 
1624     IAPDESCR(C8H_00H, 0xC8, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1625     IAPDESCR(C8H_20H, 0xC8, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1626 
1627     IAPDESCR(C9H_00H, 0xC9, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1628 
1629     IAPDESCR(CAH_00H, 0xCA, 0x00, IAP_F_FM | IAP_F_CC),
1630     IAPDESCR(CAH_01H, 0xCA, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 | IAP_F_CAS),
1631     IAPDESCR(CAH_02H, 0xCA, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1632 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1633 	IAP_F_BW | IAP_F_BWX),
1634     IAPDESCR(CAH_04H, 0xCA, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1635 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1636 	IAP_F_BW | IAP_F_BWX),
1637     IAPDESCR(CAH_08H, 0xCA, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1638 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1639 	IAP_F_BW | IAP_F_BWX),
1640     IAPDESCR(CAH_10H, 0xCA, 0x10, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1641 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1642     IAPDESCR(CAH_1EH, 0xCA, 0x1, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1643 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1644     IAPDESCR(CAH_20H, 0xCA, 0x0, IAP_F_FM | IAP_F_CAS | IAP_F_BW | IAP_F_BWX),
1645     IAPDESCR(CAH_3FH, 0xCA, 0x3F, IAP_F_CAS),
1646     IAPDESCR(CAH_50H, 0xCA, 0x50, IAP_F_CAS),
1647 
1648     IAPDESCR(CBH_01H, 0xCB, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1649 	IAP_F_I7 | IAP_F_WM | IAP_F_CAS | IAP_F_SL),
1650     IAPDESCR(CBH_02H, 0xCB, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1651 	IAP_F_I7 | IAP_F_WM),
1652     IAPDESCR(CBH_04H, 0xCB, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1653 	IAP_F_I7 | IAP_F_WM),
1654     IAPDESCR(CBH_08H, 0xCB, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1655 	IAP_F_I7 | IAP_F_WM),
1656     IAPDESCR(CBH_10H, 0xCB, 0x10, IAP_F_FM | IAP_F_CC2 | IAP_F_I7 |
1657 	IAP_F_WM),
1658     IAPDESCR(CBH_1FH, 0xCB, 0x1F, IAP_F_CAS),
1659     IAPDESCR(CBH_40H, 0xCB, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1660     IAPDESCR(CBH_80H, 0xCB, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1661 
1662     IAPDESCR(CCH_00H, 0xCC, 0x00, IAP_F_FM | IAP_F_CC),
1663     IAPDESCR(CCH_01H, 0xCC, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 |
1664 	IAP_F_I7 | IAP_F_WM),
1665     IAPDESCR(CCH_02H, 0xCC, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1666 	IAP_F_I7 | IAP_F_WM),
1667     IAPDESCR(CCH_03H, 0xCC, 0x03, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1668     IAPDESCR(CCH_20H, 0xCC, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1669 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1670 
1671     IAPDESCR(CDH_00H, 0xCD, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1672     IAPDESCR(CDH_01H, 0xCD, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1673 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS | IAP_F_HWX | IAP_F_BW | IAP_F_BWX |
1674 	IAP_F_SL),
1675     IAPDESCR(CDH_02H, 0xCD, 0x02, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1676 	IAP_F_SBX | IAP_F_IBX),
1677 
1678     IAPDESCR(CEH_00H, 0xCE, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1679     IAPDESCR(CFH_00H, 0xCF, 0x00, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1680 
1681     /* Sandy Bridge / Sandy Bridge Xeon - 11, 12, 21, 41, 42, 81, 82 */
1682     IAPDESCR(D0H_00H, 0xD0, 0x00, IAP_F_FM | IAP_F_CC),
1683     IAPDESCR(D0H_01H, 0xD0, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1684     IAPDESCR(D0H_11H, 0xD0, 0x11, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1685         IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1686     IAPDESCR(D0H_12H, 0xD0, 0x12, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1687         IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1688     IAPDESCR(D0H_21H, 0xD0, 0x21, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_BW | IAP_F_BWX |
1689 	IAP_F_SL),
1690     IAPDESCR(D0H_41H, 0xD0, 0x41, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1691         IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1692     IAPDESCR(D0H_42H, 0xD0, 0x42, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1693         IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1694     IAPDESCR(D0H_81H, 0xD0, 0x81, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1695         IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1696     IAPDESCR(D0H_82H, 0xD0, 0x82, IAP_F_FM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1697         IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1698 
1699     IAPDESCR(D1H_01H, 0xD1, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_SB |
1700 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW |
1701 	IAP_F_BWX | IAP_F_SL),
1702     IAPDESCR(D1H_02H, 0xD1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1703 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1704 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1705     IAPDESCR(D1H_04H, 0xD1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1706 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1707 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1708     IAPDESCR(D1H_08H, 0xD1, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_IB |
1709         IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1710     IAPDESCR(D1H_10H, 0xD1, 0x10, IAP_F_HW | IAP_F_IB | IAP_F_IBX | IAP_F_HWX |
1711 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1712     IAPDESCR(D1H_20H, 0xD1, 0x20, IAP_F_FM | IAP_F_SBX | IAP_F_IBX | IAP_F_IB |
1713         IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1714     IAPDESCR(D1H_40H, 0xD1, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1715 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1716 
1717     IAPDESCR(D2H_01H, 0xD2, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1718 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1719 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1720     IAPDESCR(D2H_02H, 0xD2, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1721 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1722 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1723     IAPDESCR(D2H_04H, 0xD2, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1724 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1725 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1726     IAPDESCR(D2H_08H, 0xD2, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1727 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1728 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1729     IAPDESCR(D2H_0FH, 0xD2, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1730 	IAP_F_I7 | IAP_F_WM | IAP_F_SB | IAP_F_SBX | IAP_F_IB |
1731 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1732 
1733     IAPDESCR(D2H_10H, 0xD2, 0x10, IAP_F_FM | IAP_F_CC2E),
1734 
1735     IAPDESCR(D3H_01H, 0xD3, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_SBX |
1736 	IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1737     IAPDESCR(D3H_03H, 0xD3, 0x03, IAP_F_IBX),
1738     IAPDESCR(D3H_04H, 0xD3, 0x04, IAP_F_FM | IAP_F_SBX | IAP_F_IBX),	/* Not defined for IBX */
1739     IAPDESCR(D3H_0CH, 0xD3, 0x0C, IAP_F_IBX),
1740     IAPDESCR(D3H_10H, 0xD3, 0x10, IAP_F_IBX  ),
1741     IAPDESCR(D3H_20H, 0xD3, 0x20, IAP_F_IBX  ),
1742 
1743     IAPDESCR(D4H_01H, 0xD4, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1744 	IAP_F_I7 | IAP_F_WM),
1745     IAPDESCR(D4H_02H, 0xD4, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1746 	IAP_F_SB | IAP_F_SBX),
1747     IAPDESCR(D4H_04H, 0xD4, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1748     IAPDESCR(D4H_08H, 0xD4, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1749     IAPDESCR(D4H_0FH, 0xD4, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1750 
1751     IAPDESCR(D5H_01H, 0xD5, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2 |
1752 	IAP_F_I7 | IAP_F_WM),
1753     IAPDESCR(D5H_02H, 0xD5, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1754     IAPDESCR(D5H_04H, 0xD5, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1755     IAPDESCR(D5H_08H, 0xD5, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1756     IAPDESCR(D5H_0FH, 0xD5, 0x0F, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1757 
1758     IAPDESCR(D7H_00H, 0xD7, 0x00, IAP_F_FM | IAP_F_CC),
1759 
1760     IAPDESCR(D8H_00H, 0xD8, 0x00, IAP_F_FM | IAP_F_CC),
1761     IAPDESCR(D8H_01H, 0xD8, 0x01, IAP_F_FM | IAP_F_CC),
1762     IAPDESCR(D8H_02H, 0xD8, 0x02, IAP_F_FM | IAP_F_CC),
1763     IAPDESCR(D8H_03H, 0xD8, 0x03, IAP_F_FM | IAP_F_CC),
1764     IAPDESCR(D8H_04H, 0xD8, 0x04, IAP_F_FM | IAP_F_CC),
1765 
1766     IAPDESCR(D9H_00H, 0xD9, 0x00, IAP_F_FM | IAP_F_CC),
1767     IAPDESCR(D9H_01H, 0xD9, 0x01, IAP_F_FM | IAP_F_CC),
1768     IAPDESCR(D9H_02H, 0xD9, 0x02, IAP_F_FM | IAP_F_CC),
1769     IAPDESCR(D9H_03H, 0xD9, 0x03, IAP_F_FM | IAP_F_CC),
1770 
1771     IAPDESCR(DAH_00H, 0xDA, 0x00, IAP_F_FM | IAP_F_CC),
1772     IAPDESCR(DAH_01H, 0xDA, 0x01, IAP_F_FM | IAP_F_CC),
1773     IAPDESCR(DAH_02H, 0xDA, 0x02, IAP_F_FM | IAP_F_CC),
1774 
1775     IAPDESCR(DBH_00H, 0xDB, 0x00, IAP_F_FM | IAP_F_CC),
1776     IAPDESCR(DBH_01H, 0xDB, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1777 
1778     IAPDESCR(DCH_01H, 0xDC, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1779     IAPDESCR(DCH_02H, 0xDC, 0x02, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1780     IAPDESCR(DCH_04H, 0xDC, 0x04, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1781     IAPDESCR(DCH_08H, 0xDC, 0x08, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1782     IAPDESCR(DCH_10H, 0xDC, 0x10, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1783     IAPDESCR(DCH_1FH, 0xDC, 0x1F, IAP_F_FM | IAP_F_CA | IAP_F_CC2),
1784 
1785     IAPDESCR(E0H_00H, 0xE0, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
1786     IAPDESCR(E0H_01H, 0xE0, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
1787 	IAP_F_WM),
1788 
1789     IAPDESCR(E2H_00H, 0xE2, 0x00, IAP_F_FM | IAP_F_CC),
1790 
1791     IAPDESCR(E4H_00H, 0xE4, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1792     IAPDESCR(E4H_01H, 0xE4, 0x01, IAP_F_FM | IAP_F_I7O),
1793 
1794     IAPDESCR(E5H_01H, 0xE5, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1795 
1796     IAPDESCR(E6H_00H, 0xE6, 0x00, IAP_F_FM | IAP_F_CC | IAP_F_CC2),
1797     IAPDESCR(E6H_01H, 0xE6, 0x01, IAP_F_FM | IAP_F_CA | IAP_F_I7 |
1798 	IAP_F_WM | IAP_F_SBX | IAP_F_CAS | IAP_F_SL),
1799     IAPDESCR(E6H_02H, 0xE6, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1800     IAPDESCR(E6H_08H, 0xE6, 0x08, IAP_F_CAS),
1801     IAPDESCR(E6H_10H, 0xE6, 0x10, IAP_F_CAS),
1802     IAPDESCR(E6H_1FH, 0xE6, 0x1F, IAP_F_FM | IAP_F_IB |
1803         IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
1804 
1805     IAPDESCR(E7H_01H, 0xE7, 0x01, IAP_F_CAS),
1806 
1807     IAPDESCR(E8H_01H, 0xE8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1808     IAPDESCR(E8H_02H, 0xE8, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1809     IAPDESCR(E8H_03H, 0xE8, 0x03, IAP_F_FM | IAP_F_I7O),
1810 
1811     IAPDESCR(ECH_01H, 0xEC, 0x01, IAP_F_FM | IAP_F_WM),
1812 
1813     IAPDESCR(F0H_00H, 0xF0, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1814     IAPDESCR(F0H_01H, 0xF0, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1815 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1816 	IAP_F_BW | IAP_F_BWX),
1817     IAPDESCR(F0H_02H, 0xF0, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1818 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1819 	IAP_F_BW | IAP_F_BWX),
1820     IAPDESCR(F0H_04H, 0xF0, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1821 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1822 	IAP_F_BW | IAP_F_BWX),
1823     IAPDESCR(F0H_08H, 0xF0, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1824 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1825 	IAP_F_BW | IAP_F_BWX),
1826     IAPDESCR(F0H_10H, 0xF0, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1827 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1828 	IAP_F_BW | IAP_F_BWX),
1829     IAPDESCR(F0H_20H, 0xF0, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1830 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1831 	IAP_F_BW | IAP_F_BWX),
1832     IAPDESCR(F0H_40H, 0xF0, 0x40, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1833 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1834 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1835     IAPDESCR(F0H_80H, 0xF0, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1836 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1837 	IAP_F_BW | IAP_F_BWX),
1838 
1839     IAPDESCR(F1H_01H, 0xF1, 0x01, IAP_F_FM | IAP_F_SB | IAP_F_IB |
1840 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1841     IAPDESCR(F1H_02H, 0xF1, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1842 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1843 	IAP_F_BW | IAP_F_BWX),
1844     IAPDESCR(F1H_04H, 0xF1, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1845 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1846 	IAP_F_BW | IAP_F_BWX ),
1847     IAPDESCR(F1H_07H, 0xF1, 0x07, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1848 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX |
1849 	IAP_F_BW | IAP_F_BWX | IAP_F_SL),
1850 
1851     IAPDESCR(F2H_01H, 0xF2, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1852 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
1853     IAPDESCR(F2H_02H, 0xF2, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1854 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
1855     IAPDESCR(F2H_04H, 0xF2, 0x04, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1856 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
1857     IAPDESCR(F2H_05H, 0xF2, 0x05, IAP_F_FM | IAP_F_HW | IAP_F_HWX | IAP_F_BW | IAP_F_BWX),
1858     IAPDESCR(F2H_06H, 0xF2, 0x06, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
1859     IAPDESCR(F2H_08H, 0xF2, 0x08, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1860 	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX),
1861     IAPDESCR(F2H_0AH, 0xF2, 0x0A, IAP_F_FM | IAP_F_SB | IAP_F_SBX |
1862 	IAP_F_IBX),
1863     IAPDESCR(F2H_0FH, 0xF2, 0x0F, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1864 
1865     IAPDESCR(F3H_01H, 0xF3, 0x01, IAP_F_FM | IAP_F_I7O),
1866     IAPDESCR(F3H_02H, 0xF3, 0x02, IAP_F_FM | IAP_F_I7O),
1867     IAPDESCR(F3H_04H, 0xF3, 0x04, IAP_F_FM | IAP_F_I7O),
1868     IAPDESCR(F3H_08H, 0xF3, 0x08, IAP_F_FM | IAP_F_I7O),
1869     IAPDESCR(F3H_10H, 0xF3, 0x10, IAP_F_FM | IAP_F_I7O),
1870     IAPDESCR(F3H_20H, 0xF3, 0x20, IAP_F_FM | IAP_F_I7O),
1871 
1872     IAPDESCR(F4H_01H, 0xF4, 0x01, IAP_F_FM | IAP_F_I7O),
1873     IAPDESCR(F4H_02H, 0xF4, 0x02, IAP_F_FM | IAP_F_I7O),
1874     IAPDESCR(F4H_04H, 0xF4, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7O),
1875     IAPDESCR(F4H_08H, 0xF4, 0x08, IAP_F_FM | IAP_F_I7O),
1876     IAPDESCR(F4H_10H, 0xF4, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
1877 	IAP_F_SB | IAP_F_SBX),
1878 
1879     IAPDESCR(F6H_01H, 0xF6, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
1880 
1881     IAPDESCR(F7H_01H, 0xF7, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1882     IAPDESCR(F7H_02H, 0xF7, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1883     IAPDESCR(F7H_04H, 0xF7, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1884 
1885     IAPDESCR(F8H_00H, 0xF8, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
1886     IAPDESCR(F8H_01H, 0xF8, 0x01, IAP_F_FM | IAP_F_I7O),
1887 
1888     IAPDESCR(FDH_01H, 0xFD, 0x01, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1889     IAPDESCR(FDH_02H, 0xFD, 0x02, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1890     IAPDESCR(FDH_04H, 0xFD, 0x04, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1891     IAPDESCR(FDH_08H, 0xFD, 0x08, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1892     IAPDESCR(FDH_10H, 0xFD, 0x10, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1893     IAPDESCR(FDH_20H, 0xFD, 0x20, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1894     IAPDESCR(FDH_40H, 0xFD, 0x40, IAP_F_FM | IAP_F_WM | IAP_F_I7),
1895 };
1896 
1897 static const int niap_events = sizeof(iap_events) / sizeof(iap_events[0]);
1898 
1899 static pmc_value_t
1900 iap_perfctr_value_to_reload_count(pmc_value_t v)
1901 {
1902 
1903 	/* If the PMC has overflowed, return a reload count of zero. */
1904 	if ((v & (1ULL << (core_iap_width - 1))) == 0)
1905 		return (0);
1906 	v &= (1ULL << core_iap_width) - 1;
1907 	return (1ULL << core_iap_width) - v;
1908 }
1909 
1910 static pmc_value_t
1911 iap_reload_count_to_perfctr_value(pmc_value_t rlc)
1912 {
1913 	return (1ULL << core_iap_width) - rlc;
1914 }
1915 
1916 static int
1917 iap_pmc_has_overflowed(int ri)
1918 {
1919 	uint64_t v;
1920 
1921 	/*
1922 	 * We treat a Core (i.e., Intel architecture v1) PMC as has
1923 	 * having overflowed if its MSB is zero.
1924 	 */
1925 	v = rdpmc(ri);
1926 	return ((v & (1ULL << (core_iap_width - 1))) == 0);
1927 }
1928 
1929 /*
1930  * Check an event against the set of supported architectural events.
1931  *
1932  * If the event is not architectural EV_IS_NOTARCH is returned.
1933  * If the event is architectural and supported on this CPU, the correct
1934  * event+umask mapping is returned in map, and EV_IS_ARCH_SUPP is returned.
1935  * Otherwise, the function returns EV_IS_ARCH_NOTSUPP.
1936  */
1937 
1938 static int
1939 iap_is_event_architectural(enum pmc_event pe, enum pmc_event *map)
1940 {
1941 	enum core_arch_events ae;
1942 
1943 	switch (pe) {
1944 	case PMC_EV_IAP_ARCH_UNH_COR_CYC:
1945 		ae = CORE_AE_UNHALTED_CORE_CYCLES;
1946 		*map = PMC_EV_IAP_EVENT_3CH_00H;
1947 		break;
1948 	case PMC_EV_IAP_ARCH_INS_RET:
1949 		ae = CORE_AE_INSTRUCTION_RETIRED;
1950 		*map = PMC_EV_IAP_EVENT_C0H_00H;
1951 		break;
1952 	case PMC_EV_IAP_ARCH_UNH_REF_CYC:
1953 		ae = CORE_AE_UNHALTED_REFERENCE_CYCLES;
1954 		*map = PMC_EV_IAP_EVENT_3CH_01H;
1955 		break;
1956 	case PMC_EV_IAP_ARCH_LLC_REF:
1957 		ae = CORE_AE_LLC_REFERENCE;
1958 		*map = PMC_EV_IAP_EVENT_2EH_4FH;
1959 		break;
1960 	case PMC_EV_IAP_ARCH_LLC_MIS:
1961 		ae = CORE_AE_LLC_MISSES;
1962 		*map = PMC_EV_IAP_EVENT_2EH_41H;
1963 		break;
1964 	case PMC_EV_IAP_ARCH_BR_INS_RET:
1965 		ae = CORE_AE_BRANCH_INSTRUCTION_RETIRED;
1966 		*map = PMC_EV_IAP_EVENT_C4H_00H;
1967 		break;
1968 	case PMC_EV_IAP_ARCH_BR_MIS_RET:
1969 		ae = CORE_AE_BRANCH_MISSES_RETIRED;
1970 		*map = PMC_EV_IAP_EVENT_C5H_00H;
1971 		break;
1972 
1973 	default:	/* Non architectural event. */
1974 		return (EV_IS_NOTARCH);
1975 	}
1976 
1977 	return (((core_architectural_events & (1 << ae)) == 0) ?
1978 	    EV_IS_ARCH_NOTSUPP : EV_IS_ARCH_SUPP);
1979 }
1980 
1981 static int
1982 iap_event_corei7_ok_on_counter(enum pmc_event pe, int ri)
1983 {
1984 	uint32_t mask;
1985 
1986 	switch (pe) {
1987 		/*
1988 		 * Events valid only on counter 0, 1.
1989 		 */
1990 	case PMC_EV_IAP_EVENT_40H_01H:
1991 	case PMC_EV_IAP_EVENT_40H_02H:
1992 	case PMC_EV_IAP_EVENT_40H_04H:
1993 	case PMC_EV_IAP_EVENT_40H_08H:
1994 	case PMC_EV_IAP_EVENT_40H_0FH:
1995 	case PMC_EV_IAP_EVENT_41H_02H:
1996 	case PMC_EV_IAP_EVENT_41H_04H:
1997 	case PMC_EV_IAP_EVENT_41H_08H:
1998 	case PMC_EV_IAP_EVENT_42H_01H:
1999 	case PMC_EV_IAP_EVENT_42H_02H:
2000 	case PMC_EV_IAP_EVENT_42H_04H:
2001 	case PMC_EV_IAP_EVENT_42H_08H:
2002 	case PMC_EV_IAP_EVENT_43H_01H:
2003 	case PMC_EV_IAP_EVENT_43H_02H:
2004 	case PMC_EV_IAP_EVENT_51H_01H:
2005 	case PMC_EV_IAP_EVENT_51H_02H:
2006 	case PMC_EV_IAP_EVENT_51H_04H:
2007 	case PMC_EV_IAP_EVENT_51H_08H:
2008 	case PMC_EV_IAP_EVENT_63H_01H:
2009 	case PMC_EV_IAP_EVENT_63H_02H:
2010 		mask = 0x3;
2011 		break;
2012 
2013 	default:
2014 		mask = ~0;	/* Any row index is ok. */
2015 	}
2016 
2017 	return (mask & (1 << ri));
2018 }
2019 
2020 static int
2021 iap_event_westmere_ok_on_counter(enum pmc_event pe, int ri)
2022 {
2023 	uint32_t mask;
2024 
2025 	switch (pe) {
2026 		/*
2027 		 * Events valid only on counter 0.
2028 		 */
2029 	case PMC_EV_IAP_EVENT_60H_01H:
2030 	case PMC_EV_IAP_EVENT_60H_02H:
2031 	case PMC_EV_IAP_EVENT_60H_04H:
2032 	case PMC_EV_IAP_EVENT_60H_08H:
2033 	case PMC_EV_IAP_EVENT_B3H_01H:
2034 	case PMC_EV_IAP_EVENT_B3H_02H:
2035 	case PMC_EV_IAP_EVENT_B3H_04H:
2036 		mask = 0x1;
2037 		break;
2038 
2039 		/*
2040 		 * Events valid only on counter 0, 1.
2041 		 */
2042 	case PMC_EV_IAP_EVENT_4CH_01H:
2043 	case PMC_EV_IAP_EVENT_4EH_01H:
2044 	case PMC_EV_IAP_EVENT_4EH_02H:
2045 	case PMC_EV_IAP_EVENT_4EH_04H:
2046 	case PMC_EV_IAP_EVENT_51H_01H:
2047 	case PMC_EV_IAP_EVENT_51H_02H:
2048 	case PMC_EV_IAP_EVENT_51H_04H:
2049 	case PMC_EV_IAP_EVENT_51H_08H:
2050 	case PMC_EV_IAP_EVENT_63H_01H:
2051 	case PMC_EV_IAP_EVENT_63H_02H:
2052 		mask = 0x3;
2053 		break;
2054 
2055 	default:
2056 		mask = ~0;	/* Any row index is ok. */
2057 	}
2058 
2059 	return (mask & (1 << ri));
2060 }
2061 
2062 static int
2063 iap_event_sb_sbx_ib_ibx_ok_on_counter(enum pmc_event pe, int ri)
2064 {
2065 	uint32_t mask;
2066 
2067 	switch (pe) {
2068 		/* Events valid only on counter 0. */
2069 	case PMC_EV_IAP_EVENT_B7H_01H:
2070 		mask = 0x1;
2071 		break;
2072 		/* Events valid only on counter 1. */
2073 	case PMC_EV_IAP_EVENT_C0H_01H:
2074 		mask = 0x2;
2075 		break;
2076 		/* Events valid only on counter 2. */
2077 	case PMC_EV_IAP_EVENT_48H_01H:
2078 	case PMC_EV_IAP_EVENT_A2H_02H:
2079 	case PMC_EV_IAP_EVENT_A3H_08H:
2080 		mask = 0x4;
2081 		break;
2082 		/* Events valid only on counter 3. */
2083 	case PMC_EV_IAP_EVENT_BBH_01H:
2084 	case PMC_EV_IAP_EVENT_CDH_01H:
2085 	case PMC_EV_IAP_EVENT_CDH_02H:
2086 		mask = 0x8;
2087 		break;
2088 	default:
2089 		mask = ~0;	/* Any row index is ok. */
2090 	}
2091 
2092 	return (mask & (1 << ri));
2093 }
2094 
2095 static int
2096 iap_event_ok_on_counter(enum pmc_event pe, int ri)
2097 {
2098 	uint32_t mask;
2099 
2100 	switch (pe) {
2101 		/*
2102 		 * Events valid only on counter 0.
2103 		 */
2104 	case PMC_EV_IAP_EVENT_10H_00H:
2105 	case PMC_EV_IAP_EVENT_14H_00H:
2106 	case PMC_EV_IAP_EVENT_18H_00H:
2107 	case PMC_EV_IAP_EVENT_B3H_01H:
2108 	case PMC_EV_IAP_EVENT_B3H_02H:
2109 	case PMC_EV_IAP_EVENT_B3H_04H:
2110 	case PMC_EV_IAP_EVENT_C1H_00H:
2111 	case PMC_EV_IAP_EVENT_CBH_01H:
2112 	case PMC_EV_IAP_EVENT_CBH_02H:
2113 		mask = (1 << 0);
2114 		break;
2115 
2116 		/*
2117 		 * Events valid only on counter 1.
2118 		 */
2119 	case PMC_EV_IAP_EVENT_11H_00H:
2120 	case PMC_EV_IAP_EVENT_12H_00H:
2121 	case PMC_EV_IAP_EVENT_13H_00H:
2122 		mask = (1 << 1);
2123 		break;
2124 
2125 	default:
2126 		mask = ~0;	/* Any row index is ok. */
2127 	}
2128 
2129 	return (mask & (1 << ri));
2130 }
2131 
2132 static int
2133 iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
2134     const struct pmc_op_pmcallocate *a)
2135 {
2136 	int arch, n, model;
2137 	enum pmc_event ev, map;
2138 	struct iap_event_descr *ie;
2139 	uint32_t c, caps, config, cpuflag, evsel, mask;
2140 
2141 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2142 	    ("[core,%d] illegal CPU %d", __LINE__, cpu));
2143 	KASSERT(ri >= 0 && ri < core_iap_npmc,
2144 	    ("[core,%d] illegal row-index value %d", __LINE__, ri));
2145 
2146 	/* check requested capabilities */
2147 	caps = a->pm_caps;
2148 	if ((IAP_PMC_CAPS & caps) != caps)
2149 		return (EPERM);
2150 	map = 0;	/* XXX: silent GCC warning */
2151 	arch = iap_is_event_architectural(pm->pm_event, &map);
2152 	if (arch == EV_IS_ARCH_NOTSUPP)
2153 		return (EOPNOTSUPP);
2154 	else if (arch == EV_IS_ARCH_SUPP)
2155 		ev = map;
2156 	else
2157 		ev = pm->pm_event;
2158 
2159 	/*
2160 	 * A small number of events are not supported in all the
2161 	 * processors based on a given microarchitecture.
2162 	 */
2163 	if (ev == PMC_EV_IAP_EVENT_0FH_01H || ev == PMC_EV_IAP_EVENT_0FH_80H) {
2164 		model = ((cpu_id & 0xF0000) >> 12) | ((cpu_id & 0xF0) >> 4);
2165 		if (core_cputype == PMC_CPU_INTEL_COREI7 && model != 0x2E)
2166 			return (EINVAL);
2167 	}
2168 
2169 	switch (core_cputype) {
2170 	case PMC_CPU_INTEL_COREI7:
2171 	case PMC_CPU_INTEL_NEHALEM_EX:
2172 		if (iap_event_corei7_ok_on_counter(ev, ri) == 0)
2173 			return (EINVAL);
2174 		break;
2175 	case PMC_CPU_INTEL_SKYLAKE:
2176 	case PMC_CPU_INTEL_BROADWELL:
2177 	case PMC_CPU_INTEL_BROADWELL_XEON:
2178 	case PMC_CPU_INTEL_SANDYBRIDGE:
2179 	case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
2180 	case PMC_CPU_INTEL_IVYBRIDGE:
2181 	case PMC_CPU_INTEL_IVYBRIDGE_XEON:
2182 	case PMC_CPU_INTEL_HASWELL:
2183 	case PMC_CPU_INTEL_HASWELL_XEON:
2184 		if (iap_event_sb_sbx_ib_ibx_ok_on_counter(ev, ri) == 0)
2185 			return (EINVAL);
2186 		break;
2187 	case PMC_CPU_INTEL_WESTMERE:
2188 	case PMC_CPU_INTEL_WESTMERE_EX:
2189 		if (iap_event_westmere_ok_on_counter(ev, ri) == 0)
2190 			return (EINVAL);
2191 		break;
2192 	default:
2193 		if (iap_event_ok_on_counter(ev, ri) == 0)
2194 			return (EINVAL);
2195 	}
2196 
2197 	/*
2198 	 * Look for an event descriptor with matching CPU and event id
2199 	 * fields.
2200 	 */
2201 
2202 	switch (core_cputype) {
2203 	default:
2204 	case PMC_CPU_INTEL_ATOM:
2205 		cpuflag = IAP_F_CA;
2206 		break;
2207 	case PMC_CPU_INTEL_ATOM_SILVERMONT:
2208 		cpuflag = IAP_F_CAS;
2209 		break;
2210 	case PMC_CPU_INTEL_SKYLAKE:
2211 		cpuflag = IAP_F_SL;
2212 		break;
2213 	case PMC_CPU_INTEL_BROADWELL_XEON:
2214 		cpuflag = IAP_F_BWX;
2215 		break;
2216 	case PMC_CPU_INTEL_BROADWELL:
2217 		cpuflag = IAP_F_BW;
2218 		break;
2219 	case PMC_CPU_INTEL_CORE:
2220 		cpuflag = IAP_F_CC;
2221 		break;
2222 	case PMC_CPU_INTEL_CORE2:
2223 		cpuflag = IAP_F_CC2;
2224 		break;
2225 	case PMC_CPU_INTEL_CORE2EXTREME:
2226 		cpuflag = IAP_F_CC2 | IAP_F_CC2E;
2227 		break;
2228 	case PMC_CPU_INTEL_COREI7:
2229 		cpuflag = IAP_F_I7;
2230 		break;
2231 	case PMC_CPU_INTEL_HASWELL:
2232 		cpuflag = IAP_F_HW;
2233 		break;
2234 	case PMC_CPU_INTEL_HASWELL_XEON:
2235 		cpuflag = IAP_F_HWX;
2236 		break;
2237 	case PMC_CPU_INTEL_IVYBRIDGE:
2238 		cpuflag = IAP_F_IB;
2239 		break;
2240 	case PMC_CPU_INTEL_IVYBRIDGE_XEON:
2241 		cpuflag = IAP_F_IBX;
2242 		break;
2243 	case PMC_CPU_INTEL_SANDYBRIDGE:
2244 		cpuflag = IAP_F_SB;
2245 		break;
2246 	case PMC_CPU_INTEL_SANDYBRIDGE_XEON:
2247 		cpuflag = IAP_F_SBX;
2248 		break;
2249 	case PMC_CPU_INTEL_WESTMERE:
2250 		cpuflag = IAP_F_WM;
2251 		break;
2252 	}
2253 
2254 	for (n = 0, ie = iap_events; n < niap_events; n++, ie++)
2255 		if (ie->iap_ev == ev && ie->iap_flags & cpuflag)
2256 			break;
2257 
2258 	if (n == niap_events)
2259 		return (EINVAL);
2260 
2261 	/*
2262 	 * A matching event descriptor has been found, so start
2263 	 * assembling the contents of the event select register.
2264 	 */
2265 	evsel = ie->iap_evcode;
2266 
2267 	config = a->pm_md.pm_iap.pm_iap_config & ~IAP_F_CMASK;
2268 
2269 	/*
2270 	 * If the event uses a fixed umask value, reject any umask
2271 	 * bits set by the user.
2272 	 */
2273 	if (ie->iap_flags & IAP_F_FM) {
2274 
2275 		if (IAP_UMASK(config) != 0)
2276 			return (EINVAL);
2277 
2278 		evsel |= (ie->iap_umask << 8);
2279 
2280 	} else {
2281 
2282 		/*
2283 		 * Otherwise, the UMASK value needs to be taken from
2284 		 * the MD fields of the allocation request.  Reject
2285 		 * requests that specify reserved bits.
2286 		 */
2287 
2288 		mask = 0;
2289 
2290 		if (ie->iap_umask & IAP_M_CORE) {
2291 			if ((c = (config & IAP_F_CORE)) != IAP_CORE_ALL &&
2292 			    c != IAP_CORE_THIS)
2293 				return (EINVAL);
2294 			mask |= IAP_F_CORE;
2295 		}
2296 
2297 		if (ie->iap_umask & IAP_M_AGENT)
2298 			mask |= IAP_F_AGENT;
2299 
2300 		if (ie->iap_umask & IAP_M_PREFETCH) {
2301 
2302 			if ((c = (config & IAP_F_PREFETCH)) ==
2303 			    IAP_PREFETCH_RESERVED)
2304 				return (EINVAL);
2305 
2306 			mask |= IAP_F_PREFETCH;
2307 		}
2308 
2309 		if (ie->iap_umask & IAP_M_MESI)
2310 			mask |= IAP_F_MESI;
2311 
2312 		if (ie->iap_umask & IAP_M_SNOOPRESPONSE)
2313 			mask |= IAP_F_SNOOPRESPONSE;
2314 
2315 		if (ie->iap_umask & IAP_M_SNOOPTYPE)
2316 			mask |= IAP_F_SNOOPTYPE;
2317 
2318 		if (ie->iap_umask & IAP_M_TRANSITION)
2319 			mask |= IAP_F_TRANSITION;
2320 
2321 		/*
2322 		 * If bits outside of the allowed set of umask bits
2323 		 * are set, reject the request.
2324 		 */
2325 		if (config & ~mask)
2326 			return (EINVAL);
2327 
2328 		evsel |= (config & mask);
2329 
2330 	}
2331 
2332 	/*
2333 	 * Only Atom and SandyBridge CPUs support the 'ANY' qualifier.
2334 	 */
2335 	if (core_cputype == PMC_CPU_INTEL_ATOM ||
2336 		core_cputype == PMC_CPU_INTEL_ATOM_SILVERMONT ||
2337 		core_cputype == PMC_CPU_INTEL_SANDYBRIDGE ||
2338 		core_cputype == PMC_CPU_INTEL_SANDYBRIDGE_XEON)
2339 		evsel |= (config & IAP_ANY);
2340 	else if (config & IAP_ANY)
2341 		return (EINVAL);
2342 
2343 	/*
2344 	 * Check offcore response configuration.
2345 	 */
2346 	if (a->pm_md.pm_iap.pm_iap_rsp != 0) {
2347 		if (ev != PMC_EV_IAP_EVENT_B7H_01H &&
2348 		    ev != PMC_EV_IAP_EVENT_BBH_01H)
2349 			return (EINVAL);
2350 		if (core_cputype == PMC_CPU_INTEL_COREI7 &&
2351 		    ev == PMC_EV_IAP_EVENT_BBH_01H)
2352 			return (EINVAL);
2353 		if ((core_cputype == PMC_CPU_INTEL_COREI7 ||
2354 		    core_cputype == PMC_CPU_INTEL_WESTMERE ||
2355 		    core_cputype == PMC_CPU_INTEL_NEHALEM_EX ||
2356 		    core_cputype == PMC_CPU_INTEL_WESTMERE_EX) &&
2357 		    a->pm_md.pm_iap.pm_iap_rsp & ~IA_OFFCORE_RSP_MASK_I7WM)
2358 			return (EINVAL);
2359 		else if ((core_cputype == PMC_CPU_INTEL_SANDYBRIDGE ||
2360 			core_cputype == PMC_CPU_INTEL_SANDYBRIDGE_XEON ||
2361 			core_cputype == PMC_CPU_INTEL_IVYBRIDGE ||
2362 			core_cputype == PMC_CPU_INTEL_IVYBRIDGE_XEON) &&
2363 		    a->pm_md.pm_iap.pm_iap_rsp & ~IA_OFFCORE_RSP_MASK_SBIB)
2364 			return (EINVAL);
2365 		pm->pm_md.pm_iap.pm_iap_rsp = a->pm_md.pm_iap.pm_iap_rsp;
2366 	}
2367 
2368 	if (caps & PMC_CAP_THRESHOLD)
2369 		evsel |= (a->pm_md.pm_iap.pm_iap_config & IAP_F_CMASK);
2370 	if (caps & PMC_CAP_USER)
2371 		evsel |= IAP_USR;
2372 	if (caps & PMC_CAP_SYSTEM)
2373 		evsel |= IAP_OS;
2374 	if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
2375 		evsel |= (IAP_OS | IAP_USR);
2376 	if (caps & PMC_CAP_EDGE)
2377 		evsel |= IAP_EDGE;
2378 	if (caps & PMC_CAP_INVERT)
2379 		evsel |= IAP_INV;
2380 	if (caps & PMC_CAP_INTERRUPT)
2381 		evsel |= IAP_INT;
2382 
2383 	pm->pm_md.pm_iap.pm_iap_evsel = evsel;
2384 
2385 	return (0);
2386 }
2387 
2388 static int
2389 iap_config_pmc(int cpu, int ri, struct pmc *pm)
2390 {
2391 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2392 	    ("[core,%d] illegal CPU %d", __LINE__, cpu));
2393 
2394 	KASSERT(ri >= 0 && ri < core_iap_npmc,
2395 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
2396 
2397 	PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
2398 
2399 	KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
2400 	    cpu));
2401 
2402 	core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm;
2403 
2404 	return (0);
2405 }
2406 
2407 static int
2408 iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
2409 {
2410 	int error;
2411 	struct pmc_hw *phw;
2412 	char iap_name[PMC_NAME_MAX];
2413 
2414 	phw = &core_pcpu[cpu]->pc_corepmcs[ri];
2415 
2416 	(void) snprintf(iap_name, sizeof(iap_name), "IAP-%d", ri);
2417 	if ((error = copystr(iap_name, pi->pm_name, PMC_NAME_MAX,
2418 	    NULL)) != 0)
2419 		return (error);
2420 
2421 	pi->pm_class = PMC_CLASS_IAP;
2422 
2423 	if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
2424 		pi->pm_enabled = TRUE;
2425 		*ppmc          = phw->phw_pmc;
2426 	} else {
2427 		pi->pm_enabled = FALSE;
2428 		*ppmc          = NULL;
2429 	}
2430 
2431 	return (0);
2432 }
2433 
2434 static int
2435 iap_get_config(int cpu, int ri, struct pmc **ppm)
2436 {
2437 	*ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
2438 
2439 	return (0);
2440 }
2441 
2442 static int
2443 iap_get_msr(int ri, uint32_t *msr)
2444 {
2445 	KASSERT(ri >= 0 && ri < core_iap_npmc,
2446 	    ("[iap,%d] ri %d out of range", __LINE__, ri));
2447 
2448 	*msr = ri;
2449 
2450 	return (0);
2451 }
2452 
2453 static int
2454 iap_read_pmc(int cpu, int ri, pmc_value_t *v)
2455 {
2456 	struct pmc *pm;
2457 	pmc_value_t tmp;
2458 
2459 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2460 	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
2461 	KASSERT(ri >= 0 && ri < core_iap_npmc,
2462 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
2463 
2464 	pm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
2465 
2466 	KASSERT(pm,
2467 	    ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
2468 		ri));
2469 
2470 	tmp = rdpmc(ri);
2471 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2472 		*v = iap_perfctr_value_to_reload_count(tmp);
2473 	else
2474 		*v = tmp & ((1ULL << core_iap_width) - 1);
2475 
2476 	PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
2477 	    ri, *v);
2478 
2479 	return (0);
2480 }
2481 
2482 static int
2483 iap_release_pmc(int cpu, int ri, struct pmc *pm)
2484 {
2485 	(void) pm;
2486 
2487 	PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,
2488 	    pm);
2489 
2490 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2491 	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
2492 	KASSERT(ri >= 0 && ri < core_iap_npmc,
2493 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
2494 
2495 	KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc
2496 	    == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__));
2497 
2498 	return (0);
2499 }
2500 
2501 static int
2502 iap_start_pmc(int cpu, int ri)
2503 {
2504 	struct pmc *pm;
2505 	uint32_t evsel;
2506 	struct core_cpu *cc;
2507 
2508 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2509 	    ("[core,%d] illegal CPU value %d", __LINE__, cpu));
2510 	KASSERT(ri >= 0 && ri < core_iap_npmc,
2511 	    ("[core,%d] illegal row-index %d", __LINE__, ri));
2512 
2513 	cc = core_pcpu[cpu];
2514 	pm = cc->pc_corepmcs[ri].phw_pmc;
2515 
2516 	KASSERT(pm,
2517 	    ("[core,%d] starting cpu%d,ri%d with no pmc configured",
2518 		__LINE__, cpu, ri));
2519 
2520 	PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);
2521 
2522 	evsel = pm->pm_md.pm_iap.pm_iap_evsel;
2523 
2524 	PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
2525 	    cpu, ri, IAP_EVSEL0 + ri, evsel);
2526 
2527 	/* Event specific configuration. */
2528 	switch (pm->pm_event) {
2529 	case PMC_EV_IAP_EVENT_B7H_01H:
2530 		wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp);
2531 		break;
2532 	case PMC_EV_IAP_EVENT_BBH_01H:
2533 		wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp);
2534 		break;
2535 	default:
2536 		break;
2537 	}
2538 
2539 	wrmsr(IAP_EVSEL0 + ri, evsel | IAP_EN);
2540 
2541 	if (core_cputype == PMC_CPU_INTEL_CORE)
2542 		return (0);
2543 
2544 	do {
2545 		cc->pc_resync = 0;
2546 		cc->pc_globalctrl |= (1ULL << ri);
2547 		wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
2548 	} while (cc->pc_resync != 0);
2549 
2550 	return (0);
2551 }
2552 
2553 static int
2554 iap_stop_pmc(int cpu, int ri)
2555 {
2556 	struct pmc *pm;
2557 	struct core_cpu *cc;
2558 	uint64_t msr;
2559 
2560 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2561 	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
2562 	KASSERT(ri >= 0 && ri < core_iap_npmc,
2563 	    ("[core,%d] illegal row index %d", __LINE__, ri));
2564 
2565 	cc = core_pcpu[cpu];
2566 	pm = cc->pc_corepmcs[ri].phw_pmc;
2567 
2568 	KASSERT(pm,
2569 	    ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
2570 		cpu, ri));
2571 
2572 	PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
2573 
2574 	msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
2575 	wrmsr(IAP_EVSEL0 + ri, msr);	/* stop hw */
2576 
2577 	if (core_cputype == PMC_CPU_INTEL_CORE)
2578 		return (0);
2579 
2580 	msr = 0;
2581 	do {
2582 		cc->pc_resync = 0;
2583 		cc->pc_globalctrl &= ~(1ULL << ri);
2584 		msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
2585 		wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
2586 	} while (cc->pc_resync != 0);
2587 
2588 	return (0);
2589 }
2590 
2591 static int
2592 iap_write_pmc(int cpu, int ri, pmc_value_t v)
2593 {
2594 	struct pmc *pm;
2595 	struct core_cpu *cc;
2596 
2597 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
2598 	    ("[core,%d] illegal cpu value %d", __LINE__, cpu));
2599 	KASSERT(ri >= 0 && ri < core_iap_npmc,
2600 	    ("[core,%d] illegal row index %d", __LINE__, ri));
2601 
2602 	cc = core_pcpu[cpu];
2603 	pm = cc->pc_corepmcs[ri].phw_pmc;
2604 
2605 	KASSERT(pm,
2606 	    ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
2607 		cpu, ri));
2608 
2609 	PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
2610 	    IAP_PMC0 + ri, v);
2611 
2612 	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2613 		v = iap_reload_count_to_perfctr_value(v);
2614 
2615 	/*
2616 	 * Write the new value to the counter.  The counter will be in
2617 	 * a stopped state when the pcd_write() entry point is called.
2618 	 */
2619 
2620 	wrmsr(IAP_PMC0 + ri, v & ((1ULL << core_iap_width) - 1));
2621 
2622 	return (0);
2623 }
2624 
2625 
2626 static void
2627 iap_initialize(struct pmc_mdep *md, int maxcpu, int npmc, int pmcwidth,
2628     int flags)
2629 {
2630 	struct pmc_classdep *pcd;
2631 
2632 	KASSERT(md != NULL, ("[iap,%d] md is NULL", __LINE__));
2633 
2634 	PMCDBG0(MDP,INI,1, "iap-initialize");
2635 
2636 	/* Remember the set of architectural events supported. */
2637 	core_architectural_events = ~flags;
2638 
2639 	pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP];
2640 
2641 	pcd->pcd_caps	= IAP_PMC_CAPS;
2642 	pcd->pcd_class	= PMC_CLASS_IAP;
2643 	pcd->pcd_num	= npmc;
2644 	pcd->pcd_ri	= md->pmd_npmc;
2645 	pcd->pcd_width	= pmcwidth;
2646 
2647 	pcd->pcd_allocate_pmc	= iap_allocate_pmc;
2648 	pcd->pcd_config_pmc	= iap_config_pmc;
2649 	pcd->pcd_describe	= iap_describe;
2650 	pcd->pcd_get_config	= iap_get_config;
2651 	pcd->pcd_get_msr	= iap_get_msr;
2652 	pcd->pcd_pcpu_fini	= core_pcpu_fini;
2653 	pcd->pcd_pcpu_init	= core_pcpu_init;
2654 	pcd->pcd_read_pmc	= iap_read_pmc;
2655 	pcd->pcd_release_pmc	= iap_release_pmc;
2656 	pcd->pcd_start_pmc	= iap_start_pmc;
2657 	pcd->pcd_stop_pmc	= iap_stop_pmc;
2658 	pcd->pcd_write_pmc	= iap_write_pmc;
2659 
2660 	md->pmd_npmc	       += npmc;
2661 }
2662 
2663 static int
2664 core_intr(int cpu, struct trapframe *tf)
2665 {
2666 	pmc_value_t v;
2667 	struct pmc *pm;
2668 	struct core_cpu *cc;
2669 	int error, found_interrupt, ri;
2670 	uint64_t msr;
2671 
2672 	PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
2673 	    TRAPF_USERMODE(tf));
2674 
2675 	found_interrupt = 0;
2676 	cc = core_pcpu[cpu];
2677 
2678 	for (ri = 0; ri < core_iap_npmc; ri++) {
2679 
2680 		if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL ||
2681 		    !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2682 			continue;
2683 
2684 		if (!iap_pmc_has_overflowed(ri))
2685 			continue;
2686 
2687 		found_interrupt = 1;
2688 
2689 		if (pm->pm_state != PMC_STATE_RUNNING)
2690 			continue;
2691 
2692 		error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
2693 		    TRAPF_USERMODE(tf));
2694 
2695 		v = pm->pm_sc.pm_reloadcount;
2696 		v = iap_reload_count_to_perfctr_value(v);
2697 
2698 		/*
2699 		 * Stop the counter, reload it but only restart it if
2700 		 * the PMC is not stalled.
2701 		 */
2702 		msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
2703 		wrmsr(IAP_EVSEL0 + ri, msr);
2704 		wrmsr(IAP_PMC0 + ri, v);
2705 
2706 		if (error)
2707 			continue;
2708 
2709 		wrmsr(IAP_EVSEL0 + ri, msr | (pm->pm_md.pm_iap.pm_iap_evsel |
2710 					      IAP_EN));
2711 	}
2712 
2713 	if (found_interrupt)
2714 		lapic_reenable_pmc();
2715 
2716 	atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed :
2717 	    &pmc_stats.pm_intr_ignored, 1);
2718 
2719 	return (found_interrupt);
2720 }
2721 
2722 static int
2723 core2_intr(int cpu, struct trapframe *tf)
2724 {
2725 	int error, found_interrupt, n;
2726 	uint64_t flag, intrstatus, intrenable, msr;
2727 	struct pmc *pm;
2728 	struct core_cpu *cc;
2729 	pmc_value_t v;
2730 
2731 	PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
2732 	    TRAPF_USERMODE(tf));
2733 
2734 	/*
2735 	 * The IA_GLOBAL_STATUS (MSR 0x38E) register indicates which
2736 	 * PMCs have a pending PMI interrupt.  We take a 'snapshot' of
2737 	 * the current set of interrupting PMCs and process these
2738 	 * after stopping them.
2739 	 */
2740 	intrstatus = rdmsr(IA_GLOBAL_STATUS);
2741 	intrenable = intrstatus & core_pmcmask;
2742 
2743 	PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu,
2744 	    (uintmax_t) intrstatus);
2745 
2746 	found_interrupt = 0;
2747 	cc = core_pcpu[cpu];
2748 
2749 	KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));
2750 
2751 	cc->pc_globalctrl &= ~intrenable;
2752 	cc->pc_resync = 1;	/* MSRs now potentially out of sync. */
2753 
2754 	/*
2755 	 * Stop PMCs and clear overflow status bits.
2756 	 */
2757 	msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
2758 	wrmsr(IA_GLOBAL_CTRL, msr);
2759 	wrmsr(IA_GLOBAL_OVF_CTRL, intrenable |
2760 	    IA_GLOBAL_STATUS_FLAG_OVFBUF |
2761 	    IA_GLOBAL_STATUS_FLAG_CONDCHG);
2762 
2763 	/*
2764 	 * Look for interrupts from fixed function PMCs.
2765 	 */
2766 	for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc;
2767 	     n++, flag <<= 1) {
2768 
2769 		if ((intrstatus & flag) == 0)
2770 			continue;
2771 
2772 		found_interrupt = 1;
2773 
2774 		pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc;
2775 		if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
2776 		    !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2777 			continue;
2778 
2779 		error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
2780 		    TRAPF_USERMODE(tf));
2781 		if (error)
2782 			intrenable &= ~flag;
2783 
2784 		v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
2785 
2786 		/* Reload sampling count. */
2787 		wrmsr(IAF_CTR0 + n, v);
2788 
2789 		PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", cpu,
2790 		    error, (uintmax_t) v, (uintmax_t) rdpmc(IAF_RI_TO_MSR(n)));
2791 	}
2792 
2793 	/*
2794 	 * Process interrupts from the programmable counters.
2795 	 */
2796 	for (n = 0, flag = 1; n < core_iap_npmc; n++, flag <<= 1) {
2797 		if ((intrstatus & flag) == 0)
2798 			continue;
2799 
2800 		found_interrupt = 1;
2801 
2802 		pm = cc->pc_corepmcs[n].phw_pmc;
2803 		if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
2804 		    !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2805 			continue;
2806 
2807 		error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
2808 		    TRAPF_USERMODE(tf));
2809 		if (error)
2810 			intrenable &= ~flag;
2811 
2812 		v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
2813 
2814 		PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error,
2815 		    (uintmax_t) v);
2816 
2817 		/* Reload sampling count. */
2818 		wrmsr(IAP_PMC0 + n, v);
2819 	}
2820 
2821 	/*
2822 	 * Reenable all non-stalled PMCs.
2823 	 */
2824 	PMCDBG2(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu,
2825 	    (uintmax_t) intrenable);
2826 
2827 	cc->pc_globalctrl |= intrenable;
2828 
2829 	wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl & IA_GLOBAL_CTRL_MASK);
2830 
2831 	PMCDBG5(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx "
2832 	    "ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL),
2833 	    (uintmax_t) rdmsr(IA_GLOBAL_CTRL),
2834 	    (uintmax_t) rdmsr(IA_GLOBAL_STATUS),
2835 	    (uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL));
2836 
2837 	if (found_interrupt)
2838 		lapic_reenable_pmc();
2839 
2840 	atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed :
2841 	    &pmc_stats.pm_intr_ignored, 1);
2842 
2843 	return (found_interrupt);
2844 }
2845 
2846 int
2847 pmc_core_initialize(struct pmc_mdep *md, int maxcpu, int version_override)
2848 {
2849 	int cpuid[CORE_CPUID_REQUEST_SIZE];
2850 	int ipa_version, flags, nflags;
2851 
2852 	do_cpuid(CORE_CPUID_REQUEST, cpuid);
2853 
2854 	ipa_version = (version_override > 0) ? version_override :
2855 	    cpuid[CORE_CPUID_EAX] & 0xFF;
2856 	core_cputype = md->pmd_cputype;
2857 
2858 	PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d ipa-version=%d",
2859 	    core_cputype, maxcpu, ipa_version);
2860 
2861 	if (ipa_version < 1 || ipa_version > 3 ||
2862 	    (core_cputype != PMC_CPU_INTEL_CORE && ipa_version == 1)) {
2863 		/* Unknown PMC architecture. */
2864 		printf("hwpc_core: unknown PMC architecture: %d\n",
2865 		    ipa_version);
2866 		return (EPROGMISMATCH);
2867 	}
2868 
2869 	core_pmcmask = 0;
2870 
2871 	/*
2872 	 * Initialize programmable counters.
2873 	 */
2874 	core_iap_npmc = (cpuid[CORE_CPUID_EAX] >> 8) & 0xFF;
2875 	core_iap_width = (cpuid[CORE_CPUID_EAX] >> 16) & 0xFF;
2876 
2877 	core_pmcmask |= ((1ULL << core_iap_npmc) - 1);
2878 
2879 	nflags = (cpuid[CORE_CPUID_EAX] >> 24) & 0xFF;
2880 	flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1);
2881 
2882 	iap_initialize(md, maxcpu, core_iap_npmc, core_iap_width, flags);
2883 
2884 	/*
2885 	 * Initialize fixed function counters, if present.
2886 	 */
2887 	if (core_cputype != PMC_CPU_INTEL_CORE) {
2888 		core_iaf_ri = core_iap_npmc;
2889 		core_iaf_npmc = cpuid[CORE_CPUID_EDX] & 0x1F;
2890 		core_iaf_width = (cpuid[CORE_CPUID_EDX] >> 5) & 0xFF;
2891 
2892 		iaf_initialize(md, maxcpu, core_iaf_npmc, core_iaf_width);
2893 		core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET;
2894 	}
2895 
2896 	PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask,
2897 	    core_iaf_ri);
2898 
2899 	core_pcpu = malloc(sizeof(*core_pcpu) * maxcpu, M_PMC,
2900 	    M_ZERO | M_WAITOK);
2901 
2902 	/*
2903 	 * Choose the appropriate interrupt handler.
2904 	 */
2905 	if (ipa_version == 1)
2906 		md->pmd_intr = core_intr;
2907 	else
2908 		md->pmd_intr = core2_intr;
2909 
2910 	md->pmd_pcpu_fini = NULL;
2911 	md->pmd_pcpu_init = NULL;
2912 
2913 	return (0);
2914 }
2915 
2916 void
2917 pmc_core_finalize(struct pmc_mdep *md)
2918 {
2919 	PMCDBG0(MDP,INI,1, "core-finalize");
2920 
2921 	free(core_pcpu, M_PMC);
2922 	core_pcpu = NULL;
2923 }
2924