1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #define PSMI_1_5 29 #include <sys/smp_impldefs.h> 30 #include <sys/psm.h> 31 #include <sys/psm_modctl.h> 32 #include <sys/pit.h> 33 #include <sys/cmn_err.h> 34 #include <sys/strlog.h> 35 #include <sys/clock.h> 36 #include <sys/debug.h> 37 #include <sys/rtc.h> 38 #include <sys/x86_archext.h> 39 #include <sys/cpupart.h> 40 #include <sys/cpuvar.h> 41 #include <sys/cmt.h> 42 #include <sys/cpu.h> 43 #include <sys/disp.h> 44 #include <sys/archsystm.h> 45 #include <sys/machsystm.h> 46 #include <sys/sysmacros.h> 47 #include <sys/param.h> 48 #include <sys/promif.h> 49 #include <sys/mach_intr.h> 50 #include <vm/hat_i86.h> 51 #include <sys/kdi_machimpl.h> 52 53 #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) 54 55 /* 56 * Local function prototypes 57 */ 58 static int mp_disable_intr(processorid_t cpun); 59 static void mp_enable_intr(processorid_t cpun); 60 static void mach_init(); 61 static void mach_picinit(); 62 static uint64_t mach_calchz(uint32_t pit_counter, uint64_t *processor_clks); 63 static int machhztomhz(uint64_t cpu_freq_hz); 64 static uint64_t mach_getcpufreq(void); 65 static void mach_fixcpufreq(void); 66 static int mach_clkinit(int, int *); 67 static void mach_smpinit(void); 68 static int mach_softlvl_to_vect(int ipl); 69 static void mach_get_platform(int owner); 70 static void mach_construct_info(); 71 static int mach_translate_irq(dev_info_t *dip, int irqno); 72 static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 73 psm_intr_op_t, int *); 74 static void mach_notify_error(int level, char *errmsg); 75 static hrtime_t dummy_hrtime(void); 76 static void dummy_scalehrtime(hrtime_t *); 77 static void cpu_idle(void); 78 static void cpu_wakeup(cpu_t *, int); 79 static void cpu_idle_mwait(void); 80 static void cpu_wakeup_mwait(cpu_t *, int); 81 /* 82 * External reference functions 83 */ 84 extern void return_instr(); 85 extern uint64_t freq_tsc(uint32_t *); 86 #if defined(__i386) 87 extern uint64_t freq_notsc(uint32_t *); 88 #endif 89 extern void pc_gethrestime(timestruc_t *); 90 extern int cpuid_get_coreid(cpu_t *); 91 extern int cpuid_get_chipid(cpu_t *); 92 93 /* 94 * PSM functions initialization 95 */ 96 void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; 97 void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; 98 void (*psm_notifyf)(int) = (void (*)(int))return_instr; 99 void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; 100 void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; 101 void (*psminitf)() = mach_init; 102 void (*picinitf)() = return_instr; 103 int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; 104 int (*ap_mlsetup)() = (int (*)(void))return_instr; 105 void (*send_dirintf)() = return_instr; 106 void (*setspl)(int) = (void (*)(int))return_instr; 107 int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 108 int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; 109 void (*kdisetsoftint)(int, struct av_softinfo *)= 110 (void (*)(int, struct av_softinfo *))return_instr; 111 void (*setsoftint)(int, struct av_softinfo *)= 112 (void (*)(int, struct av_softinfo *))return_instr; 113 int (*slvltovect)(int) = (int (*)(int))return_instr; 114 int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; 115 void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; 116 int (*psm_disable_intr)(int) = mp_disable_intr; 117 void (*psm_enable_intr)(int) = mp_enable_intr; 118 hrtime_t (*gethrtimef)(void) = dummy_hrtime; 119 hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; 120 void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; 121 int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; 122 void (*gethrestimef)(timestruc_t *) = pc_gethrestime; 123 void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; 124 int (*psm_get_clockirq)(int) = NULL; 125 int (*psm_get_ipivect)(int, int) = NULL; 126 127 int (*psm_clkinit)(int) = NULL; 128 void (*psm_timer_reprogram)(hrtime_t) = NULL; 129 void (*psm_timer_enable)(void) = NULL; 130 void (*psm_timer_disable)(void) = NULL; 131 void (*psm_post_cyclic_setup)(void *arg) = NULL; 132 int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, 133 int *) = mach_intr_ops; 134 135 void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; 136 void (*hrtime_tick)(void) = return_instr; 137 138 int tsc_gethrtime_enable = 1; 139 int tsc_gethrtime_initted = 0; 140 141 /* 142 * Local Static Data 143 */ 144 static struct psm_ops mach_ops; 145 static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; 146 static ushort_t mach_ver[4] = {0, 0, 0, 0}; 147 148 /* 149 * If non-zero, idle cpus will become "halted" when there's 150 * no work to do. 151 */ 152 int idle_cpu_use_hlt = 1; 153 154 /* 155 * If non-zero, idle cpus will use mwait if available to halt instead of hlt. 156 */ 157 int idle_cpu_prefer_mwait = 1; 158 159 160 /*ARGSUSED*/ 161 int 162 pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) 163 { 164 switch (hw) { 165 case PGHW_IPIPE: 166 if (x86_feature & (X86_HTT)) { 167 /* 168 * Hyper-threading is SMT 169 */ 170 return (1); 171 } else { 172 return (0); 173 } 174 case PGHW_CHIP: 175 if (x86_feature & (X86_CMP|X86_HTT)) 176 return (1); 177 else 178 return (0); 179 case PGHW_CACHE: 180 if (cpuid_get_ncpu_sharing_last_cache(cp) > 1) 181 return (1); 182 else 183 return (0); 184 default: 185 return (0); 186 } 187 } 188 189 /* 190 * Compare two CPUs and see if they have a pghw_type_t sharing relationship 191 * If pghw_type_t is an unsupported hardware type, then return -1 192 */ 193 int 194 pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) 195 { 196 id_t pgp_a, pgp_b; 197 198 pgp_a = pg_plat_hw_instance_id(cpu_a, hw); 199 pgp_b = pg_plat_hw_instance_id(cpu_b, hw); 200 201 if (pgp_a == -1 || pgp_b == -1) 202 return (-1); 203 204 return (pgp_a == pgp_b); 205 } 206 207 /* 208 * Return a physical instance identifier for known hardware sharing 209 * relationships 210 */ 211 id_t 212 pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) 213 { 214 switch (hw) { 215 case PGHW_IPIPE: 216 return (cpuid_get_coreid(cpu)); 217 case PGHW_CACHE: 218 return (cpuid_get_last_lvl_cacheid(cpu)); 219 case PGHW_CHIP: 220 return (cpuid_get_chipid(cpu)); 221 default: 222 return (-1); 223 } 224 } 225 226 int 227 pg_plat_hw_level(pghw_type_t hw) 228 { 229 int i; 230 static pghw_type_t hw_hier[] = { 231 PGHW_IPIPE, 232 PGHW_CACHE, 233 PGHW_CHIP, 234 PGHW_NUM_COMPONENTS 235 }; 236 237 for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { 238 if (hw_hier[i] == hw) 239 return (i); 240 } 241 return (-1); 242 } 243 244 /* 245 * Return 1 if CMT load balancing policies should be 246 * implemented across instances of the specified hardware 247 * sharing relationship. 248 */ 249 int 250 pg_plat_cmt_load_bal_hw(pghw_type_t hw) 251 { 252 if (hw == PGHW_IPIPE || 253 hw == PGHW_FPU || 254 hw == PGHW_CHIP || 255 hw == PGHW_CACHE) 256 return (1); 257 else 258 return (0); 259 } 260 261 262 /* 263 * Return 1 if thread affinity polices should be implemented 264 * for instances of the specifed hardware sharing relationship. 265 */ 266 int 267 pg_plat_cmt_affinity_hw(pghw_type_t hw) 268 { 269 if (hw == PGHW_CACHE) 270 return (1); 271 else 272 return (0); 273 } 274 275 id_t 276 pg_plat_get_core_id(cpu_t *cpu) 277 { 278 return ((id_t)cpuid_get_coreid(cpu)); 279 } 280 281 void 282 cmp_set_nosteal_interval(void) 283 { 284 /* Set the nosteal interval (used by disp_getbest()) to 100us */ 285 nosteal_nsec = 100000UL; 286 } 287 288 /* 289 * Routine to ensure initial callers to hrtime gets 0 as return 290 */ 291 static hrtime_t 292 dummy_hrtime(void) 293 { 294 return (0); 295 } 296 297 /* ARGSUSED */ 298 static void 299 dummy_scalehrtime(hrtime_t *ticks) 300 {} 301 302 /* 303 * Idle the present CPU until awoken via an interrupt 304 */ 305 static void 306 cpu_idle(void) 307 { 308 cpu_t *cpup = CPU; 309 processorid_t cpun = cpup->cpu_id; 310 cpupart_t *cp = cpup->cpu_part; 311 int hset_update = 1; 312 313 /* 314 * If this CPU is online, and there's multiple CPUs 315 * in the system, then we should notate our halting 316 * by adding ourselves to the partition's halted CPU 317 * bitmap. This allows other CPUs to find/awaken us when 318 * work becomes available. 319 */ 320 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 321 hset_update = 0; 322 323 /* 324 * Add ourselves to the partition's halted CPUs bitmask 325 * and set our HALTED flag, if necessary. 326 * 327 * When a thread becomes runnable, it is placed on the queue 328 * and then the halted cpuset is checked to determine who 329 * (if anyone) should be awoken. We therefore need to first 330 * add ourselves to the halted cpuset, and and then check if there 331 * is any work available. 332 * 333 * Note that memory barriers after updating the HALTED flag 334 * are not necessary since an atomic operation (updating the bitmap) 335 * immediately follows. On x86 the atomic operation acts as a 336 * memory barrier for the update of cpu_disp_flags. 337 */ 338 if (hset_update) { 339 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 340 CPUSET_ATOMIC_ADD(cp->cp_mach->mc_haltset, cpun); 341 } 342 343 /* 344 * Check to make sure there's really nothing to do. 345 * Work destined for this CPU may become available after 346 * this check. We'll be notified through the clearing of our 347 * bit in the halted CPU bitmask, and a poke. 348 */ 349 if (disp_anywork()) { 350 if (hset_update) { 351 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 352 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 353 } 354 return; 355 } 356 357 /* 358 * We're on our way to being halted. 359 * 360 * Disable interrupts now, so that we'll awaken immediately 361 * after halting if someone tries to poke us between now and 362 * the time we actually halt. 363 * 364 * We check for the presence of our bit after disabling interrupts. 365 * If it's cleared, we'll return. If the bit is cleared after 366 * we check then the poke will pop us out of the halted state. 367 * 368 * This means that the ordering of the poke and the clearing 369 * of the bit by cpu_wakeup is important. 370 * cpu_wakeup() must clear, then poke. 371 * cpu_idle() must disable interrupts, then check for the bit. 372 */ 373 cli(); 374 375 if (hset_update && !CPU_IN_SET(cp->cp_mach->mc_haltset, cpun)) { 376 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 377 sti(); 378 return; 379 } 380 381 /* 382 * The check for anything locally runnable is here for performance 383 * and isn't needed for correctness. disp_nrunnable ought to be 384 * in our cache still, so it's inexpensive to check, and if there 385 * is anything runnable we won't have to wait for the poke. 386 */ 387 if (cpup->cpu_disp->disp_nrunnable != 0) { 388 if (hset_update) { 389 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 390 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 391 } 392 sti(); 393 return; 394 } 395 396 mach_cpu_idle(); 397 398 /* 399 * We're no longer halted 400 */ 401 if (hset_update) { 402 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 403 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 404 } 405 } 406 407 408 /* 409 * If "cpu" is halted, then wake it up clearing its halted bit in advance. 410 * Otherwise, see if other CPUs in the cpu partition are halted and need to 411 * be woken up so that they can steal the thread we placed on this CPU. 412 * This function is only used on MP systems. 413 */ 414 static void 415 cpu_wakeup(cpu_t *cpu, int bound) 416 { 417 uint_t cpu_found; 418 int result; 419 cpupart_t *cp; 420 421 cp = cpu->cpu_part; 422 if (CPU_IN_SET(cp->cp_mach->mc_haltset, cpu->cpu_id)) { 423 /* 424 * Clear the halted bit for that CPU since it will be 425 * poked in a moment. 426 */ 427 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpu->cpu_id); 428 /* 429 * We may find the current CPU present in the halted cpuset 430 * if we're in the context of an interrupt that occurred 431 * before we had a chance to clear our bit in cpu_idle(). 432 * Poking ourself is obviously unnecessary, since if 433 * we're here, we're not halted. 434 */ 435 if (cpu != CPU) 436 poke_cpu(cpu->cpu_id); 437 return; 438 } else { 439 /* 440 * This cpu isn't halted, but it's idle or undergoing a 441 * context switch. No need to awaken anyone else. 442 */ 443 if (cpu->cpu_thread == cpu->cpu_idle_thread || 444 cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) 445 return; 446 } 447 448 /* 449 * No need to wake up other CPUs if the thread we just enqueued 450 * is bound. 451 */ 452 if (bound) 453 return; 454 455 456 /* 457 * See if there's any other halted CPUs. If there are, then 458 * select one, and awaken it. 459 * It's possible that after we find a CPU, somebody else 460 * will awaken it before we get the chance. 461 * In that case, look again. 462 */ 463 do { 464 CPUSET_FIND(cp->cp_mach->mc_haltset, cpu_found); 465 if (cpu_found == CPUSET_NOTINSET) 466 return; 467 468 ASSERT(cpu_found >= 0 && cpu_found < NCPU); 469 CPUSET_ATOMIC_XDEL(cp->cp_mach->mc_haltset, cpu_found, result); 470 } while (result < 0); 471 472 if (cpu_found != CPU->cpu_id) 473 poke_cpu(cpu_found); 474 } 475 476 /* 477 * Idle the present CPU until awoken via touching its monitored line 478 */ 479 static void 480 cpu_idle_mwait(void) 481 { 482 volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; 483 cpu_t *cpup = CPU; 484 processorid_t cpun = cpup->cpu_id; 485 cpupart_t *cp = cpup->cpu_part; 486 int hset_update = 1; 487 488 /* 489 * Set our mcpu_mwait here, so we can tell if anyone trys to 490 * wake us between now and when we call mwait. No other cpu will 491 * attempt to set our mcpu_mwait until we add ourself to the haltset. 492 */ 493 *mcpu_mwait = MWAIT_HALTED; 494 495 /* 496 * If this CPU is online, and there's multiple CPUs 497 * in the system, then we should notate our halting 498 * by adding ourselves to the partition's halted CPU 499 * bitmap. This allows other CPUs to find/awaken us when 500 * work becomes available. 501 */ 502 if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) 503 hset_update = 0; 504 505 /* 506 * Add ourselves to the partition's halted CPUs bitmask 507 * and set our HALTED flag, if necessary. 508 * 509 * When a thread becomes runnable, it is placed on the queue 510 * and then the halted cpuset is checked to determine who 511 * (if anyone) should be awoken. We therefore need to first 512 * add ourselves to the halted cpuset, and and then check if there 513 * is any work available. 514 * 515 * Note that memory barriers after updating the HALTED flag 516 * are not necessary since an atomic operation (updating the bitmap) 517 * immediately follows. On x86 the atomic operation acts as a 518 * memory barrier for the update of cpu_disp_flags. 519 */ 520 if (hset_update) { 521 cpup->cpu_disp_flags |= CPU_DISP_HALTED; 522 CPUSET_ATOMIC_ADD(cp->cp_mach->mc_haltset, cpun); 523 } 524 525 /* 526 * Check to make sure there's really nothing to do. 527 * Work destined for this CPU may become available after 528 * this check. We'll be notified through the clearing of our 529 * bit in the halted CPU bitmask, and a write to our mcpu_mwait. 530 * 531 * disp_anywork() checks disp_nrunnable, so we do not have to later. 532 */ 533 if (disp_anywork()) { 534 if (hset_update) { 535 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 536 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 537 } 538 return; 539 } 540 541 /* 542 * We're on our way to being halted. 543 * To avoid a lost wakeup, arm the monitor before checking if another 544 * cpu wrote to mcpu_mwait to wake us up. 545 */ 546 i86_monitor(mcpu_mwait, 0, 0); 547 if (*mcpu_mwait == MWAIT_HALTED) { 548 tlb_going_idle(); 549 i86_mwait(0, 0); 550 tlb_service(); 551 } 552 553 /* 554 * We're no longer halted 555 */ 556 if (hset_update) { 557 cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; 558 CPUSET_ATOMIC_DEL(cp->cp_mach->mc_haltset, cpun); 559 } 560 } 561 562 /* 563 * If "cpu" is halted in mwait, then wake it up clearing its halted bit in 564 * advance. Otherwise, see if other CPUs in the cpu partition are halted and 565 * need to be woken up so that they can steal the thread we placed on this CPU. 566 * This function is only used on MP systems. 567 */ 568 static void 569 cpu_wakeup_mwait(cpu_t *cp, int bound) 570 { 571 cpupart_t *cpu_part; 572 uint_t cpu_found; 573 int result; 574 575 cpu_part = cp->cpu_part; 576 577 /* 578 * Clear the halted bit for that CPU since it will be woken up 579 * in a moment. 580 */ 581 if (CPU_IN_SET(cpu_part->cp_mach->mc_haltset, cp->cpu_id)) { 582 /* 583 * Clear the halted bit for that CPU since it will be 584 * poked in a moment. 585 */ 586 CPUSET_ATOMIC_DEL(cpu_part->cp_mach->mc_haltset, cp->cpu_id); 587 /* 588 * We may find the current CPU present in the halted cpuset 589 * if we're in the context of an interrupt that occurred 590 * before we had a chance to clear our bit in cpu_idle(). 591 * Waking ourself is obviously unnecessary, since if 592 * we're here, we're not halted. 593 * 594 * monitor/mwait wakeup via writing to our cache line is 595 * harmless and less expensive than always checking if we 596 * are waking ourself which is an uncommon case. 597 */ 598 MWAIT_WAKEUP(cp); /* write to monitored line */ 599 return; 600 } else { 601 /* 602 * This cpu isn't halted, but it's idle or undergoing a 603 * context switch. No need to awaken anyone else. 604 */ 605 if (cp->cpu_thread == cp->cpu_idle_thread || 606 cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) 607 return; 608 } 609 610 /* 611 * No need to wake up other CPUs if the thread we just enqueued 612 * is bound. 613 */ 614 if (bound) 615 return; 616 617 618 /* 619 * See if there's any other halted CPUs. If there are, then 620 * select one, and awaken it. 621 * It's possible that after we find a CPU, somebody else 622 * will awaken it before we get the chance. 623 * In that case, look again. 624 */ 625 do { 626 CPUSET_FIND(cpu_part->cp_mach->mc_haltset, cpu_found); 627 if (cpu_found == CPUSET_NOTINSET) 628 return; 629 630 ASSERT(cpu_found >= 0 && cpu_found < NCPU); 631 CPUSET_ATOMIC_XDEL(cpu_part->cp_mach->mc_haltset, cpu_found, 632 result); 633 } while (result < 0); 634 635 /* 636 * Do not check if cpu_found is ourself as monitor/mwait wakeup is 637 * cheap. 638 */ 639 MWAIT_WAKEUP(cpu[cpu_found]); /* write to monitored line */ 640 } 641 642 void (*cpu_pause_handler)(volatile char *) = NULL; 643 644 static int 645 mp_disable_intr(int cpun) 646 { 647 /* 648 * switch to the offline cpu 649 */ 650 affinity_set(cpun); 651 /* 652 * raise ipl to just below cross call 653 */ 654 splx(XC_MED_PIL-1); 655 /* 656 * set base spl to prevent the next swtch to idle from 657 * lowering back to ipl 0 658 */ 659 CPU->cpu_intr_actv |= (1 << (XC_MED_PIL-1)); 660 set_base_spl(); 661 affinity_clear(); 662 return (DDI_SUCCESS); 663 } 664 665 static void 666 mp_enable_intr(int cpun) 667 { 668 /* 669 * switch to the online cpu 670 */ 671 affinity_set(cpun); 672 /* 673 * clear the interrupt active mask 674 */ 675 CPU->cpu_intr_actv &= ~(1 << (XC_MED_PIL-1)); 676 set_base_spl(); 677 (void) spl0(); 678 affinity_clear(); 679 } 680 681 static void 682 mach_get_platform(int owner) 683 { 684 void **srv_opsp; 685 void **clt_opsp; 686 int i; 687 int total_ops; 688 689 /* fix up psm ops */ 690 srv_opsp = (void **)mach_set[0]; 691 clt_opsp = (void **)mach_set[owner]; 692 if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) 693 total_ops = sizeof (struct psm_ops_ver01) / 694 sizeof (void (*)(void)); 695 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) 696 /* no psm_notify_func */ 697 total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / 698 sizeof (void (*)(void)); 699 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) 700 /* no psm_timer funcs */ 701 total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / 702 sizeof (void (*)(void)); 703 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) 704 /* no psm_preshutdown function */ 705 total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / 706 sizeof (void (*)(void)); 707 else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) 708 /* no psm_preshutdown function */ 709 total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / 710 sizeof (void (*)(void)); 711 else 712 total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); 713 714 /* 715 * Save the version of the PSM module, in case we need to 716 * bahave differently based on version. 717 */ 718 mach_ver[0] = mach_ver[owner]; 719 720 for (i = 0; i < total_ops; i++) 721 if (clt_opsp[i] != NULL) 722 srv_opsp[i] = clt_opsp[i]; 723 } 724 725 static void 726 mach_construct_info() 727 { 728 struct psm_sw *swp; 729 int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; 730 int conflict_owner = 0; 731 732 if (psmsw->psw_forw == psmsw) 733 panic("No valid PSM modules found"); 734 mutex_enter(&psmsw_lock); 735 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 736 if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) 737 continue; 738 mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; 739 mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; 740 mach_cnt[swp->psw_infop->p_owner]++; 741 } 742 mutex_exit(&psmsw_lock); 743 744 mach_get_platform(PSM_OWN_SYS_DEFAULT); 745 746 /* check to see are there any conflicts */ 747 if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) 748 conflict_owner = PSM_OWN_EXCLUSIVE; 749 if (mach_cnt[PSM_OWN_OVERRIDE] > 1) 750 conflict_owner = PSM_OWN_OVERRIDE; 751 if (conflict_owner) { 752 /* remove all psm modules except uppc */ 753 cmn_err(CE_WARN, 754 "Conflicts detected on the following PSM modules:"); 755 mutex_enter(&psmsw_lock); 756 for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { 757 if (swp->psw_infop->p_owner == conflict_owner) 758 cmn_err(CE_WARN, "%s ", 759 swp->psw_infop->p_mach_idstring); 760 } 761 mutex_exit(&psmsw_lock); 762 cmn_err(CE_WARN, 763 "Setting the system back to SINGLE processor mode!"); 764 cmn_err(CE_WARN, 765 "Please edit /etc/mach to remove the invalid PSM module."); 766 return; 767 } 768 769 if (mach_set[PSM_OWN_EXCLUSIVE]) 770 mach_get_platform(PSM_OWN_EXCLUSIVE); 771 772 if (mach_set[PSM_OWN_OVERRIDE]) 773 mach_get_platform(PSM_OWN_OVERRIDE); 774 } 775 776 static void 777 mach_init() 778 { 779 struct psm_ops *pops; 780 781 mach_construct_info(); 782 783 pops = mach_set[0]; 784 785 /* register the interrupt and clock initialization rotuines */ 786 picinitf = mach_picinit; 787 clkinitf = mach_clkinit; 788 psm_get_clockirq = pops->psm_get_clockirq; 789 790 /* register the interrupt setup code */ 791 slvltovect = mach_softlvl_to_vect; 792 addspl = pops->psm_addspl; 793 delspl = pops->psm_delspl; 794 795 if (pops->psm_translate_irq) 796 psm_translate_irq = pops->psm_translate_irq; 797 if (pops->psm_intr_ops) 798 psm_intr_ops = pops->psm_intr_ops; 799 800 #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) 801 /* 802 * Time-of-day functionality now handled in TOD modules. 803 * (Warn about PSM modules that think that we're going to use 804 * their ops vectors.) 805 */ 806 if (pops->psm_tod_get) 807 cmn_err(CE_WARN, "obsolete psm_tod_get op %p", 808 (void *)pops->psm_tod_get); 809 810 if (pops->psm_tod_set) 811 cmn_err(CE_WARN, "obsolete psm_tod_set op %p", 812 (void *)pops->psm_tod_set); 813 #endif 814 815 if (pops->psm_notify_error) { 816 psm_notify_error = mach_notify_error; 817 notify_error = pops->psm_notify_error; 818 } 819 820 (*pops->psm_softinit)(); 821 822 /* 823 * Initialize the dispatcher's function hooks 824 * to enable CPU halting when idle. 825 * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle). 826 * Allocate monitor/mwait buffer for cpu0. 827 */ 828 if (idle_cpu_use_hlt) { 829 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) { 830 CPU->cpu_m.mcpu_mwait = mach_alloc_mwait(CPU); 831 idle_cpu = cpu_idle_mwait; 832 } else { 833 idle_cpu = cpu_idle; 834 } 835 } 836 837 mach_smpinit(); 838 } 839 840 /* 841 * Return a pointer to memory suitable for monitor/mwait use. Memory must be 842 * aligned as specified by cpuid (a cache line size). 843 */ 844 uint32_t * 845 mach_alloc_mwait(cpu_t *cp) 846 { 847 size_t mwait_size = cpuid_get_mwait_size(cp); 848 uint32_t *ret; 849 850 if (mwait_size < sizeof (uint32_t) || !ISP2(mwait_size)) 851 panic("Can't handle mwait size %ld", (long)mwait_size); 852 853 /* 854 * kmem_alloc() returns cache line size aligned data for mwait_size 855 * allocations. mwait_size is currently cache line sized. Neither 856 * of these implementation details are guarantied to be true in the 857 * future. 858 * 859 * First try allocating mwait_size as kmem_alloc() currently returns 860 * correctly aligned memory. If kmem_alloc() does not return 861 * mwait_size aligned memory, then use mwait_size ROUNDUP. 862 */ 863 ret = kmem_zalloc(mwait_size, KM_SLEEP); 864 if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) { 865 *ret = MWAIT_RUNNING; 866 return (ret); 867 } else { 868 kmem_free(ret, mwait_size); 869 ret = kmem_zalloc(mwait_size * 2, KM_SLEEP); 870 ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size); 871 *ret = MWAIT_RUNNING; 872 return (ret); 873 } 874 } 875 876 static void 877 mach_smpinit(void) 878 { 879 struct psm_ops *pops; 880 processorid_t cpu_id; 881 int cnt; 882 cpuset_t cpumask; 883 884 pops = mach_set[0]; 885 886 cpu_id = -1; 887 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 888 for (cnt = 0, CPUSET_ZERO(cpumask); cpu_id != -1; cnt++) { 889 CPUSET_ADD(cpumask, cpu_id); 890 cpu_id = (*pops->psm_get_next_processorid)(cpu_id); 891 } 892 893 mp_cpus = cpumask; 894 895 /* MP related routines */ 896 ap_mlsetup = pops->psm_post_cpu_start; 897 send_dirintf = pops->psm_send_ipi; 898 899 /* optional MP related routines */ 900 if (pops->psm_shutdown) 901 psm_shutdownf = pops->psm_shutdown; 902 if (pops->psm_preshutdown) 903 psm_preshutdownf = pops->psm_preshutdown; 904 if (pops->psm_notify_func) 905 psm_notifyf = pops->psm_notify_func; 906 if (pops->psm_set_idlecpu) 907 psm_set_idle_cpuf = pops->psm_set_idlecpu; 908 if (pops->psm_unset_idlecpu) 909 psm_unset_idle_cpuf = pops->psm_unset_idlecpu; 910 911 psm_clkinit = pops->psm_clkinit; 912 913 if (pops->psm_timer_reprogram) 914 psm_timer_reprogram = pops->psm_timer_reprogram; 915 916 if (pops->psm_timer_enable) 917 psm_timer_enable = pops->psm_timer_enable; 918 919 if (pops->psm_timer_disable) 920 psm_timer_disable = pops->psm_timer_disable; 921 922 if (pops->psm_post_cyclic_setup) 923 psm_post_cyclic_setup = pops->psm_post_cyclic_setup; 924 925 /* check for multiple cpu's */ 926 if (cnt < 2) 927 return; 928 929 /* check for MP platforms */ 930 if (pops->psm_cpu_start == NULL) 931 return; 932 933 /* 934 * Set the dispatcher hook to enable cpu "wake up" 935 * when a thread becomes runnable. 936 */ 937 if (idle_cpu_use_hlt) 938 if ((x86_feature & X86_MWAIT) && idle_cpu_prefer_mwait) 939 disp_enq_thread = cpu_wakeup_mwait; 940 else 941 disp_enq_thread = cpu_wakeup; 942 943 if (pops->psm_disable_intr) 944 psm_disable_intr = pops->psm_disable_intr; 945 if (pops->psm_enable_intr) 946 psm_enable_intr = pops->psm_enable_intr; 947 948 psm_get_ipivect = pops->psm_get_ipivect; 949 950 (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_hi_intr", 951 (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), 952 (caddr_t)X_CALL_HIPRI, NULL, NULL, NULL); 953 (void) add_avintr((void *)NULL, XC_MED_PIL, xc_serv, "xc_med_intr", 954 (*pops->psm_get_ipivect)(XC_MED_PIL, PSM_INTR_IPI_LO), 955 (caddr_t)X_CALL_MEDPRI, NULL, NULL, NULL); 956 957 (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); 958 } 959 960 static void 961 mach_picinit() 962 { 963 struct psm_ops *pops; 964 965 pops = mach_set[0]; 966 967 /* register the interrupt handlers */ 968 setlvl = pops->psm_intr_enter; 969 setlvlx = pops->psm_intr_exit; 970 971 /* initialize the interrupt hardware */ 972 (*pops->psm_picinit)(); 973 974 /* set interrupt mask for current ipl */ 975 setspl = pops->psm_setspl; 976 cli(); 977 setspl(CPU->cpu_pri); 978 } 979 980 uint_t cpu_freq; /* MHz */ 981 uint64_t cpu_freq_hz; /* measured (in hertz) */ 982 983 #define MEGA_HZ 1000000 984 985 static uint64_t 986 mach_calchz(uint32_t pit_counter, uint64_t *processor_clks) 987 { 988 uint64_t cpu_hz; 989 990 if ((pit_counter == 0) || (*processor_clks == 0) || 991 (*processor_clks > (((uint64_t)-1) / PIT_HZ))) 992 return (0); 993 994 cpu_hz = ((uint64_t)PIT_HZ * *processor_clks) / pit_counter; 995 996 return (cpu_hz); 997 } 998 999 static uint64_t 1000 mach_getcpufreq(void) 1001 { 1002 uint32_t pit_counter; 1003 uint64_t processor_clks; 1004 1005 if (x86_feature & X86_TSC) { 1006 /* 1007 * We have a TSC. freq_tsc() knows how to measure the number 1008 * of clock cycles sampled against the PIT. 1009 */ 1010 ulong_t flags = clear_int_flag(); 1011 processor_clks = freq_tsc(&pit_counter); 1012 restore_int_flag(flags); 1013 return (mach_calchz(pit_counter, &processor_clks)); 1014 } else if (x86_vendor == X86_VENDOR_Cyrix || x86_type == X86_TYPE_P5) { 1015 #if defined(__amd64) 1016 panic("mach_getcpufreq: no TSC!"); 1017 #elif defined(__i386) 1018 /* 1019 * We are a Cyrix based on a 6x86 core or an Intel Pentium 1020 * for which freq_notsc() knows how to measure the number of 1021 * elapsed clock cycles sampled against the PIT 1022 */ 1023 ulong_t flags = clear_int_flag(); 1024 processor_clks = freq_notsc(&pit_counter); 1025 restore_int_flag(flags); 1026 return (mach_calchz(pit_counter, &processor_clks)); 1027 #endif /* __i386 */ 1028 } 1029 1030 /* We do not know how to calculate cpu frequency for this cpu. */ 1031 return (0); 1032 } 1033 1034 /* 1035 * If the clock speed of a cpu is found to be reported incorrectly, do not add 1036 * to this array, instead improve the accuracy of the algorithm that determines 1037 * the clock speed of the processor or extend the implementation to support the 1038 * vendor as appropriate. This is here only to support adjusting the speed on 1039 * older slower processors that mach_fixcpufreq() would not be able to account 1040 * for otherwise. 1041 */ 1042 static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; 1043 1044 /* 1045 * On fast processors the clock frequency that is measured may be off by 1046 * a few MHz from the value printed on the part. This is a combination of 1047 * the factors that for such fast parts being off by this much is within 1048 * the tolerances for manufacture and because of the difficulties in the 1049 * measurement that can lead to small error. This function uses some 1050 * heuristics in order to tweak the value that was measured to match what 1051 * is most likely printed on the part. 1052 * 1053 * Some examples: 1054 * AMD Athlon 1000 mhz measured as 998 mhz 1055 * Intel Pentium III Xeon 733 mhz measured as 731 mhz 1056 * Intel Pentium IV 1500 mhz measured as 1495mhz 1057 * 1058 * If in the future this function is no longer sufficient to correct 1059 * for the error in the measurement, then the algorithm used to perform 1060 * the measurement will have to be improved in order to increase accuracy 1061 * rather than adding horrible and questionable kludges here. 1062 * 1063 * This is called after the cyclics subsystem because of the potential 1064 * that the heuristics within may give a worse estimate of the clock 1065 * frequency than the value that was measured. 1066 */ 1067 static void 1068 mach_fixcpufreq(void) 1069 { 1070 uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; 1071 1072 freq = (uint32_t)cpu_freq; 1073 1074 /* 1075 * Find the nearest integer multiple of 200/3 (about 66) MHz to the 1076 * measured speed taking into account that the 667 MHz parts were 1077 * the first to round-up. 1078 */ 1079 mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); 1080 near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); 1081 delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); 1082 1083 /* Find the nearest integer multiple of 50 MHz to the measured speed */ 1084 mul = (freq + 25) / 50; 1085 near50 = mul * 50; 1086 delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); 1087 1088 /* Find the closer of the two */ 1089 if (delta66 < delta50) { 1090 fixed = near66; 1091 delta = delta66; 1092 } else { 1093 fixed = near50; 1094 delta = delta50; 1095 } 1096 1097 if (fixed > INT_MAX) 1098 return; 1099 1100 /* 1101 * Some older parts have a core clock frequency that is not an 1102 * integral multiple of 50 or 66 MHz. Check if one of the old 1103 * clock frequencies is closer to the measured value than any 1104 * of the integral multiples of 50 an 66, and if so set fixed 1105 * and delta appropriately to represent the closest value. 1106 */ 1107 i = sizeof (x86_cpu_freq) / sizeof (int); 1108 while (i > 0) { 1109 i--; 1110 1111 if (x86_cpu_freq[i] <= freq) { 1112 mul = freq - x86_cpu_freq[i]; 1113 1114 if (mul < delta) { 1115 fixed = x86_cpu_freq[i]; 1116 delta = mul; 1117 } 1118 1119 break; 1120 } 1121 1122 mul = x86_cpu_freq[i] - freq; 1123 1124 if (mul < delta) { 1125 fixed = x86_cpu_freq[i]; 1126 delta = mul; 1127 } 1128 } 1129 1130 /* 1131 * Set a reasonable maximum for how much to correct the measured 1132 * result by. This check is here to prevent the adjustment made 1133 * by this function from being more harm than good. It is entirely 1134 * possible that in the future parts will be made that are not 1135 * integral multiples of 66 or 50 in clock frequency or that 1136 * someone may overclock a part to some odd frequency. If the 1137 * measured value is farther from the corrected value than 1138 * allowed, then assume the corrected value is in error and use 1139 * the measured value. 1140 */ 1141 if (6 < delta) 1142 return; 1143 1144 cpu_freq = (int)fixed; 1145 } 1146 1147 1148 static int 1149 machhztomhz(uint64_t cpu_freq_hz) 1150 { 1151 uint64_t cpu_mhz; 1152 1153 /* Round to nearest MHZ */ 1154 cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; 1155 1156 if (cpu_mhz > INT_MAX) 1157 return (0); 1158 1159 return ((int)cpu_mhz); 1160 1161 } 1162 1163 1164 static int 1165 mach_clkinit(int preferred_mode, int *set_mode) 1166 { 1167 struct psm_ops *pops; 1168 int resolution; 1169 1170 pops = mach_set[0]; 1171 1172 cpu_freq_hz = mach_getcpufreq(); 1173 1174 cpu_freq = machhztomhz(cpu_freq_hz); 1175 1176 if (!(x86_feature & X86_TSC) || (cpu_freq == 0)) 1177 tsc_gethrtime_enable = 0; 1178 1179 if (tsc_gethrtime_enable) { 1180 tsc_hrtimeinit(cpu_freq_hz); 1181 gethrtimef = tsc_gethrtime; 1182 gethrtimeunscaledf = tsc_gethrtimeunscaled; 1183 scalehrtimef = tsc_scalehrtime; 1184 hrtime_tick = tsc_tick; 1185 tsc_gethrtime_initted = 1; 1186 } else { 1187 if (pops->psm_hrtimeinit) 1188 (*pops->psm_hrtimeinit)(); 1189 gethrtimef = pops->psm_gethrtime; 1190 gethrtimeunscaledf = gethrtimef; 1191 /* scalehrtimef will remain dummy */ 1192 } 1193 1194 mach_fixcpufreq(); 1195 1196 if (mach_ver[0] >= PSM_INFO_VER01_3) { 1197 if ((preferred_mode == TIMER_ONESHOT) && 1198 (tsc_gethrtime_enable)) { 1199 1200 resolution = (*pops->psm_clkinit)(0); 1201 if (resolution != 0) { 1202 *set_mode = TIMER_ONESHOT; 1203 return (resolution); 1204 } 1205 1206 } 1207 1208 /* 1209 * either periodic mode was requested or could not set to 1210 * one-shot mode 1211 */ 1212 resolution = (*pops->psm_clkinit)(hz); 1213 /* 1214 * psm should be able to do periodic, so we do not check 1215 * for return value of psm_clkinit here. 1216 */ 1217 *set_mode = TIMER_PERIODIC; 1218 return (resolution); 1219 } else { 1220 /* 1221 * PSMI interface prior to PSMI_3 does not define a return 1222 * value for psm_clkinit, so the return value is ignored. 1223 */ 1224 (void) (*pops->psm_clkinit)(hz); 1225 *set_mode = TIMER_PERIODIC; 1226 return (nsec_per_tick); 1227 } 1228 } 1229 1230 1231 /*ARGSUSED*/ 1232 static int 1233 mach_softlvl_to_vect(int ipl) 1234 { 1235 setsoftint = av_set_softint_pending; 1236 kdisetsoftint = kdi_av_set_softint_pending; 1237 1238 return (PSM_SV_SOFTWARE); 1239 } 1240 1241 #ifdef DEBUG 1242 /* 1243 * This is here to allow us to simulate cpus that refuse to start. 1244 */ 1245 cpuset_t cpufailset; 1246 #endif 1247 1248 int 1249 mach_cpu_start(struct cpu *cp, void *ctx) 1250 { 1251 struct psm_ops *pops = mach_set[0]; 1252 processorid_t id = cp->cpu_id; 1253 1254 #ifdef DEBUG 1255 if (CPU_IN_SET(cpufailset, id)) 1256 return (0); 1257 #endif 1258 return ((*pops->psm_cpu_start)(id, ctx)); 1259 } 1260 1261 /*ARGSUSED*/ 1262 static int 1263 mach_translate_irq(dev_info_t *dip, int irqno) 1264 { 1265 return (irqno); /* default to NO translation */ 1266 } 1267 1268 static void 1269 mach_notify_error(int level, char *errmsg) 1270 { 1271 /* 1272 * SL_FATAL is pass in once panicstr is set, deliver it 1273 * as CE_PANIC. Also, translate SL_ codes back to CE_ 1274 * codes for the psmi handler 1275 */ 1276 if (level & SL_FATAL) 1277 (*notify_error)(CE_PANIC, errmsg); 1278 else if (level & SL_WARN) 1279 (*notify_error)(CE_WARN, errmsg); 1280 else if (level & SL_NOTE) 1281 (*notify_error)(CE_NOTE, errmsg); 1282 else if (level & SL_CONSOLE) 1283 (*notify_error)(CE_CONT, errmsg); 1284 } 1285 1286 /* 1287 * It provides the default basic intr_ops interface for the new DDI 1288 * interrupt framework if the PSM doesn't have one. 1289 * 1290 * Input: 1291 * dip - pointer to the dev_info structure of the requested device 1292 * hdlp - pointer to the internal interrupt handle structure for the 1293 * requested interrupt 1294 * intr_op - opcode for this call 1295 * result - pointer to the integer that will hold the result to be 1296 * passed back if return value is PSM_SUCCESS 1297 * 1298 * Output: 1299 * return value is either PSM_SUCCESS or PSM_FAILURE 1300 */ 1301 static int 1302 mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, 1303 psm_intr_op_t intr_op, int *result) 1304 { 1305 struct intrspec *ispec; 1306 1307 switch (intr_op) { 1308 case PSM_INTR_OP_CHECK_MSI: 1309 *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | 1310 DDI_INTR_TYPE_MSIX); 1311 break; 1312 case PSM_INTR_OP_ALLOC_VECTORS: 1313 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1314 *result = 1; 1315 else 1316 *result = 0; 1317 break; 1318 case PSM_INTR_OP_FREE_VECTORS: 1319 break; 1320 case PSM_INTR_OP_NAVAIL_VECTORS: 1321 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) 1322 *result = 1; 1323 else 1324 *result = 0; 1325 break; 1326 case PSM_INTR_OP_XLATE_VECTOR: 1327 ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; 1328 *result = psm_translate_irq(dip, ispec->intrspec_vec); 1329 break; 1330 case PSM_INTR_OP_GET_CAP: 1331 *result = 0; 1332 break; 1333 case PSM_INTR_OP_GET_PENDING: 1334 case PSM_INTR_OP_CLEAR_MASK: 1335 case PSM_INTR_OP_SET_MASK: 1336 case PSM_INTR_OP_GET_SHARED: 1337 case PSM_INTR_OP_SET_PRI: 1338 case PSM_INTR_OP_SET_CAP: 1339 case PSM_INTR_OP_SET_CPU: 1340 case PSM_INTR_OP_GET_INTR: 1341 default: 1342 return (PSM_FAILURE); 1343 } 1344 return (PSM_SUCCESS); 1345 } 1346