1 /* 2 * Copyright 2000 ATI Technologies Inc., Markham, Ontario, and 3 * VA Linux Systems Inc., Fremont, California. 4 * Copyright 2008 Red Hat Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Original Authors: 25 * Kevin E. Martin, Rickard E. Faith, Alan Hourihane 26 * 27 * Kernel port Author: Dave Airlie 28 */ 29 30 #ifndef AMDGPU_MODE_H 31 #define AMDGPU_MODE_H 32 33 #include <drm/drm_crtc.h> 34 #include <drm/drm_edid.h> 35 #include <drm/drm_encoder.h> 36 #include <drm/drm_dp_helper.h> 37 #include <drm/drm_fixed.h> 38 #include <drm/drm_crtc_helper.h> 39 #include <drm/drm_fb_helper.h> 40 #include <drm/drm_plane_helper.h> 41 #include <linux/i2c.h> 42 #include <linux/i2c-algo-bit.h> 43 #include <linux/hrtimer.h> 44 #include "amdgpu_irq.h" 45 46 struct amdgpu_bo; 47 struct amdgpu_device; 48 struct amdgpu_encoder; 49 struct amdgpu_router; 50 struct amdgpu_hpd; 51 52 #define to_amdgpu_crtc(x) container_of(x, struct amdgpu_crtc, base) 53 #define to_amdgpu_connector(x) container_of(x, struct amdgpu_connector, base) 54 #define to_amdgpu_encoder(x) container_of(x, struct amdgpu_encoder, base) 55 #define to_amdgpu_framebuffer(x) container_of(x, struct amdgpu_framebuffer, base) 56 57 #define AMDGPU_MAX_HPD_PINS 6 58 #define AMDGPU_MAX_CRTCS 6 59 #define AMDGPU_MAX_AFMT_BLOCKS 9 60 61 enum amdgpu_rmx_type { 62 RMX_OFF, 63 RMX_FULL, 64 RMX_CENTER, 65 RMX_ASPECT 66 }; 67 68 enum amdgpu_underscan_type { 69 UNDERSCAN_OFF, 70 UNDERSCAN_ON, 71 UNDERSCAN_AUTO, 72 }; 73 74 #define AMDGPU_HPD_CONNECT_INT_DELAY_IN_MS 50 75 #define AMDGPU_HPD_DISCONNECT_INT_DELAY_IN_MS 10 76 77 enum amdgpu_hpd_id { 78 AMDGPU_HPD_1 = 0, 79 AMDGPU_HPD_2, 80 AMDGPU_HPD_3, 81 AMDGPU_HPD_4, 82 AMDGPU_HPD_5, 83 AMDGPU_HPD_6, 84 AMDGPU_HPD_LAST, 85 AMDGPU_HPD_NONE = 0xff, 86 }; 87 88 enum amdgpu_crtc_irq { 89 AMDGPU_CRTC_IRQ_VBLANK1 = 0, 90 AMDGPU_CRTC_IRQ_VBLANK2, 91 AMDGPU_CRTC_IRQ_VBLANK3, 92 AMDGPU_CRTC_IRQ_VBLANK4, 93 AMDGPU_CRTC_IRQ_VBLANK5, 94 AMDGPU_CRTC_IRQ_VBLANK6, 95 AMDGPU_CRTC_IRQ_VLINE1, 96 AMDGPU_CRTC_IRQ_VLINE2, 97 AMDGPU_CRTC_IRQ_VLINE3, 98 AMDGPU_CRTC_IRQ_VLINE4, 99 AMDGPU_CRTC_IRQ_VLINE5, 100 AMDGPU_CRTC_IRQ_VLINE6, 101 AMDGPU_CRTC_IRQ_LAST, 102 AMDGPU_CRTC_IRQ_NONE = 0xff 103 }; 104 105 enum amdgpu_pageflip_irq { 106 AMDGPU_PAGEFLIP_IRQ_D1 = 0, 107 AMDGPU_PAGEFLIP_IRQ_D2, 108 AMDGPU_PAGEFLIP_IRQ_D3, 109 AMDGPU_PAGEFLIP_IRQ_D4, 110 AMDGPU_PAGEFLIP_IRQ_D5, 111 AMDGPU_PAGEFLIP_IRQ_D6, 112 AMDGPU_PAGEFLIP_IRQ_LAST, 113 AMDGPU_PAGEFLIP_IRQ_NONE = 0xff 114 }; 115 116 enum amdgpu_flip_status { 117 AMDGPU_FLIP_NONE, 118 AMDGPU_FLIP_PENDING, 119 AMDGPU_FLIP_SUBMITTED 120 }; 121 122 #define AMDGPU_MAX_I2C_BUS 16 123 124 /* amdgpu gpio-based i2c 125 * 1. "mask" reg and bits 126 * grabs the gpio pins for software use 127 * 0=not held 1=held 128 * 2. "a" reg and bits 129 * output pin value 130 * 0=low 1=high 131 * 3. "en" reg and bits 132 * sets the pin direction 133 * 0=input 1=output 134 * 4. "y" reg and bits 135 * input pin value 136 * 0=low 1=high 137 */ 138 struct amdgpu_i2c_bus_rec { 139 bool valid; 140 /* id used by atom */ 141 uint8_t i2c_id; 142 /* id used by atom */ 143 enum amdgpu_hpd_id hpd; 144 /* can be used with hw i2c engine */ 145 bool hw_capable; 146 /* uses multi-media i2c engine */ 147 bool mm_i2c; 148 /* regs and bits */ 149 uint32_t mask_clk_reg; 150 uint32_t mask_data_reg; 151 uint32_t a_clk_reg; 152 uint32_t a_data_reg; 153 uint32_t en_clk_reg; 154 uint32_t en_data_reg; 155 uint32_t y_clk_reg; 156 uint32_t y_data_reg; 157 uint32_t mask_clk_mask; 158 uint32_t mask_data_mask; 159 uint32_t a_clk_mask; 160 uint32_t a_data_mask; 161 uint32_t en_clk_mask; 162 uint32_t en_data_mask; 163 uint32_t y_clk_mask; 164 uint32_t y_data_mask; 165 }; 166 167 #define AMDGPU_MAX_BIOS_CONNECTOR 16 168 169 /* pll flags */ 170 #define AMDGPU_PLL_USE_BIOS_DIVS (1 << 0) 171 #define AMDGPU_PLL_NO_ODD_POST_DIV (1 << 1) 172 #define AMDGPU_PLL_USE_REF_DIV (1 << 2) 173 #define AMDGPU_PLL_LEGACY (1 << 3) 174 #define AMDGPU_PLL_PREFER_LOW_REF_DIV (1 << 4) 175 #define AMDGPU_PLL_PREFER_HIGH_REF_DIV (1 << 5) 176 #define AMDGPU_PLL_PREFER_LOW_FB_DIV (1 << 6) 177 #define AMDGPU_PLL_PREFER_HIGH_FB_DIV (1 << 7) 178 #define AMDGPU_PLL_PREFER_LOW_POST_DIV (1 << 8) 179 #define AMDGPU_PLL_PREFER_HIGH_POST_DIV (1 << 9) 180 #define AMDGPU_PLL_USE_FRAC_FB_DIV (1 << 10) 181 #define AMDGPU_PLL_PREFER_CLOSEST_LOWER (1 << 11) 182 #define AMDGPU_PLL_USE_POST_DIV (1 << 12) 183 #define AMDGPU_PLL_IS_LCD (1 << 13) 184 #define AMDGPU_PLL_PREFER_MINM_OVER_MAXP (1 << 14) 185 186 struct amdgpu_pll { 187 /* reference frequency */ 188 uint32_t reference_freq; 189 190 /* fixed dividers */ 191 uint32_t reference_div; 192 uint32_t post_div; 193 194 /* pll in/out limits */ 195 uint32_t pll_in_min; 196 uint32_t pll_in_max; 197 uint32_t pll_out_min; 198 uint32_t pll_out_max; 199 uint32_t lcd_pll_out_min; 200 uint32_t lcd_pll_out_max; 201 uint32_t best_vco; 202 203 /* divider limits */ 204 uint32_t min_ref_div; 205 uint32_t max_ref_div; 206 uint32_t min_post_div; 207 uint32_t max_post_div; 208 uint32_t min_feedback_div; 209 uint32_t max_feedback_div; 210 uint32_t min_frac_feedback_div; 211 uint32_t max_frac_feedback_div; 212 213 /* flags for the current clock */ 214 uint32_t flags; 215 216 /* pll id */ 217 uint32_t id; 218 }; 219 220 struct amdgpu_i2c_chan { 221 struct i2c_adapter adapter; 222 struct drm_device *dev; 223 struct i2c_algo_bit_data bit; 224 struct amdgpu_i2c_bus_rec rec; 225 struct drm_dp_aux aux; 226 bool has_aux; 227 struct mutex mutex; 228 }; 229 230 struct amdgpu_fbdev; 231 232 struct amdgpu_afmt { 233 bool enabled; 234 int offset; 235 bool last_buffer_filled_status; 236 int id; 237 struct amdgpu_audio_pin *pin; 238 }; 239 240 /* 241 * Audio 242 */ 243 struct amdgpu_audio_pin { 244 int channels; 245 int rate; 246 int bits_per_sample; 247 u8 status_bits; 248 u8 category_code; 249 u32 offset; 250 bool connected; 251 u32 id; 252 }; 253 254 struct amdgpu_audio { 255 bool enabled; 256 struct amdgpu_audio_pin pin[AMDGPU_MAX_AFMT_BLOCKS]; 257 int num_pins; 258 }; 259 260 struct amdgpu_mode_mc_save { 261 u32 vga_render_control; 262 u32 vga_hdp_control; 263 bool crtc_enabled[AMDGPU_MAX_CRTCS]; 264 }; 265 266 struct amdgpu_display_funcs { 267 /* vga render */ 268 void (*set_vga_render_state)(struct amdgpu_device *adev, bool render); 269 /* display watermarks */ 270 void (*bandwidth_update)(struct amdgpu_device *adev); 271 /* get frame count */ 272 u32 (*vblank_get_counter)(struct amdgpu_device *adev, int crtc); 273 /* wait for vblank */ 274 void (*vblank_wait)(struct amdgpu_device *adev, int crtc); 275 /* set backlight level */ 276 void (*backlight_set_level)(struct amdgpu_encoder *amdgpu_encoder, 277 u8 level); 278 /* get backlight level */ 279 u8 (*backlight_get_level)(struct amdgpu_encoder *amdgpu_encoder); 280 /* hotplug detect */ 281 bool (*hpd_sense)(struct amdgpu_device *adev, enum amdgpu_hpd_id hpd); 282 void (*hpd_set_polarity)(struct amdgpu_device *adev, 283 enum amdgpu_hpd_id hpd); 284 u32 (*hpd_get_gpio_reg)(struct amdgpu_device *adev); 285 /* pageflipping */ 286 void (*page_flip)(struct amdgpu_device *adev, 287 int crtc_id, u64 crtc_base, bool async); 288 int (*page_flip_get_scanoutpos)(struct amdgpu_device *adev, int crtc, 289 u32 *vbl, u32 *position); 290 /* display topology setup */ 291 void (*add_encoder)(struct amdgpu_device *adev, 292 uint32_t encoder_enum, 293 uint32_t supported_device, 294 u16 caps); 295 void (*add_connector)(struct amdgpu_device *adev, 296 uint32_t connector_id, 297 uint32_t supported_device, 298 int connector_type, 299 struct amdgpu_i2c_bus_rec *i2c_bus, 300 uint16_t connector_object_id, 301 struct amdgpu_hpd *hpd, 302 struct amdgpu_router *router); 303 void (*stop_mc_access)(struct amdgpu_device *adev, 304 struct amdgpu_mode_mc_save *save); 305 void (*resume_mc_access)(struct amdgpu_device *adev, 306 struct amdgpu_mode_mc_save *save); 307 }; 308 309 struct amdgpu_mode_info { 310 struct atom_context *atom_context; 311 struct card_info *atom_card_info; 312 bool mode_config_initialized; 313 struct amdgpu_crtc *crtcs[AMDGPU_MAX_CRTCS]; 314 struct amdgpu_afmt *afmt[AMDGPU_MAX_AFMT_BLOCKS]; 315 /* DVI-I properties */ 316 struct drm_property *coherent_mode_property; 317 /* DAC enable load detect */ 318 struct drm_property *load_detect_property; 319 /* underscan */ 320 struct drm_property *underscan_property; 321 struct drm_property *underscan_hborder_property; 322 struct drm_property *underscan_vborder_property; 323 /* audio */ 324 struct drm_property *audio_property; 325 /* FMT dithering */ 326 struct drm_property *dither_property; 327 /* hardcoded DFP edid from BIOS */ 328 struct edid *bios_hardcoded_edid; 329 int bios_hardcoded_edid_size; 330 331 /* pointer to fbdev info structure */ 332 struct amdgpu_fbdev *rfbdev; 333 /* firmware flags */ 334 u16 firmware_flags; 335 /* pointer to backlight encoder */ 336 struct amdgpu_encoder *bl_encoder; 337 struct amdgpu_audio audio; /* audio stuff */ 338 int num_crtc; /* number of crtcs */ 339 int num_hpd; /* number of hpd pins */ 340 int num_dig; /* number of dig blocks */ 341 int disp_priority; 342 const struct amdgpu_display_funcs *funcs; 343 }; 344 345 #define AMDGPU_MAX_BL_LEVEL 0xFF 346 347 #if defined(CONFIG_BACKLIGHT_CLASS_DEVICE) || defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE) 348 349 struct amdgpu_backlight_privdata { 350 struct amdgpu_encoder *encoder; 351 uint8_t negative; 352 }; 353 354 #endif 355 356 struct amdgpu_atom_ss { 357 uint16_t percentage; 358 uint16_t percentage_divider; 359 uint8_t type; 360 uint16_t step; 361 uint8_t delay; 362 uint8_t range; 363 uint8_t refdiv; 364 /* asic_ss */ 365 uint16_t rate; 366 uint16_t amount; 367 }; 368 369 struct amdgpu_crtc { 370 struct drm_crtc base; 371 int crtc_id; 372 u16 lut_r[256], lut_g[256], lut_b[256]; 373 bool enabled; 374 bool can_tile; 375 uint32_t crtc_offset; 376 struct drm_gem_object *cursor_bo; 377 uint64_t cursor_addr; 378 int cursor_x; 379 int cursor_y; 380 int cursor_hot_x; 381 int cursor_hot_y; 382 int cursor_width; 383 int cursor_height; 384 int max_cursor_width; 385 int max_cursor_height; 386 enum amdgpu_rmx_type rmx_type; 387 u8 h_border; 388 u8 v_border; 389 fixed20_12 vsc; 390 fixed20_12 hsc; 391 struct drm_display_mode native_mode; 392 u32 pll_id; 393 /* page flipping */ 394 struct amdgpu_flip_work *pflip_works; 395 enum amdgpu_flip_status pflip_status; 396 int deferred_flip_completion; 397 /* pll sharing */ 398 struct amdgpu_atom_ss ss; 399 bool ss_enabled; 400 u32 adjusted_clock; 401 int bpc; 402 u32 pll_reference_div; 403 u32 pll_post_div; 404 u32 pll_flags; 405 struct drm_encoder *encoder; 406 struct drm_connector *connector; 407 /* for dpm */ 408 u32 line_time; 409 u32 wm_low; 410 u32 wm_high; 411 u32 lb_vblank_lead_lines; 412 struct drm_display_mode hw_mode; 413 /* for virtual dce */ 414 struct hrtimer vblank_timer; 415 enum amdgpu_interrupt_state vsync_timer_enabled; 416 }; 417 418 struct amdgpu_encoder_atom_dig { 419 bool linkb; 420 /* atom dig */ 421 bool coherent_mode; 422 int dig_encoder; /* -1 disabled, 0 DIGA, 1 DIGB, etc. */ 423 /* atom lvds/edp */ 424 uint32_t lcd_misc; 425 uint16_t panel_pwr_delay; 426 uint32_t lcd_ss_id; 427 /* panel mode */ 428 struct drm_display_mode native_mode; 429 struct backlight_device *bl_dev; 430 int dpms_mode; 431 uint8_t backlight_level; 432 int panel_mode; 433 struct amdgpu_afmt *afmt; 434 }; 435 436 struct amdgpu_encoder { 437 struct drm_encoder base; 438 uint32_t encoder_enum; 439 uint32_t encoder_id; 440 uint32_t devices; 441 uint32_t active_device; 442 uint32_t flags; 443 uint32_t pixel_clock; 444 enum amdgpu_rmx_type rmx_type; 445 enum amdgpu_underscan_type underscan_type; 446 uint32_t underscan_hborder; 447 uint32_t underscan_vborder; 448 struct drm_display_mode native_mode; 449 void *enc_priv; 450 int audio_polling_active; 451 bool is_ext_encoder; 452 u16 caps; 453 }; 454 455 struct amdgpu_connector_atom_dig { 456 /* displayport */ 457 u8 dpcd[DP_RECEIVER_CAP_SIZE]; 458 u8 dp_sink_type; 459 int dp_clock; 460 int dp_lane_count; 461 bool edp_on; 462 }; 463 464 struct amdgpu_gpio_rec { 465 bool valid; 466 u8 id; 467 u32 reg; 468 u32 mask; 469 u32 shift; 470 }; 471 472 struct amdgpu_hpd { 473 enum amdgpu_hpd_id hpd; 474 u8 plugged_state; 475 struct amdgpu_gpio_rec gpio; 476 }; 477 478 struct amdgpu_router { 479 u32 router_id; 480 struct amdgpu_i2c_bus_rec i2c_info; 481 u8 i2c_addr; 482 /* i2c mux */ 483 bool ddc_valid; 484 u8 ddc_mux_type; 485 u8 ddc_mux_control_pin; 486 u8 ddc_mux_state; 487 /* clock/data mux */ 488 bool cd_valid; 489 u8 cd_mux_type; 490 u8 cd_mux_control_pin; 491 u8 cd_mux_state; 492 }; 493 494 enum amdgpu_connector_audio { 495 AMDGPU_AUDIO_DISABLE = 0, 496 AMDGPU_AUDIO_ENABLE = 1, 497 AMDGPU_AUDIO_AUTO = 2 498 }; 499 500 enum amdgpu_connector_dither { 501 AMDGPU_FMT_DITHER_DISABLE = 0, 502 AMDGPU_FMT_DITHER_ENABLE = 1, 503 }; 504 505 struct amdgpu_connector { 506 struct drm_connector base; 507 uint32_t connector_id; 508 uint32_t devices; 509 struct amdgpu_i2c_chan *ddc_bus; 510 /* some systems have an hdmi and vga port with a shared ddc line */ 511 bool shared_ddc; 512 bool use_digital; 513 /* we need to mind the EDID between detect 514 and get modes due to analog/digital/tvencoder */ 515 struct edid *edid; 516 void *con_priv; 517 bool dac_load_detect; 518 bool detected_by_load; /* if the connection status was determined by load */ 519 uint16_t connector_object_id; 520 struct amdgpu_hpd hpd; 521 struct amdgpu_router router; 522 struct amdgpu_i2c_chan *router_bus; 523 enum amdgpu_connector_audio audio; 524 enum amdgpu_connector_dither dither; 525 unsigned pixelclock_for_modeset; 526 }; 527 528 struct amdgpu_framebuffer { 529 struct drm_framebuffer base; 530 struct drm_gem_object *obj; 531 }; 532 533 #define ENCODER_MODE_IS_DP(em) (((em) == ATOM_ENCODER_MODE_DP) || \ 534 ((em) == ATOM_ENCODER_MODE_DP_MST)) 535 536 /* Driver internal use only flags of amdgpu_get_crtc_scanoutpos() */ 537 #define DRM_SCANOUTPOS_VALID (1 << 0) 538 #define DRM_SCANOUTPOS_IN_VBLANK (1 << 1) 539 #define DRM_SCANOUTPOS_ACCURATE (1 << 2) 540 #define USE_REAL_VBLANKSTART (1 << 30) 541 #define GET_DISTANCE_TO_VBLANKSTART (1 << 31) 542 543 void amdgpu_link_encoder_connector(struct drm_device *dev); 544 545 struct drm_connector * 546 amdgpu_get_connector_for_encoder(struct drm_encoder *encoder); 547 struct drm_connector * 548 amdgpu_get_connector_for_encoder_init(struct drm_encoder *encoder); 549 bool amdgpu_dig_monitor_is_duallink(struct drm_encoder *encoder, 550 u32 pixel_clock); 551 552 u16 amdgpu_encoder_get_dp_bridge_encoder_id(struct drm_encoder *encoder); 553 struct drm_encoder *amdgpu_get_external_encoder(struct drm_encoder *encoder); 554 555 bool amdgpu_ddc_probe(struct amdgpu_connector *amdgpu_connector, bool use_aux); 556 557 void amdgpu_encoder_set_active_device(struct drm_encoder *encoder); 558 559 int amdgpu_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, 560 unsigned int flags, int *vpos, int *hpos, 561 ktime_t *stime, ktime_t *etime, 562 const struct drm_display_mode *mode); 563 564 int amdgpu_framebuffer_init(struct drm_device *dev, 565 struct amdgpu_framebuffer *rfb, 566 const struct drm_mode_fb_cmd2 *mode_cmd, 567 struct drm_gem_object *obj); 568 569 int amdgpufb_remove(struct drm_device *dev, struct drm_framebuffer *fb); 570 571 void amdgpu_enc_destroy(struct drm_encoder *encoder); 572 void amdgpu_copy_fb(struct drm_device *dev, struct drm_gem_object *dst_obj); 573 bool amdgpu_crtc_scaling_mode_fixup(struct drm_crtc *crtc, 574 const struct drm_display_mode *mode, 575 struct drm_display_mode *adjusted_mode); 576 void amdgpu_panel_mode_fixup(struct drm_encoder *encoder, 577 struct drm_display_mode *adjusted_mode); 578 int amdgpu_crtc_idx_to_irq_type(struct amdgpu_device *adev, int crtc); 579 580 /* fbdev layer */ 581 int amdgpu_fbdev_init(struct amdgpu_device *adev); 582 void amdgpu_fbdev_fini(struct amdgpu_device *adev); 583 void amdgpu_fbdev_set_suspend(struct amdgpu_device *adev, int state); 584 int amdgpu_fbdev_total_size(struct amdgpu_device *adev); 585 bool amdgpu_fbdev_robj_is_fb(struct amdgpu_device *adev, struct amdgpu_bo *robj); 586 void amdgpu_fbdev_restore_mode(struct amdgpu_device *adev); 587 588 void amdgpu_fb_output_poll_changed(struct amdgpu_device *adev); 589 590 591 int amdgpu_align_pitch(struct amdgpu_device *adev, int width, int bpp, bool tiled); 592 593 /* amdgpu_display.c */ 594 void amdgpu_print_display_setup(struct drm_device *dev); 595 int amdgpu_modeset_create_props(struct amdgpu_device *adev); 596 int amdgpu_crtc_set_config(struct drm_mode_set *set, 597 struct drm_modeset_acquire_ctx *ctx); 598 int amdgpu_crtc_page_flip_target(struct drm_crtc *crtc, 599 struct drm_framebuffer *fb, 600 struct drm_pending_vblank_event *event, 601 uint32_t page_flip_flags, uint32_t target, 602 struct drm_modeset_acquire_ctx *ctx); 603 extern const struct drm_mode_config_funcs amdgpu_mode_funcs; 604 605 #endif 606