Lines Matching refs:ISD
235 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT,
238 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32,
242 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
244 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
245 setOperationAction(ISD::BR_CC, XLenVT, Expand);
247 setOperationAction(ISD::BR_CC, MVT::i32, Expand);
248 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
249 setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
251 setOperationAction(ISD::SELECT_CC, MVT::i32, Expand);
254 setCondCodeAction(ISD::SETLE, XLenVT, Expand);
255 setCondCodeAction(ISD::SETGT, XLenVT, Custom);
256 setCondCodeAction(ISD::SETGE, XLenVT, Expand);
258 setCondCodeAction(ISD::SETULE, XLenVT, Expand);
259 setCondCodeAction(ISD::SETUGT, XLenVT, Custom);
260 setCondCodeAction(ISD::SETUGE, XLenVT, Expand);
263 setOperationAction(ISD::SETCC, MVT::i32, Promote);
265 setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand);
267 setOperationAction(ISD::VASTART, MVT::Other, Custom);
268 setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);
270 setOperationAction(ISD::VAARG, MVT::i32, Promote);
272 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
274 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
277 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand);
280 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);
283 setOperationAction(ISD::LOAD, MVT::i32, Custom);
284 setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
286 setOperationAction({ISD::UADDO, ISD::USUBO, ISD::UADDSAT, ISD::USUBSAT},
289 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, MVT::i32, Custom);
291 setOperationAction(ISD::SSUBO, MVT::i32, Custom);
293 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, MVT::i32, Custom);
294 setOperationAction({ISD::UADDSAT, ISD::USUBSAT}, MVT::i32, Custom);
297 setOperationAction(ISD::SADDO, MVT::i32, Custom);
300 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand);
302 setOperationAction(ISD::MUL, MVT::i32, Promote);
304 setOperationAction(ISD::MUL, MVT::i128, Custom);
306 setOperationAction(ISD::MUL, MVT::i32, Custom);
308 setOperationAction(ISD::SMULO, MVT::i32, Custom);
310 setOperationAction(ISD::MUL, MVT::i64, Custom);
314 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM},
317 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, MVT::i32,
321 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM},
326 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::i32, Expand);
328 {ISD::SDIVREM, ISD::UDIVREM, ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32,
333 {ISD::SDIVREM, ISD::UDIVREM, ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT,
336 setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT,
341 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
344 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
345 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom);
347 setOperationAction(ISD::ROTL, XLenVT, Expand);
349 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand);
351 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Expand);
356 setOperationAction(ISD::BSWAP, XLenVT,
362 setOperationAction(ISD::BSWAP, MVT::i32,
370 setOperationAction(ISD::BITREVERSE, XLenVT, Legal);
373 setOperationAction(ISD::BITREVERSE, XLenVT,
378 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT,
381 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, MVT::i32,
386 setOperationAction(ISD::CTTZ, MVT::i32, Legal);
388 setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, MVT::i32, Custom);
391 setOperationAction({ISD::CTTZ, ISD::CTPOP}, XLenVT, Expand);
393 setOperationAction({ISD::CTTZ, ISD::CTPOP}, MVT::i32, Expand);
402 setOperationAction(ISD::CTLZ, MVT::i32,
405 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Promote);
407 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom);
410 setOperationAction(ISD::CTLZ, XLenVT, Expand);
412 setOperationAction(ISD::CTLZ, MVT::i32, Expand);
417 setOperationAction(ISD::ABS, MVT::i32, Custom);
421 setOperationAction(ISD::ABS, XLenVT, Legal);
424 setOperationAction(ISD::SELECT, XLenVT, Custom);
426 setOperationAction(ISD::SELECT, MVT::i32, Promote);
430 ISD::FMINNUM, ISD::FMAXNUM, ISD::LRINT,
431 ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
432 ISD::STRICT_LRINT, ISD::STRICT_LLRINT, ISD::STRICT_LROUND,
433 ISD::STRICT_LLROUND, ISD::STRICT_FMA, ISD::STRICT_FADD,
434 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV,
435 ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS};
437 static const ISD::CondCode FPCCToExpand[] = {
438 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
439 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
440 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO};
443 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW,
444 ISD::FREM};
447 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
448 ISD::FROUNDEVEN};
451 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
454 ISD::FMINNUM, ISD::FMAXNUM, ISD::FADD,
455 ISD::FSUB, ISD::FMUL, ISD::FMA,
456 ISD::FDIV, ISD::FSQRT, ISD::FABS,
457 ISD::FNEG, ISD::STRICT_FMA, ISD::STRICT_FADD,
458 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV,
459 ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
460 ISD::SETCC, ISD::FCEIL, ISD::FFLOOR,
461 ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
462 ISD::FROUNDEVEN, ISD::SELECT};
465 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
466 setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
467 setOperationAction(ISD::FP_ROUND, MVT::bf16, Custom);
468 setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);
469 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
470 setOperationAction(ISD::ConstantFP, MVT::bf16, Expand);
471 setOperationAction(ISD::SELECT_CC, MVT::bf16, Expand);
472 setOperationAction(ISD::BR_CC, MVT::bf16, Expand);
474 setOperationAction(ISD::FREM, MVT::bf16, Promote);
477 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Expand);
485 setOperationAction(ISD::SELECT, MVT::f16, Custom);
486 setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom);
489 setOperationAction({ISD::STRICT_LRINT, ISD::STRICT_LLRINT,
490 ISD::STRICT_LROUND, ISD::STRICT_LLROUND},
494 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);
497 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
498 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
500 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
501 setOperationAction(ISD::BR_CC, MVT::f16, Expand);
503 setOperationAction(ISD::FNEARBYINT, MVT::f16,
505 setOperationAction({ISD::FREM, ISD::FPOW, ISD::FPOWI,
506 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
507 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
508 ISD::FLOG10},
513 setOperationAction({ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
514 ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
515 ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
516 ISD::STRICT_FTRUNC},
521 setOperationAction(ISD::FPOWI, MVT::i32, Custom);
523 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16,
532 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
533 setOperationAction(ISD::SELECT, MVT::f32, Custom);
534 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
536 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
538 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
540 setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom);
541 setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
542 setOperationAction(ISD::FP_TO_BF16, MVT::f32,
544 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
545 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom);
548 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
549 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Legal);
551 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Custom);
556 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
562 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
566 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
567 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Legal);
572 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Custom);
575 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
576 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
578 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
579 setOperationAction(ISD::SELECT, MVT::f64, Custom);
580 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
581 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
584 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
586 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
588 setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom);
589 setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
590 setOperationAction(ISD::FP_TO_BF16, MVT::f64,
592 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
593 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
597 setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT,
598 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
600 setOperationAction(ISD::LROUND, MVT::i32, Custom);
604 setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT,
607 setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT,
608 ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP},
612 setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT,
613 ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP},
616 setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom);
617 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
620 setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
621 ISD::JumpTable},
624 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
627 setOperationAction(ISD::Constant, MVT::i64, Custom);
631 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
633 setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64,
636 setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
637 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
639 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
642 setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
657 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
663 setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);
669 setOperationAction(ISD::VSCALE, XLenVT, Custom);
671 setOperationAction(ISD::VSCALE, MVT::i32, Custom);
675 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
676 ISD::INTRINSIC_VOID},
679 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
682 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
685 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
689 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
690 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
691 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
692 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
693 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
694 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
695 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
696 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
697 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
698 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
699 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
700 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
701 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
702 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
703 ISD::EXPERIMENTAL_VP_SPLAT};
706 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
707 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
708 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
709 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
710 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
711 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
712 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
713 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
714 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
715 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
716 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
717 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
718 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
719 ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
722 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
723 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
724 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
727 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
728 ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
733 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
738 setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
739 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
740 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
741 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
749 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
752 setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS,
753 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
754 ISD::SCALAR_TO_VECTOR},
757 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
760 setOperationAction(ISD::SELECT, VT, Custom);
762 {ISD::SELECT_CC, ISD::VSELECT, ISD::VP_MERGE, ISD::VP_SELECT}, VT,
765 setOperationAction({ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF}, VT,
768 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Custom);
771 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
775 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
782 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
783 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
784 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
785 ISD::STRICT_FP_TO_UINT},
787 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
794 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
798 setOperationAction({ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT,
799 ISD::VP_TRUNCATE, ISD::VP_SETCC},
802 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
803 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
805 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
807 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
808 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
811 ISD::VECTOR_SPLICE, VT,
819 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
820 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
823 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);
827 setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand);
829 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
832 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
835 setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
842 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
843 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
844 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
845 ISD::STRICT_FP_TO_UINT},
847 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
849 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
850 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
851 ISD::SSUBSAT, ISD::USUBSAT},
856 setOperationAction(ISD::TRUNCATE, VT, Custom);
859 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
868 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
870 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
874 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
875 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
878 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
879 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
882 setOperationAction(ISD::SELECT, VT, Custom);
883 setOperationAction(ISD::SELECT_CC, VT, Expand);
885 setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom);
889 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
893 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
894 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
897 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
900 setOperationAction(ISD::BSWAP, VT, Legal);
901 setOperationAction(ISD::VP_BSWAP, VT, Custom);
903 setOperationAction({ISD::BSWAP, ISD::VP_BSWAP}, VT, Expand);
904 setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand);
908 setOperationAction(ISD::BITREVERSE, VT, Legal);
909 setOperationAction(ISD::VP_BITREVERSE, VT, Custom);
910 setOperationAction({ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ,
911 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
914 setOperationAction({ISD::BITREVERSE, ISD::VP_BITREVERSE}, VT, Expand);
915 setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand);
916 setOperationAction({ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ,
917 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
924 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF,
925 ISD::CTTZ_ZERO_UNDEF, ISD::VP_CTLZ,
926 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
939 static const ISD::CondCode VFPCCToExpand[] = {
940 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
941 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
942 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE,
947 ISD::FMINNUM, ISD::FMAXNUM, ISD::FADD, ISD::FSUB,
948 ISD::FMUL, ISD::FMA, ISD::FDIV, ISD::FSQRT,
949 ISD::FABS, ISD::FNEG, ISD::FCOPYSIGN, ISD::FCEIL,
950 ISD::FFLOOR, ISD::FROUND, ISD::FROUNDEVEN, ISD::FRINT,
951 ISD::FNEARBYINT, ISD::IS_FPCLASS, ISD::SETCC, ISD::FMAXIMUM,
952 ISD::FMINIMUM, ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
953 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA};
957 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
958 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
959 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
960 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
961 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
962 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
963 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
964 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
965 ISD::VP_FMAXIMUM, ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
969 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
974 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
975 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
977 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
982 setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, VT, Legal);
983 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
985 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
986 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
987 ISD::IS_FPCLASS},
993 setOperationAction(ISD::FREM, VT, Expand);
994 setOperationAction(ISD::FPOW, VT, Expand);
995 setOperationAction(ISD::FCOS, VT, Expand);
996 setOperationAction(ISD::FSIN, VT, Expand);
997 setOperationAction(ISD::FSINCOS, VT, Expand);
998 setOperationAction(ISD::FEXP, VT, Expand);
999 setOperationAction(ISD::FEXP2, VT, Expand);
1000 setOperationAction(ISD::FEXP10, VT, Expand);
1001 setOperationAction(ISD::FLOG, VT, Expand);
1002 setOperationAction(ISD::FLOG2, VT, Expand);
1003 setOperationAction(ISD::FLOG10, VT, Expand);
1005 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
1007 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1009 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1013 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1014 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1017 setOperationAction(ISD::SELECT, VT, Custom);
1018 setOperationAction(ISD::SELECT_CC, VT, Expand);
1020 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1021 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1024 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1025 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1027 setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE}, VT, Custom);
1031 setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
1033 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1034 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
1036 setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
1037 ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
1038 ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1039 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1049 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
1063 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1064 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1066 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1067 setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT,
1069 setOperationAction(ISD::SELECT_CC, VT, Expand);
1070 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP,
1071 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1073 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1074 ISD::EXTRACT_SUBVECTOR},
1077 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1079 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1099 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1100 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1101 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1103 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1104 ISD::EXTRACT_SUBVECTOR},
1106 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1108 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1109 setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT,
1111 setOperationAction(ISD::SELECT_CC, VT, Expand);
1141 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1145 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1151 setOperationAction(ISD::UNDEF, VT, Custom);
1154 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
1157 setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS}, VT,
1160 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1163 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1165 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1167 setOperationAction(ISD::SETCC, VT, Custom);
1169 setOperationAction(ISD::SELECT, VT, Custom);
1171 setOperationAction(ISD::TRUNCATE, VT, Custom);
1173 setOperationAction(ISD::BITCAST, VT, Custom);
1176 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1180 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1185 ISD::SINT_TO_FP,
1186 ISD::UINT_TO_FP,
1187 ISD::FP_TO_SINT,
1188 ISD::FP_TO_UINT,
1189 ISD::STRICT_SINT_TO_FP,
1190 ISD::STRICT_UINT_TO_FP,
1191 ISD::STRICT_FP_TO_SINT,
1192 ISD::STRICT_FP_TO_UINT,
1195 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1198 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1202 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR, ISD::AND,
1203 ISD::OR, ISD::XOR},
1206 setOperationAction({ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT,
1207 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1210 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1211 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1221 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1222 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
1226 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
1228 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
1229 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1230 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1231 ISD::VP_SCATTER},
1234 setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
1235 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
1236 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
1240 {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom);
1242 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
1246 setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);
1248 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1249 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1250 ISD::SSUBSAT, ISD::USUBSAT},
1253 setOperationAction(ISD::VSELECT, VT, Custom);
1256 {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom);
1260 setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
1261 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
1262 ISD::VECREDUCE_UMIN},
1268 setOperationAction({ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Custom);
1271 setOperationAction({ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF,
1272 ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF, ISD::CTPOP},
1280 {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT,
1288 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1296 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1301 setOperationAction(ISD::UNDEF, VT, Custom);
1303 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1304 ISD::EXTRACT_SUBVECTOR},
1309 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1311 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1312 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1317 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1319 {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
1321 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP,
1322 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1324 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1327 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1331 setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom);
1345 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1347 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1349 {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
1355 setOperationAction({ISD::BUILD_VECTOR, ISD::VECTOR_SHUFFLE,
1356 ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1360 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
1362 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
1363 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1364 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1365 ISD::VP_SCATTER},
1368 setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
1369 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
1370 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
1371 ISD::IS_FPCLASS, ISD::FMAXIMUM, ISD::FMINIMUM},
1374 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1375 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT},
1380 setOperationAction(ISD::SETCC, VT, Custom);
1381 setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom);
1383 setOperationAction(ISD::BITCAST, VT, Custom);
1390 {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1391 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
1392 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
1393 ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1394 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1399 setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32}, Custom);
1401 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1403 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1405 setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1407 setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1412 setOperationAction(ISD::ATOMIC_LOAD_SUB, XLenVT, Expand);
1414 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand);
1420 {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1421 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1422 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1423 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1428 for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
1444 setIndexedLoadAction(ISD::POST_INC, MVT::i8, Legal);
1445 setIndexedLoadAction(ISD::POST_INC, MVT::i16, Legal);
1446 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
1448 setIndexedStoreAction(ISD::POST_INC, MVT::i8, Legal);
1449 setIndexedStoreAction(ISD::POST_INC, MVT::i16, Legal);
1450 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
1454 setOperationAction(ISD::ABS, XLenVT, Legal);
1455 setOperationAction(ISD::SMIN, XLenVT, Legal);
1456 setOperationAction(ISD::UMIN, XLenVT, Legal);
1457 setOperationAction(ISD::SMAX, XLenVT, Legal);
1458 setOperationAction(ISD::UMAX, XLenVT, Legal);
1459 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Legal);
1460 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Legal);
1470 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
1471 ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
1472 ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT});
1474 setTargetDAGCombine(ISD::SRA);
1477 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM});
1480 setTargetDAGCombine({ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN});
1484 setTargetDAGCombine(ISD::TRUNCATE);
1487 setTargetDAGCombine(ISD::BITREVERSE);
1489 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
1491 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
1492 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
1494 setTargetDAGCombine({ISD::FCOPYSIGN, ISD::MGATHER, ISD::MSCATTER,
1495 ISD::VP_GATHER, ISD::VP_SCATTER, ISD::SRA, ISD::SRL,
1496 ISD::SHL, ISD::STORE, ISD::SPLAT_VECTOR,
1497 ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS,
1498 ISD::EXPERIMENTAL_VP_REVERSE, ISD::MUL,
1499 ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM,
1500 ISD::INSERT_VECTOR_ELT, ISD::ABS});
1502 setTargetDAGCombine({ISD::LOAD, ISD::STORE});
1504 setTargetDAGCombine(ISD::BITCAST);
1573 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
1618 Info.opc = ISD::INTRINSIC_W_CHAIN;
1903 (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
1921 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1922 LD->getExtensionType() == ISD::ZEXTLOAD))
2037 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
2041 if (NewShiftOpcode == ISD::SRL && CC->isOne())
2195 if (Opc >= ISD::BUILTIN_OP_END)
2294 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
2388 ISD::CondCode &CC, SelectionDAG &DAG) {
2392 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
2398 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
2406 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
2416 case ISD::SETGT:
2420 CC = ISD::SETGE;
2424 case ISD::SETLT:
2429 CC = ISD::SETGE;
2439 case ISD::SETGT:
2440 case ISD::SETLE:
2441 case ISD::SETUGT:
2442 case ISD::SETULE:
2443 CC = ISD::getSetCCSwappedOperands(CC);
2594 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2595 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
2597 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
2736 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
2748 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
2915 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
2922 Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src);
2944 ISD::CondCode::SETUO);
2979 {Src, Src, DAG.getCondCode(ISD::SETNE),
3008 case ISD::FROUNDEVEN:
3009 case ISD::STRICT_FROUNDEVEN:
3010 case ISD::VP_FROUNDEVEN:
3012 case ISD::FTRUNC:
3013 case ISD::STRICT_FTRUNC:
3014 case ISD::VP_FROUNDTOZERO:
3016 case ISD::FFLOOR:
3017 case ISD::STRICT_FFLOOR:
3018 case ISD::VP_FFLOOR:
3020 case ISD::FCEIL:
3021 case ISD::STRICT_FCEIL:
3022 case ISD::VP_FCEIL:
3024 case ISD::FROUND:
3025 case ISD::STRICT_FROUND:
3026 case ISD::VP_FROUND:
3028 case ISD::FRINT:
3089 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT),
3100 case ISD::FCEIL:
3101 case ISD::VP_FCEIL:
3102 case ISD::FFLOOR:
3103 case ISD::VP_FFLOOR:
3104 case ISD::FROUND:
3105 case ISD::FROUNDEVEN:
3106 case ISD::VP_FROUND:
3107 case ISD::VP_FROUNDEVEN:
3108 case ISD::VP_FROUNDTOZERO: {
3115 case ISD::FTRUNC:
3119 case ISD::FRINT:
3120 case ISD::VP_FRINT:
3123 case ISD::FNEARBYINT:
3124 case ISD::VP_FNEARBYINT:
3171 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3198 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL});
3208 case ISD::STRICT_FCEIL:
3209 case ISD::STRICT_FFLOOR:
3210 case ISD::STRICT_FROUND:
3211 case ISD::STRICT_FROUNDEVEN: {
3219 case ISD::STRICT_FTRUNC:
3224 case ISD::STRICT_FNEARBYINT:
3233 if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
3372 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
3474 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
3565 (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) &&
3585 LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp);
3598 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
3608 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
3634 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
3639 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
3698 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
3714 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
3731 unsigned StepOpcode = ISD::MUL;
3737 StepOpcode = ISD::SHL;
3745 if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
3746 (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
3754 // Convert right out of the scalable type so we can use standard ISD
3759 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
3760 (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
3767 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
3771 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
3776 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
3786 ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
3812 SDValue Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ViaVecVT,
3817 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
3834 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
3886 Splat = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL,
3979 A = DAG.getNode(ISD::AND, SDLoc(A), XLenVT, A, Mask);
3980 B = DAG.getNode(ISD::AND, SDLoc(B), XLenVT, B, Mask);
3984 return DAG.getNode(ISD::OR, ElemDL, XLenVT, A,
3985 DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt), Flags);
3995 return DAG.getNode(ISD::BITCAST, DL, VT,
4006 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), IVT, NewOps);
4020 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
4021 ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()))
4042 Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat,
4049 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
4052 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
4061 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
4091 DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps);
4094 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubBV,
4145 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
4226 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V);
4263 NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
4270 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
4276 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
4324 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
4352 if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
4367 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru,
4369 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtractedVal,
4391 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
4412 if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
4413 V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
4626 while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
4678 ISD::EXTRACT_SUBVECTOR, DL, VT,
4782 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat);
4916 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE,
4925 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ViaBitVT, DAG.getUNDEF(ViaBitVT),
4929 DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V));
4934 Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res,
4940 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
4985 Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op);
4987 Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op,
5066 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, SrcVec,
5072 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubVec,
5098 V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1);
5100 : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2);
5103 ISD::SETNE);
5121 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
5129 if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
5147 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
5171 V = DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
5271 EvenV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, EvenV,
5276 OddV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, OddV,
5376 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
5377 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V2, V1);
5436 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
5438 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
5439 } else if (Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5440 SDValue Neg = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(0, DL, VT),
5442 Src = DAG.getNode(ISD::VP_AND, DL, VT, Src, Neg, Mask, VL);
5449 FloatVal = DAG.getNode(ISD::VP_UINT_TO_FP, DL, FloatVT, Src, Mask, VL);
5451 FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
5477 Exp = DAG.getNode(ISD::VP_SRL, DL, IntVT, Bitcast,
5481 Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
5484 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
5486 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
5492 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF)
5493 return DAG.getNode(ISD::SUB, DL, VT, Exp,
5495 if (Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5496 return DAG.getNode(ISD::VP_SUB, DL, VT, Exp,
5504 Res = DAG.getNode(ISD::VP_SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp,
5507 Res = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
5511 if (Op.getOpcode() == ISD::CTLZ)
5512 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
5513 else if (Op.getOpcode() == ISD::VP_CTLZ)
5514 Res = DAG.getNode(ISD::VP_UMIN, DL, VT, Res,
5541 {Source, AllZero, DAG.getCondCode(ISD::SETNE),
5546 if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5552 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
5554 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
5559 // responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
5588 // responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
5673 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
5681 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
5691 bool IsAdd = Op.getOpcode() == ISD::SADDSAT;
5693 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5694 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5696 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
5702 Result = DAG.getNode(ISD::SMIN, DL, MVT::i64, Result, SatMax);
5703 Result = DAG.getNode(ISD::SMAX, DL, MVT::i64, Result, SatMin);
5704 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
5715 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5716 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5718 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, WideOp);
5728 bool IsAdd = Op.getOpcode() == ISD::SADDO;
5730 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5731 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5733 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
5734 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, WideOp);
5735 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, WideOp,
5738 ISD::SETNE);
5747 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(0));
5748 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Op.getOperand(1));
5749 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
5750 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Mul);
5751 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Mul,
5754 ISD::SETNE);
5797 if (Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5806 ISD::CondCode::SETEQ);
5807 SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS,
5810 ISD::SETNE);
5817 if (Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5834 {FPCLASS, TDCMaskV, DAG.getCondCode(ISD::SETEQ),
5846 {AND, SplatZero, DAG.getCondCode(ISD::SETNE),
5852 SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV);
5854 ISD::CondCode::SETNE);
5855 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
5878 SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ);
5884 SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ);
5889 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
5918 {X, X, DAG.getCondCode(ISD::SETOEQ),
5927 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5934 Op.getOpcode() == ISD::FMAXIMUM || Op->getOpcode() == ISD::VP_FMAXIMUM
5947 case ISD::NODE: \
5950 case ISD::VP_##NODE: \
6030 case ISD::CTLZ_ZERO_UNDEF:
6031 case ISD::VP_CTLZ_ZERO_UNDEF:
6033 case ISD::CTTZ_ZERO_UNDEF:
6034 case ISD::VP_CTTZ_ZERO_UNDEF:
6036 case ISD::FMA:
6037 case ISD::VP_FMA:
6039 case ISD::STRICT_FMA:
6041 case ISD::AND:
6042 case ISD::VP_AND:
6046 case ISD::OR:
6047 case ISD::VP_OR:
6051 case ISD::XOR:
6052 case ISD::VP_XOR:
6056 case ISD::VP_SELECT:
6057 case ISD::VP_MERGE:
6059 case ISD::VP_SRA:
6061 case ISD::VP_SRL:
6063 case ISD::VP_SQRT:
6065 case ISD::VP_SIGN_EXTEND:
6067 case ISD::VP_ZERO_EXTEND:
6069 case ISD::VP_FP_TO_SINT:
6071 case ISD::VP_FP_TO_UINT:
6073 case ISD::FMINNUM:
6074 case ISD::VP_FMINNUM:
6076 case ISD::FMAXNUM:
6077 case ISD::VP_FMAXNUM:
6079 case ISD::LRINT:
6080 case ISD::VP_LRINT:
6081 case ISD::LLRINT:
6082 case ISD::VP_LLRINT:
6098 ISD::FIRST_TARGET_STRICTFP_OPCODE ==
6124 ISD::FIRST_TARGET_STRICTFP_OPCODE ==
6159 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
6163 assert(ISD::isVPOpcode(Op.getOpcode()) && "Not a VP op");
6171 if (ISD::getVPExplicitVectorLengthIdx(Op.getOpcode()) == j) {
6190 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
6238 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0),
6248 case ISD::ATOMIC_FENCE:
6250 case ISD::GlobalAddress:
6252 case ISD::BlockAddress:
6254 case ISD::ConstantPool:
6256 case ISD::JumpTable:
6258 case ISD::GlobalTLSAddress:
6260 case ISD::Constant:
6262 case ISD::SELECT:
6264 case ISD::BRCOND:
6266 case ISD::VASTART:
6268 case ISD::FRAMEADDR:
6270 case ISD::RETURNADDR:
6272 case ISD::SADDO:
6273 case ISD::SSUBO:
6275 case ISD::SMULO:
6277 case ISD::SHL_PARTS:
6279 case ISD::SRA_PARTS:
6281 case ISD::SRL_PARTS:
6283 case ISD::ROTL:
6284 case ISD::ROTR:
6296 case ISD::BITCAST: {
6304 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
6310 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
6316 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
6353 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
6367 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
6372 case ISD::INTRINSIC_WO_CHAIN:
6374 case ISD::INTRINSIC_W_CHAIN:
6376 case ISD::INTRINSIC_VOID:
6378 case ISD::IS_FPCLASS:
6380 case ISD::BITREVERSE: {
6388 assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
6390 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
6393 case ISD::TRUNCATE:
6398 case ISD::ANY_EXTEND:
6399 case ISD::ZERO_EXTEND:
6404 case ISD::SIGN_EXTEND:
6409 case ISD::SPLAT_VECTOR_PARTS:
6411 case ISD::INSERT_VECTOR_ELT:
6413 case ISD::EXTRACT_VECTOR_ELT:
6415 case ISD::SCALAR_TO_VECTOR: {
6421 SDValue V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, WideVT, Scalar);
6422 return DAG.getNode(ISD::TRUNCATE, DL, VT, V);
6428 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar);
6435 case ISD::VSCALE: {
6452 Res = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
6455 Res = DAG.getNode(ISD::SHL, DL, XLenVT, Res,
6460 Res = DAG.getNode(ISD::MUL, DL, XLenVT, Res,
6463 SDValue VScale = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
6465 Res = DAG.getNode(ISD::MUL, DL, XLenVT, VScale,
6468 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
6470 case ISD::FPOWI: {
6476 SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
6478 DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
6479 return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
6484 case ISD::FMAXIMUM:
6485 case ISD::FMINIMUM:
6491 case ISD::FP_EXTEND: {
6501 return DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, FloatVal);
6508 case ISD::FP_ROUND: {
6518 DAG.getNode(ISD::FP_ROUND, DL, MVT::f32, Op0,
6527 case ISD::STRICT_FP_ROUND:
6528 case ISD::STRICT_FP_EXTEND:
6530 case ISD::SINT_TO_FP:
6531 case ISD::UINT_TO_FP:
6544 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC,
6548 case ISD::FP_TO_SINT:
6549 case ISD::FP_TO_UINT:
6561 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1);
6566 case ISD::STRICT_FP_TO_SINT:
6567 case ISD::STRICT_FP_TO_UINT:
6568 case ISD::STRICT_SINT_TO_FP:
6569 case ISD::STRICT_UINT_TO_FP: {
6594 unsigned ExtOpcode = (Op.getOpcode() == ISD::UINT_TO_FP ||
6595 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
6596 ? ISD::ZERO_EXTEND
6597 : ISD::SIGN_EXTEND;
6643 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
6647 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
6660 case ISD::FP_TO_SINT:
6663 case ISD::FP_TO_UINT:
6666 case ISD::SINT_TO_FP:
6669 case ISD::UINT_TO_FP:
6672 case ISD::STRICT_FP_TO_SINT:
6675 case ISD::STRICT_FP_TO_UINT:
6678 case ISD::STRICT_SINT_TO_FP:
6681 case ISD::STRICT_UINT_TO_FP:
6703 case ISD::FP_TO_SINT_SAT:
6704 case ISD::FP_TO_UINT_SAT:
6706 case ISD::FP_TO_BF16: {
6720 case ISD::BF16_TO_FP: {
6725 ISD::SHL, DL, Op.getOperand(0).getValueType(), Op.getOperand(0),
6732 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res);
6735 case ISD::FP_TO_FP16: {
6749 case ISD::FP16_TO_FP: {
6764 case ISD::FTRUNC:
6765 case ISD::FCEIL:
6766 case ISD::FFLOOR:
6767 case ISD::FNEARBYINT:
6768 case ISD::FRINT:
6769 case ISD::FROUND:
6770 case ISD::FROUNDEVEN:
6772 case ISD::LRINT:
6773 case ISD::LLRINT:
6775 case ISD::VECREDUCE_ADD:
6776 case ISD::VECREDUCE_UMAX:
6777 case ISD::VECREDUCE_SMAX:
6778 case ISD::VECREDUCE_UMIN:
6779 case ISD::VECREDUCE_SMIN:
6781 case ISD::VECREDUCE_AND:
6782 case ISD::VECREDUCE_OR:
6783 case ISD::VECREDUCE_XOR:
6787 case ISD::VECREDUCE_FADD:
6788 case ISD::VECREDUCE_SEQ_FADD:
6789 case ISD::VECREDUCE_FMIN:
6790 case ISD::VECREDUCE_FMAX:
6791 case ISD::VECREDUCE_FMAXIMUM:
6792 case ISD::VECREDUCE_FMINIMUM:
6794 case ISD::VP_REDUCE_ADD:
6795 case ISD::VP_REDUCE_UMAX:
6796 case ISD::VP_REDUCE_SMAX:
6797 case ISD::VP_REDUCE_UMIN:
6798 case ISD::VP_REDUCE_SMIN:
6799 case ISD::VP_REDUCE_FADD:
6800 case ISD::VP_REDUCE_SEQ_FADD:
6801 case ISD::VP_REDUCE_FMIN:
6802 case ISD::VP_REDUCE_FMAX:
6803 case ISD::VP_REDUCE_FMINIMUM:
6804 case ISD::VP_REDUCE_FMAXIMUM:
6810 case ISD::VP_REDUCE_AND:
6811 case ISD::VP_REDUCE_OR:
6812 case ISD::VP_REDUCE_XOR:
6816 case ISD::VP_CTTZ_ELTS:
6817 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
6819 case ISD::UNDEF: {
6824 case ISD::INSERT_SUBVECTOR:
6826 case ISD::EXTRACT_SUBVECTOR:
6828 case ISD::VECTOR_DEINTERLEAVE:
6830 case ISD::VECTOR_INTERLEAVE:
6832 case ISD::STEP_VECTOR:
6834 case ISD::VECTOR_REVERSE:
6836 case ISD::VECTOR_SPLICE:
6838 case ISD::BUILD_VECTOR:
6840 case ISD::SPLAT_VECTOR:
6852 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
6854 ISD::SPLAT_VECTOR, DL,
6857 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NewSplat,
6863 case ISD::VECTOR_SHUFFLE:
6865 case ISD::CONCAT_VECTORS: {
6887 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
6889 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
6891 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
6903 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, SubVec,
6908 case ISD::LOAD:
6914 case ISD::STORE:
6920 case ISD::MLOAD:
6921 case ISD::VP_LOAD:
6923 case ISD::MSTORE:
6924 case ISD::VP_STORE:
6926 case ISD::SELECT_CC: {
6941 DAG.getNode(ISD::SETCC, DL, CCVT, Tmp1, Tmp2, CC, Op->getFlags());
6944 case ISD::SETCC: {
6950 ISD::CondCode CCVal = cast<CondCodeSDNode>(Op.getOperand(2))->get();
6951 assert((CCVal == ISD::SETGT || CCVal == ISD::SETUGT) &&
6965 if (CCVal == ISD::SETUGT && Imm == -1)
6968 CCVal = ISD::getSetCCSwappedOperands(CCVal);
6977 CCVal = ISD::getSetCCSwappedOperands(CCVal);
6988 case ISD::ADD:
6989 case ISD::SUB:
6990 case ISD::MUL:
6991 case ISD::MULHS:
6992 case ISD::MULHU:
6993 case ISD::AND:
6994 case ISD::OR:
6995 case ISD::XOR:
6996 case ISD::SDIV:
6997 case ISD::SREM:
6998 case ISD::UDIV:
6999 case ISD::UREM:
7000 case ISD::BSWAP:
7001 case ISD::CTPOP:
7003 case ISD::SHL:
7004 case ISD::SRA:
7005 case ISD::SRL:
7012 case ISD::FADD:
7013 case ISD::FSUB:
7014 case ISD::FMUL:
7015 case ISD::FDIV:
7016 case ISD::FNEG:
7017 case ISD::FABS:
7018 case ISD::FSQRT:
7019 case ISD::FMA:
7020 case ISD::FMINNUM:
7021 case ISD::FMAXNUM:
7027 case ISD::AVGFLOORS:
7028 case ISD::AVGFLOORU:
7029 case ISD::AVGCEILS:
7030 case ISD::AVGCEILU:
7031 case ISD::SMIN:
7032 case ISD::SMAX:
7033 case ISD::UMIN:
7034 case ISD::UMAX:
7036 case ISD::UADDSAT:
7037 case ISD::USUBSAT:
7041 case ISD::SADDSAT:
7042 case ISD::SSUBSAT:
7046 case ISD::ABDS:
7047 case ISD::ABDU: {
7052 bool IsSigned = Op->getOpcode() == ISD::ABDS;
7056 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
7057 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
7060 return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
7062 case ISD::ABS:
7063 case ISD::VP_ABS:
7065 case ISD::CTLZ:
7066 case ISD::CTLZ_ZERO_UNDEF:
7067 case ISD::CTTZ:
7068 case ISD::CTTZ_ZERO_UNDEF:
7071 assert(Op.getOpcode() != ISD::CTTZ);
7073 case ISD::VSELECT:
7075 case ISD::FCOPYSIGN:
7081 case ISD::STRICT_FADD:
7082 case ISD::STRICT_FSUB:
7083 case ISD::STRICT_FMUL:
7084 case ISD::STRICT_FDIV:
7085 case ISD::STRICT_FSQRT:
7086 case ISD::STRICT_FMA:
7092 case ISD::STRICT_FSETCC:
7093 case ISD::STRICT_FSETCCS:
7095 case ISD::STRICT_FCEIL:
7096 case ISD::STRICT_FRINT:
7097 case ISD::STRICT_FFLOOR:
7098 case ISD::STRICT_FTRUNC:
7099 case ISD::STRICT_FNEARBYINT:
7100 case ISD::STRICT_FROUND:
7101 case ISD::STRICT_FROUNDEVEN:
7103 case ISD::MGATHER:
7104 case ISD::VP_GATHER:
7106 case ISD::MSCATTER:
7107 case ISD::VP_SCATTER:
7109 case ISD::GET_ROUNDING:
7111 case ISD::SET_ROUNDING:
7113 case ISD::EH_DWARF_CFA:
7115 case ISD::VP_SELECT:
7116 case ISD::VP_MERGE:
7117 case ISD::VP_ADD:
7118 case ISD::VP_SUB:
7119 case ISD::VP_MUL:
7120 case ISD::VP_SDIV:
7121 case ISD::VP_UDIV:
7122 case ISD::VP_SREM:
7123 case ISD::VP_UREM:
7124 case ISD::VP_UADDSAT:
7125 case ISD::VP_USUBSAT:
7126 case ISD::VP_SADDSAT:
7127 case ISD::VP_SSUBSAT:
7128 case ISD::VP_LRINT:
7129 case ISD::VP_LLRINT:
7131 case ISD::VP_AND:
7132 case ISD::VP_OR:
7133 case ISD::VP_XOR:
7135 case ISD::VP_FADD:
7136 case ISD::VP_FSUB:
7137 case ISD::VP_FMUL:
7138 case ISD::VP_FDIV:
7139 case ISD::VP_FNEG:
7140 case ISD::VP_FABS:
7141 case ISD::VP_SQRT:
7142 case ISD::VP_FMA:
7143 case ISD::VP_FMINNUM:
7144 case ISD::VP_FMAXNUM:
7145 case ISD::VP_FCOPYSIGN:
7151 case ISD::VP_SRA:
7152 case ISD::VP_SRL:
7153 case ISD::VP_SHL:
7155 case ISD::VP_IS_FPCLASS:
7157 case ISD::VP_SIGN_EXTEND:
7158 case ISD::VP_ZERO_EXTEND:
7162 case ISD::VP_TRUNCATE:
7164 case ISD::VP_FP_EXTEND:
7165 case ISD::VP_FP_ROUND:
7167 case ISD::VP_SINT_TO_FP:
7168 case ISD::VP_UINT_TO_FP:
7181 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC,
7185 case ISD::VP_FP_TO_SINT:
7186 case ISD::VP_FP_TO_UINT:
7198 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1);
7204 case ISD::VP_SETCC:
7212 case ISD::VP_SMIN:
7213 case ISD::VP_SMAX:
7214 case ISD::VP_UMIN:
7215 case ISD::VP_UMAX:
7216 case ISD::VP_BITREVERSE:
7217 case ISD::VP_BSWAP:
7219 case ISD::VP_CTLZ:
7220 case ISD::VP_CTLZ_ZERO_UNDEF:
7224 case ISD::VP_CTTZ:
7225 case ISD::VP_CTTZ_ZERO_UNDEF:
7229 case ISD::VP_CTPOP:
7231 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7233 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7235 case ISD::VP_FCEIL:
7236 case ISD::VP_FFLOOR:
7237 case ISD::VP_FRINT:
7238 case ISD::VP_FNEARBYINT:
7239 case ISD::VP_FROUND:
7240 case ISD::VP_FROUNDEVEN:
7241 case ISD::VP_FROUNDTOZERO:
7247 case ISD::VP_FMAXIMUM:
7248 case ISD::VP_FMINIMUM:
7254 case ISD::EXPERIMENTAL_VP_SPLICE:
7256 case ISD::EXPERIMENTAL_VP_REVERSE:
7258 case ISD::EXPERIMENTAL_VP_SPLAT:
7260 case ISD::CLEAR_CACHE: {
7431 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
7537 ISD::CondCode CC, SDValue Val) {
7538 assert(Val->getOpcode() == ISD::SETCC);
7541 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val.getOperand(2))->get();
7546 if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType()))
7549 CC2 = ISD::getSetCCSwappedOperands(CC2);
7552 if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType()))
7571 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(FalseV));
7575 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
7577 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(TrueV));
7582 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
7584 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(FalseV));
7589 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(TrueV));
7599 return DAG.getNode(ISD::XOR, DL, VT, Neg, FalseV);
7605 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
7606 FalseV.getOpcode() == ISD::SETCC) {
7609 ISD::CondCode CC = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
7614 return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV,
7620 return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT,
7642 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
7647 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
7705 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
7722 if (TrueV.getOpcode() == ISD::AND &&
7725 ISD::OR, DL, VT, TrueV,
7728 if (FalseV.getOpcode() == ISD::AND &&
7731 ISD::OR, DL, VT, FalseV,
7755 return DAG.getNode(ISD::ADD, DL, VT, CMOV, RHSVal);
7762 ISD::OR, DL, VT,
7779 if (NewSel.getOpcode() == ISD::SELECT)
7792 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, CondV);
7794 SDValue XOR = DAG.getNode(ISD::XOR, DL, XLenVT, CondV,
7796 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, XOR);
7804 if (CondV.getOpcode() != ISD::SETCC ||
7807 SDValue SetNE = DAG.getCondCode(ISD::SETNE);
7821 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
7831 CCVal == ISD::SETLT) {
7835 return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV);
7837 return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV);
7842 if (isOneConstant(LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
7846 if (CCVal == ISD::SETULT) {
7848 CCVal = ISD::SETNE;
7853 if (isAllOnesConstant(RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
7864 TargetCC = DAG.getCondCode(ISD::getSetCCInverse(CCVal, LHS.getValueType()));
7876 if (CondV.getOpcode() == ISD::SETCC &&
7880 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
7891 DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
7924 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
7952 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
7981 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
7982 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
7984 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
7985 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
7987 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
7988 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
7989 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
7990 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
7992 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
7994 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
7995 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
8025 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
8031 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
8032 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
8034 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
8035 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
8037 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
8038 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
8042 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
8044 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
8046 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
8047 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
8061 if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
8065 if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
8070 SplatVal = DAG.getNode(ISD::AND, DL, SplatVal.getValueType(), SplatVal,
8074 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
8123 return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
8181 bool IsVPTrunc = Op.getOpcode() == ISD::VP_TRUNCATE;
8224 {Trunc, SplatZero, DAG.getCondCode(ISD::SETNE),
8233 bool IsVPTrunc = Op.getOpcode() == ISD::VP_TRUNCATE;
8319 unsigned InterConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
8328 unsigned ConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
8346 Op.getOpcode() == ISD::VP_FP_ROUND || Op.getOpcode() == ISD::VP_FP_EXTEND;
8348 Op.getOpcode() == ISD::VP_FP_EXTEND || Op.getOpcode() == ISD::FP_EXTEND;
8458 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
8459 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
8460 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
8502 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
8530 Val = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Val);
8534 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec,
8570 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec,
8592 DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
8603 Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, OrigContainerVT, OrigVec,
8635 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
8636 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
8660 ISD::SRL, DL, XLenVT, Idx,
8664 ISD::AND, DL, XLenVT, Idx,
8668 Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec);
8669 SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT,
8673 DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx);
8674 SDValue Res = DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight,
8676 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
8681 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
8682 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
8707 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec,
8724 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
8755 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
8760 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
8767 assert((Op.getOpcode() == ISD::INTRINSIC_VOID ||
8768 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
8769 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
8775 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
8776 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
8805 isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
8826 ScalarOp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ScalarOp);
8866 I32VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVLMAX, SEW,
8883 SDValue VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVL, AVL,
8886 DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
8980 SDValue AVL = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, N->getOperand(1));
8984 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, ID, AVL, Sew, LMul);
8985 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res);
9006 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
9017 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
9018 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
9042 isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
9114 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9116 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9128 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9130 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9133 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9148 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9152 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9161 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9163 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9167 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9175 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9177 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9179 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9189 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(1));
9191 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2));
9192 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0,
9194 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1,
9197 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
9199 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res);
9210 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
9213 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getValueType(),
9226 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
9264 {VID, SplattedIdx, DAG.getCondCode(ISD::SETEQ),
9340 SDValue NewNode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, RetVT, Operands);
9416 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
9443 DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Load->getChain(), Ptr,
9476 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
9515 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
9570 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
9600 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, Store->getVTList(),
9637 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
9673 case ISD::VP_REDUCE_ADD:
9674 case ISD::VECREDUCE_ADD:
9676 case ISD::VP_REDUCE_UMAX:
9677 case ISD::VECREDUCE_UMAX:
9679 case ISD::VP_REDUCE_SMAX:
9680 case ISD::VECREDUCE_SMAX:
9682 case ISD::VP_REDUCE_UMIN:
9683 case ISD::VECREDUCE_UMIN:
9685 case ISD::VP_REDUCE_SMIN:
9686 case ISD::VECREDUCE_SMIN:
9688 case ISD::VP_REDUCE_AND:
9689 case ISD::VECREDUCE_AND:
9691 case ISD::VP_REDUCE_OR:
9692 case ISD::VECREDUCE_OR:
9694 case ISD::VP_REDUCE_XOR:
9695 case ISD::VECREDUCE_XOR:
9697 case ISD::VP_REDUCE_FADD:
9699 case ISD::VP_REDUCE_SEQ_FADD:
9701 case ISD::VP_REDUCE_FMAX:
9702 case ISD::VP_REDUCE_FMAXIMUM:
9704 case ISD::VP_REDUCE_FMIN:
9705 case ISD::VP_REDUCE_FMINIMUM:
9717 assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
9718 Op.getOpcode() == ISD::VECREDUCE_OR ||
9719 Op.getOpcode() == ISD::VECREDUCE_XOR ||
9720 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9721 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9722 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9743 ISD::CondCode CC;
9749 case ISD::VECREDUCE_AND:
9750 case ISD::VP_REDUCE_AND: {
9755 CC = ISD::SETEQ;
9756 BaseOpc = ISD::AND;
9759 case ISD::VECREDUCE_OR:
9760 case ISD::VP_REDUCE_OR:
9763 CC = ISD::SETNE;
9764 BaseOpc = ISD::OR;
9766 case ISD::VECREDUCE_XOR:
9767 case ISD::VP_REDUCE_XOR: {
9771 Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
9772 CC = ISD::SETNE;
9773 BaseOpc = ISD::XOR;
9779 SetCC = DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), SetCC);
9822 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, M1VT, DAG.getUNDEF(M1VT),
9828 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Reduction,
9838 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
9868 case ISD::AND:
9869 case ISD::OR:
9870 case ISD::UMAX:
9871 case ISD::UMIN:
9872 case ISD::SMAX:
9873 case ISD::SMIN:
9874 StartV = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Vec,
9893 case ISD::VECREDUCE_FADD: {
9899 case ISD::VECREDUCE_SEQ_FADD:
9902 case ISD::VECREDUCE_FMINIMUM:
9903 case ISD::VECREDUCE_FMAXIMUM:
9904 case ISD::VECREDUCE_FMIN:
9905 case ISD::VECREDUCE_FMAX: {
9907 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op.getOperand(0),
9910 (Opcode == ISD::VECREDUCE_FMIN || Opcode == ISD::VECREDUCE_FMINIMUM)
9939 if (Op.getOpcode() != ISD::VECREDUCE_FMINIMUM &&
9940 Op.getOpcode() != ISD::VECREDUCE_FMAXIMUM)
9949 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9954 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
9988 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
9997 {Vec, Vec, DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(PredVT), Mask, VL});
10000 SDValue StartIsNaN = DAG.getSetCC(DL, XLenVT, Start, Start, ISD::SETUO);
10001 VCPop = DAG.getNode(ISD::OR, DL, XLenVT, VCPop, StartIsNaN);
10003 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
10051 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
10052 SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
10053 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
10056 return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
10076 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
10083 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
10200 AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
10204 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
10228 VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
10237 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVecVT, Vec, SubVec,
10288 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
10289 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
10292 return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
10320 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ContainerVT, Vec,
10335 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
10412 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
10430 WideOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op));
10442 DAG.getConstant(0, DL, WideVT), ISD::SETNE));
10469 SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
10471 SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL,
10474 SDValue Even = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
10476 SDValue Odd = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, ResLo.getValue(1),
10485 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
10509 DAG.getNode(ISD::ADD, DL, IdxVT, EvenIdx, DAG.getConstant(1, DL, IdxVT));
10518 SDValue Even = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, EvenWide,
10520 SDValue Odd = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, OddWide,
10547 SDValue ResLo = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
10549 SDValue ResHi = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL,
10552 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
10554 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
10571 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
10583 SDValue OddMask = DAG.getNode(ISD::AND, DL, IdxVT, StepVec, Ones);
10587 ISD::CondCode::SETNE);
10593 SDValue Idx = DAG.getNode(ISD::SRL, DL, IdxVT, StepVec, Ones);
10606 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VecVT, Interleaved,
10609 ISD::EXTRACT_SUBVECTOR, DL, VecVT, Interleaved,
10631 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
10636 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
10653 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, Op.getOperand(0));
10654 SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1);
10655 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Op2);
10678 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
10679 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
10683 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
10686 ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
10699 SDValue VLMinus1 = DAG.getNode(ISD::SUB, DL, XLenVT,
10737 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DownOffset);
10742 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, UpOffset);
10797 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
10823 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getConstant(0, DL, VT),
10852 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other),
10879 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
10913 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
10946 ISD::isConstantSplatVectorAllOnes(Mask.getNode()) || IsCompressingStore;
10966 Val = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
10983 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
11020 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
11026 if (Opc == ISD::STRICT_FSETCCS) {
11030 if (CCVal == ISD::SETEQ || CCVal == ISD::SETOEQ) {
11031 SDValue OLECCVal = DAG.getCondCode(ISD::SETOLE);
11032 SDValue Tmp1 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
11034 SDValue Tmp2 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op2,
11036 SDValue OutChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
11040 Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2);
11045 if (CCVal == ISD::SETNE || CCVal == ISD::SETUNE) {
11046 SDValue OEQCCVal = DAG.getCondCode(ISD::SETOEQ);
11047 SDValue OEQ = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
11065 if (Opc == ISD::STRICT_FSETCC &&
11066 (CCVal == ISD::SETLT || CCVal == ISD::SETOLT || CCVal == ISD::SETLE ||
11067 CCVal == ISD::SETOLE)) {
11073 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11077 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
11087 unsigned RVVOpc = Opc == ISD::STRICT_FSETCC ? RISCVISD::STRICT_FSETCC_VL
11106 assert((Op.getOpcode() == ISD::VP_ABS || VT.isFixedLengthVector()) &&
11107 "Unexpected type for ISD::ABS");
11116 if (Op->getOpcode() == ISD::VP_ABS) {
11232 // Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
11255 auto MaskIdx = ISD::getVPMaskIdx(Op.getOpcode());
11259 } else if (ISD::getVPExplicitVectorLengthIdx(Op.getOpcode()) ==
11261 if (Op.getOpcode() == ISD::VP_MERGE) {
11265 assert(Op.getOpcode() == ISD::VP_SELECT);
11314 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1, DL, XLenVT);
11332 ISD::CondCode Condition = cast<CondCodeSDNode>(Op.getOperand(2))->get();
11350 case ISD::SETNE:
11354 case ISD::SETEQ: {
11363 case ISD::SETGT:
11364 case ISD::SETULT: {
11372 case ISD::SETLT:
11373 case ISD::SETUGT: {
11381 case ISD::SETGE:
11382 case ISD::SETULE: {
11390 case ISD::SETLE:
11391 case ISD::SETUGE: {
11512 {Result, SplatZero, DAG.getCondCode(ISD::SETNE),
11562 EVL1 = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, EVL1);
11594 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, DownOffset);
11599 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, UpOffset);
11613 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11704 SDValue LoRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
11705 SDValue HiRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
11712 DAG.getNode(ISD::CONCAT_VECTORS, DL, GatherVT, HiRev, LoRev);
11719 SDValue Diff = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, EVL);
11729 DAG.getCondCode(ISD::SETNE),
11745 DAG.getNode(ISD::SUB, DL, XLenVT, EVL, DAG.getConstant(1, DL, XLenVT));
11757 {Result, DAG.getConstant(0, DL, GatherVT), DAG.getCondCode(ISD::SETNE),
11806 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
11828 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
11854 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
11870 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, VPNode->getVTList(),
11892 [[maybe_unused]] ISD::LoadExtType LoadExtType;
11901 LoadExtType = ISD::NON_EXTLOAD;
11918 assert(LoadExtType == ISD::NON_EXTLOAD &&
11923 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
11945 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
11965 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
12021 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
12043 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
12056 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
12082 DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
12083 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
12085 SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
12111 RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, RMValue);
12113 SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
12115 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
12117 RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
12140 case ISD::SHL:
12142 case ISD::SRA:
12144 case ISD::SRL:
12146 case ISD::SDIV:
12148 case ISD::UDIV:
12150 case ISD::UREM:
12152 case ISD::ROTL:
12154 case ISD::ROTR:
12165 unsigned ExtOpc = ISD::ANY_EXTEND) {
12172 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
12179 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12180 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12182 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
12184 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
12194 case ISD::STRICT_FP_TO_SINT:
12195 case ISD::STRICT_FP_TO_UINT:
12196 case ISD::FP_TO_SINT:
12197 case ISD::FP_TO_UINT: {
12201 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
12202 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
12213 Op0 = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
12223 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12232 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
12238 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12262 case ISD::LROUND: {
12272 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
12277 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12289 Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
12293 case ISD::READCYCLECOUNTER:
12294 case ISD::READSTEADYCOUNTER: {
12300 if (N->getOpcode() == ISD::READCYCLECOUNTER) {
12316 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
12320 case ISD::LOAD: {
12321 if (!ISD::isNON_EXTLoad(N))
12329 SDValue Res = DAG.getExtLoad(ISD::SEXTLOAD, dl, MVT::i64, Ld->getChain(),
12332 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Res));
12336 case ISD::MUL: {
12354 S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
12355 U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
12356 SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
12358 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
12375 case ISD::ADD:
12376 case ISD::SUB:
12381 case ISD::SHL:
12382 case ISD::SRA:
12383 case ISD::SRL:
12386 if (N->getOperand(1).getOpcode() != ISD::Constant) {
12388 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() &&
12395 // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
12398 if (N->getOpcode() == ISD::SHL) {
12401 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12403 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
12404 SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
12405 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
12407 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
12411 case ISD::ROTL:
12412 case ISD::ROTR:
12423 case ISD::CTTZ:
12424 case ISD::CTTZ_ZERO_UNDEF:
12425 case ISD::CTLZ:
12426 case ISD::CTLZ_ZERO_UNDEF: {
12431 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12433 N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF;
12436 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12439 case ISD::SDIV:
12440 case ISD::UDIV:
12441 case ISD::UREM: {
12449 if (N->getOperand(1).getOpcode() == ISD::Constant &&
12456 unsigned ExtOpc = ISD::ANY_EXTEND;
12458 ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
12459 : ISD::ZERO_EXTEND;
12464 case ISD::SADDO: {
12473 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12474 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
12475 SDValue Res = DAG.getNode(ISD::ADD, DL, MVT::i64, LHS, RHS);
12476 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
12488 SDValue ResultLowerThanLHS = DAG.getSetCC(DL, OType, Res, LHS, ISD::SETLT);
12489 SDValue ConditionRHS = DAG.getSetCC(DL, OType, RHS, Zero, ISD::SETLT);
12492 DAG.getNode(ISD::XOR, DL, OType, ConditionRHS, ResultLowerThanLHS);
12493 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12497 case ISD::UADDO:
12498 case ISD::USUBO: {
12501 bool IsAdd = N->getOpcode() == ISD::UADDO;
12503 SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12504 SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12506 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
12507 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
12518 DAG.getConstant(0, DL, MVT::i64), ISD::SETEQ);
12522 DAG.getConstant(0, DL, MVT::i32), ISD::SETNE);
12527 LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12529 IsAdd ? ISD::SETULT : ISD::SETUGT);
12532 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12536 case ISD::UADDSAT:
12537 case ISD::USUBSAT: {
12545 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
12547 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
12549 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12558 case ISD::SADDSAT:
12559 case ISD::SSUBSAT: {
12565 case ISD::ABS: {
12573 SDValue Src = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64,
12576 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Abs));
12581 SDValue Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
12587 SDValue SignFill = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Src,
12589 SignFill = DAG.getNode(ISD::SRA, DL, MVT::i64, SignFill,
12592 SDValue NewRes = DAG.getNode(ISD::XOR, DL, MVT::i64, Src, SignFill);
12593 NewRes = DAG.getNode(ISD::SUB, DL, MVT::i64, NewRes, SignFill);
12597 NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewRes,
12599 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
12602 case ISD::BITCAST: {
12611 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
12615 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
12620 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
12625 SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64,
12637 Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
12652 SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
12656 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes));
12659 case ISD::EXTRACT_VECTOR_ELT: {
12714 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
12717 case ISD::INTRINSIC_WO_CHAIN: {
12725 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12731 DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res));
12757 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12759 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12767 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12769 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12772 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12779 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12783 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12790 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12792 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12796 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12804 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12806 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12808 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12826 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
12828 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
12829 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0,
12831 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1,
12836 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
12838 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
12848 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
12874 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
12880 case ISD::VECREDUCE_ADD:
12881 case ISD::VECREDUCE_AND:
12882 case ISD::VECREDUCE_OR:
12883 case ISD::VECREDUCE_XOR:
12884 case ISD::VECREDUCE_SMAX:
12885 case ISD::VECREDUCE_UMAX:
12886 case ISD::VECREDUCE_SMIN:
12887 case ISD::VECREDUCE_UMIN:
12891 case ISD::VP_REDUCE_ADD:
12892 case ISD::VP_REDUCE_AND:
12893 case ISD::VP_REDUCE_OR:
12894 case ISD::VP_REDUCE_XOR:
12895 case ISD::VP_REDUCE_SMAX:
12896 case ISD::VP_REDUCE_UMAX:
12897 case ISD::VP_REDUCE_SMIN:
12898 case ISD::VP_REDUCE_UMIN:
12902 case ISD::GET_ROUNDING: {
12904 SDValue Res = DAG.getNode(ISD::GET_ROUNDING, DL, VTs, N->getOperand(0));
12912 /// Given a binary operator, return the *associative* generic ISD::VECREDUCE_OP
12918 case ISD::ADD:
12919 return ISD::VECREDUCE_ADD;
12920 case ISD::UMAX:
12921 return ISD::VECREDUCE_UMAX;
12922 case ISD::SMAX:
12923 return ISD::VECREDUCE_SMAX;
12924 case ISD::UMIN:
12925 return ISD::VECREDUCE_UMIN;
12926 case ISD::SMIN:
12927 return ISD::VECREDUCE_SMIN;
12928 case ISD::AND:
12929 return ISD::VECREDUCE_AND;
12930 case ISD::OR:
12931 return ISD::VECREDUCE_OR;
12932 case ISD::XOR:
12933 return ISD::VECREDUCE_XOR;
12934 case ISD::FADD:
12936 return ISD::VECREDUCE_FADD;
12970 (Opc != ISD::FADD || !N->getFlags().hasAllowReassociation()))
12974 assert(Opc == ISD::getVecReduceBaseOpcode(ReduceOpc) &&
12982 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
12985 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
13003 if (LHS.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
13009 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec,
13022 if (ReduceVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
13031 SDValue Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ReduceVT, SrcVec,
13049 case ISD::ADD:
13051 case ISD::UMAX:
13053 case ISD::SMAX:
13055 case ISD::UMIN:
13057 case ISD::SMIN:
13059 case ISD::AND:
13061 case ISD::OR:
13063 case ISD::XOR:
13065 case ISD::FADD:
13067 case ISD::FMAXNUM:
13069 case ISD::FMINNUM:
13075 return V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
13090 if (Opc == ISD::FADD && !N->getFlags().hasAllowReassociation())
13100 if (ScalarV.getOpcode() == ISD::INSERT_SUBVECTOR &&
13135 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ScalarVT, DAG.getUNDEF(ScalarVT),
13163 if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
13189 return DAG.getNode(ISD::SHL, DL, VT, SHADD, DAG.getConstant(Bits, DL, VT));
13217 N->getOpcode() != ISD::AND)
13221 if (Slct.getOpcode() == ISD::SELECT && !Slct.getOperand(0).hasOneUse())
13229 if ((Slct.getOpcode() != ISD::SELECT &&
13264 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
13307 if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
13341 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
13344 DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getConstant(C0, DL, VT));
13345 return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getConstant(CB, DL, VT));
13367 if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND)
13384 Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0);
13385 Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1);
13393 N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
13412 if (N0.getOpcode() != ISD::XOR || !isOneConstant(N0.getOperand(1)))
13421 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
13467 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse()) {
13470 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
13474 CCVal = ISD::getSetCCInverse(CCVal, SetCCOpVT);
13477 } else if (N1.getOpcode() == ISD::XOR && isOneConstant(N1.getOperand(1)) &&
13478 N1.getOperand(0).getOpcode() == ISD::SETCC) {
13486 return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS);
13504 if (N0.getOpcode() != ISD::SHL || N0.getOperand(0) != N1 || !N0.hasOneUse())
13527 if (isNullConstant(N0) && N1.getOpcode() == ISD::SETCC && N1.hasOneUse() &&
13529 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
13530 if (CCVal == ISD::SETLT) {
13533 return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0),
13554 bool IsAnd = N->getOpcode() == ISD::AND;
13556 if (N0.getOpcode() != ISD::XOR || N1.getOpcode() != ISD::XOR)
13590 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
13592 return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT));
13610 if (N0.getOpcode() != ISD::VSELECT || !N0.hasOneUse())
13617 if (Cond.getOpcode() != ISD::SETCC)
13622 ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
13623 if (CCVal != ISD::SETULT)
13639 if (False.getOpcode() != ISD::SIGN_EXTEND)
13644 if (False.getOpcode() != ISD::SETCC || False.getOperand(0) != True)
13651 ISD::CondCode CCVal2 = cast<CondCodeSDNode>(False.getOperand(2))->get();
13652 if (CCVal2 != ISD::SETGT)
13658 DAG.getNode(ISD::SMAX, DL, SrcVT, True, DAG.getConstant(0, DL, SrcVT));
13660 DAG.getNode(ISD::SMIN, DL, SrcVT, Max,
13662 return DAG.getNode(ISD::TRUNCATE, DL, VT, Min);
13676 N0.getValueType() == MVT::i32 && N0.getOpcode() == ISD::SRL &&
13679 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
13680 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
13681 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
13682 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Srl);
13704 N0.getOpcode() == ISD::SRL && !isa<ConstantSDNode>(N0.getOperand(1)) &&
13707 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
13708 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
13709 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
13710 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, Srl,
13712 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And);
13733 assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
13748 if (TrueV.getOpcode() != ISD::XOR || FalseV.getOpcode() != ISD::XOR ||
13761 SDValue NewOr = DAG.getNode(ISD::OR, DL, VT, NewN0, NewN1);
13762 return DAG.getNode(ISD::XOR, DL, VT, NewOr, TrueV.getOperand(1));
13802 N0.getOpcode() == ISD::SHL && isAllOnesConstant(N0.getOperand(0)) &&
13805 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
13806 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
13807 SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i64, Op0, Op1);
13809 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And);
13817 TLI.isOperationLegal(ISD::ROTL, MVT::i64)) {
13824 if (N0.getOpcode() == ISD::SETCC && isOneConstant(N1) && N0.hasOneUse()) {
13826 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
13827 if (ConstN00 && CC == ISD::SETLT) {
13840 if (N->getValueType(0) == MVT::i32 && N0.getOpcode() == ISD::TRUNCATE &&
13841 isOneConstant(N1) && N0.getOperand(0).getOpcode() == ISD::SETCC) {
13847 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
13851 return DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N->getValueType(0), Setcc);
13906 if (X.getOpcode() == ISD::AND && isa<ConstantSDNode>(X.getOperand(1)) &&
13908 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, X,
13919 return DAG.getNode(ISD::SHL, DL, VT, Mul359,
13944 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
13977 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
13978 return DAG.getNode(ISD::ADD, DL, VT, Shift1,
13989 DAG.getNode(ISD::SHL, DL, VT, X,
13994 return DAG.getNode(ISD::SUB, DL, VT, Shift1, Mul359);
14004 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14007 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
14009 return DAG.getNode(ISD::SUB, DL, VT, Shift1, Shift2);
14026 if (N->getOperand(0).getOpcode() != ISD::AND ||
14027 N->getOperand(0).getOperand(0).getOpcode() != ISD::SRL)
14034 if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), V1) ||
14035 !ISD::isConstantSplatVector(And.getOperand(1).getNode(), V2) ||
14036 !ISD::isConstantSplatVector(Srl.getOperand(1).getNode(), V3))
14048 SDValue Cast = DAG.getNode(ISD::BITCAST, DL, HalfVT, Srl.getOperand(0));
14049 SDValue Sra = DAG.getNode(ISD::SRA, DL, HalfVT, Cast,
14051 return DAG.getNode(ISD::BITCAST, DL, VT, Sra);
14073 if ((AddSubOpc == ISD::ADD || AddSubOpc == ISD::SUB) && V->hasOneUse()) {
14076 if (AddSubOpc == ISD::SUB)
14085 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper);
14090 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper);
14105 static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG) {
14122 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) {
14128 N = DAG.getNode(ISD::TRUNCATE, DL,
14135 if (N.getOpcode() != ISD::SHL)
14139 if (N0.getOpcode() != ISD::ZERO_EXTEND &&
14147 if (!ISD::isConstantSplatVector(N1.getNode(), ShAmt))
14166 N = DAG.getNode(ISD::SHL, DL, NewVT, NewExt, NewShAmtVec);
14190 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
14196 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
14212 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
14214 SDValue SExtOp = DAG.getNode(ISD::SIGN_EXTEND_INREG, N, OpVT,
14287 case ISD::ZERO_EXTEND:
14288 case ISD::SIGN_EXTEND:
14301 OrigOperand.getOpcode() == ISD::SPLAT_VECTOR;
14339 case ISD::ZERO_EXTEND:
14340 case ISD::SIGN_EXTEND:
14345 case ISD::SPLAT_VECTOR:
14352 assert(Source.getOpcode() == ISD::FP_EXTEND && "Unexpected source");
14389 case ISD::ADD:
14393 case ISD::OR:
14395 case ISD::SUB:
14400 case ISD::MUL:
14412 case ISD::ADD:
14416 case ISD::OR:
14418 case ISD::SUB:
14423 case ISD::MUL:
14426 case ISD::SHL:
14454 assert((Opcode == RISCVISD::MUL_VL || Opcode == ISD::MUL) &&
14463 case ISD::ADD:
14465 case ISD::OR:
14468 case ISD::SUB:
14494 assert((Opc == ISD::SPLAT_VECTOR || Opc == RISCVISD::VMV_V_X_VL) &&
14502 SDValue Op = Opc == ISD::SPLAT_VECTOR ? OrigOperand.getOperand(0)
14554 case ISD::ZERO_EXTEND:
14555 case ISD::SIGN_EXTEND: {
14566 SupportsZExt = Opc == ISD::ZERO_EXTEND;
14567 SupportsSExt = Opc == ISD::SIGN_EXTEND;
14579 case ISD::SPLAT_VECTOR:
14590 if (Op.getOpcode() != ISD::FP_EXTEND)
14610 case ISD::ADD:
14611 case ISD::SUB:
14612 case ISD::MUL: {
14615 case ISD::OR: {
14634 case ISD::SHL:
14691 case ISD::ADD:
14692 case ISD::SUB:
14693 case ISD::MUL:
14694 case ISD::OR:
14695 case ISD::SHL: {
14709 case ISD::ADD:
14710 case ISD::MUL:
14711 case ISD::OR:
14720 case ISD::SUB:
14726 case ISD::SHL:
14777 case ISD::ADD:
14778 case ISD::SUB:
14779 case ISD::MUL:
14780 case ISD::OR:
14781 case ISD::SHL:
14921 case ISD::ADD:
14922 case ISD::SUB:
14923 case ISD::OR:
14936 case ISD::MUL:
14943 case ISD::SHL:
15090 if (MergeOpc != RISCVISD::VMERGE_VL && MergeOpc != ISD::VSELECT)
15111 if (Z.getOpcode() == ISD::INSERT_SUBVECTOR &&
15115 if (!ISD::isConstantSplatVectorAllZeros(Z.getNode()))
15160 if (LSNode1->getOpcode() == ISD::LOAD) {
15164 Opcode = (Ext == ISD::ZEXTLOAD) ? RISCVISD::TH_LWUD : RISCVISD::TH_LWD;
15209 unsigned OpNum = LSNode1->getOpcode() == ISD::LOAD ? 1 : 2;
15217 if (Ptr->getOpcode() == ISD::ADD)
15238 if (LSNode1->getOpcode() == ISD::LOAD)
15311 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
15377 return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
15420 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
15445 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
15455 if (Src.getOpcode() != ISD::BSWAP)
15606 assert(N->getOpcode() == ISD::SRA && "Unexpected opcode");
15623 N0.getOpcode() == ISD::SIGN_EXTEND_INREG && N0.hasOneUse() &&
15625 N0.getOperand(0).getOpcode() == ISD::SHL && N0.getOperand(0).hasOneUse() &&
15630 SDValue Shl = DAG.getNode(ISD::SHL, ShlDL, MVT::i64,
15634 return DAG.getNode(ISD::SRA, DL, MVT::i64, Shl,
15649 bool IsAdd = N0.getOpcode() == ISD::ADD;
15650 if ((IsAdd || N0.getOpcode() == ISD::SUB)) {
15664 if (U->getOpcode() != ISD::SRA ||
15677 if (Shl.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Shl.getOperand(1)) ||
15697 In = DAG.getNode(ISD::ADD, DL, MVT::i64, In, ShiftedAddC);
15699 In = DAG.getNode(ISD::SUB, DL, MVT::i64, ShiftedAddC, In);
15702 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, In,
15708 ISD::SHL, DL, MVT::i64, SExt,
15717 bool IsAnd = Cond.getOpcode() == ISD::AND;
15718 if (!IsAnd && Cond.getOpcode() != ISD::OR)
15727 if (Setcc.getOpcode() != ISD::SETCC)
15730 if (Setcc.getOpcode() != ISD::SETCC || !Setcc.hasOneUse() ||
15731 Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
15753 ISD::CondCode CCVal = cast<CondCodeSDNode>(Setcc.getOperand(2))->get();
15754 if (ISD::isIntEqualitySetCC(CCVal)) {
15755 CCVal = ISD::getSetCCInverse(CCVal, SetCCOpVT);
15758 } else if (CCVal == ISD::SETLT && isNullConstant(Setcc.getOperand(0))) {
15762 } else if (CCVal == ISD::SETLT && isOneConstant(Setcc.getOperand(1))) {
15770 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
15777 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
15783 if (isNullConstant(RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
15784 LHS.getOpcode() == ISD::SRA) {
15789 if (!ISD::isIntEqualitySetCC(CCVal))
15794 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
15798 bool Invert = CCVal == ISD::SETEQ;
15801 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
15812 if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS)) {
15819 if (isNullConstant(RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
15820 LHS.getOperand(1).getOpcode() == ISD::Constant) {
15822 if (LHS0.getOpcode() == ISD::AND &&
15823 LHS0.getOperand(1).getOpcode() == ISD::Constant) {
15827 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
15834 DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS0.getOperand(0),
15845 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
15853 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
15876 case ISD::SHL:
15877 case ISD::SRA:
15878 case ISD::SRL:
15879 case ISD::SUB:
15882 case ISD::ADD:
15883 case ISD::OR:
15884 case ISD::XOR:
15926 if (Cond.getOpcode() != ISD::SETCC)
15932 ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond->getOperand(2))->get();
15933 if (CCVal == ISD::CondCode::SETEQ) {
15936 } else if (CCVal == ISD::CondCode::SETNE) {
15943 if (CountZeroes.getOpcode() == ISD::TRUNCATE ||
15944 CountZeroes.getOpcode() == ISD::ZERO_EXTEND)
15947 if (CountZeroes.getOpcode() != ISD::CTTZ &&
15948 CountZeroes.getOpcode() != ISD::CTTZ_ZERO_UNDEF &&
15949 CountZeroes.getOpcode() != ISD::CTLZ &&
15950 CountZeroes.getOpcode() != ISD::CTLZ_ZERO_UNDEF)
15960 if (CountZeroes.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
15961 CountZeroes = DAG.getNode(ISD::CTTZ, SDLoc(CountZeroes),
15963 } else if (CountZeroes.getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
15964 CountZeroes = DAG.getNode(ISD::CTLZ, SDLoc(CountZeroes),
15972 auto AndNode = DAG.getNode(ISD::AND, SDLoc(N), CountZeroes.getValueType(),
15986 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
15995 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
15996 if (CC == ISD::SETEQ && LHS.getOpcode() == ISD::AND &&
16001 DAG.getSetCC(DL, CondVT, LHS, RHS, ISD::SETNE),
16115 if (!ISD::isBuildVectorOfConstantSDNodes(InVecRHS.getNode()))
16123 SDValue LHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
16125 SDValue RHS = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
16132 if (InVec.getOpcode() != ISD::CONCAT_VECTORS)
16148 ConcatOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ConcatVT,
16154 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps);
16176 if (!BaseLd || !BaseLd->isSimple() || !ISD::isNormalLoad(BaseLd) ||
16189 Ld->getChain() != BaseLd->getChain() || !ISD::isNormalLoad(Ld) ||
16212 if (P2.getOpcode() == ISD::ADD && P2.getOperand(0) == P1)
16214 if (P1.getOpcode() == ISD::ADD && P1.getOperand(0) == P2)
16289 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD);
16327 if (N->getOpcode() == ISD::ADD) {
16359 ISD::MemIndexType &IndexType,
16380 Index = DAG.getNode(ISD::SIGN_EXTEND, DL,
16383 IndexType = ISD::UNSIGNED_SCALED;
16393 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
16395 if (!ISD::isBuildVectorOfConstantSDNodes(Index.getNode()))
16428 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
16430 if (!ISD::isBuildVectorOfConstantSDNodes(Index.getNode()))
16499 if (Op.getOpcode() != ISD::SRA || !Op.hasOneUse())
16504 if (N0.getOpcode() != ISD::SIGN_EXTEND || !N0.hasOneUse() ||
16505 N1.getOpcode() != ISD::ZERO_EXTEND || !N1.hasOneUse())
16517 DAG.getNode(ISD::SMIN, SDLoc(N1), N->getValueType(0), N10,
16519 return DAG.getNode(ISD::SRA, SDLoc(N), N->getValueType(0), N00, SMin);
16546 if (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op.getOperand(0).isUndef() &&
16549 Op.getOperand(1).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
16554 if (ISD::isConstantSplatVector(Op.getNode(), SplatVal))
16575 if (SDValue UMinOp = MatchMinMax(V, ISD::UMIN, RISCVISD::UMIN_VL, HiC))
16581 if (SDValue SMinOp = MatchMinMax(V, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
16583 MatchMinMax(SMinOp, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
16590 if (SDValue SMaxOp = MatchMinMax(V, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
16592 MatchMinMax(SMaxOp, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
16609 if (SDValue SMinOp = MatchMinMax(V, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
16611 MatchMinMax(SMinOp, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
16615 if (SDValue SMaxOp = MatchMinMax(V, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
16617 MatchMinMax(SMaxOp, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
16672 if (N->getOpcode() != ISD::DELETED_NODE)
16707 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
16716 if (Op0.getOpcode() == ISD::FNEG) {
16717 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
16721 assert(Op0.getOpcode() == ISD::FABS);
16722 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
16775 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
16781 if (Op0.getOpcode() == ISD::FNEG)
16782 return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
16785 assert(Op0.getOpcode() == ISD::FABS);
16786 return DAG.getNode(ISD::AND, DL, VT, NewFMV,
16789 case ISD::ABS: {
16794 if (VT.isVector() && N0.hasOneUse() && N0.getOpcode() == ISD::SIGN_EXTEND) {
16797 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT,
16798 DAG.getNode(ISD::ABS, DL, Src.getValueType(), Src));
16802 case ISD::ADD: {
16809 case ISD::SUB: {
16814 case ISD::AND:
16816 case ISD::OR: {
16821 case ISD::XOR:
16823 case ISD::MUL:
16827 case ISD::SDIV:
16828 case ISD::UDIV:
16829 case ISD::SREM:
16830 case ISD::UREM:
16834 case ISD::FADD:
16835 case ISD::UMAX:
16836 case ISD::UMIN:
16837 case ISD::SMAX:
16838 case ISD::SMIN:
16839 case ISD::FMAXNUM:
16840 case ISD::FMINNUM: {
16847 case ISD::SETCC:
16849 case ISD::SIGN_EXTEND_INREG:
16851 case ISD::ZERO_EXTEND:
16857 if (Src.getOpcode() == ISD::FP_TO_UINT &&
16859 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
16861 if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
16864 SDValue Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, SDLoc(N), VTs,
16877 case ISD::TRUNCATE:
16879 case ISD::SELECT:
16897 if (Cond.getOpcode() == ISD::XOR && isOneConstant(Cond.getOperand(1))) {
16907 if (Cond.getOpcode() == ISD::SETCC && isNullConstant(Cond.getOperand(1))) {
16908 ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
16909 if (ISD::isIntEqualitySetCC(CCVal))
16910 return DAG.getNode(CCVal == ISD::SETNE ? Opc : InvOpc, SDLoc(N),
16920 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
16934 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) {
16935 if (CCVal == ISD::CondCode::SETGE)
16945 DAG.getNode(ISD::SRA, DL, VT, LHS,
16948 DAG.getNode(ISD::AND, DL, VT, SRA,
16950 return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV);
16953 if (CCVal == ISD::CondCode::SETGE)
16966 return DAG.getNode(ISD::OR, DL, VT, Neg, FalseV);
16971 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT));
16973 return DAG.getNode(ISD::OR, DL, VT, Neg, TrueV);
16979 DAG.getSetCC(DL, VT, LHS, RHS, ISD::getSetCCInverse(CCVal, VT));
16981 return DAG.getNode(ISD::AND, DL, VT, Neg, FalseV);
16987 return DAG.getNode(ISD::AND, DL, VT, Neg, TrueV);
16992 CCVal == ISD::CondCode::SETNE) ||
16994 CCVal == ISD::CondCode::SETEQ)) &&
16998 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, ISD::CondCode::SETEQ);
16999 return DAG.getNode(ISD::ADD, DL, VT, LHS, C);
17007 if (TrueV.getOpcode() == ISD::XOR && FalseV.getOpcode() == ISD::XOR &&
17013 return DAG.getNode(ISD::XOR, DL, VT, NewSel, TrueV.getOperand(1));
17030 case ISD::BITREVERSE:
17032 case ISD::FP_TO_SINT:
17033 case ISD::FP_TO_UINT:
17035 case ISD::FP_TO_SINT_SAT:
17036 case ISD::FP_TO_UINT_SAT:
17038 case ISD::FCOPYSIGN: {
17051 if (In2.getOpcode() != ISD::FP_EXTEND &&
17052 (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0))
17055 if (In2.getOpcode() != ISD::FNEG)
17059 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
17060 DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
17062 case ISD::MGATHER: {
17067 ISD::MemIndexType IndexType = MGN->getIndexType();
17086 if (Index.getOpcode() == ISD::BUILD_VECTOR &&
17087 MGN->getExtensionType() == ISD::NON_EXTLOAD && isTypeLegal(VT)) {
17102 assert(IndexType == ISD::UNSIGNED_SCALED);
17103 SDValue BasePtr = DAG.getNode(ISD::ADD, DL, PtrVT, MGN->getBasePtr(),
17112 SDValue VPSelect = DAG.getNode(ISD::VP_SELECT, DL, VT, MGN->getMask(),
17120 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
17126 ISD::UNINDEXED, ISD::NON_EXTLOAD);
17132 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
17157 MGN->getMemOperand(), IndexType, ISD::NON_EXTLOAD);
17163 case ISD::MSCATTER:{
17167 ISD::MemIndexType IndexType = MSN->getIndexType();
17195 ISD::UNINDEXED, false);
17199 case ISD::VP_GATHER: {
17203 ISD::MemIndexType IndexType = VPGN->getIndexType();
17224 case ISD::VP_SCATTER: {
17228 ISD::MemIndexType IndexType = VPSN->getIndexType();
17267 case ISD::SRA:
17271 case ISD::SRL:
17272 case ISD::SHL: {
17273 if (N->getOpcode() == ISD::SHL) {
17320 case ISD::LOAD:
17321 case ISD::STORE: {
17326 if (N->getOpcode() != ISD::STORE)
17336 MemVT.isFixedLengthVector() && ISD::isNormalStore(Store) &&
17352 ISD::isBuildVectorOfConstantSDNodes(Val.getNode())) {
17382 Store->getChain() == SDValue(L, 1) && ISD::isNormalLoad(L) &&
17403 Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
17423 case ISD::SPLAT_VECTOR: {
17433 case ISD::BUILD_VECTOR:
17437 case ISD::CONCAT_VECTORS:
17441 case ISD::INSERT_VECTOR_ELT:
17486 Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
17513 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Passthru,
17517 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Passthru, Result,
17537 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, M1VT, Vec,
17543 case ISD::INTRINSIC_VOID:
17544 case ISD::INTRINSIC_W_CHAIN:
17545 case ISD::INTRINSIC_WO_CHAIN: {
17546 unsigned IntOpNo = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1;
17568 ISD::UNINDEXED, ISD::NON_EXTLOAD);
17586 ISD::UNINDEXED, false);
17609 case ISD::BITCAST: {
17622 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i1, Ops);
17624 return DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
17655 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
17656 N->getOpcode() == ISD::SRL) &&
17667 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
17717 if (Opcode != ISD::AND && Opcode != ISD::OR && Opcode != ISD::XOR)
17752 if (Opcode == ISD::AND) {
17817 assert((Opc >= ISD::BUILTIN_OP_END ||
17818 Opc == ISD::INTRINSIC_WO_CHAIN ||
17819 Opc == ISD::INTRINSIC_W_CHAIN ||
17820 Opc == ISD::INTRINSIC_VOID) &&
17917 case ISD::INTRINSIC_W_CHAIN:
17918 case ISD::INTRINSIC_WO_CHAIN: {
17920 Op.getConstantOperandVal(Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1);
18006 case ISD::INTRINSIC_W_CHAIN: {
18057 if (!ISD::isNormalLoad(Ld))
18950 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
18952 ISD::ArgFlagsTy ArgFlags2, bool EABI) {
18996 ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
19085 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
19147 ISD::ArgFlagsTy AF = PendingArgFlags[0];
19248 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
19265 ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
19286 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
19300 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
19332 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Val);
19333 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
19338 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
19349 const ISD::InputArg &In,
19398 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
19399 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Val);
19404 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
19431 ISD::LoadExtType ExtType;
19438 ExtType = ISD::NON_EXTLOAD;
19484 ISD::ArgFlagsTy ArgFlags, CCState &State,
19600 ISD::ArgFlagsTy ArgFlags, CCState &State) {
19662 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
19744 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
19745 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
19818 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
19897 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
19899 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
19939 ISD::ArgFlagsTy Flags = Outs[i].Flags;
19970 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
19992 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
20031 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
20047 DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
20073 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
20084 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
20196 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
20204 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
20217 const SmallVectorImpl<ISD::OutputArg> &Outs,
20339 if (Copy->getOpcode() == ISD::BITCAST) {
20345 if (Copy->getOpcode() != ISD::CopyToReg) {
20349 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
21135 return Extend.getOpcode() == ISD::ZERO_EXTEND &&
21190 ISD::MemIndexedMode &AM,
21196 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
21202 if (Op->getOpcode() == ISD::SUB)
21226 ISD::MemIndexedMode &AM,
21242 AM = ISD::PRE_INC;
21249 ISD::MemIndexedMode &AM,
21252 if (Op->getOpcode() != ISD::ADD)
21267 AM = ISD::POST_INC;
21289 AM = ISD::POST_INC;
21315 ISD::NodeType RISCVTargetLowering::getExtendForAtomicCmpSwapArg() const {
21317 return Subtarget.hasStdExtZacas() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
21487 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
21488 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
21489 Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
21491 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
21516 Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SameEltTypeVT,
21520 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
21523 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
21542 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
21543 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
21544 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
21569 Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
21571 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
21591 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND)
21979 ISD::CondCode Cond) const {