...
▽

Text file src/cmd/compile/internal/ssa/_gen/386.rules

Documentation: cmd/compile/internal/ssa/_gen

     1// Copyright 2016 The Go Authors. All rights reserved.
     2// Use of this source code is governed by a BSD-style
     3// license that can be found in the LICENSE file.
     4
     5// Lowering arithmetic
     6(Add(Ptr|32|16|8) ...) => (ADDL ...)
     7(Add(32|64)F ...) => (ADDS(S|D) ...)
     8(Add32carry ...) => (ADDLcarry ...)
     9(Add32withcarry ...) => (ADCL ...)
    10(Add32carrywithcarry ...) => (ADCLcarry ...)
    11
    12(Sub(Ptr|32|16|8) ...) => (SUBL ...)
    13(Sub(32|64)F ...) => (SUBS(S|D) ...)
    14(Sub32carry ...) => (SUBLcarry ...)
    15(Sub32withcarry ...) => (SBBL ...)
    16
    17(Mul(32|16|8) ...) => (MULL ...)
    18(Mul(32|64)F ...) => (MULS(S|D) ...)
    19(Mul32uhilo ...) => (MULLQU ...)
    20
    21(Select0 (Mul32uover x y)) => (Select0 <typ.UInt32> (MULLU x y))
    22(Select1 (Mul32uover x y)) => (SETO (Select1 <types.TypeFlags> (MULLU x y)))
    23
    24(Avg32u ...) => (AVGLU ...)
    25
    26(Div(32|64)F ...) => (DIVS(S|D) ...)
    27(Div(32|32u|16|16u) ...) => (DIV(L|LU|W|WU) ...)
    28(Div8   x y) => (DIVW  (SignExt8to16 x) (SignExt8to16 y))
    29(Div8u  x y) => (DIVWU (ZeroExt8to16 x) (ZeroExt8to16 y))
    30
    31(Hmul(32|32u) ...) => (HMUL(L|LU) ...)
    32
    33(Mod(32|32u|16|16u) ...) => (MOD(L|LU|W|WU) ...)
    34(Mod8   x y) => (MODW  (SignExt8to16 x) (SignExt8to16 y))
    35(Mod8u  x y) => (MODWU (ZeroExt8to16 x) (ZeroExt8to16 y))
    36
    37(And(32|16|8) ...) => (ANDL ...)
    38(Or(32|16|8) ...) => (ORL ...)
    39(Xor(32|16|8) ...) => (XORL ...)
    40
    41(Neg(32|16|8) ...) => (NEGL ...)
    42(Neg32F x) => (PXOR x (MOVSSconst <typ.Float32> [float32(math.Copysign(0, -1))]))
    43(Neg64F x) => (PXOR x (MOVSDconst <typ.Float64> [math.Copysign(0, -1)]))
    44
    45(Com(32|16|8) ...) => (NOTL ...)
    46
    47// Lowering boolean ops
    48(AndB ...) => (ANDL ...)
    49(OrB ...) => (ORL ...)
    50(Not x) => (XORLconst [1] x)
    51
    52// Lowering pointer arithmetic
    53(OffPtr [off] ptr) => (ADDLconst [int32(off)] ptr)
    54
    55(Bswap32 ...) => (BSWAPL ...)
    56(Bswap16 x) => (ROLWconst [8] x)
    57
    58(Sqrt ...) => (SQRTSD ...)
    59(Sqrt32 ...) => (SQRTSS ...)
    60
    61(Ctz8 x) => (BSFL (ORLconst <typ.UInt32> [0x100] x))
    62(Ctz8NonZero ...) => (BSFL ...)
    63(Ctz16 x) => (BSFL (ORLconst <typ.UInt32> [0x10000] x))
    64(Ctz16NonZero ...) => (BSFL ...)
    65(Ctz32 ...) => (LoweredCtz32 ...)
    66(Ctz32NonZero ...) => (BSFL ...)
    67(Ctz64On32 ...) => (LoweredCtz64 ...)
    68
    69// Lowering extension
    70(SignExt8to16  ...) => (MOVBLSX ...)
    71(SignExt8to32  ...) => (MOVBLSX ...)
    72(SignExt16to32 ...) => (MOVWLSX ...)
    73
    74(ZeroExt8to16  ...) => (MOVBLZX ...)
    75(ZeroExt8to32  ...) => (MOVBLZX ...)
    76(ZeroExt16to32 ...) => (MOVWLZX ...)
    77
    78(Signmask x) => (SARLconst x [31])
    79(Zeromask <t> x) => (XORLconst [-1] (SBBLcarrymask <t> (CMPLconst x [1])))
    80(Slicemask <t> x) => (SARLconst (NEGL <t> x) [31])
    81
    82// Lowering truncation
    83// Because we ignore high parts of registers, truncates are just copies.
    84(Trunc16to8  ...) => (Copy ...)
    85(Trunc32to8  ...) => (Copy ...)
    86(Trunc32to16 ...) => (Copy ...)
    87
    88// Lowering float-int conversions
    89(Cvt32to32F ...) => (CVTSL2SS ...)
    90(Cvt32to64F ...) => (CVTSL2SD ...)
    91
    92(Cvt32Fto32 ...) => (CVTTSS2SL ...)
    93(Cvt64Fto32 ...) => (CVTTSD2SL ...)
    94
    95(Cvt32Fto64F ...) => (CVTSS2SD ...)
    96(Cvt64Fto32F ...) => (CVTSD2SS ...)
    97
    98(Round32F ...) => (Copy ...)
    99(Round64F ...) => (Copy ...)
   100
   101(CvtBoolToUint8 ...) => (Copy ...)
   102
   103// Lowering shifts
   104// Unsigned shifts need to return 0 if shift amount is >= width of shifted value.
   105//   result = (arg << shift) & (shift >= argbits ? 0 : 0xffffffffffffffff)
   106(Lsh32x(32|16|8) <t> x y) && !shiftIsBounded(v) => (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMP(L|W|B)const y [32])))
   107(Lsh16x(32|16|8) <t> x y) && !shiftIsBounded(v) => (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMP(L|W|B)const y [32])))
   108(Lsh8x(32|16|8)  <t> x y) && !shiftIsBounded(v) => (ANDL (SHLL <t> x y) (SBBLcarrymask <t> (CMP(L|W|B)const y [32])))
   109
   110(Lsh32x(32|16|8) <t> x y) && shiftIsBounded(v) => (SHLL <t> x y)
   111(Lsh16x(32|16|8) <t> x y) && shiftIsBounded(v) => (SHLL <t> x y)
   112(Lsh8x(32|16|8)  <t> x y) && shiftIsBounded(v) => (SHLL <t> x y)
   113
   114(Rsh32Ux(32|16|8) <t> x y) && !shiftIsBounded(v) => (ANDL (SHRL <t> x y) (SBBLcarrymask <t> (CMP(L|W|B)const y [32])))
   115(Rsh16Ux(32|16|8) <t> x y) && !shiftIsBounded(v) => (ANDL (SHRW <t> x y) (SBBLcarrymask <t> (CMP(L|W|B)const y [16])))
   116(Rsh8Ux(32|16|8)  <t> x y) && !shiftIsBounded(v) => (ANDL (SHRB <t> x y) (SBBLcarrymask <t> (CMP(L|W|B)const y [8])))
   117
   118(Rsh32Ux(32|16|8) <t> x y) && shiftIsBounded(v) => (SHRL <t> x y)
   119(Rsh16Ux(32|16|8) <t> x y) && shiftIsBounded(v) => (SHRW <t> x y)
   120(Rsh8Ux(32|16|8)  <t> x y) && shiftIsBounded(v) => (SHRB <t> x y)
   121
   122// Signed right shift needs to return 0/-1 if shift amount is >= width of shifted value.
   123// We implement this by setting the shift value to -1 (all ones) if the shift value is >= width.
   124
   125(Rsh32x(32|16|8) <t> x y) && !shiftIsBounded(v) => (SARL <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMP(L|W|B)const y [32])))))
   126(Rsh16x(32|16|8) <t> x y) && !shiftIsBounded(v) => (SARW <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMP(L|W|B)const y [16])))))
   127(Rsh8x(32|16|8) <t> x y)  && !shiftIsBounded(v) => (SARB <t> x (ORL <y.Type> y (NOTL <y.Type> (SBBLcarrymask <y.Type> (CMP(L|W|B)const y [8])))))
   128
   129(Rsh32x(32|16|8) <t> x y) && shiftIsBounded(v) => (SARL x y)
   130(Rsh16x(32|16|8) <t> x y) && shiftIsBounded(v) => (SARW x y)
   131(Rsh8x(32|16|8) <t> x y)  && shiftIsBounded(v) => (SARB x y)
   132
   133// constant shifts
   134// generic opt rewrites all constant shifts to shift by Const64
   135(Lsh32x64 x (Const64 [c])) && uint64(c) < 32 => (SHLLconst x [int32(c)])
   136(Rsh32x64 x (Const64 [c])) && uint64(c) < 32 => (SARLconst x [int32(c)])
   137(Rsh32Ux64 x (Const64 [c])) && uint64(c) < 32 => (SHRLconst x [int32(c)])
   138(Lsh16x64 x (Const64 [c])) && uint64(c) < 16 => (SHLLconst x [int32(c)])
   139(Rsh16x64 x (Const64 [c])) && uint64(c) < 16 => (SARWconst x [int16(c)])
   140(Rsh16Ux64 x (Const64 [c])) && uint64(c) < 16 => (SHRWconst x [int16(c)])
   141(Lsh8x64 x (Const64 [c])) && uint64(c) < 8 => (SHLLconst x [int32(c)])
   142(Rsh8x64 x (Const64 [c])) && uint64(c) < 8 => (SARBconst x [int8(c)])
   143(Rsh8Ux64 x (Const64 [c])) && uint64(c) < 8 => (SHRBconst x [int8(c)])
   144
   145// large constant shifts
   146(Lsh32x64 _ (Const64 [c])) && uint64(c) >= 32 => (Const32 [0])
   147(Rsh32Ux64 _ (Const64 [c])) && uint64(c) >= 32 => (Const32 [0])
   148(Lsh16x64 _ (Const64 [c])) && uint64(c) >= 16 => (Const16 [0])
   149(Rsh16Ux64 _ (Const64 [c])) && uint64(c) >= 16 => (Const16 [0])
   150(Lsh8x64 _ (Const64 [c])) && uint64(c) >= 8 => (Const8 [0])
   151(Rsh8Ux64 _ (Const64 [c])) && uint64(c) >= 8 => (Const8 [0])
   152
   153// large constant signed right shift, we leave the sign bit
   154(Rsh32x64 x (Const64 [c])) && uint64(c) >= 32 => (SARLconst x [31])
   155(Rsh16x64 x (Const64 [c])) && uint64(c) >= 16 => (SARWconst x [15])
   156(Rsh8x64 x (Const64 [c])) && uint64(c) >= 8 => (SARBconst x [7])
   157
   158// rotates
   159(RotateLeft32 ...) => (ROLL ...)
   160(RotateLeft16 ...) => (ROLW ...)
   161(RotateLeft8  ...) => (ROLB ...)
   162// constant rotates
   163(ROLL x (MOVLconst [c])) => (ROLLconst [c&31] x)
   164(ROLW x (MOVLconst [c])) => (ROLWconst [int16(c&15)] x)
   165(ROLB x (MOVLconst [c])) => (ROLBconst [int8(c&7)] x)
   166
   167// Lowering comparisons
   168(Less32  x y) => (SETL (CMPL x y))
   169(Less16  x y) => (SETL (CMPW x y))
   170(Less8   x y) => (SETL (CMPB x y))
   171(Less32U x y) => (SETB (CMPL x y))
   172(Less16U x y) => (SETB (CMPW x y))
   173(Less8U  x y) => (SETB (CMPB x y))
   174// Use SETGF with reversed operands to dodge NaN case
   175(Less64F x y) => (SETGF (UCOMISD y x))
   176(Less32F x y) => (SETGF (UCOMISS y x))
   177
   178(Leq32  x y) => (SETLE (CMPL x y))
   179(Leq16  x y) => (SETLE (CMPW x y))
   180(Leq8   x y) => (SETLE (CMPB x y))
   181(Leq32U x y) => (SETBE (CMPL x y))
   182(Leq16U x y) => (SETBE (CMPW x y))
   183(Leq8U  x y) => (SETBE (CMPB x y))
   184// Use SETGEF with reversed operands to dodge NaN case
   185(Leq64F x y) => (SETGEF (UCOMISD y x))
   186(Leq32F x y) => (SETGEF (UCOMISS y x))
   187
   188(Eq32  x y) => (SETEQ (CMPL x y))
   189(Eq16  x y) => (SETEQ (CMPW x y))
   190(Eq8   x y) => (SETEQ (CMPB x y))
   191(EqB   x y) => (SETEQ (CMPB x y))
   192(EqPtr x y) => (SETEQ (CMPL x y))
   193(Eq64F x y) => (SETEQF (UCOMISD x y))
   194(Eq32F x y) => (SETEQF (UCOMISS x y))
   195
   196(Neq32  x y) => (SETNE (CMPL x y))
   197(Neq16  x y) => (SETNE (CMPW x y))
   198(Neq8   x y) => (SETNE (CMPB x y))
   199(NeqB   x y) => (SETNE (CMPB x y))
   200(NeqPtr x y) => (SETNE (CMPL x y))
   201(Neq64F x y) => (SETNEF (UCOMISD x y))
   202(Neq32F x y) => (SETNEF (UCOMISS x y))
   203
   204// Lowering loads
   205(Load <t> ptr mem) && (is32BitInt(t) || isPtr(t)) => (MOVLload ptr mem)
   206(Load <t> ptr mem) && is16BitInt(t) => (MOVWload ptr mem)
   207(Load <t> ptr mem) && (t.IsBoolean() || is8BitInt(t)) => (MOVBload ptr mem)
   208(Load <t> ptr mem) && is32BitFloat(t) => (MOVSSload ptr mem)
   209(Load <t> ptr mem) && is64BitFloat(t) => (MOVSDload ptr mem)
   210
   211// Lowering stores
   212(Store {t} ptr val mem) && t.Size() == 8 &&  t.IsFloat() => (MOVSDstore ptr val mem)
   213(Store {t} ptr val mem) && t.Size() == 4 &&  t.IsFloat() => (MOVSSstore ptr val mem)
   214(Store {t} ptr val mem) && t.Size() == 4 && !t.IsFloat() => (MOVLstore ptr val mem)
   215(Store {t} ptr val mem) && t.Size() == 2 => (MOVWstore ptr val mem)
   216(Store {t} ptr val mem) && t.Size() == 1 => (MOVBstore ptr val mem)
   217
   218// Lowering moves
   219(Move [0] _ _ mem) => mem
   220(Move [1] dst src mem) => (MOVBstore dst (MOVBload src mem) mem)
   221(Move [2] dst src mem) => (MOVWstore dst (MOVWload src mem) mem)
   222(Move [4] dst src mem) => (MOVLstore dst (MOVLload src mem) mem)
   223(Move [3] dst src mem) =>
   224	(MOVBstore [2] dst (MOVBload [2] src mem)
   225		(MOVWstore dst (MOVWload src mem) mem))
   226(Move [5] dst src mem) =>
   227	(MOVBstore [4] dst (MOVBload [4] src mem)
   228		(MOVLstore dst (MOVLload src mem) mem))
   229(Move [6] dst src mem) =>
   230	(MOVWstore [4] dst (MOVWload [4] src mem)
   231		(MOVLstore dst (MOVLload src mem) mem))
   232(Move [7] dst src mem) =>
   233	(MOVLstore [3] dst (MOVLload [3] src mem)
   234		(MOVLstore dst (MOVLload src mem) mem))
   235(Move [8] dst src mem) =>
   236	(MOVLstore [4] dst (MOVLload [4] src mem)
   237		(MOVLstore dst (MOVLload src mem) mem))
   238
   239// Adjust moves to be a multiple of 4 bytes.
   240(Move [s] dst src mem)
   241	&& s > 8 && s%4 != 0 =>
   242	(Move [s-s%4]
   243		(ADDLconst <dst.Type> dst [int32(s%4)])
   244		(ADDLconst <src.Type> src [int32(s%4)])
   245		(MOVLstore dst (MOVLload src mem) mem))
   246
   247// Medium copying uses a duff device.
   248(Move [s] dst src mem)
   249	&& s > 8 && s <= 4*128 && s%4 == 0
   250	&& logLargeCopy(v, s) =>
   251	(DUFFCOPY [10*(128-s/4)] dst src mem)
   252// 10 and 128 are magic constants.  10 is the number of bytes to encode:
   253//	MOVL	(SI), CX
   254//	ADDL	$4, SI
   255//	MOVL	CX, (DI)
   256//	ADDL	$4, DI
   257// and 128 is the number of such blocks. See src/runtime/duff_386.s:duffcopy.
   258
   259// Large copying uses REP MOVSL.
   260(Move [s] dst src mem) && s > 4*128 && s%4 == 0 && logLargeCopy(v, s) =>
   261	(REPMOVSL dst src (MOVLconst [int32(s/4)]) mem)
   262
   263// Lowering Zero instructions
   264(Zero [0] _ mem) => mem
   265(Zero [1] destptr mem) => (MOVBstoreconst [0] destptr mem)
   266(Zero [2] destptr mem) => (MOVWstoreconst [0] destptr mem)
   267(Zero [4] destptr mem) => (MOVLstoreconst [0] destptr mem)
   268
   269(Zero [3] destptr mem) =>
   270	(MOVBstoreconst [makeValAndOff(0,2)] destptr
   271		(MOVWstoreconst [makeValAndOff(0,0)] destptr mem))
   272(Zero [5] destptr mem) =>
   273	(MOVBstoreconst [makeValAndOff(0,4)] destptr
   274		(MOVLstoreconst [makeValAndOff(0,0)] destptr mem))
   275(Zero [6] destptr mem) =>
   276	(MOVWstoreconst [makeValAndOff(0,4)] destptr
   277		(MOVLstoreconst [makeValAndOff(0,0)] destptr mem))
   278(Zero [7] destptr mem) =>
   279	(MOVLstoreconst [makeValAndOff(0,3)] destptr
   280		(MOVLstoreconst [makeValAndOff(0,0)] destptr mem))
   281
   282// Strip off any fractional word zeroing.
   283(Zero [s] destptr mem) && s%4 != 0 && s > 4 =>
   284	(Zero [s-s%4] (ADDLconst destptr [int32(s%4)])
   285		(MOVLstoreconst [0] destptr mem))
   286
   287// Zero small numbers of words directly.
   288(Zero [8] destptr mem) =>
   289	(MOVLstoreconst [makeValAndOff(0,4)] destptr
   290		(MOVLstoreconst [makeValAndOff(0,0)] destptr mem))
   291(Zero [12] destptr mem) =>
   292	(MOVLstoreconst [makeValAndOff(0,8)] destptr
   293		(MOVLstoreconst [makeValAndOff(0,4)] destptr
   294			(MOVLstoreconst [makeValAndOff(0,0)] destptr mem)))
   295(Zero [16] destptr mem) =>
   296	(MOVLstoreconst [makeValAndOff(0,12)] destptr
   297		(MOVLstoreconst [makeValAndOff(0,8)] destptr
   298			(MOVLstoreconst [makeValAndOff(0,4)] destptr
   299				(MOVLstoreconst [makeValAndOff(0,0)] destptr mem))))
   300
   301// Medium zeroing uses a duff device.
   302(Zero [s] destptr mem)
   303  && s > 16 && s <= 4*128 && s%4 == 0 =>
   304	(DUFFZERO [1*(128-s/4)] destptr (MOVLconst [0]) mem)
   305// 1 and 128 are magic constants.  1 is the number of bytes to encode STOSL.
   306// 128 is the number of STOSL instructions in duffzero.
   307// See src/runtime/duff_386.s:duffzero.
   308
   309// Large zeroing uses REP STOSQ.
   310(Zero [s] destptr mem)
   311  && s > 4*128
   312  && s%4 == 0 =>
   313	(REPSTOSL destptr (MOVLconst [int32(s/4)]) (MOVLconst [0]) mem)
   314
   315
   316// Lowering constants
   317(Const8   [c]) => (MOVLconst [int32(c)])
   318(Const16  [c]) => (MOVLconst [int32(c)])
   319(Const32  ...) => (MOVLconst ...)
   320(Const(32|64)F ...) => (MOVS(S|D)const ...)
   321(ConstNil) => (MOVLconst [0])
   322(ConstBool [c]) => (MOVLconst [b2i32(c)])
   323
   324// Lowering calls
   325(StaticCall ...) => (CALLstatic ...)
   326(ClosureCall ...) => (CALLclosure ...)
   327(InterCall ...) => (CALLinter ...)
   328(TailCall ...) => (CALLtail ...)
   329(TailCallInter ...) => (CALLtailinter ...)
   330
   331// Miscellaneous
   332(IsNonNil p) => (SETNE (TESTL p p))
   333(IsInBounds idx len) => (SETB (CMPL idx len))
   334(IsSliceInBounds idx len) => (SETBE (CMPL idx len))
   335(NilCheck ...) => (LoweredNilCheck ...)
   336(GetG ...) => (LoweredGetG ...)
   337(GetClosurePtr ...) => (LoweredGetClosurePtr ...)
   338(GetCallerPC ...) => (LoweredGetCallerPC ...)
   339(GetCallerSP ...) => (LoweredGetCallerSP ...)
   340(Addr {sym} base) => (LEAL {sym} base)
   341(LocalAddr <t> {sym} base mem) && t.Elem().HasPointers() => (LEAL {sym} (SPanchored base mem))
   342(LocalAddr <t> {sym} base _)  && !t.Elem().HasPointers() => (LEAL {sym} base)
   343
   344// block rewrites
   345(If (SETL  cmp) yes no) => (LT  cmp yes no)
   346(If (SETLE cmp) yes no) => (LE  cmp yes no)
   347(If (SETG  cmp) yes no) => (GT  cmp yes no)
   348(If (SETGE cmp) yes no) => (GE  cmp yes no)
   349(If (SETEQ cmp) yes no) => (EQ  cmp yes no)
   350(If (SETNE cmp) yes no) => (NE  cmp yes no)
   351(If (SETB  cmp) yes no) => (ULT cmp yes no)
   352(If (SETBE cmp) yes no) => (ULE cmp yes no)
   353(If (SETA  cmp) yes no) => (UGT cmp yes no)
   354(If (SETAE cmp) yes no) => (UGE cmp yes no)
   355(If (SETO  cmp) yes no) => (OS cmp yes no)
   356
   357// Special case for floating point - LF/LEF not generated
   358(If (SETGF  cmp) yes no) => (UGT  cmp yes no)
   359(If (SETGEF cmp) yes no) => (UGE  cmp yes no)
   360(If (SETEQF cmp) yes no) => (EQF  cmp yes no)
   361(If (SETNEF cmp) yes no) => (NEF  cmp yes no)
   362
   363(If cond yes no) => (NE (TESTB cond cond) yes no)
   364
   365// Write barrier.
   366(WB ...) => (LoweredWB ...)
   367
   368(PanicBounds ...) => (LoweredPanicBoundsRR ...)
   369(PanicExtend ...) => (LoweredPanicExtendRR ...)
   370
   371(LoweredPanicBoundsRR [kind] x (MOVLconst [c]) mem) => (LoweredPanicBoundsRC [kind] x {PanicBoundsC{C:int64(c)}} mem)
   372(LoweredPanicBoundsRR [kind] (MOVLconst [c]) y mem) => (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(c)}} y mem)
   373(LoweredPanicBoundsRC [kind] {p} (MOVLconst [c]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(c), Cy:p.C}} mem)
   374
   375(LoweredPanicExtendRR [kind] hi lo (MOVLconst [c]) mem) => (LoweredPanicExtendRC [kind] hi lo {PanicBoundsC{C:int64(c)}} mem)
   376(LoweredPanicExtendRR [kind] (MOVLconst [hi]) (MOVLconst [lo]) y mem) => (LoweredPanicBoundsCR [kind] {PanicBoundsC{C:int64(hi)<<32 + int64(uint32(lo))}} y mem)
   377(LoweredPanicExtendRC [kind] {p} (MOVLconst [hi]) (MOVLconst [lo]) mem) => (LoweredPanicBoundsCC [kind] {PanicBoundsCC{Cx:int64(hi)<<32+int64(uint32(lo)), Cy:p.C}} mem)
   378
   379// ***************************
   380// Above: lowering rules
   381// Below: optimizations
   382// ***************************
   383// TODO: Should the optimizations be a separate pass?
   384
   385// Fold boolean tests into blocks
   386(NE (TESTB (SETL  cmp) (SETL  cmp)) yes no) => (LT  cmp yes no)
   387(NE (TESTB (SETLE cmp) (SETLE cmp)) yes no) => (LE  cmp yes no)
   388(NE (TESTB (SETG  cmp) (SETG  cmp)) yes no) => (GT  cmp yes no)
   389(NE (TESTB (SETGE cmp) (SETGE cmp)) yes no) => (GE  cmp yes no)
   390(NE (TESTB (SETEQ cmp) (SETEQ cmp)) yes no) => (EQ  cmp yes no)
   391(NE (TESTB (SETNE cmp) (SETNE cmp)) yes no) => (NE  cmp yes no)
   392(NE (TESTB (SETB  cmp) (SETB  cmp)) yes no) => (ULT cmp yes no)
   393(NE (TESTB (SETBE cmp) (SETBE cmp)) yes no) => (ULE cmp yes no)
   394(NE (TESTB (SETA  cmp) (SETA  cmp)) yes no) => (UGT cmp yes no)
   395(NE (TESTB (SETAE cmp) (SETAE cmp)) yes no) => (UGE cmp yes no)
   396(NE (TESTB (SETO cmp) (SETO cmp)) yes no) => (OS cmp yes no)
   397
   398// Special case for floating point - LF/LEF not generated
   399(NE (TESTB (SETGF  cmp) (SETGF  cmp)) yes no) => (UGT  cmp yes no)
   400(NE (TESTB (SETGEF cmp) (SETGEF cmp)) yes no) => (UGE  cmp yes no)
   401(NE (TESTB (SETEQF cmp) (SETEQF cmp)) yes no) => (EQF  cmp yes no)
   402(NE (TESTB (SETNEF cmp) (SETNEF cmp)) yes no) => (NEF  cmp yes no)
   403
   404// fold constants into instructions
   405(ADDL x (MOVLconst <t> [c])) && !t.IsPtr() => (ADDLconst [c] x)
   406(ADDLcarry x (MOVLconst [c])) => (ADDLconstcarry [c] x)
   407(ADCL x (MOVLconst [c]) f) => (ADCLconst [c] x f)
   408
   409(SUBL x (MOVLconst [c])) => (SUBLconst x [c])
   410(SUBL (MOVLconst [c]) x) => (NEGL (SUBLconst <v.Type> x [c]))
   411(SUBLcarry x (MOVLconst [c])) => (SUBLconstcarry [c] x)
   412(SBBL x (MOVLconst [c]) f) => (SBBLconst [c] x f)
   413
   414(MULL x (MOVLconst [c])) => (MULLconst [c] x)
   415(ANDL x (MOVLconst [c])) => (ANDLconst [c] x)
   416
   417(ANDLconst [c] (ANDLconst [d] x)) => (ANDLconst [c & d] x)
   418(XORLconst [c] (XORLconst [d] x)) => (XORLconst [c ^ d] x)
   419(MULLconst [c] (MULLconst [d] x)) => (MULLconst [c * d] x)
   420
   421(ORL x (MOVLconst [c])) => (ORLconst [c] x)
   422(XORL x (MOVLconst [c])) => (XORLconst [c] x)
   423
   424(SHLL x (MOVLconst [c])) => (SHLLconst [c&31] x)
   425(SHRL x (MOVLconst [c])) => (SHRLconst [c&31] x)
   426(SHRW x (MOVLconst [c])) && c&31 < 16 => (SHRWconst [int16(c&31)] x)
   427(SHRW _ (MOVLconst [c])) && c&31 >= 16 => (MOVLconst [0])
   428(SHRB x (MOVLconst [c])) && c&31 < 8 => (SHRBconst [int8(c&31)] x)
   429(SHRB _ (MOVLconst [c])) && c&31 >= 8 => (MOVLconst [0])
   430
   431(SARL x (MOVLconst [c])) => (SARLconst [c&31] x)
   432(SARW x (MOVLconst [c])) => (SARWconst [int16(min(int64(c&31),15))] x)
   433(SARB x (MOVLconst [c])) => (SARBconst [int8(min(int64(c&31),7))] x)
   434
   435(SARL x (ANDLconst [31] y)) => (SARL x y)
   436(SHLL x (ANDLconst [31] y)) => (SHLL x y)
   437(SHRL x (ANDLconst [31] y)) => (SHRL x y)
   438
   439// Constant shift simplifications
   440
   441(SHLLconst x [0]) => x
   442(SHRLconst x [0]) => x
   443(SARLconst x [0]) => x
   444
   445(SHRWconst x [0]) => x
   446(SARWconst x [0]) => x
   447
   448(SHRBconst x [0]) => x
   449(SARBconst x [0]) => x
   450
   451(ROLLconst [0] x) => x
   452(ROLWconst [0] x) => x
   453(ROLBconst [0] x) => x
   454
   455// Note: the word and byte shifts keep the low 5 bits (not the low 4 or 3 bits)
   456// because the x86 instructions are defined to use all 5 bits of the shift even
   457// for the small shifts. I don't think we'll ever generate a weird shift (e.g.
   458// (SHRW x (MOVLconst [24])), but just in case.
   459
   460(CMPL x (MOVLconst [c])) => (CMPLconst x [c])
   461(CMPL (MOVLconst [c]) x) => (InvertFlags (CMPLconst x [c]))
   462(CMPW x (MOVLconst [c])) => (CMPWconst x [int16(c)])
   463(CMPW (MOVLconst [c]) x) => (InvertFlags (CMPWconst x [int16(c)]))
   464(CMPB x (MOVLconst [c])) => (CMPBconst x [int8(c)])
   465(CMPB (MOVLconst [c]) x) => (InvertFlags (CMPBconst x [int8(c)]))
   466
   467// Canonicalize the order of arguments to comparisons - helps with CSE.
   468(CMP(L|W|B) x y) && canonLessThan(x,y) => (InvertFlags (CMP(L|W|B) y x))
   469
   470// strength reduction
   471// Assumes that the following costs from https://gmplib.org/~tege/x86-timing.pdf:
   472//    1 - addl, shll, leal, negl, subl
   473//    3 - imull
   474// This limits the rewrites to two instructions.
   475// Note that negl always operates in-place,
   476// which can require a register-register move
   477// to preserve the original value,
   478// so it must be used with care.
   479(MULLconst [-9] x) => (NEGL (LEAL8 <v.Type> x x))
   480(MULLconst [-5] x) => (NEGL (LEAL4 <v.Type> x x))
   481(MULLconst [-3] x) => (NEGL (LEAL2 <v.Type> x x))
   482(MULLconst [-1] x) => (NEGL x)
   483(MULLconst [0] _) => (MOVLconst [0])
   484(MULLconst [1] x) => x
   485(MULLconst [3] x) => (LEAL2 x x)
   486(MULLconst [5] x) => (LEAL4 x x)
   487(MULLconst [7] x) => (LEAL2 x (LEAL2 <v.Type> x x))
   488(MULLconst [9] x) => (LEAL8 x x)
   489(MULLconst [11] x) => (LEAL2 x (LEAL4 <v.Type> x x))
   490(MULLconst [13] x) => (LEAL4 x (LEAL2 <v.Type> x x))
   491(MULLconst [19] x) => (LEAL2 x (LEAL8 <v.Type> x x))
   492(MULLconst [21] x) => (LEAL4 x (LEAL4 <v.Type> x x))
   493(MULLconst [25] x) => (LEAL8 x (LEAL2 <v.Type> x x))
   494(MULLconst [27] x) => (LEAL8 (LEAL2 <v.Type> x x) (LEAL2 <v.Type> x x))
   495(MULLconst [37] x) => (LEAL4 x (LEAL8 <v.Type> x x))
   496(MULLconst [41] x) => (LEAL8 x (LEAL4 <v.Type> x x))
   497(MULLconst [45] x) => (LEAL8 (LEAL4 <v.Type> x x) (LEAL4 <v.Type> x x))
   498(MULLconst [73] x) => (LEAL8 x (LEAL8 <v.Type> x x))
   499(MULLconst [81] x) => (LEAL8 (LEAL8 <v.Type> x x) (LEAL8 <v.Type> x x))
   500
   501(MULLconst [c] x) && isPowerOfTwo(c+1) && c >= 15 => (SUBL (SHLLconst <v.Type> [int32(log32(c+1))] x) x)
   502(MULLconst [c] x) && isPowerOfTwo(c-1) && c >= 17 => (LEAL1 (SHLLconst <v.Type> [int32(log32(c-1))] x) x)
   503(MULLconst [c] x) && isPowerOfTwo(c-2) && c >= 34 => (LEAL2 (SHLLconst <v.Type> [int32(log32(c-2))] x) x)
   504(MULLconst [c] x) && isPowerOfTwo(c-4) && c >= 68 => (LEAL4 (SHLLconst <v.Type> [int32(log32(c-4))] x) x)
   505(MULLconst [c] x) && isPowerOfTwo(c-8) && c >= 136 => (LEAL8 (SHLLconst <v.Type> [int32(log32(c-8))] x) x)
   506(MULLconst [c] x) && c%3 == 0 && isPowerOfTwo(c/3) => (SHLLconst [int32(log32(c/3))] (LEAL2 <v.Type> x x))
   507(MULLconst [c] x) && c%5 == 0 && isPowerOfTwo(c/5) => (SHLLconst [int32(log32(c/5))] (LEAL4 <v.Type> x x))
   508(MULLconst [c] x) && c%9 == 0 && isPowerOfTwo(c/9) => (SHLLconst [int32(log32(c/9))] (LEAL8 <v.Type> x x))
   509
   510// combine add/shift into LEAL
   511(ADDL x (SHLLconst [3] y)) => (LEAL8 x y)
   512(ADDL x (SHLLconst [2] y)) => (LEAL4 x y)
   513(ADDL x (SHLLconst [1] y)) => (LEAL2 x y)
   514(ADDL x (ADDL y y)) => (LEAL2 x y)
   515(ADDL x (ADDL x y)) => (LEAL2 y x)
   516
   517// combine ADDL/ADDLconst into LEAL1
   518(ADDLconst [c] (ADDL x y)) => (LEAL1 [c] x y)
   519(ADDL (ADDLconst [c] x) y) => (LEAL1 [c] x y)
   520
   521// fold ADDL into LEAL
   522(ADDLconst [c] (LEAL [d] {s} x)) && is32Bit(int64(c)+int64(d)) => (LEAL [c+d] {s} x)
   523(LEAL [c] {s} (ADDLconst [d] x)) && is32Bit(int64(c)+int64(d)) => (LEAL [c+d] {s} x)
   524(ADDLconst [c] x:(SP)) => (LEAL [c] x) // so it is rematerializeable
   525(LEAL [c] {s} (ADDL x y)) && x.Op != OpSB && y.Op != OpSB => (LEAL1 [c] {s} x y)
   526(ADDL x (LEAL [c] {s} y)) && x.Op != OpSB && y.Op != OpSB => (LEAL1 [c] {s} x y)
   527
   528// fold ADDLconst into LEALx
   529(ADDLconst [c] (LEAL1 [d] {s} x y)) && is32Bit(int64(c)+int64(d)) => (LEAL1 [c+d] {s} x y)
   530(ADDLconst [c] (LEAL2 [d] {s} x y)) && is32Bit(int64(c)+int64(d)) => (LEAL2 [c+d] {s} x y)
   531(ADDLconst [c] (LEAL4 [d] {s} x y)) && is32Bit(int64(c)+int64(d)) => (LEAL4 [c+d] {s} x y)
   532(ADDLconst [c] (LEAL8 [d] {s} x y)) && is32Bit(int64(c)+int64(d)) => (LEAL8 [c+d] {s} x y)
   533(LEAL1 [c] {s} (ADDLconst [d] x) y) && is32Bit(int64(c)+int64(d))   && x.Op != OpSB => (LEAL1 [c+d] {s} x y)
   534(LEAL2 [c] {s} (ADDLconst [d] x) y) && is32Bit(int64(c)+int64(d))   && x.Op != OpSB => (LEAL2 [c+d] {s} x y)
   535(LEAL2 [c] {s} x (ADDLconst [d] y)) && is32Bit(int64(c)+2*int64(d)) && y.Op != OpSB => (LEAL2 [c+2*d] {s} x y)
   536(LEAL4 [c] {s} (ADDLconst [d] x) y) && is32Bit(int64(c)+int64(d))   && x.Op != OpSB => (LEAL4 [c+d] {s} x y)
   537(LEAL4 [c] {s} x (ADDLconst [d] y)) && is32Bit(int64(c)+4*int64(d)) && y.Op != OpSB => (LEAL4 [c+4*d] {s} x y)
   538(LEAL8 [c] {s} (ADDLconst [d] x) y) && is32Bit(int64(c)+int64(d))   && x.Op != OpSB => (LEAL8 [c+d] {s} x y)
   539(LEAL8 [c] {s} x (ADDLconst [d] y)) && is32Bit(int64(c)+8*int64(d)) && y.Op != OpSB => (LEAL8 [c+8*d] {s} x y)
   540
   541// fold shifts into LEALx
   542(LEAL1 [c] {s} x (SHLLconst [1] y)) => (LEAL2 [c] {s} x y)
   543(LEAL1 [c] {s} x (SHLLconst [2] y)) => (LEAL4 [c] {s} x y)
   544(LEAL1 [c] {s} x (SHLLconst [3] y)) => (LEAL8 [c] {s} x y)
   545(LEAL2 [c] {s} x (SHLLconst [1] y)) => (LEAL4 [c] {s} x y)
   546(LEAL2 [c] {s} x (SHLLconst [2] y)) => (LEAL8 [c] {s} x y)
   547(LEAL4 [c] {s} x (SHLLconst [1] y)) => (LEAL8 [c] {s} x y)
   548
   549// reverse ordering of compare instruction
   550(SETL (InvertFlags x)) => (SETG x)
   551(SETG (InvertFlags x)) => (SETL x)
   552(SETB (InvertFlags x)) => (SETA x)
   553(SETA (InvertFlags x)) => (SETB x)
   554(SETLE (InvertFlags x)) => (SETGE x)
   555(SETGE (InvertFlags x)) => (SETLE x)
   556(SETBE (InvertFlags x)) => (SETAE x)
   557(SETAE (InvertFlags x)) => (SETBE x)
   558(SETEQ (InvertFlags x)) => (SETEQ x)
   559(SETNE (InvertFlags x)) => (SETNE x)
   560
   561// sign extended loads
   562// Note: The combined instruction must end up in the same block
   563// as the original load. If not, we end up making a value with
   564// memory type live in two different blocks, which can lead to
   565// multiple memory values alive simultaneously.
   566// Make sure we don't combine these ops if the load has another use.
   567// This prevents a single load from being split into multiple loads
   568// which then might return different values.  See test/atomicload.go.
   569(MOVBLSX x:(MOVBload [off] {sym} ptr mem)) && x.Uses == 1 && clobber(x) => @x.Block (MOVBLSXload <v.Type> [off] {sym} ptr mem)
   570(MOVBLZX x:(MOVBload [off] {sym} ptr mem)) && x.Uses == 1 && clobber(x) => @x.Block (MOVBload <v.Type> [off] {sym} ptr mem)
   571(MOVWLSX x:(MOVWload [off] {sym} ptr mem)) && x.Uses == 1 && clobber(x) => @x.Block (MOVWLSXload <v.Type> [off] {sym} ptr mem)
   572(MOVWLZX x:(MOVWload [off] {sym} ptr mem)) && x.Uses == 1 && clobber(x) => @x.Block (MOVWload <v.Type> [off] {sym} ptr mem)
   573
   574// replace load from same location as preceding store with zero/sign extension (or copy in case of full width)
   575(MOVBload [off] {sym} ptr (MOVBstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) => (MOVBLZX x)
   576(MOVWload [off] {sym} ptr (MOVWstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) => (MOVWLZX x)
   577(MOVLload [off] {sym} ptr (MOVLstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) => x
   578(MOVBLSXload [off] {sym} ptr (MOVBstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) => (MOVBLSX x)
   579(MOVWLSXload [off] {sym} ptr (MOVWstore [off2] {sym2} ptr2 x _)) && sym == sym2 && off == off2 && isSamePtr(ptr, ptr2) => (MOVWLSX x)
   580
   581// Fold extensions and ANDs together.
   582(MOVBLZX (ANDLconst [c] x)) => (ANDLconst [c & 0xff] x)
   583(MOVWLZX (ANDLconst [c] x)) => (ANDLconst [c & 0xffff] x)
   584(MOVBLSX (ANDLconst [c] x)) && c & 0x80 == 0 => (ANDLconst [c & 0x7f] x)
   585(MOVWLSX (ANDLconst [c] x)) && c & 0x8000 == 0 => (ANDLconst [c & 0x7fff] x)
   586
   587// Don't extend before storing
   588(MOVWstore [off] {sym} ptr (MOVWL(S|Z)X x) mem) => (MOVWstore [off] {sym} ptr x mem)
   589(MOVBstore [off] {sym} ptr (MOVBL(S|Z)X x) mem) => (MOVBstore [off] {sym} ptr x mem)
   590
   591// fold constants into memory operations
   592// Note that this is not always a good idea because if not all the uses of
   593// the ADDLconst get eliminated, we still have to compute the ADDLconst and we now
   594// have potentially two live values (ptr and (ADDLconst [off] ptr)) instead of one.
   595// Nevertheless, let's do it!
   596(MOV(L|W|B|SS|SD)load  [off1] {sym} (ADDLconst [off2] ptr) mem) && is32Bit(int64(off1)+int64(off2)) =>
   597    (MOV(L|W|B|SS|SD)load  [off1+off2] {sym} ptr mem)
   598(MOV(L|W|B|SS|SD)store  [off1] {sym} (ADDLconst [off2] ptr) val mem) && is32Bit(int64(off1)+int64(off2)) =>
   599    (MOV(L|W|B|SS|SD)store  [off1+off2] {sym} ptr val mem)
   600
   601((ADD|SUB|MUL|AND|OR|XOR)Lload [off1] {sym} val (ADDLconst [off2] base) mem) && is32Bit(int64(off1)+int64(off2)) =>
   602	((ADD|SUB|MUL|AND|OR|XOR)Lload [off1+off2] {sym} val base mem)
   603((ADD|SUB|MUL|DIV)SSload [off1] {sym} val (ADDLconst [off2] base) mem) && is32Bit(int64(off1)+int64(off2)) =>
   604	((ADD|SUB|MUL|DIV)SSload [off1+off2] {sym} val base mem)
   605((ADD|SUB|MUL|DIV)SDload [off1] {sym} val (ADDLconst [off2] base) mem) && is32Bit(int64(off1)+int64(off2)) =>
   606	((ADD|SUB|MUL|DIV)SDload [off1+off2] {sym} val base mem)
   607((ADD|SUB|AND|OR|XOR)Lmodify [off1] {sym} (ADDLconst [off2] base) val mem) && is32Bit(int64(off1)+int64(off2)) =>
   608	((ADD|SUB|AND|OR|XOR)Lmodify [off1+off2] {sym} base val mem)
   609((ADD|AND|OR|XOR)Lconstmodify [valoff1] {sym} (ADDLconst [off2] base) mem) && valoff1.canAdd32(off2) =>
   610	((ADD|AND|OR|XOR)Lconstmodify [valoff1.addOffset32(off2)] {sym} base mem)
   611
   612// Fold constants into stores.
   613(MOVLstore [off] {sym} ptr (MOVLconst [c]) mem) =>
   614	(MOVLstoreconst [makeValAndOff(c,off)] {sym} ptr mem)
   615(MOVWstore [off] {sym} ptr (MOVLconst [c]) mem) =>
   616	(MOVWstoreconst [makeValAndOff(c,off)] {sym} ptr mem)
   617(MOVBstore [off] {sym} ptr (MOVLconst [c]) mem) =>
   618	(MOVBstoreconst [makeValAndOff(c,off)] {sym} ptr mem)
   619
   620// Fold address offsets into constant stores.
   621(MOV(L|W|B)storeconst [sc] {s} (ADDLconst [off] ptr) mem) && sc.canAdd32(off) =>
   622	(MOV(L|W|B)storeconst [sc.addOffset32(off)] {s} ptr mem)
   623
   624// We need to fold LEAL into the MOVx ops so that the live variable analysis knows
   625// what variables are being read/written by the ops.
   626// Note: we turn off this merging for operations on globals when building
   627// position-independent code (when Flag_shared is set).
   628// PIC needs a spare register to load the PC into.  Having the LEAL be
   629// a separate instruction gives us that register.  Having the LEAL be
   630// a separate instruction also allows it to be CSEd (which is good because
   631// it compiles to a thunk call).
   632(MOV(L|W|B|SS|SD|BLSX|WLSX)load  [off1] {sym1} (LEAL [off2] {sym2} base) mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2)
   633  && (base.Op != OpSB || !config.ctxt.Flag_shared) =>
   634        (MOV(L|W|B|SS|SD|BLSX|WLSX)load  [off1+off2] {mergeSym(sym1,sym2)} base mem)
   635
   636(MOV(L|W|B|SS|SD)store  [off1] {sym1} (LEAL [off2] {sym2} base) val mem) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2)
   637  && (base.Op != OpSB || !config.ctxt.Flag_shared) =>
   638	(MOV(L|W|B|SS|SD)store  [off1+off2] {mergeSym(sym1,sym2)} base val mem)
   639
   640(MOV(L|W|B)storeconst [sc] {sym1} (LEAL [off] {sym2} ptr) mem) && canMergeSym(sym1, sym2) && sc.canAdd32(off)
   641  && (ptr.Op != OpSB || !config.ctxt.Flag_shared) =>
   642	(MOV(L|W|B)storeconst [sc.addOffset32(off)] {mergeSym(sym1, sym2)} ptr mem)
   643
   644((ADD|SUB|MUL|AND|OR|XOR)Lload [off1] {sym1} val (LEAL [off2] {sym2} base) mem)
   645	&& is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) =>
   646	((ADD|SUB|MUL|AND|OR|XOR)Lload [off1+off2] {mergeSym(sym1,sym2)} val base mem)
   647((ADD|SUB|MUL|DIV)SSload [off1] {sym1} val (LEAL [off2] {sym2} base) mem)
   648	&& is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) =>
   649	((ADD|SUB|MUL|DIV)SSload [off1+off2] {mergeSym(sym1,sym2)} val base mem)
   650((ADD|SUB|MUL|DIV)SDload [off1] {sym1} val (LEAL [off2] {sym2} base) mem)
   651	&& is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) =>
   652	((ADD|SUB|MUL|DIV)SDload [off1+off2] {mergeSym(sym1,sym2)} val base mem)
   653((ADD|SUB|AND|OR|XOR)Lmodify [off1] {sym1} (LEAL [off2] {sym2} base) val mem)
   654	&& is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) =>
   655	((ADD|SUB|AND|OR|XOR)Lmodify [off1+off2] {mergeSym(sym1,sym2)} base val mem)
   656((ADD|AND|OR|XOR)Lconstmodify [valoff1] {sym1} (LEAL [off2] {sym2} base) mem)
   657	&& valoff1.canAdd32(off2) && canMergeSym(sym1, sym2) && (base.Op != OpSB || !config.ctxt.Flag_shared) =>
   658	((ADD|AND|OR|XOR)Lconstmodify [valoff1.addOffset32(off2)] {mergeSym(sym1,sym2)} base mem)
   659
   660// Merge load/store to op
   661((ADD|AND|OR|XOR|SUB|MUL)L x l:(MOVLload [off] {sym} ptr mem)) && canMergeLoadClobber(v, l, x) && clobber(l) => ((ADD|AND|OR|XOR|SUB|MUL)Lload x [off] {sym} ptr mem)
   662((ADD|SUB|MUL|DIV)SD x l:(MOVSDload [off] {sym} ptr mem)) && canMergeLoadClobber(v, l, x) && clobber(l) => ((ADD|SUB|MUL|DIV)SDload x [off] {sym} ptr mem)
   663((ADD|SUB|MUL|DIV)SS x l:(MOVSSload [off] {sym} ptr mem)) && canMergeLoadClobber(v, l, x) && clobber(l) => ((ADD|SUB|MUL|DIV)SSload x [off] {sym} ptr mem)
   664(MOVLstore {sym} [off] ptr y:((ADD|AND|OR|XOR)Lload x [off] {sym} ptr mem) mem) && y.Uses==1 && clobber(y) => ((ADD|AND|OR|XOR)Lmodify [off] {sym} ptr x mem)
   665(MOVLstore {sym} [off] ptr y:((ADD|SUB|AND|OR|XOR)L l:(MOVLload [off] {sym} ptr mem) x) mem) && y.Uses==1 && l.Uses==1 && clobber(y, l) =>
   666	((ADD|SUB|AND|OR|XOR)Lmodify [off] {sym} ptr x mem)
   667(MOVLstore {sym} [off] ptr y:((ADD|AND|OR|XOR)Lconst [c] l:(MOVLload [off] {sym} ptr mem)) mem)
   668	&& y.Uses==1 && l.Uses==1 && clobber(y, l) =>
   669	((ADD|AND|OR|XOR)Lconstmodify [makeValAndOff(c,off)] {sym} ptr mem)
   670
   671// fold LEALs together
   672(LEAL [off1] {sym1} (LEAL [off2] {sym2} x)) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) =>
   673      (LEAL [off1+off2] {mergeSym(sym1,sym2)} x)
   674
   675// LEAL into LEAL1
   676(LEAL1 [off1] {sym1} (LEAL [off2] {sym2} x) y) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && x.Op != OpSB =>
   677       (LEAL1 [off1+off2] {mergeSym(sym1,sym2)} x y)
   678
   679// LEAL1 into LEAL
   680(LEAL [off1] {sym1} (LEAL1 [off2] {sym2} x y)) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) =>
   681       (LEAL1 [off1+off2] {mergeSym(sym1,sym2)} x y)
   682
   683// LEAL into LEAL[248]
   684(LEAL2 [off1] {sym1} (LEAL [off2] {sym2} x) y) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && x.Op != OpSB =>
   685       (LEAL2 [off1+off2] {mergeSym(sym1,sym2)} x y)
   686(LEAL4 [off1] {sym1} (LEAL [off2] {sym2} x) y) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && x.Op != OpSB =>
   687       (LEAL4 [off1+off2] {mergeSym(sym1,sym2)} x y)
   688(LEAL8 [off1] {sym1} (LEAL [off2] {sym2} x) y) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) && x.Op != OpSB =>
   689       (LEAL8 [off1+off2] {mergeSym(sym1,sym2)} x y)
   690
   691// LEAL[248] into LEAL
   692(LEAL [off1] {sym1} (LEAL2 [off2] {sym2} x y)) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) =>
   693      (LEAL2 [off1+off2] {mergeSym(sym1,sym2)} x y)
   694(LEAL [off1] {sym1} (LEAL4 [off2] {sym2} x y)) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) =>
   695      (LEAL4 [off1+off2] {mergeSym(sym1,sym2)} x y)
   696(LEAL [off1] {sym1} (LEAL8 [off2] {sym2} x y)) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) =>
   697      (LEAL8 [off1+off2] {mergeSym(sym1,sym2)} x y)
   698
   699// LEAL[1248] into LEAL[1248]. Only some such merges are possible.
   700(LEAL1 [off1] {sym1} x (LEAL1 [off2] {sym2} y y)) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) =>
   701      (LEAL2 [off1+off2] {mergeSym(sym1, sym2)} x y)
   702(LEAL1 [off1] {sym1} x (LEAL1 [off2] {sym2} x y)) && is32Bit(int64(off1)+int64(off2)) && canMergeSym(sym1, sym2) =>
   703      (LEAL2 [off1+off2] {mergeSym(sym1, sym2)} y x)
   704(LEAL2 [off1] {sym} x (LEAL1 [off2] {nil} y y)) && is32Bit(int64(off1)+2*int64(off2)) =>
   705      (LEAL4 [off1+2*off2] {sym} x y)
   706(LEAL4 [off1] {sym} x (LEAL1 [off2] {nil} y y)) && is32Bit(int64(off1)+4*int64(off2)) =>
   707      (LEAL8 [off1+4*off2] {sym} x y)
   708
   709// Absorb InvertFlags into branches.
   710(LT (InvertFlags cmp) yes no) => (GT cmp yes no)
   711(GT (InvertFlags cmp) yes no) => (LT cmp yes no)
   712(LE (InvertFlags cmp) yes no) => (GE cmp yes no)
   713(GE (InvertFlags cmp) yes no) => (LE cmp yes no)
   714(ULT (InvertFlags cmp) yes no) => (UGT cmp yes no)
   715(UGT (InvertFlags cmp) yes no) => (ULT cmp yes no)
   716(ULE (InvertFlags cmp) yes no) => (UGE cmp yes no)
   717(UGE (InvertFlags cmp) yes no) => (ULE cmp yes no)
   718(EQ (InvertFlags cmp) yes no) => (EQ cmp yes no)
   719(NE (InvertFlags cmp) yes no) => (NE cmp yes no)
   720
   721// Constant comparisons.
   722(CMPLconst (MOVLconst [x]) [y]) && x==y                       => (FlagEQ)
   723(CMPLconst (MOVLconst [x]) [y]) && x<y && uint32(x)<uint32(y) => (FlagLT_ULT)
   724(CMPLconst (MOVLconst [x]) [y]) && x<y && uint32(x)>uint32(y) => (FlagLT_UGT)
   725(CMPLconst (MOVLconst [x]) [y]) && x>y && uint32(x)<uint32(y) => (FlagGT_ULT)
   726(CMPLconst (MOVLconst [x]) [y]) && x>y && uint32(x)>uint32(y) => (FlagGT_UGT)
   727
   728(CMPWconst (MOVLconst [x]) [y]) && int16(x)==y                       => (FlagEQ)
   729(CMPWconst (MOVLconst [x]) [y]) && int16(x)<y && uint16(x)<uint16(y) => (FlagLT_ULT)
   730(CMPWconst (MOVLconst [x]) [y]) && int16(x)<y && uint16(x)>uint16(y) => (FlagLT_UGT)
   731(CMPWconst (MOVLconst [x]) [y]) && int16(x)>y && uint16(x)<uint16(y) => (FlagGT_ULT)
   732(CMPWconst (MOVLconst [x]) [y]) && int16(x)>y && uint16(x)>uint16(y) => (FlagGT_UGT)
   733
   734(CMPBconst (MOVLconst [x]) [y]) && int8(x)==y                      => (FlagEQ)
   735(CMPBconst (MOVLconst [x]) [y]) && int8(x)<y && uint8(x)<uint8(y) => (FlagLT_ULT)
   736(CMPBconst (MOVLconst [x]) [y]) && int8(x)<y && uint8(x)>uint8(y) => (FlagLT_UGT)
   737(CMPBconst (MOVLconst [x]) [y]) && int8(x)>y && uint8(x)<uint8(y) => (FlagGT_ULT)
   738(CMPBconst (MOVLconst [x]) [y]) && int8(x)>y && uint8(x)>uint8(y) => (FlagGT_UGT)
   739
   740// Other known comparisons.
   741(CMPLconst (SHRLconst _ [c]) [n]) && 0 <= n && 0 < c && c <= 32 && (1<<uint64(32-c)) <= uint64(n) => (FlagLT_ULT)
   742(CMPLconst (ANDLconst _ [m]) [n]) && 0 <= m && m < n => (FlagLT_ULT)
   743(CMPWconst (ANDLconst _ [m]) [n]) && 0 <= int16(m) && int16(m) < n => (FlagLT_ULT)
   744(CMPBconst (ANDLconst _ [m]) [n]) && 0 <= int8(m) && int8(m) < n => (FlagLT_ULT)
   745// TODO: DIVxU also.
   746
   747// Absorb flag constants into SBB ops.
   748(SBBLcarrymask (FlagEQ)) => (MOVLconst [0])
   749(SBBLcarrymask (FlagLT_ULT)) => (MOVLconst [-1])
   750(SBBLcarrymask (FlagLT_UGT)) => (MOVLconst [0])
   751(SBBLcarrymask (FlagGT_ULT)) => (MOVLconst [-1])
   752(SBBLcarrymask (FlagGT_UGT)) => (MOVLconst [0])
   753
   754// Absorb flag constants into branches.
   755(EQ (FlagEQ) yes no) => (First yes no)
   756(EQ (FlagLT_ULT) yes no) => (First no yes)
   757(EQ (FlagLT_UGT) yes no) => (First no yes)
   758(EQ (FlagGT_ULT) yes no) => (First no yes)
   759(EQ (FlagGT_UGT) yes no) => (First no yes)
   760
   761(NE (FlagEQ) yes no) => (First no yes)
   762(NE (FlagLT_ULT) yes no) => (First yes no)
   763(NE (FlagLT_UGT) yes no) => (First yes no)
   764(NE (FlagGT_ULT) yes no) => (First yes no)
   765(NE (FlagGT_UGT) yes no) => (First yes no)
   766
   767(LT (FlagEQ) yes no) => (First no yes)
   768(LT (FlagLT_ULT) yes no) => (First yes no)
   769(LT (FlagLT_UGT) yes no) => (First yes no)
   770(LT (FlagGT_ULT) yes no) => (First no yes)
   771(LT (FlagGT_UGT) yes no) => (First no yes)
   772
   773(LE (FlagEQ) yes no) => (First yes no)
   774(LE (FlagLT_ULT) yes no) => (First yes no)
   775(LE (FlagLT_UGT) yes no) => (First yes no)
   776(LE (FlagGT_ULT) yes no) => (First no yes)
   777(LE (FlagGT_UGT) yes no) => (First no yes)
   778
   779(GT (FlagEQ) yes no) => (First no yes)
   780(GT (FlagLT_ULT) yes no) => (First no yes)
   781(GT (FlagLT_UGT) yes no) => (First no yes)
   782(GT (FlagGT_ULT) yes no) => (First yes no)
   783(GT (FlagGT_UGT) yes no) => (First yes no)
   784
   785(GE (FlagEQ) yes no) => (First yes no)
   786(GE (FlagLT_ULT) yes no) => (First no yes)
   787(GE (FlagLT_UGT) yes no) => (First no yes)
   788(GE (FlagGT_ULT) yes no) => (First yes no)
   789(GE (FlagGT_UGT) yes no) => (First yes no)
   790
   791(ULT (FlagEQ) yes no) => (First no yes)
   792(ULT (FlagLT_ULT) yes no) => (First yes no)
   793(ULT (FlagLT_UGT) yes no) => (First no yes)
   794(ULT (FlagGT_ULT) yes no) => (First yes no)
   795(ULT (FlagGT_UGT) yes no) => (First no yes)
   796
   797(ULE (FlagEQ) yes no) => (First yes no)
   798(ULE (FlagLT_ULT) yes no) => (First yes no)
   799(ULE (FlagLT_UGT) yes no) => (First no yes)
   800(ULE (FlagGT_ULT) yes no) => (First yes no)
   801(ULE (FlagGT_UGT) yes no) => (First no yes)
   802
   803(UGT (FlagEQ) yes no) => (First no yes)
   804(UGT (FlagLT_ULT) yes no) => (First no yes)
   805(UGT (FlagLT_UGT) yes no) => (First yes no)
   806(UGT (FlagGT_ULT) yes no) => (First no yes)
   807(UGT (FlagGT_UGT) yes no) => (First yes no)
   808
   809(UGE (FlagEQ) yes no) => (First yes no)
   810(UGE (FlagLT_ULT) yes no) => (First no yes)
   811(UGE (FlagLT_UGT) yes no) => (First yes no)
   812(UGE (FlagGT_ULT) yes no) => (First no yes)
   813(UGE (FlagGT_UGT) yes no) => (First yes no)
   814
   815// Absorb flag constants into SETxx ops.
   816(SETEQ (FlagEQ)) => (MOVLconst [1])
   817(SETEQ (FlagLT_ULT)) => (MOVLconst [0])
   818(SETEQ (FlagLT_UGT)) => (MOVLconst [0])
   819(SETEQ (FlagGT_ULT)) => (MOVLconst [0])
   820(SETEQ (FlagGT_UGT)) => (MOVLconst [0])
   821
   822(SETNE (FlagEQ)) => (MOVLconst [0])
   823(SETNE (FlagLT_ULT)) => (MOVLconst [1])
   824(SETNE (FlagLT_UGT)) => (MOVLconst [1])
   825(SETNE (FlagGT_ULT)) => (MOVLconst [1])
   826(SETNE (FlagGT_UGT)) => (MOVLconst [1])
   827
   828(SETL (FlagEQ)) => (MOVLconst [0])
   829(SETL (FlagLT_ULT)) => (MOVLconst [1])
   830(SETL (FlagLT_UGT)) => (MOVLconst [1])
   831(SETL (FlagGT_ULT)) => (MOVLconst [0])
   832(SETL (FlagGT_UGT)) => (MOVLconst [0])
   833
   834(SETLE (FlagEQ)) => (MOVLconst [1])
   835(SETLE (FlagLT_ULT)) => (MOVLconst [1])
   836(SETLE (FlagLT_UGT)) => (MOVLconst [1])
   837(SETLE (FlagGT_ULT)) => (MOVLconst [0])
   838(SETLE (FlagGT_UGT)) => (MOVLconst [0])
   839
   840(SETG (FlagEQ)) => (MOVLconst [0])
   841(SETG (FlagLT_ULT)) => (MOVLconst [0])
   842(SETG (FlagLT_UGT)) => (MOVLconst [0])
   843(SETG (FlagGT_ULT)) => (MOVLconst [1])
   844(SETG (FlagGT_UGT)) => (MOVLconst [1])
   845
   846(SETGE (FlagEQ)) => (MOVLconst [1])
   847(SETGE (FlagLT_ULT)) => (MOVLconst [0])
   848(SETGE (FlagLT_UGT)) => (MOVLconst [0])
   849(SETGE (FlagGT_ULT)) => (MOVLconst [1])
   850(SETGE (FlagGT_UGT)) => (MOVLconst [1])
   851
   852(SETB (FlagEQ)) => (MOVLconst [0])
   853(SETB (FlagLT_ULT)) => (MOVLconst [1])
   854(SETB (FlagLT_UGT)) => (MOVLconst [0])
   855(SETB (FlagGT_ULT)) => (MOVLconst [1])
   856(SETB (FlagGT_UGT)) => (MOVLconst [0])
   857
   858(SETBE (FlagEQ)) => (MOVLconst [1])
   859(SETBE (FlagLT_ULT)) => (MOVLconst [1])
   860(SETBE (FlagLT_UGT)) => (MOVLconst [0])
   861(SETBE (FlagGT_ULT)) => (MOVLconst [1])
   862(SETBE (FlagGT_UGT)) => (MOVLconst [0])
   863
   864(SETA (FlagEQ)) => (MOVLconst [0])
   865(SETA (FlagLT_ULT)) => (MOVLconst [0])
   866(SETA (FlagLT_UGT)) => (MOVLconst [1])
   867(SETA (FlagGT_ULT)) => (MOVLconst [0])
   868(SETA (FlagGT_UGT)) => (MOVLconst [1])
   869
   870(SETAE (FlagEQ)) => (MOVLconst [1])
   871(SETAE (FlagLT_ULT)) => (MOVLconst [0])
   872(SETAE (FlagLT_UGT)) => (MOVLconst [1])
   873(SETAE (FlagGT_ULT)) => (MOVLconst [0])
   874(SETAE (FlagGT_UGT)) => (MOVLconst [1])
   875
   876// Remove redundant *const ops
   877(ADDLconst [c] x) && c==0  => x
   878(SUBLconst [c] x) && c==0  => x
   879(ANDLconst [c] _) && c==0  => (MOVLconst [0])
   880(ANDLconst [c] x) && c==-1 => x
   881(ORLconst [c] x)  && c==0  => x
   882(ORLconst [c] _)  && c==-1 => (MOVLconst [-1])
   883(XORLconst [c] x) && c==0  => x
   884// TODO: since we got rid of the W/B versions, we might miss
   885// things like (ANDLconst [0x100] x) which were formerly
   886// (ANDBconst [0] x).  Probably doesn't happen very often.
   887// If we cared, we might do:
   888//  (ANDLconst <t> [c] x) && t.Size()==1 && int8(x)==0 => (MOVLconst [0])
   889
   890// Convert constant subtracts to constant adds
   891(SUBLconst [c] x) => (ADDLconst [-c] x)
   892
   893// generic constant folding
   894// TODO: more of this
   895(ADDLconst [c] (MOVLconst [d])) => (MOVLconst [c+d])
   896(ADDLconst [c] (ADDLconst [d] x)) => (ADDLconst [c+d] x)
   897(SARLconst [c] (MOVLconst [d])) => (MOVLconst [d>>uint64(c)])
   898(SARWconst [c] (MOVLconst [d])) => (MOVLconst [d>>uint64(c)])
   899(SARBconst [c] (MOVLconst [d])) => (MOVLconst [d>>uint64(c)])
   900(NEGL (MOVLconst [c])) => (MOVLconst [-c])
   901(MULLconst [c] (MOVLconst [d])) => (MOVLconst [c*d])
   902(ANDLconst [c] (MOVLconst [d])) => (MOVLconst [c&d])
   903(ORLconst [c] (MOVLconst [d])) => (MOVLconst [c|d])
   904(XORLconst [c] (MOVLconst [d])) => (MOVLconst [c^d])
   905(NOTL (MOVLconst [c])) => (MOVLconst [^c])
   906
   907// generic simplifications
   908// TODO: more of this
   909(ADDL x (NEGL y)) => (SUBL x y)
   910(SUBL x x) => (MOVLconst [0])
   911(ANDL x x) => x
   912(ORL x x) => x
   913(XORL x x) => (MOVLconst [0])
   914
   915// checking AND against 0.
   916(CMP(L|W|B)const l:(ANDL x y) [0]) && l.Uses==1 => (TEST(L|W|B) x y)
   917(CMPLconst l:(ANDLconst [c] x) [0]) && l.Uses==1 => (TESTLconst [c] x)
   918(CMPWconst l:(ANDLconst [c] x) [0]) && l.Uses==1 => (TESTWconst [int16(c)] x)
   919(CMPBconst l:(ANDLconst [c] x) [0]) && l.Uses==1 => (TESTBconst [int8(c)] x)
   920
   921// TEST %reg,%reg is shorter than CMP
   922(CMP(L|W|B)const x [0]) => (TEST(L|W|B) x x)
   923
   924// Convert LEAL1 back to ADDL if we can
   925(LEAL1 [0] {nil} x y) => (ADDL x y)
   926
   927// For PIC, break floating-point constant loading into two instructions so we have
   928// a register to use for holding the address of the constant pool entry.
   929(MOVSSconst [c]) && config.ctxt.Flag_shared => (MOVSSconst2 (MOVSSconst1 [c]))
   930(MOVSDconst [c]) && config.ctxt.Flag_shared => (MOVSDconst2 (MOVSDconst1 [c]))
   931
   932(CMP(L|W|B) l:(MOV(L|W|B)load {sym} [off] ptr mem) x) && canMergeLoad(v, l) && clobber(l) => (CMP(L|W|B)load {sym} [off] ptr x mem)
   933(CMP(L|W|B) x l:(MOV(L|W|B)load {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (InvertFlags (CMP(L|W|B)load {sym} [off] ptr x mem))
   934
   935(CMP(L|W|B)const l:(MOV(L|W|B)load {sym} [off] ptr mem) [c])
   936	&& l.Uses == 1
   937	&& clobber(l) =>
   938  @l.Block (CMP(L|W|B)constload {sym} [makeValAndOff(int32(c),off)] ptr mem)
   939
   940(CMPLload {sym} [off] ptr (MOVLconst [c]) mem) => (CMPLconstload {sym} [makeValAndOff(c,off)] ptr mem)
   941(CMPWload {sym} [off] ptr (MOVLconst [c]) mem) => (CMPWconstload {sym} [makeValAndOff(int32(int16(c)),off)] ptr mem)
   942(CMPBload {sym} [off] ptr (MOVLconst [c]) mem) => (CMPBconstload {sym} [makeValAndOff(int32(int8(c)),off)] ptr mem)
   943
   944(MOVBload [off] {sym} (SB) _) && symIsRO(sym) => (MOVLconst [int32(read8(sym, int64(off)))])
   945(MOVWload [off] {sym} (SB) _) && symIsRO(sym) => (MOVLconst [int32(read16(sym, int64(off), config.ctxt.Arch.ByteOrder))])
   946(MOVLload [off] {sym} (SB) _) && symIsRO(sym) => (MOVLconst [int32(read32(sym, int64(off), config.ctxt.Arch.ByteOrder))])
   947(MOVBLSXload [off] {sym} (SB) _) && symIsRO(sym) => (MOVLconst [int32(int8(read8(sym, int64(off))))])
   948(MOVWLSXload [off] {sym} (SB) _) && symIsRO(sym) => (MOVLconst [int32(int16(read16(sym, int64(off), config.ctxt.Arch.ByteOrder)))])

View as plain text