Source file src/cmd/compile/internal/arm64/ssa.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package arm64
     6  
     7  import (
     8  	"math"
     9  
    10  	"cmd/compile/internal/base"
    11  	"cmd/compile/internal/ir"
    12  	"cmd/compile/internal/logopt"
    13  	"cmd/compile/internal/objw"
    14  	"cmd/compile/internal/ssa"
    15  	"cmd/compile/internal/ssagen"
    16  	"cmd/compile/internal/types"
    17  	"cmd/internal/obj"
    18  	"cmd/internal/obj/arm64"
    19  	"internal/abi"
    20  )
    21  
    22  // loadByType returns the load instruction of the given type.
    23  func loadByType(t *types.Type) obj.As {
    24  	if t.IsSIMD() {
    25  		if t.Size() == 16 {
    26  			return arm64.AFMOVQ // Use FMOVQ (LDR Q) for 128-bit SIMD loads
    27  		}
    28  	} else if t.IsFloat() {
    29  		switch t.Size() {
    30  		case 4:
    31  			return arm64.AFMOVS
    32  		case 8:
    33  			return arm64.AFMOVD
    34  		}
    35  	} else {
    36  		switch t.Size() {
    37  		case 1:
    38  			if t.IsSigned() {
    39  				return arm64.AMOVB
    40  			} else {
    41  				return arm64.AMOVBU
    42  			}
    43  		case 2:
    44  			if t.IsSigned() {
    45  				return arm64.AMOVH
    46  			} else {
    47  				return arm64.AMOVHU
    48  			}
    49  		case 4:
    50  			if t.IsSigned() {
    51  				return arm64.AMOVW
    52  			} else {
    53  				return arm64.AMOVWU
    54  			}
    55  		case 8:
    56  			return arm64.AMOVD
    57  		}
    58  	}
    59  	panic("bad load type")
    60  }
    61  
    62  // storeByType returns the store instruction of the given type.
    63  func storeByType(t *types.Type) obj.As {
    64  	if t.IsSIMD() {
    65  		if t.Size() == 16 {
    66  			return arm64.AFMOVQ // Use FMOVQ (STR Q) for 128-bit SIMD stores
    67  		}
    68  	} else if t.IsFloat() {
    69  		switch t.Size() {
    70  		case 4:
    71  			return arm64.AFMOVS
    72  		case 8:
    73  			return arm64.AFMOVD
    74  		}
    75  	} else {
    76  		switch t.Size() {
    77  		case 1:
    78  			return arm64.AMOVB
    79  		case 2:
    80  			return arm64.AMOVH
    81  		case 4:
    82  			return arm64.AMOVW
    83  		case 8:
    84  			return arm64.AMOVD
    85  		}
    86  	}
    87  	panic("bad store type")
    88  }
    89  
    90  // loadByType2 returns an opcode that can load consecutive memory locations into 2 registers with type t.
    91  // returns obj.AXXX if no such opcode exists.
    92  func loadByType2(t *types.Type) obj.As {
    93  	if t.IsFloat() {
    94  		switch t.Size() {
    95  		case 4:
    96  			return arm64.AFLDPS
    97  		case 8:
    98  			return arm64.AFLDPD
    99  		}
   100  	} else {
   101  		switch t.Size() {
   102  		case 4:
   103  			return arm64.ALDPW
   104  		case 8:
   105  			return arm64.ALDP
   106  		}
   107  	}
   108  	return obj.AXXX
   109  }
   110  
   111  // storeByType2 returns an opcode that can store registers with type t into 2 consecutive memory locations.
   112  // returns obj.AXXX if no such opcode exists.
   113  func storeByType2(t *types.Type) obj.As {
   114  	if t.IsFloat() {
   115  		switch t.Size() {
   116  		case 4:
   117  			return arm64.AFSTPS
   118  		case 8:
   119  			return arm64.AFSTPD
   120  		}
   121  	} else {
   122  		switch t.Size() {
   123  		case 4:
   124  			return arm64.ASTPW
   125  		case 8:
   126  			return arm64.ASTP
   127  		}
   128  	}
   129  	return obj.AXXX
   130  }
   131  
   132  // makeshift encodes a register shifted by a constant, used as an Offset in Prog.
   133  func makeshift(v *ssa.Value, reg int16, typ int64, s int64) int64 {
   134  	if s < 0 || s >= 64 {
   135  		v.Fatalf("shift out of range: %d", s)
   136  	}
   137  	return int64(reg&31)<<16 | typ | (s&63)<<10
   138  }
   139  
   140  // genshift generates a Prog for r = r0 op (r1 shifted by n).
   141  func genshift(s *ssagen.State, v *ssa.Value, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog {
   142  	p := s.Prog(as)
   143  	p.From.Type = obj.TYPE_SHIFT
   144  	p.From.Offset = makeshift(v, r1, typ, n)
   145  	p.Reg = r0
   146  	if r != 0 {
   147  		p.To.Type = obj.TYPE_REG
   148  		p.To.Reg = r
   149  	}
   150  	return p
   151  }
   152  
   153  // generate the memory operand for the indexed load/store instructions.
   154  // base and idx are registers.
   155  func genIndexedOperand(op ssa.Op, base, idx int16) obj.Addr {
   156  	// Reg: base register, Index: (shifted) index register
   157  	mop := obj.Addr{Type: obj.TYPE_MEM, Reg: base}
   158  	switch op {
   159  	case ssa.OpARM64MOVDloadidx8, ssa.OpARM64MOVDstoreidx8,
   160  		ssa.OpARM64FMOVDloadidx8, ssa.OpARM64FMOVDstoreidx8:
   161  		mop.Index = arm64.REG_LSL | 3<<5 | idx&31
   162  	case ssa.OpARM64MOVWloadidx4, ssa.OpARM64MOVWUloadidx4, ssa.OpARM64MOVWstoreidx4,
   163  		ssa.OpARM64FMOVSloadidx4, ssa.OpARM64FMOVSstoreidx4:
   164  		mop.Index = arm64.REG_LSL | 2<<5 | idx&31
   165  	case ssa.OpARM64MOVHloadidx2, ssa.OpARM64MOVHUloadidx2, ssa.OpARM64MOVHstoreidx2:
   166  		mop.Index = arm64.REG_LSL | 1<<5 | idx&31
   167  	default: // not shifted
   168  		mop.Index = idx
   169  	}
   170  	return mop
   171  }
   172  
   173  // simdRegArng encodes ssa value's register with specified simd arrangement
   174  func simdRegArng(reg int16, arng int16) int16 {
   175  	if reg < arm64.REG_F0 || arm64.REG_F31 < reg {
   176  		base.Fatalf("expected fp register: r%d", reg)
   177  	}
   178  	var err error
   179  	if reg, err = arm64.RegisterArrangement(reg, arng, false); err != nil {
   180  		base.Fatalf("bad simd register arrangement: %v", err)
   181  	}
   182  	return reg
   183  }
   184  
   185  // simdRegElem encodes ssa value's reference to a vector register element
   186  func simdRegElem(reg int16, arng int16, idx int16) (res obj.Addr) {
   187  	if reg < arm64.REG_F0 || arm64.REG_F31 < reg {
   188  		base.Fatalf("expected fp register: r%d", reg)
   189  	}
   190  	elem, err := arm64.RegisterArrangement(reg, arng, true /*indexing*/)
   191  	if err != nil {
   192  		base.Fatalf("bad simd register indexing arrangement: %v", err)
   193  	}
   194  	res.Type = obj.TYPE_REG
   195  	res.Class = arm64.C_ELEM
   196  	res.Index = idx
   197  	res.Reg = elem
   198  	return
   199  }
   200  
   201  // allLanes converts an element arrangement to its 128-bit vector arrangement.
   202  // e.g., ARNG_B -> ARNG_16B, ARNG_S -> ARNG_4S
   203  func allLanes(arng int16) int16 {
   204  	switch arng {
   205  	case arm64.ARNG_B:
   206  		return arm64.ARNG_16B
   207  	case arm64.ARNG_H:
   208  		return arm64.ARNG_8H
   209  	case arm64.ARNG_S:
   210  		return arm64.ARNG_4S
   211  	case arm64.ARNG_D:
   212  		return arm64.ARNG_2D
   213  	default:
   214  		base.Fatalf("unsupported element arrangement: %d", arng)
   215  		return 0
   216  	}
   217  }
   218  
   219  // arngNarrow converts arng to its narrow (halved element width and vector width) arrangement.
   220  func arngNarrow(arng int16) int16 {
   221  	switch arng {
   222  	case arm64.ARNG_8H:
   223  		return arm64.ARNG_8B
   224  	case arm64.ARNG_4S:
   225  		return arm64.ARNG_4H
   226  	case arm64.ARNG_2D:
   227  		return arm64.ARNG_2S
   228  	default:
   229  		base.Fatalf("unsupported narrow input arrangement: %d", arng)
   230  		return 0
   231  	}
   232  }
   233  
   234  // arngLong converts a half-lane arrangement to its long (doubled element width and vector width) arrangement.
   235  func arngLong(arng int16) int16 {
   236  	switch arng {
   237  	case arm64.ARNG_8B:
   238  		return arm64.ARNG_8H
   239  	case arm64.ARNG_4H:
   240  		return arm64.ARNG_4S
   241  	case arm64.ARNG_2S:
   242  		return arm64.ARNG_2D
   243  	case arm64.ARNG_1D:
   244  		return arm64.ARNG_1Q
   245  	default:
   246  		base.Fatalf("unsupported long input arrangement: %d", arng)
   247  		return 0
   248  	}
   249  }
   250  
   251  // arngHalfLanes converts a full-width arrangement to its half-lane (64-bit) arrangement.
   252  // Same element width, half the lanes. Used for long base variant sources.
   253  func arngHalfLanes(arng int16) int16 {
   254  	switch arng {
   255  	case arm64.ARNG_16B:
   256  		return arm64.ARNG_8B
   257  	case arm64.ARNG_8H:
   258  		return arm64.ARNG_4H
   259  	case arm64.ARNG_4S:
   260  		return arm64.ARNG_2S
   261  	case arm64.ARNG_2D:
   262  		return arm64.ARNG_1D
   263  	default:
   264  		base.Fatalf("unsupported halfLanes input arrangement: %d", arng)
   265  		return 0
   266  	}
   267  }
   268  
   269  // arngTwiceLanes converts a half-lane (64-bit) arrangement to its full-width arrangement.
   270  // Same element width, double the lanes. Inverse of arngHalfLanes.
   271  func arngTwiceLanes(arng int16) int16 {
   272  	switch arng {
   273  	case arm64.ARNG_8B:
   274  		return arm64.ARNG_16B
   275  	case arm64.ARNG_4H:
   276  		return arm64.ARNG_8H
   277  	case arm64.ARNG_2S:
   278  		return arm64.ARNG_4S
   279  	default:
   280  		base.Fatalf("unsupported twiceLanes input arrangement: %d", arng)
   281  		return 0
   282  	}
   283  }
   284  
   285  // simdV01Imm generates a VMOVI-like instruction, e.g. VMOVI $0, V0.B16
   286  func simdV01Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   287  	p := s.Prog(v.Op.Asm())
   288  	p.From.Type = obj.TYPE_CONST
   289  	p.From.Offset = int64(v.AuxUInt8())
   290  	p.To.Type = obj.TYPE_REG
   291  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   292  	return p
   293  }
   294  
   295  // simdV11Asm generates element-wise unary vector operations with explicit asm, e.g. VMOV V1.B16, V0.B16
   296  func simdV11Asm(s *ssagen.State, asm obj.As, src, dst int16, arrangement int16) *obj.Prog {
   297  	p := s.Prog(asm)
   298  	p.From.Type = obj.TYPE_REG
   299  	p.From.Reg = simdRegArng(src, arrangement)
   300  	p.To.Type = obj.TYPE_REG
   301  	p.To.Reg = simdRegArng(dst, arrangement)
   302  	return p
   303  }
   304  
   305  // simdV11 generates element-wise unary vector operations, e.g. VCNT V1.B8, V0.B8
   306  func simdV11(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   307  	return simdV11Asm(s, v.Op.Asm(), v.Args[0].Reg(), v.Reg(), arrangement)
   308  }
   309  
   310  // simdV11Imm generates a unary vector operation with immediate constant,
   311  // e.g. VUSHR $3, V1.B16, V0.B16
   312  func simdV11Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   313  	p := s.Prog(v.Op.Asm())
   314  	p.From.Type = obj.TYPE_CONST
   315  	p.From.Offset = int64(v.AuxUInt8())
   316  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   317  	p.To.Type = obj.TYPE_REG
   318  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   319  	return p
   320  }
   321  
   322  // simdV11ImmIn1 generates a broadcast1ToN instruction,
   323  // e.g. VDUP V1.S[0], V0.S4 (duplicate element 0 to all lanes)
   324  // The arrangement parameter specifies the element arrangement (e.g., ARNG_S, ARNG_D)
   325  func simdV11ImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   326  	p := s.Prog(v.Op.Asm())
   327  	p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
   328  	p.To.Type = obj.TYPE_REG
   329  	p.To.Reg = simdRegArng(v.Reg(), allLanes(arrangement))
   330  	return p
   331  }
   332  
   333  // simdV11Scalar generates vector-to-scalar reduction operations, e.g. VUADDLV V1.B8, V0
   334  func simdV11Scalar(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   335  	p := s.Prog(v.Op.Asm())
   336  	p.From.Type = obj.TYPE_REG
   337  	p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   338  	p.To.Type = obj.TYPE_REG
   339  	p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
   340  	return p
   341  }
   342  
   343  // simdV11ScalarImmIn1 generates a SIMD instruction with indexed input and
   344  // scalar-in-vector-register output, e.g. VDUP V1.S[1], V0
   345  // The arrangement parameter specifies the source arrangement (e.g., S, D)
   346  func simdV11ScalarImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   347  	p := s.Prog(v.Op.Asm())
   348  	p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
   349  	p.To.Type = obj.TYPE_REG
   350  	p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
   351  	p.To.Class = arm64.C_VREG
   352  	return p
   353  }
   354  
   355  // simdV21 generates element-wise binary vector operations, e.g. VFADD V1.S4, V2.S4, V0.S4
   356  func simdV21(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   357  	p := s.Prog(v.Op.Asm())
   358  	p.From.Type = obj.TYPE_REG
   359  	p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   360  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   361  	p.To.Type = obj.TYPE_REG
   362  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   363  	return p
   364  }
   365  
   366  // simdV21Imm generates a binary instruction with immediate, e.g. EXT $imm, Vm.16B, Vn.16B, Vd.16B
   367  func simdV21Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   368  	p := s.Prog(v.Op.Asm())
   369  	p.From.Type = obj.TYPE_CONST
   370  	p.From.Offset = int64(v.AuxUInt8())
   371  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   372  	p.To.Type = obj.TYPE_REG
   373  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   374  	p.AddRestSource(obj.Addr{Type: obj.TYPE_REG, Reg: simdRegArng(v.Args[1].Reg(), arrangement)})
   375  	return p
   376  }
   377  
   378  // simdV31ResultInArg0 generates a destructive 3-register instruction,
   379  // e.g. VBIT Vm.16B, Vn.16B, Vd.16B.
   380  func simdV31ResultInArg0(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   381  	p := s.Prog(v.Op.Asm())
   382  	p.From.Type = obj.TYPE_REG
   383  	p.From.Reg = simdRegArng(v.Args[2].Reg(), arrangement)
   384  	p.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   385  	p.To.Type = obj.TYPE_REG
   386  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   387  	return p
   388  }
   389  
   390  // simdV21List generates a binary instruction with register list, e.g. TBL Vm.Ta, {Vn.B16}, Vd.Ta.
   391  func simdV21List(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   392  	if v.Op.Asm() != arm64.AVTBL { // TODO: support other instructions as needed.
   393  		panic("simdV21List: expected VTBL")
   394  	}
   395  	p := s.Prog(v.Op.Asm())
   396  	p.From.Type = obj.TYPE_REG
   397  	p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   398  	// TBL requires B16 table arrangement.
   399  	// Also, multi-element register lists are not supported by regalloc.
   400  	const listB16 = int64(1 << 30)
   401  	regList, _ := arm64.RegisterListOffset(int(v.Args[0].Reg()&31), 1, listB16, 0)
   402  	p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList})
   403  	p.To.Type = obj.TYPE_REG
   404  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   405  	return p
   406  }
   407  
   408  // simdV31ResultInArg0List generates a destructive 3-register instruction
   409  // with register list, e.g. TBX Vm.Ta, {Vn.B16}, Vd.Ta.
   410  func simdV31ResultInArg0List(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   411  	if v.Op.Asm() != arm64.AVTBX { // TODO: support other instructions as needed.
   412  		panic("simdV31ResultInArg0List: expected VTBX")
   413  	}
   414  	p := s.Prog(v.Op.Asm())
   415  	p.From.Type = obj.TYPE_REG
   416  	p.From.Reg = simdRegArng(v.Args[2].Reg(), arrangement)
   417  	// TBX requires B16 table arrangement.
   418  	// Also, multi-element register lists are not supported by regalloc.
   419  	const listB16 = int64(1 << 30)
   420  	regList, _ := arm64.RegisterListOffset(int(v.Args[1].Reg()&31), 1, listB16, 0)
   421  	p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList})
   422  	p.To.Type = obj.TYPE_REG
   423  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   424  	return p
   425  }
   426  
   427  // simdVfpvResultInArg0ImmOutIn1 generates vector floating-point SetElem,
   428  // e.g. VMOV V2.S[0], V1.S[3] (INS element instruction)
   429  // The arrangement parameter specifies the vector element arrangement (e.g., S, D)
   430  func simdVfpvResultInArg0ImmOutIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   431  	p := s.Prog(v.Op.Asm())
   432  	p.To = simdRegElem(v.Reg(), arrangement, int16(v.AuxUInt8()))
   433  	p.From = simdRegElem(v.Args[1].Reg(), arrangement, 0)
   434  	return p
   435  }
   436  
   437  // simdVgpImmIn1 generates vector GetElem instruction VMOV V1.S[2], R0
   438  // The arrangement parameter specifies the vector element arrangement (e.g., S, D)
   439  func simdVgpImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   440  	p := s.Prog(v.Op.Asm())
   441  	p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
   442  	p.To.Reg = v.Reg()
   443  	p.To.Type = obj.TYPE_REG
   444  	return p
   445  }
   446  
   447  // simdVgpvResultInArg0ImmOutIn0 generates vector SetElem, e.g. VMOV R0, V1.S[2] (INS general instruction)
   448  // The arrangement parameter specifies the vector element arrangement (e.g., S, D)
   449  func simdVgpvResultInArg0ImmOutIn0(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   450  	p := s.Prog(v.Op.Asm())
   451  	p.To = simdRegElem(v.Reg(), arrangement, int16(v.AuxUInt8()))
   452  	p.From.Reg = v.Args[1].Reg()
   453  	p.From.Type = obj.TYPE_REG
   454  	return p
   455  }
   456  
   457  // Narrow and long lowering helpers
   458  
   459  // simdV11Narrow generates a pure narrowing instruction, e.g. XTN Vn.8H, Vd.8B
   460  func simdV11Narrow(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   461  	p := s.Prog(v.Op.Asm())
   462  	p.From.Type = obj.TYPE_REG
   463  	p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   464  	p.To.Type = obj.TYPE_REG
   465  	p.To.Reg = simdRegArng(v.Reg(), arngNarrow(arrangement))
   466  	return p
   467  }
   468  
   469  // simdV21Narrow2 generates a a destructive (updating upper half only) narrow "2" instruction,
   470  // e.g. XTN2 V1.4S, V0.8H. The arrangement parameter specifies the source arrangement.
   471  func simdV21Narrow2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   472  	p := s.Prog(v.Op.Asm())
   473  	p.From.Type = obj.TYPE_REG
   474  	p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   475  	p.To.Type = obj.TYPE_REG
   476  	p.To.Reg = simdRegArng(v.Reg(), arngTwiceLanes(arngNarrow(arrangement)))
   477  	return p
   478  }
   479  
   480  // simdV11ImmNarrow generates a pure narrowing instruction with immediate, e.g. SHRN $imm, V1.4S, V0.8B
   481  // The arrangement parameter specifies the source arrangement.
   482  func simdV11ImmNarrow(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   483  	p := s.Prog(v.Op.Asm())
   484  	p.From.Type = obj.TYPE_CONST
   485  	p.From.Offset = int64(v.AuxUInt8())
   486  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   487  	p.To.Type = obj.TYPE_REG
   488  	p.To.Reg = simdRegArng(v.Reg(), arngNarrow(arrangement))
   489  	return p
   490  }
   491  
   492  // simdV21ImmNarrow2 generates a destructive (updating upper half only) narrow "2" instruction
   493  // with immediate, e.g. SHRN2 $imm, V1.4S, V0.16B. The arrangement parameter specifies the source arrangement.
   494  func simdV21ImmNarrow2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   495  	p := s.Prog(v.Op.Asm())
   496  	p.From.Type = obj.TYPE_CONST
   497  	p.From.Offset = int64(v.AuxUInt8())
   498  	p.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   499  	p.To.Type = obj.TYPE_REG
   500  	p.To.Reg = simdRegArng(v.Reg(), arngTwiceLanes(arngNarrow(arrangement)))
   501  	return p
   502  }
   503  
   504  // simdV11Long generates a unary long instruction, e.g. SXTL V1.4H, V0.8H
   505  // The instruction reads the lower half of the source, the destination has 2x element size.
   506  func simdV11Long(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   507  	src := arngHalfLanes(arrangement)
   508  	p := s.Prog(v.Op.Asm())
   509  	p.From.Type = obj.TYPE_REG
   510  	p.From.Reg = simdRegArng(v.Args[0].Reg(), src)
   511  	p.To.Type = obj.TYPE_REG
   512  	p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
   513  	return p
   514  }
   515  
   516  // simdV11Long2 generates a unary long "2" instruction, e.g. SXTL2 V1.4S, V0.2D
   517  // The instruction reads the upper half of the source, the destination has 2x element size.
   518  func simdV11Long2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   519  	p := s.Prog(v.Op.Asm())
   520  	p.From.Type = obj.TYPE_REG
   521  	p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   522  	p.To.Type = obj.TYPE_REG
   523  	p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
   524  	return p
   525  }
   526  
   527  // simdV11ImmLong generates a long instruction with immediate, e.g. USHLL $imm, V1.4H, V0.8H
   528  // The instruction reads the lower half of the source, the destination has 2x element size.
   529  func simdV11ImmLong(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   530  	src := arngHalfLanes(arrangement)
   531  	p := s.Prog(v.Op.Asm())
   532  	p.From.Type = obj.TYPE_CONST
   533  	p.From.Offset = int64(v.AuxUInt8())
   534  	p.Reg = simdRegArng(v.Args[0].Reg(), src)
   535  	p.To.Type = obj.TYPE_REG
   536  	p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
   537  	return p
   538  }
   539  
   540  // simdV11ImmLong2 generates a long "2" instruction with immediate, e.g. USHLL2 $imm, V1.4S, V0.2D
   541  // The instruction reads the upper half of the source, the destination has 2x element size.
   542  func simdV11ImmLong2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   543  	p := s.Prog(v.Op.Asm())
   544  	p.From.Type = obj.TYPE_CONST
   545  	p.From.Offset = int64(v.AuxUInt8())
   546  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   547  	p.To.Type = obj.TYPE_REG
   548  	p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
   549  	return p
   550  }
   551  
   552  // simdV21Long generates a binary long instruction, e.g. UMULL V1.4H, V2.4H, V0.8H
   553  // The instruction reads lower halves of its sources, the destination has 2x element size.
   554  func simdV21Long(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   555  	src := arngHalfLanes(arrangement)
   556  	p := s.Prog(v.Op.Asm())
   557  	p.From.Type = obj.TYPE_REG
   558  	p.From.Reg = simdRegArng(v.Args[1].Reg(), src)
   559  	p.Reg = simdRegArng(v.Args[0].Reg(), src)
   560  	p.To.Type = obj.TYPE_REG
   561  	p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
   562  	return p
   563  }
   564  
   565  // simdV21Long2 generates a binary long "2" instruction, e.g. UMULL2 V1.4S, V2.4S, V0.2D
   566  // The instruction reads upper halves of its sources, the destination has 2x element size.
   567  func simdV21Long2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   568  	p := s.Prog(v.Op.Asm())
   569  	p.From.Type = obj.TYPE_REG
   570  	p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   571  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   572  	p.To.Type = obj.TYPE_REG
   573  	p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
   574  	return p
   575  }
   576  
   577  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   578  	switch v.Op {
   579  	case ssa.OpCopy, ssa.OpARM64MOVDreg:
   580  		if v.Type.IsMemory() {
   581  			return
   582  		}
   583  		x := v.Args[0].Reg()
   584  		y := v.Reg()
   585  		if x == y {
   586  			return
   587  		}
   588  		as := arm64.AMOVD
   589  		if v.Type.IsFloat() {
   590  			switch v.Type.Size() {
   591  			case 4:
   592  				as = arm64.AFMOVS
   593  			case 8:
   594  				as = arm64.AFMOVD
   595  			default:
   596  				panic("bad float size")
   597  			}
   598  		} else if v.Type.IsSIMD() {
   599  			if v.Type.Size() == 16 {
   600  				simdV11Asm(s, arm64.AVMOV, x, y, arm64.ARNG_16B)
   601  				return
   602  			} else {
   603  				panic("bad simd size")
   604  			}
   605  		}
   606  		p := s.Prog(as)
   607  		p.From.Type = obj.TYPE_REG
   608  		p.From.Reg = x
   609  		p.To.Type = obj.TYPE_REG
   610  		p.To.Reg = y
   611  	case ssa.OpARM64MOVDnop, ssa.OpARM64ZERO:
   612  		// nothing to do
   613  	case ssa.OpARM64VMOVI16B:
   614  		simdV01Imm(s, v, arm64.ARNG_16B)
   615  	case ssa.OpLoadReg:
   616  		if v.Type.IsFlags() {
   617  			v.Fatalf("load flags not implemented: %v", v.LongString())
   618  			return
   619  		}
   620  		p := s.Prog(loadByType(v.Type))
   621  		ssagen.AddrAuto(&p.From, v.Args[0])
   622  		p.To.Type = obj.TYPE_REG
   623  		p.To.Reg = v.Reg()
   624  	case ssa.OpStoreReg:
   625  		if v.Type.IsFlags() {
   626  			v.Fatalf("store flags not implemented: %v", v.LongString())
   627  			return
   628  		}
   629  		p := s.Prog(storeByType(v.Type))
   630  		p.From.Type = obj.TYPE_REG
   631  		p.From.Reg = v.Args[0].Reg()
   632  		ssagen.AddrAuto(&p.To, v)
   633  	case ssa.OpArgIntReg, ssa.OpArgFloatReg:
   634  		ssagen.CheckArgReg(v)
   635  		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
   636  		// The loop only runs once.
   637  		args := v.Block.Func.RegArgs
   638  		if len(args) == 0 {
   639  			break
   640  		}
   641  		v.Block.Func.RegArgs = nil // prevent from running again
   642  
   643  		for i := 0; i < len(args); i++ {
   644  			a := args[i]
   645  			// Offset by size of the saved LR slot.
   646  			addr := ssagen.SpillSlotAddr(a, arm64.REGSP, base.Ctxt.Arch.FixedFrameSize)
   647  			// Look for double-register operations if we can.
   648  			if i < len(args)-1 {
   649  				b := args[i+1]
   650  				if a.Type.Size() == b.Type.Size() &&
   651  					a.Type.IsFloat() == b.Type.IsFloat() &&
   652  					b.Offset == a.Offset+a.Type.Size() {
   653  					ld := loadByType2(a.Type)
   654  					st := storeByType2(a.Type)
   655  					if ld != obj.AXXX && st != obj.AXXX {
   656  						s.FuncInfo().AddSpill(obj.RegSpill{Reg: a.Reg, Reg2: b.Reg, Addr: addr, Unspill: ld, Spill: st})
   657  						i++ // b is done also, skip it.
   658  						continue
   659  					}
   660  				}
   661  			}
   662  			// Pass the spill/unspill information along to the assembler.
   663  			s.FuncInfo().AddSpill(obj.RegSpill{Reg: a.Reg, Addr: addr, Unspill: loadByType(a.Type), Spill: storeByType(a.Type)})
   664  		}
   665  
   666  	case ssa.OpARM64ADD,
   667  		ssa.OpARM64SUB,
   668  		ssa.OpARM64AND,
   669  		ssa.OpARM64OR,
   670  		ssa.OpARM64XOR,
   671  		ssa.OpARM64BIC,
   672  		ssa.OpARM64EON,
   673  		ssa.OpARM64ORN,
   674  		ssa.OpARM64MUL,
   675  		ssa.OpARM64MULW,
   676  		ssa.OpARM64MNEG,
   677  		ssa.OpARM64MNEGW,
   678  		ssa.OpARM64MULH,
   679  		ssa.OpARM64UMULH,
   680  		ssa.OpARM64MULL,
   681  		ssa.OpARM64UMULL,
   682  		ssa.OpARM64DIV,
   683  		ssa.OpARM64UDIV,
   684  		ssa.OpARM64DIVW,
   685  		ssa.OpARM64UDIVW,
   686  		ssa.OpARM64MOD,
   687  		ssa.OpARM64UMOD,
   688  		ssa.OpARM64MODW,
   689  		ssa.OpARM64UMODW,
   690  		ssa.OpARM64SLL,
   691  		ssa.OpARM64SRL,
   692  		ssa.OpARM64SRA,
   693  		ssa.OpARM64FADDS,
   694  		ssa.OpARM64FADDD,
   695  		ssa.OpARM64FSUBS,
   696  		ssa.OpARM64FSUBD,
   697  		ssa.OpARM64FMULS,
   698  		ssa.OpARM64FMULD,
   699  		ssa.OpARM64FNMULS,
   700  		ssa.OpARM64FNMULD,
   701  		ssa.OpARM64FDIVS,
   702  		ssa.OpARM64FDIVD,
   703  		ssa.OpARM64FMINS,
   704  		ssa.OpARM64FMIND,
   705  		ssa.OpARM64FMAXS,
   706  		ssa.OpARM64FMAXD,
   707  		ssa.OpARM64ROR,
   708  		ssa.OpARM64RORW:
   709  		r := v.Reg()
   710  		r1 := v.Args[0].Reg()
   711  		r2 := v.Args[1].Reg()
   712  		p := s.Prog(v.Op.Asm())
   713  		p.From.Type = obj.TYPE_REG
   714  		p.From.Reg = r2
   715  		p.Reg = r1
   716  		p.To.Type = obj.TYPE_REG
   717  		p.To.Reg = r
   718  	case ssa.OpARM64FMADDS,
   719  		ssa.OpARM64FMADDD,
   720  		ssa.OpARM64FNMADDS,
   721  		ssa.OpARM64FNMADDD,
   722  		ssa.OpARM64FMSUBS,
   723  		ssa.OpARM64FMSUBD,
   724  		ssa.OpARM64FNMSUBS,
   725  		ssa.OpARM64FNMSUBD,
   726  		ssa.OpARM64MADD,
   727  		ssa.OpARM64MADDW,
   728  		ssa.OpARM64MSUB,
   729  		ssa.OpARM64MSUBW:
   730  		rt := v.Reg()
   731  		ra := v.Args[0].Reg()
   732  		rm := v.Args[1].Reg()
   733  		rn := v.Args[2].Reg()
   734  		p := s.Prog(v.Op.Asm())
   735  		p.Reg = ra
   736  		p.From.Type = obj.TYPE_REG
   737  		p.From.Reg = rm
   738  		p.AddRestSourceReg(rn)
   739  		p.To.Type = obj.TYPE_REG
   740  		p.To.Reg = rt
   741  	case ssa.OpARM64ADDconst,
   742  		ssa.OpARM64SUBconst,
   743  		ssa.OpARM64ANDconst,
   744  		ssa.OpARM64ORconst,
   745  		ssa.OpARM64XORconst,
   746  		ssa.OpARM64SLLconst,
   747  		ssa.OpARM64SRLconst,
   748  		ssa.OpARM64SRAconst,
   749  		ssa.OpARM64RORconst,
   750  		ssa.OpARM64RORWconst:
   751  		p := s.Prog(v.Op.Asm())
   752  		p.From.Type = obj.TYPE_CONST
   753  		p.From.Offset = v.AuxInt
   754  		p.Reg = v.Args[0].Reg()
   755  		p.To.Type = obj.TYPE_REG
   756  		p.To.Reg = v.Reg()
   757  	case ssa.OpARM64ADDSconstflags:
   758  		p := s.Prog(v.Op.Asm())
   759  		p.From.Type = obj.TYPE_CONST
   760  		p.From.Offset = v.AuxInt
   761  		p.Reg = v.Args[0].Reg()
   762  		p.To.Type = obj.TYPE_REG
   763  		p.To.Reg = v.Reg0()
   764  	case ssa.OpARM64ADCzerocarry:
   765  		p := s.Prog(v.Op.Asm())
   766  		p.From.Type = obj.TYPE_REG
   767  		p.From.Reg = arm64.REGZERO
   768  		p.Reg = arm64.REGZERO
   769  		p.To.Type = obj.TYPE_REG
   770  		p.To.Reg = v.Reg()
   771  	case ssa.OpARM64ADCSflags,
   772  		ssa.OpARM64ADDSflags,
   773  		ssa.OpARM64SBCSflags,
   774  		ssa.OpARM64SUBSflags:
   775  		r := v.Reg0()
   776  		r1 := v.Args[0].Reg()
   777  		r2 := v.Args[1].Reg()
   778  		p := s.Prog(v.Op.Asm())
   779  		p.From.Type = obj.TYPE_REG
   780  		p.From.Reg = r2
   781  		p.Reg = r1
   782  		p.To.Type = obj.TYPE_REG
   783  		p.To.Reg = r
   784  	case ssa.OpARM64NEGSflags:
   785  		p := s.Prog(v.Op.Asm())
   786  		p.From.Type = obj.TYPE_REG
   787  		p.From.Reg = v.Args[0].Reg()
   788  		p.To.Type = obj.TYPE_REG
   789  		p.To.Reg = v.Reg0()
   790  	case ssa.OpARM64NGCzerocarry:
   791  		p := s.Prog(v.Op.Asm())
   792  		p.From.Type = obj.TYPE_REG
   793  		p.From.Reg = arm64.REGZERO
   794  		p.To.Type = obj.TYPE_REG
   795  		p.To.Reg = v.Reg()
   796  	case ssa.OpARM64EXTRconst,
   797  		ssa.OpARM64EXTRWconst:
   798  		p := s.Prog(v.Op.Asm())
   799  		p.From.Type = obj.TYPE_CONST
   800  		p.From.Offset = v.AuxInt
   801  		p.AddRestSourceReg(v.Args[0].Reg())
   802  		p.Reg = v.Args[1].Reg()
   803  		p.To.Type = obj.TYPE_REG
   804  		p.To.Reg = v.Reg()
   805  	case ssa.OpARM64MVNshiftLL, ssa.OpARM64NEGshiftLL:
   806  		genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
   807  	case ssa.OpARM64MVNshiftRL, ssa.OpARM64NEGshiftRL:
   808  		genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
   809  	case ssa.OpARM64MVNshiftRA, ssa.OpARM64NEGshiftRA:
   810  		genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
   811  	case ssa.OpARM64MVNshiftRO:
   812  		genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_ROR, v.AuxInt)
   813  	case ssa.OpARM64ADDshiftLL,
   814  		ssa.OpARM64SUBshiftLL,
   815  		ssa.OpARM64ANDshiftLL,
   816  		ssa.OpARM64ORshiftLL,
   817  		ssa.OpARM64XORshiftLL,
   818  		ssa.OpARM64EONshiftLL,
   819  		ssa.OpARM64ORNshiftLL,
   820  		ssa.OpARM64BICshiftLL:
   821  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
   822  	case ssa.OpARM64ADDshiftRL,
   823  		ssa.OpARM64SUBshiftRL,
   824  		ssa.OpARM64ANDshiftRL,
   825  		ssa.OpARM64ORshiftRL,
   826  		ssa.OpARM64XORshiftRL,
   827  		ssa.OpARM64EONshiftRL,
   828  		ssa.OpARM64ORNshiftRL,
   829  		ssa.OpARM64BICshiftRL:
   830  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
   831  	case ssa.OpARM64ADDshiftRA,
   832  		ssa.OpARM64SUBshiftRA,
   833  		ssa.OpARM64ANDshiftRA,
   834  		ssa.OpARM64ORshiftRA,
   835  		ssa.OpARM64XORshiftRA,
   836  		ssa.OpARM64EONshiftRA,
   837  		ssa.OpARM64ORNshiftRA,
   838  		ssa.OpARM64BICshiftRA:
   839  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
   840  	case ssa.OpARM64ANDshiftRO,
   841  		ssa.OpARM64ORshiftRO,
   842  		ssa.OpARM64XORshiftRO,
   843  		ssa.OpARM64EONshiftRO,
   844  		ssa.OpARM64ORNshiftRO,
   845  		ssa.OpARM64BICshiftRO:
   846  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_ROR, v.AuxInt)
   847  	case ssa.OpARM64MOVDconst:
   848  		p := s.Prog(v.Op.Asm())
   849  		p.From.Type = obj.TYPE_CONST
   850  		p.From.Offset = v.AuxInt
   851  		p.To.Type = obj.TYPE_REG
   852  		p.To.Reg = v.Reg()
   853  	case ssa.OpARM64FMOVSconst,
   854  		ssa.OpARM64FMOVDconst:
   855  		p := s.Prog(v.Op.Asm())
   856  		p.From.Type = obj.TYPE_FCONST
   857  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
   858  		p.To.Type = obj.TYPE_REG
   859  		p.To.Reg = v.Reg()
   860  	case ssa.OpARM64FCMPS0,
   861  		ssa.OpARM64FCMPD0:
   862  		p := s.Prog(v.Op.Asm())
   863  		p.From.Type = obj.TYPE_FCONST
   864  		p.From.Val = math.Float64frombits(0)
   865  		p.Reg = v.Args[0].Reg()
   866  	case ssa.OpARM64CMP,
   867  		ssa.OpARM64CMPW,
   868  		ssa.OpARM64CMN,
   869  		ssa.OpARM64CMNW,
   870  		ssa.OpARM64TST,
   871  		ssa.OpARM64TSTW,
   872  		ssa.OpARM64FCMPS,
   873  		ssa.OpARM64FCMPD:
   874  		p := s.Prog(v.Op.Asm())
   875  		p.From.Type = obj.TYPE_REG
   876  		p.From.Reg = v.Args[1].Reg()
   877  		p.Reg = v.Args[0].Reg()
   878  	case ssa.OpARM64CMPconst,
   879  		ssa.OpARM64CMPWconst,
   880  		ssa.OpARM64CMNconst,
   881  		ssa.OpARM64CMNWconst,
   882  		ssa.OpARM64TSTconst,
   883  		ssa.OpARM64TSTWconst:
   884  		p := s.Prog(v.Op.Asm())
   885  		p.From.Type = obj.TYPE_CONST
   886  		p.From.Offset = v.AuxInt
   887  		p.Reg = v.Args[0].Reg()
   888  	case ssa.OpARM64CMPshiftLL, ssa.OpARM64CMNshiftLL, ssa.OpARM64TSTshiftLL:
   889  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LL, v.AuxInt)
   890  	case ssa.OpARM64CMPshiftRL, ssa.OpARM64CMNshiftRL, ssa.OpARM64TSTshiftRL:
   891  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LR, v.AuxInt)
   892  	case ssa.OpARM64CMPshiftRA, ssa.OpARM64CMNshiftRA, ssa.OpARM64TSTshiftRA:
   893  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_AR, v.AuxInt)
   894  	case ssa.OpARM64TSTshiftRO:
   895  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_ROR, v.AuxInt)
   896  	case ssa.OpARM64MOVDaddr:
   897  		p := s.Prog(arm64.AMOVD)
   898  		p.From.Type = obj.TYPE_ADDR
   899  		p.From.Reg = v.Args[0].Reg()
   900  		p.To.Type = obj.TYPE_REG
   901  		p.To.Reg = v.Reg()
   902  
   903  		var wantreg string
   904  		// MOVD $sym+off(base), R
   905  		// the assembler expands it as the following:
   906  		// - base is SP: add constant offset to SP (R13)
   907  		//               when constant is large, tmp register (R11) may be used
   908  		// - base is SB: load external address from constant pool (use relocation)
   909  		switch v.Aux.(type) {
   910  		default:
   911  			v.Fatalf("aux is of unknown type %T", v.Aux)
   912  		case *obj.LSym:
   913  			wantreg = "SB"
   914  			ssagen.AddAux(&p.From, v)
   915  		case *ir.Name:
   916  			wantreg = "SP"
   917  			ssagen.AddAux(&p.From, v)
   918  		case nil:
   919  			// No sym, just MOVD $off(SP), R
   920  			wantreg = "SP"
   921  			p.From.Offset = v.AuxInt
   922  		}
   923  		if reg := v.Args[0].RegName(); reg != wantreg {
   924  			v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg)
   925  		}
   926  	case ssa.OpARM64MOVBload,
   927  		ssa.OpARM64MOVBUload,
   928  		ssa.OpARM64MOVHload,
   929  		ssa.OpARM64MOVHUload,
   930  		ssa.OpARM64MOVWload,
   931  		ssa.OpARM64MOVWUload,
   932  		ssa.OpARM64MOVDload,
   933  		ssa.OpARM64FMOVSload,
   934  		ssa.OpARM64FMOVDload,
   935  		ssa.OpARM64FMOVQload:
   936  		p := s.Prog(v.Op.Asm())
   937  		p.From.Type = obj.TYPE_MEM
   938  		p.From.Reg = v.Args[0].Reg()
   939  		ssagen.AddAux(&p.From, v)
   940  		p.To.Type = obj.TYPE_REG
   941  		p.To.Reg = v.Reg()
   942  	case ssa.OpARM64LDP, ssa.OpARM64LDPW, ssa.OpARM64LDPSW, ssa.OpARM64FLDPD, ssa.OpARM64FLDPS, ssa.OpARM64FLDPQ:
   943  		p := s.Prog(v.Op.Asm())
   944  		p.From.Type = obj.TYPE_MEM
   945  		p.From.Reg = v.Args[0].Reg()
   946  		ssagen.AddAux(&p.From, v)
   947  		p.To.Type = obj.TYPE_REGREG
   948  		p.To.Reg = v.Reg0()
   949  		p.To.Offset = int64(v.Reg1())
   950  	case ssa.OpARM64MOVBloadidx,
   951  		ssa.OpARM64MOVBUloadidx,
   952  		ssa.OpARM64MOVHloadidx,
   953  		ssa.OpARM64MOVHUloadidx,
   954  		ssa.OpARM64MOVWloadidx,
   955  		ssa.OpARM64MOVWUloadidx,
   956  		ssa.OpARM64MOVDloadidx,
   957  		ssa.OpARM64FMOVSloadidx,
   958  		ssa.OpARM64FMOVDloadidx,
   959  		ssa.OpARM64MOVHloadidx2,
   960  		ssa.OpARM64MOVHUloadidx2,
   961  		ssa.OpARM64MOVWloadidx4,
   962  		ssa.OpARM64MOVWUloadidx4,
   963  		ssa.OpARM64MOVDloadidx8,
   964  		ssa.OpARM64FMOVDloadidx8,
   965  		ssa.OpARM64FMOVSloadidx4:
   966  		p := s.Prog(v.Op.Asm())
   967  		p.From = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg())
   968  		p.To.Type = obj.TYPE_REG
   969  		p.To.Reg = v.Reg()
   970  	case ssa.OpARM64LDAR,
   971  		ssa.OpARM64LDARB,
   972  		ssa.OpARM64LDARW:
   973  		p := s.Prog(v.Op.Asm())
   974  		p.From.Type = obj.TYPE_MEM
   975  		p.From.Reg = v.Args[0].Reg()
   976  		ssagen.AddAux(&p.From, v)
   977  		p.To.Type = obj.TYPE_REG
   978  		p.To.Reg = v.Reg0()
   979  	case ssa.OpARM64MOVBstore,
   980  		ssa.OpARM64MOVHstore,
   981  		ssa.OpARM64MOVWstore,
   982  		ssa.OpARM64MOVDstore,
   983  		ssa.OpARM64FMOVSstore,
   984  		ssa.OpARM64FMOVDstore,
   985  		ssa.OpARM64FMOVQstore,
   986  		ssa.OpARM64STLRB,
   987  		ssa.OpARM64STLR,
   988  		ssa.OpARM64STLRW:
   989  		p := s.Prog(v.Op.Asm())
   990  		p.From.Type = obj.TYPE_REG
   991  		p.From.Reg = v.Args[1].Reg()
   992  		p.To.Type = obj.TYPE_MEM
   993  		p.To.Reg = v.Args[0].Reg()
   994  		ssagen.AddAux(&p.To, v)
   995  	case ssa.OpARM64MOVBstoreidx,
   996  		ssa.OpARM64MOVHstoreidx,
   997  		ssa.OpARM64MOVWstoreidx,
   998  		ssa.OpARM64MOVDstoreidx,
   999  		ssa.OpARM64FMOVSstoreidx,
  1000  		ssa.OpARM64FMOVDstoreidx,
  1001  		ssa.OpARM64MOVHstoreidx2,
  1002  		ssa.OpARM64MOVWstoreidx4,
  1003  		ssa.OpARM64FMOVSstoreidx4,
  1004  		ssa.OpARM64MOVDstoreidx8,
  1005  		ssa.OpARM64FMOVDstoreidx8:
  1006  		p := s.Prog(v.Op.Asm())
  1007  		p.To = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg())
  1008  		p.From.Type = obj.TYPE_REG
  1009  		p.From.Reg = v.Args[2].Reg()
  1010  	case ssa.OpARM64STP, ssa.OpARM64STPW, ssa.OpARM64FSTPD, ssa.OpARM64FSTPS, ssa.OpARM64FSTPQ:
  1011  		p := s.Prog(v.Op.Asm())
  1012  		p.From.Type = obj.TYPE_REGREG
  1013  		p.From.Reg = v.Args[1].Reg()
  1014  		p.From.Offset = int64(v.Args[2].Reg())
  1015  		p.To.Type = obj.TYPE_MEM
  1016  		p.To.Reg = v.Args[0].Reg()
  1017  		ssagen.AddAux(&p.To, v)
  1018  	case ssa.OpARM64BFI,
  1019  		ssa.OpARM64BFXIL:
  1020  		p := s.Prog(v.Op.Asm())
  1021  		p.From.Type = obj.TYPE_CONST
  1022  		p.From.Offset = v.AuxInt >> 8
  1023  		p.AddRestSourceConst(v.AuxInt & 0xff)
  1024  		p.Reg = v.Args[1].Reg()
  1025  		p.To.Type = obj.TYPE_REG
  1026  		p.To.Reg = v.Reg()
  1027  	case ssa.OpARM64SBFIZ,
  1028  		ssa.OpARM64SBFX,
  1029  		ssa.OpARM64UBFIZ,
  1030  		ssa.OpARM64UBFX:
  1031  		p := s.Prog(v.Op.Asm())
  1032  		p.From.Type = obj.TYPE_CONST
  1033  		p.From.Offset = v.AuxInt >> 8
  1034  		p.AddRestSourceConst(v.AuxInt & 0xff)
  1035  		p.Reg = v.Args[0].Reg()
  1036  		p.To.Type = obj.TYPE_REG
  1037  		p.To.Reg = v.Reg()
  1038  	case ssa.OpARM64LoweredAtomicExchange64,
  1039  		ssa.OpARM64LoweredAtomicExchange32,
  1040  		ssa.OpARM64LoweredAtomicExchange8:
  1041  		// LDAXR	(Rarg0), Rout
  1042  		// STLXR	Rarg1, (Rarg0), Rtmp
  1043  		// CBNZ		Rtmp, -2(PC)
  1044  		var ld, st obj.As
  1045  		switch v.Op {
  1046  		case ssa.OpARM64LoweredAtomicExchange8:
  1047  			ld = arm64.ALDAXRB
  1048  			st = arm64.ASTLXRB
  1049  		case ssa.OpARM64LoweredAtomicExchange32:
  1050  			ld = arm64.ALDAXRW
  1051  			st = arm64.ASTLXRW
  1052  		case ssa.OpARM64LoweredAtomicExchange64:
  1053  			ld = arm64.ALDAXR
  1054  			st = arm64.ASTLXR
  1055  		}
  1056  		r0 := v.Args[0].Reg()
  1057  		r1 := v.Args[1].Reg()
  1058  		out := v.Reg0()
  1059  		p := s.Prog(ld)
  1060  		p.From.Type = obj.TYPE_MEM
  1061  		p.From.Reg = r0
  1062  		p.To.Type = obj.TYPE_REG
  1063  		p.To.Reg = out
  1064  		p1 := s.Prog(st)
  1065  		p1.From.Type = obj.TYPE_REG
  1066  		p1.From.Reg = r1
  1067  		p1.To.Type = obj.TYPE_MEM
  1068  		p1.To.Reg = r0
  1069  		p1.RegTo2 = arm64.REGTMP
  1070  		p2 := s.Prog(arm64.ACBNZ)
  1071  		p2.From.Type = obj.TYPE_REG
  1072  		p2.From.Reg = arm64.REGTMP
  1073  		p2.To.Type = obj.TYPE_BRANCH
  1074  		p2.To.SetTarget(p)
  1075  	case ssa.OpARM64LoweredAtomicExchange64Variant,
  1076  		ssa.OpARM64LoweredAtomicExchange32Variant,
  1077  		ssa.OpARM64LoweredAtomicExchange8Variant:
  1078  		var swap obj.As
  1079  		switch v.Op {
  1080  		case ssa.OpARM64LoweredAtomicExchange8Variant:
  1081  			swap = arm64.ASWPALB
  1082  		case ssa.OpARM64LoweredAtomicExchange32Variant:
  1083  			swap = arm64.ASWPALW
  1084  		case ssa.OpARM64LoweredAtomicExchange64Variant:
  1085  			swap = arm64.ASWPALD
  1086  		}
  1087  		r0 := v.Args[0].Reg()
  1088  		r1 := v.Args[1].Reg()
  1089  		out := v.Reg0()
  1090  
  1091  		// SWPALD	Rarg1, (Rarg0), Rout
  1092  		p := s.Prog(swap)
  1093  		p.From.Type = obj.TYPE_REG
  1094  		p.From.Reg = r1
  1095  		p.To.Type = obj.TYPE_MEM
  1096  		p.To.Reg = r0
  1097  		p.RegTo2 = out
  1098  
  1099  	case ssa.OpARM64LoweredAtomicAdd64,
  1100  		ssa.OpARM64LoweredAtomicAdd32:
  1101  		// LDAXR	(Rarg0), Rout
  1102  		// ADD		Rarg1, Rout
  1103  		// STLXR	Rout, (Rarg0), Rtmp
  1104  		// CBNZ		Rtmp, -3(PC)
  1105  		ld := arm64.ALDAXR
  1106  		st := arm64.ASTLXR
  1107  		if v.Op == ssa.OpARM64LoweredAtomicAdd32 {
  1108  			ld = arm64.ALDAXRW
  1109  			st = arm64.ASTLXRW
  1110  		}
  1111  		r0 := v.Args[0].Reg()
  1112  		r1 := v.Args[1].Reg()
  1113  		out := v.Reg0()
  1114  		p := s.Prog(ld)
  1115  		p.From.Type = obj.TYPE_MEM
  1116  		p.From.Reg = r0
  1117  		p.To.Type = obj.TYPE_REG
  1118  		p.To.Reg = out
  1119  		p1 := s.Prog(arm64.AADD)
  1120  		p1.From.Type = obj.TYPE_REG
  1121  		p1.From.Reg = r1
  1122  		p1.To.Type = obj.TYPE_REG
  1123  		p1.To.Reg = out
  1124  		p2 := s.Prog(st)
  1125  		p2.From.Type = obj.TYPE_REG
  1126  		p2.From.Reg = out
  1127  		p2.To.Type = obj.TYPE_MEM
  1128  		p2.To.Reg = r0
  1129  		p2.RegTo2 = arm64.REGTMP
  1130  		p3 := s.Prog(arm64.ACBNZ)
  1131  		p3.From.Type = obj.TYPE_REG
  1132  		p3.From.Reg = arm64.REGTMP
  1133  		p3.To.Type = obj.TYPE_BRANCH
  1134  		p3.To.SetTarget(p)
  1135  	case ssa.OpARM64LoweredAtomicAdd64Variant,
  1136  		ssa.OpARM64LoweredAtomicAdd32Variant:
  1137  		// LDADDAL	Rarg1, (Rarg0), Rout
  1138  		// ADD		Rarg1, Rout
  1139  		op := arm64.ALDADDALD
  1140  		if v.Op == ssa.OpARM64LoweredAtomicAdd32Variant {
  1141  			op = arm64.ALDADDALW
  1142  		}
  1143  		r0 := v.Args[0].Reg()
  1144  		r1 := v.Args[1].Reg()
  1145  		out := v.Reg0()
  1146  		p := s.Prog(op)
  1147  		p.From.Type = obj.TYPE_REG
  1148  		p.From.Reg = r1
  1149  		p.To.Type = obj.TYPE_MEM
  1150  		p.To.Reg = r0
  1151  		p.RegTo2 = out
  1152  		p1 := s.Prog(arm64.AADD)
  1153  		p1.From.Type = obj.TYPE_REG
  1154  		p1.From.Reg = r1
  1155  		p1.To.Type = obj.TYPE_REG
  1156  		p1.To.Reg = out
  1157  	case ssa.OpARM64LoweredAtomicCas64,
  1158  		ssa.OpARM64LoweredAtomicCas32:
  1159  		// LDAXR	(Rarg0), Rtmp
  1160  		// CMP		Rarg1, Rtmp
  1161  		// BNE		3(PC)
  1162  		// STLXR	Rarg2, (Rarg0), Rtmp
  1163  		// CBNZ		Rtmp, -4(PC)
  1164  		// CSET		EQ, Rout
  1165  		ld := arm64.ALDAXR
  1166  		st := arm64.ASTLXR
  1167  		cmp := arm64.ACMP
  1168  		if v.Op == ssa.OpARM64LoweredAtomicCas32 {
  1169  			ld = arm64.ALDAXRW
  1170  			st = arm64.ASTLXRW
  1171  			cmp = arm64.ACMPW
  1172  		}
  1173  		r0 := v.Args[0].Reg()
  1174  		r1 := v.Args[1].Reg()
  1175  		r2 := v.Args[2].Reg()
  1176  		out := v.Reg0()
  1177  		p := s.Prog(ld)
  1178  		p.From.Type = obj.TYPE_MEM
  1179  		p.From.Reg = r0
  1180  		p.To.Type = obj.TYPE_REG
  1181  		p.To.Reg = arm64.REGTMP
  1182  		p1 := s.Prog(cmp)
  1183  		p1.From.Type = obj.TYPE_REG
  1184  		p1.From.Reg = r1
  1185  		p1.Reg = arm64.REGTMP
  1186  		p2 := s.Prog(arm64.ABNE)
  1187  		p2.To.Type = obj.TYPE_BRANCH
  1188  		p3 := s.Prog(st)
  1189  		p3.From.Type = obj.TYPE_REG
  1190  		p3.From.Reg = r2
  1191  		p3.To.Type = obj.TYPE_MEM
  1192  		p3.To.Reg = r0
  1193  		p3.RegTo2 = arm64.REGTMP
  1194  		p4 := s.Prog(arm64.ACBNZ)
  1195  		p4.From.Type = obj.TYPE_REG
  1196  		p4.From.Reg = arm64.REGTMP
  1197  		p4.To.Type = obj.TYPE_BRANCH
  1198  		p4.To.SetTarget(p)
  1199  		p5 := s.Prog(arm64.ACSET)
  1200  		p5.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1201  		p5.From.Offset = int64(arm64.SPOP_EQ)
  1202  		p5.To.Type = obj.TYPE_REG
  1203  		p5.To.Reg = out
  1204  		p2.To.SetTarget(p5)
  1205  	case ssa.OpARM64LoweredAtomicCas64Variant,
  1206  		ssa.OpARM64LoweredAtomicCas32Variant:
  1207  		// Rarg0: ptr
  1208  		// Rarg1: old
  1209  		// Rarg2: new
  1210  		// MOV  	Rarg1, Rtmp
  1211  		// CASAL	Rtmp, (Rarg0), Rarg2
  1212  		// CMP  	Rarg1, Rtmp
  1213  		// CSET 	EQ, Rout
  1214  		cas := arm64.ACASALD
  1215  		cmp := arm64.ACMP
  1216  		mov := arm64.AMOVD
  1217  		if v.Op == ssa.OpARM64LoweredAtomicCas32Variant {
  1218  			cas = arm64.ACASALW
  1219  			cmp = arm64.ACMPW
  1220  			mov = arm64.AMOVW
  1221  		}
  1222  		r0 := v.Args[0].Reg()
  1223  		r1 := v.Args[1].Reg()
  1224  		r2 := v.Args[2].Reg()
  1225  		out := v.Reg0()
  1226  
  1227  		// MOV  	Rarg1, Rtmp
  1228  		p := s.Prog(mov)
  1229  		p.From.Type = obj.TYPE_REG
  1230  		p.From.Reg = r1
  1231  		p.To.Type = obj.TYPE_REG
  1232  		p.To.Reg = arm64.REGTMP
  1233  
  1234  		// CASAL	Rtmp, (Rarg0), Rarg2
  1235  		p1 := s.Prog(cas)
  1236  		p1.From.Type = obj.TYPE_REG
  1237  		p1.From.Reg = arm64.REGTMP
  1238  		p1.To.Type = obj.TYPE_MEM
  1239  		p1.To.Reg = r0
  1240  		p1.RegTo2 = r2
  1241  
  1242  		// CMP  	Rarg1, Rtmp
  1243  		p2 := s.Prog(cmp)
  1244  		p2.From.Type = obj.TYPE_REG
  1245  		p2.From.Reg = r1
  1246  		p2.Reg = arm64.REGTMP
  1247  
  1248  		// CSET 	EQ, Rout
  1249  		p3 := s.Prog(arm64.ACSET)
  1250  		p3.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1251  		p3.From.Offset = int64(arm64.SPOP_EQ)
  1252  		p3.To.Type = obj.TYPE_REG
  1253  		p3.To.Reg = out
  1254  
  1255  	case ssa.OpARM64LoweredAtomicAnd64,
  1256  		ssa.OpARM64LoweredAtomicOr64,
  1257  		ssa.OpARM64LoweredAtomicAnd32,
  1258  		ssa.OpARM64LoweredAtomicOr32,
  1259  		ssa.OpARM64LoweredAtomicAnd8,
  1260  		ssa.OpARM64LoweredAtomicOr8:
  1261  		// LDAXR[BW] (Rarg0), Rout
  1262  		// AND/OR	Rarg1, Rout, tmp1
  1263  		// STLXR[BW] tmp1, (Rarg0), Rtmp
  1264  		// CBNZ		Rtmp, -3(PC)
  1265  		ld := arm64.ALDAXR
  1266  		st := arm64.ASTLXR
  1267  		if v.Op == ssa.OpARM64LoweredAtomicAnd32 || v.Op == ssa.OpARM64LoweredAtomicOr32 {
  1268  			ld = arm64.ALDAXRW
  1269  			st = arm64.ASTLXRW
  1270  		}
  1271  		if v.Op == ssa.OpARM64LoweredAtomicAnd8 || v.Op == ssa.OpARM64LoweredAtomicOr8 {
  1272  			ld = arm64.ALDAXRB
  1273  			st = arm64.ASTLXRB
  1274  		}
  1275  		r0 := v.Args[0].Reg()
  1276  		r1 := v.Args[1].Reg()
  1277  		out := v.Reg0()
  1278  		tmp := v.RegTmp()
  1279  		p := s.Prog(ld)
  1280  		p.From.Type = obj.TYPE_MEM
  1281  		p.From.Reg = r0
  1282  		p.To.Type = obj.TYPE_REG
  1283  		p.To.Reg = out
  1284  		p1 := s.Prog(v.Op.Asm())
  1285  		p1.From.Type = obj.TYPE_REG
  1286  		p1.From.Reg = r1
  1287  		p1.Reg = out
  1288  		p1.To.Type = obj.TYPE_REG
  1289  		p1.To.Reg = tmp
  1290  		p2 := s.Prog(st)
  1291  		p2.From.Type = obj.TYPE_REG
  1292  		p2.From.Reg = tmp
  1293  		p2.To.Type = obj.TYPE_MEM
  1294  		p2.To.Reg = r0
  1295  		p2.RegTo2 = arm64.REGTMP
  1296  		p3 := s.Prog(arm64.ACBNZ)
  1297  		p3.From.Type = obj.TYPE_REG
  1298  		p3.From.Reg = arm64.REGTMP
  1299  		p3.To.Type = obj.TYPE_BRANCH
  1300  		p3.To.SetTarget(p)
  1301  
  1302  	case ssa.OpARM64LoweredAtomicAnd8Variant,
  1303  		ssa.OpARM64LoweredAtomicAnd32Variant,
  1304  		ssa.OpARM64LoweredAtomicAnd64Variant:
  1305  		atomic_clear := arm64.ALDCLRALD
  1306  		if v.Op == ssa.OpARM64LoweredAtomicAnd32Variant {
  1307  			atomic_clear = arm64.ALDCLRALW
  1308  		}
  1309  		if v.Op == ssa.OpARM64LoweredAtomicAnd8Variant {
  1310  			atomic_clear = arm64.ALDCLRALB
  1311  		}
  1312  		r0 := v.Args[0].Reg()
  1313  		r1 := v.Args[1].Reg()
  1314  		out := v.Reg0()
  1315  
  1316  		// MNV       Rarg1 Rtemp
  1317  		p := s.Prog(arm64.AMVN)
  1318  		p.From.Type = obj.TYPE_REG
  1319  		p.From.Reg = r1
  1320  		p.To.Type = obj.TYPE_REG
  1321  		p.To.Reg = arm64.REGTMP
  1322  
  1323  		// LDCLRAL[BDW]  Rtemp, (Rarg0), Rout
  1324  		p1 := s.Prog(atomic_clear)
  1325  		p1.From.Type = obj.TYPE_REG
  1326  		p1.From.Reg = arm64.REGTMP
  1327  		p1.To.Type = obj.TYPE_MEM
  1328  		p1.To.Reg = r0
  1329  		p1.RegTo2 = out
  1330  
  1331  	case ssa.OpARM64LoweredAtomicOr8Variant,
  1332  		ssa.OpARM64LoweredAtomicOr32Variant,
  1333  		ssa.OpARM64LoweredAtomicOr64Variant:
  1334  		atomic_or := arm64.ALDORALD
  1335  		if v.Op == ssa.OpARM64LoweredAtomicOr32Variant {
  1336  			atomic_or = arm64.ALDORALW
  1337  		}
  1338  		if v.Op == ssa.OpARM64LoweredAtomicOr8Variant {
  1339  			atomic_or = arm64.ALDORALB
  1340  		}
  1341  		r0 := v.Args[0].Reg()
  1342  		r1 := v.Args[1].Reg()
  1343  		out := v.Reg0()
  1344  
  1345  		// LDORAL[BDW]  Rarg1, (Rarg0), Rout
  1346  		p := s.Prog(atomic_or)
  1347  		p.From.Type = obj.TYPE_REG
  1348  		p.From.Reg = r1
  1349  		p.To.Type = obj.TYPE_MEM
  1350  		p.To.Reg = r0
  1351  		p.RegTo2 = out
  1352  
  1353  	case ssa.OpARM64MOVBreg,
  1354  		ssa.OpARM64MOVBUreg,
  1355  		ssa.OpARM64MOVHreg,
  1356  		ssa.OpARM64MOVHUreg,
  1357  		ssa.OpARM64MOVWreg,
  1358  		ssa.OpARM64MOVWUreg:
  1359  		a := v.Args[0]
  1360  		for a.Op == ssa.OpCopy || a.Op == ssa.OpARM64MOVDreg {
  1361  			a = a.Args[0]
  1362  		}
  1363  		if a.Op == ssa.OpLoadReg {
  1364  			t := a.Type
  1365  			switch {
  1366  			case v.Op == ssa.OpARM64MOVBreg && t.Size() == 1 && t.IsSigned(),
  1367  				v.Op == ssa.OpARM64MOVBUreg && t.Size() == 1 && !t.IsSigned(),
  1368  				v.Op == ssa.OpARM64MOVHreg && t.Size() == 2 && t.IsSigned(),
  1369  				v.Op == ssa.OpARM64MOVHUreg && t.Size() == 2 && !t.IsSigned(),
  1370  				v.Op == ssa.OpARM64MOVWreg && t.Size() == 4 && t.IsSigned(),
  1371  				v.Op == ssa.OpARM64MOVWUreg && t.Size() == 4 && !t.IsSigned():
  1372  				// arg is a proper-typed load, already zero/sign-extended, don't extend again
  1373  				if v.Reg() == v.Args[0].Reg() {
  1374  					return
  1375  				}
  1376  				p := s.Prog(arm64.AMOVD)
  1377  				p.From.Type = obj.TYPE_REG
  1378  				p.From.Reg = v.Args[0].Reg()
  1379  				p.To.Type = obj.TYPE_REG
  1380  				p.To.Reg = v.Reg()
  1381  				return
  1382  			default:
  1383  			}
  1384  		}
  1385  		fallthrough
  1386  	case ssa.OpARM64MVN,
  1387  		ssa.OpARM64NEG,
  1388  		ssa.OpARM64FABSD,
  1389  		ssa.OpARM64FABSS,
  1390  		ssa.OpARM64FMOVDfpgp,
  1391  		ssa.OpARM64FMOVDgpfp,
  1392  		ssa.OpARM64FMOVSfpgp,
  1393  		ssa.OpARM64FMOVSgpfp,
  1394  		ssa.OpARM64FNEGS,
  1395  		ssa.OpARM64FNEGD,
  1396  		ssa.OpARM64FSQRTS,
  1397  		ssa.OpARM64FSQRTD,
  1398  		ssa.OpARM64FCVTZSSW,
  1399  		ssa.OpARM64FCVTZSDW,
  1400  		ssa.OpARM64FCVTZUSW,
  1401  		ssa.OpARM64FCVTZUDW,
  1402  		ssa.OpARM64FCVTZSS,
  1403  		ssa.OpARM64FCVTZSD,
  1404  		ssa.OpARM64FCVTZUS,
  1405  		ssa.OpARM64FCVTZUD,
  1406  		ssa.OpARM64SCVTFWS,
  1407  		ssa.OpARM64SCVTFWD,
  1408  		ssa.OpARM64SCVTFS,
  1409  		ssa.OpARM64SCVTFD,
  1410  		ssa.OpARM64UCVTFWS,
  1411  		ssa.OpARM64UCVTFWD,
  1412  		ssa.OpARM64UCVTFS,
  1413  		ssa.OpARM64UCVTFD,
  1414  		ssa.OpARM64FCVTSD,
  1415  		ssa.OpARM64FCVTDS,
  1416  		ssa.OpARM64REV,
  1417  		ssa.OpARM64REVW,
  1418  		ssa.OpARM64REV16,
  1419  		ssa.OpARM64REV16W,
  1420  		ssa.OpARM64RBIT,
  1421  		ssa.OpARM64RBITW,
  1422  		ssa.OpARM64CLZ,
  1423  		ssa.OpARM64CLZW,
  1424  		ssa.OpARM64FRINTAD,
  1425  		ssa.OpARM64FRINTMD,
  1426  		ssa.OpARM64FRINTND,
  1427  		ssa.OpARM64FRINTPD,
  1428  		ssa.OpARM64FRINTZD,
  1429  		ssa.OpARM64FRINTAS,
  1430  		ssa.OpARM64FRINTMS,
  1431  		ssa.OpARM64FRINTNS,
  1432  		ssa.OpARM64FRINTPS,
  1433  		ssa.OpARM64FRINTZS:
  1434  		p := s.Prog(v.Op.Asm())
  1435  		p.From.Type = obj.TYPE_REG
  1436  		p.From.Reg = v.Args[0].Reg()
  1437  		p.To.Type = obj.TYPE_REG
  1438  		p.To.Reg = v.Reg()
  1439  	case ssa.OpARM64LoweredRound32F, ssa.OpARM64LoweredRound64F:
  1440  		// input is already rounded
  1441  	case ssa.OpARM64VCNT:
  1442  		simdV11(s, v, arm64.ARNG_8B)
  1443  	case ssa.OpARM64VUADDLV:
  1444  		simdV11Scalar(s, v, arm64.ARNG_8B)
  1445  	case ssa.OpARM64CSEL, ssa.OpARM64CSEL0:
  1446  		r1 := int16(arm64.REGZERO)
  1447  		if v.Op != ssa.OpARM64CSEL0 {
  1448  			r1 = v.Args[1].Reg()
  1449  		}
  1450  		p := s.Prog(v.Op.Asm())
  1451  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1452  		condCode := condBits[ssa.Op(v.AuxInt)]
  1453  		p.From.Offset = int64(condCode)
  1454  		p.Reg = v.Args[0].Reg()
  1455  		p.AddRestSourceReg(r1)
  1456  		p.To.Type = obj.TYPE_REG
  1457  		p.To.Reg = v.Reg()
  1458  	case ssa.OpARM64CSINC, ssa.OpARM64CSINV, ssa.OpARM64CSNEG:
  1459  		p := s.Prog(v.Op.Asm())
  1460  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1461  		condCode := condBits[ssa.Op(v.AuxInt)]
  1462  		p.From.Offset = int64(condCode)
  1463  		p.Reg = v.Args[0].Reg()
  1464  		p.AddRestSourceReg(v.Args[1].Reg())
  1465  		p.To.Type = obj.TYPE_REG
  1466  		p.To.Reg = v.Reg()
  1467  	case ssa.OpARM64CSETM:
  1468  		p := s.Prog(arm64.ACSETM)
  1469  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1470  		condCode := condBits[ssa.Op(v.AuxInt)]
  1471  		p.From.Offset = int64(condCode)
  1472  		p.To.Type = obj.TYPE_REG
  1473  		p.To.Reg = v.Reg()
  1474  	case ssa.OpARM64CCMP,
  1475  		ssa.OpARM64CCMN,
  1476  		ssa.OpARM64CCMPconst,
  1477  		ssa.OpARM64CCMNconst,
  1478  		ssa.OpARM64CCMPW,
  1479  		ssa.OpARM64CCMNW,
  1480  		ssa.OpARM64CCMPWconst,
  1481  		ssa.OpARM64CCMNWconst:
  1482  		p := s.Prog(v.Op.Asm())
  1483  		p.Reg = v.Args[0].Reg()
  1484  		params := v.AuxArm64ConditionalParams()
  1485  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1486  		p.From.Offset = int64(condBits[params.Cond()])
  1487  		constValue, ok := params.ConstValue()
  1488  		if ok {
  1489  			p.AddRestSourceConst(constValue)
  1490  		} else {
  1491  			p.AddRestSourceReg(v.Args[1].Reg())
  1492  		}
  1493  		p.To.Type = obj.TYPE_CONST
  1494  		p.To.Offset = params.Nzcv()
  1495  	case ssa.OpARM64LoweredZero:
  1496  		ptrReg := v.Args[0].Reg()
  1497  		n := v.AuxInt
  1498  		if n < 16 {
  1499  			v.Fatalf("Zero too small %d", n)
  1500  		}
  1501  
  1502  		// Generate zeroing instructions.
  1503  		var off int64
  1504  		for n >= 16 {
  1505  			//  STP     (ZR, ZR), off(ptrReg)
  1506  			zero16(s, ptrReg, off, false)
  1507  			off += 16
  1508  			n -= 16
  1509  		}
  1510  		// Write any fractional portion.
  1511  		// An overlapping 16-byte write can't be used here
  1512  		// because STP's offsets must be a multiple of 8.
  1513  		if n > 8 {
  1514  			//  MOVD    ZR, off(ptrReg)
  1515  			zero8(s, ptrReg, off)
  1516  			off += 8
  1517  			n -= 8
  1518  		}
  1519  		if n != 0 {
  1520  			//  MOVD    ZR, off+n-8(ptrReg)
  1521  			// TODO: for n<=4 we could use a smaller write.
  1522  			zero8(s, ptrReg, off+n-8)
  1523  		}
  1524  	case ssa.OpARM64LoweredZeroLoop:
  1525  		ptrReg := v.Args[0].Reg()
  1526  		countReg := v.RegTmp()
  1527  		n := v.AuxInt
  1528  		loopSize := int64(64)
  1529  		if n < 3*loopSize {
  1530  			// - a loop count of 0 won't work.
  1531  			// - a loop count of 1 is useless.
  1532  			// - a loop count of 2 is a code size ~tie
  1533  			//     3 instructions to implement the loop
  1534  			//     4 instructions in the loop body
  1535  			//   vs
  1536  			//     8 instructions in the straightline code
  1537  			//   Might as well use straightline code.
  1538  			v.Fatalf("ZeroLoop size too small %d", n)
  1539  		}
  1540  
  1541  		// Put iteration count in a register.
  1542  		//   MOVD    $n, countReg
  1543  		p := s.Prog(arm64.AMOVD)
  1544  		p.From.Type = obj.TYPE_CONST
  1545  		p.From.Offset = n / loopSize
  1546  		p.To.Type = obj.TYPE_REG
  1547  		p.To.Reg = countReg
  1548  		cntInit := p
  1549  
  1550  		// Zero loopSize bytes starting at ptrReg.
  1551  		// Increment ptrReg by loopSize as a side effect.
  1552  		for range loopSize / 16 {
  1553  			//  STP.P   (ZR, ZR), 16(ptrReg)
  1554  			zero16(s, ptrReg, 0, true)
  1555  			// TODO: should we use the postincrement form,
  1556  			// or use a separate += 64 instruction?
  1557  			// postincrement saves an instruction, but maybe
  1558  			// it requires more integer units to do the +=16s.
  1559  		}
  1560  		// Decrement loop count.
  1561  		//   SUB     $1, countReg
  1562  		p = s.Prog(arm64.ASUB)
  1563  		p.From.Type = obj.TYPE_CONST
  1564  		p.From.Offset = 1
  1565  		p.To.Type = obj.TYPE_REG
  1566  		p.To.Reg = countReg
  1567  		// Jump to loop header if we're not done yet.
  1568  		//   CBNZ    head
  1569  		p = s.Prog(arm64.ACBNZ)
  1570  		p.From.Type = obj.TYPE_REG
  1571  		p.From.Reg = countReg
  1572  		p.To.Type = obj.TYPE_BRANCH
  1573  		p.To.SetTarget(cntInit.Link)
  1574  
  1575  		// Multiples of the loop size are now done.
  1576  		n %= loopSize
  1577  
  1578  		// Write any fractional portion.
  1579  		var off int64
  1580  		for n >= 16 {
  1581  			//  STP     (ZR, ZR), off(ptrReg)
  1582  			zero16(s, ptrReg, off, false)
  1583  			off += 16
  1584  			n -= 16
  1585  		}
  1586  		if n > 8 {
  1587  			// Note: an overlapping 16-byte write can't be used
  1588  			// here because STP's offsets must be a multiple of 8.
  1589  			//  MOVD    ZR, off(ptrReg)
  1590  			zero8(s, ptrReg, off)
  1591  			off += 8
  1592  			n -= 8
  1593  		}
  1594  		if n != 0 {
  1595  			//  MOVD    ZR, off+n-8(ptrReg)
  1596  			// TODO: for n<=4 we could use a smaller write.
  1597  			zero8(s, ptrReg, off+n-8)
  1598  		}
  1599  		// TODO: maybe we should use the count register to instead
  1600  		// hold an end pointer and compare against that?
  1601  		//   ADD $n, ptrReg, endReg
  1602  		// then
  1603  		//   CMP ptrReg, endReg
  1604  		//   BNE loop
  1605  		// There's a past-the-end pointer here, any problem with that?
  1606  
  1607  	case ssa.OpARM64LoweredMove:
  1608  		dstReg := v.Args[0].Reg()
  1609  		srcReg := v.Args[1].Reg()
  1610  		if dstReg == srcReg {
  1611  			break
  1612  		}
  1613  		tmpReg1 := int16(arm64.REG_R25)
  1614  		tmpFReg1 := int16(arm64.REG_F16)
  1615  		tmpFReg2 := int16(arm64.REG_F17)
  1616  		n := v.AuxInt
  1617  		if n < 16 {
  1618  			v.Fatalf("Move too small %d", n)
  1619  		}
  1620  
  1621  		// Generate copying instructions.
  1622  		var off int64
  1623  		for n >= 32 {
  1624  			//  FLDPQ   off(srcReg), (tmpFReg1, tmpFReg2)
  1625  			//  FSTPQ   (tmpFReg1, tmpFReg2), off(dstReg)
  1626  			move32(s, srcReg, dstReg, tmpFReg1, tmpFReg2, off, false)
  1627  			off += 32
  1628  			n -= 32
  1629  		}
  1630  		for n >= 16 {
  1631  			//  FMOVQ   off(src), tmpFReg1
  1632  			//  FMOVQ   tmpFReg1, off(dst)
  1633  			move16(s, srcReg, dstReg, tmpFReg1, off, false)
  1634  			off += 16
  1635  			n -= 16
  1636  		}
  1637  		if n > 8 {
  1638  			//  MOVD    off(srcReg), tmpReg1
  1639  			//  MOVD    tmpReg1, off(dstReg)
  1640  			move8(s, srcReg, dstReg, tmpReg1, off)
  1641  			off += 8
  1642  			n -= 8
  1643  		}
  1644  		if n != 0 {
  1645  			//  MOVD    off+n-8(srcReg), tmpReg1
  1646  			//  MOVD    tmpReg1, off+n-8(dstReg)
  1647  			move8(s, srcReg, dstReg, tmpReg1, off+n-8)
  1648  		}
  1649  	case ssa.OpARM64LoweredMoveLoop:
  1650  		dstReg := v.Args[0].Reg()
  1651  		srcReg := v.Args[1].Reg()
  1652  		if dstReg == srcReg {
  1653  			break
  1654  		}
  1655  		countReg := int16(arm64.REG_R24)
  1656  		tmpReg1 := int16(arm64.REG_R25)
  1657  		tmpFReg1 := int16(arm64.REG_F16)
  1658  		tmpFReg2 := int16(arm64.REG_F17)
  1659  		n := v.AuxInt
  1660  		loopSize := int64(64)
  1661  		if n < 3*loopSize {
  1662  			// - a loop count of 0 won't work.
  1663  			// - a loop count of 1 is useless.
  1664  			// - a loop count of 2 is a code size ~tie
  1665  			//     3 instructions to implement the loop
  1666  			//     4 instructions in the loop body
  1667  			//   vs
  1668  			//     8 instructions in the straightline code
  1669  			//   Might as well use straightline code.
  1670  			v.Fatalf("ZeroLoop size too small %d", n)
  1671  		}
  1672  
  1673  		// Put iteration count in a register.
  1674  		//   MOVD    $n, countReg
  1675  		p := s.Prog(arm64.AMOVD)
  1676  		p.From.Type = obj.TYPE_CONST
  1677  		p.From.Offset = n / loopSize
  1678  		p.To.Type = obj.TYPE_REG
  1679  		p.To.Reg = countReg
  1680  		cntInit := p
  1681  
  1682  		// Move loopSize bytes starting at srcReg to dstReg.
  1683  		// Increment srcReg and destReg by loopSize as a side effect.
  1684  		for range loopSize / 32 {
  1685  			// FLDPQ.P 32(srcReg), (tmpFReg1, tmpFReg2)
  1686  			// FSTPQ.P (tmpFReg1, tmpFReg2), 32(dstReg)
  1687  			move32(s, srcReg, dstReg, tmpFReg1, tmpFReg2, 0, true)
  1688  		}
  1689  		// Decrement loop count.
  1690  		//   SUB     $1, countReg
  1691  		p = s.Prog(arm64.ASUB)
  1692  		p.From.Type = obj.TYPE_CONST
  1693  		p.From.Offset = 1
  1694  		p.To.Type = obj.TYPE_REG
  1695  		p.To.Reg = countReg
  1696  		// Jump to loop header if we're not done yet.
  1697  		//   CBNZ    head
  1698  		p = s.Prog(arm64.ACBNZ)
  1699  		p.From.Type = obj.TYPE_REG
  1700  		p.From.Reg = countReg
  1701  		p.To.Type = obj.TYPE_BRANCH
  1702  		p.To.SetTarget(cntInit.Link)
  1703  
  1704  		// Multiples of the loop size are now done.
  1705  		n %= loopSize
  1706  
  1707  		// Copy any fractional portion.
  1708  		var off int64
  1709  		for n >= 32 {
  1710  			//  FLDPQ   off(srcReg), (tmpFReg1, tmpFReg2)
  1711  			//  FSTPQ   (tmpFReg1, tmpFReg2), off(dstReg)
  1712  			move32(s, srcReg, dstReg, tmpFReg1, tmpFReg2, off, false)
  1713  			off += 32
  1714  			n -= 32
  1715  		}
  1716  		for n >= 16 {
  1717  			//  FMOVQ   off(src), tmpFReg1
  1718  			//  FMOVQ   tmpFReg1, off(dst)
  1719  			move16(s, srcReg, dstReg, tmpFReg1, off, false)
  1720  			off += 16
  1721  			n -= 16
  1722  		}
  1723  		if n > 8 {
  1724  			//  MOVD    off(srcReg), tmpReg1
  1725  			//  MOVD    tmpReg1, off(dstReg)
  1726  			move8(s, srcReg, dstReg, tmpReg1, off)
  1727  			off += 8
  1728  			n -= 8
  1729  		}
  1730  		if n != 0 {
  1731  			//  MOVD    off+n-8(srcReg), tmpReg1
  1732  			//  MOVD    tmpReg1, off+n-8(dstReg)
  1733  			move8(s, srcReg, dstReg, tmpReg1, off+n-8)
  1734  		}
  1735  
  1736  	case ssa.OpARM64CALLstatic, ssa.OpARM64CALLclosure, ssa.OpARM64CALLinter:
  1737  		s.Call(v)
  1738  	case ssa.OpARM64CALLtail, ssa.OpARM64CALLtailinter:
  1739  		s.TailCall(v)
  1740  	case ssa.OpARM64LoweredWB:
  1741  		p := s.Prog(obj.ACALL)
  1742  		p.To.Type = obj.TYPE_MEM
  1743  		p.To.Name = obj.NAME_EXTERN
  1744  		// AuxInt encodes how many buffer entries we need.
  1745  		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
  1746  	case ssa.OpARM64LoweredMemEq:
  1747  		p := s.Prog(obj.ACALL)
  1748  		p.To.Type = obj.TYPE_MEM
  1749  		p.To.Name = obj.NAME_EXTERN
  1750  		p.To.Sym = ir.Syms.Memequal
  1751  
  1752  	case ssa.OpARM64LoweredPanicBoundsRR, ssa.OpARM64LoweredPanicBoundsRC, ssa.OpARM64LoweredPanicBoundsCR, ssa.OpARM64LoweredPanicBoundsCC:
  1753  		// Compute the constant we put in the PCData entry for this call.
  1754  		code, signed := ssa.BoundsKind(v.AuxInt).Code()
  1755  		xIsReg := false
  1756  		yIsReg := false
  1757  		xVal := 0
  1758  		yVal := 0
  1759  		switch v.Op {
  1760  		case ssa.OpARM64LoweredPanicBoundsRR:
  1761  			xIsReg = true
  1762  			xVal = int(v.Args[0].Reg() - arm64.REG_R0)
  1763  			yIsReg = true
  1764  			yVal = int(v.Args[1].Reg() - arm64.REG_R0)
  1765  		case ssa.OpARM64LoweredPanicBoundsRC:
  1766  			xIsReg = true
  1767  			xVal = int(v.Args[0].Reg() - arm64.REG_R0)
  1768  			c := v.Aux.(ssa.PanicBoundsC).C
  1769  			if c >= 0 && c <= abi.BoundsMaxConst {
  1770  				yVal = int(c)
  1771  			} else {
  1772  				// Move constant to a register
  1773  				yIsReg = true
  1774  				if yVal == xVal {
  1775  					yVal = 1
  1776  				}
  1777  				p := s.Prog(arm64.AMOVD)
  1778  				p.From.Type = obj.TYPE_CONST
  1779  				p.From.Offset = c
  1780  				p.To.Type = obj.TYPE_REG
  1781  				p.To.Reg = arm64.REG_R0 + int16(yVal)
  1782  			}
  1783  		case ssa.OpARM64LoweredPanicBoundsCR:
  1784  			yIsReg = true
  1785  			yVal = int(v.Args[0].Reg() - arm64.REG_R0)
  1786  			c := v.Aux.(ssa.PanicBoundsC).C
  1787  			if c >= 0 && c <= abi.BoundsMaxConst {
  1788  				xVal = int(c)
  1789  			} else {
  1790  				// Move constant to a register
  1791  				if xVal == yVal {
  1792  					xVal = 1
  1793  				}
  1794  				p := s.Prog(arm64.AMOVD)
  1795  				p.From.Type = obj.TYPE_CONST
  1796  				p.From.Offset = c
  1797  				p.To.Type = obj.TYPE_REG
  1798  				p.To.Reg = arm64.REG_R0 + int16(xVal)
  1799  			}
  1800  		case ssa.OpARM64LoweredPanicBoundsCC:
  1801  			c := v.Aux.(ssa.PanicBoundsCC).Cx
  1802  			if c >= 0 && c <= abi.BoundsMaxConst {
  1803  				xVal = int(c)
  1804  			} else {
  1805  				// Move constant to a register
  1806  				xIsReg = true
  1807  				p := s.Prog(arm64.AMOVD)
  1808  				p.From.Type = obj.TYPE_CONST
  1809  				p.From.Offset = c
  1810  				p.To.Type = obj.TYPE_REG
  1811  				p.To.Reg = arm64.REG_R0 + int16(xVal)
  1812  			}
  1813  			c = v.Aux.(ssa.PanicBoundsCC).Cy
  1814  			if c >= 0 && c <= abi.BoundsMaxConst {
  1815  				yVal = int(c)
  1816  			} else {
  1817  				// Move constant to a register
  1818  				yIsReg = true
  1819  				yVal = 1
  1820  				p := s.Prog(arm64.AMOVD)
  1821  				p.From.Type = obj.TYPE_CONST
  1822  				p.From.Offset = c
  1823  				p.To.Type = obj.TYPE_REG
  1824  				p.To.Reg = arm64.REG_R0 + int16(yVal)
  1825  			}
  1826  		}
  1827  		c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
  1828  
  1829  		p := s.Prog(obj.APCDATA)
  1830  		p.From.SetConst(abi.PCDATA_PanicBounds)
  1831  		p.To.SetConst(int64(c))
  1832  		p = s.Prog(obj.ACALL)
  1833  		p.To.Type = obj.TYPE_MEM
  1834  		p.To.Name = obj.NAME_EXTERN
  1835  		p.To.Sym = ir.Syms.PanicBounds
  1836  
  1837  	case ssa.OpARM64LoweredNilCheck:
  1838  		// Issue a load which will fault if arg is nil.
  1839  		p := s.Prog(arm64.AMOVB)
  1840  		p.From.Type = obj.TYPE_MEM
  1841  		p.From.Reg = v.Args[0].Reg()
  1842  		ssagen.AddAux(&p.From, v)
  1843  		p.To.Type = obj.TYPE_REG
  1844  		p.To.Reg = arm64.REGTMP
  1845  		if logopt.Enabled() {
  1846  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  1847  		}
  1848  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Line==1 in generated wrappers
  1849  			base.WarnfAt(v.Pos, "generated nil check")
  1850  		}
  1851  	case ssa.OpARM64Equal,
  1852  		ssa.OpARM64NotEqual,
  1853  		ssa.OpARM64LessThan,
  1854  		ssa.OpARM64LessEqual,
  1855  		ssa.OpARM64GreaterThan,
  1856  		ssa.OpARM64GreaterEqual,
  1857  		ssa.OpARM64LessThanU,
  1858  		ssa.OpARM64LessEqualU,
  1859  		ssa.OpARM64GreaterThanU,
  1860  		ssa.OpARM64GreaterEqualU,
  1861  		ssa.OpARM64LessThanF,
  1862  		ssa.OpARM64LessEqualF,
  1863  		ssa.OpARM64GreaterThanF,
  1864  		ssa.OpARM64GreaterEqualF,
  1865  		ssa.OpARM64NotLessThanF,
  1866  		ssa.OpARM64NotLessEqualF,
  1867  		ssa.OpARM64NotGreaterThanF,
  1868  		ssa.OpARM64NotGreaterEqualF,
  1869  		ssa.OpARM64LessThanNoov,
  1870  		ssa.OpARM64GreaterEqualNoov:
  1871  		// generate boolean values using CSET
  1872  		p := s.Prog(arm64.ACSET)
  1873  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1874  		condCode := condBits[v.Op]
  1875  		p.From.Offset = int64(condCode)
  1876  		p.To.Type = obj.TYPE_REG
  1877  		p.To.Reg = v.Reg()
  1878  	case ssa.OpARM64PRFM:
  1879  		p := s.Prog(v.Op.Asm())
  1880  		p.From.Type = obj.TYPE_MEM
  1881  		p.From.Reg = v.Args[0].Reg()
  1882  		p.To.Type = obj.TYPE_CONST
  1883  		p.To.Offset = v.AuxInt
  1884  	case ssa.OpARM64LoweredGetClosurePtr:
  1885  		// Closure pointer is R26 (arm64.REGCTXT).
  1886  		ssagen.CheckLoweredGetClosurePtr(v)
  1887  	case ssa.OpARM64LoweredGetCallerSP:
  1888  		// caller's SP is FixedFrameSize below the address of the first arg
  1889  		p := s.Prog(arm64.AMOVD)
  1890  		p.From.Type = obj.TYPE_ADDR
  1891  		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize
  1892  		p.From.Name = obj.NAME_PARAM
  1893  		p.To.Type = obj.TYPE_REG
  1894  		p.To.Reg = v.Reg()
  1895  	case ssa.OpARM64LoweredGetCallerPC:
  1896  		p := s.Prog(obj.AGETCALLERPC)
  1897  		p.To.Type = obj.TYPE_REG
  1898  		p.To.Reg = v.Reg()
  1899  	case ssa.OpARM64DMB:
  1900  		p := s.Prog(v.Op.Asm())
  1901  		p.From.Type = obj.TYPE_CONST
  1902  		p.From.Offset = v.AuxInt
  1903  	case ssa.OpARM64FlagConstant:
  1904  		v.Fatalf("FlagConstant op should never make it to codegen %v", v.LongString())
  1905  	case ssa.OpARM64InvertFlags:
  1906  		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
  1907  	case ssa.OpClobber:
  1908  		// MOVW	$0xdeaddead, REGTMP
  1909  		// MOVW	REGTMP, (slot)
  1910  		// MOVW	REGTMP, 4(slot)
  1911  		p := s.Prog(arm64.AMOVW)
  1912  		p.From.Type = obj.TYPE_CONST
  1913  		p.From.Offset = 0xdeaddead
  1914  		p.To.Type = obj.TYPE_REG
  1915  		p.To.Reg = arm64.REGTMP
  1916  		p = s.Prog(arm64.AMOVW)
  1917  		p.From.Type = obj.TYPE_REG
  1918  		p.From.Reg = arm64.REGTMP
  1919  		p.To.Type = obj.TYPE_MEM
  1920  		p.To.Reg = arm64.REGSP
  1921  		ssagen.AddAux(&p.To, v)
  1922  		p = s.Prog(arm64.AMOVW)
  1923  		p.From.Type = obj.TYPE_REG
  1924  		p.From.Reg = arm64.REGTMP
  1925  		p.To.Type = obj.TYPE_MEM
  1926  		p.To.Reg = arm64.REGSP
  1927  		ssagen.AddAux2(&p.To, v, v.AuxInt+4)
  1928  	case ssa.OpClobberReg:
  1929  		x := uint64(0xdeaddeaddeaddead)
  1930  		p := s.Prog(arm64.AMOVD)
  1931  		p.From.Type = obj.TYPE_CONST
  1932  		p.From.Offset = int64(x)
  1933  		p.To.Type = obj.TYPE_REG
  1934  		p.To.Reg = v.Reg()
  1935  	default:
  1936  		if !ssaGenSIMDValue(s, v) {
  1937  			v.Fatalf("genValue not implemented: %s", v.LongString())
  1938  		}
  1939  	}
  1940  }
  1941  
  1942  var condBits = map[ssa.Op]arm64.SpecialOperand{
  1943  	ssa.OpARM64Equal:         arm64.SPOP_EQ,
  1944  	ssa.OpARM64NotEqual:      arm64.SPOP_NE,
  1945  	ssa.OpARM64LessThan:      arm64.SPOP_LT,
  1946  	ssa.OpARM64LessThanU:     arm64.SPOP_LO,
  1947  	ssa.OpARM64LessEqual:     arm64.SPOP_LE,
  1948  	ssa.OpARM64LessEqualU:    arm64.SPOP_LS,
  1949  	ssa.OpARM64GreaterThan:   arm64.SPOP_GT,
  1950  	ssa.OpARM64GreaterThanU:  arm64.SPOP_HI,
  1951  	ssa.OpARM64GreaterEqual:  arm64.SPOP_GE,
  1952  	ssa.OpARM64GreaterEqualU: arm64.SPOP_HS,
  1953  	ssa.OpARM64LessThanF:     arm64.SPOP_MI, // Less than
  1954  	ssa.OpARM64LessEqualF:    arm64.SPOP_LS, // Less than or equal to
  1955  	ssa.OpARM64GreaterThanF:  arm64.SPOP_GT, // Greater than
  1956  	ssa.OpARM64GreaterEqualF: arm64.SPOP_GE, // Greater than or equal to
  1957  
  1958  	// The following condition codes have unordered to handle comparisons related to NaN.
  1959  	ssa.OpARM64NotLessThanF:     arm64.SPOP_PL, // Greater than, equal to, or unordered
  1960  	ssa.OpARM64NotLessEqualF:    arm64.SPOP_HI, // Greater than or unordered
  1961  	ssa.OpARM64NotGreaterThanF:  arm64.SPOP_LE, // Less than, equal to or unordered
  1962  	ssa.OpARM64NotGreaterEqualF: arm64.SPOP_LT, // Less than or unordered
  1963  
  1964  	ssa.OpARM64LessThanNoov:     arm64.SPOP_MI, // Less than but without honoring overflow
  1965  	ssa.OpARM64GreaterEqualNoov: arm64.SPOP_PL, // Greater than or equal to but without honoring overflow
  1966  }
  1967  
  1968  var blockJump = map[ssa.BlockKind]struct {
  1969  	asm, invasm obj.As
  1970  }{
  1971  	ssa.BlockARM64EQ:     {arm64.ABEQ, arm64.ABNE},
  1972  	ssa.BlockARM64NE:     {arm64.ABNE, arm64.ABEQ},
  1973  	ssa.BlockARM64LT:     {arm64.ABLT, arm64.ABGE},
  1974  	ssa.BlockARM64GE:     {arm64.ABGE, arm64.ABLT},
  1975  	ssa.BlockARM64LE:     {arm64.ABLE, arm64.ABGT},
  1976  	ssa.BlockARM64GT:     {arm64.ABGT, arm64.ABLE},
  1977  	ssa.BlockARM64ULT:    {arm64.ABLO, arm64.ABHS},
  1978  	ssa.BlockARM64UGE:    {arm64.ABHS, arm64.ABLO},
  1979  	ssa.BlockARM64UGT:    {arm64.ABHI, arm64.ABLS},
  1980  	ssa.BlockARM64ULE:    {arm64.ABLS, arm64.ABHI},
  1981  	ssa.BlockARM64Z:      {arm64.ACBZ, arm64.ACBNZ},
  1982  	ssa.BlockARM64NZ:     {arm64.ACBNZ, arm64.ACBZ},
  1983  	ssa.BlockARM64ZW:     {arm64.ACBZW, arm64.ACBNZW},
  1984  	ssa.BlockARM64NZW:    {arm64.ACBNZW, arm64.ACBZW},
  1985  	ssa.BlockARM64TBZ:    {arm64.ATBZ, arm64.ATBNZ},
  1986  	ssa.BlockARM64TBNZ:   {arm64.ATBNZ, arm64.ATBZ},
  1987  	ssa.BlockARM64FLT:    {arm64.ABMI, arm64.ABPL},
  1988  	ssa.BlockARM64FGE:    {arm64.ABGE, arm64.ABLT},
  1989  	ssa.BlockARM64FLE:    {arm64.ABLS, arm64.ABHI},
  1990  	ssa.BlockARM64FGT:    {arm64.ABGT, arm64.ABLE},
  1991  	ssa.BlockARM64LTnoov: {arm64.ABMI, arm64.ABPL},
  1992  	ssa.BlockARM64GEnoov: {arm64.ABPL, arm64.ABMI},
  1993  }
  1994  
  1995  // To model a 'LEnoov' ('<=' without overflow checking) branching.
  1996  var leJumps = [2][2]ssagen.IndexJump{
  1997  	{{Jump: arm64.ABEQ, Index: 0}, {Jump: arm64.ABPL, Index: 1}}, // next == b.Succs[0]
  1998  	{{Jump: arm64.ABMI, Index: 0}, {Jump: arm64.ABEQ, Index: 0}}, // next == b.Succs[1]
  1999  }
  2000  
  2001  // To model a 'GTnoov' ('>' without overflow checking) branching.
  2002  var gtJumps = [2][2]ssagen.IndexJump{
  2003  	{{Jump: arm64.ABMI, Index: 1}, {Jump: arm64.ABEQ, Index: 1}}, // next == b.Succs[0]
  2004  	{{Jump: arm64.ABEQ, Index: 1}, {Jump: arm64.ABPL, Index: 0}}, // next == b.Succs[1]
  2005  }
  2006  
  2007  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  2008  	switch b.Kind {
  2009  	case ssa.BlockPlain, ssa.BlockDefer:
  2010  		if b.Succs[0].Block() != next {
  2011  			p := s.Prog(obj.AJMP)
  2012  			p.To.Type = obj.TYPE_BRANCH
  2013  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  2014  		}
  2015  
  2016  	case ssa.BlockExit, ssa.BlockRetJmp:
  2017  
  2018  	case ssa.BlockRet:
  2019  		s.Prog(obj.ARET)
  2020  
  2021  	case ssa.BlockARM64EQ, ssa.BlockARM64NE,
  2022  		ssa.BlockARM64LT, ssa.BlockARM64GE,
  2023  		ssa.BlockARM64LE, ssa.BlockARM64GT,
  2024  		ssa.BlockARM64ULT, ssa.BlockARM64UGT,
  2025  		ssa.BlockARM64ULE, ssa.BlockARM64UGE,
  2026  		ssa.BlockARM64Z, ssa.BlockARM64NZ,
  2027  		ssa.BlockARM64ZW, ssa.BlockARM64NZW,
  2028  		ssa.BlockARM64FLT, ssa.BlockARM64FGE,
  2029  		ssa.BlockARM64FLE, ssa.BlockARM64FGT,
  2030  		ssa.BlockARM64LTnoov, ssa.BlockARM64GEnoov:
  2031  		jmp := blockJump[b.Kind]
  2032  		var p *obj.Prog
  2033  		switch next {
  2034  		case b.Succs[0].Block():
  2035  			p = s.Br(jmp.invasm, b.Succs[1].Block())
  2036  		case b.Succs[1].Block():
  2037  			p = s.Br(jmp.asm, b.Succs[0].Block())
  2038  		default:
  2039  			if b.Likely != ssa.BranchUnlikely {
  2040  				p = s.Br(jmp.asm, b.Succs[0].Block())
  2041  				s.Br(obj.AJMP, b.Succs[1].Block())
  2042  			} else {
  2043  				p = s.Br(jmp.invasm, b.Succs[1].Block())
  2044  				s.Br(obj.AJMP, b.Succs[0].Block())
  2045  			}
  2046  		}
  2047  		if !b.Controls[0].Type.IsFlags() {
  2048  			p.From.Type = obj.TYPE_REG
  2049  			p.From.Reg = b.Controls[0].Reg()
  2050  		}
  2051  	case ssa.BlockARM64TBZ, ssa.BlockARM64TBNZ:
  2052  		jmp := blockJump[b.Kind]
  2053  		var p *obj.Prog
  2054  		switch next {
  2055  		case b.Succs[0].Block():
  2056  			p = s.Br(jmp.invasm, b.Succs[1].Block())
  2057  		case b.Succs[1].Block():
  2058  			p = s.Br(jmp.asm, b.Succs[0].Block())
  2059  		default:
  2060  			if b.Likely != ssa.BranchUnlikely {
  2061  				p = s.Br(jmp.asm, b.Succs[0].Block())
  2062  				s.Br(obj.AJMP, b.Succs[1].Block())
  2063  			} else {
  2064  				p = s.Br(jmp.invasm, b.Succs[1].Block())
  2065  				s.Br(obj.AJMP, b.Succs[0].Block())
  2066  			}
  2067  		}
  2068  		p.From.Offset = b.AuxInt
  2069  		p.From.Type = obj.TYPE_CONST
  2070  		p.Reg = b.Controls[0].Reg()
  2071  
  2072  	case ssa.BlockARM64LEnoov:
  2073  		s.CombJump(b, next, &leJumps)
  2074  	case ssa.BlockARM64GTnoov:
  2075  		s.CombJump(b, next, &gtJumps)
  2076  
  2077  	case ssa.BlockARM64JUMPTABLE:
  2078  		// MOVD	(TABLE)(IDX<<3), Rtmp
  2079  		// JMP	(Rtmp)
  2080  		p := s.Prog(arm64.AMOVD)
  2081  		p.From = genIndexedOperand(ssa.OpARM64MOVDloadidx8, b.Controls[1].Reg(), b.Controls[0].Reg())
  2082  		p.To.Type = obj.TYPE_REG
  2083  		p.To.Reg = arm64.REGTMP
  2084  		p = s.Prog(obj.AJMP)
  2085  		p.To.Type = obj.TYPE_MEM
  2086  		p.To.Reg = arm64.REGTMP
  2087  		// Save jump tables for later resolution of the target blocks.
  2088  		s.JumpTables = append(s.JumpTables, b)
  2089  
  2090  	default:
  2091  		b.Fatalf("branch not implemented: %s", b.LongString())
  2092  	}
  2093  }
  2094  
  2095  func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2096  	p := s.Prog(loadByType(t))
  2097  	p.From.Type = obj.TYPE_MEM
  2098  	p.From.Name = obj.NAME_AUTO
  2099  	p.From.Sym = n.Linksym()
  2100  	p.From.Offset = n.FrameOffset() + off
  2101  	p.To.Type = obj.TYPE_REG
  2102  	p.To.Reg = reg
  2103  	return p
  2104  }
  2105  
  2106  func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2107  	p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
  2108  	p.To.Name = obj.NAME_PARAM
  2109  	p.To.Sym = n.Linksym()
  2110  	p.Pos = p.Pos.WithNotStmt()
  2111  	return p
  2112  }
  2113  
  2114  // zero16 zeroes 16 bytes at reg+off.
  2115  // If postInc is true, increment reg by 16.
  2116  func zero16(s *ssagen.State, reg int16, off int64, postInc bool) {
  2117  	//   STP     (ZR, ZR), off(reg)
  2118  	p := s.Prog(arm64.ASTP)
  2119  	p.From.Type = obj.TYPE_REGREG
  2120  	p.From.Reg = arm64.REGZERO
  2121  	p.From.Offset = int64(arm64.REGZERO)
  2122  	p.To.Type = obj.TYPE_MEM
  2123  	p.To.Reg = reg
  2124  	p.To.Offset = off
  2125  	if postInc {
  2126  		if off != 0 {
  2127  			panic("can't postinc with non-zero offset")
  2128  		}
  2129  		//   STP.P  (ZR, ZR), 16(reg)
  2130  		p.Scond = arm64.C_XPOST
  2131  		p.To.Offset = 16
  2132  	}
  2133  }
  2134  
  2135  // zero8 zeroes 8 bytes at reg+off.
  2136  func zero8(s *ssagen.State, reg int16, off int64) {
  2137  	//   MOVD     ZR, off(reg)
  2138  	p := s.Prog(arm64.AMOVD)
  2139  	p.From.Type = obj.TYPE_REG
  2140  	p.From.Reg = arm64.REGZERO
  2141  	p.To.Type = obj.TYPE_MEM
  2142  	p.To.Reg = reg
  2143  	p.To.Offset = off
  2144  }
  2145  
  2146  // move32 copies 32 bytes at src+off to dst+off.
  2147  // Uses registers tmp1 and tmp2.
  2148  // If postInc is true, increment src and dst by 32.
  2149  func move32(s *ssagen.State, src, dst, tmp1, tmp2 int16, off int64, postInc bool) {
  2150  	// FLDPQ   off(src), (tmp1, tmp2)
  2151  	ld := s.Prog(arm64.AFLDPQ)
  2152  	ld.From.Type = obj.TYPE_MEM
  2153  	ld.From.Reg = src
  2154  	ld.From.Offset = off
  2155  	ld.To.Type = obj.TYPE_REGREG
  2156  	ld.To.Reg = tmp1
  2157  	ld.To.Offset = int64(tmp2)
  2158  	// FSTPQ   (tmp1, tmp2), off(dst)
  2159  	st := s.Prog(arm64.AFSTPQ)
  2160  	st.From.Type = obj.TYPE_REGREG
  2161  	st.From.Reg = tmp1
  2162  	st.From.Offset = int64(tmp2)
  2163  	st.To.Type = obj.TYPE_MEM
  2164  	st.To.Reg = dst
  2165  	st.To.Offset = off
  2166  	if postInc {
  2167  		if off != 0 {
  2168  			panic("can't postinc with non-zero offset")
  2169  		}
  2170  		ld.Scond = arm64.C_XPOST
  2171  		st.Scond = arm64.C_XPOST
  2172  		ld.From.Offset = 32
  2173  		st.To.Offset = 32
  2174  	}
  2175  }
  2176  
  2177  // move16 copies 16 bytes at src+off to dst+off.
  2178  // Uses register tmp1
  2179  // If postInc is true, increment src and dst by 16.
  2180  func move16(s *ssagen.State, src, dst, tmp1 int16, off int64, postInc bool) {
  2181  	// FMOVQ     off(src), tmp1
  2182  	ld := s.Prog(arm64.AFMOVQ)
  2183  	ld.From.Type = obj.TYPE_MEM
  2184  	ld.From.Reg = src
  2185  	ld.From.Offset = off
  2186  	ld.To.Type = obj.TYPE_REG
  2187  	ld.To.Reg = tmp1
  2188  	// FMOVQ     tmp1, off(dst)
  2189  	st := s.Prog(arm64.AFMOVQ)
  2190  	st.From.Type = obj.TYPE_REG
  2191  	st.From.Reg = tmp1
  2192  	st.To.Type = obj.TYPE_MEM
  2193  	st.To.Reg = dst
  2194  	st.To.Offset = off
  2195  	if postInc {
  2196  		if off != 0 {
  2197  			panic("can't postinc with non-zero offset")
  2198  		}
  2199  		ld.Scond = arm64.C_XPOST
  2200  		st.Scond = arm64.C_XPOST
  2201  		ld.From.Offset = 16
  2202  		st.To.Offset = 16
  2203  	}
  2204  }
  2205  
  2206  // move8 copies 8 bytes at src+off to dst+off.
  2207  // Uses register tmp.
  2208  func move8(s *ssagen.State, src, dst, tmp int16, off int64) {
  2209  	// MOVD    off(src), tmp
  2210  	ld := s.Prog(arm64.AMOVD)
  2211  	ld.From.Type = obj.TYPE_MEM
  2212  	ld.From.Reg = src
  2213  	ld.From.Offset = off
  2214  	ld.To.Type = obj.TYPE_REG
  2215  	ld.To.Reg = tmp
  2216  	// MOVD    tmp, off(dst)
  2217  	st := s.Prog(arm64.AMOVD)
  2218  	st.From.Type = obj.TYPE_REG
  2219  	st.From.Reg = tmp
  2220  	st.To.Type = obj.TYPE_MEM
  2221  	st.To.Reg = dst
  2222  	st.To.Offset = off
  2223  }
  2224  

View as plain text