Source file src/cmd/compile/internal/amd64/ssa.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package amd64
     6  
     7  import (
     8  	"fmt"
     9  	"math"
    10  
    11  	"cmd/compile/internal/base"
    12  	"cmd/compile/internal/ir"
    13  	"cmd/compile/internal/logopt"
    14  	"cmd/compile/internal/objw"
    15  	"cmd/compile/internal/ssa"
    16  	"cmd/compile/internal/ssagen"
    17  	"cmd/compile/internal/types"
    18  	"cmd/internal/obj"
    19  	"cmd/internal/obj/x86"
    20  	"internal/abi"
    21  )
    22  
    23  // ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags.
    24  func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {
    25  	flive := b.FlagsLiveAtEnd
    26  	for _, c := range b.ControlValues() {
    27  		flive = c.Type.IsFlags() || flive
    28  	}
    29  	for i := len(b.Values) - 1; i >= 0; i-- {
    30  		v := b.Values[i]
    31  		if flive && (v.Op == ssa.OpAMD64MOVLconst || v.Op == ssa.OpAMD64MOVQconst) {
    32  			// The "mark" is any non-nil Aux value.
    33  			v.Aux = ssa.AuxMark
    34  		}
    35  		if v.Type.IsFlags() {
    36  			flive = false
    37  		}
    38  		for _, a := range v.Args {
    39  			if a.Type.IsFlags() {
    40  				flive = true
    41  			}
    42  		}
    43  	}
    44  }
    45  
    46  func isGPReg(r int16) bool {
    47  	return x86.REG_AL <= r && r <= x86.REG_R15
    48  }
    49  
    50  func isFPReg(r int16) bool {
    51  	return x86.REG_X0 <= r && r <= x86.REG_Z31
    52  }
    53  
    54  func isKReg(r int16) bool {
    55  	return x86.REG_K0 <= r && r <= x86.REG_K7
    56  }
    57  
    58  func isLowFPReg(r int16) bool {
    59  	return x86.REG_X0 <= r && r <= x86.REG_X15
    60  }
    61  
    62  func isHighFPReg(r int16) bool {
    63  	return x86.REG_X16 <= r && r <= x86.REG_X31 || x86.REG_Y16 <= r && r <= x86.REG_Y31 || x86.REG_Z16 <= r && r <= x86.REG_Z31
    64  }
    65  
    66  // loadByRegWidth returns the load instruction of the given register of a given width.
    67  func loadByRegWidth(r int16, width int64) obj.As {
    68  	// Avoid partial register write for GPR
    69  	if !isFPReg(r) && !isKReg(r) {
    70  		switch width {
    71  		case 1:
    72  			return x86.AMOVBLZX
    73  		case 2:
    74  			return x86.AMOVWLZX
    75  		}
    76  	}
    77  	// Otherwise, there's no difference between load and store opcodes.
    78  	return storeByRegWidth(r, width)
    79  }
    80  
    81  // storeByRegWidth returns the store instruction of the given register of a given width.
    82  // It's also used for loading const to a reg.
    83  func storeByRegWidth(r int16, width int64) obj.As {
    84  	if isHighFPReg(r) {
    85  		// High registers require AVX512 instruction
    86  		return x86.AVMOVDQU64
    87  	}
    88  	if isFPReg(r) {
    89  		switch width {
    90  		case 4:
    91  			return x86.AMOVSS
    92  		case 8:
    93  			return x86.AMOVSD
    94  		case 16:
    95  			// int128s are in SSE registers
    96  			return x86.AMOVUPS
    97  		case 32:
    98  			return x86.AVMOVDQU
    99  		case 64:
   100  			return x86.AVMOVDQU64
   101  		}
   102  	}
   103  	if isKReg(r) {
   104  		return x86.AKMOVQ
   105  	}
   106  	// gp
   107  	switch width {
   108  	case 1:
   109  		return x86.AMOVB
   110  	case 2:
   111  		return x86.AMOVW
   112  	case 4:
   113  		return x86.AMOVL
   114  	case 8:
   115  		return x86.AMOVQ
   116  	}
   117  	panic(fmt.Sprintf("bad store reg=%v, width=%d", r, width))
   118  }
   119  
   120  // moveByRegsWidth returns the reg->reg move instruction of the given dest/src registers of a given width.
   121  func moveByRegsWidth(dest, src int16, width int64) obj.As {
   122  	// fp -> fp
   123  	if isFPReg(dest) && isFPReg(src) {
   124  		if isHighFPReg(src) || isHighFPReg(dest) {
   125  			// High registers require AVX512 instruction
   126  			return x86.AVMOVDQU64
   127  		}
   128  		// Moving the whole sse2 register is faster
   129  		// than moving just the correct low portion of it.
   130  		// There is no xmm->xmm move with 1 byte opcode,
   131  		// so use movups, which has 2 byte opcode.
   132  		if width <= 16 {
   133  			return x86.AMOVUPS
   134  		}
   135  		if width <= 32 {
   136  			return x86.AVMOVDQU
   137  		}
   138  		return x86.AVMOVDQU64
   139  	}
   140  	// k -> gp, gp -> k, k -> k
   141  	if isKReg(dest) || isKReg(src) {
   142  		if isFPReg(dest) || isFPReg(src) {
   143  			panic(fmt.Sprintf("bad move, src=%v, dest=%v, width=%d", src, dest, width))
   144  		}
   145  		return x86.AKMOVQ
   146  	}
   147  	// gp -> fp, fp -> gp, gp -> gp
   148  	switch width {
   149  	case 1:
   150  		// Avoids partial register write
   151  		return x86.AMOVL
   152  	case 2:
   153  		return x86.AMOVL
   154  	case 4:
   155  		return x86.AMOVL
   156  	case 8:
   157  		return x86.AMOVQ
   158  	case 16:
   159  		// int128s are in SSE registers
   160  		return x86.AMOVUPS
   161  	case 32:
   162  		return x86.AVMOVDQU
   163  	case 64:
   164  		return x86.AVMOVDQU64
   165  	}
   166  	panic(fmt.Sprintf("bad move, src=%v, dest=%v, width=%d", src, dest, width))
   167  }
   168  
   169  // opregreg emits instructions for
   170  //
   171  //	dest := dest(To) op src(From)
   172  //
   173  // and also returns the created obj.Prog so it
   174  // may be further adjusted (offset, scale, etc).
   175  func opregreg(s *ssagen.State, op obj.As, dest, src int16) *obj.Prog {
   176  	p := s.Prog(op)
   177  	p.From.Type = obj.TYPE_REG
   178  	p.To.Type = obj.TYPE_REG
   179  	p.To.Reg = dest
   180  	p.From.Reg = src
   181  	return p
   182  }
   183  
   184  // memIdx fills out a as an indexed memory reference for v.
   185  // It assumes that the base register and the index register
   186  // are v.Args[0].Reg() and v.Args[1].Reg(), respectively.
   187  // The caller must still use gc.AddAux/gc.AddAux2 to handle v.Aux as necessary.
   188  func memIdx(a *obj.Addr, v *ssa.Value) {
   189  	r, i := v.Args[0].Reg(), v.Args[1].Reg()
   190  	a.Type = obj.TYPE_MEM
   191  	a.Scale = v.Op.Scale()
   192  	if a.Scale == 1 && i == x86.REG_SP {
   193  		r, i = i, r
   194  	}
   195  	a.Reg = r
   196  	a.Index = i
   197  }
   198  
   199  func getgFromTLS(s *ssagen.State, r int16) {
   200  	// See the comments in cmd/internal/obj/x86/obj6.go
   201  	// near CanUse1InsnTLS for a detailed explanation of these instructions.
   202  	if x86.CanUse1InsnTLS(base.Ctxt) {
   203  		// MOVQ (TLS), r
   204  		p := s.Prog(x86.AMOVQ)
   205  		p.From.Type = obj.TYPE_MEM
   206  		p.From.Reg = x86.REG_TLS
   207  		p.To.Type = obj.TYPE_REG
   208  		p.To.Reg = r
   209  	} else {
   210  		// MOVQ TLS, r
   211  		// MOVQ (r)(TLS*1), r
   212  		p := s.Prog(x86.AMOVQ)
   213  		p.From.Type = obj.TYPE_REG
   214  		p.From.Reg = x86.REG_TLS
   215  		p.To.Type = obj.TYPE_REG
   216  		p.To.Reg = r
   217  		q := s.Prog(x86.AMOVQ)
   218  		q.From.Type = obj.TYPE_MEM
   219  		q.From.Reg = r
   220  		q.From.Index = x86.REG_TLS
   221  		q.From.Scale = 1
   222  		q.To.Type = obj.TYPE_REG
   223  		q.To.Reg = r
   224  	}
   225  }
   226  
   227  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   228  	switch v.Op {
   229  	case ssa.OpAMD64VFMADD231SD, ssa.OpAMD64VFMADD231SS:
   230  		p := s.Prog(v.Op.Asm())
   231  		p.From = obj.Addr{Type: obj.TYPE_REG, Reg: v.Args[2].Reg()}
   232  		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.Reg()}
   233  		p.AddRestSourceReg(v.Args[1].Reg())
   234  	case ssa.OpAMD64ADDQ, ssa.OpAMD64ADDL:
   235  		r := v.Reg()
   236  		r1 := v.Args[0].Reg()
   237  		r2 := v.Args[1].Reg()
   238  		switch {
   239  		case r == r1:
   240  			p := s.Prog(v.Op.Asm())
   241  			p.From.Type = obj.TYPE_REG
   242  			p.From.Reg = r2
   243  			p.To.Type = obj.TYPE_REG
   244  			p.To.Reg = r
   245  		case r == r2:
   246  			p := s.Prog(v.Op.Asm())
   247  			p.From.Type = obj.TYPE_REG
   248  			p.From.Reg = r1
   249  			p.To.Type = obj.TYPE_REG
   250  			p.To.Reg = r
   251  		default:
   252  			var asm obj.As
   253  			if v.Op == ssa.OpAMD64ADDQ {
   254  				asm = x86.ALEAQ
   255  			} else {
   256  				asm = x86.ALEAL
   257  			}
   258  			p := s.Prog(asm)
   259  			p.From.Type = obj.TYPE_MEM
   260  			p.From.Reg = r1
   261  			p.From.Scale = 1
   262  			p.From.Index = r2
   263  			p.To.Type = obj.TYPE_REG
   264  			p.To.Reg = r
   265  		}
   266  	// 2-address opcode arithmetic
   267  	case ssa.OpAMD64SUBQ, ssa.OpAMD64SUBL,
   268  		ssa.OpAMD64MULQ, ssa.OpAMD64MULL,
   269  		ssa.OpAMD64ANDQ, ssa.OpAMD64ANDL,
   270  		ssa.OpAMD64ORQ, ssa.OpAMD64ORL,
   271  		ssa.OpAMD64XORQ, ssa.OpAMD64XORL,
   272  		ssa.OpAMD64SHLQ, ssa.OpAMD64SHLL,
   273  		ssa.OpAMD64SHRQ, ssa.OpAMD64SHRL, ssa.OpAMD64SHRW, ssa.OpAMD64SHRB,
   274  		ssa.OpAMD64SARQ, ssa.OpAMD64SARL, ssa.OpAMD64SARW, ssa.OpAMD64SARB,
   275  		ssa.OpAMD64ROLQ, ssa.OpAMD64ROLL, ssa.OpAMD64ROLW, ssa.OpAMD64ROLB,
   276  		ssa.OpAMD64RORQ, ssa.OpAMD64RORL, ssa.OpAMD64RORW, ssa.OpAMD64RORB,
   277  		ssa.OpAMD64ADDSS, ssa.OpAMD64ADDSD, ssa.OpAMD64SUBSS, ssa.OpAMD64SUBSD,
   278  		ssa.OpAMD64MULSS, ssa.OpAMD64MULSD, ssa.OpAMD64DIVSS, ssa.OpAMD64DIVSD,
   279  		ssa.OpAMD64MINSS, ssa.OpAMD64MINSD,
   280  		ssa.OpAMD64POR, ssa.OpAMD64PXOR,
   281  		ssa.OpAMD64BTSL, ssa.OpAMD64BTSQ,
   282  		ssa.OpAMD64BTCL, ssa.OpAMD64BTCQ,
   283  		ssa.OpAMD64BTRL, ssa.OpAMD64BTRQ,
   284  		ssa.OpAMD64PCMPEQB, ssa.OpAMD64PSIGNB,
   285  		ssa.OpAMD64PUNPCKLBW:
   286  		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   287  
   288  	case ssa.OpAMD64PSHUFLW:
   289  		p := s.Prog(v.Op.Asm())
   290  		imm := v.AuxInt
   291  		if imm < 0 || imm > 255 {
   292  			v.Fatalf("Invalid source selection immediate")
   293  		}
   294  		p.From.Offset = imm
   295  		p.From.Type = obj.TYPE_CONST
   296  		p.AddRestSourceReg(v.Args[0].Reg())
   297  		p.To.Type = obj.TYPE_REG
   298  		p.To.Reg = v.Reg()
   299  
   300  	case ssa.OpAMD64PSHUFBbroadcast:
   301  		// PSHUFB with a control mask of zero copies byte 0 to all
   302  		// bytes in the register.
   303  		//
   304  		// X15 is always zero with ABIInternal.
   305  		if s.ABI != obj.ABIInternal {
   306  			// zero X15 manually
   307  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
   308  		}
   309  
   310  		p := s.Prog(v.Op.Asm())
   311  		p.From.Type = obj.TYPE_REG
   312  		p.To.Type = obj.TYPE_REG
   313  		p.To.Reg = v.Reg()
   314  		p.From.Reg = x86.REG_X15
   315  
   316  	case ssa.OpAMD64BLSIQ, ssa.OpAMD64BLSIL,
   317  		ssa.OpAMD64BLSMSKQ, ssa.OpAMD64BLSMSKL,
   318  		ssa.OpAMD64BLSRQ, ssa.OpAMD64BLSRL:
   319  		p := s.Prog(v.Op.Asm())
   320  		p.From.Type = obj.TYPE_REG
   321  		p.From.Reg = v.Args[0].Reg()
   322  		p.To.Type = obj.TYPE_REG
   323  		switch v.Op {
   324  		case ssa.OpAMD64BLSRQ, ssa.OpAMD64BLSRL:
   325  			p.To.Reg = v.Reg0()
   326  		default:
   327  			p.To.Reg = v.Reg()
   328  		}
   329  
   330  	case ssa.OpAMD64ANDNQ, ssa.OpAMD64ANDNL:
   331  		p := s.Prog(v.Op.Asm())
   332  		p.From.Type = obj.TYPE_REG
   333  		p.From.Reg = v.Args[0].Reg()
   334  		p.To.Type = obj.TYPE_REG
   335  		p.To.Reg = v.Reg()
   336  		p.AddRestSourceReg(v.Args[1].Reg())
   337  
   338  	case ssa.OpAMD64SARXL, ssa.OpAMD64SARXQ,
   339  		ssa.OpAMD64SHLXL, ssa.OpAMD64SHLXQ,
   340  		ssa.OpAMD64SHRXL, ssa.OpAMD64SHRXQ:
   341  		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   342  		p.AddRestSourceReg(v.Args[0].Reg())
   343  
   344  	case ssa.OpAMD64SHLXLload, ssa.OpAMD64SHLXQload,
   345  		ssa.OpAMD64SHRXLload, ssa.OpAMD64SHRXQload,
   346  		ssa.OpAMD64SARXLload, ssa.OpAMD64SARXQload:
   347  		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   348  		m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
   349  		ssagen.AddAux(&m, v)
   350  		p.AddRestSource(m)
   351  
   352  	case ssa.OpAMD64SHLXLloadidx1, ssa.OpAMD64SHLXLloadidx4, ssa.OpAMD64SHLXLloadidx8,
   353  		ssa.OpAMD64SHRXLloadidx1, ssa.OpAMD64SHRXLloadidx4, ssa.OpAMD64SHRXLloadidx8,
   354  		ssa.OpAMD64SARXLloadidx1, ssa.OpAMD64SARXLloadidx4, ssa.OpAMD64SARXLloadidx8,
   355  		ssa.OpAMD64SHLXQloadidx1, ssa.OpAMD64SHLXQloadidx8,
   356  		ssa.OpAMD64SHRXQloadidx1, ssa.OpAMD64SHRXQloadidx8,
   357  		ssa.OpAMD64SARXQloadidx1, ssa.OpAMD64SARXQloadidx8:
   358  		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[2].Reg())
   359  		m := obj.Addr{Type: obj.TYPE_MEM}
   360  		memIdx(&m, v)
   361  		ssagen.AddAux(&m, v)
   362  		p.AddRestSource(m)
   363  
   364  	case ssa.OpAMD64DIVQU, ssa.OpAMD64DIVLU, ssa.OpAMD64DIVWU:
   365  		// Arg[0] (the dividend) is in AX.
   366  		// Arg[1] (the divisor) can be in any other register.
   367  		// Result[0] (the quotient) is in AX.
   368  		// Result[1] (the remainder) is in DX.
   369  		r := v.Args[1].Reg()
   370  
   371  		// Zero extend dividend.
   372  		opregreg(s, x86.AXORL, x86.REG_DX, x86.REG_DX)
   373  
   374  		// Issue divide.
   375  		p := s.Prog(v.Op.Asm())
   376  		p.From.Type = obj.TYPE_REG
   377  		p.From.Reg = r
   378  
   379  	case ssa.OpAMD64DIVQ, ssa.OpAMD64DIVL, ssa.OpAMD64DIVW:
   380  		// Arg[0] (the dividend) is in AX.
   381  		// Arg[1] (the divisor) can be in any other register.
   382  		// Result[0] (the quotient) is in AX.
   383  		// Result[1] (the remainder) is in DX.
   384  		r := v.Args[1].Reg()
   385  
   386  		var opCMP, opNEG, opSXD obj.As
   387  		switch v.Op {
   388  		case ssa.OpAMD64DIVQ:
   389  			opCMP, opNEG, opSXD = x86.ACMPQ, x86.ANEGQ, x86.ACQO
   390  		case ssa.OpAMD64DIVL:
   391  			opCMP, opNEG, opSXD = x86.ACMPL, x86.ANEGL, x86.ACDQ
   392  		case ssa.OpAMD64DIVW:
   393  			opCMP, opNEG, opSXD = x86.ACMPW, x86.ANEGW, x86.ACWD
   394  		}
   395  
   396  		// CPU faults upon signed overflow, which occurs when the most
   397  		// negative int is divided by -1. Handle divide by -1 as a special case.
   398  		var j1, j2 *obj.Prog
   399  		if ssa.DivisionNeedsFixUp(v) {
   400  			c := s.Prog(opCMP)
   401  			c.From.Type = obj.TYPE_REG
   402  			c.From.Reg = r
   403  			c.To.Type = obj.TYPE_CONST
   404  			c.To.Offset = -1
   405  
   406  			// Divisor is not -1, proceed with normal division.
   407  			j1 = s.Prog(x86.AJNE)
   408  			j1.To.Type = obj.TYPE_BRANCH
   409  
   410  			// Divisor is -1, manually compute quotient and remainder via fixup code.
   411  			// n / -1 = -n
   412  			n1 := s.Prog(opNEG)
   413  			n1.To.Type = obj.TYPE_REG
   414  			n1.To.Reg = x86.REG_AX
   415  
   416  			// n % -1 == 0
   417  			opregreg(s, x86.AXORL, x86.REG_DX, x86.REG_DX)
   418  
   419  			// TODO(khr): issue only the -1 fixup code we need.
   420  			// For instance, if only the quotient is used, no point in zeroing the remainder.
   421  
   422  			// Skip over normal division.
   423  			j2 = s.Prog(obj.AJMP)
   424  			j2.To.Type = obj.TYPE_BRANCH
   425  		}
   426  
   427  		// Sign extend dividend and perform division.
   428  		p := s.Prog(opSXD)
   429  		if j1 != nil {
   430  			j1.To.SetTarget(p)
   431  		}
   432  		p = s.Prog(v.Op.Asm())
   433  		p.From.Type = obj.TYPE_REG
   434  		p.From.Reg = r
   435  
   436  		if j2 != nil {
   437  			j2.To.SetTarget(s.Pc())
   438  		}
   439  
   440  	case ssa.OpAMD64HMULQ, ssa.OpAMD64HMULL, ssa.OpAMD64HMULQU, ssa.OpAMD64HMULLU:
   441  		// the frontend rewrites constant division by 8/16/32 bit integers into
   442  		// HMUL by a constant
   443  		// SSA rewrites generate the 64 bit versions
   444  
   445  		// Arg[0] is already in AX as it's the only register we allow
   446  		// and DX is the only output we care about (the high bits)
   447  		p := s.Prog(v.Op.Asm())
   448  		p.From.Type = obj.TYPE_REG
   449  		p.From.Reg = v.Args[1].Reg()
   450  
   451  		// IMULB puts the high portion in AH instead of DL,
   452  		// so move it to DL for consistency
   453  		if v.Type.Size() == 1 {
   454  			m := s.Prog(x86.AMOVB)
   455  			m.From.Type = obj.TYPE_REG
   456  			m.From.Reg = x86.REG_AH
   457  			m.To.Type = obj.TYPE_REG
   458  			m.To.Reg = x86.REG_DX
   459  		}
   460  
   461  	case ssa.OpAMD64MULQU, ssa.OpAMD64MULLU:
   462  		// Arg[0] is already in AX as it's the only register we allow
   463  		// results lo in AX
   464  		p := s.Prog(v.Op.Asm())
   465  		p.From.Type = obj.TYPE_REG
   466  		p.From.Reg = v.Args[1].Reg()
   467  
   468  	case ssa.OpAMD64MULQU2:
   469  		// Arg[0] is already in AX as it's the only register we allow
   470  		// results hi in DX, lo in AX
   471  		p := s.Prog(v.Op.Asm())
   472  		p.From.Type = obj.TYPE_REG
   473  		p.From.Reg = v.Args[1].Reg()
   474  
   475  	case ssa.OpAMD64DIVQU2:
   476  		// Arg[0], Arg[1] are already in Dx, AX, as they're the only registers we allow
   477  		// results q in AX, r in DX
   478  		p := s.Prog(v.Op.Asm())
   479  		p.From.Type = obj.TYPE_REG
   480  		p.From.Reg = v.Args[2].Reg()
   481  
   482  	case ssa.OpAMD64AVGQU:
   483  		// compute (x+y)/2 unsigned.
   484  		// Do a 64-bit add, the overflow goes into the carry.
   485  		// Shift right once and pull the carry back into the 63rd bit.
   486  		p := s.Prog(x86.AADDQ)
   487  		p.From.Type = obj.TYPE_REG
   488  		p.To.Type = obj.TYPE_REG
   489  		p.To.Reg = v.Reg()
   490  		p.From.Reg = v.Args[1].Reg()
   491  		p = s.Prog(x86.ARCRQ)
   492  		p.From.Type = obj.TYPE_CONST
   493  		p.From.Offset = 1
   494  		p.To.Type = obj.TYPE_REG
   495  		p.To.Reg = v.Reg()
   496  
   497  	case ssa.OpAMD64ADDQcarry, ssa.OpAMD64ADCQ:
   498  		r := v.Reg0()
   499  		r0 := v.Args[0].Reg()
   500  		r1 := v.Args[1].Reg()
   501  		switch r {
   502  		case r0:
   503  			p := s.Prog(v.Op.Asm())
   504  			p.From.Type = obj.TYPE_REG
   505  			p.From.Reg = r1
   506  			p.To.Type = obj.TYPE_REG
   507  			p.To.Reg = r
   508  		case r1:
   509  			p := s.Prog(v.Op.Asm())
   510  			p.From.Type = obj.TYPE_REG
   511  			p.From.Reg = r0
   512  			p.To.Type = obj.TYPE_REG
   513  			p.To.Reg = r
   514  		default:
   515  			v.Fatalf("output not in same register as an input %s", v.LongString())
   516  		}
   517  
   518  	case ssa.OpAMD64SUBQborrow, ssa.OpAMD64SBBQ:
   519  		p := s.Prog(v.Op.Asm())
   520  		p.From.Type = obj.TYPE_REG
   521  		p.From.Reg = v.Args[1].Reg()
   522  		p.To.Type = obj.TYPE_REG
   523  		p.To.Reg = v.Reg0()
   524  
   525  	case ssa.OpAMD64ADDQconstcarry, ssa.OpAMD64ADCQconst, ssa.OpAMD64SUBQconstborrow, ssa.OpAMD64SBBQconst:
   526  		p := s.Prog(v.Op.Asm())
   527  		p.From.Type = obj.TYPE_CONST
   528  		p.From.Offset = v.AuxInt
   529  		p.To.Type = obj.TYPE_REG
   530  		p.To.Reg = v.Reg0()
   531  
   532  	case ssa.OpAMD64ADDQconst, ssa.OpAMD64ADDLconst:
   533  		r := v.Reg()
   534  		a := v.Args[0].Reg()
   535  		if r == a {
   536  			switch v.AuxInt {
   537  			case 1:
   538  				var asm obj.As
   539  				// Software optimization manual recommends add $1,reg.
   540  				// But inc/dec is 1 byte smaller. ICC always uses inc
   541  				// Clang/GCC choose depending on flags, but prefer add.
   542  				// Experiments show that inc/dec is both a little faster
   543  				// and make a binary a little smaller.
   544  				if v.Op == ssa.OpAMD64ADDQconst {
   545  					asm = x86.AINCQ
   546  				} else {
   547  					asm = x86.AINCL
   548  				}
   549  				p := s.Prog(asm)
   550  				p.To.Type = obj.TYPE_REG
   551  				p.To.Reg = r
   552  				return
   553  			case -1:
   554  				var asm obj.As
   555  				if v.Op == ssa.OpAMD64ADDQconst {
   556  					asm = x86.ADECQ
   557  				} else {
   558  					asm = x86.ADECL
   559  				}
   560  				p := s.Prog(asm)
   561  				p.To.Type = obj.TYPE_REG
   562  				p.To.Reg = r
   563  				return
   564  			case 0x80:
   565  				// 'SUBQ $-0x80, r' is shorter to encode than
   566  				// and functionally equivalent to 'ADDQ $0x80, r'.
   567  				asm := x86.ASUBL
   568  				if v.Op == ssa.OpAMD64ADDQconst {
   569  					asm = x86.ASUBQ
   570  				}
   571  				p := s.Prog(asm)
   572  				p.From.Type = obj.TYPE_CONST
   573  				p.From.Offset = -0x80
   574  				p.To.Type = obj.TYPE_REG
   575  				p.To.Reg = r
   576  				return
   577  
   578  			}
   579  			p := s.Prog(v.Op.Asm())
   580  			p.From.Type = obj.TYPE_CONST
   581  			p.From.Offset = v.AuxInt
   582  			p.To.Type = obj.TYPE_REG
   583  			p.To.Reg = r
   584  			return
   585  		}
   586  		var asm obj.As
   587  		if v.Op == ssa.OpAMD64ADDQconst {
   588  			asm = x86.ALEAQ
   589  		} else {
   590  			asm = x86.ALEAL
   591  		}
   592  		p := s.Prog(asm)
   593  		p.From.Type = obj.TYPE_MEM
   594  		p.From.Reg = a
   595  		p.From.Offset = v.AuxInt
   596  		p.To.Type = obj.TYPE_REG
   597  		p.To.Reg = r
   598  
   599  	case ssa.OpAMD64CMOVQEQ, ssa.OpAMD64CMOVLEQ, ssa.OpAMD64CMOVWEQ,
   600  		ssa.OpAMD64CMOVQLT, ssa.OpAMD64CMOVLLT, ssa.OpAMD64CMOVWLT,
   601  		ssa.OpAMD64CMOVQNE, ssa.OpAMD64CMOVLNE, ssa.OpAMD64CMOVWNE,
   602  		ssa.OpAMD64CMOVQGT, ssa.OpAMD64CMOVLGT, ssa.OpAMD64CMOVWGT,
   603  		ssa.OpAMD64CMOVQLE, ssa.OpAMD64CMOVLLE, ssa.OpAMD64CMOVWLE,
   604  		ssa.OpAMD64CMOVQGE, ssa.OpAMD64CMOVLGE, ssa.OpAMD64CMOVWGE,
   605  		ssa.OpAMD64CMOVQHI, ssa.OpAMD64CMOVLHI, ssa.OpAMD64CMOVWHI,
   606  		ssa.OpAMD64CMOVQLS, ssa.OpAMD64CMOVLLS, ssa.OpAMD64CMOVWLS,
   607  		ssa.OpAMD64CMOVQCC, ssa.OpAMD64CMOVLCC, ssa.OpAMD64CMOVWCC,
   608  		ssa.OpAMD64CMOVQCS, ssa.OpAMD64CMOVLCS, ssa.OpAMD64CMOVWCS,
   609  		ssa.OpAMD64CMOVQGTF, ssa.OpAMD64CMOVLGTF, ssa.OpAMD64CMOVWGTF,
   610  		ssa.OpAMD64CMOVQGEF, ssa.OpAMD64CMOVLGEF, ssa.OpAMD64CMOVWGEF:
   611  		p := s.Prog(v.Op.Asm())
   612  		p.From.Type = obj.TYPE_REG
   613  		p.From.Reg = v.Args[1].Reg()
   614  		p.To.Type = obj.TYPE_REG
   615  		p.To.Reg = v.Reg()
   616  
   617  	case ssa.OpAMD64CMOVQNEF, ssa.OpAMD64CMOVLNEF, ssa.OpAMD64CMOVWNEF:
   618  		// Flag condition: ^ZERO || PARITY
   619  		// Generate:
   620  		//   CMOV*NE  SRC,DST
   621  		//   CMOV*PS  SRC,DST
   622  		p := s.Prog(v.Op.Asm())
   623  		p.From.Type = obj.TYPE_REG
   624  		p.From.Reg = v.Args[1].Reg()
   625  		p.To.Type = obj.TYPE_REG
   626  		p.To.Reg = v.Reg()
   627  		var q *obj.Prog
   628  		if v.Op == ssa.OpAMD64CMOVQNEF {
   629  			q = s.Prog(x86.ACMOVQPS)
   630  		} else if v.Op == ssa.OpAMD64CMOVLNEF {
   631  			q = s.Prog(x86.ACMOVLPS)
   632  		} else {
   633  			q = s.Prog(x86.ACMOVWPS)
   634  		}
   635  		q.From.Type = obj.TYPE_REG
   636  		q.From.Reg = v.Args[1].Reg()
   637  		q.To.Type = obj.TYPE_REG
   638  		q.To.Reg = v.Reg()
   639  
   640  	case ssa.OpAMD64CMOVQEQF, ssa.OpAMD64CMOVLEQF, ssa.OpAMD64CMOVWEQF:
   641  		// Flag condition: ZERO && !PARITY
   642  		// Generate:
   643  		//   MOV      SRC,TMP
   644  		//   CMOV*NE  DST,TMP
   645  		//   CMOV*PC  TMP,DST
   646  		//
   647  		// TODO(rasky): we could generate:
   648  		//   CMOV*NE  DST,SRC
   649  		//   CMOV*PC  SRC,DST
   650  		// But this requires a way for regalloc to know that SRC might be
   651  		// clobbered by this instruction.
   652  		t := v.RegTmp()
   653  		opregreg(s, moveByRegsWidth(t, v.Args[1].Reg(), v.Type.Size()), t, v.Args[1].Reg())
   654  
   655  		p := s.Prog(v.Op.Asm())
   656  		p.From.Type = obj.TYPE_REG
   657  		p.From.Reg = v.Reg()
   658  		p.To.Type = obj.TYPE_REG
   659  		p.To.Reg = t
   660  		var q *obj.Prog
   661  		if v.Op == ssa.OpAMD64CMOVQEQF {
   662  			q = s.Prog(x86.ACMOVQPC)
   663  		} else if v.Op == ssa.OpAMD64CMOVLEQF {
   664  			q = s.Prog(x86.ACMOVLPC)
   665  		} else {
   666  			q = s.Prog(x86.ACMOVWPC)
   667  		}
   668  		q.From.Type = obj.TYPE_REG
   669  		q.From.Reg = t
   670  		q.To.Type = obj.TYPE_REG
   671  		q.To.Reg = v.Reg()
   672  
   673  	case ssa.OpAMD64MULQconst, ssa.OpAMD64MULLconst:
   674  		r := v.Reg()
   675  		p := s.Prog(v.Op.Asm())
   676  		p.From.Type = obj.TYPE_CONST
   677  		p.From.Offset = v.AuxInt
   678  		p.To.Type = obj.TYPE_REG
   679  		p.To.Reg = r
   680  		p.AddRestSourceReg(v.Args[0].Reg())
   681  
   682  	case ssa.OpAMD64ANDQconst:
   683  		asm := v.Op.Asm()
   684  		// If the constant is positive and fits into 32 bits, use ANDL.
   685  		// This saves a few bytes of encoding.
   686  		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {
   687  			asm = x86.AANDL
   688  		}
   689  		p := s.Prog(asm)
   690  		p.From.Type = obj.TYPE_CONST
   691  		p.From.Offset = v.AuxInt
   692  		p.To.Type = obj.TYPE_REG
   693  		p.To.Reg = v.Reg()
   694  
   695  	case ssa.OpAMD64SUBQconst, ssa.OpAMD64SUBLconst,
   696  		ssa.OpAMD64ANDLconst,
   697  		ssa.OpAMD64ORQconst, ssa.OpAMD64ORLconst,
   698  		ssa.OpAMD64XORQconst, ssa.OpAMD64XORLconst:
   699  		p := s.Prog(v.Op.Asm())
   700  		p.From.Type = obj.TYPE_CONST
   701  		p.From.Offset = v.AuxInt
   702  		p.To.Type = obj.TYPE_REG
   703  		p.To.Reg = v.Reg()
   704  
   705  	case ssa.OpAMD64SHLQconst, ssa.OpAMD64SHLLconst,
   706  		ssa.OpAMD64SHRQconst, ssa.OpAMD64SHRLconst, ssa.OpAMD64SHRWconst, ssa.OpAMD64SHRBconst,
   707  		ssa.OpAMD64SARQconst, ssa.OpAMD64SARLconst, ssa.OpAMD64SARWconst, ssa.OpAMD64SARBconst,
   708  		ssa.OpAMD64ROLQconst, ssa.OpAMD64ROLLconst, ssa.OpAMD64ROLWconst, ssa.OpAMD64ROLBconst:
   709  		var maxShift int64
   710  		switch v.Op {
   711  		case ssa.OpAMD64SHLQconst, ssa.OpAMD64SHRQconst, ssa.OpAMD64SARQconst, ssa.OpAMD64ROLQconst:
   712  			maxShift = 63
   713  		case ssa.OpAMD64SHLLconst, ssa.OpAMD64SHRLconst, ssa.OpAMD64SARLconst, ssa.OpAMD64ROLLconst:
   714  			maxShift = 31
   715  		case ssa.OpAMD64SHRWconst, ssa.OpAMD64SARWconst, ssa.OpAMD64ROLWconst:
   716  			maxShift = 15
   717  		case ssa.OpAMD64SHRBconst, ssa.OpAMD64SARBconst, ssa.OpAMD64ROLBconst:
   718  			maxShift = 7
   719  		default:
   720  			panic("unreachable")
   721  		}
   722  		if v.AuxInt < 0 || v.AuxInt > maxShift {
   723  			v.Fatalf("shift amount out of range [0,%d]: %d", maxShift, v.AuxInt)
   724  		}
   725  		p := s.Prog(v.Op.Asm())
   726  		p.From.Type = obj.TYPE_CONST
   727  		p.From.Offset = v.AuxInt
   728  		p.To.Type = obj.TYPE_REG
   729  		p.To.Reg = v.Reg()
   730  	case ssa.OpAMD64SBBQcarrymask, ssa.OpAMD64SBBLcarrymask:
   731  		r := v.Reg()
   732  		p := s.Prog(v.Op.Asm())
   733  		p.From.Type = obj.TYPE_REG
   734  		p.From.Reg = r
   735  		p.To.Type = obj.TYPE_REG
   736  		p.To.Reg = r
   737  	case ssa.OpAMD64LEAQ1, ssa.OpAMD64LEAQ2, ssa.OpAMD64LEAQ4, ssa.OpAMD64LEAQ8,
   738  		ssa.OpAMD64LEAL1, ssa.OpAMD64LEAL2, ssa.OpAMD64LEAL4, ssa.OpAMD64LEAL8,
   739  		ssa.OpAMD64LEAW1, ssa.OpAMD64LEAW2, ssa.OpAMD64LEAW4, ssa.OpAMD64LEAW8:
   740  		p := s.Prog(v.Op.Asm())
   741  		memIdx(&p.From, v)
   742  		o := v.Reg()
   743  		p.To.Type = obj.TYPE_REG
   744  		p.To.Reg = o
   745  		if v.AuxInt != 0 && v.Aux == nil {
   746  			// Emit an additional LEA to add the displacement instead of creating a slow 3 operand LEA.
   747  			switch v.Op {
   748  			case ssa.OpAMD64LEAQ1, ssa.OpAMD64LEAQ2, ssa.OpAMD64LEAQ4, ssa.OpAMD64LEAQ8:
   749  				p = s.Prog(x86.ALEAQ)
   750  			case ssa.OpAMD64LEAL1, ssa.OpAMD64LEAL2, ssa.OpAMD64LEAL4, ssa.OpAMD64LEAL8:
   751  				p = s.Prog(x86.ALEAL)
   752  			case ssa.OpAMD64LEAW1, ssa.OpAMD64LEAW2, ssa.OpAMD64LEAW4, ssa.OpAMD64LEAW8:
   753  				p = s.Prog(x86.ALEAW)
   754  			}
   755  			p.From.Type = obj.TYPE_MEM
   756  			p.From.Reg = o
   757  			p.To.Type = obj.TYPE_REG
   758  			p.To.Reg = o
   759  		}
   760  		ssagen.AddAux(&p.From, v)
   761  	case ssa.OpAMD64LEAQ, ssa.OpAMD64LEAL, ssa.OpAMD64LEAW:
   762  		p := s.Prog(v.Op.Asm())
   763  		p.From.Type = obj.TYPE_MEM
   764  		p.From.Reg = v.Args[0].Reg()
   765  		ssagen.AddAux(&p.From, v)
   766  		p.To.Type = obj.TYPE_REG
   767  		p.To.Reg = v.Reg()
   768  	case ssa.OpAMD64CMPQ, ssa.OpAMD64CMPL, ssa.OpAMD64CMPW, ssa.OpAMD64CMPB,
   769  		ssa.OpAMD64TESTQ, ssa.OpAMD64TESTL, ssa.OpAMD64TESTW, ssa.OpAMD64TESTB,
   770  		ssa.OpAMD64BTL, ssa.OpAMD64BTQ:
   771  		opregreg(s, v.Op.Asm(), v.Args[1].Reg(), v.Args[0].Reg())
   772  	case ssa.OpAMD64UCOMISS, ssa.OpAMD64UCOMISD:
   773  		// Go assembler has swapped operands for UCOMISx relative to CMP,
   774  		// must account for that right here.
   775  		opregreg(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg())
   776  	case ssa.OpAMD64CMPQconst, ssa.OpAMD64CMPLconst, ssa.OpAMD64CMPWconst, ssa.OpAMD64CMPBconst:
   777  		p := s.Prog(v.Op.Asm())
   778  		p.From.Type = obj.TYPE_REG
   779  		p.From.Reg = v.Args[0].Reg()
   780  		p.To.Type = obj.TYPE_CONST
   781  		p.To.Offset = v.AuxInt
   782  	case ssa.OpAMD64BTLconst, ssa.OpAMD64BTQconst,
   783  		ssa.OpAMD64TESTQconst, ssa.OpAMD64TESTLconst, ssa.OpAMD64TESTWconst, ssa.OpAMD64TESTBconst,
   784  		ssa.OpAMD64BTSQconst,
   785  		ssa.OpAMD64BTCQconst,
   786  		ssa.OpAMD64BTRQconst:
   787  		op := v.Op
   788  		if op == ssa.OpAMD64BTQconst && v.AuxInt < 32 {
   789  			// Emit 32-bit version because it's shorter
   790  			op = ssa.OpAMD64BTLconst
   791  		}
   792  		p := s.Prog(op.Asm())
   793  		p.From.Type = obj.TYPE_CONST
   794  		p.From.Offset = v.AuxInt
   795  		p.To.Type = obj.TYPE_REG
   796  		p.To.Reg = v.Args[0].Reg()
   797  	case ssa.OpAMD64CMPQload, ssa.OpAMD64CMPLload, ssa.OpAMD64CMPWload, ssa.OpAMD64CMPBload:
   798  		p := s.Prog(v.Op.Asm())
   799  		p.From.Type = obj.TYPE_MEM
   800  		p.From.Reg = v.Args[0].Reg()
   801  		ssagen.AddAux(&p.From, v)
   802  		p.To.Type = obj.TYPE_REG
   803  		p.To.Reg = v.Args[1].Reg()
   804  	case ssa.OpAMD64CMPQconstload, ssa.OpAMD64CMPLconstload, ssa.OpAMD64CMPWconstload, ssa.OpAMD64CMPBconstload:
   805  		sc := v.AuxValAndOff()
   806  		p := s.Prog(v.Op.Asm())
   807  		p.From.Type = obj.TYPE_MEM
   808  		p.From.Reg = v.Args[0].Reg()
   809  		ssagen.AddAux2(&p.From, v, sc.Off64())
   810  		p.To.Type = obj.TYPE_CONST
   811  		p.To.Offset = sc.Val64()
   812  	case ssa.OpAMD64CMPQloadidx8, ssa.OpAMD64CMPQloadidx1, ssa.OpAMD64CMPLloadidx4, ssa.OpAMD64CMPLloadidx1, ssa.OpAMD64CMPWloadidx2, ssa.OpAMD64CMPWloadidx1, ssa.OpAMD64CMPBloadidx1:
   813  		p := s.Prog(v.Op.Asm())
   814  		memIdx(&p.From, v)
   815  		ssagen.AddAux(&p.From, v)
   816  		p.To.Type = obj.TYPE_REG
   817  		p.To.Reg = v.Args[2].Reg()
   818  	case ssa.OpAMD64CMPQconstloadidx8, ssa.OpAMD64CMPQconstloadidx1, ssa.OpAMD64CMPLconstloadidx4, ssa.OpAMD64CMPLconstloadidx1, ssa.OpAMD64CMPWconstloadidx2, ssa.OpAMD64CMPWconstloadidx1, ssa.OpAMD64CMPBconstloadidx1:
   819  		sc := v.AuxValAndOff()
   820  		p := s.Prog(v.Op.Asm())
   821  		memIdx(&p.From, v)
   822  		ssagen.AddAux2(&p.From, v, sc.Off64())
   823  		p.To.Type = obj.TYPE_CONST
   824  		p.To.Offset = sc.Val64()
   825  	case ssa.OpAMD64MOVLconst, ssa.OpAMD64MOVQconst:
   826  		x := v.Reg()
   827  
   828  		// If flags aren't live (indicated by v.Aux == nil),
   829  		// then we can rewrite MOV $0, AX into XOR AX, AX.
   830  		if v.AuxInt == 0 && v.Aux == nil {
   831  			opregreg(s, x86.AXORL, x, x)
   832  			break
   833  		}
   834  
   835  		asm := v.Op.Asm()
   836  		// Use MOVL to move a small constant into a register
   837  		// when the constant is positive and fits into 32 bits.
   838  		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {
   839  			// The upper 32bit are zeroed automatically when using MOVL.
   840  			asm = x86.AMOVL
   841  		}
   842  		p := s.Prog(asm)
   843  		p.From.Type = obj.TYPE_CONST
   844  		p.From.Offset = v.AuxInt
   845  		p.To.Type = obj.TYPE_REG
   846  		p.To.Reg = x
   847  
   848  	case ssa.OpAMD64MOVSSconst, ssa.OpAMD64MOVSDconst:
   849  		x := v.Reg()
   850  		if !isFPReg(x) && v.AuxInt == 0 && v.Aux == nil {
   851  			opregreg(s, x86.AXORL, x, x)
   852  			break
   853  		}
   854  		p := s.Prog(storeByRegWidth(x, v.Type.Size()))
   855  		p.From.Type = obj.TYPE_FCONST
   856  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
   857  		p.To.Type = obj.TYPE_REG
   858  		p.To.Reg = x
   859  	case ssa.OpAMD64MOVQload, ssa.OpAMD64MOVLload, ssa.OpAMD64MOVWload, ssa.OpAMD64MOVBload, ssa.OpAMD64MOVOload,
   860  		ssa.OpAMD64MOVSSload, ssa.OpAMD64MOVSDload, ssa.OpAMD64MOVBQSXload, ssa.OpAMD64MOVWQSXload, ssa.OpAMD64MOVLQSXload,
   861  		ssa.OpAMD64MOVBEQload, ssa.OpAMD64MOVBELload:
   862  		p := s.Prog(v.Op.Asm())
   863  		p.From.Type = obj.TYPE_MEM
   864  		p.From.Reg = v.Args[0].Reg()
   865  		ssagen.AddAux(&p.From, v)
   866  		p.To.Type = obj.TYPE_REG
   867  		p.To.Reg = v.Reg()
   868  	case ssa.OpAMD64MOVBloadidx1, ssa.OpAMD64MOVWloadidx1, ssa.OpAMD64MOVLloadidx1, ssa.OpAMD64MOVQloadidx1, ssa.OpAMD64MOVSSloadidx1, ssa.OpAMD64MOVSDloadidx1,
   869  		ssa.OpAMD64MOVQloadidx8, ssa.OpAMD64MOVSDloadidx8, ssa.OpAMD64MOVLloadidx8, ssa.OpAMD64MOVLloadidx4, ssa.OpAMD64MOVSSloadidx4, ssa.OpAMD64MOVWloadidx2,
   870  		ssa.OpAMD64MOVBELloadidx1, ssa.OpAMD64MOVBELloadidx4, ssa.OpAMD64MOVBELloadidx8, ssa.OpAMD64MOVBEQloadidx1, ssa.OpAMD64MOVBEQloadidx8:
   871  		p := s.Prog(v.Op.Asm())
   872  		memIdx(&p.From, v)
   873  		ssagen.AddAux(&p.From, v)
   874  		p.To.Type = obj.TYPE_REG
   875  		p.To.Reg = v.Reg()
   876  	case ssa.OpAMD64MOVQstore, ssa.OpAMD64MOVSSstore, ssa.OpAMD64MOVSDstore, ssa.OpAMD64MOVLstore, ssa.OpAMD64MOVWstore, ssa.OpAMD64MOVBstore, ssa.OpAMD64MOVOstore,
   877  		ssa.OpAMD64ADDQmodify, ssa.OpAMD64SUBQmodify, ssa.OpAMD64ANDQmodify, ssa.OpAMD64ORQmodify, ssa.OpAMD64XORQmodify,
   878  		ssa.OpAMD64ADDLmodify, ssa.OpAMD64SUBLmodify, ssa.OpAMD64ANDLmodify, ssa.OpAMD64ORLmodify, ssa.OpAMD64XORLmodify,
   879  		ssa.OpAMD64MOVBEQstore, ssa.OpAMD64MOVBELstore, ssa.OpAMD64MOVBEWstore:
   880  		p := s.Prog(v.Op.Asm())
   881  		p.From.Type = obj.TYPE_REG
   882  		p.From.Reg = v.Args[1].Reg()
   883  		p.To.Type = obj.TYPE_MEM
   884  		p.To.Reg = v.Args[0].Reg()
   885  		ssagen.AddAux(&p.To, v)
   886  	case ssa.OpAMD64MOVBstoreidx1, ssa.OpAMD64MOVWstoreidx1, ssa.OpAMD64MOVLstoreidx1, ssa.OpAMD64MOVQstoreidx1, ssa.OpAMD64MOVSSstoreidx1, ssa.OpAMD64MOVSDstoreidx1,
   887  		ssa.OpAMD64MOVQstoreidx8, ssa.OpAMD64MOVSDstoreidx8, ssa.OpAMD64MOVLstoreidx8, ssa.OpAMD64MOVSSstoreidx4, ssa.OpAMD64MOVLstoreidx4, ssa.OpAMD64MOVWstoreidx2,
   888  		ssa.OpAMD64ADDLmodifyidx1, ssa.OpAMD64ADDLmodifyidx4, ssa.OpAMD64ADDLmodifyidx8, ssa.OpAMD64ADDQmodifyidx1, ssa.OpAMD64ADDQmodifyidx8,
   889  		ssa.OpAMD64SUBLmodifyidx1, ssa.OpAMD64SUBLmodifyidx4, ssa.OpAMD64SUBLmodifyidx8, ssa.OpAMD64SUBQmodifyidx1, ssa.OpAMD64SUBQmodifyidx8,
   890  		ssa.OpAMD64ANDLmodifyidx1, ssa.OpAMD64ANDLmodifyidx4, ssa.OpAMD64ANDLmodifyidx8, ssa.OpAMD64ANDQmodifyidx1, ssa.OpAMD64ANDQmodifyidx8,
   891  		ssa.OpAMD64ORLmodifyidx1, ssa.OpAMD64ORLmodifyidx4, ssa.OpAMD64ORLmodifyidx8, ssa.OpAMD64ORQmodifyidx1, ssa.OpAMD64ORQmodifyidx8,
   892  		ssa.OpAMD64XORLmodifyidx1, ssa.OpAMD64XORLmodifyidx4, ssa.OpAMD64XORLmodifyidx8, ssa.OpAMD64XORQmodifyidx1, ssa.OpAMD64XORQmodifyidx8,
   893  		ssa.OpAMD64MOVBEWstoreidx1, ssa.OpAMD64MOVBEWstoreidx2, ssa.OpAMD64MOVBELstoreidx1, ssa.OpAMD64MOVBELstoreidx4, ssa.OpAMD64MOVBELstoreidx8, ssa.OpAMD64MOVBEQstoreidx1, ssa.OpAMD64MOVBEQstoreidx8:
   894  		p := s.Prog(v.Op.Asm())
   895  		p.From.Type = obj.TYPE_REG
   896  		p.From.Reg = v.Args[2].Reg()
   897  		memIdx(&p.To, v)
   898  		ssagen.AddAux(&p.To, v)
   899  	case ssa.OpAMD64ADDQconstmodify, ssa.OpAMD64ADDLconstmodify:
   900  		sc := v.AuxValAndOff()
   901  		off := sc.Off64()
   902  		val := sc.Val()
   903  		if val == 1 || val == -1 {
   904  			var asm obj.As
   905  			if v.Op == ssa.OpAMD64ADDQconstmodify {
   906  				if val == 1 {
   907  					asm = x86.AINCQ
   908  				} else {
   909  					asm = x86.ADECQ
   910  				}
   911  			} else {
   912  				if val == 1 {
   913  					asm = x86.AINCL
   914  				} else {
   915  					asm = x86.ADECL
   916  				}
   917  			}
   918  			p := s.Prog(asm)
   919  			p.To.Type = obj.TYPE_MEM
   920  			p.To.Reg = v.Args[0].Reg()
   921  			ssagen.AddAux2(&p.To, v, off)
   922  			break
   923  		}
   924  		fallthrough
   925  	case ssa.OpAMD64ANDQconstmodify, ssa.OpAMD64ANDLconstmodify, ssa.OpAMD64ORQconstmodify, ssa.OpAMD64ORLconstmodify,
   926  		ssa.OpAMD64XORQconstmodify, ssa.OpAMD64XORLconstmodify,
   927  		ssa.OpAMD64BTSQconstmodify, ssa.OpAMD64BTRQconstmodify, ssa.OpAMD64BTCQconstmodify:
   928  		sc := v.AuxValAndOff()
   929  		off := sc.Off64()
   930  		val := sc.Val64()
   931  		p := s.Prog(v.Op.Asm())
   932  		p.From.Type = obj.TYPE_CONST
   933  		p.From.Offset = val
   934  		p.To.Type = obj.TYPE_MEM
   935  		p.To.Reg = v.Args[0].Reg()
   936  		ssagen.AddAux2(&p.To, v, off)
   937  
   938  	case ssa.OpAMD64MOVQstoreconst, ssa.OpAMD64MOVLstoreconst, ssa.OpAMD64MOVWstoreconst, ssa.OpAMD64MOVBstoreconst:
   939  		p := s.Prog(v.Op.Asm())
   940  		p.From.Type = obj.TYPE_CONST
   941  		sc := v.AuxValAndOff()
   942  		p.From.Offset = sc.Val64()
   943  		p.To.Type = obj.TYPE_MEM
   944  		p.To.Reg = v.Args[0].Reg()
   945  		ssagen.AddAux2(&p.To, v, sc.Off64())
   946  	case ssa.OpAMD64MOVOstoreconst:
   947  		sc := v.AuxValAndOff()
   948  		if sc.Val() != 0 {
   949  			v.Fatalf("MOVO for non zero constants not implemented: %s", v.LongString())
   950  		}
   951  
   952  		if s.ABI != obj.ABIInternal {
   953  			// zero X15 manually
   954  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
   955  		}
   956  		p := s.Prog(v.Op.Asm())
   957  		p.From.Type = obj.TYPE_REG
   958  		p.From.Reg = x86.REG_X15
   959  		p.To.Type = obj.TYPE_MEM
   960  		p.To.Reg = v.Args[0].Reg()
   961  		ssagen.AddAux2(&p.To, v, sc.Off64())
   962  
   963  	case ssa.OpAMD64MOVQstoreconstidx1, ssa.OpAMD64MOVQstoreconstidx8, ssa.OpAMD64MOVLstoreconstidx1, ssa.OpAMD64MOVLstoreconstidx4, ssa.OpAMD64MOVWstoreconstidx1, ssa.OpAMD64MOVWstoreconstidx2, ssa.OpAMD64MOVBstoreconstidx1,
   964  		ssa.OpAMD64ADDLconstmodifyidx1, ssa.OpAMD64ADDLconstmodifyidx4, ssa.OpAMD64ADDLconstmodifyidx8, ssa.OpAMD64ADDQconstmodifyidx1, ssa.OpAMD64ADDQconstmodifyidx8,
   965  		ssa.OpAMD64ANDLconstmodifyidx1, ssa.OpAMD64ANDLconstmodifyidx4, ssa.OpAMD64ANDLconstmodifyidx8, ssa.OpAMD64ANDQconstmodifyidx1, ssa.OpAMD64ANDQconstmodifyidx8,
   966  		ssa.OpAMD64ORLconstmodifyidx1, ssa.OpAMD64ORLconstmodifyidx4, ssa.OpAMD64ORLconstmodifyidx8, ssa.OpAMD64ORQconstmodifyidx1, ssa.OpAMD64ORQconstmodifyidx8,
   967  		ssa.OpAMD64XORLconstmodifyidx1, ssa.OpAMD64XORLconstmodifyidx4, ssa.OpAMD64XORLconstmodifyidx8, ssa.OpAMD64XORQconstmodifyidx1, ssa.OpAMD64XORQconstmodifyidx8:
   968  		p := s.Prog(v.Op.Asm())
   969  		p.From.Type = obj.TYPE_CONST
   970  		sc := v.AuxValAndOff()
   971  		p.From.Offset = sc.Val64()
   972  		switch {
   973  		case p.As == x86.AADDQ && p.From.Offset == 1:
   974  			p.As = x86.AINCQ
   975  			p.From.Type = obj.TYPE_NONE
   976  		case p.As == x86.AADDQ && p.From.Offset == -1:
   977  			p.As = x86.ADECQ
   978  			p.From.Type = obj.TYPE_NONE
   979  		case p.As == x86.AADDL && p.From.Offset == 1:
   980  			p.As = x86.AINCL
   981  			p.From.Type = obj.TYPE_NONE
   982  		case p.As == x86.AADDL && p.From.Offset == -1:
   983  			p.As = x86.ADECL
   984  			p.From.Type = obj.TYPE_NONE
   985  		}
   986  		memIdx(&p.To, v)
   987  		ssagen.AddAux2(&p.To, v, sc.Off64())
   988  	case ssa.OpAMD64MOVLQSX, ssa.OpAMD64MOVWQSX, ssa.OpAMD64MOVBQSX, ssa.OpAMD64MOVLQZX, ssa.OpAMD64MOVWQZX, ssa.OpAMD64MOVBQZX,
   989  		ssa.OpAMD64CVTTSS2SL, ssa.OpAMD64CVTTSD2SL, ssa.OpAMD64CVTTSS2SQ, ssa.OpAMD64CVTTSD2SQ,
   990  		ssa.OpAMD64CVTSS2SD, ssa.OpAMD64CVTSD2SS, ssa.OpAMD64VPBROADCASTB, ssa.OpAMD64PMOVMSKB:
   991  		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[0].Reg())
   992  	case ssa.OpAMD64CVTSL2SD, ssa.OpAMD64CVTSQ2SD, ssa.OpAMD64CVTSQ2SS, ssa.OpAMD64CVTSL2SS:
   993  		r := v.Reg()
   994  		// Break false dependency on destination register.
   995  		opregreg(s, x86.AXORPS, r, r)
   996  		opregreg(s, v.Op.Asm(), r, v.Args[0].Reg())
   997  	case ssa.OpAMD64MOVQi2f, ssa.OpAMD64MOVQf2i, ssa.OpAMD64MOVLi2f, ssa.OpAMD64MOVLf2i:
   998  		var p *obj.Prog
   999  		switch v.Op {
  1000  		case ssa.OpAMD64MOVQi2f, ssa.OpAMD64MOVQf2i:
  1001  			p = s.Prog(x86.AMOVQ)
  1002  		case ssa.OpAMD64MOVLi2f, ssa.OpAMD64MOVLf2i:
  1003  			p = s.Prog(x86.AMOVL)
  1004  		}
  1005  		p.From.Type = obj.TYPE_REG
  1006  		p.From.Reg = v.Args[0].Reg()
  1007  		p.To.Type = obj.TYPE_REG
  1008  		p.To.Reg = v.Reg()
  1009  	case ssa.OpAMD64ADDQload, ssa.OpAMD64ADDLload, ssa.OpAMD64SUBQload, ssa.OpAMD64SUBLload,
  1010  		ssa.OpAMD64ANDQload, ssa.OpAMD64ANDLload, ssa.OpAMD64ORQload, ssa.OpAMD64ORLload,
  1011  		ssa.OpAMD64XORQload, ssa.OpAMD64XORLload, ssa.OpAMD64ADDSDload, ssa.OpAMD64ADDSSload,
  1012  		ssa.OpAMD64SUBSDload, ssa.OpAMD64SUBSSload, ssa.OpAMD64MULSDload, ssa.OpAMD64MULSSload,
  1013  		ssa.OpAMD64DIVSDload, ssa.OpAMD64DIVSSload:
  1014  		p := s.Prog(v.Op.Asm())
  1015  		p.From.Type = obj.TYPE_MEM
  1016  		p.From.Reg = v.Args[1].Reg()
  1017  		ssagen.AddAux(&p.From, v)
  1018  		p.To.Type = obj.TYPE_REG
  1019  		p.To.Reg = v.Reg()
  1020  	case ssa.OpAMD64ADDLloadidx1, ssa.OpAMD64ADDLloadidx4, ssa.OpAMD64ADDLloadidx8, ssa.OpAMD64ADDQloadidx1, ssa.OpAMD64ADDQloadidx8,
  1021  		ssa.OpAMD64SUBLloadidx1, ssa.OpAMD64SUBLloadidx4, ssa.OpAMD64SUBLloadidx8, ssa.OpAMD64SUBQloadidx1, ssa.OpAMD64SUBQloadidx8,
  1022  		ssa.OpAMD64ANDLloadidx1, ssa.OpAMD64ANDLloadidx4, ssa.OpAMD64ANDLloadidx8, ssa.OpAMD64ANDQloadidx1, ssa.OpAMD64ANDQloadidx8,
  1023  		ssa.OpAMD64ORLloadidx1, ssa.OpAMD64ORLloadidx4, ssa.OpAMD64ORLloadidx8, ssa.OpAMD64ORQloadidx1, ssa.OpAMD64ORQloadidx8,
  1024  		ssa.OpAMD64XORLloadidx1, ssa.OpAMD64XORLloadidx4, ssa.OpAMD64XORLloadidx8, ssa.OpAMD64XORQloadidx1, ssa.OpAMD64XORQloadidx8,
  1025  		ssa.OpAMD64ADDSSloadidx1, ssa.OpAMD64ADDSSloadidx4, ssa.OpAMD64ADDSDloadidx1, ssa.OpAMD64ADDSDloadidx8,
  1026  		ssa.OpAMD64SUBSSloadidx1, ssa.OpAMD64SUBSSloadidx4, ssa.OpAMD64SUBSDloadidx1, ssa.OpAMD64SUBSDloadidx8,
  1027  		ssa.OpAMD64MULSSloadidx1, ssa.OpAMD64MULSSloadidx4, ssa.OpAMD64MULSDloadidx1, ssa.OpAMD64MULSDloadidx8,
  1028  		ssa.OpAMD64DIVSSloadidx1, ssa.OpAMD64DIVSSloadidx4, ssa.OpAMD64DIVSDloadidx1, ssa.OpAMD64DIVSDloadidx8:
  1029  		p := s.Prog(v.Op.Asm())
  1030  
  1031  		r, i := v.Args[1].Reg(), v.Args[2].Reg()
  1032  		p.From.Type = obj.TYPE_MEM
  1033  		p.From.Scale = v.Op.Scale()
  1034  		if p.From.Scale == 1 && i == x86.REG_SP {
  1035  			r, i = i, r
  1036  		}
  1037  		p.From.Reg = r
  1038  		p.From.Index = i
  1039  
  1040  		ssagen.AddAux(&p.From, v)
  1041  		p.To.Type = obj.TYPE_REG
  1042  		p.To.Reg = v.Reg()
  1043  
  1044  	case ssa.OpAMD64LoweredZero:
  1045  		if s.ABI != obj.ABIInternal {
  1046  			// zero X15 manually
  1047  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  1048  		}
  1049  		ptrReg := v.Args[0].Reg()
  1050  		n := v.AuxInt
  1051  		if n < 16 {
  1052  			v.Fatalf("Zero too small %d", n)
  1053  		}
  1054  		zero16 := func(off int64) {
  1055  			zero16(s, ptrReg, off)
  1056  		}
  1057  
  1058  		// Generate zeroing instructions.
  1059  		var off int64
  1060  		for n >= 16 {
  1061  			zero16(off)
  1062  			off += 16
  1063  			n -= 16
  1064  		}
  1065  		if n != 0 {
  1066  			// use partially overlapped write.
  1067  			// TODO: n <= 8, use smaller write?
  1068  			zero16(off + n - 16)
  1069  		}
  1070  
  1071  	case ssa.OpAMD64LoweredZeroLoop:
  1072  		if s.ABI != obj.ABIInternal {
  1073  			// zero X15 manually
  1074  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  1075  		}
  1076  		ptrReg := v.Args[0].Reg()
  1077  		countReg := v.RegTmp()
  1078  		n := v.AuxInt
  1079  		loopSize := int64(64)
  1080  		if n < 3*loopSize {
  1081  			// - a loop count of 0 won't work.
  1082  			// - a loop count of 1 is useless.
  1083  			// - a loop count of 2 is a code size ~tie
  1084  			//     4 instructions to implement the loop
  1085  			//     4 instructions in the loop body
  1086  			//   vs
  1087  			//     8 instructions in the straightline code
  1088  			//   Might as well use straightline code.
  1089  			v.Fatalf("ZeroLoop size too small %d", n)
  1090  		}
  1091  		zero16 := func(off int64) {
  1092  			zero16(s, ptrReg, off)
  1093  		}
  1094  
  1095  		// Put iteration count in a register.
  1096  		//   MOVL    $n, countReg
  1097  		p := s.Prog(x86.AMOVL)
  1098  		p.From.Type = obj.TYPE_CONST
  1099  		p.From.Offset = n / loopSize
  1100  		p.To.Type = obj.TYPE_REG
  1101  		p.To.Reg = countReg
  1102  		cntInit := p
  1103  
  1104  		// Zero loopSize bytes starting at ptrReg.
  1105  		for i := range loopSize / 16 {
  1106  			zero16(i * 16)
  1107  		}
  1108  		//   ADDQ    $loopSize, ptrReg
  1109  		p = s.Prog(x86.AADDQ)
  1110  		p.From.Type = obj.TYPE_CONST
  1111  		p.From.Offset = loopSize
  1112  		p.To.Type = obj.TYPE_REG
  1113  		p.To.Reg = ptrReg
  1114  		//   DECL    countReg
  1115  		p = s.Prog(x86.ADECL)
  1116  		p.To.Type = obj.TYPE_REG
  1117  		p.To.Reg = countReg
  1118  		// Jump to first instruction in loop if we're not done yet.
  1119  		//   JNE     head
  1120  		p = s.Prog(x86.AJNE)
  1121  		p.To.Type = obj.TYPE_BRANCH
  1122  		p.To.SetTarget(cntInit.Link)
  1123  
  1124  		// Multiples of the loop size are now done.
  1125  		n %= loopSize
  1126  
  1127  		// Write any fractional portion.
  1128  		var off int64
  1129  		for n >= 16 {
  1130  			zero16(off)
  1131  			off += 16
  1132  			n -= 16
  1133  		}
  1134  		if n != 0 {
  1135  			// Use partially-overlapping write.
  1136  			// TODO: n <= 8, use smaller write?
  1137  			zero16(off + n - 16)
  1138  		}
  1139  
  1140  	case ssa.OpAMD64LoweredMove:
  1141  		dstReg := v.Args[0].Reg()
  1142  		srcReg := v.Args[1].Reg()
  1143  		if dstReg == srcReg {
  1144  			break
  1145  		}
  1146  		tmpReg := int16(x86.REG_X14)
  1147  		n := v.AuxInt
  1148  		if n < 16 {
  1149  			v.Fatalf("Move too small %d", n)
  1150  		}
  1151  		// move 16 bytes from srcReg+off to dstReg+off.
  1152  		move16 := func(off int64) {
  1153  			move16(s, srcReg, dstReg, tmpReg, off)
  1154  		}
  1155  
  1156  		// Generate copying instructions.
  1157  		var off int64
  1158  		for n >= 16 {
  1159  			move16(off)
  1160  			off += 16
  1161  			n -= 16
  1162  		}
  1163  		if n != 0 {
  1164  			// use partially overlapped read/write.
  1165  			// TODO: use smaller operations when we can?
  1166  			move16(off + n - 16)
  1167  		}
  1168  
  1169  	case ssa.OpAMD64LoweredMoveLoop:
  1170  		dstReg := v.Args[0].Reg()
  1171  		srcReg := v.Args[1].Reg()
  1172  		if dstReg == srcReg {
  1173  			break
  1174  		}
  1175  		countReg := v.RegTmp()
  1176  		tmpReg := int16(x86.REG_X14)
  1177  		n := v.AuxInt
  1178  		loopSize := int64(64)
  1179  		if n < 3*loopSize {
  1180  			// - a loop count of 0 won't work.
  1181  			// - a loop count of 1 is useless.
  1182  			// - a loop count of 2 is a code size ~tie
  1183  			//     4 instructions to implement the loop
  1184  			//     4 instructions in the loop body
  1185  			//   vs
  1186  			//     8 instructions in the straightline code
  1187  			//   Might as well use straightline code.
  1188  			v.Fatalf("ZeroLoop size too small %d", n)
  1189  		}
  1190  		// move 16 bytes from srcReg+off to dstReg+off.
  1191  		move16 := func(off int64) {
  1192  			move16(s, srcReg, dstReg, tmpReg, off)
  1193  		}
  1194  
  1195  		// Put iteration count in a register.
  1196  		//   MOVL    $n, countReg
  1197  		p := s.Prog(x86.AMOVL)
  1198  		p.From.Type = obj.TYPE_CONST
  1199  		p.From.Offset = n / loopSize
  1200  		p.To.Type = obj.TYPE_REG
  1201  		p.To.Reg = countReg
  1202  		cntInit := p
  1203  
  1204  		// Copy loopSize bytes starting at srcReg to dstReg.
  1205  		for i := range loopSize / 16 {
  1206  			move16(i * 16)
  1207  		}
  1208  		//   ADDQ    $loopSize, srcReg
  1209  		p = s.Prog(x86.AADDQ)
  1210  		p.From.Type = obj.TYPE_CONST
  1211  		p.From.Offset = loopSize
  1212  		p.To.Type = obj.TYPE_REG
  1213  		p.To.Reg = srcReg
  1214  		//   ADDQ    $loopSize, dstReg
  1215  		p = s.Prog(x86.AADDQ)
  1216  		p.From.Type = obj.TYPE_CONST
  1217  		p.From.Offset = loopSize
  1218  		p.To.Type = obj.TYPE_REG
  1219  		p.To.Reg = dstReg
  1220  		//   DECL    countReg
  1221  		p = s.Prog(x86.ADECL)
  1222  		p.To.Type = obj.TYPE_REG
  1223  		p.To.Reg = countReg
  1224  		// Jump to loop header if we're not done yet.
  1225  		//   JNE     head
  1226  		p = s.Prog(x86.AJNE)
  1227  		p.To.Type = obj.TYPE_BRANCH
  1228  		p.To.SetTarget(cntInit.Link)
  1229  
  1230  		// Multiples of the loop size are now done.
  1231  		n %= loopSize
  1232  
  1233  		// Copy any fractional portion.
  1234  		var off int64
  1235  		for n >= 16 {
  1236  			move16(off)
  1237  			off += 16
  1238  			n -= 16
  1239  		}
  1240  		if n != 0 {
  1241  			// Use partially-overlapping copy.
  1242  			move16(off + n - 16)
  1243  		}
  1244  
  1245  	case ssa.OpCopy: // TODO: use MOVQreg for reg->reg copies instead of OpCopy?
  1246  		if v.Type.IsMemory() {
  1247  			return
  1248  		}
  1249  		arg := v.Args[0]
  1250  		x := arg.Reg()
  1251  		y := v.Reg()
  1252  		if v.Type.IsSIMD() {
  1253  			x = simdOrMaskReg(arg)
  1254  			y = simdOrMaskReg(v)
  1255  		}
  1256  		if x != y {
  1257  			width := v.Type.Size()
  1258  			if width == 8 && isGPReg(y) && ssa.ZeroUpper32Bits(arg, 3) {
  1259  				// The source was naturally zext-ed from 32 to 64 bits,
  1260  				// but we are asked to do a full 64-bit copy.
  1261  				// Save the REX prefix byte in I-CACHE by using a 32-bit move,
  1262  				// since it zeroes the upper 32 bits anyway.
  1263  				width = 4
  1264  			}
  1265  			opregreg(s, moveByRegsWidth(y, x, width), y, x)
  1266  		}
  1267  	case ssa.OpLoadReg:
  1268  		if v.Type.IsFlags() {
  1269  			v.Fatalf("load flags not implemented: %v", v.LongString())
  1270  			return
  1271  		}
  1272  		r := v.Reg()
  1273  		p := s.Prog(loadByRegWidth(r, v.Type.Size()))
  1274  		ssagen.AddrAuto(&p.From, v.Args[0])
  1275  		p.To.Type = obj.TYPE_REG
  1276  		if v.Type.IsSIMD() {
  1277  			r = simdOrMaskReg(v)
  1278  		}
  1279  		p.To.Reg = r
  1280  
  1281  	case ssa.OpStoreReg:
  1282  		if v.Type.IsFlags() {
  1283  			v.Fatalf("store flags not implemented: %v", v.LongString())
  1284  			return
  1285  		}
  1286  		r := v.Args[0].Reg()
  1287  		if v.Type.IsSIMD() {
  1288  			r = simdOrMaskReg(v.Args[0])
  1289  		}
  1290  		p := s.Prog(storeByRegWidth(r, v.Type.Size()))
  1291  		p.From.Type = obj.TYPE_REG
  1292  		p.From.Reg = r
  1293  		ssagen.AddrAuto(&p.To, v)
  1294  	case ssa.OpAMD64LoweredHasCPUFeature:
  1295  		p := s.Prog(x86.AMOVBLZX)
  1296  		p.From.Type = obj.TYPE_MEM
  1297  		ssagen.AddAux(&p.From, v)
  1298  		p.To.Type = obj.TYPE_REG
  1299  		p.To.Reg = v.Reg()
  1300  	case ssa.OpArgIntReg, ssa.OpArgFloatReg:
  1301  		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
  1302  		// The loop only runs once.
  1303  		for _, ap := range v.Block.Func.RegArgs {
  1304  			// Pass the spill/unspill information along to the assembler, offset by size of return PC pushed on stack.
  1305  			addr := ssagen.SpillSlotAddr(ap, x86.REG_SP, v.Block.Func.Config.PtrSize)
  1306  			reg := ap.Reg
  1307  			t := ap.Type
  1308  			sz := t.Size()
  1309  			if t.IsSIMD() {
  1310  				reg = simdRegBySize(reg, sz)
  1311  			}
  1312  			s.FuncInfo().AddSpill(
  1313  				obj.RegSpill{Reg: reg, Addr: addr, Unspill: loadByRegWidth(reg, sz), Spill: storeByRegWidth(reg, sz)})
  1314  		}
  1315  		v.Block.Func.RegArgs = nil
  1316  		ssagen.CheckArgReg(v)
  1317  	case ssa.OpAMD64LoweredGetClosurePtr:
  1318  		// Closure pointer is DX.
  1319  		ssagen.CheckLoweredGetClosurePtr(v)
  1320  	case ssa.OpAMD64LoweredGetG:
  1321  		if s.ABI == obj.ABIInternal {
  1322  			v.Fatalf("LoweredGetG should not appear in ABIInternal")
  1323  		}
  1324  		r := v.Reg()
  1325  		getgFromTLS(s, r)
  1326  	case ssa.OpAMD64CALLstatic, ssa.OpAMD64CALLtail, ssa.OpAMD64CALLtailinter:
  1327  		if s.ABI == obj.ABI0 && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABIInternal {
  1328  			// zeroing X15 when entering ABIInternal from ABI0
  1329  			zeroX15(s)
  1330  			// set G register from TLS
  1331  			getgFromTLS(s, x86.REG_R14)
  1332  		}
  1333  		if v.Op == ssa.OpAMD64CALLtail || v.Op == ssa.OpAMD64CALLtailinter {
  1334  			s.TailCall(v)
  1335  			break
  1336  		}
  1337  		s.Call(v)
  1338  		if s.ABI == obj.ABIInternal && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABI0 {
  1339  			// zeroing X15 when entering ABIInternal from ABI0
  1340  			zeroX15(s)
  1341  			// set G register from TLS
  1342  			getgFromTLS(s, x86.REG_R14)
  1343  		}
  1344  	case ssa.OpAMD64CALLclosure, ssa.OpAMD64CALLinter:
  1345  		s.Call(v)
  1346  
  1347  	case ssa.OpAMD64LoweredGetCallerPC:
  1348  		p := s.Prog(x86.AMOVQ)
  1349  		p.From.Type = obj.TYPE_MEM
  1350  		p.From.Offset = -8 // PC is stored 8 bytes below first parameter.
  1351  		p.From.Name = obj.NAME_PARAM
  1352  		p.To.Type = obj.TYPE_REG
  1353  		p.To.Reg = v.Reg()
  1354  
  1355  	case ssa.OpAMD64LoweredGetCallerSP:
  1356  		// caller's SP is the address of the first arg
  1357  		mov := x86.AMOVQ
  1358  		if types.PtrSize == 4 {
  1359  			mov = x86.AMOVL
  1360  		}
  1361  		p := s.Prog(mov)
  1362  		p.From.Type = obj.TYPE_ADDR
  1363  		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize // 0 on amd64, just to be consistent with other architectures
  1364  		p.From.Name = obj.NAME_PARAM
  1365  		p.To.Type = obj.TYPE_REG
  1366  		p.To.Reg = v.Reg()
  1367  
  1368  	case ssa.OpAMD64LoweredWB:
  1369  		p := s.Prog(obj.ACALL)
  1370  		p.To.Type = obj.TYPE_MEM
  1371  		p.To.Name = obj.NAME_EXTERN
  1372  		// AuxInt encodes how many buffer entries we need.
  1373  		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
  1374  
  1375  	case ssa.OpAMD64LoweredPanicBoundsRR, ssa.OpAMD64LoweredPanicBoundsRC, ssa.OpAMD64LoweredPanicBoundsCR, ssa.OpAMD64LoweredPanicBoundsCC:
  1376  		// Compute the constant we put in the PCData entry for this call.
  1377  		code, signed := ssa.BoundsKind(v.AuxInt).Code()
  1378  		xIsReg := false
  1379  		yIsReg := false
  1380  		xVal := 0
  1381  		yVal := 0
  1382  		switch v.Op {
  1383  		case ssa.OpAMD64LoweredPanicBoundsRR:
  1384  			xIsReg = true
  1385  			xVal = int(v.Args[0].Reg() - x86.REG_AX)
  1386  			yIsReg = true
  1387  			yVal = int(v.Args[1].Reg() - x86.REG_AX)
  1388  		case ssa.OpAMD64LoweredPanicBoundsRC:
  1389  			xIsReg = true
  1390  			xVal = int(v.Args[0].Reg() - x86.REG_AX)
  1391  			c := v.Aux.(ssa.PanicBoundsC).C
  1392  			if c >= 0 && c <= abi.BoundsMaxConst {
  1393  				yVal = int(c)
  1394  			} else {
  1395  				// Move constant to a register
  1396  				yIsReg = true
  1397  				if yVal == xVal {
  1398  					yVal = 1
  1399  				}
  1400  				p := s.Prog(x86.AMOVQ)
  1401  				p.From.Type = obj.TYPE_CONST
  1402  				p.From.Offset = c
  1403  				p.To.Type = obj.TYPE_REG
  1404  				p.To.Reg = x86.REG_AX + int16(yVal)
  1405  			}
  1406  		case ssa.OpAMD64LoweredPanicBoundsCR:
  1407  			yIsReg = true
  1408  			yVal = int(v.Args[0].Reg() - x86.REG_AX)
  1409  			c := v.Aux.(ssa.PanicBoundsC).C
  1410  			if c >= 0 && c <= abi.BoundsMaxConst {
  1411  				xVal = int(c)
  1412  			} else {
  1413  				// Move constant to a register
  1414  				xIsReg = true
  1415  				if xVal == yVal {
  1416  					xVal = 1
  1417  				}
  1418  				p := s.Prog(x86.AMOVQ)
  1419  				p.From.Type = obj.TYPE_CONST
  1420  				p.From.Offset = c
  1421  				p.To.Type = obj.TYPE_REG
  1422  				p.To.Reg = x86.REG_AX + int16(xVal)
  1423  			}
  1424  		case ssa.OpAMD64LoweredPanicBoundsCC:
  1425  			c := v.Aux.(ssa.PanicBoundsCC).Cx
  1426  			if c >= 0 && c <= abi.BoundsMaxConst {
  1427  				xVal = int(c)
  1428  			} else {
  1429  				// Move constant to a register
  1430  				xIsReg = true
  1431  				p := s.Prog(x86.AMOVQ)
  1432  				p.From.Type = obj.TYPE_CONST
  1433  				p.From.Offset = c
  1434  				p.To.Type = obj.TYPE_REG
  1435  				p.To.Reg = x86.REG_AX + int16(xVal)
  1436  			}
  1437  			c = v.Aux.(ssa.PanicBoundsCC).Cy
  1438  			if c >= 0 && c <= abi.BoundsMaxConst {
  1439  				yVal = int(c)
  1440  			} else {
  1441  				// Move constant to a register
  1442  				yIsReg = true
  1443  				yVal = 1
  1444  				p := s.Prog(x86.AMOVQ)
  1445  				p.From.Type = obj.TYPE_CONST
  1446  				p.From.Offset = c
  1447  				p.To.Type = obj.TYPE_REG
  1448  				p.To.Reg = x86.REG_AX + int16(yVal)
  1449  			}
  1450  		}
  1451  		c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
  1452  
  1453  		p := s.Prog(obj.APCDATA)
  1454  		p.From.SetConst(abi.PCDATA_PanicBounds)
  1455  		p.To.SetConst(int64(c))
  1456  		p = s.Prog(obj.ACALL)
  1457  		p.To.Type = obj.TYPE_MEM
  1458  		p.To.Name = obj.NAME_EXTERN
  1459  		p.To.Sym = ir.Syms.PanicBounds
  1460  
  1461  	case ssa.OpAMD64NEGQ, ssa.OpAMD64NEGL,
  1462  		ssa.OpAMD64BSWAPQ, ssa.OpAMD64BSWAPL,
  1463  		ssa.OpAMD64NOTQ, ssa.OpAMD64NOTL:
  1464  		p := s.Prog(v.Op.Asm())
  1465  		p.To.Type = obj.TYPE_REG
  1466  		p.To.Reg = v.Reg()
  1467  
  1468  	case ssa.OpAMD64NEGLflags:
  1469  		p := s.Prog(v.Op.Asm())
  1470  		p.To.Type = obj.TYPE_REG
  1471  		p.To.Reg = v.Reg0()
  1472  
  1473  	case ssa.OpAMD64ADDQconstflags, ssa.OpAMD64ADDLconstflags:
  1474  		p := s.Prog(v.Op.Asm())
  1475  		p.From.Type = obj.TYPE_CONST
  1476  		p.From.Offset = v.AuxInt
  1477  		// Note: the inc/dec instructions do not modify
  1478  		// the carry flag like add$1 / sub$1 do.
  1479  		// We currently never use the CF/OF flags from
  1480  		// these instructions, so that is ok.
  1481  		switch {
  1482  		case p.As == x86.AADDQ && p.From.Offset == 1:
  1483  			p.As = x86.AINCQ
  1484  			p.From.Type = obj.TYPE_NONE
  1485  		case p.As == x86.AADDQ && p.From.Offset == -1:
  1486  			p.As = x86.ADECQ
  1487  			p.From.Type = obj.TYPE_NONE
  1488  		case p.As == x86.AADDL && p.From.Offset == 1:
  1489  			p.As = x86.AINCL
  1490  			p.From.Type = obj.TYPE_NONE
  1491  		case p.As == x86.AADDL && p.From.Offset == -1:
  1492  			p.As = x86.ADECL
  1493  			p.From.Type = obj.TYPE_NONE
  1494  		}
  1495  		p.To.Type = obj.TYPE_REG
  1496  		p.To.Reg = v.Reg0()
  1497  
  1498  	case ssa.OpAMD64BSFQ, ssa.OpAMD64BSRQ, ssa.OpAMD64BSFL, ssa.OpAMD64BSRL, ssa.OpAMD64SQRTSD, ssa.OpAMD64SQRTSS:
  1499  		p := s.Prog(v.Op.Asm())
  1500  		p.From.Type = obj.TYPE_REG
  1501  		p.From.Reg = v.Args[0].Reg()
  1502  		p.To.Type = obj.TYPE_REG
  1503  		switch v.Op {
  1504  		case ssa.OpAMD64BSFQ, ssa.OpAMD64BSRQ:
  1505  			p.To.Reg = v.Reg0()
  1506  		case ssa.OpAMD64BSFL, ssa.OpAMD64BSRL, ssa.OpAMD64SQRTSD, ssa.OpAMD64SQRTSS:
  1507  			p.To.Reg = v.Reg()
  1508  		}
  1509  	case ssa.OpAMD64LoweredRound32F, ssa.OpAMD64LoweredRound64F:
  1510  		// input is already rounded
  1511  	case ssa.OpAMD64ROUNDSD, ssa.OpAMD64ROUNDSS:
  1512  		p := s.Prog(v.Op.Asm())
  1513  		val := v.AuxInt
  1514  		// 0 means math.RoundToEven, 1 Floor, 2 Ceil, 3 Trunc
  1515  		if val < 0 || val > 3 {
  1516  			v.Fatalf("Invalid rounding mode")
  1517  		}
  1518  		p.From.Offset = val
  1519  		p.From.Type = obj.TYPE_CONST
  1520  		p.AddRestSourceReg(v.Args[0].Reg())
  1521  		p.To.Type = obj.TYPE_REG
  1522  		p.To.Reg = v.Reg()
  1523  	case ssa.OpAMD64POPCNTQ, ssa.OpAMD64POPCNTL,
  1524  		ssa.OpAMD64TZCNTQ, ssa.OpAMD64TZCNTL,
  1525  		ssa.OpAMD64LZCNTQ, ssa.OpAMD64LZCNTL:
  1526  		if v.Args[0].Reg() != v.Reg() {
  1527  			// POPCNT/TZCNT/LZCNT have a false dependency on the destination register on Intel cpus.
  1528  			// TZCNT/LZCNT problem affects pre-Skylake models. See discussion at https://gcc.gnu.org/bugzilla/show_bug.cgi?id=62011#c7.
  1529  			// Xor register with itself to break the dependency.
  1530  			opregreg(s, x86.AXORL, v.Reg(), v.Reg())
  1531  		}
  1532  		p := s.Prog(v.Op.Asm())
  1533  		p.From.Type = obj.TYPE_REG
  1534  		p.From.Reg = v.Args[0].Reg()
  1535  		p.To.Type = obj.TYPE_REG
  1536  		p.To.Reg = v.Reg()
  1537  
  1538  	case ssa.OpAMD64SETEQ, ssa.OpAMD64SETNE,
  1539  		ssa.OpAMD64SETL, ssa.OpAMD64SETLE,
  1540  		ssa.OpAMD64SETG, ssa.OpAMD64SETGE,
  1541  		ssa.OpAMD64SETGF, ssa.OpAMD64SETGEF,
  1542  		ssa.OpAMD64SETB, ssa.OpAMD64SETBE,
  1543  		ssa.OpAMD64SETORD, ssa.OpAMD64SETNAN,
  1544  		ssa.OpAMD64SETA, ssa.OpAMD64SETAE,
  1545  		ssa.OpAMD64SETO:
  1546  		p := s.Prog(v.Op.Asm())
  1547  		p.To.Type = obj.TYPE_REG
  1548  		p.To.Reg = v.Reg()
  1549  
  1550  	case ssa.OpAMD64SETEQstore, ssa.OpAMD64SETNEstore,
  1551  		ssa.OpAMD64SETLstore, ssa.OpAMD64SETLEstore,
  1552  		ssa.OpAMD64SETGstore, ssa.OpAMD64SETGEstore,
  1553  		ssa.OpAMD64SETBstore, ssa.OpAMD64SETBEstore,
  1554  		ssa.OpAMD64SETAstore, ssa.OpAMD64SETAEstore:
  1555  		p := s.Prog(v.Op.Asm())
  1556  		p.To.Type = obj.TYPE_MEM
  1557  		p.To.Reg = v.Args[0].Reg()
  1558  		ssagen.AddAux(&p.To, v)
  1559  
  1560  	case ssa.OpAMD64SETEQstoreidx1, ssa.OpAMD64SETNEstoreidx1,
  1561  		ssa.OpAMD64SETLstoreidx1, ssa.OpAMD64SETLEstoreidx1,
  1562  		ssa.OpAMD64SETGstoreidx1, ssa.OpAMD64SETGEstoreidx1,
  1563  		ssa.OpAMD64SETBstoreidx1, ssa.OpAMD64SETBEstoreidx1,
  1564  		ssa.OpAMD64SETAstoreidx1, ssa.OpAMD64SETAEstoreidx1:
  1565  		p := s.Prog(v.Op.Asm())
  1566  		memIdx(&p.To, v)
  1567  		ssagen.AddAux(&p.To, v)
  1568  
  1569  	case ssa.OpAMD64SETNEF:
  1570  		t := v.RegTmp()
  1571  		p := s.Prog(v.Op.Asm())
  1572  		p.To.Type = obj.TYPE_REG
  1573  		p.To.Reg = v.Reg()
  1574  		q := s.Prog(x86.ASETPS)
  1575  		q.To.Type = obj.TYPE_REG
  1576  		q.To.Reg = t
  1577  		// ORL avoids partial register write and is smaller than ORQ, used by old compiler
  1578  		opregreg(s, x86.AORL, v.Reg(), t)
  1579  
  1580  	case ssa.OpAMD64SETEQF:
  1581  		t := v.RegTmp()
  1582  		p := s.Prog(v.Op.Asm())
  1583  		p.To.Type = obj.TYPE_REG
  1584  		p.To.Reg = v.Reg()
  1585  		q := s.Prog(x86.ASETPC)
  1586  		q.To.Type = obj.TYPE_REG
  1587  		q.To.Reg = t
  1588  		// ANDL avoids partial register write and is smaller than ANDQ, used by old compiler
  1589  		opregreg(s, x86.AANDL, v.Reg(), t)
  1590  
  1591  	case ssa.OpAMD64InvertFlags:
  1592  		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
  1593  	case ssa.OpAMD64FlagEQ, ssa.OpAMD64FlagLT_ULT, ssa.OpAMD64FlagLT_UGT, ssa.OpAMD64FlagGT_ULT, ssa.OpAMD64FlagGT_UGT:
  1594  		v.Fatalf("Flag* ops should never make it to codegen %v", v.LongString())
  1595  	case ssa.OpAMD64AddTupleFirst32, ssa.OpAMD64AddTupleFirst64:
  1596  		v.Fatalf("AddTupleFirst* should never make it to codegen %v", v.LongString())
  1597  	case ssa.OpAMD64REPSTOSQ:
  1598  		s.Prog(x86.AREP)
  1599  		s.Prog(x86.ASTOSQ)
  1600  	case ssa.OpAMD64REPMOVSQ:
  1601  		s.Prog(x86.AREP)
  1602  		s.Prog(x86.AMOVSQ)
  1603  	case ssa.OpAMD64LoweredNilCheck:
  1604  		// Issue a load which will fault if the input is nil.
  1605  		// TODO: We currently use the 2-byte instruction TESTB AX, (reg).
  1606  		// Should we use the 3-byte TESTB $0, (reg) instead? It is larger
  1607  		// but it doesn't have false dependency on AX.
  1608  		// Or maybe allocate an output register and use MOVL (reg),reg2 ?
  1609  		// That trades clobbering flags for clobbering a register.
  1610  		p := s.Prog(x86.ATESTB)
  1611  		p.From.Type = obj.TYPE_REG
  1612  		p.From.Reg = x86.REG_AX
  1613  		p.To.Type = obj.TYPE_MEM
  1614  		p.To.Reg = v.Args[0].Reg()
  1615  		if logopt.Enabled() {
  1616  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  1617  		}
  1618  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos.Line()==1 in generated wrappers
  1619  			base.WarnfAt(v.Pos, "generated nil check")
  1620  		}
  1621  	case ssa.OpAMD64MOVBatomicload, ssa.OpAMD64MOVLatomicload, ssa.OpAMD64MOVQatomicload:
  1622  		p := s.Prog(v.Op.Asm())
  1623  		p.From.Type = obj.TYPE_MEM
  1624  		p.From.Reg = v.Args[0].Reg()
  1625  		ssagen.AddAux(&p.From, v)
  1626  		p.To.Type = obj.TYPE_REG
  1627  		p.To.Reg = v.Reg0()
  1628  	case ssa.OpAMD64XCHGB, ssa.OpAMD64XCHGL, ssa.OpAMD64XCHGQ:
  1629  		p := s.Prog(v.Op.Asm())
  1630  		p.From.Type = obj.TYPE_REG
  1631  		p.From.Reg = v.Reg0()
  1632  		p.To.Type = obj.TYPE_MEM
  1633  		p.To.Reg = v.Args[1].Reg()
  1634  		ssagen.AddAux(&p.To, v)
  1635  	case ssa.OpAMD64XADDLlock, ssa.OpAMD64XADDQlock:
  1636  		s.Prog(x86.ALOCK)
  1637  		p := s.Prog(v.Op.Asm())
  1638  		p.From.Type = obj.TYPE_REG
  1639  		p.From.Reg = v.Reg0()
  1640  		p.To.Type = obj.TYPE_MEM
  1641  		p.To.Reg = v.Args[1].Reg()
  1642  		ssagen.AddAux(&p.To, v)
  1643  	case ssa.OpAMD64CMPXCHGLlock, ssa.OpAMD64CMPXCHGQlock:
  1644  		if v.Args[1].Reg() != x86.REG_AX {
  1645  			v.Fatalf("input[1] not in AX %s", v.LongString())
  1646  		}
  1647  		s.Prog(x86.ALOCK)
  1648  		p := s.Prog(v.Op.Asm())
  1649  		p.From.Type = obj.TYPE_REG
  1650  		p.From.Reg = v.Args[2].Reg()
  1651  		p.To.Type = obj.TYPE_MEM
  1652  		p.To.Reg = v.Args[0].Reg()
  1653  		ssagen.AddAux(&p.To, v)
  1654  		p = s.Prog(x86.ASETEQ)
  1655  		p.To.Type = obj.TYPE_REG
  1656  		p.To.Reg = v.Reg0()
  1657  	case ssa.OpAMD64ANDBlock, ssa.OpAMD64ANDLlock, ssa.OpAMD64ANDQlock, ssa.OpAMD64ORBlock, ssa.OpAMD64ORLlock, ssa.OpAMD64ORQlock:
  1658  		// Atomic memory operations that don't need to return the old value.
  1659  		s.Prog(x86.ALOCK)
  1660  		p := s.Prog(v.Op.Asm())
  1661  		p.From.Type = obj.TYPE_REG
  1662  		p.From.Reg = v.Args[1].Reg()
  1663  		p.To.Type = obj.TYPE_MEM
  1664  		p.To.Reg = v.Args[0].Reg()
  1665  		ssagen.AddAux(&p.To, v)
  1666  	case ssa.OpAMD64LoweredAtomicAnd64, ssa.OpAMD64LoweredAtomicOr64, ssa.OpAMD64LoweredAtomicAnd32, ssa.OpAMD64LoweredAtomicOr32:
  1667  		// Atomic memory operations that need to return the old value.
  1668  		// We need to do these with compare-and-exchange to get access to the old value.
  1669  		// loop:
  1670  		// MOVQ mask, tmp
  1671  		// MOVQ (addr), AX
  1672  		// ANDQ AX, tmp
  1673  		// LOCK CMPXCHGQ tmp, (addr) : note that AX is implicit old value to compare against
  1674  		// JNE loop
  1675  		// : result in AX
  1676  		mov := x86.AMOVQ
  1677  		op := x86.AANDQ
  1678  		cmpxchg := x86.ACMPXCHGQ
  1679  		switch v.Op {
  1680  		case ssa.OpAMD64LoweredAtomicOr64:
  1681  			op = x86.AORQ
  1682  		case ssa.OpAMD64LoweredAtomicAnd32:
  1683  			mov = x86.AMOVL
  1684  			op = x86.AANDL
  1685  			cmpxchg = x86.ACMPXCHGL
  1686  		case ssa.OpAMD64LoweredAtomicOr32:
  1687  			mov = x86.AMOVL
  1688  			op = x86.AORL
  1689  			cmpxchg = x86.ACMPXCHGL
  1690  		}
  1691  		addr := v.Args[0].Reg()
  1692  		mask := v.Args[1].Reg()
  1693  		tmp := v.RegTmp()
  1694  		p1 := s.Prog(mov)
  1695  		p1.From.Type = obj.TYPE_REG
  1696  		p1.From.Reg = mask
  1697  		p1.To.Type = obj.TYPE_REG
  1698  		p1.To.Reg = tmp
  1699  		p2 := s.Prog(mov)
  1700  		p2.From.Type = obj.TYPE_MEM
  1701  		p2.From.Reg = addr
  1702  		ssagen.AddAux(&p2.From, v)
  1703  		p2.To.Type = obj.TYPE_REG
  1704  		p2.To.Reg = x86.REG_AX
  1705  		p3 := s.Prog(op)
  1706  		p3.From.Type = obj.TYPE_REG
  1707  		p3.From.Reg = x86.REG_AX
  1708  		p3.To.Type = obj.TYPE_REG
  1709  		p3.To.Reg = tmp
  1710  		s.Prog(x86.ALOCK)
  1711  		p5 := s.Prog(cmpxchg)
  1712  		p5.From.Type = obj.TYPE_REG
  1713  		p5.From.Reg = tmp
  1714  		p5.To.Type = obj.TYPE_MEM
  1715  		p5.To.Reg = addr
  1716  		ssagen.AddAux(&p5.To, v)
  1717  		p6 := s.Prog(x86.AJNE)
  1718  		p6.To.Type = obj.TYPE_BRANCH
  1719  		p6.To.SetTarget(p1)
  1720  	case ssa.OpAMD64PrefetchT0, ssa.OpAMD64PrefetchNTA:
  1721  		p := s.Prog(v.Op.Asm())
  1722  		p.From.Type = obj.TYPE_MEM
  1723  		p.From.Reg = v.Args[0].Reg()
  1724  	case ssa.OpClobber:
  1725  		p := s.Prog(x86.AMOVL)
  1726  		p.From.Type = obj.TYPE_CONST
  1727  		p.From.Offset = 0xdeaddead
  1728  		p.To.Type = obj.TYPE_MEM
  1729  		p.To.Reg = x86.REG_SP
  1730  		ssagen.AddAux(&p.To, v)
  1731  		p = s.Prog(x86.AMOVL)
  1732  		p.From.Type = obj.TYPE_CONST
  1733  		p.From.Offset = 0xdeaddead
  1734  		p.To.Type = obj.TYPE_MEM
  1735  		p.To.Reg = x86.REG_SP
  1736  		ssagen.AddAux(&p.To, v)
  1737  		p.To.Offset += 4
  1738  	case ssa.OpClobberReg:
  1739  		x := uint64(0xdeaddeaddeaddead)
  1740  		p := s.Prog(x86.AMOVQ)
  1741  		p.From.Type = obj.TYPE_CONST
  1742  		p.From.Offset = int64(x)
  1743  		p.To.Type = obj.TYPE_REG
  1744  		p.To.Reg = v.Reg()
  1745  
  1746  	// SIMD ops
  1747  	case ssa.OpAMD64VZEROUPPER, ssa.OpAMD64VZEROALL:
  1748  		s.Prog(v.Op.Asm())
  1749  
  1750  	case ssa.OpAMD64Zero128: // no code emitted
  1751  
  1752  	case ssa.OpAMD64Zero256, ssa.OpAMD64Zero512:
  1753  		p := s.Prog(v.Op.Asm())
  1754  		p.From.Type = obj.TYPE_REG
  1755  		p.From.Reg = simdReg(v)
  1756  		p.AddRestSourceReg(simdReg(v))
  1757  		p.To.Type = obj.TYPE_REG
  1758  		p.To.Reg = simdReg(v)
  1759  
  1760  	case ssa.OpAMD64VMOVSSf2v, ssa.OpAMD64VMOVSDf2v:
  1761  		// These are for initializing the least 32/64 bits of a SIMD register from a "float".
  1762  		p := s.Prog(v.Op.Asm())
  1763  		p.From.Type = obj.TYPE_REG
  1764  		p.From.Reg = v.Args[0].Reg()
  1765  		p.AddRestSourceReg(x86.REG_X15)
  1766  		p.To.Type = obj.TYPE_REG
  1767  		p.To.Reg = simdReg(v)
  1768  
  1769  	case ssa.OpAMD64VMOVQload, ssa.OpAMD64VMOVDload,
  1770  		ssa.OpAMD64VMOVSSload, ssa.OpAMD64VMOVSDload:
  1771  		p := s.Prog(v.Op.Asm())
  1772  		p.From.Type = obj.TYPE_MEM
  1773  		p.From.Reg = v.Args[0].Reg()
  1774  		ssagen.AddAux(&p.From, v)
  1775  		p.To.Type = obj.TYPE_REG
  1776  		p.To.Reg = simdReg(v)
  1777  
  1778  	case ssa.OpAMD64VMOVSSconst, ssa.OpAMD64VMOVSDconst:
  1779  		// for loading constants directly into SIMD registers
  1780  		x := simdReg(v)
  1781  		p := s.Prog(v.Op.Asm())
  1782  		p.From.Type = obj.TYPE_FCONST
  1783  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
  1784  		p.To.Type = obj.TYPE_REG
  1785  		p.To.Reg = x
  1786  
  1787  	case ssa.OpAMD64VMOVD, ssa.OpAMD64VMOVQ:
  1788  		// These are for initializing the least 32/64 bits of a SIMD register from an "int".
  1789  		p := s.Prog(v.Op.Asm())
  1790  		p.From.Type = obj.TYPE_REG
  1791  		p.From.Reg = v.Args[0].Reg()
  1792  		p.To.Type = obj.TYPE_REG
  1793  		p.To.Reg = simdReg(v)
  1794  
  1795  	case ssa.OpAMD64VMOVDQUload128, ssa.OpAMD64VMOVDQUload256, ssa.OpAMD64VMOVDQUload512,
  1796  		ssa.OpAMD64KMOVBload, ssa.OpAMD64KMOVWload, ssa.OpAMD64KMOVDload, ssa.OpAMD64KMOVQload:
  1797  		p := s.Prog(v.Op.Asm())
  1798  		p.From.Type = obj.TYPE_MEM
  1799  		p.From.Reg = v.Args[0].Reg()
  1800  		ssagen.AddAux(&p.From, v)
  1801  		p.To.Type = obj.TYPE_REG
  1802  		p.To.Reg = simdOrMaskReg(v)
  1803  	case ssa.OpAMD64VMOVDQUstore128, ssa.OpAMD64VMOVDQUstore256, ssa.OpAMD64VMOVDQUstore512,
  1804  		ssa.OpAMD64KMOVBstore, ssa.OpAMD64KMOVWstore, ssa.OpAMD64KMOVDstore, ssa.OpAMD64KMOVQstore:
  1805  		p := s.Prog(v.Op.Asm())
  1806  		p.From.Type = obj.TYPE_REG
  1807  		p.From.Reg = simdOrMaskReg(v.Args[1])
  1808  		p.To.Type = obj.TYPE_MEM
  1809  		p.To.Reg = v.Args[0].Reg()
  1810  		ssagen.AddAux(&p.To, v)
  1811  
  1812  	case ssa.OpAMD64VPMASK32load128, ssa.OpAMD64VPMASK64load128, ssa.OpAMD64VPMASK32load256, ssa.OpAMD64VPMASK64load256:
  1813  		p := s.Prog(v.Op.Asm())
  1814  		p.From.Type = obj.TYPE_MEM
  1815  		p.From.Reg = v.Args[0].Reg()
  1816  		ssagen.AddAux(&p.From, v)
  1817  		p.To.Type = obj.TYPE_REG
  1818  		p.To.Reg = simdReg(v)
  1819  		p.AddRestSourceReg(simdReg(v.Args[1])) // masking simd reg
  1820  
  1821  	case ssa.OpAMD64VPMASK32store128, ssa.OpAMD64VPMASK64store128, ssa.OpAMD64VPMASK32store256, ssa.OpAMD64VPMASK64store256:
  1822  		p := s.Prog(v.Op.Asm())
  1823  		p.From.Type = obj.TYPE_REG
  1824  		p.From.Reg = simdReg(v.Args[2])
  1825  		p.To.Type = obj.TYPE_MEM
  1826  		p.To.Reg = v.Args[0].Reg()
  1827  		ssagen.AddAux(&p.To, v)
  1828  		p.AddRestSourceReg(simdReg(v.Args[1])) // masking simd reg
  1829  
  1830  	case ssa.OpAMD64VPMASK64load512, ssa.OpAMD64VPMASK32load512, ssa.OpAMD64VPMASK16load512, ssa.OpAMD64VPMASK8load512:
  1831  		p := s.Prog(v.Op.Asm())
  1832  		p.From.Type = obj.TYPE_MEM
  1833  		p.From.Reg = v.Args[0].Reg()
  1834  		ssagen.AddAux(&p.From, v)
  1835  		p.To.Type = obj.TYPE_REG
  1836  		p.To.Reg = simdReg(v)
  1837  		p.AddRestSourceReg(v.Args[1].Reg()) // simd mask reg
  1838  		x86.ParseSuffix(p, "Z")             // must be zero if not in mask
  1839  
  1840  	case ssa.OpAMD64KANDB, ssa.OpAMD64KANDW, ssa.OpAMD64KANDD, ssa.OpAMD64KANDQ,
  1841  		ssa.OpAMD64KORB, ssa.OpAMD64KORW, ssa.OpAMD64KORD, ssa.OpAMD64KORQ,
  1842  		ssa.OpAMD64KXORB, ssa.OpAMD64KXORW, ssa.OpAMD64KXORD, ssa.OpAMD64KXORQ,
  1843  		ssa.OpAMD64KXNORB, ssa.OpAMD64KXNORW, ssa.OpAMD64KXNORD, ssa.OpAMD64KXNORQ: // XNOR == EQ
  1844  		p := s.Prog(v.Op.Asm())
  1845  		p.From.Type = obj.TYPE_REG
  1846  		p.From.Reg = v.Args[0].Reg()
  1847  		p.To.Type = obj.TYPE_REG
  1848  		p.To.Reg = v.Reg()
  1849  		p.AddRestSourceReg(v.Args[1].Reg()) // masking simd reg
  1850  
  1851  	case ssa.OpAMD64VPMASK64store512, ssa.OpAMD64VPMASK32store512, ssa.OpAMD64VPMASK16store512, ssa.OpAMD64VPMASK8store512:
  1852  		p := s.Prog(v.Op.Asm())
  1853  		p.From.Type = obj.TYPE_REG
  1854  		p.From.Reg = simdReg(v.Args[2])
  1855  		p.To.Type = obj.TYPE_MEM
  1856  		p.To.Reg = v.Args[0].Reg()
  1857  		ssagen.AddAux(&p.To, v)
  1858  		p.AddRestSourceReg(v.Args[1].Reg()) // simd mask reg
  1859  
  1860  	case ssa.OpAMD64VPMOVMToVec8x16,
  1861  		ssa.OpAMD64VPMOVMToVec8x32,
  1862  		ssa.OpAMD64VPMOVMToVec8x64,
  1863  		ssa.OpAMD64VPMOVMToVec16x8,
  1864  		ssa.OpAMD64VPMOVMToVec16x16,
  1865  		ssa.OpAMD64VPMOVMToVec16x32,
  1866  		ssa.OpAMD64VPMOVMToVec32x4,
  1867  		ssa.OpAMD64VPMOVMToVec32x8,
  1868  		ssa.OpAMD64VPMOVMToVec32x16,
  1869  		ssa.OpAMD64VPMOVMToVec64x2,
  1870  		ssa.OpAMD64VPMOVMToVec64x4,
  1871  		ssa.OpAMD64VPMOVMToVec64x8:
  1872  		p := s.Prog(v.Op.Asm())
  1873  		p.From.Type = obj.TYPE_REG
  1874  		p.From.Reg = v.Args[0].Reg()
  1875  		p.To.Type = obj.TYPE_REG
  1876  		p.To.Reg = simdReg(v)
  1877  
  1878  	case ssa.OpAMD64VPMOVVec8x16ToM,
  1879  		ssa.OpAMD64VPMOVVec8x32ToM,
  1880  		ssa.OpAMD64VPMOVVec8x64ToM,
  1881  		ssa.OpAMD64VPMOVVec16x8ToM,
  1882  		ssa.OpAMD64VPMOVVec16x16ToM,
  1883  		ssa.OpAMD64VPMOVVec16x32ToM,
  1884  		ssa.OpAMD64VPMOVVec32x4ToM,
  1885  		ssa.OpAMD64VPMOVVec32x8ToM,
  1886  		ssa.OpAMD64VPMOVVec32x16ToM,
  1887  		ssa.OpAMD64VPMOVVec64x2ToM,
  1888  		ssa.OpAMD64VPMOVVec64x4ToM,
  1889  		ssa.OpAMD64VPMOVVec64x8ToM,
  1890  		ssa.OpAMD64VPMOVMSKB128,
  1891  		ssa.OpAMD64VPMOVMSKB256,
  1892  		ssa.OpAMD64VMOVMSKPS128,
  1893  		ssa.OpAMD64VMOVMSKPS256,
  1894  		ssa.OpAMD64VMOVMSKPD128,
  1895  		ssa.OpAMD64VMOVMSKPD256:
  1896  		p := s.Prog(v.Op.Asm())
  1897  		p.From.Type = obj.TYPE_REG
  1898  		p.From.Reg = simdReg(v.Args[0])
  1899  		p.To.Type = obj.TYPE_REG
  1900  		p.To.Reg = v.Reg()
  1901  
  1902  	case ssa.OpAMD64KMOVQk, ssa.OpAMD64KMOVDk, ssa.OpAMD64KMOVWk, ssa.OpAMD64KMOVBk,
  1903  		ssa.OpAMD64KMOVQi, ssa.OpAMD64KMOVDi, ssa.OpAMD64KMOVWi, ssa.OpAMD64KMOVBi:
  1904  		// See also ssa.OpAMD64KMOVQload
  1905  		p := s.Prog(v.Op.Asm())
  1906  		p.From.Type = obj.TYPE_REG
  1907  		p.From.Reg = v.Args[0].Reg()
  1908  		p.To.Type = obj.TYPE_REG
  1909  		p.To.Reg = v.Reg()
  1910  	case ssa.OpAMD64VPTEST:
  1911  		// Some instructions setting flags put their second operand into the destination reg.
  1912  		// See also CMP[BWDQ].
  1913  		p := s.Prog(v.Op.Asm())
  1914  		p.From.Type = obj.TYPE_REG
  1915  		p.From.Reg = simdReg(v.Args[0])
  1916  		p.To.Type = obj.TYPE_REG
  1917  		p.To.Reg = simdReg(v.Args[1])
  1918  
  1919  	default:
  1920  		if !ssaGenSIMDValue(s, v) {
  1921  			v.Fatalf("genValue not implemented: %s", v.LongString())
  1922  		}
  1923  	}
  1924  }
  1925  
  1926  // zeroX15 zeroes the X15 register.
  1927  func zeroX15(s *ssagen.State) {
  1928  	opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  1929  }
  1930  
  1931  // Example instruction: VRSQRTPS X1, X1
  1932  func simdV11(s *ssagen.State, v *ssa.Value) *obj.Prog {
  1933  	p := s.Prog(v.Op.Asm())
  1934  	p.From.Type = obj.TYPE_REG
  1935  	p.From.Reg = simdReg(v.Args[0])
  1936  	p.To.Type = obj.TYPE_REG
  1937  	p.To.Reg = simdReg(v)
  1938  	return p
  1939  }
  1940  
  1941  // Example instruction: VPSUBD X1, X2, X3
  1942  func simdV21(s *ssagen.State, v *ssa.Value) *obj.Prog {
  1943  	p := s.Prog(v.Op.Asm())
  1944  	p.From.Type = obj.TYPE_REG
  1945  	// Vector registers operands follows a right-to-left order.
  1946  	// e.g. VPSUBD X1, X2, X3 means X3 = X2 - X1.
  1947  	p.From.Reg = simdReg(v.Args[1])
  1948  	p.AddRestSourceReg(simdReg(v.Args[0]))
  1949  	p.To.Type = obj.TYPE_REG
  1950  	p.To.Reg = simdReg(v)
  1951  	return p
  1952  }
  1953  
  1954  // This function is to accustomize the shifts.
  1955  // The 2nd arg is an XMM, and this function merely checks that.
  1956  // Example instruction: VPSLLQ Z1, X1, Z2
  1957  func simdVfpv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  1958  	p := s.Prog(v.Op.Asm())
  1959  	p.From.Type = obj.TYPE_REG
  1960  	// Vector registers operands follows a right-to-left order.
  1961  	// e.g. VPSUBD X1, X2, X3 means X3 = X2 - X1.
  1962  	p.From.Reg = v.Args[1].Reg()
  1963  	p.AddRestSourceReg(simdReg(v.Args[0]))
  1964  	p.To.Type = obj.TYPE_REG
  1965  	p.To.Reg = simdReg(v)
  1966  	return p
  1967  }
  1968  
  1969  // Example instruction: VPCMPEQW Z26, Z30, K4
  1970  func simdV2k(s *ssagen.State, v *ssa.Value) *obj.Prog {
  1971  	p := s.Prog(v.Op.Asm())
  1972  	p.From.Type = obj.TYPE_REG
  1973  	p.From.Reg = simdReg(v.Args[1])
  1974  	p.AddRestSourceReg(simdReg(v.Args[0]))
  1975  	p.To.Type = obj.TYPE_REG
  1976  	p.To.Reg = maskReg(v)
  1977  	return p
  1978  }
  1979  
  1980  // Example instruction: VPMINUQ X21, X3, K3, X31
  1981  func simdV2kv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  1982  	p := s.Prog(v.Op.Asm())
  1983  	p.From.Type = obj.TYPE_REG
  1984  	p.From.Reg = simdReg(v.Args[1])
  1985  	p.AddRestSourceReg(simdReg(v.Args[0]))
  1986  	// These "simd*" series of functions assumes:
  1987  	// Any "K" register that serves as the write-mask
  1988  	// or "predicate" for "predicated AVX512 instructions"
  1989  	// sits right at the end of the operand list.
  1990  	// TODO: verify this assumption.
  1991  	p.AddRestSourceReg(maskReg(v.Args[2]))
  1992  	p.To.Type = obj.TYPE_REG
  1993  	p.To.Reg = simdReg(v)
  1994  	return p
  1995  }
  1996  
  1997  // Example instruction: VPABSB X1, X2, K3 (masking merging)
  1998  func simdV2kvResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  1999  	p := s.Prog(v.Op.Asm())
  2000  	p.From.Type = obj.TYPE_REG
  2001  	p.From.Reg = simdReg(v.Args[1])
  2002  	// These "simd*" series of functions assumes:
  2003  	// Any "K" register that serves as the write-mask
  2004  	// or "predicate" for "predicated AVX512 instructions"
  2005  	// sits right at the end of the operand list.
  2006  	// TODO: verify this assumption.
  2007  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2008  	p.To.Type = obj.TYPE_REG
  2009  	p.To.Reg = simdReg(v)
  2010  	return p
  2011  }
  2012  
  2013  // This function is to accustomize the shifts.
  2014  // The 2nd arg is an XMM, and this function merely checks that.
  2015  // Example instruction: VPSLLQ Z1, X1, K1, Z2
  2016  func simdVfpkv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2017  	p := s.Prog(v.Op.Asm())
  2018  	p.From.Type = obj.TYPE_REG
  2019  	p.From.Reg = v.Args[1].Reg()
  2020  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2021  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2022  	p.To.Type = obj.TYPE_REG
  2023  	p.To.Reg = simdReg(v)
  2024  	return p
  2025  }
  2026  
  2027  // Example instruction: VPCMPEQW Z26, Z30, K1, K4
  2028  func simdV2kk(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2029  	p := s.Prog(v.Op.Asm())
  2030  	p.From.Type = obj.TYPE_REG
  2031  	p.From.Reg = simdReg(v.Args[1])
  2032  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2033  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2034  	p.To.Type = obj.TYPE_REG
  2035  	p.To.Reg = maskReg(v)
  2036  	return p
  2037  }
  2038  
  2039  // Example instruction: VPOPCNTB X14, K4, X16
  2040  func simdVkv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2041  	p := s.Prog(v.Op.Asm())
  2042  	p.From.Type = obj.TYPE_REG
  2043  	p.From.Reg = simdReg(v.Args[0])
  2044  	p.AddRestSourceReg(maskReg(v.Args[1]))
  2045  	p.To.Type = obj.TYPE_REG
  2046  	p.To.Reg = simdReg(v)
  2047  	return p
  2048  }
  2049  
  2050  // Example instruction: VROUNDPD $7, X2, X2
  2051  func simdV11Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2052  	p := s.Prog(v.Op.Asm())
  2053  	p.From.Offset = int64(v.AuxUInt8())
  2054  	p.From.Type = obj.TYPE_CONST
  2055  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2056  	p.To.Type = obj.TYPE_REG
  2057  	p.To.Reg = simdReg(v)
  2058  	return p
  2059  }
  2060  
  2061  // Example instruction: VREDUCEPD $126, X1, K3, X31
  2062  func simdVkvImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2063  	p := s.Prog(v.Op.Asm())
  2064  	p.From.Offset = int64(v.AuxUInt8())
  2065  	p.From.Type = obj.TYPE_CONST
  2066  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2067  	p.AddRestSourceReg(maskReg(v.Args[1]))
  2068  	p.To.Type = obj.TYPE_REG
  2069  	p.To.Reg = simdReg(v)
  2070  	return p
  2071  }
  2072  
  2073  // Example instruction: VCMPPS $7, X2, X9, X2
  2074  func simdV21Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2075  	p := s.Prog(v.Op.Asm())
  2076  	p.From.Offset = int64(v.AuxUInt8())
  2077  	p.From.Type = obj.TYPE_CONST
  2078  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2079  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2080  	p.To.Type = obj.TYPE_REG
  2081  	p.To.Reg = simdReg(v)
  2082  	return p
  2083  }
  2084  
  2085  // Example instruction: VPINSRB $3, DX, X0, X0
  2086  func simdVgpvImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2087  	p := s.Prog(v.Op.Asm())
  2088  	p.From.Offset = int64(v.AuxUInt8())
  2089  	p.From.Type = obj.TYPE_CONST
  2090  	p.AddRestSourceReg(v.Args[1].Reg())
  2091  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2092  	p.To.Type = obj.TYPE_REG
  2093  	p.To.Reg = simdReg(v)
  2094  	return p
  2095  }
  2096  func simdVgpvImm(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2097  	// within simdgen, the choice of intrinsic shape and the output
  2098  	// intruction format are linked.  In the case of VgpImm, there is
  2099  	// a difference in the intrinsic, but no difference in the
  2100  	// instruction, it is just like VgpvImm8.
  2101  	//
  2102  	// See also, simdVgpImm and simdVgpImm8
  2103  	return simdVgpvImm8(s, v)
  2104  }
  2105  
  2106  // Example instruction: VPCMPD $1, Z1, Z2, K1
  2107  func simdV2kImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2108  	p := s.Prog(v.Op.Asm())
  2109  	p.From.Offset = int64(v.AuxUInt8())
  2110  	p.From.Type = obj.TYPE_CONST
  2111  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2112  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2113  	p.To.Type = obj.TYPE_REG
  2114  	p.To.Reg = maskReg(v)
  2115  	return p
  2116  }
  2117  
  2118  // Example instruction: VPCMPD $1, Z1, Z2, K2, K1
  2119  func simdV2kkImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2120  	p := s.Prog(v.Op.Asm())
  2121  	p.From.Offset = int64(v.AuxUInt8())
  2122  	p.From.Type = obj.TYPE_CONST
  2123  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2124  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2125  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2126  	p.To.Type = obj.TYPE_REG
  2127  	p.To.Reg = maskReg(v)
  2128  	return p
  2129  }
  2130  
  2131  func simdV2kvImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2132  	p := s.Prog(v.Op.Asm())
  2133  	p.From.Offset = int64(v.AuxUInt8())
  2134  	p.From.Type = obj.TYPE_CONST
  2135  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2136  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2137  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2138  	p.To.Type = obj.TYPE_REG
  2139  	p.To.Reg = simdReg(v)
  2140  	return p
  2141  }
  2142  
  2143  // Example instruction: VFMADD213PD Z2, Z1, Z0
  2144  func simdV31ResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2145  	p := s.Prog(v.Op.Asm())
  2146  	p.From.Type = obj.TYPE_REG
  2147  	p.From.Reg = simdReg(v.Args[2])
  2148  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2149  	p.To.Type = obj.TYPE_REG
  2150  	p.To.Reg = simdReg(v)
  2151  	return p
  2152  }
  2153  
  2154  func simdV31ResultInArg0Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2155  	p := s.Prog(v.Op.Asm())
  2156  	p.From.Offset = int64(v.AuxUInt8())
  2157  	p.From.Type = obj.TYPE_CONST
  2158  
  2159  	p.AddRestSourceReg(simdReg(v.Args[2]))
  2160  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2161  	// p.AddRestSourceReg(x86.REG_K0)
  2162  	p.To.Type = obj.TYPE_REG
  2163  	p.To.Reg = simdReg(v)
  2164  	return p
  2165  }
  2166  
  2167  // v31loadResultInArg0Imm8
  2168  // Example instruction:
  2169  // for (VPTERNLOGD128load {sym} [makeValAndOff(int32(int8(c)),off)]  x y ptr mem)
  2170  func simdV31loadResultInArg0Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2171  	sc := v.AuxValAndOff()
  2172  	p := s.Prog(v.Op.Asm())
  2173  
  2174  	p.From.Type = obj.TYPE_CONST
  2175  	p.From.Offset = sc.Val64()
  2176  
  2177  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[2].Reg()}
  2178  	ssagen.AddAux2(&m, v, sc.Off64())
  2179  	p.AddRestSource(m)
  2180  
  2181  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2182  	return p
  2183  }
  2184  
  2185  // Example instruction: VFMADD213PD Z2, Z1, K1, Z0
  2186  func simdV3kvResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2187  	p := s.Prog(v.Op.Asm())
  2188  	p.From.Type = obj.TYPE_REG
  2189  	p.From.Reg = simdReg(v.Args[2])
  2190  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2191  	p.AddRestSourceReg(maskReg(v.Args[3]))
  2192  	p.To.Type = obj.TYPE_REG
  2193  	p.To.Reg = simdReg(v)
  2194  	return p
  2195  }
  2196  
  2197  func simdVgpImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2198  	p := s.Prog(v.Op.Asm())
  2199  	p.From.Offset = int64(v.AuxUInt8())
  2200  	p.From.Type = obj.TYPE_CONST
  2201  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2202  	p.To.Type = obj.TYPE_REG
  2203  	p.To.Reg = v.Reg()
  2204  	return p
  2205  }
  2206  
  2207  func simdVgpImm(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2208  	// within simdgen, the choice of intrinsic shape and the output
  2209  	// intruction format are linked.  In the case of VgpImm, there is
  2210  	// a difference in the intrinsic, but no difference in the
  2211  	// instruction, it is just like VgpImm8.
  2212  	return simdVgpImm8(s, v)
  2213  }
  2214  
  2215  // Currently unused
  2216  func simdV31(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2217  	p := s.Prog(v.Op.Asm())
  2218  	p.From.Type = obj.TYPE_REG
  2219  	p.From.Reg = simdReg(v.Args[2])
  2220  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2221  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2222  	p.To.Type = obj.TYPE_REG
  2223  	p.To.Reg = simdReg(v)
  2224  	return p
  2225  }
  2226  
  2227  // Currently unused
  2228  func simdV3kv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2229  	p := s.Prog(v.Op.Asm())
  2230  	p.From.Type = obj.TYPE_REG
  2231  	p.From.Reg = simdReg(v.Args[2])
  2232  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2233  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2234  	p.AddRestSourceReg(maskReg(v.Args[3]))
  2235  	p.To.Type = obj.TYPE_REG
  2236  	p.To.Reg = simdReg(v)
  2237  	return p
  2238  }
  2239  
  2240  // Example instruction: VRCP14PS (DI), K6, X22
  2241  func simdVkvload(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2242  	p := s.Prog(v.Op.Asm())
  2243  	p.From.Type = obj.TYPE_MEM
  2244  	p.From.Reg = v.Args[0].Reg()
  2245  	ssagen.AddAux(&p.From, v)
  2246  	p.AddRestSourceReg(maskReg(v.Args[1]))
  2247  	p.To.Type = obj.TYPE_REG
  2248  	p.To.Reg = simdReg(v)
  2249  	return p
  2250  }
  2251  
  2252  // Example instruction: VPSLLVD (DX), X7, X18
  2253  func simdV21load(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2254  	p := s.Prog(v.Op.Asm())
  2255  	p.From.Type = obj.TYPE_MEM
  2256  	p.From.Reg = v.Args[1].Reg()
  2257  	ssagen.AddAux(&p.From, v)
  2258  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2259  	p.To.Type = obj.TYPE_REG
  2260  	p.To.Reg = simdReg(v)
  2261  	return p
  2262  }
  2263  
  2264  // Example instruction: VPDPWSSD (SI), X24, X18
  2265  func simdV31loadResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2266  	p := s.Prog(v.Op.Asm())
  2267  	p.From.Type = obj.TYPE_MEM
  2268  	p.From.Reg = v.Args[2].Reg()
  2269  	ssagen.AddAux(&p.From, v)
  2270  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2271  	p.To.Type = obj.TYPE_REG
  2272  	p.To.Reg = simdReg(v)
  2273  	return p
  2274  }
  2275  
  2276  // Example instruction: VPDPWSSD (SI), X24, K1, X18
  2277  func simdV3kvloadResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2278  	p := s.Prog(v.Op.Asm())
  2279  	p.From.Type = obj.TYPE_MEM
  2280  	p.From.Reg = v.Args[2].Reg()
  2281  	ssagen.AddAux(&p.From, v)
  2282  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2283  	p.AddRestSourceReg(maskReg(v.Args[3]))
  2284  	p.To.Type = obj.TYPE_REG
  2285  	p.To.Reg = simdReg(v)
  2286  	return p
  2287  }
  2288  
  2289  // Example instruction: VPSLLVD (SI), X1, K1, X2
  2290  func simdV2kvload(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2291  	p := s.Prog(v.Op.Asm())
  2292  	p.From.Type = obj.TYPE_MEM
  2293  	p.From.Reg = v.Args[1].Reg()
  2294  	ssagen.AddAux(&p.From, v)
  2295  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2296  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2297  	p.To.Type = obj.TYPE_REG
  2298  	p.To.Reg = simdReg(v)
  2299  	return p
  2300  }
  2301  
  2302  // Example instruction: VPCMPEQD (SI), X1, K1
  2303  func simdV2kload(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2304  	p := s.Prog(v.Op.Asm())
  2305  	p.From.Type = obj.TYPE_MEM
  2306  	p.From.Reg = v.Args[1].Reg()
  2307  	ssagen.AddAux(&p.From, v)
  2308  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2309  	p.To.Type = obj.TYPE_REG
  2310  	p.To.Reg = maskReg(v)
  2311  	return p
  2312  }
  2313  
  2314  // Example instruction: VCVTTPS2DQ (BX), X2
  2315  func simdV11load(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2316  	p := s.Prog(v.Op.Asm())
  2317  	p.From.Type = obj.TYPE_MEM
  2318  	p.From.Reg = v.Args[0].Reg()
  2319  	ssagen.AddAux(&p.From, v)
  2320  	p.To.Type = obj.TYPE_REG
  2321  	p.To.Reg = simdReg(v)
  2322  	return p
  2323  }
  2324  
  2325  // Example instruction: VPSHUFD $7, (BX), X11
  2326  func simdV11loadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2327  	sc := v.AuxValAndOff()
  2328  	p := s.Prog(v.Op.Asm())
  2329  	p.From.Type = obj.TYPE_CONST
  2330  	p.From.Offset = sc.Val64()
  2331  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
  2332  	ssagen.AddAux2(&m, v, sc.Off64())
  2333  	p.AddRestSource(m)
  2334  	p.To.Type = obj.TYPE_REG
  2335  	p.To.Reg = simdReg(v)
  2336  	return p
  2337  }
  2338  
  2339  // Example instruction: VPRORD $81, -15(R14), K7, Y1
  2340  func simdVkvloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2341  	sc := v.AuxValAndOff()
  2342  	p := s.Prog(v.Op.Asm())
  2343  	p.From.Type = obj.TYPE_CONST
  2344  	p.From.Offset = sc.Val64()
  2345  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
  2346  	ssagen.AddAux2(&m, v, sc.Off64())
  2347  	p.AddRestSource(m)
  2348  	p.AddRestSourceReg(maskReg(v.Args[1]))
  2349  	p.To.Type = obj.TYPE_REG
  2350  	p.To.Reg = simdReg(v)
  2351  	return p
  2352  }
  2353  
  2354  // Example instruction: VPSHLDD $82, 7(SI), Y21, Y3
  2355  func simdV21loadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2356  	sc := v.AuxValAndOff()
  2357  	p := s.Prog(v.Op.Asm())
  2358  	p.From.Type = obj.TYPE_CONST
  2359  	p.From.Offset = sc.Val64()
  2360  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
  2361  	ssagen.AddAux2(&m, v, sc.Off64())
  2362  	p.AddRestSource(m)
  2363  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2364  	p.To.Type = obj.TYPE_REG
  2365  	p.To.Reg = simdReg(v)
  2366  	return p
  2367  }
  2368  
  2369  // Example instruction: VCMPPS $81, -7(DI), Y16, K3
  2370  func simdV2kloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2371  	sc := v.AuxValAndOff()
  2372  	p := s.Prog(v.Op.Asm())
  2373  	p.From.Type = obj.TYPE_CONST
  2374  	p.From.Offset = sc.Val64()
  2375  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
  2376  	ssagen.AddAux2(&m, v, sc.Off64())
  2377  	p.AddRestSource(m)
  2378  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2379  	p.To.Type = obj.TYPE_REG
  2380  	p.To.Reg = maskReg(v)
  2381  	return p
  2382  }
  2383  
  2384  // Example instruction: VCMPPS $81, -7(DI), Y16, K1, K3
  2385  func simdV2kkloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2386  	sc := v.AuxValAndOff()
  2387  	p := s.Prog(v.Op.Asm())
  2388  	p.From.Type = obj.TYPE_CONST
  2389  	p.From.Offset = sc.Val64()
  2390  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
  2391  	ssagen.AddAux2(&m, v, sc.Off64())
  2392  	p.AddRestSource(m)
  2393  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2394  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2395  	p.To.Type = obj.TYPE_REG
  2396  	p.To.Reg = maskReg(v)
  2397  	return p
  2398  }
  2399  
  2400  // Example instruction: VGF2P8AFFINEINVQB $64, -17(BP), X31, K3, X26
  2401  func simdV2kvloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2402  	sc := v.AuxValAndOff()
  2403  	p := s.Prog(v.Op.Asm())
  2404  	p.From.Type = obj.TYPE_CONST
  2405  	p.From.Offset = sc.Val64()
  2406  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
  2407  	ssagen.AddAux2(&m, v, sc.Off64())
  2408  	p.AddRestSource(m)
  2409  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2410  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2411  	p.To.Type = obj.TYPE_REG
  2412  	p.To.Reg = simdReg(v)
  2413  	return p
  2414  }
  2415  
  2416  // Example instruction: SHA1NEXTE X2, X2
  2417  func simdV21ResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2418  	p := s.Prog(v.Op.Asm())
  2419  	p.From.Type = obj.TYPE_REG
  2420  	p.From.Reg = simdReg(v.Args[1])
  2421  	p.To.Type = obj.TYPE_REG
  2422  	p.To.Reg = simdReg(v)
  2423  	return p
  2424  }
  2425  
  2426  // Example instruction: SHA1RNDS4 $1, X2, X2
  2427  func simdV21ResultInArg0Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2428  	p := s.Prog(v.Op.Asm())
  2429  	p.From.Offset = int64(v.AuxUInt8())
  2430  	p.From.Type = obj.TYPE_CONST
  2431  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2432  	p.To.Type = obj.TYPE_REG
  2433  	p.To.Reg = simdReg(v)
  2434  	return p
  2435  }
  2436  
  2437  // Example instruction: SHA256RNDS2 X0, X11, X2
  2438  func simdV31x0AtIn2ResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2439  	return simdV31ResultInArg0(s, v)
  2440  }
  2441  
  2442  var blockJump = [...]struct {
  2443  	asm, invasm obj.As
  2444  }{
  2445  	ssa.BlockAMD64EQ:  {x86.AJEQ, x86.AJNE},
  2446  	ssa.BlockAMD64NE:  {x86.AJNE, x86.AJEQ},
  2447  	ssa.BlockAMD64LT:  {x86.AJLT, x86.AJGE},
  2448  	ssa.BlockAMD64GE:  {x86.AJGE, x86.AJLT},
  2449  	ssa.BlockAMD64LE:  {x86.AJLE, x86.AJGT},
  2450  	ssa.BlockAMD64GT:  {x86.AJGT, x86.AJLE},
  2451  	ssa.BlockAMD64OS:  {x86.AJOS, x86.AJOC},
  2452  	ssa.BlockAMD64OC:  {x86.AJOC, x86.AJOS},
  2453  	ssa.BlockAMD64ULT: {x86.AJCS, x86.AJCC},
  2454  	ssa.BlockAMD64UGE: {x86.AJCC, x86.AJCS},
  2455  	ssa.BlockAMD64UGT: {x86.AJHI, x86.AJLS},
  2456  	ssa.BlockAMD64ULE: {x86.AJLS, x86.AJHI},
  2457  	ssa.BlockAMD64ORD: {x86.AJPC, x86.AJPS},
  2458  	ssa.BlockAMD64NAN: {x86.AJPS, x86.AJPC},
  2459  }
  2460  
  2461  var eqfJumps = [2][2]ssagen.IndexJump{
  2462  	{{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPS, Index: 1}}, // next == b.Succs[0]
  2463  	{{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPC, Index: 0}}, // next == b.Succs[1]
  2464  }
  2465  var nefJumps = [2][2]ssagen.IndexJump{
  2466  	{{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPC, Index: 1}}, // next == b.Succs[0]
  2467  	{{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPS, Index: 0}}, // next == b.Succs[1]
  2468  }
  2469  
  2470  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  2471  	switch b.Kind {
  2472  	case ssa.BlockPlain, ssa.BlockDefer:
  2473  		if b.Succs[0].Block() != next {
  2474  			p := s.Prog(obj.AJMP)
  2475  			p.To.Type = obj.TYPE_BRANCH
  2476  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  2477  		}
  2478  	case ssa.BlockExit, ssa.BlockRetJmp:
  2479  	case ssa.BlockRet:
  2480  		s.Prog(obj.ARET)
  2481  
  2482  	case ssa.BlockAMD64EQF:
  2483  		s.CombJump(b, next, &eqfJumps)
  2484  
  2485  	case ssa.BlockAMD64NEF:
  2486  		s.CombJump(b, next, &nefJumps)
  2487  
  2488  	case ssa.BlockAMD64EQ, ssa.BlockAMD64NE,
  2489  		ssa.BlockAMD64LT, ssa.BlockAMD64GE,
  2490  		ssa.BlockAMD64LE, ssa.BlockAMD64GT,
  2491  		ssa.BlockAMD64OS, ssa.BlockAMD64OC,
  2492  		ssa.BlockAMD64ULT, ssa.BlockAMD64UGT,
  2493  		ssa.BlockAMD64ULE, ssa.BlockAMD64UGE:
  2494  		jmp := blockJump[b.Kind]
  2495  		switch next {
  2496  		case b.Succs[0].Block():
  2497  			s.Br(jmp.invasm, b.Succs[1].Block())
  2498  		case b.Succs[1].Block():
  2499  			s.Br(jmp.asm, b.Succs[0].Block())
  2500  		default:
  2501  			if b.Likely != ssa.BranchUnlikely {
  2502  				s.Br(jmp.asm, b.Succs[0].Block())
  2503  				s.Br(obj.AJMP, b.Succs[1].Block())
  2504  			} else {
  2505  				s.Br(jmp.invasm, b.Succs[1].Block())
  2506  				s.Br(obj.AJMP, b.Succs[0].Block())
  2507  			}
  2508  		}
  2509  
  2510  	case ssa.BlockAMD64JUMPTABLE:
  2511  		// JMP      *(TABLE)(INDEX*8)
  2512  		p := s.Prog(obj.AJMP)
  2513  		p.To.Type = obj.TYPE_MEM
  2514  		p.To.Reg = b.Controls[1].Reg()
  2515  		p.To.Index = b.Controls[0].Reg()
  2516  		p.To.Scale = 8
  2517  		// Save jump tables for later resolution of the target blocks.
  2518  		s.JumpTables = append(s.JumpTables, b)
  2519  
  2520  	default:
  2521  		b.Fatalf("branch not implemented: %s", b.LongString())
  2522  	}
  2523  }
  2524  
  2525  func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2526  	p := s.Prog(loadByRegWidth(reg, t.Size()))
  2527  	p.From.Type = obj.TYPE_MEM
  2528  	p.From.Name = obj.NAME_AUTO
  2529  	p.From.Sym = n.Linksym()
  2530  	p.From.Offset = n.FrameOffset() + off
  2531  	p.To.Type = obj.TYPE_REG
  2532  	p.To.Reg = reg
  2533  	return p
  2534  }
  2535  
  2536  func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2537  	p = pp.Append(p, storeByRegWidth(reg, t.Size()), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
  2538  	p.To.Name = obj.NAME_PARAM
  2539  	p.To.Sym = n.Linksym()
  2540  	p.Pos = p.Pos.WithNotStmt()
  2541  	return p
  2542  }
  2543  
  2544  // zero 16 bytes at reg+off.
  2545  func zero16(s *ssagen.State, reg int16, off int64) {
  2546  	//   MOVUPS  X15, off(ptrReg)
  2547  	p := s.Prog(x86.AMOVUPS)
  2548  	p.From.Type = obj.TYPE_REG
  2549  	p.From.Reg = x86.REG_X15
  2550  	p.To.Type = obj.TYPE_MEM
  2551  	p.To.Reg = reg
  2552  	p.To.Offset = off
  2553  }
  2554  
  2555  // move 16 bytes from src+off to dst+off using temporary register tmp.
  2556  func move16(s *ssagen.State, src, dst, tmp int16, off int64) {
  2557  	//   MOVUPS  off(srcReg), tmpReg
  2558  	//   MOVUPS  tmpReg, off(dstReg)
  2559  	p := s.Prog(x86.AMOVUPS)
  2560  	p.From.Type = obj.TYPE_MEM
  2561  	p.From.Reg = src
  2562  	p.From.Offset = off
  2563  	p.To.Type = obj.TYPE_REG
  2564  	p.To.Reg = tmp
  2565  	p = s.Prog(x86.AMOVUPS)
  2566  	p.From.Type = obj.TYPE_REG
  2567  	p.From.Reg = tmp
  2568  	p.To.Type = obj.TYPE_MEM
  2569  	p.To.Reg = dst
  2570  	p.To.Offset = off
  2571  }
  2572  
  2573  // XXX maybe make this part of v.Reg?
  2574  // On the other hand, it is architecture-specific.
  2575  func simdReg(v *ssa.Value) int16 {
  2576  	t := v.Type
  2577  	if !t.IsSIMD() {
  2578  		base.Fatalf("simdReg: not a simd type; v=%s, b=b%d, f=%s", v.LongString(), v.Block.ID, v.Block.Func.Name)
  2579  	}
  2580  	return simdRegBySize(v.Reg(), t.Size())
  2581  }
  2582  
  2583  func simdRegBySize(reg int16, size int64) int16 {
  2584  	switch size {
  2585  	case 16:
  2586  		return reg
  2587  	case 32:
  2588  		return reg + (x86.REG_Y0 - x86.REG_X0)
  2589  	case 64:
  2590  		return reg + (x86.REG_Z0 - x86.REG_X0)
  2591  	}
  2592  	panic("simdRegBySize: bad size")
  2593  }
  2594  
  2595  // XXX k mask
  2596  func maskReg(v *ssa.Value) int16 {
  2597  	t := v.Type
  2598  	if !t.IsSIMD() {
  2599  		base.Fatalf("maskReg: not a simd type; v=%s, b=b%d, f=%s", v.LongString(), v.Block.ID, v.Block.Func.Name)
  2600  	}
  2601  	switch t.Size() {
  2602  	case 8:
  2603  		return v.Reg()
  2604  	}
  2605  	panic("unreachable")
  2606  }
  2607  
  2608  // XXX k mask + vec
  2609  func simdOrMaskReg(v *ssa.Value) int16 {
  2610  	t := v.Type
  2611  	if t.Size() <= 8 {
  2612  		return maskReg(v)
  2613  	}
  2614  	return simdReg(v)
  2615  }
  2616  
  2617  // XXX this is used for shift operations only.
  2618  // regalloc will issue OpCopy with incorrect type, but the assigned
  2619  // register should be correct, and this function is merely checking
  2620  // the sanity of this part.
  2621  func simdCheckRegOnly(v *ssa.Value, regStart, regEnd int16) int16 {
  2622  	if v.Reg() > regEnd || v.Reg() < regStart {
  2623  		panic("simdCheckRegOnly: not the desired register")
  2624  	}
  2625  	return v.Reg()
  2626  }
  2627  

View as plain text