Source file src/cmd/compile/internal/ssagen/intrinsics.go

     1  // Copyright 2024 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssagen
     6  
     7  import (
     8  	"fmt"
     9  	"internal/abi"
    10  	"internal/buildcfg"
    11  
    12  	"cmd/compile/internal/base"
    13  	"cmd/compile/internal/ir"
    14  	"cmd/compile/internal/ssa"
    15  	"cmd/compile/internal/typecheck"
    16  	"cmd/compile/internal/types"
    17  	"cmd/internal/sys"
    18  )
    19  
    20  var intrinsics intrinsicBuilders
    21  
    22  // An intrinsicBuilder converts a call node n into an ssa value that
    23  // implements that call as an intrinsic. args is a list of arguments to the func.
    24  type intrinsicBuilder func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value
    25  
    26  type intrinsicKey struct {
    27  	arch *sys.Arch
    28  	pkg  string
    29  	fn   string
    30  }
    31  
    32  // intrinsicBuildConfig specifies the config to use for intrinsic building.
    33  type intrinsicBuildConfig struct {
    34  	instrumenting bool
    35  
    36  	go386     string
    37  	goamd64   int
    38  	goarm     buildcfg.GoarmFeatures
    39  	goarm64   buildcfg.Goarm64Features
    40  	gomips    string
    41  	gomips64  string
    42  	goppc64   int
    43  	goriscv64 int
    44  }
    45  
    46  type intrinsicBuilders map[intrinsicKey]intrinsicBuilder
    47  
    48  // add adds the intrinsic builder b for pkg.fn for the given architecture.
    49  func (ib intrinsicBuilders) add(arch *sys.Arch, pkg, fn string, b intrinsicBuilder) {
    50  	if _, found := ib[intrinsicKey{arch, pkg, fn}]; found {
    51  		panic(fmt.Sprintf("intrinsic already exists for %v.%v on %v", pkg, fn, arch.Name))
    52  	}
    53  	ib[intrinsicKey{arch, pkg, fn}] = b
    54  }
    55  
    56  // addForArchs adds the intrinsic builder b for pkg.fn for the given architectures.
    57  func (ib intrinsicBuilders) addForArchs(pkg, fn string, b intrinsicBuilder, archs ...*sys.Arch) {
    58  	for _, arch := range archs {
    59  		ib.add(arch, pkg, fn, b)
    60  	}
    61  }
    62  
    63  // addForFamilies does the same as addForArchs but operates on architecture families.
    64  func (ib intrinsicBuilders) addForFamilies(pkg, fn string, b intrinsicBuilder, archFamilies ...sys.ArchFamily) {
    65  	for _, arch := range sys.Archs {
    66  		if arch.InFamily(archFamilies...) {
    67  			intrinsics.add(arch, pkg, fn, b)
    68  		}
    69  	}
    70  }
    71  
    72  // alias aliases pkg.fn to targetPkg.targetFn for all architectures in archs
    73  // for which targetPkg.targetFn already exists.
    74  func (ib intrinsicBuilders) alias(pkg, fn, targetPkg, targetFn string, archs ...*sys.Arch) {
    75  	// TODO(jsing): Consider making this work even if the alias is added
    76  	// before the intrinsic.
    77  	aliased := false
    78  	for _, arch := range archs {
    79  		if b := intrinsics.lookup(arch, targetPkg, targetFn); b != nil {
    80  			intrinsics.add(arch, pkg, fn, b)
    81  			aliased = true
    82  		}
    83  	}
    84  	if !aliased {
    85  		panic(fmt.Sprintf("attempted to alias undefined intrinsic: %s.%s", pkg, fn))
    86  	}
    87  }
    88  
    89  // lookup looks up the intrinsic for a pkg.fn on the specified architecture.
    90  func (ib intrinsicBuilders) lookup(arch *sys.Arch, pkg, fn string) intrinsicBuilder {
    91  	return intrinsics[intrinsicKey{arch, pkg, fn}]
    92  }
    93  
    94  func initIntrinsics(cfg *intrinsicBuildConfig) {
    95  	if cfg == nil {
    96  		cfg = &intrinsicBuildConfig{
    97  			instrumenting: base.Flag.Cfg.Instrumenting,
    98  			go386:         buildcfg.GO386,
    99  			goamd64:       buildcfg.GOAMD64,
   100  			goarm:         buildcfg.GOARM,
   101  			goarm64:       buildcfg.GOARM64,
   102  			gomips:        buildcfg.GOMIPS,
   103  			gomips64:      buildcfg.GOMIPS64,
   104  			goppc64:       buildcfg.GOPPC64,
   105  			goriscv64:     buildcfg.GORISCV64,
   106  		}
   107  	}
   108  	intrinsics = intrinsicBuilders{}
   109  
   110  	var p4 []*sys.Arch
   111  	var p8 []*sys.Arch
   112  	var lwatomics []*sys.Arch
   113  	for _, a := range sys.Archs {
   114  		if a.PtrSize == 4 {
   115  			p4 = append(p4, a)
   116  		} else {
   117  			p8 = append(p8, a)
   118  		}
   119  		if a.Family != sys.PPC64 {
   120  			lwatomics = append(lwatomics, a)
   121  		}
   122  	}
   123  	all := sys.Archs[:]
   124  
   125  	add := func(pkg, fn string, b intrinsicBuilder, archs ...*sys.Arch) {
   126  		intrinsics.addForArchs(pkg, fn, b, archs...)
   127  	}
   128  	addF := func(pkg, fn string, b intrinsicBuilder, archFamilies ...sys.ArchFamily) {
   129  		intrinsics.addForFamilies(pkg, fn, b, archFamilies...)
   130  	}
   131  	alias := func(pkg, fn, pkg2, fn2 string, archs ...*sys.Arch) {
   132  		intrinsics.alias(pkg, fn, pkg2, fn2, archs...)
   133  	}
   134  
   135  	/******** runtime ********/
   136  	if !cfg.instrumenting {
   137  		add("runtime", "slicebytetostringtmp",
   138  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   139  				// Compiler frontend optimizations emit OBYTES2STRTMP nodes
   140  				// for the backend instead of slicebytetostringtmp calls
   141  				// when not instrumenting.
   142  				return s.newValue2(ssa.OpStringMake, n.Type(), args[0], args[1])
   143  			},
   144  			all...)
   145  	}
   146  	addF("internal/runtime/math", "MulUintptr",
   147  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   148  			if s.config.PtrSize == 4 {
   149  				return s.newValue2(ssa.OpMul32uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1])
   150  			}
   151  			return s.newValue2(ssa.OpMul64uover, types.NewTuple(types.Types[types.TUINT], types.Types[types.TUINT]), args[0], args[1])
   152  		},
   153  		sys.AMD64, sys.I386, sys.Loong64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.ARM64)
   154  	add("runtime", "KeepAlive",
   155  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   156  			data := s.newValue1(ssa.OpIData, s.f.Config.Types.BytePtr, args[0])
   157  			s.vars[memVar] = s.newValue2(ssa.OpKeepAlive, types.TypeMem, data, s.mem())
   158  			return nil
   159  		},
   160  		all...)
   161  
   162  	addF("runtime", "publicationBarrier",
   163  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   164  			s.vars[memVar] = s.newValue1(ssa.OpPubBarrier, types.TypeMem, s.mem())
   165  			return nil
   166  		},
   167  		sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64)
   168  
   169  	/******** internal/runtime/sys ********/
   170  	add("internal/runtime/sys", "GetCallerPC",
   171  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   172  			return s.newValue0(ssa.OpGetCallerPC, s.f.Config.Types.Uintptr)
   173  		},
   174  		all...)
   175  
   176  	add("internal/runtime/sys", "GetCallerSP",
   177  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   178  			return s.newValue1(ssa.OpGetCallerSP, s.f.Config.Types.Uintptr, s.mem())
   179  		},
   180  		all...)
   181  
   182  	add("internal/runtime/sys", "GetClosurePtr",
   183  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   184  			return s.newValue0(ssa.OpGetClosurePtr, s.f.Config.Types.Uintptr)
   185  		},
   186  		all...)
   187  
   188  	addF("internal/runtime/sys", "Bswap32",
   189  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   190  			return s.newValue1(ssa.OpBswap32, types.Types[types.TUINT32], args[0])
   191  		},
   192  		sys.AMD64, sys.I386, sys.ARM64, sys.ARM, sys.Loong64, sys.S390X)
   193  	addF("internal/runtime/sys", "Bswap64",
   194  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   195  			return s.newValue1(ssa.OpBswap64, types.Types[types.TUINT64], args[0])
   196  		},
   197  		sys.AMD64, sys.I386, sys.ARM64, sys.ARM, sys.Loong64, sys.S390X)
   198  
   199  	addF("runtime", "memequal",
   200  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   201  			return s.newValue4(ssa.OpMemEq, s.f.Config.Types.Bool, args[0], args[1], args[2], s.mem())
   202  		},
   203  		sys.ARM64)
   204  
   205  	if cfg.goppc64 >= 10 {
   206  		// Use only on Power10 as the new byte reverse instructions that Power10 provide
   207  		// make it worthwhile as an intrinsic
   208  		addF("internal/runtime/sys", "Bswap32",
   209  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   210  				return s.newValue1(ssa.OpBswap32, types.Types[types.TUINT32], args[0])
   211  			},
   212  			sys.PPC64)
   213  		addF("internal/runtime/sys", "Bswap64",
   214  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   215  				return s.newValue1(ssa.OpBswap64, types.Types[types.TUINT64], args[0])
   216  			},
   217  			sys.PPC64)
   218  	}
   219  
   220  	if cfg.goriscv64 >= 22 {
   221  		addF("internal/runtime/sys", "Bswap32",
   222  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   223  				return s.newValue1(ssa.OpBswap32, types.Types[types.TUINT32], args[0])
   224  			},
   225  			sys.RISCV64)
   226  		addF("internal/runtime/sys", "Bswap64",
   227  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   228  				return s.newValue1(ssa.OpBswap64, types.Types[types.TUINT64], args[0])
   229  			},
   230  			sys.RISCV64)
   231  	}
   232  
   233  	/****** Prefetch ******/
   234  	makePrefetchFunc := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   235  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   236  			s.vars[memVar] = s.newValue2(op, types.TypeMem, args[0], s.mem())
   237  			return nil
   238  		}
   239  	}
   240  
   241  	// Make Prefetch intrinsics for supported platforms
   242  	// On the unsupported platforms stub function will be eliminated
   243  	addF("internal/runtime/sys", "Prefetch", makePrefetchFunc(ssa.OpPrefetchCache),
   244  		sys.AMD64, sys.ARM64, sys.Loong64, sys.PPC64)
   245  	addF("internal/runtime/sys", "PrefetchStreamed", makePrefetchFunc(ssa.OpPrefetchCacheStreamed),
   246  		sys.AMD64, sys.ARM64, sys.Loong64, sys.PPC64)
   247  
   248  	/******** internal/runtime/atomic ********/
   249  	type atomicOpEmitter func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool)
   250  
   251  	addF("internal/runtime/atomic", "Load",
   252  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   253  			v := s.newValue2(ssa.OpAtomicLoad32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], s.mem())
   254  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   255  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v)
   256  		},
   257  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   258  	addF("internal/runtime/atomic", "Load8",
   259  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   260  			v := s.newValue2(ssa.OpAtomicLoad8, types.NewTuple(types.Types[types.TUINT8], types.TypeMem), args[0], s.mem())
   261  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   262  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], v)
   263  		},
   264  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   265  	addF("internal/runtime/atomic", "Load64",
   266  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   267  			v := s.newValue2(ssa.OpAtomicLoad64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], s.mem())
   268  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   269  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v)
   270  		},
   271  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   272  	addF("internal/runtime/atomic", "LoadAcq",
   273  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   274  			v := s.newValue2(ssa.OpAtomicLoadAcq32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], s.mem())
   275  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   276  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v)
   277  		},
   278  		sys.PPC64)
   279  	addF("internal/runtime/atomic", "LoadAcq64",
   280  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   281  			v := s.newValue2(ssa.OpAtomicLoadAcq64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], s.mem())
   282  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   283  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v)
   284  		},
   285  		sys.PPC64)
   286  	addF("internal/runtime/atomic", "Loadp",
   287  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   288  			v := s.newValue2(ssa.OpAtomicLoadPtr, types.NewTuple(s.f.Config.Types.BytePtr, types.TypeMem), args[0], s.mem())
   289  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   290  			return s.newValue1(ssa.OpSelect0, s.f.Config.Types.BytePtr, v)
   291  		},
   292  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   293  
   294  	addF("internal/runtime/atomic", "Store",
   295  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   296  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStore32, types.TypeMem, args[0], args[1], s.mem())
   297  			return nil
   298  		},
   299  		sys.AMD64, sys.ARM64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   300  	addF("internal/runtime/atomic", "Store8",
   301  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   302  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStore8, types.TypeMem, args[0], args[1], s.mem())
   303  			return nil
   304  		},
   305  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   306  	addF("internal/runtime/atomic", "Store64",
   307  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   308  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStore64, types.TypeMem, args[0], args[1], s.mem())
   309  			return nil
   310  		},
   311  		sys.AMD64, sys.ARM64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   312  	addF("internal/runtime/atomic", "StorepNoWB",
   313  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   314  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStorePtrNoWB, types.TypeMem, args[0], args[1], s.mem())
   315  			return nil
   316  		},
   317  		sys.AMD64, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.RISCV64, sys.S390X)
   318  	addF("internal/runtime/atomic", "StoreRel",
   319  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   320  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStoreRel32, types.TypeMem, args[0], args[1], s.mem())
   321  			return nil
   322  		},
   323  		sys.PPC64)
   324  	addF("internal/runtime/atomic", "StoreRel64",
   325  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   326  			s.vars[memVar] = s.newValue3(ssa.OpAtomicStoreRel64, types.TypeMem, args[0], args[1], s.mem())
   327  			return nil
   328  		},
   329  		sys.PPC64)
   330  
   331  	makeAtomicStoreGuardedIntrinsicLoong64 := func(op0, op1 ssa.Op, typ types.Kind, emit atomicOpEmitter) intrinsicBuilder {
   332  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   333  			// Target Atomic feature is identified by dynamic detection
   334  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.Loong64HasDBAR_HINTS, s.sb)
   335  			v := s.load(types.Types[types.TBOOL], addr)
   336  			b := s.endBlock()
   337  			b.Kind = ssa.BlockIf
   338  			b.SetControl(v)
   339  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   340  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   341  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   342  			b.AddEdgeTo(bTrue)
   343  			b.AddEdgeTo(bFalse)
   344  			b.Likely = ssa.BranchLikely
   345  
   346  			// most loong64 machines support the finer-grained DBAR hints
   347  			s.startBlock(bTrue)
   348  			emit(s, n, args, op0, typ, false)
   349  			s.endBlock().AddEdgeTo(bEnd)
   350  
   351  			// Use original instruction sequence.
   352  			s.startBlock(bFalse)
   353  			emit(s, n, args, op1, typ, false)
   354  			s.endBlock().AddEdgeTo(bEnd)
   355  
   356  			// Merge results.
   357  			s.startBlock(bEnd)
   358  
   359  			return nil
   360  		}
   361  	}
   362  
   363  	atomicStoreEmitterLoong64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool) {
   364  		v := s.newValue3(op, types.NewTuple(types.Types[typ], types.TypeMem), args[0], args[1], s.mem())
   365  		s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   366  		if needReturn {
   367  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v)
   368  		}
   369  	}
   370  
   371  	addF("internal/runtime/atomic", "Store",
   372  		makeAtomicStoreGuardedIntrinsicLoong64(ssa.OpAtomicStore32, ssa.OpAtomicStore32Variant, types.TUINT8, atomicStoreEmitterLoong64),
   373  		sys.Loong64)
   374  	addF("internal/runtime/atomic", "Store64",
   375  		makeAtomicStoreGuardedIntrinsicLoong64(ssa.OpAtomicStore64, ssa.OpAtomicStore64Variant, types.TUINT8, atomicStoreEmitterLoong64),
   376  		sys.Loong64)
   377  
   378  	addF("internal/runtime/atomic", "Xchg8",
   379  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   380  			v := s.newValue3(ssa.OpAtomicExchange8, types.NewTuple(types.Types[types.TUINT8], types.TypeMem), args[0], args[1], s.mem())
   381  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   382  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], v)
   383  		},
   384  		sys.AMD64, sys.PPC64)
   385  	addF("internal/runtime/atomic", "Xchg",
   386  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   387  			v := s.newValue3(ssa.OpAtomicExchange32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
   388  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   389  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v)
   390  		},
   391  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   392  	addF("internal/runtime/atomic", "Xchg64",
   393  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   394  			v := s.newValue3(ssa.OpAtomicExchange64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem())
   395  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   396  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v)
   397  		},
   398  		sys.AMD64, sys.Loong64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   399  
   400  	makeAtomicGuardedIntrinsicARM64common := func(op0, op1 ssa.Op, typ types.Kind, emit atomicOpEmitter, needReturn bool) intrinsicBuilder {
   401  
   402  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   403  			if cfg.goarm64.LSE {
   404  				emit(s, n, args, op1, typ, needReturn)
   405  			} else {
   406  				// Target Atomic feature is identified by dynamic detection
   407  				addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARM64HasATOMICS, s.sb)
   408  				v := s.load(types.Types[types.TBOOL], addr)
   409  				b := s.endBlock()
   410  				b.Kind = ssa.BlockIf
   411  				b.SetControl(v)
   412  				bTrue := s.f.NewBlock(ssa.BlockPlain)
   413  				bFalse := s.f.NewBlock(ssa.BlockPlain)
   414  				bEnd := s.f.NewBlock(ssa.BlockPlain)
   415  				b.AddEdgeTo(bTrue)
   416  				b.AddEdgeTo(bFalse)
   417  				b.Likely = ssa.BranchLikely
   418  
   419  				// We have atomic instructions - use it directly.
   420  				s.startBlock(bTrue)
   421  				emit(s, n, args, op1, typ, needReturn)
   422  				s.endBlock().AddEdgeTo(bEnd)
   423  
   424  				// Use original instruction sequence.
   425  				s.startBlock(bFalse)
   426  				emit(s, n, args, op0, typ, needReturn)
   427  				s.endBlock().AddEdgeTo(bEnd)
   428  
   429  				// Merge results.
   430  				s.startBlock(bEnd)
   431  			}
   432  			if needReturn {
   433  				return s.variable(n, types.Types[typ])
   434  			} else {
   435  				return nil
   436  			}
   437  		}
   438  	}
   439  	makeAtomicGuardedIntrinsicARM64 := func(op0, op1 ssa.Op, typ types.Kind, emit atomicOpEmitter) intrinsicBuilder {
   440  		return makeAtomicGuardedIntrinsicARM64common(op0, op1, typ, emit, true)
   441  	}
   442  	makeAtomicGuardedIntrinsicARM64old := func(op0, op1 ssa.Op, typ types.Kind, emit atomicOpEmitter) intrinsicBuilder {
   443  		return makeAtomicGuardedIntrinsicARM64common(op0, op1, typ, emit, false)
   444  	}
   445  
   446  	atomicEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool) {
   447  		v := s.newValue3(op, types.NewTuple(types.Types[typ], types.TypeMem), args[0], args[1], s.mem())
   448  		s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   449  		if needReturn {
   450  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v)
   451  		}
   452  	}
   453  	addF("internal/runtime/atomic", "Xchg8",
   454  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange8, ssa.OpAtomicExchange8Variant, types.TUINT8, atomicEmitterARM64),
   455  		sys.ARM64)
   456  	addF("internal/runtime/atomic", "Xchg",
   457  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange32, ssa.OpAtomicExchange32Variant, types.TUINT32, atomicEmitterARM64),
   458  		sys.ARM64)
   459  	addF("internal/runtime/atomic", "Xchg64",
   460  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicExchange64, ssa.OpAtomicExchange64Variant, types.TUINT64, atomicEmitterARM64),
   461  		sys.ARM64)
   462  
   463  	makeAtomicXchg8GuardedIntrinsicLoong64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   464  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   465  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.Loong64HasLAM_BH, s.sb)
   466  			v := s.load(types.Types[types.TBOOL], addr)
   467  			b := s.endBlock()
   468  			b.Kind = ssa.BlockIf
   469  			b.SetControl(v)
   470  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   471  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   472  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   473  			b.AddEdgeTo(bTrue)
   474  			b.AddEdgeTo(bFalse)
   475  			b.Likely = ssa.BranchLikely // most loong64 machines support the amswapdb.b
   476  
   477  			// We have the intrinsic - use it directly.
   478  			s.startBlock(bTrue)
   479  			s.vars[n] = s.newValue3(op, types.NewTuple(types.Types[types.TUINT8], types.TypeMem), args[0], args[1], s.mem())
   480  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, s.vars[n])
   481  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[types.TUINT8], s.vars[n])
   482  			s.endBlock().AddEdgeTo(bEnd)
   483  
   484  			// Call the pure Go version.
   485  			s.startBlock(bFalse)
   486  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TUINT8]
   487  			s.endBlock().AddEdgeTo(bEnd)
   488  
   489  			// Merge results.
   490  			s.startBlock(bEnd)
   491  			return s.variable(n, types.Types[types.TUINT8])
   492  		}
   493  	}
   494  	addF("internal/runtime/atomic", "Xchg8",
   495  		makeAtomicXchg8GuardedIntrinsicLoong64(ssa.OpAtomicExchange8Variant),
   496  		sys.Loong64)
   497  
   498  	addF("internal/runtime/atomic", "Xadd",
   499  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   500  			v := s.newValue3(ssa.OpAtomicAdd32, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
   501  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   502  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT32], v)
   503  		},
   504  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   505  	addF("internal/runtime/atomic", "Xadd64",
   506  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   507  			v := s.newValue3(ssa.OpAtomicAdd64, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem())
   508  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   509  			return s.newValue1(ssa.OpSelect0, types.Types[types.TUINT64], v)
   510  		},
   511  		sys.AMD64, sys.Loong64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   512  
   513  	addF("internal/runtime/atomic", "Xadd",
   514  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAdd32, ssa.OpAtomicAdd32Variant, types.TUINT32, atomicEmitterARM64),
   515  		sys.ARM64)
   516  	addF("internal/runtime/atomic", "Xadd64",
   517  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAdd64, ssa.OpAtomicAdd64Variant, types.TUINT64, atomicEmitterARM64),
   518  		sys.ARM64)
   519  
   520  	addF("internal/runtime/atomic", "Cas",
   521  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   522  			v := s.newValue4(ssa.OpAtomicCompareAndSwap32, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   523  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   524  			return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v)
   525  		},
   526  		sys.AMD64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   527  	addF("internal/runtime/atomic", "Cas64",
   528  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   529  			v := s.newValue4(ssa.OpAtomicCompareAndSwap64, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   530  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   531  			return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v)
   532  		},
   533  		sys.AMD64, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   534  	addF("internal/runtime/atomic", "CasRel",
   535  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   536  			v := s.newValue4(ssa.OpAtomicCompareAndSwap32, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   537  			s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   538  			return s.newValue1(ssa.OpSelect0, types.Types[types.TBOOL], v)
   539  		},
   540  		sys.PPC64)
   541  
   542  	atomicCasEmitterARM64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool) {
   543  		v := s.newValue4(op, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   544  		s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   545  		if needReturn {
   546  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v)
   547  		}
   548  	}
   549  
   550  	addF("internal/runtime/atomic", "Cas",
   551  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicCompareAndSwap32, ssa.OpAtomicCompareAndSwap32Variant, types.TBOOL, atomicCasEmitterARM64),
   552  		sys.ARM64)
   553  	addF("internal/runtime/atomic", "Cas64",
   554  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicCompareAndSwap64, ssa.OpAtomicCompareAndSwap64Variant, types.TBOOL, atomicCasEmitterARM64),
   555  		sys.ARM64)
   556  
   557  	atomicCasEmitterLoong64 := func(s *state, n *ir.CallExpr, args []*ssa.Value, op ssa.Op, typ types.Kind, needReturn bool) {
   558  		v := s.newValue4(op, types.NewTuple(types.Types[types.TBOOL], types.TypeMem), args[0], args[1], args[2], s.mem())
   559  		s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v)
   560  		if needReturn {
   561  			s.vars[n] = s.newValue1(ssa.OpSelect0, types.Types[typ], v)
   562  		}
   563  	}
   564  
   565  	makeAtomicCasGuardedIntrinsicLoong64 := func(op0, op1 ssa.Op, emit atomicOpEmitter) intrinsicBuilder {
   566  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   567  			// Target Atomic feature is identified by dynamic detection
   568  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.Loong64HasLAMCAS, s.sb)
   569  			v := s.load(types.Types[types.TBOOL], addr)
   570  			b := s.endBlock()
   571  			b.Kind = ssa.BlockIf
   572  			b.SetControl(v)
   573  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   574  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   575  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   576  			b.AddEdgeTo(bTrue)
   577  			b.AddEdgeTo(bFalse)
   578  			b.Likely = ssa.BranchLikely
   579  
   580  			// We have atomic instructions - use it directly.
   581  			s.startBlock(bTrue)
   582  			emit(s, n, args, op1, types.TBOOL, true)
   583  			s.endBlock().AddEdgeTo(bEnd)
   584  
   585  			// Use original instruction sequence.
   586  			s.startBlock(bFalse)
   587  			emit(s, n, args, op0, types.TBOOL, true)
   588  			s.endBlock().AddEdgeTo(bEnd)
   589  
   590  			// Merge results.
   591  			s.startBlock(bEnd)
   592  
   593  			return s.variable(n, types.Types[types.TBOOL])
   594  		}
   595  	}
   596  
   597  	addF("internal/runtime/atomic", "Cas",
   598  		makeAtomicCasGuardedIntrinsicLoong64(ssa.OpAtomicCompareAndSwap32, ssa.OpAtomicCompareAndSwap32Variant, atomicCasEmitterLoong64),
   599  		sys.Loong64)
   600  	addF("internal/runtime/atomic", "Cas64",
   601  		makeAtomicCasGuardedIntrinsicLoong64(ssa.OpAtomicCompareAndSwap64, ssa.OpAtomicCompareAndSwap64Variant, atomicCasEmitterLoong64),
   602  		sys.Loong64)
   603  
   604  	// Old-style atomic logical operation API (all supported archs except arm64).
   605  	addF("internal/runtime/atomic", "And8",
   606  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   607  			s.vars[memVar] = s.newValue3(ssa.OpAtomicAnd8, types.TypeMem, args[0], args[1], s.mem())
   608  			return nil
   609  		},
   610  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   611  	addF("internal/runtime/atomic", "And",
   612  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   613  			s.vars[memVar] = s.newValue3(ssa.OpAtomicAnd32, types.TypeMem, args[0], args[1], s.mem())
   614  			return nil
   615  		},
   616  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   617  	addF("internal/runtime/atomic", "Or8",
   618  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   619  			s.vars[memVar] = s.newValue3(ssa.OpAtomicOr8, types.TypeMem, args[0], args[1], s.mem())
   620  			return nil
   621  		},
   622  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   623  	addF("internal/runtime/atomic", "Or",
   624  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   625  			s.vars[memVar] = s.newValue3(ssa.OpAtomicOr32, types.TypeMem, args[0], args[1], s.mem())
   626  			return nil
   627  		},
   628  		sys.AMD64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X)
   629  
   630  	// arm64 always uses the new-style atomic logical operations, for both the
   631  	// old and new style API.
   632  	addF("internal/runtime/atomic", "And8",
   633  		makeAtomicGuardedIntrinsicARM64old(ssa.OpAtomicAnd8value, ssa.OpAtomicAnd8valueVariant, types.TUINT8, atomicEmitterARM64),
   634  		sys.ARM64)
   635  	addF("internal/runtime/atomic", "Or8",
   636  		makeAtomicGuardedIntrinsicARM64old(ssa.OpAtomicOr8value, ssa.OpAtomicOr8valueVariant, types.TUINT8, atomicEmitterARM64),
   637  		sys.ARM64)
   638  	addF("internal/runtime/atomic", "And64",
   639  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAnd64value, ssa.OpAtomicAnd64valueVariant, types.TUINT64, atomicEmitterARM64),
   640  		sys.ARM64)
   641  	addF("internal/runtime/atomic", "And32",
   642  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicAnd32value, ssa.OpAtomicAnd32valueVariant, types.TUINT32, atomicEmitterARM64),
   643  		sys.ARM64)
   644  	addF("internal/runtime/atomic", "And",
   645  		makeAtomicGuardedIntrinsicARM64old(ssa.OpAtomicAnd32value, ssa.OpAtomicAnd32valueVariant, types.TUINT32, atomicEmitterARM64),
   646  		sys.ARM64)
   647  	addF("internal/runtime/atomic", "Or64",
   648  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicOr64value, ssa.OpAtomicOr64valueVariant, types.TUINT64, atomicEmitterARM64),
   649  		sys.ARM64)
   650  	addF("internal/runtime/atomic", "Or32",
   651  		makeAtomicGuardedIntrinsicARM64(ssa.OpAtomicOr32value, ssa.OpAtomicOr32valueVariant, types.TUINT32, atomicEmitterARM64),
   652  		sys.ARM64)
   653  	addF("internal/runtime/atomic", "Or",
   654  		makeAtomicGuardedIntrinsicARM64old(ssa.OpAtomicOr32value, ssa.OpAtomicOr32valueVariant, types.TUINT32, atomicEmitterARM64),
   655  		sys.ARM64)
   656  
   657  	// New-style atomic logical operations, which return the old memory value.
   658  	addF("internal/runtime/atomic", "And64",
   659  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   660  			v := s.newValue3(ssa.OpAtomicAnd64value, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem())
   661  			p0, p1 := s.split(v)
   662  			s.vars[memVar] = p1
   663  			return p0
   664  		},
   665  		sys.AMD64, sys.Loong64)
   666  	addF("internal/runtime/atomic", "And32",
   667  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   668  			v := s.newValue3(ssa.OpAtomicAnd32value, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
   669  			p0, p1 := s.split(v)
   670  			s.vars[memVar] = p1
   671  			return p0
   672  		},
   673  		sys.AMD64, sys.Loong64)
   674  	addF("internal/runtime/atomic", "Or64",
   675  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   676  			v := s.newValue3(ssa.OpAtomicOr64value, types.NewTuple(types.Types[types.TUINT64], types.TypeMem), args[0], args[1], s.mem())
   677  			p0, p1 := s.split(v)
   678  			s.vars[memVar] = p1
   679  			return p0
   680  		},
   681  		sys.AMD64, sys.Loong64)
   682  	addF("internal/runtime/atomic", "Or32",
   683  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   684  			v := s.newValue3(ssa.OpAtomicOr32value, types.NewTuple(types.Types[types.TUINT32], types.TypeMem), args[0], args[1], s.mem())
   685  			p0, p1 := s.split(v)
   686  			s.vars[memVar] = p1
   687  			return p0
   688  		},
   689  		sys.AMD64, sys.Loong64)
   690  
   691  	// Aliases for atomic load operations
   692  	alias("internal/runtime/atomic", "Loadint32", "internal/runtime/atomic", "Load", all...)
   693  	alias("internal/runtime/atomic", "Loadint64", "internal/runtime/atomic", "Load64", all...)
   694  	alias("internal/runtime/atomic", "Loaduintptr", "internal/runtime/atomic", "Load", p4...)
   695  	alias("internal/runtime/atomic", "Loaduintptr", "internal/runtime/atomic", "Load64", p8...)
   696  	alias("internal/runtime/atomic", "Loaduint", "internal/runtime/atomic", "Load", p4...)
   697  	alias("internal/runtime/atomic", "Loaduint", "internal/runtime/atomic", "Load64", p8...)
   698  	alias("internal/runtime/atomic", "LoadAcq", "internal/runtime/atomic", "Load", lwatomics...)
   699  	alias("internal/runtime/atomic", "LoadAcq64", "internal/runtime/atomic", "Load64", lwatomics...)
   700  	alias("internal/runtime/atomic", "LoadAcquintptr", "internal/runtime/atomic", "LoadAcq", p4...)
   701  	alias("internal/runtime/atomic", "LoadAcquintptr", "internal/runtime/atomic", "LoadAcq64", p8...)
   702  
   703  	// Aliases for atomic store operations
   704  	alias("internal/runtime/atomic", "Storeint32", "internal/runtime/atomic", "Store", all...)
   705  	alias("internal/runtime/atomic", "Storeint64", "internal/runtime/atomic", "Store64", all...)
   706  	alias("internal/runtime/atomic", "Storeuintptr", "internal/runtime/atomic", "Store", p4...)
   707  	alias("internal/runtime/atomic", "Storeuintptr", "internal/runtime/atomic", "Store64", p8...)
   708  	alias("internal/runtime/atomic", "StoreRel", "internal/runtime/atomic", "Store", lwatomics...)
   709  	alias("internal/runtime/atomic", "StoreRel64", "internal/runtime/atomic", "Store64", lwatomics...)
   710  	alias("internal/runtime/atomic", "StoreReluintptr", "internal/runtime/atomic", "StoreRel", p4...)
   711  	alias("internal/runtime/atomic", "StoreReluintptr", "internal/runtime/atomic", "StoreRel64", p8...)
   712  
   713  	// Aliases for atomic swap operations
   714  	alias("internal/runtime/atomic", "Xchgint32", "internal/runtime/atomic", "Xchg", all...)
   715  	alias("internal/runtime/atomic", "Xchgint64", "internal/runtime/atomic", "Xchg64", all...)
   716  	alias("internal/runtime/atomic", "Xchguintptr", "internal/runtime/atomic", "Xchg", p4...)
   717  	alias("internal/runtime/atomic", "Xchguintptr", "internal/runtime/atomic", "Xchg64", p8...)
   718  
   719  	// Aliases for atomic add operations
   720  	alias("internal/runtime/atomic", "Xaddint32", "internal/runtime/atomic", "Xadd", all...)
   721  	alias("internal/runtime/atomic", "Xaddint64", "internal/runtime/atomic", "Xadd64", all...)
   722  	alias("internal/runtime/atomic", "Xadduintptr", "internal/runtime/atomic", "Xadd", p4...)
   723  	alias("internal/runtime/atomic", "Xadduintptr", "internal/runtime/atomic", "Xadd64", p8...)
   724  
   725  	// Aliases for atomic CAS operations
   726  	alias("internal/runtime/atomic", "Casint32", "internal/runtime/atomic", "Cas", all...)
   727  	alias("internal/runtime/atomic", "Casint64", "internal/runtime/atomic", "Cas64", all...)
   728  	alias("internal/runtime/atomic", "Casuintptr", "internal/runtime/atomic", "Cas", p4...)
   729  	alias("internal/runtime/atomic", "Casuintptr", "internal/runtime/atomic", "Cas64", p8...)
   730  	alias("internal/runtime/atomic", "Casp1", "internal/runtime/atomic", "Cas", p4...)
   731  	alias("internal/runtime/atomic", "Casp1", "internal/runtime/atomic", "Cas64", p8...)
   732  	alias("internal/runtime/atomic", "CasRel", "internal/runtime/atomic", "Cas", lwatomics...)
   733  
   734  	// Aliases for atomic And/Or operations
   735  	alias("internal/runtime/atomic", "Anduintptr", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchLoong64)
   736  	alias("internal/runtime/atomic", "Oruintptr", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchLoong64)
   737  
   738  	/******** math ********/
   739  	addF("math", "sqrt",
   740  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   741  			return s.newValue1(ssa.OpSqrt, types.Types[types.TFLOAT64], args[0])
   742  		},
   743  		sys.I386, sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.MIPS64, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm)
   744  	addF("math", "Trunc",
   745  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   746  			return s.newValue1(ssa.OpTrunc, types.Types[types.TFLOAT64], args[0])
   747  		},
   748  		sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm)
   749  	addF("math", "Ceil",
   750  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   751  			return s.newValue1(ssa.OpCeil, types.Types[types.TFLOAT64], args[0])
   752  		},
   753  		sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm)
   754  	addF("math", "Floor",
   755  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   756  			return s.newValue1(ssa.OpFloor, types.Types[types.TFLOAT64], args[0])
   757  		},
   758  		sys.ARM64, sys.PPC64, sys.S390X, sys.Wasm)
   759  	addF("math", "Round",
   760  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   761  			return s.newValue1(ssa.OpRound, types.Types[types.TFLOAT64], args[0])
   762  		},
   763  		sys.ARM64, sys.PPC64, sys.S390X)
   764  	addF("math", "RoundToEven",
   765  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   766  			return s.newValue1(ssa.OpRoundToEven, types.Types[types.TFLOAT64], args[0])
   767  		},
   768  		sys.ARM64, sys.S390X, sys.Wasm)
   769  	addF("math", "Abs",
   770  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   771  			return s.newValue1(ssa.OpAbs, types.Types[types.TFLOAT64], args[0])
   772  		},
   773  		sys.ARM64, sys.ARM, sys.Loong64, sys.PPC64, sys.RISCV64, sys.Wasm, sys.MIPS, sys.MIPS64)
   774  	addF("math", "Copysign",
   775  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   776  			return s.newValue2(ssa.OpCopysign, types.Types[types.TFLOAT64], args[0], args[1])
   777  		},
   778  		sys.Loong64, sys.PPC64, sys.RISCV64, sys.Wasm)
   779  	addF("math", "FMA",
   780  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   781  			return s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2])
   782  		},
   783  		sys.ARM64, sys.Loong64, sys.PPC64, sys.RISCV64, sys.S390X)
   784  	addF("math", "FMA",
   785  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   786  			if cfg.goamd64 >= 3 {
   787  				return s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2])
   788  			}
   789  
   790  			v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasFMA)
   791  			b := s.endBlock()
   792  			b.Kind = ssa.BlockIf
   793  			b.SetControl(v)
   794  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   795  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   796  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   797  			b.AddEdgeTo(bTrue)
   798  			b.AddEdgeTo(bFalse)
   799  			b.Likely = ssa.BranchLikely // >= haswell cpus are common
   800  
   801  			// We have the intrinsic - use it directly.
   802  			s.startBlock(bTrue)
   803  			s.vars[n] = s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2])
   804  			s.endBlock().AddEdgeTo(bEnd)
   805  
   806  			// Call the pure Go version.
   807  			s.startBlock(bFalse)
   808  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64]
   809  			s.endBlock().AddEdgeTo(bEnd)
   810  
   811  			// Merge results.
   812  			s.startBlock(bEnd)
   813  			return s.variable(n, types.Types[types.TFLOAT64])
   814  		},
   815  		sys.AMD64)
   816  	addF("math", "FMA",
   817  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   818  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.ARMHasVFPv4, s.sb)
   819  			v := s.load(types.Types[types.TBOOL], addr)
   820  			b := s.endBlock()
   821  			b.Kind = ssa.BlockIf
   822  			b.SetControl(v)
   823  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   824  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   825  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   826  			b.AddEdgeTo(bTrue)
   827  			b.AddEdgeTo(bFalse)
   828  			b.Likely = ssa.BranchLikely
   829  
   830  			// We have the intrinsic - use it directly.
   831  			s.startBlock(bTrue)
   832  			s.vars[n] = s.newValue3(ssa.OpFMA, types.Types[types.TFLOAT64], args[0], args[1], args[2])
   833  			s.endBlock().AddEdgeTo(bEnd)
   834  
   835  			// Call the pure Go version.
   836  			s.startBlock(bFalse)
   837  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64]
   838  			s.endBlock().AddEdgeTo(bEnd)
   839  
   840  			// Merge results.
   841  			s.startBlock(bEnd)
   842  			return s.variable(n, types.Types[types.TFLOAT64])
   843  		},
   844  		sys.ARM)
   845  
   846  	makeRoundAMD64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   847  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   848  			if cfg.goamd64 >= 2 {
   849  				return s.newValue1(op, types.Types[types.TFLOAT64], args[0])
   850  			}
   851  
   852  			v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasSSE41)
   853  			b := s.endBlock()
   854  			b.Kind = ssa.BlockIf
   855  			b.SetControl(v)
   856  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   857  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   858  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   859  			b.AddEdgeTo(bTrue)
   860  			b.AddEdgeTo(bFalse)
   861  			b.Likely = ssa.BranchLikely // most machines have sse4.1 nowadays
   862  
   863  			// We have the intrinsic - use it directly.
   864  			s.startBlock(bTrue)
   865  			s.vars[n] = s.newValue1(op, types.Types[types.TFLOAT64], args[0])
   866  			s.endBlock().AddEdgeTo(bEnd)
   867  
   868  			// Call the pure Go version.
   869  			s.startBlock(bFalse)
   870  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64]
   871  			s.endBlock().AddEdgeTo(bEnd)
   872  
   873  			// Merge results.
   874  			s.startBlock(bEnd)
   875  			return s.variable(n, types.Types[types.TFLOAT64])
   876  		}
   877  	}
   878  	addF("math", "RoundToEven",
   879  		makeRoundAMD64(ssa.OpRoundToEven),
   880  		sys.AMD64)
   881  	addF("math", "Floor",
   882  		makeRoundAMD64(ssa.OpFloor),
   883  		sys.AMD64)
   884  	addF("math", "Ceil",
   885  		makeRoundAMD64(ssa.OpCeil),
   886  		sys.AMD64)
   887  	addF("math", "Trunc",
   888  		makeRoundAMD64(ssa.OpTrunc),
   889  		sys.AMD64)
   890  
   891  	makeRoundLoong64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   892  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   893  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.Loong64HasLSX, s.sb)
   894  			v := s.load(types.Types[types.TBOOL], addr)
   895  			b := s.endBlock()
   896  			b.Kind = ssa.BlockIf
   897  			b.SetControl(v)
   898  			bTrue := s.f.NewBlock(ssa.BlockPlain)
   899  			bFalse := s.f.NewBlock(ssa.BlockPlain)
   900  			bEnd := s.f.NewBlock(ssa.BlockPlain)
   901  			b.AddEdgeTo(bTrue)
   902  			b.AddEdgeTo(bFalse)
   903  			b.Likely = ssa.BranchLikely // most loong64 machines support the LSX
   904  
   905  			// We have the intrinsic - use it directly.
   906  			s.startBlock(bTrue)
   907  			s.vars[n] = s.newValue1(op, types.Types[types.TFLOAT64], args[0])
   908  			s.endBlock().AddEdgeTo(bEnd)
   909  
   910  			// Call the pure Go version.
   911  			s.startBlock(bFalse)
   912  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TFLOAT64]
   913  			s.endBlock().AddEdgeTo(bEnd)
   914  
   915  			// Merge results.
   916  			s.startBlock(bEnd)
   917  			return s.variable(n, types.Types[types.TFLOAT64])
   918  		}
   919  	}
   920  	addF("math", "RoundToEven",
   921  		makeRoundLoong64(ssa.OpRoundToEven),
   922  		sys.Loong64)
   923  	addF("math", "Floor",
   924  		makeRoundLoong64(ssa.OpFloor),
   925  		sys.Loong64)
   926  	addF("math", "Ceil",
   927  		makeRoundLoong64(ssa.OpCeil),
   928  		sys.Loong64)
   929  	addF("math", "Trunc",
   930  		makeRoundLoong64(ssa.OpTrunc),
   931  		sys.Loong64)
   932  
   933  	/******** math/bits ********/
   934  	addF("math/bits", "TrailingZeros64",
   935  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   936  			return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], args[0])
   937  		},
   938  		sys.AMD64, sys.ARM64, sys.ARM, sys.Loong64, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm)
   939  	addF("math/bits", "TrailingZeros64",
   940  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   941  			lo := s.newValue1(ssa.OpInt64Lo, types.Types[types.TUINT32], args[0])
   942  			hi := s.newValue1(ssa.OpInt64Hi, types.Types[types.TUINT32], args[0])
   943  			return s.newValue2(ssa.OpCtz64On32, types.Types[types.TINT], lo, hi)
   944  		},
   945  		sys.I386)
   946  	addF("math/bits", "TrailingZeros32",
   947  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   948  			return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], args[0])
   949  		},
   950  		sys.AMD64, sys.I386, sys.ARM64, sys.ARM, sys.Loong64, sys.S390X, sys.MIPS, sys.PPC64, sys.Wasm)
   951  	addF("math/bits", "TrailingZeros16",
   952  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   953  			return s.newValue1(ssa.OpCtz16, types.Types[types.TINT], args[0])
   954  		},
   955  		sys.AMD64, sys.ARM, sys.ARM64, sys.I386, sys.MIPS, sys.Loong64, sys.PPC64, sys.S390X, sys.Wasm)
   956  	addF("math/bits", "TrailingZeros8",
   957  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   958  			return s.newValue1(ssa.OpCtz8, types.Types[types.TINT], args[0])
   959  		},
   960  		sys.AMD64, sys.ARM, sys.ARM64, sys.I386, sys.MIPS, sys.Loong64, sys.PPC64, sys.S390X, sys.Wasm)
   961  
   962  	if cfg.goriscv64 >= 22 {
   963  		addF("math/bits", "TrailingZeros64",
   964  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   965  				return s.newValue1(ssa.OpCtz64, types.Types[types.TINT], args[0])
   966  			},
   967  			sys.RISCV64)
   968  		addF("math/bits", "TrailingZeros32",
   969  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   970  				return s.newValue1(ssa.OpCtz32, types.Types[types.TINT], args[0])
   971  			},
   972  			sys.RISCV64)
   973  		addF("math/bits", "TrailingZeros16",
   974  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   975  				return s.newValue1(ssa.OpCtz16, types.Types[types.TINT], args[0])
   976  			},
   977  			sys.RISCV64)
   978  		addF("math/bits", "TrailingZeros8",
   979  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   980  				return s.newValue1(ssa.OpCtz8, types.Types[types.TINT], args[0])
   981  			},
   982  			sys.RISCV64)
   983  	}
   984  
   985  	// ReverseBytes inlines correctly, no need to intrinsify it.
   986  	alias("math/bits", "ReverseBytes64", "internal/runtime/sys", "Bswap64", all...)
   987  	alias("math/bits", "ReverseBytes32", "internal/runtime/sys", "Bswap32", all...)
   988  	// Nothing special is needed for targets where ReverseBytes16 lowers to a rotate
   989  	addF("math/bits", "ReverseBytes16",
   990  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   991  			return s.newValue1(ssa.OpBswap16, types.Types[types.TUINT16], args[0])
   992  		},
   993  		sys.Loong64)
   994  	if cfg.goppc64 >= 10 {
   995  		// On Power10, 16-bit rotate is not available so use BRH instruction
   996  		addF("math/bits", "ReverseBytes16",
   997  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
   998  				return s.newValue1(ssa.OpBswap16, types.Types[types.TUINT], args[0])
   999  			},
  1000  			sys.PPC64)
  1001  	}
  1002  	if cfg.goriscv64 >= 22 {
  1003  		addF("math/bits", "ReverseBytes16",
  1004  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1005  				return s.newValue1(ssa.OpBswap16, types.Types[types.TUINT16], args[0])
  1006  			},
  1007  			sys.RISCV64)
  1008  	}
  1009  
  1010  	addF("math/bits", "Len64",
  1011  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1012  			return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], args[0])
  1013  		},
  1014  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.PPC64, sys.S390X, sys.Wasm)
  1015  	addF("math/bits", "Len32",
  1016  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1017  			return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0])
  1018  		},
  1019  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.PPC64, sys.S390X, sys.Wasm)
  1020  	addF("math/bits", "Len16",
  1021  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1022  			return s.newValue1(ssa.OpBitLen16, types.Types[types.TINT], args[0])
  1023  		},
  1024  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.PPC64, sys.S390X, sys.Wasm)
  1025  	addF("math/bits", "Len8",
  1026  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1027  			return s.newValue1(ssa.OpBitLen8, types.Types[types.TINT], args[0])
  1028  		},
  1029  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.MIPS, sys.PPC64, sys.S390X, sys.Wasm)
  1030  
  1031  	if cfg.goriscv64 >= 22 {
  1032  		addF("math/bits", "Len64",
  1033  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1034  				return s.newValue1(ssa.OpBitLen64, types.Types[types.TINT], args[0])
  1035  			},
  1036  			sys.RISCV64)
  1037  		addF("math/bits", "Len32",
  1038  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1039  				return s.newValue1(ssa.OpBitLen32, types.Types[types.TINT], args[0])
  1040  			},
  1041  			sys.RISCV64)
  1042  		addF("math/bits", "Len16",
  1043  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1044  				return s.newValue1(ssa.OpBitLen16, types.Types[types.TINT], args[0])
  1045  			},
  1046  			sys.RISCV64)
  1047  		addF("math/bits", "Len8",
  1048  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1049  				return s.newValue1(ssa.OpBitLen8, types.Types[types.TINT], args[0])
  1050  			},
  1051  			sys.RISCV64)
  1052  	}
  1053  
  1054  	alias("math/bits", "Len", "math/bits", "Len64", p8...)
  1055  	alias("math/bits", "Len", "math/bits", "Len32", p4...)
  1056  
  1057  	// LeadingZeros is handled because it trivially calls Len.
  1058  	addF("math/bits", "Reverse64",
  1059  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1060  			return s.newValue1(ssa.OpBitRev64, types.Types[types.TUINT64], args[0])
  1061  		},
  1062  		sys.ARM64, sys.Loong64)
  1063  	addF("math/bits", "Reverse32",
  1064  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1065  			return s.newValue1(ssa.OpBitRev32, types.Types[types.TUINT32], args[0])
  1066  		},
  1067  		sys.ARM64, sys.Loong64)
  1068  	addF("math/bits", "Reverse16",
  1069  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1070  			return s.newValue1(ssa.OpBitRev16, types.Types[types.TUINT16], args[0])
  1071  		},
  1072  		sys.ARM64, sys.Loong64)
  1073  	addF("math/bits", "Reverse8",
  1074  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1075  			return s.newValue1(ssa.OpBitRev8, types.Types[types.TUINT8], args[0])
  1076  		},
  1077  		sys.ARM64, sys.Loong64)
  1078  	addF("math/bits", "Reverse",
  1079  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1080  			return s.newValue1(ssa.OpBitRev64, types.Types[types.TUINT], args[0])
  1081  		},
  1082  		sys.ARM64, sys.Loong64)
  1083  	addF("math/bits", "RotateLeft8",
  1084  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1085  			return s.newValue2(ssa.OpRotateLeft8, types.Types[types.TUINT8], args[0], args[1])
  1086  		},
  1087  		sys.AMD64, sys.RISCV64)
  1088  	addF("math/bits", "RotateLeft16",
  1089  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1090  			return s.newValue2(ssa.OpRotateLeft16, types.Types[types.TUINT16], args[0], args[1])
  1091  		},
  1092  		sys.AMD64, sys.RISCV64)
  1093  	addF("math/bits", "RotateLeft32",
  1094  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1095  			return s.newValue2(ssa.OpRotateLeft32, types.Types[types.TUINT32], args[0], args[1])
  1096  		},
  1097  		sys.AMD64, sys.ARM, sys.ARM64, sys.Loong64, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm)
  1098  	addF("math/bits", "RotateLeft64",
  1099  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1100  			return s.newValue2(ssa.OpRotateLeft64, types.Types[types.TUINT64], args[0], args[1])
  1101  		},
  1102  		sys.AMD64, sys.ARM64, sys.Loong64, sys.PPC64, sys.RISCV64, sys.S390X, sys.Wasm)
  1103  	alias("math/bits", "RotateLeft", "math/bits", "RotateLeft64", p8...)
  1104  
  1105  	makeOnesCountAMD64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1106  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1107  			if cfg.goamd64 >= 2 {
  1108  				return s.newValue1(op, types.Types[types.TINT], args[0])
  1109  			}
  1110  
  1111  			v := s.entryNewValue0A(ssa.OpHasCPUFeature, types.Types[types.TBOOL], ir.Syms.X86HasPOPCNT)
  1112  			b := s.endBlock()
  1113  			b.Kind = ssa.BlockIf
  1114  			b.SetControl(v)
  1115  			bTrue := s.f.NewBlock(ssa.BlockPlain)
  1116  			bFalse := s.f.NewBlock(ssa.BlockPlain)
  1117  			bEnd := s.f.NewBlock(ssa.BlockPlain)
  1118  			b.AddEdgeTo(bTrue)
  1119  			b.AddEdgeTo(bFalse)
  1120  			b.Likely = ssa.BranchLikely // most machines have popcnt nowadays
  1121  
  1122  			// We have the intrinsic - use it directly.
  1123  			s.startBlock(bTrue)
  1124  			s.vars[n] = s.newValue1(op, types.Types[types.TINT], args[0])
  1125  			s.endBlock().AddEdgeTo(bEnd)
  1126  
  1127  			// Call the pure Go version.
  1128  			s.startBlock(bFalse)
  1129  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TINT]
  1130  			s.endBlock().AddEdgeTo(bEnd)
  1131  
  1132  			// Merge results.
  1133  			s.startBlock(bEnd)
  1134  			return s.variable(n, types.Types[types.TINT])
  1135  		}
  1136  	}
  1137  
  1138  	makeOnesCountLoong64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1139  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1140  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.Loong64HasLSX, s.sb)
  1141  			v := s.load(types.Types[types.TBOOL], addr)
  1142  			b := s.endBlock()
  1143  			b.Kind = ssa.BlockIf
  1144  			b.SetControl(v)
  1145  			bTrue := s.f.NewBlock(ssa.BlockPlain)
  1146  			bFalse := s.f.NewBlock(ssa.BlockPlain)
  1147  			bEnd := s.f.NewBlock(ssa.BlockPlain)
  1148  			b.AddEdgeTo(bTrue)
  1149  			b.AddEdgeTo(bFalse)
  1150  			b.Likely = ssa.BranchLikely // most loong64 machines support the LSX
  1151  
  1152  			// We have the intrinsic - use it directly.
  1153  			s.startBlock(bTrue)
  1154  			s.vars[n] = s.newValue1(op, types.Types[types.TINT], args[0])
  1155  			s.endBlock().AddEdgeTo(bEnd)
  1156  
  1157  			// Call the pure Go version.
  1158  			s.startBlock(bFalse)
  1159  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TINT]
  1160  			s.endBlock().AddEdgeTo(bEnd)
  1161  
  1162  			// Merge results.
  1163  			s.startBlock(bEnd)
  1164  			return s.variable(n, types.Types[types.TINT])
  1165  		}
  1166  	}
  1167  
  1168  	makeOnesCountRISCV64 := func(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1169  		return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1170  			if cfg.goriscv64 >= 22 {
  1171  				return s.newValue1(op, types.Types[types.TINT], args[0])
  1172  			}
  1173  
  1174  			addr := s.entryNewValue1A(ssa.OpAddr, types.Types[types.TBOOL].PtrTo(), ir.Syms.RISCV64HasZbb, s.sb)
  1175  			v := s.load(types.Types[types.TBOOL], addr)
  1176  			b := s.endBlock()
  1177  			b.Kind = ssa.BlockIf
  1178  			b.SetControl(v)
  1179  			bTrue := s.f.NewBlock(ssa.BlockPlain)
  1180  			bFalse := s.f.NewBlock(ssa.BlockPlain)
  1181  			bEnd := s.f.NewBlock(ssa.BlockPlain)
  1182  			b.AddEdgeTo(bTrue)
  1183  			b.AddEdgeTo(bFalse)
  1184  			b.Likely = ssa.BranchLikely // Majority of RISC-V support Zbb.
  1185  
  1186  			// We have the intrinsic - use it directly.
  1187  			s.startBlock(bTrue)
  1188  			s.vars[n] = s.newValue1(op, types.Types[types.TINT], args[0])
  1189  			s.endBlock().AddEdgeTo(bEnd)
  1190  
  1191  			// Call the pure Go version.
  1192  			s.startBlock(bFalse)
  1193  			s.vars[n] = s.callResult(n, callNormal) // types.Types[TINT]
  1194  			s.endBlock().AddEdgeTo(bEnd)
  1195  
  1196  			// Merge results.
  1197  			s.startBlock(bEnd)
  1198  			return s.variable(n, types.Types[types.TINT])
  1199  		}
  1200  	}
  1201  
  1202  	addF("math/bits", "OnesCount64",
  1203  		makeOnesCountAMD64(ssa.OpPopCount64),
  1204  		sys.AMD64)
  1205  	addF("math/bits", "OnesCount64",
  1206  		makeOnesCountLoong64(ssa.OpPopCount64),
  1207  		sys.Loong64)
  1208  	addF("math/bits", "OnesCount64",
  1209  		makeOnesCountRISCV64(ssa.OpPopCount64),
  1210  		sys.RISCV64)
  1211  	addF("math/bits", "OnesCount64",
  1212  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1213  			return s.newValue1(ssa.OpPopCount64, types.Types[types.TINT], args[0])
  1214  		},
  1215  		sys.PPC64, sys.ARM64, sys.S390X, sys.Wasm)
  1216  	addF("math/bits", "OnesCount32",
  1217  		makeOnesCountAMD64(ssa.OpPopCount32),
  1218  		sys.AMD64)
  1219  	addF("math/bits", "OnesCount32",
  1220  		makeOnesCountLoong64(ssa.OpPopCount32),
  1221  		sys.Loong64)
  1222  	addF("math/bits", "OnesCount32",
  1223  		makeOnesCountRISCV64(ssa.OpPopCount32),
  1224  		sys.RISCV64)
  1225  	addF("math/bits", "OnesCount32",
  1226  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1227  			return s.newValue1(ssa.OpPopCount32, types.Types[types.TINT], args[0])
  1228  		},
  1229  		sys.PPC64, sys.ARM64, sys.S390X, sys.Wasm)
  1230  	addF("math/bits", "OnesCount16",
  1231  		makeOnesCountAMD64(ssa.OpPopCount16),
  1232  		sys.AMD64)
  1233  	addF("math/bits", "OnesCount16",
  1234  		makeOnesCountLoong64(ssa.OpPopCount16),
  1235  		sys.Loong64)
  1236  	addF("math/bits", "OnesCount16",
  1237  		makeOnesCountRISCV64(ssa.OpPopCount16),
  1238  		sys.RISCV64)
  1239  	addF("math/bits", "OnesCount16",
  1240  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1241  			return s.newValue1(ssa.OpPopCount16, types.Types[types.TINT], args[0])
  1242  		},
  1243  		sys.ARM64, sys.S390X, sys.PPC64, sys.Wasm)
  1244  	addF("math/bits", "OnesCount8",
  1245  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1246  			return s.newValue1(ssa.OpPopCount8, types.Types[types.TINT], args[0])
  1247  		},
  1248  		sys.S390X, sys.PPC64, sys.Wasm)
  1249  
  1250  	if cfg.goriscv64 >= 22 {
  1251  		addF("math/bits", "OnesCount8",
  1252  			makeOnesCountRISCV64(ssa.OpPopCount8),
  1253  			sys.RISCV64)
  1254  	}
  1255  
  1256  	alias("math/bits", "OnesCount", "math/bits", "OnesCount64", p8...)
  1257  
  1258  	add("math/bits", "Mul64",
  1259  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1260  			return s.newValue2(ssa.OpMul64uhilo, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1])
  1261  		},
  1262  		all...)
  1263  	alias("math/bits", "Mul", "math/bits", "Mul64", p8...)
  1264  	addF("math/bits", "Add64",
  1265  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1266  			return s.newValue3(ssa.OpAdd64carry, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2])
  1267  		},
  1268  		sys.AMD64, sys.ARM64, sys.PPC64, sys.S390X, sys.RISCV64, sys.Loong64, sys.MIPS64)
  1269  	alias("math/bits", "Add", "math/bits", "Add64", p8...)
  1270  	alias("internal/runtime/math", "Add64", "math/bits", "Add64", all...)
  1271  	addF("math/bits", "Sub64",
  1272  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1273  			return s.newValue3(ssa.OpSub64borrow, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2])
  1274  		},
  1275  		sys.AMD64, sys.ARM64, sys.PPC64, sys.S390X, sys.RISCV64, sys.Loong64, sys.MIPS64)
  1276  	alias("math/bits", "Sub", "math/bits", "Sub64", p8...)
  1277  	addF("math/bits", "Div64",
  1278  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1279  			// check for divide-by-zero/overflow and panic with appropriate message
  1280  			cmpZero := s.newValue2(s.ssaOp(ir.ONE, types.Types[types.TUINT64]), types.Types[types.TBOOL], args[2], s.zeroVal(types.Types[types.TUINT64]))
  1281  			s.check(cmpZero, ir.Syms.Panicdivide)
  1282  			cmpOverflow := s.newValue2(s.ssaOp(ir.OLT, types.Types[types.TUINT64]), types.Types[types.TBOOL], args[0], args[2])
  1283  			s.check(cmpOverflow, ir.Syms.Panicoverflow)
  1284  			return s.newValue3(ssa.OpDiv128u, types.NewTuple(types.Types[types.TUINT64], types.Types[types.TUINT64]), args[0], args[1], args[2])
  1285  		},
  1286  		sys.AMD64)
  1287  	alias("math/bits", "Div", "math/bits", "Div64", sys.ArchAMD64)
  1288  
  1289  	alias("internal/runtime/sys", "TrailingZeros8", "math/bits", "TrailingZeros8", all...)
  1290  	alias("internal/runtime/sys", "TrailingZeros32", "math/bits", "TrailingZeros32", all...)
  1291  	alias("internal/runtime/sys", "TrailingZeros64", "math/bits", "TrailingZeros64", all...)
  1292  	alias("internal/runtime/sys", "Len8", "math/bits", "Len8", all...)
  1293  	alias("internal/runtime/sys", "Len64", "math/bits", "Len64", all...)
  1294  	alias("internal/runtime/sys", "OnesCount64", "math/bits", "OnesCount64", all...)
  1295  
  1296  	/******** sync/atomic ********/
  1297  
  1298  	// Note: these are disabled by flag_race in findIntrinsic below.
  1299  	alias("sync/atomic", "LoadInt32", "internal/runtime/atomic", "Load", all...)
  1300  	alias("sync/atomic", "LoadInt64", "internal/runtime/atomic", "Load64", all...)
  1301  	alias("sync/atomic", "LoadPointer", "internal/runtime/atomic", "Loadp", all...)
  1302  	alias("sync/atomic", "LoadUint32", "internal/runtime/atomic", "Load", all...)
  1303  	alias("sync/atomic", "LoadUint64", "internal/runtime/atomic", "Load64", all...)
  1304  	alias("sync/atomic", "LoadUintptr", "internal/runtime/atomic", "Load", p4...)
  1305  	alias("sync/atomic", "LoadUintptr", "internal/runtime/atomic", "Load64", p8...)
  1306  
  1307  	alias("sync/atomic", "StoreInt32", "internal/runtime/atomic", "Store", all...)
  1308  	alias("sync/atomic", "StoreInt64", "internal/runtime/atomic", "Store64", all...)
  1309  	// Note: not StorePointer, that needs a write barrier.  Same below for {CompareAnd}Swap.
  1310  	alias("sync/atomic", "StoreUint32", "internal/runtime/atomic", "Store", all...)
  1311  	alias("sync/atomic", "StoreUint64", "internal/runtime/atomic", "Store64", all...)
  1312  	alias("sync/atomic", "StoreUintptr", "internal/runtime/atomic", "Store", p4...)
  1313  	alias("sync/atomic", "StoreUintptr", "internal/runtime/atomic", "Store64", p8...)
  1314  
  1315  	alias("sync/atomic", "SwapInt32", "internal/runtime/atomic", "Xchg", all...)
  1316  	alias("sync/atomic", "SwapInt64", "internal/runtime/atomic", "Xchg64", all...)
  1317  	alias("sync/atomic", "SwapUint32", "internal/runtime/atomic", "Xchg", all...)
  1318  	alias("sync/atomic", "SwapUint64", "internal/runtime/atomic", "Xchg64", all...)
  1319  	alias("sync/atomic", "SwapUintptr", "internal/runtime/atomic", "Xchg", p4...)
  1320  	alias("sync/atomic", "SwapUintptr", "internal/runtime/atomic", "Xchg64", p8...)
  1321  
  1322  	alias("sync/atomic", "CompareAndSwapInt32", "internal/runtime/atomic", "Cas", all...)
  1323  	alias("sync/atomic", "CompareAndSwapInt64", "internal/runtime/atomic", "Cas64", all...)
  1324  	alias("sync/atomic", "CompareAndSwapUint32", "internal/runtime/atomic", "Cas", all...)
  1325  	alias("sync/atomic", "CompareAndSwapUint64", "internal/runtime/atomic", "Cas64", all...)
  1326  	alias("sync/atomic", "CompareAndSwapUintptr", "internal/runtime/atomic", "Cas", p4...)
  1327  	alias("sync/atomic", "CompareAndSwapUintptr", "internal/runtime/atomic", "Cas64", p8...)
  1328  
  1329  	alias("sync/atomic", "AddInt32", "internal/runtime/atomic", "Xadd", all...)
  1330  	alias("sync/atomic", "AddInt64", "internal/runtime/atomic", "Xadd64", all...)
  1331  	alias("sync/atomic", "AddUint32", "internal/runtime/atomic", "Xadd", all...)
  1332  	alias("sync/atomic", "AddUint64", "internal/runtime/atomic", "Xadd64", all...)
  1333  	alias("sync/atomic", "AddUintptr", "internal/runtime/atomic", "Xadd", p4...)
  1334  	alias("sync/atomic", "AddUintptr", "internal/runtime/atomic", "Xadd64", p8...)
  1335  
  1336  	alias("sync/atomic", "AndInt32", "internal/runtime/atomic", "And32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1337  	alias("sync/atomic", "AndUint32", "internal/runtime/atomic", "And32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1338  	alias("sync/atomic", "AndInt64", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1339  	alias("sync/atomic", "AndUint64", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1340  	alias("sync/atomic", "AndUintptr", "internal/runtime/atomic", "And64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1341  	alias("sync/atomic", "OrInt32", "internal/runtime/atomic", "Or32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1342  	alias("sync/atomic", "OrUint32", "internal/runtime/atomic", "Or32", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1343  	alias("sync/atomic", "OrInt64", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1344  	alias("sync/atomic", "OrUint64", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1345  	alias("sync/atomic", "OrUintptr", "internal/runtime/atomic", "Or64", sys.ArchARM64, sys.ArchAMD64, sys.ArchLoong64)
  1346  
  1347  	/******** math/big ********/
  1348  	alias("math/big", "mulWW", "math/bits", "Mul64", p8...)
  1349  
  1350  	/******** internal/runtime/maps ********/
  1351  
  1352  	// Important: The intrinsic implementations below return a packed
  1353  	// bitset, while the portable Go implementation uses an unpacked
  1354  	// representation (one bit set in each byte).
  1355  	//
  1356  	// Thus we must replace most bitset methods with implementations that
  1357  	// work with the packed representation.
  1358  	//
  1359  	// TODO(prattmic): The bitset implementations don't use SIMD, so they
  1360  	// could be handled with build tags (though that would break
  1361  	// -d=ssa/intrinsics/off=1).
  1362  
  1363  	// With a packed representation we no longer need to shift the result
  1364  	// of TrailingZeros64.
  1365  	alias("internal/runtime/maps", "bitsetFirst", "internal/runtime/sys", "TrailingZeros64", sys.ArchAMD64)
  1366  
  1367  	addF("internal/runtime/maps", "bitsetRemoveBelow",
  1368  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1369  			b := args[0]
  1370  			i := args[1]
  1371  
  1372  			// Clear the lower i bits in b.
  1373  			//
  1374  			// out = b &^ ((1 << i) - 1)
  1375  
  1376  			one := s.constInt64(types.Types[types.TUINT64], 1)
  1377  
  1378  			mask := s.newValue2(ssa.OpLsh8x8, types.Types[types.TUINT64], one, i)
  1379  			mask = s.newValue2(ssa.OpSub64, types.Types[types.TUINT64], mask, one)
  1380  			mask = s.newValue1(ssa.OpCom64, types.Types[types.TUINT64], mask)
  1381  
  1382  			return s.newValue2(ssa.OpAnd64, types.Types[types.TUINT64], b, mask)
  1383  		},
  1384  		sys.AMD64)
  1385  
  1386  	addF("internal/runtime/maps", "bitsetLowestSet",
  1387  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1388  			b := args[0]
  1389  
  1390  			// Test the lowest bit in b.
  1391  			//
  1392  			// out = (b & 1) == 1
  1393  
  1394  			one := s.constInt64(types.Types[types.TUINT64], 1)
  1395  			and := s.newValue2(ssa.OpAnd64, types.Types[types.TUINT64], b, one)
  1396  			return s.newValue2(ssa.OpEq64, types.Types[types.TBOOL], and, one)
  1397  		},
  1398  		sys.AMD64)
  1399  
  1400  	addF("internal/runtime/maps", "bitsetShiftOutLowest",
  1401  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1402  			b := args[0]
  1403  
  1404  			// Right shift out the lowest bit in b.
  1405  			//
  1406  			// out = b >> 1
  1407  
  1408  			one := s.constInt64(types.Types[types.TUINT64], 1)
  1409  			return s.newValue2(ssa.OpRsh64Ux64, types.Types[types.TUINT64], b, one)
  1410  		},
  1411  		sys.AMD64)
  1412  
  1413  	addF("internal/runtime/maps", "ctrlGroupMatchH2",
  1414  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1415  			g := args[0]
  1416  			h := args[1]
  1417  
  1418  			// Explicit copies to fp registers. See
  1419  			// https://go.dev/issue/70451.
  1420  			gfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, g)
  1421  			hfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, h)
  1422  
  1423  			// Broadcast h2 into each byte of a word.
  1424  			var broadcast *ssa.Value
  1425  			if buildcfg.GOAMD64 >= 4 {
  1426  				// VPBROADCASTB saves 1 instruction vs PSHUFB
  1427  				// because the input can come from a GP
  1428  				// register, while PSHUFB requires moving into
  1429  				// an FP register first.
  1430  				//
  1431  				// Nominally PSHUFB would require a second
  1432  				// additional instruction to load the control
  1433  				// mask into a FP register. But broadcast uses
  1434  				// a control mask of 0, and the register ABI
  1435  				// already defines X15 as a zero register.
  1436  				broadcast = s.newValue1(ssa.OpAMD64VPBROADCASTB, types.TypeInt128, h) // use gp copy of h
  1437  			} else if buildcfg.GOAMD64 >= 2 {
  1438  				// PSHUFB performs a byte broadcast when given
  1439  				// a control input of 0.
  1440  				broadcast = s.newValue1(ssa.OpAMD64PSHUFBbroadcast, types.TypeInt128, hfp)
  1441  			} else {
  1442  				// No direct byte broadcast. First we must
  1443  				// duplicate the lower byte and then do a
  1444  				// 16-bit broadcast.
  1445  
  1446  				// "Unpack" h2 with itself. This duplicates the
  1447  				// input, resulting in h2 in the lower two
  1448  				// bytes.
  1449  				unpack := s.newValue2(ssa.OpAMD64PUNPCKLBW, types.TypeInt128, hfp, hfp)
  1450  
  1451  				// Copy the lower 16-bits of unpack into every
  1452  				// 16-bit slot in the lower 64-bits of the
  1453  				// output register. Note that immediate 0
  1454  				// selects the low word as the source for every
  1455  				// destination slot.
  1456  				broadcast = s.newValue1I(ssa.OpAMD64PSHUFLW, types.TypeInt128, 0, unpack)
  1457  
  1458  				// No need to broadcast into the upper 64-bits,
  1459  				// as we don't use those.
  1460  			}
  1461  
  1462  			// Compare each byte of the control word with h2. Each
  1463  			// matching byte has every bit set.
  1464  			eq := s.newValue2(ssa.OpAMD64PCMPEQB, types.TypeInt128, broadcast, gfp)
  1465  
  1466  			// Construct a "byte mask": each output bit is equal to
  1467  			// the sign bit each input byte.
  1468  			//
  1469  			// This results in a packed output (bit N set means
  1470  			// byte N matched).
  1471  			//
  1472  			// NOTE: See comment above on bitsetFirst.
  1473  			out := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT8], eq)
  1474  
  1475  			// g is only 64-bits so the upper 64-bits of the
  1476  			// 128-bit register will be zero. If h2 is also zero,
  1477  			// then we'll get matches on those bytes. Truncate the
  1478  			// upper bits to ignore such matches.
  1479  			ret := s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], out)
  1480  
  1481  			return ret
  1482  		},
  1483  		sys.AMD64)
  1484  
  1485  	addF("internal/runtime/maps", "ctrlGroupMatchEmpty",
  1486  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1487  			// An empty slot is   1000 0000
  1488  			// A deleted slot is  1111 1110
  1489  			// A full slot is     0??? ????
  1490  
  1491  			g := args[0]
  1492  
  1493  			// Explicit copy to fp register. See
  1494  			// https://go.dev/issue/70451.
  1495  			gfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, g)
  1496  
  1497  			if buildcfg.GOAMD64 >= 2 {
  1498  				// "PSIGNB negates each data element of the
  1499  				// destination operand (the first operand) if
  1500  				// the signed integer value of the
  1501  				// corresponding data element in the source
  1502  				// operand (the second operand) is less than
  1503  				// zero. If the signed integer value of a data
  1504  				// element in the source operand is positive,
  1505  				// the corresponding data element in the
  1506  				// destination operand is unchanged. If a data
  1507  				// element in the source operand is zero, the
  1508  				// corresponding data element in the
  1509  				// destination operand is set to zero" - Intel SDM
  1510  				//
  1511  				// If we pass the group control word as both
  1512  				// arguments:
  1513  				// - Full slots are unchanged.
  1514  				// - Deleted slots are negated, becoming
  1515  				//   0000 0010.
  1516  				// - Empty slots are negated, becoming
  1517  				//   1000 0000 (unchanged!).
  1518  				//
  1519  				// The result is that only empty slots have the
  1520  				// sign bit set. We then use PMOVMSKB to
  1521  				// extract the sign bits.
  1522  				sign := s.newValue2(ssa.OpAMD64PSIGNB, types.TypeInt128, gfp, gfp)
  1523  
  1524  				// Construct a "byte mask": each output bit is
  1525  				// equal to the sign bit each input byte. The
  1526  				// sign bit is only set for empty or deleted
  1527  				// slots.
  1528  				//
  1529  				// This results in a packed output (bit N set
  1530  				// means byte N matched).
  1531  				//
  1532  				// NOTE: See comment above on bitsetFirst.
  1533  				ret := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT64], sign)
  1534  
  1535  				// g is only 64-bits so the upper 64-bits of
  1536  				// the 128-bit register will be zero. PSIGNB
  1537  				// will keep all of these bytes zero, so no
  1538  				// need to truncate.
  1539  
  1540  				return ret
  1541  			}
  1542  
  1543  			// No PSIGNB, simply do byte equality with ctrlEmpty.
  1544  
  1545  			// Load ctrlEmpty into each byte of a control word.
  1546  			var ctrlsEmpty uint64 = abi.MapCtrlEmpty
  1547  			e := s.constInt64(types.Types[types.TUINT64], int64(ctrlsEmpty))
  1548  			// Explicit copy to fp register. See
  1549  			// https://go.dev/issue/70451.
  1550  			efp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, e)
  1551  
  1552  			// Compare each byte of the control word with ctrlEmpty. Each
  1553  			// matching byte has every bit set.
  1554  			eq := s.newValue2(ssa.OpAMD64PCMPEQB, types.TypeInt128, efp, gfp)
  1555  
  1556  			// Construct a "byte mask": each output bit is equal to
  1557  			// the sign bit each input byte.
  1558  			//
  1559  			// This results in a packed output (bit N set means
  1560  			// byte N matched).
  1561  			//
  1562  			// NOTE: See comment above on bitsetFirst.
  1563  			out := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT8], eq)
  1564  
  1565  			// g is only 64-bits so the upper 64-bits of the
  1566  			// 128-bit register will be zero. The upper 64-bits of
  1567  			// efp are also zero, so we'll get matches on those
  1568  			// bytes. Truncate the upper bits to ignore such
  1569  			// matches.
  1570  			return s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], out)
  1571  		},
  1572  		sys.AMD64)
  1573  
  1574  	addF("internal/runtime/maps", "ctrlGroupMatchEmptyOrDeleted",
  1575  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1576  			// An empty slot is   1000 0000
  1577  			// A deleted slot is  1111 1110
  1578  			// A full slot is     0??? ????
  1579  			//
  1580  			// A slot is empty or deleted iff bit 7 (sign bit) is
  1581  			// set.
  1582  
  1583  			g := args[0]
  1584  
  1585  			// Explicit copy to fp register. See
  1586  			// https://go.dev/issue/70451.
  1587  			gfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, g)
  1588  
  1589  			// Construct a "byte mask": each output bit is equal to
  1590  			// the sign bit each input byte. The sign bit is only
  1591  			// set for empty or deleted slots.
  1592  			//
  1593  			// This results in a packed output (bit N set means
  1594  			// byte N matched).
  1595  			//
  1596  			// NOTE: See comment above on bitsetFirst.
  1597  			ret := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT64], gfp)
  1598  
  1599  			// g is only 64-bits so the upper 64-bits of the
  1600  			// 128-bit register will be zero. Zero will never match
  1601  			// ctrlEmpty or ctrlDeleted, so no need to truncate.
  1602  
  1603  			return ret
  1604  		},
  1605  		sys.AMD64)
  1606  
  1607  	addF("internal/runtime/maps", "ctrlGroupMatchFull",
  1608  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1609  			// An empty slot is   1000 0000
  1610  			// A deleted slot is  1111 1110
  1611  			// A full slot is     0??? ????
  1612  			//
  1613  			// A slot is full iff bit 7 (sign bit) is unset.
  1614  
  1615  			g := args[0]
  1616  
  1617  			// Explicit copy to fp register. See
  1618  			// https://go.dev/issue/70451.
  1619  			gfp := s.newValue1(ssa.OpAMD64MOVQi2f, types.TypeInt128, g)
  1620  
  1621  			// Construct a "byte mask": each output bit is equal to
  1622  			// the sign bit each input byte. The sign bit is only
  1623  			// set for empty or deleted slots.
  1624  			//
  1625  			// This results in a packed output (bit N set means
  1626  			// byte N matched).
  1627  			//
  1628  			// NOTE: See comment above on bitsetFirst.
  1629  			mask := s.newValue1(ssa.OpAMD64PMOVMSKB, types.Types[types.TUINT8], gfp)
  1630  
  1631  			// Invert the mask to set the bits for the full slots.
  1632  			out := s.newValue1(ssa.OpCom8, types.Types[types.TUINT8], mask)
  1633  
  1634  			// g is only 64-bits so the upper 64-bits of the
  1635  			// 128-bit register will be zero, with bit 7 unset.
  1636  			// Truncate the upper bits to ignore these.
  1637  			return s.newValue1(ssa.OpZeroExt8to64, types.Types[types.TUINT64], out)
  1638  		},
  1639  		sys.AMD64)
  1640  
  1641  	/******** crypto/internal/constanttime ********/
  1642  	// We implement a superset of the Select promise:
  1643  	// Select returns x if v != 0 and y if v == 0.
  1644  	hasCMOV := []*sys.Arch{sys.ArchAMD64, sys.ArchARM64, sys.ArchLoong64, sys.ArchPPC64, sys.ArchPPC64LE, sys.ArchWasm}
  1645  	if cfg.goriscv64 >= 23 {
  1646  		hasCMOV = append(hasCMOV, sys.ArchRISCV64)
  1647  	}
  1648  	add("crypto/internal/constanttime", "Select",
  1649  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1650  			v, x, y := args[0], args[1], args[2]
  1651  
  1652  			var checkOp ssa.Op
  1653  			var zero *ssa.Value
  1654  			switch s.config.PtrSize {
  1655  			case 8:
  1656  				checkOp = ssa.OpNeq64
  1657  				zero = s.constInt64(types.Types[types.TINT], 0)
  1658  			case 4:
  1659  				checkOp = ssa.OpNeq32
  1660  				zero = s.constInt32(types.Types[types.TINT], 0)
  1661  			default:
  1662  				panic("unreachable")
  1663  			}
  1664  			check := s.newValue2(checkOp, types.Types[types.TBOOL], zero, v)
  1665  
  1666  			return s.newValue3(ssa.OpCondSelect, types.Types[types.TINT], x, y, check)
  1667  		}, hasCMOV...) // all with CMOV support.
  1668  	add("crypto/internal/constanttime", "boolToUint8",
  1669  		func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1670  			return s.newValue1(ssa.OpCvtBoolToUint8, types.Types[types.TUINT8], args[0])
  1671  		},
  1672  		all...)
  1673  
  1674  	if buildcfg.Experiment.SIMD {
  1675  		// Only enable intrinsics, if SIMD experiment.
  1676  		simdAMD64Intrinsics(addF)
  1677  		simdARM64Intrinsics(addF)
  1678  		initWasmSIMD()
  1679  
  1680  		addF(simdPackage, "ClearAVXUpperBits",
  1681  			func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1682  				s.vars[memVar] = s.newValue1(ssa.OpAMD64VZEROUPPER, types.TypeMem, s.mem())
  1683  				return nil
  1684  			},
  1685  			sys.AMD64)
  1686  
  1687  		addF(simdPackage, "Int8x16.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1688  		addF(simdPackage, "Int16x8.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1689  		addF(simdPackage, "Int32x4.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1690  		addF(simdPackage, "Int64x2.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1691  		addF(simdPackage, "Uint8x16.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1692  		addF(simdPackage, "Uint16x8.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1693  		addF(simdPackage, "Uint32x4.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1694  		addF(simdPackage, "Uint64x2.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1695  		addF(simdPackage, "Int8x32.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1696  		addF(simdPackage, "Int16x16.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1697  		addF(simdPackage, "Int32x8.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1698  		addF(simdPackage, "Int64x4.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1699  		addF(simdPackage, "Uint8x32.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1700  		addF(simdPackage, "Uint16x16.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1701  		addF(simdPackage, "Uint32x8.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1702  		addF(simdPackage, "Uint64x4.IsZero", opLen1(ssa.OpIsZeroVec, types.Types[types.TBOOL]), sys.AMD64)
  1703  		addF(simdPackage, "Float32x4.IsNaN", opLen1(ssa.OpIsNaNFloat32x4, types.TypeVec128), sys.AMD64)
  1704  		addF(simdPackage, "Float32x8.IsNaN", opLen1(ssa.OpIsNaNFloat32x8, types.TypeVec256), sys.AMD64)
  1705  		addF(simdPackage, "Float32x16.IsNaN", opLen1(ssa.OpIsNaNFloat32x16, types.TypeVec512), sys.AMD64)
  1706  		addF(simdPackage, "Float64x2.IsNaN", opLen1(ssa.OpIsNaNFloat64x2, types.TypeVec128), sys.AMD64)
  1707  		addF(simdPackage, "Float64x4.IsNaN", opLen1(ssa.OpIsNaNFloat64x4, types.TypeVec256), sys.AMD64)
  1708  		addF(simdPackage, "Float64x8.IsNaN", opLen1(ssa.OpIsNaNFloat64x8, types.TypeVec512), sys.AMD64)
  1709  
  1710  		// sfp4 is intrinsic-if-constant, but otherwise it's complicated enough to just implement in Go.
  1711  		sfp4 := func(method string, hwop ssa.Op, vectype *types.Type) {
  1712  			addF(simdPackage, method,
  1713  				func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1714  					x, a, b, c, d, y := args[0], args[1], args[2], args[3], args[4], args[5]
  1715  					if a.Op == ssa.OpConst8 && b.Op == ssa.OpConst8 && c.Op == ssa.OpConst8 && d.Op == ssa.OpConst8 {
  1716  						z := select4FromPair(x, a, b, c, d, y, s, hwop, vectype)
  1717  						if z != nil {
  1718  							return z
  1719  						}
  1720  					}
  1721  					return s.callResult(n, callNormal)
  1722  				},
  1723  				sys.AMD64)
  1724  		}
  1725  
  1726  		sfp4("Int32x4.ConcatPermuteScalars", ssa.OpconcatSelectedConstantInt32x4, types.TypeVec128)
  1727  		sfp4("Uint32x4.ConcatPermuteScalars", ssa.OpconcatSelectedConstantUint32x4, types.TypeVec128)
  1728  		sfp4("Float32x4.ConcatPermuteScalars", ssa.OpconcatSelectedConstantFloat32x4, types.TypeVec128)
  1729  
  1730  		sfp4("Int32x8.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedInt32x8, types.TypeVec256)
  1731  		sfp4("Uint32x8.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedUint32x8, types.TypeVec256)
  1732  		sfp4("Float32x8.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedFloat32x8, types.TypeVec256)
  1733  
  1734  		sfp4("Int32x16.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedInt32x16, types.TypeVec512)
  1735  		sfp4("Uint32x16.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedUint32x16, types.TypeVec512)
  1736  		sfp4("Float32x16.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedFloat32x16, types.TypeVec512)
  1737  
  1738  		// sfp2 is intrinsic-if-constant, but otherwise it's complicated enough to just implement in Go.
  1739  		sfp2 := func(method string, hwop ssa.Op, vectype *types.Type, cscimm func(i, j uint8) int64) {
  1740  			addF(simdPackage, method,
  1741  				func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1742  					x, a, b, y := args[0], args[1], args[2], args[3]
  1743  					if a.Op == ssa.OpConst8 && b.Op == ssa.OpConst8 {
  1744  						z := select2FromPair(x, a, b, y, s, hwop, vectype, cscimm)
  1745  						if z != nil {
  1746  							return z
  1747  						}
  1748  					}
  1749  					return s.callResult(n, callNormal)
  1750  				},
  1751  				sys.AMD64)
  1752  		}
  1753  
  1754  		sfp2("Uint64x2.ConcatPermuteScalars", ssa.OpconcatSelectedConstantUint64x2, types.TypeVec128, cscimm2)
  1755  		sfp2("Int64x2.ConcatPermuteScalars", ssa.OpconcatSelectedConstantInt64x2, types.TypeVec128, cscimm2)
  1756  		sfp2("Float64x2.ConcatPermuteScalars", ssa.OpconcatSelectedConstantFloat64x2, types.TypeVec128, cscimm2)
  1757  
  1758  		sfp2("Uint64x4.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedUint64x4, types.TypeVec256, cscimm2g2)
  1759  		sfp2("Int64x4.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedInt64x4, types.TypeVec256, cscimm2g2)
  1760  		sfp2("Float64x4.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedFloat64x4, types.TypeVec256, cscimm2g2)
  1761  
  1762  		sfp2("Uint64x8.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedUint64x8, types.TypeVec512, cscimm2g4)
  1763  		sfp2("Int64x8.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedInt64x8, types.TypeVec512, cscimm2g4)
  1764  		sfp2("Float64x8.ConcatPermuteScalarsGrouped", ssa.OpconcatSelectedConstantGroupedFloat64x8, types.TypeVec512, cscimm2g4)
  1765  
  1766  	}
  1767  }
  1768  
  1769  const simdPackage = "simd/archsimd"
  1770  
  1771  func cscimm4(a, b, c, d uint8) int64 {
  1772  	return se(a + b<<2 + c<<4 + d<<6)
  1773  }
  1774  
  1775  func cscimm2(a, b uint8) int64 {
  1776  	return se(a + b<<1)
  1777  }
  1778  
  1779  func cscimm2g2(a, b uint8) int64 {
  1780  	g := cscimm2(a, b)
  1781  	return int64(int8(g + g<<2))
  1782  }
  1783  
  1784  func cscimm2g4(a, b uint8) int64 {
  1785  	g := cscimm2g2(a, b)
  1786  	return int64(int8(g + g<<4))
  1787  }
  1788  
  1789  const (
  1790  	_LLLL = iota
  1791  	_HLLL
  1792  	_LHLL
  1793  	_HHLL
  1794  	_LLHL
  1795  	_HLHL
  1796  	_LHHL
  1797  	_HHHL
  1798  	_LLLH
  1799  	_HLLH
  1800  	_LHLH
  1801  	_HHLH
  1802  	_LLHH
  1803  	_HLHH
  1804  	_LHHH
  1805  	_HHHH
  1806  )
  1807  
  1808  const (
  1809  	_LL = iota
  1810  	_HL
  1811  	_LH
  1812  	_HH
  1813  )
  1814  
  1815  func select2FromPair(x, _a, _b, y *ssa.Value, s *state, op ssa.Op, t *types.Type, csc func(a, b uint8) int64) *ssa.Value {
  1816  	a, b := uint8(_a.AuxInt8()), uint8(_b.AuxInt8())
  1817  	if a > 3 || b > 3 {
  1818  		return nil
  1819  	}
  1820  	pattern := (a&2)>>1 + (b & 2)
  1821  	a, b = a&1, b&1
  1822  
  1823  	switch pattern {
  1824  	case _LL:
  1825  		return s.newValue2I(op, t, csc(a, b), x, x)
  1826  	case _HH:
  1827  		return s.newValue2I(op, t, csc(a, b), y, y)
  1828  	case _LH:
  1829  		return s.newValue2I(op, t, csc(a, b), x, y)
  1830  	case _HL:
  1831  		return s.newValue2I(op, t, csc(a, b), y, x)
  1832  	}
  1833  	panic("The preceding switch should have been exhaustive")
  1834  }
  1835  
  1836  func select4FromPair(x, _a, _b, _c, _d, y *ssa.Value, s *state, op ssa.Op, t *types.Type) *ssa.Value {
  1837  	a, b, c, d := uint8(_a.AuxInt8()), uint8(_b.AuxInt8()), uint8(_c.AuxInt8()), uint8(_d.AuxInt8())
  1838  	if a > 7 || b > 7 || c > 7 || d > 7 {
  1839  		return nil
  1840  	}
  1841  	pattern := a>>2 + (b&4)>>1 + (c & 4) + (d&4)<<1
  1842  
  1843  	a, b, c, d = a&3, b&3, c&3, d&3
  1844  
  1845  	switch pattern {
  1846  	case _LLLL:
  1847  		// TODO DETECT 0,1,2,3, 0,0,0,0
  1848  		return s.newValue2I(op, t, cscimm4(a, b, c, d), x, x)
  1849  	case _HHHH:
  1850  		// TODO DETECT 0,1,2,3, 0,0,0,0
  1851  		return s.newValue2I(op, t, cscimm4(a, b, c, d), y, y)
  1852  	case _LLHH:
  1853  		return s.newValue2I(op, t, cscimm4(a, b, c, d), x, y)
  1854  	case _HHLL:
  1855  		return s.newValue2I(op, t, cscimm4(a, b, c, d), y, x)
  1856  
  1857  	case _HLLL:
  1858  		z := s.newValue2I(op, t, cscimm4(a, a, b, b), y, x)
  1859  		return s.newValue2I(op, t, cscimm4(0, 2, c, d), z, x)
  1860  	case _LHLL:
  1861  		z := s.newValue2I(op, t, cscimm4(a, a, b, b), x, y)
  1862  		return s.newValue2I(op, t, cscimm4(0, 2, c, d), z, x)
  1863  	case _HLHH:
  1864  		z := s.newValue2I(op, t, cscimm4(a, a, b, b), y, x)
  1865  		return s.newValue2I(op, t, cscimm4(0, 2, c, d), z, y)
  1866  	case _LHHH:
  1867  		z := s.newValue2I(op, t, cscimm4(a, a, b, b), x, y)
  1868  		return s.newValue2I(op, t, cscimm4(0, 2, c, d), z, y)
  1869  
  1870  	case _LLLH:
  1871  		z := s.newValue2I(op, t, cscimm4(c, c, d, d), x, y)
  1872  		return s.newValue2I(op, t, cscimm4(a, b, 0, 2), x, z)
  1873  	case _LLHL:
  1874  		z := s.newValue2I(op, t, cscimm4(c, c, d, d), y, x)
  1875  		return s.newValue2I(op, t, cscimm4(a, b, 0, 2), x, z)
  1876  
  1877  	case _HHLH:
  1878  		z := s.newValue2I(op, t, cscimm4(c, c, d, d), x, y)
  1879  		return s.newValue2I(op, t, cscimm4(a, b, 0, 2), y, z)
  1880  
  1881  	case _HHHL:
  1882  		z := s.newValue2I(op, t, cscimm4(c, c, d, d), y, x)
  1883  		return s.newValue2I(op, t, cscimm4(a, b, 0, 2), y, z)
  1884  
  1885  	case _LHLH:
  1886  		z := s.newValue2I(op, t, cscimm4(a, c, b, d), x, y)
  1887  		return s.newValue2I(op, t, se(0b11_01_10_00), z, z)
  1888  	case _HLHL:
  1889  		z := s.newValue2I(op, t, cscimm4(b, d, a, c), x, y)
  1890  		return s.newValue2I(op, t, se(0b01_11_00_10), z, z)
  1891  	case _HLLH:
  1892  		z := s.newValue2I(op, t, cscimm4(b, c, a, d), x, y)
  1893  		return s.newValue2I(op, t, se(0b11_01_00_10), z, z)
  1894  	case _LHHL:
  1895  		z := s.newValue2I(op, t, cscimm4(a, d, b, c), x, y)
  1896  		return s.newValue2I(op, t, se(0b01_11_10_00), z, z)
  1897  	}
  1898  	panic("The preceding switch should have been exhaustive")
  1899  }
  1900  
  1901  // se smears the not-really-a-sign bit of a uint8 to conform to the conventions
  1902  // for representing AuxInt in ssa.
  1903  func se(x uint8) int64 {
  1904  	return int64(int8(x))
  1905  }
  1906  
  1907  func opLen1(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1908  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1909  		return s.newValue1(op, t, args[0])
  1910  	}
  1911  }
  1912  
  1913  func opLen2(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1914  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1915  		return s.newValue2(op, t, args[0], args[1])
  1916  	}
  1917  }
  1918  
  1919  func opLen2_21(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1920  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1921  		return s.newValue2(op, t, args[1], args[0])
  1922  	}
  1923  }
  1924  
  1925  func opLen3(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1926  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1927  		return s.newValue3(op, t, args[0], args[1], args[2])
  1928  	}
  1929  }
  1930  
  1931  var ssaVecBySize = map[int64]*types.Type{
  1932  	16: types.TypeVec128,
  1933  	32: types.TypeVec256,
  1934  	64: types.TypeVec512,
  1935  }
  1936  
  1937  func opLen3_31Zero3(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1938  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1939  		if t, ok := ssaVecBySize[args[1].Type.Size()]; !ok {
  1940  			panic("unknown simd vector size")
  1941  		} else {
  1942  			return s.newValue3(op, t, s.newValue0(ssa.OpZeroSIMD, t), args[1], args[0])
  1943  		}
  1944  	}
  1945  }
  1946  
  1947  func opLen3_21(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1948  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1949  		return s.newValue3(op, t, args[1], args[0], args[2])
  1950  	}
  1951  }
  1952  
  1953  func opLen3_231(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1954  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1955  		return s.newValue3(op, t, args[2], args[0], args[1])
  1956  	}
  1957  }
  1958  
  1959  func opLen4(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1960  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1961  		return s.newValue4(op, t, args[0], args[1], args[2], args[3])
  1962  	}
  1963  }
  1964  
  1965  func opLen4_231(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1966  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1967  		return s.newValue4(op, t, args[2], args[0], args[1], args[3])
  1968  	}
  1969  }
  1970  
  1971  func opLen4_31(op ssa.Op, t *types.Type) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1972  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  1973  		return s.newValue4(op, t, args[2], args[1], args[0], args[3])
  1974  	}
  1975  }
  1976  
  1977  func immJumpTable(s *state, idx *ssa.Value, intrinsicCall *ir.CallExpr, genOp func(*state, int)) *ssa.Value {
  1978  	if !idx.Type.IsKind(types.TUINT8) && !idx.Type.IsKind(types.TUINT64) {
  1979  		panic("immJumpTable expects uint8 or uint64 value")
  1980  	}
  1981  	if idx.Type.IsKind(types.TUINT64) {
  1982  		// Match the constant path and keep the jump table index in range.
  1983  		idx = s.conv(nil, idx, idx.Type, types.Types[types.TUINT8])
  1984  	}
  1985  
  1986  	if base.Ctxt.Retpoline {
  1987  		// Note spectre=all implies retpoline which requires binary search instead of table switch.
  1988  		return branchTableImm8(s, idx, intrinsicCall, genOp)
  1989  	}
  1990  
  1991  	// Make blocks we'll need.
  1992  	bEnd := s.f.NewBlock(ssa.BlockPlain)
  1993  
  1994  	// We will exhaust 0-255, so no need to check the bounds.
  1995  	t := types.Types[types.TUINTPTR]
  1996  	idx = s.conv(nil, idx, idx.Type, t)
  1997  
  1998  	b := s.curBlock
  1999  	b.Kind = ssa.BlockJumpTable
  2000  	b.Pos = intrinsicCall.Pos()
  2001  
  2002  	b.SetControl(idx)
  2003  	targets := [256]*ssa.Block{}
  2004  	for i := range 256 {
  2005  		t := s.f.NewBlock(ssa.BlockPlain)
  2006  		targets[i] = t
  2007  		b.AddEdgeTo(t)
  2008  	}
  2009  	s.endBlock()
  2010  
  2011  	for i, t := range targets {
  2012  		s.startBlock(t)
  2013  		genOp(s, i)
  2014  		if t.Kind != ssa.BlockExit {
  2015  			t.AddEdgeTo(bEnd)
  2016  		}
  2017  		s.endBlock()
  2018  	}
  2019  
  2020  	s.startBlock(bEnd)
  2021  	ret := s.variable(intrinsicCall, intrinsicCall.Type())
  2022  	return ret
  2023  }
  2024  
  2025  func branchTableImm8(s *state, idx *ssa.Value, intrinsicCall *ir.CallExpr, genOp func(*state, int)) *ssa.Value {
  2026  	return branchTableN(s, idx, intrinsicCall, genOp, 256, true)
  2027  }
  2028  
  2029  func branchTableN(s *state, idx *ssa.Value, intrinsicCall *ir.CallExpr, genOp func(*state, int), immLimit uint64, preChecked bool) *ssa.Value {
  2030  	// Make blocks we'll need.
  2031  	bEnd := s.f.NewBlock(ssa.BlockPlain)
  2032  	bPanic := s.f.NewBlock(ssa.BlockPlain)
  2033  
  2034  	jt := s.f.NewBlock(ssa.BlockPlain)
  2035  
  2036  	t := types.Types[types.TUINTPTR]
  2037  	idx = s.conv(nil, idx, idx.Type, t)
  2038  
  2039  	if !preChecked {
  2040  		// Begin with a bounds check
  2041  		width := s.uintptrConstant(immLimit)
  2042  		cmp := s.newValue2(s.ssaOp(ir.OLT, t), types.Types[types.TBOOL], idx, width)
  2043  		bb := s.endBlock()
  2044  		bb.Kind = ssa.BlockIf
  2045  		bb.SetControl(cmp)
  2046  		bb.AddEdgeTo(jt)             // in range - use jump table
  2047  		bb.AddEdgeTo(bPanic)         // out of range - panic
  2048  		bb.Likely = ssa.BranchLikely // panic is unlikely
  2049  
  2050  		s.startBlock(bPanic)
  2051  		s.rtcall(ir.Syms.PanicSimdImm, false, nil)
  2052  	}
  2053  	if s.curBlock != nil {
  2054  		bb := s.endBlock()
  2055  		bb.AddEdgeTo(jt)
  2056  	}
  2057  
  2058  	s.startBlock(jt)
  2059  	jt.Kind = ssa.BlockPlain
  2060  	jt.Pos = intrinsicCall.Pos()
  2061  
  2062  	branchTableNInner(s, idx, 0, immLimit, genOp, bEnd)
  2063  
  2064  	s.startBlock(bEnd)
  2065  	ret := s.variable(intrinsicCall, intrinsicCall.Type())
  2066  	return ret
  2067  }
  2068  
  2069  func branchTableNInner(s *state, idx *ssa.Value, lowInclusive, len uint64, genOp func(*state, int), bEnd *ssa.Block) {
  2070  	t := types.Types[types.TUINTPTR]
  2071  	if len == 0 {
  2072  		panic("empty branch table")
  2073  	}
  2074  	if len == 1 {
  2075  		genOp(s, int(lowInclusive+len-1))
  2076  		if s.curBlock != nil { // if genOp was "panic" then curBlock is already ended and nil
  2077  			if s.curBlock.Kind != ssa.BlockExit {
  2078  				s.curBlock.AddEdgeTo(bEnd)
  2079  			}
  2080  			s.endBlock()
  2081  		}
  2082  		return
  2083  	}
  2084  
  2085  	s.curBlock.Kind = ssa.BlockIf
  2086  	cmp := s.newValue2(s.ssaOp(ir.OLT, t), types.Types[types.TBOOL], idx, s.uintptrConstant(lowInclusive+len/2))
  2087  	bb := s.endBlock()
  2088  	bb.Kind = ssa.BlockIf
  2089  	bb.SetControl(cmp)
  2090  	bMatch := s.f.NewBlock(ssa.BlockPlain)
  2091  	bNext := s.f.NewBlock(ssa.BlockPlain)
  2092  	bb.AddEdgeTo(bMatch)
  2093  	bb.AddEdgeTo(bNext)
  2094  	s.startBlock(bMatch)
  2095  	branchTableNInner(s, idx, lowInclusive, len/2, genOp, bEnd)
  2096  	s.startBlock(bNext)
  2097  	branchTableNInner(s, idx, lowInclusive+len/2, len-len/2, genOp, bEnd)
  2098  }
  2099  
  2100  // immJumpTableN emits a jump table to one of a number of indexed cases, from zero to n-1.
  2101  // an index of n or larger will panic
  2102  func immJumpTableN(s *state, idx *ssa.Value, intrinsicCall *ir.CallExpr, immLimit uint64, genOp func(*state, int)) *ssa.Value {
  2103  
  2104  	if !idx.Type.IsKind(types.TUINT8) && !idx.Type.IsKind(types.TUINT64) {
  2105  		s.Fatalf("immJumpTable expects uint8 or uint64 value, saw %v instead, val=%s", idx.Type.String(), idx.LongString())
  2106  	}
  2107  
  2108  	if base.Flag.N != 0 || !Arch.LinkArch.CanJumpTable || base.Ctxt.Retpoline {
  2109  		return branchTableN(s, idx, intrinsicCall, genOp, immLimit, false)
  2110  	}
  2111  
  2112  	// Make blocks we'll need.
  2113  	bEnd := s.f.NewBlock(ssa.BlockPlain)
  2114  	bPanic := s.f.NewBlock(ssa.BlockPlain)
  2115  
  2116  	jt := s.f.NewBlock(ssa.BlockJumpTable)
  2117  
  2118  	t := types.Types[types.TUINTPTR]
  2119  	idx = s.conv(nil, idx, idx.Type, t)
  2120  	width := s.uintptrConstant(immLimit)
  2121  
  2122  	// Begin with a bounds check
  2123  	cmp := s.newValue2(s.ssaOp(ir.OLT, t), types.Types[types.TBOOL], idx, width)
  2124  	bb := s.endBlock()
  2125  	bb.Kind = ssa.BlockIf
  2126  	bb.SetControl(cmp)
  2127  	bb.AddEdgeTo(jt)             // in range - use jump table
  2128  	bb.AddEdgeTo(bPanic)         // out of range - panic
  2129  	bb.Likely = ssa.BranchLikely // panic is unlikely
  2130  
  2131  	s.startBlock(bPanic)
  2132  	s.rtcall(ir.Syms.PanicSimdImm, false, nil)
  2133  	s.endBlock()
  2134  
  2135  	s.startBlock(jt)
  2136  	jt.Kind = ssa.BlockJumpTable
  2137  	jt.Pos = intrinsicCall.Pos()
  2138  	if base.Flag.Cfg.SpectreIndex {
  2139  		// Potential Spectre vulnerability hardening?
  2140  		idx = s.newValue2(ssa.OpSpectreSliceIndex, t, idx, s.uintptrConstant(immLimit-1))
  2141  	}
  2142  	jt.SetControl(idx)
  2143  	targets := make([]*ssa.Block, immLimit, immLimit)
  2144  	for i := range immLimit {
  2145  		t := s.f.NewBlock(ssa.BlockPlain)
  2146  		targets[i] = t
  2147  		jt.AddEdgeTo(t)
  2148  	}
  2149  	s.endBlock()
  2150  
  2151  	for i, t := range targets {
  2152  		s.startBlock(t)
  2153  		genOp(s, i)
  2154  		if t.Kind != ssa.BlockExit {
  2155  			t.AddEdgeTo(bEnd)
  2156  		}
  2157  		s.endBlock()
  2158  	}
  2159  
  2160  	s.startBlock(bEnd)
  2161  	ret := s.variable(intrinsicCall, intrinsicCall.Type())
  2162  	return ret
  2163  }
  2164  
  2165  func opLen1Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2166  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2167  		if args[1].Op == ssa.OpConst8 || args[1].Op == ssa.OpConst64 {
  2168  			return s.newValue1I(op, t, int64(int8(args[1].AuxInt<<int64(offset))), args[0])
  2169  		}
  2170  		return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
  2171  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2172  			s.vars[n] = sNew.newValue1I(op, t, int64(int8(idx<<offset)), args[0])
  2173  		})
  2174  	}
  2175  }
  2176  
  2177  func opLen2Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2178  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2179  		if args[1].Op == ssa.OpConst8 || args[1].Op == ssa.OpConst64 {
  2180  			return s.newValue2I(op, t, int64(int8(args[1].AuxInt<<int64(offset))), args[0], args[2])
  2181  		}
  2182  		return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
  2183  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2184  			s.vars[n] = sNew.newValue2I(op, t, int64(int8(idx<<offset)), args[0], args[2])
  2185  		})
  2186  	}
  2187  }
  2188  
  2189  func opLen3Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2190  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2191  		if args[1].Op == ssa.OpConst8 || args[1].Op == ssa.OpConst64 {
  2192  			return s.newValue3I(op, t, int64(int8(args[1].AuxInt<<int64(offset))), args[0], args[2], args[3])
  2193  		}
  2194  		return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
  2195  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2196  			s.vars[n] = sNew.newValue3I(op, t, int64(int8(idx<<offset)), args[0], args[2], args[3])
  2197  		})
  2198  	}
  2199  }
  2200  
  2201  func opLen2Imm8_2I(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2202  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2203  		if args[2].Op == ssa.OpConst8 || args[2].Op == ssa.OpConst64 {
  2204  			return s.newValue2I(op, t, int64(int8(args[2].AuxInt<<int64(offset))), args[0], args[1])
  2205  		}
  2206  		return immJumpTable(s, args[2], n, func(sNew *state, idx int) {
  2207  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2208  			s.vars[n] = sNew.newValue2I(op, t, int64(int8(idx<<offset)), args[0], args[1])
  2209  		})
  2210  	}
  2211  }
  2212  
  2213  // Two immediates instead of just 1.  Offset is ignored, so it is a _ parameter instead.
  2214  func opLen2Imm8_II(op ssa.Op, t *types.Type, _ int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2215  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2216  		if (args[1].Op == ssa.OpConst8 || args[1].Op == ssa.OpConst64) && (args[2].Op == ssa.OpConst8 || args[2].Op == ssa.OpConst64) && args[1].AuxInt & ^3 == 0 && args[2].AuxInt & ^3 == 0 {
  2217  			i1, i2 := args[1].AuxInt, args[2].AuxInt
  2218  			return s.newValue2I(op, t, int64(int8(i1+i2<<4)), args[0], args[3])
  2219  		}
  2220  		four := s.constInt64(types.Types[types.TUINT8], 4)
  2221  		shifted := s.newValue2(ssa.OpLsh8x8, types.Types[types.TUINT8], args[2], four)
  2222  		combined := s.newValue2(ssa.OpAdd8, types.Types[types.TUINT8], args[1], shifted)
  2223  		return immJumpTable(s, combined, n, func(sNew *state, idx int) {
  2224  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2225  			// TODO for "zeroing" values, panic instead.
  2226  			if idx & ^(3+3<<4) == 0 {
  2227  				s.vars[n] = sNew.newValue2I(op, t, int64(int8(idx)), args[0], args[3])
  2228  			} else {
  2229  				sNew.rtcall(ir.Syms.PanicSimdImm, false, nil)
  2230  			}
  2231  		})
  2232  	}
  2233  }
  2234  
  2235  // The assembler requires the imm value of a SHA1RNDS4 instruction to be one of 0,1,2,3...
  2236  func opLen2Imm8_SHA1RNDS4(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2237  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2238  		if args[1].Op == ssa.OpConst8 || args[1].Op == ssa.OpConst64 {
  2239  			return s.newValue2I(op, t, int64(int8((args[1].AuxInt<<int64(offset))&0b11)), args[0], args[2])
  2240  		}
  2241  		return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
  2242  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2243  			s.vars[n] = sNew.newValue2I(op, t, int64(int8(idx<<offset))&0b11, args[0], args[2])
  2244  		})
  2245  	}
  2246  }
  2247  
  2248  func opLen1Imm(op ssa.Op, t *types.Type, offset int, immMax uint64) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2249  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2250  		if (args[1].Op == ssa.OpConst8 || args[1].Op == ssa.OpConst64) && uint64(args[1].AuxInt) <= immMax {
  2251  			return s.newValue1I(op, t, int64(int8(args[1].AuxInt<<int64(offset))), args[0])
  2252  		}
  2253  		return immJumpTableN(s, args[1], n, immMax+1, func(sNew *state, idx int) {
  2254  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2255  			s.vars[n] = sNew.newValue1I(op, t, int64(int8(idx<<offset)), args[0])
  2256  		})
  2257  	}
  2258  }
  2259  
  2260  func opLen2Imm(op ssa.Op, t *types.Type, offset int, immMax uint64) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2261  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2262  		if (args[1].Op == ssa.OpConst8 || args[1].Op == ssa.OpConst64) && uint64(args[1].AuxInt) <= immMax {
  2263  			return s.newValue2I(op, t, int64(int8(args[1].AuxInt<<int64(offset))), args[0], args[2])
  2264  		}
  2265  		return immJumpTableN(s, args[1], n, immMax+1, func(sNew *state, idx int) {
  2266  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2267  			s.vars[n] = sNew.newValue2I(op, t, int64(int8(idx<<offset)), args[0], args[2])
  2268  		})
  2269  	}
  2270  }
  2271  
  2272  func opLen3Imm(op ssa.Op, t *types.Type, offset int, immMax uint64) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2273  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2274  		if (args[1].Op == ssa.OpConst8 || args[1].Op == ssa.OpConst64) && uint64(args[1].AuxInt) <= immMax {
  2275  			return s.newValue3I(op, t, int64(int8(args[1].AuxInt<<int64(offset))), args[0], args[2], args[3])
  2276  		}
  2277  		return immJumpTableN(s, args[1], n, immMax+1, func(sNew *state, idx int) {
  2278  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2279  			s.vars[n] = sNew.newValue3I(op, t, int64(int8(idx<<offset)), args[0], args[2], args[3])
  2280  		})
  2281  	}
  2282  }
  2283  
  2284  func opLen2Imm_2I(op ssa.Op, t *types.Type, offset int, immMax uint64) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2285  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2286  		if (args[2].Op == ssa.OpConst8 || args[2].Op == ssa.OpConst64) && uint64(args[2].AuxInt) <= immMax {
  2287  			return s.newValue2I(op, t, int64(int8(args[2].AuxInt<<int64(offset))), args[0], args[1])
  2288  		}
  2289  		return immJumpTableN(s, args[2], n, immMax+1, func(sNew *state, idx int) {
  2290  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2291  			s.vars[n] = sNew.newValue2I(op, t, int64(int8(idx<<offset)), args[0], args[1])
  2292  		})
  2293  	}
  2294  }
  2295  
  2296  func opLen3Imm8_2I(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2297  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2298  		if args[2].Op == ssa.OpConst8 || args[2].Op == ssa.OpConst64 {
  2299  			return s.newValue3I(op, t, int64(int8(args[2].AuxInt<<int64(offset))), args[0], args[1], args[3])
  2300  		}
  2301  		return immJumpTable(s, args[2], n, func(sNew *state, idx int) {
  2302  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2303  			s.vars[n] = sNew.newValue3I(op, t, int64(int8(idx<<offset)), args[0], args[1], args[3])
  2304  		})
  2305  	}
  2306  }
  2307  
  2308  func opLen4Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2309  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2310  		if args[1].Op == ssa.OpConst8 || args[1].Op == ssa.OpConst64 {
  2311  			return s.newValue4I(op, t, int64(int8(args[1].AuxInt<<int64(offset))), args[0], args[2], args[3], args[4])
  2312  		}
  2313  		return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
  2314  			// Encode as int8 due to requirement of AuxInt, check its comment for details.
  2315  			s.vars[n] = sNew.newValue4I(op, t, int64(int8(idx<<offset)), args[0], args[2], args[3], args[4])
  2316  		})
  2317  	}
  2318  }
  2319  
  2320  func simdBroadcast(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2321  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2322  		return s.newValue2(op, n.Type(), args[0], s.mem())
  2323  	}
  2324  }
  2325  
  2326  func simdLoad() func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2327  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2328  		return s.newValue2(ssa.OpLoad, n.Type(), args[0], s.mem())
  2329  	}
  2330  }
  2331  
  2332  func simdStore() func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2333  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2334  		s.store(args[0].Type, args[1], args[0])
  2335  		return nil
  2336  	}
  2337  }
  2338  
  2339  var cvtVToMaskOpcodes = map[int]map[int]ssa.Op{
  2340  	8:  {16: ssa.OpCvt16toMask8x16, 32: ssa.OpCvt32toMask8x32, 64: ssa.OpCvt64toMask8x64},
  2341  	16: {8: ssa.OpCvt8toMask16x8, 16: ssa.OpCvt16toMask16x16, 32: ssa.OpCvt32toMask16x32},
  2342  	32: {4: ssa.OpCvt8toMask32x4, 8: ssa.OpCvt8toMask32x8, 16: ssa.OpCvt16toMask32x16},
  2343  	64: {2: ssa.OpCvt8toMask64x2, 4: ssa.OpCvt8toMask64x4, 8: ssa.OpCvt8toMask64x8},
  2344  }
  2345  
  2346  var cvtMaskToVOpcodes = map[int]map[int]ssa.Op{
  2347  	8:  {16: ssa.OpCvtMask8x16to16, 32: ssa.OpCvtMask8x32to32, 64: ssa.OpCvtMask8x64to64},
  2348  	16: {8: ssa.OpCvtMask16x8to8, 16: ssa.OpCvtMask16x16to16, 32: ssa.OpCvtMask16x32to32},
  2349  	32: {4: ssa.OpCvtMask32x4to8, 8: ssa.OpCvtMask32x8to8, 16: ssa.OpCvtMask32x16to16},
  2350  	64: {2: ssa.OpCvtMask64x2to8, 4: ssa.OpCvtMask64x4to8, 8: ssa.OpCvtMask64x8to8},
  2351  }
  2352  
  2353  func simdCvtVToMask(elemBits, lanes int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2354  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2355  		op := cvtVToMaskOpcodes[elemBits][lanes]
  2356  		if op == 0 {
  2357  			panic(fmt.Sprintf("Unknown mask shape: Mask%dx%d", elemBits, lanes))
  2358  		}
  2359  		return s.newValue1(op, types.TypeMask, args[0])
  2360  	}
  2361  }
  2362  
  2363  func simdCvtMaskToV(elemBits, lanes int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2364  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2365  		op := cvtMaskToVOpcodes[elemBits][lanes]
  2366  		if op == 0 {
  2367  			panic(fmt.Sprintf("Unknown mask shape: Mask%dx%d", elemBits, lanes))
  2368  		}
  2369  		return s.newValue1(op, n.Type(), args[0])
  2370  	}
  2371  }
  2372  
  2373  func simdMaskedLoad(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2374  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2375  		return s.newValue3(op, n.Type(), args[0], args[1], s.mem())
  2376  	}
  2377  }
  2378  
  2379  func simdMaskedStore(op ssa.Op) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2380  	return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
  2381  		s.vars[memVar] = s.newValue4A(op, types.TypeMem, args[0].Type, args[1], args[2], args[0], s.mem())
  2382  		return nil
  2383  	}
  2384  }
  2385  
  2386  // findIntrinsic returns a function which builds the SSA equivalent of the
  2387  // function identified by the symbol sym.  If sym is not an intrinsic call, returns nil.
  2388  func findIntrinsic(sym *types.Sym) intrinsicBuilder {
  2389  	if sym == nil || sym.Pkg == nil {
  2390  		return nil
  2391  	}
  2392  	pkg := sym.Pkg.Path
  2393  	if sym.Pkg == ir.Pkgs.Runtime {
  2394  		pkg = "runtime"
  2395  	}
  2396  	if base.Flag.Race && pkg == "sync/atomic" {
  2397  		// The race detector needs to be able to intercept these calls.
  2398  		// We can't intrinsify them.
  2399  		return nil
  2400  	}
  2401  	// Skip intrinsifying math functions (which may contain hard-float
  2402  	// instructions) when soft-float
  2403  	if Arch.SoftFloat && pkg == "math" {
  2404  		return nil
  2405  	}
  2406  
  2407  	fn := sym.Name
  2408  	if ssa.IntrinsicsDisable {
  2409  		if pkg == "internal/runtime/sys" && (fn == "GetCallerPC" || fn == "GetCallerSP" || fn == "GetClosurePtr") ||
  2410  			pkg == simdPackage {
  2411  			// These runtime functions don't have definitions, must be intrinsics.
  2412  		} else {
  2413  			return nil
  2414  		}
  2415  	}
  2416  	return intrinsics.lookup(Arch.LinkArch.Arch, pkg, fn)
  2417  }
  2418  
  2419  func IsIntrinsicCall(n *ir.CallExpr) bool {
  2420  	if n == nil {
  2421  		return false
  2422  	}
  2423  	name, ok := n.Fun.(*ir.Name)
  2424  	if !ok {
  2425  		if n.Fun.Op() == ir.OMETHEXPR {
  2426  			if meth := ir.MethodExprName(n.Fun); meth != nil {
  2427  				if fn := meth.Func; fn != nil {
  2428  					return IsIntrinsicSym(fn.Sym())
  2429  				}
  2430  			}
  2431  		}
  2432  		return false
  2433  	}
  2434  	return IsIntrinsicSym(name.Sym())
  2435  }
  2436  
  2437  func IsIntrinsicSym(sym *types.Sym) bool {
  2438  	return findIntrinsic(sym) != nil
  2439  }
  2440  
  2441  // GenIntrinsicBody generates the function body for a bodyless intrinsic.
  2442  // This is used when the intrinsic is used in a non-call context, e.g.
  2443  // as a function pointer, or (for a method) being referenced from the type
  2444  // descriptor.
  2445  //
  2446  // The compiler already recognizes a call to fn as an intrinsic and can
  2447  // directly generate code for it. So we just fill in the body with a call
  2448  // to fn.
  2449  func GenIntrinsicBody(fn *ir.Func) {
  2450  	if ir.CurFunc != nil {
  2451  		base.FatalfAt(fn.Pos(), "enqueueFunc %v inside %v", fn, ir.CurFunc)
  2452  	}
  2453  
  2454  	if base.Flag.LowerR != 0 {
  2455  		fmt.Println("generate intrinsic for", ir.FuncName(fn))
  2456  	}
  2457  
  2458  	pos := fn.Pos()
  2459  	ft := fn.Type()
  2460  	var ret ir.Node
  2461  
  2462  	// For a method, it usually starts with an ODOTMETH (pre-typecheck) or
  2463  	// OMETHEXPR (post-typecheck) referencing the method symbol without the
  2464  	// receiver type, and Walk rewrites it to a call directly to the
  2465  	// type-qualified method symbol, moving the receiver to an argument.
  2466  	// Here fn has already the type-qualified method symbol, and it is hard
  2467  	// to get the unqualified symbol. So we just generate the post-Walk form
  2468  	// and mark it typechecked and Walked.
  2469  	call := ir.NewCallExpr(pos, ir.OCALLFUNC, fn.Nname, nil)
  2470  	call.Args = ir.RecvParamNames(ft)
  2471  	call.IsDDD = ft.IsVariadic()
  2472  	typecheck.Exprs(call.Args)
  2473  	call.SetTypecheck(1)
  2474  	call.SetWalked(true)
  2475  	ret = call
  2476  	if ft.NumResults() > 0 {
  2477  		if ft.NumResults() == 1 {
  2478  			call.SetType(ft.Result(0).Type)
  2479  		} else {
  2480  			call.SetType(ft.ResultsTuple())
  2481  		}
  2482  		n := ir.NewReturnStmt(base.Pos, nil)
  2483  		n.Results = []ir.Node{call}
  2484  		ret = n
  2485  	}
  2486  	fn.Body.Append(ret)
  2487  
  2488  	if base.Flag.LowerR != 0 {
  2489  		ir.DumpList("generate intrinsic body", fn.Body)
  2490  	}
  2491  
  2492  	ir.CurFunc = fn
  2493  	typecheck.Stmts(fn.Body)
  2494  	ir.CurFunc = nil // we know CurFunc is nil at entry
  2495  }
  2496  

View as plain text