Source file src/cmd/compile/internal/ssa/rewrite.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssa
     6  
     7  import (
     8  	"cmd/compile/internal/base"
     9  	"cmd/compile/internal/ir"
    10  	"cmd/compile/internal/logopt"
    11  	"cmd/compile/internal/reflectdata"
    12  	"cmd/compile/internal/rttype"
    13  	"cmd/compile/internal/typecheck"
    14  	"cmd/compile/internal/types"
    15  	"cmd/internal/obj"
    16  	"cmd/internal/obj/s390x"
    17  	"cmd/internal/objabi"
    18  	"cmd/internal/src"
    19  	"encoding/binary"
    20  	"fmt"
    21  	"internal/buildcfg"
    22  	"io"
    23  	"math"
    24  	"math/bits"
    25  	"os"
    26  	"path/filepath"
    27  	"strings"
    28  )
    29  
    30  type deadValueChoice bool
    31  
    32  const (
    33  	leaveDeadValues  deadValueChoice = false
    34  	removeDeadValues                 = true
    35  
    36  	repZeroThreshold = 1408 // size beyond which we use REP STOS for zeroing
    37  	repMoveThreshold = 1408 // size beyond which we use REP MOVS for copying
    38  )
    39  
    40  // deadcode indicates whether rewrite should try to remove any values that become dead.
    41  func applyRewrite(f *Func, rb blockRewriter, rv valueRewriter, deadcode deadValueChoice) {
    42  	// repeat rewrites until we find no more rewrites
    43  	pendingLines := f.cachedLineStarts // Holds statement boundaries that need to be moved to a new value/block
    44  	pendingLines.clear()
    45  	debug := f.pass.debug
    46  	if debug > 1 {
    47  		fmt.Printf("%s: rewriting for %s\n", f.pass.name, f.Name)
    48  	}
    49  	// if the number of rewrite iterations reaches itersLimit we will
    50  	// at that point turn on cycle detection. Instead of a fixed limit,
    51  	// size the limit according to func size to allow for cases such
    52  	// as the one in issue #66773.
    53  	itersLimit := f.NumBlocks()
    54  	if itersLimit < 20 {
    55  		itersLimit = 20
    56  	}
    57  	var iters int
    58  	var states map[string]bool
    59  	for {
    60  		if debug > 1 {
    61  			fmt.Printf("%s: iter %d\n", f.pass.name, iters)
    62  		}
    63  		change := false
    64  		deadChange := false
    65  		for _, b := range f.Blocks {
    66  			var b0 *Block
    67  			if debug > 1 {
    68  				fmt.Printf("%s: start block\n", f.pass.name)
    69  				b0 = new(Block)
    70  				*b0 = *b
    71  				b0.Succs = append([]Edge{}, b.Succs...) // make a new copy, not aliasing
    72  			}
    73  			for i, c := range b.ControlValues() {
    74  				for c.Op == OpCopy {
    75  					c = c.Args[0]
    76  					b.ReplaceControl(i, c)
    77  				}
    78  			}
    79  			if rb(b) {
    80  				change = true
    81  				if debug > 1 {
    82  					fmt.Printf("rewriting %s  ->  %s\n", b0.LongString(), b.LongString())
    83  				}
    84  			}
    85  			for j, v := range b.Values {
    86  				if debug > 1 {
    87  					fmt.Printf("%s: consider %v\n", f.pass.name, v.LongString())
    88  				}
    89  				var v0 *Value
    90  				if debug > 1 {
    91  					v0 = new(Value)
    92  					*v0 = *v
    93  					v0.Args = append([]*Value{}, v.Args...) // make a new copy, not aliasing
    94  				}
    95  				if v.Uses == 0 && v.removeable() {
    96  					if v.Op != OpInvalid && deadcode == removeDeadValues {
    97  						// Reset any values that are now unused, so that we decrement
    98  						// the use count of all of its arguments.
    99  						// Not quite a deadcode pass, because it does not handle cycles.
   100  						// But it should help Uses==1 rules to fire.
   101  						v.reset(OpInvalid)
   102  						deadChange = true
   103  					}
   104  					// No point rewriting values which aren't used.
   105  					continue
   106  				}
   107  
   108  				vchange := phielimValue(v)
   109  				if vchange && debug > 1 {
   110  					fmt.Printf("rewriting %s  ->  %s\n", v0.LongString(), v.LongString())
   111  				}
   112  
   113  				// Eliminate copy inputs.
   114  				// If any copy input becomes unused, mark it
   115  				// as invalid and discard its argument. Repeat
   116  				// recursively on the discarded argument.
   117  				// This phase helps remove phantom "dead copy" uses
   118  				// of a value so that a x.Uses==1 rule condition
   119  				// fires reliably.
   120  				for i, a := range v.Args {
   121  					if a.Op != OpCopy {
   122  						continue
   123  					}
   124  					aa := copySource(a)
   125  					v.SetArg(i, aa)
   126  					// If a, a copy, has a line boundary indicator, attempt to find a new value
   127  					// to hold it.  The first candidate is the value that will replace a (aa),
   128  					// if it shares the same block and line and is eligible.
   129  					// The second option is v, which has a as an input.  Because aa is earlier in
   130  					// the data flow, it is the better choice.
   131  					if a.Pos.IsStmt() == src.PosIsStmt {
   132  						if aa.Block == a.Block && aa.Pos.Line() == a.Pos.Line() && aa.Pos.IsStmt() != src.PosNotStmt {
   133  							aa.Pos = aa.Pos.WithIsStmt()
   134  						} else if v.Block == a.Block && v.Pos.Line() == a.Pos.Line() && v.Pos.IsStmt() != src.PosNotStmt {
   135  							v.Pos = v.Pos.WithIsStmt()
   136  						} else {
   137  							// Record the lost line and look for a new home after all rewrites are complete.
   138  							// TODO: it's possible (in FOR loops, in particular) for statement boundaries for the same
   139  							// line to appear in more than one block, but only one block is stored, so if both end
   140  							// up here, then one will be lost.
   141  							pendingLines.set(a.Pos, int32(a.Block.ID))
   142  						}
   143  						a.Pos = a.Pos.WithNotStmt()
   144  					}
   145  					vchange = true
   146  					for a.Uses == 0 {
   147  						b := a.Args[0]
   148  						a.reset(OpInvalid)
   149  						a = b
   150  					}
   151  				}
   152  				if vchange && debug > 1 {
   153  					fmt.Printf("rewriting %s  ->  %s\n", v0.LongString(), v.LongString())
   154  				}
   155  
   156  				// apply rewrite function
   157  				if rv(v) {
   158  					vchange = true
   159  					// If value changed to a poor choice for a statement boundary, move the boundary
   160  					if v.Pos.IsStmt() == src.PosIsStmt {
   161  						if k := nextGoodStatementIndex(v, j, b); k != j {
   162  							v.Pos = v.Pos.WithNotStmt()
   163  							b.Values[k].Pos = b.Values[k].Pos.WithIsStmt()
   164  						}
   165  					}
   166  				}
   167  
   168  				change = change || vchange
   169  				if vchange && debug > 1 {
   170  					fmt.Printf("rewriting %s  ->  %s\n", v0.LongString(), v.LongString())
   171  				}
   172  			}
   173  		}
   174  		if !change && !deadChange {
   175  			break
   176  		}
   177  		iters++
   178  		if (iters > itersLimit || debug >= 2) && change {
   179  			// We've done a suspiciously large number of rewrites (or we're in debug mode).
   180  			// As of Sep 2021, 90% of rewrites complete in 4 iterations or fewer
   181  			// and the maximum value encountered during make.bash is 12.
   182  			// Start checking for cycles. (This is too expensive to do routinely.)
   183  			// Note: we avoid this path for deadChange-only iterations, to fix #51639.
   184  			if states == nil {
   185  				states = make(map[string]bool)
   186  			}
   187  			h := f.rewriteHash()
   188  			if _, ok := states[h]; ok {
   189  				// We've found a cycle.
   190  				// To diagnose it, set debug to 2 and start again,
   191  				// so that we'll print all rules applied until we complete another cycle.
   192  				// If debug is already >= 2, we've already done that, so it's time to crash.
   193  				if debug < 2 {
   194  					debug = 2
   195  					states = make(map[string]bool)
   196  				} else {
   197  					f.Fatalf("rewrite cycle detected")
   198  				}
   199  			}
   200  			states[h] = true
   201  		}
   202  	}
   203  	// remove clobbered values
   204  	for _, b := range f.Blocks {
   205  		j := 0
   206  		for i, v := range b.Values {
   207  			vl := v.Pos
   208  			if v.Op == OpInvalid {
   209  				if v.Pos.IsStmt() == src.PosIsStmt {
   210  					pendingLines.set(vl, int32(b.ID))
   211  				}
   212  				f.freeValue(v)
   213  				continue
   214  			}
   215  			if v.Pos.IsStmt() != src.PosNotStmt && !notStmtBoundary(v.Op) {
   216  				if pl, ok := pendingLines.get(vl); ok && pl == int32(b.ID) {
   217  					pendingLines.remove(vl)
   218  					v.Pos = v.Pos.WithIsStmt()
   219  				}
   220  			}
   221  			if i != j {
   222  				b.Values[j] = v
   223  			}
   224  			j++
   225  		}
   226  		if pl, ok := pendingLines.get(b.Pos); ok && pl == int32(b.ID) {
   227  			b.Pos = b.Pos.WithIsStmt()
   228  			pendingLines.remove(b.Pos)
   229  		}
   230  		b.truncateValues(j)
   231  	}
   232  }
   233  
   234  // Common functions called from rewriting rules
   235  
   236  func is64BitFloat(t *types.Type) bool {
   237  	return t.Size() == 8 && t.IsFloat()
   238  }
   239  
   240  func is32BitFloat(t *types.Type) bool {
   241  	return t.Size() == 4 && t.IsFloat()
   242  }
   243  
   244  func is64BitInt(t *types.Type) bool {
   245  	return t.Size() == 8 && t.IsInteger()
   246  }
   247  
   248  func is32BitInt(t *types.Type) bool {
   249  	return t.Size() == 4 && t.IsInteger()
   250  }
   251  
   252  func is16BitInt(t *types.Type) bool {
   253  	return t.Size() == 2 && t.IsInteger()
   254  }
   255  
   256  func is8BitInt(t *types.Type) bool {
   257  	return t.Size() == 1 && t.IsInteger()
   258  }
   259  
   260  func isPtr(t *types.Type) bool {
   261  	return t.IsPtrShaped()
   262  }
   263  
   264  func copyCompatibleType(t1, t2 *types.Type) bool {
   265  	if t1.Size() != t2.Size() {
   266  		return false
   267  	}
   268  	if t1.IsInteger() {
   269  		return t2.IsInteger()
   270  	}
   271  	if isPtr(t1) {
   272  		return isPtr(t2)
   273  	}
   274  	return t1.Compare(t2) == types.CMPeq
   275  }
   276  
   277  // mergeSym merges two symbolic offsets. There is no real merging of
   278  // offsets, we just pick the non-nil one.
   279  func mergeSym(x, y Sym) Sym {
   280  	if x == nil {
   281  		return y
   282  	}
   283  	if y == nil {
   284  		return x
   285  	}
   286  	panic(fmt.Sprintf("mergeSym with two non-nil syms %v %v", x, y))
   287  }
   288  
   289  func canMergeSym(x, y Sym) bool {
   290  	return x == nil || y == nil
   291  }
   292  
   293  // canMergeLoadClobber reports whether the load can be merged into target without
   294  // invalidating the schedule.
   295  // It also checks that the other non-load argument x is something we
   296  // are ok with clobbering.
   297  func canMergeLoadClobber(target, load, x *Value) bool {
   298  	// The register containing x is going to get clobbered.
   299  	// Don't merge if we still need the value of x.
   300  	// We don't have liveness information here, but we can
   301  	// approximate x dying with:
   302  	//  1) target is x's only use.
   303  	//  2) target is not in a deeper loop than x.
   304  	switch {
   305  	case x.Uses == 2 && x.Op == OpPhi && len(x.Args) == 2 && (x.Args[0] == target || x.Args[1] == target) && target.Uses == 1:
   306  		// This is a simple detector to determine that x is probably
   307  		// not live after target. (It does not need to be perfect,
   308  		// regalloc will issue a reg-reg move to save it if we are wrong.)
   309  		// We have:
   310  		//   x = Phi(?, target)
   311  		//   target = Op(load, x)
   312  		// Because target has only one use as a Phi argument, we can schedule it
   313  		// very late. Hopefully, later than the other use of x. (The other use died
   314  		// between x and target, or exists on another branch entirely).
   315  	case x.Uses > 1:
   316  		return false
   317  	}
   318  	loopnest := x.Block.Func.loopnest()
   319  	if loopnest.depth(target.Block.ID) > loopnest.depth(x.Block.ID) {
   320  		return false
   321  	}
   322  	return canMergeLoad(target, load)
   323  }
   324  
   325  // canMergeLoad reports whether the load can be merged into target without
   326  // invalidating the schedule.
   327  func canMergeLoad(target, load *Value) bool {
   328  	if target.Block.ID != load.Block.ID {
   329  		// If the load is in a different block do not merge it.
   330  		return false
   331  	}
   332  
   333  	// We can't merge the load into the target if the load
   334  	// has more than one use.
   335  	if load.Uses != 1 {
   336  		return false
   337  	}
   338  
   339  	mem := load.MemoryArg()
   340  
   341  	// We need the load's memory arg to still be alive at target. That
   342  	// can't be the case if one of target's args depends on a memory
   343  	// state that is a successor of load's memory arg.
   344  	//
   345  	// For example, it would be invalid to merge load into target in
   346  	// the following situation because newmem has killed oldmem
   347  	// before target is reached:
   348  	//     load = read ... oldmem
   349  	//   newmem = write ... oldmem
   350  	//     arg0 = read ... newmem
   351  	//   target = add arg0 load
   352  	//
   353  	// If the argument comes from a different block then we can exclude
   354  	// it immediately because it must dominate load (which is in the
   355  	// same block as target).
   356  	var args []*Value
   357  	for _, a := range target.Args {
   358  		if a != load && a.Block.ID == target.Block.ID {
   359  			args = append(args, a)
   360  		}
   361  	}
   362  
   363  	// memPreds contains memory states known to be predecessors of load's
   364  	// memory state. It is lazily initialized.
   365  	var memPreds map[*Value]bool
   366  	for i := 0; len(args) > 0; i++ {
   367  		const limit = 100
   368  		if i >= limit {
   369  			// Give up if we have done a lot of iterations.
   370  			return false
   371  		}
   372  		v := args[len(args)-1]
   373  		args = args[:len(args)-1]
   374  		if target.Block.ID != v.Block.ID {
   375  			// Since target and load are in the same block
   376  			// we can stop searching when we leave the block.
   377  			continue
   378  		}
   379  		if v.Op == OpPhi {
   380  			// A Phi implies we have reached the top of the block.
   381  			// The memory phi, if it exists, is always
   382  			// the first logical store in the block.
   383  			continue
   384  		}
   385  		if v.Type.IsTuple() && v.Type.FieldType(1).IsMemory() {
   386  			// We could handle this situation however it is likely
   387  			// to be very rare.
   388  			return false
   389  		}
   390  		if v.Op.SymEffect()&SymAddr != 0 {
   391  			// This case prevents an operation that calculates the
   392  			// address of a local variable from being forced to schedule
   393  			// before its corresponding VarDef.
   394  			// See issue 28445.
   395  			//   v1 = LOAD ...
   396  			//   v2 = VARDEF
   397  			//   v3 = LEAQ
   398  			//   v4 = CMPQ v1 v3
   399  			// We don't want to combine the CMPQ with the load, because
   400  			// that would force the CMPQ to schedule before the VARDEF, which
   401  			// in turn requires the LEAQ to schedule before the VARDEF.
   402  			return false
   403  		}
   404  		if v.Type.IsMemory() {
   405  			if memPreds == nil {
   406  				// Initialise a map containing memory states
   407  				// known to be predecessors of load's memory
   408  				// state.
   409  				memPreds = make(map[*Value]bool)
   410  				m := mem
   411  				const limit = 50
   412  				for i := 0; i < limit; i++ {
   413  					if m.Op == OpPhi {
   414  						// The memory phi, if it exists, is always
   415  						// the first logical store in the block.
   416  						break
   417  					}
   418  					if m.Block.ID != target.Block.ID {
   419  						break
   420  					}
   421  					if !m.Type.IsMemory() {
   422  						break
   423  					}
   424  					memPreds[m] = true
   425  					if len(m.Args) == 0 {
   426  						break
   427  					}
   428  					m = m.MemoryArg()
   429  				}
   430  			}
   431  
   432  			// We can merge if v is a predecessor of mem.
   433  			//
   434  			// For example, we can merge load into target in the
   435  			// following scenario:
   436  			//      x = read ... v
   437  			//    mem = write ... v
   438  			//   load = read ... mem
   439  			// target = add x load
   440  			if memPreds[v] {
   441  				continue
   442  			}
   443  			return false
   444  		}
   445  		if len(v.Args) > 0 && v.Args[len(v.Args)-1] == mem {
   446  			// If v takes mem as an input then we know mem
   447  			// is valid at this point.
   448  			continue
   449  		}
   450  		for _, a := range v.Args {
   451  			if target.Block.ID == a.Block.ID {
   452  				args = append(args, a)
   453  			}
   454  		}
   455  	}
   456  
   457  	return true
   458  }
   459  
   460  // isSameCall reports whether aux is the same as the given named symbol.
   461  func isSameCall(aux Aux, name string) bool {
   462  	fn := aux.(*AuxCall).Fn
   463  	return fn != nil && fn.String() == name
   464  }
   465  
   466  func isMalloc(aux Aux) bool {
   467  	return isNewObject(aux) || isSpecializedMalloc(aux)
   468  }
   469  
   470  func isNewObject(aux Aux) bool {
   471  	fn := aux.(*AuxCall).Fn
   472  	return fn != nil && fn.String() == "runtime.newobject"
   473  }
   474  
   475  func isSpecializedMalloc(aux Aux) bool {
   476  	fn := aux.(*AuxCall).Fn
   477  	if fn == nil {
   478  		return false
   479  	}
   480  	name := fn.String()
   481  	return strings.HasPrefix(name, "runtime.mallocgcSmallNoScanSC") ||
   482  		strings.HasPrefix(name, "runtime.mallocgcSmallScanNoHeaderSC") ||
   483  		strings.HasPrefix(name, "runtime.mallocgcTinySC")
   484  }
   485  
   486  // canLoadUnaligned reports if the architecture supports unaligned load operations.
   487  func canLoadUnaligned(c *Config) bool {
   488  	return c.ctxt.Arch.Alignment == 1
   489  }
   490  
   491  // nlzX returns the number of leading zeros.
   492  func nlz64(x int64) int { return bits.LeadingZeros64(uint64(x)) }
   493  func nlz32(x int32) int { return bits.LeadingZeros32(uint32(x)) }
   494  func nlz16(x int16) int { return bits.LeadingZeros16(uint16(x)) }
   495  func nlz8(x int8) int   { return bits.LeadingZeros8(uint8(x)) }
   496  
   497  // ntzX returns the number of trailing zeros.
   498  func ntz64(x int64) int { return bits.TrailingZeros64(uint64(x)) }
   499  func ntz32(x int32) int { return bits.TrailingZeros32(uint32(x)) }
   500  func ntz16(x int16) int { return bits.TrailingZeros16(uint16(x)) }
   501  func ntz8(x int8) int   { return bits.TrailingZeros8(uint8(x)) }
   502  
   503  // oneBit reports whether x contains exactly one set bit.
   504  func oneBit[T int8 | int16 | int32 | int64](x T) bool {
   505  	return x&(x-1) == 0 && x != 0
   506  }
   507  
   508  // nto returns the number of trailing ones.
   509  func nto(x int64) int64 {
   510  	return int64(ntz64(^x))
   511  }
   512  
   513  // logX returns logarithm of n base 2.
   514  // n must be a positive power of 2 (isPowerOfTwoX returns true).
   515  func log8(n int8) int64   { return log8u(uint8(n)) }
   516  func log16(n int16) int64 { return log16u(uint16(n)) }
   517  func log32(n int32) int64 { return log32u(uint32(n)) }
   518  func log64(n int64) int64 { return log64u(uint64(n)) }
   519  
   520  // logXu returns the logarithm of n base 2.
   521  // n must be a power of 2 (isPowerOfTwo returns true)
   522  func log8u(n uint8) int64   { return int64(bits.Len8(n)) - 1 }
   523  func log16u(n uint16) int64 { return int64(bits.Len16(n)) - 1 }
   524  func log32u(n uint32) int64 { return int64(bits.Len32(n)) - 1 }
   525  func log64u(n uint64) int64 { return int64(bits.Len64(n)) - 1 }
   526  
   527  // isPowerOfTwoX functions report whether n is a power of 2.
   528  func isPowerOfTwo[T int8 | int16 | int32 | int64 | uint8 | uint16 | uint32 | uint64](n T) bool {
   529  	return n > 0 && n&(n-1) == 0
   530  }
   531  
   532  // is32Bit reports whether n can be represented as a signed 32 bit integer.
   533  func is32Bit(n int64) bool {
   534  	return n == int64(int32(n))
   535  }
   536  
   537  // is16Bit reports whether n can be represented as a signed 16 bit integer.
   538  func is16Bit(n int64) bool {
   539  	return n == int64(int16(n))
   540  }
   541  
   542  // is8Bit reports whether n can be represented as a signed 8 bit integer.
   543  func is8Bit(n int64) bool {
   544  	return n == int64(int8(n))
   545  }
   546  
   547  // isU8Bit reports whether n can be represented as an unsigned 8 bit integer.
   548  func isU8Bit(n int64) bool {
   549  	return n == int64(uint8(n))
   550  }
   551  
   552  // is12Bit reports whether n can be represented as a signed 12 bit integer.
   553  func is12Bit(n int64) bool {
   554  	return -(1<<11) <= n && n < (1<<11)
   555  }
   556  
   557  // isU12Bit reports whether n can be represented as an unsigned 12 bit integer.
   558  func isU12Bit(n int64) bool {
   559  	return 0 <= n && n < (1<<12)
   560  }
   561  
   562  // isU16Bit reports whether n can be represented as an unsigned 16 bit integer.
   563  func isU16Bit(n int64) bool {
   564  	return n == int64(uint16(n))
   565  }
   566  
   567  // isU32Bit reports whether n can be represented as an unsigned 32 bit integer.
   568  func isU32Bit(n int64) bool {
   569  	return n == int64(uint32(n))
   570  }
   571  
   572  // is20Bit reports whether n can be represented as a signed 20 bit integer.
   573  func is20Bit(n int64) bool {
   574  	return -(1<<19) <= n && n < (1<<19)
   575  }
   576  
   577  // b2i translates a boolean value to 0 or 1 for assigning to auxInt.
   578  func b2i(b bool) int64 {
   579  	if b {
   580  		return 1
   581  	}
   582  	return 0
   583  }
   584  
   585  // b2i32 translates a boolean value to 0 or 1.
   586  func b2i32(b bool) int32 {
   587  	if b {
   588  		return 1
   589  	}
   590  	return 0
   591  }
   592  
   593  func canMulStrengthReduce(config *Config, x int64) bool {
   594  	_, ok := config.mulRecipes[x]
   595  	return ok
   596  }
   597  func canMulStrengthReduce32(config *Config, x int32) bool {
   598  	_, ok := config.mulRecipes[int64(x)]
   599  	return ok
   600  }
   601  
   602  // mulStrengthReduce returns v*x evaluated at the location
   603  // (block and source position) of m.
   604  // canMulStrengthReduce must have returned true.
   605  func mulStrengthReduce(m *Value, v *Value, x int64) *Value {
   606  	return v.Block.Func.Config.mulRecipes[x].build(m, v)
   607  }
   608  
   609  // mulStrengthReduce32 returns v*x evaluated at the location
   610  // (block and source position) of m.
   611  // canMulStrengthReduce32 must have returned true.
   612  // The upper 32 bits of m might be set to junk.
   613  func mulStrengthReduce32(m *Value, v *Value, x int32) *Value {
   614  	return v.Block.Func.Config.mulRecipes[int64(x)].build(m, v)
   615  }
   616  
   617  // shiftIsBounded reports whether (left/right) shift Value v is known to be bounded.
   618  // A shift is bounded if it is shifting by less than the width of the shifted value.
   619  func shiftIsBounded(v *Value) bool {
   620  	return v.AuxInt != 0
   621  }
   622  
   623  // canonLessThan returns whether x is "ordered" less than y, for purposes of normalizing
   624  // generated code as much as possible.
   625  func canonLessThan(x, y *Value) bool {
   626  	if x.Op != y.Op {
   627  		return x.Op < y.Op
   628  	}
   629  	if !x.Pos.SameFileAndLine(y.Pos) {
   630  		return x.Pos.Before(y.Pos)
   631  	}
   632  	return x.ID < y.ID
   633  }
   634  
   635  // truncate64Fto32F converts a float64 value to a float32 preserving the bit pattern
   636  // of the mantissa. It will panic if the truncation results in lost information.
   637  func truncate64Fto32F(f float64) float32 {
   638  	if !isExactFloat32(f) {
   639  		panic("truncate64Fto32F: truncation is not exact")
   640  	}
   641  	if !math.IsNaN(f) {
   642  		return float32(f)
   643  	}
   644  	// NaN bit patterns aren't necessarily preserved across conversion
   645  	// instructions so we need to do the conversion manually.
   646  	b := math.Float64bits(f)
   647  	m := b & ((1 << 52) - 1) // mantissa (a.k.a. significand)
   648  	//          | sign                  | exponent   | mantissa       |
   649  	r := uint32(((b >> 32) & (1 << 31)) | 0x7f800000 | (m >> (52 - 23)))
   650  	return math.Float32frombits(r)
   651  }
   652  
   653  // DivisionNeedsFixUp reports whether the division needs fix-up code.
   654  func DivisionNeedsFixUp(v *Value) bool {
   655  	return v.AuxInt == 0
   656  }
   657  
   658  // auxTo32F decodes a float32 from the AuxInt value provided.
   659  func auxTo32F(i int64) float32 {
   660  	return truncate64Fto32F(math.Float64frombits(uint64(i)))
   661  }
   662  
   663  func auxIntToBool(i int64) bool {
   664  	if i == 0 {
   665  		return false
   666  	}
   667  	return true
   668  }
   669  func auxIntToInt8(i int64) int8 {
   670  	return int8(i)
   671  }
   672  func auxIntToInt16(i int64) int16 {
   673  	return int16(i)
   674  }
   675  func auxIntToInt32(i int64) int32 {
   676  	return int32(i)
   677  }
   678  func auxIntToInt64(i int64) int64 {
   679  	return i
   680  }
   681  func auxIntToUint8(i int64) uint8 {
   682  	return uint8(i)
   683  }
   684  func auxIntToUint64(i int64) uint64 {
   685  	return uint64(i)
   686  }
   687  func auxIntToFloat32(i int64) float32 {
   688  	return float32(math.Float64frombits(uint64(i)))
   689  }
   690  func auxIntToFloat64(i int64) float64 {
   691  	return math.Float64frombits(uint64(i))
   692  }
   693  func auxIntToValAndOff(i int64) ValAndOff {
   694  	return ValAndOff(i)
   695  }
   696  func auxIntToArm64BitField(i int64) arm64BitField {
   697  	return arm64BitField(i)
   698  }
   699  func auxIntToArm64ConditionalParams(i int64) arm64ConditionalParams {
   700  	var params arm64ConditionalParams
   701  	params.cond = Op(i & 0xffff)
   702  	i >>= 16
   703  	params.nzcv = uint8(i & 0x0f)
   704  	i >>= 4
   705  	params.constValue = uint8(i & 0x1f)
   706  	i >>= 5
   707  	params.ind = i == 1
   708  	return params
   709  }
   710  func auxIntToFlagConstant(x int64) flagConstant {
   711  	return flagConstant(x)
   712  }
   713  
   714  func auxIntToOp(cc int64) Op {
   715  	return Op(cc)
   716  }
   717  
   718  func boolToAuxInt(b bool) int64 {
   719  	if b {
   720  		return 1
   721  	}
   722  	return 0
   723  }
   724  func int8ToAuxInt(i int8) int64 {
   725  	return int64(i)
   726  }
   727  func int16ToAuxInt(i int16) int64 {
   728  	return int64(i)
   729  }
   730  func int32ToAuxInt(i int32) int64 {
   731  	return int64(i)
   732  }
   733  func int64ToAuxInt(i int64) int64 {
   734  	return i
   735  }
   736  func uint8ToAuxInt(i uint8) int64 {
   737  	return int64(int8(i))
   738  }
   739  func uint64ToAuxInt(i uint64) int64 {
   740  	return int64(i)
   741  }
   742  func float32ToAuxInt(f float32) int64 {
   743  	return int64(math.Float64bits(float64(f)))
   744  }
   745  func float64ToAuxInt(f float64) int64 {
   746  	return int64(math.Float64bits(f))
   747  }
   748  func valAndOffToAuxInt(v ValAndOff) int64 {
   749  	return int64(v)
   750  }
   751  func arm64BitFieldToAuxInt(v arm64BitField) int64 {
   752  	return int64(v)
   753  }
   754  func arm64ConditionalParamsToAuxInt(v arm64ConditionalParams) int64 {
   755  	if v.cond&^0xffff != 0 {
   756  		panic("condition value exceeds 16 bits")
   757  	}
   758  
   759  	var i int64
   760  	if v.ind {
   761  		i = 1 << 25
   762  	}
   763  	i |= int64(v.constValue) << 20
   764  	i |= int64(v.nzcv) << 16
   765  	i |= int64(v.cond)
   766  	return i
   767  }
   768  
   769  func flagConstantToAuxInt(x flagConstant) int64 {
   770  	return int64(x)
   771  }
   772  
   773  func opToAuxInt(o Op) int64 {
   774  	return int64(o)
   775  }
   776  
   777  // Aux is an interface to hold miscellaneous data in Blocks and Values.
   778  type Aux interface {
   779  	CanBeAnSSAAux()
   780  }
   781  
   782  // for now only used to mark moves that need to avoid clobbering flags
   783  type auxMark bool
   784  
   785  func (auxMark) CanBeAnSSAAux() {}
   786  
   787  var AuxMark auxMark
   788  
   789  // stringAux wraps string values for use in Aux.
   790  type stringAux string
   791  
   792  func (stringAux) CanBeAnSSAAux() {}
   793  
   794  func auxToString(i Aux) string {
   795  	return string(i.(stringAux))
   796  }
   797  
   798  type int64Aux int64
   799  
   800  func (int64Aux) CanBeAnSSAAux() {}
   801  
   802  func int64ToAux(v int64) Aux {
   803  	return int64Aux(v)
   804  }
   805  func auxToSym(i Aux) Sym {
   806  	// TODO: kind of a hack - allows nil interface through
   807  	s, _ := i.(Sym)
   808  	return s
   809  }
   810  func auxToType(i Aux) *types.Type {
   811  	return i.(*types.Type)
   812  }
   813  func auxToCall(i Aux) *AuxCall {
   814  	return i.(*AuxCall)
   815  }
   816  func auxToS390xCCMask(i Aux) s390x.CCMask {
   817  	return i.(s390x.CCMask)
   818  }
   819  func auxToS390xRotateParams(i Aux) s390x.RotateParams {
   820  	return i.(s390x.RotateParams)
   821  }
   822  
   823  func StringToAux(s string) Aux {
   824  	return stringAux(s)
   825  }
   826  func symToAux(s Sym) Aux {
   827  	return s
   828  }
   829  func callToAux(s *AuxCall) Aux {
   830  	return s
   831  }
   832  func typeToAux(t *types.Type) Aux {
   833  	return t
   834  }
   835  func s390xCCMaskToAux(c s390x.CCMask) Aux {
   836  	return c
   837  }
   838  func s390xRotateParamsToAux(r s390x.RotateParams) Aux {
   839  	return r
   840  }
   841  
   842  // uaddOvf reports whether unsigned a+b would overflow.
   843  func uaddOvf(a, b int64) bool {
   844  	return uint64(a)+uint64(b) < uint64(a)
   845  }
   846  
   847  func devirtLECall(v *Value, sym *obj.LSym) *Value {
   848  	v.Op = OpStaticLECall
   849  	auxcall := v.Aux.(*AuxCall)
   850  	auxcall.Fn = sym
   851  	// Remove first arg
   852  	v.Args[0].Uses--
   853  	copy(v.Args[0:], v.Args[1:])
   854  	v.Args[len(v.Args)-1] = nil // aid GC
   855  	v.Args = v.Args[:len(v.Args)-1]
   856  	if f := v.Block.Func; f.pass.debug > 0 {
   857  		f.Warnl(v.Pos, "de-virtualizing call")
   858  	}
   859  	return v
   860  }
   861  
   862  // isSamePtr reports whether p1 and p2 point to the same address.
   863  func isSamePtr(p1, p2 *Value) bool {
   864  	if p1 == p2 {
   865  		return true
   866  	}
   867  	if p1.Op != p2.Op {
   868  		for p1.Op == OpOffPtr && p1.AuxInt == 0 {
   869  			p1 = p1.Args[0]
   870  		}
   871  		for p2.Op == OpOffPtr && p2.AuxInt == 0 {
   872  			p2 = p2.Args[0]
   873  		}
   874  		if p1 == p2 {
   875  			return true
   876  		}
   877  		if p1.Op != p2.Op {
   878  			return false
   879  		}
   880  	}
   881  	switch p1.Op {
   882  	case OpOffPtr:
   883  		return p1.AuxInt == p2.AuxInt && isSamePtr(p1.Args[0], p2.Args[0])
   884  	case OpAddr, OpLocalAddr:
   885  		return p1.Aux == p2.Aux
   886  	case OpAddPtr:
   887  		return p1.Args[1] == p2.Args[1] && isSamePtr(p1.Args[0], p2.Args[0])
   888  	}
   889  	return false
   890  }
   891  
   892  func isStackPtr(v *Value) bool {
   893  	for v.Op == OpOffPtr || v.Op == OpAddPtr {
   894  		v = v.Args[0]
   895  	}
   896  	return v.Op == OpSP || v.Op == OpLocalAddr
   897  }
   898  
   899  // disjoint reports whether the memory region specified by [p1:p1+n1)
   900  // does not overlap with [p2:p2+n2).
   901  // A return value of false does not imply the regions overlap.
   902  func disjoint(p1 *Value, n1 int64, p2 *Value, n2 int64) bool {
   903  	if n1 == 0 || n2 == 0 {
   904  		return true
   905  	}
   906  	if p1 == p2 {
   907  		return false
   908  	}
   909  	baseAndOffset := func(ptr *Value) (base *Value, offset int64) {
   910  		base, offset = ptr, 0
   911  		for base.Op == OpOffPtr {
   912  			offset += base.AuxInt
   913  			base = base.Args[0]
   914  		}
   915  		if opcodeTable[base.Op].nilCheck {
   916  			base = base.Args[0]
   917  		}
   918  		return base, offset
   919  	}
   920  
   921  	// Run types-based analysis
   922  	if disjointTypes(p1.Type, p2.Type) {
   923  		return true
   924  	}
   925  
   926  	p1, off1 := baseAndOffset(p1)
   927  	p2, off2 := baseAndOffset(p2)
   928  	if isSamePtr(p1, p2) {
   929  		return !overlap(off1, n1, off2, n2)
   930  	}
   931  	// p1 and p2 are not the same, so if they are both OpAddrs then
   932  	// they point to different variables.
   933  	// If one pointer is on the stack and the other is an argument
   934  	// then they can't overlap.
   935  	switch p1.Op {
   936  	case OpAddr, OpLocalAddr:
   937  		if p2.Op == OpAddr || p2.Op == OpLocalAddr || p2.Op == OpSP {
   938  			return true
   939  		}
   940  		return (p2.Op == OpArg || p2.Op == OpArgIntReg) && p1.Args[0].Op == OpSP
   941  	case OpArg, OpArgIntReg:
   942  		if p2.Op == OpSP || p2.Op == OpLocalAddr {
   943  			return true
   944  		}
   945  	case OpSP:
   946  		return p2.Op == OpAddr || p2.Op == OpLocalAddr || p2.Op == OpArg || p2.Op == OpArgIntReg || p2.Op == OpSP
   947  	}
   948  	return false
   949  }
   950  
   951  // disjointTypes reports whether a memory region pointed to by a pointer of type
   952  // t1 does not overlap with a memory region pointed to by a pointer of type t2 --
   953  // based on type aliasing rules.
   954  func disjointTypes(t1 *types.Type, t2 *types.Type) bool {
   955  	// Unsafe pointer can alias with anything.
   956  	if t1.IsUnsafePtr() || t2.IsUnsafePtr() {
   957  		return false
   958  	}
   959  
   960  	if !t1.IsPtr() || !t2.IsPtr() {
   961  		// Treat non-pointer types (such as TFUNC, TMAP, uintptr) conservatively.
   962  		return false
   963  	}
   964  
   965  	t1 = t1.Elem()
   966  	t2 = t2.Elem()
   967  
   968  	// Not-in-heap types are not supported -- they are rare and non-important; also,
   969  	// type.HasPointers check doesn't work for them correctly.
   970  	if t1.NotInHeap() || t2.NotInHeap() {
   971  		return false
   972  	}
   973  
   974  	isPtrShaped := func(t *types.Type) bool { return int(t.Size()) == types.PtrSize && t.HasPointers() }
   975  
   976  	// Pointers and non-pointers are disjoint (https://pkg.go.dev/unsafe#Pointer).
   977  	if (isPtrShaped(t1) && !t2.HasPointers()) ||
   978  		(isPtrShaped(t2) && !t1.HasPointers()) {
   979  		return true
   980  	}
   981  
   982  	return false
   983  }
   984  
   985  // moveSize returns the number of bytes an aligned MOV instruction moves.
   986  func moveSize(align int64, c *Config) int64 {
   987  	switch {
   988  	case align%8 == 0 && c.PtrSize == 8:
   989  		return 8
   990  	case align%4 == 0:
   991  		return 4
   992  	case align%2 == 0:
   993  		return 2
   994  	}
   995  	return 1
   996  }
   997  
   998  // mergePoint finds a block among a's blocks which dominates b and is itself
   999  // dominated by all of a's blocks. Returns nil if it can't find one.
  1000  // Might return nil even if one does exist.
  1001  func mergePoint(b *Block, a ...*Value) *Block {
  1002  	// Walk backward from b looking for one of the a's blocks.
  1003  
  1004  	// Max distance
  1005  	d := 100
  1006  
  1007  	for d > 0 {
  1008  		for _, x := range a {
  1009  			if b == x.Block {
  1010  				goto found
  1011  			}
  1012  		}
  1013  		if len(b.Preds) > 1 {
  1014  			// Don't know which way to go back. Abort.
  1015  			return nil
  1016  		}
  1017  		b = b.Preds[0].b
  1018  		d--
  1019  	}
  1020  	return nil // too far away
  1021  found:
  1022  	// At this point, r is the first value in a that we find by walking backwards.
  1023  	// if we return anything, r will be it.
  1024  	r := b
  1025  
  1026  	// Keep going, counting the other a's that we find. They must all dominate r.
  1027  	na := 0
  1028  	for d > 0 {
  1029  		for _, x := range a {
  1030  			if b == x.Block {
  1031  				na++
  1032  			}
  1033  		}
  1034  		if na == len(a) {
  1035  			// Found all of a in a backwards walk. We can return r.
  1036  			return r
  1037  		}
  1038  		if len(b.Preds) > 1 {
  1039  			return nil
  1040  		}
  1041  		b = b.Preds[0].b
  1042  		d--
  1043  
  1044  	}
  1045  	return nil // too far away
  1046  }
  1047  
  1048  // clobber invalidates values. Returns true.
  1049  // clobber is used by rewrite rules to:
  1050  //
  1051  //	A) make sure the values are really dead and never used again.
  1052  //	B) decrement use counts of the values' args.
  1053  func clobber(vv ...*Value) bool {
  1054  	for _, v := range vv {
  1055  		v.reset(OpInvalid)
  1056  		// Note: leave v.Block intact.  The Block field is used after clobber.
  1057  	}
  1058  	return true
  1059  }
  1060  
  1061  // resetCopy resets v to be a copy of arg.
  1062  // Always returns true.
  1063  func resetCopy(v *Value, arg *Value) bool {
  1064  	v.reset(OpCopy)
  1065  	v.AddArg(arg)
  1066  	return true
  1067  }
  1068  
  1069  // clobberIfDead resets v when use count is 1. Returns true.
  1070  // clobberIfDead is used by rewrite rules to decrement
  1071  // use counts of v's args when v is dead and never used.
  1072  func clobberIfDead(v *Value) bool {
  1073  	if v.Uses == 1 {
  1074  		v.reset(OpInvalid)
  1075  	}
  1076  	// Note: leave v.Block intact.  The Block field is used after clobberIfDead.
  1077  	return true
  1078  }
  1079  
  1080  // noteRule is an easy way to track if a rule is matched when writing
  1081  // new ones.  Make the rule of interest also conditional on
  1082  //
  1083  //	noteRule("note to self: rule of interest matched")
  1084  //
  1085  // and that message will print when the rule matches.
  1086  func noteRule(s string) bool {
  1087  	fmt.Println(s)
  1088  	return true
  1089  }
  1090  
  1091  // countRule increments Func.ruleMatches[key].
  1092  // If Func.ruleMatches is non-nil at the end
  1093  // of compilation, it will be printed to stdout.
  1094  // This is intended to make it easier to find which functions
  1095  // which contain lots of rules matches when developing new rules.
  1096  func countRule(v *Value, key string) bool {
  1097  	f := v.Block.Func
  1098  	if f.ruleMatches == nil {
  1099  		f.ruleMatches = make(map[string]int)
  1100  	}
  1101  	f.ruleMatches[key]++
  1102  	return true
  1103  }
  1104  
  1105  // warnRule generates compiler debug output with string s when
  1106  // v is not in autogenerated code, cond is true and the rule has fired.
  1107  func warnRule(cond bool, v *Value, s string) bool {
  1108  	if pos := v.Pos; pos.Line() > 1 && cond {
  1109  		v.Block.Func.Warnl(pos, s)
  1110  	}
  1111  	return true
  1112  }
  1113  
  1114  // for a pseudo-op like (LessThan x), extract x.
  1115  func flagArg(v *Value) *Value {
  1116  	if len(v.Args) != 1 || !v.Args[0].Type.IsFlags() {
  1117  		return nil
  1118  	}
  1119  	return v.Args[0]
  1120  }
  1121  
  1122  // amd64CapAVXShift caps an AMD64 AVX vector shift amount c so that over-shifts
  1123  // always result in 0.
  1124  //
  1125  // These instructions have room for an 8-bit immediate and any value larger than
  1126  // the element width will result in 0 or -1 (for an arithmetic right shift).
  1127  // Thus, we simply cap this at 255.
  1128  func amd64CapAVXShift(auxInt int64) uint8 {
  1129  	u := auxIntToUint64(auxInt)
  1130  	if u > 255 {
  1131  		return 255
  1132  	}
  1133  	return uint8(u)
  1134  }
  1135  
  1136  // arm64Negate finds the complement to an ARM64 condition code,
  1137  // for example !Equal -> NotEqual or !LessThan -> GreaterEqual
  1138  //
  1139  // For floating point, it's more subtle because NaN is unordered. We do
  1140  // !LessThanF -> NotLessThanF, the latter takes care of NaNs.
  1141  func arm64Negate(op Op) Op {
  1142  	switch op {
  1143  	case OpARM64LessThan:
  1144  		return OpARM64GreaterEqual
  1145  	case OpARM64LessThanU:
  1146  		return OpARM64GreaterEqualU
  1147  	case OpARM64GreaterThan:
  1148  		return OpARM64LessEqual
  1149  	case OpARM64GreaterThanU:
  1150  		return OpARM64LessEqualU
  1151  	case OpARM64LessEqual:
  1152  		return OpARM64GreaterThan
  1153  	case OpARM64LessEqualU:
  1154  		return OpARM64GreaterThanU
  1155  	case OpARM64GreaterEqual:
  1156  		return OpARM64LessThan
  1157  	case OpARM64GreaterEqualU:
  1158  		return OpARM64LessThanU
  1159  	case OpARM64Equal:
  1160  		return OpARM64NotEqual
  1161  	case OpARM64NotEqual:
  1162  		return OpARM64Equal
  1163  	case OpARM64LessThanF:
  1164  		return OpARM64NotLessThanF
  1165  	case OpARM64NotLessThanF:
  1166  		return OpARM64LessThanF
  1167  	case OpARM64LessEqualF:
  1168  		return OpARM64NotLessEqualF
  1169  	case OpARM64NotLessEqualF:
  1170  		return OpARM64LessEqualF
  1171  	case OpARM64GreaterThanF:
  1172  		return OpARM64NotGreaterThanF
  1173  	case OpARM64NotGreaterThanF:
  1174  		return OpARM64GreaterThanF
  1175  	case OpARM64GreaterEqualF:
  1176  		return OpARM64NotGreaterEqualF
  1177  	case OpARM64NotGreaterEqualF:
  1178  		return OpARM64GreaterEqualF
  1179  	default:
  1180  		panic("unreachable")
  1181  	}
  1182  }
  1183  
  1184  // arm64Invert evaluates (InvertFlags op), which
  1185  // is the same as altering the condition codes such
  1186  // that the same result would be produced if the arguments
  1187  // to the flag-generating instruction were reversed, e.g.
  1188  // (InvertFlags (CMP x y)) -> (CMP y x)
  1189  func arm64Invert(op Op) Op {
  1190  	switch op {
  1191  	case OpARM64LessThan:
  1192  		return OpARM64GreaterThan
  1193  	case OpARM64LessThanU:
  1194  		return OpARM64GreaterThanU
  1195  	case OpARM64GreaterThan:
  1196  		return OpARM64LessThan
  1197  	case OpARM64GreaterThanU:
  1198  		return OpARM64LessThanU
  1199  	case OpARM64LessEqual:
  1200  		return OpARM64GreaterEqual
  1201  	case OpARM64LessEqualU:
  1202  		return OpARM64GreaterEqualU
  1203  	case OpARM64GreaterEqual:
  1204  		return OpARM64LessEqual
  1205  	case OpARM64GreaterEqualU:
  1206  		return OpARM64LessEqualU
  1207  	case OpARM64Equal, OpARM64NotEqual:
  1208  		return op
  1209  	case OpARM64LessThanF:
  1210  		return OpARM64GreaterThanF
  1211  	case OpARM64GreaterThanF:
  1212  		return OpARM64LessThanF
  1213  	case OpARM64LessEqualF:
  1214  		return OpARM64GreaterEqualF
  1215  	case OpARM64GreaterEqualF:
  1216  		return OpARM64LessEqualF
  1217  	case OpARM64NotLessThanF:
  1218  		return OpARM64NotGreaterThanF
  1219  	case OpARM64NotGreaterThanF:
  1220  		return OpARM64NotLessThanF
  1221  	case OpARM64NotLessEqualF:
  1222  		return OpARM64NotGreaterEqualF
  1223  	case OpARM64NotGreaterEqualF:
  1224  		return OpARM64NotLessEqualF
  1225  	default:
  1226  		panic("unreachable")
  1227  	}
  1228  }
  1229  
  1230  // evaluate an ARM64 op against a flags value
  1231  // that is potentially constant; return 1 for true,
  1232  // -1 for false, and 0 for not constant.
  1233  func ccARM64Eval(op Op, flags *Value) int {
  1234  	fop := flags.Op
  1235  	if fop == OpARM64InvertFlags {
  1236  		return -ccARM64Eval(op, flags.Args[0])
  1237  	}
  1238  	if fop != OpARM64FlagConstant {
  1239  		return 0
  1240  	}
  1241  	fc := flagConstant(flags.AuxInt)
  1242  	b2i := func(b bool) int {
  1243  		if b {
  1244  			return 1
  1245  		}
  1246  		return -1
  1247  	}
  1248  	switch op {
  1249  	case OpARM64Equal:
  1250  		return b2i(fc.eq())
  1251  	case OpARM64NotEqual:
  1252  		return b2i(fc.ne())
  1253  	case OpARM64LessThan:
  1254  		return b2i(fc.lt())
  1255  	case OpARM64LessThanU:
  1256  		return b2i(fc.ult())
  1257  	case OpARM64GreaterThan:
  1258  		return b2i(fc.gt())
  1259  	case OpARM64GreaterThanU:
  1260  		return b2i(fc.ugt())
  1261  	case OpARM64LessEqual:
  1262  		return b2i(fc.le())
  1263  	case OpARM64LessEqualU:
  1264  		return b2i(fc.ule())
  1265  	case OpARM64GreaterEqual:
  1266  		return b2i(fc.ge())
  1267  	case OpARM64GreaterEqualU:
  1268  		return b2i(fc.uge())
  1269  	}
  1270  	return 0
  1271  }
  1272  
  1273  // logRule logs the use of the rule s. This will only be enabled if
  1274  // rewrite rules were generated with the -log option, see _gen/rulegen.go.
  1275  func logRule(s string) {
  1276  	if ruleFile == nil {
  1277  		// Open a log file to write log to. We open in append
  1278  		// mode because all.bash runs the compiler lots of times,
  1279  		// and we want the concatenation of all of those logs.
  1280  		// This means, of course, that users need to rm the old log
  1281  		// to get fresh data.
  1282  		// TODO: all.bash runs compilers in parallel. Need to synchronize logging somehow?
  1283  		w, err := os.OpenFile(filepath.Join(os.Getenv("GOROOT"), "src", "rulelog"),
  1284  			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
  1285  		if err != nil {
  1286  			panic(err)
  1287  		}
  1288  		ruleFile = w
  1289  	}
  1290  	// Ignore errors in case of multiple processes fighting over the file.
  1291  	fmt.Fprintln(ruleFile, s)
  1292  }
  1293  
  1294  var ruleFile io.Writer
  1295  
  1296  func isConstZero(v *Value) bool {
  1297  	switch v.Op {
  1298  	case OpConstNil:
  1299  		return true
  1300  	case OpConst64, OpConst32, OpConst16, OpConst8, OpConstBool, OpConst32F, OpConst64F:
  1301  		return v.AuxInt == 0
  1302  	case OpStringMake, OpIMake, OpComplexMake:
  1303  		return isConstZero(v.Args[0]) && isConstZero(v.Args[1])
  1304  	case OpSliceMake:
  1305  		return isConstZero(v.Args[0]) && isConstZero(v.Args[1]) && isConstZero(v.Args[2])
  1306  	case OpStringPtr, OpStringLen, OpSlicePtr, OpSliceLen, OpSliceCap, OpITab, OpIData, OpComplexReal, OpComplexImag:
  1307  		return isConstZero(v.Args[0])
  1308  	}
  1309  	return false
  1310  }
  1311  
  1312  // reciprocalExact64 reports whether 1/c is exactly representable.
  1313  func reciprocalExact64(c float64) bool {
  1314  	b := math.Float64bits(c)
  1315  	man := b & (1<<52 - 1)
  1316  	if man != 0 {
  1317  		return false // not a power of 2, denormal, or NaN
  1318  	}
  1319  	exp := b >> 52 & (1<<11 - 1)
  1320  	// exponent bias is 0x3ff.  So taking the reciprocal of a number
  1321  	// changes the exponent to 0x7fe-exp.
  1322  	switch exp {
  1323  	case 0:
  1324  		return false // ±0
  1325  	case 0x7ff:
  1326  		return false // ±inf
  1327  	case 0x7fe:
  1328  		return false // exponent is not representable
  1329  	default:
  1330  		return true
  1331  	}
  1332  }
  1333  
  1334  // reciprocalExact32 reports whether 1/c is exactly representable.
  1335  func reciprocalExact32(c float32) bool {
  1336  	b := math.Float32bits(c)
  1337  	man := b & (1<<23 - 1)
  1338  	if man != 0 {
  1339  		return false // not a power of 2, denormal, or NaN
  1340  	}
  1341  	exp := b >> 23 & (1<<8 - 1)
  1342  	// exponent bias is 0x7f.  So taking the reciprocal of a number
  1343  	// changes the exponent to 0xfe-exp.
  1344  	switch exp {
  1345  	case 0:
  1346  		return false // ±0
  1347  	case 0xff:
  1348  		return false // ±inf
  1349  	case 0xfe:
  1350  		return false // exponent is not representable
  1351  	default:
  1352  		return true
  1353  	}
  1354  }
  1355  
  1356  // check if an immediate can be directly encoded into an ARM's instruction.
  1357  func isARMImmRot(v uint32) bool {
  1358  	for i := 0; i < 16; i++ {
  1359  		if v&^0xff == 0 {
  1360  			return true
  1361  		}
  1362  		v = v<<2 | v>>30
  1363  	}
  1364  
  1365  	return false
  1366  }
  1367  
  1368  // overlap reports whether the ranges given by the given offset and
  1369  // size pairs overlap.
  1370  func overlap(offset1, size1, offset2, size2 int64) bool {
  1371  	if offset1 >= offset2 && offset2+size2 > offset1 {
  1372  		return true
  1373  	}
  1374  	if offset2 >= offset1 && offset1+size1 > offset2 {
  1375  		return true
  1376  	}
  1377  	return false
  1378  }
  1379  
  1380  // check if value zeroes out upper 32-bit of 64-bit register.
  1381  // depth limits recursion depth. In AMD64.rules 3 is used as limit,
  1382  // because it catches same amount of cases as 4.
  1383  func ZeroUpper32Bits(x *Value, depth int) bool {
  1384  	if x.Type.IsSigned() && x.Type.Size() < 8 {
  1385  		// If the value is signed, it might get re-sign-extended
  1386  		// during spill and restore. See issue 68227.
  1387  		return false
  1388  	}
  1389  	switch x.Op {
  1390  	case OpAMD64MOVLconst, OpAMD64MOVLload, OpAMD64MOVLQZX, OpAMD64MOVLloadidx1,
  1391  		OpAMD64MOVWload, OpAMD64MOVWloadidx1, OpAMD64MOVBload, OpAMD64MOVBloadidx1,
  1392  		OpAMD64MOVLloadidx4, OpAMD64ADDLload, OpAMD64SUBLload, OpAMD64ANDLload,
  1393  		OpAMD64ORLload, OpAMD64XORLload, OpAMD64CVTTSD2SL,
  1394  		OpAMD64ADDL, OpAMD64ADDLconst, OpAMD64SUBL, OpAMD64SUBLconst,
  1395  		OpAMD64ANDL, OpAMD64ANDLconst, OpAMD64ORL, OpAMD64ORLconst,
  1396  		OpAMD64XORL, OpAMD64XORLconst, OpAMD64NEGL, OpAMD64NOTL,
  1397  		OpAMD64SHRL, OpAMD64SHRLconst, OpAMD64SARL, OpAMD64SARLconst,
  1398  		OpAMD64SHLL, OpAMD64SHLLconst:
  1399  		return true
  1400  	case OpAMD64MOVQconst:
  1401  		return uint64(uint32(x.AuxInt)) == uint64(x.AuxInt)
  1402  	case OpARM64REV16W, OpARM64REVW, OpARM64RBITW, OpARM64CLZW, OpARM64EXTRWconst,
  1403  		OpARM64MULW, OpARM64MNEGW, OpARM64UDIVW, OpARM64DIVW, OpARM64UMODW,
  1404  		OpARM64MADDW, OpARM64MSUBW, OpARM64RORW, OpARM64RORWconst:
  1405  		return true
  1406  	case OpArg: // note: but not ArgIntReg
  1407  		// amd64 always loads args from the stack unsigned.
  1408  		// most other architectures load them sign/zero extended based on the type.
  1409  		return x.Type.Size() == 4 && x.Block.Func.Config.arch == "amd64"
  1410  	case OpPhi, OpSelect0, OpSelect1:
  1411  		// Phis can use each-other as an arguments, instead of tracking visited values,
  1412  		// just limit recursion depth.
  1413  		if depth <= 0 {
  1414  			return false
  1415  		}
  1416  		for i := range x.Args {
  1417  			if !ZeroUpper32Bits(x.Args[i], depth-1) {
  1418  				return false
  1419  			}
  1420  		}
  1421  		return true
  1422  
  1423  	}
  1424  	return false
  1425  }
  1426  
  1427  // ZeroUpper48Bits is similar to ZeroUpper32Bits, but for upper 48 bits.
  1428  func ZeroUpper48Bits(x *Value, depth int) bool {
  1429  	if x.Type.IsSigned() && x.Type.Size() < 8 {
  1430  		return false
  1431  	}
  1432  	switch x.Op {
  1433  	case OpAMD64MOVWQZX, OpAMD64MOVWload, OpAMD64MOVWloadidx1, OpAMD64MOVWloadidx2:
  1434  		return true
  1435  	case OpAMD64MOVQconst, OpAMD64MOVLconst:
  1436  		return uint64(uint16(x.AuxInt)) == uint64(x.AuxInt)
  1437  	case OpArg: // note: but not ArgIntReg
  1438  		return x.Type.Size() == 2 && x.Block.Func.Config.arch == "amd64"
  1439  	case OpPhi, OpSelect0, OpSelect1:
  1440  		// Phis can use each-other as an arguments, instead of tracking visited values,
  1441  		// just limit recursion depth.
  1442  		if depth <= 0 {
  1443  			return false
  1444  		}
  1445  		for i := range x.Args {
  1446  			if !ZeroUpper48Bits(x.Args[i], depth-1) {
  1447  				return false
  1448  			}
  1449  		}
  1450  		return true
  1451  
  1452  	}
  1453  	return false
  1454  }
  1455  
  1456  // ZeroUpper56Bits is similar to ZeroUpper32Bits, but for upper 56 bits.
  1457  func ZeroUpper56Bits(x *Value, depth int) bool {
  1458  	if x.Type.IsSigned() && x.Type.Size() < 8 {
  1459  		return false
  1460  	}
  1461  	switch x.Op {
  1462  	case OpAMD64MOVBQZX, OpAMD64MOVBload, OpAMD64MOVBloadidx1:
  1463  		return true
  1464  	case OpAMD64MOVQconst, OpAMD64MOVLconst:
  1465  		return uint64(uint8(x.AuxInt)) == uint64(x.AuxInt)
  1466  	case OpArg: // note: but not ArgIntReg
  1467  		return x.Type.Size() == 1 && x.Block.Func.Config.arch == "amd64"
  1468  	case OpPhi, OpSelect0, OpSelect1:
  1469  		// Phis can use each-other as an arguments, instead of tracking visited values,
  1470  		// just limit recursion depth.
  1471  		if depth <= 0 {
  1472  			return false
  1473  		}
  1474  		for i := range x.Args {
  1475  			if !ZeroUpper56Bits(x.Args[i], depth-1) {
  1476  				return false
  1477  			}
  1478  		}
  1479  		return true
  1480  
  1481  	}
  1482  	return false
  1483  }
  1484  
  1485  func isInlinableMemclr(c *Config, sz int64) bool {
  1486  	if sz < 0 {
  1487  		return false
  1488  	}
  1489  	// TODO: expand this check to allow other architectures
  1490  	// see CL 454255 and issue 56997
  1491  	switch c.arch {
  1492  	case "amd64", "arm64":
  1493  		return true
  1494  	case "ppc64le", "ppc64", "loong64":
  1495  		return sz < 512
  1496  	}
  1497  	return false
  1498  }
  1499  
  1500  // isInlinableMemmove reports whether the given arch performs a Move of the given size
  1501  // faster than memmove. It will only return true if replacing the memmove with a Move is
  1502  // safe, either because Move will do all of its loads before any of its stores, or
  1503  // because the arguments are known to be disjoint.
  1504  // This is used as a check for replacing memmove with Move ops.
  1505  func isInlinableMemmove(dst, src *Value, sz int64, c *Config) bool {
  1506  	// It is always safe to convert memmove into Move when its arguments are disjoint.
  1507  	// Move ops may or may not be faster for large sizes depending on how the platform
  1508  	// lowers them, so we only perform this optimization on platforms that we know to
  1509  	// have fast Move ops.
  1510  	switch c.arch {
  1511  	case "amd64":
  1512  		return sz <= 16 || (sz < 1024 && disjoint(dst, sz, src, sz))
  1513  	case "arm64":
  1514  		return sz <= 64 || (sz <= 1024 && disjoint(dst, sz, src, sz))
  1515  	case "loong64":
  1516  		return sz <= 16 || (sz <= 64 && disjoint(dst, sz, src, sz))
  1517  	case "386":
  1518  		return sz <= 8
  1519  	case "s390x", "ppc64", "ppc64le":
  1520  		return sz <= 8 || disjoint(dst, sz, src, sz)
  1521  	case "arm", "mips", "mips64", "mipsle", "mips64le":
  1522  		return sz <= 4
  1523  	}
  1524  	return false
  1525  }
  1526  func IsInlinableMemmove(dst, src *Value, sz int64, c *Config) bool {
  1527  	return isInlinableMemmove(dst, src, sz, c)
  1528  }
  1529  
  1530  // logLargeCopy logs the occurrence of a large copy.
  1531  // The best place to do this is in the rewrite rules where the size of the move is easy to find.
  1532  // "Large" is arbitrarily chosen to be 128 bytes; this may change.
  1533  func logLargeCopy(v *Value, s int64) bool {
  1534  	if s < 128 {
  1535  		return true
  1536  	}
  1537  	if logopt.Enabled() {
  1538  		logopt.LogOpt(v.Pos, "copy", "lower", v.Block.Func.Name, fmt.Sprintf("%d bytes", s))
  1539  	}
  1540  	return true
  1541  }
  1542  func LogLargeCopy(funcName string, pos src.XPos, s int64) {
  1543  	if s < 128 {
  1544  		return
  1545  	}
  1546  	if logopt.Enabled() {
  1547  		logopt.LogOpt(pos, "copy", "lower", funcName, fmt.Sprintf("%d bytes", s))
  1548  	}
  1549  }
  1550  
  1551  // hasSmallRotate reports whether the architecture has rotate instructions
  1552  // for sizes < 32-bit.  This is used to decide whether to promote some rotations.
  1553  func hasSmallRotate(c *Config) bool {
  1554  	switch c.arch {
  1555  	case "amd64", "386":
  1556  		return true
  1557  	default:
  1558  		return false
  1559  	}
  1560  }
  1561  
  1562  func supportsPPC64PCRel() bool {
  1563  	// PCRel is currently supported for >= power10, linux only
  1564  	// Internal and external linking supports this on ppc64le; internal linking on ppc64.
  1565  	return buildcfg.GOPPC64 >= 10 && buildcfg.GOOS == "linux"
  1566  }
  1567  
  1568  func newPPC64ShiftAuxInt(sh, mb, me, sz int64) int32 {
  1569  	if sh < 0 || sh >= sz {
  1570  		panic("PPC64 shift arg sh out of range")
  1571  	}
  1572  	if mb < 0 || mb >= sz {
  1573  		panic("PPC64 shift arg mb out of range")
  1574  	}
  1575  	if me < 0 || me >= sz {
  1576  		panic("PPC64 shift arg me out of range")
  1577  	}
  1578  	return int32(sh<<16 | mb<<8 | me)
  1579  }
  1580  
  1581  func GetPPC64Shiftsh(auxint int64) int64 {
  1582  	return int64(int8(auxint >> 16))
  1583  }
  1584  
  1585  func GetPPC64Shiftmb(auxint int64) int64 {
  1586  	return int64(int8(auxint >> 8))
  1587  }
  1588  
  1589  // Test if this value can encoded as a mask for a rlwinm like
  1590  // operation.  Masks can also extend from the msb and wrap to
  1591  // the lsb too.  That is, the valid masks are 32 bit strings
  1592  // of the form: 0..01..10..0 or 1..10..01..1 or 1...1
  1593  //
  1594  // Note: This ignores the upper 32 bits of the input. When a
  1595  // zero extended result is desired (e.g a 64 bit result), the
  1596  // user must verify the upper 32 bits are 0 and the mask is
  1597  // contiguous (that is, non-wrapping).
  1598  func isPPC64WordRotateMask(v64 int64) bool {
  1599  	// Isolate rightmost 1 (if none 0) and add.
  1600  	v := uint32(v64)
  1601  	vp := (v & -v) + v
  1602  	// Likewise, for the wrapping case.
  1603  	vn := ^v
  1604  	vpn := (vn & -vn) + vn
  1605  	return (v&vp == 0 || vn&vpn == 0) && v != 0
  1606  }
  1607  
  1608  // Test if this mask is a valid, contiguous bitmask which can be
  1609  // represented by a RLWNM mask and also clears the upper 32 bits
  1610  // of the register.
  1611  func isPPC64WordRotateMaskNonWrapping(v64 int64) bool {
  1612  	// Isolate rightmost 1 (if none 0) and add.
  1613  	v := uint32(v64)
  1614  	vp := (v & -v) + v
  1615  	return (v&vp == 0) && v != 0 && uint64(uint32(v64)) == uint64(v64)
  1616  }
  1617  
  1618  // Compress mask and shift into single value of the form
  1619  // me | mb<<8 | rotate<<16 | nbits<<24 where me and mb can
  1620  // be used to regenerate the input mask.
  1621  func encodePPC64RotateMask(rotate, mask, nbits int64) int64 {
  1622  	var mb, me, mbn, men int
  1623  
  1624  	// Determine boundaries and then decode them
  1625  	if mask == 0 || ^mask == 0 || rotate >= nbits {
  1626  		panic(fmt.Sprintf("invalid PPC64 rotate mask: %x %d %d", uint64(mask), rotate, nbits))
  1627  	} else if nbits == 32 {
  1628  		mb = bits.LeadingZeros32(uint32(mask))
  1629  		me = 32 - bits.TrailingZeros32(uint32(mask))
  1630  		mbn = bits.LeadingZeros32(^uint32(mask))
  1631  		men = 32 - bits.TrailingZeros32(^uint32(mask))
  1632  	} else {
  1633  		mb = bits.LeadingZeros64(uint64(mask))
  1634  		me = 64 - bits.TrailingZeros64(uint64(mask))
  1635  		mbn = bits.LeadingZeros64(^uint64(mask))
  1636  		men = 64 - bits.TrailingZeros64(^uint64(mask))
  1637  	}
  1638  	// Check for a wrapping mask (e.g bits at 0 and 63)
  1639  	if mb == 0 && me == int(nbits) {
  1640  		// swap the inverted values
  1641  		mb, me = men, mbn
  1642  	}
  1643  
  1644  	return int64(me) | int64(mb<<8) | rotate<<16 | nbits<<24
  1645  }
  1646  
  1647  // Merge (RLDICL [encoded] (SRDconst [s] x)) into (RLDICL [new_encoded] x)
  1648  // SRDconst on PPC64 is an extended mnemonic of RLDICL. If the input to an
  1649  // RLDICL is an SRDconst, and the RLDICL does not rotate its value, the two
  1650  // operations can be combined. This functions assumes the two opcodes can
  1651  // be merged, and returns an encoded rotate+mask value of the combined RLDICL.
  1652  func mergePPC64RLDICLandSRDconst(encoded, s int64) int64 {
  1653  	mb := s
  1654  	r := 64 - s
  1655  	// A larger mb is a smaller mask.
  1656  	if (encoded>>8)&0xFF < mb {
  1657  		encoded = (encoded &^ 0xFF00) | mb<<8
  1658  	}
  1659  	// The rotate is expected to be 0.
  1660  	if (encoded & 0xFF0000) != 0 {
  1661  		panic("non-zero rotate")
  1662  	}
  1663  	return encoded | r<<16
  1664  }
  1665  
  1666  // DecodePPC64RotateMask is the inverse operation of encodePPC64RotateMask.  The values returned as
  1667  // mb and me satisfy the POWER ISA definition of MASK(x,y) where MASK(mb,me) = mask.
  1668  func DecodePPC64RotateMask(sauxint int64) (rotate, mb, me int64, mask uint64) {
  1669  	auxint := uint64(sauxint)
  1670  	rotate = int64((auxint >> 16) & 0xFF)
  1671  	mb = int64((auxint >> 8) & 0xFF)
  1672  	me = int64((auxint >> 0) & 0xFF)
  1673  	nbits := int64((auxint >> 24) & 0xFF)
  1674  	mask = ((1 << uint(nbits-mb)) - 1) ^ ((1 << uint(nbits-me)) - 1)
  1675  	if mb > me {
  1676  		mask = ^mask
  1677  	}
  1678  	if nbits == 32 {
  1679  		mask = uint64(uint32(mask))
  1680  	}
  1681  
  1682  	// Fixup ME to match ISA definition.  The second argument to MASK(..,me)
  1683  	// is inclusive.
  1684  	me = (me - 1) & (nbits - 1)
  1685  	return
  1686  }
  1687  
  1688  // This verifies that the mask is a set of
  1689  // consecutive bits including the least
  1690  // significant bit.
  1691  func isPPC64ValidShiftMask(v int64) bool {
  1692  	if (v != 0) && ((v+1)&v) == 0 {
  1693  		return true
  1694  	}
  1695  	return false
  1696  }
  1697  
  1698  func getPPC64ShiftMaskLength(v int64) int64 {
  1699  	return int64(bits.Len64(uint64(v)))
  1700  }
  1701  
  1702  // Decompose a shift right into an equivalent rotate/mask,
  1703  // and return mask & m.
  1704  func mergePPC64RShiftMask(m, s, nbits int64) int64 {
  1705  	smask := uint64((1<<uint(nbits))-1) >> uint(s)
  1706  	return m & int64(smask)
  1707  }
  1708  
  1709  // Combine (ANDconst [m] (SRWconst [s])) into (RLWINM [y]) or return 0
  1710  func mergePPC64AndSrwi(m, s int64) int64 {
  1711  	mask := mergePPC64RShiftMask(m, s, 32)
  1712  	if !isPPC64WordRotateMask(mask) {
  1713  		return 0
  1714  	}
  1715  	return encodePPC64RotateMask((32-s)&31, mask, 32)
  1716  }
  1717  
  1718  // Combine (ANDconst [m] (SRDconst [s])) into (RLWINM [y]) or return 0
  1719  func mergePPC64AndSrdi(m, s int64) int64 {
  1720  	mask := mergePPC64RShiftMask(m, s, 64)
  1721  
  1722  	// Verify the rotate and mask result only uses the lower 32 bits.
  1723  	rv := bits.RotateLeft64(0xFFFFFFFF00000000, -int(s))
  1724  	if rv&uint64(mask) != 0 {
  1725  		return 0
  1726  	}
  1727  	if !isPPC64WordRotateMaskNonWrapping(mask) {
  1728  		return 0
  1729  	}
  1730  	return encodePPC64RotateMask((32-s)&31, mask, 32)
  1731  }
  1732  
  1733  // Combine (ANDconst [m] (SLDconst [s])) into (RLWINM [y]) or return 0
  1734  func mergePPC64AndSldi(m, s int64) int64 {
  1735  	mask := -1 << s & m
  1736  
  1737  	// Verify the rotate and mask result only uses the lower 32 bits.
  1738  	rv := bits.RotateLeft64(0xFFFFFFFF00000000, int(s))
  1739  	if rv&uint64(mask) != 0 {
  1740  		return 0
  1741  	}
  1742  	if !isPPC64WordRotateMaskNonWrapping(mask) {
  1743  		return 0
  1744  	}
  1745  	return encodePPC64RotateMask(s&31, mask, 32)
  1746  }
  1747  
  1748  // Test if a word shift right feeding into a CLRLSLDI can be merged into RLWINM.
  1749  // Return the encoded RLWINM constant, or 0 if they cannot be merged.
  1750  func mergePPC64ClrlsldiSrw(sld, srw int64) int64 {
  1751  	mask_1 := uint64(0xFFFFFFFF >> uint(srw))
  1752  	// for CLRLSLDI, it's more convenient to think of it as a mask left bits then rotate left.
  1753  	mask_2 := uint64(0xFFFFFFFFFFFFFFFF) >> uint(GetPPC64Shiftmb(sld))
  1754  
  1755  	// Rewrite mask to apply after the final left shift.
  1756  	mask_3 := (mask_1 & mask_2) << uint(GetPPC64Shiftsh(sld))
  1757  
  1758  	r_1 := 32 - srw
  1759  	r_2 := GetPPC64Shiftsh(sld)
  1760  	r_3 := (r_1 + r_2) & 31 // This can wrap.
  1761  
  1762  	if uint64(uint32(mask_3)) != mask_3 || mask_3 == 0 {
  1763  		return 0
  1764  	}
  1765  	return encodePPC64RotateMask(r_3, int64(mask_3), 32)
  1766  }
  1767  
  1768  // Test if a doubleword shift right feeding into a CLRLSLDI can be merged into RLWINM.
  1769  // Return the encoded RLWINM constant, or 0 if they cannot be merged.
  1770  func mergePPC64ClrlsldiSrd(sld, srd int64) int64 {
  1771  	mask_1 := uint64(0xFFFFFFFFFFFFFFFF) >> uint(srd)
  1772  	// for CLRLSLDI, it's more convenient to think of it as a mask left bits then rotate left.
  1773  	mask_2 := uint64(0xFFFFFFFFFFFFFFFF) >> uint(GetPPC64Shiftmb(sld))
  1774  
  1775  	// Rewrite mask to apply after the final left shift.
  1776  	mask_3 := (mask_1 & mask_2) << uint(GetPPC64Shiftsh(sld))
  1777  
  1778  	r_1 := 64 - srd
  1779  	r_2 := GetPPC64Shiftsh(sld)
  1780  	r_3 := (r_1 + r_2) & 63 // This can wrap.
  1781  
  1782  	if uint64(uint32(mask_3)) != mask_3 || mask_3 == 0 {
  1783  		return 0
  1784  	}
  1785  	// This combine only works when selecting and shifting the lower 32 bits.
  1786  	v1 := bits.RotateLeft64(0xFFFFFFFF00000000, int(r_3))
  1787  	if v1&mask_3 != 0 {
  1788  		return 0
  1789  	}
  1790  	return encodePPC64RotateMask(r_3&31, int64(mask_3), 32)
  1791  }
  1792  
  1793  // Test if a RLWINM feeding into a CLRLSLDI can be merged into RLWINM.  Return
  1794  // the encoded RLWINM constant, or 0 if they cannot be merged.
  1795  func mergePPC64ClrlsldiRlwinm(sld int32, rlw int64) int64 {
  1796  	r_1, _, _, mask_1 := DecodePPC64RotateMask(rlw)
  1797  	// for CLRLSLDI, it's more convenient to think of it as a mask left bits then rotate left.
  1798  	mask_2 := uint64(0xFFFFFFFFFFFFFFFF) >> uint(GetPPC64Shiftmb(int64(sld)))
  1799  
  1800  	// combine the masks, and adjust for the final left shift.
  1801  	mask_3 := (mask_1 & mask_2) << uint(GetPPC64Shiftsh(int64(sld)))
  1802  	r_2 := GetPPC64Shiftsh(int64(sld))
  1803  	r_3 := (r_1 + r_2) & 31 // This can wrap.
  1804  
  1805  	// Verify the result is still a valid bitmask of <= 32 bits.
  1806  	if !isPPC64WordRotateMask(int64(mask_3)) || uint64(uint32(mask_3)) != mask_3 {
  1807  		return 0
  1808  	}
  1809  	return encodePPC64RotateMask(r_3, int64(mask_3), 32)
  1810  }
  1811  
  1812  // Test if RLWINM feeding into an ANDconst can be merged. Return the encoded RLWINM constant,
  1813  // or 0 if they cannot be merged.
  1814  func mergePPC64AndRlwinm(mask uint32, rlw int64) int64 {
  1815  	r, _, _, mask_rlw := DecodePPC64RotateMask(rlw)
  1816  	mask_out := (mask_rlw & uint64(mask))
  1817  
  1818  	// Verify the result is still a valid bitmask of <= 32 bits.
  1819  	if !isPPC64WordRotateMask(int64(mask_out)) {
  1820  		return 0
  1821  	}
  1822  	return encodePPC64RotateMask(r, int64(mask_out), 32)
  1823  }
  1824  
  1825  // Test if RLWINM opcode rlw clears the upper 32 bits of the
  1826  // result. Return rlw if it does, 0 otherwise.
  1827  func mergePPC64MovwzregRlwinm(rlw int64) int64 {
  1828  	_, mb, me, _ := DecodePPC64RotateMask(rlw)
  1829  	if mb > me {
  1830  		return 0
  1831  	}
  1832  	return rlw
  1833  }
  1834  
  1835  // Test if AND feeding into an ANDconst can be merged. Return the encoded RLWINM constant,
  1836  // or 0 if they cannot be merged.
  1837  func mergePPC64RlwinmAnd(rlw int64, mask uint32) int64 {
  1838  	r, _, _, mask_rlw := DecodePPC64RotateMask(rlw)
  1839  
  1840  	// Rotate the input mask, combine with the rlwnm mask, and test if it is still a valid rlwinm mask.
  1841  	r_mask := bits.RotateLeft32(mask, int(r))
  1842  
  1843  	mask_out := (mask_rlw & uint64(r_mask))
  1844  
  1845  	// Verify the result is still a valid bitmask of <= 32 bits.
  1846  	if !isPPC64WordRotateMask(int64(mask_out)) {
  1847  		return 0
  1848  	}
  1849  	return encodePPC64RotateMask(r, int64(mask_out), 32)
  1850  }
  1851  
  1852  // Test if RLWINM feeding into SRDconst can be merged. Return the encoded RLIWNM constant,
  1853  // or 0 if they cannot be merged.
  1854  func mergePPC64SldiRlwinm(sldi, rlw int64) int64 {
  1855  	r_1, mb, me, mask_1 := DecodePPC64RotateMask(rlw)
  1856  	if mb > me || mb < sldi {
  1857  		// Wrapping masks cannot be merged as the upper 32 bits are effectively undefined in this case.
  1858  		// Likewise, if mb is less than the shift amount, it cannot be merged.
  1859  		return 0
  1860  	}
  1861  	// combine the masks, and adjust for the final left shift.
  1862  	mask_3 := mask_1 << sldi
  1863  	r_3 := (r_1 + sldi) & 31 // This can wrap.
  1864  
  1865  	// Verify the result is still a valid bitmask of <= 32 bits.
  1866  	if uint64(uint32(mask_3)) != mask_3 {
  1867  		return 0
  1868  	}
  1869  	return encodePPC64RotateMask(r_3, int64(mask_3), 32)
  1870  }
  1871  
  1872  // Compute the encoded RLWINM constant from combining (SLDconst [sld] (SRWconst [srw] x)),
  1873  // or return 0 if they cannot be combined.
  1874  func mergePPC64SldiSrw(sld, srw int64) int64 {
  1875  	if sld > srw || srw >= 32 {
  1876  		return 0
  1877  	}
  1878  	mask_r := uint32(0xFFFFFFFF) >> uint(srw)
  1879  	mask_l := uint32(0xFFFFFFFF) >> uint(sld)
  1880  	mask := (mask_r & mask_l) << uint(sld)
  1881  	return encodePPC64RotateMask((32-srw+sld)&31, int64(mask), 32)
  1882  }
  1883  
  1884  // Convert a PPC64 opcode from the Op to OpCC form. This converts (op x y)
  1885  // to (Select0 (opCC x y)) without having to explicitly fixup every user
  1886  // of op.
  1887  //
  1888  // E.g consider the case:
  1889  // a = (ADD x y)
  1890  // b = (CMPconst [0] a)
  1891  // c = (OR a z)
  1892  //
  1893  // A rule like (CMPconst [0] (ADD x y)) => (CMPconst [0] (Select0 (ADDCC x y)))
  1894  // would produce:
  1895  // a  = (ADD x y)
  1896  // a' = (ADDCC x y)
  1897  // a” = (Select0 a')
  1898  // b  = (CMPconst [0] a”)
  1899  // c  = (OR a z)
  1900  //
  1901  // which makes it impossible to rewrite the second user. Instead the result
  1902  // of this conversion is:
  1903  // a' = (ADDCC x y)
  1904  // a  = (Select0 a')
  1905  // b  = (CMPconst [0] a)
  1906  // c  = (OR a z)
  1907  //
  1908  // Which makes it trivial to rewrite b using a lowering rule.
  1909  func convertPPC64OpToOpCC(op *Value) *Value {
  1910  	ccOpMap := map[Op]Op{
  1911  		OpPPC64ADD:      OpPPC64ADDCC,
  1912  		OpPPC64ADDconst: OpPPC64ADDCCconst,
  1913  		OpPPC64AND:      OpPPC64ANDCC,
  1914  		OpPPC64ANDN:     OpPPC64ANDNCC,
  1915  		OpPPC64ANDconst: OpPPC64ANDCCconst,
  1916  		OpPPC64CNTLZD:   OpPPC64CNTLZDCC,
  1917  		OpPPC64MULHDU:   OpPPC64MULHDUCC,
  1918  		OpPPC64NEG:      OpPPC64NEGCC,
  1919  		OpPPC64NOR:      OpPPC64NORCC,
  1920  		OpPPC64OR:       OpPPC64ORCC,
  1921  		OpPPC64RLDICL:   OpPPC64RLDICLCC,
  1922  		OpPPC64SUB:      OpPPC64SUBCC,
  1923  		OpPPC64XOR:      OpPPC64XORCC,
  1924  	}
  1925  	b := op.Block
  1926  	opCC := b.NewValue0I(op.Pos, ccOpMap[op.Op], types.NewTuple(op.Type, types.TypeFlags), op.AuxInt)
  1927  	opCC.AddArgs(op.Args...)
  1928  	op.reset(OpSelect0)
  1929  	op.AddArgs(opCC)
  1930  	return op
  1931  }
  1932  
  1933  // Try converting a RLDICL to ANDCC. If successful, return the mask otherwise 0.
  1934  func convertPPC64RldiclAndccconst(sauxint int64) int64 {
  1935  	r, _, _, mask := DecodePPC64RotateMask(sauxint)
  1936  	if r != 0 || mask&0xFFFF != mask {
  1937  		return 0
  1938  	}
  1939  	return int64(mask)
  1940  }
  1941  
  1942  // Convenience function to rotate a 32 bit constant value by another constant.
  1943  func rotateLeft32(v, rotate int64) int64 {
  1944  	return int64(bits.RotateLeft32(uint32(v), int(rotate)))
  1945  }
  1946  
  1947  func rotateRight64(v, rotate int64) int64 {
  1948  	return int64(bits.RotateLeft64(uint64(v), int(-rotate)))
  1949  }
  1950  
  1951  // encodes the lsb and width for arm(64) bitfield ops into the expected auxInt format.
  1952  func armBFAuxInt(lsb, width int64) arm64BitField {
  1953  	if lsb < 0 || lsb > 63 {
  1954  		panic("ARM(64) bit field lsb constant out of range")
  1955  	}
  1956  	if width < 1 || lsb+width > 64 {
  1957  		panic("ARM(64) bit field width constant out of range")
  1958  	}
  1959  	return arm64BitField(width | lsb<<8)
  1960  }
  1961  
  1962  // returns the lsb part of the auxInt field of arm64 bitfield ops.
  1963  func (bfc arm64BitField) lsb() int64 {
  1964  	return int64(uint64(bfc) >> 8)
  1965  }
  1966  
  1967  // returns the width part of the auxInt field of arm64 bitfield ops.
  1968  func (bfc arm64BitField) width() int64 {
  1969  	return int64(bfc) & 0xff
  1970  }
  1971  
  1972  // checks if mask >> rshift applied at lsb is a valid arm64 bitfield op mask.
  1973  func isARM64BFMask(lsb, mask, rshift int64) bool {
  1974  	shiftedMask := int64(uint64(mask) >> uint64(rshift))
  1975  	return shiftedMask != 0 && isPowerOfTwo(shiftedMask+1) && nto(shiftedMask)+lsb < 64
  1976  }
  1977  
  1978  // returns the bitfield width of mask >> rshift for arm64 bitfield ops.
  1979  func arm64BFWidth(mask, rshift int64) int64 {
  1980  	shiftedMask := int64(uint64(mask) >> uint64(rshift))
  1981  	if shiftedMask == 0 {
  1982  		panic("ARM64 BF mask is zero")
  1983  	}
  1984  	return nto(shiftedMask)
  1985  }
  1986  
  1987  // encodes condition code and NZCV flags into result.
  1988  func arm64ConditionalParamsAuxInt(cond Op, nzcv uint8) arm64ConditionalParams {
  1989  	if cond < OpARM64Equal || cond > OpARM64GreaterEqualU {
  1990  		panic("Wrong conditional operation")
  1991  	}
  1992  	if nzcv&0x0f != nzcv {
  1993  		panic("Wrong value of NZCV flag")
  1994  	}
  1995  	return arm64ConditionalParams{cond, nzcv, 0, false}
  1996  }
  1997  
  1998  // encodes condition code, NZCV flags and constant value into auxint.
  1999  func arm64ConditionalParamsAuxIntWithValue(cond Op, nzcv uint8, value uint8) arm64ConditionalParams {
  2000  	if value&0x1f != value {
  2001  		panic("Wrong value of constant")
  2002  	}
  2003  	params := arm64ConditionalParamsAuxInt(cond, nzcv)
  2004  	params.constValue = value
  2005  	params.ind = true
  2006  	return params
  2007  }
  2008  
  2009  // extracts condition code from auxint.
  2010  func (condParams arm64ConditionalParams) Cond() Op {
  2011  	return condParams.cond
  2012  }
  2013  
  2014  // extracts NZCV flags from auxint.
  2015  func (condParams arm64ConditionalParams) Nzcv() int64 {
  2016  	return int64(condParams.nzcv)
  2017  }
  2018  
  2019  // extracts constant value from auxint if present.
  2020  func (condParams arm64ConditionalParams) ConstValue() (int64, bool) {
  2021  	return int64(condParams.constValue), condParams.ind
  2022  }
  2023  
  2024  // registerizable reports whether t is a primitive type that fits in
  2025  // a register. It assumes float64 values will always fit into registers
  2026  // even if that isn't strictly true.
  2027  func registerizable(b *Block, typ *types.Type) bool {
  2028  	if typ.IsPtrShaped() || typ.IsFloat() || typ.IsBoolean() {
  2029  		return true
  2030  	}
  2031  	if typ.IsInteger() {
  2032  		return typ.Size() <= b.Func.Config.RegSize
  2033  	}
  2034  	return false
  2035  }
  2036  
  2037  // needRaceCleanup reports whether this call to racefuncenter/exit isn't needed.
  2038  func needRaceCleanup(sym *AuxCall, v *Value) bool {
  2039  	f := v.Block.Func
  2040  	if !f.Config.Race {
  2041  		return false
  2042  	}
  2043  	if !isSameCall(sym, "runtime.racefuncenter") && !isSameCall(sym, "runtime.racefuncexit") {
  2044  		return false
  2045  	}
  2046  	for _, b := range f.Blocks {
  2047  		for _, v := range b.Values {
  2048  			switch v.Op {
  2049  			case OpStaticCall, OpStaticLECall:
  2050  				// Check for racefuncenter will encounter racefuncexit and vice versa.
  2051  				// Allow calls to panic*
  2052  				s := v.Aux.(*AuxCall).Fn.String()
  2053  				switch s {
  2054  				case "runtime.racefuncenter", "runtime.racefuncexit",
  2055  					"runtime.panicdivide", "runtime.panicwrap",
  2056  					"runtime.panicshift":
  2057  					continue
  2058  				}
  2059  				// If we encountered any call, we need to keep racefunc*,
  2060  				// for accurate stacktraces.
  2061  				return false
  2062  			case OpPanicBounds, OpPanicExtend:
  2063  				// Note: these are panic generators that are ok (like the static calls above).
  2064  			case OpClosureCall, OpInterCall, OpClosureLECall, OpInterLECall:
  2065  				// We must keep the race functions if there are any other call types.
  2066  				return false
  2067  			}
  2068  		}
  2069  	}
  2070  	if isSameCall(sym, "runtime.racefuncenter") {
  2071  		// TODO REGISTER ABI this needs to be cleaned up.
  2072  		// If we're removing racefuncenter, remove its argument as well.
  2073  		if v.Args[0].Op != OpStore {
  2074  			if v.Op == OpStaticLECall {
  2075  				// there is no store, yet.
  2076  				return true
  2077  			}
  2078  			return false
  2079  		}
  2080  		mem := v.Args[0].Args[2]
  2081  		v.Args[0].reset(OpCopy)
  2082  		v.Args[0].AddArg(mem)
  2083  	}
  2084  	return true
  2085  }
  2086  
  2087  // symIsRO reports whether sym is a read-only global.
  2088  func symIsRO(sym Sym) bool {
  2089  	lsym := sym.(*obj.LSym)
  2090  	return lsym.Type == objabi.SRODATA && len(lsym.R) == 0
  2091  }
  2092  
  2093  // symIsROZero reports whether sym is a read-only global whose data contains all zeros.
  2094  func symIsROZero(sym Sym) bool {
  2095  	lsym := sym.(*obj.LSym)
  2096  	if lsym.Type != objabi.SRODATA || len(lsym.R) != 0 {
  2097  		return false
  2098  	}
  2099  	for _, b := range lsym.P {
  2100  		if b != 0 {
  2101  			return false
  2102  		}
  2103  	}
  2104  	return true
  2105  }
  2106  
  2107  // isFixedLoad returns true if the load can be resolved to fixed address or constant,
  2108  // and can be rewritten by rewriteFixedLoad.
  2109  func isFixedLoad(v *Value, sym Sym, off int64) bool {
  2110  	lsym := sym.(*obj.LSym)
  2111  	if (v.Type.IsPtrShaped() || v.Type.IsUintptr()) && lsym.Type == objabi.SRODATA {
  2112  		for _, r := range lsym.R {
  2113  			if (r.Type == objabi.R_ADDR || r.Type == objabi.R_WEAKADDR) && int64(r.Off) == off && r.Add == 0 {
  2114  				return true
  2115  			}
  2116  		}
  2117  		return false
  2118  	}
  2119  
  2120  	if ti := lsym.TypeInfo(); ti != nil {
  2121  		// Type symbols do not contain information about their fields, unlike the cases above.
  2122  		// Hand-implement field accesses.
  2123  		// TODO: can this be replaced with reflectdata.writeType and just use the code above?
  2124  
  2125  		t := ti.Type.(*types.Type)
  2126  
  2127  		for _, f := range rttype.Type.Fields() {
  2128  			if f.Offset == off && copyCompatibleType(v.Type, f.Type) {
  2129  				switch f.Sym.Name {
  2130  				case "Size_", "PtrBytes", "Hash", "Kind_", "GCData":
  2131  					return true
  2132  				default:
  2133  					// fmt.Println("unknown field", f.Sym.Name)
  2134  					return false
  2135  				}
  2136  			}
  2137  		}
  2138  
  2139  		if t.IsPtr() && off == rttype.PtrType.OffsetOf("Elem") {
  2140  			return true
  2141  		}
  2142  
  2143  		return false
  2144  	}
  2145  
  2146  	return false
  2147  }
  2148  
  2149  // rewriteFixedLoad rewrites a load to a fixed address or constant, if isFixedLoad returns true.
  2150  func rewriteFixedLoad(v *Value, sym Sym, sb *Value, off int64) *Value {
  2151  	b := v.Block
  2152  	f := b.Func
  2153  
  2154  	lsym := sym.(*obj.LSym)
  2155  	if (v.Type.IsPtrShaped() || v.Type.IsUintptr()) && lsym.Type == objabi.SRODATA {
  2156  		for _, r := range lsym.R {
  2157  			if (r.Type == objabi.R_ADDR || r.Type == objabi.R_WEAKADDR) && int64(r.Off) == off && r.Add == 0 {
  2158  				if strings.HasPrefix(r.Sym.Name, "type:") {
  2159  					// In case we're loading a type out of a dictionary, we need to record
  2160  					// that the containing function might put that type in an interface.
  2161  					// That information is currently recorded in relocations in the dictionary,
  2162  					// but if we perform this load at compile time then the dictionary
  2163  					// might be dead.
  2164  					reflectdata.MarkTypeSymUsedInInterface(r.Sym, f.fe.Func().Linksym())
  2165  				} else if strings.HasPrefix(r.Sym.Name, "go:itab") {
  2166  					// Same, but if we're using an itab we need to record that the
  2167  					// itab._type might be put in an interface.
  2168  					reflectdata.MarkTypeSymUsedInInterface(r.Sym, f.fe.Func().Linksym())
  2169  				}
  2170  				v.reset(OpAddr)
  2171  				v.Aux = symToAux(r.Sym)
  2172  				v.AddArg(sb)
  2173  				return v
  2174  			}
  2175  		}
  2176  		base.Fatalf("fixedLoad data not known for %s:%d", sym, off)
  2177  	}
  2178  
  2179  	if ti := lsym.TypeInfo(); ti != nil {
  2180  		// Type symbols do not contain information about their fields, unlike the cases above.
  2181  		// Hand-implement field accesses.
  2182  		// TODO: can this be replaced with reflectdata.writeType and just use the code above?
  2183  
  2184  		t := ti.Type.(*types.Type)
  2185  
  2186  		ptrSizedOpConst := OpConst64
  2187  		if f.Config.PtrSize == 4 {
  2188  			ptrSizedOpConst = OpConst32
  2189  		}
  2190  
  2191  		for _, f := range rttype.Type.Fields() {
  2192  			if f.Offset == off && copyCompatibleType(v.Type, f.Type) {
  2193  				switch f.Sym.Name {
  2194  				case "Size_":
  2195  					v.reset(ptrSizedOpConst)
  2196  					v.AuxInt = t.Size()
  2197  					return v
  2198  				case "PtrBytes":
  2199  					v.reset(ptrSizedOpConst)
  2200  					v.AuxInt = types.PtrDataSize(t)
  2201  					return v
  2202  				case "Hash":
  2203  					v.reset(OpConst32)
  2204  					v.AuxInt = int64(int32(types.TypeHash(t)))
  2205  					return v
  2206  				case "Kind_":
  2207  					v.reset(OpConst8)
  2208  					v.AuxInt = int64(int8(reflectdata.ABIKindOfType(t)))
  2209  					return v
  2210  				case "GCData":
  2211  					gcdata, _ := reflectdata.GCSym(t, true)
  2212  					v.reset(OpAddr)
  2213  					v.Aux = symToAux(gcdata)
  2214  					v.AddArg(sb)
  2215  					return v
  2216  				default:
  2217  					base.Fatalf("unknown field %s for fixedLoad of %s at offset %d", f.Sym.Name, lsym.Name, off)
  2218  				}
  2219  			}
  2220  		}
  2221  
  2222  		if t.IsPtr() && off == rttype.PtrType.OffsetOf("Elem") {
  2223  			elemSym := reflectdata.TypeLinksym(t.Elem())
  2224  			reflectdata.MarkTypeSymUsedInInterface(elemSym, f.fe.Func().Linksym())
  2225  			v.reset(OpAddr)
  2226  			v.Aux = symToAux(elemSym)
  2227  			v.AddArg(sb)
  2228  			return v
  2229  		}
  2230  
  2231  		base.Fatalf("fixedLoad data not known for %s:%d", sym, off)
  2232  	}
  2233  
  2234  	base.Fatalf("fixedLoad data not known for %s:%d", sym, off)
  2235  	return nil
  2236  }
  2237  
  2238  // read8 reads one byte from the read-only global sym at offset off.
  2239  func read8(sym Sym, off int64) uint8 {
  2240  	lsym := sym.(*obj.LSym)
  2241  	if off >= int64(len(lsym.P)) || off < 0 {
  2242  		// Invalid index into the global sym.
  2243  		// This can happen in dead code, so we don't want to panic.
  2244  		// Just return any value, it will eventually get ignored.
  2245  		// See issue 29215.
  2246  		return 0
  2247  	}
  2248  	return lsym.P[off]
  2249  }
  2250  
  2251  // read16 reads two bytes from the read-only global sym at offset off.
  2252  func read16(sym Sym, off int64, byteorder binary.ByteOrder) uint16 {
  2253  	lsym := sym.(*obj.LSym)
  2254  	// lsym.P is written lazily.
  2255  	// Bytes requested after the end of lsym.P are 0.
  2256  	var src []byte
  2257  	if 0 <= off && off < int64(len(lsym.P)) {
  2258  		src = lsym.P[off:]
  2259  	}
  2260  	buf := make([]byte, 2)
  2261  	copy(buf, src)
  2262  	return byteorder.Uint16(buf)
  2263  }
  2264  
  2265  // read32 reads four bytes from the read-only global sym at offset off.
  2266  func read32(sym Sym, off int64, byteorder binary.ByteOrder) uint32 {
  2267  	lsym := sym.(*obj.LSym)
  2268  	var src []byte
  2269  	if 0 <= off && off < int64(len(lsym.P)) {
  2270  		src = lsym.P[off:]
  2271  	}
  2272  	buf := make([]byte, 4)
  2273  	copy(buf, src)
  2274  	return byteorder.Uint32(buf)
  2275  }
  2276  
  2277  // read64 reads eight bytes from the read-only global sym at offset off.
  2278  func read64(sym Sym, off int64, byteorder binary.ByteOrder) uint64 {
  2279  	lsym := sym.(*obj.LSym)
  2280  	var src []byte
  2281  	if 0 <= off && off < int64(len(lsym.P)) {
  2282  		src = lsym.P[off:]
  2283  	}
  2284  	buf := make([]byte, 8)
  2285  	copy(buf, src)
  2286  	return byteorder.Uint64(buf)
  2287  }
  2288  
  2289  // sequentialAddresses reports true if it can prove that x + n == y
  2290  func sequentialAddresses(x, y *Value, n int64) bool {
  2291  	if x == y && n == 0 {
  2292  		return true
  2293  	}
  2294  	if x.Op == Op386ADDL && y.Op == Op386LEAL1 && y.AuxInt == n && y.Aux == nil &&
  2295  		(x.Args[0] == y.Args[0] && x.Args[1] == y.Args[1] ||
  2296  			x.Args[0] == y.Args[1] && x.Args[1] == y.Args[0]) {
  2297  		return true
  2298  	}
  2299  	if x.Op == Op386LEAL1 && y.Op == Op386LEAL1 && y.AuxInt == x.AuxInt+n && x.Aux == y.Aux &&
  2300  		(x.Args[0] == y.Args[0] && x.Args[1] == y.Args[1] ||
  2301  			x.Args[0] == y.Args[1] && x.Args[1] == y.Args[0]) {
  2302  		return true
  2303  	}
  2304  	if x.Op == OpAMD64ADDQ && y.Op == OpAMD64LEAQ1 && y.AuxInt == n && y.Aux == nil &&
  2305  		(x.Args[0] == y.Args[0] && x.Args[1] == y.Args[1] ||
  2306  			x.Args[0] == y.Args[1] && x.Args[1] == y.Args[0]) {
  2307  		return true
  2308  	}
  2309  	if x.Op == OpAMD64LEAQ1 && y.Op == OpAMD64LEAQ1 && y.AuxInt == x.AuxInt+n && x.Aux == y.Aux &&
  2310  		(x.Args[0] == y.Args[0] && x.Args[1] == y.Args[1] ||
  2311  			x.Args[0] == y.Args[1] && x.Args[1] == y.Args[0]) {
  2312  		return true
  2313  	}
  2314  	return false
  2315  }
  2316  
  2317  // flagConstant represents the result of a compile-time comparison.
  2318  // The sense of these flags does not necessarily represent the hardware's notion
  2319  // of a flags register - these are just a compile-time construct.
  2320  // We happen to match the semantics to those of arm/arm64.
  2321  // Note that these semantics differ from x86: the carry flag has the opposite
  2322  // sense on a subtraction!
  2323  //
  2324  //	On amd64, C=1 represents a borrow, e.g. SBB on amd64 does x - y - C.
  2325  //	On arm64, C=0 represents a borrow, e.g. SBC on arm64 does x - y - ^C.
  2326  //	 (because it does x + ^y + C).
  2327  //
  2328  // See https://en.wikipedia.org/wiki/Carry_flag#Vs._borrow_flag
  2329  type flagConstant uint8
  2330  
  2331  // N reports whether the result of an operation is negative (high bit set).
  2332  func (fc flagConstant) N() bool {
  2333  	return fc&1 != 0
  2334  }
  2335  
  2336  // Z reports whether the result of an operation is 0.
  2337  func (fc flagConstant) Z() bool {
  2338  	return fc&2 != 0
  2339  }
  2340  
  2341  // C reports whether an unsigned add overflowed (carry), or an
  2342  // unsigned subtract did not underflow (borrow).
  2343  func (fc flagConstant) C() bool {
  2344  	return fc&4 != 0
  2345  }
  2346  
  2347  // V reports whether a signed operation overflowed or underflowed.
  2348  func (fc flagConstant) V() bool {
  2349  	return fc&8 != 0
  2350  }
  2351  
  2352  func (fc flagConstant) eq() bool {
  2353  	return fc.Z()
  2354  }
  2355  func (fc flagConstant) ne() bool {
  2356  	return !fc.Z()
  2357  }
  2358  func (fc flagConstant) lt() bool {
  2359  	return fc.N() != fc.V()
  2360  }
  2361  func (fc flagConstant) le() bool {
  2362  	return fc.Z() || fc.lt()
  2363  }
  2364  func (fc flagConstant) gt() bool {
  2365  	return !fc.Z() && fc.ge()
  2366  }
  2367  func (fc flagConstant) ge() bool {
  2368  	return fc.N() == fc.V()
  2369  }
  2370  func (fc flagConstant) ult() bool {
  2371  	return !fc.C()
  2372  }
  2373  func (fc flagConstant) ule() bool {
  2374  	return fc.Z() || fc.ult()
  2375  }
  2376  func (fc flagConstant) ugt() bool {
  2377  	return !fc.Z() && fc.uge()
  2378  }
  2379  func (fc flagConstant) uge() bool {
  2380  	return fc.C()
  2381  }
  2382  
  2383  func (fc flagConstant) ltNoov() bool {
  2384  	return fc.lt() && !fc.V()
  2385  }
  2386  func (fc flagConstant) leNoov() bool {
  2387  	return fc.le() && !fc.V()
  2388  }
  2389  func (fc flagConstant) gtNoov() bool {
  2390  	return fc.gt() && !fc.V()
  2391  }
  2392  func (fc flagConstant) geNoov() bool {
  2393  	return fc.ge() && !fc.V()
  2394  }
  2395  
  2396  func (fc flagConstant) String() string {
  2397  	return fmt.Sprintf("N=%v,Z=%v,C=%v,V=%v", fc.N(), fc.Z(), fc.C(), fc.V())
  2398  }
  2399  
  2400  type flagConstantBuilder struct {
  2401  	N bool
  2402  	Z bool
  2403  	C bool
  2404  	V bool
  2405  }
  2406  
  2407  func (fcs flagConstantBuilder) encode() flagConstant {
  2408  	var fc flagConstant
  2409  	if fcs.N {
  2410  		fc |= 1
  2411  	}
  2412  	if fcs.Z {
  2413  		fc |= 2
  2414  	}
  2415  	if fcs.C {
  2416  		fc |= 4
  2417  	}
  2418  	if fcs.V {
  2419  		fc |= 8
  2420  	}
  2421  	return fc
  2422  }
  2423  
  2424  // Note: addFlags(x,y) != subFlags(x,-y) in some situations:
  2425  //  - the results of the C flag are different
  2426  //  - the results of the V flag when y==minint are different
  2427  
  2428  // addFlags64 returns the flags that would be set from computing x+y.
  2429  func addFlags64(x, y int64) flagConstant {
  2430  	var fcb flagConstantBuilder
  2431  	fcb.Z = x+y == 0
  2432  	fcb.N = x+y < 0
  2433  	fcb.C = uint64(x+y) < uint64(x)
  2434  	fcb.V = x >= 0 && y >= 0 && x+y < 0 || x < 0 && y < 0 && x+y >= 0
  2435  	return fcb.encode()
  2436  }
  2437  
  2438  // subFlags64 returns the flags that would be set from computing x-y.
  2439  func subFlags64(x, y int64) flagConstant {
  2440  	var fcb flagConstantBuilder
  2441  	fcb.Z = x-y == 0
  2442  	fcb.N = x-y < 0
  2443  	fcb.C = uint64(y) <= uint64(x) // This code follows the arm carry flag model.
  2444  	fcb.V = x >= 0 && y < 0 && x-y < 0 || x < 0 && y >= 0 && x-y >= 0
  2445  	return fcb.encode()
  2446  }
  2447  
  2448  // addFlags32 returns the flags that would be set from computing x+y.
  2449  func addFlags32(x, y int32) flagConstant {
  2450  	var fcb flagConstantBuilder
  2451  	fcb.Z = x+y == 0
  2452  	fcb.N = x+y < 0
  2453  	fcb.C = uint32(x+y) < uint32(x)
  2454  	fcb.V = x >= 0 && y >= 0 && x+y < 0 || x < 0 && y < 0 && x+y >= 0
  2455  	return fcb.encode()
  2456  }
  2457  
  2458  // subFlags32 returns the flags that would be set from computing x-y.
  2459  func subFlags32(x, y int32) flagConstant {
  2460  	var fcb flagConstantBuilder
  2461  	fcb.Z = x-y == 0
  2462  	fcb.N = x-y < 0
  2463  	fcb.C = uint32(y) <= uint32(x) // This code follows the arm carry flag model.
  2464  	fcb.V = x >= 0 && y < 0 && x-y < 0 || x < 0 && y >= 0 && x-y >= 0
  2465  	return fcb.encode()
  2466  }
  2467  
  2468  // logicFlags64 returns flags set to the sign/zeroness of x.
  2469  // C and V are set to false.
  2470  func logicFlags64(x int64) flagConstant {
  2471  	var fcb flagConstantBuilder
  2472  	fcb.Z = x == 0
  2473  	fcb.N = x < 0
  2474  	return fcb.encode()
  2475  }
  2476  
  2477  // logicFlags32 returns flags set to the sign/zeroness of x.
  2478  // C and V are set to false.
  2479  func logicFlags32(x int32) flagConstant {
  2480  	var fcb flagConstantBuilder
  2481  	fcb.Z = x == 0
  2482  	fcb.N = x < 0
  2483  	return fcb.encode()
  2484  }
  2485  
  2486  func makeJumpTableSym(b *Block) *obj.LSym {
  2487  	s := base.Ctxt.Lookup(fmt.Sprintf("%s.jump%d", b.Func.fe.Func().LSym.Name, b.ID))
  2488  	// The jump table symbol is accessed only from the function symbol.
  2489  	s.Set(obj.AttrStatic, true)
  2490  	return s
  2491  }
  2492  
  2493  // canRotate reports whether the architecture supports
  2494  // rotates of integer registers with the given number of bits.
  2495  func canRotate(c *Config, bits int64) bool {
  2496  	if bits > c.PtrSize*8 {
  2497  		// Don't rewrite to rotates bigger than the machine word.
  2498  		return false
  2499  	}
  2500  	switch c.arch {
  2501  	case "386", "amd64", "arm64", "loong64", "riscv64":
  2502  		return true
  2503  	case "arm", "s390x", "ppc64", "ppc64le", "wasm":
  2504  		return bits >= 32
  2505  	default:
  2506  		return false
  2507  	}
  2508  }
  2509  
  2510  // isARM64bitcon reports whether a constant can be encoded into a logical instruction.
  2511  func isARM64bitcon(x uint64) bool {
  2512  	if x == 1<<64-1 || x == 0 {
  2513  		return false
  2514  	}
  2515  	// determine the period and sign-extend a unit to 64 bits
  2516  	switch {
  2517  	case x != x>>32|x<<32:
  2518  		// period is 64
  2519  		// nothing to do
  2520  	case x != x>>16|x<<48:
  2521  		// period is 32
  2522  		x = uint64(int64(int32(x)))
  2523  	case x != x>>8|x<<56:
  2524  		// period is 16
  2525  		x = uint64(int64(int16(x)))
  2526  	case x != x>>4|x<<60:
  2527  		// period is 8
  2528  		x = uint64(int64(int8(x)))
  2529  	default:
  2530  		// period is 4 or 2, always true
  2531  		// 0001, 0010, 0100, 1000 -- 0001 rotate
  2532  		// 0011, 0110, 1100, 1001 -- 0011 rotate
  2533  		// 0111, 1011, 1101, 1110 -- 0111 rotate
  2534  		// 0101, 1010             -- 01   rotate, repeat
  2535  		return true
  2536  	}
  2537  	return sequenceOfOnes(x) || sequenceOfOnes(^x)
  2538  }
  2539  
  2540  // sequenceOfOnes tests whether a constant is a sequence of ones in binary, with leading and trailing zeros.
  2541  func sequenceOfOnes(x uint64) bool {
  2542  	y := x & -x // lowest set bit of x. x is good iff x+y is a power of 2
  2543  	y += x
  2544  	return (y-1)&y == 0
  2545  }
  2546  
  2547  // isARM64addcon reports whether x can be encoded as the immediate value in an ADD or SUB instruction.
  2548  func isARM64addcon(v int64) bool {
  2549  	/* uimm12 or uimm24? */
  2550  	if v < 0 {
  2551  		return false
  2552  	}
  2553  	if (v & 0xFFF) == 0 {
  2554  		v >>= 12
  2555  	}
  2556  	return v <= 0xFFF
  2557  }
  2558  
  2559  // setPos sets the position of v to pos, then returns true.
  2560  // Useful for setting the result of a rewrite's position to
  2561  // something other than the default.
  2562  func setPos(v *Value, pos src.XPos) bool {
  2563  	v.Pos = pos
  2564  	return true
  2565  }
  2566  
  2567  // isNonNegative reports whether v is known to be greater or equal to zero.
  2568  // Note that this is pretty simplistic. The prove pass generates more detailed
  2569  // nonnegative information about values.
  2570  func isNonNegative(v *Value) bool {
  2571  	if !v.Type.IsInteger() {
  2572  		v.Fatalf("isNonNegative bad type: %v", v.Type)
  2573  	}
  2574  	// TODO: return true if !v.Type.IsSigned()
  2575  	// SSA isn't type-safe enough to do that now (issue 37753).
  2576  	// The checks below depend only on the pattern of bits.
  2577  
  2578  	switch v.Op {
  2579  	case OpConst64:
  2580  		return v.AuxInt >= 0
  2581  
  2582  	case OpConst32:
  2583  		return int32(v.AuxInt) >= 0
  2584  
  2585  	case OpConst16:
  2586  		return int16(v.AuxInt) >= 0
  2587  
  2588  	case OpConst8:
  2589  		return int8(v.AuxInt) >= 0
  2590  
  2591  	case OpStringLen, OpSliceLen, OpSliceCap,
  2592  		OpZeroExt8to64, OpZeroExt16to64, OpZeroExt32to64,
  2593  		OpZeroExt8to32, OpZeroExt16to32, OpZeroExt8to16,
  2594  		OpCtz64, OpCtz32, OpCtz16, OpCtz8,
  2595  		OpCtz64NonZero, OpCtz32NonZero, OpCtz16NonZero, OpCtz8NonZero,
  2596  		OpBitLen64, OpBitLen32, OpBitLen16, OpBitLen8:
  2597  		return true
  2598  
  2599  	case OpRsh64Ux64, OpRsh32Ux64:
  2600  		by := v.Args[1]
  2601  		return by.Op == OpConst64 && by.AuxInt > 0
  2602  
  2603  	case OpRsh64x64, OpRsh32x64, OpRsh8x64, OpRsh16x64, OpRsh32x32, OpRsh64x32,
  2604  		OpSignExt32to64, OpSignExt16to64, OpSignExt8to64, OpSignExt16to32, OpSignExt8to32:
  2605  		return isNonNegative(v.Args[0])
  2606  
  2607  	case OpAnd64, OpAnd32, OpAnd16, OpAnd8:
  2608  		return isNonNegative(v.Args[0]) || isNonNegative(v.Args[1])
  2609  
  2610  	case OpMod64, OpMod32, OpMod16, OpMod8,
  2611  		OpDiv64, OpDiv32, OpDiv16, OpDiv8,
  2612  		OpOr64, OpOr32, OpOr16, OpOr8,
  2613  		OpXor64, OpXor32, OpXor16, OpXor8:
  2614  		return isNonNegative(v.Args[0]) && isNonNegative(v.Args[1])
  2615  
  2616  		// We could handle OpPhi here, but the improvements from doing
  2617  		// so are very minor, and it is neither simple nor cheap.
  2618  	}
  2619  	return false
  2620  }
  2621  
  2622  func rewriteStructLoad(v *Value) *Value {
  2623  	b := v.Block
  2624  	ptr := v.Args[0]
  2625  	mem := v.Args[1]
  2626  
  2627  	t := v.Type
  2628  	args := make([]*Value, t.NumFields())
  2629  	for i := range args {
  2630  		ft := t.FieldType(i)
  2631  		addr := b.NewValue1I(v.Pos, OpOffPtr, ft.PtrTo(), t.FieldOff(i), ptr)
  2632  		args[i] = b.NewValue2(v.Pos, OpLoad, ft, addr, mem)
  2633  	}
  2634  
  2635  	v.reset(OpStructMake)
  2636  	v.AddArgs(args...)
  2637  	return v
  2638  }
  2639  
  2640  func rewriteStructStore(v *Value) *Value {
  2641  	b := v.Block
  2642  	dst := v.Args[0]
  2643  	x := v.Args[1]
  2644  	if x.Op != OpStructMake {
  2645  		base.Fatalf("invalid struct store: %v", x)
  2646  	}
  2647  	mem := v.Args[2]
  2648  
  2649  	t := x.Type
  2650  	for i, arg := range x.Args {
  2651  		ft := t.FieldType(i)
  2652  
  2653  		addr := b.NewValue1I(v.Pos, OpOffPtr, ft.PtrTo(), t.FieldOff(i), dst)
  2654  		mem = b.NewValue3A(v.Pos, OpStore, types.TypeMem, typeToAux(ft), addr, arg, mem)
  2655  	}
  2656  
  2657  	return mem
  2658  }
  2659  
  2660  // isDirectAndComparableType reports whether v represents a type
  2661  // (a *runtime._type) whose value is stored directly in an
  2662  // interface (i.e., is pointer or pointer-like) and is comparable.
  2663  func isDirectAndComparableType(v *Value) bool {
  2664  	return isDirectAndComparableType1(v)
  2665  }
  2666  
  2667  // v is a type
  2668  func isDirectAndComparableType1(v *Value) bool {
  2669  	switch v.Op {
  2670  	case OpITab:
  2671  		return isDirectAndComparableType2(v.Args[0])
  2672  	case OpAddr:
  2673  		lsym := v.Aux.(*obj.LSym)
  2674  		if ti := lsym.TypeInfo(); ti != nil {
  2675  			t := ti.Type.(*types.Type)
  2676  			return types.IsDirectIface(t) && types.IsComparable(t)
  2677  		}
  2678  	}
  2679  	return false
  2680  }
  2681  
  2682  // v is an empty interface
  2683  func isDirectAndComparableType2(v *Value) bool {
  2684  	switch v.Op {
  2685  	case OpIMake:
  2686  		return isDirectAndComparableType1(v.Args[0])
  2687  	}
  2688  	return false
  2689  }
  2690  
  2691  // isDirectAndComparableIface reports whether v represents an itab
  2692  // (a *runtime._itab) for a type whose value is stored directly
  2693  // in an interface (i.e., is pointer or pointer-like) and is comparable.
  2694  func isDirectAndComparableIface(v *Value) bool {
  2695  	return isDirectAndComparableIface1(v, 9)
  2696  }
  2697  
  2698  // v is an itab
  2699  func isDirectAndComparableIface1(v *Value, depth int) bool {
  2700  	if depth == 0 {
  2701  		return false
  2702  	}
  2703  	switch v.Op {
  2704  	case OpITab:
  2705  		return isDirectAndComparableIface2(v.Args[0], depth-1)
  2706  	case OpAddr:
  2707  		lsym := v.Aux.(*obj.LSym)
  2708  		if ii := lsym.ItabInfo(); ii != nil {
  2709  			t := ii.Type.(*types.Type)
  2710  			return types.IsDirectIface(t) && types.IsComparable(t)
  2711  		}
  2712  	case OpConstNil:
  2713  		// We can treat this as direct, because if the itab is
  2714  		// nil, the data field must be nil also.
  2715  		return true
  2716  	}
  2717  	return false
  2718  }
  2719  
  2720  // v is an interface
  2721  func isDirectAndComparableIface2(v *Value, depth int) bool {
  2722  	if depth == 0 {
  2723  		return false
  2724  	}
  2725  	switch v.Op {
  2726  	case OpIMake:
  2727  		return isDirectAndComparableIface1(v.Args[0], depth-1)
  2728  	case OpPhi:
  2729  		for _, a := range v.Args {
  2730  			if !isDirectAndComparableIface2(a, depth-1) {
  2731  				return false
  2732  			}
  2733  		}
  2734  		return true
  2735  	}
  2736  	return false
  2737  }
  2738  
  2739  func bitsAdd64(x, y, carry int64) (r struct{ sum, carry int64 }) {
  2740  	s, c := bits.Add64(uint64(x), uint64(y), uint64(carry))
  2741  	r.sum, r.carry = int64(s), int64(c)
  2742  	return
  2743  }
  2744  
  2745  func bitsMulU64(x, y int64) (r struct{ hi, lo int64 }) {
  2746  	hi, lo := bits.Mul64(uint64(x), uint64(y))
  2747  	r.hi, r.lo = int64(hi), int64(lo)
  2748  	return
  2749  }
  2750  func bitsMulU32(x, y int32) (r struct{ hi, lo int32 }) {
  2751  	hi, lo := bits.Mul32(uint32(x), uint32(y))
  2752  	r.hi, r.lo = int32(hi), int32(lo)
  2753  	return
  2754  }
  2755  
  2756  // flagify rewrites v which is (X ...) to (Select0 (Xflags ...)).
  2757  func flagify(v *Value) bool {
  2758  	var flagVersion Op
  2759  	switch v.Op {
  2760  	case OpAMD64ADDQconst:
  2761  		flagVersion = OpAMD64ADDQconstflags
  2762  	case OpAMD64ADDLconst:
  2763  		flagVersion = OpAMD64ADDLconstflags
  2764  	default:
  2765  		base.Fatalf("can't flagify op %s", v.Op)
  2766  	}
  2767  	inner := v.copyInto(v.Block)
  2768  	inner.Op = flagVersion
  2769  	inner.Type = types.NewTuple(v.Type, types.TypeFlags)
  2770  	v.reset(OpSelect0)
  2771  	v.AddArg(inner)
  2772  	return true
  2773  }
  2774  
  2775  // PanicBoundsC contains a constant for a bounds failure.
  2776  type PanicBoundsC struct {
  2777  	C int64
  2778  }
  2779  
  2780  // PanicBoundsCC contains 2 constants for a bounds failure.
  2781  type PanicBoundsCC struct {
  2782  	Cx int64
  2783  	Cy int64
  2784  }
  2785  
  2786  func (p PanicBoundsC) CanBeAnSSAAux() {
  2787  }
  2788  func (p PanicBoundsCC) CanBeAnSSAAux() {
  2789  }
  2790  
  2791  func auxToPanicBoundsC(i Aux) PanicBoundsC {
  2792  	return i.(PanicBoundsC)
  2793  }
  2794  func auxToPanicBoundsCC(i Aux) PanicBoundsCC {
  2795  	return i.(PanicBoundsCC)
  2796  }
  2797  func panicBoundsCToAux(p PanicBoundsC) Aux {
  2798  	return p
  2799  }
  2800  func panicBoundsCCToAux(p PanicBoundsCC) Aux {
  2801  	return p
  2802  }
  2803  
  2804  func isDictArgSym(sym Sym) bool {
  2805  	return sym.(*ir.Name).Sym().Name == typecheck.LocalDictName
  2806  }
  2807  
  2808  // When v is (IMake typ (StructMake ...)), convert to
  2809  // (IMake typ arg) where arg is the pointer-y argument to
  2810  // the StructMake (there must be exactly one).
  2811  func imakeOfStructMake(v *Value) *Value {
  2812  	var arg *Value
  2813  	for _, a := range v.Args[1].Args {
  2814  		if a.Type.Size() > 0 {
  2815  			arg = a
  2816  			break
  2817  		}
  2818  	}
  2819  	return v.Block.NewValue2(v.Pos, OpIMake, v.Type, v.Args[0], arg)
  2820  }
  2821  
  2822  // bool2int converts bool to int: true to 1, false to 0
  2823  func bool2int(x bool) int {
  2824  	var b int
  2825  	if x {
  2826  		b = 1
  2827  	}
  2828  	return b
  2829  }
  2830  
  2831  // rewriteCondSelectIntoMath reports whether x OP (y * constant) should be used instead of a CondSelect.
  2832  // x arbitrary, y in [0,1]
  2833  func rewriteCondSelectIntoMath(config *Config, op Op, constant int64) bool {
  2834  	switch config.arch {
  2835  	case "amd64":
  2836  		if constant == 1 {
  2837  			return true
  2838  		}
  2839  		switch op {
  2840  		case OpAdd64, OpAdd32, OpAdd16, OpAdd8:
  2841  			switch constant {
  2842  			case 2, 4, 8:
  2843  				// Implemented with LEA a + b * displacement form
  2844  				return true
  2845  			}
  2846  		}
  2847  	case "arm64":
  2848  		switch op {
  2849  		case OpAdd64, OpAdd32, OpAdd16, OpAdd8:
  2850  			if constant == 1 {
  2851  				return false // better done as CSINC
  2852  			}
  2853  			fallthrough
  2854  		case OpSub64, OpSub32, OpSub16, OpSub8,
  2855  			OpAnd64, OpAnd32, OpAnd16, OpAnd8,
  2856  			OpOr64, OpOr32, OpOr16, OpOr8,
  2857  			OpXor64, OpXor32, OpXor16, OpXor8:
  2858  			// Implemented using an inline LSL
  2859  			return isPowerOfTwo(uint64(constant))
  2860  		default:
  2861  			if constant == 1 {
  2862  				return true
  2863  			}
  2864  		}
  2865  	default:
  2866  		// TODO: fine tune for other architectures.
  2867  		return constant == 1
  2868  	}
  2869  	return false
  2870  }
  2871  
  2872  func addToSub(op Op) Op {
  2873  	switch op {
  2874  	case OpAdd64:
  2875  		return OpSub64
  2876  	case OpAdd32:
  2877  		return OpSub32
  2878  	case OpAdd16:
  2879  		return OpSub16
  2880  	case OpAdd8:
  2881  		return OpSub8
  2882  	default:
  2883  		panic(fmt.Sprintf("unexpected op %v", op))
  2884  	}
  2885  }
  2886  
  2887  func modularMultiplicativeInverse(x uint64) (y uint64) {
  2888  	if x%2 != 1 {
  2889  		panic("even numbers in a power-of-two modulus do not have a multiplicative inverse")
  2890  	}
  2891  	// we start with 3 bits of precision because each odd number is its own multiplicative inverse mod 8
  2892  	y = x // 3 bits
  2893  
  2894  	// now use the Newton-Raphson method to double the number of correct bits in each iteration.
  2895  	y *= 2 - x*y // 6 bits
  2896  	y *= 2 - x*y // 12 bits
  2897  	y *= 2 - x*y // 24 bits
  2898  	y *= 2 - x*y // 48 bits
  2899  	y *= 2 - x*y // 96 bits; good enough
  2900  	return
  2901  }
  2902  

View as plain text