main.go

     1  // Copyright 2025 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // simdgen is an experiment in generating Go <-> asm SIMD mappings.
     6  //
     7  // Usage: simdgen [-xedPath=path | -arm64Path=path] [-q=query] input.yaml...
     8  //
     9  // Only one of -xedPath or -arm64Path may be specified.
    10  //
    11  // If -xedPath is provided, one of the inputs is a sum of op-code definitions
    12  // generated from the Intel XED data at path.
    13  //
    14  // If -arm64Path is provided, one of the inputs is a set of instruction
    15  // definitions parsed from ARM64 ISA XML files at path (obtained from
    16  // https://developer.arm.com/-/cdn-downloads/permalink/Exploration-Tools-A64-ISA/ISA_A64/ISA_A64_xml_A_profile-2025-12.tar.gz).
    17  //
    18  // If input YAML files are provided, each file is read as an input value. See
    19  // [unify.Closure.UnmarshalYAML] or "go doc unify.Closure.UnmarshalYAML" for the
    20  // format of these files.
    21  //
    22  // TODO: Example definitions and values.
    23  //
    24  // The command unifies across all of the inputs and prints all possible results
    25  // of this unification.
    26  //
    27  // If the -q flag is provided, its string value is parsed as a value and treated
    28  // as another input to unification. This is intended as a way to "query" the
    29  // result, typically by narrowing it down to a small subset of results.
    30  //
    31  // Typical usage:
    32  //
    33  //	go run . -xedPath $XEDPATH *.yaml
    34  //
    35  // To see just the definitions generated from XED, run:
    36  //
    37  //	go run . -xedPath $XEDPATH
    38  //
    39  // (This works because if there's only one input, there's nothing to unify it
    40  // with, so the result is simply itself.)
    41  //
    42  // To see just the definitions for VPADDQ:
    43  //
    44  //	go run . -xedPath $XEDPATH -q '{asm: VPADDQ}'
    45  //
    46  // For VADD.S4 on ARM64:
    47  //
    48  //	go run . -arm64Path $ARM64_ISA_PATH -o yaml -q '{asm: VADD, arrangement: "4S"}'
    49  //
    50  // simdgen can also generate Go definitions of SIMD mappings:
    51  // To generate go files to the go root, run:
    52  //
    53  //	go run . -xedPath $XEDPATH -o godefs -goroot $PATH/TO/go go_amd64.yaml categories.yaml types.yaml
    54  //
    55  // For ARM64:
    56  //
    57  //	go run . -arm64Path $ARM64_ISA_PATH -o godefs -goroot $PATH/TO/go go_arm64.yaml categories.yaml types.yaml
    58  //
    59  // types.yaml is already written, it specifies the shapes of vectors.
    60  // categories.yaml and go_<arch>.yaml contain definitions that unify with types.yaml and
    61  // XED/ARM64 ISA data, you can find an example in ops/AddSub/.
    62  //
    63  // When generating Go definitions, simdgen do 3 "magic"s:
    64  // - It splits masked operations(with op's [Masked] field set) to const and non const:
    65  //   - One is a normal masked operation, the original
    66  //   - The other has its mask operand's [Const] fields set to "K0".
    67  //   - This way the user does not need to provide a separate "K0"-masked operation def.
    68  //
    69  // - It deduplicates intrinsic names that have duplicates:
    70  //   - If there are two operations that shares the same signature, one is AVX512 the other
    71  //     is before AVX512, the other will be selected.
    72  //   - This happens often when some operations are defined both before AVX512 and after.
    73  //     This way the user does not need to provide a separate "K0" operation for the
    74  //     AVX512 counterpart.
    75  //
    76  // - It copies the op's [ConstImm] field to its immediate operand's [Const] field.
    77  //   - This way the user does not need to provide verbose op definition while only
    78  //     the const immediate field is different. This is useful to reduce verbosity of
    79  //     compares with imm control predicates.
    80  //
    81  // These 3 magics could be disabled by enabling -nosplitmask, -nodedup or
    82  // -noconstimmporting flags.
    83  //
    84  // simdgen supports amd64 and arm64 architectures.
    85  package main
    86  
    87  // Big TODOs:
    88  //
    89  // - This can produce duplicates, which can also lead to less efficient
    90  // environment merging. Add hashing and use it for deduplication. Be careful
    91  // about how this shows up in debug traces, since it could make things
    92  // confusing if we don't show it happening.
    93  //
    94  // - Do I need Closure, Value, and Domain? It feels like I should only need two
    95  // types.
    96  
    97  import (
    98  	"cmp"
    99  	"flag"
   100  	"fmt"
   101  	"log"
   102  	"maps"
   103  	"os"
   104  	"path/filepath"
   105  	"runtime/pprof"
   106  	"slices"
   107  	"strings"
   108  
   109  	"simd/archsimd/_gen/simdgen/arm64"
   110  	"simd/archsimd/_gen/unify"
   111  
   112  	"gopkg.in/yaml.v3"
   113  )
   114  
   115  var (
   116  	xedPath               = flag.String("xedPath", "", "load XED datafiles from `path`")
   117  	arm64Path             = flag.String("arm64Path", "", "load ARM64 instruction xml definitions from `path`")
   118  	flagQ                 = flag.String("q", "", "query: read `def` as another input (skips final validation)")
   119  	flagO                 = flag.String("o", "yaml", "output type: yaml, godefs (generate definitions into a Go source tree")
   120  	flagGoDefRoot         = flag.String("goroot", ".", "the path to the Go dev directory that will receive the generated files")
   121  	FlagNoDedup           = flag.Bool("nodedup", false, "disable deduplicating godefs of 2 qualifying operations from different extensions")
   122  	FlagNoConstImmPorting = flag.Bool("noconstimmporting", false, "disable const immediate porting from op to imm operand")
   123  
   124  	// FlagArch must be pre-initialized to a bogus value because there have been initializations that depended on it
   125  	FlagArch = flag.String("arch", "must be specified, amd64 or arm64", "the target architecture")
   126  
   127  	Verbose = flag.Bool("v", false, "verbose")
   128  
   129  	flagDebugXED   = flag.Bool("debug-xed", false, "show XED instructions")
   130  	flagDebugUnify = flag.Bool("debug-unify", false, "print unification trace")
   131  	flagDebugHTML  = flag.String("debug-html", "", "write unification trace to `file.html`")
   132  	FlagReportDup  = flag.Bool("reportdup", false, "report the duplicate godefs")
   133  
   134  	flagCPUProfile = flag.String("cpuprofile", "", "write CPU profile to `file`")
   135  	flagMemProfile = flag.String("memprofile", "", "write memory profile to `file`")
   136  )
   137  
   138  const simdPackage = "simd/archsimd"
   139  
   140  func main() {
   141  	flag.Parse()
   142  
   143  	if *flagCPUProfile != "" {
   144  		f, err := os.Create(*flagCPUProfile)
   145  		if err != nil {
   146  			log.Fatalf("-cpuprofile: %s", err)
   147  		}
   148  		defer f.Close()
   149  		pprof.StartCPUProfile(f)
   150  		defer pprof.StopCPUProfile()
   151  	}
   152  	if *flagMemProfile != "" {
   153  		f, err := os.Create(*flagMemProfile)
   154  		if err != nil {
   155  			log.Fatalf("-memprofile: %s", err)
   156  		}
   157  		defer func() {
   158  			pprof.WriteHeapProfile(f)
   159  			f.Close()
   160  		}()
   161  	}
   162  
   163  	// Default -arch to arm64 when -arm64Path is specified.
   164  	if *arm64Path != "" && *FlagArch != "arm64" {
   165  		if *xedPath != "" {
   166  			log.Fatalf("both -xedPath and -arm64Path specified")
   167  		}
   168  		// *FlagArch = "arm64"
   169  	}
   170  
   171  	// Load instructions into the architecture-specific defs set.
   172  	var defs []*unify.Value
   173  	switch *FlagArch {
   174  	case "amd64":
   175  		if *xedPath != "" {
   176  			defs = loadXED(*xedPath)
   177  		}
   178  	case "arm64":
   179  		if *arm64Path != "" {
   180  			var err error
   181  			defs, err = arm64.Load(*arm64Path)
   182  			if err != nil {
   183  				log.Fatalf("loading ARM64 instructions: %s", err)
   184  			}
   185  		}
   186  	default:
   187  		log.Fatalf("simdgen only supports amd64 and arm64")
   188  	}
   189  
   190  	var inputs []unify.Closure
   191  	inputs = append(inputs, unify.NewSum(defs...))
   192  
   193  	// Load query.
   194  	if *flagQ != "" {
   195  		r := strings.NewReader(*flagQ)
   196  		def, err := unify.Read(r, "<query>", unify.ReadOpts{})
   197  		if err != nil {
   198  			log.Fatalf("parsing -q: %s", err)
   199  		}
   200  		inputs = append(inputs, def)
   201  	}
   202  
   203  	// Load defs files.
   204  	must := make(map[*unify.Value]struct{})
   205  	for _, path := range flag.Args() {
   206  		defs, err := unify.ReadFile(path, unify.ReadOpts{})
   207  		if err != nil {
   208  			log.Fatal(err)
   209  		}
   210  		inputs = append(inputs, defs)
   211  
   212  		base := filepath.Base(path)
   213  		if base == "go_amd64.yaml" || base == "go_arm64.yaml" {
   214  			// These must all be used in the final result
   215  			for def := range defs.Summands() {
   216  				must[def] = struct{}{}
   217  			}
   218  		}
   219  	}
   220  
   221  	// Prepare for unification
   222  	if *flagDebugUnify {
   223  		unify.Debug.UnifyLog = os.Stderr
   224  	}
   225  	if *flagDebugHTML != "" {
   226  		f, err := os.Create(*flagDebugHTML)
   227  		if err != nil {
   228  			log.Fatal(err)
   229  		}
   230  		unify.Debug.HTML = f
   231  		defer f.Close()
   232  	}
   233  
   234  	// Unify!
   235  	unified, err := unify.Unify(inputs...)
   236  	if err != nil {
   237  		log.Fatal(err)
   238  	}
   239  
   240  	ok := true
   241  
   242  	// Validate results.
   243  	//
   244  	// Don't validate if this is a command-line query because that tends to
   245  	// eliminate lots of required defs and is used in cases where maybe defs
   246  	// aren't enumerable anyway.
   247  	if *flagQ == "" && len(must) > 0 {
   248  		ok = validate(unified, must)
   249  	}
   250  
   251  	// Print results.
   252  	switch *flagO {
   253  	case "yaml":
   254  		// Produce a result that looks like encoding a slice, but stream it.
   255  		fmt.Println("!sum")
   256  		var val1 [1]*unify.Value
   257  		for val := range unified.All() {
   258  			val1[0] = val
   259  			// We have to make a new encoder each time or it'll print a document
   260  			// separator between each object.
   261  			enc := yaml.NewEncoder(os.Stdout)
   262  			if err := enc.Encode(val1); err != nil {
   263  				log.Fatal(err)
   264  			}
   265  			enc.Close()
   266  		}
   267  	case "godefs":
   268  		if err := writeGoDefs(*flagGoDefRoot, unified); err != nil {
   269  			log.Fatalf("Failed writing godefs: %+v", err)
   270  		}
   271  	}
   272  
   273  	if !*Verbose && *xedPath != "" {
   274  		if operandRemarks == 0 {
   275  			fmt.Fprintf(os.Stderr, "XED decoding generated no errors, which is unusual.\n")
   276  		} else {
   277  			fmt.Fprintf(os.Stderr, "XED decoding generated %d \"errors\" which is not cause for alarm, use -v for details.\n", operandRemarks)
   278  		}
   279  	}
   280  	if !ok {
   281  		os.Exit(1)
   282  	}
   283  }
   284  
   285  func validate(cl unify.Closure, required map[*unify.Value]struct{}) bool {
   286  	ok := true
   287  	// Validate that:
   288  	// 1. All final defs are exact
   289  	// 2. All required defs are used
   290  	for def := range cl.All() {
   291  		if _, ok := def.Domain.(unify.Def); !ok {
   292  			fmt.Fprintf(os.Stderr, "%s: expected Def, got %T\n", def.PosString(), def.Domain)
   293  			continue
   294  		}
   295  
   296  		if !def.Exact() {
   297  			fmt.Fprintf(os.Stderr, "%s: def not reduced to an exact value, why is %s:\n", def.PosString(), def.WhyNotExact())
   298  			fmt.Fprintf(os.Stderr, "\t%s\n", strings.ReplaceAll(def.String(), "\n", "\n\t"))
   299  		}
   300  
   301  		for root := range def.Provenance() {
   302  			delete(required, root)
   303  		}
   304  	}
   305  	// Report unused defs
   306  	unused := slices.SortedFunc(maps.Keys(required),
   307  		func(a, b *unify.Value) int {
   308  			return cmp.Or(
   309  				cmp.Compare(a.Pos().Path, b.Pos().Path),
   310  				cmp.Compare(a.Pos().Line, b.Pos().Line),
   311  			)
   312  		})
   313  	for _, def := range unused {
   314  		// TODO: Can we say anything more actionable? This is always a problem
   315  		// with unification: if it fails, it's very hard to point a finger at
   316  		// any particular reason. We could go back and try unifying this again
   317  		// with each subset of the inputs (starting with individual inputs) to
   318  		// at least say "it doesn't unify with anything in x.yaml". That's a lot
   319  		// of work, but if we have trouble debugging unification failure it may
   320  		// be worth it.
   321  		fmt.Fprintf(os.Stderr, "%s: def required, but did not unify (%v)\n",
   322  			def.PosString(), def)
   323  		ok = false
   324  	}
   325  	return ok
   326  }
   327
View as plain text