diff --git a/src/bytes/bytes_test.go b/src/bytes/bytes_test.go index 20d333217e81e2..d821d2f1ca5e5b 100644 --- a/src/bytes/bytes_test.go +++ b/src/bytes/bytes_test.go @@ -762,7 +762,7 @@ func BenchmarkEqual(b *testing.B) { } }) - sizes := []int{1, 6, 9, 15, 16, 20, 32, 4 << 10, 4 << 20, 64 << 20} + sizes := []int{1, 6, 9, 15, 16, 20, 32, 65, 96, 100, 127, 4 << 10, 4 << 20, 64 << 20} b.Run("same", func(b *testing.B) { benchBytes(b, sizes, bmEqual(func(a, b []byte) bool { return Equal(a, a) })) diff --git a/src/cmd/compile/internal/noder/reader.go b/src/cmd/compile/internal/noder/reader.go index fab311e7e7a48d..d088f24ec5cf82 100644 --- a/src/cmd/compile/internal/noder/reader.go +++ b/src/cmd/compile/internal/noder/reader.go @@ -11,6 +11,7 @@ import ( "internal/buildcfg" "internal/pkgbits" "path/filepath" + "slices" "strings" "cmd/compile/internal/base" @@ -3542,6 +3543,51 @@ func (r *reader) pkgInitOrder(target *ir.Package) { typecheck.DeclFunc(fn) r.curfn = fn + var varInitFns []*ir.Func + if len(initOrder) <= maxInitStatements { + fn.Body = r.doPkgInitOrder(initOrder) + } else { + varInitFns = r.splitLargeInitOrder(initOrder) + calls := make([]ir.Node, len(varInitFns)) + for i, varInitFn := range varInitFns { + ir.WithFunc(fn, func() { + calls[i] = typecheck.Call(varInitFn.Pos(), varInitFn.Nname, nil, false) + }) + } + fn.Body = calls + } + + typecheck.FinishFuncBody() + r.curfn = nil + r.locals = nil + + // Outline (if legal/profitable) global map inits. + staticinit.OutlineMapInits(fn) + for _, varInitFn := range varInitFns { + staticinit.OutlineMapInits(varInitFn) + } + + target.Inits = append(target.Inits, fn) + target.Inits = append(target.Inits, varInitFns...) +} + +const maxInitStatements = 1000 + +func (r *reader) generateVarInitFunc(body []ir.Node) *ir.Func { + fn := staticinit.GenerateVarInitFunc() + typecheck.DeclFunc(fn) + + old := r.curfn + r.curfn = fn + fn.Body = r.doPkgInitOrder(body) + r.curfn = old + + typecheck.FinishFuncBody() + + return fn +} + +func (r *reader) doPkgInitOrder(initOrder []ir.Node) []ir.Node { for i := range initOrder { lhs := make([]ir.Node, r.Len()) for j := range lhs { @@ -3563,17 +3609,15 @@ func (r *reader) pkgInitOrder(target *ir.Package) { initOrder[i] = as } + return initOrder +} - fn.Body = initOrder - - typecheck.FinishFuncBody() - r.curfn = nil - r.locals = nil - - // Outline (if legal/profitable) global map inits. - staticinit.OutlineMapInits(fn) - - target.Inits = append(target.Inits, fn) +func (r *reader) splitLargeInitOrder(initOrder []ir.Node) []*ir.Func { + var initFuncs []*ir.Func + for chunk := range slices.Chunk(initOrder, maxInitStatements) { + initFuncs = append(initFuncs, r.generateVarInitFunc(chunk)) + } + return initFuncs } func (r *reader) pkgDecls(target *ir.Package) { diff --git a/src/cmd/compile/internal/pkginit/init.go b/src/cmd/compile/internal/pkginit/init.go index 04f6a5d3509f85..9be15860a664da 100644 --- a/src/cmd/compile/internal/pkginit/init.go +++ b/src/cmd/compile/internal/pkginit/init.go @@ -87,10 +87,7 @@ func MakeTask() { // Record user init functions. for _, fn := range typecheck.Target.Inits { - if fn.Sym().Name == "init" { - // Synthetic init function for initialization of package-scope - // variables. We can use staticinit to optimize away static - // assignments. + if staticinit.CanOptimize(fn) { s := staticinit.Schedule{ Plans: make(map[ir.Node]*staticinit.Plan), Temps: make(map[ir.Node]*ir.Name), diff --git a/src/cmd/compile/internal/ssa/_gen/generic.rules b/src/cmd/compile/internal/ssa/_gen/generic.rules index 80e7c8e9b6ead3..fc7169eccd5d8b 100644 --- a/src/cmd/compile/internal/ssa/_gen/generic.rules +++ b/src/cmd/compile/internal/ssa/_gen/generic.rules @@ -74,6 +74,13 @@ (PopCount32 (Const32 [c])) && config.PtrSize == 4 => (Const32 [int32(bits.OnesCount32(uint32(c)))]) (PopCount16 (Const16 [c])) && config.PtrSize == 4 => (Const32 [int32(bits.OnesCount16(uint16(c)))]) (PopCount8 (Const8 [c])) && config.PtrSize == 4 => (Const32 [int32(bits.OnesCount8(uint8(c)))]) +(Bswap64 (Const64 [c])) => (Const64 [int64(bits.ReverseBytes64(uint64(c)))]) +(Bswap32 (Const32 [c])) => (Const32 [int32(bits.ReverseBytes32(uint32(c)))]) +(Bswap16 (Const16 [c])) => (Const16 [int16(bits.ReverseBytes16(uint16(c)))]) +(BitRev64 (Const64 [c])) => (Const64 [int64(bits.Reverse64(uint64(c)))]) +(BitRev32 (Const32 [c])) => (Const32 [int32(bits.Reverse32(uint32(c)))]) +(BitRev16 (Const16 [c])) => (Const16 [int16(bits.Reverse16(uint16(c)))]) +(BitRev8 (Const8 [c])) => (Const8 [int8(bits.Reverse8(uint8(c)))]) (Add64carry (Const64 [x]) (Const64 [y]) (Const64 [c])) && c >= 0 && c <= 1 => (MakeTuple (Const64 [bitsAdd64(x, y, c).sum]) (Const64 [bitsAdd64(x, y, c).carry])) (Trunc16to8 (ZeroExt8to16 x)) => x @@ -144,6 +151,13 @@ (Mul32uover (Const32 [c]) (Const32 [d])) => (MakeTuple (Const32 [bitsMulU32(c, d).lo]) (ConstBool [bitsMulU32(c,d).hi != 0])) (Mul64uover (Const64 [c]) (Const64 [d])) => (MakeTuple (Const64 [bitsMulU64(c, d).lo]) (ConstBool [bitsMulU64(c,d).hi != 0])) +// bits.Mul64(x, 1<> (64-s), x << s) +(Mul64uhilo x (Const64 [c])) && c > 0 && isPowerOfTwo(uint64(c)) => + (MakeTuple + (Rsh64Ux64 x (Const64 [64 - log64u(uint64(c))])) + (Lsh64x64 x (Const64 [log64u(uint64(c))]))) + (And8 (Const8 [c]) (Const8 [d])) => (Const8 [c&d]) (And16 (Const16 [c]) (Const16 [d])) => (Const16 [c&d]) (And32 (Const32 [c]) (Const32 [d])) => (Const32 [c&d]) @@ -260,12 +274,12 @@ (IsInBounds (ZeroExt8to64 (Rsh8Ux64 _ (Const64 [c]))) (Const64 [d])) && 0 < c && c < 8 && 1< (ConstBool [true]) (IsInBounds (ZeroExt8to32 (Rsh8Ux64 _ (Const64 [c]))) (Const32 [d])) && 0 < c && c < 8 && 1< (ConstBool [true]) (IsInBounds (ZeroExt8to16 (Rsh8Ux64 _ (Const64 [c]))) (Const16 [d])) && 0 < c && c < 8 && 1< (ConstBool [true]) -(IsInBounds (Rsh8Ux64 _ (Const64 [c])) (Const64 [d])) && 0 < c && c < 8 && 1< (ConstBool [true]) +(IsInBounds (Rsh8Ux64 _ (Const64 [c])) (Const8 [d])) && 0 < c && c < 8 && 1< (ConstBool [true]) (IsInBounds (ZeroExt16to64 (Rsh16Ux64 _ (Const64 [c]))) (Const64 [d])) && 0 < c && c < 16 && 1< (ConstBool [true]) -(IsInBounds (ZeroExt16to32 (Rsh16Ux64 _ (Const64 [c]))) (Const64 [d])) && 0 < c && c < 16 && 1< (ConstBool [true]) -(IsInBounds (Rsh16Ux64 _ (Const64 [c])) (Const64 [d])) && 0 < c && c < 16 && 1< (ConstBool [true]) +(IsInBounds (ZeroExt16to32 (Rsh16Ux64 _ (Const64 [c]))) (Const32 [d])) && 0 < c && c < 16 && 1< (ConstBool [true]) +(IsInBounds (Rsh16Ux64 _ (Const64 [c])) (Const16 [d])) && 0 < c && c < 16 && 1< (ConstBool [true]) (IsInBounds (ZeroExt32to64 (Rsh32Ux64 _ (Const64 [c]))) (Const64 [d])) && 0 < c && c < 32 && 1< (ConstBool [true]) -(IsInBounds (Rsh32Ux64 _ (Const64 [c])) (Const64 [d])) && 0 < c && c < 32 && 1< (ConstBool [true]) +(IsInBounds (Rsh32Ux64 _ (Const64 [c])) (Const32 [d])) && 0 < c && c < 32 && 1< (ConstBool [true]) (IsInBounds (Rsh64Ux64 _ (Const64 [c])) (Const64 [d])) && 0 < c && c < 64 && 1< (ConstBool [true]) (IsSliceInBounds x x) => (ConstBool [true]) @@ -2408,3 +2422,11 @@ // Canonicalize sext(int(bool)) => zext(int(bool)) (SignExt8to(64|32|16) cvt:(CvtBoolToUint8 bool)) => (ZeroExt8to(64|32|16) cvt) + +// int(bool)^1 => int(!bool) +(Xor8 (CvtBoolToUint8 bool) (Const8 [1])) && invertibleBool(bool.Op) => (CvtBoolToUint8 (Not bool)) +(Xor(64|32|16) (ZeroExt8to(64|32|16) (CvtBoolToUint8 bool)) (Const(64|32|16) [1])) && invertibleBool(bool.Op) => (ZeroExt8to(64|32|16) (CvtBoolToUint8 (Not bool))) + +// int(!bool)^c => int(bool)^(c^1) +(Xor8 (CvtBoolToUint8 (Not bool)) (Const8 [c])) && c != 1 => (Xor8 (CvtBoolToUint8 bool) (Const8 [c^1])) +(Xor(64|32|16) (ZeroExt8to(64|32|16) (CvtBoolToUint8 (Not bool))) (Const(64|32|16) [c])) && c != 1 => (Xor(64|32|16) (ZeroExt8to(64|32|16) (CvtBoolToUint8 bool)) (Const(64|32|16) [c^1])) diff --git a/src/cmd/compile/internal/ssa/memcombine.go b/src/cmd/compile/internal/ssa/memcombine.go index 32b9b7a290a9ca..288b7f145ab423 100644 --- a/src/cmd/compile/internal/ssa/memcombine.go +++ b/src/cmd/compile/internal/ssa/memcombine.go @@ -90,7 +90,54 @@ func memcombineLoads(f *Func) { // idx may be nil, in which case it is treated as 0. type BaseAddress struct { ptr *Value - idx *Value + idx Index +} + +// Index represents an address index in the form exp< 0 && isPowerOfTwo(uint64(c)) + // result: (MakeTuple (Rsh64Ux64 x (Const64 [64 - log64u(uint64(c))])) (Lsh64x64 x (Const64 [log64u(uint64(c))]))) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + x := v_0 + if v_1.Op != OpConst64 { + continue + } + c := auxIntToInt64(v_1.AuxInt) + if !(c > 0 && isPowerOfTwo(uint64(c))) { + continue + } + v.reset(OpMakeTuple) + v0 := b.NewValue0(v.Pos, OpRsh64Ux64, typ.UInt64) + v1 := b.NewValue0(v.Pos, OpConst64, typ.UInt64) + v1.AuxInt = int64ToAuxInt(64 - log64u(uint64(c))) + v0.AddArg2(x, v1) + v2 := b.NewValue0(v.Pos, OpLsh64x64, typ.UInt64) + v3 := b.NewValue0(v.Pos, OpConst64, typ.UInt64) + v3.AuxInt = int64ToAuxInt(log64u(uint64(c))) + v2.AddArg2(x, v3) + v.AddArg2(v0, v2) + return true + } + break + } return false } func rewriteValuegeneric_OpMul64uover(v *Value) bool { @@ -37580,6 +37726,7 @@ func rewriteValuegeneric_OpXor16(v *Value) bool { v_0 := v.Args[0] b := v.Block config := b.Func.Config + typ := &b.Func.Config.Types // match: (Xor16 (Const16 [c]) (Const16 [d])) // result: (Const16 [c^d]) for { @@ -38044,6 +38191,71 @@ func rewriteValuegeneric_OpXor16(v *Value) bool { } break } + // match: (Xor16 (ZeroExt8to16 (CvtBoolToUint8 bool)) (Const16 [1])) + // cond: invertibleBool(bool.Op) + // result: (ZeroExt8to16 (CvtBoolToUint8 (Not bool))) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpZeroExt8to16 { + continue + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpCvtBoolToUint8 { + continue + } + cvtT := v_0_0.Type + bool := v_0_0.Args[0] + if v_1.Op != OpConst16 || auxIntToInt16(v_1.AuxInt) != 1 || !(invertibleBool(bool.Op)) { + continue + } + v.reset(OpZeroExt8to16) + v0 := b.NewValue0(v.Pos, OpCvtBoolToUint8, cvtT) + v1 := b.NewValue0(v.Pos, OpNot, bool.Type) + v1.AddArg(bool) + v0.AddArg(v1) + v.AddArg(v0) + return true + } + break + } + // match: (Xor16 (ZeroExt8to16 (CvtBoolToUint8 (Not bool))) (Const16 [c])) + // cond: c != 1 + // result: (Xor16 (ZeroExt8to16 (CvtBoolToUint8 bool)) (Const16 [c^1])) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpZeroExt8to16 { + continue + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpCvtBoolToUint8 { + continue + } + cvtT := v_0_0.Type + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpNot { + continue + } + bool := v_0_0_0.Args[0] + if v_1.Op != OpConst16 { + continue + } + constT := v_1.Type + c := auxIntToInt16(v_1.AuxInt) + if !(c != 1) { + continue + } + v.reset(OpXor16) + v0 := b.NewValue0(v.Pos, OpZeroExt8to16, typ.UInt16) + v1 := b.NewValue0(v.Pos, OpCvtBoolToUint8, cvtT) + v1.AddArg(bool) + v0.AddArg(v1) + v2 := b.NewValue0(v.Pos, OpConst16, constT) + v2.AuxInt = int16ToAuxInt(c ^ 1) + v.AddArg2(v0, v2) + return true + } + break + } return false } func rewriteValuegeneric_OpXor32(v *Value) bool { @@ -38051,6 +38263,7 @@ func rewriteValuegeneric_OpXor32(v *Value) bool { v_0 := v.Args[0] b := v.Block config := b.Func.Config + typ := &b.Func.Config.Types // match: (Xor32 (Const32 [c]) (Const32 [d])) // result: (Const32 [c^d]) for { @@ -38515,6 +38728,71 @@ func rewriteValuegeneric_OpXor32(v *Value) bool { } break } + // match: (Xor32 (ZeroExt8to32 (CvtBoolToUint8 bool)) (Const32 [1])) + // cond: invertibleBool(bool.Op) + // result: (ZeroExt8to32 (CvtBoolToUint8 (Not bool))) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpZeroExt8to32 { + continue + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpCvtBoolToUint8 { + continue + } + cvtT := v_0_0.Type + bool := v_0_0.Args[0] + if v_1.Op != OpConst32 || auxIntToInt32(v_1.AuxInt) != 1 || !(invertibleBool(bool.Op)) { + continue + } + v.reset(OpZeroExt8to32) + v0 := b.NewValue0(v.Pos, OpCvtBoolToUint8, cvtT) + v1 := b.NewValue0(v.Pos, OpNot, bool.Type) + v1.AddArg(bool) + v0.AddArg(v1) + v.AddArg(v0) + return true + } + break + } + // match: (Xor32 (ZeroExt8to32 (CvtBoolToUint8 (Not bool))) (Const32 [c])) + // cond: c != 1 + // result: (Xor32 (ZeroExt8to32 (CvtBoolToUint8 bool)) (Const32 [c^1])) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpZeroExt8to32 { + continue + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpCvtBoolToUint8 { + continue + } + cvtT := v_0_0.Type + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpNot { + continue + } + bool := v_0_0_0.Args[0] + if v_1.Op != OpConst32 { + continue + } + constT := v_1.Type + c := auxIntToInt32(v_1.AuxInt) + if !(c != 1) { + continue + } + v.reset(OpXor32) + v0 := b.NewValue0(v.Pos, OpZeroExt8to32, typ.UInt32) + v1 := b.NewValue0(v.Pos, OpCvtBoolToUint8, cvtT) + v1.AddArg(bool) + v0.AddArg(v1) + v2 := b.NewValue0(v.Pos, OpConst32, constT) + v2.AuxInt = int32ToAuxInt(c ^ 1) + v.AddArg2(v0, v2) + return true + } + break + } return false } func rewriteValuegeneric_OpXor64(v *Value) bool { @@ -38522,6 +38800,7 @@ func rewriteValuegeneric_OpXor64(v *Value) bool { v_0 := v.Args[0] b := v.Block config := b.Func.Config + typ := &b.Func.Config.Types // match: (Xor64 (Const64 [c]) (Const64 [d])) // result: (Const64 [c^d]) for { @@ -38986,6 +39265,71 @@ func rewriteValuegeneric_OpXor64(v *Value) bool { } break } + // match: (Xor64 (ZeroExt8to64 (CvtBoolToUint8 bool)) (Const64 [1])) + // cond: invertibleBool(bool.Op) + // result: (ZeroExt8to64 (CvtBoolToUint8 (Not bool))) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpZeroExt8to64 { + continue + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpCvtBoolToUint8 { + continue + } + cvtT := v_0_0.Type + bool := v_0_0.Args[0] + if v_1.Op != OpConst64 || auxIntToInt64(v_1.AuxInt) != 1 || !(invertibleBool(bool.Op)) { + continue + } + v.reset(OpZeroExt8to64) + v0 := b.NewValue0(v.Pos, OpCvtBoolToUint8, cvtT) + v1 := b.NewValue0(v.Pos, OpNot, bool.Type) + v1.AddArg(bool) + v0.AddArg(v1) + v.AddArg(v0) + return true + } + break + } + // match: (Xor64 (ZeroExt8to64 (CvtBoolToUint8 (Not bool))) (Const64 [c])) + // cond: c != 1 + // result: (Xor64 (ZeroExt8to64 (CvtBoolToUint8 bool)) (Const64 [c^1])) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpZeroExt8to64 { + continue + } + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpCvtBoolToUint8 { + continue + } + cvtT := v_0_0.Type + v_0_0_0 := v_0_0.Args[0] + if v_0_0_0.Op != OpNot { + continue + } + bool := v_0_0_0.Args[0] + if v_1.Op != OpConst64 { + continue + } + constT := v_1.Type + c := auxIntToInt64(v_1.AuxInt) + if !(c != 1) { + continue + } + v.reset(OpXor64) + v0 := b.NewValue0(v.Pos, OpZeroExt8to64, typ.UInt64) + v1 := b.NewValue0(v.Pos, OpCvtBoolToUint8, cvtT) + v1.AddArg(bool) + v0.AddArg(v1) + v2 := b.NewValue0(v.Pos, OpConst64, constT) + v2.AuxInt = int64ToAuxInt(c ^ 1) + v.AddArg2(v0, v2) + return true + } + break + } return false } func rewriteValuegeneric_OpXor8(v *Value) bool { @@ -39457,6 +39801,58 @@ func rewriteValuegeneric_OpXor8(v *Value) bool { } break } + // match: (Xor8 (CvtBoolToUint8 bool) (Const8 [1])) + // cond: invertibleBool(bool.Op) + // result: (CvtBoolToUint8 (Not bool)) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpCvtBoolToUint8 { + continue + } + bool := v_0.Args[0] + if v_1.Op != OpConst8 || auxIntToInt8(v_1.AuxInt) != 1 || !(invertibleBool(bool.Op)) { + continue + } + v.reset(OpCvtBoolToUint8) + v0 := b.NewValue0(v.Pos, OpNot, bool.Type) + v0.AddArg(bool) + v.AddArg(v0) + return true + } + break + } + // match: (Xor8 (CvtBoolToUint8 (Not bool)) (Const8 [c])) + // cond: c != 1 + // result: (Xor8 (CvtBoolToUint8 bool) (Const8 [c^1])) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + if v_0.Op != OpCvtBoolToUint8 { + continue + } + cvtT := v_0.Type + v_0_0 := v_0.Args[0] + if v_0_0.Op != OpNot { + continue + } + bool := v_0_0.Args[0] + if v_1.Op != OpConst8 { + continue + } + constT := v_1.Type + c := auxIntToInt8(v_1.AuxInt) + if !(c != 1) { + continue + } + v.reset(OpXor8) + v0 := b.NewValue0(v.Pos, OpCvtBoolToUint8, cvtT) + v0.AddArg(bool) + v1 := b.NewValue0(v.Pos, OpConst8, constT) + v1.AuxInt = int8ToAuxInt(c ^ 1) + v.AddArg2(v0, v1) + return true + } + break + } return false } func rewriteValuegeneric_OpZero(v *Value) bool { diff --git a/src/cmd/compile/internal/staticinit/sched.go b/src/cmd/compile/internal/staticinit/sched.go index c9546313bb50aa..3f840217ccd5dd 100644 --- a/src/cmd/compile/internal/staticinit/sched.go +++ b/src/cmd/compile/internal/staticinit/sched.go @@ -1247,3 +1247,33 @@ func OutlineMapInits(fn *ir.Func) { fmt.Fprintf(os.Stderr, "=-= outlined %v map initializations\n", outlined) } } + +// varInitGen is a counter used to uniquify compiler-generated functions for initializing variables. +var varInitGen int + +const varInitFuncPrefix = "init.part." + +// GenerateVarInitFunc create a new function that will (eventually) have this form: +// +// func init.part.%d() { +// ... +// } +func GenerateVarInitFunc() *ir.Func { + pos := base.AutogeneratedPos + base.Pos = pos + + sym := typecheck.LookupNum(varInitFuncPrefix, varInitGen) + varInitGen++ + + fn := ir.NewFunc(pos, pos, sym, types.NewSignature(nil, nil, nil)) + fn.SetInlinabilityChecked(true) // suppress inlining; otherwise, we end up with giant init eventually. + fn.SetWrapper(true) // less disruptive on backtraces. + + return fn +} + +// CanOptimize reports whether the given fn can be optimized for static assignments. +func CanOptimize(fn *ir.Func) bool { + name := fn.Sym().Name + return name == "init" || strings.HasPrefix(name, varInitFuncPrefix) +} diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index 43ac003c7907c7..6786131a4f7ff5 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -314,14 +314,13 @@ func testName(pkg, variant string) string { // goTest represents all options to a "go test" command. The final command will // combine configuration from goTest and tester flags. type goTest struct { - timeout time.Duration // If non-zero, override timeout - short bool // If true, force -short - tags []string // Build tags - race bool // Force -race - bench bool // Run benchmarks (briefly), not tests. - runTests string // Regexp of tests to run - cpu string // If non-empty, -cpu flag - skip string // If non-empty, -skip flag + short bool // If true, force -short + tags []string // Build tags + race bool // Force -race + bench bool // Run benchmarks (briefly), not tests. + runTests string // Regexp of tests to run + cpu string // If non-empty, -cpu flag + skip string // If non-empty, -skip flag gcflags string // If non-empty, build with -gcflags=all=X ldflags string // If non-empty, build with -ldflags=X @@ -329,6 +328,16 @@ type goTest struct { env []string // Environment variables to add, as KEY=VAL. KEY= unsets a variable + // timeout optionally raises the per-package test timeout to be at least this long. + // The zero value means to stay with the default test timeout. + // When adding new tests, this field generally doesn't need to be set, not unless + // the go commmand's default test timeout proves to be insufficient. + // + // In either case, the per-package test timeout get scaled by a multiplier, + // and applied only if the end result is longer than the go command's default + // test timeout. + timeout time.Duration + runOnHost bool // When cross-compiling, run this test on the host instead of guest // variant, if non-empty, is a name used to distinguish different @@ -358,6 +367,18 @@ func (opts *goTest) compileOnly() bool { return opts.runTests == "^$" && !opts.bench } +// scaledTimeout reports the per-package test timeout scaled by t.timeoutScale. +func (opts *goTest) scaledTimeout(t *tester) time.Duration { + d := goTestDefaultTimeout + if opts.timeout != 0 { + d = opts.timeout + } + d *= time.Duration(t.timeoutScale) + return d +} + +const goTestDefaultTimeout = 10 * time.Minute // Default value of go test -timeout flag. + // bgCommand returns a go test Cmd and a post-Run flush function. The result // will write its output to stdout and stderr. If stdout==stderr, bgCommand // ensures Writes are serialized. The caller should call flush() after Cmd exits. @@ -425,13 +446,9 @@ func (opts *goTest) run(t *tester) error { // The caller must call setupCmd on the resulting exec.Cmd to set its directory // and environment. func (opts *goTest) buildArgs(t *tester) (build, run, pkgs, testFlags []string, setupCmd func(*exec.Cmd)) { - run = append(run, "-count=1") // Disallow caching - if opts.timeout != 0 { - d := opts.timeout * time.Duration(t.timeoutScale) + run = append(run, "-count=1") // Disallow caching. + if d := opts.scaledTimeout(t); d > goTestDefaultTimeout { run = append(run, "-timeout="+d.String()) - } else if t.timeoutScale != 1 { - const goTestDefaultTimeout = 10 * time.Minute // Default value of go test -timeout flag. - run = append(run, "-timeout="+(goTestDefaultTimeout*time.Duration(t.timeoutScale)).String()) } if opts.short || t.short { run = append(run, "-short") @@ -571,26 +588,7 @@ func (t *tester) registerStdTest(pkg string) { defer timelog("end", dt.name) ranGoTest = true - timeoutSec := 180 * time.Second - for _, pkg := range stdMatches { - switch pkg { - case "cmd/go": - timeoutSec *= 3 - case "cmd/cgo/internal/testshared": - // This package can take 2-3 minutes to test, so 3 min timeout causes - // flaky failures, like https://ci.chromium.org/b/8679277370961616529. - // Use the default timeout for it rather than the custom 3 minute one. - timeoutSec = 0 - case "internal/godebugs": - // This package can take 5-6 minutes to test when the asan mode is on. - // The asan modifier scales the timeout by 2, but even 3*2 minutes is - // sometimes not enough. See go.dev/issue/78392. - // Use the default timeout for it rather than the custom 3 minute one. - timeoutSec = 0 - } - } return (&goTest{ - timeout: timeoutSec, gcflags: gcflags, pkgs: stdMatches, }).run(t) @@ -617,7 +615,7 @@ func (t *tester) registerRaceBenchTest(pkg string) { // This makes the test targets distinct, allowing our build system to record // elapsed time for each one, which is useful for load-balancing test shards. omitVariant: false, - timeout: 1200 * time.Second, // longer timeout for race with benchmarks + timeout: 20 * time.Minute, // longer timeout for race with benchmarks race: true, bench: true, cpu: "4", @@ -708,7 +706,6 @@ func (t *tester) registerTests() { t.registerTest("os/user with tag osusergo", &goTest{ variant: "osusergo", - timeout: 300 * time.Second, tags: []string{"osusergo"}, pkg: "os/user", }) @@ -799,7 +796,6 @@ func (t *tester) registerTests() { t.registerTest("GOOS=ios on darwin/amd64", &goTest{ variant: "amd64ios", - timeout: 300 * time.Second, runTests: "SystemRoots", env: []string{"GOOS=ios", "CGO_ENABLED=1"}, pkg: "crypto/x509", @@ -813,7 +809,6 @@ func (t *tester) registerTests() { t.registerTest("GODEBUG=gcstoptheworld=2 archive/zip", &goTest{ variant: "gcstoptheworld2", - timeout: 300 * time.Second, short: true, env: []string{"GODEBUG=gcstoptheworld=2"}, pkg: "archive/zip", @@ -821,7 +816,6 @@ func (t *tester) registerTests() { t.registerTest("GODEBUG=gccheckmark=1 runtime", &goTest{ variant: "gccheckmark", - timeout: 300 * time.Second, short: true, env: []string{"GODEBUG=gccheckmark=1"}, pkg: "runtime", @@ -878,7 +872,6 @@ func (t *tester) registerTests() { t.registerTest("maymorestack="+hook, &goTest{ variant: hook, - timeout: 600 * time.Second, short: true, env: []string{"GOFLAGS=" + goFlags}, pkgs: []string{"runtime", "reflect", "sync"}, @@ -924,7 +917,6 @@ func (t *tester) registerTests() { t.registerTest("internal linking, -buildmode=pie", &goTest{ variant: "pie_internal", - timeout: 60 * time.Second, buildmode: "pie", ldflags: "-linkmode=internal", env: []string{"CGO_ENABLED=0"}, @@ -933,7 +925,6 @@ func (t *tester) registerTests() { t.registerTest("internal linking, -buildmode=pie", &goTest{ variant: "pie_internal", - timeout: 60 * time.Second, buildmode: "pie", ldflags: "-linkmode=internal", env: []string{"CGO_ENABLED=0"}, @@ -945,7 +936,6 @@ func (t *tester) registerTests() { t.registerTest("internal linking, -buildmode=pie", &goTest{ variant: "pie_internal", - timeout: 60 * time.Second, buildmode: "pie", ldflags: "-linkmode=internal", pkg: "os/user", @@ -958,7 +948,6 @@ func (t *tester) registerTests() { t.registerTest("external linking, -buildmode=exe", &goTest{ variant: "exe_external", - timeout: 60 * time.Second, buildmode: "exe", ldflags: "-linkmode=external", env: []string{"CGO_ENABLED=1"}, @@ -970,7 +959,6 @@ func (t *tester) registerTests() { t.registerTest("external linking, -buildmode=pie", &goTest{ variant: "pie_external", - timeout: 60 * time.Second, buildmode: "pie", ldflags: "-linkmode=external", env: []string{"CGO_ENABLED=1"}, @@ -985,7 +973,6 @@ func (t *tester) registerTests() { t.registerTest("sync -cpu=10", &goTest{ variant: "cpu10", - timeout: 120 * time.Second, cpu: "10", pkg: "sync", }) @@ -1001,7 +988,6 @@ func (t *tester) registerTests() { &goTest{ variant: "host", pkg: "internal/runtime/wasitest", - timeout: 1 * time.Minute, runOnHost: true, }) } @@ -1017,7 +1003,7 @@ func (t *tester) registerTests() { // TODO: remove the exclusion of goexperiment simd right before dev.simd branch is merged to master. if goos == "darwin" || ((goos == "linux" || goos == "windows") && (goarch == "amd64" && !strings.Contains(goexperiment, "simd"))) { t.registerTest("API release note check", &goTest{variant: "check", pkg: "cmd/relnote", testFlags: []string{"-check"}}) - t.registerTest("API check", &goTest{variant: "check", pkg: "cmd/api", timeout: 5 * time.Minute, testFlags: []string{"-check"}}) + t.registerTest("API check", &goTest{variant: "check", pkg: "cmd/api", testFlags: []string{"-check"}}) } // Runtime CPU tests. @@ -1026,7 +1012,6 @@ func (t *tester) registerTests() { t.registerTest(fmt.Sprintf("GOMAXPROCS=2 runtime -cpu=%d -quick", i), &goTest{ variant: "cpu" + strconv.Itoa(i), - timeout: 300 * time.Second, cpu: strconv.Itoa(i), gcflags: gogcflags, short: true, diff --git a/src/cmd/distpack/pack.go b/src/cmd/distpack/pack.go index 09c3a331195f58..53e9dbd73b64ca 100644 --- a/src/cmd/distpack/pack.go +++ b/src/cmd/distpack/pack.go @@ -113,6 +113,7 @@ func main() { ".gitattributes", ".github/**", ".gitignore", + ".jj/**", "VERSION.cache", "misc/cgo/*/_obj/**", "**/.DS_Store", diff --git a/src/cmd/distpack/test.go b/src/cmd/distpack/test.go index 7479bd77f55951..7fd2dad57be7f1 100644 --- a/src/cmd/distpack/test.go +++ b/src/cmd/distpack/test.go @@ -30,6 +30,7 @@ var srcRules = []testRule{ {name: "go/.git", exclude: true}, {name: "go/.gitattributes", exclude: true}, {name: "go/.github", exclude: true}, + {name: "go/.jj", exclude: true}, {name: "go/VERSION.cache", exclude: true}, {name: "go/bin/**", exclude: true}, {name: "go/pkg/**", exclude: true}, @@ -48,6 +49,7 @@ var zipRules = []testRule{ {name: "go/.git", exclude: true}, {name: "go/.gitattributes", exclude: true}, {name: "go/.github", exclude: true}, + {name: "go/.jj", exclude: true}, {name: "go/VERSION.cache", exclude: true}, {name: "go/bin", exclude: true}, {name: "go/pkg", exclude: true}, @@ -84,6 +86,7 @@ var modRules = []testRule{ {name: "golang.org/toolchain@*/.git", exclude: true}, {name: "golang.org/toolchain@*/.gitattributes", exclude: true}, {name: "golang.org/toolchain@*/.github", exclude: true}, + {name: "golang.org/toolchain@*/.jj", exclude: true}, {name: "golang.org/toolchain@*/VERSION.cache", exclude: true}, {name: "golang.org/toolchain@*/bin", exclude: true}, {name: "golang.org/toolchain@*/pkg", exclude: true}, diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 66a51c5e4f4c2c..3c7f99a8d4265e 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -719,7 +719,22 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { // Do we need to run a build to gather information? needStale := (listJson && listJsonFields.needAny("Stale", "StaleReason")) || strings.Contains(*listFmt, ".Stale") - if needStale || *listExport || *listCompiled { + var buildPkgs []*load.Package + if needStale || *listExport || (*listCompiled && cfg.BuildCover) { + buildPkgs = pkgs + } else if *listCompiled { + // In the non-cover case, for pure-Go packages, package loading already knows the complete set + // of files passed to the compiler, so no build action is needed. + for _, p := range pkgs { + if p.UsesCgo() || p.UsesSwig() { + // Keep using the builder for packages that generate Go sources. + buildPkgs = append(buildPkgs, p) + continue + } + p.CompiledGoFiles = str.StringList(p.GoFiles) + } + } + if len(buildPkgs) > 0 { b := work.NewBuilder("", moduleLoader.VendorDirOrEmpty) if *listE { b.AllowErrors = true @@ -738,7 +753,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } a := &work.Action{} // TODO: Use pkgsFilter? - for _, p := range pkgs { + for _, p := range buildPkgs { if len(p.GoFiles)+len(p.CgoFiles) > 0 { a.Deps = append(a.Deps, b.AutoAction(moduleLoader, work.ModeInstall, work.ModeInstall, p)) } diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 2985cff67cec0b..0de472927ffba6 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -377,7 +377,6 @@ func (p *Package) Resolve(s *modload.Loader, imports []string) []string { // CoverSetup holds parameters related to coverage setup for a given package (covermode, etc). type CoverSetup struct { Mode string // coverage mode for this package - Cfg string // path to config file to pass to "go tool cover" GenMeta bool // ask cover tool to emit a static meta data if set } diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go index 8b3897e215ed57..c2a0e0c16e43c3 100644 --- a/src/cmd/go/internal/work/action.go +++ b/src/cmd/go/internal/work/action.go @@ -9,7 +9,6 @@ package work import ( "bufio" "bytes" - "cmd/internal/cov/covcmd" "cmd/internal/par" "container/heap" "context" @@ -540,8 +539,7 @@ type checkCacheProvider struct { // and cgo compile actions earlier because they wouldn't depend on the builds of the // dependencies of the package they belong to. type checkCacheActor struct { - covMetaFileName string - buildAction *Action + buildAction *Action } func (cca *checkCacheActor) Act(b *Builder, ctx context.Context, a *Action) error { @@ -551,7 +549,7 @@ func (cca *checkCacheActor) Act(b *Builder, ctx context.Context, a *Action) erro // making the true build action its dependency. Fetch the build action in that case. buildAction = buildAction.Deps[0] } - pr, err := b.checkCacheForBuild(a, buildAction, cca.covMetaFileName) + pr, err := b.checkCacheForBuild(a, buildAction) if err != nil { return err } @@ -560,28 +558,16 @@ func (cca *checkCacheActor) Act(b *Builder, ctx context.Context, a *Action) erro } type coverProvider struct { - goSources, cgoSources []string // The go and cgo sources generated by the cover tool, which should be used instead of the raw sources on the package. -} - -// The actor to run the cover tool to produce instrumented source files for cover -// builds. In the case of a package with no test files, we store some additional state -// information in the build actor to help with reporting. -type coverActor struct { - // name of static meta-data file fragment emitted by the cover + // name of static metadata file fragment emitted by the cover // tool as part of the package cover action, for selected // "go test -cover" runs. covMetaFileName string - buildAction *Action -} + // coverageConfig is the path to the json-serialized covcmd.CoverPkgConfig + // provided to the cover tool. The config is created by coverConfig. + coverageConfig string -func (ca *coverActor) Act(b *Builder, ctx context.Context, a *Action) error { - pr, err := b.runCover(a, ca.buildAction, a.Objdir, a.Package.GoFiles, a.Package.CgoFiles) - if err != nil { - return err - } - a.Provider = pr - return nil + goSources, cgoSources []string // The go and cgo sources generated by the cover tool, which should be used instead of the raw sources on the package. } // runCgoActor implements the Actor interface for running the cgo command for the package. @@ -597,7 +583,6 @@ func (c runCgoActor) Act(b *Builder, ctx context.Context, a *Action) error { } } need := cacheProvider.need - need &^= needCovMetaFile // handled by cover action if need == 0 { return nil } @@ -691,22 +676,6 @@ func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Actio } } - // Determine the covmeta file name. - var covMetaFileName string - if p.Internal.Cover.GenMeta { - covMetaFileName = covcmd.MetaFileForPackage(p.ImportPath) - } - - // Create a cache action. - cacheAction := &Action{ - Mode: "build check cache", - Package: p, - Actor: &checkCacheActor{buildAction: a, covMetaFileName: covMetaFileName}, - Objdir: a.Objdir, - Deps: a.Deps, // Need outputs of dependency build actions to generate action id. - } - a.Deps = append(a.Deps, cacheAction) - // Create a cover action if we need to instrument the code for coverage. // The cover action always runs in the same go build invocation as the build, // and is not cached separately, so it can use the same objdir. @@ -716,14 +685,23 @@ func (b *Builder) CompileAction(mode, depMode BuildMode, p *load.Package) *Actio return &Action{ Mode: "cover", Package: p, - Actor: &coverActor{buildAction: a, covMetaFileName: covMetaFileName}, + Actor: ActorFunc((*Builder).runCover), Objdir: a.Objdir, - Deps: []*Action{cacheAction}, } }) a.Deps = append(a.Deps, coverAction) } + // Create a cache action. + cacheAction := &Action{ + Mode: "build check cache", + Package: p, + Actor: &checkCacheActor{buildAction: a}, + Objdir: a.Objdir, + Deps: a.Deps, // Need outputs of dependency build actions to generate action id. + } + a.Deps = append(a.Deps, cacheAction) + // Create actions to run swig and cgo if needed. These actions always run in the // same go build invocation as the build action and their actions are not cached // separately, so they can use the same objdir. diff --git a/src/cmd/go/internal/work/cover.go b/src/cmd/go/internal/work/cover.go index be090ce4c3b50b..86f935690a5c6c 100644 --- a/src/cmd/go/internal/work/cover.go +++ b/src/cmd/go/internal/work/cover.go @@ -7,9 +7,9 @@ package work import ( + "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/str" - "cmd/internal/cov/covcmd" "context" "encoding/json" "fmt" @@ -42,8 +42,18 @@ func BuildActionCoverMetaFile(runAct *Action) (string, error) { if pred.Mode != "build" || pred.Package == nil { continue } + var coverAction *Action + for _, act := range pred.Deps { + if act.Mode == "cover" { + coverAction = act + break + } + } + if coverAction == nil { + base.Fatalf("internal error: cover action not found for %s", p.ImportPath) + } if pred.Package.ImportPath == p.ImportPath { - metaFile := pred.Objdir + covcmd.MetaFileForPackage(p.ImportPath) + metaFile := coverAction.Provider.(*coverProvider).covMetaFileName if cfg.BuildN { return metaFile, nil } @@ -118,7 +128,17 @@ func WriteCoverMetaFilesFile(b *Builder, ctx context.Context, a *Action) error { if dep.Mode != "build" { panic("unexpected mode " + dep.Mode) } - metaFilesFile := dep.Objdir + covcmd.MetaFileForPackage(dep.Package.ImportPath) + var coverAction *Action + for _, act := range dep.Deps { + if act.Mode == "cover" { + coverAction = act + } + } + if coverAction == nil { + // No coverage data for this package. + continue + } + metaFilesFile := coverAction.Provider.(*coverProvider).covMetaFileName // Check to make sure the meta-data file fragment exists // and has content (may be empty if package has no functions). if fi, err := os.Stat(metaFilesFile); err != nil { diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index ae2768bc37d1a4..883177058b0164 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -510,7 +510,6 @@ const ( needCgoHdr needVet needCompiledGoFiles - needCovMetaFile needStale ) @@ -518,7 +517,7 @@ const ( // what work needs to be done by it and the actions preceding it. a is the action // currently being run, which has an actor of type *checkCacheActor and is a dependency // of the buildAction. -func (b *Builder) checkCacheForBuild(a, buildAction *Action, covMetaFileName string) (_ *checkCacheProvider, err error) { +func (b *Builder) checkCacheForBuild(a, buildAction *Action) (_ *checkCacheProvider, err error) { p := buildAction.Package sh := b.Shell(a) @@ -530,11 +529,9 @@ func (b *Builder) checkCacheForBuild(a, buildAction *Action, covMetaFileName str } cachedBuild := false - needCovMeta := p.Internal.Cover.GenMeta need := bit(needBuild, !b.IsCmdList && buildAction.needBuild || b.NeedExport) | bit(needCgoHdr, b.needCgoHdr(buildAction)) | bit(needVet, buildAction.needVet) | - bit(needCovMetaFile, needCovMeta) | bit(needCompiledGoFiles, b.NeedCompiledGoFiles) if !p.BinaryOnly { @@ -617,14 +614,6 @@ func (b *Builder) checkCacheForBuild(a, buildAction *Action, covMetaFileName str } } - // Load cached coverage meta-data file fragment, but only if we're - // skipping the main build (cachedBuild==true). - if cachedBuild && need&needCovMetaFile != 0 { - if err := b.loadCachedObjdirFile(buildAction, cache.Default(), covMetaFileName); err == nil { - need &^= needCovMetaFile - } - } - // Load cached vet config, but only if that's all we have left // (need == needVet, not testing just the one bit). // If we are going to do a full build anyway, @@ -638,31 +627,28 @@ func (b *Builder) checkCacheForBuild(a, buildAction *Action, covMetaFileName str return &checkCacheProvider{need: need}, nil } -func (b *Builder) runCover(a, buildAction *Action, objdir string, gofiles, cgofiles []string) (*coverProvider, error) { +func (b *Builder) runCover(ctx context.Context, a *Action) error { p := a.Package sh := b.Shell(a) - var cacheProvider *checkCacheProvider - for _, dep := range a.Deps { - if pr, ok := dep.Provider.(*checkCacheProvider); ok { - cacheProvider = pr - } + // Determine the covmeta file name. + var covMetaFileName string + if a.Package.Internal.Cover.GenMeta { + covMetaFileName = a.Objdir + covcmd.MetaFileForPackage(a.Package.ImportPath) } - if cacheProvider == nil { - base.Fatalf("internal error: could not find checkCacheProvider") - } - need := cacheProvider.need - if need == 0 { - return nil, nil + if err := sh.Mkdir(a.Objdir); err != nil { + return err } - if err := sh.Mkdir(a.Objdir); err != nil { - return nil, err + a.actionID = b.coverActionID(a, covMetaFileName) + if pr, err := b.loadCachedCoverOutputs(a); err == nil { + a.Provider = pr + return nil } - gofiles = slices.Clone(gofiles) - cgofiles = slices.Clone(cgofiles) + gofiles := slices.Clone(a.Package.GoFiles) + cgofiles := slices.Clone(a.Package.CgoFiles) outfiles := []string{} infiles := []string{} @@ -677,10 +663,10 @@ func (b *Builder) runCover(a, buildAction *Action, objdir string, gofiles, cgofi // cgo files have absolute paths base = filepath.Base(base) sourceFile = file - coverFile = objdir + base + ".cgo1.go" + coverFile = a.Objdir + base + ".cgo1.go" } else { sourceFile = filepath.Join(p.Dir, file) - coverFile = objdir + file + coverFile = a.Objdir + file } coverFile = strings.TrimSuffix(coverFile, ".go") + ".cover.go" infiles = append(infiles, sourceFile) @@ -692,6 +678,7 @@ func (b *Builder) runCover(a, buildAction *Action, objdir string, gofiles, cgofi } } + var coverCfg string if len(infiles) != 0 { // Coverage instrumentation creates new top level // variables in the target package for things like @@ -708,17 +695,25 @@ func (b *Builder) runCover(a, buildAction *Action, objdir string, gofiles, cgofi if mode == "" { panic("covermode should be set at this point") } - if newoutfiles, err := b.cover(a, infiles, outfiles, coverVar, mode); err != nil { - return nil, err + coverCfg = a.Objdir + "coveragecfg" + if newoutfiles, err := b.cover(a, infiles, outfiles, coverVar, mode, covMetaFileName, coverCfg); err != nil { + return err } else { outfiles = newoutfiles gofiles = append([]string{newoutfiles[0]}, gofiles...) } - if ca, ok := a.Actor.(*coverActor); ok && ca.covMetaFileName != "" { - b.cacheObjdirFile(buildAction, cache.Default(), ca.covMetaFileName) + } + + pr := &coverProvider{covMetaFileName, coverCfg, gofiles, cgofiles} + a.Provider = pr + + if !cfg.BuildN { + if err := b.cacheCoverOutputs(a, pr); err != nil { + return err } } - return &coverProvider{gofiles, cgofiles}, nil + + return nil } // build is the action for building a single package. @@ -745,8 +740,7 @@ func (b *Builder) build(ctx context.Context, a *Action) (err error) { } need := cacheProvider.need - need &^= needCovMetaFile // handled by cover action - need &^= needCgoHdr // handled by run cgo action // TODO: accumulate "negative" need bits from actions + need &^= needCgoHdr // handled by run cgo action // TODO: accumulate "negative" need bits from actions if need == 0 { return @@ -918,6 +912,11 @@ func (b *Builder) build(ctx context.Context, a *Action) (err error) { pgoProfile = a1.built } + var coverageConfig string + if coverPr != nil { + coverageConfig = coverPr.coverageConfig + } + if p.Internal.BuildInfo != nil && cfg.ModulesEnabled { prog := modload.ModInfoProg(p.Internal.BuildInfo.String(), cfg.BuildToolchainName == "gccgo") if len(prog) > 0 { @@ -930,7 +929,7 @@ func (b *Builder) build(ctx context.Context, a *Action) (err error) { // Compile Go. objpkg := objdir + "_pkg_.a" - ofile, out, err := BuildToolchain.gc(b, a, objpkg, icfg.Bytes(), embedcfg, symabis, len(sfiles) > 0, pgoProfile, gofiles) + ofile, out, err := BuildToolchain.gc(b, a, objpkg, icfg.Bytes(), embedcfg, symabis, len(sfiles) > 0, pgoProfile, coverageConfig, gofiles) if len(out) > 0 && (p.UsesCgo() || p.UsesSwig()) && !cfg.BuildX { // Fix up output referring to cgo-generated code to be more readable. // Replace *[100]_Ctype_foo with *[100]C.foo. @@ -1132,11 +1131,134 @@ func (b *Builder) cacheSrcFiles(a *Action, srcfiles []string) { cache.PutBytes(c, cache.Subkey(a.actionID, "srcfiles"), buf.Bytes()) } -func (b *Builder) loadCachedVet(a *Action, vetDeps []*Action) error { +// coverProviderCached is the structure we'll use to represent a coverProvider +// in the action cache. It has the same fields as a coverProvider but they'll contain +// different contents: we replace the acutal objdir name with $OBJDIR so that +// the cached provider doesn't depend on the objdir used. +type coverProviderCached struct { + CovMetaFile string + CoverageConfig string + GoSources, CgoSources []string +} + +func (b *Builder) cacheCoverOutputs(a *Action, pr *coverProvider) error { + c := cache.Default() + + cacheSrcFileName := func(a *Action, file string) string { + if name, ok := strings.CutPrefix(file, a.Objdir); ok { + return name + } + return "./" + file + } + + b.cacheSrcFiles(a, str.StringList(pr.goSources, pr.cgoSources)) + var cached coverProviderCached + if pr.covMetaFileName != "" { + cached.CovMetaFile = strings.TrimPrefix(pr.covMetaFileName, a.Objdir) + if err := b.cacheObjdirFile(a, c, cached.CovMetaFile); err != nil { + return err + } + } + if pr.coverageConfig != "" { + cached.CoverageConfig = strings.TrimPrefix(pr.coverageConfig, a.Objdir) + if err := b.cacheObjdirFile(a, c, cached.CoverageConfig); err != nil { + return err + } + } + + for _, fn := range pr.goSources { + cached.GoSources = append(cached.GoSources, cacheSrcFileName(a, fn)) + } + for _, fn := range pr.cgoSources { + cached.CgoSources = append(cached.CgoSources, cacheSrcFileName(a, fn)) + } + js, err := json.Marshal(cached) + if err != nil { + return err + } + cache.PutBytes(c, cache.Subkey(a.actionID, "coverprovider"), js) + + return nil +} + +func (b *Builder) loadCachedCoverOutputs(a *Action) (*coverProvider, error) { + c := cache.Default() + + if _, err := b.loadCachedSrcFiles(a); err != nil { + return nil, err + } + + var cached coverProviderCached + if js, _, err := cache.GetBytes(c, cache.Subkey(a.actionID, "coverprovider")); err != nil { + return nil, err + } else if err := json.Unmarshal(js, &cached); err != nil { + return nil, err + } + + covMetaFile := cached.CovMetaFile + if covMetaFile != "" { + if err := b.loadCachedObjdirFile(a, c, covMetaFile); err != nil { + return nil, err + } + covMetaFile = a.Objdir + covMetaFile + } + coverageConfig := cached.CoverageConfig + if coverageConfig != "" { + if err := b.loadCachedObjdirFile(a, c, coverageConfig); err != nil { + return nil, err + } + coverageConfig = a.Objdir + coverageConfig + } + + var goSources, cgoSources []string + for _, file := range cached.GoSources { + name, ok := strings.CutPrefix(file, "./") + if !ok { + name = a.Objdir + name + } + goSources = append(goSources, name) + } + for _, file := range cached.CgoSources { + name, ok := strings.CutPrefix(file, "./") + if !ok { + name = a.Objdir + name + } + cgoSources = append(cgoSources, name) + } + + pr := &coverProvider{ + covMetaFileName: covMetaFile, + coverageConfig: coverageConfig, + goSources: goSources, + cgoSources: cgoSources, + } + + return pr, nil +} + +func (b *Builder) coverActionID(a *Action, covMetaFileName string) cache.ActionID { + p := a.Package + h := cache.NewHash("cover " + p.ImportPath) + fmt.Fprintf(h, "cover %q\n", b.toolID("cover")) + + // Input files for cover. + fmt.Fprintf(h, "setup %s %v\n", p.Internal.Cover.Mode, p.Internal.Cover.GenMeta) + fmt.Fprintf(h, "config ") + if err := json.NewEncoder(h).Encode(coverConfig(p, filepath.Base(covMetaFileName), "")); err != nil { + base.Fatal(err) + } + for _, file := range str.StringList(p.GoFiles, p.CgoFiles) { + fmt.Fprintf(h, "file %s %s\n", file, b.fileHash(filepath.Join(p.Dir, file))) + } + + return h.Sum() +} + +func (b *Builder) loadCachedSrcFiles(a *Action) ([]string, error) { c := cache.Default() list, _, err := cache.GetBytes(c, cache.Subkey(a.actionID, "srcfiles")) if err != nil { - return fmt.Errorf("reading srcfiles list: %w", err) + return nil, fmt.Errorf("reading srcfiles list: %w", err) } var srcfiles []string for name := range strings.SplitSeq(string(list), "\n") { @@ -1148,10 +1270,18 @@ func (b *Builder) loadCachedVet(a *Action, vetDeps []*Action) error { continue } if err := b.loadCachedObjdirFile(a, c, name); err != nil { - return err + return nil, err } srcfiles = append(srcfiles, a.Objdir+name) } + return srcfiles, nil +} + +func (b *Builder) loadCachedVet(a *Action, vetDeps []*Action) error { + srcfiles, err := b.loadCachedSrcFiles(a) + if err != nil { + return err + } buildVetConfig(a, srcfiles, vetDeps) return nil } @@ -2090,13 +2220,13 @@ func (b *Builder) installHeader(ctx context.Context, a *Action) error { // regular outputs (instrumented source files) the cover tool also // writes a separate file (appearing first in the list of outputs) // that will contain coverage counters and meta-data. -func (b *Builder) cover(a *Action, infiles, outfiles []string, varName string, mode string) ([]string, error) { +func (b *Builder) cover(a *Action, infiles, outfiles []string, varName, mode, covMetaFileName, coverCfg string) ([]string, error) { pkgcfg := a.Objdir + "pkgcfg.txt" covoutputs := a.Objdir + "coveroutfiles.txt" odir := filepath.Dir(outfiles[0]) cv := filepath.Join(odir, "covervars.go") outfiles = append([]string{cv}, outfiles...) - if err := b.writeCoverPkgInputs(a, pkgcfg, covoutputs, outfiles); err != nil { + if err := b.writeCoverPkgInputs(a, pkgcfg, covMetaFileName, coverCfg, covoutputs, outfiles); err != nil { return nil, err } args := []string{base.Tool("cover"), @@ -2113,10 +2243,7 @@ func (b *Builder) cover(a *Action, infiles, outfiles []string, varName string, m return outfiles, nil } -func (b *Builder) writeCoverPkgInputs(a *Action, pconfigfile string, covoutputsfile string, outfiles []string) error { - sh := b.Shell(a) - p := a.Package - p.Internal.Cover.Cfg = a.Objdir + "coveragecfg" +func coverConfig(p *load.Package, covMetaFileName, outConfig string) covcmd.CoverPkgConfig { pcfg := covcmd.CoverPkgConfig{ PkgPath: p.ImportPath, PkgName: p.Name, @@ -2124,16 +2251,21 @@ func (b *Builder) writeCoverPkgInputs(a *Action, pconfigfile string, covoutputsf // 'perblock'; there isn't a way using "go build -cover" or "go // test -cover" to select it. This may change in the future // depending on user demand. - Granularity: "perblock", - OutConfig: p.Internal.Cover.Cfg, - Local: p.Internal.Local, - } - if ca, ok := a.Actor.(*coverActor); ok && ca.covMetaFileName != "" { - pcfg.EmitMetaFile = a.Objdir + ca.covMetaFileName + Granularity: "perblock", + OutConfig: outConfig, + Local: p.Internal.Local, + EmitMetaFile: covMetaFileName, } - if a.Package.Module != nil { - pcfg.ModulePath = a.Package.Module.Path + if p.Module != nil { + pcfg.ModulePath = p.Module.Path } + return pcfg +} + +func (b *Builder) writeCoverPkgInputs(a *Action, pconfigfile, covMetaFileName, coverCfg, covoutputsfile string, outfiles []string) error { + sh := b.Shell(a) + p := a.Package + pcfg := coverConfig(p, covMetaFileName, coverCfg) data, err := json.Marshal(pcfg) if err != nil { return err @@ -2209,7 +2341,7 @@ func mkAbs(dir, f string) string { type toolchain interface { // gc runs the compiler in a specific directory on a set of files // and returns the name of the generated output file. - gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, pgoProfile string, gofiles []string) (ofile string, out []byte, err error) + gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, pgoProfile, coverCfg string, gofiles []string) (ofile string, out []byte, err error) // cc runs the toolchain's C compiler in a directory on a C file // to produce an output file. cc(b *Builder, a *Action, ofile, cfile string) error @@ -2249,7 +2381,7 @@ func (noToolchain) linker() string { return "" } -func (noToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, pgoProfile string, gofiles []string) (ofile string, out []byte, err error) { +func (noToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, pgoProfile, coverCfg string, gofiles []string) (ofile string, out []byte, err error) { return "", nil, noCompiler() } @@ -3482,7 +3614,7 @@ func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { p := load.GoFilesPackage(modload.NewLoader(), context.TODO(), load.PackageOpts{}, srcs) - if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, nil, "", false, "", srcs); e != nil { + if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, nil, "", false, "", "", srcs); e != nil { return "32", nil } return "64", nil diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index 49b26e15609bb8..cee93bc1e53c3c 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -53,7 +53,7 @@ func pkgPath(a *Action) string { return ppath } -func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, pgoProfile string, gofiles []string) (ofile string, output []byte, err error) { +func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, pgoProfile, coverCfg string, gofiles []string) (ofile string, output []byte, err error) { p := a.Package sh := b.Shell(a) objdir := a.Objdir @@ -112,8 +112,8 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg if strings.HasPrefix(ToolchainVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") { defaultGcFlags = append(defaultGcFlags, "-goversion", ToolchainVersion) } - if p.Internal.Cover.Cfg != "" { - defaultGcFlags = append(defaultGcFlags, "-coveragecfg="+p.Internal.Cover.Cfg) + if coverCfg != "" { + defaultGcFlags = append(defaultGcFlags, "-coveragecfg="+coverCfg) } if pgoProfile != "" { defaultGcFlags = append(defaultGcFlags, "-pgoprofile="+pgoProfile) diff --git a/src/cmd/go/internal/work/gccgo.go b/src/cmd/go/internal/work/gccgo.go index 276e082b71ba55..e0b238836422af 100644 --- a/src/cmd/go/internal/work/gccgo.go +++ b/src/cmd/go/internal/work/gccgo.go @@ -64,7 +64,7 @@ func checkGccgoBin() { base.Exit() } -func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, pgoProfile string, gofiles []string) (ofile string, output []byte, err error) { +func (tools gccgoToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, pgoProfile, coverCfg string, gofiles []string) (ofile string, output []byte, err error) { p := a.Package sh := b.Shell(a) objdir := a.Objdir diff --git a/src/cmd/go/testdata/script/build_GOTMPDIR.txt b/src/cmd/go/testdata/script/build_GOTMPDIR.txt index 6ad0157e5f629b..0ce4504b2fdbe1 100644 --- a/src/cmd/go/testdata/script/build_GOTMPDIR.txt +++ b/src/cmd/go/testdata/script/build_GOTMPDIR.txt @@ -25,6 +25,9 @@ stderr '^go: creating work dir: \w+ '$GOTMPDIR go list -x . ! stderr 'creating work dir' stdout m +go list -compiled -x -f '{{.CompiledGoFiles}}' . +! stderr 'creating work dir' +stdout '\[hello.go\]' go list -m all stdout m ! go list -x -export . diff --git a/src/cmd/link/internal/ld/elf.go b/src/cmd/link/internal/ld/elf.go index 949ca6a3f698ae..aea8d2b2e74539 100644 --- a/src/cmd/link/internal/ld/elf.go +++ b/src/cmd/link/internal/ld/elf.go @@ -114,7 +114,10 @@ const ( ELF32RELSIZE = 8 ) -var elfstrdat []byte +var ( + elfstrdat []byte + elfstroff map[string]int +) // ELFRESERVE is the total amount of space to reserve at the // start of the file for Header, PHeaders, SHeaders, and interp. diff --git a/src/cmd/link/internal/ld/symtab.go b/src/cmd/link/internal/ld/symtab.go index e1bddecb708460..5dc06237811283 100644 --- a/src/cmd/link/internal/ld/symtab.go +++ b/src/cmd/link/internal/ld/symtab.go @@ -45,14 +45,19 @@ import ( // Symbol table. func putelfstr(s string) int { - if len(elfstrdat) == 0 && s != "" { + if len(elfstrdat) == 0 { // first entry must be empty string - putelfstr("") + elfstrdat = append(elfstrdat, 0) + elfstroff = map[string]int{"": 0} + } + if off, ok := elfstroff[s]; ok { + return off } off := len(elfstrdat) elfstrdat = append(elfstrdat, s...) elfstrdat = append(elfstrdat, 0) + elfstroff[s] = off return off } diff --git a/src/crypto/internal/fips140/bigmod/nat.go b/src/crypto/internal/fips140/bigmod/nat.go index 0233f12639c00c..decb14c7ebd4c0 100644 --- a/src/crypto/internal/fips140/bigmod/nat.go +++ b/src/crypto/internal/fips140/bigmod/nat.go @@ -640,7 +640,7 @@ func (x *Nat) shiftIn(y uint, m *Modulus) *Nat { } // Like in maybeSubtractModulus, we need the subtraction if either it // didn't underflow (meaning 2x + b > m) or if computing 2x + b - // overflowed (meaning 2x + b > 2^_W*n > m). + // overflowed (meaning 2x + b > 2^(_W * n) > m). needSubtraction = not(choice(borrow)) | choice(carry) } return x.assign(needSubtraction, d) @@ -699,7 +699,7 @@ func (out *Nat) resetFor(m *Modulus) *Nat { // range for results computed by higher level operations. // // always is usually a carry that indicates that the operation that produced x -// overflowed its size, meaning abstractly x > 2^_W*n > m even if x < m. +// overflowed its size, meaning abstractly x > 2^(_W * n) > m even if x < m. // // x and m operands must have the same announced length. // @@ -1096,7 +1096,8 @@ func (x *Nat) GCDVarTime(a, b *Nat) (*Nat, error) { return x.set(u), nil } -// extendedGCD computes u and A such that u = GCD(a, m) = A*a - B*m. +// extendedGCD computes u = GCD(a, m). Additionally, if a < m, it computes A +// such that u = A*a - B*m. If a >= m, A is undefined. // // u will have the size of the larger of a and m, and A will have the size of m. // @@ -1125,6 +1126,8 @@ func extendedGCD(a, m *Nat) (u, A *Nat, err error) { // value. // // Note this algorithm does not handle either input being zero. + // + // See https://go.dev/issue/78218 for a Gobra proof of this implementation. if a.IsZero() == yes || m.IsZero() == yes { return nil, nil, errors.New("extendedGCD: a or m is zero") @@ -1146,29 +1149,34 @@ func extendedGCD(a, m *Nat) (u, A *Nat, err error) { // Before and after each loop iteration, the following hold: // - // u = A*a - B*m - // v = D*m - C*a - // 0 < u <= a - // 0 <= v <= m - // 0 <= A < m - // 0 <= B <= a - // 0 <= C < m - // 0 <= D <= a + // 0 < m + // 0 < u <= a + // 0 <= v <= m + // a or m is odd + // u or v is odd + // gcd(u, v) = gcd(a, m) + // + // If a < m, then the following also hold: // - // After each loop iteration, u and v only get smaller, and at least one of - // them shrinks by at least a factor of two. + // 0 <= A < m + // 0 <= B < a + // 0 <= C < m + // 0 <= D <= a + // u = A*a - B*m + // v = D*m - C*a + // + // After each loop iteration, u + v only gets smaller, and at least one of + // u and v shrinks by at least a factor of two. for { // If both u and v are odd, subtract the smaller from the larger. // If u = v, we need to subtract from v to hit the modified exit condition. if u.IsOdd() == yes && v.IsOdd() == yes { if v.cmpGeq(u) == no { u.sub(v) - A.Add(C, &Modulus{nat: m}) - B.Add(D, &Modulus{nat: a}) + syncAdd(A, C, B, D, m, a) } else { v.sub(u) - C.Add(A, &Modulus{nat: m}) - D.Add(B, &Modulus{nat: a}) + syncAdd(C, A, D, B, m, a) } } @@ -1199,11 +1207,29 @@ func extendedGCD(a, m *Nat) (u, A *Nat, err error) { } if v.IsZero() == yes { + // Base case: v = 0 -> gcd(a, m) = gcd(u, 0) = u. return u, A, nil } } } +// syncAdd adds Y to X and W to Z, then subtracts m from X and a from Z if +// X + Y >= m. This is synchronized single-subtraction modular reduction: +// X = (X + Y) mod m, with Z tracking the same wrap/no-wrap. +// +//go:norace +func syncAdd(X, Y, Z, W, m, a *Nat) { + c := X.add(Y) + Z.add(W) + + // Like in maybeSubtractModulus, we need the subtraction if either + // X + Y >= m, or if X + Y overflowed (meaning X + Y >= 2^(_W * n) > m). + if choice(c) == yes || X.cmpGeq(m) == yes { + X.sub(m) + Z.sub(a) + } +} + //go:norace func rshift1(a *Nat, carry uint) { size := len(a.limbs) diff --git a/src/internal/bytealg/compare_arm64.s b/src/internal/bytealg/compare_arm64.s index cc02c464e8b274..852a535779b985 100644 --- a/src/internal/bytealg/compare_arm64.s +++ b/src/internal/bytealg/compare_arm64.s @@ -39,11 +39,17 @@ TEXT cmpbody<>(SB),NOSPLIT|NOFRAME,$0-0 CSEL LT, R3, R1, R6 // R6 is min(R1, R3) CBZ R6, samebytes - BIC $0xf, R6, R10 - CBZ R10, small // length < 16 - ADD R0, R10 // end of chunk16 - // length >= 16 -chunk16_loop: + BIC $0x1f, R6, R10 + CBZ R10, chunk16 // length < 32, try single chunk16 + ADD R0, R10 // end of chunk32 + // length >= 32 +chunk32_loop: + LDP.P 16(R0), (R4, R8) + LDP.P 16(R2), (R5, R9) + CMP R4, R5 + BNE cmp + CMP R8, R9 + BNE cmpnext LDP.P 16(R0), (R4, R8) LDP.P 16(R2), (R5, R9) CMP R4, R5 @@ -51,7 +57,20 @@ chunk16_loop: CMP R8, R9 BNE cmpnext CMP R10, R0 - BNE chunk16_loop + BNE chunk32_loop + AND $0x1f, R6, R6 // remaining 0-31 bytes + CBZ R6, samebytes + // handle remaining 0-31 bytes: one possible 16-byte chunk then tail + TBZ $4, R6, small_direct +chunk16: + TBZ $4, R6, small // length < 16 + LDP.P 16(R0), (R4, R8) + LDP.P 16(R2), (R5, R9) + CMP R4, R5 + BNE cmp + CMP R8, R9 + BNE cmpnext +small_direct: AND $0xf, R6, R6 CBZ R6, samebytes SUBS $8, R6 diff --git a/src/internal/bytealg/equal_arm64.s b/src/internal/bytealg/equal_arm64.s index 408ab374e62994..8bc290da061dd9 100644 --- a/src/internal/bytealg/equal_arm64.s +++ b/src/internal/bytealg/equal_arm64.s @@ -32,83 +32,65 @@ TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25 CMP $16, R2 // handle specially if length < 16 BLO tail - BIC $0x3f, R2, R3 - CBZ R3, chunk16 - // work with 64-byte chunks - ADD R3, R0, R6 // end of chunks -chunk64_loop: - VLD1.P (R0), [V0.D2, V1.D2, V2.D2, V3.D2] - VLD1.P (R1), [V4.D2, V5.D2, V6.D2, V7.D2] - VCMEQ V0.D2, V4.D2, V8.D2 - VCMEQ V1.D2, V5.D2, V9.D2 - VCMEQ V2.D2, V6.D2, V10.D2 - VCMEQ V3.D2, V7.D2, V11.D2 - VAND V8.B16, V9.B16, V8.B16 - VAND V8.B16, V10.B16, V8.B16 - VAND V8.B16, V11.B16, V8.B16 - CMP R0, R6 - VMOV V8.D[0], R4 - VMOV V8.D[1], R5 - CBZ R4, not_equal - CBZ R5, not_equal - BNE chunk64_loop - AND $0x3f, R2, R2 - CBZ R2, equal -chunk16: - // work with 16-byte chunks - BIC $0xf, R2, R3 - CBZ R3, tail - ADD R3, R0, R6 // end of chunks -chunk16_loop: - LDP.P 16(R0), (R4, R5) - LDP.P 16(R1), (R7, R9) - EOR R4, R7 - CBNZ R7, not_equal - EOR R5, R9 - CBNZ R9, not_equal - CMP R0, R6 - BNE chunk16_loop - AND $0xf, R2, R2 - CBZ R2, equal + CMP $33, R2 + BHS large +pairwise_16_32: + // use pairwise loads for 16 <= len <= 32 + LDP (R0), (R16, R17) + LDP (R1), (R24, R26) + CMP R16, R24 + CCMP EQ, R17, R26, $0 + BNE not_equal + SUB $16, R2, R16 + CBZ R16, equal + ADD R0, R16, R24 + ADD R1, R16, R25 + LDP (R24), (R16, R17) + LDP (R25), (R24, R26) + CMP R16, R24 + CCMP EQ, R17, R26, $0 + CSET EQ, R0 + RET + PCALIGN $16 tail: // special compare of tail with length < 16 TBZ $3, R2, lt_8 - MOVD (R0), R4 - MOVD (R1), R5 - EOR R4, R5 - CBNZ R5, not_equal - SUB $8, R2, R6 // offset of the last 8 bytes - MOVD (R0)(R6), R4 - MOVD (R1)(R6), R5 - EOR R4, R5 - CBNZ R5, not_equal - B equal + MOVD (R0), R16 + MOVD (R1), R17 + CMP R16, R17 + BNE not_equal + SUB $8, R2, R26 // offset of the last 8 bytes + MOVD (R0)(R26), R16 + MOVD (R1)(R26), R17 + CMP R16, R17 + CSET EQ, R0 + RET PCALIGN $16 lt_8: TBZ $2, R2, lt_4 - MOVWU (R0), R4 - MOVWU (R1), R5 - EOR R4, R5 - CBNZ R5, not_equal - SUB $4, R2, R6 // offset of the last 4 bytes - MOVWU (R0)(R6), R4 - MOVWU (R1)(R6), R5 - EOR R4, R5 - CBNZ R5, not_equal - B equal + MOVWU (R0), R16 + MOVWU (R1), R17 + CMP R16, R17 + BNE not_equal + SUB $4, R2, R26 // offset of the last 4 bytes + MOVWU (R0)(R26), R16 + MOVWU (R1)(R26), R17 + CMP R16, R17 + CSET EQ, R0 + RET PCALIGN $16 lt_4: TBZ $1, R2, lt_2 - MOVHU.P 2(R0), R4 - MOVHU.P 2(R1), R5 - CMP R4, R5 + MOVHU.P 2(R0), R16 + MOVHU.P 2(R1), R17 + CMP R16, R17 BNE not_equal lt_2: TBZ $0, R2, equal one: - MOVBU (R0), R4 - MOVBU (R1), R5 - CMP R4, R5 + MOVBU (R0), R16 + MOVBU (R1), R17 + CMP R16, R17 BNE not_equal equal: MOVD $1, R0 @@ -116,3 +98,54 @@ equal: not_equal: MOVB ZR, R0 RET +large: + BIC $0x3f, R2, R26 + CBZ R26, remainder_33_64 + // work with 64-byte chunks + ADD R0, R26 // end of chunks +chunk64_loop: + VLD1.P (R0), [V21.D2, V22.D2, V23.D2, V24.D2] + VLD1.P (R1), [V25.D2, V26.D2, V27.D2, V28.D2] + VCMEQ V21.D2, V25.D2, V21.D2 + VCMEQ V22.D2, V26.D2, V22.D2 + VCMEQ V23.D2, V27.D2, V23.D2 + VCMEQ V24.D2, V28.D2, V24.D2 + VAND V21.B16, V22.B16, V21.B16 + VAND V23.B16, V24.B16, V23.B16 + VAND V21.B16, V23.B16, V21.B16 + CMP R0, R26 + VMOV V21.D[0], R16 + VMOV V21.D[1], R17 + CBZ R16, not_equal + CBZ R17, not_equal + BNE chunk64_loop + AND $0x3f, R2, R2 + CBZ R2, equal + CMP $16, R2 + BLO tail + CMP $33, R2 + BLO pairwise_16_32 +remainder_33_64: + // 33 <= len < 64 + VLD1 (R0), [V21.D2, V22.D2] + VLD1 (R1), [V23.D2, V24.D2] + SUB $32, R2, R26 + ADD R0, R26, R24 + ADD R1, R26, R25 + VEOR V23.B16, V21.B16, V21.B16 + VEOR V24.B16, V22.B16, V22.B16 + VORR V22.B16, V21.B16, V21.B16 + VMOV V21.D[0], R16 + VMOV V21.D[1], R17 + ORR R17, R16, R16 + CBNZ R16, not_equal + VLD1 (R24), [V21.D2, V22.D2] + VLD1 (R25), [V23.D2, V24.D2] + VEOR V23.B16, V21.B16, V21.B16 + VEOR V24.B16, V22.B16, V22.B16 + VORR V22.B16, V21.B16, V21.B16 + VMOV V21.D[0], R16 + VMOV V21.D[1], R17 + ORR R17, R16, R16 + CBNZ R16, not_equal + B equal diff --git a/src/net/netip/netip.go b/src/net/netip/netip.go index 57b8c6f70a3698..e14cf59b352cff 100644 --- a/src/net/netip/netip.go +++ b/src/net/netip/netip.go @@ -967,22 +967,22 @@ func (ip Addr) AppendText(b []byte) ([]byte, error) { // The encoding is the same as returned by [Addr.String], with one exception: // If ip is the zero [Addr], the encoding is the empty string. func (ip Addr) MarshalText() ([]byte, error) { - buf := []byte{} + var b []byte switch ip.z { case z0: case z4: - const maxCap = len("255.255.255.255") - buf = make([]byte, 0, maxCap) + const max = len("255.255.255.255") + b = make([]byte, 0, max) default: if ip.Is4In6() { - const maxCap = len("::ffff:255.255.255.255%enp5s0") - buf = make([]byte, 0, maxCap) - break + const max = len("::ffff:255.255.255.255%enp5s0") + b = make([]byte, 0, max) + } else { + const max = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0") + b = make([]byte, 0, max) } - const maxCap = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0") - buf = make([]byte, 0, maxCap) } - return ip.AppendText(buf) + return ip.AppendText(b) } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. @@ -1211,17 +1211,22 @@ func (p AddrPort) AppendText(b []byte) ([]byte, error) { // The encoding is the same as returned by [AddrPort.String], with one exception: // If p.Addr() is the zero [Addr], the encoding is the empty string. func (p AddrPort) MarshalText() ([]byte, error) { - buf := []byte{} + var b []byte switch p.ip.z { case z0: case z4: - const maxCap = len("255.255.255.255:65535") - buf = make([]byte, 0, maxCap) + const max = len("255.255.255.255:65535") + b = make([]byte, 0, max) default: - const maxCap = len("[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535") - buf = make([]byte, 0, maxCap) + if p.ip.Is4In6() { + const max = len("[::ffff:255.255.255.255%enp5s0]:65535") + b = make([]byte, 0, max) + } else { + const max = len("[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535") + b = make([]byte, 0, max) + } } - return p.AppendText(buf) + return p.AppendText(b) } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. diff --git a/src/net/parse.go b/src/net/parse.go index 106a303dfadae4..36b6e2aaa28067 100644 --- a/src/net/parse.go +++ b/src/net/parse.go @@ -182,7 +182,7 @@ func xtoi2(s string, e byte) (byte, bool) { // hasUpperCase tells whether the given string contains at least one upper-case. func hasUpperCase(s string) bool { - for i := range s { + for i := range len(s) { if 'A' <= s[i] && s[i] <= 'Z' { return true } diff --git a/src/path/filepath/path.go b/src/path/filepath/path.go index cd0fcc9c342464..579fe5d7a15ff4 100644 --- a/src/path/filepath/path.go +++ b/src/path/filepath/path.go @@ -225,6 +225,11 @@ func Rel(basePath, targPath string) (string, error) { if ti < tl { ti++ } + // If the last elements matched and both paths are now exhausted, + // the paths are equivalent. Without this check, Rel can loop forever. + if bi == bl && ti == tl { + return ".", nil + } b0 = bi t0 = ti } diff --git a/src/path/filepath/path_windows_test.go b/src/path/filepath/path_windows_test.go index d60903f62ebb53..ac7b2d7c6459d3 100644 --- a/src/path/filepath/path_windows_test.go +++ b/src/path/filepath/path_windows_test.go @@ -702,3 +702,45 @@ func TestAbsWindows(t *testing.T) { } } } + +func TestRelUNC(t *testing.T) { + tests := []struct { + base string + targ string + want string + wantErr bool + }{ + {`\\host\share`, `\\host\share`, `.`, false}, + {`\\host\share`, `\\host\share\`, `.`, false}, + {`\\host\share`, filepath.Clean(`\\host\share\`), `.`, false}, + {`\\host\share`, `\\host\share\file.txt`, `file.txt`, false}, + {`\\host\share\`, `\\host\share\file.txt`, `file.txt`, false}, + {`\\HOST\SHARE`, `\\host\share\file.txt`, `file.txt`, false}, + {`\\host\share\dir`, `\\host\share\dir\file.txt`, `file.txt`, false}, + {`\\host\share\dir`, `\\host\share\file.txt`, `..\file.txt`, false}, + {`\\host\share\dir\file.txt`, `\\host\share\dir`, `..`, false}, + {`\\host\share`, `\\host\other\file.txt`, ``, true}, + {`\\host\share`, filepath.Clean(`\\host\other\`), ``, true}, + {`\\host\share`, filepath.Clean(`\\other\share\`), ``, true}, + // TODO: These cases should be supported, but currently fail. + {`\\host\share\`, `\\host\share`, `.`, true}, + {`\\host\share\dir`, `\\host\share`, `..`, true}, + } + + for _, test := range tests { + got, err := filepath.Rel(test.base, test.targ) + if test.wantErr { + if err == nil { + t.Errorf("Rel(%q, %q) = %q, want error", test.base, test.targ, got) + } + continue + } + if err != nil { + t.Errorf("Rel(%q, %q): want %q, got error: %s", test.base, test.targ, test.want, err) + continue + } + if got != test.want { + t.Errorf("Rel(%q, %q) = %q, want %q", test.base, test.targ, got, test.want) + } + } +} diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index 42b04505968f11..ab9c5b436d265a 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -101,6 +101,7 @@ nocgo: ADD $16, X2 // start this M + CALL runtime·stackcheck(SB) // fault if stack check is wrong CALL runtime·mstart(SB) WORD $0 // crash if reached @@ -274,6 +275,17 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 MOV ZERO, CTXT JMP runtime·morestack(SB) +// check that SP is in range [g->stack.lo, g->stack.hi] +TEXT runtime·stackcheck(SB), NOSPLIT|NOFRAME, $0-0 + MOV (g_stack+stack_hi)(g), A0 + BGEU A0, X2, 2(PC) + CALL runtime·abort(SB) + + MOV (g_stack+stack_lo)(g), A0 + BGTU X2, A0, 2(PC) + CALL runtime·abort(SB) + RET + // restore state from Gobuf; longjmp // func gogo(buf *gobuf) diff --git a/src/runtime/cpuprof.go b/src/runtime/cpuprof.go index ea4d3a8cb0bb33..e36a2880cbe15d 100644 --- a/src/runtime/cpuprof.go +++ b/src/runtime/cpuprof.go @@ -66,6 +66,13 @@ var cpuprof cpuProfile // the [testing] package's -test.cpuprofile flag instead of calling // SetCPUProfileRate directly. func SetCPUProfileRate(hz int) { + setCPUProfileRate(hz, true) +} + +// setCPUProfileRate sets the CPU profiling rate to hz. Setting a non-zero rate +// when the rate is already non-zero is a no-op. An error is printed in this case +// if warn is true. +func setCPUProfileRate(hz int, warn bool) { // Clamp hz to something reasonable. if hz < 0 { hz = 0 @@ -77,7 +84,9 @@ func SetCPUProfileRate(hz int) { lock(&cpuprof.lock) if hz > 0 { if cpuprof.on || cpuprof.log != nil { - print("runtime: cannot set cpu profile rate until previous profile has finished.\n") + if warn { + print("runtime: cannot set cpu profile rate until previous profile has finished.\n") + } unlock(&cpuprof.lock) return } @@ -209,6 +218,15 @@ func CPUProfile() []byte { panic("CPUProfile no longer available") } +// pprof_setCPUProfileRate is provided to runtime/pprof.StartCPUProfile, +// to enable CPU profiling without logging an error if the user has already +// configured the profiling rate. +// +//go:linkname pprof_setCPUProfileRate +func pprof_setCPUProfileRate(hz int) { + setCPUProfileRate(hz, false) +} + // runtime/pprof.runtime_cyclesPerSecond should be an internal detail, // but widely used packages access it using linkname. // Notable members of the hall of shame include: diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 8c5ec383f7b2f8..42399c6f8ea7b6 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -1180,10 +1180,6 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { if asanenabled { // Poison the space between the end of the requested size of x // and the end of the slot. Unpoison the requested allocation. - frag := elemsize - size - if typ != nil && typ.Pointers() && !heapBitsInSpan(elemsize) && size <= maxSmallSize-gc.MallocHeaderSize { - frag -= gc.MallocHeaderSize - } asanpoison(unsafe.Add(x, size-asanRZ), asanRZ) asanunpoison(x, size-asanRZ) } diff --git a/src/runtime/pprof/pprof.go b/src/runtime/pprof/pprof.go index 1c116ad21dcd7b..3a82ff49cbe654 100644 --- a/src/runtime/pprof/pprof.go +++ b/src/runtime/pprof/pprof.go @@ -904,7 +904,7 @@ func StartCPUProfile(w io.Writer) error { return fmt.Errorf("cpu profiling already in use") } cpu.profiling = true - runtime.SetCPUProfileRate(hz) + pprof_setCPUProfileRate(hz) go profileWriter(w) return nil } @@ -952,7 +952,7 @@ func StopCPUProfile() { return } cpu.profiling = false - runtime.SetCPUProfileRate(0) + pprof_setCPUProfileRate(0) <-cpu.done } @@ -1055,3 +1055,6 @@ func pprof_fpunwindExpand(dst, src []uintptr) int //go:linkname pprof_makeProfStack runtime.pprof_makeProfStack func pprof_makeProfStack() []uintptr + +//go:linkname pprof_setCPUProfileRate runtime.pprof_setCPUProfileRate +func pprof_setCPUProfileRate(hz int) diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index 02ae875c5445b1..ac6e77765c3fe2 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -663,6 +663,45 @@ func TestCPUProfileWithFork(t *testing.T) { } } +func TestCPUProfileRateErrorLog(t *testing.T) { + testenv.MustHaveExec(t) + + switch os.Getenv("GO_TEST_CPU_PROFILE_RATE_SCENARIO") { + case "DoubleSet": + runtime.SetCPUProfileRate(100) + runtime.SetCPUProfileRate(500) // should log error: profile already active + runtime.SetCPUProfileRate(0) + return + case "SetThenStart": + runtime.SetCPUProfileRate(500) + if err := StartCPUProfile(io.Discard); err != nil { + fmt.Fprintf(os.Stderr, "StartCPUProfile: unexpected error: %v\n", err) + } + StopCPUProfile() + return + } + + for _, tc := range []struct { + scenario string + want string + }{ + {"DoubleSet", "runtime: cannot set cpu profile rate until previous profile has finished."}, + {"SetThenStart", ""}, + } { + t.Run(tc.scenario, func(t *testing.T) { + cmd := testenv.CleanCmdEnv(testenv.Command(t, testenv.Executable(t), + "-test.run="+"^"+t.Name()+"$")) + cmd.Env = append(cmd.Env, "GO_TEST_CPU_PROFILE_RATE_SCENARIO="+tc.scenario) + var stderr bytes.Buffer + cmd.Stderr = &stderr + cmd.Run() + if got := strings.TrimSpace(stderr.String()); got != tc.want { + t.Errorf("got error output %q, wanted %q", got, tc.want) + } + }) + } +} + // Test that profiler does not observe runtime.gogo as "user" goroutine execution. // If it did, it would see inconsistent state and would either record an incorrect stack // or crash because the stack was malformed. diff --git a/src/runtime/proc.go b/src/runtime/proc.go index b8d4d15041a54e..9edb1f57bb853d 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1118,7 +1118,7 @@ func (mp *m) hasCgoOnStack() bool { const ( // osHasLowResTimer indicates that the platform's internal timer system has a low resolution, // typically on the order of 1 ms or more. - osHasLowResTimer = GOOS == "windows" || GOOS == "openbsd" || GOOS == "netbsd" + osHasLowResTimer = GOOS == "windows" || GOOS == "openbsd" || GOOS == "netbsd" || GOOS == "plan9" // osHasLowResClockInt is osHasLowResClock but in integer form, so it can be used to create // constants conditionally. diff --git a/src/runtime/sys_freebsd_riscv64.s b/src/runtime/sys_freebsd_riscv64.s index d01bfa314c09b6..14b3a883e08d9e 100644 --- a/src/runtime/sys_freebsd_riscv64.s +++ b/src/runtime/sys_freebsd_riscv64.s @@ -79,7 +79,7 @@ TEXT runtime·thr_start(SB),NOSPLIT,$0 // set up g MOV m_g0(A0), g MOV A0, g_m(g) - CALL emptyfunc<>(SB) // fault if stack check is wrong + CALL runtime·stackcheck(SB) // fault if stack check is wrong CALL runtime·mstart(SB) WORD $0 // crash diff --git a/src/runtime/sys_linux_riscv64.s b/src/runtime/sys_linux_riscv64.s index 0d523667627151..72534f4128689c 100644 --- a/src/runtime/sys_linux_riscv64.s +++ b/src/runtime/sys_linux_riscv64.s @@ -488,6 +488,7 @@ good: // In child, set up new stack MOV T0, g_m(T1) MOV T1, g + CALL runtime·stackcheck(SB) // fault if stack check is wrong nog: // Call fn diff --git a/src/runtime/sys_openbsd_riscv64.s b/src/runtime/sys_openbsd_riscv64.s index bfdd9e17385a10..318e01931247a8 100644 --- a/src/runtime/sys_openbsd_riscv64.s +++ b/src/runtime/sys_openbsd_riscv64.s @@ -50,6 +50,7 @@ TEXT runtime·mstart_stub(SB),NOSPLIT,$200 MOV m_g0(X10), g CALL runtime·save_g(SB) + CALL runtime·stackcheck(SB) // fault if stack check is wrong CALL runtime·mstart(SB) // Restore callee-save registers. diff --git a/src/simd/archsimd/_gen/go.mod b/src/simd/archsimd/_gen/go.mod index f297b790c9b061..515f54f3b321ec 100644 --- a/src/simd/archsimd/_gen/go.mod +++ b/src/simd/archsimd/_gen/go.mod @@ -1,6 +1,6 @@ module simd/archsimd/_gen -go 1.25.0 +go 1.26.5 require ( golang.org/x/arch v0.26.0 diff --git a/src/simd/archsimd/_gen/midway/intersect_simd_ops.go b/src/simd/archsimd/_gen/midway/intersect_simd_ops.go index 141ccc24923173..4df2892bed956b 100644 --- a/src/simd/archsimd/_gen/midway/intersect_simd_ops.go +++ b/src/simd/archsimd/_gen/midway/intersect_simd_ops.go @@ -415,7 +415,7 @@ func main() { pf := func(f string, s ...any) { fmt.Fprintf(w, f, s...) } - fmt.Fprintln(w, + fmt.Fprint(w, `// Code generated by 'go run -C $GOROOT/src/simd/archsimd/_gen/midway'; DO NOT EDIT. //go:build goexperiment.simd @@ -444,7 +444,7 @@ type _simd bridge.ZeroSized pf := func(f string, s ...any) { fmt.Fprintf(w, f, s...) } nl := func() { fmt.Fprintln(w) } - fmt.Fprintln(w, + fmt.Fprint(w, `// Code generated by 'go run -C $GOROOT/src/simd/archsimd/_gen/midway'; DO NOT EDIT. //go:build goexperiment.simd && (amd64 || wasm || arm64) diff --git a/src/simd/archsimd/_gen/simdgen/arm64/instruction.go b/src/simd/archsimd/_gen/simdgen/arm64/instruction.go index c968b2c2c4118d..598b6284ff58b1 100644 --- a/src/simd/archsimd/_gen/simdgen/arm64/instruction.go +++ b/src/simd/archsimd/_gen/simdgen/arm64/instruction.go @@ -7,6 +7,7 @@ package arm64 import ( "fmt" "regexp" + "slices" "sort" "strconv" "strings" @@ -132,10 +133,8 @@ func (instruction *Instruction) ResultInArg0() bool { // Check pseudocode for reading destination register for _, PsSection := range instruction.PsSections { for _, Ps := range PsSection.Ps { - for _, pstext := range Ps.PSText { - if resultInArg0Re.MatchString(pstext) { - return true - } + if slices.ContainsFunc(Ps.PSText, resultInArg0Re.MatchString) { + return true } } } @@ -257,16 +256,16 @@ func (instruction *Instruction) extractFixedArrangements() []string { for _, class := range instruction.Classes.Iclass { for _, encoding := range class.Encodings { - var templateStr string + var templateStr strings.Builder for _, content := range encoding.AsmTemplate.TextA { if content.Link == "sa_t" || content.Link == "sa_ta" { return []string{} } - templateStr += content.Value + templateStr.WriteString(content.Value) } // Extract hardcoded arrangements like ".16B", ".4S", ".8H", ".2D" - matches := fixedArrangementRe.FindAllStringSubmatch(templateStr, -1) + matches := fixedArrangementRe.FindAllStringSubmatch(templateStr.String(), -1) for _, match := range matches { if len(match) > 1 { arrangement := match[1] diff --git a/src/simd/archsimd/_gen/simdgen/arm64/instruction_test.go b/src/simd/archsimd/_gen/simdgen/arm64/instruction_test.go index 0524e54e450160..48a5aa7748de3f 100644 --- a/src/simd/archsimd/_gen/simdgen/arm64/instruction_test.go +++ b/src/simd/archsimd/_gen/simdgen/arm64/instruction_test.go @@ -16,7 +16,7 @@ import ( var arm64Path = flag.String("arm64Path", "", "Path to ARM64 XML definitions") -func requireEqual(t *testing.T, expected, actual interface{}) bool { +func requireEqual(t *testing.T, expected, actual any) bool { t.Helper() if !reflect.DeepEqual(expected, actual) { t.Errorf("❌ expected %v, got %v", expected, actual) @@ -32,7 +32,7 @@ func requireEqual(t *testing.T, expected, actual interface{}) bool { return true } -func matchEqual(t *testing.T, expected, actual interface{}) bool { +func matchEqual(t *testing.T, expected, actual any) bool { t.Helper() eq := reflect.DeepEqual(expected, actual) if eq && expected != nil { @@ -57,7 +57,7 @@ func arngs(t *testing.T, instr *Instruction, expectedArrangements []string, expe requireEqual(t, expectedShape, actualShape) } -func ops(t *testing.T, instr *Instruction, expectedOps []string, equal func(*testing.T, interface{}, interface{}) bool) bool { +func ops(t *testing.T, instr *Instruction, expectedOps []string, equal func(*testing.T, any, any) bool) bool { t.Helper() templates := instr.templates() if !equal(t, 1, len(templates)) { diff --git a/src/simd/archsimd/_gen/simdgen/arm64/operands.go b/src/simd/archsimd/_gen/simdgen/arm64/operands.go index aa9e84acbae54a..e6e49fdc923883 100644 --- a/src/simd/archsimd/_gen/simdgen/arm64/operands.go +++ b/src/simd/archsimd/_gen/simdgen/arm64/operands.go @@ -86,10 +86,7 @@ func (op *Operand) instantiate(arrangement Arrangement, ashape ArngShape, vregPo op.Lanes = arrangement.bits / (arrangement.elemBits / 2) case ashape == LongArngs && vregPos == 0: op.ElemBits = arrangement.elemBits * 2 - op.Bits = arrangement.bits * 2 - if op.Bits > 128 { - op.Bits = 128 - } + op.Bits = min(arrangement.bits*2, 128) op.Lanes = arrangement.bits / op.ElemBits case ashape == WideArngs && vregPos == 2: op.ElemBits = arrangement.elemBits / 2 @@ -203,8 +200,8 @@ func tokenizeTemplate(template string) []token { // stripMnemonic removes the instruction mnemonic from the template string. // For example, "ADD ., ., ." becomes "., ., .". func stripMnemonic(template string) string { - if idx := strings.Index(template, " "); idx >= 0 { - return strings.TrimSpace(template[idx+1:]) + if _, after, ok := strings.Cut(template, " "); ok { + return strings.TrimSpace(after) } return template } diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdrules.go b/src/simd/archsimd/_gen/simdgen/gen_simdrules.go index a09b208486e7d5..4b844759cc9be7 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdrules.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdrules.go @@ -131,13 +131,13 @@ func compareTplRuleData(x, y tplRuleData) int { // lowering for an instruction which doesn't support zero immediate encoding: // (VUSRA4S [a] x y) && a==0 => (VADD4S x y) func parseAsmRule(rule string) (bool, string, string) { - arrowIndex := strings.Index(rule, "=>") - if arrowIndex == -1 { + before, after, ok := strings.Cut(rule, "=>") + if !ok { return false, "", "" } - condPart := rule[:arrowIndex] - outPart := rule[arrowIndex+len("=>"):] + condPart := before + outPart := after // Check if condPart starts with "if" followed by at least one space. cond := strings.TrimPrefix(condPart, "if") diff --git a/src/simd/archsimd/_gen/unify/env.go b/src/simd/archsimd/_gen/unify/env.go index 870f1307699ff2..f5c0d487518d5c 100644 --- a/src/simd/archsimd/_gen/unify/env.go +++ b/src/simd/archsimd/_gen/unify/env.go @@ -8,6 +8,7 @@ import ( "fmt" "iter" "reflect" + "slices" "strings" "sync/atomic" ) @@ -389,10 +390,8 @@ type smallSet[T comparable] struct { // Has returns whether val is in set. func (s *smallSet[T]) Has(val T) bool { arr := s.array[:s.n] - for i := range arr { - if arr[i] == val { - return true - } + if slices.Contains(arr, val) { + return true } _, ok := s.m[val] return ok diff --git a/src/simd/archsimd/_gen/unify/value.go b/src/simd/archsimd/_gen/unify/value.go index ffc25b8728bc51..e4e688eb7406c6 100644 --- a/src/simd/archsimd/_gen/unify/value.go +++ b/src/simd/archsimd/_gen/unify/value.go @@ -140,7 +140,7 @@ type Decoder interface { DecodeUnified(v *Value) error } -var decoderType = reflect.TypeOf((*Decoder)(nil)).Elem() +var decoderType = reflect.TypeFor[Decoder]() // Provenance iterates over all of the source Values that have contributed to // this Value. diff --git a/src/simd/archsimd/_gen/wasmgen/main.go b/src/simd/archsimd/_gen/wasmgen/main.go index 0e39c5632a0f88..4c23183022c676 100644 --- a/src/simd/archsimd/_gen/wasmgen/main.go +++ b/src/simd/archsimd/_gen/wasmgen/main.go @@ -263,15 +263,15 @@ var allFlags = []OpFlags{ func (o OpFlags) String() string { sep := "" - ret := "" + var ret strings.Builder for _, x := range allFlags { if x&o != 0 { - ret += sep + x.OneString() + ret.WriteString(sep + x.OneString()) sep = "+" } } - return ret + return ret.String() } // wasmOp represents a WebAssembly SIMD instruction. @@ -337,7 +337,7 @@ func (o *wasmOp) ImmName() string { func snakeToCamel(s string) string { capnext := true - result := "" + var result strings.Builder for _, c := range s { if c == '_' { capnext = true @@ -351,9 +351,9 @@ func snakeToCamel(s string) string { capnext = false } } - result += string(c) + result.WriteString(string(c)) } - return result + return result.String() } // Op returns the snakeToCamel version of the WASM operation, diff --git a/test/codegen/bool.go b/test/codegen/bool.go index 22c86ba77956b9..86d73de1d4a213 100644 --- a/test/codegen/bool.go +++ b/test/codegen/bool.go @@ -328,3 +328,20 @@ func boolCompare2(p, q *bool) int { } return 7 } + +func branchlessBoolToUint8(b bool) (r uint8) { + if b { + r = 1 + } + return +} + +func xorBranchlessBoolToUintFoldsIntoCompare(x, y int) uint8 { + // amd64:"SETNE" "CMPQ" -"XOR" + return branchlessBoolToUint8(x == y) ^ 1 +} + +func otherXorBranchlessBoolToUintRemovesNot(a bool) uint8 { + // amd64:"XORL [$]2" -"XORL [$]1" -"XORL [$]3" + return branchlessBoolToUint8(!a) ^ 3 +} diff --git a/test/codegen/condmove.go b/test/codegen/condmove.go index 93f0e06eb9fc86..8366961726a9d4 100644 --- a/test/codegen/condmove.go +++ b/test/codegen/condmove.go @@ -535,10 +535,30 @@ func cmovmathadd8else(a uint, b bool) uint { return a } +func cmovmathadd16(a uint, b bool) uint { + if b { + a += 16 + } + // amd64:"ADDQ" -"CMOV" + // arm64:"ADD R[0-9]+<<4" -"CSEL" -"MUL" + // ppc64x: "ISEL" -"MUL" + return a +} +func cmovmathadd16else(a uint, b bool) uint { + if !b { + a += 16 + } + // amd64:"ADDQ" -"CMOV" + // arm64:"ADD R[0-9]+<<4" -"CSEL" -"MUL" + // ppc64x: "ISEL" -"MUL" + return a +} + func cmovmathadd9223372036854775808(a uint, b bool) uint { if b { a += 1 << 63 } + // amd64:"ADDQ" -"CMOV" // arm64:"ADD R[0-9]+<<63" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a @@ -547,6 +567,7 @@ func cmovmathadd9223372036854775808else(a uint, b bool) uint { if !b { a += 1 << 63 } + // amd64:"ADDQ" -"CMOV" // arm64:"ADD R[0-9]+<<63" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a @@ -577,6 +598,7 @@ func cmovmathsub2(a uint, b bool) uint { if b { a -= 2 } + // amd64:"SUBQ" -"CMOV" // arm64 :"SUB R[0-9]+<<1" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a @@ -585,6 +607,7 @@ func cmovmathsub2else(a uint, b bool) uint { if !b { a -= 2 } + // amd64:"SUBQ" -"CMOV" // arm64 :"SUB R[0-9]+<<1" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a @@ -596,6 +619,7 @@ func cmovmathsub9223372036854775808(a uint, b bool) uint { if b { a -= 1 << 63 } + // amd64:"(SUBQ|ADDQ)" -"CMOV" // arm64:"(SUB|ADD) R[0-9]+<<63" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a @@ -604,6 +628,7 @@ func cmovmathsub9223372036854775808else(a uint, b bool) uint { if !b { a -= 1 << 63 } + // amd64:"(SUBQ|ADDQ)" -"CMOV" // arm64:"(SUB|ADD) R[0-9]+<<63" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a @@ -699,6 +724,7 @@ func cmovmathor2(a uint, b bool) uint { if b { a |= 2 } + // amd64:"ORQ" -"CMOV" // arm64:"ORR R[0-9]+<<1" -"CSEL" -"MUL" // ppc64x:"ISEL" -"MUL" return a @@ -707,6 +733,7 @@ func cmovmathor2else(a uint, b bool) uint { if !b { a |= 2 } + // amd64:"ORQ" -"CMOV" // arm64:"ORR R[0-9]+<<1" -"CSEL" -"MUL" // ppc64x:"ISEL" -"MUL" return a @@ -716,6 +743,7 @@ func cmovmathor9223372036854775808(a uint, b bool) uint { if b { a |= 1 << 63 } + // amd64:"ORQ" -"CMOV" // arm64:"ORR R[0-9]+<<63" -"CSEL" -"MUL" // ppc64x:"ISEL" -"MUL" return a @@ -724,6 +752,7 @@ func cmovmathor9223372036854775808else(a uint, b bool) uint { if !b { a |= 1 << 63 } + // amd64:"ORQ" -"CMOV" // arm64:"ORR R[0-9]+<<63" -"CSEL" -"MUL" // ppc64x:"ISEL" -"MUL" return a @@ -754,6 +783,7 @@ func cmovmathxor2(a uint, b bool) uint { if b { a ^= 2 } + // amd64:"XORQ" -"CMOV" // arm64:"EOR R[0-9]+<<1" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a @@ -762,6 +792,7 @@ func cmovmathxor2else(a uint, b bool) uint { if !b { a ^= 2 } + // amd64:"XORQ" -"CMOV" // arm64:"EOR R[0-9]+<<1" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a @@ -771,6 +802,7 @@ func cmovmathxor9223372036854775808(a uint, b bool) uint { if b { a ^= 1 << 63 } + // amd64:"XORQ" -"CMOV" // arm64:"EOR R[0-9]+<<63" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a @@ -779,6 +811,7 @@ func cmovmathxor9223372036854775808else(a uint, b bool) uint { if !b { a ^= 1 << 63 } + // amd64:"XORQ" -"CMOV" // arm64:"EOR R[0-9]+<<63" -"CSEL" -"MUL" // ppc64x: "ISEL" -"MUL" return a diff --git a/test/codegen/mathbits.go b/test/codegen/mathbits.go index 4d02bfb96c240b..60db94d6a93f2a 100644 --- a/test/codegen/mathbits.go +++ b/test/codegen/mathbits.go @@ -261,6 +261,28 @@ func Reverse8(n uint8) uint8 { return bits.Reverse8(n) } +func Reverse64Const() uint64 { + // arm64:-"RBIT" + // loong64:-"BITREV" + return bits.Reverse64(0x0102030405060708) +} + +func Reverse32Const() uint32 { + // arm64:-"RBITW" + // loong64:-"BITREV" + return bits.Reverse32(0x01020304) +} + +func Reverse16Const() uint16 { + // loong64:-"BITREV" + return bits.Reverse16(0x0102) +} + +func Reverse8Const() uint8 { + // loong64:-"BITREV" + return bits.Reverse8(0x12) +} + // ----------------------- // // bits.ReverseBytes // // ----------------------- // @@ -309,6 +331,26 @@ func ReverseBytes16(n uint16) uint16 { return bits.ReverseBytes16(n) } +func ReverseBytes64Const() uint64 { + // amd64:-"BSWAPQ" + // arm64:-"REV" + // s390x:-"MOVDBR" + return bits.ReverseBytes64(0x0102030405060708) +} + +func ReverseBytes32Const() uint32 { + // amd64:-"BSWAPL" + // arm64:-"REVW" + // s390x:-"MOVWBR" + return bits.ReverseBytes32(0x01020304) +} + +func ReverseBytes16Const() uint16 { + // amd64:-"ROLW" + // arm64:-"REV16W" + return bits.ReverseBytes16(0x0102) +} + // --------------------- // // bits.RotateLeft // // --------------------- // @@ -981,6 +1023,12 @@ func Mul64Const() (uint64, uint64) { return bits.Mul64(99+88<<8, 1<<56) } +func Mul64ConstPow2(x uint64) (uint64, uint64) { + // amd64:-"MULQ" + // arm64:-"UMULH" -"\tMUL\t" + return bits.Mul64(x, 1<<40) +} + func MulUintOverflow(p *uint64) []uint64 { // arm64:"CMP [$]72" return unsafe.Slice(p, 9) diff --git a/test/codegen/memcombine.go b/test/codegen/memcombine.go index 55af97870761ff..f71c806ade9dcc 100644 --- a/test/codegen/memcombine.go +++ b/test/codegen/memcombine.go @@ -344,6 +344,52 @@ func load_be_byte8_uint64_idx8(s []byte, idx int) uint64 { return uint64(s[idx<<3])<<56 | uint64(s[(idx<<3)+1])<<48 | uint64(s[(idx<<3)+2])<<40 | uint64(s[(idx<<3)+3])<<32 | uint64(s[(idx<<3)+4])<<24 | uint64(s[(idx<<3)+5])<<16 | uint64(s[(idx<<3)+6])<<8 | uint64(s[(idx<<3)+7]) } +// Check combining of different int types + +func load_le_2uint16_uint32(s []uint16) uint32 { + // arm64:`MOVWU \(R[0-9]+\)` -`ORR` -`MOVHU` + // 386:`MOVL \([A-Z]+\)` -`MOVWLZX` -`ORL` + // amd64:`MOVL \([A-Z]+\)` -`MOVWLZX` -`ORL` + // ppc64le:`MOVWZ \(R[0-9]+\)` -`MOVHZ` + return uint32(s[0]) | uint32(s[1])<<16 +} + +func load_le_2uint16_uint32_idx(s []uint16, idx int) uint32 { + // arm64:`MOVWU \(R[0-9]+\)` -`ORR` -`MOVHU` + // 386:`MOVL \([A-Z]+\)` -`MOVWLZX` -`ORL` + // amd64:`MOVL \([A-Z]+\)` -`MOVWLZX` -`ORL` + // ppc64le:`MOVWZ \(R[0-9]+\)` -`MOVHZ` + return uint32(s[idx]) | uint32(s[idx+1])<<16 +} + +func load_le_4uint16_uint64(s []uint16) uint64 { + // arm64:`MOVD \(R[0-9]+\)` -`ORR` -`MOVHU` + // amd64:`MOVQ \([A-Z]+\)` -`ORQ` -`MOVWLZX` + // ppc64le:`MOVD \(R[0-9]+\)` -`MOVHZ` + return uint64(s[0]) | uint64(s[1])<<16 | uint64(s[2])<<32 | uint64(s[3])<<48 +} + +func load_le_4uint16_uint64_idx(s []uint16, idx int) uint64 { + // arm64:`MOVD \(R[0-9]+\)` -`ORR` -`MOVHU` + // amd64:`MOVQ \([A-Z]+\)` -`ORQ` -`MOVWLZX` + // ppc64le:`MOVD \(R[0-9]+\)` -`MOVHZ` + return uint64(s[idx]) | uint64(s[idx+1])<<16 | uint64(s[idx+2])<<32 | uint64(s[idx+3])<<48 +} + +func load_le_2uint32_uint64(s []uint32) uint64 { + // arm64:`MOVD \(R[0-9]+\)` -`ORR` -`LDPW` -`MOVWU` + // amd64:`MOVQ \([A-Z]+\)` -`ORQ` -`MOVL` + // ppc64le:`MOVD \(R[0-9]+\)` -`MOVWZ` + return uint64(s[0]) | uint64(s[1])<<32 +} + +func load_le_2uint32_uint64_idx(s []uint32, idx int) uint64 { + // arm64:`MOVD \(R[0-9]+\)` -`ORR` -`LDPW` -`MOVWU` + // amd64:`MOVQ \([A-Z]+\)` -`ORQ` -`MOVL` + // ppc64le:`MOVD \(R[0-9]+\)` -`MOVWZ` + return uint64(s[idx]) | uint64(s[idx+1])<<32 +} + // Some tougher cases for the memcombine pass. func reassoc_load_uint32(b []byte) uint32 { diff --git a/test/fixedbugs/issue80141.go b/test/fixedbugs/issue80141.go new file mode 100644 index 00000000000000..365cdc3598c6cc --- /dev/null +++ b/test/fixedbugs/issue80141.go @@ -0,0 +1,1016 @@ +// compile + +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type T [16]byte + +func f() (T, error) { return T{}, nil } +func g(t T, err error) T { return t } + +var ( + v0 = "x" + v1 = "x" + v2 = "x" + v3 = "x" + v4 = "x" + v5 = "x" + v6 = "x" + v7 = "x" + v8 = "x" + v9 = "x" + v10 = "x" + v11 = "x" + v12 = "x" + v13 = "x" + v14 = "x" + v15 = "x" + v16 = "x" + v17 = "x" + v18 = "x" + v19 = "x" + v20 = "x" + v21 = "x" + v22 = "x" + v23 = "x" + v24 = "x" + v25 = "x" + v26 = "x" + v27 = "x" + v28 = "x" + v29 = "x" + v30 = "x" + v31 = "x" + v32 = "x" + v33 = "x" + v34 = "x" + v35 = "x" + v36 = "x" + v37 = "x" + v38 = "x" + v39 = "x" + v40 = "x" + v41 = "x" + v42 = "x" + v43 = "x" + v44 = "x" + v45 = "x" + v46 = "x" + v47 = "x" + v48 = "x" + v49 = "x" + v50 = "x" + v51 = "x" + v52 = "x" + v53 = "x" + v54 = "x" + v55 = "x" + v56 = "x" + v57 = "x" + v58 = "x" + v59 = "x" + v60 = "x" + v61 = "x" + v62 = "x" + v63 = "x" + v64 = "x" + v65 = "x" + v66 = "x" + v67 = "x" + v68 = "x" + v69 = "x" + v70 = "x" + v71 = "x" + v72 = "x" + v73 = "x" + v74 = "x" + v75 = "x" + v76 = "x" + v77 = "x" + v78 = "x" + v79 = "x" + v80 = "x" + v81 = "x" + v82 = "x" + v83 = "x" + v84 = "x" + v85 = "x" + v86 = "x" + v87 = "x" + v88 = "x" + v89 = "x" + v90 = "x" + v91 = "x" + v92 = "x" + v93 = "x" + v94 = "x" + v95 = "x" + v96 = "x" + v97 = "x" + v98 = "x" + v99 = "x" + v100 = "x" + v101 = "x" + v102 = "x" + v103 = "x" + v104 = "x" + v105 = "x" + v106 = "x" + v107 = "x" + v108 = "x" + v109 = "x" + v110 = "x" + v111 = "x" + v112 = "x" + v113 = "x" + v114 = "x" + v115 = "x" + v116 = "x" + v117 = "x" + v118 = "x" + v119 = "x" + v120 = "x" + v121 = "x" + v122 = "x" + v123 = "x" + v124 = "x" + v125 = "x" + v126 = "x" + v127 = "x" + v128 = "x" + v129 = "x" + v130 = "x" + v131 = "x" + v132 = "x" + v133 = "x" + v134 = "x" + v135 = "x" + v136 = "x" + v137 = "x" + v138 = "x" + v139 = "x" + v140 = "x" + v141 = "x" + v142 = "x" + v143 = "x" + v144 = "x" + v145 = "x" + v146 = "x" + v147 = "x" + v148 = "x" + v149 = "x" + v150 = "x" + v151 = "x" + v152 = "x" + v153 = "x" + v154 = "x" + v155 = "x" + v156 = "x" + v157 = "x" + v158 = "x" + v159 = "x" + v160 = "x" + v161 = "x" + v162 = "x" + v163 = "x" + v164 = "x" + v165 = "x" + v166 = "x" + v167 = "x" + v168 = "x" + v169 = "x" + v170 = "x" + v171 = "x" + v172 = "x" + v173 = "x" + v174 = "x" + v175 = "x" + v176 = "x" + v177 = "x" + v178 = "x" + v179 = "x" + v180 = "x" + v181 = "x" + v182 = "x" + v183 = "x" + v184 = "x" + v185 = "x" + v186 = "x" + v187 = "x" + v188 = "x" + v189 = "x" + v190 = "x" + v191 = "x" + v192 = "x" + v193 = "x" + v194 = "x" + v195 = "x" + v196 = "x" + v197 = "x" + v198 = "x" + v199 = "x" + v200 = "x" + v201 = "x" + v202 = "x" + v203 = "x" + v204 = "x" + v205 = "x" + v206 = "x" + v207 = "x" + v208 = "x" + v209 = "x" + v210 = "x" + v211 = "x" + v212 = "x" + v213 = "x" + v214 = "x" + v215 = "x" + v216 = "x" + v217 = "x" + v218 = "x" + v219 = "x" + v220 = "x" + v221 = "x" + v222 = "x" + v223 = "x" + v224 = "x" + v225 = "x" + v226 = "x" + v227 = "x" + v228 = "x" + v229 = "x" + v230 = "x" + v231 = "x" + v232 = "x" + v233 = "x" + v234 = "x" + v235 = "x" + v236 = "x" + v237 = "x" + v238 = "x" + v239 = "x" + v240 = "x" + v241 = "x" + v242 = "x" + v243 = "x" + v244 = "x" + v245 = "x" + v246 = "x" + v247 = "x" + v248 = "x" + v249 = "x" + v250 = "x" + v251 = "x" + v252 = "x" + v253 = "x" + v254 = "x" + v255 = "x" + v256 = "x" + v257 = "x" + v258 = "x" + v259 = "x" + v260 = "x" + v261 = "x" + v262 = "x" + v263 = "x" + v264 = "x" + v265 = "x" + v266 = "x" + v267 = "x" + v268 = "x" + v269 = "x" + v270 = "x" + v271 = "x" + v272 = "x" + v273 = "x" + v274 = "x" + v275 = "x" + v276 = "x" + v277 = "x" + v278 = "x" + v279 = "x" + v280 = "x" + v281 = "x" + v282 = "x" + v283 = "x" + v284 = "x" + v285 = "x" + v286 = "x" + v287 = "x" + v288 = "x" + v289 = "x" + v290 = "x" + v291 = "x" + v292 = "x" + v293 = "x" + v294 = "x" + v295 = "x" + v296 = "x" + v297 = "x" + v298 = "x" + v299 = "x" + v300 = "x" + v301 = "x" + v302 = "x" + v303 = "x" + v304 = "x" + v305 = "x" + v306 = "x" + v307 = "x" + v308 = "x" + v309 = "x" + v310 = "x" + v311 = "x" + v312 = "x" + v313 = "x" + v314 = "x" + v315 = "x" + v316 = "x" + v317 = "x" + v318 = "x" + v319 = "x" + v320 = "x" + v321 = "x" + v322 = "x" + v323 = "x" + v324 = "x" + v325 = "x" + v326 = "x" + v327 = "x" + v328 = "x" + v329 = "x" + v330 = "x" + v331 = "x" + v332 = "x" + v333 = "x" + v334 = "x" + v335 = "x" + v336 = "x" + v337 = "x" + v338 = "x" + v339 = "x" + v340 = "x" + v341 = "x" + v342 = "x" + v343 = "x" + v344 = "x" + v345 = "x" + v346 = "x" + v347 = "x" + v348 = "x" + v349 = "x" + v350 = "x" + v351 = "x" + v352 = "x" + v353 = "x" + v354 = "x" + v355 = "x" + v356 = "x" + v357 = "x" + v358 = "x" + v359 = "x" + v360 = "x" + v361 = "x" + v362 = "x" + v363 = "x" + v364 = "x" + v365 = "x" + v366 = "x" + v367 = "x" + v368 = "x" + v369 = "x" + v370 = "x" + v371 = "x" + v372 = "x" + v373 = "x" + v374 = "x" + v375 = "x" + v376 = "x" + v377 = "x" + v378 = "x" + v379 = "x" + v380 = "x" + v381 = "x" + v382 = "x" + v383 = "x" + v384 = "x" + v385 = "x" + v386 = "x" + v387 = "x" + v388 = "x" + v389 = "x" + v390 = "x" + v391 = "x" + v392 = "x" + v393 = "x" + v394 = "x" + v395 = "x" + v396 = "x" + v397 = "x" + v398 = "x" + v399 = "x" + v400 = "x" + v401 = "x" + v402 = "x" + v403 = "x" + v404 = "x" + v405 = "x" + v406 = "x" + v407 = "x" + v408 = "x" + v409 = "x" + v410 = "x" + v411 = "x" + v412 = "x" + v413 = "x" + v414 = "x" + v415 = "x" + v416 = "x" + v417 = "x" + v418 = "x" + v419 = "x" + v420 = "x" + v421 = "x" + v422 = "x" + v423 = "x" + v424 = "x" + v425 = "x" + v426 = "x" + v427 = "x" + v428 = "x" + v429 = "x" + v430 = "x" + v431 = "x" + v432 = "x" + v433 = "x" + v434 = "x" + v435 = "x" + v436 = "x" + v437 = "x" + v438 = "x" + v439 = "x" + v440 = "x" + v441 = "x" + v442 = "x" + v443 = "x" + v444 = "x" + v445 = "x" + v446 = "x" + v447 = "x" + v448 = "x" + v449 = "x" + v450 = "x" + v451 = "x" + v452 = "x" + v453 = "x" + v454 = "x" + v455 = "x" + v456 = "x" + v457 = "x" + v458 = "x" + v459 = "x" + v460 = "x" + v461 = "x" + v462 = "x" + v463 = "x" + v464 = "x" + v465 = "x" + v466 = "x" + v467 = "x" + v468 = "x" + v469 = "x" + v470 = "x" + v471 = "x" + v472 = "x" + v473 = "x" + v474 = "x" + v475 = "x" + v476 = "x" + v477 = "x" + v478 = "x" + v479 = "x" + v480 = "x" + v481 = "x" + v482 = "x" + v483 = "x" + v484 = "x" + v485 = "x" + v486 = "x" + v487 = "x" + v488 = "x" + v489 = "x" + v490 = "x" + v491 = "x" + v492 = "x" + v493 = "x" + v494 = "x" + v495 = "x" + v496 = "x" + v497 = "x" + v498 = "x" + v499 = "x" + v500 = "x" + v501 = "x" + v502 = "x" + v503 = "x" + v504 = "x" + v505 = "x" + v506 = "x" + v507 = "x" + v508 = "x" + v509 = "x" + v510 = "x" + v511 = "x" + v512 = "x" + v513 = "x" + v514 = "x" + v515 = "x" + v516 = "x" + v517 = "x" + v518 = "x" + v519 = "x" + v520 = "x" + v521 = "x" + v522 = "x" + v523 = "x" + v524 = "x" + v525 = "x" + v526 = "x" + v527 = "x" + v528 = "x" + v529 = "x" + v530 = "x" + v531 = "x" + v532 = "x" + v533 = "x" + v534 = "x" + v535 = "x" + v536 = "x" + v537 = "x" + v538 = "x" + v539 = "x" + v540 = "x" + v541 = "x" + v542 = "x" + v543 = "x" + v544 = "x" + v545 = "x" + v546 = "x" + v547 = "x" + v548 = "x" + v549 = "x" + v550 = "x" + v551 = "x" + v552 = "x" + v553 = "x" + v554 = "x" + v555 = "x" + v556 = "x" + v557 = "x" + v558 = "x" + v559 = "x" + v560 = "x" + v561 = "x" + v562 = "x" + v563 = "x" + v564 = "x" + v565 = "x" + v566 = "x" + v567 = "x" + v568 = "x" + v569 = "x" + v570 = "x" + v571 = "x" + v572 = "x" + v573 = "x" + v574 = "x" + v575 = "x" + v576 = "x" + v577 = "x" + v578 = "x" + v579 = "x" + v580 = "x" + v581 = "x" + v582 = "x" + v583 = "x" + v584 = "x" + v585 = "x" + v586 = "x" + v587 = "x" + v588 = "x" + v589 = "x" + v590 = "x" + v591 = "x" + v592 = "x" + v593 = "x" + v594 = "x" + v595 = "x" + v596 = "x" + v597 = "x" + v598 = "x" + v599 = "x" + v600 = "x" + v601 = "x" + v602 = "x" + v603 = "x" + v604 = "x" + v605 = "x" + v606 = "x" + v607 = "x" + v608 = "x" + v609 = "x" + v610 = "x" + v611 = "x" + v612 = "x" + v613 = "x" + v614 = "x" + v615 = "x" + v616 = "x" + v617 = "x" + v618 = "x" + v619 = "x" + v620 = "x" + v621 = "x" + v622 = "x" + v623 = "x" + v624 = "x" + v625 = "x" + v626 = "x" + v627 = "x" + v628 = "x" + v629 = "x" + v630 = "x" + v631 = "x" + v632 = "x" + v633 = "x" + v634 = "x" + v635 = "x" + v636 = "x" + v637 = "x" + v638 = "x" + v639 = "x" + v640 = "x" + v641 = "x" + v642 = "x" + v643 = "x" + v644 = "x" + v645 = "x" + v646 = "x" + v647 = "x" + v648 = "x" + v649 = "x" + v650 = "x" + v651 = "x" + v652 = "x" + v653 = "x" + v654 = "x" + v655 = "x" + v656 = "x" + v657 = "x" + v658 = "x" + v659 = "x" + v660 = "x" + v661 = "x" + v662 = "x" + v663 = "x" + v664 = "x" + v665 = "x" + v666 = "x" + v667 = "x" + v668 = "x" + v669 = "x" + v670 = "x" + v671 = "x" + v672 = "x" + v673 = "x" + v674 = "x" + v675 = "x" + v676 = "x" + v677 = "x" + v678 = "x" + v679 = "x" + v680 = "x" + v681 = "x" + v682 = "x" + v683 = "x" + v684 = "x" + v685 = "x" + v686 = "x" + v687 = "x" + v688 = "x" + v689 = "x" + v690 = "x" + v691 = "x" + v692 = "x" + v693 = "x" + v694 = "x" + v695 = "x" + v696 = "x" + v697 = "x" + v698 = "x" + v699 = "x" + v700 = "x" + v701 = "x" + v702 = "x" + v703 = "x" + v704 = "x" + v705 = "x" + v706 = "x" + v707 = "x" + v708 = "x" + v709 = "x" + v710 = "x" + v711 = "x" + v712 = "x" + v713 = "x" + v714 = "x" + v715 = "x" + v716 = "x" + v717 = "x" + v718 = "x" + v719 = "x" + v720 = "x" + v721 = "x" + v722 = "x" + v723 = "x" + v724 = "x" + v725 = "x" + v726 = "x" + v727 = "x" + v728 = "x" + v729 = "x" + v730 = "x" + v731 = "x" + v732 = "x" + v733 = "x" + v734 = "x" + v735 = "x" + v736 = "x" + v737 = "x" + v738 = "x" + v739 = "x" + v740 = "x" + v741 = "x" + v742 = "x" + v743 = "x" + v744 = "x" + v745 = "x" + v746 = "x" + v747 = "x" + v748 = "x" + v749 = "x" + v750 = "x" + v751 = "x" + v752 = "x" + v753 = "x" + v754 = "x" + v755 = "x" + v756 = "x" + v757 = "x" + v758 = "x" + v759 = "x" + v760 = "x" + v761 = "x" + v762 = "x" + v763 = "x" + v764 = "x" + v765 = "x" + v766 = "x" + v767 = "x" + v768 = "x" + v769 = "x" + v770 = "x" + v771 = "x" + v772 = "x" + v773 = "x" + v774 = "x" + v775 = "x" + v776 = "x" + v777 = "x" + v778 = "x" + v779 = "x" + v780 = "x" + v781 = "x" + v782 = "x" + v783 = "x" + v784 = "x" + v785 = "x" + v786 = "x" + v787 = "x" + v788 = "x" + v789 = "x" + v790 = "x" + v791 = "x" + v792 = "x" + v793 = "x" + v794 = "x" + v795 = "x" + v796 = "x" + v797 = "x" + v798 = "x" + v799 = "x" + v800 = "x" + v801 = "x" + v802 = "x" + v803 = "x" + v804 = "x" + v805 = "x" + v806 = "x" + v807 = "x" + v808 = "x" + v809 = "x" + v810 = "x" + v811 = "x" + v812 = "x" + v813 = "x" + v814 = "x" + v815 = "x" + v816 = "x" + v817 = "x" + v818 = "x" + v819 = "x" + v820 = "x" + v821 = "x" + v822 = "x" + v823 = "x" + v824 = "x" + v825 = "x" + v826 = "x" + v827 = "x" + v828 = "x" + v829 = "x" + v830 = "x" + v831 = "x" + v832 = "x" + v833 = "x" + v834 = "x" + v835 = "x" + v836 = "x" + v837 = "x" + v838 = "x" + v839 = "x" + v840 = "x" + v841 = "x" + v842 = "x" + v843 = "x" + v844 = "x" + v845 = "x" + v846 = "x" + v847 = "x" + v848 = "x" + v849 = "x" + v850 = "x" + v851 = "x" + v852 = "x" + v853 = "x" + v854 = "x" + v855 = "x" + v856 = "x" + v857 = "x" + v858 = "x" + v859 = "x" + v860 = "x" + v861 = "x" + v862 = "x" + v863 = "x" + v864 = "x" + v865 = "x" + v866 = "x" + v867 = "x" + v868 = "x" + v869 = "x" + v870 = "x" + v871 = "x" + v872 = "x" + v873 = "x" + v874 = "x" + v875 = "x" + v876 = "x" + v877 = "x" + v878 = "x" + v879 = "x" + v880 = "x" + v881 = "x" + v882 = "x" + v883 = "x" + v884 = "x" + v885 = "x" + v886 = "x" + v887 = "x" + v888 = "x" + v889 = "x" + v890 = "x" + v891 = "x" + v892 = "x" + v893 = "x" + v894 = "x" + v895 = "x" + v896 = "x" + v897 = "x" + v898 = "x" + v899 = "x" + v900 = "x" + v901 = "x" + v902 = "x" + v903 = "x" + v904 = "x" + v905 = "x" + v906 = "x" + v907 = "x" + v908 = "x" + v909 = "x" + v910 = "x" + v911 = "x" + v912 = "x" + v913 = "x" + v914 = "x" + v915 = "x" + v916 = "x" + v917 = "x" + v918 = "x" + v919 = "x" + v920 = "x" + v921 = "x" + v922 = "x" + v923 = "x" + v924 = "x" + v925 = "x" + v926 = "x" + v927 = "x" + v928 = "x" + v929 = "x" + v930 = "x" + v931 = "x" + v932 = "x" + v933 = "x" + v934 = "x" + v935 = "x" + v936 = "x" + v937 = "x" + v938 = "x" + v939 = "x" + v940 = "x" + v941 = "x" + v942 = "x" + v943 = "x" + v944 = "x" + v945 = "x" + v946 = "x" + v947 = "x" + v948 = "x" + v949 = "x" + v950 = "x" + v951 = "x" + v952 = "x" + v953 = "x" + v954 = "x" + v955 = "x" + v956 = "x" + v957 = "x" + v958 = "x" + v959 = "x" + v960 = "x" + v961 = "x" + v962 = "x" + v963 = "x" + v964 = "x" + v965 = "x" + v966 = "x" + v967 = "x" + v968 = "x" + v969 = "x" + v970 = "x" + v971 = "x" + v972 = "x" + v973 = "x" + v974 = "x" + v975 = "x" + v976 = "x" + v977 = "x" + v978 = "x" + v979 = "x" + v980 = "x" + v981 = "x" + v982 = "x" + v983 = "x" + v984 = "x" + v985 = "x" + v986 = "x" + v987 = "x" + v988 = "x" + v989 = "x" + v990 = "x" + v991 = "x" + v992 = "x" + v993 = "x" + v994 = "x" + v995 = "x" + v996 = "x" + v997 = "x" + v998 = "x" + v999 = "x" + trigger = g(f()) +)