diff --git a/cmd/tl2gen/main.go b/cmd/tl2gen/main.go index 8a1cfc0ce..76160ee2a 100644 --- a/cmd/tl2gen/main.go +++ b/cmd/tl2gen/main.go @@ -22,6 +22,7 @@ import ( "github.com/VKCOM/tl/internal/puregen" "github.com/VKCOM/tl/internal/puregen/gencanonical" "github.com/VKCOM/tl/internal/puregen/gengo" + "github.com/VKCOM/tl/internal/puregen/geninfo" "github.com/VKCOM/tl/internal/puregen/genphp" "github.com/VKCOM/tl/internal/puregen/genrust" "github.com/VKCOM/tl/internal/puregen/gentljsonhtml" @@ -41,6 +42,7 @@ var languages = map[string]func(kernel *pure.Kernel, options *puregen.Options) e "tljson.html": gentljsonhtml.Generate, "tlo": gentlo.Generate, "rust": genrust.Generate, + "info": geninfo.Generate, } func languagesString() string { diff --git a/internal/puregen/geninfo/geninfo.go b/internal/puregen/geninfo/geninfo.go new file mode 100644 index 000000000..a9cb69d39 --- /dev/null +++ b/internal/puregen/geninfo/geninfo.go @@ -0,0 +1,364 @@ +// Copyright 2025 V Kontakte LLC +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package geninfo + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "sort" + "strings" + "time" + + "github.com/VKCOM/tl/internal/pure" + "github.com/VKCOM/tl/internal/puregen" + "github.com/VKCOM/tl/internal/tlast" + "github.com/VKCOM/tl/internal/utils" +) + +const ( + statshouseView = "/view" + statshouseQuery = "/api/query" + + cookieEnvVar = "STATSHOUSE_VKUTH_DATA" + cookieKey = "vkuth_data" + cookieSessionEnvVar = "STATSHOUSE_VKUTH_SESSION" + cookieSessionKey = "vkuth_session" + + sectionLengthInSeconds = "60" +) + +const ( + PartSize = 200 +) + +type PartitioningMode int + +const ( + PartsByChunks PartitioningMode = iota + 1 + PartByNamespaces +) + +func statshouseViewLink(shURL string, queries []tlast.Name) string { + v := url.Values{} + v.Set("t", "0") + v.Set("f", "-7200") + v.Set("s", "vkcom_rpc_get_time") + v.Set("g", sectionLengthInSeconds) + v.Add("qb", "15") + v.Add("qb", "16") + v.Set("qf", "15-kphp") + v.Set("dg", "0") + v.Set("dl", "0.0.12.12") + for _, entry := range queries { + v.Add("qf", "11-"+entry.String()) + } + return shURL + statshouseView + "?" + v.Encode() +} + +func statshouseQueryLink(shURL string, queries []tlast.Name) string { + v := url.Values{} + v.Set("qw", "count_norm") + v.Set("t", "0") + v.Set("f", "-7200") + v.Set("w", sectionLengthInSeconds+"s") + v.Set("v", "2") + v.Add("qb", "15") + v.Add("qb", "16") + v.Set("ep", "1") + v.Set("qv", "0") + v.Set("s", "vkcom_rpc_get_time") + v.Set("n", "5") + v.Set("priority", "1") + for _, entry := range queries { + v.Add("qf", "11-"+entry.String()) + } + return shURL + statshouseQuery + "?" + v.Encode() +} + +type reportPart struct { + Index int `json:"index"` + QueryCount int `json:"query_count"` + ViewURL string `json:"view_url"` + QueryURL string `json:"query_url"` + Values map[string]float64 `json:"values"` + Namespaces []string `json:"namespaces"` +} + +type report struct { + Table string `json:"table"` + Values map[string]float64 `json:"values"` + Parts []reportPart `json:"parts"` +} + +type queryResponse struct { + Data struct { + Series struct { + Time []int64 `json:"time"` + SeriesMeta []struct { + Tags map[string]struct { + Value string `json:"value"` + } `json:"tags"` + } `json:"series_meta"` + SeriesData [][]*float64 `json:"series_data"` + } `json:"series"` + } `json:"data"` +} + +func fetchCountNorm(link string, dataCookie, sessionCookie string) (map[string]float64, error) { + req, err := http.NewRequest("GET", link, nil) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + cookieHeader := cookieKey + "=" + dataCookie + if sessionCookie != "" { + cookieHeader += "; " + cookieSessionKey + "=" + sessionCookie + } + req.Header.Set("Cookie", cookieHeader) + + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("status %d: %s", resp.StatusCode, string(body)) + } + + var result queryResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode json: %w", err) + } + + seriesMap := make(map[string]float64) + for i, meta := range result.Data.Series.SeriesMeta { + if i >= len(result.Data.Series.SeriesData) { + break + } + // строим ключ из тегов (отсортированные ключи для стабильности) + tagKeys := make([]string, 0, len(meta.Tags)) + for k := range meta.Tags { + tagKeys = append(tagKeys, k) + } + sort.Strings(tagKeys) + + var keyParts []string + for _, k := range tagKeys { + val := meta.Tags[k].Value + if val == " 0" { + val = "?" + } + keyParts = append(keyParts, val) + } + key := "" + if len(keyParts) > 0 { + key = keyParts[0] + for i := 1; i < len(keyParts); i++ { + key += "," + keyParts[i] + } + } + + var sum float64 + var count int + for _, v := range result.Data.Series.SeriesData[i] { + if v != nil { + sum += *v + count++ + } + } + if count > 0 { + seriesMap[key] = sum / float64(count) + } else { + seriesMap[key] = 0 + } + } + return seriesMap, nil +} + +func Generate(kernel *pure.Kernel, options *puregen.Options) error { + shURL := options.Info.StatshouseURL + if shURL == "" { + return fmt.Errorf("info-statshouse-url is required") + } + + var mode PartitioningMode + + switch options.Info.PartitioningMode { + case 1: + mode = PartsByChunks + case 2: + mode = PartByNamespaces + default: + mode = PartsByChunks + } + + namespaces := map[string][]tlast.Name{} + for _, comb := range kernel.TL1() { + if comb.IsFunction { + namespaces[comb.Construct.Name.Namespace] = append(namespaces[comb.Construct.Name.Namespace], comb.Construct.Name) + } + } + ns := utils.Keys(namespaces) + sort.Strings(ns) + sort.Slice(ns, func(i, j int) bool { + return len(namespaces[ns[i]])-len(namespaces[ns[j]]) > 0 + }) + + sum := 0 + for _, n := range ns { + //fmt.Println(n, len(namespaces[n])) + sum += len(namespaces[n]) + } + //fmt.Println(sum) + + parts, partsNs := extractParts(ns, namespaces, mode) + + cookieValue := os.Getenv(cookieEnvVar) + sessionCookie := os.Getenv(cookieSessionEnvVar) + if cookieValue == "" { + fmt.Fprintf(os.Stderr, "Warning: %s not set, printing view links only\n", cookieEnvVar) + for _, part := range parts { + fmt.Println(statshouseViewLink(shURL, part)) + } + return nil + } + + totalSeries := make(map[string]float64) + var reportParts []reportPart + for i, part := range parts { + queryLink := statshouseQueryLink(shURL, part) + viewLink := statshouseViewLink(shURL, part) + + seriesMap, err := fetchCountNorm(queryLink, cookieValue, sessionCookie) + if err != nil { + return fmt.Errorf("failed to fetch part %d (%d queries): %w", i, len(part), err) + } + fmt.Printf("Part %d (%d queries): series = %d\n", i, len(part), len(seriesMap)) + for k, v := range seriesMap { + totalSeries[k] += v + } + + reportParts = append(reportParts, reportPart{ + Index: i, + QueryCount: len(part), + ViewURL: viewLink, + QueryURL: queryLink, + Values: seriesMap, + Namespaces: partsNs[i], + }) + } + + type item struct { + key string + value float64 + } + var items []item + maxKeyLen := 3 // "Key" + for k, v := range totalSeries { + items = append(items, item{key: k, value: v}) + if len(k) > maxKeyLen { + maxKeyLen = len(k) + } + } + sort.Slice(items, func(i, j int) bool { + return items[i].value > items[j].value + }) + + totalSum := 0.0 + for _, v := range totalSeries { + totalSum += v + } + + var tableBuilder strings.Builder + fmt.Fprintf(&tableBuilder, "%-*s %10s %10s\n", maxKeyLen, "Key", "Avg", "Percent") + tableBuilder.WriteString(strings.Repeat("-", maxKeyLen+2+10+2+10) + "\n") + for _, it := range items { + percent := 0.0 + if totalSum > 0 { + percent = it.value / totalSum * 100 + } + fmt.Fprintf(&tableBuilder, "%-*s %10.2f %9.2f%%\n", maxKeyLen, it.key, it.value, percent) + } + tableStr := tableBuilder.String() + fmt.Print(tableStr) + + sort.Slice(reportParts, func(i, j int) bool { + sumI, sumJ := 0.0, 0.0 + for _, v := range reportParts[i].Values { + sumI += v + } + for _, v := range reportParts[j].Values { + sumJ += v + } + return sumI > sumJ + }) + + rep := report{ + Table: tableStr, + Values: totalSeries, + Parts: reportParts, + } + + const infoMarker = "info.json" + + outputInfo := strings.Builder{} + enc := json.NewEncoder(&outputInfo) + enc.SetIndent("", " ") + if err := enc.Encode(rep); err != nil { + return fmt.Errorf("failed to encode report: %w", err) + } + + out := puregen.OutDir{} + if err := out.AddCodeFile(infoMarker, outputInfo.String()); err != nil { + return fmt.Errorf("failed to write report: %w", err) + } + + if err := out.Write(options, infoMarker); err != nil { + return err + } + + return nil +} + +func extractParts(ns []string, namespaces map[string][]tlast.Name, mode PartitioningMode) (parts [][]tlast.Name, partsNs [][]string) { + parts = make([][]tlast.Name, 0) + partsNs = make([][]string, 0) + + switch mode { + case PartsByChunks: + parts = append(parts, []tlast.Name{}) + partsNs = append(partsNs, []string{}) + + for _, n := range ns { + i := len(parts) - 1 + parts[i] = append(parts[i], namespaces[n]...) + partsNs[i] = append(partsNs[i], n) + if len(parts[i]) >= PartSize { + parts = append(parts, []tlast.Name{}) + partsNs = append(partsNs, []string{}) + } + } + case PartByNamespaces: + for _, n := range ns { + i := len(parts) + + parts = append(parts, []tlast.Name{}) + partsNs = append(partsNs, []string{}) + + parts[i] = append(parts[i], namespaces[n]...) + partsNs[i] = append(partsNs[i], n) + } + } + + return parts, partsNs +} diff --git a/internal/puregen/options.go b/internal/puregen/options.go index e3c767cd8..9603b5feb 100644 --- a/internal/puregen/options.go +++ b/internal/puregen/options.go @@ -41,6 +41,7 @@ type Options struct { Go OptionsGo PHP OptionsPHP Rust OptionsRust + Info OptionsInfo ProfileCPU string @@ -94,6 +95,7 @@ func (opt *Options) Bind(f *flag.FlagSet, languagesString string) { opt.Go.Bind(f) opt.PHP.Bind(f) opt.Rust.Bind(f) + opt.Info.Bind(f) } func (opt *Options) Validate() error { diff --git a/internal/puregen/options_info.go b/internal/puregen/options_info.go new file mode 100644 index 000000000..fcac4052c --- /dev/null +++ b/internal/puregen/options_info.go @@ -0,0 +1,15 @@ +package puregen + +import "flag" + +type OptionsInfo struct { + StatshouseURL string + PartitioningMode int +} + +func (opt *OptionsInfo) Bind(f *flag.FlagSet) { + f.StringVar(&opt.StatshouseURL, "info-statshouse-url", "", + `url to statshouse (required)`) + f.IntVar(&opt.PartitioningMode, "info-partitioning-mode", 1, + `mode how to partitioning namespaces for queries (1 - by chunks of 200 methods, 2 - by namespace)`) +}