From 9c3c07a4a80773aee18bf5be1fdfd57d475fcea8 Mon Sep 17 00:00:00 2001 From: M-Vivek-Juniper Date: Fri, 24 May 2024 02:25:46 +0530 Subject: [PATCH 01/24] Print del_path --- internal_jtimon.go | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/internal_jtimon.go b/internal_jtimon.go index fa4c1e0..d0dd3f6 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -96,6 +96,27 @@ func isInternalJtimonLogging(jctx *JCtx) bool { return jctx.config.InternalJtimon.logger != nil } +func getPath(prefixPath string, pathElements []*gnmi.PathElem) string { + for _, pe := range pathElements { + peName := pe.GetName() + prefixPath += gXPathTokenPathSep + peName + is_key := false + for k, v := range pe.GetKey() { + if is_key { + prefixPath += " and " + k + "='" + v + "'" + } else { + prefixPath += "[" + k + "='" + v + "'" + is_key = true + } + } + if is_key { + prefixPath += "]" + } + } + + return prefixPath +} + func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *gnmi.SubscribeResponse) { if jctx.config.InternalJtimon.logger == nil { return @@ -160,27 +181,18 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g prefix := notif.GetPrefix() if prefix != nil { - for _, pe := range prefix.GetElem() { - peName := pe.GetName() - prefixPath += gXPathTokenPathSep + peName - is_key := false - for k, v := range pe.GetKey() { - if is_key { - prefixPath += " and " + k + "='" + v + "'" - } else { - prefixPath += "[" + k + "='" + v + "'" - is_key = true - } - } - if is_key { - prefixPath += "]" - } - } + prefixPath = getPath(prefixPath, prefix.GetElem()) } s += fmt.Sprintf( "Update {\n\ttimestamp: %d\n\tprefix: %v\n", notif.GetTimestamp(), prefixPath) + // Parse all the deletes here + for _, d := range notif.Delete { + delPath := getPath(prefixPath, d.GetElem()) + s += fmt.Sprintf("del_path: %s", delPath) + } + // Parse all the updates here for _, u := range notif.Update { notifString := u.String() From 46fa0e4d61aceb7f96178d9d4bdb3abb8e2b34e4 Mon Sep 17 00:00:00 2001 From: amohit Date: Mon, 10 Jun 2024 21:33:14 -0700 Subject: [PATCH 02/24] fixed panic error when parseOutput.jHeader is nill --- internal_jtimon.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal_jtimon.go b/internal_jtimon.go index 74a91f3..80d3e75 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -110,7 +110,7 @@ func getPath(prefixPath string, pathElements []*gnmi.PathElem) string { } func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *gnmi.SubscribeResponse) { - if jctx.config.InternalJtimon.logger == nil { + if jctx.config.InternalJtimon.logger == nil || parseOutput.jHeader == nil { return } From a8c2d5d4c7725bd8346fa327d7fb822c2dfd5cc6 Mon Sep 17 00:00:00 2001 From: amohit Date: Thu, 20 Jun 2024 13:43:18 -0700 Subject: [PATCH 03/24] added override control of eos in internal jtimon configs --- internal_jtimon.go | 2 ++ subscribe_gnmi.go | 9 +++++++-- subscribe_juniper_junos.go | 4 ++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/internal_jtimon.go b/internal_jtimon.go index 80d3e75..2fdcaf3 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -21,6 +21,8 @@ type InternalJtimonConfig struct { logger *log.Logger preGnmiLogger *log.Logger csvLogger *log.Logger + GnmiEOS bool `json:"gnmi-eos"` + PreGnmiEOS bool `json:"pre-gnmi-eos"` } func internalJtimonLogInit(jctx *JCtx) { diff --git a/subscribe_gnmi.go b/subscribe_gnmi.go index 3c9fe7f..d35509e 100644 --- a/subscribe_gnmi.go +++ b/subscribe_gnmi.go @@ -291,7 +291,8 @@ func gnmiHandleResponse(jctx *JCtx, rsp *gnmi.SubscribeResponse) error { parseOutput = &tmpParseOp err error - hostname = jctx.config.Host + ":" + strconv.Itoa(jctx.config.Port) + eosEnabled = false + hostname = jctx.config.Host + ":" + strconv.Itoa(jctx.config.Port) ) // Update packet stats @@ -316,7 +317,11 @@ func gnmiHandleResponse(jctx *JCtx, rsp *gnmi.SubscribeResponse) error { updateStatsKV(jctx, true, parseOutput.inKvs) // Ignore all packets till sync response is received. - if !jctx.config.EOS { + eosEnabled = jctx.config.EOS + if isInternalJtimonLogging(jctx) { + eosEnabled = jctx.config.InternalJtimon.GnmiEOS + } + if !eosEnabled { if !jctx.receivedSyncRsp { if parseOutput.jHeader != nil { // For juniper packets, ignore only the packets which are numbered in initial sync sequence range diff --git a/subscribe_juniper_junos.go b/subscribe_juniper_junos.go index 4357224..dd089c3 100644 --- a/subscribe_juniper_junos.go +++ b/subscribe_juniper_junos.go @@ -237,6 +237,10 @@ func subscribeJunos(conn *grpc.ClientConn, jctx *JCtx, cfg Config, paths []Paths subReqM.PathList = append(subReqM.PathList, &pathM) } additionalConfigM.NeedEos = jctx.config.EOS + // Override EOS if InternalJtimon is configured + if isInternalJtimonLogging(jctx) { + additionalConfigM.NeedEos = jctx.config.InternalJtimon.PreGnmiEOS + } subReqM.AdditionalConfig = &additionalConfigM return subSendAndReceive(conn, jctx, subReqM) From d17b94b4c1deb6b2c9f792f76cd8e050990665db Mon Sep 17 00:00:00 2001 From: amohit Date: Thu, 20 Jun 2024 23:12:35 -0700 Subject: [PATCH 04/24] added auto unit tests update --- .../cisco-ios-xr/config/xr-all-influx.log | 3 +- .../config/xr-wdsysmon-influx.log | 3 +- .../config/jtisim-influx-alias.log | 14 ++++----- .../juniper-junos/config/jtisim-influx.log | 29 ++++++++++--------- .../config/jtisim-interfaces-1.log | 29 ++++++++++--------- .../config/jtisim-interfaces-2.log | 29 ++++++++++--------- .../config/jtisim-interfaces-3.log | 19 ++++++------ .../config/jtisim-interfaces-4.log | 8 ++--- .../config/jtisim-interfaces-6.log | 7 +++-- .../config/jtisim-prometheus.log | 9 +++--- 10 files changed, 77 insertions(+), 73 deletions(-) diff --git a/tests/data/cisco-ios-xr/config/xr-all-influx.log b/tests/data/cisco-ios-xr/config/xr-all-influx.log index 09f6d8c..7bdc701 100644 --- a/tests/data/cisco-ios-xr/config/xr-all-influx.log +++ b/tests/data/cisco-ios-xr/config/xr-all-influx.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { diff --git a/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log b/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log index 6c85db2..889cca6 100644 --- a/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log +++ b/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { diff --git a/tests/data/juniper-junos/config/jtisim-influx-alias.log b/tests/data/juniper-junos/config/jtisim-influx-alias.log index 35938f1..a30ecd5 100644 --- a/tests/data/juniper-junos/config/jtisim-influx-alias.log +++ b/tests/data/juniper-junos/config/jtisim-influx-alias.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { @@ -91,22 +92,19 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Fri May 31 02:22:48 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:52 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Fri May 31 02:22:50 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:54 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:22:52 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:56 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:22:54 PDT 2024 | 1980 | 40 | 87418 | 87618 | - - -Collector Stats for 127.0.0.1:50051 (Run time : 8.007675042s) +Collector Stats for 127.0.0.1:50051 (Run time : 8.003406166s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-influx.log b/tests/data/juniper-junos/config/jtisim-influx.log index 39b8fb2..448cdca 100644 --- a/tests/data/juniper-junos/config/jtisim-influx.log +++ b/tests/data/juniper-junos/config/jtisim-influx.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { @@ -92,50 +93,50 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Fri May 31 02:22:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Fri May 31 02:22:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:22:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:22:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:33 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:22:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:35 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 -| Fri May 31 02:22:33 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:57:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | Batch write successful! Post batch write available points: 0 -| Fri May 31 02:22:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:57:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:22:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:57:41 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:22:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:57:43 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:22:41 PDT 2024 | 5367 | 109 | 236698 | 237243 | +| Thu Jun 20 22:57:45 PDT 2024 | 4158 | 84 | 183580 | 184000 | Batch processing: #packets:40 #points:40 -| Fri May 31 02:22:43 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Thu Jun 20 22:57:47 PDT 2024 | 5940 | 120 | 262258 | 262858 | Batch write successful! Post batch write available points: 0 -| Fri May 31 02:22:45 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Thu Jun 20 22:57:49 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.004155208s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.00566875s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-1.log b/tests/data/juniper-junos/config/jtisim-interfaces-1.log index b25388f..ab332cb 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-1.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-1.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { @@ -89,44 +90,44 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Fri May 31 02:23:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:33 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:31 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:33 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:41 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:43 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:41 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Thu Jun 20 22:58:45 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Fri May 31 02:23:43 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Thu Jun 20 22:58:47 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.002534292s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.001561125s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-2.log b/tests/data/juniper-junos/config/jtisim-interfaces-2.log index 20ec4ac..5a0570e 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-2.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-2.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { @@ -89,44 +90,44 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Fri May 31 02:23:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:58:33 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:23:31 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:33 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:41 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:58:43 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:23:41 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Thu Jun 20 22:58:45 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Fri May 31 02:23:43 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Thu Jun 20 22:58:47 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.00119025s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.000711s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-3.log b/tests/data/juniper-junos/config/jtisim-interfaces-3.log index b60e621..a56228e 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-3.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-3.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { @@ -89,29 +90,29 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Fri May 31 02:21:37 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:56:41 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:21:39 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:56:43 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:21:41 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:56:45 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:21:43 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:56:47 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:21:45 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:56:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:21:47 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:56:51 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Fri May 31 02:21:49 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Thu Jun 20 22:56:53 PDT 2024 | 3960 | 80 | 174838 | 175238 | -Collector Stats for 127.0.0.1:50051 (Run time : 15.001363375s) +Collector Stats for 127.0.0.1:50051 (Run time : 15.001743459s) 80 : in-packets 3960 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-4.log b/tests/data/juniper-junos/config/jtisim-interfaces-4.log index 519110c..72b6b62 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-4.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-4.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { @@ -82,11 +83,9 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:90052 subscribe returns, reconnecting after 10s for worker tests/data/juniper-junos/config/jtisim-interfaces-4.json -compression = none -Reconnecting to 127.0.0.1:90052 -Collector Stats for 127.0.0.1:90052 (Run time : 10.004200459s) +Collector Stats for 127.0.0.1:90052 (Run time : 10.00228125s) 0 : in-packets 0 : data points (KV pairs) 0 : in-header wirelength (bytes) @@ -96,6 +95,5 @@ Collector Stats for 127.0.0.1:90052 (Run time : 10.004200459s) Streaming for host 127.0.0.1 will be stopped (SIGINT) -subscribe returns, reconnecting after 10s for worker tests/data/juniper-junos/config/jtisim-interfaces-4.json compression = none Connection for 127.0.0.1:90052 has been interrupted diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-6.log b/tests/data/juniper-junos/config/jtisim-interfaces-6.log index 29bf1d0..2997d4d 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-6.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-6.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { @@ -89,11 +90,11 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Fri May 31 02:21:33 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:56:37 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 3.99416075s) +Collector Stats for 127.0.0.1:50051 (Run time : 3.996386375s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-prometheus.log b/tests/data/juniper-junos/config/jtisim-prometheus.log index 358bff4..567cb11 100644 --- a/tests/data/juniper-junos/config/jtisim-prometheus.log +++ b/tests/data/juniper-junos/config/jtisim-prometheus.log @@ -48,7 +48,8 @@ Running config of JTIMON: "internal-jtimon": { "data-log-file": "", "csv-log-file": "", - "csv-stats": false + "gnmi-eos": false, + "pre-gnmi-eos": false }, "paths": [ { @@ -89,14 +90,14 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Fri May 31 02:22:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Fri May 31 02:22:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Thu Jun 20 22:57:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 6.005650209s) +Collector Stats for 127.0.0.1:50051 (Run time : 6.002609541s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) From dd154db63bf693ef5ade1fe3355b74be726da463 Mon Sep 17 00:00:00 2001 From: amohit Date: Tue, 25 Jun 2024 11:43:59 -0700 Subject: [PATCH 05/24] fixed eos enhancement and subscription issue for single path --- config.go | 2 ++ internal_jtimon.go | 11 ++++++++ multi_vendor.go | 8 ++++-- .../config/jtisim-influx-alias.log | 11 +++++--- .../juniper-junos/config/jtisim-influx.log | 28 +++++++++---------- .../config/jtisim-interfaces-1.log | 28 +++++++++---------- .../config/jtisim-interfaces-2.log | 26 ++++++++--------- .../config/jtisim-interfaces-3.log | 16 +++++------ .../config/jtisim-interfaces-4.log | 5 +++- .../config/jtisim-interfaces-6.log | 6 ++-- .../config/jtisim-prometheus.log | 6 ++-- 11 files changed, 85 insertions(+), 62 deletions(-) diff --git a/config.go b/config.go index f48856d..20efddf 100644 --- a/config.go +++ b/config.go @@ -323,6 +323,7 @@ func HandleConfigChange(jctx *JCtx, config Config, restart *bool) error { jctx.config = config logInit(jctx) internalJtimonLogInit(jctx) + initInternalJtimon(jctx) if restart != nil { jLog(jctx, fmt.Sprintf("Restarting worker process to spawn new device connection for: %s", jctx.file)) *restart = true @@ -348,6 +349,7 @@ func ConfigRead(jctx *JCtx, init bool, restart *bool) error { jctx.config = config logInit(jctx) internalJtimonLogInit(jctx) + initInternalJtimon(jctx) b, err := json.MarshalIndent(jctx.config, "", " ") if err != nil { return fmt.Errorf("config parsing error (json marshal) for %s: %v", jctx.file, err) diff --git a/internal_jtimon.go b/internal_jtimon.go index 2fdcaf3..f5a3586 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -25,6 +25,17 @@ type InternalJtimonConfig struct { PreGnmiEOS bool `json:"pre-gnmi-eos"` } +func initInternalJtimon(jctx *JCtx) { + // if Internal Jtimon EOS value is not set, + // then take the EOS value from parent config + if !jctx.config.InternalJtimon.GnmiEOS { + jctx.config.InternalJtimon.GnmiEOS = jctx.config.EOS + } + if !jctx.config.InternalJtimon.PreGnmiEOS { + jctx.config.InternalJtimon.PreGnmiEOS = jctx.config.EOS + } +} + func internalJtimonLogInit(jctx *JCtx) { if jctx.config.InternalJtimon.DataLog == "" { return diff --git a/multi_vendor.go b/multi_vendor.go index 0949f38..2aa5918 100644 --- a/multi_vendor.go +++ b/multi_vendor.go @@ -84,12 +84,16 @@ func subscribePrePostGNMI(conn *grpc.ClientConn, jctx *JCtx, cfg Config, paths [ // Launch goroutines for each subscription function go func() { gnmiPaths := getGnmiPaths(cfg) - gnmiResultCh <- subscribegNMI(conn, jctx, cfg, gnmiPaths) + if len(gnmiPaths) > 0 { + gnmiResultCh <- subscribegNMI(conn, jctx, cfg, gnmiPaths) + } }() go func() { preGnmiPaths := getPreGnmiPaths(cfg) - junosResultCh <- subscribeJunos(conn, jctx, cfg, preGnmiPaths) + if len(preGnmiPaths) > 0 { + junosResultCh <- subscribeJunos(conn, jctx, cfg, preGnmiPaths) + } }() // Use select to wait for the first result to be available diff --git a/tests/data/juniper-junos/config/jtisim-influx-alias.log b/tests/data/juniper-junos/config/jtisim-influx-alias.log index a30ecd5..bed703a 100644 --- a/tests/data/juniper-junos/config/jtisim-influx-alias.log +++ b/tests/data/juniper-junos/config/jtisim-influx-alias.log @@ -92,19 +92,22 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Jun 20 22:57:52 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Thu Jun 20 22:57:54 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:33 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:57:56 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:35 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:37 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 8.003406166s) + + +Collector Stats for 127.0.0.1:50051 (Run time : 8.010581416s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-influx.log b/tests/data/juniper-junos/config/jtisim-influx.log index 448cdca..cd96421 100644 --- a/tests/data/juniper-junos/config/jtisim-influx.log +++ b/tests/data/juniper-junos/config/jtisim-influx.log @@ -86,57 +86,57 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - content-type: [application/grpc] jtisim: [yes] + content-type: [application/grpc] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Jun 20 22:57:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:06 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Thu Jun 20 22:57:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:08 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:57:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:10 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:57:33 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:12 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:57:35 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:14 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 -| Thu Jun 20 22:57:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:38:16 PDT 2024 | 3960 | 80 | 174838 | 175238 | Batch write successful! Post batch write available points: 0 -| Thu Jun 20 22:57:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:38:18 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:57:41 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:38:20 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:57:43 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:38:22 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:57:45 PDT 2024 | 4158 | 84 | 183580 | 184000 | +| Tue Jun 25 11:38:24 PDT 2024 | 4040 | 82 | 178196 | 178606 | Batch processing: #packets:40 #points:40 -| Thu Jun 20 22:57:47 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Tue Jun 25 11:38:26 PDT 2024 | 5940 | 120 | 262258 | 262858 | Batch write successful! Post batch write available points: 0 -| Thu Jun 20 22:57:49 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Tue Jun 25 11:38:28 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.00566875s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.004049042s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-1.log b/tests/data/juniper-junos/config/jtisim-interfaces-1.log index ab332cb..c560d3e 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-1.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-1.log @@ -83,51 +83,51 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - content-type: [application/grpc] jtisim: [yes] + content-type: [application/grpc] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Jun 20 22:58:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:04 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:06 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:08 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:10 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:33 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:12 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:14 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:16 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:18 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:41 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:20 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:43 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:22 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:45 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Tue Jun 25 11:39:24 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Thu Jun 20 22:58:47 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Tue Jun 25 11:39:26 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.001561125s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.001662209s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-2.log b/tests/data/juniper-junos/config/jtisim-interfaces-2.log index 5a0570e..e775a66 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-2.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-2.log @@ -90,44 +90,44 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Jun 20 22:58:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:04 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:06 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:08 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:10 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:33 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:39:12 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:58:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:14 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:16 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:18 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:41 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:20 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:43 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:39:22 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:58:45 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Tue Jun 25 11:39:24 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Thu Jun 20 22:58:47 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Tue Jun 25 11:39:26 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.000711s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.000787458s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-3.log b/tests/data/juniper-junos/config/jtisim-interfaces-3.log index a56228e..9bc32b8 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-3.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-3.log @@ -90,29 +90,29 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Jun 20 22:56:41 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:37:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:56:43 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:37:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:56:45 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:37:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:56:47 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:37:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:56:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:37:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:56:51 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:37:29 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Thu Jun 20 22:56:53 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Tue Jun 25 11:37:31 PDT 2024 | 3960 | 80 | 174838 | 175238 | -Collector Stats for 127.0.0.1:50051 (Run time : 15.001743459s) +Collector Stats for 127.0.0.1:50051 (Run time : 15.00162425s) 80 : in-packets 3960 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-4.log b/tests/data/juniper-junos/config/jtisim-interfaces-4.log index 72b6b62..9fbbcfb 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-4.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-4.log @@ -83,9 +83,11 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:90052 subscribe returns, reconnecting after 10s for worker tests/data/juniper-junos/config/jtisim-interfaces-4.json +compression = none +Reconnecting to 127.0.0.1:90052 -Collector Stats for 127.0.0.1:90052 (Run time : 10.00228125s) +Collector Stats for 127.0.0.1:90052 (Run time : 10.003748166s) 0 : in-packets 0 : data points (KV pairs) 0 : in-header wirelength (bytes) @@ -95,5 +97,6 @@ Collector Stats for 127.0.0.1:90052 (Run time : 10.00228125s) Streaming for host 127.0.0.1 will be stopped (SIGINT) +subscribe returns, reconnecting after 10s for worker tests/data/juniper-junos/config/jtisim-interfaces-4.json compression = none Connection for 127.0.0.1:90052 has been interrupted diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-6.log b/tests/data/juniper-junos/config/jtisim-interfaces-6.log index 2997d4d..3b265b3 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-6.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-6.log @@ -83,18 +83,18 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - content-type: [application/grpc] jtisim: [yes] + content-type: [application/grpc] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Jun 20 22:56:37 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:37:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 3.996386375s) +Collector Stats for 127.0.0.1:50051 (Run time : 3.997711875s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-prometheus.log b/tests/data/juniper-junos/config/jtisim-prometheus.log index 567cb11..08146bd 100644 --- a/tests/data/juniper-junos/config/jtisim-prometheus.log +++ b/tests/data/juniper-junos/config/jtisim-prometheus.log @@ -90,14 +90,14 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Jun 20 22:57:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:37:59 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Thu Jun 20 22:57:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Tue Jun 25 11:38:01 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 6.002609541s) +Collector Stats for 127.0.0.1:50051 (Run time : 6.003748958s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) From 5c00b2408f263a2992c141a2ebb61dce8133daec Mon Sep 17 00:00:00 2001 From: amohit Date: Wed, 3 Jul 2024 13:00:31 -0700 Subject: [PATCH 06/24] output stat summary to terminal if internal jtimon csv stats is enabled --- statshandler.go | 5 ++++ .../config/jtisim-influx-alias.log | 11 +++----- .../juniper-junos/config/jtisim-influx.log | 28 +++++++++---------- .../config/jtisim-interfaces-1.log | 28 +++++++++---------- .../config/jtisim-interfaces-2.log | 26 ++++++++--------- .../config/jtisim-interfaces-3.log | 16 +++++------ .../config/jtisim-interfaces-4.log | 2 +- .../config/jtisim-interfaces-6.log | 6 ++-- .../config/jtisim-prometheus.log | 6 ++-- 9 files changed, 65 insertions(+), 63 deletions(-) diff --git a/statshandler.go b/statshandler.go index 5784b55..0d37e05 100644 --- a/statshandler.go +++ b/statshandler.go @@ -264,6 +264,11 @@ func printSummary(jctx *JCtx) { s += fmt.Sprintf("\n") jLog(jctx, fmt.Sprintf("\n%s\n", s)) + + // Print Summary to terminal for internal Jitmon + if jctx.config.InternalJtimon.CsvLog != "" { + fmt.Println(s) + } } func isCsvStatsEnabled(jctx *JCtx) bool { diff --git a/tests/data/juniper-junos/config/jtisim-influx-alias.log b/tests/data/juniper-junos/config/jtisim-influx-alias.log index bed703a..64b0e92 100644 --- a/tests/data/juniper-junos/config/jtisim-influx-alias.log +++ b/tests/data/juniper-junos/config/jtisim-influx-alias.log @@ -92,22 +92,19 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Tue Jun 25 11:38:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:38 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Tue Jun 25 11:38:33 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:40 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:38:35 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:42 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:38:37 PDT 2024 | 1980 | 40 | 87418 | 87618 | - - -Collector Stats for 127.0.0.1:50051 (Run time : 8.010581416s) +Collector Stats for 127.0.0.1:50051 (Run time : 8.008566792s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-influx.log b/tests/data/juniper-junos/config/jtisim-influx.log index cd96421..6829b51 100644 --- a/tests/data/juniper-junos/config/jtisim-influx.log +++ b/tests/data/juniper-junos/config/jtisim-influx.log @@ -86,57 +86,57 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - jtisim: [yes] content-type: [application/grpc] + jtisim: [yes] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Tue Jun 25 11:38:06 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:13 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Tue Jun 25 11:38:08 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:38:10 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:38:12 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:38:14 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 -| Tue Jun 25 11:38:16 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:13:23 PDT 2024 | 3960 | 80 | 174838 | 175238 | Batch write successful! Post batch write available points: 0 -| Tue Jun 25 11:38:18 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:13:25 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:38:20 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:13:27 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:38:22 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:13:29 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:38:24 PDT 2024 | 4040 | 82 | 178196 | 178606 | +| Wed Jul 3 12:13:31 PDT 2024 | 4040 | 82 | 178196 | 178606 | Batch processing: #packets:40 #points:40 -| Tue Jun 25 11:38:26 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Jul 3 12:13:33 PDT 2024 | 5940 | 120 | 262258 | 262858 | Batch write successful! Post batch write available points: 0 -| Tue Jun 25 11:38:28 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Jul 3 12:13:35 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.004049042s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.002141625s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-1.log b/tests/data/juniper-junos/config/jtisim-interfaces-1.log index c560d3e..0027868 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-1.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-1.log @@ -83,51 +83,51 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - jtisim: [yes] content-type: [application/grpc] + jtisim: [yes] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Tue Jun 25 11:39:04 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:11 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:06 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:13 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:08 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:10 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:12 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:14 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:21 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:39:16 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:23 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:39:18 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:25 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:39:20 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:27 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:39:22 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:29 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:39:24 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Jul 3 12:14:31 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Tue Jun 25 11:39:26 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Jul 3 12:14:33 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.001662209s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.009377916s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-2.log b/tests/data/juniper-junos/config/jtisim-interfaces-2.log index e775a66..06c91a1 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-2.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-2.log @@ -90,44 +90,44 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Tue Jun 25 11:39:04 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:11 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:06 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:13 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:08 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:10 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:12 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:14:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:39:14 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:21 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:39:16 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:23 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:39:18 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:25 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:39:20 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:27 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:39:22 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:14:29 PDT 2024 | 4238 | 86 | 186938 | 187368 | -| Tue Jun 25 11:39:24 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Jul 3 12:14:31 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Tue Jun 25 11:39:26 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Jul 3 12:14:33 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.000787458s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.00636325s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-3.log b/tests/data/juniper-junos/config/jtisim-interfaces-3.log index 9bc32b8..e5fbebb 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-3.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-3.log @@ -90,29 +90,29 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Tue Jun 25 11:37:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:12:26 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:37:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:12:28 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:37:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:12:30 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:37:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:12:32 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:37:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:12:34 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:37:29 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:12:36 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Tue Jun 25 11:37:31 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Jul 3 12:12:38 PDT 2024 | 3960 | 80 | 174838 | 175238 | -Collector Stats for 127.0.0.1:50051 (Run time : 15.00162425s) +Collector Stats for 127.0.0.1:50051 (Run time : 15.001616292s) 80 : in-packets 3960 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-4.log b/tests/data/juniper-junos/config/jtisim-interfaces-4.log index 9fbbcfb..58a0758 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-4.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-4.log @@ -87,7 +87,7 @@ compression = none Reconnecting to 127.0.0.1:90052 -Collector Stats for 127.0.0.1:90052 (Run time : 10.003748166s) +Collector Stats for 127.0.0.1:90052 (Run time : 10.0034215s) 0 : in-packets 0 : data points (KV pairs) 0 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-6.log b/tests/data/juniper-junos/config/jtisim-interfaces-6.log index 3b265b3..9fa4b4a 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-6.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-6.log @@ -83,18 +83,18 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - jtisim: [yes] content-type: [application/grpc] + jtisim: [yes] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Tue Jun 25 11:37:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:12:22 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 3.997711875s) +Collector Stats for 127.0.0.1:50051 (Run time : 3.989988833s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-prometheus.log b/tests/data/juniper-junos/config/jtisim-prometheus.log index 08146bd..0f7181e 100644 --- a/tests/data/juniper-junos/config/jtisim-prometheus.log +++ b/tests/data/juniper-junos/config/jtisim-prometheus.log @@ -90,14 +90,14 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Tue Jun 25 11:37:59 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:06 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Tue Jun 25 11:38:01 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Jul 3 12:13:08 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 6.003748958s) +Collector Stats for 127.0.0.1:50051 (Run time : 6.003954292s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) From 4fa5ee0233f8a97b519c13e70581091fccb0a422 Mon Sep 17 00:00:00 2001 From: amohit Date: Sat, 21 Sep 2024 14:23:34 -0700 Subject: [PATCH 07/24] added support for leaf-list for gnmi and pre-gnmi --- influx.go | 11 ++++++++ subscribe_gnmi.go | 11 +++++++- .../config/jtisim-influx-alias.log | 8 +++--- .../juniper-junos/config/jtisim-influx.log | 26 ++++++++--------- .../config/jtisim-interfaces-1.log | 28 +++++++++---------- .../config/jtisim-interfaces-2.log | 26 ++++++++--------- .../config/jtisim-interfaces-3.log | 16 +++++------ .../config/jtisim-interfaces-4.log | 2 +- .../config/jtisim-interfaces-6.log | 6 ++-- .../config/jtisim-prometheus.log | 6 ++-- 10 files changed, 80 insertions(+), 60 deletions(-) diff --git a/influx.go b/influx.go index 0d9fe40..e16f625 100644 --- a/influx.go +++ b/influx.go @@ -503,6 +503,17 @@ func addIDB(ocData *na_pb.OpenConfigData, jctx *JCtx, rtime time.Time) { value32 := v.GetFloatValue() checkAndCeilFloatValues(&value32, nil, &floatVal) kv[xmlpath] = floatVal + case *na_pb.KeyValue_LeaflistValue: + e := v.GetLeaflistValue().Element + var leafListStr string + for _, elem := range e { + switch elem.Value.(type) { + case *na_pb.TypedValue_LeaflistStrValue: + llStrValue := elem.GetLeaflistStrValue() + leafListStr = leafListStr + llStrValue + "," + } + } + kv[xmlpath] = leafListStr[:len(leafListStr)-1] default: } diff --git a/subscribe_gnmi.go b/subscribe_gnmi.go index d35509e..124b2d9 100644 --- a/subscribe_gnmi.go +++ b/subscribe_gnmi.go @@ -99,7 +99,7 @@ func publishToPrometheus(jctx *JCtx, parseOutput *gnmiParseOutputT) { } /* - * Publish parsed output to Influx. Make sure there are only inegers, + * Publish parsed output to Influx. Make sure there are only integers, * floats and strings. Influx Line Protocol doesn't support other types */ func publishToInflux(jctx *JCtx, mName string, prefixPath string, kvpairs map[string]string, xpaths map[string]interface{}) error { @@ -107,6 +107,15 @@ func publishToInflux(jctx *JCtx, mName string, prefixPath string, kvpairs map[st return nil } + // Convert leaf-list values if present in xpaths for influxdb Point write + for key, value := range xpaths { + // Check if the value is a slice + if members, ok := value.([]string); ok { + // Join the slice elements into a single string separated by commas + xpaths[key] = strings.Join(members, ", ") + } + } + pt, err := client.NewPoint(mName, kvpairs, xpaths, time.Now()) if err != nil { msg := fmt.Sprintf("New point creation failed for (key: %v, xpaths: %v): %v", kvpairs, xpaths, err) diff --git a/tests/data/juniper-junos/config/jtisim-influx-alias.log b/tests/data/juniper-junos/config/jtisim-influx-alias.log index 64b0e92..9b8d1b2 100644 --- a/tests/data/juniper-junos/config/jtisim-influx-alias.log +++ b/tests/data/juniper-junos/config/jtisim-influx-alias.log @@ -92,19 +92,19 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Wed Jul 3 12:13:38 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:14 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Wed Jul 3 12:13:40 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:16 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:13:42 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:18 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 8.008566792s) +Collector Stats for 127.0.0.1:50051 (Run time : 8.00172925s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-influx.log b/tests/data/juniper-junos/config/jtisim-influx.log index 6829b51..dbd6f5a 100644 --- a/tests/data/juniper-junos/config/jtisim-influx.log +++ b/tests/data/juniper-junos/config/jtisim-influx.log @@ -93,50 +93,50 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Wed Jul 3 12:13:13 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Wed Jul 3 12:13:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:51 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:13:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:13:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:13:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:57 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 -| Wed Jul 3 12:13:23 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:57:59 PDT 2024 | 3960 | 80 | 174838 | 175238 | Batch write successful! Post batch write available points: 0 -| Wed Jul 3 12:13:25 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:58:01 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:13:27 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:58:03 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:13:29 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:58:05 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:13:31 PDT 2024 | 4040 | 82 | 178196 | 178606 | +| Sat Sep 21 13:58:07 PDT 2024 | 5940 | 120 | 262258 | 262858 | Batch processing: #packets:40 #points:40 -| Wed Jul 3 12:13:33 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Sat Sep 21 13:58:09 PDT 2024 | 5940 | 120 | 262258 | 262858 | Batch write successful! Post batch write available points: 0 -| Wed Jul 3 12:13:35 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Sat Sep 21 13:58:11 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.002141625s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.002847916s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-1.log b/tests/data/juniper-junos/config/jtisim-interfaces-1.log index 0027868..dcca0fb 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-1.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-1.log @@ -83,51 +83,51 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - content-type: [application/grpc] jtisim: [yes] + content-type: [application/grpc] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Wed Jul 3 12:14:11 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:47 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:13 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:51 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:21 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:58:57 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:23 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:58:59 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:25 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:59:01 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:27 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:59:03 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:29 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:59:05 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:31 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Sat Sep 21 13:59:07 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Wed Jul 3 12:14:33 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Sat Sep 21 13:59:09 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.009377916s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.004069667s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-2.log b/tests/data/juniper-junos/config/jtisim-interfaces-2.log index 06c91a1..28f4e1c 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-2.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-2.log @@ -90,44 +90,44 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Wed Jul 3 12:14:11 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:47 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:13 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:51 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:58:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:14:21 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:58:57 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:23 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:58:59 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:25 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:59:01 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:27 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:59:03 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:29 PDT 2024 | 4238 | 86 | 186938 | 187368 | +| Sat Sep 21 13:59:05 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:14:31 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Sat Sep 21 13:59:07 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Wed Jul 3 12:14:33 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Sat Sep 21 13:59:09 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.00636325s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.00104625s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-3.log b/tests/data/juniper-junos/config/jtisim-interfaces-3.log index e5fbebb..855ca94 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-3.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-3.log @@ -90,29 +90,29 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Wed Jul 3 12:12:26 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:03 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:12:28 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:05 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:12:30 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:07 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:12:32 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:09 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:12:34 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:11 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:12:36 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:57:13 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Wed Jul 3 12:12:38 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Sat Sep 21 13:57:15 PDT 2024 | 3960 | 80 | 174838 | 175238 | -Collector Stats for 127.0.0.1:50051 (Run time : 15.001616292s) +Collector Stats for 127.0.0.1:50051 (Run time : 15.000963042s) 80 : in-packets 3960 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-4.log b/tests/data/juniper-junos/config/jtisim-interfaces-4.log index 58a0758..9dbdc60 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-4.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-4.log @@ -87,7 +87,7 @@ compression = none Reconnecting to 127.0.0.1:90052 -Collector Stats for 127.0.0.1:90052 (Run time : 10.0034215s) +Collector Stats for 127.0.0.1:90052 (Run time : 10.004244167s) 0 : in-packets 0 : data points (KV pairs) 0 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-6.log b/tests/data/juniper-junos/config/jtisim-interfaces-6.log index 9fa4b4a..4c90d38 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-6.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-6.log @@ -83,18 +83,18 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - content-type: [application/grpc] jtisim: [yes] + content-type: [application/grpc] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Wed Jul 3 12:12:22 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:56:59 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 3.989988833s) +Collector Stats for 127.0.0.1:50051 (Run time : 3.993855375s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-prometheus.log b/tests/data/juniper-junos/config/jtisim-prometheus.log index 0f7181e..75cf4e7 100644 --- a/tests/data/juniper-junos/config/jtisim-prometheus.log +++ b/tests/data/juniper-junos/config/jtisim-prometheus.log @@ -90,14 +90,14 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Wed Jul 3 12:13:06 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:43 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Wed Jul 3 12:13:08 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Sat Sep 21 13:57:45 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 6.003954292s) +Collector Stats for 127.0.0.1:50051 (Run time : 6.003976125s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) From 1cd29787437145c218fa24233c739c4888ead1ff Mon Sep 17 00:00:00 2001 From: amohit Date: Sat, 21 Sep 2024 14:37:36 -0700 Subject: [PATCH 08/24] updated parserd string --- subscribe_gnmi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subscribe_gnmi.go b/subscribe_gnmi.go index 124b2d9..3c5b144 100644 --- a/subscribe_gnmi.go +++ b/subscribe_gnmi.go @@ -112,7 +112,7 @@ func publishToInflux(jctx *JCtx, mName string, prefixPath string, kvpairs map[st // Check if the value is a slice if members, ok := value.([]string); ok { // Join the slice elements into a single string separated by commas - xpaths[key] = strings.Join(members, ", ") + xpaths[key] = strings.Join(members, ",") } } From 0e1bc073871d87e387ca6decd39d762330e83672 Mon Sep 17 00:00:00 2001 From: Mohit Agrawal Date: Wed, 16 Oct 2024 21:42:33 -0700 Subject: [PATCH 09/24] enable csv logging without internal-jtimon logging --- grpc.go | 2 +- internal_jtimon.go | 72 ++++++++++++++++----------------- main.go | 2 +- statshandler.go | 19 ++++----- subscribe_juniper_junos.go | 2 +- subscribe_juniper_junos_test.go | 12 +++--- 6 files changed, 55 insertions(+), 54 deletions(-) diff --git a/grpc.go b/grpc.go index 7795ba3..a14f0de 100644 --- a/grpc.go +++ b/grpc.go @@ -47,7 +47,7 @@ func getGPRCDialOptions(jctx *JCtx, vendor *vendor) ([]grpc.DialOption, error) { return nil, err } - if *stateHandler { + if *statsHandler { opts = append(opts, grpc.WithStatsHandler(&statshandler{jctx: jctx})) if isCsvStatsEnabled(jctx) { jctx.config.InternalJtimon.csvLogger.Printf(fmt.Sprintf("%s,%s,%s,%s,%s,%s,%s,%s,%s\n", diff --git a/internal_jtimon.go b/internal_jtimon.go index f5a3586..a072a4a 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -3,12 +3,13 @@ package main import ( "encoding/json" "fmt" - gnmi "github.com/Juniper/jtimon/gnmi/gnmi" - na_pb "github.com/Juniper/jtimon/telemetry" "log" "os" "regexp" "strings" + + gnmi "github.com/Juniper/jtimon/gnmi/gnmi" + na_pb "github.com/Juniper/jtimon/telemetry" ) // InternalJtimonConfig type @@ -37,46 +38,45 @@ func initInternalJtimon(jctx *JCtx) { } func internalJtimonLogInit(jctx *JCtx) { - if jctx.config.InternalJtimon.DataLog == "" { - return - } - var out *os.File - - var err error - // Gnmi - out, err = os.OpenFile(jctx.config.InternalJtimon.DataLog, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - log.Printf("Could not create internal jtimon log file(%s): %v\n", jctx.config.InternalJtimon.DataLog, err) - } + if jctx.config.InternalJtimon.DataLog != "" { + var out *os.File + + var err error + // Gnmi + out, err = os.OpenFile(jctx.config.InternalJtimon.DataLog, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + log.Printf("Could not create internal jtimon log file(%s): %v\n", jctx.config.InternalJtimon.DataLog, err) + } - if out != nil { - flags := 0 + if out != nil { + flags := 0 - jctx.config.InternalJtimon.logger = log.New(out, "", flags) - jctx.config.InternalJtimon.out = out + jctx.config.InternalJtimon.logger = log.New(out, "", flags) + jctx.config.InternalJtimon.out = out - log.Printf("logging in %s for %s:%d [in the format of internal jtimon tool]\n", - jctx.config.InternalJtimon.DataLog, jctx.config.Host, jctx.config.Port) - } + log.Printf("logging in %s for %s:%d [in the format of internal jtimon tool]\n", + jctx.config.InternalJtimon.DataLog, jctx.config.Host, jctx.config.Port) + } - // Pre-gnmi - var outPreGnmi *os.File - outPreGnmi, err = os.OpenFile(fmt.Sprintf("%s_pre-gnmi", jctx.config.InternalJtimon.DataLog), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - log.Printf("Could not create internal jtimon log file(%s_pre-gnmi): %v\n", jctx.config.InternalJtimon.DataLog, err) - } + // Pre-gnmi + var outPreGnmi *os.File + outPreGnmi, err = os.OpenFile(fmt.Sprintf("%s_pre-gnmi", jctx.config.InternalJtimon.DataLog), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + log.Printf("Could not create internal jtimon log file(%s_pre-gnmi): %v\n", jctx.config.InternalJtimon.DataLog, err) + } - if outPreGnmi != nil { - flags := 0 + if outPreGnmi != nil { + flags := 0 - jctx.config.InternalJtimon.preGnmiLogger = log.New(outPreGnmi, "", flags) - jctx.config.InternalJtimon.preGnmiOut = outPreGnmi + jctx.config.InternalJtimon.preGnmiLogger = log.New(outPreGnmi, "", flags) + jctx.config.InternalJtimon.preGnmiOut = outPreGnmi - log.Printf("logging in %s_pre-gnmi for %s:%d [in the format of internal jtimon tool]\n", - jctx.config.InternalJtimon.DataLog, jctx.config.Host, jctx.config.Port) + log.Printf("logging in %s_pre-gnmi for %s:%d [in the format of internal jtimon tool]\n", + jctx.config.InternalJtimon.DataLog, jctx.config.Host, jctx.config.Port) + } } - if *stateHandler && jctx.config.InternalJtimon.CsvLog != "" { + if *statsHandler && jctx.config.InternalJtimon.CsvLog != "" { csvStatsLogInit(jctx) } } @@ -92,7 +92,7 @@ func internalJtimonLogStop(jctx *JCtx) { jctx.config.InternalJtimon.preGnmiOut = nil jctx.config.InternalJtimon.preGnmiLogger = nil } - if *stateHandler && jctx.config.InternalJtimon.CsvLog != "" { + if *statsHandler && jctx.config.InternalJtimon.CsvLog != "" { csvStatsLogStop(jctx) } } @@ -123,7 +123,7 @@ func getPath(prefixPath string, pathElements []*gnmi.PathElem) string { } func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *gnmi.SubscribeResponse) { - if jctx.config.InternalJtimon.logger == nil || parseOutput.jHeader == nil { + if jctx.config.InternalJtimon.logger == nil || parseOutput.jHeader == nil || jctx.config.InternalJtimon.DataLog != "" { return } @@ -226,7 +226,7 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g } func jLogInternalJtimonForPreGnmi(jctx *JCtx, ocdata *na_pb.OpenConfigData, outString string) { - if jctx.config.InternalJtimon.logger == nil { + if jctx.config.InternalJtimon.logger == nil || jctx.config.InternalJtimon.DataLog != "" { return } diff --git a/main.go b/main.go index 9f6a11d..44655bc 100644 --- a/main.go +++ b/main.go @@ -17,7 +17,7 @@ var ( outJSON = flag.Bool("json", false, "Convert telemetry packet into JSON") logMux = flag.Bool("log-mux-stdout", false, "All logs to stdout") maxRun = flag.Int64("max-run", 0, "Max run time in seconds") - stateHandler = flag.Bool("stats-handler", false, "Use GRPC statshandler") + statsHandler = flag.Bool("stats-handler", false, "Use GRPC statshandler") versionOnly = flag.Bool("version", false, "Print version and build-time of the binary and exit") compression = flag.String("compression", "", "Enable HTTP/2 compression (gzip)") prom = flag.Bool("prometheus", false, "Stats for prometheus monitoring system") diff --git a/statshandler.go b/statshandler.go index 0d37e05..d29a0ca 100644 --- a/statshandler.go +++ b/statshandler.go @@ -2,13 +2,14 @@ package main import ( "fmt" - gnmi_ext1 "github.com/Juniper/jtimon/gnmi/gnmi_ext" - gnmi_juniper_header_ext "github.com/Juniper/jtimon/gnmi/gnmi_juniper_header_ext" "log" "os" "sync" "time" + gnmi_ext1 "github.com/Juniper/jtimon/gnmi/gnmi_ext" + gnmi_juniper_header_ext "github.com/Juniper/jtimon/gnmi/gnmi_juniper_header_ext" + gnmi_pb "github.com/Juniper/jtimon/gnmi/gnmi" na_pb "github.com/Juniper/jtimon/telemetry" proto "github.com/golang/protobuf/proto" @@ -72,7 +73,7 @@ func (h *statshandler) HandleRPC(ctx context.Context, s stats.RPCStats) { case *stats.InPayload: h.jctx.stats.totalInPayloadLength += uint64(s.(*stats.InPayload).Length) h.jctx.stats.totalInPayloadWireLength += uint64(s.(*stats.InPayload).WireLength) - if *stateHandler && h.jctx.config.InternalJtimon.CsvLog != "" { + if *statsHandler && h.jctx.config.InternalJtimon.CsvLog != "" { switch v := (s.(*stats.InPayload).Payload).(type) { case *na_pb.OpenConfigData: updateStats(h.jctx, v, false) @@ -178,7 +179,7 @@ func (h *statshandler) getKPIStats(subResponse *gnmi_pb.SubscribeResponse) *kpiS } func updateStats(jctx *JCtx, ocData *na_pb.OpenConfigData, needLock bool) { - if !*stateHandler { + if !*statsHandler { return } if needLock { @@ -189,7 +190,7 @@ func updateStats(jctx *JCtx, ocData *na_pb.OpenConfigData, needLock bool) { } func updateStatsKV(jctx *JCtx, needLock bool, count uint64) { - if !*stateHandler { + if !*statsHandler { return } @@ -201,7 +202,7 @@ func updateStatsKV(jctx *JCtx, needLock bool, count uint64) { } func periodicStats(jctx *JCtx) { - if !*stateHandler { + if !*statsHandler { return } pstats := jctx.config.Log.PeriodicStats @@ -245,7 +246,7 @@ func periodicStats(jctx *JCtx) { } func printSummary(jctx *JCtx) { - if !*stateHandler { + if !*statsHandler { return } @@ -272,14 +273,14 @@ func printSummary(jctx *JCtx) { } func isCsvStatsEnabled(jctx *JCtx) bool { - if *stateHandler && jctx.config.InternalJtimon.CsvLog != "" { + if *statsHandler && jctx.config.InternalJtimon.CsvLog != "" { return true } return false } func csvStatsLogInit(jctx *JCtx) { - if !*stateHandler && jctx.config.InternalJtimon.CsvLog == "" { + if !*statsHandler && jctx.config.InternalJtimon.CsvLog == "" { return } var out *os.File diff --git a/subscribe_juniper_junos.go b/subscribe_juniper_junos.go index dd089c3..fb81701 100644 --- a/subscribe_juniper_junos.go +++ b/subscribe_juniper_junos.go @@ -175,7 +175,7 @@ func subSendAndReceive(conn *grpc.ClientConn, jctx *JCtx, } } - if *print || *stateHandler || IsVerboseLogging(jctx) || isInternalJtimonLogging(jctx) { + if *print || *statsHandler || IsVerboseLogging(jctx) || isInternalJtimonLogging(jctx) { handleOnePacket(ocData, jctx) } diff --git a/subscribe_juniper_junos_test.go b/subscribe_juniper_junos_test.go index a8a03bf..35f4790 100644 --- a/subscribe_juniper_junos_test.go +++ b/subscribe_juniper_junos_test.go @@ -17,7 +17,7 @@ import ( func TestJTISIMSigHup(t *testing.T) { flag.Parse() *noppgoroutines = true - *stateHandler = true + *statsHandler = true *prefixCheck = true config := "tests/data/juniper-junos/config/jtisim-interfaces-file-list-sig.json" @@ -152,7 +152,7 @@ func TestJTISIMSigHup(t *testing.T) { func TestJTISIMSigHupChanged(t *testing.T) { flag.Parse() *noppgoroutines = true - *stateHandler = true + *statsHandler = true *prefixCheck = true config := "tests/data/juniper-junos/config/jtisim-interfaces-file-list-sig.json" @@ -284,7 +284,7 @@ func TestJTISIMSigHupChanged(t *testing.T) { func TestJTISIMSigInt(t *testing.T) { flag.Parse() *noppgoroutines = true - *stateHandler = true + *statsHandler = true *prefixCheck = true config := "tests/data/juniper-junos/config/jtisim-interfaces-file-list-sig.json" @@ -323,7 +323,7 @@ func TestJTISIMSigInt(t *testing.T) { func TestJTISIMRetrySigInt(t *testing.T) { flag.Parse() *noppgoroutines = true - *stateHandler = true + *statsHandler = true *prefixCheck = true config := "tests/data/juniper-junos/config/jtisim-interfaces-file-list-sig-int.json" @@ -385,7 +385,7 @@ func TestPrometheus(t *testing.T) { port := 8090 *noppgoroutines = true - *stateHandler = true + *statsHandler = true *prom = true exporter = promInit() @@ -528,7 +528,7 @@ func TestJTISIMMaxRun(t *testing.T) { t.Run(test.name+test.compression, func(t *testing.T) { flag.Parse() *noppgoroutines = true - *stateHandler = true + *statsHandler = true *prefixCheck = true *compression = test.compression From f1c4bc12e2d94d22c2de6e9d2f2fa82a05e4eac5 Mon Sep 17 00:00:00 2001 From: Mohit Agrawal Date: Wed, 16 Oct 2024 22:08:56 -0700 Subject: [PATCH 10/24] updated new tests --- .../config/jtisim-influx-alias.log | 11 +++++--- .../juniper-junos/config/jtisim-influx.log | 26 ++++++++--------- .../config/jtisim-interfaces-1.log | 26 ++++++++--------- .../config/jtisim-interfaces-2.log | 28 +++++++++---------- .../config/jtisim-interfaces-3.log | 16 +++++------ .../config/jtisim-interfaces-4.log | 2 +- .../config/jtisim-interfaces-6.log | 6 ++-- .../config/jtisim-prometheus.log | 6 ++-- 8 files changed, 62 insertions(+), 59 deletions(-) diff --git a/tests/data/juniper-junos/config/jtisim-influx-alias.log b/tests/data/juniper-junos/config/jtisim-influx-alias.log index 9b8d1b2..dc85cd1 100644 --- a/tests/data/juniper-junos/config/jtisim-influx-alias.log +++ b/tests/data/juniper-junos/config/jtisim-influx-alias.log @@ -92,19 +92,22 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:58:14 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Sat Sep 21 13:58:16 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:18 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 8.00172925s) + + +Collector Stats for 127.0.0.1:50051 (Run time : 8.005373428s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-influx.log b/tests/data/juniper-junos/config/jtisim-influx.log index dbd6f5a..06c60ba 100644 --- a/tests/data/juniper-junos/config/jtisim-influx.log +++ b/tests/data/juniper-junos/config/jtisim-influx.log @@ -93,50 +93,50 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:57:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:52 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Sat Sep 21 13:57:51 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:54 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:56 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:58 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:57 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:00 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 -| Sat Sep 21 13:57:59 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:07:02 PDT 2024 | 3960 | 80 | 174838 | 175238 | Batch write successful! Post batch write available points: 0 -| Sat Sep 21 13:58:01 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:07:04 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:03 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:07:06 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:05 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:07:08 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:07 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:07:10 PDT 2024 | 4356 | 88 | 192322 | 192762 | Batch processing: #packets:40 #points:40 -| Sat Sep 21 13:58:09 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:07:12 PDT 2024 | 5940 | 120 | 262258 | 262858 | Batch write successful! Post batch write available points: 0 -| Sat Sep 21 13:58:11 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:07:14 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.002847916s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.003040495s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-1.log b/tests/data/juniper-junos/config/jtisim-interfaces-1.log index dcca0fb..805322a 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-1.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-1.log @@ -90,44 +90,44 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:58:47 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:50 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:52 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:51 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:54 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:56 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:58 PDT 2024 | 2317 | 47 | 102210 | 102445 | -| Sat Sep 21 13:58:57 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:00 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:59 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:02 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:01 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:04 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:03 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:06 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:05 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:08 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Sat Sep 21 13:59:07 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:08:10 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Sat Sep 21 13:59:09 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:08:12 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.004069667s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.001059871s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-2.log b/tests/data/juniper-junos/config/jtisim-interfaces-2.log index 28f4e1c..19b1893 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-2.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-2.log @@ -83,51 +83,51 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - content-type: [application/grpc] jtisim: [yes] + content-type: [application/grpc] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:58:47 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:50 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:52 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:51 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:54 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:56 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:07:58 PDT 2024 | 2258 | 46 | 99518 | 99748 | -| Sat Sep 21 13:58:57 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:00 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:59 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:02 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:01 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:04 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:03 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:06 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:05 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:08:08 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Sat Sep 21 13:59:07 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:08:10 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Sat Sep 21 13:59:09 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:08:12 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.00104625s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.000529607s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-3.log b/tests/data/juniper-junos/config/jtisim-interfaces-3.log index 855ca94..521b73b 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-3.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-3.log @@ -90,29 +90,29 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:57:03 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:06 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:05 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:08 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:07 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:10 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:09 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:12 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:11 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:14 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:13 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:06:16 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:57:15 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:06:18 PDT 2024 | 3960 | 80 | 174838 | 175238 | -Collector Stats for 127.0.0.1:50051 (Run time : 15.000963042s) +Collector Stats for 127.0.0.1:50051 (Run time : 15.000798143s) 80 : in-packets 3960 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-4.log b/tests/data/juniper-junos/config/jtisim-interfaces-4.log index 9dbdc60..a002bda 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-4.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-4.log @@ -87,7 +87,7 @@ compression = none Reconnecting to 127.0.0.1:90052 -Collector Stats for 127.0.0.1:90052 (Run time : 10.004244167s) +Collector Stats for 127.0.0.1:90052 (Run time : 10.00120697s) 0 : in-packets 0 : data points (KV pairs) 0 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-6.log b/tests/data/juniper-junos/config/jtisim-interfaces-6.log index 4c90d38..91ba878 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-6.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-6.log @@ -83,18 +83,18 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - jtisim: [yes] content-type: [application/grpc] + jtisim: [yes] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:56:59 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:02 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 3.993855375s) +Collector Stats for 127.0.0.1:50051 (Run time : 3.999050845s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-prometheus.log b/tests/data/juniper-junos/config/jtisim-prometheus.log index 75cf4e7..4f8e119 100644 --- a/tests/data/juniper-junos/config/jtisim-prometheus.log +++ b/tests/data/juniper-junos/config/jtisim-prometheus.log @@ -90,14 +90,14 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:57:43 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:46 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:45 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:06:48 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 6.003976125s) +Collector Stats for 127.0.0.1:50051 (Run time : 6.000915187s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) From dfc0441753cb3eaf2b238ec42cba0dec4281c1d2 Mon Sep 17 00:00:00 2001 From: Mohit Agrawal Date: Wed, 16 Oct 2024 22:48:06 -0700 Subject: [PATCH 11/24] added hex and float fix --- gnmi_utils.go | 64 +++++++++++-------- jtisim/gnmi_utils.go | 19 ++++-- .../config/jtisim-influx-alias.log | 11 ++-- .../juniper-junos/config/jtisim-influx.log | 28 ++++---- .../config/jtisim-interfaces-1.log | 28 ++++---- .../config/jtisim-interfaces-2.log | 26 ++++---- .../config/jtisim-interfaces-3.log | 16 ++--- .../config/jtisim-interfaces-4.log | 2 +- .../config/jtisim-interfaces-6.log | 6 +- .../config/jtisim-prometheus.log | 6 +- 10 files changed, 115 insertions(+), 91 deletions(-) diff --git a/gnmi_utils.go b/gnmi_utils.go index 6dbf45c..ad32e6f 100644 --- a/gnmi_utils.go +++ b/gnmi_utils.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "encoding/binary" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -72,48 +74,47 @@ type gnmiParseOutputT struct { inKvs uint64 } - // parseKeyValuePairs takes an input string in the format `prefix[key = "value"]` // and returns the prefix as a string and the key-value pairs as a map. // // The input string can contain multiple key-value pairs in the same or different brackets. // The keys and values are trimmed of quotes and spaces. // -// If the input string does not contain any brackets, the function returns the entire string +// If the input string does not contain any brackets, the function returns the entire string // as the prefix and an empty map. func parseKeyValuePairs(input string) (string, map[string]string) { - // Initialize a map to hold the key-value pairs - result := make(map[string]string) + // Initialize a map to hold the key-value pairs + result := make(map[string]string) - // Split the input string by the first occurrence of [" to get the prefix and the rest - parts := strings.SplitN(input, "[", 2) - prefix := parts[0] + // Split the input string by the first occurrence of [" to get the prefix and the rest + parts := strings.SplitN(input, "[", 2) + prefix := parts[0] - // Match anything inside square brackets - re := regexp.MustCompile(`\[(.*?)\]`) - matches := re.FindAllString(input, -1) + // Match anything inside square brackets + re := regexp.MustCompile(`\[(.*?)\]`) + matches := re.FindAllString(input, -1) - for _, match := range matches { - // Remove the square brackets - match = strings.Trim(match, "[]") + for _, match := range matches { + // Remove the square brackets + match = strings.Trim(match, "[]") - // Split by "and" to support multiple key-value pairs in the same brackets - pairs := strings.Split(match, "and") + // Split by "and" to support multiple key-value pairs in the same brackets + pairs := strings.Split(match, "and") - for _, pair := range pairs { - // Split by "=" to separate the key and value - kv := strings.Split(pair, "=") + for _, pair := range pairs { + // Split by "=" to separate the key and value + kv := strings.Split(pair, "=") - // Trim the quotes and spaces from the key and value - key := strings.Trim(kv[0], " \"") - value := strings.Trim(kv[1], " \"") + // Trim the quotes and spaces from the key and value + key := strings.Trim(kv[0], " \"") + value := strings.Trim(kv[1], " \"") - // Add the key-value pair to the result map - result[key] = value - } - } + // Add the key-value pair to the result map + result[key] = value + } + } - return prefix, result + return prefix, result } // Convert xpath to gNMI path @@ -359,7 +360,16 @@ func gnmiParseValue(gnmiValue *gnmi.TypedValue, ts bool, enableUint bool) (inter case *gnmi.TypedValue_BoolVal: value = gnmiValue.GetBoolVal() case *gnmi.TypedValue_BytesVal: - value = gnmiValue.GetBytesVal() + byteVal := gnmiValue.GetBytesVal() + if len(byteVal) == 4 { + var double_val float32 + if err := binary.Read(bytes.NewReader(byteVal), binary.LittleEndian, &double_val); err != nil { + value = hex.EncodeToString(byteVal) + } + value = fmt.Sprintf("%0.2f", double_val) + } else { + value = hex.EncodeToString(byteVal) + } case *gnmi.TypedValue_AsciiVal: value = gnmiValue.GetAsciiVal() case *gnmi.TypedValue_AnyVal: diff --git a/jtisim/gnmi_utils.go b/jtisim/gnmi_utils.go index 0301bce..91f7b1b 100644 --- a/jtisim/gnmi_utils.go +++ b/jtisim/gnmi_utils.go @@ -2,19 +2,21 @@ package jtisim import ( "bytes" + "encoding/binary" + "encoding/hex" "encoding/json" "errors" "fmt" "math" "strings" - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" - google_protobuf "github.com/golang/protobuf/ptypes/any" gnmi "github.com/Juniper/jtimon/gnmi/gnmi" gnmi_ext1 "github.com/Juniper/jtimon/gnmi/gnmi_ext" gnmi_juniper_header "github.com/Juniper/jtimon/gnmi/gnmi_juniper_header" gnmi_juniper_header_ext "github.com/Juniper/jtimon/gnmi/gnmi_juniper_header_ext" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + google_protobuf "github.com/golang/protobuf/ptypes/any" ) const ( @@ -300,7 +302,16 @@ func gnmiParseValue(gnmiValue *gnmi.TypedValue, ts bool) (interface{}, error) { case *gnmi.TypedValue_BoolVal: value = gnmiValue.GetBoolVal() case *gnmi.TypedValue_BytesVal: - value = gnmiValue.GetBytesVal() + byteVal := gnmiValue.GetBytesVal() + if len(byteVal) == 4 { + var double_val float32 + if err := binary.Read(bytes.NewReader(byteVal), binary.LittleEndian, &double_val); err != nil { + value = hex.EncodeToString(byteVal) + } + value = fmt.Sprintf("%0.2f", double_val) + } else { + value = hex.EncodeToString(byteVal) + } case *gnmi.TypedValue_AsciiVal: value = gnmiValue.GetAsciiVal() case *gnmi.TypedValue_AnyVal: diff --git a/tests/data/juniper-junos/config/jtisim-influx-alias.log b/tests/data/juniper-junos/config/jtisim-influx-alias.log index 9b8d1b2..13bf70a 100644 --- a/tests/data/juniper-junos/config/jtisim-influx-alias.log +++ b/tests/data/juniper-junos/config/jtisim-influx-alias.log @@ -92,19 +92,22 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:58:14 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:24 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Sat Sep 21 13:58:16 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:26 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:18 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:28 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:30 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 8.00172925s) + + +Collector Stats for 127.0.0.1:50051 (Run time : 8.003507949s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-influx.log b/tests/data/juniper-junos/config/jtisim-influx.log index dbd6f5a..c26d341 100644 --- a/tests/data/juniper-junos/config/jtisim-influx.log +++ b/tests/data/juniper-junos/config/jtisim-influx.log @@ -93,50 +93,50 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:57:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:42:59 PDT 2024 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Sat Sep 21 13:57:51 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:01 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:03 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:05 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:57 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:07 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Batch processing: #packets:40 #points:40 -| Sat Sep 21 13:57:59 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:43:09 PDT 2024 | 3960 | 80 | 174838 | 175238 | +Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Sat Sep 21 13:58:01 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:43:11 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:03 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:43:13 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:05 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:43:15 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:07 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:43:17 PDT 2024 | 4158 | 84 | 183580 | 184000 | Batch processing: #packets:40 #points:40 -| Sat Sep 21 13:58:09 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:43:19 PDT 2024 | 5940 | 120 | 262258 | 262858 | Batch write successful! Post batch write available points: 0 -| Sat Sep 21 13:58:11 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:43:21 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.002847916s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.003309177s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-1.log b/tests/data/juniper-junos/config/jtisim-interfaces-1.log index dcca0fb..ef7d122 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-1.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-1.log @@ -83,51 +83,51 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - jtisim: [yes] content-type: [application/grpc] + jtisim: [yes] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:58:47 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:57 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:59 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:51 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:44:01 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:44:03 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:44:05 PDT 2024 | 2178 | 44 | 96160 | 96380 | -| Sat Sep 21 13:58:57 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:07 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:59 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:09 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:01 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:11 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:03 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:13 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:05 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:15 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:07 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:44:17 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Sat Sep 21 13:59:09 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:44:19 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.004069667s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.001419905s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-2.log b/tests/data/juniper-junos/config/jtisim-interfaces-2.log index 28f4e1c..bdb4c1f 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-2.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-2.log @@ -90,44 +90,44 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:58:47 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:57 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:49 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:43:59 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:51 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:44:01 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:44:03 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:58:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:44:05 PDT 2024 | 2178 | 44 | 96160 | 96380 | -| Sat Sep 21 13:58:57 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:07 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:58:59 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:09 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:01 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:11 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:03 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:13 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:59:05 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:44:15 PDT 2024 | 5624 | 114 | 248132 | 248702 | -| Sat Sep 21 13:59:07 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:44:17 PDT 2024 | 5940 | 120 | 262258 | 262858 | -| Sat Sep 21 13:59:09 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Wed Oct 16 22:44:19 PDT 2024 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.00104625s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.001005316s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-3.log b/tests/data/juniper-junos/config/jtisim-interfaces-3.log index 855ca94..dd029ac 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-3.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-3.log @@ -90,29 +90,29 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:57:03 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:42:13 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:05 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:42:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:07 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:42:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:09 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:42:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:11 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:42:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:13 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:42:23 PDT 2024 | 3960 | 80 | 174838 | 175238 | -| Sat Sep 21 13:57:15 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Wed Oct 16 22:42:25 PDT 2024 | 3960 | 80 | 174838 | 175238 | -Collector Stats for 127.0.0.1:50051 (Run time : 15.000963042s) +Collector Stats for 127.0.0.1:50051 (Run time : 15.001410909s) 80 : in-packets 3960 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-4.log b/tests/data/juniper-junos/config/jtisim-interfaces-4.log index 9dbdc60..ad3fe72 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-4.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-4.log @@ -87,7 +87,7 @@ compression = none Reconnecting to 127.0.0.1:90052 -Collector Stats for 127.0.0.1:90052 (Run time : 10.004244167s) +Collector Stats for 127.0.0.1:90052 (Run time : 10.00158514s) 0 : in-packets 0 : data points (KV pairs) 0 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-6.log b/tests/data/juniper-junos/config/jtisim-interfaces-6.log index 4c90d38..9483561 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-6.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-6.log @@ -83,18 +83,18 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - jtisim: [yes] content-type: [application/grpc] + jtisim: [yes] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:56:59 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:42:09 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 3.993855375s) +Collector Stats for 127.0.0.1:50051 (Run time : 3.998631006s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-prometheus.log b/tests/data/juniper-junos/config/jtisim-prometheus.log index 75cf4e7..3dbcde6 100644 --- a/tests/data/juniper-junos/config/jtisim-prometheus.log +++ b/tests/data/juniper-junos/config/jtisim-prometheus.log @@ -90,14 +90,14 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Sat Sep 21 13:57:43 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:42:53 PDT 2024 | 1980 | 40 | 87418 | 87618 | -| Sat Sep 21 13:57:45 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Wed Oct 16 22:42:55 PDT 2024 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 6.003976125s) +Collector Stats for 127.0.0.1:50051 (Run time : 6.000813866s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) From a29d4b0bea9ee260f9ad07b3baa85f913a65c75d Mon Sep 17 00:00:00 2001 From: amohit Date: Thu, 17 Oct 2024 21:11:26 -0700 Subject: [PATCH 12/24] when json output is configured, print in json format --- internal_jtimon.go | 77 +++++++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/internal_jtimon.go b/internal_jtimon.go index a072a4a..a4b5984 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -123,22 +123,11 @@ func getPath(prefixPath string, pathElements []*gnmi.PathElem) string { } func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *gnmi.SubscribeResponse) { - if jctx.config.InternalJtimon.logger == nil || parseOutput.jHeader == nil || jctx.config.InternalJtimon.DataLog != "" { + if jctx.config.InternalJtimon.logger == nil || parseOutput.jHeader == nil || jctx.config.InternalJtimon.DataLog == "" { return } - // Log here in the format of internal jtimon - //var ( - // jxpaths map[string]interface{} - // jGnmiHdr string - //) - - s := "" - //if parseOutput.jHeader.hdr != nil { - // s += fmt.Sprintf("system_id: %s\n", parseOutput.jHeader.hdr.String()) - //} else { - // s += fmt.Sprintf("system_id: %s\n", parseOutput.jHeader.hdrExt.String()) - //} + s := "" // Keep the original string format var jHeaderData map[string]interface{} jGnmiHdrExt, err := json.Marshal(parseOutput.jHeader.hdrExt) if err != nil { @@ -151,6 +140,9 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g return } + // Prepare outputData for JSON format + outputData := make(map[string]interface{}) // Map to hold the JSON output + outJHeaderKeys := []string{ "system_id", "component_id", @@ -161,6 +153,7 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g "sequence_number", "export_timestamp", } + for _, v := range outJHeaderKeys { if _, ok := jHeaderData[v]; ok { strVal := convertToString(jHeaderData[v]) @@ -169,13 +162,17 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g "to Streamed path. Unable to convert extension value: %v to string. ", v, jHeaderData[v])) continue } + // Add to plain string format s += fmt.Sprintf("%s: %s\n", v, strVal) + + // Add to JSON structure + outputData[v] = strVal } } notif := rsp.GetUpdate() if notif != nil { - // Form an xpath for prefix here, as the internal jtimon tool does it this way. + // Plain text formatting prefixPath := "" if !jctx.config.Vendor.RemoveNS { prefixPath = notif.Prefix.GetOrigin() @@ -192,37 +189,69 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g s += fmt.Sprintf( "Update {\n\ttimestamp: %d\n\tprefix: %v\n", notif.GetTimestamp(), prefixPath) - // Parse all the deletes here + // Create a map to hold notification data for JSON output + notifData := make(map[string]interface{}) + notifData["timestamp"] = notif.GetTimestamp() + notifData["prefix"] = prefixPath + + // Parse delete paths (both for string and JSON) + var delPaths []string for _, d := range notif.Delete { delPath := getPath(prefixPath, d.GetElem()) - s += fmt.Sprintf("del_path: %s", delPath) + s += fmt.Sprintf("del_path: %s\n", delPath) + delPaths = append(delPaths, delPath) } + notifData["del_paths"] = delPaths - // Parse all the updates here + // Parse updates (both for string and JSON) + var updates []map[string]interface{} for _, u := range notif.Update { - notifString := u.String() s += fmt.Sprintf("Update {\n\tpath {\n") + + update := make(map[string]interface{}) + re := regexp.MustCompile(`name:\"(.*?)\"`) matches := re.FindAllStringSubmatch(u.String(), -1) + var elems []string for _, match := range matches { - s += fmt.Sprintf("\t\telem {\n\t\t\t") - s += fmt.Sprintf("name: %s\n\t\t}\n", match[1]) + s += fmt.Sprintf("\t\telem {\n\t\t\tname: %s\n\t\t}\n", match[1]) + elems = append(elems, match[1]) } + update["elems"] = elems - // Define regular expression pattern to match "key:val" + // Extract key-value pairs for the update + notifString := u.String() re = regexp.MustCompile(`val:\{(.*?)\}`) result := re.FindStringSubmatch(notifString) if len(result) > 1 { keyVal := strings.Split(result[1], ":") - s += fmt.Sprintf("\t\tval {\n\t\t\t") - s += fmt.Sprintf("%s: %s\n\t\t}\n", keyVal[0], keyVal[1]) + s += fmt.Sprintf("\t\tval {\n\t\t\t%s: %s\n\t\t}\n", keyVal[0], keyVal[1]) + update["key"] = keyVal[0] + update["value"] = keyVal[1] } + updates = append(updates, update) s += fmt.Sprintf("\t}\n") } s += fmt.Sprintf("}\n") + notifData["updates"] = updates // Add update data to JSON output + + outputData["notification"] = notifData // Add notification to JSON output } - jctx.config.InternalJtimon.logger.Printf(s) + + if *outJSON { + // Marshal the JSON data and print it + jsonOutput, err := json.MarshalIndent(outputData, "", " ") + if err != nil { + jLog(jctx, "Error marshaling output to JSON") + return + } + jctx.config.InternalJtimon.logger.Printf("%s\n", jsonOutput) + } else { + // Print the plain text output + jctx.config.InternalJtimon.logger.Print(s) + } + } func jLogInternalJtimonForPreGnmi(jctx *JCtx, ocdata *na_pb.OpenConfigData, outString string) { From 4a3c21827ad4ca19d8e7317c5e4db91d290948d4 Mon Sep 17 00:00:00 2001 From: amohit Date: Thu, 17 Oct 2024 21:23:25 -0700 Subject: [PATCH 13/24] remove extra escape if present and fixed regression for internal jtimon pre-gnmi logging --- internal_jtimon.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal_jtimon.go b/internal_jtimon.go index a4b5984..d497350 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -227,7 +227,7 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g keyVal := strings.Split(result[1], ":") s += fmt.Sprintf("\t\tval {\n\t\t\t%s: %s\n\t\t}\n", keyVal[0], keyVal[1]) update["key"] = keyVal[0] - update["value"] = keyVal[1] + update["value"] = strings.Trim(keyVal[1], "\"") } updates = append(updates, update) @@ -255,7 +255,7 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g } func jLogInternalJtimonForPreGnmi(jctx *JCtx, ocdata *na_pb.OpenConfigData, outString string) { - if jctx.config.InternalJtimon.logger == nil || jctx.config.InternalJtimon.DataLog != "" { + if jctx.config.InternalJtimon.logger == nil || jctx.config.InternalJtimon.DataLog == "" { return } From c2c365651b81a66a0ceacaceafc3928cd58360ee Mon Sep 17 00:00:00 2001 From: amohit Date: Fri, 18 Oct 2024 00:32:25 -0700 Subject: [PATCH 14/24] added json out flag check and printing json output for pre-gnmi for internal-jtimon --- subscribe_juniper_junos.go | 88 +++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 16 deletions(-) diff --git a/subscribe_juniper_junos.go b/subscribe_juniper_junos.go index fb81701..e20969d 100644 --- a/subscribe_juniper_junos.go +++ b/subscribe_juniper_junos.go @@ -32,61 +32,103 @@ const ( func handleOnePacket(ocData *na_pb.OpenConfigData, jctx *JCtx) { updateStats(jctx, ocData, true) + // Struct to hold the data for JSON output + logData := make(map[string]interface{}) + + // String format logging s := "" if *print || (IsVerboseLogging(jctx) && !*print) || (isInternalJtimonLogging(jctx)) { - s += fmt.Sprintf("system_id: %s\n", ocData.SystemId) - s += fmt.Sprintf("component_id: %d\n", ocData.ComponentId) - s += fmt.Sprintf("sub_component_id: %d\n", ocData.SubComponentId) - s += fmt.Sprintf("path: %s\n", ocData.Path) - s += fmt.Sprintf("sequence_number: %d\n", ocData.SequenceNumber) - s += fmt.Sprintf("timestamp: %d\n", ocData.Timestamp) - s += fmt.Sprintf("sync_response: %v\n", ocData.SyncResponse) - if ocData.SyncResponse { + // Add data to both the string and map + systemId := ocData.SystemId + componentId := ocData.ComponentId + subComponentId := ocData.SubComponentId + path := ocData.Path + sequenceNumber := ocData.SequenceNumber + timestamp := ocData.Timestamp + syncResponse := ocData.SyncResponse + + // String logging + s += fmt.Sprintf("system_id: %s\n", systemId) + s += fmt.Sprintf("component_id: %d\n", componentId) + s += fmt.Sprintf("sub_component_id: %d\n", subComponentId) + s += fmt.Sprintf("path: %s\n", path) + s += fmt.Sprintf("sequence_number: %d\n", sequenceNumber) + s += fmt.Sprintf("timestamp: %d\n", timestamp) + s += fmt.Sprintf("sync_response: %v\n", syncResponse) + if syncResponse { s += "Received sync_response\n" } + // Add data to map for JSON + logData["system_id"] = systemId + logData["component_id"] = componentId + logData["sub_component_id"] = subComponentId + logData["path"] = path + logData["sequence_number"] = sequenceNumber + logData["timestamp"] = timestamp + logData["sync_response"] = syncResponse + del := ocData.GetDelete() + deletePaths := []string{} for _, d := range del { - s += fmt.Sprintf("Delete: %s\n", d.GetPath()) + path := d.GetPath() + s += fmt.Sprintf("Delete: %s\n", path) + deletePaths = append(deletePaths, path) } + logData["delete_paths"] = deletePaths } prefixSeen := false + kvData := []map[string]interface{}{} + for _, kv := range ocData.Kv { updateStatsKV(jctx, true, 1) + kvItem := make(map[string]interface{}) + kvItem["key"] = kv.Key + if *print || (IsVerboseLogging(jctx) && !*print) || (isInternalJtimonLogging(jctx)) { s += fmt.Sprintf(" key: %s\n", kv.Key) switch value := kv.Value.(type) { case *na_pb.KeyValue_DoubleValue: s += fmt.Sprintf(" double_value: %v\n", value.DoubleValue) + kvItem["double_value"] = value.DoubleValue case *na_pb.KeyValue_IntValue: s += fmt.Sprintf(" int_value: %d\n", value.IntValue) + kvItem["int_value"] = value.IntValue case *na_pb.KeyValue_UintValue: s += fmt.Sprintf(" uint_value: %d\n", value.UintValue) + kvItem["uint_value"] = value.UintValue case *na_pb.KeyValue_SintValue: s += fmt.Sprintf(" sint_value: %d\n", value.SintValue) + kvItem["sint_value"] = value.SintValue case *na_pb.KeyValue_BoolValue: s += fmt.Sprintf(" bool_value: %v\n", value.BoolValue) + kvItem["bool_value"] = value.BoolValue case *na_pb.KeyValue_StrValue: s += fmt.Sprintf(" str_value: %s\n", value.StrValue) + kvItem["str_value"] = value.StrValue case *na_pb.KeyValue_BytesValue: s += fmt.Sprintf(" bytes_value: %s\n", value.BytesValue) + kvItem["bytes_value"] = value.BytesValue case *na_pb.KeyValue_LeaflistValue: s += fmt.Sprintf(" leaf_list_value: %s\n", value.LeaflistValue) e := kv.GetLeaflistValue().Element + elements := []string{} for _, elem := range e { - switch elem.Value.(type) { - case *na_pb.TypedValue_LeaflistStrValue: - llStrValue := elem.GetLeaflistStrValue() + if llStrValue := elem.GetLeaflistStrValue(); llStrValue != "" { s += fmt.Sprintf(" leaf_list_value(element): %s\n", llStrValue) + elements = append(elements, llStrValue) } } - + kvItem["leaf_list_elements"] = elements default: s += fmt.Sprintf(" default: %v\n", value) + kvItem["default"] = value } + + kvData = append(kvData, kvItem) } if kv.Key == "__prefix__" { @@ -100,12 +142,26 @@ func handleOnePacket(ocData *na_pb.OpenConfigData, jctx *JCtx) { } } - if s != "" && isInternalJtimonLogging(jctx) { - jLogInternalJtimonForPreGnmi(jctx, nil, s) - } + logData["kv"] = kvData + if s != "" && (*print || (IsVerboseLogging(jctx) && !*print)) { jLog(jctx, s) } + + if isInternalJtimonLogging(jctx) { + if *outJSON { + jsonOutput, err := json.MarshalIndent(logData, "", " ") + if err != nil { + jLog(jctx, fmt.Sprintf("Error marshaling to JSON: %v", err)) + return + } + jLogInternalJtimonForPreGnmi(jctx, nil, string(jsonOutput)) + } else { + if s != "" { + jLogInternalJtimonForPreGnmi(jctx, nil, s) + } + } + } } // subSendAndReceive handles the following From 9d216edfaa0372c7efa4c0cb24fb178d7d51dfb2 Mon Sep 17 00:00:00 2001 From: amohit Date: Wed, 26 Feb 2025 10:45:58 -0800 Subject: [PATCH 15/24] fixed incomplete outputs --- internal_jtimon.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal_jtimon.go b/internal_jtimon.go index d497350..b1041e5 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -221,10 +221,10 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g // Extract key-value pairs for the update notifString := u.String() - re = regexp.MustCompile(`val:\{(.*?)\}`) + re = regexp.MustCompile(`val:\{(.*)\}`) result := re.FindStringSubmatch(notifString) if len(result) > 1 { - keyVal := strings.Split(result[1], ":") + keyVal := strings.SplitN(result[1], ":", 2) s += fmt.Sprintf("\t\tval {\n\t\t\t%s: %s\n\t\t}\n", keyVal[0], keyVal[1]) update["key"] = keyVal[0] update["value"] = strings.Trim(keyVal[1], "\"") From f7c600c48d77b41dcc473406d11cc8b72fbec434 Mon Sep 17 00:00:00 2001 From: Nilesh Simaria Date: Thu, 12 Jun 2025 12:42:49 -0700 Subject: [PATCH 16/24] add sample config for gnmi target-defined --- sample-config/gnmi-target-defined.json | 62 ++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 sample-config/gnmi-target-defined.json diff --git a/sample-config/gnmi-target-defined.json b/sample-config/gnmi-target-defined.json new file mode 100644 index 0000000..89ce3af --- /dev/null +++ b/sample-config/gnmi-target-defined.json @@ -0,0 +1,62 @@ +{ + "host": "host", + "port": 32767, + "user": "user", + "password": "password", + "cid": "unique-cid", + "eos": true, + "grpc": { + "ws": 10485760 + }, + "influx": { + "server": "ip", + "port": 8086, + "dbname": "db-name", + "user": "user", + "password": "password", + "recreate": true, + "measurement": "measurement-name" + }, + "vendor": { + "gnmi": { + "encoding": "protobuf" + } + }, + "paths": [ + { + "path": "/network-instances/", + "mode": "target-defined", + "freq": 30000 + }, + { + "path": "/system/", + "mode": "target-defined", + "freq": 30000 + }, + { + "path": "/interfaces/", + "mode": "target-defined", + "freq": 30000 + }, + { + "path": "/components/", + "mode": "target-defined", + "freq": 30000 + }, + { + "path": "/lacp/", + "mode": "target-defined", + "freq": 30000 + }, + { + "path": "/lldp/", + "mode": "target-defined", + "freq": 30000 + }, + { + "path": "/qos/", + "mode": "target-defined", + "freq": 30000 + } + ] +} \ No newline at end of file From f1c02d7a8794def349ea273cef654fb64edc33ff Mon Sep 17 00:00:00 2001 From: Nilesh Simaria Date: Tue, 24 Jun 2025 15:19:50 -0700 Subject: [PATCH 17/24] Support IPv6 --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/README.md b/README.md index a563d76..d92436b 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,63 @@ eos : end of sync. Tell Junos to send end of sync for on-change subscriptions. grpc/ws : window size of grpc for slower clients + +## IPv6 Support + +jtimon supports both IPv4 and IPv6 addresses for connecting to network devices. + +### IPv4 Configuration +```json +{ + "host": "192.168.1.1", + "port": 32767, + "user": "username", + "password": "password", + "paths": [ + { + "path": "/interfaces", + "freq": 10000 + } + ] +} +``` + +### IPv6 Configuration +When using IPv6 addresses, wrap the address in square brackets `[]`: + +```json +{ + "host": "[2001:db8::1]", + "port": 32767, + "user": "username", + "password": "password", + "paths": [ + { + "path": "/interfaces", + "freq": 10000 + } + ] +} +``` + +**Note**: The square brackets are required for IPv6 addresses to properly distinguish the address from the port number, following RFC 3986 standards. + +### Examples of Valid IPv6 Formats +- `"[::1]"` - IPv6 loopback +- `"[2001:db8::1]"` - Full IPv6 address +- `"[fe80::1%eth0]"` - Link-local with zone identifier +- `"[::ffff:192.0.2.1]"` - IPv4-mapped IPv6 address + +### Command Line Usage +When using command line options for dial-out or server mode: +```bash +# IPv4 +./jtimon --host 127.0.0.1 --port 32767 + +# IPv6 +./jtimon --host ::1 --port 32767 +``` + ## Kafka Publish To publish gRPC/Openconfig JTI data to Kafka, use the following json config. From 3cef8bbaccd8775757c8bd59c368554f149308c2 Mon Sep 17 00:00:00 2001 From: amohit Date: Mon, 13 Oct 2025 11:53:58 +0530 Subject: [PATCH 18/24] added support for target and heartbeat interval --- config.go | 14 ++++---- subscribe_gnmi.go | 34 +++++++++++++++++++ .../cisco-ios-xr/config/xr-all-influx.log | 4 ++- .../config/xr-wdsysmon-influx.log | 4 ++- .../config/jtisim-influx-alias.log | 14 ++++---- .../juniper-junos/config/jtisim-influx.log | 30 ++++++++-------- .../config/jtisim-interfaces-1.log | 32 +++++++++-------- .../config/jtisim-interfaces-2.log | 32 +++++++++-------- .../config/jtisim-interfaces-3.log | 20 ++++++----- .../config/jtisim-interfaces-4.log | 6 ++-- .../config/jtisim-interfaces-6.log | 8 +++-- .../config/jtisim-prometheus.log | 13 ++++--- 12 files changed, 135 insertions(+), 76 deletions(-) diff --git a/config.go b/config.go index 20efddf..6ba2e7b 100644 --- a/config.go +++ b/config.go @@ -95,12 +95,14 @@ type TLSConfig struct { // PathsConfig to specify subscription path, reporting-interval (freq), etc,. type PathsConfig struct { - Path string `json:"path"` - Freq uint64 `json:"freq"` - Mode string `json:"mode"` - Origin string `json:"origin"` - PreGnmi bool `json:"pre-gnmi"` - Gnmi bool `json:"gnmi"` + Path string `json:"path"` + Freq uint64 `json:"freq"` + Mode string `json:"mode"` + Origin string `json:"origin"` + Target string `json:"target"` + PreGnmi bool `json:"pre-gnmi"` + Gnmi bool `json:"gnmi"` + Gnmi_heartbeat_interval uint64 `json:"gnmi_heartbeat_interval"` } // NewJTIMONConfigFilelist to return configfilelist object diff --git a/subscribe_gnmi.go b/subscribe_gnmi.go index 3c5b144..a63d5c5 100644 --- a/subscribe_gnmi.go +++ b/subscribe_gnmi.go @@ -282,6 +282,26 @@ func gnmiParseNotification(parseOrigin bool, rsp *gnmi.SubscribeResponse, parseO parseOutput.inKvs += uint64(len(parseOutput.jXpaths.xPaths)) } + // Extract target and origin from prefix and add to kvpairs + prefix := notif.GetPrefix() + if prefix != nil { + target := prefix.GetTarget() + if target != "" { + if parseOutput.kvpairs == nil { + parseOutput.kvpairs = make(map[string]string) + } + parseOutput.kvpairs["target"] = target + } + + origin := prefix.GetOrigin() + if origin != "" { + if parseOutput.kvpairs == nil { + parseOutput.kvpairs = make(map[string]string) + } + parseOutput.kvpairs["origin"] = origin + } + } + parseOutput, err = gnmiParseHeader(rsp, parseOutput) if err != nil { errMsg = fmt.Sprintf("gnmiParseHeader failed: %v", err) @@ -505,6 +525,20 @@ func subscribegNMI(conn *grpc.ClientConn, jctx *JCtx, cfg Config, paths []PathsC return SubRcConnRetry } + // Set target in prefix if any path has a target configured + var target string + for _, path := range paths { + if path.Target != "" { + target = path.Target + break + } + } + if target != "" { + subs.Prefix = &gnmi.Path{ + Target: target, + } + } + // 2. Subscribe if jctx.config.User != "" && jctx.config.Password != "" { md := metadata.New(map[string]string{"username": jctx.config.User, "password": jctx.config.Password}) diff --git a/tests/data/cisco-ios-xr/config/xr-all-influx.log b/tests/data/cisco-ios-xr/config/xr-all-influx.log index 7bdc701..d187911 100644 --- a/tests/data/cisco-ios-xr/config/xr-all-influx.log +++ b/tests/data/cisco-ios-xr/config/xr-all-influx.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 10000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { diff --git a/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log b/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log index 889cca6..be17dcf 100644 --- a/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log +++ b/tests/data/cisco-ios-xr/config/xr-wdsysmon-influx.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 10000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { diff --git a/tests/data/juniper-junos/config/jtisim-influx-alias.log b/tests/data/juniper-junos/config/jtisim-influx-alias.log index 823883b..d5372e2 100644 --- a/tests/data/juniper-junos/config/jtisim-influx-alias.log +++ b/tests/data/juniper-junos/config/jtisim-influx-alias.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 10000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { @@ -92,22 +94,22 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Oct 17 12:07:46 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:17 IST 2025 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Thu Oct 17 12:07:48 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:19 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:07:50 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:21 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:07:52 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:23 IST 2025 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 8.003061376s) +Collector Stats for 127.0.0.1:50051 (Run time : 8.003715875s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-influx.log b/tests/data/juniper-junos/config/jtisim-influx.log index a5a88c4..5e21403 100644 --- a/tests/data/juniper-junos/config/jtisim-influx.log +++ b/tests/data/juniper-junos/config/jtisim-influx.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 10000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { @@ -93,50 +95,50 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Oct 17 12:07:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:52 IST 2025 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Thu Oct 17 12:07:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:54 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:07:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:56 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:07:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:58 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:07:29 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:00 IST 2025 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 -| Thu Oct 17 12:07:31 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:49:02 IST 2025 | 3960 | 80 | 174838 | 175238 | Batch write successful! Post batch write available points: 0 -| Thu Oct 17 12:07:33 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:49:04 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:07:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:49:06 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:07:37 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:49:08 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:07:39 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:49:10 IST 2025 | 4040 | 82 | 178196 | 178606 | Batch processing: #packets:40 #points:40 -| Thu Oct 17 12:07:41 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Mon Oct 13 11:49:12 IST 2025 | 5940 | 120 | 262258 | 262858 | Batch write successful! Post batch write available points: 0 -| Thu Oct 17 12:07:43 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Mon Oct 13 11:49:14 IST 2025 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.003715882s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.004904292s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-1.log b/tests/data/juniper-junos/config/jtisim-interfaces-1.log index 2a69395..c72b300 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-1.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-1.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 10000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { @@ -83,51 +85,51 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - content-type: [application/grpc] jtisim: [yes] + content-type: [application/grpc] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Oct 17 12:08:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:50 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:52 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:54 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:56 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:58 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:29 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:50:00 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:31 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:50:02 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:33 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:50:04 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:50:06 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:37 PDT 2024 | 4634 | 94 | 204422 | 204892 | +| Mon Oct 13 11:50:08 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:39 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Mon Oct 13 11:50:10 IST 2025 | 5940 | 120 | 262258 | 262858 | -| Thu Oct 17 12:08:41 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Mon Oct 13 11:50:12 IST 2025 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.00183539s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.005915709s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-2.log b/tests/data/juniper-junos/config/jtisim-interfaces-2.log index 9254ae1..f7c8c5b 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-2.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-2.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 10000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { @@ -83,51 +85,51 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - jtisim: [yes] content-type: [application/grpc] + jtisim: [yes] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Oct 17 12:08:19 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:50 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:21 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:52 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:23 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:54 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:25 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:56 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:27 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:49:58 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:08:29 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:50:00 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:31 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:50:02 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:33 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:50:04 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:35 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:50:06 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:37 PDT 2024 | 4554 | 92 | 201064 | 201524 | +| Mon Oct 13 11:50:08 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:08:39 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Mon Oct 13 11:50:10 IST 2025 | 5940 | 120 | 262258 | 262858 | -| Thu Oct 17 12:08:41 PDT 2024 | 5940 | 120 | 262258 | 262858 | +| Mon Oct 13 11:50:12 IST 2025 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.001281279s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.003235166s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-3.log b/tests/data/juniper-junos/config/jtisim-interfaces-3.log index 27b552b..82128b0 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-3.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-3.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 10000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { @@ -90,29 +92,29 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Oct 17 12:06:35 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:06 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:06:37 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:08 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:06:39 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:10 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:06:41 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:12 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:06:43 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:14 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:06:45 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:48:16 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Thu Oct 17 12:06:47 PDT 2024 | 3960 | 80 | 174838 | 175238 | +| Mon Oct 13 11:48:18 IST 2025 | 3960 | 80 | 174838 | 175238 | -Collector Stats for 127.0.0.1:50051 (Run time : 15.001529329s) +Collector Stats for 127.0.0.1:50051 (Run time : 15.001798542s) 80 : in-packets 3960 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-4.log b/tests/data/juniper-junos/config/jtisim-interfaces-4.log index 92205e6..2439860 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-4.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-4.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 10000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { @@ -87,7 +89,7 @@ compression = none Reconnecting to 127.0.0.1:90052 -Collector Stats for 127.0.0.1:90052 (Run time : 10.001517155s) +Collector Stats for 127.0.0.1:90052 (Run time : 10.003341292s) 0 : in-packets 0 : data points (KV pairs) 0 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-6.log b/tests/data/juniper-junos/config/jtisim-interfaces-6.log index 67f26a9..483b28d 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-6.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-6.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 20000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { @@ -90,11 +92,11 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Oct 17 12:06:31 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:02 IST 2025 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 3.998590619s) +Collector Stats for 127.0.0.1:50051 (Run time : 3.997328917s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-prometheus.log b/tests/data/juniper-junos/config/jtisim-prometheus.log index 1a63724..c7dfd45 100644 --- a/tests/data/juniper-junos/config/jtisim-prometheus.log +++ b/tests/data/juniper-junos/config/jtisim-prometheus.log @@ -57,8 +57,10 @@ Running config of JTIMON: "freq": 20000, "mode": "", "origin": "", + "target": "", "pre-gnmi": false, - "gnmi": false + "gnmi": false, + "gnmi_heartbeat_interval": 0 } ], "log": { @@ -90,14 +92,17 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Thu Oct 17 12:07:15 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:46 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Thu Oct 17 12:07:17 PDT 2024 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:48 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Mon Oct 13 11:48:50 IST 2025 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 6.0011324s) + + +Collector Stats for 127.0.0.1:50051 (Run time : 6.007747792s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) From fded1677120a925d9ed28b4dac46bd37d3187baf Mon Sep 17 00:00:00 2001 From: amohit Date: Wed, 22 Oct 2025 21:16:49 +0530 Subject: [PATCH 19/24] fix heartbeat interval --- gnmi_utils.go | 5 +++++ subscribe_gnmi.go | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/gnmi_utils.go b/gnmi_utils.go index ad32e6f..64d2226 100644 --- a/gnmi_utils.go +++ b/gnmi_utils.go @@ -539,3 +539,8 @@ func formJuniperTelemetryHdr(jXpaths *jnprXpathDetails, gnmiExt []*gnmi_ext1.Ext return &juniperHdrDetails, true, nil } + +func gnmiHeartBeat(hb uint64) uint64 { + hb_val := (hb * gGnmiFreqUnits) / 10000 + return hb_val +} diff --git a/subscribe_gnmi.go b/subscribe_gnmi.go index a63d5c5..e5042e2 100644 --- a/subscribe_gnmi.go +++ b/subscribe_gnmi.go @@ -465,7 +465,8 @@ func xPathsTognmiSubscription(pathsCfg []PathsConfig, dialOutpathsCfg []*dialout mode, freq := gnmiFreq(mode, p.Freq) gp.Origin = p.Origin - subs = append(subs, &gnmi.Subscription{Path: gp, Mode: mode, SampleInterval: freq}) + gnmiHb := gnmiHeartBeat(p.Gnmi_heartbeat_interval) + subs = append(subs, &gnmi.Subscription{Path: gp, Mode: mode, SampleInterval: freq, HeartbeatInterval: gnmiHb}) } } else { for _, p := range dialOutpathsCfg { From 60840f54677bf9077bd9a57e30868c794c8be3b1 Mon Sep 17 00:00:00 2001 From: amohit Date: Fri, 24 Oct 2025 13:15:22 +0530 Subject: [PATCH 20/24] support once subscription --- config.go | 1 + subscribe_gnmi.go | 4 +-- .../config/jtisim-influx-alias.log | 10 +++---- .../juniper-junos/config/jtisim-influx.log | 28 +++++++++---------- .../config/jtisim-interfaces-1.log | 26 ++++++++--------- .../config/jtisim-interfaces-2.log | 26 ++++++++--------- .../config/jtisim-interfaces-3.log | 18 ++++++------ .../config/jtisim-interfaces-4.log | 2 +- .../config/jtisim-interfaces-6.log | 4 +-- .../config/jtisim-prometheus.log | 11 +++----- 10 files changed, 63 insertions(+), 67 deletions(-) diff --git a/config.go b/config.go index 6ba2e7b..42e5a2b 100644 --- a/config.go +++ b/config.go @@ -42,6 +42,7 @@ type Config struct { // GnmiConfig definition type GnmiConfig struct { Encoding string + Mode int16 } // VendorConfig definition diff --git a/subscribe_gnmi.go b/subscribe_gnmi.go index e5042e2..d9cebdf 100644 --- a/subscribe_gnmi.go +++ b/subscribe_gnmi.go @@ -502,9 +502,7 @@ func subscribegNMI(conn *grpc.ClientConn, jctx *JCtx, cfg Config, paths []PathsC ) // 1. Form request - - // Support only STREAM - subs.Mode = gnmi.SubscriptionList_STREAM + subs.Mode = gnmi.SubscriptionList_Mode(cfg.Vendor.Gnmi.Mode) // PROTO encoding if jctx.config.Vendor.Gnmi != nil { diff --git a/tests/data/juniper-junos/config/jtisim-influx-alias.log b/tests/data/juniper-junos/config/jtisim-influx-alias.log index d5372e2..3718770 100644 --- a/tests/data/juniper-junos/config/jtisim-influx-alias.log +++ b/tests/data/juniper-junos/config/jtisim-influx-alias.log @@ -94,22 +94,22 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Mon Oct 13 11:49:17 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:33 IST 2025 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Mon Oct 13 11:49:19 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:35 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:21 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:37 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:23 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:39 IST 2025 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 8.003715875s) +Collector Stats for 127.0.0.1:50051 (Run time : 8.004479666s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-influx.log b/tests/data/juniper-junos/config/jtisim-influx.log index 5e21403..74eafac 100644 --- a/tests/data/juniper-junos/config/jtisim-influx.log +++ b/tests/data/juniper-junos/config/jtisim-influx.log @@ -91,54 +91,54 @@ gRPC headers from host 127.0.0.1:50051 content-type: [application/grpc] jtisim: [yes] Receiving telemetry data from 127.0.0.1:50051 +Batch processing: #packets:40 #points:40 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Mon Oct 13 11:48:52 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:08 IST 2025 | 1980 | 40 | 87418 | 87618 | -Batch processing: #packets:40 #points:40 Batch write successful! Post batch write available points: 0 -| Mon Oct 13 11:48:54 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:10 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:48:56 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:12 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:48:58 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:14 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:00 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:16 IST 2025 | 1980 | 40 | 87418 | 87618 | Batch processing: #packets:40 #points:40 -| Mon Oct 13 11:49:02 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:09:18 IST 2025 | 3960 | 80 | 174838 | 175238 | Batch write successful! Post batch write available points: 0 -| Mon Oct 13 11:49:04 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:09:20 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:49:06 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:09:22 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:49:08 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:09:24 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:49:10 IST 2025 | 4040 | 82 | 178196 | 178606 | +| Fri Oct 24 13:09:26 IST 2025 | 5940 | 120 | 262258 | 262858 | Batch processing: #packets:40 #points:40 -| Mon Oct 13 11:49:12 IST 2025 | 5940 | 120 | 262258 | 262858 | +| Fri Oct 24 13:09:28 IST 2025 | 5940 | 120 | 262258 | 262858 | Batch write successful! Post batch write available points: 0 -| Mon Oct 13 11:49:14 IST 2025 | 5940 | 120 | 262258 | 262858 | +| Fri Oct 24 13:09:30 IST 2025 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.004904292s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.002738375s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-1.log b/tests/data/juniper-junos/config/jtisim-interfaces-1.log index c72b300..4c46cf2 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-1.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-1.log @@ -92,44 +92,44 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Mon Oct 13 11:49:50 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:06 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:52 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:08 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:54 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:10 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:56 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:12 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:58 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:14 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:50:00 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:16 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:50:02 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:18 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:50:04 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:20 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:50:06 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:22 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:50:08 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:24 IST 2025 | 4158 | 84 | 183580 | 184000 | -| Mon Oct 13 11:50:10 IST 2025 | 5940 | 120 | 262258 | 262858 | +| Fri Oct 24 13:10:26 IST 2025 | 5940 | 120 | 262258 | 262858 | -| Mon Oct 13 11:50:12 IST 2025 | 5940 | 120 | 262258 | 262858 | +| Fri Oct 24 13:10:28 IST 2025 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.005915709s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.002623584s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-2.log b/tests/data/juniper-junos/config/jtisim-interfaces-2.log index f7c8c5b..236cf4f 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-2.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-2.log @@ -92,44 +92,44 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Mon Oct 13 11:49:50 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:06 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:52 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:08 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:54 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:10 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:56 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:12 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:49:58 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:10:14 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:50:00 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:16 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:50:02 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:18 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:50:04 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:20 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:50:06 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:22 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:50:08 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:10:24 IST 2025 | 4238 | 86 | 186938 | 187368 | -| Mon Oct 13 11:50:10 IST 2025 | 5940 | 120 | 262258 | 262858 | +| Fri Oct 24 13:10:26 IST 2025 | 5940 | 120 | 262258 | 262858 | -| Mon Oct 13 11:50:12 IST 2025 | 5940 | 120 | 262258 | 262858 | +| Fri Oct 24 13:10:28 IST 2025 | 5940 | 120 | 262258 | 262858 | -Collector Stats for 127.0.0.1:50051 (Run time : 25.003235166s) +Collector Stats for 127.0.0.1:50051 (Run time : 25.001174625s) 120 : in-packets 5940 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-3.log b/tests/data/juniper-junos/config/jtisim-interfaces-3.log index 82128b0..f2a9674 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-3.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-3.log @@ -85,36 +85,36 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - content-type: [application/grpc] jtisim: [yes] + content-type: [application/grpc] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Mon Oct 13 11:48:06 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:08:22 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:48:08 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:08:24 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:48:10 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:08:26 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:48:12 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:08:28 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:48:14 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:08:30 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:48:16 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:08:32 IST 2025 | 3960 | 80 | 174838 | 175238 | -| Mon Oct 13 11:48:18 IST 2025 | 3960 | 80 | 174838 | 175238 | +| Fri Oct 24 13:08:34 IST 2025 | 3960 | 80 | 174838 | 175238 | -Collector Stats for 127.0.0.1:50051 (Run time : 15.001798542s) +Collector Stats for 127.0.0.1:50051 (Run time : 15.001638458s) 80 : in-packets 3960 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-4.log b/tests/data/juniper-junos/config/jtisim-interfaces-4.log index 2439860..a416b80 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-4.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-4.log @@ -89,7 +89,7 @@ compression = none Reconnecting to 127.0.0.1:90052 -Collector Stats for 127.0.0.1:90052 (Run time : 10.003341292s) +Collector Stats for 127.0.0.1:90052 (Run time : 10.001932916s) 0 : in-packets 0 : data points (KV pairs) 0 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-interfaces-6.log b/tests/data/juniper-junos/config/jtisim-interfaces-6.log index 483b28d..79387cf 100644 --- a/tests/data/juniper-junos/config/jtisim-interfaces-6.log +++ b/tests/data/juniper-junos/config/jtisim-interfaces-6.log @@ -92,11 +92,11 @@ Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Mon Oct 13 11:48:02 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:08:18 IST 2025 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 3.997328917s) +Collector Stats for 127.0.0.1:50051 (Run time : 3.992528209s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) diff --git a/tests/data/juniper-junos/config/jtisim-prometheus.log b/tests/data/juniper-junos/config/jtisim-prometheus.log index c7dfd45..0df1e6e 100644 --- a/tests/data/juniper-junos/config/jtisim-prometheus.log +++ b/tests/data/juniper-junos/config/jtisim-prometheus.log @@ -85,24 +85,21 @@ New alias creation failed for 127.0.0.1, err: open : no such file or directory compression = none Connecting to 127.0.0.1:50051 gRPC headers from host 127.0.0.1:50051 - jtisim: [yes] content-type: [application/grpc] + jtisim: [yes] Receiving telemetry data from 127.0.0.1:50051 +------------------------------+--------------------+--------------------+--------------------+--------------------+ | Timestamp | KV | Packets | Bytes | Bytes(wire) | +------------------------------+--------------------+--------------------+--------------------+--------------------+ -| Mon Oct 13 11:48:46 IST 2025 | 1980 | 40 | 87418 | 87618 | - - -| Mon Oct 13 11:48:48 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:02 IST 2025 | 1980 | 40 | 87418 | 87618 | -| Mon Oct 13 11:48:50 IST 2025 | 1980 | 40 | 87418 | 87618 | +| Fri Oct 24 13:09:04 IST 2025 | 1980 | 40 | 87418 | 87618 | -Collector Stats for 127.0.0.1:50051 (Run time : 6.007747792s) +Collector Stats for 127.0.0.1:50051 (Run time : 6.001081666s) 40 : in-packets 1980 : data points (KV pairs) 25 : in-header wirelength (bytes) From e9608cb9124825c1af4d5b4295b8727d28bebc53 Mon Sep 17 00:00:00 2001 From: amohit Date: Wed, 14 Jan 2026 09:50:52 +0530 Subject: [PATCH 21/24] added formatting output --- internal_jtimon.go | 63 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/internal_jtimon.go b/internal_jtimon.go index b1041e5..b21da16 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -186,8 +186,33 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g prefixPath = getPath(prefixPath, prefix.GetElem()) } - s += fmt.Sprintf( - "Update {\n\ttimestamp: %d\n\tprefix: %v\n", notif.GetTimestamp(), prefixPath) + // Format prefix in structured protobuf style + s += fmt.Sprintf("Update {\n\ttimestamp: %d\n", notif.GetTimestamp()) + + if prefix != nil { + s += "\tprefix {\n" + + // Add origin if present + if prefix.GetOrigin() != "" { + s += fmt.Sprintf("\t\torigin: %s\n", prefix.GetOrigin()) + } + + // Add path elements + for _, elem := range prefix.GetElem() { + s += fmt.Sprintf("\t\telem {\n\t\t\tname: %s\n", elem.GetName()) + + // Add keys if present + if len(elem.GetKey()) > 0 { + for k, v := range elem.GetKey() { + s += fmt.Sprintf("\t\t\tkey {\n\t\t\t\tkey: %s\n\t\t\t\tvalue: %s\n\t\t\t}\n", k, v) + } + } + + s += "\t\t}\n" + } + + s += "\t}\n" + } // Create a map to hold notification data for JSON output notifData := make(map[string]interface{}) @@ -206,7 +231,7 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g // Parse updates (both for string and JSON) var updates []map[string]interface{} for _, u := range notif.Update { - s += fmt.Sprintf("Update {\n\tpath {\n") + s += "Update {\n\tpath {\n" update := make(map[string]interface{}) @@ -225,15 +250,39 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g result := re.FindStringSubmatch(notifString) if len(result) > 1 { keyVal := strings.SplitN(result[1], ":", 2) - s += fmt.Sprintf("\t\tval {\n\t\t\t%s: %s\n\t\t}\n", keyVal[0], keyVal[1]) - update["key"] = keyVal[0] + fieldName := keyVal[0] + + // Convert full protobuf field names to shortened format for output + fieldNameMap := map[string]string{ + "string_val": "str_val", + "int_val": "int_val", + "uint_val": "uint_val", + "bool_val": "bool_val", + "bytes_val": "bytes_val", + "float_val": "float_val", + "double_val": "double_val", + "decimal_val": "decimal_val", + "leaflist_val": "leaflist_val", + "any_val": "any_val", + "json_val": "json_val", + "json_ietf_val": "json_ietf_val", + "ascii_val": "ascii_val", + "proto_bytes": "proto_bytes", + } + + if shortName, ok := fieldNameMap[fieldName]; ok { + fieldName = shortName + } + + s += fmt.Sprintf("\t\tval {\n\t\t\t%s: %s\n\t\t}\n", fieldName, keyVal[1]) + update["key"] = fieldName update["value"] = strings.Trim(keyVal[1], "\"") } updates = append(updates, update) - s += fmt.Sprintf("\t}\n") + s += "\t}\n" } - s += fmt.Sprintf("}\n") + s += "}\n" notifData["updates"] = updates // Add update data to JSON output outputData["notification"] = notifData // Add notification to JSON output From 203c84bb7e189bc29dd5206a416d6b58ec3ff97b Mon Sep 17 00:00:00 2001 From: amohit Date: Wed, 14 Jan 2026 10:53:18 +0530 Subject: [PATCH 22/24] testing --- subscribe_gnmi.go | 8 +++++++- subscribe_juniper_junos.go | 1 + workers.go | 13 +++++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/subscribe_gnmi.go b/subscribe_gnmi.go index d9cebdf..b9a6acf 100644 --- a/subscribe_gnmi.go +++ b/subscribe_gnmi.go @@ -503,7 +503,7 @@ func subscribegNMI(conn *grpc.ClientConn, jctx *JCtx, cfg Config, paths []PathsC // 1. Form request subs.Mode = gnmi.SubscriptionList_Mode(cfg.Vendor.Gnmi.Mode) - + jLog(jctx, fmt.Sprintf("test: gNMI host: %v, Subscription mode: %v", hostname, subs.Mode.String())) // PROTO encoding if jctx.config.Vendor.Gnmi != nil { switch jctx.config.Vendor.Gnmi.Encoding { @@ -571,6 +571,12 @@ func subscribegNMI(conn *grpc.ClientConn, jctx *JCtx, cfg Config, paths []PathsC rsp, err1 = gNMISubHandle.Recv() if err1 == io.EOF { printSummary(jctx) + // For ONCE mode, EOF after sync response indicates successful completion + if subs.Mode == gnmi.SubscriptionList_ONCE && jctx.receivedSyncRsp { + jLog(jctx, fmt.Sprintf("gNMI host: %v, ONCE subscription completed", hostname)) + datach <- SubRcOnceComplete + return + } jLog(jctx, fmt.Sprintf("gNMI host: %v, received eof", hostname)) datach <- SubRcConnRetry return diff --git a/subscribe_juniper_junos.go b/subscribe_juniper_junos.go index e20969d..eb078e7 100644 --- a/subscribe_juniper_junos.go +++ b/subscribe_juniper_junos.go @@ -27,6 +27,7 @@ const ( SubRcSighupRestart SubRcSighupNoRestart SubRcRPCFailedNoRetry + SubRcOnceComplete ) func handleOnePacket(ocData *na_pb.OpenConfigData, jctx *JCtx) { diff --git a/workers.go b/workers.go index cc2966a..c008e9e 100644 --- a/workers.go +++ b/workers.go @@ -601,7 +601,7 @@ func workTunnel(jctx *JCtx, statusch chan struct{}) error { jLog(jctx, fmt.Sprintf("Calling subscribe() ::: %s\n", jctx.file)) subscribeConfig := jctx.config code := vendor.subscribe(conn, jctx, subscribeConfig, subscribeConfig.Paths) - jLog(jctx, fmt.Sprintf("Returns subscribe() ::: %s CODE ::: %d\n", jctx.file, code)) + jLog(jctx, fmt.Sprintf("First Returns subscribe() ::: %s CODE ::: %d\n", jctx.file, code)) // close the current connection and retry conn.Close() @@ -637,6 +637,11 @@ func workTunnel(jctx *JCtx, statusch chan struct{}) error { statusch <- struct{}{} errCh <- err return + case SubRcOnceComplete: + jLog(jctx, fmt.Sprintf("ONCE subscription completed successfully for worker %s", jctx.file)) + statusch <- struct{}{} + errCh <- err + return } }() @@ -765,7 +770,7 @@ connect: fmt.Println("Calling subscribe() :::", jctx.file) subscribeConfig := jctx.config code := vendor.subscribe(conn, jctx, subscribeConfig, subscribeConfig.Paths) - fmt.Println("Returns subscribe() :::", jctx.file, "CODE ::: ", code) + fmt.Println("Second Returns subscribe() :::", jctx.file, "CODE ::: ", code) // close the current connection and retry conn.Close() @@ -801,5 +806,9 @@ connect: jLog(jctx, fmt.Sprintf("not reconnecting for worker %s", jctx.file)) statusch <- struct{}{} return + case SubRcOnceComplete: + jLog(jctx, fmt.Sprintf("ONCE subscription completed successfully for worker %s", jctx.file)) + statusch <- struct{}{} + return } } From 04b0711d565caa1a6945e8793728caea35f36bf4 Mon Sep 17 00:00:00 2001 From: amohit Date: Wed, 14 Jan 2026 11:16:26 +0530 Subject: [PATCH 23/24] testing new --- internal_jtimon.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal_jtimon.go b/internal_jtimon.go index b21da16..7857d42 100644 --- a/internal_jtimon.go +++ b/internal_jtimon.go @@ -251,10 +251,10 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g if len(result) > 1 { keyVal := strings.SplitN(result[1], ":", 2) fieldName := keyVal[0] - + // Convert full protobuf field names to shortened format for output fieldNameMap := map[string]string{ - "string_val": "str_val", + "string_val": "string_val", "int_val": "int_val", "uint_val": "uint_val", "bool_val": "bool_val", @@ -269,11 +269,11 @@ func jLogInternalJtimonForGnmi(jctx *JCtx, parseOutput *gnmiParseOutputT, rsp *g "ascii_val": "ascii_val", "proto_bytes": "proto_bytes", } - + if shortName, ok := fieldNameMap[fieldName]; ok { fieldName = shortName } - + s += fmt.Sprintf("\t\tval {\n\t\t\t%s: %s\n\t\t}\n", fieldName, keyVal[1]) update["key"] = fieldName update["value"] = strings.Trim(keyVal[1], "\"") From 648d78f1de31bd796780319c9763b0f22a5f151c Mon Sep 17 00:00:00 2001 From: amohit Date: Wed, 14 Jan 2026 11:51:23 +0530 Subject: [PATCH 24/24] Added formatting and once subscription support --- subscribe_gnmi.go | 1 - workers.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/subscribe_gnmi.go b/subscribe_gnmi.go index b9a6acf..7a752cc 100644 --- a/subscribe_gnmi.go +++ b/subscribe_gnmi.go @@ -503,7 +503,6 @@ func subscribegNMI(conn *grpc.ClientConn, jctx *JCtx, cfg Config, paths []PathsC // 1. Form request subs.Mode = gnmi.SubscriptionList_Mode(cfg.Vendor.Gnmi.Mode) - jLog(jctx, fmt.Sprintf("test: gNMI host: %v, Subscription mode: %v", hostname, subs.Mode.String())) // PROTO encoding if jctx.config.Vendor.Gnmi != nil { switch jctx.config.Vendor.Gnmi.Encoding { diff --git a/workers.go b/workers.go index c008e9e..301b0d8 100644 --- a/workers.go +++ b/workers.go @@ -601,7 +601,7 @@ func workTunnel(jctx *JCtx, statusch chan struct{}) error { jLog(jctx, fmt.Sprintf("Calling subscribe() ::: %s\n", jctx.file)) subscribeConfig := jctx.config code := vendor.subscribe(conn, jctx, subscribeConfig, subscribeConfig.Paths) - jLog(jctx, fmt.Sprintf("First Returns subscribe() ::: %s CODE ::: %d\n", jctx.file, code)) + jLog(jctx, fmt.Sprintf("Returns subscribe() ::: %s CODE ::: %d\n", jctx.file, code)) // close the current connection and retry conn.Close() @@ -770,7 +770,7 @@ connect: fmt.Println("Calling subscribe() :::", jctx.file) subscribeConfig := jctx.config code := vendor.subscribe(conn, jctx, subscribeConfig, subscribeConfig.Paths) - fmt.Println("Second Returns subscribe() :::", jctx.file, "CODE ::: ", code) + fmt.Println("Returns subscribe() :::", jctx.file, "CODE ::: ", code) // close the current connection and retry conn.Close()