Support Questions
Find answers, ask questions, and share your expertise
Announcements
Alert: Welcome to the Unified Cloudera Community. Former HCC members be sure to read and learn how to activate your account here.

increasing streaming JSON file with multiple path and objects to CSV in NiFi

increasing streaming JSON file with multiple path and objects to CSV in NiFi

New Contributor

Hi I have network telemetry data collected by Pipeline SW and dumped into a streaming JSON file, this file is continuously increasing with new JSON path and object.

I have created the flow to convert one JSON path to csv , but i am not able to find way to process the file which has multiple paths and new paths getting upended. When i try tp read using getfile/fetchfile it says not a valide JSON

input file looks like below, json path/object (Summary) keep getting upended in the file.

------- 2018-01-30 09:51:25.840227915 +0000 GMT -------
Summary: GPB(common) Message [10.91.143.118:20673(ar8.BLB)/Cisco-IOS-XR-infra-statsd-oper:infra-statistics/interfaces/interface/latest/generic-counters msg len: 24692]
{
   "Source": "10.91.143.118:54839",
    "Telemetry": {
        "node_id_str": "ar8.BLB",
        "subscription_id_str": "Sub1",
        "encoding_path": "Cisco-IOS-XR-infra-statsd-oper:infra-statistics/interfaces/interface/latest/generic-counters",
        "collection_id": 5179,
        "collection_start_time": 1507099332858,
        "msg_timestamp": 1507099332858,
        "collection_end_time": 0
    },
    "Rows": [
        {
            "Timestamp": 1507099332865,
            "Keys": {
                "interface-name": "Bundle-Ether16"
            },
            "Content": {
                "applique": 0,
                "availability-flag": 0,
                "broadcast-packets-received": 31,
                "broadcast-packets-sent": 8184,
                "bytes-received": 138560885,
                "bytes-sent": 2600833998,
                "carrier-transitions": 0,
                "crc-errors": 0,
                "framing-errors-received": 0,
                "giant-packets-received": 0,
                "input-aborts": 0,
                "input-drops": 0,
                "input-errors": 0,
                "input-ignored-packets": 0,
                "input-overruns": 0,
                "input-queue-drops": 0,
                "last-data-time": 1507099282,
                "last-discontinuity-time": 1501238152,
                "multicast-packets-received": 750969,
                "multicast-packets-sent": 14001536,
                "output-buffer-failures": 0,
                "output-buffers-swapped-out": 0,
                "output-drops": 0,
                "output-errors": 0,
                "output-queue-drops": 0,
                "output-underruns": 0,
                "packets-received": 805430,
                "packets-sent": 14059281,
                "parity-packets-received": 0,
                "resets": 0,
                "runt-packets-received": 0,
                "seconds-since-last-clear-counters": 0,
                "seconds-since-packet-received": 0,
                "seconds-since-packet-sent": 0,
                "throttled-packets-received": 0,
                "unknown-protocol-packets-received": 0
            }
        },
        {
            "Timestamp": 1507099332865,
            "Keys": {
                "interface-name": "Bundle-Ether60"
            },
            "Content": {
                "applique": 0,
                "availability-flag": 0,
                "broadcast-packets-received": 105131,
                "broadcast-packets-sent": 8184,
                "bytes-received": 209346805,
                "bytes-sent": 2609502395,
                "carrier-transitions": 0,
                "crc-errors": 0,
                "framing-errors-received": 0,
                "giant-packets-received": 0,
                "input-aborts": 0,
                "input-drops": 0,
                "input-errors": 1,
                "input-ignored-packets": 0,
                "input-overruns": 0,
                "input-queue-drops": 0,
                "last-data-time": 1507099282,
                "last-discontinuity-time": 1501238152,
                "multicast-packets-received": 750742,
                "multicast-packets-sent": 14000436,
                "output-buffer-failures": 0,
                "output-buffers-swapped-out": 0,
                "output-drops": 0,
                "output-errors": 0,
                "output-queue-drops": 0,
                "output-underruns": 0,
                "packets-received": 1445210,
                "packets-sent": 14192537,
                "parity-packets-received": 0,
                "resets": 0,
                "runt-packets-received": 0,
                "seconds-since-last-clear-counters": 0,
                "seconds-since-packet-received": 0,
                "seconds-since-packet-sent": 0,
                "throttled-packets-received": 0,
                "unknown-protocol-packets-received": 0
            }
        }
    ]
}
------- 2018-01-30 09:51:57.392014438 +0000 GMT -------
Summary: GPB(common) Message [10.91.143.118:20673(ar8.BLB)/Cisco-IOS-XR-infra-statsd-oper:infra-statistics/interfaces/interface/latest/generic-counters msg len: 65686]
{
    "Source": "10.91.143.118:20673",
    "Telemetry": {
        "node_id_str": "ar8.BLB",
        "subscription_id_str": "Sub1",
        "encoding_path": "Cisco-IOS-XR-infra-statsd-oper:infra-statistics/interfaces/interface/latest/generic-counters",
        "collection_id": 197015,
        "collection_start_time": 1517305306246,
        "msg_timestamp": 1517305306246,
        "collection_end_time": 0
    },
        {
            "Timestamp": 1517305306253,
            "Keys": {
                "interface-name": "Bundle-Ether1"
            },
            "Content": {
                "applique": 0,
                "availability-flag": 0,
                "broadcast-packets-received": 5175,
                "broadcast-packets-sent": 1233250,
                "bytes-received": 48793498884,
                "bytes-sent": 48920794690,
                "carrier-transitions": 0,
                "crc-errors": 0,
                "framing-errors-received": 0,
                "giant-packets-received": 0,
                "input-aborts": 0,
                "input-drops": 0,
                "input-errors": 0,
                "input-ignored-packets": 0,
                "input-overruns": 0,
                "input-queue-drops": 0,
                "last-data-time": 1517305306,
                "last-discontinuity-time": 1515563824,
                "multicast-packets-received": 5243922,
                "multicast-packets-sent": 5696926,
                "output-buffer-failures": 0,
                "output-buffers-swapped-out": 0,
                "output-drops": 0,
                "output-errors": 0,
                "output-queue-drops": 0,
                "output-underruns": 0,
                "packets-received": 33428894,
                "packets-sent": 35097536,
                "parity-packets-received": 0,
                "resets": 0,
                "runt-packets-received": 0,
                "seconds-since-last-clear-counters": 1723718,
                "seconds-since-packet-received": 0,
                "seconds-since-packet-sent": 0,
                "throttled-packets-received": 0,
                "unknown-protocol-packets-received": 0
            }
        },
        {
            "Timestamp": 1517305306253,
            "Keys": {
                "interface-name": "MgmtEth0/RP0/CPU0/0"
            },
            "Content": {
                "applique": 0,
                "availability-flag": 0,
                "broadcast-packets-received": 8544181,
                "broadcast-packets-sent": 0,
                "bytes-received": 2962121926,
                "bytes-sent": 9925254981,
                "carrier-transitions": 2,
                "crc-errors": 0,
                "framing-errors-received": 0,
                "giant-packets-received": 0,
                "input-aborts": 0,
                "input-drops": 0,
                "input-errors": 0,
                "input-ignored-packets": 0,
                "input-overruns": 0,
                "input-queue-drops": 0,
                "last-data-time": 1517305306,
                "last-discontinuity-time": 1513592240,
                "multicast-packets-received": 1531861,
                "multicast-packets-sent": 0,
                "output-buffer-failures": 0,
                "output-buffers-swapped-out": 0,
                "output-drops": 0,
                "output-errors": 0,
                "output-queue-drops": 0,
                "output-underruns": 0,
                "packets-received": 13446969,
                "packets-sent": 6861876,
                "parity-packets-received": 0,
                "resets": 0,
                "runt-packets-received": 0,
                "seconds-since-last-clear-counters": 1723718,
                "seconds-since-packet-received": 0,
                "seconds-since-packet-sent": 0,
                "throttled-packets-received": 0,
                "unknown-protocol-packets-received": 0
            }
        }
    ]
}
 

How to process this file?

I have created the following flow to process one json path to csv.

WOrking flow for Input

{
   "Source": "10.91.143.118:54839",
    "Telemetry": {
        "node_id_str": "ar8.BLB",
        "subscription_id_str": "Sub1",
        "encoding_path": "Cisco-IOS-XR-infra-statsd-oper:infra-statistics/interfaces/interface/latest/generic-counters",
        "collection_id": 5179,
        "collection_start_time": 1507099332858,
        "msg_timestamp": 1507099332858,
        "collection_end_time": 0
    },
    "Rows": [
        {
            "Timestamp": 1507099332865,
            "Keys": {
                "interface-name": "Null0"
            },
            "Content": {
                "applique": 0,
                "availability-flag": 0,
                "broadcast-packets-received": 0,
                "broadcast-packets-sent": 0,
                "bytes-received": 0,
                "bytes-sent": 0,
                "carrier-transitions": 0,
                "crc-errors": 0,
                "framing-errors-received": 0,
                "giant-packets-received": 0,
                "input-aborts": 0,
                "input-drops": 0,
                "input-errors": 0,
                "input-ignored-packets": 0,
                "input-overruns": 0,
                "input-queue-drops": 0,
                "last-data-time": 1507099332,
                "last-discontinuity-time": 1501238142,
                "multicast-packets-received": 0,
                "multicast-packets-sent": 0,
                "output-buffer-failures": 0,
                "output-buffers-swapped-out": 0,
                "output-drops": 0,
                "output-errors": 0,
                "output-queue-drops": 0,
                "output-underruns": 0,
                "packets-received": 0,
                "packets-sent": 0,
                "parity-packets-received": 0,
                "resets": 0,
                "runt-packets-received": 0,
                "seconds-since-last-clear-counters": 0,
                "seconds-since-packet-received": 4294967295,
                "seconds-since-packet-sent": 4294967295,
                "throttled-packets-received": 0,
                "unknown-protocol-packets-received": 0
            }
        }
    ]
}


62867-n1-6.jpg

Don't have an account?
Coming from Hortonworks? Activate your account here