Logstash嵌套数组JSON解析

时间:2017-09-20 05:40:54

标签: json elasticsearch logstash kibana logstash-configuration

我的应用程序的JSON日志包含嵌套数组形式的元素,以下是它的示例: -

{
    "msgs": [{
        "ts": "2017-09-04T07:07:45.6229372Z",
        "tid": 25,
        "eid": 1,
        "lvl": "Information",
        "cat": "Microsoft.AspNetCore.Hosting.Internal.WebHost",
        "msg": {
            "cnt": "Request starting HTTP/1.1 POST http://localhost:20001/Processor text/xml; charset=utf-8 685",
            "Protocol": "HTTP/1.1",
            "Method": "POST",
            "ContentType": "text/xml; charset=utf-8",
            "ContentLength": 685,
            "Scheme": "http",
            "Host": "localhost:20001",
            "PathBase": "",
            "Path": "/Processor",
            "QueryString": ""
        }
    },
    {
        "ts": "2017-09-04T07:07:45.6229372Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.Service.ServiceHostMiddleware",
        "msg": {
            "cnt": "REQ"
        },
        "data": {
            "Headers": {
                "Connection": "Keep-Alive",
                "Content-Length": "685",
                "Content-Type": "text/xml; charset=utf-8",
                "Accept-Encoding": "gzip, deflate",
                "Expect": "100-continue",
                "Host": "localhost:20001",
                "SOAPAction": "\"http://servereps.mtxeps.com/TransactionService/SendTransaction\""
            }
        }
    },
    {
        "ts": "2017-09-04T07:07:45.6239372Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.GatewayService.ProcessorTransactionService",
        "msg": {
            "cnt": "REQ"
        },
        "data": {
            "Trace": "Aa10031<1C>Ab1<1C>Ac000101<1C>Ad20170616145857<1C>Ae10011<1C>AhVT00009<1C>ArY<1C>Be1812<1C>BfS<1C>BnDB<1C>Bo400296<1C>Bp4803<1C>BqDebit Card<1C>Br16<1C>Da300<1C>Dc0<1C>Dk840<1C>Ga01<1C>Gb1<1C>GfG<1C>GhB/GsF>O<1C>GkOE1<1C>Ia7325980B6B284759<1C>Ib8765432100324A00313<1C>Ic0043<1C>Ig3|1|MX2015-06<1C>IjgnkA7MdNlE2EB1c2B3jlz1G3Kf2U5S3x2H9W6ldIY7QpmlUzYcNHGg==|FFFF987654323D2007CD<1C>Nc5<1C>Oa18"
        }
    },
    {
        "ts": "2017-09-04T07:07:45.6249373Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.GatewayService.TransactionProcessingEngine",
        "msg": {
            "cnt": "REQ"
        },
        "data": {
            "request": {
                "Version": "1.0",
                "Server": {
                    "LogicalDatacenterId": 0,
                    "PhysicalDatacenterId": 0,
                    "UniqueId": "G00008D4F35EDADCD368",
                    "UniversalTimestamp": "2017-09-04T07:07:45.6249373Z"
                },
                "State": {
                    "Status": "InFlight"
                },
                "Tenant": {
                    "CompanyNumber": 10031,
                    "StoreNumber": 1,
                    "HostType": {
                        "Code": 7,
                        "Name": "ProdConcordHC"
                    },
                    "MerchantNumber": "",
                    "TerminalId": ""
                },
                "PointOfInteraction": {
                    "Client": {
                        "Type": "OpenEPS",
                        "CardType": "DB",
                        "CardName": "Debit Card",
                        "CommandSequence": "B/GsF>O"
                    },
                    "Lane": {
                        "Type": "Attended",
                        "Number": 1
                    },
                    "Terminal": {
                        "EMVCapabilities": "FullEMV",
                        "SerialNumber": "0043"
                    },
                    "PointOfSale": {
                        "ReferenceNumber": "VT00009",
                        "CashierNumber": "1"
                    },
                    "LocalTimestamp": "2017-06-16T14:58:57"
                },
                "Request": {
                    "MessageType": {
                        "Code": 0,
                        "Name": "Standard"
                    },
                    "TenderType": {
                        "Code": 1,
                        "Name": "Debit"
                    },
                    "TransactionType": {
                        "Code": 1,
                        "Name": "Purchase"
                    },
                    "EntryMode": "Swiped",
                    "AuditId": 10011,
                    "CardType": {
                        "TenderType": {
                            "Code": 0,
                            "Name": "Unknown"
                        },
                        "Code": 0,
                        "Name": "Unknown"
                    },
                    "AccountNumberFirstSix": "400296",
                    "AccountNumberLastFour": "4803",
                    "AccountNumberLength": 16,
                    "Card": {
                        "EncryptedTrack": {
                            "EncryptionType": "Hardware",
                            "EncryptionKeySerialNumber": "FFFF987654323D2007CD",
                            "EncryptedValue": "gnkA7MdNlE2EB1c2B3jlz1G3Kf2U5S3x2H9W6ldIY7QpmlUzYcNHGg=="
                        },
                        "EncryptedPIN": {
                            "EncryptionType": "PassThrough",
                            "EncryptionKeySerialNumber": "8765432100324A00313",
                            "EncryptedValue": "7325980B6B284759"
                        }
                    },
                    "Currency": {
                        "Number": 840
                    },
                    "Amount": 3.0,
                    "CashbackAmount": 0.0,
                    "AllowPartialAuthorization": true,
                    "Host": {
                        "HostType": {
                            "Code": 7,
                            "Name": "ProdConcordHC"
                        },
                        "Values": {
                            "HostProfileId": 1,
                            "CompanyNumber": 10031,
                            "StoreNumber": 1,
                            "StoreProfileId": 1,
                            "NumericStateCode": 0,
                            "AlphaStateCode": "NH",
                            "CheckAuthService": "3"
                        }
                    }
                },
                "Trace": {
                    "Path": [{
                        "Type": "OpenEPS",
                        "Value": "OE1"
                    },
                    {
                        "NodeId": 0,
                        "Type": "Gateway",
                        "Value": "GAT"
                    },
                    {
                        "NodeId": 0,
                        "ElapsedSeconds": 0.7458587,
                        "Type": "Tenant",
                        "Value": "TEN"
                    }]
                }
            },
            "timeoutInMilliseconds": 45000
        }
    },
    {
        "ts": "2017-09-04T07:07:45.6259373Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.SDK.TenantResolutionClient",
        "msg": {
            "cnt": "SEND"
        },
        "data": {
            "RequestUri": "http://153.71.66.148:20002/TenantConfigurationService/v1/Resolution",
            "Method": "POST",
            "Headers": {
                "Date": "Mon, 04 Sep 2017 07:07:45 GMT",
                "User-Agent": "ConnectedPayments/1.0",
                "x-ms-request-root-id": "6627de44-43a027c7567ddccd",
                "x-ms-request-id": "|6627de44-43a027c7567ddccd.1.",
                "Request-Id": "|6627de44-43a027c7567ddccd.1.",
                "Correlation-Context": "UniqueId=G00008D4F35EDADCD368",
                "Content-Type": "application/json; charset=utf-8",
                "Content-Length": "181"
            }
        }
    },
    {
        "ts": "2017-09-04T07:07:46.37098Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.SDK.TenantResolutionClient",
        "msg": {
            "cnt": "RECV"
        },
        "data": {
            "StatusCode": 200,
            "ReasonPhrase": "OK",
            "Headers": {
                "Date": "Mon, 04 Sep 2017 07:07:46 GMT",
                "Transfer-Encoding": "chunked",
                "X-Unique-Id": "G00008D4F35EDADCD368",
                "X-Node-Id": "0",
                "Content-Type": "application/json; charset=utf-8"
            }
        }
    },
    {
        "ts": "2017-09-04T07:07:46.37098Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.GatewayService.TransactionProcessingEngine",
        "msg": {
            "cnt": "Resolution performed."
        },
        "data": {
            "Response": {
                "Content": {
                    "Company": {
                        "Number": 10031,
                        "Name": "Sprouts"
                    },
                    "TransactionProfile": {
                        "TransactionProfileId": 1,
                        "CompanyNumber": 10031,
                        "Number": 1000,
                        "Description": "Transaction Profile 1000"
                    },
                    "TransactionValidation": {
                        "TransactionValidationId": 1,
                        "TransactionProfileId": 1,
                        "MessageTypeCode": 0,
                        "TenderTypeCode": 1,
                        "TransactionTypeCode": 1,
                        "CardTypeCode": 0
                    },
                    "Store": {
                        "Number": 1,
                        "Name": "Store 1",
                        "QueueLaneNumber": 0
                    },
                    "StoreProfile": {
                        "StoreProfileId": 1,
                        "CompanyNumber": 10031,
                        "StoreNumber": 1,
                        "TransactionProfileId": 1,
                        "HostTypeCode": 7,
                        "MerchantNumber": "",
                        "TerminalId": ""
                    },
                    "HostProfile": {
                        "HostProfileId": 1,
                        "CompanyNumber": 10031,
                        "StoreNumber": 1,
                        "StoreProfileId": 1,
                        "NumericStateCode": 0,
                        "AlphaStateCode": "NH",
                        "CheckAuthService": "3"
                    }
                },
                "NodeId": 0,
                "StatusCode": 200
            },
            "ElapsedSeconds": 0.7458587
        }
    },
    {
        "ts": "2017-09-04T07:07:46.37198Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.SDK.TransactionAuthorizationClient",
        "msg": {
            "cnt": "SEND"
        },
        "data": {
            "RequestUri": "http://ser22vvm211:8082/OrchestrationService/v1/Authorization",
            "Method": "POST",
            "Headers": {
                "Date": "Mon, 04 Sep 2017 07:07:46 GMT",
                "User-Agent": "ConnectedPayments/1.0",
                "x-ms-request-root-id": "6627de44-43a027c7567ddccd",
                "x-ms-request-id": "|6627de44-43a027c7567ddccd.2.",
                "Request-Id": "|6627de44-43a027c7567ddccd.2.",
                "Correlation-Context": "UniqueId=G00008D4F35EDADCD368",
                "Content-Type": "application/json; charset=utf-8",
                "Content-Length": "1821"
            }
        }
    },
    {
        "ts": "2017-09-04T07:07:49.8941815Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.SDK.TransactionAuthorizationClient",
        "msg": {
            "cnt": "RECV"
        },
        "data": {
            "StatusCode": 200,
            "ReasonPhrase": "OK",
            "Headers": {
                "Date": "Mon, 04 Sep 2017 07:07:49 GMT",
                "Server": "Apache-Coyote/1.1",
                "Content-Length": "1672",
                "Content-Type": "application/json"
            }
        }
    },
    {
        "ts": "2017-09-04T07:07:49.8951815Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.GatewayService.TransactionProcessingEngine",
        "msg": {
            "cnt": "RSP"
        },
        "data": {
            "ResponseTransaction": {
                "Version": null,
                "Server": {
                    "LogicalDatacenterId": 0,
                    "PhysicalDatacenterId": 0,
                    "TransactionId": 10909,
                    "UniqueId": "G00008D4F35EDADCD368",
                    "UniversalTimestamp": "2017-09-04T07:07:45.6249373Z"
                },
                "State": null,
                "Tenant": {
                    "CompanyNumber": 10031,
                    "StoreNumber": 1,
                    "HostType": {
                        "Code": 7,
                        "Name": "ProdConcordHC"
                    },
                    "MerchantNumber": "",
                    "TerminalId": ""
                },
                "PointOfInteraction": {
                    "Client": {
                        "Type": "B/GsF>O",
                        "CardType": "DB",
                        "CardName": "Debit Card",
                        "CommandSequence": "B/GsF>O",
                        "Version": null
                    },
                    "Lane": {
                        "Type": "Attended",
                        "Number": 1
                    },
                    "Terminal": {
                        "EMVCapabilities": "Contact, ContactlessDisabled",
                        "Type": null,
                        "SerialNumber": null,
                        "EMVKernelVersion": null,
                        "EMVIdentifierCAPK": null,
                        "PINCapabilities": null
                    },
                    "PointOfSale": {
                        "CashierNumber": "1",
                        "ReferenceNumber": null
                    },
                    "LocalTimestamp": "2017-06-16T14:58:57",
                    "LocalTimeZoneOffset": null
                },
                "Request": {
                    "MessageType": null,
                    "EntryMode": null,
                    "CashbackAmount": 0.00,
                    "AllowPartialAuthorization": true,
                    "TenderType": null,
                    "TransactionType": null,
                    "ReversalType": null,
                    "ReferenceId": null,
                    "AuditId": null,
                    "AccountNumberFirstSix": null,
                    "AccountNumberLastFour": null,
                    "AccountNumberLength": null,
                    "CardType": null,
                    "PreviousTransaction": null,
                    "Card": null,
                    "Check": null,
                    "Identification": null,
                    "Currency": null,
                    "Amount": null,
                    "TipAmount": null,
                    "Host": null
                },
                "Response": {
                    "ErrorCode": 96,
                    "ErrorMessage": "E2G",
                    "IsApproved": false,
                    "ResponseCode": "96",
                    "HostResponseCode": "E2G",
                    "HostResponseMessage": "EDIT ER:OPT DATA",
                    "ApprovedAmount": 0.00,
                    "ApprovedCashbackAmount": 0.0,
                    "IsApprovedLocally": false,
                    "AuthorizationCode": null,
                    "ResponseMessage": null,
                    "BalanceAmount": null,
                    "Check": null,
                    "Currency": null,
                    "Card": null,
                    "Host": null
                },
                "Trace": {
                    "ElapsedSeconds": 4.2694351,
                    "Path": [{
                        "NodeId": 0,
                        "ElapsedSeconds": 4.2694351,
                        "Type": "Gateway",
                        "Value": "GAT"
                    }]
                },
                "Queue": null,
                "SensitiveData": null,
                "PreviousTransaction": null,
                "Trace": null
            }
        }
    },
    {
        "ts": "2017-09-04T07:07:49.8951815Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.GatewayService.ProcessorTransactionService",
        "msg": {
            "cnt": "RSP"
        },
        "data": {
            "Trace": "Aa10031<1C>Ab1<1C>Ac000101<1C>Ad20170616145857<1C>Ae10011<1C>Af96<1C>ArY<1C>BnDB<1C>BqDebit Card<1C>Db0<1C>Dc0<1C>Ga1<1C>Gb1<1C>GfG<1C>GhB/GsF>O<1C>GkGAT<1C>Jb96<1C>Ka7<1C>MbE2G<1C>Mg0<1C>Nc5<1C>Ya10909<1C>Yb0<1C>YcG00008D4F35EDADCD368"
        }
    },
    {
        "ts": "2017-09-04T07:07:49.8951815Z",
        "tid": 25,
        "lvl": "Information",
        "cat": "NCR.CP.Service.ServiceHostMiddleware",
        "msg": {
            "cnt": "RSP"
        },
        "data": {
            "Headers": {
                "Date": "Mon, 04 Sep 2017 07:07:49 GMT",
                "Transfer-Encoding": "chunked",
                "Content-Type": "text/xml; charset=utf-8",
                "X-Unique-Id": "G00008D4F35EDADCD368",
                "X-Node-Id": "0"
            }
        }
    },
    {
        "ts": "2017-09-04T07:07:49.8951815Z",
        "tid": 25,
        "eid": 2,
        "lvl": "Information",
        "cat": "Microsoft.AspNetCore.Hosting.Internal.WebHost",
        "msg": {
            "cnt": "Request finished in 4272.8738ms 200 text/xml; charset=utf-8",
            "ElapsedMilliseconds": 4272.8738,
            "StatusCode": 200,
            "ContentType": "text/xml; charset=utf-8"
        }
    }],
    "RequestId": "G00008D4F35EDADCD368",
    "RequestPath": "/Processor",
    "Action": "http://servereps.mtxeps.com/TransactionService/SendTransaction",
    "Contract": "NCR.CP.GatewayService.IProcessorTransactionContract",
    "OperationName": "SendTransaction",
    "MethodName": "SendTransaction"
}

这里&#34; msgs&#34; element包含嵌套的数组对象,所以要创建数组对象内的每个元素的字段,我遵循mutate和split方法: -

变异过滤器: -

    mutate {
            add_field => {
                    "ts"  => "%{[doc][msgs][0][ts]}"
                    "tid1" => "%{[doc][msgs][0][tid]}"
                    "eid1" => "%{[doc][msgs][0][eid]}"
                    "lvl1" => "%{[doc][msgs][0][lvl]}"  
                    "cat1" => "%{[doc][msgs][0][cat]}"
                    "msg1" => "%{[doc][msgs][0][msg]}"
                    "data" => "%{[doc][msgs][1][data]}"
                    "actual-message" =>"%{[doc][msgs][3][data][Trace]}"
                    "error" =>"%{[doc][msgs][5][ex][exs][0][ec]}"
                    "error-type" =>"%{[doc][msgs][5][ex][exs][0][typ]}"
                    }
    }

这种方法是静态的,因为我的日志是动态发生的,即错误的位置不固定,因为它可能出现在第6个数组元素中,或者可能是第7个或第8个, 所以我忽略了这种方法并遵循分割过滤器方法: -

在logstash配置中拆分过滤器: -

input{
  file{
  path=>"C:\Logs\GatewayService\GatewayService-Processor.Transactions-20170830.slog"
  }

}

split {
    field=>"msgs"
}

}
output {
 elasticsearch { 
    hosts => ["localhost:9200"] 
    index => "g_index"
    }
 }

现在它显示错误:

  [2017-09-19T17:40:07,557][WARN ][logstash.filters.split   ] Only String and Array types are splittable. field:msg is of type = NilClass

那么我该怎样才能将此JSON中的每个元素都检索为一个字段,以便通过在仪表板中应用指标来在kibana中进行可视化。

0 个答案:

没有答案