Question

我使用C＃和Nest存储在ElasticSearch和Windows应用程序中的日志，它正在对ElasticSearch执行搜索。 ElasticSearch中的映射如下所示：

"mappings": {
    "qns": {
        "properties": {
            "@timestamp": {
                "format": "strict_date_optional_time||epoch_millis",
                "type": "date"
            },
            "Error_Description": {
                "index": "not_analyzed",
                "type": "string"
            },
            "Thread_Id": {
                "index": "not_analyzed",
                "type": "string"
            },
            "Error_Description_Analyzed": {
                "type": "string"
            },
            "Error_Source": {
                "index": "not_analyzed",
                "type": "string"
            },
            "record": {
                "type": "string"
            },
            "@version": {
                "type": "string"
            },
            "Log_Level": {
                "type": "string"
            },
            "Record": {
                "type": "string"
            },
            "id": {
                "type": "long"
            },
            "Error_Source_Analyzed": {
                "type": "string"
            },
            "Timestamp": {
                "format": "strict_date_optional_time||epoch_millis",
                "type": "date"
            }
        }
    }
}

相应的C＃类如下：

[ElasticsearchType(IdProperty = "Id", Name = "qns")]
public class QNS
{
    [Number(NumberType.Long, Name = "id")]
    public long Id { get; set; }

    [Date(Name = "Timestamp")]
    public DateTime Timestamp { get; set; }

    [String(Name = "Error_Description", Index = FieldIndexOption.NotAnalyzed)]
    public string ErrorDescriptionKeyword { get; set; }

    [String(Name = "Error_Description_Analyzed")]
    public string ErrorDescriptionAnalyzed { get; set; }

    [String(Name = "Error_Source", Index = FieldIndexOption.NotAnalyzed)]
    public string ErrorSourceKeyword { get; set; }

    [String(Name = "Error_Source_Analyzed")]
    public string ErrorSourceAnalyzed { get; set; }

    [String(Name = "Thread_Id", Index = FieldIndexOption.NotAnalyzed)]
    public string ThreadId { get; set; }

    [String(Name = "Log_Level")]
    public string LogLevel { get; set; }

    [String(Index = FieldIndexOption.NotAnalyzed)]
    public string Record { get; set; }
}

我需要一种方法来搜索属于日期时间范围并匹配特定范围模式的不同错误记录。虽然我能够得到结果，但我也得到满足搜索的所有文档，而我只需要不同的错误字符串。对于Distinct查询，我使用的是FluentNest（https://github.com/hoonzis/fluentnest）。检索结果的代码如下：

    private List<string> FindDistinctErrorsByPatternAndTimeRangeInternal(DateTime fromDateTime, DateTime toDateTime, List<pattern> patterns, string indexName, string type)
    {
        var documents = new List<QNS>();

        var fromTime = fromDateTime.ToString(Constants.IndexSearch.ES_DATETIME_FORMAT);
        var toTime = toDateTime.ToString(Constants.IndexSearch.ES_DATETIME_FORMAT);

        var patternQueries = new List<QueryContainer>();

        foreach (var p in patterns)
        {
            var pType = PatternType.unknown;
            if (Enum.TryParse<PatternType>(p.Pattern_Type.ToLowerInvariant(), out pType))
            {
                switch (pType)
                {
                    case PatternType.word:
                        patternQueries.Add(Query<QNS>.Regexp(r =>
                            r.Field(f =>
                                f.ErrorDescriptionAnalyzed)
                                .Value(p.Pattern_Description)
                            )
                       );
                        break;
                    case PatternType.phrase:
                        patternQueries.Add(Query<QNS>.MatchPhrase(m =>
                            m.Field(f =>
                                f.ErrorDescriptionAnalyzed)
                                .Query(p.Pattern_Description)
                            )
                        );
                        break;
                    case PatternType.unknown:
                    default:
                        break;
                }
            }
        }

        var datetimeQuery = Query<QNS>.QueryString(q =>
                                q.DefaultField(f =>
                                    f.Timestamp).Query($"[{fromTime} TO {toTime}]")
                                );

        var searchResults = client.Search<QNS>(s => s.Index(indexName)
           .Type(type)
           .Query(q =>
               q.Filtered(f =>
                   f.Filter(fq =>
                       fq.Bool(b =>
                           b.MinimumShouldMatch(1).Should(patternQueries.ToArray())
                       )
                   )
                   .Query(qd =>
                       qd.Bool(b =>
                           b.Must(datetimeQuery)
                       )
                   )
               )
            )
           .Sort(sort => sort.Ascending(SortSpecialField.DocumentIndexOrder))
           .Aggregations(agg => agg.DistinctBy(q => q.ErrorDescriptionKeyword)));

        var results = searchResults.Aggs.AsContainer<QNS>().GetDistinct(d => d.ErrorDescriptionKeyword);

        return results.ToList();
    }

我需要修改此代码，只返回不同的错误字符串，而不是整个结果集。查询中的命中数大约为3500，并且只存在2个不同的错误字符串。因此，将所有这些记录恢复是没有意义的，因为我不打算使用它。有人可以使用日期范围和模式正则表达式/短语匹配来帮助我获得正确的聚合查询，以便仅使用Nest或Nest / FluentNest返回不同的错误记录。

Answer 1

我认为您正在寻找terms聚合。

但是你的整个查询有点奇怪。你有遗留的要求吗？

首先你有两个字段ErrorDescriptionAnalyzed和ErrorDescriptionKeyword你是在做一个不同的字段只是为了分析一个而不是一个？为什么不使用multi-fields

第二个过滤方法已经过时了一段时间。

以下是我希望能提供帮助的快速示例

ElasticClient db = new ElasticClient(uri);

            db.DeleteIndex(indexName);

            var mappings = new CreateIndexDescriptor(indexName).Mappings(ms => ms.Map<A>(map => map.
                AutoMap().
                Properties(props =>
                    props.String(p =>
                        p.Name(a => a.Text).
                        Fields(fields =>
                            fields.String(pr => pr.Name("raw").NotAnalyzed()))))));

            db.CreateIndex(mappings);

            foreach (var item in Enumerable.Range(0, 10).Select(i => new A
            {
                Price1 = random.NextDouble() * 1000,
                Date = i % 3 == 0 ? new DateTime(1900, 1, 1) : DateTime.Now,
                Text = i % 2 == 0 ? "ABC" : "EFG"
            }))
            {
                db.Index(item, inx => inx.Index(indexName));
            }

            var toDate = DateTime.Now + TimeSpan.FromDays(1);
            var fromDate = DateTime.Now - TimeSpan.FromDays(30);

            var data = db.Search<A>(s => 
                s.Index(indexName)
                .Query(q=>
                        q.DateRange(r => r.Field(f => f.Date).GreaterThan(fromDate).LessThanOrEquals(toDate))
                        &&
                        (
                        //term query is for finding words by default all words are lowercase but you can set a different analyzer
                        q.Term(t => t.Field(f => f.Text).Value("ABC".ToLower()))
                        ||
//Raw field is not analysed so no need to lower case you can use you query here if you want
                        q.Term(t => t.Field("text.raw").Value("EFG"))
                        )
                ).Aggregations(aggr => aggr.Terms("distinct", aterm => aterm.Field("text.raw"))));

需要在没有使用C＃和Nest的完整搜索结果文档的情况下从ElasticSearch返回Distinct字段

1 个答案: