我正在调用第三方API,以便在名为 filteredList 的列表中获取大约58,000行。类 T 包含在整个列表中重复的属性。因此,我的要求需要首先根据重复值对列表项进行分组(可能有20000个重复项):
Dim duplicateEntries = filteredList.GroupBy(Function(x) x.CustomMemberId)
一旦我知道重复列表,我就必须附加一个后缀(使用以下代码片段在集合中的每个值之后的数字,因此CustomMemberId属性没有重复。例如,1组重复值可能有34992078作为CustomMemberId。我的要求是将第一个CustomMemberId设置为34992078,并且具有相同成员ID的每个后续记录将其设置为(在运行以下代码之后):
Old CustomMemberId New CustomMemberId
34992078 349920781
34992078 349920782
34992078 349920783
34992078 349920784
34992078 349920785
For Each item As Object In duplicateEntries
Dim num As Integer = 0
Dim tempClubs As IOrderedEnumerable(Of ClubInformation) = filteredList.Where(Function(x) x.CustomMemberId = item.Key).OrderBy(Function(x) Convert.ToDateTime(x.createTimestamp))
For Each club As ClubInformation In tempClubs
club.CustomMemberId = If(num > 0, club.CustomMemberId + num.ToString(), club.CustomMemberId)
num += 1
Next
Next
逻辑似乎工作得很好,因为运行整个逻辑需要大约2个小时,这是完全不可接受的。我无法将逻辑更改为其他任何内容(必须在一组中的每个重复条目后附加一个数字,所以请不要建议更改它),所以我需要帮助才能进行查询快点。任何指针都将非常感激。
更新 - 添加代码(从API获取数据后)以便更好地理解
#Region "Remove ineligible types"
LogAuditEntry("Starting removal of Ineligible types")
Dim types = GetIneligibleMembershipTypes(_clientId)
Dim typeList As List(Of String) = New List(Of String)
If Not String.IsNullOrEmpty(types) Then
typeList = types.Split(","c).ToList()
End If
fetched = 0
totalFetched = 0
Dim filteredList As New List(Of ClubInformation)()
Do
Dim count = list.Skip(totalFetched).Take(1000).Count
Dim filtered = (From item In list.Skip(totalFetched).Take(1000)
Where Not typeList.Any(Function(x) item.MembershipType.Equals(x)) 'dont consider ineligible membership types)
Select New ClubInformation() With {
.MemberId = item.MemberId,
.ReferringMemberId = item.ReferringMemberId,
.AgreementNumber = item.AgreementNumber,
.BeginDate = item.BeginDate,
.MembershipType = item.MembershipType,
.CustomMemberId = item.CustomMemberId,
.FirstName = item.FirstName,
.LastName = item.LastName,
.AddressLine1 = item.AddressLine1,
.AddressLine2 = item.AddressLine2,
.City = item.City,
.State = item.State,
.Zip = item.Zip,
.Country = item.Country,
.Email = item.Email,
.Telephone = item.Telephone,
.DateofBirth = item.DateofBirth,
.Status = item.Status,
.memberStatus = item.memberStatus,
.agreementEntrySource = item.agreementEntrySource,
.lastModifiedTimestamp = item.lastModifiedTimestamp,
.nextBillingDate = item.nextBillingDate,
.signDate = item.signDate,
.firstPaymentDate = item.firstPaymentDate,
.sinceDate = item.sinceDate,
.agreementEntrySourceReportName = item.agreementEntrySourceReportName,
.queueTimestamp = item.queueTimestamp,
.agreementPaymentMethod = item.agreementPaymentMethod,
.barcode = item.barcode,
.createTimestamp = item.createTimestamp,
.currentQueue = item.currentQueue,
.downPayment = item.downPayment,
.emergencyContactName = item.emergencyContactName,
.emergencyExt = item.emergencyExt,
.emergencyPhone = item.emergencyPhone,
.firstCheckInTimestamp = item.firstCheckInTimestamp,
.gender = item.gender,
.hasPhoto = item.hasPhoto,
.homeClub = item.homeClub,
.isActive = item.isActive,
.isConvertedProspect = item.isConvertedProspect,
.isNonMember = item.isNonMember,
.isPastDue = item.isPastDue,
.isPrimaryMember = item.isPrimaryMember,
.joinStatus = item.joinStatus,
.lastCheckInTimestamp = item.lastCheckInTimestamp,
.lateFeeAmount = item.lateFeeAmount,
.managedType = item.managedType,
.memberStatusDate = item.memberStatusDate,
.memberStatusReason = item.memberStatusReason,
.nextDueAmount = item.nextDueAmount,
.pastDueBalance = item.pastDueBalance,
.paymentFrequency = item.paymentFrequency,
.paymentPlan = item.paymentPlan,
.renewalType = item.renewalType,
.salesPersonHomeClub = item.salesPersonHomeClub,
.salesPersonId = item.salesPersonId,
.salesPersonName = item.salesPersonName,
.serviceFeeAmount = item.serviceFeeAmount,
.term = item.term,
.totalCheckInCount = item.totalCheckInCount,
.totalPastDueBalance = item.totalPastDueBalance,
.workPhonExt = item.workPhonExt
}).ToList()
'Add to list
filteredList.AddRange(filtered)
fetched = count
totalFetched += fetched
Loop While fetched > 0
#End Region
LogAuditEntry("Removed Ineligible types")
#Region "Appends number after duplicate club id + agreement no"
Dim duplicateEntries = filteredList.GroupBy(Function(x) x.CustomMemberId)
For Each item As Object In duplicateEntries
Dim num As Integer = 0
Dim tempClubs As IOrderedEnumerable(Of ClubInformation) = filteredList.Where(Function(x) x.CustomMemberId = item.Key).OrderBy(Function(x) Convert.ToDateTime(x.createTimestamp))
For Each club As ClubInformation In tempClubs
club.CustomMemberId = If(num > 0, club.CustomMemberId + num.ToString(), club.CustomMemberId)
num += 1
Next
Next
LogAuditEntry("Removed duplicates and added custom member id")
#End Region
更新2 - 时间表
时间:16-Apr-16 3:33:22 AM
=============================================== ======================== 时间:16-Apr-16 5:37:20 AM
=============================================== =======================
答案 0 :(得分:1)
因为你使用的是late binding,所以速度很慢。别。通过早期绑定,您的查询将执行更快的数量级。
更改
For Each item As Object In duplicateEntries
到
For Each item In duplicateEntries
解决问题。