我已经使用Excel VBA构建了一个执行以下操作的Web抓取工具：
我遇到的问题是，当我点击运行时，我会收到消息“ Excel没有响应”，但是代码仍在后台运行。该代码可以正常工作，并且考虑到我要执行的操作非常快，但是显然我需要对其进行更多优化，以防止Excel重载。寻找避免在每次迭代中将解析的HTML写入Excel的方法将很有帮助，但是，我不知道如何不这样做就能以所需的格式写入数据。数组解决方案很棒，但是在将数组数据写入Excel之前，必须对数组中的数据进行大量操作，甚至可能对数组进行子集/切片。我已经用尽了全部知识，因此需要帮助，并且在构建此应用程序的过程中进行了大量研究。我什至对使用其他技术（例如python和beautifulsoup库）持开放态度，我只是不知道如何将表数据以我需要的格式输出到csv文件。预先感谢！
以下是文件： TrustTable_Parse.xlsb
免责声明：我有B.S.在数学方面，我通过使用每种语言实现我自己的许多项目来教自己如何在VBA，SQL和R中进行编码。关键是，如果我的代码看起来很奇怪，或者您认为我的工作效率低下，那是因为我已经多年没有编码了，而且我不知道更好，大声笑。
下面是我的代码：
Option Explicit
Sub Final_Parse_TrustTables()
Dim HTML As New HTMLDocument
Dim http As Object
Dim links As Object
Dim Url, Trst As String
Dim link As HTMLHtmlElement
Dim i As Long
Dim http2 As Object
Dim HTML2 As New HTMLDocument
Dim tbl As Object
Dim ele As HTMLHtmlElement
Dim wb As Workbook
Dim ws, ws_2 As Worksheet
'sets ScreenUpdating to false _
turns off event triggers, ect.
OptimizeCode_Begin
Set wb = ThisWorkbook
Set ws = wb.Sheets("CIK_Links")
'Creates this object to see if Trust table exists
Set http = CreateObject("MSXML2.ServerXMLhttp.6.0")
'Loops through the list of URL's _
in the 'CIK_Links' Sheet
For i = 2 To 3000
'List of URL's
Url = ws.Range("C1").Cells(i, 1).Value2
'Gets webpage to check _
if Trust table exists
On Error Resume Next
http.Open "GET", Url, False
http.send
'Runs code If the website sent a valid response to our request _
for FIRST http object
If Err.Number = 0 Then
If http.Status = 200 Then
'If the website sent a valid response to our request _
for SECOND http object "http2"
If Err.Number = 0 Then
If http2.Status = 200 Then
HTML.body.innerHTML = http.responseText
Set links = HTML.getElementsByTagName("a")
'Determines if there is a trust table and if so _
then it creates the http2 object and gets the _
trust table responsetext
Trst = "(List all Funds and Classes/Contracts for"
For Each link In links
'Link is returned in responsetext with "about:/" at _
the beginning instead of https://www.sec.gov/, so I _
used this to replace the "about:/"
If InStr(link.innerHTML, Trst) > 0 Then
link = Replace(link, "about:/", "https://www.sec.gov/")
Debug.Print link
'Creates this object to go to trust table webpage
Set http2 = CreateObject("MSXML2.ServerXMLhttp.6.0")
'Gets webpage to parse _
trust table
On Error Resume Next
http2.Open "GET", link, False
http2.send
HTML2.body.innerHTML = http2.responseText
'If there exists a Trust, then this refers to the _
3rd table on the trust table webpage; _
note ("table")(3)
On Error Resume Next
Set tbl = HTML2.getElementsByTagName("table")(3)
Set ws_2 = wb.Sheets("Parsed_Tables")
With ws_2
For Each ele In tbl.getElementsByTagName("tr")
'First finds rows with Class/Con numbers
If InStr(ele.innerText, "C00") Then
'Pulls Class/Con Numbers, note children(2)
'output to col E sheet
.Cells(Rows.Count, "E"). _
End(xlUp).Offset(1, 0).Value2 = ele.Children(2).innerText
'Outputs Share Class, children(3)
'Output to col F sheet
.Cells(Rows.Count, "F"). _
End(xlUp).Offset(1, 0).Value2 = ele.Children(3).innerText
'Not not all Funds have Ticker _
so this keeps the module from _
asking for object to be set
On Error Resume Next
'Outputs Ticker to excel
'Reads the last value in Col F and offsets Ticker to _
to show directly in adjacent cel in Col G
.Cells(Rows.Count, "F"). _
End(xlUp).Offset(0, 1).Value2 = ele.Children(4).innerText
'Pulls SIC number
ElseIf InStr(ele.innerText, "S00") Then
'Offsets from col F to be placed in col C
.Cells(Rows.Count, "F"). _
End(xlUp).Offset(1, -3).Value2 = ele.Children(1).innerText
'Pulls Fund Name
'Offsets from col F to col D
.Cells(Rows.Count, "F"). _
End(xlUp).Offset(1, -2).Value2 = ele.Children(2).innerText
'Pulls CIK number
ElseIf InStr(ele.Children(0).innerText, "000") Then
'Offset from col F to col A
.Cells(Rows.Count, "F"). _
End(xlUp).Offset(1, -5).Value2 = ele.Children(0).innerText
'Pulls Trust Name
'Offsets from col F to col B
.Cells(Rows.Count, "F"). _
End(xlUp).Offset(1, -4).Value2 = ele.Children(1).innerText
End If
'Counts the number of iterations of the loop _
and places it in the lower left corner of the _
workbook
Application.StatusBar = "Current Iteration: " & i
Next
End With
End If
Next
End If
Else
MsgBox "Error loading webpage", vbExclamation, "Alert!!!"
Exit Sub
End If
On Error GoTo 0
End If
Else
MsgBox "Error loading webpage", vbExclamation, "Alert!!!"
Exit Sub
End If
On Error GoTo 0
If i Mod 1000 = 0 Then
ActiveWorkbook.Save
Application.Wait (Now + TimeValue("0:00:03"))
End If
Next i
'sets everything back to normal after running code
OptimizeCode_End
End Sub
以下是CIK_Links工作表中列出的链接的示例：
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=2&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=3&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=11&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=13&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=14&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=17&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=18&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=2110&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=2135&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=2145&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=2663&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=2664&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=2691&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=2768&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=3521&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=3794&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=4123&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=4405&owner=include&count=02
https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=4568&owner=include&count=02
答案 0 :(得分：0)
除非您至少有一个On Error Resume Next隐藏一些运行时错误，否则我认为您的代码将无法运行。例如，在实例化http2对象之前，您拥有FirebaseRecyclerOptions<MessagesHelper> options =
new FirebaseRecyclerOptions.Builder<MessagesHelper>()
.setQuery(mFetchingMessages, MessagesHelper.class)
.build();
firebaseRecyclerAdapter = new FirebaseRecyclerAdapter<MessagesHelper, Chat.MessagesViewHolder>(options) {
@Override
protected void onBindViewHolder(@NonNull final Chat.MessagesViewHolder holder, final int position, @NonNull final MessagesHelper model) {
holder.setMessage(model.getMessage());
final String userId = getRef(position).getKey();
final DatabaseReference mTimeReference = FirebaseDatabase.getInstance().getReference().child("Messages").child(MessageSenderId).child(MessageRecieverId);
Query messageQuery = mTimeReference.limitToLast(10);
messageQuery.addChildEventListener(new ChildEventListener() {
@Override
public void onChildAdded(DataSnapshot dataSnapshot, String s) {
messagesList.add(message);
messageList.scrollToPosition(messagesList.size() - 1);
}
@Override
public void onChildChanged(DataSnapshot dataSnapshot, String s) {
}
@Override
public void onChildRemoved(DataSnapshot dataSnapshot) {
}
@Override
public void onChildMoved(DataSnapshot dataSnapshot, String s) {
}
@Override
public void onCancelled(DatabaseError databaseError) {
}
});
}
@Override
public Chat.MessagesViewHolder onCreateViewHolder(ViewGroup parent, int viewType) {
View view = LayoutInflater.from(parent.getContext())
.inflate(R.layout.custom_activity_chat, parent, false);
return new Chat.MessagesViewHolder(view);
}
};
messageList.setAdapter(firebaseRecyclerAdapter);
firebaseRecyclerAdapter.startListening();
。
下面是一个绝对可以改进的方法，但是使用一个类来保存xmlhttp对象，并提供了用于检索所需信息的方法。所需表的布局使解析实际网页特别复杂。您可能希望继续这样做。我选择按原样使用表结构。也许，这至少可以为您提供一个框架。您可以将自定义优化子调用添加到其中。
待办事项：
查看是否可以对可以容纳所有结果的超大结果数组进行估算，而不是对数组进行估算，以便可以随时进行写出。如果有时间的话，我会进行修正。
clsHTTP类
If http2.Status = 200 Then标准模块1
Option Explicit
Private http As Object
Const SEARCH_TERM As String = "(List all Funds and Classes/Contracts"
Private Sub Class_Initialize()
Set http = CreateObject("MSXML2.XMLHTTP")
End Sub
Public Function GetString(ByVal Url As String, Optional ByVal search As Boolean = False) As String
Dim sResponse As String
searchTermFound = False
With http
.Open "GET", Url, False
.setRequestHeader "If-Modified-Since", "Sat, 1 Jan 2000 00:00:00 GMT"
.send
sResponse = StrConv(.responseBody, vbUnicode)
If InStr(sResponse, SEARCH_TERM) > 0 Then searchTermFound = True
GetString = sResponse
End With
End Function
Public Function GetLink(ByVal html As HTMLDocument) As String
Dim i As Long, nodeList As Object
Set nodeList = html.querySelectorAll("a")
GetLink = vbNullString
For i = 0 To nodeList.Length - 1
If InStr(nodeList.item(i).innerText, SEARCH_TERM) > 0 Then
GetLink = Replace$(nodeList.item(i).href, "about:/", "https://www.sec.gov/")
Exit For
End If
Next
End Function
Public Function GetInfo(ByVal html As HTMLDocument) As Variant
Dim CIK As String, table As HTMLTable, tables As Object, tRows As Object
Dim arr(), tr As Object, td As Object, r As Long, c As Long
Set tables = html.querySelectorAll("table")
If tables.Length > 3 Then
CIK = "'" & html.querySelector(".search").innerText
Set table = tables.item(3)
Set tRows = table.getElementsByTagName("tr")
ReDim arr(1 To tRows.Length, 1 To 6)
Dim numColumns As Long, numBlanks As Long
For Each tr In tRows
numColumns = tr.getElementsByTagName("td").Length
r = r + 1: c = 2: numBlanks = 0
If r > 4 Then
arr(r - 4, 1) = CIK
For Each td In tr.getElementsByTagName("td")
If td.innerText = vbNullString Then numBlanks = numBlanks + 1
arr(r - 4, c) = td.innerText
c = c + 1
Next td
If numBlanks = numColumns Then Exit For
End If
Next
Else
ReDim arr(1, 1)
GetInfo = arr
Exit Function
End If
arr = Application.Transpose(arr)
ReDim Preserve arr(1 To 6, 1 To r - 4)
arr = Application.Transpose(arr)
GetInfo = arr
End Function