Question

我试图从维基百科下载数据。我发送一个GET请求，但返回状态 - ＆＃34; HTTP 301重定向＆＃34;。现在我需要解析它以获取目标URL，并从那里我可以访问文章内容。我做解析，但我无法访问文章内容。如果您能帮助我查看文章内容，感谢您的帮助。

代码 -

#include <winsock2.h>
#include <WS2tcpip.h>
#include <windows.h>
#include <iostream>
#include <vector>
#include <ostream>

using namespace std;

int main(){

string query = "Google";
//int depth = 0;

// Initialize Dependencies to the Windows Socket.
WSADATA wsaData;
if (WSAStartup(MAKEWORD(2, 2), &wsaData) != 0) {
    cout << "WSAStartup failed.\n";
    system("pause");
}

struct addrinfo hints;
ZeroMemory(&hints, sizeof(hints));
hints.ai_family = AF_INET;
hints.ai_protocol = IPPROTO_TCP;
hints.ai_socktype = SOCK_STREAM;

/* connect to wikipedia //and download the article */
static const char wiki_host[] = "en.wikipedia.org";
struct addrinfo* targetAdressInfo = NULL;
DWORD getAddrRes = getaddrinfo(wiki_host, NULL, &hints, &targetAdressInfo);
if (getAddrRes != 0 || targetAdressInfo == NULL)
{
    cout << "Could not resolve the Host Name" << endl;
    system("pause");
    WSACleanup();
    return -1;
}

SOCKADDR_IN sockAddr;
sockAddr.sin_addr = ((struct sockaddr_in*) targetAdressInfo->ai_addr)->sin_addr;
sockAddr.sin_family = AF_INET;
sockAddr.sin_port = htons(80);

freeaddrinfo(targetAdressInfo);

SOCKET webSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (webSocket == INVALID_SOCKET)
{
    cout << "Creation of the Socket Failed" << endl;
    system("pause");
    WSACleanup();
    return -1;
}

if (connect(webSocket, (SOCKADDR*)&sockAddr, sizeof(sockAddr)) != 0)
{
    cout << "Could not connect";
    system("pause");
    closesocket(webSocket);
    WSACleanup();
    return -1;
}

string http_query = "GET /wiki/" + query + " HTTP/1.1\r\n";
http_query += string("Host: ") + wiki_host + "\r\n";
http_query += "\r\n\r\n";
if (send(webSocket, http_query.c_str(), http_query.length(), 0) == -1) {
    cout << "Could not send the request to the Server" << endl;
    system("pause");
    closesocket(webSocket);
    WSACleanup();
    return -1;
}

/* prepare to fetch the wiki article */

string response = "";

while (true) {
    static char recv_buffer[4096];
    const int bytes_read = recv(webSocket, recv_buffer, sizeof(recv_buffer) - 1, 0);
    if (!bytes_read) {
        break;
    }
    if (bytes_read == -1) {
        closesocket(webSocket);
        WSACleanup();
    }
    recv_buffer[bytes_read] = '\0';
    response += recv_buffer;
};

/* finished with the socket */

closesocket(webSocket);
WSACleanup();

/* parse the http response headers */

size_t cursor = 0;
string response_content;
vector<std::string> response_headers;
const size_t headers_end = response.find("\r\n\r\n");

while (true) {
    const size_t line_end = response.find("\r\n", cursor);
    if (line_end == std::string::npos) { /* probably due to http error */
        break;
    }
    response_headers.push_back(response.substr(cursor, line_end - cursor));
    if (line_end == headers_end) { /* found content */
        response_content = response.substr(headers_end + 4); /* skip \r\n\r\n */
        break;
    }
    cursor = line_end + 2; /* skip \r\n */
}


// print the respone 
for (int i = 0; i < sizeof(response); i++){
    cout << response[i];
}

/* print response_content not work 
for (int i = 0; i < sizeof(response); i++){
    cout << response_content[i];
}*/
system("pause");
return 0;
}

Answer 1

我不确定你的意思是说“我做了解析，但我无法访问文章内容”。当您获得301状态时，您需要做的是查看您收到的“位置”标题。该标头包含可用于获取您要查找的内容的URL。

从维基百科页面下载数据

1 个答案: