Question

我正在尝试做一个简单的IdHttp.get，但是我得到的响应为我提供了CloudFlare的页面，上面写着Checking your browser before accessing...
我该如何处理？，我尝试了可能想到的所有选项，甚至尝试进行Sleep(6000)并重复IdHttp.get，因为CloudFlare消息中说wait for 5 second
这是我的代码：

var 
  mIdHttp: TIdHttp;  
  URL: String;  
  memoryStream: TMemoryStream;
Begin  
  mIdHttp := TIdHttp.create(nil);
  mIdHttp.AllowCookies := true;
  mIdHttp.HandleRedirects := true;
  mIdHttp.Request.UserAgent := 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 OPR/44.0.2510.1457';
  mIdHttp.Request.Accept := 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8';
  mIdHttp.Request.AcceptEncoding := 'gzip, deflate';
  mIdHttp.Request.AcceptLanguage := 'en-US,en;q=0.9';
  mIdHttp.Request.Host := 'somesite.com/'';  
  URL := 'https://somesite.com'';  
  //Both ssleay32.dll and libeay32.dll are beside the application.
  mIdHttp.get(URL, memoryStream);  
  memoryStream.saveToFile('response.txt');  
End;

Answer 1

I solved this problem，first update the indy version to 10.6.2.0,then follow my request code below:

function Request(Method,URL:String;RequestHeaders,SendString:String;TreadTLog:TLogger;ContentType:string='application/x-www-form-urlencoded'
  ;SSLVersion:TIdSSLVersion=sslvSSLv23;UserAgent:string='Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0'):string;
var
  SendStream,GetSStream: TStringStream;
  IdHTTP:TIdHTTP;
  List:TStringList;
  LHandler: TIdSSLIOHandlerSocketOpenSSL;
  EvHandler:TEventHandlers;
  IdConnectionIntercept:TIdConnectionIntercept;
  i:Integer;
  RetStr,S,KEY,VALUE:string;  
begin
  Result:='';
  if URL='' then Exit;
  IdHTTP:=TIdHTTP.Create(nil);
  List := TStringList.Create;  
  SendStream:=TStringStream.Create('');
  GetSStream:=TStringStream.Create('');
  try

    ExtractStrings(['&'],[],pchar(RequestHeaders),List);
    SendStream.WriteString(SendString);
    try
      IdConnectionIntercept:= TIdConnectionIntercept.Create(nil);
      IdConnectionIntercept.OnReceive := EvHandler.IdConnectionInterceptReceive;
      IdConnectionIntercept.OnSend := EvHandler.IdConnectionInterceptSend;
      IDHTTP.Intercept := IdConnectionIntercept;   

      if (pos('HTTPS',UPPERCASE(URL))>0) then
      begin
        LHandler := TIdSSLIOHandlerSocketOpenSSL.Create(IDHTTP);
        IdHTTP.IOHandler:=LHandler;
        LHandler.OnVerifyPeer:=EvHandler.LHandlerVerifyPeer;
        LHandler.SSLOptions.Method := SSLVersion;
        LHandler.SSLOptions.SSLVersions:=[sslvSSLv23,sslvSSLv2, sslvSSLv3, sslvTLSv1,sslvTLSv1_1,sslvTLSv1_2];
        LHandler.SSLOptions.Mode := sslmUnassigned;
        LHandler.SSLOptions.VerifyMode := LHandler.SSLOptions.VerifyMode + [sslvrfPeer];;
        LHandler.SSLOptions.VerifyDepth := 0;   
      end
      else IdHTTP.IOHandler:=nil;
      for i:=0 to List.Count-1 do
      begin
        S:=Trim(List.Strings[i]);
        if S<>'' then
        begin
          KEY:=Copy(S,1,Pos('=',S)-1);
          VALUE:=Copy(S,Pos('=',S)+1,Length(S));
          IdHTTP.Request.CustomHeaders.Add(KEY+':'+VALUE);
        end;
      end;
      IdHTTP.Request.ContentType :=ContentType;
      IdHTTP.Request.UserAgent:=UserAgent;

      IdHTTP.HandleRedirects := True;
      IdHTTP.AllowCookies := True;
      IdHTTP.Request.Connection:='keep-alive';
      IdHTTP.Request.BasicAuthentication := False;
      IdHTTP.Request.Accept:='text/html, */*';
      IdHTTP.Request.AcceptEncoding:='identity';
      //IdHTTP.ReadTimeout:=MySysPM.PMA06;
      //IdHTTP.ConnectTimeout:=MySysPM.PMA06;
      IdHTTP.HTTPOptions:=IdHTTP.HTTPOptions+[hoKeepOrigProtocol];
      IdHTTP.ProtocolVersion:=pv1_1;
      IdHTTP.Request.Referer:='';  

      IF UpperCase(Method)='POST' then
      BEGIN
        IdHTTP.Post(URL,SendStream,GetSStream);
        RetStr:=Utf8ToAnsi(GetSStream.DataString);
      end
      else if UpperCase(Method)='GET' then
      begin
        RetStr:=IdHTTP.Get(URL);
      end
      else begin
        IdHTTP.Delete(URL);
      end;
      IdHTTP.Disconnect;
      Result:=RetStr
    except
      on E:exception do
      begin
        TreadTLog.WriteLog('Request:'+e.Message,1);
      end;
    end;
  finally
    try
      if Assigned(IdConnectionIntercept) then FreeAndNil(IdConnectionIntercept);
      FreeAndNil(LHandler);
      FreeAndNil(SendStream);
      FreeAndNil(GetSStream);
      FreeAndNil(List);
      FreeAndNil(IdHTTP);
    except
    end;
  end;
end;

Answer 2

Cloudflare实现了针对机器人（DDoS攻击等）的防护，这就是等待5秒钟的时间。

Redirect to a website with cloudflare 5 second protection C#

您的应用不是执行Javascript的网络浏览器，因此将其视为机器人。

Cloudflare使用Javascript发送一个挑战，必须计算该挑战并将其发送回Cloudflare，以获得一个cookie，然后该cookie可以用来绕过后续请求的保护。

How can I get html from page with cloudflare ddos portection?

以上链接适用于C＃。您将必须使用Indy和所需的任何Javascript / Regex库在Delphi中复制类似的解决方案。

如何处理Clouflare，我执行Idhttp.get时得到Cloudflare而不是实际的网站源

2 个答案: