Question

使用以下代码抓取https网站，在大多数网站上正常运行但在某些网站（例如grubhub.com）上无效。

import java.io.IOException;
import java.security.cert.X509Certificate;

import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.conn.ssl.X509HostnameVerifier;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;

public class TestSsl {

    HttpClient httpclient = new DefaultHttpClient();

    public static void main(String[] args) throws Exception {

        TestSsl t = new TestSsl();
        t.init();
        t.queryData();
    }

    private void init() {
        httpclient = wrapClient(httpclient);
        httpclient.getParams()
                .setIntParameter("http.socket.timeout", 30 * 1000);
    }

    /**
     * 
     * @throws Exception
     */
    private void queryData() throws Exception {
        HttpGet httpget = new HttpGet("https://www.grubhub.com/browse/");
        HttpResponse response = httpclient.execute(httpget);

        HttpEntity entity = response.getEntity();
        String result = EntityUtils.toString(entity);
        System.out.println(result);
        EntityUtils.consume(entity);

    }

    /**
     * 
     * @param base
     * @return
     */
    public static HttpClient wrapClient(HttpClient base) {
        try {
            SSLContext ctx = SSLContext.getInstance("SSL");
            X509TrustManager tm = new X509TrustManager() {
                public void checkClientTrusted(X509Certificate[] xcs,
                        String string) {
                }

                public void checkServerTrusted(X509Certificate[] xcs,
                        String string) {
                }

                public X509Certificate[] getAcceptedIssuers() {
                    return null;
                }
            };
            X509HostnameVerifier verifier = new X509HostnameVerifier() {

                @Override
                public void verify(String string, SSLSocket ssls) throws IOException {
                }

                @Override
                public void verify(String string, X509Certificate xc) throws SSLException {
                }

                @Override
                public void verify(String string, String[] strings, String[] strings1) throws SSLException {
                }

                @Override
                public boolean verify(String string, SSLSession ssls) {
                    return true;
                }
            };

            ctx.init(null, new TrustManager[] { tm }, null);
            SSLSocketFactory ssf = new SSLSocketFactory(ctx);
            ssf.setHostnameVerifier(verifier);
            ClientConnectionManager ccm = base.getConnectionManager();
            SchemeRegistry sr = ccm.getSchemeRegistry();
            sr.register(new Scheme("https", ssf, 443));
            return new DefaultHttpClient(ccm, base.getParams());
        } catch (Exception ex) {
            ex.printStackTrace();
            return null;
        }
    }
}

我甚至在jvm上设置了以下内容：-Djavax.net.debug = all -Dhttps.protocols = SSLv3 -Dforce.http.jre.executor = true

main，READ：SSLv3警报，长度= 2 main，RECV SSLv3 ALERT：致命，bad_record_mac main，名为closeSocket（） main，处理异常：javax.net.ssl.SSLException：收到致命警报：bad_record_mac main，叫close（） main，叫做closeInternal（true） main，叫close（） main，叫做closeInternal（true） main，叫close（） main，叫做closeInternal（true）线程“main”中的异常javax.net.ssl.SSLException：收到致命警报：bad_record_mac

我已经研究过并在代码中放了一堆东西，但仍然没有运气。调试输出太大而无法在此处发布，但可以通过在IDE中创建新类并运行它来轻松重现。

非常感谢任何帮助。

Answer 1

你遇到一个已知问题，jre试图将TLS与不支持它的服务器对话。查看更多信息here。

建议的工作是强制sslv3。不幸的是，尝试使用系统属性不起作用，所以你需要通过在每个套接字上将启用的协议设置为SSLv3来实现。

这是一个适用于您的主机的小应用程序：

String urlString = "https://www.grubhub.com/browse/";
URL url = new URL(urlString);
SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault();
SSLSocket socket = (SSLSocket) factory.createSocket(url.getHost(), 443);
socket.setEnabledProtocols(new String[]{"SSLv3"}); // <--- THIS IS THE WORK-AROUND
PrintWriter out = new PrintWriter(
        new OutputStreamWriter(
                socket.getOutputStream()));
out.println("GET " + urlString + " HTTP/1.1");
out.println();
out.flush();

BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream()));

String line;
while ((line = in.readLine()) != null) {
    System.out.println(line);
}

out.close();
in.close();

你需要覆盖HttpClient的SSLSocketFactory，并配置像这样配置每个套接字的那个：

socket.setEnabledProtocols(new String[]{"SSLv3"});

Java使用httpclient 4.1抓取https get bad_record_mac

1 个答案: