Question

我有问题。我有一个CSV文件URLs（~12000）巫婆响应代码我需要检查。我的程序已逐一检查URLs。但经过34次迭代后，我的代码每次都会下降。 34行的URL不存在。如何修改我的代码以便下次遇到此错误程序跳过一个并继续工作。我写了一段代码：包lt.local;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;

import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;

public class OffersUrlChecker {
    private static final String SAMPLE_CSV_FILE_PATH = "d:\\LENAL\\ZUD\\load\\offers.csv";
    public static void main(String[] args) throws IOException {
        try (
            Reader reader = Files.newBufferedReader(Paths.get(SAMPLE_CSV_FILE_PATH));
            CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT)
        ){
         Iterable<CSVRecord> csvRecords = csvParser.getRecords();
            for (CSVRecord csvRecord : csvRecords) {
                String url = csvRecord.get(0);
                URL myUrl = new URL(url);
                HttpURLConnection myUrlCon = (HttpURLConnection) myUrl.openConnection();
                int code = myUrlCon.getResponseCode();
                System.out.println(csvRecord.getRecordNumber()+" - "+code+" " + url);
                }
            }/*catch (ConnectException e){
            System.out.println("java.net.ConnectException: Connection refused: connect");
        }*/
    }
}

Exception in thread "main" java.net.ConnectException: Connection refused: connect
    at java.net.DualStackPlainSocketImpl.connect0(Native Method)
    at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79)
    at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
    at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
    at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
    at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172)
    at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
    at java.net.Socket.connect(Socket.java:589)
    at sun.security.ssl.SSLSocketImpl.connect(SSLSocketImpl.java:673)
    at sun.security.ssl.BaseSSLSocketImpl.connect(BaseSSLSocketImpl.java:173)
    at sun.net.NetworkClient.doConnect(NetworkClient.java:180)
    at sun.net.www.http.HttpClient.openServer(HttpClient.java:463)
    at sun.net.www.http.HttpClient.openServer(HttpClient.java:558)
    at sun.net.www.protocol.https.HttpsClient.<init>(HttpsClient.java:264)
    at sun.net.www.protocol.https.HttpsClient.New(HttpsClient.java:367)
    at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.getNewHttpClient(AbstractDelegateHttpsURLConnection.java:191)
    at sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1138)
    at sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1032)
    at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:177)
    at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1546)
    at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1474)
    at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480)
    at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:338)
    at lt.zudykkaina.OffersUrlChecker.main(OffersUrlChecker.java:29)

Process finished with exit code 1

Answer 1

克服这个问题的三点建议：

将调用嵌入try catch块中。
关闭网址连接以释放资源。你可能正在积累开放的套接字。
在每次出错之后稍等一会儿，然后继续。某些服务器可能会因为以非常快的速度访问其URL而阻止您。

这些方面的东西：

public class OffersUrlChecker {

    private static final String SAMPLE_CSV_FILE_PATH = "d:\\LENAL\\ZUD\\load\\offers.csv";

    public static void main(String[] args) throws IOException {
        try (
                Reader reader = Files.newBufferedReader(Paths.get(SAMPLE_CSV_FILE_PATH));
                CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT)
        ) {
            Iterable<CSVRecord> csvRecords = csvParser.getRecords();
            for (CSVRecord csvRecord : csvRecords) {
                String url = csvRecord.get(0);
                URL myUrl = new URL(url);
                HttpURLConnection myUrlCon = null;
                try {
                    myUrlCon = (HttpURLConnection) myUrl.openConnection();
                    int code = myUrlCon.getResponseCode();
                    System.out.println(csvRecord.getRecordNumber() + " - " + code + " " + url);
                } catch (Exception e) {
                    Logger.getLogger(OffersUrlChecker.class.getName()).log(Level.SEVERE,
                            String.format("Could not read from %s", url), e);
                    // sleep a bit
                    try {
                        TimeUnit.SECONDS.sleep(10);
                    } catch (InterruptedException e1) {
                        Logger.getLogger(OffersUrlChecker.class.getName()).log(Level.WARNING,
                                "Sleep interrupted", e);
                    }
                } finally {
                    if (myUrlCon != null) {
                        myUrlCon.disconnect(); // Make sure you close the sockets
                    }
                }
            }
        }
    }
}

跳过由HttpURLConnection

1 个答案: