Question

我有一个应该发出非常快速的http请求的程序。请求应异步进行，以便它不会阻塞主线程。

所以我创建了一个队列，该队列由10个单独的线程观察，这些线程发出http请求。如果在队列中插入了某些内容，那么获取数据的第一个线程将发出请求并处理结果。

队列中充满了数以千计的项目，所以多线程非常需要尽可能快地获得响应。

由于我有很多代码，我会举一个简短的例子。

主要课程

package fasthttp;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;

public class FastHTTP {

    public static void main(String[] args) {
        ExecutorService executor = Executors.newFixedThreadPool(10);

        for (int i = 0; i < 10; i++) {
            LinkedBlockingQueue queue = new LinkedBlockingQueue();
            queue.add("http://www.lennar.eu/ip.php");//for example
            executor.execute(new HTTPworker(queue));
        }
    }

}

FastHTTP课程

package fasthttp;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.LinkedBlockingQueue;

public class HTTPworker implements Runnable {

    private final LinkedBlockingQueue queue;

    public HTTPworker(LinkedBlockingQueue queue) {        
        this.queue = queue;
    }

    private String getResponse(String url) throws IOException {

        URL obj = new URL(url);
        HttpURLConnection con = (HttpURLConnection) obj.openConnection();

        StringBuilder response;
        try (BufferedReader in = new BufferedReader(
                new InputStreamReader(con.getInputStream()))) {
            String inputLine;
            response = new StringBuilder();
            while ((inputLine = in.readLine()) != null) {
                response.append(inputLine);
            }
        }
        return response.toString();
    }

    @Override
    public void run() {
        while (true) {
            try {
                String data = (String) queue.take();
                String response = getResponse(data);
                //Do something with response
                System.out.println(response);
            } catch (InterruptedException | IOException ex) {
                //Handle exception
            }
        }
    }
}

是否有更好或更快的方式异步进行数千个http请求响应处理？速度和性能是我追求的目标。

Answer 1

回答我自己的问题。尝试Apaches异步http客户端，但过了一段时间我开始使用Ning的异步客户端，我很高兴。

Answer 2

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import java.util.stream.Collectors;

import org.apache.http.client.methods.HttpGet;

import java.util.Iterator;

import org.apache.http.impl.client.BasicResponseHandler;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;

public class RestService {
    private final static Executor executor = Executors.newCachedThreadPool();
    private final static CloseableHttpClient closeableHttpClient = HttpClientBuilder.create().build();

    public static String sendSyncGet(final String url) {
        return sendAsyncGet(List.of(url)).get(0);
    }

    public static List<String> sendAsyncGet(final List<String> urls){
        List<GetRequestTask> tasks = urls.stream().map(url ->  new GetRequestTask(url, executor)).collect(Collectors.toList());
        List<String> responses = new ArrayList<>();
        while(!tasks.isEmpty()) {
            for(Iterator<GetRequestTask> it = tasks.iterator(); it.hasNext();) {
                final GetRequestTask task = it.next();
                if(task.isDone()) {
                    responses.add(task.getResponse());
                    it.remove();
                }
            }
            //if(!tasks.isEmpty()) Thread.sleep(100); //avoid tight loop in "main" thread
        }
        return responses;
    }

    private static class GetRequestTask {
        private final FutureTask<String> task;

        public GetRequestTask(String url, Executor executor) {
            GetRequestWork work = new GetRequestWork(url);
            this.task = new FutureTask<>(work);
            executor.execute(this.task);
        }

        public boolean isDone() {
            return this.task.isDone();
        }

        public String getResponse() {
            try {
                return this.task.get();
            } catch(Exception e) {
                throw new RuntimeException(e);
            }
        }
    }

    private static class GetRequestWork implements Callable<String> {
        private final String url;

        public GetRequestWork(String url) {
            this.url = url;
        }
        public String getUrl() {
            return this.url;
        }
        public String call() throws Exception {
            return closeableHttpClient.execute(new HttpGet(getUrl()), new BasicResponseHandler());
        }
    }
}

在JAVA中制作多个http请求的快速和异步方式

2 个答案: