我正在用Java构建一个收集程序，它使用api从网站收集数据。我遇到了这个问题，它将挂在http调用上。我尝试通过执行器服务执行http调用并超时来解决它。这似乎不起作用，因为它会保持超时和重试。我认为它可能与API有关，因此在重试之后我将重新初始化每个网站API的全新对象。仍然没有解决方案我试图找出这个的根本原因，但似乎无法把手指放在上面。
以下是我的flickr管理器类，它处理对flickr的调用。
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.Collection;
import java.util.Collections;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.scribe.exceptions.OAuthConnectionException;
import com.flickr4java.flickr.Flickr;
import com.flickr4java.flickr.FlickrException;
import com.flickr4java.flickr.FlickrRuntimeException;
import com.flickr4java.flickr.REST;
import com.flickr4java.flickr.RequestContext;
import com.flickr4java.flickr.auth.Auth;
import com.flickr4java.flickr.auth.Permission;
import com.flickr4java.flickr.people.User;
import com.flickr4java.flickr.photos.Exif;
import com.flickr4java.flickr.photos.Extras;
import com.flickr4java.flickr.photos.Photo;
import com.flickr4java.flickr.photos.PhotoList;
import com.flickr4java.flickr.photos.SearchParameters;
import com.flickr4java.flickr.photos.Size;
import com.google.common.util.concurrent.RateLimiter;
public class FlickrManager {
private final static Logger LOG = Logger.getLogger(FlickrManager.class.getName());
private final static ExecutorService executorService = Executors.newSingleThreadExecutor();
private Flickr flickr;
private final int MAX_PER_PAGE = 500;
private final RateLimiter rateLimiter;
private String ApiKey;
private String ApiSecret;
private String authToken;
private String authTokenSecret;
private Integer hostPort;
private String hostAddress;
private String httpScheme;
public FlickrManager(Flickr flickr, double apiCallsPerSecond) throws FlickrException {
this.flickr = flickr;
flickr.getTestInterface().echo(Collections.emptyMap());
//get flickr info to reinitialize flickr object if necessary
this.ApiKey = flickr.getApiKey();
this.ApiSecret = flickr.getSharedSecret();
this.hostPort = flickr.getTransport().getPort();
this.hostAddress = flickr.getTransport().getHost();
this.httpScheme = flickr.getTransport().getScheme();
if(flickr.getAuth() != null){
this.authToken = flickr.getAuth().getToken();
this.authTokenSecret = flickr.getAuth().getTokenSecret();
}
this.rateLimiter = RateLimiter.create(apiCallsPerSecond);
}
private void initialize(){
this.flickr = null;
REST rest = new REST(this.hostAddress,this.hostPort);
rest.setScheme(this.httpScheme);
this.flickr = new Flickr(this.ApiKey, this.ApiSecret,rest);
if(this.authToken != null && this.authTokenSecret != null){
RequestContext requestContext = RequestContext.getRequestContext();
Auth auth = new Auth();
auth.setPermission(Permission.READ);
auth.setToken(this.authToken);
auth.setTokenSecret(this.authTokenSecret);
requestContext.setAuth(auth);
flickr.setAuth(auth);
}
}
public User getUserInfo(String flickrProfileId) throws FlickrException{
return doFlickrAction(new CallableFlickrTask<User>(){
@Override
public User execute() throws FlickrException {
return flickr.getPeopleInterface().getInfo(flickrProfileId);
}
});
}
public PhotoList<Photo> search(SearchParameters params, int page) throws FlickrException{
return doFlickrAction(new CallableFlickrTask<PhotoList<Photo>>(){
@Override
public PhotoList<Photo> execute() throws FlickrException {
return flickr.getPhotosInterface().search(params, MAX_PER_PAGE, page);
}
});
}
public PhotoList<Photo> getUserPhotos(String userNSID, int page) throws FlickrException{
return doFlickrAction(new CallableFlickrTask<PhotoList<Photo>>(){
@Override
public PhotoList<Photo> execute() throws FlickrException {
return flickr.getPeopleInterface().getPhotos(
userNSID,
null, null, null, null, null,
Flickr.CONTENTTYPE_PHOTO, null,
Extras.ALL_EXTRAS, 100, page);
}
});
}
//Catch the execption inside the function for failure to get exif
public Collection<Exif> getPhotoExif(Photo photo) throws FlickrException, FlickrRuntimeException {
return doFlickrAction(new CallableFlickrTask<Collection<Exif>>(){
@Override
public Collection<Exif> execute() throws FlickrException {
return flickr.getPhotosInterface().getExif(photo.getId(),photo.getSecret());
}
});
}
public Collection<Size> getAvailablePhotoSizes(Photo photo) throws FlickrException{
return doFlickrAction(new CallableFlickrTask<Collection<Size>>(){
@Override
public Collection<Size> execute() throws FlickrException {
return flickr.getPhotosInterface().getSizes(photo.getId());
}
});
}
private abstract class CallableFlickrTask<T> {
public abstract T execute() throws FlickrException, FlickrRuntimeException;
}
private <T> T doFlickrAction(CallableFlickrTask<T> callable) throws FlickrException {
while(true){
rateLimiter.acquire();
Future<T> future = executorService.submit(new Callable<T>() {
@Override
public T call() throws Exception {
return callable.execute();
}});
try {
return future.get(5, TimeUnit.MINUTES);
} catch (InterruptedException e) {
LOG.log(Level.INFO,"Interrupted exception: {0}",e.getMessage());
initialize(); //initialize if it's been interupted
} catch (ExecutionException e) {
Throwable cause = e.getCause();
if( cause instanceof UnknownHostException ||
cause instanceof SocketException ||
cause instanceof OAuthConnectionException ){
//sleep and retry
LOG.log(Level.INFO,"Unknown Host or Socket exception. Retry: {0}",e.getMessage());
try {
Thread.sleep(10000);
initialize();
} catch (InterruptedException ex) {
LOG.log(Level.INFO, "Thread sleep was interrupted exception: {0}", ex.getMessage());
}
}
//if it's not of the above exceptions, then rethrow
else if (cause instanceof FlickrException) {
throw (FlickrException) cause;
}
else {
throw new IllegalStateException(e);
}
} catch (TimeoutException e) {
LOG.log(Level.INFO,"Timeout Exception: {0}",e.getMessage());
initialize(); //initialize again after timeout
}
}
}
}
我还使用了jvisualvm来查看集合在挂起时正在做什么。线程转储在此处：Thread dump