我正在研究将大量对象排队的一对类，然后使用Java Serialization / ObjectOutputStream和ObjectInputStream允许对该队列进行迭代。队列中需要的数据量超过50GB，因此需要文件IO。问题在于，写入需要2毫秒的文件需要大约2秒钟才能读取。我已经尝试过BufferedInputStream，并且每个阅读器都有一个专用线程来保持输入缓冲区已满。
private File file;
private FileInputStream fileStream;
private BufferedInputStream buffer;
private ObjectInputStream stream;
......
this.file = file;
this.fileStream = new FileInputStream(this.file);
this.buffer = new BufferedInputStream(this.fileStream, 2^20); //Tried multiple depths, does not change behavior
this.stream = new ObjectInputStream(this.buffer);
.....
long startTime = System.currentTimeMillis();
LinkedList<T> list = (LinkedList<T>) stream.readUnshared();
long endTime = System.currentTimeMillis();
totalReadTime += endTime - startTime;
填充序列化文件的类正在插入具有10000个元素的LinkedLists。使用Integers进行测试时，stream.readUnshared（）调用的平均时间为214.7ms。为了进行比较，任何给定测试的总写入时间最多为3毫秒，其中测试通常向文件中写入10倍的10,000个元素的LinkedList。
我已经包括了所有源代码和性能测试程序。这是我的示例输出：
Hello
This
Is
A
Total time spent reading: 0
Final
Test
Average time spent reading: 0.0
Starting test: 0
Threads Created.
Total write time: 3
Total time spent reading: 2256
Average time spent reading: 225.6
Total read time: 2030
Threads Joined. Total time: 2265
Starting test: 1
Threads Created.
Total write time: 2
Total time spent reading: 2147
Average time spent reading: 214.7
Total read time: 1961
Threads Joined. Total time: 2153
Starting test: 2
Threads Created.
Total write time: 1
Total time spent reading: 2143
Average time spent reading: 214.3
Total read time: 1957
Threads Joined. Total time: 2149
我不是专业人士，可以接受建设性的反馈。谢谢您的关注。
//Serial Object Writer Class
package utils;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.LinkedList;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
public class SerialObjFileCollector<T extends Serializable> extends Thread {
private ObjectOutputStream stream;
private FileOutputStream fileStream;
private File tempFile;
private AtomicBoolean isOpen;
private AtomicInteger flushCount;
private AtomicLong count;
private LinkedList<T> objects;
private int sendThreshold = 10000;
private ConcurrentLinkedQueue<LinkedList<T>> sendQueue;
public SerialObjFileCollector() {
this.initialize();
}
private void initialize() {
try {
tempFile = File.createTempFile("binary_data", ".SerialObjFileCollector");
tempFile.deleteOnExit();
fileStream = new FileOutputStream(tempFile);
stream = new ObjectOutputStream(fileStream);
isOpen = new AtomicBoolean(true);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
isOpen = new AtomicBoolean(false);
}
objects = new LinkedList<T>();
sendQueue = new ConcurrentLinkedQueue<LinkedList<T>>();
flushCount = new AtomicInteger(0);
count = new AtomicLong(0);
this.start();
}
public void writeObject(T object) {
if(!isOpen.compareAndExchange(false, false)) throw new UnsupportedOperationException("Cannot write objects to a closed SerialObjFileCollector!");
objects.add(object);
if(objects.size() > sendThreshold) {
this.sendQueue.add(objects);
synchronized (sendQueue) {sendQueue.notifyAll();}
objects = new LinkedList<T>();
}
}
public void writeObjects(Iterable<T> objects) {
for(T object : objects) {
this.writeObject(object);
}
}
public long size() {
return count.get();
}
public SerialObjFileReader<T> getReader() {
if(isOpen.compareAndExchange(true, true)) {
flush(true);
try {
fileStream.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
return new SerialObjFileReader<T>(tempFile,count.get(),sendThreshold);
}
public void flush() {
this.flush(false);
}
private void flush(boolean close) {
this.sendQueue.add(objects);
flushCount.incrementAndGet();
isOpen.set(!close);
objects = new LinkedList<T>();
try {
synchronized (sendQueue) {sendQueue.notifyAll();}
synchronized (flushCount) {flushCount.wait();}
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void run(){
//Keep the thread open even when closing and flushing
while(isOpen.compareAndExchange(true, true) || flushCount.get() > 0) {
if(sendQueue.isEmpty() && flushCount.get() == 0) try {
synchronized (sendQueue) {sendQueue.wait();}
} catch (InterruptedException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
//Write out the Queue
while(!this.sendQueue.isEmpty()) {
LinkedList<T> toSend = sendQueue.poll();
try {
stream.writeUnshared(toSend);
stream.reset();
count.addAndGet(toSend.size());
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
};
}
//Flush the streams
if(flushCount.get() > 0) {
try {
stream.flush();
fileStream.flush();
flushCount.decrementAndGet();
synchronized (flushCount) {flushCount.notifyAll();}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
synchronized (flushCount) {flushCount.notifyAll();}
}
}
//Serial Object Reader Class
package utils;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.Serializable;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicLong;
public class SerialObjFileReader<T extends Serializable> implements Iterable<T> {
private File file;
private long count;
private int maxSize;
public SerialObjFileReader(File file, long count, int maxSize) {
this.initialize(file,count,maxSize);
}
private void initialize(File file, long count, int maxSize) {
this.file = file;
this.count = count;
this.maxSize = maxSize;
}
@Override
public Iterator<T> iterator() {
return new SerialObjFileIterator<T>(file,count,maxSize);
}
public long size() {
return count;
}
public static class SerialObjFileIterator<T> extends Thread implements Iterator<T> {
private File file;
private FileInputStream fileStream;
private BufferedInputStream buffer;
private ObjectInputStream stream;
private long count;
private int maxSize;
private AtomicLong dequeCount;
private Iterator<T> localIterator;
private ConcurrentLinkedQueue<Iterator<T>> rcvQueue;
private long totalReadTime = 0;
private int readCount = 0;
public SerialObjFileIterator(File file, long count, int maxSize) {
this.initialize(file,count,maxSize);
}
private void initialize(File file, long count, int maxSize) {
this.count = 0;
try {
this.file = file;
this.fileStream = new FileInputStream(this.file);
this.buffer = new BufferedInputStream(this.fileStream, 2^20);
this.stream = new ObjectInputStream(this.buffer);
this.count = count;
this.maxSize = maxSize;
this.dequeCount = new AtomicLong(count);
this.rcvQueue = new ConcurrentLinkedQueue<Iterator<T>>();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if(this.count > 0) localIterator = readNewIterator();
this.start();
}
@Override
public boolean hasNext() {
return remaining() > 0 || (localIterator == null ? false : localIterator.hasNext()); //Probably some redundancy here
}
@Override
public T next() {
T result = localIterator.next();
count--;
if(!localIterator.hasNext() && count > 0) localIterator = pullNewIterator();
return result;
}
private Iterator<T> pullNewIterator() {
synchronized (rcvQueue) {rcvQueue.notifyAll();} //Wake the thread regardless
if(!rcvQueue.isEmpty()) {
return rcvQueue.poll();
} else if (count > 0){
try {
synchronized (dequeCount) {dequeCount.wait();}
return rcvQueue.poll();
} catch (InterruptedException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
return null;
}
private Iterator<T> readNewIterator() {
if(dequeCount.get() > 0) try {
long startTime = System.currentTimeMillis();
LinkedList<T> list = (LinkedList<T>) stream.readUnshared();
long endTime = System.currentTimeMillis();
totalReadTime += endTime - startTime;
readCount++;
dequeCount.set( Math.max(dequeCount.get() - maxSize, 0));
return list.iterator();
} catch (ClassNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
System.exit(1);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
System.exit(1);
}
return null;
}
public long remaining() {
return count;
}
public void run() {
while(dequeCount.get() > 0) {
while(rcvQueue.size() < 3 && dequeCount.get() > 0) {
rcvQueue.add(readNewIterator());
}
synchronized (dequeCount) {dequeCount.notifyAll();} //Wake the thread regardless
if(dequeCount.get() > 0) try {
synchronized (rcvQueue) {rcvQueue.wait();}
} catch (InterruptedException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
System.out.println("\t\t\tTotal time spent reading: " + totalReadTime);
System.out.println("\t\t\tAverage time spent reading: " + ((double) totalReadTime / ((double) readCount)));
}
}
}
//Serial Object Performance Test Class
package utils;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.concurrent.atomic.AtomicReference;
public class SerialObjTester {
public static void main(String[] args) {
SerialObjFileCollector<String> collector = new SerialObjFileCollector<String>();
collector.writeObject("Hello");
collector.writeObject("This");
collector.writeObject("Is");
collector.writeObject("A");
collector.writeObject("Final");
collector.writeObject("Test");
for(String str : collector.getReader()) {
System.out.println(str);
}
//Throughput Tests
final int LIST_SIZE = 100000;
final int WRITE_COUNT = 100000;
SerialObjFileCollector<LinkedList<Integer>> intCollector = new SerialObjFileCollector<LinkedList<Integer>>();
LinkedList<Integer> intList = new LinkedList<Integer>();
for(int count = 0; count < LIST_SIZE; count++) intList.add(count);
//Populate initial collector
for(int count = 0; count < WRITE_COUNT; count++) intCollector.writeObject(intList);
AtomicReference<SerialObjFileCollector<LinkedList<Integer>>> intCollectorRef = new AtomicReference<SerialObjFileCollector<LinkedList<Integer>>>(intCollector);
for(int testCount = 0; testCount < 100; testCount++) {
System.out.println("Starting test: " + testCount);
SerialObjFileReader<LinkedList<Integer>> intReader = intCollectorRef.get().getReader();
Thread readerThread = new Thread() {
public void run() {
Iterator<LinkedList<Integer>> intListIter = intReader.iterator();
LinkedList<Integer> list = null;
final long startTime = System.currentTimeMillis();
while(intListIter.hasNext()) {
list = intListIter.next();
}
final long endTime = System.currentTimeMillis();
int testInt = 0;
for(int compareInt : list) if(compareInt != testInt++) System.err.println("\t\tInteger List Inconsistency found!");
System.out.println("\tTotal read time: " + (endTime - startTime) );
}
};
Thread writerThread = new Thread() {
public SerialObjFileCollector<LinkedList<Integer>> localIntCollector;
public void run() {
localIntCollector = new SerialObjFileCollector<LinkedList<Integer>>();
final long startTime = System.currentTimeMillis();
for(int count = 0; count < WRITE_COUNT; count++) localIntCollector.writeObject(intList);
final long endTime = System.currentTimeMillis();
intCollectorRef.set(localIntCollector);
System.out.println("\tTotal write time: " + (endTime - startTime) );
}
};
System.out.println("\tThreads Created.");
final long startTime = System.currentTimeMillis();
readerThread.start();
writerThread.start();
try {
readerThread.join();
writerThread.join();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
final long endTime = System.currentTimeMillis();
System.out.println("\tThreads Joined. Total time: " + (endTime - startTime) );
}
}
}