我正在尝试提取大小约为500MB且包含大约250K文件的zip文件的内容。
这是我想要做的事情 -
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import de.schlichtherle.truezip.file.TFile;
import de.schlichtherle.truezip.file.TFileInputStream;
public class ArchiveReaderExecutor {
private final ExecutorService pool;
public ArchiveReaderExecutor() {
pool = Executors.newFixedThreadPool(8);
}
/**
* Splits the archive file into list of lists as provided in the batch size
* variable
*
* @param archive
*
* @return
*/
public List<List<TFile>> splitArchiveFile(final File archive) {
final TFile tFile = new TFile(archive.getAbsolutePath());
final ArrayList<TFile> individualFiles = new ArrayList<TFile>();
recursivelyReadLeafnodes(tFile, individualFiles);
final List<List<TFile>> returnList = new ArrayList<List<TFile>>();
/*
* Splitting the entire list into list of objects for batch processing
*/
int count = 0;
List<TFile> innerList = null;
for (TFile splitFile : individualFiles) {
if (count == 0) {
innerList = new ArrayList<TFile>();
returnList.add(innerList);
}
if (count < 100) {
++count;
} else {
count = 0;
}
innerList.add(splitFile);
}
return returnList;
}
public List<TFile> recursivelyReadLeafnodes(TFile inputTFile,
ArrayList<TFile> individualFiles) {
TFile[] tfiles = null;
if (inputTFile.isArchive() || inputTFile.isDirectory()) {
tfiles = inputTFile.listFiles();
} else {
tfiles = new TFile[0];
tfiles[0] = inputTFile;
}
for (final TFile tFile : tfiles) {
if (tFile.isFile() && !tFile.getName().startsWith(".")) {
individualFiles.add(tFile);
} else if (tFile.isDirectory()) {
recursivelyReadLeafnodes(tFile, individualFiles);
}
}
return individualFiles;
}
public void runExtraction() {
File src = new File("Really_Big_File.zip");
List<List<TFile>> files = splitArchiveFile(src);
for (List<TFile> list : files) {
pool.execute(new FileExtractorSavor(list));
}
pool.shutdown();
}
class FileExtractorSavor implements Runnable{
List<TFile> files;
public FileExtractorSavor(List<TFile> files) {
this.files = files;
}
@Override
public void run() {
File file = null;
TFileInputStream in = null;
for (TFile tFile : files) {
try {
in = new TFileInputStream(tFile);
file = new File("Target_Location"+tFile.getName());
FileUtils.writeStringToFile(file, IOUtils.toString(in));
} catch (IOException e) {
e.printStackTrace();
} finally {
IOUtils.closeQuietly(in);
}
}
}
}
public static void main(String[] args) {
new ArchiveReaderExecutor().runExtraction();
}
}
当我同时运行此代码时,有很多线程处于等待/阻塞状态,这里是线程转储:
"pool-1-thread-7" prio=5 tid=7fd8093dd000 nid=0x11d3f3000 waiting for monitor entry [11d3f2000]
java.lang.Thread.State: BLOCKED (on object monitor)
at de.schlichtherle.truezip.socket.ConcurrentInputShop$SynchronizedConcurrentInputStream.close(ConcurrentInputShop.java:223)
- waiting to lock <785460200> (a de.schlichtherle.truezip.fs.archive.FsDefaultArchiveController$Input)
at de.schlichtherle.truezip.io.DecoratingInputStream.close(DecoratingInputStream.java:79)
at org.apache.commons.io.IOUtils.closeQuietly(IOUtils.java:178)
at ArchiveReaderExecutor$FileExtractorSavor.run(ArchiveReaderExecutor.java:136)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918)
at java.lang.Thread.run(Thread.java:695)
Locked ownable synchronizers:
- <79ed370e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
"pool-1-thread-5" prio=5 tid=7fd8093db800 nid=0x11d1ed000 waiting for monitor entry [11d1ec000]
java.lang.Thread.State: BLOCKED (on object monitor)
at de.schlichtherle.truezip.socket.ConcurrentInputShop$SynchronizedConcurrentInputStream.close(ConcurrentInputShop.java:223)
- waiting to lock <785460200> (a de.schlichtherle.truezip.fs.archive.FsDefaultArchiveController$Input)
at de.schlichtherle.truezip.io.DecoratingInputStream.close(DecoratingInputStream.java:79)
at org.apache.commons.io.IOUtils.closeQuietly(IOUtils.java:178)
at ArchiveReaderExecutor$FileExtractorSavor.run(ArchiveReaderExecutor.java:136)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:895)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:918)
at java.lang.Thread.run(Thread.java:695)
Locked ownable synchronizers:
- <79ed46468> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)
我也尝试过使用:
TFile.cp_r(src, dst, TArchiveDetector.NULL, TArchiveDetector.NULL);
因为它在单个线程上运行所花费的时间更长。
我的问题是,使用TrueZip在java中提取zip文件内容的快速,最佳和最佳方法是什么?
答案 0 :(得分:1)
这里没有错。 TrueZIP / TrueVFS为每个已安装的归档文件维护单个文件描述符。当多个线程同时读取归档文件的内容时,TrueZIP / TrueVFS内核序列化所有访问,以便只有一个线程正在使用文件描述符并随时更新其位置。所有其他线程将同时被阻止。