Question

当我尝试对目录中的文件数量和Java中的内存映射文件进行一些性能测试时，我遇到了一些我无法解释的奇怪结果。我在Windows 10，Java 8上运行了这个测试。

1.在每个目录中添加100,1000,10000个文件时，结果非常一致。但是当我将所有300,000个文件存储在一个目录中时，结果慢了2-10倍。嗯，这就是我想要测试的，所以至少我们可以说在一个目录中放置太多文件会影响内存映射文件的性能。但是，摆动过多，具体取决于运行这些buildFiles方法的顺序。

2.使用内存映射文件创建文件以写入4k数据比单独打开文件（checkFiles方法）更快，而不做任何事情。这毫无意义。一种可能的解释可能是操作系统延迟编写元数据。

我希望有人可以在我的Linus盒子上进行测试。

在尝试运行测试之前，请检查数据目录，这样就不会覆盖任何数据文件。每次调用buildFiles方法也可以向磁盘添加300,000个文件，因此请确保您有足够的空间。

package eval.files;

import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.text.NumberFormat;
import java.util.Random;

public class DirFileLimitTest {
    Path testDir = Paths.get("/AppData/"+getClass().getPackage().getName());
    int dataSize = 4096;
    byte[] data = new byte[dataSize];
    int totalFiles = 300_000;
    NumberFormat f = NumberFormat.getIntegerInstance();

    boolean memoryMap = true;

    public static void main(String...as) throws Exception {
        new DirFileLimitTest().runTest();
    }

    DirFileLimitTest() {
        new Random().nextBytes(data);
    }

    void runTest() throws Exception {
        clearDir(testDir, false);

        buildFiles(100);
//      buildFiles(1000);
//      buildFiles(10000);
        buildFiles(totalFiles);

        checkFiles(totalFiles);
//      checkFiles(10000);
//      checkFiles(1000);
        checkFiles(100);
    }

    void buildFiles(int maxChild) throws IOException {
        Path dir = testDir.resolve("child"+maxChild);
        ensureDir(dir);
        int[] levelFiles = computeLevelFiles(maxChild);

        long start = System.currentTimeMillis();
        for(int i=0; i<totalFiles; i++) {
            buildFile(dir, levelFiles, i);
        }
        System.out.println("buildFiles("+maxChild+"): 
        "+f.format(System.currentTimeMillis()-start));
    }

    void buildFile(Path root, int[] levelFiles, int fn) throws IOException {
        Path filePath = resolve(root, levelFiles, fn);
        ensureDir(filePath.getParent());

        FileChannel fileChannel = FileChannel.open(filePath,
                StandardOpenOption.READ, StandardOpenOption.WRITE,
                StandardOpenOption.CREATE);

        if(memoryMap) {
            MappedByteBuffer mem = 
               fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, dataSize);
            mem.put(data);
        }
        else
            fileChannel.write(ByteBuffer.wrap(data));
        fileChannel.close();
    }


    void checkFiles(int maxChild) throws IOException {
        Path dir = testDir.resolve("child"+maxChild);
        int[] levelFiles = computeLevelFiles(maxChild);

        long start = System.currentTimeMillis();
        for(int i=0; i<totalFiles; i++) {
            checkFile(dir, levelFiles, i);
        }
        System.out.println("checkFiles("+maxChild+"): 
            "+f.format(System.currentTimeMillis()-start));
    }

    void checkFile(Path root, int[] levelFiles, int fn) throws IOException {
        Path filePath = resolve(root, levelFiles, fn);

        FileChannel fileChannel = FileChannel.open(filePath,
                StandardOpenOption.READ, StandardOpenOption.WRITE,
                StandardOpenOption.CREATE);
        fileChannel.close();
    }

    int[] computeLevelFiles(int maxChild) {
        int levels = 1;
        int childCount = totalFiles;
        while(childCount > maxChild) {
            childCount = childCount / maxChild;
            levels++;
        }
        int[] levelFiles = new int[levels];
        int files = 1;
        for(int l=levels-1; l>=0; l--) {
            levelFiles[l] = files;
            files *= maxChild;
        }
        return levelFiles;
    }

    Path resolve(Path root, int[] levelFiles, int fn) {
        Path path = root;
        int remainder = fn;
        for(int level=0; level<levelFiles.length; level++) {
            path = 
               path.resolve(Integer.toString(remainder/levelFiles[level]));
            remainder = remainder % levelFiles[level];
        }
        return path;
    }

    public static void clearDir(Path dirPath, boolean deleteSelf) {
        try {
            Files.walkFileTree(dirPath, new SimpleFileVisitor<Path>() {
                @Override
                public FileVisitResult visitFile(Path file, 
BasicFileAttributes attrs) throws IOException {
                    Files.delete(file);
                    return FileVisitResult.CONTINUE;
                }

                @Override
                public FileVisitResult postVisitDirectory(Path dir, 
IOException exc) throws IOException {
                    if(dir != dirPath || deleteSelf)
                        Files.delete(dir);
                    return FileVisitResult.CONTINUE;
                }
            });
        } catch(IOException x) {
            throw new IllegalStateException("Cannot clear: "+dirPath, x);
        }
    }
    public static File ensureDir(Path dirPath) {
        File dir = dirPath.toFile();
        if(!dir.exists()) {
            if(!dir.mkdirs())
                throw new IllegalStateException("Cannot create directory: 
             "+dir);
        }
        else {
            if(dir.isFile())
                throw new IllegalArgumentException("Not a directory: "+dir);
        }
        return dir;
    }

}

目录中的文件数如何影响Java内存映射文件的性能？

0 个答案: