如何为orientdb嵌入式设置jmh测试？

Question

我正在尝试为orient db embedded设置jmh测试。测试套件如下：

@State(Scope.Benchmark)
public class OrientDbTest {
    private OObjectDatabaseTx db;
    private Person[] personList;

    @Setup
    public void setUp() throws IOException {
        deleteDir("/tmp/orientdb/");
        db = new OObjectDatabaseTx("plocal:/tmp/orientdb/person").create();
        ODatabaseRecordThreadLocal.INSTANCE.set(db.getUnderlying());
        loadData();
    }

    @TearDown
    public void cleanUp() {
        if (db != null) {
            ODatabaseRecordThreadLocal.INSTANCE.set(db.getUnderlying());
            db.commit();
            db.drop();
            db.close();
        }
    }

    @Benchmark
    @BenchmarkMode(Mode.AverageTime)
    @OutputTimeUnit(TimeUnit.MICROSECONDS)
    public void benchmarkInsertCompany() {
        ODatabaseRecordThreadLocal.INSTANCE.set(db.getUnderlying());
        db.getEntityManager().registerEntityClass(Person.class);

        for (Person person : personList) {
            db.save(person);
        }
    }

    void loadData() throws IOException {
        InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream("data.json");
        ObjectMapper objectMapper = new ObjectMapper();
        personList = objectMapper.readValue(inputStream, Person[].class);
    }

    void deleteDir(String dirName) {
        File file = new File(dirName);
        if (file.exists()) {
            File[] files = file.listFiles();
            if (files != null) {
                for (File child : files) {
                    if (child.isDirectory()) {
                        deleteDir(child.getAbsolutePath());
                    } else {
                        child.delete();
                    }
                }
            } else {
                file.delete();
            }
        }
    }
}

这个项目是一个gradle项目，我正在使用gradle-jmh插件运行。以下是build.gradle文件中的jmh设置：

jmh {
    jmhVersion = '1.14'
    iterations = 10 // Number of measurement iterations to do.
    fork = 2 // How many times to forks a single benchmark. Use 0 to disable forking altogether
    jvmArgs = '-server -XX:MaxDirectMemorySize=15986m'
    resultsFile = project.file("${project.buildDir}/reports/jmh/results.txt") // results file
    profilers = ['cl', 'gc', 'hs_thr'] // Use profilers to collect additional data. Supported profilers: [cl, comp, gc, stack, perf, perfnorm, perfasm, xperf, xperfasm, hs_cl, hs_comp, hs_gc, hs_rt, hs_thr]
    resultFormat = 'CSV' // Result format type (one of CSV, JSON, NONE, SCSV, TEXT)
    threads = 4 // Number of worker threads to run with.
    timeUnit = 'ms' // Output time unit. Available time units are: [m, s, ms, us, ns].
    warmupForks = 2 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks.
    warmupIterations = 10 // Number of warmup iterations to do.
}

当我运行测试时，我收到以下错误：

INFO: OrientDB auto-config DISKCACHE=10,384MB (heap=3,554MB direct=15,986MB os=15,986MB)
<failure>

com.orientechnologies.orient.core.exception.OStorageExistsException: Cannot create new storage 'plocal:/tmp/orientdb/person' because it is not closed
        DB name="person"
        at com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.create(OAbstractPaginatedStorage.java:423)
        at com.orientechnologies.orient.core.storage.impl.local.paginated.OLocalPaginatedStorage.create(OLocalPaginatedStorage.java:125)
        at com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx.create(ODatabaseDocumentTx.java:429)
        at com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx.create(ODatabaseDocumentTx.java:389)
        at com.orientechnologies.orient.core.db.ODatabaseWrapperAbstract.create(ODatabaseWrapperAbstract.java:75)
        at org.dizitart.no2.benchmark.OrientDbTest.setUp(OrientDbTest.java:24)
        at org.dizitart.no2.benchmark.generated.OrientDbTest_benchmarkInsertCompany_jmhTest._jmh_tryInit_f_orientdbtest0_G(OrientDbTest_benchmarkInsertCompany_jmhTest.java:400)
        at org.dizitart.no2.benchmark.generated.OrientDbTest_benchmarkInsertCompany_jmhTest.benchmarkInsertCompany_AverageTime(OrientDbTest_benchmarkInsertCompany_jmhTest.java:149)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:430)
        at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:412)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

我在这里的设置中做错了什么？

修改：

阅读this后，我按如下方式修改了代码：

public class OrientDbTest {

    @State(Scope.Benchmark)
    public static class TestState {
        private OObjectDatabaseTx db;
        private Person[] personList;
        private BenchmarkTestHelper testHelper = new BenchmarkTestHelper();

        @Setup(Level.Trial)
        public void setUp() throws IOException {
            System.out.println("started setup code");
            testHelper.deleteDir("/tmp/orientdb/");

            try {
                db = new OObjectDatabaseTx("plocal:/tmp/orientdb/person").open(null, null);
                db.getEntityManager().registerEntityClass(Person.class);
                personList = testHelper.loadData();
            } finally {
                if (db != null) {
                    db.close();
                }
            }
        }

        @TearDown(Level.Trial)
        public void cleanUp() {
            System.out.println("started cleanup code");
            if (db != null) {
                ODatabaseRecordThreadLocal.INSTANCE.set(db.getUnderlying());
                db.commit();
                db.drop();
                db.close();
            }
        }
    }

    @Benchmark
    @BenchmarkMode(Mode.AverageTime)
    @OutputTimeUnit(TimeUnit.MICROSECONDS)
    public void benchmarkInsertCompany(TestState state, Blackhole blackhole) {
        OObjectDatabaseTx db = state.db;
        Person[] personList = state.personList;

        ODatabaseRecordThreadLocal.INSTANCE.set(db.getUnderlying());
        for (Person person : personList) {
            blackhole.consume(db.save(person));
        }
    }
}


class BenchmarkTestHelper {
    Person[] loadData() throws IOException {
        InputStream inputStream = Thread.currentThread().getContextClassLoader().getResourceAsStream("data.json");
        ObjectMapper objectMapper = new ObjectMapper();
        return objectMapper.readValue(inputStream, Person[].class);
    }

    void deleteDir(String dirName) {
        File file = new File(dirName);
        if (file.exists()) {
            File[] files = file.listFiles();
            if (files != null) {
                for (File child : files) {
                    if (child.isDirectory()) {
                        deleteDir(child.getAbsolutePath());
                    } else {
                        child.delete();
                    }
                }
                file.delete();
            } else {                   
                file.delete();
            }
        }
    }
}

在新设置之后，我收到以下错误：

# JMH 1.14 (released 19 days ago)
# VM version: JDK 1.8.0_77, VM 25.77-b03
# VM invoker: /home/anindya/app/jdk1.8.0_77/jre/bin/java
# VM options: -server -XX:MaxDirectMemorySize=15986m
# Warmup: 10 iterations, 1 s each
# Measurement: 10 iterations, 1 s each
# Timeout: 10 min per iteration
# Threads: 4 threads, will synchronize iterations
# Benchmark mode: Average time, time/op
# Benchmark: org.dizitart.no2.benchmark.OrientDbTest.benchmarkInsertCompany

# Run progress: 50.00% complete, ETA 00:01:28
# Warmup Fork: 1 of 2
# Warmup Iteration   1: started setup code
Sep 26, 2016 11:15:57 AM com.orientechnologies.common.log.OLogManager log
INFO: OrientDB auto-config DISKCACHE=10,384MB (heap=3,554MB direct=15,986MB os=15,986MB)
started setup code
started setup code
started setup code
<failure>

com.orientechnologies.orient.core.exception.OStorageExistsException: Cannot create new storage 'plocal:/tmp/orientdb/person' because it is not closed
        DB name="person"
        at com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage.create(OAbstractPaginatedStorage.java:423)
        at com.orientechnologies.orient.core.storage.impl.local.paginated.OLocalPaginatedStorage.create(OLocalPaginatedStorage.java:125)
        at com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx.create(ODatabaseDocumentTx.java:429)
        at com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx.create(ODatabaseDocumentTx.java:389)
        at com.orientechnologies.orient.core.db.ODatabaseWrapperAbstract.create(ODatabaseWrapperAbstract.java:75)
        at org.dizitart.no2.benchmark.OrientDbTest$TestState.setUp(OrientDbTest.java:28)
        at org.dizitart.no2.benchmark.generated.OrientDbTest_benchmarkInsertCompany_jmhTest._jmh_tryInit_f_teststate1_G(OrientDbTest_benchmarkInsertCompany_jmhTest.java:409)
        at org.dizitart.no2.benchmark.generated.OrientDbTest_benchmarkInsertCompany_jmhTest.benchmarkInsertCompany_AverageTime(OrientDbTest_benchmarkInsertCompany_jmhTest.java:153)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:430)
        at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:412)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)

...

# Run progress: 87.50% complete, ETA 00:00:18
# Fork: 2 of 2
# Warmup Iteration   1: started setup code
Sep 26, 2016 11:16:38 AM com.orientechnologies.common.log.OLogManager log
INFO: OrientDB auto-config DISKCACHE=10,384MB (heap=3,554MB direct=15,986MB os=15,986MB)
started setup code
started setup code
started setup code
<failure>

java.lang.ClassCastException: com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx cannot be cast to com.orientechnologies.orient.object.db.OObjectDatabaseTx
        at org.dizitart.no2.benchmark.OrientDbTest$TestState.setUp(OrientDbTest.java:28)
        at org.dizitart.no2.benchmark.generated.OrientDbTest_benchmarkInsertCompany_jmhTest._jmh_tryInit_f_teststate1_G(OrientDbTest_benchmarkInsertCompany_jmhTest.java:409)
        at org.dizitart.no2.benchmark.generated.OrientDbTest_benchmarkInsertCompany_jmhTest.benchmarkInsertCompany_AverageTime(OrientDbTest_benchmarkInsertCompany_jmhTest.java:153)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:430)
        at org.openjdk.jmh.runner.BenchmarkHandler$BenchmarkTask.call(BenchmarkHandler.java:412)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)


Sep 26, 2016 11:16:49 AM com.orientechnologies.common.log.OLogManager log
INFO: Orient Engine is shutting down...
Sep 26, 2016 11:16:49 AM com.orientechnologies.common.log.OLogManager log
INFO: - shutdown storage: person...

Answer 1

好的，所以有几个麻烦：

a）input { couchdb_changes { db => "users" host => "localhost" port => "5984" } } output { elasticsearch { hosts => ["127.0.0.1:9200"] index => "playtym" } } 方法本身就是错误的，它抛出：

setUp()

b）java.lang.ClassCastException: com.orientechnologies.orient.core.db.document.ODatabaseDocumentTx cannot be cast to com.orientechnologies.orient.object.db.OObjectDatabaseTx at jmh.demo.OrientDbTest$TestState.setUp(OrientDbTest.java:30) at jmh.demo.generated.OrientDbTest_benchmarkInsert_jmhTest._jmh_tryInit_f_teststate1_G(OrientDbTest_benchmarkInsert_jmhTest.java:409) at jmh.demo.generated.OrientDbTest_benchmarkInsert_jmhTest.benchmarkInsert_AverageTime(OrientDbTest_benchmarkInsert_jmhTest.java:153) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ... 的{​​{1}}方法应该只执行一次。但是，由于bug in JMH，如果第一个线程因上述异常而失败，那么其他线程也尝试输入@Setup，然后他们将尝试打开数据库，但是失败了，因为它已经开放了？

事实上，即使在原始帖子中，失败（a）也是可见的，并且从多个线程进入@State(Benchmark)报告的异常将取决于纯粹的运气（这在JMH 1.14.1中变得更好）。如果设置threads = 1，它会以正确的异常可靠地失败。此外，@Setup中的清理路径与@Setup不一致，这可以解释为什么重新输入的线程无法打开数据库。

底线：在进行多线程测试之前，尝试执行单线程。

Answer 2

db.drop();未执行，因此您收到错误：Cannot create new storage because it is not closed

Answer 3

Aleksey的回答帮助我弄清楚在测试中使用 fork 是错误的。我在这里发布了工作代码，以避免在问题中更加混乱。

public class OrientDbTest {

    @State(Scope.Benchmark)
    public static class TestState {
        private OObjectDatabaseTx db;
        private Person[] personList;

        @Setup(Level.Trial)
        public void setUp() throws IOException {
            System.out.println("started setup code");
            try {
                personList = loadData();
                deleteDir("/tmp/orientdb/");
                db = new OObjectDatabaseTx("plocal:/tmp/orientdb/person");
                if (db.exists()) {
                    db.open("admin", "admin");
                    db.drop();
                }
                db.create();
                db.getEntityManager().registerEntityClass(Person.class);
                db.getEntityManager().registerEntityClass(Address.class);
                db.getEntityManager().registerEntityClass(PrivateData.class);
            } catch (Throwable e) {
                System.out.println("error in creating db ");
                e.printStackTrace();
            }
        }

        @TearDown(Level.Trial)
        public void cleanUp() {
            System.out.println("started cleanup code");
            if (db != null) {
                ODatabaseRecordThreadLocal.INSTANCE.set(db.getUnderlying());
                db.commit();
                db.close();
            }
        }

        private void deleteDir(String dirName) {
            File file = new File(dirName);
            if (file.exists()) {
                File[] files = file.listFiles();
                if (files != null) {
                    for (File child : files) {
                        if (child.isDirectory()) {
                            deleteDir(child.getAbsolutePath());
                        } else {
                            child.delete();
                        }
                    }
                    file.delete();
                } else {
                    file.delete();
                }
            }
        }

        private Person[] loadData() throws IOException {
            InputStream inputStream = Thread.currentThread()
                    .getContextClassLoader().getResourceAsStream("data.json");
            ObjectMapper objectMapper = new ObjectMapper();
            return objectMapper.readValue(inputStream, Person[].class);
        }
    }

    @Benchmark
    @BenchmarkMode(Mode.AverageTime)
    @OutputTimeUnit(TimeUnit.MICROSECONDS)
    @Fork(0)
    public void benchmarkInsert(TestState state, Blackhole blackhole) {
        OObjectDatabaseTx db = state.db;
        Person[] personList = state.personList;

        if (db == null) {
            System.out.println("db null.. exiting");
            System.exit(0);
        }

        ODatabaseRecordThreadLocal.INSTANCE.set(db.getUnderlying());
        for (Person person : personList) {
            blackhole.consume(db.save(person));
        }
    }