Monet DB JDBC并行executeBatch失败

时间：2015-10-31 11:35:32

标签： jdbc parallel-processing monetdb

在我的应用程序中，我想并行地在不同的表中插入多行。为此，我在每个线程中创建预备语句，并使用具有10K批量大小的exceuteBatch。我将自动提交设为false。在每个executeBatch之后，我使用connection.commit提交事务。在单线程中，这段代码工作正常，但在多线程中，当它开始插入不同的表（每个线程中不同的表）时，就会出现提交失败异常。 请指导如何进行并行插入（请注意，所有在不同表中工作的线程彼此之间没有链接）。

谢谢， 维卡斯

1 个答案:

答案 0 :(得分：0)

问题

这是我为从monetDB驱动程序中获取并发冲突错误而创建的代码。

public static class TestRunner implements Runnable {
private final static String problemHere = "jdbc:monetdb://localhost/test-db";

static {
    try {
    Class.forName("nl.cwi.monetdb.jdbc.MonetDriver");
    } catch (ClassNotFoundException e) {
    // TODO Auto-generated catch block
    e.printStackTrace();
    }
}

private static Connection createConnection() throws SQLException {
    // happens one time per thread
    return DriverManager.getConnection(problemHere, "myID", "myPass");
}

private static int fieldCount = 10;
private static int[] colType = new int[fieldCount];
private static final int[] types = { Types.VARCHAR, Types.TIMESTAMP, Types.DECIMAL };

static {
    Random random = new Random();
    // initialize column types
    for (int i = 0; i < fieldCount; i++) {
    colType[i] = types[random.nextInt(types.length)];
    }
}

private final String name;
private Connection con;
private int batchCount;
private int batchSize;
private String create;
private String drop;
private String insert;

public TestRunner(String string, int bs, int bc) throws SQLException {
    this.name = string;
    this.con = createConnection();
    this.batchCount = bc;
    this.batchSize = bs;
    this.create = "create table " + name + " (";

    for (int i = 0; i < fieldCount; i++) {
    create += (i == 0 ? "" : ",") + "col" + i + " " + getType(colType[i]);
    }
    this.create += ")";
    this.insert = "insert into " + name + " values (";
    for (int i = 0; i < fieldCount; i++) {
    insert += (i == 0 ? "" : ",") + "?";
    }
    this.insert += ")";
    this.drop = "drop table " + name;
}

private static String getType(int i) {
    switch (i) {
    case Types.DECIMAL:
    return "decimal(18,9)";
    case Types.VARCHAR:
    return "varchar(30000)";
    case Types.TIMESTAMP:
    return "timestamp";
    }
    return null;
}

protected void finalize() throws Throwable {
    if (con != null) {
    con.close();
    }
};

public void run() {
    System.out.format("%s started.%ncreate: %s%ninsert: %s%ndrop: %s%n", name, create, insert, drop);
    try (Statement stmt = con.createStatement()) {
    // this will throw the exception of concurrency conflict
    System.out.format("%d created %s%n", stmt.executeUpdate(create), name);
    if (!con.getAutoCommit()) {
        con.commit();
    }
    } catch (SQLException e1) {
    e1.printStackTrace();
    return;
    }
    try (PreparedStatement stmt = con.prepareStatement(insert)) {
    while (batchCount-- > 0) {
        for (int i = batchSize; i > 0; i--) {
        setBatchInput(stmt);
        stmt.addBatch();
        }
        System.out.format("%s - submitting batch ...%n", name);
        stmt.executeBatch();
        if (!con.getAutoCommit()) {
        con.commit();
        }
    }
    } catch (Exception e) {
    e.printStackTrace();
    } finally {
    if (con != null) {
        try (Statement stmt = con.createStatement()) {
        System.out.format("%d deleted - %s%n", stmt.executeUpdate(drop), name);
        if (!con.getAutoCommit()) {
            con.commit();
        }
        } catch (SQLException e1) {
        e1.printStackTrace();
        }
        try {
        con.close();
        con = null;
        } catch (SQLException e) {
        e.printStackTrace();
        }
    }
    }
    System.out.format("%s finished.", name);
}

private static void setBatchInput(PreparedStatement stmt) throws SQLException {
    for (int i = 1; i <= fieldCount; i++) {
    stmt.setObject(i, getRandomFieldValue(colType[i - 1]));
    }
}

private static Object getRandomFieldValue(int type) {
    switch (type) {
    case Types.DECIMAL:
    return 0;
    case Types.VARCHAR:
    return "null";
    case Types.TIMESTAMP:
    return new Timestamp(System.currentTimeMillis());
    }
    return null;
}
}

public static void main(String[] args) {
int num = 2;
List<Thread> ts = new ArrayList<>();
while (num-- > 0) {
    Thread t = null;
    try {
    (t = new Thread(new TestRunner("runner_" + num, 10000, 1))).start();
    } catch (SQLException e) {
    e.printStackTrace();
    }
    ts.add(t);
}
for (Thread t : ts) {
    try {
    t.join();
    } catch (InterruptedException e) {
    e.printStackTrace();
    }
}
}

这是我得到的例外

java.sql.SQLException: COMMIT: transaction is aborted because of concurrency conflicts, will ROLLBACK instead
at nl.cwi.monetdb.jdbc.MonetConnection$ResponseList.executeQuery(MonetConnection.java:2597)
at nl.cwi.monetdb.jdbc.MonetConnection$ResponseList.processQuery(MonetConnection.java:2345)
at nl.cwi.monetdb.jdbc.MonetStatement.internalExecute(MonetStatement.java:507)
at nl.cwi.monetdb.jdbc.MonetStatement.execute(MonetStatement.java:345)
at nl.cwi.monetdb.jdbc.MonetStatement.executeUpdate(MonetStatement.java:545)
at monet.test.BatchTest$TestRunner.run(BatchTest.java:92)
at java.lang.Thread.run(Thread.java:745)

如果我同步创建，则在一个线程中存在批处理失败 -

  
    

java.sql.SQLException：EXEC：没有预编译语句，带有id：2no预编译语句，ID为：

  

根据this thread！

  

如果您执行架构更新/更改，MonetDB   释放所有准备好的手柄，因为它们可能不再存在   正确。   您需要重新执行prepare命令。

解决方案

使用MonetDB bulk data load的功能！您可以直接将数据写入服务器而不是使用JDBC（即使并行打开连接） 以下是可以让您直接测试的代码

public static class TestRunner implements Runnable {
private final static String problemHere = "jdbc:monetdb://localhost:50000/test-db";

static {
    try {
    Class.forName("nl.cwi.monetdb.jdbc.MonetDriver");
    } catch (ClassNotFoundException e) {
    // TODO Auto-generated catch block
    e.printStackTrace();
    }
}

private static Connection createConnection() throws SQLException {
    // happens one time per thread
    return DriverManager.getConnection(problemHere, "monetdb", "monetdb");
}

private static int fieldCount = 10;
private static int[] colType = new int[fieldCount];
private static final int[] types = { Types.VARCHAR, Types.TIMESTAMP, Types.DECIMAL };

static {
    Random random = new Random();
    // initialize column types
    for (int i = 0; i < fieldCount; i++) {
    colType[i] = types[random.nextInt(types.length)];
    }
}

private final String name;
private Connection con;
private int batchCount;
private int batchSize;
private String create;
private String drop;
private String insert;

public TestRunner(String string, int bs, int bc) throws SQLException {
    this.name = string;
    this.con = createConnection();
    this.batchCount = bc;
    this.batchSize = bs;
    this.create = "create table " + name + " (";

    for (int i = 0; i < fieldCount; i++) {
    create += (i == 0 ? "" : ",") + "col" + i + " " + getType(colType[i]);
    }
    this.create += ")";
    this.insert = "insert into " + name + " values (";
    for (int i = 0; i < fieldCount; i++) {
    insert += (i == 0 ? "" : ",") + "?";
    }
    this.insert += ")";
    this.drop = "drop table " + name;
}

private static String getType(int i) {
    switch (i) {
    case Types.DECIMAL:
    return "decimal(18,9)";
    case Types.VARCHAR:
    return "varchar(30000)";
    case Types.TIMESTAMP:
    return "timestamp";
    }
    return null;
}

protected void finalize() throws Throwable {
    if (con != null) {
    con.close();
    }
};

public void run() {
    System.out.format("%s started.%ncreate: %s%ninsert: %s%ndrop: %s%n", name, create, insert, drop);
    try (Statement stmt = con.createStatement()) {
    // this will throw the exception of concurrency con
    synchronized (problemHere) {
        System.out.format("%d created %s%n", stmt.executeUpdate(create), name);
        if (!con.getAutoCommit()) {
        con.commit();
        }
    }
    } catch (SQLException e1) {
    e1.printStackTrace();
    return;
    }
    MapiSocket server = null;
    try {
    server = new MapiSocket();
    server.setDatabase("test-db");
    server.setLanguage("sql");

    List<String> warning = server.connect("localhost", 50000, "monetdb", "monetdb");
    if (warning != null) {
        for (Iterator<String> it = warning.iterator(); it.hasNext();) {
        System.out.println(it.next().toString());
        }
    }


    String error = in.waitForPrompt();
    if (error != null)
        throw new Exception(error);
    // try (PreparedStatement stmt = con.prepareStatement(insert)) {
    while (batchCount-- > 0) {
    BufferedMCLReader in = server.getReader();
    BufferedMCLWriter out = server.getWriter();
        String query = "COPY INTO " + name + " FROM STDIN USING DELIMITERS ',','\\n';";
        // the leading 's' is essential, since it is a protocol
        // marker that should not be omitted, likewise the
        // trailing semicolon
        out.write('s');
        out.write(query);
        out.newLine();
        for (int i = batchSize; i > 0; i--) {
        // setBatchInput(stmt);
        // stmt.addBatch();
        insertBatchElement(out);
        out.newLine();
        }
        System.out.format("%s - submitting batch ...%n", name);
        // stmt.executeBatch();
        out.writeLine(""); // need this one for synchronisation over
        // flush()
        out.flush();
        error = in.waitForPrompt();
        if (error != null)
        throw new Exception(error);
        out.close();
        in.close();
        if (!con.getAutoCommit()) {
        con.commit();
        }
    }
    } catch (Exception e) {
    e.printStackTrace();
    } finally {
    if (con != null) {
        try (Statement stmt = con.createStatement()) {
        synchronized (problemHere) {
            // System.out.format("%d deleted - %s%n",
            // stmt.executeUpdate(drop), name);
            if (!con.getAutoCommit()) {
            con.commit();
            }
        }
        } catch (SQLException e1) {
        e1.printStackTrace();
        }
        try {
        con.close();
        con = null;
        } catch (SQLException e) {
        e.printStackTrace();
        }
    }
    if (server != null) {
        server.close();
    }
    }
    System.out.format("%s finished.", name);
}

private void insertBatchElement(BufferedMCLWriter out) throws IOException {
    String data = "";
    for (int i = 1; i <= fieldCount; i++) {
    data += (i == 1 ? "" : ",") + String.valueOf(getRandomFieldValue(colType[i - 1]));
    }
    out.write(data);
}

private static void setBatchInput(PreparedStatement stmt) throws SQLException {
    for (int i = 1; i <= fieldCount; i++) {
    stmt.setObject(i, getRandomFieldValue(colType[i - 1]));
    }
}

private static Object getRandomFieldValue(int type) {
    switch (type) {
    case Types.DECIMAL:
    return 0;
    case Types.VARCHAR:
    return "null";
    case Types.TIMESTAMP:
    return new Timestamp(System.currentTimeMillis());
    }
    return null;
}
}

public static void main(String[] args) {
int num = 2;
List<Thread> ts = new ArrayList<>();
while (num-- > 0) {
    Thread t = null;
    try {
    (t = new Thread(new TestRunner("runner_" + num, 10000, 1))).start();
    } catch (SQLException e) {
    e.printStackTrace();
    }
    ts.add(t);
}
for (Thread t : ts) {
    try {
    t.join();
    } catch (InterruptedException e) {
    e.printStackTrace();
    }
}
}
相关问题
最新问题