Question

在spring批处理项目中，我使用JdbcCursorItemReader读取数据以并行处理它们。我可以在本地运行批处理，而不会出现任何问题。

我还听说针对JdbcCursorItemReader进行并行处理时，建议使用JdbcPagingItemReader，因为游标阅读器将保持连接时间太长，而分页阅读器可以在达到页面大小后释放连接。

然后在步骤2中切换到JdbcPagingItemReader，但不出意外，我在本地运行时遇到了以下异常。

原因：java.sql.SQLTransientConnectionException：HikariPool-1- 连接不可用，请求在300001ms后超时。

但是，似乎上述异常发生在执行步骤2中的页面读取器之前的步骤1中，并且这是唯一的更改。请阐明为什么会引发异常，以及在并行处理中使用分页阅读器而不是游标是一种好习惯。非常感谢您的帮助！

代码段粘贴在下面：

@Bean
@StepScope
public Flow createParallelSubFlow() {
    List<Flow> subFlowList = new ArrayList<>();

    List<Stream> streamList;

    try {
        streamList = dataSourceConfig.streamMapper().
                getStreamListByStatus(Constants.PENDING_STATUS_CD);
    } catch (Exception e) {
    }
    streamList.forEach(stream -> {
        long id = stream.getStreamId();
        String flowName = "stream" + id + "_flow";
        Flow subFlow = new FlowBuilder<Flow>(flowName)
                .start(step1(id))
                .next(step2(id))
                .end();

        subFlowList.add(subFlow);
    });

    return new FlowBuilder<Flow>("splitFlow").split(new SimpleAsyncTaskExecutor())
            .add(subFlowList.toArray(new Flow[0])).build();

}

public Step step1(long id) {
    return stepBuilderFactory.get("step1")
            .<Domain, Domain>chunk(100)
            .reader(reader1(id))
            .writer(writer1())
            .build();
}

//@StepScope
//@Bean
public Step step2(long id) {

    return stepBuilderFactory.get("step2")
            .<Domain, Domain>chunk(100)
            .reader(cursorReader2(id))
            .processor(processor2)
            .writer(writer2())
            .build();
}


public JdbcCursorItemReader<Domain> cursorReader2(Long id) {

    return new JdbcCursorItemReaderBuilder<Domain>()
            .dataSource(dataSourceConfig.dataSource())
            .name("cursorReader")
            .sql(Constants.QUERY_SQL)
            .preparedStatementSetter(new PreparedStatementSetter() {

                @Override
                public void setValues(PreparedStatement ps) throws SQLException {
                    ps.setLong(1, id);
                }})
            .rowMapper(new RowMapper())
            .build();
}


//Switch from cursorReader2 to pagingReader2 in step2
public JdbcPagingItemReader<Domain> pagingReader2(Long id) {

    return new JdbcPagingItemReaderBuilder<Domain>()
            .dataSource(dataSourceConfig.dataSource())
            .name("pagingReader")
            .queryProvider(queryProvider())
            .parameterValues(parameterValues(id))                
            .rowMapper(new RowMapper())
            .pageSize(100)
            .build();
}

@Bean
public PagingQueryProvider queryProvider() {
    SqlPagingQueryProviderFactoryBean providerFactory = new SqlPagingQueryProviderFactoryBean();

    Map<String, Order> sortKeys = new HashMap<>(2);
    sortKeys.put("ID", Order.ASCENDING);

    providerFactory.setDataSource(dataSourceConfig.dataSource());
    providerFactory.setSelectClause("SELECT Clause");
    providerFactory.setFromClause("FROM Clause");
    providerFactory.setWhereClause("WHERE Clause");
    providerFactory.setSortKeys(sortKeys);

    PagingQueryProvider pagingQueryProvider = null;
    try {
        pagingQueryProvider = providerFactory.getObject();
    } catch (Exception e) {
        logger.error("Failed to get PagingQueryProvider", e);
        throw new RuntimeException("Failed to get PagingQueryProvider", e);
    }

    return pagingQueryProvider;
}

private Map<String, Object> parameterValues(Long id) {

    Map<String, Object> parameterValues = new HashMap<>();
    parameterValues.put("1", id);

    return parameterValues;

}

使用JdbcCursorItemReader或JdbcPagingItemReader

0 个答案: