Spring Batch：来自处理器的DAO调用减慢了批处理过程 - 我可以重用数据库连接吗？

Question

当我在下面的MdwValidatingItemProcessor中添加一个DAO调用时，我的性能受到严重影响。在添加额外的DAO调用之前，2500条记录在40秒内读取/处理/写入。当我添加一个额外的DAO调用时（lobPolicyMapper.getPolicyOrigin（item.getLob（），item.getRgn_cd（）） ），它变成120秒。 lob_policy表中只有一条记录。我计划在我的处理器中有许多这样的DAO调用来对该项进行额外的验证。我可以重复使用DAO的连接，因此对于验证项目的每个处理器，连接不必不断打开和关闭吗？ AsyncItemProcessor委托给MdwValidatingItemProcessor：

public class MdwValidatingItemProcessor implements
    ItemProcessor<MecMdw, MecMdw> {

@Autowired
DataSourceTransactionManager txManager;

@Autowired
LobPolicyMapper lobPolicyMapper;

@Autowired
MecUtils mecUtils;

@Autowired 
LobShopMapper lobShopMapper;

@Autowired
KaiserIssuerMapper kaiserIssuerMapper;

private Validator validator;

public void setValidator(Validator validator) {
    this.validator = validator;
}

private List<Map> validationReasons;

@BeforeStep
public void beforeStep(StepExecution stepExecution) {
    validationReasons = (List<Map>) stepExecution.getJobExecution().getExecutionContext().get("validationReasons");
}


public MecMdw process(MecMdw item) throws Exception {

    DefaultTransactionDefinition def = new DefaultTransactionDefinition();
    def.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
    TransactionStatus status = txManager.getTransaction(def);

    BindingResult results = BindAndValidate(item);

    try {
        if (results.hasErrors()) {

            buildValidationException(results, item);

            return setAsKickOut(item);

        }

        mecUtils.checkForPaddingAndValidateSSN(item);

        //if both SSN and DOB are valid, set Rep_DOB_or_SSN to be SSN           
        if (item.getInvalid_ssn().equals(MecConstants.Y_FLAG) && mecUtils.isDOBValid(item)) {
            item.setRep_doborssn("SSN");
        }
        else if(mecUtils.isDOBValid(item)){
            item.setRep_doborssn("DOB");
        }
        else{
            List<String> listOfErrors = new ArrayList<String>();

            listOfErrors.add("BothSSNAndDOBInvalid");

            item.setValidationErrors(listOfErrors);

            return setAsKickOut(item);
        }                       

        // Policy Origin code not found in MEC database based on Region /
        // LOB.
        LobPolicy lobPolicy = lobPolicyMapper.getPolicyOrigin(
                item.getLob(), item.getRgn_cd());

        if (lobPolicy == null || ("").equals(lobPolicy.getLob())
                || lobPolicy.getLob() == null) {
            return setAsKickOut(item);
        }

        //set the Rep_PolicyOgnCd
        item.setRep_policy_ogn_cd(lobPolicy.getPolicyOrigin());

        //If origin of policy = SHOP, look for Shop Identifier.(mec_lob_shop table)
        if(("SHOP").equals(lobPolicy.getPolicyOrigin())){
            if(lobShopMapper.getValidationShopIdentifier(item) == null){
                return setAsKickOut(item);
            }               
        }

        //Validation Shop Identifier not found in MEC database based on Region/LOB/PID
        if(lobShopMapper.getValidationShopIdentifier(item) == null){
            return setAsKickOut(item);
        }

        //Kaiser Issuer EIN not found in MEC database based on Region/LOB.
        Integer kaiserIssuer = kaiserIssuerMapper.getKaiserIssuerIdWithLobAndRegion(item);
        if(kaiserIssuer  == 0){
            return setAsKickOut(item);
        }
        else{
            item.setRep_kaiser_issuer_id(kaiserIssuer.toString() );
        }

    } catch (Exception ex) {
        ex.printStackTrace();
    } finally {
        txManager.commit(status);
    }

    return item;

}

private MecMdw setAsKickOut(MecMdw item) {
    item.setKick_out_fl('Y');           

    return item;
}

private BindingResult BindAndValidate(MecMdw item) {

    DataBinder binder = new DataBinder(item);

    binder.setValidator(validator);

    binder.validate();

    return binder.getBindingResult();

}

private void buildValidationException(BindingResult results, MecMdw item) {

    List<String> listOfErrors = new ArrayList<String>();

    for (ObjectError error : results.getAllErrors()) {
        String[] codes = error.getCodes();
        listOfErrors.add(codes[1]);
    }

    item.setValidationErrors(listOfErrors);
    // throw new ValidationException(msg.toString());
}

我有一个使用AsyncItemProcessor和AsyncItemWriter的批处理作业，如下所示：

<job id="mecmdwvalidatorJob" xmlns="http://www.springframework.org/schema/batch">
    <step id="mdwvalidatorStep1">
        <tasklet>
            <chunk reader="pageItemReader" processor="asyncItemProcessor"
                writer="asynchItemWriter" commit-interval="1000" skip-limit="2147483647">
                <skippable-exception-classes> <!-- TODO -->
                    <include class="java.lang.Exception" />
                </skippable-exception-classes>
            </chunk>
        </tasklet>
    </step>
</job>

<bean id="pageItemReader"
    class="org.springframework.batch.item.database.JdbcPagingItemReader">
    <property name="dataSource" ref="dataSource" />
    <property name="queryProvider">
        <bean
            class="org.springframework.batch.item.database.support.SqlPagingQueryProviderFactoryBean">
            <property name="dataSource" ref="dataSource" />
            <property name="selectClause"
                value="select MDW_ID,FK_LOG_FILE_ID,TAX_YEAR,SUBS_TYPE_CD,SUB_FIRST_NM,SUB_MIDDLE_NM,SUB_LAST_NM,SUB_SUFFIX,SUB_DOB,SUB_ADDR1,SUB_ADDR2,SUB_CITY,SUB_STATE,SUB_PROVINCE,SUB_ZIP,SUB_ZIP4,SUB_COUNTRY_CD,SUB_COUNTRY,SUB_F_POSTAL_CD,LOB,SUB_SSN,GRP_EMP_NAME1,GRP_EMP_NAME2,GRP_EIN,GRP_ADDR1,GRP_ADDR2,GRP_CITY,GRP_STATE,GRP_PROVINCE,GRP_ZIP,GRP_ZIP4,GRP_COUNTRY_CD,GRP_COUNTRY,GRP_F_POSTAL_CD,ISSUER_NAME1,ISSUER_NAME2,ISSUER_PHONE,ISSUER_ADDR1,ISSUER_ADDR2,ISSUER_CITY,ISSUER_PROVINCE,ISSUER_ZIP,ISSUER_ZIP4,ISSUER_COUNTRY_CD,ISSUER_COUNTRY,ISSUER_F_POSTAL_CD,MEM_FIRST_NM,MEM_MIDDLE_NM,MEM_LAST_NM,MEM_SUFFIX,MEM_SSN,MEM_DOB,MEM_START_DATE,MEM_END_DATE,REGION_CD,SUB_MRN,SUB_MRN_PREFIX,MEM_MRN,MRN_PREFIX,PID,SUB_GRP_ID,SUB_GRP_NAME,INVALID_ADDR_FL" />
            <property name="fromClause"
                value="from MEC_MDW JOIN MEC_FILE_LOG on MEC_FILE_LOG.LOG_FILE_ID=MEC_MDW.FK_LOG_FILE_ID  " />
            <property name="whereClause" value="where MEC_FILE_LOG.STATUS=:status" />
            <property name="sortKey" value="MDW_ID" />
        </bean>
    </property>
    <property name="parameterValues">
        <map>
            <entry key="status" value="READY TO VALIDATE" />
        </map>
    </property>
    <property name="pageSize" value="1000" />
    <property name="rowMapper" ref="mdwRowMapper" />
</bean>
<bean id="mdwRowMapper" class="org.my.rowmapper.MdwRowMapper" />

<bean id="asyncItemProcessor"
    class="org.springframework.batch.integration.async.AsyncItemProcessor">
    <property name="delegate">
        <bean
            class="org.my.itemprocessor.MdwValidatingItemProcessor">
            <property name="validator">
                <bean
                    class="org.springframework.validation.beanvalidation.LocalValidatorFactoryBean" />
            </property>
        </bean>
    </property>
    <property name="taskExecutor" ref="taskExecutor" />
</bean>

<task:executor id="taskExecutor" pool-size="10" />

<bean id="asynchItemWriter"
    class="org.springframework.batch.integration.async.AsyncItemWriter">
    <property name="delegate" ref="customerCompositeWriter">
    </property>
</bean>

<bean id="customerCompositeWriter"
    class="org.springframework.batch.item.support.CompositeItemWriter">
    <property name="delegates">
        <list>
            <ref bean="itemWriter1" />
            <ref bean="itemWriter2" />
        </list>
    </property>
</bean>

<bean id="itemWriter1" class="org.my.writer.MdwWriter" />
<bean id="itemWriter2" class="org.my.writer.KickoutWriter" />

事务管理器和MyBatix SqlSessionTemplate：

<!-- transaction manager, use JtaTransactionManager for global tx -->
<bean id="transactionManager"
    class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
    <property name="dataSource" ref="dataSource" />
</bean>

<!--  define SqlSessionFactory as BATCH execution -->
<bean id="sqlSession" class="org.mybatis.spring.SqlSessionTemplate">
    <constructor-arg index="0" ref="sqlSessionFactory" />
    <constructor-arg index="1" value="BATCH" />
</bean>

<!-- stored job-meta in database -->
<bean id="jobRepository"
    class="org.springframework.batch.core.repository.support.JobRepositoryFactoryBean">
    <property name="dataSource" ref="dataSource" />
    <property name="transactionManager" ref="batchTransactionManager" />
    <property name="databaseType" value="sqlserver" />
</bean>

<bean id="jobLauncher"
    class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
    <property name="jobRepository" ref="jobRepository" />
</bean>

DAO打电话：

public interface LobPolicyMapper {

public abstract LobPolicy getPolicyOrigin(@Param("lob") String lob, @Param("regionCd") String regionCd);

更新 我在MyBatis XML中添加了ehcache。这将有助于重复查询。但同样，我正在寻找一种方法来跨AsyncItemProcessors分享这个相同的DAO连接：

<mapper namespace="org.mybatis.LobPolicyMapper">

<cache type="org.mybatis.caches.ehcache.EhcacheCache"/>

<select id="getPolicyOrigin" parameterType="hashmap" resultType='org.my.domain.LobPolicy'>
    SELECT 
    l.lob_id as lobId,
    l.lob as lob,
    l.policy_origin as policyOrigin,
    l.region_cd as regionCd
    from mec_lob_policy l
    where lob= #{lob} and region_cd=#{regionCd}
</select>

我在项目中添加了ehcache

    <dependency>
        <groupId>org.mybatis</groupId>
        <artifactId>mybatis-ehcache</artifactId>
        <version>1.0.0</version>
    </dependency>

我想我需要一个数据库池。以下是我在退出处理器的日志中看到的内容：

 2015-08-27 17:09:54,194 [taskExecutor-3] DEBUG org.mybatis.spring.SqlSessionUtils - Releasing transactional SqlSession [org.apache.ibatis.session.defaults.DefaultSqlSession@77b027f]
 2015-08-27 17:10:06,674 [taskExecutor-3] DEBUG org.mybatis.spring.SqlSessionUtils - Transaction synchronization committing SqlSession [org.apache.ibatis.session.defaults.DefaultSqlSession@77b027f]
2015-08-27 17:10:06,675 [taskExecutor-3] DEBUG org.mybatis.spring.SqlSessionUtils - Transaction synchronization deregistering SqlSession [org.apache.ibatis.session.defaults.DefaultSqlSession@77b027f]
2015-08-27 17:10:06,675 [taskExecutor-3] DEBUG org.mybatis.spring.SqlSessionUtils - Transaction synchronization closing SqlSession [org.apache.ibatis.session.defaults.DefaultSqlSession@77b027f]
2015-08-27 17:10:06,675 [taskExecutor-3] DEBUG org.springframework.jdbc.datasource.DataSourceTransactionManager - Initiating transaction commit
2015-08-27 17:10:06,675 [taskExecutor-3] DEBUG org.springframework.jdbc.datasource.DataSourceTransactionManager - Committing JDBC transaction on Connection [ConnectionID:14 ClientConnectionId: 1b95dc0e-83c0-487b-af59-f5be52931818]
2015-08-27 17:10:06,805 [taskExecutor-3] DEBUG org.springframework.jdbc.datasource.DataSourceTransactionManager - Releasing JDBC Connection [ConnectionID:14 ClientConnectionId: 1b95dc0e-83c0-487b-af59-f5be52931818] after transaction
2015-08-27 17:10:06,805 [taskExecutor-3] DEBUG org.springframework.jdbc.datasource.DataSourceUtils - Returning JDBC Connection to DataSource

以下是我在进入处理器的日志中看到的内容。创建新事务并获取新的数据库连接：

2015-08-27 17:10:06,805 [taskExecutor-3] DEBUG org.springframework.jdbc.datasource.DataSourceTransactionManager - Creating new transaction with name [null]: PROPAGATION_REQUIRED,ISOLATION_DEFAULT
2015-08-27 17:10:06,805 [taskExecutor-3] DEBUG org.springframework.jdbc.datasource.DriverManagerDataSource - Creating new JDBC DriverManager Connection to [jdbc:sqlserver://blah]
2015-08-27 17:10:07,115 [taskExecutor-3] DEBUG org.springframework.jdbc.datasource.DataSourceTransactionManager - Acquired Connection [ConnectionID:23 ClientConnectionId: d1f016e6-3e9d-4b0e-a34d-14298c292a65] for JDBC transaction
2015-08-27 17:10:07,115 [taskExecutor-3] DEBUG org.springframework.jdbc.datasource.DataSourceTransactionManager - Switching JDBC Connection [ConnectionID:23 ClientConnectionId: d1f016e6-3e9d-4b0e-a34d-14298c292a65] to manual commit