Spring Batch问题与记录计数的页脚预告片

Question

我正在开发一个Spring Batch程序，但是在页脚中遇到困难，该页脚包含要写入文件的记录数。我已经从该实际公司程序中简化了该程序，以演示该问题。该程序将读取一个文件，并为每个记录写出两个记录。输入文件可以包含重复项，因此应忽略重复记录。

记录数远未达到。看来这两个记录合为一体。

testInputfile.csv

Id, Name
1, "John Doe"
2, "Jane Doe"
3, "John Smith"
4, "Jane Smith"
5, "Ed Johnson"
5, "Ed Johnson"
6, "Dana Johnson"
7, "Pam Parker"
8, "Julie Walker"
9, "Sam Smith"
10, "Paul Hammond"
11, "Walter Barker"

testOutputfile.csv

1|John Doe
1|JOHN DOE
2|Jane Doe
2|JANE DOE
3|John Smith
3|JOHN SMITH
4|Jane Smith
4|JANE SMITH
5|Ed Johnson
5|ED JOHNSON
6|Dana Johnson
6|DANA JOHNSON
7|Pam Parker
7|PAM PARKER
8|Julie Walker
8|JULIE WALKER
9|Sam Smith
9|SAM SMITH
10|Paul Hammond
10|PAUL HAMMOND
11|Walter Barker
11|WALTER BARKER
Total: 12

JobConfiguration.java

package com.example;

import java.util.List;

import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.file.FlatFileFooterCallback;
import org.springframework.batch.item.file.FlatFileItemReader;
import org.springframework.batch.item.file.mapping.DefaultLineMapper;
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.FileSystemResource;

@Configuration
public class JobConfig {
    @Autowired
    private JobBuilderFactory jobBuilderFactory;

    @Autowired
    private StepBuilderFactory stepBuilderFactory;

    private String inputFileName = "/data/testInputfile.csv";
    private String outputFileName = "/data/testOutputfile.csv";
    private FlatFileFooterCallback footerCallback = new FooterCallback();

    @Bean
    public FlatFileItemReader<InputRecord> myItemReader() {
        FlatFileItemReader<InputRecord> reader = new FlatFileItemReader<>();
        reader.setLinesToSkip(1);
        reader.setResource(new FileSystemResource(inputFileName));

        DefaultLineMapper<InputRecord> scFundsLineMapper = new DefaultLineMapper<>();

        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
        tokenizer.setNames(new String[] {
                "id",
                "name"
                });

        scFundsLineMapper.setLineTokenizer(tokenizer);
        scFundsLineMapper.setFieldSetMapper(new FieldMapper());
        scFundsLineMapper.afterPropertiesSet();

        reader.setLineMapper(scFundsLineMapper);
        return reader;
    }

    @Bean
    public MyItemProcessor myItemProcessor() {
        return new MyItemProcessor();
    }

    @Bean
    public ItemWriter<List<OutputRecord>> myItemWriter() {
        return new MyItemWriter(outputFileName, footerCallback);
    }

    @Bean
    public Step step() {
        return stepBuilderFactory.get("step")
                .<InputRecord, List<OutputRecord>>chunk(10)
                .listener(footerCallback)
                .reader(myItemReader())
                .processor(myItemProcessor())
                .writer(myItemWriter())
                .build();
    }

    @Bean
    public Job job() {
        return jobBuilderFactory
                .get("job")
                .start(step())
                .build();
    }
}

MyItemProcessor.java

package com.example;

import java.util.ArrayList;
import java.util.List;

import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.springframework.batch.item.ItemProcessor;

public class MyItemProcessor implements ItemProcessor<InputRecord, List<OutputRecord>> {

    private static final Logger logger = LogManager.getLogger(MyItemWriter.class);

    private List<Integer> list1 = new ArrayList<>();
    private List<Integer> list2 = new ArrayList<>();

    @Override
    public List<OutputRecord> process(InputRecord inputRecord) {
        List <OutputRecord> outputRecords = new ArrayList<>();

        // Process for first output record
        if (list1.contains(inputRecord.getId())) {
            logger.warn("Duplicate for List 1: ID=" + inputRecord.getId());
        } else {
            OutputRecord outputRecord = new OutputRecord();

            outputRecord.setId(inputRecord.getId());
            outputRecord.setName(inputRecord.getName());

            outputRecords.add(outputRecord);
            list1.add(inputRecord.getId());
        }

        // Process for second output record
        if (list2.contains(inputRecord.getId())) {
            logger.warn("Duplicate for List 2: ID=" + inputRecord.getId());
        } else {
            OutputRecord outputRecord = new OutputRecord();

            outputRecord.setId(inputRecord.getId());
            outputRecord.setName(inputRecord.getName().toUpperCase());

            outputRecords.add(outputRecord);
            list2.add(inputRecord.getId());
        }

        return outputRecords;
    }
}

FieldMapper.java

package com.example;

import org.springframework.batch.item.file.mapping.FieldSetMapper;
import org.springframework.batch.item.file.transform.FieldSet;

public class FieldMapper implements FieldSetMapper<InputRecord> {

    @Override
    public InputRecord mapFieldSet(FieldSet fieldSet) {
        InputRecord inputRecord = new InputRecord();

        inputRecord.setId(Integer.parseInt(fieldSet.readString("id")));
        inputRecord.setName(fieldSet.readString("name"));

        return inputRecord;
    }

}

MyItemWriter.java

package com.example;

import java.util.List;

import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.annotation.AfterStep;
import org.springframework.batch.item.ExecutionContext;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.file.FlatFileFooterCallback;
import org.springframework.batch.item.file.FlatFileItemWriter;
import org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor;
import org.springframework.batch.item.file.transform.DelimitedLineAggregator;
import org.springframework.batch.item.support.AbstractItemStreamItemWriter;
import org.springframework.core.io.FileSystemResource;

public class MyItemWriter extends AbstractItemStreamItemWriter<List<OutputRecord>>
        implements ItemWriter<List<OutputRecord>> {

    private static final FlatFileItemWriter<OutputRecord> delegate = new FlatFileItemWriter<>();

    private String outputFileName;
    private FlatFileFooterCallback footerCallback = new FooterCallback();

    public MyItemWriter(String outputFileName, FlatFileFooterCallback footerCallback) {
        this.outputFileName = outputFileName;
        this.footerCallback = footerCallback;
    }

    @AfterStep
    public void afterStep(StepExecution stepExecution) {
        delegate.close();
    }

    @Override
    public void write(List<? extends List<OutputRecord>> lists) throws Exception {
        delegate.setFooterCallback(footerCallback);
        delegate.setResource(new FileSystemResource(outputFileName));

        DelimitedLineAggregator<OutputRecord> delimitedLineAggregator = new DelimitedLineAggregator<>();
        delimitedLineAggregator.setDelimiter("|");
        BeanWrapperFieldExtractor<OutputRecord> extractor = new BeanWrapperFieldExtractor<>();
        extractor.setNames(new String[] { "id", "name" });

        delimitedLineAggregator.setFieldExtractor(extractor);
        delegate.setLineAggregator(delimitedLineAggregator);

        delegate.open(new ExecutionContext());
        for (final List<OutputRecord> list : lists) {
            if (list.size() != 0) {
                delegate.write(list);
            }
        }
    }

}

FooterCallback.java

package com.example;

import java.io.IOException;
import java.io.Writer;

import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.springframework.batch.core.StepExecution;
import org.springframework.batch.core.annotation.BeforeStep;
import org.springframework.batch.core.listener.StepExecutionListenerSupport;
import org.springframework.batch.item.file.FlatFileFooterCallback;

public class FooterCallback extends StepExecutionListenerSupport implements FlatFileFooterCallback {
    private static final Logger logger = LogManager.getLogger(FooterCallback.class);
    private StepExecution stepExecution;
    static private int totalCount = 0;

    public void writeFooter(Writer writer) throws IOException {
        int count = stepExecution.getWriteCount();

        if (stepExecution.getStepName().equals("step")) {   // last step
            writer.write("Total: " + (totalCount + count));
        }
        else {
            totalCount += count;
        }

    }

    @Override
    @BeforeStep
    public void beforeStep(StepExecution stepExecution) {
        this.stepExecution = stepExecution;
    }

}

InputRecord.java

package com.example;

public class InputRecord {
    private int id;
    private String name;

    public int getId() {
        return id;
    }

    public void setId(int id) {
        this.id = id;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

}

OutputRecord

package com.example;

public class OutputRecord {
    private int id;
    private String name;

    public int getId() {
        return id;
    }

    public void setId(int id) {
        this.id = id;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

}

TrailerTestApplication.java

package com.example;

import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;

@SpringBootApplication
@EnableBatchProcessing
@EnableAutoConfiguration(exclude={DataSourceAutoConfiguration.class})
public class TrailerTestApplication {

    public static void main(String[] args) {
        SpringApplication.run(TrailerTestApplication.class, args);
    }
}