Question

我有一个文件，其中包含这样的多行数据。 DataID是新记录的开始。例如一个记录是ID的组合，并在下一行连接直到新记录开始。

    >DataID1
    Line1asdfsafsdgdsfghfghfghjfgjghjgxcvmcxnvm
    Line2asdfsafsdgdsfghfghfghjfgjghjgxcvmcxnvm
    Line3asdfsafsdgdsfghfghfghjfgjghjgxcvmcxnvm
    >DataID2
    DataID2asdfsafsdgdsfghfghfghjfgjghjgxcvmcxnvm
    >DataID3
    DataID2asdfsafsdgdsfghfghfghjfgjghjgxcvmcxnvm

我能够使用SingleItemPeekableItemReader来实现这一点，并且工作正常。

我不是要实现分区，因为我们需要处理多个文件。我不确定分区程序如何将文件信息传递给客户阅读器，以及如何确保SingleItemPeekableItemReader线程安全（因为它无法正常工作）

目前我需要一些输入

java-config

@Bean
      public Partitioner partitioner() {
          MultiResourcePartitioner partitioner = new MultiResourcePartitioner();
          partitioner.setResources(resources);          
          partitioner.partition(10);      
          return partitioner;
      }
      @Bean
      public TaskExecutor taskExecutor() {
          ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
          taskExecutor.setMaxPoolSize(4);
          taskExecutor.setCorePoolSize(4);
          taskExecutor.setQueueCapacity(8);
          taskExecutor.afterPropertiesSet();
          return taskExecutor;
      }   

      @Bean
      @Qualifier("masterStep")
      public Step masterStep() {
          return stepBuilderFactory.get("masterStep")                  
                  .partitioner("step1",partitioner())
                  .step(step1())
                  .taskExecutor(taskExecutor())                  
                  .build();
      }

     @Bean
      public MultiResourceItemReader<FieldSet> multiResourceItemReader() {
        log.info("Total Number of Files to be process {}",resources.length);
        report.setFileCount(resources.length);
        MultiResourceItemReader<FieldSet> resourceItemReader = new MultiResourceItemReader<FieldSet>();     
        resourceItemReader.setResources(resources);     
        resourceItemReader.setDelegate(reader());       
        return resourceItemReader;
      }

    @Bean
    public FlatFileItemReader<FieldSet> reader() {
         FlatFileItemReader<FieldSet> build = new FlatFileItemReaderBuilder<FieldSet>().name("fileReader")              
                .lineTokenizer(orderFileTokenizer())
                .fieldSetMapper(new FastFieldSetMapper())                   
                .recordSeparatorPolicy(new BlankLineRecordSeparatorPolicy())
                .build();        
         build.setBufferedReaderFactory(gzipBufferedReaderFactory);
         return build;
    }

    @Bean
    public SingleItemPeekableItemReader<FieldSet> readerPeek() {
        SingleItemPeekableItemReader<FieldSet> reader = new SingleItemPeekableItemReader<>();
        reader.setDelegate(multiResourceItemReader());
        return reader;
    }

    @Bean
    public MultiLineFastaItemReader itemReader() {
        MultiLineFastaItemReader itemReader = new MultiLineFastaItemReader(multiResourceItemReader());
        itemReader.setSingalPeekable(readerPeek());     
        return itemReader;
    }

    @Bean
    public PatternMatchingCompositeLineTokenizer orderFileTokenizer() {
        PatternMatchingCompositeLineTokenizer tokenizer = new PatternMatchingCompositeLineTokenizer();
        Map<String, LineTokenizer> tokenizers = new HashMap<>(2);
        tokenizers.put(">*", head());
        tokenizers.put("*", tail());
        tokenizer.setTokenizers(tokenizers);
        return tokenizer;
    }

    public DelimitedLineTokenizer head() {
        DelimitedLineTokenizer token = new DelimitedLineTokenizer();
        token.setNames("sequenceIdentifier");
        token.setDelimiter(" ");
        token.setStrict(false);
        return token;
    }

    public DelimitedLineTokenizer tail() {
        DelimitedLineTokenizer token = new DelimitedLineTokenizer();
        token.setNames("sequences");
        token.setDelimiter(" ");
        return token;
    }

    @Bean
    public FastReportWriter writer() {
        return new FastReportWriter();
    }

    @Bean
    public Job importUserJob(JobCompletionNotificationListener listener, Step step1) {
        return jobBuilderFactory.get("importUserJob")
                .incrementer(new RunIdIncrementer())
                .listener(listener)
                .flow(masterStep())
                //.flow(step1)
                .next(step2())
                .end()
                .build();
    }

    @Bean
    public Step step1() {
        return stepBuilderFactory.get("step1")
                .<Fasta, Fasta>chunk(5000)
                .reader(itemReader())
                .processor(new FastaIteamProcessor())
                //.processor(new PassThroughItemProcessor<>())
                .writer(writer())
                .build();
    }

public class MultiLineFastaItemReader implements ItemReader<Fasta>, ItemStream {

    private static final Logger log = LoggerFactory.getLogger(MultiLineFastaItemReader.class);
    private SingleItemPeekableItemReader<FieldSet> singalPeekable;

    AtomicInteger iteamCounter = new AtomicInteger(0);

    ConcurrentHashMap<String, AtomicInteger> fileNameAndCounterMap = new ConcurrentHashMap<>();

    @Autowired
    private SequenceFastaReport sequenceFastaReport;

    private MultiResourceItemReader<FieldSet> resourceItemReader;

    public MultiLineFastaItemReader(MultiResourceItemReader<FieldSet> multiResourceItemReader) {

        this.resourceItemReader = multiResourceItemReader;
    }

    public SingleItemPeekableItemReader<FieldSet> getSingalPeekable() {
        return singalPeekable;
    }

    public void setSingalPeekable(SingleItemPeekableItemReader<FieldSet> singalPeekable) {
        this.singalPeekable = singalPeekable;

    }

    @Override
    public Fasta read() throws Exception {
        FieldSet item = singalPeekable.read();
        if (item == null) {
            return null;
        }
        Fasta fastaObject = new Fasta();

        log.info("ID {} fileName {}", item.readString(0), resourceItemReader.getCurrentResource());
        fastaObject.setSequenceIdentifier(item.readString(0)
                .toUpperCase());
        fastaObject.setFileName(resourceItemReader.getCurrentResource()
                .getFilename());

        if (!fileNameAndCounterMap.containsKey(fastaObject.getFileName())) {
            fileNameAndCounterMap.put(fastaObject.getFileName(), new AtomicInteger(0));

        }

        while (true) {

            FieldSet possibleRelatedObject = singalPeekable.peek();
            if (possibleRelatedObject == null) {
                if (fastaObject.getSequenceIdentifier()
                        .length() < 1)
                    throw new InvalidParameterException("Somwthing Wrong in file");
                sequenceFastaReport.addToReport(fileNameAndCounterMap.get(fastaObject.getFileName())
                        .incrementAndGet(), fastaObject.getSequences());
                return fastaObject;
            }

            if (possibleRelatedObject.readString(0)
                    .startsWith(">")) {
                if (fastaObject.getSequenceIdentifier()
                        .length() < 1)
                    throw new InvalidParameterException("Somwthing Wrong in file");

                sequenceFastaReport.addToReport(fileNameAndCounterMap.get(fastaObject.getFileName())
                        .incrementAndGet(), fastaObject.getSequences());

                return fastaObject;
            }
            String data = fastaObject.getSequences()
                    .toUpperCase();
            fastaObject.setSequences(data + singalPeekable.read()
                    .readString(0)
                    .toUpperCase());

        }

    }

    @Override
    public void close() {
        this.singalPeekable.close();
    }

    @Override
    public void open(ExecutionContext executionContext) {
        this.singalPeekable.open(executionContext);

    }

    @Override
    public void update(ExecutionContext executionContext) {

        this.singalPeekable.update(executionContext);
    }

}

Answer 1

我不确定分区程序如何将文件信息传递给我的客户阅读器

分区器将在步骤执行上下文中创建分区元数据，您的读者应从中读取该元数据。在您的示例中，您无需在分区程序上调用partition，Spring Batch将做到这一点。您需要在分区程序上设置分区键，例如：

  @Bean
  public Partitioner partitioner() {
      MultiResourcePartitioner partitioner = new MultiResourcePartitioner();
      partitioner.setResources(resources);          
      partitioner.setKeyName("file");     
      return partitioner;
  }

这将为每个文件创建一个分区，并带有键file，您可以从步骤执行上下文中将其插入阅读器：

@Bean
@StepScope
public FlatFileItemReader reader(@Value("#{stepExecutionContext['file']}") String file) {
    // define your reader 
}

请注意，读者应在使用此功能的范围内进行调整。此处有更多详细信息：https://docs.spring.io/spring-batch/4.0.x/reference/html/step.html#late-binding

带有MultiResourcePartitioner的Spring Batch MultiLineItemReader

1 个答案: