title: Processing a large file using spring batch
body: <p>I have a large file which may contain 100K to 500K records. I am planning to use chunk oriented processing and my thought is</p>
  
  <p>1) Split the large file into smaller based on the count let say 10K in each file.</p>
  
  <p>2) If there are 100K records then I will get 10 files each containing 10K reocrds</p>
  
  <p>3) I would like to partition these 10 files and would like to process using 5 threads. I am thinking to use custom <strong>MultiResourcePartioner</strong></p>
  
  <p>4) The 5 threads should process all the 10 files created in split process.</p>
  
  <p>5) I don't want to create same number of threads equal to file count as in that case I may face memory issues. What I am looking is whatever the number of files I would like to process them using only 5 threads (I can increase based on my requirements).</p>
  
  <p>Expert could you let me know this can be achieved using spring batch? If yes could you please share pointers or reference implementations</p>
  
  <p>Thanks in advance</p>
  
  <p>The working job-config xml</p>
  
  <pre><code>&lt;description&gt;Spring Batch File Chunk Processing&lt;/description&gt;
  
  &lt;import resource="../config/batch-context.xml" /&gt;
  
  &lt;batch:job id="file-partition-batch" job-repository="jobRepository" restartable="false"&gt;        
      &lt;batch:step id="master"&gt;
          &lt;batch:partition partitioner="partitioner" handler="partitionHandler" /&gt;
      &lt;/batch:step&gt;
  &lt;/batch:job&gt;
  
  &lt;batch:step id="slave"&gt;
      &lt;batch:tasklet&gt;
          &lt;batch:chunk reader="reader" processor="compositeProcessor"
              writer="compositeWriter" commit-interval="5"&gt;
          &lt;/batch:chunk&gt;
      &lt;/batch:tasklet&gt;
  &lt;/batch:step&gt;
  
  &lt;bean id="partitionHandler" class="org.springframework.batch.core.partition.support.TaskExecutorPartitionHandler"&gt;
      &lt;property name="taskExecutor" ref="taskExecutor"/&gt;
      &lt;property name="step" ref="slave" /&gt;
      &lt;property name="gridSize" value="5" /&gt;
  &lt;/bean&gt;
  
  &lt;bean id="partitioner" class="com.poc.partitioner.FileMultiResourcePartitioner"&gt;
      &lt;property name="resources" value="file:/Users/anupghosh/Documents/Spring_Batch/FilePartitionBatch/*.txt" /&gt;
      &lt;property name="threadName" value="feed-processor" /&gt;
  &lt;/bean&gt;
  
  &lt;bean id="taskExecutor" class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor"&gt;
      &lt;property name="corePoolSize" value="5" /&gt;
      &lt;property name="maxPoolSize" value="5" /&gt;
  &lt;/bean&gt;
  
  &lt;bean id="reader" class="org.springframework.batch.item.file.FlatFileItemReader" scope="step"&gt;
      &lt;property name="resource" value="#{stepExecutionContext['fileName']}" /&gt;
  
      &lt;property name="lineMapper"&gt;
          &lt;bean class="org.springframework.batch.item.file.mapping.DefaultLineMapper"&gt;
          &lt;property name="lineTokenizer"&gt;
              &lt;bean class="org.springframework.batch.item.file.transform.DelimitedLineTokenizer"&gt;
                  &lt;property name="delimiter" value="|"/&gt;
                  &lt;property name="names" value="key,docName,docTypCD,itemType,itemNum,launchDate,status" /&gt;
              &lt;/bean&gt;
          &lt;/property&gt;
          &lt;property name="fieldSetMapper"&gt;
              &lt;bean class="com.poc.mapper.FileRowMapper" /&gt;
          &lt;/property&gt;
          &lt;/bean&gt;
      &lt;/property&gt;
  &lt;/bean&gt;
  
  &lt;bean id="validatingProcessor" class="org.springframework.batch.item.validator.ValidatingItemProcessor"&gt;
      &lt;constructor-arg ref="feedRowValidator" /&gt;
  &lt;/bean&gt;
  
  &lt;bean id="feedProcesor" class="com.poc.processor.FeedProcessor" /&gt;
  
  &lt;bean id="compositeProcessor" class="org.springframework.batch.item.support.CompositeItemProcessor" scope="step"&gt;
      &lt;property name="delegates"&gt;
          &lt;list&gt;
              &lt;ref bean="validatingProcessor" /&gt;
              &lt;ref bean="feedProcesor" /&gt;
          &lt;/list&gt;
      &lt;/property&gt;
  &lt;/bean&gt;
  
  &lt;bean id="recordDecWriter" class="com.poc.writer.RecordDecWriter" /&gt;
  
  &lt;bean id="reconFlatFileCustomWriter" class="com.poc.writer.ReconFileWriter"&gt;
      &lt;property name="reconFlatFileWriter" ref="reconFlatFileWriter" /&gt;
  &lt;/bean&gt;
  
  &lt;bean id="reconFlatFileWriter" class="org.springframework.batch.item.file.FlatFileItemWriter" scope="step"&gt;
      &lt;property name="resource" value="file:/Users/anupghosh/Documents/Spring_Batch/recon-#{stepExecutionContext[threadName]}.txt" /&gt;
      &lt;property name="shouldDeleteIfExists" value="true" /&gt;
      &lt;property name="lineAggregator"&gt;
          &lt;bean class="org.springframework.batch.item.file.transform.DelimitedLineAggregator"&gt;
              &lt;property name="delimiter" value="|" /&gt;
              &lt;property name="fieldExtractor"&gt;
                  &lt;bean class="org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor"&gt;
                      &lt;property name="names" value="validationError" /&gt;
                  &lt;/bean&gt;
              &lt;/property&gt;
          &lt;/bean&gt;
      &lt;/property&gt;
  &lt;/bean&gt;
  
  &lt;bean id="compositeWriter" class="org.springframework.batch.item.support.CompositeItemWriter"&gt;
      &lt;property name="delegates"&gt;
          &lt;list&gt;
              &lt;ref bean="recordDecWriter" /&gt;
              &lt;ref bean="reconFlatFileCustomWriter" /&gt;
          &lt;/list&gt;
      &lt;/property&gt;
  &lt;/bean&gt;
  
  &lt;bean id="feedRowValidator" class="org.springframework.batch.item.validator.SpringValidator"&gt;
      &lt;property name="validator"&gt;
          &lt;bean class="com.poc.validator.FeedRowValidator"/&gt;
      &lt;/property&gt;
  &lt;/bean&gt;     
  </code></pre>
  
  <p></p>