spring boot Batch 的简单学习

批量处理
pom.xml文件

<!-- Spring Boot 启动父依赖 核心模块，包括自动配置支持、日志和YAML-->    <parent>        <groupId>org.springframework.boot</groupId>        <artifactId>spring-boot-starter-parent</artifactId>        <version>1.5.1.RELEASE</version>    </parent>    <dependencies>        <!--start-web-->        <dependency>            <groupId>org.springframework.boot</groupId>            <artifactId>spring-boot-starter-web</artifactId>        </dependency>        <!--batch-->        <dependency>            <groupId>org.springframework.boot</groupId>            <artifactId>spring-boot-starter-batch</artifactId>            <exclusions>                <exclusion>                    <groupId>org.hsqldb</groupId>                    <artifactId>hsqldb</artifactId>                </exclusion>            </exclusions>        </dependency>        <!--mysql-->        <dependency>            <groupId>mysql</groupId>            <artifactId>mysql-connector-java</artifactId>            <version>5.1.39</version>        </dependency>        <!--jdbc-->        <dependency>            <groupId>org.springframework.boot</groupId>            <artifactId>spring-boot-starter-jdbc</artifactId>        </dependency>        <!--hibernate-validator 数据校验-->        <dependency>            <groupId>org.hibernate</groupId>            <artifactId>hibernate-validator</artifactId>        </dependency>        <!--jpa-->        <dependency>            <groupId>org.springframework.boot</groupId>            <artifactId>spring-boot-starter-data-jpa</artifactId>        </dependency>    </dependencies>

application.properties配置文件

# 端口号server.port=8080# 日志级别logging.level.root=info# 数据源配置spring.datasource.driver-class-name=com.mysql.jdbc.Driverspring.datasource.url=jdbc:mysql://ip:3306/dbname?useUnicode=true&characterEncoding=UTF-8spring.datasource.username= spring.datasource.password= # 让控制器输出json字符串格式spring.jackson.serialization.indent-output=truespring.jpa.hibernate.ddl-auto=createspring.jpa.show-sql=true

people.csv

汪某某,11,汉族,合肥张某某,12,汉族,上海李某某,13,非汉族,武汉刘某,14,非汉族,南京欧阳某某,115,汉族,北京

domian

import javax.persistence.Entity;import javax.persistence.GeneratedValue;import javax.persistence.Id;import javax.validation.constraints.Size;@Entity(name = "batch_person")public class BatchPerson {    @Id    @GeneratedValue    private Integer id;    @Size(max = 4, min = 2)    private String name;    private String age;    private String nation;    private String address;    public Integer getId() {        return id;    }    public void setId(Integer id) {        this.id = id;    }    public String getName() {        return name;    }    public void setName(String name) {        this.name = name;    }    public String getAge() {        return age;    }    public void setAge(String age) {        this.age = age;    }    public String getNation() {        return nation;    }    public void setNation(String nation) {        this.nation = nation;    }    public String getAddress() {        return address;    }    public void setAddress(String address) {        this.address = address;    }}

数据处理和校验
CsvItemValidation(校验)

import java.util.Set;import javax.validation.ConstraintViolation;import javax.validation.Validation;import javax.validation.ValidatorFactory;import org.springframework.batch.item.validator.ValidationException;import org.springframework.batch.item.validator.Validator;import org.springframework.beans.factory.InitializingBean;public class CsvItemValidation<T> implements Validator<T>,InitializingBean {    private javax.validation.Validator validator;    @Override    public void afterPropertiesSet() throws Exception { //1        ValidatorFactory validatorFactory = Validation.buildDefaultValidatorFactory();        validator = validatorFactory.usingContext().getValidator();    }    @Override    public void validate(T value) throws ValidationException {        Set<ConstraintViolation<T>> constraintViolations = validator.validate(value); //2        if(constraintViolations.size()>0){            StringBuilder message = new StringBuilder();            for (ConstraintViolation<T> constraintViolation : constraintViolations) {                message.append(constraintViolation.getMessage() + "\n");            }            throw new ValidationException(message.toString());        }    }}

CsvItemProcessor(数据处理)

import cn.zfs.spring.batch.domian.BatchPerson;import org.springframework.batch.item.validator.ValidatingItemProcessor;import org.springframework.batch.item.validator.ValidationException;/** * 数据处理 */public class CsvItemProcessor extends ValidatingItemProcessor<BatchPerson>{    @Override    public BatchPerson process(BatchPerson item) throws ValidationException {        super.process(item);        if(item.getNation().equals("汉族")){            item.setNation("01");        }else {            item.setNation("02");        }        return item;    }}

listener

import org.springframework.batch.core.JobExecution;import org.springframework.batch.core.JobExecutionListener;public class CsvJobListener implements JobExecutionListener {    @Override    public void beforeJob(JobExecution jobExecution) {        System.out.println("任务处理开始");    }    @Override    public void afterJob(JobExecution jobExecution) {        System.out.println("任务处理结束");    }}

到这里就要区分一下项目是要手动触发还是自动触发了
– 自动触发
配置文件

import javax.sql.DataSource;import cn.zfs.spring.batch.domian.BatchPerson;import cn.zfs.spring.batch.listener.CsvJobListener;import cn.zfs.spring.batch.processorvalidation.CsvItemProcessor;import cn.zfs.spring.batch.processorvalidation.CsvItemValidation;import org.springframework.batch.core.Job;import org.springframework.batch.core.Step;import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;import org.springframework.batch.core.launch.support.RunIdIncrementer;import org.springframework.batch.core.launch.support.SimpleJobLauncher;import org.springframework.batch.core.repository.JobRepository;import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean;import org.springframework.batch.item.ItemProcessor;import org.springframework.batch.item.ItemReader;import org.springframework.batch.item.ItemWriter;import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;import org.springframework.batch.item.database.JdbcBatchItemWriter;import org.springframework.batch.item.file.FlatFileItemReader;import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper;import org.springframework.batch.item.file.mapping.DefaultLineMapper;import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;import org.springframework.batch.item.validator.Validator;import org.springframework.context.annotation.Bean;import org.springframework.context.annotation.Configuration;import org.springframework.core.io.ClassPathResource;import org.springframework.transaction.PlatformTransactionManager;@Configuration@EnableBatchProcessingpublic class CsvBatchConfig {    @Bean    public ItemReader<BatchPerson> reader() throws Exception {        FlatFileItemReader<BatchPerson> reader = new FlatFileItemReader<BatchPerson>(); //1        reader.setResource(new ClassPathResource("people.csv")); //2            reader.setLineMapper(new DefaultLineMapper<BatchPerson>() {{ //3                setLineTokenizer(new DelimitedLineTokenizer() {{                    setNames(new String[] { "name","age", "nation" ,"address"});                }});                setFieldSetMapper(new BeanWrapperFieldSetMapper<BatchPerson>() {{                    setTargetType(BatchPerson.class);                }});            }});            return reader;    }    @Bean    public ItemProcessor<BatchPerson, BatchPerson> processor() {        CsvItemProcessor processor = new CsvItemProcessor(); //1        processor.setValidator(csvBeanValidator()); //2        return processor;    }    @Bean    public ItemWriter<BatchPerson> writer(DataSource dataSource) {//1        JdbcBatchItemWriter<BatchPerson> writer = new JdbcBatchItemWriter<BatchPerson>(); //2        writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<BatchPerson>());        String sql = "insert into batch_person " + "( name,age,nation,address) "                + "values( :name, :age, :nation,:address)";        writer.setSql(sql); //3        writer.setDataSource(dataSource);        return writer;    }    @Bean    public JobRepository jobRepository(DataSource dataSource, PlatformTransactionManager transactionManager)            throws Exception {        JobRepositoryFactoryBean jobRepositoryFactoryBean = new JobRepositoryFactoryBean();        jobRepositoryFactoryBean.setDataSource(dataSource);        jobRepositoryFactoryBean.setTransactionManager(transactionManager);        jobRepositoryFactoryBean.setDatabaseType("mysql");        return jobRepositoryFactoryBean.getObject();    }    @Bean    public SimpleJobLauncher jobLauncher(DataSource dataSource, PlatformTransactionManager transactionManager)            throws Exception {        SimpleJobLauncher jobLauncher = new SimpleJobLauncher();        jobLauncher.setJobRepository(jobRepository(dataSource, transactionManager));        return jobLauncher;    }    @Bean    public Job importJob(JobBuilderFactory jobs, Step s1) {        return jobs.get("importJob")                .incrementer(new RunIdIncrementer())                .flow(s1) //1                .end()                .listener(csvJobListener()) //2                .build();    }    @Bean    public Step step1(StepBuilderFactory stepBuilderFactory, ItemReader<BatchPerson> reader, ItemWriter<BatchPerson> writer,            ItemProcessor<BatchPerson,BatchPerson> processor) {        return stepBuilderFactory                .get("step1")                .<BatchPerson, BatchPerson>chunk(65000) //1                .reader(reader) //2                .processor(processor) //3                .writer(writer) //4                .build();    }    @Bean    public CsvJobListener csvJobListener() {        return new CsvJobListener();    }    @Bean    public Validator<BatchPerson> csvBeanValidator() {        return new CsvItemValidation<BatchPerson>();    }}

ok,启动项目,你会发现项目的启动日志里面有listener中的打印字段
– 手动触发
配置文件

import cn.zfs.spring.batch.domian.BatchPerson;import cn.zfs.spring.batch.listener.CsvJobListener;import cn.zfs.spring.batch.processorvalidation.CsvItemProcessor;import cn.zfs.spring.batch.processorvalidation.CsvItemValidation;import org.springframework.batch.core.Job;import org.springframework.batch.core.Step;import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;import org.springframework.batch.core.launch.support.RunIdIncrementer;import org.springframework.batch.core.launch.support.SimpleJobLauncher;import org.springframework.batch.core.repository.JobRepository;import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean;import org.springframework.batch.item.ItemProcessor;import org.springframework.batch.item.ItemReader;import org.springframework.batch.item.ItemWriter;import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;import org.springframework.batch.item.database.JdbcBatchItemWriter;import org.springframework.batch.item.file.FlatFileItemReader;import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper;import org.springframework.batch.item.file.mapping.DefaultLineMapper;import org.springframework.batch.item.file.transform.DelimitedLineTokenizer;import org.springframework.batch.item.validator.Validator;import org.springframework.beans.factory.annotation.Value;import org.springframework.context.annotation.Bean;import org.springframework.context.annotation.Configuration;import org.springframework.core.io.ClassPathResource;import org.springframework.transaction.PlatformTransactionManager;import javax.sql.DataSource;//@Configuration@EnableBatchProcessingpublic class TriggerBatchConfig {    @Bean    public FlatFileItemReader<BatchPerson> reader(@Value("#{jobParameters['input.file.name']}") String pathToFile) throws Exception {        FlatFileItemReader<BatchPerson> reader = new FlatFileItemReader<BatchPerson>(); //1        reader.setResource(new ClassPathResource(pathToFile)); //2            reader.setLineMapper(new DefaultLineMapper<BatchPerson>() {{ //3                setLineTokenizer(new DelimitedLineTokenizer() {{                    setNames(new String[] { "name","age", "nation" ,"address"});                }});                setFieldSetMapper(new BeanWrapperFieldSetMapper<BatchPerson>() {{                    setTargetType(BatchPerson.class);                }});            }});            return reader;    }    @Bean    public ItemProcessor<BatchPerson, BatchPerson> processor() {        CsvItemProcessor processor = new CsvItemProcessor(); //1        processor.setValidator(csvBeanValidator()); //2        return processor;    }    @Bean    public ItemWriter<BatchPerson> writer(DataSource dataSource) {//1        JdbcBatchItemWriter<BatchPerson> writer = new JdbcBatchItemWriter<BatchPerson>(); //2        writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<BatchPerson>());        String sql = "insert into batch_person " + "( name,age,nation,address) "                + "values( :name, :age, :nation,:address)";        writer.setSql(sql); //3        writer.setDataSource(dataSource);        return writer;    }    @Bean    public JobRepository jobRepository(DataSource dataSource, PlatformTransactionManager transactionManager)            throws Exception {        JobRepositoryFactoryBean jobRepositoryFactoryBean = new JobRepositoryFactoryBean();        jobRepositoryFactoryBean.setDataSource(dataSource);        jobRepositoryFactoryBean.setTransactionManager(transactionManager);        jobRepositoryFactoryBean.setDatabaseType("mysql");        return jobRepositoryFactoryBean.getObject();    }    @Bean    public SimpleJobLauncher jobLauncher(DataSource dataSource, PlatformTransactionManager transactionManager)            throws Exception {        SimpleJobLauncher jobLauncher = new SimpleJobLauncher();        jobLauncher.setJobRepository(jobRepository(dataSource, transactionManager));        return jobLauncher;    }    @Bean    public Job importJob(JobBuilderFactory jobs, Step s1) {        return jobs.get("importJob")                .incrementer(new RunIdIncrementer())                .flow(s1) //1                .end()                .listener(csvJobListener()) //2                .build();    }    @Bean    public Step step1(StepBuilderFactory stepBuilderFactory, ItemReader<BatchPerson> reader, ItemWriter<BatchPerson> writer,            ItemProcessor<BatchPerson,BatchPerson> processor) {        return stepBuilderFactory                .get("step1")                .<BatchPerson, BatchPerson>chunk(65000) //1                .reader(reader) //2                .processor(processor) //3                .writer(writer) //4                .build();    }    @Bean    public CsvJobListener csvJobListener() {        return new CsvJobListener();    }    @Bean    public Validator<BatchPerson> csvBeanValidator() {        return new CsvItemValidation<BatchPerson>();    }}

Controller

import org.springframework.batch.core.Job;import org.springframework.batch.core.JobParameters;import org.springframework.batch.core.JobParametersBuilder;import org.springframework.batch.core.JobParametersInvalidException;import org.springframework.batch.core.launch.JobLauncher;import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;import org.springframework.batch.core.repository.JobRestartException;import org.springframework.beans.factory.annotation.Autowired;import org.springframework.web.bind.annotation.RequestMapping;import org.springframework.web.bind.annotation.RestController;@RestControllerpublic class BatchController {    @Autowired    JobLauncher jobLauncher;    @Autowired    Job importJob;    public JobParameters jobParameters;    @RequestMapping("/imp")    public String imp(String fileName){        String path = fileName+".csv";        jobParameters = new JobParametersBuilder()                .addLong("time",System.currentTimeMillis())                .addString("input.file.name",path)                .toJobParameters();        try {            jobLauncher.run(importJob,jobParameters);        } catch (JobExecutionAlreadyRunningException e) {            e.printStackTrace();        } catch (JobRestartException e) {            e.printStackTrace();        } catch (JobInstanceAlreadyCompleteException e) {            e.printStackTrace();        } catch (JobParametersInvalidException e) {            e.printStackTrace();        }        return "ok";    }}

ok,接下来启动项目,访问localhost:8080/imp?fileName=people,就可以手动触发批量处理