多线程实现大批量数据导入

来源：互联网发布：mysql 中文怎么设置编辑：程序博客网时间：2024/05/21 12:45

最近在做大批量数据导入问题，当数据量很大的时候（例如：20万条），程序在执行的时候，需要花很长的时间，为了提高程序的执行效率，现采用多线程的方法实现数据导入功能，具体实现如下：

【实现】[java]
package com.mypack.thread;

public class Student {

    private String name;
    private String sex;
    private int age;

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public String getSex() {
        return sex;
    }

    public void setSex(String sex) {
        this.sex = sex;
    }

    public int getAge() {
        return age;
    }

    public void setAge(int age) {
        this.age = age;
    }

}

    package com.mypack.thread;

    public class Student {

        private String name;
        private String sex;
        private int age;

        public String getName() {
            return name;
        }

        public void setName(String name) {
            this.name = name;
        }

        public String getSex() {
            return sex;
        }

        public void setSex(String sex) {
            this.sex = sex;
        }

        public int getAge() {
            return age;
        }

        public void setAge(int age) {
            this.age = age;
        }

    }
[java]
package com.mypack.thread;

import java.util.ArrayList;
import java.util.List;

/**
* <ul>
* 学生线程
* </ul>
*
* @author liudong
*
*/
public class StudentThread extends Thread {

    private List<Student> students = new ArrayList<Student>();

    public StudentThread(List<Student> students) {
        this.students = students;

    }

    public void run() {

        for (Student student : students) {
            System.out.println("当前线程：" + Thread.currentThread().getName());
            System.out.println("name = " + student.getName() + " age = "
                    + student.getAge() + " sex = " + student.getSex());
            // 导入学生信息到数据库中
        }
    }

}

    package com.mypack.thread;

    import java.util.ArrayList;
    import java.util.List;

    /**
     * <ul>
     * 学生线程
     * </ul>
     *
     * @author liudong
     *
     */
    public class StudentThread extends Thread {

        private List<Student> students = new ArrayList<Student>();

        public StudentThread(List<Student> students) {
            this.students = students;

        }

        public void run() {

            for (Student student : students) {
                System.out.println("当前线程：" + Thread.currentThread().getName());
                System.out.println("name = " + student.getName() + " age = "
                        + student.getAge() + " sex = " + student.getSex());
                // 导入学生信息到数据库中
            }
        }

    }
[java]
package com.mypack.thread;

import java.util.ArrayList;
import java.util.List;

/**
*
* <ul>
* 测试类
* </ul>
*
* @author liudong
*
*/
public class Test {

    public static void main(String[] args) {

        Test test = new Test();
        // 查询所有的学生信息
        List<Student> students = test.getAllStudents();
        // 定义一个临时集合，用于存放学生信息
        List<Student> tempStudents = new ArrayList<Student>();

        for (int i = 0; i < students.size(); i++) {
            tempStudents.add(students.get(i));
            if (i != 0 && i % 50000 == 0) {
                test.createThread4ExportStudentInfo(tempStudents, i);
            }
        }

        test.createThread4ExportStudentInfo(tempStudents, students.size());

    }

    /**
     * <li>创建线程，导入学生信息</li>
     *
     * @param tempStudents
     *            每个线程，需要导入的学生信息的数量
     * @param i
     */
    public void createThread4ExportStudentInfo(List<Student> tempStudents, int i) {

        List<Student> students = new ArrayList<Student>();

        for (Student student : tempStudents) {
            students.add(student);
        }

        StudentThread studentThread = new StudentThread(students);
        // 设置线程名称
        studentThread.setName("Thread-" + i);
        // 启动线程
        studentThread.start();

        // 重新生成一个新的临时学生集合。
        tempStudents = new ArrayList<Student>();
    }

    /**
     * <li>查询所有的学生信息</li>
     *
     * @return
     */
    public List<Student> getAllStudents() {

        List<Student> students = new ArrayList<Student>();

        for (int i = 1; i <= 100000; i++) {
            Student student = new Student();

            student.setName("liudong" + i);
            student.setAge(i);
            student.setSex(i + "");

            students.add(student);
        }

        return students;
    }
}

0 0