Java jdbc多线程写数据入库

一、背景：
今天下午在做模拟数据时发现200万条数据写入数据库，时间很长，思考通过使用多线程进行数据批量写入操作。

二、实现代码：

1、数据库工具类

package com.general.data.utils;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;

/**
  * @description 数据库工具类
  * @author 砌墙民工
  * @date 2020年1月18日 上午12:56:40
  * @version 1.0
  *
  *                    修改信息说明：
  * @updateDescription
  * @updateAuthor
  * @updateDate
  */
public class DataBaseUtils {

    private static final String url = "jdbc:oracle:thin:@127.0.0.1:1521:orcl";
    private static final String driver = "oracle.jdbc.driver.OracleDriver";
    private static final String userName = "simulation";
    private static final String password = "simulation";

    /**
      * 获取数据库连接
      * @return 数据库连接
     */
    public static Connection getConnection() {
        try {
            Class.forName(driver);
            return DriverManager.getConnection(url, userName, password);
        } catch (SQLException e) {
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }
        return null;
    }

    /**
      * 关闭数据库操作资源
      * @param conn 数据库连接
      * @param stmt 数据库游标
      * @param rs   结果集
     */
    public static void closeDataBaseResources(Connection conn, Statement stmt, ResultSet rs) {
        if (null != rs) {
            try {
                rs.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
            rs = null;
        }

        if (null != stmt) {
            try {
                stmt.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
            stmt = null;
        }

        if (null != conn) {
            try {
                conn.close();
            } catch (SQLException e) {
                e.printStackTrace();
            }
            conn = null;
        }
    }

    /**
      * 读取表信息
      * @param tableName
      * @return
     */
    public static List<Map<String, String>> readTable(String tableName) {
        Connection conn = null;
        Statement stmt = null;
        ResultSet rs = null;
        List<Map<String, String>> row = null;
        try {
            conn = getConnection();
            //ResultSet.TYPE_SCROLL_INSENSITIVE 结果集的游标可以上下移动，当数据库变化时，当前结果集不变。
            //ResultSet.CONCUR_READ_ONLY 不能用结果集更新数据库中的表。
            stmt = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_UPDATABLE);
            String sql = "select * from " + tableName;
            rs = stmt.executeQuery(sql);
            ResultSetMetaData rsmd = rs.getMetaData();
            int columnCount = rsmd.getColumnCount(); //ResultSet的总列数

            row = new ArrayList<Map<String, String>>();
            while (rs.next()) {
                Map<String, String> column = new HashMap<String, String>();
                for (int columnIndex = 1; columnIndex <= columnCount; columnIndex++) {
                    column.put(rsmd.getColumnName(columnIndex).toLowerCase(), rs.getString(columnIndex));
                }
                row.add(column);
            }
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            closeDataBaseResources(conn, stmt, rs);
        }
        return row;
    }

    /**
      * 多线程将集合map数据写入表
      * @param data 集合map数据
      * @param tableName    表名
     */
    public static void multiThreadWriteTable(List<Map<String, String>> data, String tableName) {
        long startTime = System.currentTimeMillis();
        System.out.println("数据总量：" + data.size());
        List<String> sqlList = new ArrayList<String>();
        for (int i = 0; i < data.size(); i++) {
            Map<String, String> entity = data.get(i);
            StringBuffer frontSql = new StringBuffer();
            StringBuffer values = new StringBuffer();
            frontSql.append("insert into ");
            frontSql.append(tableName);
            frontSql.append(" (");
            for (String key : entity.keySet()) {
                frontSql.append(key + ",");
                values.append("'" + entity.get(key) + "',");
            }
            String sql = frontSql.substring(0, frontSql.length() - 1) + ") values(" + values.substring(0, values.length() - 1) + ")";
            sqlList.add(sql);
        }

        int threadCount = 20;
        int every = data.size() / threadCount;
        final CountDownLatch latch = new CountDownLatch(threadCount);

        for (int i = 0; i < threadCount; i++) {
            if (i == threadCount - 1) {
                new Thread(new ThreadWorker(latch, i * every, (i + 1) * every + (data.size() % threadCount), sqlList)).start();
            } else {
                new Thread(new ThreadWorker(latch, i * every, (i + 1) * every, sqlList)).start();
            }
        }
        try {
            latch.await();
            long endTimes = System.currentTimeMillis();
            System.out.println("所有线程执行完毕:" + (endTimes - startTime));
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}

2、线程工作空间

package com.general.data.utils;

import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.concurrent.CountDownLatch;

/**
  * @description 线程工作空间
  * @author 砌墙民工
  * @date 2020年1月18日 下午3:11:55
  * @version 1.0
  *
  *                    修改信息说明：
  * @updateDescription
  * @updateAuthor
  * @updateDate
  */
public class ThreadWorker implements Runnable {

    private int start = 0;
    private int end = 0;
    private CountDownLatch latch;
    private List<String> sqlList;

    public ThreadWorker(CountDownLatch latch, int start, int end, List<String> sqlList) {
        this.start = start;
        this.end = end;
        this.latch = latch;
        this.sqlList = sqlList;
    }

    public void run() {
        System.out.println("线程" + Thread.currentThread().getName() + "正在执行。。线程数据量为:" + (end - start));
        long startTime = System.currentTimeMillis();
        Connection conn = null;
        Statement stmt = null;

        try {
            conn = DataBaseUtils.getConnection();
            boolean autoCommit = conn.getAutoCommit();
            conn.setAutoCommit(false);
            stmt = conn.createStatement();

            int rowNum = 0;
            for (int i = start; i < end; i++) {
                stmt.addBatch(sqlList.get(i));
                if (++rowNum % 10000 == 0) {
                    stmt.executeBatch();
                    conn.commit();
                    System.out.println("线程" + Thread.currentThread().getName() + "现已完成" + rowNum + "条记录插入");
                }
            }
            stmt.executeBatch();
            // 提交修改
            conn.commit();
            conn.setAutoCommit(autoCommit);
        } catch (SQLException e) {
            e.printStackTrace();
        } finally {
            DataBaseUtils.closeDataBaseResources(conn, stmt, null);
            long endTime = System.currentTimeMillis();
            System.out.println("线程" + Thread.currentThread().getName() + "数据写入时间：" + (endTime - startTime) + "ms");
        }

        latch.countDown();
    }

}