1. 技术文章
  2. 【ShardingSphere】4. Spring Boot整合Sharding-JDBC实现分库分表+读写分离+雪花算法生成ID

【ShardingSphere】4. Spring Boot整合Sharding-JDBC实现分库分表+读写分离+雪花算法生成ID

由 三世轮回 提交于 2020-04-06 01:48:13

架构

在数据量不是很多的情况下，我们可以将数据库进行读写分离，以应对高并发的需求，通过水平扩展从库，来缓解查询的压力。如下：

在数据量达到500万的时候，这时数据量预估千万级别，我们可以将数据进行分表存储。

在数据量继续扩大，这时可以考虑分库分表，将数据存储在不同数据库的不同表中，如下：

案例详解

本案例有6个数据库，两个主库，四个从库，信息如下：

数据库类型 数据库 IP
主 cool 47.98.183.0
从 cool 101.37.175.23 
从 cool 120.27.250.228
主 cool2 47.98.183.0
从 cool2 101.37.175.23 
从 cool2 120.27.250.228

 在主库主机的Mysql执行以下脚本，分别为数据库cool和cool2创建5个表，这5个表分别为user_0、user_1、user_2、user_3、user_4。

分库需要注意ID的自增问题，我这里直接使用分布式生成ID
执行的脚本如下：

USE `cool`;


/*Table structure for table `user_0` */

DROP TABLE IF EXISTS `user_0`;

CREATE TABLE `user_0` (
  `id` varchar(20) NOT NULL,
  `username` varchar(12) NOT NULL,
  `password` varchar(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

/*Table structure for table `user_1` */

DROP TABLE IF EXISTS `user_1`;

CREATE TABLE `user_1` (
  `id` varchar(20) NOT NULL,
  `username` varchar(12) NOT NULL,
  `password` varchar(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

/*Table structure for table `user_2` */

DROP TABLE IF EXISTS `user_2`;

CREATE TABLE `user_2` (
  `id` varchar(20) NOT NULL,
  `username` varchar(12) NOT NULL,
  `password` varchar(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

/*Table structure for table `user_3` */

DROP TABLE IF EXISTS `user_3`;

CREATE TABLE `user_3` (
  `id` varchar(20) NOT NULL,
  `username` varchar(12) NOT NULL,
  `password` varchar(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
CREATE TABLE `user_4` (
  `id` varchar(20) NOT NULL,
  `username` VARCHAR(12) NOT NULL,
  `password` VARCHAR(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=INNODB DEFAULT CHARSET=utf8;


USE `cool2`;


/*Table structure for table `user_0` */

DROP TABLE IF EXISTS `user_0`;

CREATE TABLE `user_0` (
  `id` varchar(20) NOT NULL,
  `username` varchar(12) NOT NULL,
  `password` varchar(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

/*Table structure for table `user_1` */

DROP TABLE IF EXISTS `user_1`;

CREATE TABLE `user_1` (
  `id` varchar(20) NOT NULL,
  `username` varchar(12) NOT NULL,
  `password` varchar(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

/*Table structure for table `user_2` */

DROP TABLE IF EXISTS `user_2`;

CREATE TABLE `user_2` (
  `id` varchar(20) NOT NULL,
  `username` varchar(12) NOT NULL,
  `password` varchar(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

/*Table structure for table `user_3` */

DROP TABLE IF EXISTS `user_3`;

CREATE TABLE `user_3` (
  `id` varchar(20) NOT NULL,
  `username` varchar(12) NOT NULL,
  `password` varchar(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

CREATE TABLE `user_4` (
  `id` varchar(20) NOT NULL,
  `username` VARCHAR(12) NOT NULL,
  `password` VARCHAR(30) NOT NULL,
  PRIMARY KEY (`id`),
  KEY `idx-username` (`username`)
) ENGINE=INNODB DEFAULT CHARSET=utf8;

案例的工程是在上一篇文章的工程基础上进行改造，其中pom文件的依赖包不变。

在工程的application中做sharding-jdbc的分库分表配置，代码如下：

sharding:
  jdbc:
    config:
      sharding:
        default-database-strategy:
          inline:
            algorithm-expression: ds_$->{id % 2}  #取余2
            sharding-column: id
        master-slave-rules:
          ds_0:
            master-data-source-name: ds-master-0
            slave-data-source-names: ds-master-0-slave-0, ds-master-0-slave-1
          ds_1:
            master-data-source-name: ds-master-1
            slave-data-source-names: ds-master-1-slave-0, ds-master-1-slave-1
        tables:
          user:
            actual-data-nodes: ds_$->{0..1}.user_$->{0..4}
            key-generator-column-name: id
            table-strategy:
              inline:
                algorithm-expression: user_$->{id % 5}
                sharding-column: id
    datasource:
      ds-master-0:
        driver-class-name: com.mysql.jdbc.Driver
        password: 'Mysql@123'
        type: com.alibaba.druid.pool.DruidDataSource
        url: jdbc:mysql://47.98.183.0:3306/cool?useUnicode=true&characterEncoding=utf8&tinyInt1isBit=false&useSSL=false&serverTimezone=GMT
        username: root
      ds-master-0-slave-0:
        driver-class-name: com.mysql.jdbc.Driver
        password: 'Mysql@123'
        type: com.alibaba.druid.pool.DruidDataSource
        url: jdbc:mysql://101.37.175.23:3306/cool?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false&serverTimezone=GMT
        username: root
      ds-master-0-slave-1:
        driver-class-name: com.mysql.jdbc.Driver
        password: 'Mysql@123'
        type: com.alibaba.druid.pool.DruidDataSource
        url: jdbc:mysql://120.27.250.228:3306/cool?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false&serverTimezone=GMT
        username: root

      ds-master-1:
        driver-class-name: com.mysql.jdbc.Driver
        password: 'Mysql@123'
        type: com.alibaba.druid.pool.DruidDataSource
        url: jdbc:mysql://47.98.183.0:3306/cool2?useUnicode=true&characterEncoding=utf8&tinyInt1isBit=false&useSSL=false&serverTimezone=GMT
        username: root
      ds-master-1-slave-0:
        driver-class-name: com.mysql.jdbc.Driver
        password: 'Mysql@123'
        type: com.alibaba.druid.pool.DruidDataSource
        url: jdbc:mysql://101.37.175.23:3306/cool2?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false&serverTimezone=GMT
        username: root
      ds-master-1-slave-1:
        driver-class-name: com.mysql.jdbc.Driver
        password: 'Mysql@123'
        type: com.alibaba.druid.pool.DruidDataSource
        url: jdbc:mysql://120.27.250.228:3306/cool2?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&useSSL=false&serverTimezone=GMT
        username: root
      names: ds-master-0,ds-master-1,ds-master-0-slave-0,ds-master-0-slave-1,ds-master-1-slave-0,ds-master-1-slave-1

mybatis:
  type-aliases-package: com.springjdbc.netity #扫描实体类
  configLocation: classpath:mybatis/mybatis-config.xml
  mapperLocations: classpath*:mapper/**/*Mapper.xml
spring:
  main:
    allow-bean-definition-overriding: true #当遇到同样名字的时候，是否允许覆盖注册
  • 在上面的配置中，其中sharding.jdbc.datasource部分是配置数据库的信息，配置了6个数据库。
  • sharding.jdbc.config.sharding.master-slave-rules.ds_0.master-data-source-name配置的是ds_0区的的主库名称，同理ds_1。
  • sharding.jdbc.config.sharding.master-slave-rules.ds_0.slave-data-source-names配置的是ds_0区的的从库名称，同理ds_1。
  • sharding.jdbc.config.sharding.default-database-strategy.inline.sharding-column配置的分库的字段，本案例是根据id进行分。
  • sharding.jdbc.config.sharding.default-database-strategy.inline.algorithm-expression配置的分库的逻辑，根据id%2进行分。
  • sharding.jdbc.config.sharding.tables.user.actual-data-nodes配置的是user表在真实数据库中的位置，ds_KaTeX parse error: Expected group after '_' at position 14: ->{0..1}.user_̲->{0…4}表示
  • 数据在ds_0和ds_1中的user_0、user_1、user_2、user_3、user_4中。
  • sharding.jdbc.config.sharding.tables.user.table-strategy.inline.sharding-column，配置user表数据切分的字段
  • sharding.jdbc.config.sharding.tables.user.table-strategy.inline.algorithm-expression=user_$->{id % 5}，配置user表数据切分的策略。
  • sharding.jdbc.config.sharding.tables.user.key-generator-column-name=id 自动生成id。

然后在Spring Boot启动类的注解@SpringBootApplication，加上exclude={DataSourceAutoConfiguration.class}，代码如下：

package com.springjdbc;

import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;

@MapperScan("com.springjdbc.**.mapper")
@SpringBootApplication(exclude={DataSourceAutoConfiguration.class})
public class ShardingsphereDemoApplication {

    public static void main(String[] args) {
        SpringApplication.run(ShardingsphereDemoApplication.class, args);
    }

}

雪花算法

package com.springjdbc.util;


/**
 * 描述: Twitter的分布式自增ID雪花算法snowflake (Java版)
 *
 * @author
 * @create 2018-03-13 12:37
 **/
public class SnowFlake {

    /**
     * 起始的时间戳
     */
    private final static long START_STMP = 1480166465631L;

    /**
     * 每一部分占用的位数
     */
    private final static long SEQUENCE_BIT = 12; //序列号占用的位数
    private final static long MACHINE_BIT = 5;   //机器标识占用的位数
    private final static long DATACENTER_BIT = 5;//数据中心占用的位数

    /**
     * 每一部分的最大值
     */
    private final static long MAX_DATACENTER_NUM = -1L ^ (-1L << DATACENTER_BIT);
    private final static long MAX_MACHINE_NUM = -1L ^ (-1L << MACHINE_BIT);
    private final static long MAX_SEQUENCE = -1L ^ (-1L << SEQUENCE_BIT);

    /**
     * 每一部分向左的位移
     */
    private final static long MACHINE_LEFT = SEQUENCE_BIT;
    private final static long DATACENTER_LEFT = SEQUENCE_BIT + MACHINE_BIT;
    private final static long TIMESTMP_LEFT = DATACENTER_LEFT + DATACENTER_BIT;

    private long datacenterId;  //数据中心
    private long machineId;     //机器标识
    private long sequence = 0L; //序列号
    private long lastStmp = -1L;//上一次时间戳

    public SnowFlake(long datacenterId, long machineId) {
        if (datacenterId > MAX_DATACENTER_NUM || datacenterId < 0) {
            throw new IllegalArgumentException("datacenterId can't be greater than MAX_DATACENTER_NUM or less than 0");
        }
        if (machineId > MAX_MACHINE_NUM || machineId < 0) {
            throw new IllegalArgumentException("machineId can't be greater than MAX_MACHINE_NUM or less than 0");
        }
        this.datacenterId = datacenterId;
        this.machineId = machineId;
    }

    /**
     * 产生下一个ID
     *
     * @param ifEvenNum 是否偶数 true 时间不连续全是偶数  时间连续 奇数偶数 false 时间不连续 奇偶都有  所以一般建议用false

     * @return
     */
    public synchronized long nextId(boolean ifEvenNum) {
        long currStmp = getNewstmp();
        if (currStmp < lastStmp) {
            throw new RuntimeException("Clock moved backwards.  Refusing to generate id");
        }
        /**
         * 时间不连续出来全是偶数
         */
        if(ifEvenNum){
            if (currStmp == lastStmp) {
                //相同毫秒内，序列号自增
                sequence = (sequence + 1) & MAX_SEQUENCE;
                //同一毫秒的序列数已经达到最大
                if (sequence == 0L) {
                    currStmp = getNextMill();
                }
            } else {
                //不同毫秒内，序列号置为0
                sequence = 0L;
            }
        }else {
            //相同毫秒内，序列号自增
            sequence = (sequence + 1) & MAX_SEQUENCE;
        }

        lastStmp = currStmp;

        return (currStmp - START_STMP) << TIMESTMP_LEFT //时间戳部分
                | datacenterId << DATACENTER_LEFT       //数据中心部分
                | machineId << MACHINE_LEFT             //机器标识部分
                | sequence;                             //序列号部分
    }

    private long getNextMill() {
        long mill = getNewstmp();
        while (mill <= lastStmp) {
            mill = getNewstmp();
        }
        return mill;
    }

    private long getNewstmp() {
        return System.currentTimeMillis();
    }

    public static void main(String[] args)  throws Exception{
        /**
         * 分布式数据中心id
         * 机器id
         */
        SnowFlake snowFlake = new SnowFlake(5, 6);
        for (int i = 0; i < 10; i++) {
            /**
             * 时间连续 奇数偶数都有
             */
//            System.out.println(snowFlake.nextId(true));
//            System.out.println(snowFlake.nextId(true));
            /**
             * 时间不连续 原版 获取的id全是偶数
             */
//            Thread.sleep(1);
//            long snowFlakeId = snowFlake.nextId(false);
//            System.out.println(snowFlakeId);
//            /**
//             * 时间不连续 改版 获取的id为奇偶数
//             */
//            Thread.sleep(1);

        }
    }
}

修改批量添加方法

    /**
     * 批量添加
     *
     * @return
     */
    @GetMapping("/user/batchAdd")
    public Object add() {
        SnowFlake snowFlake = new SnowFlake(5, 6);
        for (int i = 0; i < 50; i++) {
            long id = snowFlake.nextId(false);
            User user = new User();
            user.setId(id);
            user.setUsername("batchName" + i);
            user.setPassword("12345" + i);
            userService.insertUser(user);
        }
        return "ok";
    }

实体类

package com.springjdbc.netity;

import lombok.Data;

/**
 * 用户实体类
 */
@Data
public class User {
    public Long id;

    public String username;

    public String password;


}

UserMapper.xml

<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
        PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.springjdbc.mapper.UserMapper">
    <resultMap type="User" id="UserResult">
        <result property="id" javaType="Long" column="id"/>
        <result property="username" column="username"/>
        <result property="password" column="password"/>
    </resultMap>

    <sql id="selectUserVo">
        select id,username,password
        from user
    </sql>

    <insert id="insertUser" parameterType="User">
		insert into user(id,username, password)
        values (#{id},#{username}, #{password})
	</insert>

    <select id="selectUserList" parameterType="User" resultMap="UserResult">
        <include refid="selectUserVo"/>
        <where>
            <if test="username != null and username != ''">
                AND username = #{username}
            </if>
            <if test="password != null and password != ''">
                AND password = #{password}
            </if>
        </where>
    </select>

</mapper>

启动项目测试：

id取余2为0的数据会分配到cool库

id取余2为1的数据会分配到cool2库

查询接口：

源码：https://gitee.com/hekang_admin/shardingsphere-demo.git

标签
Spring
Spring Boot
Sharding-JDBC
易学教程内所有资源均来自网络或用户发布的内容，如有违反法律规定的内容欢迎反馈！
该文章没有解决你所遇到的问题?点击提问,说说你的问题,让更多的人一起探讨吧!