增量capacity分配的ByteBuffer实现

文章目录

• 前言
• Ozone内部的增量ByteBuffer实现
• 引用

前言

---

对于Java nio ByteBuffer，我们常常会拿来做缓冲数据的处理。如果我们就为了图方便，每次数据读写操作专门allocate一个比较大capacity的ByteBuffer，这样会造成不必要的JVM heap的浪费。但是如果我们转而变为多个小ByteBuffer的动态申请，又会加大ByteBuffer的管理协调 操作。那么有什么办法能结合上述二者的特点，做到既不浪费JVM heap空间的使用，又不用在业务上执行复杂的ByteBuffer逻辑。本文笔者介绍一个Ozone内部实现的增量ByteBuffer的实现。增量ByeBuffer在外部使用上和ByteBuffe原生r操作方法语义完全兼容，内部增量allocate capacity操作对调用方而言完全透明。

Ozone内部的增量ByteBuffer实现

---

这里简单介绍下Ozone的背景，Ozone作为对象存储系统，在存储对象文件的时候，会涉及到大量小数据对象的写入，以Chunk文件的形式进行物理存储。在读写chunk数据的过程中，Ozone使用了ByteBuffer做中间数据的存储。在初始实现中，Ozone内部初始的ByteBuffer allocate的capacity是比较大的，也不管用户写入的数据有多大。为此，后来社区实现了能够指定目标capacity，同时指定动态增量分配大小的ByteBuffer，名为IncrementalChunkBuffer。

以下为IncrementalChunkBuffer的实现代码，摘自Ozone项目：

/**
 * Use a list of {@link ByteBuffer} to implement a single {@link ChunkBuffer}
 * so that the buffer can be allocated incrementally.
 */
final class IncrementalChunkBuffer implements ChunkBuffer {
  /**
   * 全局Buffer的limit边界值
   */
  private final int limit;
  /** ByteBuffer的增量容量. */
  private final int increment;
  /** BytesBuffer下标数组边界值. */
  private final int limitIndex;
  /** 增量分配ByteBuffer数组 */
  private final List<ByteBuffer> buffers;
  /** Is this a duplicated buffer? (for debug only) */
  private final boolean isDuplicated;
  
  // 增量ByteBuffer总capacity(传入的limit值)，每次动态增量increment值大小
  IncrementalChunkBuffer(int limit, int increment, boolean isDuplicated) {
    Preconditions.checkArgument(limit >= 0);
    Preconditions.checkArgument(increment > 0);
    // 初始化边界值，增量ByteBuffer值
    this.limit = limit;
    this.increment = increment;
    // 计算ByteBuffer数组下标最大值
    this.limitIndex = limit/increment;
    // 初始空ByteBuffer数组
    this.buffers = new ArrayList<>(limitIndex + (limit%increment == 0? 0: 1));
    this.isDuplicated = isDuplicated;
  }

  /** @return the capacity for the buffer at the given index. */
  private int getBufferCapacityAtIndex(int i) {
    Preconditions.checkArgument(i >= 0);
    Preconditions.checkArgument(i <= limitIndex);
    return i < limitIndex? increment: limit%increment;
  }

  private void assertInt(int expected, int computed, String name, int i) {
    ChunkBuffer.assertInt(expected, computed,
        () -> this + ": Unexpected " + name + " at index " + i);
  }

  /** @return the i-th buffer if it exists; otherwise, return null. */
  private ByteBuffer getAtIndex(int i) {
    Preconditions.checkArgument(i >= 0);
    Preconditions.checkArgument(i <= limitIndex);
    final ByteBuffer ith = i < buffers.size() ? buffers.get(i) : null;
    if (ith != null) {
      // assert limit/capacity
      if (!isDuplicated) {
        assertInt(getBufferCapacityAtIndex(i), ith.capacity(), "capacity", i);
      } else {
        if (i < limitIndex) {
          assertInt(increment, ith.capacity(), "capacity", i);
        } else if (i == limitIndex) {
          assertInt(getBufferCapacityAtIndex(i), ith.limit(), "capacity", i);
        } else {
          assertInt(0, ith.limit(), "capacity", i);
        }
      }
    }
    return ith;
  }

  /** @return the i-th buffer. It may allocate buffers. */
  private ByteBuffer getAndAllocateAtIndex(int index) {
    Preconditions.checkArgument(index >= 0);
    // never allocate over limit
    if (limit % increment == 0) {
      Preconditions.checkArgument(index < limitIndex);
    } else {
      Preconditions.checkArgument(index <= limitIndex);
    }

    int i = buffers.size();
    if (index < i) {
      return getAtIndex(index);
    }

    // allocate upto the given index
    ByteBuffer b = null;
    for (; i <= index; i++) {
      b = ByteBuffer.allocate(getBufferCapacityAtIndex(i));
      buffers.add(b);
    }
    return b;
  }

  /** @return the buffer containing the position. It may allocate buffers. */
  private ByteBuffer getAndAllocateAtPosition(int position) {
    Preconditions.checkArgument(position >= 0);
    Preconditions.checkArgument(position < limit);
    // 计算需要获取ByteBuffer的下标
    final int i = position / increment;
    // 得到此下标对应的ByteBuffer，如果没有创建，则进行ByteBuffer的创建操作
    final ByteBuffer ith = getAndAllocateAtIndex(i);
    assertInt(position%increment, ith.position(), "position", i);
    return ith;
  }

  /** @return the index of the first non-full buffer. */
  private int firstNonFullIndex() {
    for (int i = 0; i < buffers.size(); i++) {
      if (getAtIndex(i).position() != increment) {
        return i;
      }
    }
    return buffers.size();
  }

  @Override
  public int position() {
    // The buffers list must be in the following orders:
    // full buffers, buffer containing the position, empty buffers, null buffers
    // 1）寻找第一个不满的ByteBuffer下标
    final int i = firstNonFullIndex();
    // 2）获取此ByteBuffer项
    final ByteBuffer ith = getAtIndex(i);
    // 3）计算当前全局position位置，前面i个满ByteBuffer的长度+当前不满的ByteBuffer的position位置
    final int position = i * increment + Optional.ofNullable(ith)
        .map(ByteBuffer::position).orElse(0);
    // remaining buffers must be empty
    assert assertRemainingList(ith, i);
    return position;
  }

  private boolean assertRemainingList(ByteBuffer ith, int i) {
    if (ith != null) {
      // buffers must be empty
      for (i++; i < buffers.size(); i++) {
        ith = getAtIndex(i);
        if (ith == null) {
          break; // found the first non-null
        }
        assertInt(0, ith.position(), "position", i);
      }
    }
    final int j = i;
    ChunkBuffer.assertInt(buffers.size(), i,
        () -> "i = " + j + " != buffers.size() = " + buffers.size());
    return true;
  }

  @Override
  public int remaining() {
    // remaining操作和原ByteBuffer语义一致
    return limit - position();
  }

  @Override
  public int limit() {
    return limit;
  }

  @Override
  public ChunkBuffer rewind() {
    buffers.forEach(ByteBuffer::rewind);
    return this;
  }

  @Override
  public ChunkBuffer clear() {
    buffers.forEach(ByteBuffer::clear);
    return this;
  }

  @Override
  public ChunkBuffer put(ByteBuffer that) {
    // 1）判断待put操作的ByteBuffer中的剩余数据空间是否超过增量ByteBuffer当前剩余空间
    // 如果超过，则抛出BufferOverflowException异常
    if (that.remaining() > this.remaining()) {
      final BufferOverflowException boe = new BufferOverflowException();
      boe.initCause(new IllegalArgumentException(
          "Failed to put since that.remaining() = " + that.remaining()
              + " > this.remaining() = " + this.remaining()));
      throw boe;
    }

    // 2）得到待写入ByteBuffer的边界值
    final int thatLimit = that.limit();
    for(int p = position(); that.position() < thatLimit;) {
      // 3）得到当前增量Buffer内正在提供写操作的ByteBuffer
      final ByteBuffer b = getAndAllocateAtPosition(p);
      // 4）比较剩余数据空间的大小，选取上述2个Buffer的较小值
      final int min = Math.min(b.remaining(), thatLimit - that.position());
      that.limit(that.position() + min);
      // 进行ByteBuffer的写入操作
      b.put(that);
      // 更新增量ByteBuffer当前的position位置
      p += min;
    }
    return this;
  }

  @Override
  public ChunkBuffer duplicate(int newPosition, int newLimit) {
    Preconditions.checkArgument(newPosition >= 0);
    Preconditions.checkArgument(newPosition <= newLimit);
    Preconditions.checkArgument(newLimit <= limit);
    final IncrementalChunkBuffer duplicated = new IncrementalChunkBuffer(
        newLimit, increment, true);

    final int pi = newPosition / increment;
    final int pr = newPosition % increment;
    final int li = newLimit / increment;
    final int lr = newLimit % increment;
    final int newSize = lr == 0? li: li + 1;

    for (int i = 0; i < newSize; i++) {
      final int pos = i < pi ? increment : i == pi ? pr : 0;
      final int lim = i < li ? increment : i == li ? lr : 0;
      duplicated.buffers.add(duplicate(i, pos, lim));
    }
    return duplicated;
  }

  private ByteBuffer duplicate(int i, int pos, int lim) {
    final ByteBuffer ith = getAtIndex(i);
    Objects.requireNonNull(ith, () -> "buffers[" + i + "] == null");
    final ByteBuffer b = ith.duplicate();
    b.position(pos).limit(lim);
    return b;
  }

  /** Support only when bufferSize == increment. */
  @Override
  public Iterable<ByteBuffer> iterate(int bufferSize) {
    if (bufferSize != increment) {
      throw new UnsupportedOperationException(
          "Buffer size and increment mismatched: bufferSize = " + bufferSize
          + " but increment = " + increment);
    }
    return asByteBufferList();
  }

  @Override
  public List<ByteBuffer> asByteBufferList() {
    return Collections.unmodifiableList(buffers);
  }

  @Override
  public long writeTo(GatheringByteChannel channel) throws IOException {
    return channel.write(buffers.toArray(new ByteBuffer[0]));
  }

  @Override
  public ByteString toByteStringImpl(Function<ByteBuffer, ByteString> f) {
    return buffers.stream().map(f).reduce(ByteString.EMPTY, ByteString::concat);
  }

  @Override
  public boolean equals(Object obj) {
    if (this == obj) {
      return true;
    } else if (!(obj instanceof IncrementalChunkBuffer)) {
      return false;
    }
    final IncrementalChunkBuffer that = (IncrementalChunkBuffer)obj;
    return this.limit == that.limit && this.buffers.equals(that.buffers);
  }

  @Override
  public int hashCode() {
    return buffers.hashCode();
  }

  @Override
  public String toString() {
    return getClass().getSimpleName()
        + ":limit=" + limit + ",increment=" + increment;
  }
}

引用

---

[1].https://github.com/apache/hadoop-ozone/blob/master/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/IncrementalChunkBuffer.java

来源：CSDN

作者：Android路上的人

链接：https://blog.csdn.net/Androidlushangderen/article/details/104808634