Efficient FIFO queue for arbitrarily sized chunks of bytes in Python

后端 未结 4 924
既然无缘
既然无缘 2021-02-05 09:19

How do I implement a FIFO buffer to which I can efficiently add arbitrarily sized chunks of bytes to the head and from which I can efficiently pop arbitrarily sized chunks of by

4条回答
  •  心在旅途
    2021-02-05 09:38

    Update: Here's an implementation of the circular buffer technique from vartec's answer (building on my original answer, preserved below for those curious):

    from cStringIO import StringIO
    
    class FifoFileBuffer(object):
        def __init__(self):
            self.buf = StringIO()
            self.available = 0    # Bytes available for reading
            self.size = 0
            self.write_fp = 0
    
        def read(self, size = None):
            """Reads size bytes from buffer"""
            if size is None or size > self.available:
                size = self.available
            size = max(size, 0)
    
            result = self.buf.read(size)
            self.available -= size
    
            if len(result) < size:
                self.buf.seek(0)
                result += self.buf.read(size - len(result))
    
            return result
    
    
        def write(self, data):
            """Appends data to buffer"""
            if self.size < self.available + len(data):
                # Expand buffer
                new_buf = StringIO()
                new_buf.write(self.read())
                self.write_fp = self.available = new_buf.tell()
                read_fp = 0
                while self.size <= self.available + len(data):
                    self.size = max(self.size, 1024) * 2
                new_buf.write('0' * (self.size - self.write_fp))
                self.buf = new_buf
            else:
                read_fp = self.buf.tell()
    
            self.buf.seek(self.write_fp)
            written = self.size - self.write_fp
            self.buf.write(data[:written])
            self.write_fp += len(data)
            self.available += len(data)
            if written < len(data):
                self.write_fp -= self.size
                self.buf.seek(0)
                self.buf.write(data[written:])
            self.buf.seek(read_fp)
    

    Original answer (superseded by the one above):

    You can use a buffer and track the start index (read file pointer), occasionally compacting it when it gets too large (this should yield pretty good amortized performance).

    For example, wrap a StringIO object like so:

    from cStringIO import StringIO
    class FifoBuffer(object):
        def __init__(self):
            self.buf = StringIO()
    
        def read(self, *args, **kwargs):
            """Reads data from buffer"""
            self.buf.read(*args, **kwargs)
    
        def write(self, *args, **kwargs):
            """Appends data to buffer"""
            current_read_fp = self.buf.tell()
            if current_read_fp > 10 * 1024 * 1024:
                # Buffer is holding 10MB of used data, time to compact
                new_buf = StringIO()
                new_buf.write(self.buf.read())
                self.buf = new_buf
                current_read_fp = 0
    
            self.buf.seek(0, 2)    # Seek to end
            self.buf.write(*args, **kwargs)
    
            self.buf.seek(current_read_fp)
    

提交回复
热议问题