Compress an InputStream with gzip

Question

I would like to compress an input stream in java using Gzip compression.

Let\'s say we have an input stream (1GB of data..) not compressed. I want as a result a comp

Answer 1

DeflaterInputStream is not what you want because it lacks gzip header/trailer and uses a slightly different compression.

If you change from OutputStream (push) to InputStream (pull) you need to do things different.

What GzipOutputStream does is:

• write a static gzip header
• write a deflated stream using DeflaterOutputStream. While the stream is written, a CRC32 checksum is built from the uncompressed data and the number of bytes is count
• write a trailer containing the CRC32 Checksum and the number of bytes.

If you want to do the same with InputStreams, you need a stream that contains:

• the header
• the deflated content
• the trailer

The best way to do this is to provide 3 different streams and combine them to one. Fortunately there is SequenceInputStream that does the combining of the streams for you.

Here's my implementation plus a simple unit test:

import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.SequenceInputStream;
import java.util.Enumeration;
import java.util.zip.CRC32;
import java.util.zip.Deflater;
import java.util.zip.DeflaterInputStream;
import java.util.zip.DeflaterOutputStream;

/**
 * @author mwyraz
 * Wraps an input stream and compresses it's contents. Similiar to DeflateInputStream but adds GZIP-header and trailer
 * See GzipOutputStream for details.
 * LICENSE: Free to use. Contains some lines from GzipOutputStream, so oracle's license might apply as well!
 */
public class GzipCompressingInputStream extends SequenceInputStream
{
    public GzipCompressingInputStream(InputStream in) throws IOException
    {
        this(in,512);
    }
    public GzipCompressingInputStream(InputStream in, int bufferSize) throws IOException
    {
        super(new StatefullGzipStreamEnumerator(in,bufferSize));
    }

    static enum StreamState
    {
        HEADER,
        CONTENT,
        TRAILER
    }

    protected static class StatefullGzipStreamEnumerator implements Enumeration<InputStream>
    {

        protected final InputStream in;
        protected final int bufferSize;
        protected StreamState state;

        public StatefullGzipStreamEnumerator(InputStream in, int bufferSize)
        {
            this.in=in;
            this.bufferSize=bufferSize;
            state=StreamState.HEADER;
        }

        public boolean hasMoreElements()
        {
            return state!=null;
        }
        public InputStream nextElement()
        {
            switch (state)
            {
                case HEADER:
                    state=StreamState.CONTENT;
                    return createHeaderStream();
                case CONTENT:
                    state=StreamState.TRAILER;
                    return createContentStream();
                case TRAILER:
                    state=null;
                    return createTrailerStream();
            }
            return null;
        }

        static final int GZIP_MAGIC = 0x8b1f;
        static final byte[] GZIP_HEADER=new byte[] {
                (byte) GZIP_MAGIC,        // Magic number (short)
                (byte)(GZIP_MAGIC >> 8),  // Magic number (short)
                Deflater.DEFLATED,        // Compression method (CM)
                0,                        // Flags (FLG)
                0,                        // Modification time MTIME (int)
                0,                        // Modification time MTIME (int)
                0,                        // Modification time MTIME (int)
                0,                        // Modification time MTIME (int)
                0,                        // Extra flags (XFLG)
                0                         // Operating system (OS)
        };
        protected InputStream createHeaderStream()
        {
            return new ByteArrayInputStream(GZIP_HEADER);
        }
        protected InternalGzipCompressingInputStream contentStream;
        protected InputStream createContentStream()
        {
            contentStream=new InternalGzipCompressingInputStream(new CRC32InputStream(in), bufferSize);
            return contentStream;
        }
        protected InputStream createTrailerStream()
        {
            return new ByteArrayInputStream(contentStream.createTrailer());
        }
    }

    /**
     * Internal stream without header/trailer  
     */
    protected static class CRC32InputStream extends FilterInputStream
    {
        protected CRC32 crc = new CRC32();
        protected long byteCount;
        public CRC32InputStream(InputStream in)
        {
            super(in);
        }

        @Override
        public int read() throws IOException
        {
            int val=super.read();
            if (val>=0)
            {
                crc.update(val);
                byteCount++;
            }
            return val;
        }
        @Override
        public int read(byte[] b, int off, int len) throws IOException
        {
            len=super.read(b, off, len);
            if (len>=0)
            {
                crc.update(b,off,len);
                byteCount+=len;
            }
            return len;
        }
        public long getCrcValue()
        {
            return crc.getValue();
        }
        public long getByteCount()
        {
            return byteCount;
        }
    }

    /**
     * Internal stream without header/trailer  
     */
    protected static class InternalGzipCompressingInputStream extends DeflaterInputStream
    {
        protected final CRC32InputStream crcIn;
        public InternalGzipCompressingInputStream(CRC32InputStream in, int bufferSize)
        {
            super(in, new Deflater(Deflater.DEFAULT_COMPRESSION, true),bufferSize);
            crcIn=in;
        }
        public void close() throws IOException
        {
            if (in != null)
            {
                try
                {
                    def.end();
                    in.close();
                }
                finally
                {
                    in = null;
                }
            }
        }

        protected final static int TRAILER_SIZE = 8;

        public byte[] createTrailer()
        {
            byte[] trailer= new byte[TRAILER_SIZE];
            writeTrailer(trailer, 0);
            return trailer;
        }

        /*
         * Writes GZIP member trailer to a byte array, starting at a given
         * offset.
         */
        private void writeTrailer(byte[] buf, int offset)
        {
            writeInt((int)crcIn.getCrcValue(), buf, offset); // CRC-32 of uncompr. data
            writeInt((int)crcIn.getByteCount(), buf, offset + 4); // Number of uncompr. bytes
        }

        /*
         * Writes integer in Intel byte order to a byte array, starting at a
         * given offset.
         */
        private void writeInt(int i, byte[] buf, int offset)
        {
            writeShort(i & 0xffff, buf, offset);
            writeShort((i >> 16) & 0xffff, buf, offset + 2);
        }

        /*
         * Writes short integer in Intel byte order to a byte array, starting
         * at a given offset
         */
        private void writeShort(int s, byte[] buf, int offset)
        {
            buf[offset] = (byte)(s & 0xff);
            buf[offset + 1] = (byte)((s >> 8) & 0xff);
        }
    }

}

---

import static org.junit.Assert.*;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.zip.CRC32;
import java.util.zip.GZIPInputStream;

import org.junit.Test;

public class TestGzipCompressingInputStream
{

    @Test
    public void test() throws Exception
    {
        testCompressor("test1 test2 test3");
        testCompressor("1MB binary data",createTestPattern(1024*1024));
        for (int i=0;i<4096;i++)
        {
            testCompressor(i+" bytes of binary data",createTestPattern(i));
        }
    }

    protected byte[] createTestPattern(int size)
    {
        byte[] data=new byte[size];
        byte pattern=0;
        for (int i=0;i<size;i++)
        {
            data[i]=pattern++;
        }
        return data;
    }

    protected void testCompressor(String data) throws IOException
    {
        testCompressor("String: "+data,data.getBytes());
    }
    protected void testCompressor(String dataInfo, byte[] data) throws IOException
    {
        InputStream uncompressedIn=new ByteArrayInputStream(data);
        InputStream compressedIn=new GzipCompressingInputStream(uncompressedIn);
        InputStream uncompressedOut=new GZIPInputStream(compressedIn);

        byte[] result=StreamHelper.readBinaryStream(uncompressedOut);

        assertTrue("Test failed for: "+dataInfo,Arrays.equals(data,result));

    }

}

Answer 2

A working example of a compressing input stream can be found in the popular open source ESB Mule: GZIPCompressorInputStream.

It uses the DeflaterInputStream provided by the JRE for compression, prepends the gzip header and appends the gzip trailer (aka footer).

Unfortunately, it is under CPA License, which does not seem to be very common. In addition, there seems to be no unit test.

Answer 3

If you dont want to load content into large byte array and need true streaming solution :

package x.y.z;

import org.apache.commons.io.IOUtils;

import java.io.*;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipOutputStream;

/**
 * Stream Compression Utility
 *
 * @author Thamme Gowda N
 */
public enum CompressionUtil {
    INSTANCE;

    public static final int NUM_THREADS = 5;
    private final ExecutorService pool;

    CompressionUtil(){
        this.pool = Executors.newFixedThreadPool(NUM_THREADS);
    }

    public static CompressionUtil getInstance(){
        return INSTANCE;
    }

    /**
     * Supported compression type names
     */
    public static enum CompressionType {
        GZIP,
        ZIP
    }

    /**
     * Wraps the given stream in a Compressor stream based on given type
     * @param sourceStream : Stream to be wrapped
     * @param type         : Compression type
     * @return source stream wrapped in a compressor stream
     * @throws IOException when some thing bad happens
     */
    public static OutputStream getCompressionWrapper(OutputStream sourceStream,
                                     CompressionType type) throws IOException {

        switch (type) {
            case GZIP:
                return new GZIPOutputStream(sourceStream);
            case ZIP:
                return new ZipOutputStream(sourceStream);
            default:
                throw new IllegalArgumentException("Possible values :"
                        + Arrays.toString(CompressionType.values()));
        }
    }

    /**
     * Gets Compressed Stream for given input Stream
     * @param sourceStream  : Input Stream to be compressed to
     * @param type: Compression types such as GZIP
     * @return  Compressed Stream
     * @throws IOException when some thing bad happens
     */
    public static InputStream getCompressedStream(final InputStream sourceStream,
                                    CompressionType type ) throws IOException {

        if(sourceStream == null) {
            throw new IllegalArgumentException("Source Stream cannot be NULL");
        }

        /**
         *  sourceStream --> zipperOutStream(->intermediateStream -)--> resultStream
         */
        final PipedInputStream resultStream = new PipedInputStream();
        final PipedOutputStream intermediateStream = new PipedOutputStream(resultStream);
        final OutputStream zipperOutStream = getCompressionWrapper(intermediateStream, type);

        Runnable copyTask = new Runnable() {

            @Override
            public void run() {
                try {
                    int c;
                    while((c = sourceStream.read()) >= 0) {
                        zipperOutStream.write(c);
                    }
                    zipperOutStream.flush();
                } catch (IOException e) {
                    IOUtils.closeQuietly(resultStream);  // close it on error case only
                    throw new RuntimeException(e);
                } finally {
                    // close source stream and intermediate streams
                    IOUtils.closeQuietly(sourceStream);
                    IOUtils.closeQuietly(zipperOutStream);
                    IOUtils.closeQuietly(intermediateStream);
                }
            }
        };
        getInstance().pool.submit(copyTask);
        return resultStream;
    }

    public static void main(String[] args) throws IOException {
        String input = "abcdefghij";
        InputStream sourceStream = new ByteArrayInputStream(input.getBytes());
        InputStream compressedStream =
                getCompressedStream(sourceStream, CompressionType.GZIP);

        GZIPInputStream decompressedStream = new GZIPInputStream(compressedStream);
        List<String> lines = IOUtils.readLines(decompressedStream);
        String output = lines.get(0);
        System.out.println("test passed ? " + input.equals(output));

    }
}

Answer 4

You can use EasyStream.

try(final InputStreamFromOutputStream<Void> isOs = new InputStreamFromOutputStream<Void>() {
    @Override
    protected void produce(final OutputStream dataSink) throws Exception {
        InputStream in = new GZIPInputStream(unCompressedStream);
        IOUtils.copy(in, dataSink);
    }
}) {        

    //You can use the compressed input stream here

} catch (final IOException e) {
    //Handle exceptions here
} 

Answer 5

I would recommend using GzipCompressorInputStream from Apache Commons Compress.

Answer 6

There is no DeflatingGZIPInputStream in the JRE. To deflate with the "deflate" compression format, use java.util.zip.DeflaterInputStream and java.util.zip.DeflaterOutputStream:

public InputStream getCompressedStream(InputStream unCompressedStream) {
    return new DeflaterInputStream(unCompressedStream); 
}

You could derive a class from java.util.zip.DeflaterInputStream that deflates in GZIP format by looking at the the source of java.util.zip.GZIPOutputStream.