How to use `numpy.savez` in a loop for save more than one array?

Question

From a loop I\'m getting an array. I want to save this arrays in a tempfile. The problem is that np.savez only saves the last array from the loop.

Answer 1

It is also possible to use custom keys by using ** operator.

import numpy as np

a1 = [1,2,3]
a2 = [10,20,30]

savez_dict = dict()

for i in ['a1', 'a2']:
    savez_dict['key_'+i] = i 

np.savez("t.npz", **savez_dict)

Answer 2

Sorry for my English in advance.

Because the function savez opens the file, writes all variables, then close the file, data are over-written when it called.

savez is simple. you can find the code at https://github.com/numpy/numpy/blob/master/numpy/lib/npyio.py

how about implementing "your_own_savez", then use the following code.

tmp = TemporaryFile()
f = my_savez(tmp)    
for i in range(10):
    array = getarray[i]  #demo purpose
    f.savez(array)
f.close()

tmp.seek(0)
tmp_read = np.load(tmp)
print tmp_read.files

Here is my quick and dirty code.

import numpy as np
import tempfile

class my_savez(object):
    def __init__(self, file):
        # Import is postponed to here since zipfile depends on gzip, an optional
        # component of the so-called standard library.
        import zipfile
        # Import deferred for startup time improvement
        import tempfile
        import os

        if isinstance(file, basestring):
            if not file.endswith('.npz'):
                file = file + '.npz'

        compression = zipfile.ZIP_STORED

        zip = self.zipfile_factory(file, mode="w", compression=compression)

        # Stage arrays in a temporary file on disk, before writing to zip.
        fd, tmpfile = tempfile.mkstemp(suffix='-numpy.npy')
        os.close(fd)

        self.tmpfile = tmpfile
        self.zip = zip
        self.i = 0

    def zipfile_factory(self, *args, **kwargs):
        import zipfile
        import sys
        if sys.version_info >= (2, 5):
            kwargs['allowZip64'] = True
        return zipfile.ZipFile(*args, **kwargs)

    def savez(self, *args, **kwds):
        import os
        import numpy.lib.format as format

        namedict = kwds
        for val in args:
            key = 'arr_%d' % self.i
            if key in namedict.keys():
                raise ValueError("Cannot use un-named variables and keyword %s" % key)
            namedict[key] = val
            self.i += 1

        try:
            for key, val in namedict.iteritems():
                fname = key + '.npy'
                fid = open(self.tmpfile, 'wb')
                try:
                    format.write_array(fid, np.asanyarray(val))
                    fid.close()
                    fid = None
                    self.zip.write(self.tmpfile, arcname=fname)
                finally:
                    if fid:
                        fid.close()
        finally:
            os.remove(self.tmpfile)

    def close(self):
        self.zip.close()

tmp = tempfile.TemporaryFile()
f = my_savez(tmp)
for i in range(10):
  array = np.zeros(10)
  f.savez(array)
f.close()

tmp.seek(0)

tmp_read = np.load(tmp)
print tmp_read.files
for k, v in tmp_read.iteritems():
     print k, v

Answer 3

I am not an experienced programmer, but this is the way I did it (just in case it may help someone in the future). In addition, it is the first time that I am posting here, so I apologize if I am not following some kind of standard ;)

Creating the npz file:

import numpy as np

tmp = file("C:\\Windows\\Temp\\temp_npz.npz",'wb')

# some variables
a= [23,4,67,7]
b= ['w','ww','wwww']
c= np.ones((2,6))

# a lit containing the name of your variables
var_list=['a','b','c']

# save the npz file with the variables you selected
str_exec_save = "np.savez(tmp,"    
for i in range(len(var_list)):    
    str_exec_save += "%s = %s," % (var_list[i],var_list[i])
str_exec_save += ")"
exec(str_exec_save)

tmp.close

Loading the variables with their original names:

import numpy as np
import tempfile

tmp = open("C:\\Windows\\Temp\\temp_npz.npz",'rb')

# loading of the saved variables
var_load = np.load(tmp)

# getting the name of the variables
files = var_load.files

# loading then with their original names
for i in range(len(files)):
    exec("%s = var_load['%s']" % (files[i],files[i]) )

The only difference is that the variables will become numpy variables.

Answer 4

You can use the *args arguments to save many arrays in only one temp file.

np.savez(tmp, *getarray[:10])

or:

np.savez(tmp, *[getarray[0], getarray[1], getarray[8]])