NumPy reading file with filtering lines on the fly

Question

I have a large array of numbers written in a CSV file and need to load only a slice of that array. Conceptually I want to call np.genfromtxt() and then row-slic

Answer 1

Trying to demonstrate comment to OP.

def fread(name, cond):
    with open(name) as file:
        for line in file:
            if cond(line):
                yield line.split()

def a_genfromtxt_cond(fname, cond=(lambda str: True)):
    """Seems to work without need to convert to float."""
    return np.array(list(fread(fname, cond)), dtype=np.float64)

def b_genfromtxt_cond(fname, cond=(lambda str: True)):
    r = [[int(float(i)) for i in l] for l in fread(fname, cond)]
    return np.array(r, dtype=np.integer)


a = a_genfromtxt_cond("tar.data")
print a
aa = b_genfromtxt_cond("tar.data")
print aa

Output

[[ 1.   2.3  4.5]
 [ 4.7  9.2  6.7]
 [ 4.7  1.8  4.3]]
[[1 2 4]
 [4 9 6]
 [4 1 4]]