Keeping large dictionary in Python affects application performance

Question

I\'m having some difficulties understanding (and ultimately solving) why having a large dictionary in memory makes creation of other dictionaries longer.

Here\'s the tes

Answer 1

Modify program like this to inspect bytecode :

import time
import dis
import inspect

def create_dict():
    return {x:[x]*125 for x in xrange(0, 100000)}


class Foo(object):
    @staticmethod
    def dict_init():
        start = time.clock()
        Foo.sample_dict = create_dict()
        print "dict_init in Foo took {0} sec".format(time.clock() - start)
        dis.dis(inspect.currentframe().f_code)

if __name__ == '__main__':
    Foo.dict_init()
    for x in xrange(0, 1):
        start = time.clock()
        create_dict()
        print "Run {0} took {1} seconds".format(x, time.clock() - start)
        dis.dis(inspect.currentframe().f_code)

Here is the output :

dict_init in Foo took 0.44164 sec
 12           0 LOAD_GLOBAL              0 (time)
              3 LOAD_ATTR                1 (clock)
              6 CALL_FUNCTION            0
              9 STORE_FAST               0 (start)

 13          12 LOAD_GLOBAL              2 (create_dict)
             15 CALL_FUNCTION            0
             18 LOAD_GLOBAL              3 (Foo)
             21 STORE_ATTR               4 (sample_dict)

 14          24 LOAD_CONST               1 ('dict_init in Foo took {0} sec')
             27 LOAD_ATTR                5 (format)
             30 LOAD_GLOBAL              0 (time)
             33 LOAD_ATTR                1 (clock)
             36 CALL_FUNCTION            0
             39 LOAD_FAST                0 (start)
             42 BINARY_SUBTRACT     
             43 CALL_FUNCTION            1
             46 PRINT_ITEM          
             47 PRINT_NEWLINE       

 15          48 LOAD_GLOBAL              6 (dis)
             51 LOAD_ATTR                6 (dis)
             54 LOAD_GLOBAL              7 (inspect)
             57 LOAD_ATTR                8 (currentframe)
             60 CALL_FUNCTION            0
             63 LOAD_ATTR                9 (f_code)
             66 CALL_FUNCTION            1
             69 POP_TOP             
             70 LOAD_CONST               0 (None)
             73 RETURN_VALUE        
Run 0 took 0.641144 seconds
  1           0 LOAD_CONST               0 (-1)
              3 LOAD_CONST               1 (None)
              6 IMPORT_NAME              0 (time)
              9 STORE_NAME               0 (time)

  2          12 LOAD_CONST               0 (-1)
             15 LOAD_CONST               1 (None)
             18 IMPORT_NAME              1 (dis)
             21 STORE_NAME               1 (dis)

  3          24 LOAD_CONST               0 (-1)
             27 LOAD_CONST               1 (None)
             30 IMPORT_NAME              2 (inspect)
             33 STORE_NAME               2 (inspect)

  5          36 LOAD_CONST               2 ()
             39 MAKE_FUNCTION            0
             42 STORE_NAME               3 (create_dict)

  9          45 LOAD_CONST               3 ('Foo')
             48 LOAD_NAME                4 (object)
             51 BUILD_TUPLE              1
             54 LOAD_CONST               4 ()
             57 MAKE_FUNCTION            0
             60 CALL_FUNCTION            0
             63 BUILD_CLASS         
             64 STORE_NAME               5 (Foo)

 17          67 LOAD_NAME                6 (__name__)
             70 LOAD_CONST               5 ('__main__')
             73 COMPARE_OP               2 (==)
             76 POP_JUMP_IF_FALSE      186

 18          79 LOAD_NAME                5 (Foo)
             82 LOAD_ATTR                7 (dict_init)
             85 CALL_FUNCTION            0
             88 POP_TOP             

 19          89 SETUP_LOOP              94 (to 186)
             92 LOAD_NAME                8 (xrange)
             95 LOAD_CONST               6 (0)
             98 LOAD_CONST               7 (1)
            101 CALL_FUNCTION            2
            104 GET_ITER            
        >>  105 FOR_ITER                74 (to 182)
            108 STORE_NAME               9 (x)

 20         111 LOAD_NAME                0 (time)
            114 LOAD_ATTR               10 (clock)
            117 CALL_FUNCTION            0
            120 STORE_NAME              11 (start)

 21         123 LOAD_NAME                3 (create_dict)
            126 CALL_FUNCTION            0
            129 POP_TOP             

 22         130 LOAD_CONST               8 ('Run {0} took {1} seconds')
            133 LOAD_ATTR               12 (format)
            136 LOAD_NAME                9 (x)
            139 LOAD_NAME                0 (time)
            142 LOAD_ATTR               10 (clock)
            145 CALL_FUNCTION            0
            148 LOAD_NAME               11 (start)
            151 BINARY_SUBTRACT     
            152 CALL_FUNCTION            2
            155 PRINT_ITEM          
            156 PRINT_NEWLINE       

 23         157 LOAD_NAME                1 (dis)
            160 LOAD_ATTR                1 (dis)
            163 LOAD_NAME                2 (inspect)
            166 LOAD_ATTR               13 (currentframe)
            169 CALL_FUNCTION            0
            172 LOAD_ATTR               14 (f_code)
            175 CALL_FUNCTION            1
            178 POP_TOP             
            179 JUMP_ABSOLUTE          105
        >>  182 POP_BLOCK           
            183 JUMP_FORWARD             0 (to 186)
        >>  186 LOAD_CONST               1 (None)
            189 RETURN_VALUE        

Maybe it is the difference in the format of the string that is causing the difference when garbage collection is off.