This is the output of the memory profiler of a function in my code, using xarray (v.0.16.1) datasets:
Line # Mem usage Increment Line Contents
================================================
139 94.195 MiB 94.195 MiB @profile
140 def getMaps(ncfile):
141 335.914 MiB 241.719 MiB myCMEMSdata = xr.open_dataset(ncfile).resample(time='3H').reduce(np.mean)
142
143 335.945 MiB 0.031 MiB plt.figure(figsize=(20.48, 10.24))
144
145 # projection, lat/lon extents and resolution of polygons to draw
146 # resolutions: c - crude, l - low, i - intermediate, h - high, f - full
147 336.809 MiB 0.863 MiB map = Basemap(projection='merc', llcrnrlon=-10.,
148 335.945 MiB 0.000 MiB llcrnrlat=30., urcrnrlon=36.5, urcrnrlat=46.)
149
150
151 339.773 MiB 2.965 MiB X, Y = np.meshgrid(myCMEMSdata.longitude.values,
152 336.809 MiB 0.000 MiB myCMEMSdata.latitude.values)
153 348.023 MiB 8.250 MiB x, y = map(X, Y)
154
155 # reduce arrows density (1 out of 15)
156 348.023 MiB 0.000 MiB yy = np.arange(0, y.shape[0], 15)
157 348.023 MiB 0.000 MiB xx = np.arange(0, x.shape[1], 15)
158 348.023 MiB 0.000 MiB points = np.meshgrid(yy,xx)
159
160 #cycle time to save maps
161 348.023 MiB 0.000 MiB i=0
162 742.566 MiB 0.000 MiB while i < myCMEMSdata.time.values.size:
163 742.566 MiB 305.996 MiB map.shadedrelief(scale=0.65)
164 #waves height
165 742.566 MiB 0.000 MiB waveH = myCMEMSdata.VHM0.values[i, :, :]
166 742.566 MiB 0.000 MiB my_cmap = plt.get_cmap('rainbow')
167 742.566 MiB 0.043 MiB map.pcolormesh(x, y, waveH, cmap=my_cmap, norm=matplotlib.colors.LogNorm(vmin=0.07, vmax=4.,clip=True))
168 # waves direction
169 742.566 MiB 0.000 MiB wDir = myCMEMSdata.VMDR.values[i, :, :]
170 742.566 MiB 0.242 MiB map.quiver(x[tuple(points)],y[tuple(points)],np.cos(np.deg2rad(270-wDir[tuple(points)])),np.sin(np.deg2rad(270-wDir[tuple(points)])),
171 742.566 MiB 0.000 MiB edgecolor='lightgray', minshaft=4, width=0.007, headwidth=3., headlength=4., linewidth=.5)
172 # save plot
173 742.566 MiB 0.000 MiB filename = pd.to_datetime(myCMEMSdata.time[i].values).strftime("%Y-%m-%d_%H")
174 742.566 MiB 0.086 MiB plt.show()
175 742.566 MiB 39.406 MiB plt.savefig(TEMPDIR+filename+".jpg", quality=75)
176 742.566 MiB 0.000 MiB plt.clf()
177 742.566 MiB 0.000 MiB del wDir
178 742.566 MiB 0.000 MiB del waveH
179 742.566 MiB 0.000 MiB i += 1
180
181 #out of loop
182 581.840 MiB 0.000 MiB plt.close("all")
183 581.840 MiB 0.000 MiB myCMEMSdata.close()
184 441.961 MiB 0.000 MiB del myCMEMSdata
As you can see the allocated memory is not freed up, and after many runs of the program, it simply fails ("Killed") for low memory.
How can I free the memory allocated by the dataset? I am using either dataset.close() and deleting the variable, with no success.