Hi José, The code is somewhat longish for a pure visual inspection, but my advice is that you install memory profiler ( https://pypi.python.org/pypi/memory_profiler). This will help you determine which line or lines are hugging the memory the most.
Saludos, Francesc On Fri, Aug 23, 2013 at 3:58 PM, Josè Luis Mietta < [email protected]> wrote: > Hi ecperts. I need your help with a RAM porblem during execution of my > script. > I wrote the next code. I use SAGE. In 1-2 hours of execution time the RAM > of my laptop (8gb) is filled and the sistem crash: > > from scipy.stats import uniform import numpy as np > > cant_de_cadenas =[700,800,900] > > cantidad_de_cadenas=np.array([]) > for kkkkk in cant_de_cadenas: > cantidad_de_cadenas=np.append(cantidad_de_cadenas,kkkkk) > > cantidad_de_cadenas=np.transpose(cantidad_de_cadenas) > > b=10 > h=bLongitud=1 > numero_experimentos=150 > > densidad_de_cadenas =cantidad_de_cadenas/(b**2) > > prob_perc=np.array([]) > > tiempos=np.array([]) > > S_int=np.array([]) > > S_medio=np.array([]) > > desviacion_standard=np.array([]) > > desviacion_standard_nuevo=np.array([]) > > anisotropia_macroscopica_porcentual=np.array([]) > > componente_y=np.array([]) > > componente_x=np.array([]) > import time > for N in cant_de_cadenas: > > empieza=time.clock() > > PERCOLACION=np.array([]) > > size_medio_intuitivo = np.array([]) > size_medio_nuevo = np.array([]) > std_dev_size_medio_intuitivo = np.array([]) > std_dev_size_medio_nuevo = np.array([]) > comp_y = np.array([]) > comp_x = np.array([]) > > > for u in xrange(numero_experimentos): > > > perco = False > > array_x1=uniform.rvs(loc=-b/2, scale=b, size=N) > array_y1=uniform.rvs(loc=-h/2, scale=h, size=N) > array_angle=uniform.rvs(loc=-0.5*(np.pi), scale=np.pi, size=N) > > > array_pendiente_x=1./np.tan(array_angle) > > random=uniform.rvs(loc=-1, scale=2, size=N) > lambda_sign=np.zeros([N]) > for t in xrange(N): > if random[t]<0: > lambda_sign[t]=-1 > else: > lambda_sign[t]=1 > array_lambdas=(lambda_sign*Longitud)/np.sqrt(1+array_pendiente_x**2) > > > array_x2= array_x1 + array_lambdas*array_pendiente_x > array_y2= array_y1 + array_lambdas*1 > > array_x1 = np.append(array_x1, [-b/2, b/2, -b/2, -b/2]) > array_y1 = np.append(array_y1, [-h/2, -h/2, -h/2, h/2]) > array_x2 = np.append(array_x2, [-b/2, b/2, b/2, b/2]) > array_y2 = np.append(array_y2, [h/2, h/2, -h/2, h/2]) > > M = np.zeros([N+4,N+4]) > > for j in xrange(N+4): > if j>0: > x_A1B1 = array_x2[j]-array_x1[j] > y_A1B1 = array_y2[j]-array_y1[j] > x_A1A2 = array_x1[0:j]-array_x1[j] > y_A1A2 = array_y1[0:j]-array_y1[j] > x_A2A1 = -1*x_A1A2 > y_A2A1 = -1*y_A1A2 > x_A2B2 = array_x2[0:j]-array_x1[0:j] > y_A2B2 = array_y2[0:j]-array_y1[0:j] > x_A1B2 = array_x2[0:j]-array_x1[j] > y_A1B2 = array_y2[0:j]-array_y1[j] > x_A2B1 = array_x2[j]-array_x1[0:j] > y_A2B1 = array_y2[j]-array_y1[0:j] > > p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2 > p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2 > p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1 > p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1 > > condicion_1=p1*p2 > condicion_2=p3*p4 > > for k in xrange (j): > if condicion_1[k]<=0 and condicion_2[k]<=0: > M[j,k]=1 > del condicion_1 > del condicion_2 > > > if j+1<N+4: > x_A1B1 = array_x2[j]-array_x1[j] > y_A1B1 = array_y2[j]-array_y1[j] > x_A1A2 = array_x1[j+1:]-array_x1[j] > y_A1A2 = array_y1[j+1:]-array_y1[j] > x_A2A1 = -1*x_A1A2 > y_A2A1 = -1*y_A1A2 > x_A2B2 = array_x2[j+1:]-array_x1[j+1:] > y_A2B2 = array_y2[j+1:]-array_y1[j+1:] > x_A1B2 = array_x2[j+1:]-array_x1[j] > y_A1B2 = array_y2[j+1:]-array_y1[j] > x_A2B1 = array_x2[j]-array_x1[j+1:] > y_A2B1 = array_y2[j]-array_y1[j+1:] > > p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2 > p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2 > p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1 > p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1 > > condicion_1=p1*p2 > condicion_2=p3*p4 > > for k in xrange ((N+4)-j-1): > if condicion_1[k]<=0 and condicion_2[k]<=0: > M[j,k+j+1]=1 > del condicion_1 > del condicion_2 > > M[N,N+2]=0 > M[N,N+3]=0 > M[N+1,N+2]=0 > M[N+1,N+3]=0 > M[N+2,N]=0 > M[N+2,N+1]=0 > M[N+3,N]=0 > M[N+3,N+1]=0 > > > CD=np.array([]) > > POPOPO=[] > for g in xrange(N): > > lala=0 > r=False > while lala<=len(POPOPO)-1: > esta= g in POPOPO[lala] > > if esta is True: > lala=len(POPOPO) > r=True > else: > lala=lala+1 > if r is False: > > > > L=np.array([g]) > for s in xrange(N): > if M[g,s] != 0: > L=np.append(L,s) > > x=0 > while x<= N: > for l in xrange(N): > z= l in L > d=L[x] > if z is False and M[d,l] != 0: > L=np.append(L,l) > if x+1<len(L): > x+=1 > else: > x=N+1. > > q= len (L) > CD=np.append(CD, q) > POPOPO.append(L) > > > M_horizontal=M.copy() > M_horizontal[:,N+2] = np.zeros(N+4) > M_horizontal[:,N+3] = np.zeros(N+4) > M_horizontal[N+2] = np.zeros(N+4) > M_horizontal[N+3] = np.zeros(N+4) > > > > L=np.array([N]) > for s in xrange(N+4): > if M_horizontal[N,s] != 0: > L=np.append(L,s) > > x=0 > while x<= N+4: > for l in xrange(N+4): > z= l in L > d=L[x] > if z is False and M_horizontal[d,l] != 0: > L=np.append(L,l) > if x+1<len(L): > x+=1 > else: > x=(N+4)+1. > > LV1_in_L = N in L > LV2_in_L= (N+1) in L > > if LV1_in_L is True and LV2_in_L is True: > perc_horiz=True > else: > perc_horiz=False > > > M_vertical=M.copy() > > M_vertical[:,N] = np.zeros(N+4) > M_vertical[:,N+1] = np.zeros(N+4) > M_vertical[N] = np.zeros(N+4) > M_vertical[N+1] = np.zeros(N+4) > > > > L=np.array([N+2]) > for s in xrange(N+4): > if M_vertical[N+2,s] != 0: > L=np.append(L,s) > > x=0 > while x<= N+4: > for l in xrange(N+4): > z= l in L > d=L[x] > if z is False and M_vertical[d,l] != 0: > L=np.append(L,l) > if x+1<len(L): > x+=1 > else: > x=(N+4)+1. > > LH1_in_L = (N+2) in L > LH2_in_L= (N+3) in L > > if LH1_in_L is True and LH2_in_L is True: > perc_ver = True > else: > perc_ver = False > > > > if perc_ver is True or perc_horiz is True: > PERCOLACION=np.append(PERCOLACION,1) > perco=True > > > D = np.array([]) > W = np.array([]) > > for c in xrange (int(min(CD)), int(max(CD)+1),1): > D=np.append(D,c) > frec = sum (CD == c) > W = np.append(W,frec) > > if perco is True: > posicion=np.argmax(D) > D=np.delete(D,posicion) > W=np.delete(W,posicion) > > if len(D) == 0 and len(W)==0: > S_medio_intuitivo_exp_u=0 > S_medio_nuevo_exp_u = 0 > std_dev_exp_u = 0 > std_dev_nuevo_exp_u = 0 > else: > > S_medio_intuitivo_exp_u = np.average (D,weights=W) > > peso_nuevo=D*W > > S_medio_nuevo_exp_u = np.average (D,weights=peso_nuevo) > > > tipos=sum(W) > X=W*((D-S_medio_intuitivo_exp_u)**2) > S=sum(X) > > std_dev_exp_u = np.sqrt(S/(tipos-1.)) > > tipos_nuevo=sum(peso_nuevo) > X_nuevo=peso_nuevo*((D-S_medio_nuevo_exp_u)**2) > S_nuevo=sum(X_nuevo) > > std_dev_nuevo_exp_u = np.sqrt(S_nuevo/(tipos_nuevo-1.)) > > > componente_longitudinal=Longitud*np.abs(np.cos(array_angle)) > comp_y=np.append(comp_y, sum(componente_longitudinal)/N) > > componente_transversal=Longitud*np.abs(np.sin(array_angle)) > comp_x=np.append(comp_x, sum(componente_transversal)/N) > > std_dev_size_medio_intuitivo=np.append(std_dev_size_medio_intuitivo, > std_dev_exp_u) > > std_dev_size_medio_nuevo=np.append(std_dev_size_medio_nuevo, > std_dev_nuevo_exp_u) > > size_medio_intuitivo=np.append(size_medio_intuitivo, > S_medio_intuitivo_exp_u) > > size_medio_nuevo=np.append(size_medio_nuevo, S_medio_nuevo_exp_u) > > > percolation_probability=sum(PERCOLACION)/numero_experimentos > > prob_perc=np.append(prob_perc, percolation_probability) > > S_int = np.append (S_int, sum(size_medio_intuitivo)/numero_experimentos) > > S_medio=np.append (S_medio, sum(size_medio_nuevo)/numero_experimentos) > > desviacion_standard = np.append (desviacion_standard, > sum(std_dev_size_medio_intuitivo)/numero_experimentos) > > desviacion_standard_nuevo=np.append (desviacion_standard_nuevo, > sum(std_dev_size_medio_nuevo)/numero_experimentos) > > tiempos=np.append(tiempos, time.clock()-empieza) > > componente_y=np.append(componente_y, sum(comp_y)/numero_experimentos) > componente_x=np.append(componente_x, sum(comp_x)/numero_experimentos) > > anisotropia_macroscopica_porcentual=100*(1-(componente_y/componente_x)) > > I tryed with gc and gc.collect() and 'del'command for deleting arrays > after his use and nothing work! > > What am I doing wrong? Why the memory becomes full while running (starts > with 10% of RAM used and in 1-2hour is totally full used)? > > Please help me, I'm totally stuck! > Thanks a lot! > > _______________________________________________ > NumPy-Discussion mailing list > [email protected] > http://mail.scipy.org/mailman/listinfo/numpy-discussion > > -- Francesc Alted
_______________________________________________ NumPy-Discussion mailing list [email protected] http://mail.scipy.org/mailman/listinfo/numpy-discussion
