我正在执行一些数据分析任务,使用此python脚本可以得到我想要的结果,但是它可能非常慢,可能是由于for循环所致,我必须处理数百万个数据行,有没有办法将此脚本更改为快速?
df=df.sort_values(by='ts')
df = df.set_index(pd.DatetimeIndex(df['ts']))
df = df.rename(columns={'ts': 'Time'})
x2=df.groupby(pd.Grouper(freq='1D',base=30,label='right'))
for name,df1 in x2:
df1_split=np.array_split(df1,2)
df_first=df1_split[0]
df_second=df1_split[1]
length_1=len(df_first)
length_2=len(df_second)
if len(df_first)>=5000:
df_first_diff_max=abs(df_first['A'].diff(periods=1)).max()
if df_first_diff_max<=10:
time_first=df_first['Time'].values[0]
time_first=pd.DataFrame([time_first],columns=['start_time'])
time_first['End_Time']=df_first['Time'].values[-1]
time_first['flag']=1
time_first['mean_B']=np.mean(df_first['B'])
time_first['mean_C']=np.mean(df_first['C'])
time_first['mean_D']=np.mean(df_first['D'])
time_first['E']=df_first['E'].values[0]
time_first['F']=df_first['F'].values[0]
result.append(time_first)
if len(df_second)>=5000:
df_second_diff_max=abs(df_second['A'].diff(periods=1)).max()
if df_second_diff_max<=10:
print('2')
time_first=df_second['Time'].values[0]
time_first=pd.DataFrame([time_first],columns=['start_time'])
time_first['End_Time']=df_second['Time'].values[-1]
time_first['flag']=2
time_first['mean_B']=np.mean(df_second['B'])
time_first['mean_C']=np.mean(df_second['C'])
time_first['mean_D']=np.mean(df_second['D'])
time_first['E']=df_second['E'].values[0]
time_first['F']=df_second['F'].values[0]
result.append(time_first)
final=pd.concat(result)