广告行业中那些趣事系列51：超牛的kaggle比赛Favorita Grocery Sales Forecasting冠军方案

# 计算不同时间窗口的特征
def get_timespan(df, dt, minus, periods,freq='D'):
   df_result = df[pd.date_range(dt - timedelta(days=minus),periods=periods, freq=freq)]
   return df_result
 
def prepare_dataset(df, promo_df, t2017,is_train=True, name_prefix=None):
    X= {
       # 以t2017为起点，最近14/60/140天促销汇总
       "promo_14_2017": get_timespan(promo_df, t2017, 14,14).sum(axis=1).values,
       "promo_60_2017": get_timespan(promo_df, t2017, 60,60).sum(axis=1).values,
       "promo_140_2017": get_timespan(promo_df, t2017, 140,140).sum(axis=1).values,
       # 以t2017为起点，后3/7/14天促销汇总
       "promo_3_2017_aft": get_timespan(promo_df, t2017 +timedelta(days=16), 15, 3).sum(axis=1).values,
       "promo_7_2017_aft": get_timespan(promo_df, t2017 +timedelta(days=16), 15, 7).sum(axis=1).values,
       "promo_14_2017_aft": get_timespan(promo_df, t2017 +timedelta(days=16), 15, 14).sum(axis=1).values,
    }
   
    #t2017为起点
   for i in [3, 7, 14, 30, 60, 140]:
       tmp = get_timespan(df, t2017, i, i)
       # 最近i天里和前一天销量差值的均值
       X['diff_%s_mean' % i] = tmp.diff(axis=1).mean(axis=1).values
       # 最近i天里销量每天按0.9衰减之后汇总       *******************************
       X['mean_%s_decay' % i] = (tmp * np.power(0.9,np.arange(i)[::-1])).sum(axis=1).values
       # 最近i天里均值、中位数、最小值、最大值和标准偏差
       X['mean_%s' % i] = tmp.mean(axis=1).values
       X['median_%s' % i] = tmp.median(axis=1).values
       X['min_%s' % i] = tmp.min(axis=1).values
       X['max_%s' % i] = tmp.max(axis=1).values
       X['std_%s' % i] = tmp.std(axis=1).values
   
    #t2017上一周，前i天各指标值，和上面是一样的
   for i in [3, 7, 14, 30, 60, 140]:
       tmp = get_timespan(df, t2017 + timedelta(days=-7), i, i)
       X['diff_%s_mean_2' % i] = tmp.diff(axis=1).mean(axis=1).values
       X['mean_%s_decay_2' % i] = (tmp * np.power(0.9,np.arange(i)[::-1])).sum(axis=1).values
       X['mean_%s_2' % i] = tmp.mean(axis=1).values
       X['median_%s_2' % i] =tmp.median(axis=1).values
       X['min_%s_2' % i] = tmp.min(axis=1).values
       X['max_%s_2' % i] = tmp.max(axis=1).values
       X['std_%s_2' % i] = tmp.std(axis=1).values
   
    #t2017为起点，最近i天内有销量/促销的天数、距离上次有销量的天数、距离最早有销量的天数
   for i in [7, 14, 30, 60, 140]:
       tmp = get_timespan(df, t2017, i, i)
       # 最近i天内有销量的天数
       X['has_sales_days_in_last_%s' % i] = (tmp > 0).sum(axis=1).values
       # 最近i天内距离上次有销量的天数，如果都没有销量则为i
       X['last_has_sales_day_in_last_%s' % i] = i - ((tmp > 0) *np.arange(i)).max(axis=1).values
       # 最近i天内距离最早有销量的天数
       X['first_has_sales_day_in_last_%s' % i] = ((tmp > 0) * np.arange(i,0, -1)).max(axis=1).values
       tmp = get_timespan(promo_df, t2017, i, i)
       X['has_promo_days_in_last_%s' % i] = (tmp > 0).sum(axis=1).values
       X['last_has_promo_day_in_last_%s' % i] = i - ((tmp > 0) *np.arange(i)).max(axis=1).values
       X['first_has_promo_day_in_last_%s' % i] = ((tmp > 0) * np.arange(i,0, -1)).max(axis=1).values
   
    #t2017为起点，未来15天内有促销的天数、距离上次有促销的天数、距离最早有促销的天数
   tmp = get_timespan(promo_df, t2017 + timedelta(days=16), 15, 15)
   X['has_promo_days_in_after_15_days'] = (tmp > 0).sum(axis=1).values
   X['last_has_promo_day_in_after_15_days'] = i - ((tmp > 0) *np.arange(15)).max(axis=1).values
   X['first_has_promo_day_in_after_15_days'] = ((tmp > 0) *np.arange(15, 0, -1)).max(axis=1).values
   
    #t2017为起点，前i天当天的销量
   for i in range(1, 16):
       X['day_%s_2017' % i] = get_timespan(df, t2017, i, 1).values.ravel()
   
    #t2017为起点，最近4/20周时间窗口为(每周1-每周日)的销量均值，比如最近4周每周周1的均值；
   for i in range(7):
       X['mean_4_dow{}_2017'.format(i)] = get_timespan(df, t2017, 28-i, 4,freq='7D').mean(axis=1).values
       X['mean_20_dow{}_2017'.format(i)] = get_timespan(df, t2017, 140-i, 20,freq='7D').mean(axis=1).values
   
    #t2017为起点，前后16天当天促销
   for i in range(-16, 16):
       # 需要把t2017 + timedelta(days=i) 转化成str格式，否则会报错
       X["promo_{}".format(i)] = promo_df[str(t2017 +timedelta(days=i))].values.astype(np.uint8)
 
    X= pd.DataFrame(X)
    #y是未来16天当天销量
   if is_train:
       y = df[pd.date_range(t2017, periods=16)].values
       return X, y
   if name_prefix is not None:
        X.columns = ['%s_%s' % (name_prefix, c) forc in X.columns]
   return X
 
print("Preparing dataset...")
#num_days = 8
num_days = 2
t2017 = date(2017, 5, 31)
X_l, y_l = [], []
for i in range(num_days):
   delta = timedelta(days=7 * i)
    #store_nbr-item_nbr粒度
   X_tmp, y_tmp = prepare_dataset(df_2017, promo_2017, t2017 + delta)
    #item_nbr粒度
   X_tmp2 = prepare_dataset(df_2017_item, promo_2017_item, t2017 + delta,is_train=False, name_prefix='item')
   X_tmp2.index = df_2017_item.index
   X_tmp2 =X_tmp2.reindex(df_2017.index.get_level_values(1)).reset_index(drop=True)
    #store-class粒度
   X_tmp3 = prepare_dataset(df_2017_store_class, df_2017_promo_store_class,t2017 + delta, is_train=False, name_prefix='store_class')
    X_tmp3.index = df_2017_store_class.index
   
    #构建多重索引必须从这里pd.MultiIndex.from_frame，源代码会报错
   X_tmp3 =X_tmp3.reindex(pd.MultiIndex.from_frame(df_2017_store_class_index)).reset_index(drop=True)
   X_tmp3
   
    #将不同粒度的训练数据合并
   X_tmp = pd.concat([X_tmp, X_tmp2, X_tmp3, items.reset_index(),stores.reset_index()], axis=1)
   
   X_l.append(X_tmp)
y_l.append(y_tmp)

def build_model():
   model = Sequential()
   model.add(LSTM(512, input_shape=(X_train.shape[1],X_train.shape[2])))
   model.add(BatchNormalization())
   model.add(Dropout(.2))
 
   model.add(Dense(256))
   model.add(PReLU())
   model.add(BatchNormalization())
   model.add(Dropout(.1))
 
   model.add(Dense(256))
   model.add(PReLU())
   model.add(BatchNormalization())
   model.add(Dropout(.1))
 
   model.add(Dense(128))
   model.add(PReLU())
   model.add(BatchNormalization())
   model.add(Dropout(.05))
 
   model.add(Dense(64))
   model.add(PReLU())
   model.add(BatchNormalization())
   model.add(Dropout(.05))
 
   model.add(Dense(32))
   model.add(PReLU())
   model.add(BatchNormalization())
   model.add(Dropout(.05))
 
   model.add(Dense(16))
   model.add(PReLU())
   model.add(BatchNormalization())
   model.add(Dropout(.05))
   model.add(Dense(1))
   return model