算法集锦（17） | 推荐系统 | 基于机器学习的商品定价系统

 1import gc
 2import time
 3import numpy as np
 4import pandas as pd
 5import matplotlib.pyplot as plt
 6import seaborn as sns
 7from scipy.sparse import csr_matrix, hstack
 8from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
 9from sklearn.preprocessing import LabelBinarizer
10from sklearn.model_selection import train_test_split, cross_val_score
11from sklearn.metrics import mean_squared_error
12import lightgbm as lgb
13df = pd.read_csv('train.tsv', sep = '\t')

1msk = np.random.rand(len(df)) < 0.8
2train = df[msk]
3test = df[~msk]
4train.shape, test.shape
5train.head()

1train.info()

1train.price.describe()

1plt.subplot(1, 2, 1)
2(train['price']).plot.hist(bins=50, figsize=(12, 6), edgecolor = 'white', range = [0, 250])
3plt.xlabel('price', fontsize=12)
4plt.title('Price Distribution', fontsize=12)
5plt.subplot(1, 2, 2)
6np.log(train['price']+1).plot.hist(bins=50, figsize=(12,6), edgecolor='white')
7plt.xlabel('log(price+1)', fontsize=12)
8plt.title('Price Distribution', fontsize=12)

1train['shipping'].value_counts() / len(train)

 1shipping_fee_by_buyer = train.loc[df['shipping'] == 0, 'price']
 2shipping_fee_by_seller = train.loc[df['shipping'] == 1, 'price']
 3fig, ax = plt.subplots(figsize=(18,8))
 4ax.hist(shipping_fee_by_seller, color='#8CB4E1', alpha=1.0, bins=50, range = [0, 100],
 5       label='Price when Seller pays Shipping')
 6ax.hist(shipping_fee_by_buyer, color='#007D00', alpha=0.7, bins=50, range = [0, 100],
 7       label='Price when Buyer pays Shipping')
 8plt.xlabel('price', fontsize=12)
 9plt.ylabel('frequency', fontsize=12)
10plt.title('Price Distribution by Shipping Type', fontsize=15)
11plt.tick_params(labelsize=12)
12plt.legend()
13plt.show()

1print('The average price is {}'.format(round(shipping_fee_by_seller.mean(), 2)), 'if seller pays shipping');
2print('The average price is {}'.format(round(shipping_fee_by_buyer.mean(), 2)), 'if buyer pays shipping')

 1fig, ax = plt.subplots(figsize=(18,8))
 2ax.hist(np.log(shipping_fee_by_seller+1), color='#8CB4E1', alpha=1.0, bins=50,
 3       label='Price when Seller pays Shipping')
 4ax.hist(np.log(shipping_fee_by_buyer+1), color='#007D00', alpha=0.7, bins=50,
 5       label='Price when Buyer pays Shipping')
 6plt.xlabel('log(price+1)', fontsize=12)
 7plt.ylabel('frequency', fontsize=12)
 8plt.title('Price Distribution by Shipping Type', fontsize=15)
 9plt.tick_params(labelsize=12)
10plt.legend()
11plt.show()

1print('There are', train['category_name'].nunique(), 'unique values in category name column')

1train['category_name'].value_counts()[:10]

1sns.boxplot(x = 'item_condition_id', y = np.log(train['price']+1), data = train, palette = sns.color_palette('RdBu',5))

1NUM_BRANDS = 4000
2NUM_CATEGORIES = 1000
3NAME_MIN_DF = 10
4MAX_FEATURES_ITEM_DESCRIPTION = 50000

1print('There are %d items that do not have a category name.' %train['category_name'].isnull().sum())

1print('There are %d items that do not have a brand name.' %train['brand_name'].isnull().sum())

1print('There are %d items that do not have a description.' %train['item_description'].isnull().sum())

 1def handle_missing_inplace(dataset): 
 2    dataset['category_name'].fillna(value='missing', inplace=True) 
 3    dataset['brand_name'].fillna(value='missing', inplace=True) 
 4    dataset['item_description'].replace('No description yet,''missing', inplace=True) 
 5    dataset['item_description'].fillna(value='missing', inplace=True)
 6def cutting(dataset):
 7    pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
 8    dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'
 9    pop_category = dataset['category_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
10def to_categorical(dataset):
11    dataset['category_name'] = dataset['category_name'].astype('category')
12    dataset['brand_name'] = dataset['brand_name'].astype('category')
13    dataset['item_condition_id'] = dataset['item_condition_id'].astype('category')

1df = pd.read_csv('train.tsv', sep = '\t')
2msk = np.random.rand(len(df)) < 0.8
3train = df[msk]
4test = df[~msk]
5test_new = test.drop('price', axis=1)
6y_test = np.log1p(test["price"])
7train = train[train.price != 0].reset_index(drop=True)

1nrow_train = train.shape[0]
2y = np.log1p(train["price"])
3merge: pd.DataFrame = pd.concat([train, test_new])

1handle_missing_inplace(merge)
2cutting(merge)
3to_categorical(merge)

1cv = CountVectorizer(min_df=NAME_MIN_DF)
2X_name = cv.fit_transform(merge['name'])
3cv = CountVectorizer()
4X_category = cv.fit_transform(merge['category_name'])

1tv = TfidfVectorizer(max_features=MAX_FEATURES_ITEM_DESCRIPTION, ngram_range=(1, 3), stop_words='english')
2X_description = tv.fit_transform(merge['item_description'])

1lb = LabelBinarizer(sparse_output=True)
2X_brand = lb.fit_transform(merge['brand_name'])

1X_dummies = csr_matrix(pd.get_dummies(merge[['item_condition_id', 'shipping']], sparse=True).values)

1sparse_merge = hstack((X_dummies, X_description, X_brand, X_category, X_name)).tocsr()

1mask = np.array(np.clip(sparse_merge.getnnz(axis=0) - 1, 0, 1), dtype=bool)
2sparse_merge = sparse_merge[:, mask]

1X = sparse_merge[:nrow_train]
2X_test = sparse_merge[nrow_train:]

1train_X = lgb.Dataset(X, label=y)

1params = {
2        'learning_rate': 0.75,
3        'application': 'regression',
4        'max_depth': 3,
5        'num_leaves': 100,
6        'verbosity': -1,
7        'metric': 'RMSE',
8    }

1gbm = lgb.train(params, train_set=train_X, num_boost_round=3200, verbose_eval=100)

1y_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)

1from sklearn.metrics import mean_squared_error
2print('The rmse of prediction is:', mean_squared_error(y_test, y_pred) ** 0.5)