基于关联规则算法的电商数据挖掘

import pandas as pd
import numpy as np

# 显示所有列
# pd.set_option('display.max_columns', None)
# 显示所有行
# pd.set_option('display.max_rows', None)
# 设置value的显示长度为100，默认为50
# pd.set_option('max_colwidth',100)

import time
import os
from datetime import datetime
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
#设置中文编码和负号的正常显示
plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus']=False

import missingno as ms

from pyecharts.globals import CurrentConfig, OnlineHostType
from pyecharts import options as opts  # 配置项
from pyecharts.charts import Bar, Scatter, Pie, Line,Map, WordCloud, Grid, Page  # 各个图形的类
from pyecharts.commons.utils import JsCode
from pyecharts.globals import ThemeType,SymbolType

import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots  # 画子图


import jieba
from snownlp import SnowNLP

from sklearn.cluster import KMeans
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler

import warnings
warnings.filterwarnings("ignore")

# 数据中存在中文，指定读取的编码格式

df = pd.read_csv("ic_sale.csv",
                 encoding="gb18030",  # windows系统需要指定类型；mac不需要
                 converters={"order_id":str,"product_id":str,"category_id":str,"user_id":str}
                )  
df.head()

# 1、数据shape

df.shape   

(564169, 11)

# 2、数据字段类型

df.dtypes

event_time        object
order_id          object
product_id        object
category_id       object
category_code     object
brand             object
price            float64
user_id           object
age                int64
sex               object
local             object
dtype: object

# 3、数据描述统计信息

df.describe()

# 4、总共多少个不同客户

df["user_id"].nunique()

6908

df.shape  # 去重前

(564169, 11)

df.drop_duplicates(ignore_index=True,inplace=True)

df.shape # 去重后

(561214, 11)

stats = []
for col in df.columns:
    stats.append((col, 
                  df[col].nunique(), 
                  round(df[col].isnull().sum() * 100 / df.shape[0], 3), 
                  round(df[col].value_counts(normalize=True, dropna=False).values[0] * 100,3), 
                  df[col].dtype)
                )
    
stats_df = pd.DataFrame(stats, 
                        columns=['特征名', '属性个数', '缺失值占比', '最大属性占比', '特征类型'])

stats_df.sort_values('缺失值占比', ascending=False, ignore_index=True)

df = df[df["price"] > 0]

df.isnull().sum()

event_time            0
order_id              0
product_id            0
category_id           0
category_code    128662
brand             27132
price                 0
user_id               0
age                   0
sex                   0
local                 0
dtype: int64

ms.bar(df,color="red")  # 缺失值可视化

plt.show()

df.fillna("missing",inplace=True)  # 填充missing

df["event_time"].value_counts()

1970-01-01 00:33:40 UTC    1302
2020-04-09 16:30:01 UTC      51
2020-04-08 16:30:01 UTC      49
2020-04-06 16:30:01 UTC      46
2020-04-05 16:30:01 UTC      44
                           ... 
2020-07-28 13:10:35 UTC       1
2020-07-28 13:10:21 UTC       1
2020-07-28 13:09:37 UTC       1
2020-07-28 13:08:23 UTC       1
2020-08-13 17:16:24 UTC       1
Name: event_time, Length: 389813, dtype: int64

# 去掉最后的UTC
df["event_time"] = df["event_time"].apply(lambda x: x[:19])  

# 时间数据类型转化：字符类型---->指定时间格式
df['event_time'] = pd.to_datetime(df['event_time'], format="%Y-%m-%d %H:%M:%S")

# 提取多个时间相关字段
# df['month']=df['event_time'].dt.month
# df['day'] = df['event_time'].dt.day
# df['dayofweek']=df['event_time'].dt.dayofweek
# df['hour']=df['event_time'].dt.hour

# 不同性别下的年龄分布

fig = px.box(df,y=["age"], color="sex")

fig.show()

# 不同年龄段人数统计

fig = plt.figure(figsize=(12,6))
sns.countplot(df["age"])

plt.title("Counts of Different Age")

plt.show()

df["age"] = pd.cut(df["age"],bins=4,precision=0)

df["age"]  # 分段之后的age字段显示

0         (16.0, 24.0]
1         (33.0, 42.0]
2         (24.0, 33.0]
3         (16.0, 24.0]
4         (16.0, 24.0]
              ...     
561209    (16.0, 24.0]
561210    (16.0, 24.0]
561211    (16.0, 24.0]
561212    (16.0, 24.0]
561213    (16.0, 24.0]
Name: age, Length: 561175, dtype: category
Categories (4, interval[float64, right]): [(16.0, 24.0] < (24.0, 33.0] < (33.0, 42.0] < (42.0, 50.0]]

fig = px.scatter(df[df["brand"] != "missing"],  # 除去missing数据
#                  x="local",
                 y="price",
                 facet_col="age",
                 color="local",
                 size="price"           
                )

fig.show()

age_brand = df.groupby(["age","sex","brand"]).size().reset_index().rename(columns={0:"number"})

age_brand.head()

# 实现排序功能-降序

age_brand = age_brand.sort_values(["age","number"],ascending=[True,False],ignore_index=True)

age_brand.head()

# 条件筛选

age_brand = age_brand.query("number > 0 & brand != 'missing'")

fig = px.treemap(
    age_brand,  # 传入数据
    path=[px.Constant("all"),"age","sex","brand"],  # 传递数据路径
    values="number"  # 数值显示
)

fig.update_traces(root_color="lightskyblue")

fig.update_layout(margin=dict(t=30,l=30,r=25,b=30))

fig.show()

age_brand.head()

brand_list = age_brand["brand"].value_counts().reset_index()

brand_list.columns=["word","number"]

brand_list.head(10)

information_zip = [tuple(z) for z in zip(brand_list["word"].tolist(), brand_list["number"].tolist())]

# 绘图
c = (
    WordCloud()
    .add("", information_zip, word_size_range=[20, 80], shape=SymbolType.DIAMOND)
    .set_global_opts(title_opts=opts.TitleOpts(title="品牌词云图"))
)

c.render_notebook()

df["category_code"].value_counts()  

missing                             128662
electronics.smartphone              101502
computers.notebook                   25917
appliances.kitchen.refrigerators     20296
electronics.audio.headphone          20049
                                     ...  
kids.swing                               8
country_yard.watering                    5
sport.snowboard                          3
apparel.costume                          2
apparel.shoes                            2
Name: category_code, Length: 124, dtype: int64

fig = px.bar(df["category_code"].value_counts()[1:30])  # 前30个category_code

fig.show()

df = df[df["category_code"] != "missing"]  # 去除missing部分
df = df[["category_code", "brand","age", "sex", "local"]]

df["category_code"] = df["category_code"].apply(lambda x: x.split(".") if "." in x else [x])

df.head()

data = df["category_code"].tolist()
data[:3]

[['electronics', 'tablet'],
 ['electronics', 'audio', 'headphone'],
 ['furniture', 'kitchen', 'table']]

import itertools

# 通过chain方法从可迭代对象中生成；展开成列表

sum_data = list(itertools.chain.from_iterable(data))
sum_data[:10]

['electronics',
 'tablet',
 'electronics',
 'audio',
 'headphone',
 'furniture',
 'kitchen',
 'table',
 'electronics',
 'smartphone']

category_code_number = pd.value_counts(sum_data).to_frame().reset_index()

category_code_number.columns=["category_code","number"]

category_code_number.head()

information_zip = [tuple(z) for z in zip(category_code_number["category_code"].tolist(), category_code_number["number"].tolist())]

# 绘图
c = (
    WordCloud()
    .add("", information_zip, word_size_range=[20, 80], shape=SymbolType.DIAMOND)
    .set_global_opts(title_opts=opts.TitleOpts(title="商品种类词云图"))
)

c.render_notebook()

male = df[df["sex"] == "男"]
male.head()

import efficient_apriori as ea

male_list = male["category_code"].tolist()

# itemsets：频繁项  rules：关联规则
itemsets, rules = ea.apriori(male_list,
                min_support=0.005,
                min_confidence=1
               )

len(itemsets[1])  

60

itemsets[1]  # 一个频繁项集

# 字典的值value的降序排列

dict(sorted(itemsets[1].items(), key=lambda x: x[1], reverse=True))

len(itemsets[2])  # 总个数

84

# 两个频繁项集

dict(sorted(itemsets[2].items(), key=lambda x: x[1], reverse=True))

len(itemsets[3])  # 总个数

32

# 三个频繁项集

dict(sorted(itemsets[3].items(), key=lambda x: x[1], reverse=True))

{('appliances', 'kitchen', 'refrigerators'): 10209,
 ('audio', 'electronics', 'headphone'): 10154,
 ('electronics', 'tv', 'video'): 8876,
 ('appliances', 'environment', 'vacuum'): 8069,
 ('appliances', 'kitchen', 'washer'): 7235,
 ('appliances', 'kettle', 'kitchen'): 6389,
 ('computers', 'mouse', 'peripherals'): 6359,
 ('furniture', 'kitchen', 'table'): 5626,
 ('appliances', 'hood', 'kitchen'): 4487,
 ('appliances', 'blender', 'kitchen'): 4439,
 ('appliances', 'kitchen', 'microwave'): 3830,
 ('air_conditioner', 'appliances', 'environment'): 3806,
 ('appliances', 'personal', 'scales'): 3423,
 ('computers', 'network', 'router'): 3318,
 ('components', 'computers', 'hdd'): 2598,
 ('appliances', 'kitchen', 'meat_grinder'): 2361,
 ('components', 'computers', 'cpu'): 2055,
 ('appliances', 'kitchen', 'oven'): 1958,
 ('appliances', 'environment', 'fan'): 1952,
 ('computers', 'keyboard', 'peripherals'): 1940,
 ('computers', 'peripherals', 'printer'): 1802,
 ('appliances', 'environment', 'water_heater'): 1753,
 ('computers', 'monitor', 'peripherals'): 1733,
 ('components', 'computers', 'cooler'): 1717,
 ('cabinet', 'furniture', 'living_room'): 1550,
 ('chair', 'furniture', 'kitchen'): 1513,
 ('appliances', 'hair_cutter', 'personal'): 1388,
 ('air_heater', 'appliances', 'environment'): 1341,
 ('appliances', 'dishwasher', 'kitchen'): 1329,
 ('furniture', 'living_room', 'shelving'): 1314,
 ('appliances', 'kitchen', 'mixer'): 1288,
 ('construction', 'screw', 'tools'): 1194}

female = df[df["sex"] == "女"]
female.head()

import efficient_apriori as ea

female_list = male["category_code"].tolist()

# itemsets：频繁项  rules：关联规则
itemsets, rules = ea.apriori(female_list,
                min_support=0.005,
                min_confidence=1
               )

len(itemsets[1])  # 总个数

60

# 一个频繁项集

dict(sorted(itemsets[1].items(), key=lambda x: x[1], reverse=True))

# 两个频繁项集

dict(sorted(itemsets[2].items(), key=lambda x: x[1], reverse=True))

# 三个频繁项集

dict(sorted(itemsets[3].items(), key=lambda x: x[1], reverse=True))

brand_category = df.groupby(["brand"])["category_code"].sum().reset_index()
brand_category

# 去重功能-set
brand_category["category_code"] = brand_category["category_code"].apply(lambda x: list(set(x)))

brand_category

import efficient_apriori as ea

brand_list = brand_category["category_code"].tolist()

# itemsets：频繁项  rules：关联规则
itemsets, rules = ea.apriori(
    brand_list,
    min_support=0.05,
    min_confidence=1
)

# 三个频繁项集
dict(sorted(itemsets[3].items(), key=lambda x: x[1], reverse=True))

# 两个频繁项集
dict(sorted(itemsets[2].items(), key=lambda x: x[1], reverse=True))

# 一个频繁项集
dict(sorted(itemsets[1].items(), key=lambda x: x[1], reverse=True))