【Python】数据分析优秀案例&项目经历-用数据分析能力构建高分学生人群画像

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots 
import scipy.stats as stats
import scipy
from scipy.stats import ttest_ind
from scipy.stats import chisquare
from scipy.stats import chi2_contingency
import numpy as np

data = pd.read_csv('Student Performance new.csv')

data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1000 entries, 0 to 999
Data columns (total 9 columns):
 #   Column                       Non-Null Count  Dtype  
---  ------                       --------------  -----  
 0   Unnamed: 0                   1000 non-null   int64  
 1   race/ethnicity               1000 non-null   object 
 2   parental level of education  1000 non-null   object 
 3   lunch                        1000 non-null   object 
 4   test preparation course      1000 non-null   object 
 5   math percentage              1000 non-null   float64
 6   reading score percentage     1000 non-null   float64
 7   writing score percentage     1000 non-null   float64
 8   sex                          1000 non-null   object 
dtypes: float64(3), int64(1), object(5)
memory usage: 70.4+ KB

data.sample(n=5) # 随机抽取数据查看

# 数据Unnamed列为学生编号，我们将其舍弃
data = data.drop(columns=['Unnamed: 0'], axis=1)

labels = ['race/ethnicity', 'parental level of education', 'lunch', 'test preparation course']
for label in labels:
    print(f'标签{ 
     label}情况：')
    print(f'共有{ 
     data[label].nunique()}个变量，分别为')
    print(data[label].unique())
    print('*'*20)

标签race/ethnicity情况：
共有5个变量，分别为
['group B' 'group C' 'group A' 'group D' 'group E']
********************
标签parental level of education情况：
共有6个变量，分别为
["bachelor's degree" 'some college' "master's degree" "associate's degree"
 'high school' 'some high school']
********************
标签lunch情况：
共有2个变量，分别为
['standard' 'free/reduced']
********************
标签test preparation course情况：
共有2个变量，分别为
['none' 'completed']
********************

average_score = data[['math percentage', 'reading score percentage', 'writing score percentage']].mean(axis=1)
data.insert(7, 'average score', np.round(average_score, 2))

performance_level = pd.cut(data['average score'], bins=[0, 0.59, 0.69, 0.79, 0.89, 100], 
                                   labels=['F', 'D', 'C', 'B', 'A'] )
data.insert(8, 'performance level', performance_level)

data.groupby('sex')['sex'].count()

sex
F    518
M    482
Name: sex, dtype: int64

expected_number = [500, 500] # 理论上1000个样本中男女人数
observed_number = [482, 518] # 实际观察到的男女人数
result = stats.chisquare(f_obs=observed_number, f_exp=expected_number)
print(f'卡方拟合度检验的P值为：{ 
     result[1]}')

卡方拟合度检验的P值为：0.25494516431731784

data.sample(n=5)

fig = plt.figure(figsize=(5,10))
sns.set_style('darkgrid')
subjects = ['math percentage', 'reading score percentage', 'writing score percentage', 'average score']
color = ['green', 'blue', 'orange', 'purple']
column = 1
for subject in subjects:
    plt.subplot(len(subjects), 1, column)
    sns.kdeplot(data=data, x=subject, color=color[column - 1])
    column = column + 1
plt.tight_layout()
plt.show()

lower_bound = data['average score'].mean() - data['average score'].std()*2
upper_bound = data['average score'].mean() + data['average score'].std()*2
print(f'95%的学生成绩分布下限：{ 
     lower_bound}')
print(f'95%的学生成绩分布上限：{ 
     upper_bound}')

95%的学生成绩分布下限：0.3924529214957264
95%的学生成绩分布上限：0.9627870785042743

data[subjects].corr()

fig = plt.figure()
plt.subplot()
sns.heatmap(data[subjects].corr(), annot=True)
plt.show()

# 将数学成绩进行分箱，分箱方式与前述对平均成绩的分箱方式相关
math_grading = pd.cut(data['math percentage'], bins=[0, 0.59, 0.69, 0.79, 0.89, 100], 
                                   labels=['F', 'D', 'C', 'B', 'A'] )
crosstab = pd.crosstab(math_grading, data['sex'])

# 引入卡方独立性检验
result = stats.chi2_contingency(crosstab)
print(f'p值为:{ 
     result[1]}')

p值为:3.052111718576621e-05

data.groupby('sex')[['math percentage']].agg([np.mean, np.median])

fig, ax  = plt.subplots(1,2, figsize=(10, 5))
sns.boxplot(data=data, y='math percentage', x='sex', palette='summer', ax=ax[0])
sns.histplot(data=data, x='math percentage', hue='sex', fill=True, ax=ax[1], stat='probability')
plt.show()

data.groupby('sex')[['writing score percentage']].agg([np.mean, np.median])

fig = px.histogram(
    data, x='writing score percentage', 
    marginal='box', opacity=0.6,
    color='sex',
    histnorm='probability',
    title='男生与女生在文科上的表现',
    template='plotly_white'
)

fig.update_layout(barmode='overlay', width=800)
fig.show()

honor_students = data.loc[data['average score']>=0.9] # 选取均分高于0.9的学生，组成子数据集honor_students

honor_count = honor_students['parental level of education'].value_counts().sort_index()
total_count = data['parental level of education'].value_counts().sort_index()

fig = make_subplots(rows=1, cols=2, specs=[[dict(type='domain'),{ 
   'type':'domain'}]])

fig.add_pie(
    values=total_count.values, hole=0.4, labels=total_count.index, 
            row=1, col=1, name='整体学生父母受教育程度'
)
fig.add_pie(
    values=honor_count.values, hole=0.4, labels=honor_count.index, 
            row=1, col=2, name='高分学生父母受教育程度'
)

fig.update_layout(
    title_text="学生父母受教育程度",
    annotations=[dict(text='整体父母', x=0.15, y=0.5, font_size=20, showarrow=False),
                 dict(text='高分父母', x=0.85, y=0.5, font_size=20, showarrow=False)],
    width=900
)
fig.show()

data.groupby('parental level of education')['average score'].mean().sort_values()

parental level of education
high school           0.631224
some high school      0.650726
some college          0.684469
associate's degree    0.695586
bachelor's degree     0.719492
master's degree       0.735763
Name: average score, dtype: float64

parents = data['parental level of education'].unique()
sample_data = []
for parent in parents:
    member = data[data['parental level of education'] == parent].sample(n=30, random_state=1)
    sample_data.append(member)
parent_data = pd.concat(sample_data, axis=0)

fig = px.scatter(
    parent_data, x='math percentage', y='writing score percentage',
    color='parental level of education',
    size='average score',
    title='父母教育程度对学生成绩影响',
    template='plotly_dark'
)
fig.show()

crosstab = pd.crosstab(data['parental level of education'], data['performance level'])

result = stats.chi2_contingency(crosstab)
if result[1] < 0.05:
    print(f'p值为{ 
     result[1]}, 原假设不成立!')
else:
    print(f'p值为{ 
     result[1]}, 原假设成立!')

p值为0.00014895457933147155, 原假设不成立!

honor_students.groupby('sex')['sex'].count()

sex
F    40
M    14
Name: sex, dtype: int64

condition = ((data['math percentage']+data['writing score percentage'])/2 >= 0.9)
data[condition].groupby('sex')['sex'].count()

sex
F    35
M    18
Name: sex, dtype: int64

honor_index = honor_students.groupby('sex')['sex'].count().index
honor_value = honor_students.groupby('sex')['sex'].count().values
math_writing_index = data[condition].groupby('sex')['sex'].count().index
math_writing_value = data[condition].groupby('sex')['sex'].count().values
fig = go.Figure(data=[
    go.Bar(x=honor_index, y=honor_value, name='三科均分高于90分', text=honor_value,),
    go.Bar(x=math_writing_index, y=math_writing_value, name='数学写作均分高于90分', text=math_writing_value)
])
fig.update_layout(width=500, height=400)
fig.show()

data[data['math percentage']>0.9].groupby('sex')['math percentage'].mean()

sex
F    0.952857
M    0.948621
Name: math percentage, dtype: float64

data[data['math percentage']>0.9].groupby('sex')['math percentage'].std()

sex
F    0.031803
M    0.032373
Name: math percentage, dtype: float64

honored_male_math = data.loc[(data['math percentage'] >= 0.9) & (data['sex'] == 'M'),'math percentage']
honored_female_math = data.loc[(data['math percentage'] >= 0.9) & (data['sex'] == 'F'),'math percentage']
result = ttest_ind(honored_male_math.values, honored_female_math.values, equal_var=True)
if result[1]>0.95:
    print(f'两者的均值一致，P值为{ 
     result[1]}')
else:
    print(f'两者的均值不一致，P值为{ 
     result[1]}')

两者的均值一致，P值为1.0