导入包
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
读文件
df=pd.read_csv(r'C:\Users\MSI\Desktop\1.csv')
查看数据
df.head()
查看基本信息
df.info()
一共有九个字段,22739条数据,数据全为字符串,不存在数据为空的情况,因此不需要进行对缺少数据的处理
对重复数据进行处理,删除职位和公司重复值
df.drop_duplicates(['PositionName','CompanyName'],keep='first', inplace=True)
查看处理后的信息
df.info()
剩余21851条记录
查看薪资的分布的频率,发现面议有较大的比重
df['Salary'].str[0:].value_counts(normalize = True)
自定义函数drops,删除薪资中的面议
def drops(col, tag):
df.drop(df[df[col].str.contains(tag)].index, inplace=True)
drops('Salary', '面议')
自定义函数cutWord求平均薪资
def cutWord(word,method):
position=word.find("-")
length = len(word)
if position != -1:
bottomSalary = word[:position]
topSalary = word[position + 1:length - 1]
if method == 'bottom':
return bottomSalary
else:
return topSalary
df['topSalary']=df.Salary.apply(cutWord,method='top')
df['bottomSalary']=df.Salary.apply(cutWord,method='bottom')
df.topSalary=df.topSalary.astype("int")
df.bottomSalary=df.bottomSalary.astype("int")
df['avgSalary']=df.apply(lambda x:(x.bottomSalary+x.topSalary)/2,axis=1)
df['avgSalary'].value_counts
由于各个仅统计各个省份,但所给数据中含有地级市及区等,因此对数据进行处理,仅保留省份/直辖市
自定义函数newCity
def newCity(city):
if(len(str(city))>2):
newcity = city[:2]
else:
newcity=city
return newcity
df['newcity']=df.City.apply(newCity)
数据基本处理完成,保存为df_clean
df_clean = df[["PositionName", "CompanyName", "newcity", "Experience", "JobWords", "avgSalary"]]
df_clean.head()
查看数据的描述性信息
print(df_clean.describe())
平均薪资:21.85W,中位数:19W,最高:177.5W
薪资分布情况图
plt.rcParams['font.sans-serif']=['SimHei']
df_clean.avgSalary.hist(bins=20)
plt.show()
分割experience,不知道为什么这里分割了八个出来,我就定义了8列。不太懂我觉的这里应该四列才对,8列弄出来之后再把多的删了
info_split=df_clean['Experience'].str.split(' ',expand=True)
info_split.columns=['education','experience','language','age','1','2','3','4']
newExp=info_split.drop(['1','2','3','4'],axis=1)
display(newExp)
display(df_clean)
然后把两个二维表进行链接,再保存为new_df,最开始是链接之后删除experience,但是不知道为什么链接之后删除newcity就变成了city,之前的city白处理了。然后就直接保存了
newDF=pd.concat([df_clean, newExp], axis=1)
new_df = newDF[["PositionName", "CompanyName", "newcity",'education','experience','language','age' , "JobWords", "avgSalary"]]
display(new_df)
转换分类数据,这里发现本科有两个,然后其他数据不是很直观,后续有对这个数据进行了处理
new_df.education.astype('category')
自定义newEdu处理教育水平,写的有点复杂,之前的写法不知道为什么最后的结构只剩下本科和硕士。
def newEdu(education):
if education == "硕士及以上":
new_edu = "硕士"
elif education == "统招本科":
new_edu = "本科"
elif education == "本科及以上":
new_edu = "本科"
elif education== "学历不限":
new_edu = "不限"
elif education== "大专及以上":
new_edu = "大专"
elif education == "中专/中技及以上":
new_edu = "中专"
else:
new_edu="博士"
return new_edu
new_df['new_edu'] = new_df.education.apply(newEdu)
new_df.new_edu.astype('category')
选用线箱进行比较。其最大的优点就是不受异常值的影响,可以以一种相对稳定的方式描述数据的平均水平、波动程度和异常值分布情况。
new_df.new_edu=new_df.new_edu.astype('category')
new_df.new_edu.cat.set_categories(["中专", "博士", "大专", "不限", "本科", "硕士", ],inplace=True)
ax=new_df.boxplot(column='avgSalary',by='new_edu',figsize=(9,6))
print(new_df.groupby(new_df.new_edu).avgSalary.mean().sort_values(ascending=False))
如图1,本科中位数薪资高于硕士生,容易误以为本科薪资高于硕士生,但同时结合图2,可见硕士生的平均薪资水平远高于本科生,由此可知,学历越高,薪资越高,知识改变命运。
转化数据类型(工作年限)创建线箱进行比较
new_df.experience.astype('category')
new_df.boxplot(column='avgSalary',by='experience',figsize=(9,6))
工作年限和薪资的比较
print(new_df.groupby(new_df.experience).avgSalary.mean().sort_values(ascending=False))
薪资与工作年限有很大关系,但优秀员工薪资明显超越年限限制。
北京和上海这两座城市,学历对薪资的影响
df_sz_bj=new_df[new_df['newcity'].isin(['上海','北京'])]
df_sz_bj.boxplot(column='avgSalary',by=['new_edu','newcity'],figsize=[14,6])
plt.show()
薪资与工作区域有很大关系,北京薪资不管什么学历都高于同等学历的薪资状况
不同城市,招聘数据分析需求前五的公司
自定义了函数topN,将传入的数据计数,并且从大到小返回前五的数据。然后以newcity聚合分组,因为求的是前5的公司,所以对CompanyName调用topN函数。
new_df.groupby('CompanyName').avgSalary.agg(lambda x:max(x)-min(x))
def topN(df,n=5):
counts=df.value_counts()
return counts.sort_values(ascending=False)[:n]
print(new_df.groupby('newcity').CompanyName.apply(topN))
职位需求的前五,以计算机行业为主
print(new_df.groupby('newcity').PositionName.apply(topN))
将上海和北京的薪资数据以直方图的形式进行对比
plt.hist(x=new_df[new_df.newcity=='上海'].avgSalary,
bins=15,
density=1,
facecolor='blue',
alpha=0.5)
plt.hist(x=new_df[new_df.newcity=='北京'].avgSalary,
bins=15,
density=1,
facecolor='red',
alpha=0.5)
plt.show()
做一个所需要做的工作的词云,先下载wordcloud库
在anaconda下载第三方库还挺麻烦的,镜像还不能用,只能下载之后导包
查看数据进行处理
print(new_df.JobWords)
重置索引然后作词云
df_word_counts=df_word.unstack().dropna().reset_index().groupby('level_0').count()
from wordcloud import WordCloud
df_word_counts.index=df_word_counts.index.str.replace("'","")
wc=WordCloud(font_path=r'C:\Windows\Fonts\FZSTK.TTF',width=900,height=400,background_color='white')
fig,ax=plt.subplots(figsize=(20,15))
wc.fit_words(df_word_counts.level_1)
ax=plt.imshow(wc)
plt.axis('off')
plt.show()
上图可见对统计分析,数学,英语和office使用还是有一定的要求。
#!/usr/bin/env python
# coding: utf-8
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
df=pd.read_csv(r'C:\Users\MSI\Desktop\1.csv')
df.head()
df.info()
df.drop_duplicates(['PositionName','CompanyName'],keep='first', inplace=True)
df.info()
df['Salary'].str[0:].value_counts(normalize = True)
def drops(col, tag):
df.drop(df[df[col].str.contains(tag)].index, inplace=True)
drops('Salary', '面议')
df['Salary'].str[0:].value_counts(normalize = True)
def cutWord(word,method):
position=word.find("-")
length = len(word)
if position != -1:
bottomSalary = word[:position]
topSalary = word[position + 1:length - 1]
if method == 'bottom':
return bottomSalary
else:
return topSalary
df['topSalary']=df.Salary.apply(cutWord,method='top')
df['bottomSalary']=df.Salary.apply(cutWord,method='bottom')
df.topSalary=df.topSalary.astype("int")
df.bottomSalary=df.bottomSalary.astype("int")
df['avgSalary']=df.apply(lambda x:(x.bottomSalary+x.topSalary)/2,axis=1)
df['avgSalary'].value_counts
def newCity(city):
if(len(str(city))>2):
newcity = city[:2]
else:
newcity=city
return newcity
df['newcity']=df.City.apply(newCity)
df_clean = df[["PositionName", "CompanyName", "newcity", "Experience", "JobWords", "avgSalary"]]
df_clean.head()
print(df_clean.describe())
plt.rcParams['font.sans-serif']=['SimHei']
df_clean.avgSalary.hist(bins=20)
plt.show()
info_split=df_clean['Experience'].str.split(' ',expand=True)
info_split.columns=['education','experience','language','age','1','2','3','4']
newExp=info_split.drop(['1','2','3','4'],axis=1)
display(newExp)
display(df_clean)
newDF=pd.concat([df_clean, newExp], axis=1)
new_df = newDF[["PositionName", "CompanyName", "newcity",'education','experience','language','age' , "JobWords", "avgSalary"]]
display(new_df)
new_df.education.astype('category')
def newEdu(education):
if education == "硕士及以上":
new_edu = "硕士"
elif education == "统招本科":
new_edu = "本科"
elif education == "本科及以上":
new_edu = "本科"
elif education== "学历不限":
new_edu = "不限"
elif education== "大专及以上":
new_edu = "大专"
elif education == "中专/中技及以上":
new_edu = "中专"
else:
new_edu="博士"
return new_edu
new_df['new_edu'] = new_df.education.apply(newEdu)
new_df.new_edu.astype('category')
new_df.new_edu=new_df.new_edu.astype('category')
new_df.new_edu.cat.set_categories(["中专", "博士", "大专", "不限", "本科", "硕士", ],inplace=True)
ax=new_df.boxplot(column='avgSalary',by='new_edu',figsize=(9,6))
print(new_df.groupby(new_df.new_edu).avgSalary.mean().sort_values(ascending=False))
new_df.experience.astype('category')
new_df.boxplot(column='avgSalary',by='experience',figsize=(9,6))
print(new_df.groupby(new_df.experience).avgSalary.mean().sort_values(ascending=False))
df_sz_bj=new_df[new_df['newcity'].isin(['上海','北京'])]
df_sz_bj.boxplot(column='avgSalary',by=['new_edu','newcity'],figsize=[14,6])
plt.show()
new_df.groupby('CompanyName').avgSalary.agg(lambda x:max(x)-min(x))
def topN(df,n=5):
counts=df.value_counts()
return counts.sort_values(ascending=False)[:n]
print(new_df.groupby('newcity').CompanyName.apply(topN))
print(new_df.groupby('newcity').PositionName.apply(topN))
plt.hist(x=new_df[new_df.newcity=='上海'].avgSalary,
bins=15,
density=1,
facecolor='blue',
alpha=0.5)
plt.hist(x=new_df[new_df.newcity=='北京'].avgSalary,
bins=15,
density=1,
facecolor='red',
alpha=0.5)
plt.show()
print(new_df.JobWords)
df_word_counts=df_word.unstack().dropna().reset_index().groupby('level_0').count()
from wordcloud import WordCloud
df_word_counts.index=df_word_counts.index.str.replace("'","")
wc=WordCloud(font_path=r'C:\Windows\Fonts\FZSTK.TTF',width=900,height=400,background_color='white')
fig,ax=plt.subplots(figsize=(20,15))
wc.fit_words(df_word_counts.level_1)
ax=plt.imshow(wc)
plt.axis('off')
plt.show()
参考资料:
手机扫一扫
移动阅读更方便
你可能感兴趣的文章