自然语言处理——词频统计

#要统计词频的段落
para='This is a test. Beautiful is better than ugly. Explicit is better than implicit. Simple is better than complex. Now is better than never. Is this a complex thing?'
#将.和？替换成空，然后转成小写
para_sub=tolower(gsub("\\.|\\?","",para))

#按照空格分词，统计词频
count=sort(table(unlist(strsplit(para_sub," "))),decreasing = T)
#保留出现一次以上的单词，作为关键词
keys=count[count>1]
#统计关键词的长度
keylen=sum(nchar(names(keys))*as.numeric(keys))
#统计关键词占整段文字的百分比
percent=keylen/nchar(para)

#！/usr/bin/perl  
$para='This is a test. Beautiful is better than ugly. Explicit is better than implicit. Simple is better than complex. Now is better than never. Is this a complex thing?';
$para_sub=$para;
#将.和？替换成空，
$para_sub=~s/\.|\?//g;
#转成小写
$para_sub=lc($para_sub);
#分词
my @array=split " ",$para_sub;

#统计词频
foreach $word (@array){
 $hash{$word}++;
}

#计算关键词的总长度
foreach $i (sort {$hash{$b}<=>$hash{$a}} keys %hash){
 if($hash{$i}>1){
  print "$i\t$hash{$i}\n";
  $key_len+=length($i)*$hash{$i};
 }
}

#统计关键词占整段文字的百分比
$percent=$key_len/length($para);
$percent=sprintf("%.4f",$percent);
print "keyword percent: $percent";