使用Python中对情态动词进行NLP分析

import nltk import os def get_genres(): yield 'legal' for genre in brown.categories(): yield genre modals = ['can', 'could', 'may', 'might', 'must', 'will', 'would', 'should'] def get_words(genre): if (genre == 'legal'): grams = open('1gram', 'rU') for line in grams: vals = line.split(' ') word = vals[0] count = int(vals[1]) if (word in modals): for index in range(0, count): yield word else: yield word grams.close() else: for word in brown.words(categories=genre): yield word

cfd = nltk.ConditionalFreqDist( (genre, word) for genre in get_genres() for word in get_words(genre) ) genres = [g for g in get_genres()] cfd.tabulate(conditions=genres, samples=modals) cfd.tabulate(conditions=genres, samples=modals)

can

could

may

might

must

will

would

should

def tabulate(cfd, conditions, samples): max_len = max(len(w) for w in conditions) sys.stdout.write(" " * (max_len + 1)) for c in samples: sys.stdout.write("%-s\t" % c) sys.stdout.write("\n") for c in conditions: sys.stdout.write(" " * (max_len - len(c))) sys.stdout.write("%-s" % c) sys.stdout.write(" ") dist = cfd[c] norm = sum([dist[w] for w in modals]) for s in samples: value = 100 * dist[s] / norm sys.stdout.write("%-d\t" % value) sys.stdout.write("\n") tabulate(cfd, genres, modals)

can

could

may

might

must

will

would

should

def tabulate(cfd, conditions, samples): max_len = max(len(w) for w in conditions) sys.stdout.write(" " * (max_len + 1)) for c in samples: sys.stdout.write("%-s\t" % c) sys.stdout.write("\n") for c in conditions: sys.stdout.write(" " * (max_len - len(c))) sys.stdout.write("%-s" % c) sys.stdout.write(" ") dist = cfd[c] norm = sum([dist[w] for w in modals]) for s in samples: value = 10 * float(dist[s]) / norm sys.stdout.write("%.1f\t" % value) sys.stdout.write("\n") tabulate(cfd, genres, modals)

can

could

may

might

must

will

would

should

import math def distance(cfd, conditions, samples, base): base_cond = cfd[base] base_vector = [base_cond[w] for w in samples] base_length = math.sqrt(sum(a * a for a in base_vector)) for c in conditions: cond = cfd[c] cond_vector = [cond[w] for w in samples] dotp = sum(a * b for (a,b) in zip(base_vector, cond_vector)) cond_length = math.sqrt(sum(a * a for a in cond_vector)) angle = math.acos(dotp / (cond_length * base_length)) percent = (math.pi / 2 - angle) / (math.pi / 2) * 100 print "%-s similarity to %-s: %-.1f" % (c, base, percent)

def distance(cfd, conditions, samples): base_vector = [0.0 for w in conditions] norm = {} for c_i in range(0, len(conditions)): cond_name = conditions[c_i] cond = cfd[cond_name] norm[cond_name] = float(sum(cond[s] for s in samples)) for s in samples: base_vector[c_i] = base_vector[c_i] + float(cond[s]) / norm[cond_name] base_length = math.sqrt(sum(a * a for a in base_vector)) for s in samples: # compute each vector - which, might, etc sample_vector = [] for c in conditions: # find condition for each vector sample_vector.append(cfd[c][s] / norm[c]) dotp = sum(a * b for (a,b) in zip(base_vector, sample_vector)) sample_length = math.sqrt(sum(a * a for a in sample_vector)) angle = math.acos(dotp / (sample_length * base_length)) percent = (math.pi / 2 - angle) / (math.pi / 2) * 100 print "%-s similarity to mean: %-.1f" % (s, percent) distance(cfd, genres, modals)