自然语言处理| NLTK库的详解

pip install nltk

# 新建一个ipython，输入
import nltk 
nltk.download()

>>> import nltk
>>> sentence = "hello,,world"
>>> tokens = nltk.word_tokenize(sentence)
>>> tokens
['hello', ',', ',world']

>>> import nltk
>>> sentence = """At eight o'clock on Thursday morning
... Arthur didn't feel very good."""
>>> tokens = nltk.word_tokenize(sentence)
>>> tokens
['At', 'eight', "o'clock", 'on', 'Thursday', 'morning',
'Arthur', 'did', "n't", 'feel', 'very', 'good', '.']
>>> tagged = nltk.pos_tag(tokens)  # 标记词性
>>> tagged[0:6]
[('At', 'IN'), ('eight', 'CD'), ("o'clock", 'JJ'), ('on', 'IN'),
('Thursday', 'NNP'), ('morning', 'NN')]

>>> from nltk.tokenize import word_tokenize 
>>> from nltk.text import Text
>>> input_str = "Today's weather is good, very windy and sunny, we have no classes in the afternoon,We have to play basketball tomorrow."
>>> tokens = word_tokenize(input_str)
>>> tokens[:5]
['Today', "'s", 'weather', 'is', 'good']
>>> tokens = [word.lower() for word in tokens] #小写
>>> tokens[:5]
['today', "'s", 'weather', 'is', 'good']

>>> t = Text(tokens)
>>> t.count('good')
1
>>> t.index('good')
4

t.plot(8)

from nltk.corpus import stopwords
stopwords.fileids() # 具体的语言

###  果然没有中文
['arabic', 'azerbaijani', 'danish', 'dutch', 'english', 'finnish', 'french', 'german', 'greek',
 'hungarian', 'italian', 'kazakh', 'nepali', 'norwegian', 'portuguese', 'romanian', 'russian',
  'spanish', 'swedish', 'turkish']
  ```

"i me my myself we our ours ourselves you you're you've you'll you'd your yours yourself yourselves he him his himself she she's her hers herself it it's its itself they them their theirs themselves what which who whom this that that'll these those am is are was were be been being have has had having do does did doing a an the and but if or because as until while of at by for with about against between into through during before after above below to from up down in out on off over under again further then once here there when where why how all any both each few more most other some such no nor not only own same so than too very s t can will just don don't should should've now d ll m o re ve y ain aren aren't couldn couldn't didn didn't doesn doesn't hadn hadn't hasn hasn't haven haven't isn isn't ma mightn mightn't mustn mustn't needn needn't shan shan't shouldn shouldn't wasn wasn't weren weren't won won't wouldn wouldn't "

test_words = [word.lower() for word in tokens] #  tokens是上面的句子
test_words_set = set(test_words) # 集合
test_words_set.intersection(set(stopwords.words('english')))
>>>{'and', 'have', 'in', 'is', 'no', 'the', 'to', 'very', 'we'}

filtered = [w for w in test_words_set if(w not in stopwords.words('english'))]
filtered

['today',
 'good',
 'windy',
 'sunny',
 'afternoon',
 'play',
 'basketball',
 'tomorrow',
 'weather',
 'classes',
 ',',
 '.',
 "'s"]

from nltk import pos_tag
tags = pos_tag(tokens)
tags

[('Today', 'NN'),
 ("'s", 'POS'),
 ('weather', 'NN'),
 ('is', 'VBZ'),
 ('good', 'JJ'),
 (',', ','),
 ('very', 'RB'),
 ('windy', 'JJ'),
 ('and', 'CC'),
 ('sunny', 'JJ'),
 (',', ','),
 ('we', 'PRP'),
 ('have', 'VBP'),
 ('no', 'DT'),
 ('classes', 'NNS'),
 ('in', 'IN'),
 ('the', 'DT'),
 ('afternoon', 'NN'),
 (',', ','),
 ('We', 'PRP'),
 ('have', 'VBP'),
 ('to', 'TO'),
 ('play', 'VB'),
 ('basketball', 'NN'),
 ('tomorrow', 'NN'),
 ('.', '.')]

from nltk.chunk import RegexpParser
sentence = [('the','DT'),('little','JJ'),('yellow','JJ'),('dog','NN'),('died','VBD')]
grammer = "MY_NP: {<DT>?<JJ>*<NN>}"
cp = nltk.RegexpParser(grammer) #生成规则
result = cp.parse(sentence) #进行分块
print(result)
out:
result.draw() #调用matplotlib库画出来

from nltk import ne_chunk
sentence = "Edison went to Tsinghua University today."
print(ne_chunk(pos_tag(word_tokenize(sentence))))

(S
  (PERSON Edison/NNP)
  went/VBD
  to/TO
  (ORGANIZATION Tsinghua/NNP University/NNP)
  today/NN
  ./.)