NLTK 2 获得文本语料和词汇资源

1 获得文本语料库

1.1 古登堡语料库（Project Gutenberg）

#nltk包含gutenberg的一小部分文本
import nltk
nltk.corpus.gutenberg.fileids()
emma=nltk.corpus.gutenberg.words('austen-emma.txt')
print len(emma)                    #192427

form nltk.corpus import gutenberg
gutenberg.fileids()
for fileid in gutenberg.fileids():
num_chars=len(gutenberg.raw(fileid))
num_words=len(gutenberg.words(fileid))
num_sents=len(gutenberg.sents(fileid))
num_vocab=len(set([w.lower() for w in gutenberg.words(fileid)]))
print int(num_chars/num_words),int(num_words/num_sents),int(num_words/num_vocab),fileid

1.2 网络和聊天文本

from nltk.corpus import webtext
#还有from nltk.corpus import nps_chat
for fileid in webtext.fileids():
print fileid, webtext.raw(fileid)[:65], '...'

1.3 布朗语料库

from nltk.corpus import brown

1.4 路透社语料库

from nltk.corpus import reuters

1.5 就职演说语料库

from nltk.corpus import inaugural

1.6 标注文本语料库

1.7 其他预言的语料库

from nltk.corpus import cess_esp.words()