2-3 표제어와 어간

표제어 추출

import spacy
nlp = spacy.load('en')
doc = nlp(u"he was running late")
for token in doc:
		print('{} --> {}'.format(token, token.lemma_))   #format() 함수는 따옴표 속 {} 안에 지정해준 변수들을 차례대로 넣어줍니다.

he --> he
was --> be
running --> run
late --> late

어간 추출

import nltk
from nltk.stem import PorterStemmer
from nltk.tokenize import word_tokenize
s = PorterStemmer()
text = "This was not the map we found in Billy Bones's chest, but an accurate copy, complete in all things--names and heights and soundings--with the single exception of the red crosses and the written notes."
words = word_tokenize(text)
print(words)
  #문장의 단어들을 나눔
['This', 'was', 'not', 'the', 'map', 'we', 'found', 'in', 'Billy', 'Bones', "'s", 'chest', ',', 'but', 'an', 'accurate', 'copy', ',', 'complete', 'in', 'all', 'things', '--', 'names', 'and', 'heights', 'and', 'soundings', '--', 'with', 'the', 'single', 'exception', 'of', 'the', 'red', 'crosses', 'and', 'the', 'written', 'notes', '.']
  #각 단어들이 어간 추출 과정을 거친 결과
[s.stem(w) for w in words]
['thi', 'wa', 'not', 'the', 'map', 'we', 'found', 'in', 'billi', 'bone', "'s", 'chest', ',', 'but', 'an', 'accur', 'copi', ',', 'complet', 'in', 'all', 'thing', '--', 'name', 'and', 'height',
 'and', 'sound', '--', 'with', 'the', 'singl', 'except', 'of',
 'the', 'red', 'cross', 'and', 'the', 'written', 'note', '.']