Applying Machine Learning to Sentiment Analysis

1. Obtaining the IMDb movie review dataset ：

A compressed archive of the movie review dataset  ---- http://ai.stanford.edu/~amaas/data/sentiment/

import pandas as pddf = pd.read_csv('./datasets/movie/movie_data.csv')print('Excerpt of the movie dataset', df.head(3))

('Excerpt of the movie dataset',                                               review  sentiment
0  In 1974, the teenager Martha Moxley (Maggie Gr...          1
1  OK... so... I really like Kris Kristofferson a...          0
2  ***SPOILER*** Do not read this, if you think a...          0)

2. Introducing the bag-of-words model

2.1 Transforming words into feature vectors

import numpy as npfrom sklearn.feature_extraction.text import CountVectorizercount = CountVectorizer()docs = np.array(['The sun is shining',                 'The weather is sweet',                 'The sun is shining and the weather is sweet'])bag = count.fit_transform(docs)print('Vocabulary', count.vocabulary_)print('bag.toarray()', bag.toarray())

('Vocabulary', {u'and': 0, u'weather': 6, u'sweet': 4, u'sun': 3, u'is': 1, u'the': 5, u'shining': 2})
('bag.toarray()', array([[0, 1, 1, 1, 0, 1, 0],
       [0, 1, 0, 0, 1, 1, 1],
       [1, 2, 1, 1, 1, 2, 1]], dtype=int64))

2.2 Assessing word relevancy via term frequency-inverse document frequency

from sklearn.feature_extraction.text import TfidfTransformernp.set_printoptions(precision=2)tfidf = TfidfTransformer(use_idf=True, norm='l2', smooth_idf=True)print(tfidf.fit_transform(count.fit_transform(docs)).toarray())

[[ 0.    0.43  0.56  0.56  0.    0.43  0.  ]
  [ 0.    0.43  0.    0.    0.56  0.43  0.56]
  [ 0.4   0.48  0.31  0.31  0.31  0.48  0.31]]

2.3 Cleaning text data

print('Excerpt:\n\n', df.loc[0, 'review'][-50:])

('Excerpt:\n\n', 'is seven.<br /><br />Title (Brazil): Not Available')

def preprocessor(text):    text = re.sub('<[^>]*>', '', text)    emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)', text)    text = re.sub('[\W]+', ' ', text.lower()) +\        ' '.join(emoticons).replace('-', '')    return textprint('Preprocessor on Excerpt:\n\n', preprocessor(df.loc[0, 'review'][-50:]))

('Preprocessor on Excerpt:\n\n', 'is seven title brazil not available')

res = preprocessor("</a>This :) is :( a test :-)!")print('Preprocessor on "</a>This :) is :( a test :-)!":\n\n', res)df['review'] = df['review'].apply(preprocessor)

('Preprocessor on "</a>This :) is :( a test :-)!":\n\n', 'this is a test :) :( :)')

2.4 Processing documents into tokens

from nltk.stem.porter import PorterStemmerporter = PorterStemmer()def tokenizer(text):    return text.split()def tokenizer_porter(text):    return [porter.stem(word) for word in text.split()]t1 = tokenizer('runners like running and thus they run')print("Tokenize: 'runners like running and thus they run'")print(t1)t2 = tokenizer_porter('runners like running and thus they run')print("\nPorter-Tokenize: 'runners like running and thus they run'")print(t2)

Tokenize: 'runners like running and thus they run'
['runners', 'like', 'running', 'and', 'thus', 'they', 'run']

Porter-Tokenize: 'runners like running and thus they run'
[u'runner', 'like', u'run', 'and', u'thu', 'they', 'run']

3. Training a logistic regression model for document classifcation

from sklearn.model_selection import GridSearchCVfrom sklearn.pipeline import Pipelinefrom sklearn.linear_model import LogisticRegressionfrom sklearn.feature_extraction.text import TfidfVectorizernltk.download('stopwords')stop = stopwords.words('english')X_train = df.loc[:25000, 'review'].valuesy_train = df.loc[:25000, 'sentiment'].valuesX_test = df.loc[25000:, 'review'].valuesy_test = df.loc[25000:, 'sentiment'].valuestfidf = TfidfVectorizer(strip_accents=None,                        lowercase=False,                        preprocessor=None)param_grid = [{'vect__ngram_range': [(1, 1)],               'vect__stop_words': [stop, None],               'vect__tokenizer': [tokenizer, tokenizer_porter],               'clf__penalty': ['l1', 'l2'],               'clf__C': [1.0, 10.0, 100.0]},              {'vect__ngram_range': [(1, 1)],               'vect__stop_words': [stop, None],               'vect__tokenizer': [tokenizer, tokenizer_porter],               'vect__use_idf':[False],               'vect__norm':[None],               'clf__penalty': ['l1', 'l2'],               'clf__C': [1.0, 10.0, 100.0]},              ]lr_tfidf = Pipeline([('vect', tfidf),                     ('clf', LogisticRegression(random_state=0))])gs_lr_tfidf = GridSearchCV(lr_tfidf, param_grid,                           scoring='accuracy',                           cv=5,                           verbose=1,                           n_jobs=-1)gs_lr_tfidf.fit(X_train, y_train)print('CV Accuracy: %.3f' % gs_lr_tfidf.best_score_)

CV Accuracy: 0.897

clf = gs_lr_tfidf.best_estimator_print('Test Accuracy: %.3f' % clf.score(X_test, y_test))

Test Accuracy: 0.899

Reference：《Python Machine Learning》