BuildingMachineLearningSystemsWithPython/ch06/04_sent.py at master · luispedro/BuildingMachineLearningSystemsWithPython · GitHub

History

349 lines (270 loc) · 9.89 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

# This code is supporting material for the book

# Building Machine Learning Systems with Python

# by Willi Richert and Luis Pedro Coelho

# published by PACKT Publishing

# It is made available under the MIT License

# This script trains tries to tweak hyperparameters to improve P/R AUC

import time

start_time = time.time()

import re

import nltk

import numpy as np

from sklearn.metrics import precision_recall_curve, roc_curve, auc

from sklearn.cross_validation import ShuffleSplit

from utils import plot_pr

from utils import load_sanders_data

from utils import tweak_labels

from utils import log_false_positives

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.pipeline import Pipeline, FeatureUnion

from sklearn.grid_search import GridSearchCV

from sklearn.metrics import f1_score

from sklearn.base import BaseEstimator

from sklearn.naive_bayes import MultinomialNB

from utils import load_sent_word_net

sent_word_net = load_sent_word_net()

phase = "04"

import json

poscache_filename = "poscache.json"

try:

poscache = json.load(open(poscache_filename, "r"))

except IOError:

poscache = {}

class LinguisticVectorizer(BaseEstimator):

def get_feature_names(self):

return np.array(['sent_neut', 'sent_pos', 'sent_neg',

'nouns', 'adjectives', 'verbs', 'adverbs',

'allcaps', 'exclamation', 'question'])

def fit(self, documents, y=None):

return self

def _get_sentiments(self, d):

# http://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html

sent = tuple(nltk.word_tokenize(d))

if poscache is not None:

if d in poscache:

tagged = poscache[d]

else:

poscache[d] = tagged = nltk.pos_tag(sent)

else:

tagged = nltk.pos_tag(sent)

pos_vals = []

neg_vals = []

nouns = 0.

adjectives = 0.

verbs = 0.

adverbs = 0.

for w, t in tagged:

p, n = 0, 0

sent_pos_type = None

if t.startswith("NN"):

sent_pos_type = "n"

nouns += 1

elif t.startswith("JJ"):

sent_pos_type = "a"

adjectives += 1

elif t.startswith("VB"):

sent_pos_type = "v"

verbs += 1

elif t.startswith("RB"):

sent_pos_type = "r"

adverbs += 1

if sent_pos_type is not None:

sent_word = "%s/%s" % (sent_pos_type, w)

if sent_word in sent_word_net:

p, n = sent_word_net[sent_word]

pos_vals.append(p)

neg_vals.append(n)

l = len(sent)

avg_pos_val = np.mean(pos_vals)

avg_neg_val = np.mean(neg_vals)

return [1 - avg_pos_val - avg_neg_val, avg_pos_val, avg_neg_val,

nouns / l, adjectives / l, verbs / l, adverbs / l]

def transform(self, documents):

obj_val, pos_val, neg_val, nouns, adjectives, verbs, adverbs = np.array(

[self._get_sentiments(d) for d in documents]).T

allcaps = []

exclamation = []

question = []

for d in documents:

allcaps.append(

np.sum([t.isupper() for t in d.split() if len(t) > 2]))

exclamation.append(d.count("!"))

question.append(d.count("?"))

result = np.array(

[obj_val, pos_val, neg_val, nouns, adjectives, verbs, adverbs, allcaps,

exclamation, question]).T

return result

emo_repl = {

# positive emoticons

"<3": " good ",

":d": " good ", # :D in lower case

":dd": " good ", # :DD in lower case

"8)": " good ",

":-)": " good ",

":)": " good ",

";)": " good ",

"(-:": " good ",

"(:": " good ",

# negative emoticons:

":/": " bad ",

":>": " sad ",

":')": " sad ",

":-(": " bad ",

":(": " bad ",

":S": " bad ",

":-S": " bad ",

}

emo_repl_order = [k for (k_len, k) in reversed(

sorted([(len(k), k) for k in list(emo_repl.keys())]))]

re_repl = {

r"\br\b": "are",

r"\bu\b": "you",

r"\bhaha\b": "ha",

r"\bhahaha\b": "ha",

r"\bdon't\b": "do not",

r"\bdoesn't\b": "does not",

r"\bdidn't\b": "did not",

r"\bhasn't\b": "has not",

r"\bhaven't\b": "have not",

r"\bhadn't\b": "had not",

r"\bwon't\b": "will not",

r"\bwouldn't\b": "would not",

r"\bcan't\b": "can not",

r"\bcannot\b": "can not",

}

def create_union_model(params=None):

def preprocessor(tweet):

tweet = tweet.lower()

for k in emo_repl_order:

tweet = tweet.replace(k, emo_repl[k])

for r, repl in re_repl.items():

tweet = re.sub(r, repl, tweet)

return tweet.replace("-", " ").replace("_", " ")

tfidf_ngrams = TfidfVectorizer(preprocessor=preprocessor,

analyzer="word")

ling_stats = LinguisticVectorizer()

all_features = FeatureUnion(

[('ling', ling_stats), ('tfidf', tfidf_ngrams)])

#all_features = FeatureUnion([('tfidf', tfidf_ngrams)])

#all_features = FeatureUnion([('ling', ling_stats)])

clf = MultinomialNB()

pipeline = Pipeline([('all', all_features), ('clf', clf)])

if params:

pipeline.set_params(**params)

return pipeline

def __grid_search_model(clf_factory, X, Y):

cv = ShuffleSplit(

n=len(X), n_iter=10, test_size=0.3, random_state=0)

param_grid = dict(vect__ngram_range=[(1, 1), (1, 2), (1, 3)],

vect__min_df=[1, 2],

vect__smooth_idf=[False, True],

vect__use_idf=[False, True],

vect__sublinear_tf=[False, True],

vect__binary=[False, True],

clf__alpha=[0, 0.01, 0.05, 0.1, 0.5, 1],

)

grid_search = GridSearchCV(clf_factory(),

param_grid=param_grid,

cv=cv,

score_func=f1_score,

verbose=10)

grid_search.fit(X, Y)

clf = grid_search.best_estimator_

print(clf)

return clf

def train_model(clf, X, Y, name="NB ngram", plot=False):

# create it again for plotting

cv = ShuffleSplit(

n=len(X), n_iter=10, test_size=0.3, random_state=0)

train_errors = []

test_errors = []

scores = []

pr_scores = []

precisions, recalls, thresholds = [], [], []

clfs = [] # just to later get the median

for train, test in cv:

X_train, y_train = X[train], Y[train]

X_test, y_test = X[test], Y[test]

clf.fit(X_train, y_train)

clfs.append(clf)

train_score = clf.score(X_train, y_train)

test_score = clf.score(X_test, y_test)

train_errors.append(1 - train_score)

test_errors.append(1 - test_score)

scores.append(test_score)

proba = clf.predict_proba(X_test)

fpr, tpr, roc_thresholds = roc_curve(y_test, proba[:, 1])

precision, recall, pr_thresholds = precision_recall_curve(

y_test, proba[:, 1])

pr_scores.append(auc(recall, precision))

precisions.append(precision)

recalls.append(recall)

thresholds.append(pr_thresholds)

if plot:

scores_to_sort = pr_scores

median = np.argsort(scores_to_sort)[len(scores_to_sort) / 2]

plot_pr(pr_scores[median], name, phase, precisions[median],

recalls[median], label=name)

log_false_positives(clfs[median], X_test, y_test, name)

summary = (np.mean(scores), np.std(scores),

np.mean(pr_scores), np.std(pr_scores))

print("%.3f\t%.3f\t%.3f\t%.3f\t" % summary)

return np.mean(train_errors), np.mean(test_errors)

def print_incorrect(clf, X, Y):

Y_hat = clf.predict(X)

wrong_idx = Y_hat != Y

X_wrong = X[wrong_idx]

Y_wrong = Y[wrong_idx]

Y_hat_wrong = Y_hat[wrong_idx]

for idx in range(len(X_wrong)):

print("clf.predict('%s')=%i instead of %i" %

(X_wrong[idx], Y_hat_wrong[idx], Y_wrong[idx]))

def get_best_model():

best_params = dict(all__tfidf__ngram_range=(1, 2),

all__tfidf__min_df=1,

all__tfidf__stop_words=None,

all__tfidf__smooth_idf=False,

all__tfidf__use_idf=False,

all__tfidf__sublinear_tf=True,

all__tfidf__binary=False,

clf__alpha=0.01,

)

best_clf = create_union_model(best_params)

return best_clf

if __name__ == "__main__":

X_orig, Y_orig = load_sanders_data()

#from sklearn.utils import shuffle

# print "shuffle, sample"

#X_orig, Y_orig = shuffle(X_orig, Y_orig)

#X_orig = X_orig[:100,]

#Y_orig = Y_orig[:100,]

classes = np.unique(Y_orig)

for c in classes:

print("#%s: %i" % (c, sum(Y_orig == c)))

print("== Pos vs. neg ==")

pos_neg = np.logical_or(Y_orig == "positive", Y_orig == "negative")

X = X_orig[pos_neg]

Y = Y_orig[pos_neg]

Y = tweak_labels(Y, ["positive"])

train_model(get_best_model(), X, Y, name="pos vs neg", plot=True)

print("== Pos/neg vs. irrelevant/neutral ==")

X = X_orig

Y = tweak_labels(Y_orig, ["positive", "negative"])

# best_clf = grid_search_model(create_union_model, X, Y, name="sent vs

# rest", plot=True)

train_model(get_best_model(), X, Y, name="pos+neg vs rest", plot=True)

print("== Pos vs. rest ==")

X = X_orig

Y = tweak_labels(Y_orig, ["positive"])

train_model(get_best_model(), X, Y, name="pos vs rest",

plot=True)

print("== Neg vs. rest ==")

X = X_orig

Y = tweak_labels(Y_orig, ["negative"])

train_model(get_best_model(), X, Y, name="neg vs rest",

plot=True)

print("time spent:", time.time() - start_time)

json.dump(poscache, open(poscache_filename, "w"))

BuildingMachineLearningSystemsWithPython/ch06/04_sent.py at master · luispedro/BuildingMachineLearningSystemsWithPython · GitHub

Expand file tree

Breadcrumbs

04_sent.py

Latest commit

History

Breadcrumbs

04_sent.py

File metadata and controls