python-visualization-crawler/WeiboTopicScrapy.py at main · COOLMSF/python-visualization-crawler · GitHub

History

494 lines (446 loc) · 20.7 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

# -*- coding: utf-8 -*-

import requests

from lxml import etree

from collections import OrderedDict

from urllib.parse import quote

import csv

import traceback

import random

import re

from time import sleep

import os

from datetime import datetime, timedelta

import sys

Cookie = 'SCF=AtXLglXmCgR8AbTdUcSocE6mcnl_Mk232fDtiB0lfc4_t4daI7YiDvfSi29hqDxN2krttq2Vd36tu7BF_d493vQ.; login_sid_t=3a25024492e5a6807c7531609f5e8af2; cross_origin_proto=SSL; _s_tentry=cn.bing.com; Apache=3573941561831.7373.1618126514882; SINAGLOBAL=3573941561831.7373.1618126514882; ULV=1618126514898:1:1:1:3573941561831.7373.1618126514882:; TC-V-WEIBO-G0=35846f552801987f8c1e8f7cec0e2230; XSRF-TOKEN=ZBlPsUguNTD9xnw1UZibOqsk; SSOLoginState=1618378415; SUB=_2A25Ncg7_DeRhGeBO6FMU9CbEyjuIHXVunJK3rDV8PUJbkNANLRDakW1NSgy_LWAhRW3ewHUQ0DtF2AtdQONlb1PI; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W5ajGj7bbOjdh3e96f3nQCF5NHD95QcehepSKBR1h2NWs4DqcjiqcvadsxLM._T; wvr=6; UOR=cn.bing.com,weibo.com,www.baidu.com; wb_view_log_6031548817=1366*7681.75; webim_unReadCount=%7B%22time%22%3A1618480799471%2C%22dm_pub_total%22%3A0%2C%22chat_group_client%22%3A0%2C%22chat_group_notice%22%3A0%2C%22allcountNum%22%3A50%2C%22msgbox%22%3A0%7D; wb_timefeed_6031548817=1'

User_Agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0'

class WeiboTopicScrapy():

def __init__(self, keyword, filter, start_time, end_time):

self.headers = {

'Cookie': Cookie,

'User_Agent': User_Agent

}

self.keyword = keyword

self.filter = filter # 1: 原创微博； 0：所有微博

self.start_time = time_params_formatter(start_time, offset_hour=-8)

self.end_time = time_params_formatter(end_time, offset_day=-1, offset_hour=-7)

self.got_num = 0 # 爬取到的微博数

self.weibo = [] # 存储爬取到的所有微博信息

if not os.path.exists('topic'):

os.mkdir('topic')

self.run()

def deal_html(self, url):

"""处理html"""

try:

html = requests.get(url, headers=self.headers).content

selector = etree.HTML(html)

return selector

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def deal_garbled(self, info):

"""处理乱码"""

try:

info = (info.xpath('string(.)').replace(u'\u200b', '').encode(

sys.stdout.encoding, 'ignore').decode(sys.stdout.encoding))

return info

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_long_weibo(self, weibo_link):

"""获取长原创微博"""

try:

selector = self.deal_html(weibo_link)

info = selector.xpath("//div[@class='c']")[1]

wb_content = self.deal_garbled(info)

wb_time = info.xpath("//span[@class='ct']/text()")[0]

weibo_content = wb_content[wb_content.find(':') +

1:wb_content.rfind(wb_time)]

return weibo_content

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_original_weibo(self, info, weibo_id):

"""获取原创微博"""

try:

weibo_content = self.deal_garbled(info)

weibo_content = weibo_content[:weibo_content.rfind(u'赞')]

a_text = info.xpath('div//a/text()')

if u'全文' in a_text:

weibo_link = 'https://weibo.cn/comment/' + weibo_id

wb_content = self.get_long_weibo(weibo_link)

if wb_content:

weibo_content = wb_content

return weibo_content

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_long_retweet(self, weibo_link):

"""获取长转发微博"""

try:

wb_content = self.get_long_weibo(weibo_link)

weibo_content = wb_content[:wb_content.rfind('原文转发')]

return weibo_content

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_retweet(self, info, weibo_id):

"""获取转发微博"""

try:

original_user = info.xpath("div/span[@class='cmt']/a/text()")

if not original_user:

wb_content = '转发微博已被删除'

return wb_content

else:

original_user = original_user[0]

wb_content = self.deal_garbled(info)

wb_content = wb_content[wb_content.find(':') +

1:wb_content.rfind('赞')]

wb_content = wb_content[:wb_content.rfind('赞')]

a_text = info.xpath('div//a/text()')

if '全文' in a_text:

weibo_link = 'https://weibo.cn/comment/' + weibo_id

weibo_content = self.get_long_retweet(weibo_link)

if weibo_content:

wb_content = weibo_content

retweet_reason = self.deal_garbled(info.xpath('div')[-1])

retweet_reason = retweet_reason[:retweet_reason.rindex('赞')]

wb_content = (retweet_reason + '\n' + '原始用户: ' + original_user +

'\n' + '转发内容: ' + wb_content)

return wb_content

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_weibo_content(self, info, is_original):

"""获取微博内容"""

try:

weibo_id = info.xpath('@id')[0][2:]

if is_original:

weibo_content = self.get_original_weibo(info, weibo_id)

else:

weibo_content = self.get_retweet(info, weibo_id)

print(weibo_content)

return weibo_content

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_publish_place(self, info):

"""获取微博发布位置"""

try:

div_first = info.xpath('div')[0]

a_list = div_first.xpath('a')

publish_place = '无'

for a in a_list:

if ('place.weibo.com' in a.xpath('@href')[0]

and a.xpath('text()')[0] == '显示地图'):

weibo_a = div_first.xpath("span[@class='ctt']/a")

if len(weibo_a) >= 1:

publish_place = weibo_a[-1]

if ('视频' == div_first.xpath(

"span[@class='ctt']/a/text()")[-1][-2:]):

if len(weibo_a) >= 2:

publish_place = weibo_a[-2]

else:

publish_place = '无'

publish_place = self.deal_garbled(publish_place)

break

print('微博发布位置: ' + publish_place)

return publish_place

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_publish_time(self, info):

"""获取微博发布时间"""

try:

str_time = info.xpath("div/span[@class='ct']")

str_time = self.deal_garbled(str_time[0])

publish_time = str_time.split('来自')[0]

if '刚刚' in publish_time:

publish_time = datetime.now().strftime('%Y-%m-%d %H:%M')

elif '分钟' in publish_time:

minute = publish_time[:publish_time.find('分钟')]

minute = timedelta(minutes=int(minute))

publish_time = (datetime.now() -

minute).strftime('%Y-%m-%d %H:%M')

elif '今天' in publish_time:

today = datetime.now().strftime('%Y-%m-%d')

time = publish_time[3:]

publish_time = today + ' ' + time

elif '月' in publish_time:

year = datetime.now().strftime('%Y')

month = publish_time[0:2]

day = publish_time[3:5]

time = publish_time[7:12]

publish_time = year + '-' + month + '-' + day + ' ' + time

else:

publish_time = publish_time[:16]

print('微博发布时间: ' + publish_time)

return publish_time

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_publish_tool(self, info):

"""获取微博发布工具"""

try:

str_time = info.xpath("div/span[@class='ct']")

str_time = self.deal_garbled(str_time[0])

if len(str_time.split('来自')) > 1:

publish_tool = str_time.split(u'来自')[1]

else:

publish_tool = '无'

print('微博发布工具: ' + publish_tool)

return publish_tool

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_weibo_footer(self, info):

"""获取微博点赞数、转发数、评论数"""

try:

footer = {}

pattern = r'\d+'

str_footer = info.xpath('div')[-1]

str_footer = self.deal_garbled(str_footer)

str_footer = str_footer[str_footer.rfind('赞'):]

weibo_footer = re.findall(pattern, str_footer, re.M)

up_num = int(weibo_footer[0])

print('点赞数: ' + str(up_num))

footer['up_num'] = up_num

retweet_num = int(weibo_footer[1])

print('转发数: ' + str(retweet_num))

footer['retweet_num'] = retweet_num

comment_num = int(weibo_footer[2])

print('评论数: ' + str(comment_num))

footer['comment_num'] = comment_num

return footer

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def extract_picture_urls(self, info, weibo_id):

print('开始提取图片 URL')

"""提取微博原始图片url"""

try:

a_list = info.xpath('./div/a/@href')

all_pic = 'https://weibo.cn/mblog/picAll/' + weibo_id + '?rl=1'

if all_pic in a_list:

selector = self.deal_html(all_pic)

preview_picture_list = selector.xpath('//img/@src')

picture_list = [

p.replace('/thumb180/', '/large/')

for p in preview_picture_list

]

picture_urls = ','.join(picture_list)

print(picture_urls)

else:

picture_urls = '无'

# return picture_urls

# if info.xpath('.//img/@src'):

# preview_picture = info.xpath('.//img/@src')[-1]

# picture_urls = preview_picture.replace(

# '/wap180/', '/large/')

# else:

# sys.exit(

# "爬虫微博可能被设置成了'不显示图片'，请前往"

# "'https://weibo.cn/account/customize/pic'，修改为'显示'"

# )

return picture_urls

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_picture_urls(self, info, is_original):

"""获取微博原始图片url"""

try:

weibo_id = info.xpath('@id')[0][2:]

picture_urls = {}

if is_original:

original_pictures = self.extract_picture_urls(info, weibo_id)

picture_urls['original_pictures'] = original_pictures

if not self.filter:

picture_urls['retweet_pictures'] = '无'

else:

retweet_url = info.xpath("div/a[@class='cc']/@href")[0]

retweet_id = retweet_url.split('/')[-1].split('?')[0]

retweet_pictures = self.extract_picture_urls(info, retweet_id)

picture_urls['retweet_pictures'] = retweet_pictures

a_list = info.xpath('div[last()]/a/@href')

original_picture = '无'

for a in a_list:

if a.endswith(('.gif', '.jpeg', '.jpg', '.png')):

original_picture = a

break

picture_urls['original_pictures'] = original_picture

return picture_urls

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def get_publisher_info(self, link):

response = requests.get(url=link, headers=self.headers)

html = self.deal_html(link)

user_info = html.xpath('//div[@class="ut"]/span[@class="ctt"]')[0]

user_info = user_info.xpath('string(.)').strip()

# 去掉 \xa0

user_info = ' '.join(user_info.split())

kindex = user_info.index(' ')

username = user_info[:kindex]

sex = user_info[kindex + 1:user_info.index('/')]

province = user_info[user_info.index('/') + 1:user_info.rindex(' ')]

following = html.xpath('//div[@class="tip2"]/a[1]/text()')[0]

following = following[3:-1]

followd = html.xpath('//div[@class="tip2"]/a[2]/text()')[0]

followd = followd[3:-1]

# print(link, followd)

return username, sex, province, following, followd

def get_one_weibo(self, info):

"""获取一条微博的全部信息"""

try:

weibo = OrderedDict()

is_original = False if len(info.xpath("div/span[@class='cmt']")) > 3 else True

if (not self.filter) or is_original:

weibo['id'] = info.xpath('@id')[0][2:]

# weibo['publisher'] = info.xpath('div/a/text()')[0]

publisher_link = info.xpath('div/a/@href')[0]

weibo['publisher_name'], weibo['publisher_sex'], weibo['publisher_province'], \

weibo['publisher_following'], weibo['publisher_followed'] = self.get_publisher_info(publisher_link)

weibo['content'] = self.get_weibo_content(info,

is_original) # 微博内容

weibo['publish_place'] = self.get_publish_place(info) # 微博发布位置

weibo['publish_time'] = self.get_publish_time(info) # 微博发布时间

weibo['publish_tool'] = self.get_publish_tool(info) # 微博发布工具

footer = self.get_weibo_footer(info)

weibo['up_num'] = footer['up_num'] # 微博点赞数

weibo['retweet_num'] = footer['retweet_num'] # 转发数

weibo['comment_num'] = footer['comment_num'] # 评论数

picture_urls = self.get_picture_urls(info, is_original)

weibo['original_pictures'] = picture_urls[

'original_pictures'] # 原创图片url

if not self.filter:

weibo['retweet_pictures'] = picture_urls[

'retweet_pictures'] # 转发图片url

weibo['original'] = is_original # 是否原创微博

else:

weibo = None

return weibo

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def write_csv(self, wrote_num):

"""将爬取的信息写入csv文件"""

try:

result_headers = [

'微博id',

'发布者昵称',

'发布者性别',

'发布者地区',

'发布者关注数',

'发布者粉丝数',

'微博正文',

'原始图片url',

'发布时间',

'发布工具',

'点赞数',

'转发数',

'评论数',

'发布位置',

]

if not self.filter:

result_headers.insert(14, '被转发微博原始图片url')

result_headers.insert(15, '是否为原创微博')

result_data = [w.values() for w in self.weibo][wrote_num:]

with open('topic/' + self.keyword + '.csv', 'a', encoding='utf-8-sig', newline='') as f:

writer = csv.writer(f)

if wrote_num == 0:

writer.writerows([result_headers])

writer.writerows(result_data)

print('%d条微博写入csv文件完毕:' % self.got_num)

except Exception as e:

print('Error: ', e)

traceback.print_exc()

def run(self):

wrote_num = 0

page1 = 0

random_pages = random.randint(1, 5)

# pageNum = 1000000

pageNum = 3

for page in range(1, pageNum):

print('\n\n第{}页....\n'.format(page))

Referer = 'https://weibo.cn/search/mblog?hideSearchFrame=&keyword={}&page={}'.format(quote(self.keyword),

page - 1)

headers = {

'Cookie': Cookie,

'User-Agent': User_Agent,

'Referer': Referer

}

params = {

'hideSearchFrame': '',

'keyword': self.keyword,

'advancedfilter': '1',

'starttime': self.start_time,

'endtime': self.end_time,

'sort': 'time',

'page': page

}

res = requests.get(url='https://weibo.cn/search/mblog', params=params, headers=headers)

html = etree.HTML(res.text.encode('utf-8'))

try:

weibos = html.xpath("//div[@class='c' and @id]")

# 自动结束

if len(weibos) == 0:

print('自动结束，大概率是因为内容爬完了，也请留意是否是 cookie 失效等情况\n')

break

for i in range(0, len(weibos)):

aweibo = self.get_one_weibo(info=weibos[i])

if aweibo:

self.weibo.append(aweibo)

self.got_num += 1

print('-' * 100)

if page % 3 == 0 and self.got_num > wrote_num: # 每爬3页写入一次文件

self.write_csv(wrote_num)

wrote_num = self.got_num

# 通过加入随机等待避免被限制。爬虫速度过快容易被系统限制(一段时间后限

# 制会自动解除)，加入随机等待模拟人的操作，可降低被系统限制的风险。默

# 认是每爬取1到5页随机等待6到10秒，如果仍然被限，可适当增加sleep时间

if page - page1 == random_pages and page < pageNum:

sleep(random.randint(6, 10))

page1 = page

# random_pages = random.randint(1, 5)

random_pages = random.randint(1, 3)

except:

print(traceback.format_exc())

# print(res.text)

if self.got_num > wrote_num:

self.write_csv(wrote_num) # 将剩余不足3页的微博写入文件

if not self.filter:

print('共爬取' + str(self.got_num) + '条微博')

else:

print('共爬取' + str(self.got_num) + '条原创微博')

def get_offset_date(days):

# 获取当前时间

today = datetime.now()

base_day = datetime(year=2016, month=1, day=1)

# 计算偏移量

offset = timedelta(days=days)

# 获取想要的日期的时间

res_date = (base_day + offset).strftime('%Y%m%d')

return res_date

# days = 205

# WeiboTopicScrapy(keyword=keyword, filter=1, start_time=get_offset_date(days=days), end_time=get_offset_date(days=days+5))

def time_params_formatter(params_time, offset_day=0, offset_hour=-8):

[temp_year, temp_month, temp_day, temp_hour] = [int(e) for e in params_time.split('-')]

print(temp_year)

temp_date = datetime(year=temp_year, month=temp_month, day=temp_day, hour=temp_hour)

temp_offset = timedelta(days=offset_day, hours=offset_hour)

res_time = (temp_date + temp_offset).strftime('%Y-%m-%d-%H')

return res_time

if __name__ == '__main__':

if len(sys.argv) != 2:

print("Usage: %s topic" % sys.argv[0])

sys.exit(-1)

# filter = 0 爬取所有微博，filter = 1 爬取原创微博

keyword = sys.argv[1]

# 时间是从 start_time 到 end_time 这样

# 程序是从 end_time 到 start_time 这样爬

# end_time + 1 day + 8 hour

# start_time + 8hour

# 时间尽量短一点，不然到时后分析评论感情的时候会很慢

start_time, end_time = '2020-10-31-04', '2021-10-31-05'

if start_time >= end_time:

raise Exception('start_time 是离现在更远的那个时间，必须小于 end_time')

WeiboTopicScrapy(keyword=keyword, filter=1, start_time=start_time, end_time=end_time)

python-visualization-crawler/WeiboTopicScrapy.py at main · COOLMSF/python-visualization-crawler · GitHub

Expand file tree

Breadcrumbs

WeiboTopicScrapy.py

Latest commit

History

Breadcrumbs

WeiboTopicScrapy.py

File metadata and controls