HTML2Markdown/html2markdown.py at master · junninho/HTML2Markdown · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
import json
import html2text
import re
import xlsxwriter

h = html2text.HTML2Text()
h.body_width = 0

input_file = open('json_original.json').read()
output_file = open('output.txt', 'w')
object_file = open('output.json', 'w')

workbook = xlsxwriter.Workbook('extracted_banners.xlsx')
worksheet = workbook.add_worksheet()

row = 0
col = 0

data = json.loads(input_file)
i = 0


for post in data['data']['posts']:
	post['markdown'] = h.handle(post['html'])
	banners = []
	title = 'https://www.policygenius.com/blog/'+post['slug']
	search = '\[banner-.*?]', '\[Banner-.*?]', '\[video-.*?]', '\[recommended-posts .*?]', '\[widget-.*?]', '\[hotspot .*?]'
	for ban in search:
		banners += re.findall(ban, post['markdown'])
	if banners:
		worksheet.write(row, col,     title)
		worksheet.write(row, col + 1,     ','.join(banners))
		row += 1

	post['markdown'] = re.sub('\[banner-.*?]', ' ', post['markdown'])
	post['markdown'] = re.sub('\[Banner-.*?]', ' ', post['markdown'])
	post['markdown'] = re.sub('\[video-.*?]', ' ', post['markdown'])
	post['markdown'] = re.sub('\[recommended-posts .*?]', ' ', post['markdown'])
	post['markdown'] = re.sub('\[widget-.*?]', ' ', post['markdown'])
	post['markdown'] = re.sub('\[hotspot .*?]', ' ', post['markdown'])
	post['image'] = re.sub('https://pg-lib', 'https://policygenius-blog', str(post['image']))

	output_file.write(post['markdown'])
object_file.write(json.dumps(data))
workbook.close()


print("Done")