pdf2md/syntax.py at develop · OlegVic/pdf2md · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
import re


class Syntax(object):
    def __init__(self):
        pass

    def pattern(self):
        return 'plain-text'

    def newline(self):
        return True


class UrbanSyntax(Syntax):
    def __init__(self):
        pass

    def pattern(self, text):
        content = text.get_text().strip()
        # content = text.get_text().encode('utf8').strip()

        if not content:
            return 'none'

        if content.isdigit():  # page number
            return 'none'

        if 130 < text.x0 and text.x1 < 480:
            if text.size == 12:
                return 'heading-2'
            if text.size < 12:
                return 'heading-3'
            if text.size > 12:
                return 'heading-1'

        if text.size == 18:
            return 'heading-4'

        if text.size == 16:
            return 'heading-3'

        if text.size == 20:
            return 'heading-2'

        if content == content.upper():
            if text.bold:  # special case for neihu page 2
                return 'heading-2'
            else:
                return 'heading-3'

        mo = re.search(r'^(I|II|III|IV|V|VI|VII|VIII|IX|X).', content)
        if mo:
            return 'heading-4'

        mo = re.search(r'^(（|\()(I|II|III|IV|V|VI|VII|VIII|IX|X)(）|\))', content)
        if mo:
            return 'heading-5'

        mo = re.search(r'^(\d+\.)+', content)
        if mo:
            return 'ordered-list-item'

        if text.x0 < 90.1:  # special case for neihu page 2
            return 'unordered-list-item'

        return 'plain-text'

    def newline(self, text):
        # content = text.get_text().encode('utf8').strip()
        content = text.get_text() #.strip()

        if text.x0 < 90.1:  # special case for neihu page 2
            return True

        mo = re.search('\n\n$', content)
        if mo:
            return True

        mo = re.search('\.$', content)
        if mo:
            return True

        if text.x1 > 505.0:  # reach the right margin
            return False

        return False

    def purify(self, text):
        # content = text.get_text().encode('utf8').strip()
        content = text.get_text().strip()

        mo = re.match(r'(I|II|III|IV|V|VI|VII|VIII|IX|X). (.*)', content)
        if mo:
            return mo.group(2)

        mo = re.match(r'(（|\()(I|II|III|IV|V|VI|VII|VIII|IX|X)(）|\))(.*)', content)
        if mo:
            return mo.group(4)

        mo = re.match(r'^\d+、(.*)', content)
        if mo:
            return mo.group(1)

        return content