Skip to content

Commit 69e2c02

Browse files
IPQualityScoreIPQualityScore
committed
Fix bug with reader sometimes returning the wrong data for IPs. Add support for blacklist files.
1 parent 17f13bb commit 69e2c02

3 files changed

Lines changed: 60 additions & 42 deletions

File tree

‎IPQualityScore/BinaryOption.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,17 @@ def Create(value):
1717
result.SetData(value)
1818
return result
1919

20-
IPV4MAP = 0B0001
20+
IPV4MAP = 1 << 0
2121
IPV6MAP = 1 << 1
22+
BLACKLISTFILE = 1 << 2
23+
BINARYDATA = 1 << 7
24+
2225
TREEDATA = 1 << 2
2326
STRINGDATA = 1 << 3
2427
SMALLINTDATA = 1 << 4
2528
INTDATA = 1 << 5
2629
FLOATDATA = 1 << 6
27-
BINARYDATA = 1 << 7
30+
2831
ISPROXY = 1 << 0
2932
ISVPN = 1 << 1
3033
ISTOR = 1 << 2
@@ -33,6 +36,7 @@ def Create(value):
3336
RECENTABUSE = 1 << 5
3437
ISBLACKLISTED = 1 << 6
3538
ISPRIVATE = 1 << 7
39+
3640
ISMOBILE = 1 << 0
3741
HASOPENPORTS = 1 << 1
3842
ISHOSTINGPROVIDER = 1 << 2
@@ -41,6 +45,7 @@ def Create(value):
4145
PUBLICACCESSPOINT = 1 << 5
4246
RESERVEDONE = 1 << 6
4347
RESERVEDTWO = 1 << 7
48+
4449
RESERVEDTHREE = 1 << 0
4550
RESERVEDFOUR = 1 << 1
4651
RESERVEDFIVE = 1 << 2

‎IPQualityScore/DBReader.py‎

Lines changed: 52 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
import os
32
from binascii import hexlify
43
from socket import error, inet_aton, inet_pton, socket
@@ -28,6 +27,7 @@ class DBReader:
2827
record_bytes = None
2928
columns = []
3029
ipv6 = False
30+
blacklistfile = False
3131

3232
def __init__(self, filename:str):
3333
if not os.path.isfile(filename):
@@ -54,58 +54,60 @@ def Fetch(self, ip:str):
5454

5555
v_literal = self.IP2Literal(ip)
5656
position = 0
57-
57+
previous = {}
5858
file_position = self.tree_start + self.BASE_TREE_BYTES
59-
while True:
59+
60+
# Loop over tree. Will abort if we try too many times.
61+
for _ in range(257):
62+
previous[position] = file_position
6063
if len(v_literal) <= position:
6164
raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 8)")
62-
63-
next = 0
64-
try:
65-
if v_literal[position] == "0":
66-
pos = self.ReadAt(file_position, self.TREE_BYTE_WIDTH)
67-
if len(pos) == 4:
68-
next = unpack("<L", pos)[0]
69-
else:
70-
pos = self.ReadAt(file_position + 4, self.TREE_BYTE_WIDTH)
71-
if len(pos) == 4:
72-
next = unpack("<L", pos)[0]
7365

74-
except Exception:
75-
raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 9)")
76-
77-
try:
78-
if next == 0:
79-
if v_literal[position] == "0":
80-
pos = self.ReadAt(file_position + 4, self.TREE_BYTE_WIDTH)
81-
if len(pos) == 4:
82-
next = unpack("<L", pos)[0]
83-
84-
if next == 0:
85-
raise IPNotFoundException()
86-
else:
87-
pos = self.ReadAt(file_position, self.TREE_BYTE_WIDTH)
88-
if len(pos) == 4:
89-
next = unpack("<L", pos)[0]
90-
91-
if next == 0:
92-
raise IPNotFoundException()
93-
except Exception:
94-
raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 10)")
95-
96-
file_position = next
66+
if v_literal[position] == "0":
67+
pos = self.ReadAt(file_position, self.TREE_BYTE_WIDTH)
68+
if len(pos) == 4:
69+
file_position = unpack("<L", pos)[0]
70+
else:
71+
pos = self.ReadAt(file_position + 4, self.TREE_BYTE_WIDTH)
72+
if len(pos) == 4:
73+
file_position = unpack("<L", pos)[0]
74+
75+
if(self.blacklistfile == False):
76+
if(file_position == 0):
77+
for i in range(position):
78+
if(v_literal[position - i] == "1"):
79+
l = list(v_literal)
80+
l[position - i] = "0"
81+
v_literal = ''.join(l)
82+
83+
for n in range(position - i + 1, len(v_literal)):
84+
l = list(v_literal)
85+
l[n] = "1"
86+
v_literal = ''.join(l)
87+
88+
position = position - i
89+
file_position = previous[position]
90+
break
91+
continue
92+
9793
if(file_position < self.tree_end):
94+
if(file_position == 0):
95+
break
9896
position += 1
9997
continue
98+
10099
try:
101100
raw = self.ReadAt(file_position, self.record_bytes)
102101
except Exception:
103102
raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 11)")
104103

105104
try:
106105
return self.CreateRecord(raw)
106+
107107
except Exception:
108108
raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 12)")
109+
raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 13)")
110+
109111
def GetColumns(self):
110112
return self.columns
111113

@@ -140,9 +142,12 @@ def SetupHeaders(self):
140142
if file_type.Has(BinaryOption.IPV6MAP):
141143
self.valid = True
142144
self.ipv6 = True
143-
145+
144146
if file_type.Has(BinaryOption.BINARYDATA):
145147
self.binary_data = True
148+
149+
if file_type.Has(BinaryOption.BLACKLISTFILE):
150+
self.blacklistfile = True
146151

147152
if(self.valid == False):
148153
raise FileReaderException("Invalid file format, invalid first byte. EID 1.")
@@ -221,22 +226,27 @@ def CreateRecord(self,raw):
221226
value = unpack("<L", raw[current_byte:current_byte+4])[0]
222227
record.ASN(int(value))
223228
current_byte += 4
229+
224230
elif column.Name() == "Latitude":
225231
value = unpack("<f", raw[current_byte:current_byte+4])[0]
226232
record.Latitude(float(value))
227233
current_byte += 4
234+
228235
elif column.Name() == "Longitude":
229236
value = unpack("<f", raw[current_byte:current_byte+4])[0]
230237
record.Longitude(float(value))
231238
current_byte += 4
239+
232240
elif column.Name() == "ZeroFraudScore":
233241
value = unpack("B", raw[current_byte:current_byte+1])[0]
234242
record.SetFraudScore(0, int(value))
235243
current_byte += 1
244+
236245
elif column.Name() == "OneFraudScore":
237246
value = unpack("B", raw[current_byte:current_byte+1])[0]
238-
record.ASN(float(value))
247+
record.SetFraudScore(1, int(value))
239248
current_byte += 1
249+
240250
else:
241251
if (column.Type()).Has(BinaryOption.STRINGDATA):
242252
value = self.GetRangedStringValue(unpack("<L", raw[current_byte:current_byte+4])[0])
@@ -264,14 +274,17 @@ def IsIPv4(self, ip:str):
264274
try:
265275
inet_pton(AF_INET, ip)
266276
return True
277+
267278
except error:
268279
return False
269280

270281
def IsIPv6(self, ip:str):
271282
try:
272283
inet_pton(AF_INET6, ip)
273284
return True
285+
274286
except error:
275287
return False
288+
276289
def __del__(self):
277290
self.handler.close()

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
setuptools.setup(
77
name="PythonIPQSDBReader",
8-
version="0.0.1",
8+
version="1.0.2",
99
author="IPQualityScore",
1010
author_email="support@ipqualityscore.com",
1111
description="IPQualityScore IP Address Reputation & Proxy Detection DB Reader",

0 commit comments

Comments
 (0)