v1.0

Author: IPQualityScore

IPQualityScore/.gitignore

@@ -0,0 +1,2 @@
+*.pyc
+__pycache__

IPQualityScore/BinaryOption.py

@@ -0,0 +1,51 @@
+class BinaryOption:
+    def __init__(self):
+        self.__data = None
+    
+    def SetData(self, bit):
+        self.__data = bit
+    
+    def Has(self, value):
+        return (self.__data & value)
+    
+    def Set(self, value):
+        self.__data = self.__data | value
+    
+    @staticmethod
+    def Create(value):
+        result = BinaryOption()
+        result.SetData(value)
+        return result
+    
+    IPV4MAP             = 0B0001
+    IPV6MAP             = 1 << 1
+    TREEDATA            = 1 << 2
+    STRINGDATA          = 1 << 3
+    SMALLINTDATA        = 1 << 4
+    INTDATA             = 1 << 5
+    FLOATDATA           = 1 << 6
+    BINARYDATA          = 1 << 7
+    ISPROXY             = 1 << 0
+    ISVPN               = 1 << 1
+    ISTOR               = 1 << 2
+    ISCRAWLER           = 1 << 3
+    ISBOT               = 1 << 4
+    RECENTABUSE         = 1 << 5
+    ISBLACKLISTED       = 1 << 6
+    ISPRIVATE           = 1 << 7
+    ISMOBILE            = 1 << 0
+    HASOPENPORTS        = 1 << 1
+    ISHOSTINGPROVIDER   = 1 << 2
+    ACTIVEVPN           = 1 << 3
+    ACTIVETOR           = 1 << 4
+    PUBLICACCESSPOINT   = 1 << 5
+    RESERVEDONE         = 1 << 6
+    RESERVEDTWO         = 1 << 7
+    RESERVEDTHREE       = 1 << 0
+    RESERVEDFOUR        = 1 << 1
+    RESERVEDFIVE        = 1 << 2
+    CONNECTIONTYPEONE   = 1 << 3
+    CONNECTIONTYPETWO   = 1 << 4
+    CONNECTIONTYPETHREE = 1 << 5
+    ABUSEVELOCITYONE    = 1 << 6
+    ABUSEVELOCITYTWO    = 1 << 7

IPQualityScore/Columns.py

@@ -0,0 +1,34 @@
+from IPQualityScore.BinaryOption import BinaryOption
+
+class Column:
+
+    def __init__(self):
+        self.__Name     = None
+        self.__RawValue = None
+        self.__Type     = None
+        self.__Name     = None
+        self.__Value    = None
+
+    @staticmethod
+    def Create(name, stype, value= None):
+        C = Column()
+        C.Name(name)
+        C.Type(stype)
+        C.Value(value)
+
+        return C
+    
+    def Name(self, value = None):
+        if value != None:
+            self.__Name = value
+        return self.__Name
+
+    def Type(self, value:BinaryOption = None):
+        if value != None:
+            self.__Type  = value
+        return self.__Type
+    
+    def Value(self, value = None):
+        if value != None:
+            self.__Value = value
+        return self.__Value

IPQualityScore/DBReader.py

@@ -0,0 +1,277 @@
+
+import os
+from binascii import hexlify
+from socket import error, inet_aton, inet_pton, socket
+from socket import AF_INET
+from socket import AF_INET6
+from struct import unpack, unpack_from, error as struck_err
+from IPQualityScore.Columns         import Column
+from IPQualityScore.Exceptions      import FileReaderException
+from IPQualityScore.Exceptions      import IPNotFoundException
+from IPQualityScore.BinaryOption    import BinaryOption
+from IPQualityScore.IPQSRecord      import IPQSRecord
+import IPQualityScore.Utilities as Utilities
+
+class DBReader:
+    READER_VERSION = 1
+    BASE_TREE_BYTES = 5
+    TREE_BYTE_WIDTH = 4
+    BASE_HEADER_BYTES = 11
+    BINARY_DATA_HEADER = "<BBB"
+    NONBINARY_DATA_HEADER = "<B"
+    COLUNN_HEADER = "<P23xB" #"x%s/a23name/Cvalue" null 23null-padded_string UnsigedChar <x23xB
+    HEADERS = "<BB3BHL"
+
+    handler  = None
+    tree_start = None
+    tree_end = None
+    record_bytes = None
+    columns = []
+    ipv6 = False
+
+    def __init__(self, filename:str):
+        if not os.path.isfile(filename):
+            raise FileReaderException('Invalid or non existant file name specified. Please check the file and try again')
+        
+        try:
+            self.handler = open(filename,'rb')
+        except IOError:
+            raise FileReaderException('Invalid or non existant file name specified. Please check the file and try again')
+
+        self.SetupHeaders()
+        self.SetupColumns()
+        self.SetupTreeHeaders()
+
+    def Fetch(self, ip:str):
+        if not self.IsIPv4(ip) and not self.IsIPv6(ip):
+            raise FileReaderException("Attemtped to look up invalid IP address. Aborting.")
+        
+        if self.ipv6 and not self.IsIPv6(ip):
+            raise FileReaderException("Attemtped to look up IPv4 using IPv6 database file. Aborting.")
+        
+        if self.ipv6 == False and not self.IsIPv4(ip):
+            raise FileReaderException("Attemtped to look up IPv6 using IPv4 database file. Aborting.")
+        
+        v_literal = self.IP2Literal(ip)
+        position = 0
+
+        file_position  =  self.tree_start + self.BASE_TREE_BYTES
+        while True:
+            if len(v_literal) <= position:
+                raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 8)")
+            
+            next = 0
+            try:
+                if v_literal[position] == "0":
+                    pos = self.ReadAt(file_position, self.TREE_BYTE_WIDTH)
+                    if len(pos) == 4:
+                        next = unpack("<L", pos)[0]
+                else:
+                    pos = self.ReadAt(file_position + 4, self.TREE_BYTE_WIDTH)
+                    if len(pos) == 4:
+                        next = unpack("<L", pos)[0]
+
+            except Exception:
+                raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 9)")
+            
+            try:
+                if next == 0:
+                    if v_literal[position] == "0":
+                        pos = self.ReadAt(file_position + 4, self.TREE_BYTE_WIDTH)
+                        if len(pos) == 4:
+                            next = unpack("<L", pos)[0]
+                        
+                        if next == 0:
+                            raise IPNotFoundException()
+                    else:
+                        pos = self.ReadAt(file_position, self.TREE_BYTE_WIDTH)
+                        if len(pos) == 4:
+                            next = unpack("<L", pos)[0]
+
+                        if next == 0:
+                            raise IPNotFoundException()
+            except Exception:
+                raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 10)")
+            
+            file_position = next
+            if(file_position < self.tree_end):
+                position += 1
+                continue
+            try:
+                raw = self.ReadAt(file_position, self.record_bytes)
+            except Exception:
+                raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 11)")
+
+            try:
+                return self.CreateRecord(raw)
+            except Exception:
+                raise IPNotFoundException("Invalid or nonexistant IP address specified for lookup. (EID: 12)")
+    def GetColumns(self):
+        return self.columns
+
+    def Read(self, blenght):
+        try:
+            data = self.handler.read(blenght)
+        except IOError:
+            raise IPNotFoundException("Unknown file format. Please check the file's integrity. EID 13")
+        return data
+    
+    def ReadAt(self, position, b):
+        self.handler.seek(position)
+        d = self.handler.read(b)
+        if d == False:
+            raise IPNotFoundException("Unknown file format. Please check the file's integrity. EID 14")
+        return d
+
+    def SetupHeaders(self):
+        try:
+            headers ={}
+            headers["file_type"], headers["version"], headers["header_bytes1"], headers["header_bytes2"], headers["header_bytes3"], headers["record_bytes"], headers["file_bytes"] = unpack(self.HEADERS, self.Read(self.BASE_HEADER_BYTES))
+        except Exception:
+            raise FileReaderException("Invalid file format, unable to read first 11 bytes. EID 1")
+        if headers["file_type"] == None:
+            raise FileReaderException("Invalid file format, unable to read first 11 bytes. EID 1")
+        
+        file_type = BinaryOption.Create(headers["file_type"])
+        if file_type.Has(BinaryOption.IPV4MAP):
+            self.valid = True
+            self.ipv6 = False
+        
+        if file_type.Has(BinaryOption.IPV6MAP):
+            self.valid = True
+            self.ipv6 = True
+        
+        if file_type.Has(BinaryOption.BINARYDATA):
+            self.binary_data = True
+        
+        if(self.valid == False):
+            raise FileReaderException("Invalid file format, invalid first byte. EID 1.")
+        
+        if headers['version'] != DBReader.READER_VERSION:
+            raise FileReaderException("Invalid file version, EID 2.")
+        
+        
+        self.tree_start = Utilities.uVarInt([headers['header_bytes1'], headers['header_bytes2'], headers['header_bytes3']])
+        if self.tree_start == 0:
+            raise FileReaderException("Invalid file format, invalid header bytes. EID 2")
+        
+        if headers['record_bytes'] == 0:
+            raise FileReaderException("Invalid file format, invalid record size. EID 3")
+        
+        self.record_bytes = headers['record_bytes']
+
+        if headers['file_bytes'] == 0 :
+            raise FileReaderException("Invalid file format, invalid file size. EID 3")
+    
+    def SetupColumns(self):
+        length = self.tree_start - self.BASE_HEADER_BYTES
+        column_data = self.Read(length)
+        values = {}
+        for i in range(0, int(length/24)):
+            s = i * 24
+            values['name'], values['value'] = unpack_from("<{0}x23sB".format(i*24), column_data)
+            BO = BinaryOption.Create(values['value'])
+            C = Column.Create(values['name'].decode("utf-8").rstrip('\x00'), BO)
+            self.columns.append(C)
+        if len(self.columns) == 0:
+            raise FileReaderException("File does not appear to be valid, no column data found. EID: 5")
+    
+    def SetupTreeHeaders(self):
+        tree = {}
+        tree['header'], tree['tree_bytes'] = unpack("<BL", self.Read(5))
+
+        if tree['tree_bytes'] == 0 or BinaryOption.Create(tree['header']).Has(BinaryOption.TREEDATA) == False:
+            raise FileReaderException("File does not appear to be valid, bad binary tree. EID: 6")
+        
+        self.tree_end = tree['tree_bytes'] + self.tree_start
+    
+    def IP2Literal(self,ip):
+        result = ""
+        if(self.ipv6):
+            for block in self.Expand(ip).split(":"):
+                result += bin(int(block,16))[2:]
+        else:
+            for block in ip.split("."):
+                result += bin(int(block,10))[2:].zfill(8)
+
+        return result
+
+    def CreateRecord(self,raw):
+        bdata = {}
+        record = IPQSRecord()
+        current_byte = 0
+        bdata['one'], bdata["two"], bdata["three"] = unpack("{0}B".format(len(raw)), raw)[:3]
+        if self.binary_data:
+            record.ParseFirstByte(BinaryOption.Create(bdata['one']))
+            record.ParseSecondByte(BinaryOption.Create(bdata['two']))
+            
+            third = BinaryOption.Create(bdata['three'])
+            record.ConnectionTypeRaw(third)
+            record.AbuseVelocityRaw(third)
+            current_byte = 3
+        else:
+            first = BinaryOption.Create(bdata['one'])
+            record.ConnectionTypeRaw(first)
+            record.AbuseVelocityRaw(first)
+            current_byte = 1
+        
+        for column in self.columns:
+            value = ""
+            if column.Name() == "ASN":
+                value = unpack("<L", raw[current_byte:current_byte+4])[0]
+                record.ASN(int(value))
+                current_byte += 4
+            elif column.Name() == "Latitude":
+                value = unpack("<f", raw[current_byte:current_byte+4])[0]
+                record.Latitude(float(value))
+                current_byte += 4
+            elif column.Name() == "Longitude":
+                value = unpack("<f", raw[current_byte:current_byte+4])[0]
+                record.Longitude(float(value))
+                current_byte += 4
+            elif column.Name() == "ZeroFraudScore":
+                value = unpack("B", raw[current_byte:current_byte+1])[0]
+                record.SetFraudScore(0, int(value))
+                current_byte += 1
+            elif column.Name() == "OneFraudScore":
+                value = unpack("B", raw[current_byte:current_byte+1])[0]
+                record.ASN(float(value))
+                current_byte += 1
+            else:
+                if (column.Type()).Has(BinaryOption.STRINGDATA):
+                    value = self.GetRangedStringValue(unpack("<L", raw[current_byte:current_byte+4])[0])
+                    if type(value) == tuple:
+                        value = value[0].decode('utf-8')
+                    current_byte += 4
+                    try:
+                        m = getattr(record, column.Name())
+                        m(value)
+                    except AttributeError:
+                        pass
+            
+            record.AddColumns(Column.Create(column.Name(),column.Type(), value))
+        
+        return record
+    def GetRangedStringValue(self, position):
+        b = unpack("<B", self.ReadAt(position, 1))[0]
+        return unpack(("<{0}s".format(b)), self.Read(b))
+
+    def Expand(self, ip):
+            l_hex = list(hexlify(inet_pton(AF_INET6, ip)).decode("utf-8"))
+            return''.join(l + ':' * (n % 4 == 3) for n, l in enumerate(l_hex))[:-1]
+
+    def IsIPv4(self, ip:str):
+        try:
+            inet_pton(AF_INET, ip)
+            return True
+        except error:
+            return False
+
+    def IsIPv6(self, ip:str):
+        try:
+            inet_pton(AF_INET6, ip)
+            return True
+        except error:
+            return False
+    def __del__(self):
+        self.handler.close()

IPQualityScore/Exceptions.py

@@ -0,0 +1,5 @@
+class FileReaderException(Exception):
+    pass
+    
+class IPNotFoundException(Exception):
+    pass