From 831d77cd84de12072e8760e8a239c0937cbd7fe0 Mon Sep 17 00:00:00 2001 From: IP2Location Date: Tue, 6 Jul 2021 09:23:43 +0800 Subject: [PATCH] Added provider field and exception handling for incorrect BIN database. --- LICENSE.TXT | 2 +- README.md | 7 +- dub.json | 2 +- source/app.d | 6 +- source/ip2proxy-d/ip2proxy.d | 132 ++++++++++++++++------------------- 5 files changed, 72 insertions(+), 77 deletions(-) diff --git a/LICENSE.TXT b/LICENSE.TXT index 1f446b9..a54dedb 100644 --- a/LICENSE.TXT +++ b/LICENSE.TXT @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 IP2Location.com +Copyright (c) 2021 IP2Location.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 8c2ac9e..34cd715 100644 --- a/README.md +++ b/README.md @@ -40,15 +40,16 @@ Below are the methods supported in this library. |get_as|Return the autonomous system name of the proxy.| |get_last_seen|Return the number of days that the proxy was last seen.| |get_threat|Return the threat type of the proxy.| +|get_provider|Return the provider of the proxy.| ## Usage ```d import std.stdio; -import ip2proxy; +import ip2proxy : ip2proxy; int main() { - string db = "./IP2PROXY-IP-PROXYTYPE-COUNTRY-REGION-CITY-ISP-DOMAIN-USAGETYPE-ASN-LASTSEEN-THREAT-RESIDENTIAL.BIN"; + string db = "./IP2PROXY-IP-PROXYTYPE-COUNTRY-REGION-CITY-ISP-DOMAIN-USAGETYPE-ASN-LASTSEEN-THREAT-RESIDENTIAL-PROVIDER.BIN"; auto prox = new ip2proxy(); if (prox.open(db) == 0) { @@ -72,6 +73,7 @@ int main() { writeln("AS: ", prox.get_as(ip)); writeln("LastSeen: ", prox.get_last_seen(ip)); writeln("Threat: ", prox.get_threat(ip)); + writeln("Provider: ", prox.get_provider(ip)); // function for all fields auto all = prox.get_all(ip); @@ -88,6 +90,7 @@ int main() { writeln("AS: ", all["AS"]); writeln("LastSeen: ", all["LastSeen"]); writeln("Threat: ", all["Threat"]); + writeln("Provider: ", all["Provider"]); } else { writeln("Error reading BIN file."); diff --git a/dub.json b/dub.json index 777bb2d..2649604 100644 --- a/dub.json +++ b/dub.json @@ -5,7 +5,7 @@ ], "description": "IP2Proxy D Library", "homepage": "https://www.ip2location.com/development-libraries/ip2proxy/d", - "copyright": "Copyright © 2020, IP2Location", + "copyright": "Copyright © 2021, IP2Location", "license": "MIT", "targetType": "library" } \ No newline at end of file diff --git a/source/app.d b/source/app.d index 85d7773..82c823f 100644 --- a/source/app.d +++ b/source/app.d @@ -1,8 +1,8 @@ import std.stdio; -import ip2proxy; +import ip2proxy : ip2proxy; int main() { - string db = "./IP2PROXY-IP-PROXYTYPE-COUNTRY-REGION-CITY-ISP-DOMAIN-USAGETYPE-ASN-LASTSEEN-THREAT-RESIDENTIAL.BIN"; + string db = "./IP2PROXY-IP-PROXYTYPE-COUNTRY-REGION-CITY-ISP-DOMAIN-USAGETYPE-ASN-LASTSEEN-THREAT-RESIDENTIAL-PROVIDER.BIN"; auto prox = new ip2proxy(); if (prox.open(db) == 0) { @@ -26,6 +26,7 @@ int main() { writeln("AS: ", prox.get_as(ip)); writeln("LastSeen: ", prox.get_last_seen(ip)); writeln("Threat: ", prox.get_threat(ip)); + writeln("Provider: ", prox.get_provider(ip)); // function for all fields auto all = prox.get_all(ip); @@ -42,6 +43,7 @@ int main() { writeln("AS: ", all["AS"]); writeln("LastSeen: ", all["LastSeen"]); writeln("Threat: ", all["Threat"]); + writeln("Provider: ", all["Provider"]); } else { writeln("Error reading BIN file."); diff --git a/source/ip2proxy-d/ip2proxy.d b/source/ip2proxy-d/ip2proxy.d index 85b466d..8b35063 100644 --- a/source/ip2proxy-d/ip2proxy.d +++ b/source/ip2proxy-d/ip2proxy.d @@ -21,6 +21,9 @@ protected struct ip2proxymeta { uint ipv6indexbaseaddr; uint ipv4columnsize; uint ipv6columnsize; + ubyte productcode; + ubyte producttype; + uint filesize; } protected struct ip2proxyrecord { @@ -36,6 +39,7 @@ protected struct ip2proxyrecord { string as = "-"; string last_seen = "-"; string threat = "-"; + string provider = "-"; byte is_proxy = -1; } @@ -45,19 +49,20 @@ protected struct ipv { uint ipindex = 0; } -const ubyte[11] COUNTRY_POSITION = [0, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3]; -const ubyte[11] REGION_POSITION = [0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4]; -const ubyte[11] CITY_POSITION = [0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5]; -const ubyte[11] ISP_POSITION = [0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6]; -const ubyte[11] PROXYTYPE_POSITION = [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2]; -const ubyte[11] DOMAIN_POSITION = [0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7]; -const ubyte[11] USAGETYPE_POSITION = [0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8]; -const ubyte[11] ASN_POSITION = [0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9]; -const ubyte[11] AS_POSITION = [0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 10]; -const ubyte[11] LASTSEEN_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 11]; -const ubyte[11] THREAT_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12]; +const ubyte[12] COUNTRY_POSITION = [0, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]; +const ubyte[12] REGION_POSITION = [0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4]; +const ubyte[12] CITY_POSITION = [0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5]; +const ubyte[12] ISP_POSITION = [0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6]; +const ubyte[12] PROXYTYPE_POSITION = [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]; +const ubyte[12] DOMAIN_POSITION = [0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7]; +const ubyte[12] USAGETYPE_POSITION = [0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8]; +const ubyte[12] ASN_POSITION = [0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9]; +const ubyte[12] AS_POSITION = [0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 10, 10]; +const ubyte[12] LASTSEEN_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 11, 11]; +const ubyte[12] THREAT_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 12]; +const ubyte[12] PROVIDER_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13]; -protected const string MODULE_VERSION = "3.0.0"; +protected const string MODULE_VERSION = "3.1.0"; protected const BigInt MAX_IPV4_RANGE = BigInt("4294967295"); protected const BigInt MAX_IPV6_RANGE = BigInt("340282366920938463463374607431768211455"); @@ -80,13 +85,15 @@ protected const uint ASN = 0X00200; protected const uint AS = 0X00400; protected const uint LASTSEEN = 0X00800; protected const uint THREAT = 0X01000; +protected const uint PROVIDER = 0X02000; -protected const uint ALL = COUNTRYSHORT | COUNTRYLONG | REGION | CITY | ISP | PROXYTYPE | ISPROXY | DOMAIN | USAGETYPE | ASN | AS | LASTSEEN | THREAT; +protected const uint ALL = COUNTRYSHORT | COUNTRYLONG | REGION | CITY | ISP | PROXYTYPE | ISPROXY | DOMAIN | USAGETYPE | ASN | AS | LASTSEEN | THREAT | PROVIDER; protected const string MSG_NOT_SUPPORTED = "NOT SUPPORTED"; protected const string MSG_INVALID_IP = "INVALID IP ADDRESS"; protected const string MSG_MISSING_FILE = "MISSING FILE"; protected const string MSG_IPV6_UNSUPPORTED = "IPV6 ADDRESS MISSING IN IPV4 BIN"; +protected const string MSG_INVALID_BIN = "Incorrect IP2Proxy BIN file format. Please make sure that you are using the latest IP2Proxy BIN file."; class ip2proxy { protected MmFile db; @@ -105,6 +112,7 @@ class ip2proxy { private uint as_position_offset; private uint lastseen_position_offset; private uint threat_position_offset; + private uint provider_position_offset; private bool country_enabled; private bool region_enabled; @@ -117,6 +125,7 @@ class ip2proxy { private bool as_enabled; private bool lastseen_enabled; private bool threat_enabled; + private bool provider_enabled; // constructor public this() { @@ -159,6 +168,9 @@ class ip2proxy { meta.ipv6indexbaseaddr = 0; meta.ipv4columnsize = 0; meta.ipv6columnsize = 0; + meta.productcode = 0; + meta.producttype = 0; + meta.filesize = 0; metaok = false; country_position_offset = 0; region_position_offset = 0; @@ -171,6 +183,7 @@ class ip2proxy { as_position_offset = 0; lastseen_position_offset = 0; threat_position_offset = 0; + provider_position_offset = 0; country_enabled = false; region_enabled = false; city_enabled = false; @@ -182,6 +195,7 @@ class ip2proxy { as_enabled = false; lastseen_enabled = false; threat_enabled = false; + provider_enabled = false; destroy(db); @@ -228,27 +242,6 @@ class ip2proxy { return result; } - // read float from row - private float readfloat_row(ref ubyte[] row, uint index) { - ubyte[4] buf = row[index .. (index + 4)]; - float result = 0.0; - result = littleEndianToNative!float(buf); - return result; - } - - // read float - private float readfloat(uint index) { - uint pos = index - 1; - ubyte[4] fl; - float result = 0.0; - for (int x = 0; x < 4; x++) { - fl[x] = cast(ubyte)db[pos + x]; - } - - result = littleEndianToNative!float(fl); - return result; - } - // read BIN file metadata private byte readmeta() { if (binfile.length == 0) { @@ -273,25 +266,21 @@ class ip2proxy { meta.ipv6databaseaddr = readuint(18); meta.ipv4indexbaseaddr = readuint(22); meta.ipv6indexbaseaddr = readuint(26); + meta.productcode = db[29]; + // below 2 fields just read for now, not being used yet + meta.producttype = db[30]; + meta.filesize = readuint(32); + + // check if is correct BIN (should be 2 for IP2Proxy BIN file), also checking for zipped file (PK being the first 2 chars) + if ((meta.productcode != 2 && meta.databaseyear >= 21) || (meta.databasetype == 80 && meta.databasecolumn == 75)) { // only BINs from Jan 2021 onwards have this byte set + throw new Exception(MSG_INVALID_BIN); + } + meta.ipv4columnsize = meta.databasecolumn << 2; // 4 bytes each column meta.ipv6columnsize = 16 + ((meta.databasecolumn - 1) << 2); // 4 bytes each column, except IPFrom column which is 16 bytes uint dbt = meta.databasetype; - // since both IPv4 and IPv6 use 4 bytes for the below columns, can just do it once here - // country_position_offset = (COUNTRY_POSITION[dbt] != 0) ? (COUNTRY_POSITION[dbt] - 1) << 2 : 0; - // region_position_offset = (REGION_POSITION[dbt] != 0) ? (REGION_POSITION[dbt] - 1) << 2 : 0; - // city_position_offset = (CITY_POSITION[dbt] != 0) ? (CITY_POSITION[dbt] - 1) << 2 : 0; - // isp_position_offset = (ISP_POSITION[dbt] != 0) ? (ISP_POSITION[dbt] - 1) << 2 : 0; - // proxytype_position_offset = (PROXYTYPE_POSITION[dbt] != 0) ? (PROXYTYPE_POSITION[dbt] - 1) << 2 : 0; - // domain_position_offset = (DOMAIN_POSITION[dbt] != 0) ? (DOMAIN_POSITION[dbt] - 1) << 2 : 0; - // usagetype_position_offset = (USAGETYPE_POSITION[dbt] != 0) ? (USAGETYPE_POSITION[dbt] - 1) << 2 : 0; - // asn_position_offset = (ASN_POSITION[dbt] != 0) ? (ASN_POSITION[dbt] - 1) << 2 : 0; - // as_position_offset = (AS_POSITION[dbt] != 0) ? (AS_POSITION[dbt] - 1) << 2 : 0; - // lastseen_position_offset = (LASTSEEN_POSITION[dbt] != 0) ? (LASTSEEN_POSITION[dbt] - 1) << 2 : 0; - // threat_position_offset = (THREAT_POSITION[dbt] != 0) ? (THREAT_POSITION[dbt] - 1) << 2 : 0; - - // offset slightly different when reading by row country_position_offset = (COUNTRY_POSITION[dbt] != 0) ? (COUNTRY_POSITION[dbt] - 2) << 2 : 0; region_position_offset = (REGION_POSITION[dbt] != 0) ? (REGION_POSITION[dbt] - 2) << 2 : 0; city_position_offset = (CITY_POSITION[dbt] != 0) ? (CITY_POSITION[dbt] - 2) << 2 : 0; @@ -303,18 +292,20 @@ class ip2proxy { as_position_offset = (AS_POSITION[dbt] != 0) ? (AS_POSITION[dbt] - 2) << 2 : 0; lastseen_position_offset = (LASTSEEN_POSITION[dbt] != 0) ? (LASTSEEN_POSITION[dbt] - 2) << 2 : 0; threat_position_offset = (THREAT_POSITION[dbt] != 0) ? (THREAT_POSITION[dbt] - 2) << 2 : 0; + provider_position_offset = (PROVIDER_POSITION[dbt] != 0) ? (PROVIDER_POSITION[dbt] - 2) << 2 : 0; - country_enabled = (COUNTRY_POSITION[dbt] != 0) ? true : false; - region_enabled = (REGION_POSITION[dbt] != 0) ? true : false; - city_enabled = (CITY_POSITION[dbt] != 0) ? true : false; - isp_enabled = (ISP_POSITION[dbt] != 0) ? true : false; - proxytype_enabled = (PROXYTYPE_POSITION[dbt] != 0) ? true : false; - domain_enabled = (DOMAIN_POSITION[dbt] != 0) ? true : false; - usagetype_enabled = (USAGETYPE_POSITION[dbt] != 0) ? true : false; - asn_enabled = (ASN_POSITION[dbt] != 0) ? true : false; - as_enabled = (AS_POSITION[dbt] != 0) ? true : false; - lastseen_enabled = (LASTSEEN_POSITION[dbt] != 0) ? true : false; - threat_enabled = (THREAT_POSITION[dbt] != 0) ? true : false; + country_enabled = (COUNTRY_POSITION[dbt] != 0); + region_enabled = (REGION_POSITION[dbt] != 0); + city_enabled = (CITY_POSITION[dbt] != 0); + isp_enabled = (ISP_POSITION[dbt] != 0); + proxytype_enabled = (PROXYTYPE_POSITION[dbt] != 0); + domain_enabled = (DOMAIN_POSITION[dbt] != 0); + usagetype_enabled = (USAGETYPE_POSITION[dbt] != 0); + asn_enabled = (ASN_POSITION[dbt] != 0); + as_enabled = (AS_POSITION[dbt] != 0); + lastseen_enabled = (LASTSEEN_POSITION[dbt] != 0); + threat_enabled = (THREAT_POSITION[dbt] != 0); + provider_enabled = (PROVIDER_POSITION[dbt] != 0); metaok = true; } @@ -436,6 +427,7 @@ class ip2proxy { x["AS"] = data.as; x["LastSeen"] = data.last_seen; x["Threat"] = data.threat; + x["Provider"] = data.provider; return x; } @@ -512,6 +504,12 @@ class ip2proxy { return data.threat; } + // get provider + public auto get_provider(const string ipaddress) { + auto data = query(ipaddress, PROVIDER); + return data.provider; + } + // is proxy public auto is_proxy(const string ipaddress) { auto data = query(ipaddress, ISPROXY); @@ -594,20 +592,17 @@ class ip2proxy { uint firstcol = 4; // 4 bytes for ip from if (ipdata.iptype == 6) { firstcol = 16; // 16 bytes for ipv6 - // rowoffset = rowoffset + 12; // coz below is assuming all columns are 4 bytes, so got 12 left to go to make 16 bytes total } ubyte[] row = cast(ubyte[])db[(rowoffset + firstcol - 1) .. (rowoffset + colsize - 1)]; if (proxytype_enabled) { if ((mode & PROXYTYPE) || (mode & ISPROXY)) { - // x.proxy_type = readstr(readuint(rowoffset + proxytype_position_offset)); x.proxy_type = readstr(readuint_row(row, proxytype_position_offset)); } } if (country_enabled) { if ((mode & COUNTRYSHORT) || (mode & COUNTRYLONG) || (mode & ISPROXY)) { - // countrypos = readuint(rowoffset + country_position_offset); countrypos = readuint_row(row, country_position_offset); } if ((mode & COUNTRYSHORT) || (mode & ISPROXY)) { @@ -619,50 +614,45 @@ class ip2proxy { } if ((mode & REGION) && (region_enabled)) { - // x.region = readstr(readuint(rowoffset + region_position_offset)); x.region = readstr(readuint_row(row, region_position_offset)); } if ((mode & CITY) && (city_enabled)) { - // x.city = readstr(readuint(rowoffset + city_position_offset)); x.city = readstr(readuint_row(row, city_position_offset)); } if ((mode & ISP) && (isp_enabled)) { - // x.isp = readstr(readuint(rowoffset + isp_position_offset)); x.isp = readstr(readuint_row(row, isp_position_offset)); } if ((mode & DOMAIN) && (domain_enabled)) { - // x.domain = readstr(readuint(rowoffset + domain_position_offset)); x.domain = readstr(readuint_row(row, domain_position_offset)); } if ((mode & USAGETYPE) && (usagetype_enabled)) { - // x.usage_type = readstr(readuint(rowoffset + usagetype_position_offset)); x.usage_type = readstr(readuint_row(row, usagetype_position_offset)); } if ((mode & ASN) && (asn_enabled)) { - // x.asn = readstr(readuint(rowoffset + asn_position_offset)); x.asn = readstr(readuint_row(row, asn_position_offset)); } if ((mode & AS) && (as_enabled)) { - // x.as = readstr(readuint(rowoffset + as_position_offset)); x.as = readstr(readuint_row(row, as_position_offset)); } if ((mode & LASTSEEN) && (lastseen_enabled)) { - // x.last_seen = readstr(readuint(rowoffset + lastseen_position_offset)); x.last_seen = readstr(readuint_row(row, lastseen_position_offset)); } if ((mode & THREAT) && (threat_enabled)) { - // x.threat = readstr(readuint(rowoffset + threat_position_offset)); x.threat = readstr(readuint_row(row, threat_position_offset)); } + if ((mode & PROVIDER) && (provider_enabled)) { + x.provider = readstr(readuint_row(row, provider_position_offset)); + } + if ((x.country_short == "-") || (x.proxy_type == "-")) { x.is_proxy = 0; }