Skip to content

Commit

Permalink
Added provider field and exception handling for incorrect BIN database.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ip2location
ip2location committed Jul 6, 2021
1 parent e11d1bb commit 831d77c
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 77 deletions.
2 changes: 1 addition & 1 deletion LICENSE.TXT
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
MIT License

Copyright (c) 2020 IP2Location.com
Copyright (c) 2021 IP2Location.com

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
Expand Down
7 changes: 5 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,16 @@ Below are the methods supported in this library.
|get_as|Return the autonomous system name of the proxy.|
|get_last_seen|Return the number of days that the proxy was last seen.|
|get_threat|Return the threat type of the proxy.|
|get_provider|Return the provider of the proxy.|

## Usage

```d
import std.stdio;
import ip2proxy;
import ip2proxy : ip2proxy;
int main() {
string db = "./IP2PROXY-IP-PROXYTYPE-COUNTRY-REGION-CITY-ISP-DOMAIN-USAGETYPE-ASN-LASTSEEN-THREAT-RESIDENTIAL.BIN";
string db = "./IP2PROXY-IP-PROXYTYPE-COUNTRY-REGION-CITY-ISP-DOMAIN-USAGETYPE-ASN-LASTSEEN-THREAT-RESIDENTIAL-PROVIDER.BIN";
auto prox = new ip2proxy();
if (prox.open(db) == 0) {
Expand All @@ -72,6 +73,7 @@ int main() {
writeln("AS: ", prox.get_as(ip));
writeln("LastSeen: ", prox.get_last_seen(ip));
writeln("Threat: ", prox.get_threat(ip));
writeln("Provider: ", prox.get_provider(ip));
// function for all fields
auto all = prox.get_all(ip);
Expand All @@ -88,6 +90,7 @@ int main() {
writeln("AS: ", all["AS"]);
writeln("LastSeen: ", all["LastSeen"]);
writeln("Threat: ", all["Threat"]);
writeln("Provider: ", all["Provider"]);
}
else {
writeln("Error reading BIN file.");
Expand Down
2 changes: 1 addition & 1 deletion dub.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
],
"description": "IP2Proxy D Library",
"homepage": "https://www.ip2location.com/development-libraries/ip2proxy/d",
"copyright": "Copyright © 2020, IP2Location",
"copyright": "Copyright © 2021, IP2Location",
"license": "MIT",
"targetType": "library"
}
6 changes: 4 additions & 2 deletions source/app.d
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import std.stdio;
import ip2proxy;
import ip2proxy : ip2proxy;

int main() {
string db = "./IP2PROXY-IP-PROXYTYPE-COUNTRY-REGION-CITY-ISP-DOMAIN-USAGETYPE-ASN-LASTSEEN-THREAT-RESIDENTIAL.BIN";
string db = "./IP2PROXY-IP-PROXYTYPE-COUNTRY-REGION-CITY-ISP-DOMAIN-USAGETYPE-ASN-LASTSEEN-THREAT-RESIDENTIAL-PROVIDER.BIN";
auto prox = new ip2proxy();

if (prox.open(db) == 0) {
Expand All @@ -26,6 +26,7 @@ int main() {
writeln("AS: ", prox.get_as(ip));
writeln("LastSeen: ", prox.get_last_seen(ip));
writeln("Threat: ", prox.get_threat(ip));
writeln("Provider: ", prox.get_provider(ip));

// function for all fields
auto all = prox.get_all(ip);
Expand All @@ -42,6 +43,7 @@ int main() {
writeln("AS: ", all["AS"]);
writeln("LastSeen: ", all["LastSeen"]);
writeln("Threat: ", all["Threat"]);
writeln("Provider: ", all["Provider"]);
}
else {
writeln("Error reading BIN file.");
Expand Down
132 changes: 61 additions & 71 deletions source/ip2proxy-d/ip2proxy.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ protected struct ip2proxymeta {
uint ipv6indexbaseaddr;
uint ipv4columnsize;
uint ipv6columnsize;
ubyte productcode;
ubyte producttype;
uint filesize;
}

protected struct ip2proxyrecord {
Expand All @@ -36,6 +39,7 @@ protected struct ip2proxyrecord {
string as = "-";
string last_seen = "-";
string threat = "-";
string provider = "-";
byte is_proxy = -1;
}

Expand All @@ -45,19 +49,20 @@ protected struct ipv {
uint ipindex = 0;
}

const ubyte[11] COUNTRY_POSITION = [0, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3];
const ubyte[11] REGION_POSITION = [0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4];
const ubyte[11] CITY_POSITION = [0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5];
const ubyte[11] ISP_POSITION = [0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6];
const ubyte[11] PROXYTYPE_POSITION = [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2];
const ubyte[11] DOMAIN_POSITION = [0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7];
const ubyte[11] USAGETYPE_POSITION = [0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8];
const ubyte[11] ASN_POSITION = [0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9];
const ubyte[11] AS_POSITION = [0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 10];
const ubyte[11] LASTSEEN_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 11];
const ubyte[11] THREAT_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12];
const ubyte[12] COUNTRY_POSITION = [0, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3];
const ubyte[12] REGION_POSITION = [0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4];
const ubyte[12] CITY_POSITION = [0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5];
const ubyte[12] ISP_POSITION = [0, 0, 0, 0, 6, 6, 6, 6, 6, 6, 6, 6];
const ubyte[12] PROXYTYPE_POSITION = [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2];
const ubyte[12] DOMAIN_POSITION = [0, 0, 0, 0, 0, 7, 7, 7, 7, 7, 7, 7];
const ubyte[12] USAGETYPE_POSITION = [0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8];
const ubyte[12] ASN_POSITION = [0, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9];
const ubyte[12] AS_POSITION = [0, 0, 0, 0, 0, 0, 0, 10, 10, 10, 10, 10];
const ubyte[12] LASTSEEN_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 11, 11, 11, 11];
const ubyte[12] THREAT_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 12];
const ubyte[12] PROVIDER_POSITION = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13];

protected const string MODULE_VERSION = "3.0.0";
protected const string MODULE_VERSION = "3.1.0";

protected const BigInt MAX_IPV4_RANGE = BigInt("4294967295");
protected const BigInt MAX_IPV6_RANGE = BigInt("340282366920938463463374607431768211455");
Expand All @@ -80,13 +85,15 @@ protected const uint ASN = 0X00200;
protected const uint AS = 0X00400;
protected const uint LASTSEEN = 0X00800;
protected const uint THREAT = 0X01000;
protected const uint PROVIDER = 0X02000;

protected const uint ALL = COUNTRYSHORT | COUNTRYLONG | REGION | CITY | ISP | PROXYTYPE | ISPROXY | DOMAIN | USAGETYPE | ASN | AS | LASTSEEN | THREAT;
protected const uint ALL = COUNTRYSHORT | COUNTRYLONG | REGION | CITY | ISP | PROXYTYPE | ISPROXY | DOMAIN | USAGETYPE | ASN | AS | LASTSEEN | THREAT | PROVIDER;

protected const string MSG_NOT_SUPPORTED = "NOT SUPPORTED";
protected const string MSG_INVALID_IP = "INVALID IP ADDRESS";
protected const string MSG_MISSING_FILE = "MISSING FILE";
protected const string MSG_IPV6_UNSUPPORTED = "IPV6 ADDRESS MISSING IN IPV4 BIN";
protected const string MSG_INVALID_BIN = "Incorrect IP2Proxy BIN file format. Please make sure that you are using the latest IP2Proxy BIN file.";

class ip2proxy {
protected MmFile db;
Expand All @@ -105,6 +112,7 @@ class ip2proxy {
private uint as_position_offset;
private uint lastseen_position_offset;
private uint threat_position_offset;
private uint provider_position_offset;

private bool country_enabled;
private bool region_enabled;
Expand All @@ -117,6 +125,7 @@ class ip2proxy {
private bool as_enabled;
private bool lastseen_enabled;
private bool threat_enabled;
private bool provider_enabled;

// constructor
public this() {
Expand Down Expand Up @@ -159,6 +168,9 @@ class ip2proxy {
meta.ipv6indexbaseaddr = 0;
meta.ipv4columnsize = 0;
meta.ipv6columnsize = 0;
meta.productcode = 0;
meta.producttype = 0;
meta.filesize = 0;
metaok = false;
country_position_offset = 0;
region_position_offset = 0;
Expand All @@ -171,6 +183,7 @@ class ip2proxy {
as_position_offset = 0;
lastseen_position_offset = 0;
threat_position_offset = 0;
provider_position_offset = 0;
country_enabled = false;
region_enabled = false;
city_enabled = false;
Expand All @@ -182,6 +195,7 @@ class ip2proxy {
as_enabled = false;
lastseen_enabled = false;
threat_enabled = false;
provider_enabled = false;

destroy(db);

Expand Down Expand Up @@ -228,27 +242,6 @@ class ip2proxy {
return result;
}

// read float from row
private float readfloat_row(ref ubyte[] row, uint index) {
ubyte[4] buf = row[index .. (index + 4)];
float result = 0.0;
result = littleEndianToNative!float(buf);
return result;
}

// read float
private float readfloat(uint index) {
uint pos = index - 1;
ubyte[4] fl;
float result = 0.0;
for (int x = 0; x < 4; x++) {
fl[x] = cast(ubyte)db[pos + x];
}

result = littleEndianToNative!float(fl);
return result;
}

// read BIN file metadata
private byte readmeta() {
if (binfile.length == 0) {
Expand All @@ -273,25 +266,21 @@ class ip2proxy {
meta.ipv6databaseaddr = readuint(18);
meta.ipv4indexbaseaddr = readuint(22);
meta.ipv6indexbaseaddr = readuint(26);
meta.productcode = db[29];
// below 2 fields just read for now, not being used yet
meta.producttype = db[30];
meta.filesize = readuint(32);

// check if is correct BIN (should be 2 for IP2Proxy BIN file), also checking for zipped file (PK being the first 2 chars)
if ((meta.productcode != 2 && meta.databaseyear >= 21) || (meta.databasetype == 80 && meta.databasecolumn == 75)) { // only BINs from Jan 2021 onwards have this byte set
throw new Exception(MSG_INVALID_BIN);
}

meta.ipv4columnsize = meta.databasecolumn << 2; // 4 bytes each column
meta.ipv6columnsize = 16 + ((meta.databasecolumn - 1) << 2); // 4 bytes each column, except IPFrom column which is 16 bytes

uint dbt = meta.databasetype;

// since both IPv4 and IPv6 use 4 bytes for the below columns, can just do it once here
// country_position_offset = (COUNTRY_POSITION[dbt] != 0) ? (COUNTRY_POSITION[dbt] - 1) << 2 : 0;
// region_position_offset = (REGION_POSITION[dbt] != 0) ? (REGION_POSITION[dbt] - 1) << 2 : 0;
// city_position_offset = (CITY_POSITION[dbt] != 0) ? (CITY_POSITION[dbt] - 1) << 2 : 0;
// isp_position_offset = (ISP_POSITION[dbt] != 0) ? (ISP_POSITION[dbt] - 1) << 2 : 0;
// proxytype_position_offset = (PROXYTYPE_POSITION[dbt] != 0) ? (PROXYTYPE_POSITION[dbt] - 1) << 2 : 0;
// domain_position_offset = (DOMAIN_POSITION[dbt] != 0) ? (DOMAIN_POSITION[dbt] - 1) << 2 : 0;
// usagetype_position_offset = (USAGETYPE_POSITION[dbt] != 0) ? (USAGETYPE_POSITION[dbt] - 1) << 2 : 0;
// asn_position_offset = (ASN_POSITION[dbt] != 0) ? (ASN_POSITION[dbt] - 1) << 2 : 0;
// as_position_offset = (AS_POSITION[dbt] != 0) ? (AS_POSITION[dbt] - 1) << 2 : 0;
// lastseen_position_offset = (LASTSEEN_POSITION[dbt] != 0) ? (LASTSEEN_POSITION[dbt] - 1) << 2 : 0;
// threat_position_offset = (THREAT_POSITION[dbt] != 0) ? (THREAT_POSITION[dbt] - 1) << 2 : 0;

// offset slightly different when reading by row
country_position_offset = (COUNTRY_POSITION[dbt] != 0) ? (COUNTRY_POSITION[dbt] - 2) << 2 : 0;
region_position_offset = (REGION_POSITION[dbt] != 0) ? (REGION_POSITION[dbt] - 2) << 2 : 0;
city_position_offset = (CITY_POSITION[dbt] != 0) ? (CITY_POSITION[dbt] - 2) << 2 : 0;
Expand All @@ -303,18 +292,20 @@ class ip2proxy {
as_position_offset = (AS_POSITION[dbt] != 0) ? (AS_POSITION[dbt] - 2) << 2 : 0;
lastseen_position_offset = (LASTSEEN_POSITION[dbt] != 0) ? (LASTSEEN_POSITION[dbt] - 2) << 2 : 0;
threat_position_offset = (THREAT_POSITION[dbt] != 0) ? (THREAT_POSITION[dbt] - 2) << 2 : 0;
provider_position_offset = (PROVIDER_POSITION[dbt] != 0) ? (PROVIDER_POSITION[dbt] - 2) << 2 : 0;

country_enabled = (COUNTRY_POSITION[dbt] != 0) ? true : false;
region_enabled = (REGION_POSITION[dbt] != 0) ? true : false;
city_enabled = (CITY_POSITION[dbt] != 0) ? true : false;
isp_enabled = (ISP_POSITION[dbt] != 0) ? true : false;
proxytype_enabled = (PROXYTYPE_POSITION[dbt] != 0) ? true : false;
domain_enabled = (DOMAIN_POSITION[dbt] != 0) ? true : false;
usagetype_enabled = (USAGETYPE_POSITION[dbt] != 0) ? true : false;
asn_enabled = (ASN_POSITION[dbt] != 0) ? true : false;
as_enabled = (AS_POSITION[dbt] != 0) ? true : false;
lastseen_enabled = (LASTSEEN_POSITION[dbt] != 0) ? true : false;
threat_enabled = (THREAT_POSITION[dbt] != 0) ? true : false;
country_enabled = (COUNTRY_POSITION[dbt] != 0);
region_enabled = (REGION_POSITION[dbt] != 0);
city_enabled = (CITY_POSITION[dbt] != 0);
isp_enabled = (ISP_POSITION[dbt] != 0);
proxytype_enabled = (PROXYTYPE_POSITION[dbt] != 0);
domain_enabled = (DOMAIN_POSITION[dbt] != 0);
usagetype_enabled = (USAGETYPE_POSITION[dbt] != 0);
asn_enabled = (ASN_POSITION[dbt] != 0);
as_enabled = (AS_POSITION[dbt] != 0);
lastseen_enabled = (LASTSEEN_POSITION[dbt] != 0);
threat_enabled = (THREAT_POSITION[dbt] != 0);
provider_enabled = (PROVIDER_POSITION[dbt] != 0);

metaok = true;
}
Expand Down Expand Up @@ -436,6 +427,7 @@ class ip2proxy {
x["AS"] = data.as;
x["LastSeen"] = data.last_seen;
x["Threat"] = data.threat;
x["Provider"] = data.provider;

return x;
}
Expand Down Expand Up @@ -512,6 +504,12 @@ class ip2proxy {
return data.threat;
}

// get provider
public auto get_provider(const string ipaddress) {
auto data = query(ipaddress, PROVIDER);
return data.provider;
}

// is proxy
public auto is_proxy(const string ipaddress) {
auto data = query(ipaddress, ISPROXY);
Expand Down Expand Up @@ -594,20 +592,17 @@ class ip2proxy {
uint firstcol = 4; // 4 bytes for ip from
if (ipdata.iptype == 6) {
firstcol = 16; // 16 bytes for ipv6
// rowoffset = rowoffset + 12; // coz below is assuming all columns are 4 bytes, so got 12 left to go to make 16 bytes total
}
ubyte[] row = cast(ubyte[])db[(rowoffset + firstcol - 1) .. (rowoffset + colsize - 1)];

if (proxytype_enabled) {
if ((mode & PROXYTYPE) || (mode & ISPROXY)) {
// x.proxy_type = readstr(readuint(rowoffset + proxytype_position_offset));
x.proxy_type = readstr(readuint_row(row, proxytype_position_offset));
}
}

if (country_enabled) {
if ((mode & COUNTRYSHORT) || (mode & COUNTRYLONG) || (mode & ISPROXY)) {
// countrypos = readuint(rowoffset + country_position_offset);
countrypos = readuint_row(row, country_position_offset);
}
if ((mode & COUNTRYSHORT) || (mode & ISPROXY)) {
Expand All @@ -619,50 +614,45 @@ class ip2proxy {
}

if ((mode & REGION) && (region_enabled)) {
// x.region = readstr(readuint(rowoffset + region_position_offset));
x.region = readstr(readuint_row(row, region_position_offset));
}

if ((mode & CITY) && (city_enabled)) {
// x.city = readstr(readuint(rowoffset + city_position_offset));
x.city = readstr(readuint_row(row, city_position_offset));
}

if ((mode & ISP) && (isp_enabled)) {
// x.isp = readstr(readuint(rowoffset + isp_position_offset));
x.isp = readstr(readuint_row(row, isp_position_offset));
}

if ((mode & DOMAIN) && (domain_enabled)) {
// x.domain = readstr(readuint(rowoffset + domain_position_offset));
x.domain = readstr(readuint_row(row, domain_position_offset));
}

if ((mode & USAGETYPE) && (usagetype_enabled)) {
// x.usage_type = readstr(readuint(rowoffset + usagetype_position_offset));
x.usage_type = readstr(readuint_row(row, usagetype_position_offset));
}

if ((mode & ASN) && (asn_enabled)) {
// x.asn = readstr(readuint(rowoffset + asn_position_offset));
x.asn = readstr(readuint_row(row, asn_position_offset));
}

if ((mode & AS) && (as_enabled)) {
// x.as = readstr(readuint(rowoffset + as_position_offset));
x.as = readstr(readuint_row(row, as_position_offset));
}

if ((mode & LASTSEEN) && (lastseen_enabled)) {
// x.last_seen = readstr(readuint(rowoffset + lastseen_position_offset));
x.last_seen = readstr(readuint_row(row, lastseen_position_offset));
}

if ((mode & THREAT) && (threat_enabled)) {
// x.threat = readstr(readuint(rowoffset + threat_position_offset));
x.threat = readstr(readuint_row(row, threat_position_offset));
}

if ((mode & PROVIDER) && (provider_enabled)) {
x.provider = readstr(readuint_row(row, provider_position_offset));
}

if ((x.country_short == "-") || (x.proxy_type == "-")) {
x.is_proxy = 0;
}
Expand Down

0 comments on commit 831d77c

Please sign in to comment.