From 6c010f9fa4edcc331c4543d8b1be6a5054ed8258 Mon Sep 17 00:00:00 2001 From: sherlockcatyyds <91426047+sherlock1cat@users.noreply.github.com> Date: Thu, 13 Jul 2023 01:00:56 +0800 Subject: [PATCH] fix GetAsnInfoByIp issue --- cmd/httpxUtilz.go | 43 ++++++++++++++++++++++++++++++++----------- utilz/asn.go | 25 +++++++++++-------------- utilz/cdnInfo.go | 21 +++++++++------------ utilz/cname.go | 6 +++--- utilz/httpx.go | 10 +++++----- 5 files changed, 60 insertions(+), 45 deletions(-) diff --git a/cmd/httpxUtilz.go b/cmd/httpxUtilz.go index bb7c294..252a395 100644 --- a/cmd/httpxUtilz.go +++ b/cmd/httpxUtilz.go @@ -7,6 +7,7 @@ import ( "fmt" httpxUtilz "httpxUtilz/utilz" "log" + "net" "os" "reflect" "sync" @@ -14,18 +15,18 @@ import ( ) type PassiveResult struct { - CName string `json:"cname"` - IP string `json:"ip"` + CName []string `json:"cname"` + IP []string `json:"ip"` Cdn int `json:"cdn"` CdnByIP bool `json:"cdn_by_ip"` CdnByHeader []string `json:"cdn_by_header"` CdnByCidr bool `json:"cdn_by_cidr"` CdnByAsn bool `json:"cdn_by_asn"` CdnByCName bool `json:"cdn_by_cname"` - Cidr string `json:"cidr"` - Asn string `json:"asn"` - Org string `json:"org"` - Addr string `json:"addr"` + Cidr []string `json:"cidr"` + Asn []string `json:"asn"` + Org []string `json:"org"` + Addr []string `json:"addr"` } type ResponseResult struct { @@ -150,6 +151,26 @@ func WriteBufferToFile(buffer *bytes.Buffer, filePath string) error { return err } +func UniquerIps(cnameIps, resolveIps []string) (ips []string) { + uniqueIPs := make(map[string]bool) + + for _, ip := range cnameIps { + uniqueIPs[ip] = true + } + + for _, ip := range resolveIps { + uniqueIPs[ip] = true + } + + for ip := range uniqueIPs { + // check ipv6 + if ipAddr := net.ParseIP(ip); ipAddr != nil { + ips = append(ips, ip) + } + } + return +} + func processURL(params ProcessUrlParams) (result Result) { config := httpxUtilz.RequestClientConfig{ ProxyURL: params.Proxy, @@ -220,7 +241,7 @@ func processURL(params ProcessUrlParams) (result Result) { if params.Passive { cname, cnameIps := config.GetCNameIPByDomain(params.Url, "./data/vaildResolvers.txt") resolveIps := config.GetIpsByAsnmap(params.Url) - ips := cnameIps + resolveIps + ips := UniquerIps(cnameIps, resolveIps) if len(ips) == 0 { return Result{} } @@ -237,10 +258,10 @@ func processURL(params ProcessUrlParams) (result Result) { } cdn, cdnbyip, cdnbyheader, cdnbycidr, cdnbyasn, cdnbycname = config.GetCdnInfoByAll( - resp, ips, "./data/cdn_header_keys.json", - cidr, "./data/cdn_ip_cidr.json", - asn, "./data/cdn_asn_list.json", - cname, "./data/cdn_cname_keywords.json") + resp, ips, cidr, asn, cname, "./data/cdn_header_keys.json", + "./data/cdn_ip_cidr.json", + "./data/cdn_asn_list.json", + "./data/cdn_cname_keywords.json") } passiveInfos = PassiveResult{ diff --git a/utilz/asn.go b/utilz/asn.go index 312539a..37e539d 100644 --- a/utilz/asn.go +++ b/utilz/asn.go @@ -4,7 +4,6 @@ import ( "encoding/json" asnmap "github.com/projectdiscovery/asnmap/libs" "log" - "strings" ) type AsnData struct { @@ -38,7 +37,7 @@ func handleInput(client *asnmap.Client, item string) *AsnData { return &data } -func GetAsnInfoByIps(ips string, proxy string) (cidr, asn, org, addr string) { +func GetAsnInfoByIps(ips []string, proxy string) (cidr, asn, org, addr []string) { client, err := asnmap.NewClient() if proxy != "" { proxys := []string{proxy} @@ -48,37 +47,35 @@ func GetAsnInfoByIps(ips string, proxy string) (cidr, asn, org, addr string) { log.Println("GetAsnInfoByIps> asnmap new client ", err) return } - items := strings.Split(ips, ",") - for _, item := range items { // Retrieve the first result. + for _, item := range ips { // Retrieve result have value break. data := handleInput(client, item) - if data != nil { - cidr += strings.Join(data.AsRange, ",") - asn += data.AsNumber - org += data.AsName - addr += data.AsCountry + if data != nil && len(cidr) == 0 && len(asn) == 0 && len(org) == 0 && len(addr) == 0 { + cidr = data.AsRange + asn = append(asn, data.AsNumber) + org = append(org, data.AsName) + addr = append(addr, data.AsCountry) } else { - log.Printf("GetAsnInfoByIps> asnmap can't get data by %s", ips) + log.Printf("GetAsnInfoByIps> asnmap can't get data by %s", item) //cidr = "Na" //asn = "Na" //org = "Na" //addr = "Na" } - return } return } -func GetIpsByAsnmap(url string) (ips string) { +func GetIpsByAsnmap(url string) (ips []string) { domain, err := GetSubDomain(url) if err != nil { log.Printf("GetIpsByAsnmap> %s getsubdomain failed, check url format.", url) return } - resolvedIps, err := asnmap.ResolveDomain(domain) + ips, err = asnmap.ResolveDomain(domain) if err != nil { log.Fatal(err) } - ips = strings.Join(resolvedIps, ",") + return } diff --git a/utilz/cdnInfo.go b/utilz/cdnInfo.go index 5476702..c438c9a 100644 --- a/utilz/cdnInfo.go +++ b/utilz/cdnInfo.go @@ -44,11 +44,10 @@ func ReadCNameJSONFile(filename string) (map[string]string, error) { return value, nil } -func GetCDNInfoByIps(ips string) (cdn int, cdnbyip bool) { +func GetCDNInfoByIps(ips []string) (cdn int, cdnbyip bool) { client := cdncheck.New() - ipList := strings.Split(ips, ",") - for _, ipStr := range ipList { + for _, ipStr := range ips { ip := net.ParseIP(strings.TrimSpace(ipStr)) if ip == nil { //log.Println("GetCDNInfoByIps: Invalid IP address: ", ipStr) @@ -83,14 +82,14 @@ func GetCDNInfoByHeader(resp *Response, CdnHeaderfilename string) (cdn int, cdnb return } -func GetCDNInfoByCidr(cidr string, CdnCidrfilename string) (cdn int, cdnbycidr bool) { +func GetCDNInfoByCidr(cidr []string, CdnCidrfilename string) (cdn int, cdnbycidr bool) { cdnCidrs, err := ReadJSONFile(CdnCidrfilename) if err != nil { log.Fatal("GetCDNInfoByCidr: Encountered an error while processing the JSON file：", err) return } - cidrRange := strings.Split(cidr, ",") - for _, checkCidr := range cidrRange { + + for _, checkCidr := range cidr { for _, value := range cdnCidrs { if value == checkCidr && value != "" { cdnbycidr = true @@ -102,15 +101,14 @@ func GetCDNInfoByCidr(cidr string, CdnCidrfilename string) (cdn int, cdnbycidr b return } -func GetCDNInfoByAsn(asn, CdnAsnfilename string) (cdn int, cdnbyasn bool) { +func GetCDNInfoByAsn(asn []string, CdnAsnfilename string) (cdn int, cdnbyasn bool) { cdnAsn, err := ReadJSONFile(CdnAsnfilename) if err != nil { log.Fatal("GetCDNInfoByAsn: Failed to handle the JSON file：", err) return } - asnRange := strings.Split(asn, ",") - for _, checkAsn := range asnRange { + for _, checkAsn := range asn { for _, value := range cdnAsn { if value == checkAsn && value != "" { cdnbyasn = true @@ -122,15 +120,14 @@ func GetCDNInfoByAsn(asn, CdnAsnfilename string) (cdn int, cdnbyasn bool) { return } -func GetCDNInfoByCName(cname, CdnCNamefilename string) (cdn int, cdnbycname bool) { +func GetCDNInfoByCName(cname []string, CdnCNamefilename string) (cdn int, cdnbycname bool) { cnameMap, err := ReadCNameJSONFile(CdnCNamefilename) if err != nil { log.Println("GetCDNInfoByAsn: Failed to handle the JSON file：", err) return } - cnameRange := strings.Split(cname, ",") - for _, checkCName := range cnameRange { + for _, checkCName := range cname { _, ok := cnameMap[checkCName] if !ok { continue diff --git a/utilz/cname.go b/utilz/cname.go index 34d3a54..12a38a0 100644 --- a/utilz/cname.go +++ b/utilz/cname.go @@ -50,7 +50,7 @@ func DnsxClient(domain string, resolversFile string) *dnsx.DNSX { return dnsxClient } -func GetCnameIPsByDomain(url string, resolversFile string) (cname string, ips string) { +func GetCnameIPsByDomain(url string, resolversFile string) (cname, ips []string) { domain, err := GetSubDomain(url) if err != nil { @@ -63,8 +63,8 @@ func GetCnameIPsByDomain(url string, resolversFile string) (cname string, ips st CName := dnsxResult.CNAME IPs := dnsxResult.A - cname = strings.Join(CName, ",") - ips = strings.Join(IPs, ",") + cname = CName + ips = IPs return } diff --git a/utilz/httpx.go b/utilz/httpx.go index bb662f3..e47f994 100644 --- a/utilz/httpx.go +++ b/utilz/httpx.go @@ -112,20 +112,20 @@ func (config *RequestClientConfig) GetContentLengthAllBodyByResponse(resp *Respo return } -func (config *RequestClientConfig) GetCNameIPByDomain(domain string, resolversFile string) (cname string, ips string) { +func (config *RequestClientConfig) GetCNameIPByDomain(domain string, resolversFile string) (cname, ips []string) { cname, ips = GetCnameIPsByDomain(domain, resolversFile) if len(cname) == 0 { - cname = "NA" + cname = []string{"Na"} } return } -func (config *RequestClientConfig) GetIpsByAsnmap(domain string) (ips string) { +func (config *RequestClientConfig) GetIpsByAsnmap(domain string) (ips []string) { ips = GetIpsByAsnmap(domain) return } -func (config *RequestClientConfig) GetCdnInfoByAll(resp *Response, ips, CdnHeaderfilename, cidr, CdnCidrfilename, asn, CdnAsnfilename, cname, CdnCNamefilename string) (cdn int, cdnbyip bool, cdnbyheader []string, cdnbycidr, cdnbyasn, cdnbycname bool) { +func (config *RequestClientConfig) GetCdnInfoByAll(resp *Response, ips, cidr, asn, cname []string, CdnHeaderfilename, CdnCidrfilename, CdnAsnfilename, CdnCNamefilename string) (cdn int, cdnbyip bool, cdnbyheader []string, cdnbycidr, cdnbyasn, cdnbycname bool) { _, cdnbyip = GetCDNInfoByIps(ips) _, cdnbyheader = GetCDNInfoByHeader(resp, CdnHeaderfilename) @@ -143,7 +143,7 @@ func (config *RequestClientConfig) GetCdnInfoByAll(resp *Response, ips, CdnHeade return } -func (config *RequestClientConfig) GetAsnInfoByIp(ips string, proxy string) (cidr, asn, org, addr string) { +func (config *RequestClientConfig) GetAsnInfoByIp(ips []string, proxy string) (cidr, asn, org, addr []string) { cidr, asn, org, addr = GetAsnInfoByIps(ips, proxy) return }