diff --git a/modules/webproxy/request.go b/modules/webproxy/request.go index e02e8497..8b3a9570 100644 --- a/modules/webproxy/request.go +++ b/modules/webproxy/request.go @@ -1,12 +1,12 @@ package webproxy import ( + "bytes" "fmt" "log" "net" "net/url" "slices" - "strings" "github.com/zmap/zgrab2/lib/http" ) @@ -77,9 +77,8 @@ func (b *RequestBuilder) SetMethod(method string) *RequestBuilder { func (builder *RequestBuilder) SetHeaders(headers http.Header) { builder.headers = http.Header{ - "Accept": {"*/*"}, - "Proxy-Connection": {"close"}, - "Cache-Control": {"no-store"}, + "Accept": {"*/*"}, + "Content-Type": {"application/x-www-form-urlencoded"}, } if headers == nil { @@ -91,17 +90,17 @@ func (builder *RequestBuilder) SetHeaders(headers http.Header) { } } -func (builder *RequestBuilder) Build(token string) (*http.Request, error) { - // Create the request - req, err := http.NewRequest(builder.method, builder.url.String(), strings.NewReader(token)) +func (builder *RequestBuilder) Build(tokenHash string) (*http.Request, error) { + t := fmt.Sprintf("token=%s", tokenHash) + req, err := http.NewRequest(builder.method, builder.url.String(), bytes.NewBufferString(t)) if err != nil { - return nil, err + return nil, fmt.Errorf("RequestBuilder.Build(): failed to create request: %v", err) } // Slug token if needed if builder.slug { q := req.URL.Query() - q.Add("token", token) + q.Add("token", tokenHash) req.URL.RawQuery = q.Encode() } diff --git a/modules/webproxy/scan.go b/modules/webproxy/scan.go index 1236a635..221f9417 100644 --- a/modules/webproxy/scan.go +++ b/modules/webproxy/scan.go @@ -3,25 +3,27 @@ package webproxy import ( "bytes" "context" + "crypto/md5" "crypto/sha256" + "errors" "fmt" "io" "net" "net/url" - "strings" "time" log "github.com/sirupsen/logrus" "github.com/zmap/zgrab2" "github.com/zmap/zgrab2/lib/http" "golang.org/x/net/html/charset" + "golang.org/x/net/proxy" ) type Scan struct { target *zgrab2.ScanTarget scanner *Scanner client *http.Client - useTLS bool + scheme string maxRead int64 proxy *url.URL @@ -40,39 +42,33 @@ func readBody(contentType string, body io.ReadCloser, maxReadLen int64) (string, encoder, encoding, certain := charset.DetermineEncoding(buf.Bytes(), contentType) decoder := encoder.NewDecoder() - // Return early if the body was empty. No reason to go on from here. - // A bit weird, but it may happen! - bts := buf.Bytes() - if len(bts) == 0 { - return "", nil, nil - } + bodyText := "" + decodedSuccessfully := false - var b strings.Builder - switch { - //"windows-1252" is the default value and will likely not decode correctly - case certain || encoding != "windows-1252": - decoded, err := decoder.Bytes(bts) - if err != nil { - return "", nil, fmt.Errorf("error while decoding the body: %v", err) + if certain || encoding != "windows-1252" { + decoded, decErr := decoder.Bytes(buf.Bytes()) + + if decErr == nil { + bodyText = string(decoded) + decodedSuccessfully = true } + } - b.Write(decoded) - default: - b.Write(bts) + if !decodedSuccessfully { + bodyText = buf.String() } - bString := b.String() // re-enforce readlen - if int64(len(bString)) > maxReadLen { - bString = bString[:int(maxReadLen)] + if int64(len(bodyText)) > maxReadLen { + bodyText = bodyText[:int(maxReadLen)] } // Calculate the hash of the body m := sha256.New() - m.Write(bts) + m.Write(buf.Bytes()) h := m.Sum(nil) - return bString, h, nil + return bodyText, h, nil } // Get a context whose deadline is the earliest of the context's deadline (if it has one) and the @@ -134,6 +130,33 @@ func (scan *Scan) dialContext(ctx context.Context, network string, addr string) return conn, nil } +func (scan *Scan) socks5DialContext() (func(ctx context.Context, network, addr string) (net.Conn, error), error) { + u, err := scan.getProxyURL() + if err != nil { + return nil, err + } + + socksDialer, err := proxy.FromURL(u, proxy.Direct) + if err != nil { + return nil, fmt.Errorf("failed to create SOCKS5 dialer: %v", err) + } + + dc := socksDialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + + return func(ctx context.Context, network string, address string) (net.Conn, error) { + timeoutContext, _ := context.WithTimeout(ctx, scan.client.Timeout) + conn, err := dc.DialContext(scan.withDeadlineContext(timeoutContext), network, address) + if err != nil { + return nil, err + } + + scan.connections = append(scan.connections, conn) + return conn, nil + }, nil +} + // getTLSDialer returns a Dial function that connects using the // zgrab2.GetTLSConnection() func (scan *Scan) getTLSDialer(t *zgrab2.ScanTarget) func(network, addr string) (net.Conn, error) { @@ -173,10 +196,7 @@ func (scan *Scan) getTLSDialer(t *zgrab2.ScanTarget) func(network, addr string) } } -func (scan *Scan) SetProxyUrl() error { - transport := scan.client.Transport.(*http.Transport) - transport.DialContext = scan.dialContext - +func (scan *Scan) getProxyURL() (*url.URL, error) { host := scan.target.Domain if host == "" { host = scan.target.IP.String() @@ -187,22 +207,36 @@ func (scan *Scan) SetProxyUrl() error { port = &scan.scanner.config.BaseFlags.Port } - var schema string - switch { - case scan.useTLS: - schema = "https" - transport.DialTLS = scan.getTLSDialer(scan.target) - default: - schema = "http" - } + addr := fmt.Sprintf("%s://%s:%d", scan.scheme, host, *port) + return url.Parse(addr) +} - addr := fmt.Sprintf("%s://%s:%d", schema, host, *port) - proxy, err := url.Parse(addr) +// Note: `scan.target` is the proxy. +func (scan *Scan) SetProxyUrl() error { + proxy, err := scan.getProxyURL() if err != nil { - return fmt.Errorf(`failed to parse proxy address "%s": %v`, addr, err) + return err } scan.proxy = proxy - transport.Proxy = http.ProxyURL(proxy) + + transport := scan.client.Transport.(*http.Transport) + switch scan.scheme { + case "https": + transport.DialTLS = scan.getTLSDialer(scan.target) + transport.Proxy = http.ProxyURL(proxy) + case "socks5": + // This dialer does not need the Proxy + // value in the transport, since it makes + // connections through the proxy already. + dialCtx, err := scan.socks5DialContext() + if err != nil { + return err + } + transport.DialContext = dialCtx + default: + transport.DialContext = scan.dialContext + transport.Proxy = http.ProxyURL(proxy) + } return nil } @@ -219,7 +253,10 @@ func (scan *Scan) Grab() *zgrab2.ScanError { } // Build the request - req, err := scan.scanner.requestBuilder.Build(tkn) + hash := md5.New() + hash.Write([]byte(tkn)) + tknHash := fmt.Sprintf("%x", hash.Sum(nil)) + req, err := scan.scanner.requestBuilder.Build(tknHash) if err != nil { return zgrab2.NewScanError(zgrab2.SCAN_UNKNOWN_ERROR, err) } @@ -232,6 +269,7 @@ func (scan *Scan) Grab() *zgrab2.ScanError { // Put the response as is scan.Results.Token = tkn + scan.Results.TokenMD5 = tknHash scan.Results.Response = resp scan.Results.Target = scan.proxy.String() if err != nil { @@ -243,8 +281,8 @@ func (scan *Scan) Grab() *zgrab2.ScanError { // NOTE: we will not handle responses with unknown content length // or supposedly "empty". This can be done offline - if resp.ContentLength <= 0 { - return nil + if resp.ContentLength == 0 { + return zgrab2.NewScanError(zgrab2.SCAN_UNKNOWN_ERROR, errors.New("empty content")) } // Parse the body until the assigned number of bytes to read @@ -255,14 +293,12 @@ func (scan *Scan) Grab() *zgrab2.ScanError { cType := resp.Header.Get("content-type") bodyText, h, err := readBody(cType, resp.Body, maxReadLen) - if err != nil { - return zgrab2.NewScanError(zgrab2.SCAN_APPLICATION_ERROR, err) - } - // Assign the parsed body resp.BodyText = bodyText resp.BodySHA256 = h - + if err != nil { + return zgrab2.NewScanError(zgrab2.SCAN_UNKNOWN_ERROR, err) + } return nil } @@ -290,13 +326,6 @@ type ScanBuilder struct { maxRead int64 } -func (b *ScanBuilder) getTLSDialer(t *zgrab2.ScanTarget) func(network, addr string) (net.Conn, error) { - return func(network, addr string) (net.Conn, error) { - log.Fatal("not implemented yet") - return nil, nil - } -} - func (b *ScanBuilder) SetClient() *ScanBuilder { t := b.scanner.config.Timeout if t == 0 { @@ -323,7 +352,7 @@ func (b *ScanBuilder) SetMaxRead() *ScanBuilder { switch mr { // this is a replacement for nil values, i.e., default case 0: - b.maxRead = 256 + b.maxRead = 4096 // it may be that we do not want to read anything. case -1: b.maxRead = 0 @@ -333,17 +362,14 @@ func (b *ScanBuilder) SetMaxRead() *ScanBuilder { return b } -func (b *ScanBuilder) Build(t *zgrab2.ScanTarget, useTLS bool) *Scan { +func (b *ScanBuilder) Build(t *zgrab2.ScanTarget, scheme string) *Scan { scan := &Scan{ scanner: b.scanner, target: t, - useTLS: useTLS, + scheme: scheme, deadline: time.Now().Add(b.client.Timeout), client: &b.client, maxRead: b.maxRead, } - - // TODO: handle SOCKS - // if useSOCKS {} return scan } diff --git a/modules/webproxy/scanner.go b/modules/webproxy/scanner.go index 9c3f0348..f5c2a4a0 100644 --- a/modules/webproxy/scanner.go +++ b/modules/webproxy/scanner.go @@ -48,8 +48,8 @@ type Flags struct { RetryTLS bool `long:"retry-tls" description:"If the initial request fails, reconnect and try with HTTPS."` // TODO: not implemented yet! - //UseSOCKS bool `long:"use-tls" description:"Perform a SOCKS connection on the initial host"` - //RetrySOCKS bool `long:"retry-socks" description:"If the initial request fails, reconnect and try with SOCKS."` + UseSOCKS bool `long:"use-socks" description:"Perform a SOCKS connection on the initial host"` + RetrySOCKS bool `long:"retry-socks" description:"If the initial request fails, reconnect and try with SOCKS."` RawHeaders bool `long:"raw-headers" description:"Extract raw response up through headers"` } @@ -62,8 +62,9 @@ type Results struct { // It contains all redirect response prior to the final response. RedirectResponseChain []*http.Response `json:"redirect_response_chain,omitempty"` - Token string `json:"token"` - Target string `json:"target"` + Token string `json:"token"` + TokenMD5 string `json:"token-md5"` + Target string `json:"target"` } // Module is an implementation of the zgrab2.Module interface. @@ -160,28 +161,49 @@ func (scanner *Scanner) Init(flags zgrab2.ScanFlags) error { return nil } -func (scanner *Scanner) Scan(t zgrab2.ScanTarget) (zgrab2.ScanStatus, interface{}, error) { - scan := scanner.scanBuilder.Build(&t, scanner.config.UseTLS) +func (s *Scanner) scan(t *zgrab2.ScanTarget, scheme string) (zgrab2.ScanStatus, interface{}, error) { + scan := s.scanBuilder.Build(t, scheme) defer scan.Cleanup() if err := scan.Grab(); err != nil { - if scanner.config.RetryTLS && !scanner.config.UseTLS { - scan.Cleanup() - retry := scanner.scanBuilder.Build(&t, true) - defer retry.Cleanup() - - if rErr := retry.Grab(); rErr != nil { - return rErr.Unpack(scan.Results) - } - return zgrab2.SCAN_SUCCESS, retry.Results, nil - } return err.Unpack(scan.Results) } return zgrab2.SCAN_SUCCESS, scan.Results, nil } -// RegisterModule is called by modules/http.go to register this module with the -// zgrab2 framework. +func (s *Scanner) getRetryIterator() []string { + var schemes []string + var base string + switch { + case s.config.UseTLS: + base = "https" + case s.config.UseSOCKS: + base = "socks5" + default: + base = "http" + } + schemes = append(schemes, base) + if s.config.RetryTLS && !s.config.UseTLS { + schemes = append(schemes, "https") + } + + if s.config.RetrySOCKS && !s.config.UseSOCKS { + schemes = append(schemes, "socks5") + } + + return schemes +} + +func (s *Scanner) Scan(t zgrab2.ScanTarget) (status zgrab2.ScanStatus, results interface{}, err error) { + schemes := s.getRetryIterator() + for _, scheme := range schemes { + if status, results, err = s.scan(&t, scheme); status == zgrab2.SCAN_SUCCESS { + return + } + } + return +} + func RegisterModule() { var module Module diff --git a/modules/webproxy/webproxy_test.go b/modules/webproxy/webproxy_test.go index 692bb079..67357b77 100644 --- a/modules/webproxy/webproxy_test.go +++ b/modules/webproxy/webproxy_test.go @@ -1,6 +1,7 @@ package webproxy import ( + "context" "fmt" "io" "net" @@ -10,6 +11,7 @@ import ( "github.com/golang-jwt/jwt" "github.com/zmap/zgrab2" "github.com/zmap/zgrab2/lib/http" + "golang.org/x/net/proxy" ) type webproxyTester struct { @@ -45,13 +47,14 @@ func (cfg *webproxyTester) getScanner() (*Scanner, error) { var module Module flags := module.NewFlags().(*Flags) flags.Method = "POST" - flags.UserAgent = "Mozilla/5.0 HTTP proxy zgrab/0.1.6" + flags.UserAgent = "Mozilla/5.0 HTTPproxy/1.0 zgrab/0.1.6" flags.Port = cfg.pport flags.Timeout = 30 * time.Second flags.Endpoint = fmt.Sprintf("%s:%d", cfg.laddress, cfg.lport) flags.HmacKey = cfg.hmackey flags.SlugToken = cfg.slug - flags.UseTLS = true + flags.UseSOCKS = true + flags.MaxSize = 4096 scanner := module.NewScanner() if err := scanner.Init(flags); err != nil { @@ -104,10 +107,10 @@ func (cfg *webproxyTester) runTest(t *testing.T, testName string) { var tests = map[string]*webproxyTester{ "success": { - paddress: "", - laddress: "", - pport: 8888, - lport: 8081, + paddress: "82.165.198.169", + laddress: "130.226.254.28", + pport: 41569, + lport: 80, bChan: make(chan string, 1), hmackey: "gz13WcqhVBy09Mnw7ZZYNCqqlWvyRfJx", }, @@ -139,6 +142,45 @@ func TestProxy(t *testing.T) { } } +func TestClient(t *testing.T) { + proxyURL := "98.181.137.80:4145" + dialer, err := proxy.SOCKS5("tcp", proxyURL, nil, proxy.Direct) + if err != nil { + fmt.Printf("Failed to create dialer: %v\n", err) + return + } + + dc := dialer.(interface { + DialContext(ctx context.Context, network, addr string) (net.Conn, error) + }) + + dfunc := func(ctx context.Context, network string, address string) (net.Conn, error) { + t := 10 * time.Second + tc, _ := context.WithTimeout(ctx, t) + cded, _ := context.WithDeadline(tc, time.Now().Add(t)) + conn, err := dc.DialContext(cded, network, address) + return conn, err + } + + transport := &http.Transport{ + DialContext: dfunc, + } + + client := &http.Client{ + Timeout: 10 * time.Second, + Transport: transport, + UserAgent: "Go-http-client/1.1", + } + + req, _ := http.NewRequest("POST", "http://130.226.254.28:80", nil) + + resp, err := client.Do(req) + if err != nil { + t.Error(err) + } + defer resp.Body.Close() +} + func TestRequestBuilder(t *testing.T) { b := NewRequestBuilder("POST", "localhost:8080", true, http.Header{"cookie": {"123test"}}) var times = 3