Merge pull request #7 from liamg/liamg-add-redirect-detection

Add redirect detection

pkg/scan/options.go

@@ -24,8 +24,10 @@ type Options struct {
 }
 
 type Result struct {
-	URL        url.URL
-	StatusCode int
+	URL               url.URL
+	StatusCode        int
+	ExtraWork         []string
+	SupplementaryOnly bool
 }
 
 var DefaultOptions = Options{
@@ -39,6 +41,8 @@ var DefaultOptions = Options{
 		http.StatusMethodNotAllowed,
 		http.StatusNoContent,
 		http.StatusUnauthorized,
+		http.StatusMovedPermanently,
+		http.StatusFound,
 	},
 	Timeout:     time.Second * 5,
 	Parallelism: 10,

pkg/scan/scanner.go

@@ -28,6 +28,9 @@ func NewScanner(opt *Options) *Scanner {
 
 	client := &http.Client{
 		Timeout: opt.Timeout,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
 	}
 
 	if opt.SkipSSLVerification {
@@ -67,8 +70,16 @@ func (scanner *Scanner) Scan() ([]url.URL, error) {
 	waitChan := make(chan struct{})
 	var foundURLs []url.URL
 
+	var extraWork []string
+
 	go func() {
 		for result := range results {
+			if result.ExtraWork != nil {
+				extraWork = append(extraWork, result.ExtraWork...)
+			}
+			if result.SupplementaryOnly {
+				continue
+			}
 			if scanner.options.ResultChan != nil {
 				scanner.options.ResultChan <- result
 			}
@@ -108,6 +119,7 @@ func (scanner *Scanner) Scan() ([]url.URL, error) {
 			}
 		}
 	}
+
 	close(jobs)
 
 	logrus.Debug("Waiting for workers to complete...")
@@ -123,6 +135,14 @@ func (scanner *Scanner) Scan() ([]url.URL, error) {
 
 	<-waitChan
 
+	logrus.Debug("Supplementing results...")
+
+	for _, work := range extraWork {
+		if result := scanner.checkURL(work); result != nil {
+			foundURLs = append(foundURLs, result.URL)
+		}
+	}
+
 	logrus.Debug("Complete!")
 
 	return foundURLs, nil
@@ -144,6 +164,7 @@ func (scanner *Scanner) checkURL(uri string) *Result {
 	}
 
 	var code int
+	var location string
 	if err := retry.Do(func() error {
 		resp, err := scanner.client.Get(uri)
 		if err != nil {
@@ -152,11 +173,26 @@ func (scanner *Scanner) checkURL(uri string) *Result {
 		_, _ = io.Copy(ioutil.Discard, resp.Body)
 		resp.Body.Close()
 		code = resp.StatusCode
+		location = resp.Header.Get("Location")
 		return nil
 	}, retry.Attempts(10), retry.DelayType(retry.BackOffDelay)); err != nil {
 		return nil
 	}
 
+	var extraWork []string
+
+	if location != "" {
+		if !strings.Contains(location, "://") {
+			if parsed, err := url.Parse(uri); err == nil {
+				if relative, err := url.Parse(location); err == nil {
+					extraWork = append(extraWork, parsed.ResolveReference(relative).String())
+				}
+			}
+		} else {
+			extraWork = append(extraWork, location)
+		}
+	}
+
 	for _, status := range scanner.options.PositiveStatusCodes {
 		if status == code {
 			parsedURL, err := url.Parse(uri)
@@ -167,9 +203,17 @@ func (scanner *Scanner) checkURL(uri string) *Result {
 			return &Result{
 				StatusCode: code,
 				URL:        *parsedURL,
+				ExtraWork:  extraWork,
 			}
 		}
 	}
 
+	if len(extraWork) > 0 {
+		return &Result{
+			SupplementaryOnly: true,
+			ExtraWork:         extraWork,
+		}
+	}
+
 	return nil
 }

pkg/scan/scanner_test.go

@@ -1,11 +1,14 @@
 package scan
 
 import (
+	"bytes"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"testing"
 
+	"github.com/liamg/scout/pkg/wordlist"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -41,3 +44,39 @@ func TestScanner(t *testing.T) {
 	assert.Equal(t, results[0].String(), server.URL+"/login.php")
 
 }
+
+func TestRedirects(t *testing.T) {
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/very-secret-file.php":
+			w.WriteHeader(http.StatusOK)
+		case "/login.php":
+			http.Redirect(w, r, "/very-secret-file.php", http.StatusFound)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer server.Close()
+
+	parsed, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	scanner := NewScanner(&Options{
+		TargetURL:           *parsed,
+		Parallelism:         100,
+		PositiveStatusCodes: []int{http.StatusOK},
+		Wordlist:            wordlist.FromReader(bytes.NewReader([]byte("login.php"))),
+	})
+
+	results, err := scanner.Scan()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	require.Equal(t, len(results), 1)
+	assert.Equal(t, results[0].String(), server.URL+"/very-secret-file.php")
+
+}