Added spidering and improved a bunch of shiz

README.md

@@ -2,7 +2,7 @@
 
 [![Travis Build Status](https://travis-ci.org/liamg/scout.svg?branch=master)](https://travis-ci.org/liamg/scout)
 
-Scout is a URL fuzzer for discovering undisclosed VHOSTS, files and directories on a web server. 
+Scout is a URL fuzzer and spider for discovering undisclosed VHOSTS, files and directories on a web server. 
 
 <p align="center">
   <img width="929" height="502" src="./demo.gif" />
@@ -14,13 +14,13 @@ A full word list is included in the binary, meaning maximum portability and mini
 
 ```bash
 
-
 Usage:
   scout [command]
 
 Available Commands:
   help        Help about any command
   url         Discover URLs on a given web server.
+  version     Display scout version.
   vhost       Discover VHOSTs on a given web server.
 
 Flags:
@@ -31,11 +31,8 @@ Flags:
   -k, --skip-ssl-verify   Skip SSL certificate verification.
   -w, --wordlist string   Path to wordlist file. If this is not specified an internal wordlist will be used.
 
-Use "scout [command] --help" for more information about a command.
-
 ```
 
-
 ### Discover URLs
 
 #### Flags
@@ -52,10 +49,14 @@ Filename to seek in the directory being searched. Useful when all directories re
 
 Extra header to send with requests e.g. `-H "Cookie: PHPSESSID=blah"` 
 
-##### `-s, --status-codes`
+##### `-c, --status-codes`
 
 HTTP status codes which indicate a positive find. (default `200,400,403,500,405,204,401,301,302`)
 
+##### `-s, --spider`
+
+Scan page content for links and confirm their existence.
+
 #### Full example
 
 ```bash

assets/wordlist.txt

@@ -1,3 +1,16 @@
+.git
+.github
+.travis.yml
+vendor
+.gitignore
+Dockerfile
+.viminfo
+.rhosts
+.rhost
+.profile
+.maintenance
+.canna
+.backup
 images
 css
 LC_MESSAGES
@@ -35558,11 +35571,4 @@ ABBA
 000002
 0.8
 0.11
-0.10
-.viminfo
-.rhosts
-.rhost
-.profile
-.maintenance
-.canna
-.backup
+0.10

cmd/scout/root.go

@@ -1,29 +1,49 @@
 package main
 
 import (
+	"fmt"
+	"net/http"
 	"strconv"
 
-	"github.com/liamg/scout/pkg/scan"
+	"github.com/liamg/scout/internal/app/scout/version"
 	"github.com/spf13/cobra"
 )
 
-var parallelism = scan.DefaultURLOptions.Parallelism
+var parallelism = 10
 var noColours = false
 var wordlistPath string
 var debug bool
 var skipSSLVerification bool
+var positiveStatusCodes = []int{
+	http.StatusOK,
+	http.StatusBadRequest,
+	http.StatusInternalServerError,
+	http.StatusMethodNotAllowed,
+	http.StatusNoContent,
+	http.StatusUnauthorized,
+	http.StatusForbidden,
+	http.StatusFound,
+	http.StatusMovedPermanently,
+}
 
 var rootCmd = &cobra.Command{
 	Use:   "scout",
-	Short: "Scout is a portable URL fuzzer",
-	Long:  `A fast and portable url fuzzer - see https://github.com/liamg/scout for more information`,
+	Short: "Scout is a portable URL fuzzer and spider",
+	Long:  `A fast and portable url fuzzer and spider - see https://github.com/liamg/scout for more information`,
 	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		fmt.Printf(`
+                          __ 
+   ______________  __  __/ /_   
+  / ___/ ___/ __ \/ / / / __/    %s
+ (__  ) /__/ /_/ / /_/ / /_      http://github.com/liamg/scout
+/____/\___/\____/\__,_/\__/
 
+`, version.Version)
 	},
 }
 
 func init() {
-	for _, code := range scan.DefaultURLOptions.PositiveStatusCodes {
+	for _, code := range positiveStatusCodes {
 		statusCodes = append(statusCodes, strconv.Itoa(code))
 	}
 

cmd/scout/url.go

@@ -19,7 +19,8 @@ import (
 var statusCodes []string
 var filename string
 var header string
-var extensions = scan.DefaultURLOptions.Extensions
+var extensions = []string{"php", "htm", "html", "txt"}
+var enableSpidering bool
 
 var urlCmd = &cobra.Command{
 	Use:   "url [url]",
@@ -62,41 +63,42 @@ var urlCmd = &cobra.Command{
 			intStatusCodes = append(intStatusCodes, i)
 		}
 
-		options := &scan.URLOptions{
-			PositiveStatusCodes: intStatusCodes,
-			TargetURL:           *parsedURL,
-			ResultChan:          resultChan,
-			BusyChan:            busyChan,
-			Parallelism:         parallelism,
-			Extensions:          extensions,
-			Filename:            filename,
-			SkipSSLVerification: skipSSLVerification,
-			ExtraHeader:         header,
+		options := []scan.URLOption{
+			scan.WithPositiveStatusCodes(intStatusCodes),
+			scan.WithTargetURL(*parsedURL),
+			scan.WithResultChan(resultChan),
+			scan.WithBusyChan(busyChan),
+			scan.WithParallelism(parallelism),
+			scan.WithExtensions(extensions),
+			scan.WithFilename(filename),
+			scan.WithSkipSSLVerification(skipSSLVerification),
+			scan.WithExtraHeader(header),
+			scan.WithSpidering(enableSpidering),
 		}
+
 		if wordlistPath != "" {
-			options.Wordlist, err = wordlist.FromFile(wordlistPath)
+			words, err := wordlist.FromFile(wordlistPath)
 			if err != nil {
 				tml.Printf("<bold><red>Error:</red></bold> %s\n", err)
 				os.Exit(1)
 			}
+			options = append(options, scan.WithWordlist(words))
 		}
-		options.Inherit()
 
 		tml.Printf(
-			`
-<blue>[</blue><yellow>+</yellow><blue>] Target URL</blue><yellow>      %s
+			`<blue>[</blue><yellow>+</yellow><blue>] Target URL</blue><yellow>      %s
 <blue>[</blue><yellow>+</yellow><blue>] Routines</blue><yellow>        %d 
 <blue>[</blue><yellow>+</yellow><blue>] Extensions</blue><yellow>      %s 
 <blue>[</blue><yellow>+</yellow><blue>] Positive Codes</blue><yellow>  %s
 
 `,
-			options.TargetURL.String(),
-			options.Parallelism,
-			strings.Join(options.Extensions, ","),
+			parsedURL.String(),
+			parallelism,
+			strings.Join(extensions, ","),
 			strings.Join(statusCodes, ","),
 		)
 
-		scanner := scan.NewURLScanner(options)
+		scanner := scan.NewURLScanner(options...)
 
 		waitChan := make(chan struct{})
 
@@ -173,9 +175,10 @@ func clearLine() {
 
 func init() {
 	urlCmd.Flags().StringVarP(&filename, "filename", "f", filename, "Filename to seek in the directory being searched. Useful when all directories report 404 status.")
-	urlCmd.Flags().StringSliceVarP(&statusCodes, "status-codes", "s", statusCodes, "HTTP status codes which indicate a positive find.")
+	urlCmd.Flags().StringSliceVarP(&statusCodes, "status-codes", "c", statusCodes, "HTTP status codes which indicate a positive find.")
 	urlCmd.Flags().StringSliceVarP(&extensions, "extensions", "x", extensions, "File extensions to detect.")
 	urlCmd.Flags().StringVarP(&header, "header", "H", header, "Extra header to send with requests.")
+	urlCmd.Flags().BoolVarP(&enableSpidering, "spider", "s", enableSpidering, "Spider links within page content")
 
 	rootCmd.AddCommand(urlCmd)
 }

cmd/scout/version.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Display scout version.",
+	Run: func(cmd *cobra.Command, args []string) {
+
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}

cmd/scout/vhost.go

@@ -101,8 +101,7 @@ var vhostCmd = &cobra.Command{
 		options.Inherit()
 
 		tml.Printf(
-			`
-<blue>[</blue><yellow>+</yellow><blue>] Base Domain</blue><yellow>     %s
+			`<blue>[</blue><yellow>+</yellow><blue>] Base Domain</blue><yellow>     %s
 <blue>[</blue><yellow>+</yellow><blue>] Routines</blue><yellow>        %d 
 <blue>[</blue><yellow>+</yellow><blue>] IP</blue><yellow>              %s 
 <blue>[</blue><yellow>+</yellow><blue>] Port</blue><yellow>            %s 

internal/app/scout/version/version.go

@@ -0,0 +1,3 @@
+package version
+
+var Version = "0.0.0"