diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..394a937 Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index b172266..2c4baac 100644 --- a/.gitignore +++ b/.gitignore @@ -18,10 +18,16 @@ # Dependency directories (remove the comment below to include it) # vendor/ -*.db +*.db* ds_store exe # Go workspace file go.work go.sum +.history/ +dist/ +logs/ +failed-download.txt +config.json +data/ \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml index a2bfabf..6bf3717 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -7,14 +7,14 @@ builds: env: - CC=x86_64-w64-mingw32-gcc - CXX=x86_64-w64-mingw32-g++ - - CGO_ENABLED=1 + - CGO_ENABLED=0 goos: - windows goarch: - amd64 - id: linux env: - - CGO_ENABLED=1 + - CGO_ENABLED=0 goos: - linux goarch: diff --git a/config/config.go b/config/config.go index 125727c..3afecc1 100644 --- a/config/config.go +++ b/config/config.go @@ -1,23 +1,26 @@ package config import ( - "asmr-downloader/log" - "asmr-downloader/utils" "encoding/json" - "go.uber.org/zap" "io" "net/http" "os" "path/filepath" "strconv" + + "go.uber.org/zap" + + "asmr-downloader/log" + "asmr-downloader/utils" ) const ConfigFileName = "config.json" const MetaDataDb = "asmr.db" // AsmroneStartPageUrl https://api.asmr.one/api/works?order=create_date&sort=desc&page=1&seed=92&subtitle=0 -const AsmrOneStartPageUrl = "https://api.asmr.one" +// const AsmrOneStartPageUrl = "https://api.asmr.one" const Asmr100StartPageUrl = "https://api.asmr-100.com" +const Asmr200StartPageUrl = "https://api.asmr-200.com" var AsmrBaseApiUrl = "" @@ -29,11 +32,11 @@ func init() { resp, err := client.Do(req) if err != nil || resp.StatusCode != 200 { log.AsmrLog.Error("尝试访问asmr.one失败: ", zap.String("error", err.Error())) - log.AsmrLog.Info("当前使用asmr-100.com访问") - AsmrBaseApiUrl = Asmr100StartPageUrl + log.AsmrLog.Info("当前使用asmr-200.com访问") + AsmrBaseApiUrl = Asmr200StartPageUrl } else { - log.AsmrLog.Info("当前使用asmr.one访问...") - AsmrBaseApiUrl = AsmrOneStartPageUrl + log.AsmrLog.Info("当前使用asmr-100.com访问...") + AsmrBaseApiUrl = Asmr100StartPageUrl } utils.Client.Put(client) defer resp.Body.Close() diff --git a/go.mod b/go.mod index 17944bd..b3855e1 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,34 @@ module asmr-downloader go 1.20 require ( - github.com/EDDYCJY/fake-useragent v0.2.0 - github.com/mattn/go-sqlite3 v2.0.1+incompatible github.com/melbahja/got v0.7.0 github.com/xxjwxc/gowp v0.0.0-20220528192505-f87b7668d4ff go.uber.org/zap v1.10.0 + golang.org/x/text v0.3.3 + modernc.org/sqlite v1.27.0 ) require ( - github.com/PuerkitoBio/goquery v1.8.0 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/xxjwxc/public v0.0.0-20210518123934-6cc0965f0bc5 // indirect go.uber.org/atomic v1.4.0 // indirect go.uber.org/multierr v1.1.0 // indirect - golang.org/x/net v0.5.0 // indirect + golang.org/x/mod v0.3.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/eapache/queue.v1 v1.1.0 // indirect + lukechampine.com/uint128 v1.2.0 // indirect + modernc.org/cc/v3 v3.40.0 // indirect + modernc.org/ccgo/v3 v3.16.13 // indirect + modernc.org/libc v1.29.0 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.2 // indirect + modernc.org/opt v0.1.3 // indirect + modernc.org/strutil v1.1.3 // indirect + modernc.org/token v1.0.1 // indirect ) diff --git a/main.go b/main.go index 40333a0..b3ecdd4 100644 --- a/main.go +++ b/main.go @@ -1,19 +1,21 @@ package main import ( - "asmr-downloader/config" - "asmr-downloader/log" - "asmr-downloader/model" - "asmr-downloader/spider" - "asmr-downloader/storage" - "asmr-downloader/utils" "database/sql" "fmt" - "go.uber.org/zap" "os" "strings" "sync" "time" + + "go.uber.org/zap" + + "asmr-downloader/config" + "asmr-downloader/log" + "asmr-downloader/model" + "asmr-downloader/spider" + "asmr-downloader/storage" + "asmr-downloader/utils" ) var pageDataChannel = make(chan model.PageResult, 4) diff --git a/patch/patch.go b/patch/patch.go index 56daf40..7d02e61 100644 --- a/patch/patch.go +++ b/patch/patch.go @@ -9,13 +9,14 @@ import ( "os" "strconv" "strings" + + _ "modernc.org/sqlite" ) -import _ "github.com/mattn/go-sqlite3" var haveDoneTxt = "have-download.txt" func PatchHavenDownload2DB() { - db, err := sql.Open("sqlite3", "../asmr.db") + db, err := sql.Open("sqlite", "../asmr.db") _ = db if err != nil { log.Fatal(err) diff --git a/spider/spider.go b/spider/spider.go index 2918e96..1191198 100644 --- a/spider/spider.go +++ b/spider/spider.go @@ -1,23 +1,24 @@ package spider import ( - "asmr-downloader/config" - "asmr-downloader/log" - "asmr-downloader/model" - "asmr-downloader/utils" "bytes" "context" "encoding/json" "fmt" - browser "github.com/EDDYCJY/fake-useragent" - "github.com/xxjwxc/gowp/workpool" - "go.uber.org/zap" "io" "net/http" "os" "path/filepath" "runtime" "strings" + + "github.com/xxjwxc/gowp/workpool" + "go.uber.org/zap" + + "asmr-downloader/config" + "asmr-downloader/log" + "asmr-downloader/model" + "asmr-downloader/utils" ) var ctx = context.Background() @@ -48,6 +49,18 @@ func NewASMRClient(maxWorker int, globalConfig *config.Config) *ASMRClient { } } +func HeadersInit(r *http.Request) *http.Request { + r.Header.Set("Referer", "https://www.asmr.one/") + r.Header.Set("Origin", "https://www.asmr.one") + r.Header.Set("Host", strings.Split(config.AsmrBaseApiUrl, "//")[1]) + r.Header.Set("Connection", "keep-alive") + r.Header.Set("Sec-Fetch-Mode", "cors") + r.Header.Set("Sec-Fetch-Site", "cross-site") + r.Header.Set("Sec-Fetch-Dest", "empty") + r.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") + return r +} + // Login 登入获取授权信息 func (asmrClient *ASMRClient) Login() error { payload, err := json.Marshal(map[string]string{ @@ -61,8 +74,7 @@ func (asmrClient *ASMRClient) Login() error { client := utils.Client.Get().(*http.Client) req, _ := http.NewRequest("POST", config.AsmrBaseApiUrl+"/api/auth/me", bytes.NewBuffer(payload)) req.Header.Set("Content-Type", "application/json") - req.Header.Set("Referer", "https://www.asmr.one/") - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36") + req = HeadersInit(req) resp, err := client.Do(req) utils.Client.Put(client) if err != nil { @@ -85,8 +97,7 @@ func (asmrClient *ASMRClient) GetVoiceTracks(id string) ([]track, error) { client := utils.Client.Get().(*http.Client) req, _ := http.NewRequest("GET", config.AsmrBaseApiUrl+"/api/tracks/"+id, nil) req.Header.Set("Authorization", asmrClient.Authorization) - req.Header.Set("Referer", "https://www.asmr.one/") - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36") + req = HeadersInit(req) resp, err := client.Do(req) utils.Client.Put(client) if err != nil { @@ -200,7 +211,6 @@ func (asmrClient *ASMRClient) DownloadFile(url string, dirPath string, fileName // @return error func GetPerPageInfo(authorStr string, pageIndex int, subtitleFlag int) (*model.PageResult, error) { var seed int = utils.GenerateReqSeed() - randomUserAgent := browser.Random() //log.Printf("Random: %s\n", randomUserAgent) //var reqUrl = "https://api.asmr.one/api/works?order=create_date&sort=desc&page=1&seed=" + strconv.Itoa(seed) + "&subtitle=0" var reqUrl = "" @@ -216,21 +226,8 @@ func GetPerPageInfo(authorStr string, pageIndex int, subtitleFlag int) (*model.P // Handle error // ignore here } - req.Header.Set("Accept", "application/json, text/plain, */*") - //req.Header.Set("Accept-Encoding", "gzip, deflate, br") - req.Header.Set("Accept-Language", "zh,en;q=0.9,zh-TW;q=0.8,zh-CN;q=0.7,ja;q=0.6") + req = HeadersInit(req) req.Header.Set("Authorization", authorStr) - req.Header.Set("Cache-Control", "no-cache") - req.Header.Set("Origin", "https://www.asmr.one") - req.Header.Set("Pragma", "no-cache") - req.Header.Set("Referer", "https://www.asmr.one/") - req.Header.Set("Sec-Ch-UA", `"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"`) - req.Header.Set("Sec-Ch-Ua-Mobile", "?0") - req.Header.Set("Sec-Ch-Ua-Platform", "macOS") - req.Header.Set("Sec-Fetch-Dest", "empty") - req.Header.Set("Sec-Fetch-Mode", "cors") - req.Header.Set("Sec-Fetch-Site", "same-site") - req.Header.Set("User-Agent", randomUserAgent) respond, respError := client.Do(req.WithContext(context.Background())) utils.Client.Put(client) diff --git a/storage/storage.go b/storage/storage.go index 8ea5eb3..9f57c57 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,13 +1,15 @@ package storage import ( - "asmr-downloader/config" - "asmr-downloader/log" "database/sql" - "go.uber.org/zap" "sync" + + "go.uber.org/zap" + _ "modernc.org/sqlite" + + "asmr-downloader/config" + "asmr-downloader/log" ) -import _ "github.com/mattn/go-sqlite3" var StoreDb *SqliteStoreEngine @@ -18,7 +20,7 @@ var once sync.Once // @Description: 单例存储实例 // @return *SqliteStoreEngine func GetDbInstance() *SqliteStoreEngine { - db, err := sql.Open("sqlite3", config.MetaDataDb) + db, err := sql.Open("sqlite", config.MetaDataDb) if err != nil { log.AsmrLog.Error("", zap.String("error", err.Error())) return nil diff --git a/utils/utils.go b/utils/utils.go index 2f4186a..c8b5b04 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,13 +1,9 @@ package utils import ( - "asmr-downloader/log" "bufio" "crypto/tls" "fmt" - "github.com/melbahja/got" - "github.com/xxjwxc/gowp/workpool" - "go.uber.org/zap" "io" "math/rand" "net/http" @@ -16,6 +12,13 @@ import ( "strings" "sync" "time" + + "github.com/melbahja/got" + "github.com/xxjwxc/gowp/workpool" + "go.uber.org/zap" + "golang.org/x/text/unicode/norm" + + "asmr-downloader/log" ) const FailedDownloadFileName = "failed-download.txt" @@ -44,16 +47,22 @@ var Client = sync.Pool{ }, } -// FileOrDirExists 判断所给路径文件/文件夹是否存在 +// FileOrDirExists 判断所给路径文件/文件夹是否存在 after unicode normalization func FileOrDirExists(path string) bool { - _, err := os.Stat(path) //os.Stat获取文件信息 + path = norm.NFC.String(path) + //path = strings.ReplaceAll(path, "/jfs/", "/ASMR/") + files, err := os.ReadDir(filepath.Dir(path)) if err != nil { - if os.IsExist(err) { + return false + } + + for _, file := range files { + if norm.NFC.String(file.Name()) == norm.NFC.String(filepath.Base(path)) { return true } - return false } - return true + return false + } // PromotForInput 获取用户输入