Merge pull request #9 from maikirakiwi/master

修复至可用状态

.gitignore

@@ -18,10 +18,16 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 
-*.db
+*.db*
 ds_store
 exe
 # Go workspace file
 go.work
 go.sum
 
+.history/
+dist/
+logs/
+failed-download.txt
+config.json
+data/

.goreleaser.yaml

@@ -7,14 +7,14 @@ builds:
     env:
       - CC=x86_64-w64-mingw32-gcc
       - CXX=x86_64-w64-mingw32-g++
-      - CGO_ENABLED=1
+      - CGO_ENABLED=0
     goos:
       - windows
     goarch:
       - amd64
   - id: linux
     env:
-      - CGO_ENABLED=1
+      - CGO_ENABLED=0
     goos:
       - linux
     goarch:

config/config.go

@@ -1,23 +1,26 @@
 package config
 
 import (
-	"asmr-downloader/log"
-	"asmr-downloader/utils"
 	"encoding/json"
-	"go.uber.org/zap"
 	"io"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
+
+	"go.uber.org/zap"
+
+	"asmr-downloader/log"
+	"asmr-downloader/utils"
 )
 
 const ConfigFileName = "config.json"
 const MetaDataDb = "asmr.db"
 
 // AsmroneStartPageUrl https://api.asmr.one/api/works?order=create_date&sort=desc&page=1&seed=92&subtitle=0
-const AsmrOneStartPageUrl = "https://api.asmr.one"
+// const AsmrOneStartPageUrl = "https://api.asmr.one"
 const Asmr100StartPageUrl = "https://api.asmr-100.com"
+const Asmr200StartPageUrl = "https://api.asmr-200.com"
 
 var AsmrBaseApiUrl = ""
 
@@ -29,11 +32,11 @@ func init() {
 	resp, err := client.Do(req)
 	if err != nil || resp.StatusCode != 200 {
 		log.AsmrLog.Error("尝试访问asmr.one失败: ", zap.String("error", err.Error()))
-		log.AsmrLog.Info("当前使用asmr-100.com访问")
-		AsmrBaseApiUrl = Asmr100StartPageUrl
+		log.AsmrLog.Info("当前使用asmr-200.com访问")
+		AsmrBaseApiUrl = Asmr200StartPageUrl
 	} else {
-		log.AsmrLog.Info("当前使用asmr.one访问...")
-		AsmrBaseApiUrl = AsmrOneStartPageUrl
+		log.AsmrLog.Info("当前使用asmr-100.com访问...")
+		AsmrBaseApiUrl = Asmr100StartPageUrl
 	}
 	utils.Client.Put(client)
 	defer resp.Body.Close()

go.mod

@@ -3,19 +3,34 @@ module asmr-downloader
 go 1.20
 
 require (
-	github.com/EDDYCJY/fake-useragent v0.2.0
-	github.com/mattn/go-sqlite3 v2.0.1+incompatible
 	github.com/melbahja/got v0.7.0
 	github.com/xxjwxc/gowp v0.0.0-20220528192505-f87b7668d4ff
 	go.uber.org/zap v1.10.0
+	golang.org/x/text v0.3.3
+	modernc.org/sqlite v1.27.0
 )
 
 require (
-	github.com/PuerkitoBio/goquery v1.8.0 // indirect
-	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/mattn/go-isatty v0.0.16 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/xxjwxc/public v0.0.0-20210518123934-6cc0965f0bc5 // indirect
 	go.uber.org/atomic v1.4.0 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
-	golang.org/x/net v0.5.0 // indirect
+	golang.org/x/mod v0.3.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/eapache/queue.v1 v1.1.0 // indirect
+	lukechampine.com/uint128 v1.2.0 // indirect
+	modernc.org/cc/v3 v3.40.0 // indirect
+	modernc.org/ccgo/v3 v3.16.13 // indirect
+	modernc.org/libc v1.29.0 // indirect
+	modernc.org/mathutil v1.6.0 // indirect
+	modernc.org/memory v1.7.2 // indirect
+	modernc.org/opt v0.1.3 // indirect
+	modernc.org/strutil v1.1.3 // indirect
+	modernc.org/token v1.0.1 // indirect
 )

main.go

@@ -1,19 +1,21 @@
 package main
 
 import (
-	"asmr-downloader/config"
-	"asmr-downloader/log"
-	"asmr-downloader/model"
-	"asmr-downloader/spider"
-	"asmr-downloader/storage"
-	"asmr-downloader/utils"
 	"database/sql"
 	"fmt"
-	"go.uber.org/zap"
 	"os"
 	"strings"
 	"sync"
 	"time"
+
+	"go.uber.org/zap"
+
+	"asmr-downloader/config"
+	"asmr-downloader/log"
+	"asmr-downloader/model"
+	"asmr-downloader/spider"
+	"asmr-downloader/storage"
+	"asmr-downloader/utils"
 )
 
 var pageDataChannel = make(chan model.PageResult, 4)

patch/patch.go

@@ -9,13 +9,14 @@ import (
 	"os"
 	"strconv"
 	"strings"
+
+	_ "modernc.org/sqlite"
 )
-import _ "github.com/mattn/go-sqlite3"
 
 var haveDoneTxt = "have-download.txt"
 
 func PatchHavenDownload2DB() {
-	db, err := sql.Open("sqlite3", "../asmr.db")
+	db, err := sql.Open("sqlite", "../asmr.db")
 	_ = db
 	if err != nil {
 		log.Fatal(err)

spider/spider.go

@@ -1,23 +1,24 @@
 package spider
 
 import (
-	"asmr-downloader/config"
-	"asmr-downloader/log"
-	"asmr-downloader/model"
-	"asmr-downloader/utils"
 	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	browser "github.com/EDDYCJY/fake-useragent"
-	"github.com/xxjwxc/gowp/workpool"
-	"go.uber.org/zap"
 	"io"
 	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
+
+	"github.com/xxjwxc/gowp/workpool"
+	"go.uber.org/zap"
+
+	"asmr-downloader/config"
+	"asmr-downloader/log"
+	"asmr-downloader/model"
+	"asmr-downloader/utils"
 )
 
 var ctx = context.Background()
@@ -48,6 +49,18 @@ func NewASMRClient(maxWorker int, globalConfig *config.Config) *ASMRClient {
 	}
 }
 
+func HeadersInit(r *http.Request) *http.Request {
+	r.Header.Set("Referer", "https://www.asmr.one/")
+	r.Header.Set("Origin", "https://www.asmr.one")
+	r.Header.Set("Host", strings.Split(config.AsmrBaseApiUrl, "//")[1])
+	r.Header.Set("Connection", "keep-alive")
+	r.Header.Set("Sec-Fetch-Mode", "cors")
+	r.Header.Set("Sec-Fetch-Site", "cross-site")
+	r.Header.Set("Sec-Fetch-Dest", "empty")
+	r.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
+	return r
+}
+
 // Login 登入获取授权信息
 func (asmrClient *ASMRClient) Login() error {
 	payload, err := json.Marshal(map[string]string{
@@ -61,8 +74,7 @@ func (asmrClient *ASMRClient) Login() error {
 	client := utils.Client.Get().(*http.Client)
 	req, _ := http.NewRequest("POST", config.AsmrBaseApiUrl+"/api/auth/me", bytes.NewBuffer(payload))
 	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Referer", "https://www.asmr.one/")
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
+	req = HeadersInit(req)
 	resp, err := client.Do(req)
 	utils.Client.Put(client)
 	if err != nil {
@@ -85,8 +97,7 @@ func (asmrClient *ASMRClient) GetVoiceTracks(id string) ([]track, error) {
 	client := utils.Client.Get().(*http.Client)
 	req, _ := http.NewRequest("GET", config.AsmrBaseApiUrl+"/api/tracks/"+id, nil)
 	req.Header.Set("Authorization", asmrClient.Authorization)
-	req.Header.Set("Referer", "https://www.asmr.one/")
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
+	req = HeadersInit(req)
 	resp, err := client.Do(req)
 	utils.Client.Put(client)
 	if err != nil {
@@ -200,7 +211,6 @@ func (asmrClient *ASMRClient) DownloadFile(url string, dirPath string, fileName
 //	@return error
 func GetPerPageInfo(authorStr string, pageIndex int, subtitleFlag int) (*model.PageResult, error) {
 	var seed int = utils.GenerateReqSeed()
-	randomUserAgent := browser.Random()
 	//log.Printf("Random: %s\n", randomUserAgent)
 	//var reqUrl = "https://api.asmr.one/api/works?order=create_date&sort=desc&page=1&seed=" + strconv.Itoa(seed) + "&subtitle=0"
 	var reqUrl = ""
@@ -216,21 +226,8 @@ func GetPerPageInfo(authorStr string, pageIndex int, subtitleFlag int) (*model.P
 		// Handle error
 		// ignore here
 	}
-	req.Header.Set("Accept", "application/json, text/plain, */*")
-	//req.Header.Set("Accept-Encoding", "gzip, deflate, br")
-	req.Header.Set("Accept-Language", "zh,en;q=0.9,zh-TW;q=0.8,zh-CN;q=0.7,ja;q=0.6")
+	req = HeadersInit(req)
 	req.Header.Set("Authorization", authorStr)
-	req.Header.Set("Cache-Control", "no-cache")
-	req.Header.Set("Origin", "https://www.asmr.one")
-	req.Header.Set("Pragma", "no-cache")
-	req.Header.Set("Referer", "https://www.asmr.one/")
-	req.Header.Set("Sec-Ch-UA", `"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"`)
-	req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
-	req.Header.Set("Sec-Ch-Ua-Platform", "macOS")
-	req.Header.Set("Sec-Fetch-Dest", "empty")
-	req.Header.Set("Sec-Fetch-Mode", "cors")
-	req.Header.Set("Sec-Fetch-Site", "same-site")
-	req.Header.Set("User-Agent", randomUserAgent)
 
 	respond, respError := client.Do(req.WithContext(context.Background()))
 	utils.Client.Put(client)

storage/storage.go

@@ -1,13 +1,15 @@
 package storage
 
 import (
-	"asmr-downloader/config"
-	"asmr-downloader/log"
 	"database/sql"
-	"go.uber.org/zap"
 	"sync"
+
+	"go.uber.org/zap"
+	_ "modernc.org/sqlite"
+
+	"asmr-downloader/config"
+	"asmr-downloader/log"
 )
-import _ "github.com/mattn/go-sqlite3"
 
 var StoreDb *SqliteStoreEngine
 
@@ -18,7 +20,7 @@ var once sync.Once
 //	@Description: 单例存储实例
 //	@return *SqliteStoreEngine
 func GetDbInstance() *SqliteStoreEngine {
-	db, err := sql.Open("sqlite3", config.MetaDataDb)
+	db, err := sql.Open("sqlite", config.MetaDataDb)
 	if err != nil {
 		log.AsmrLog.Error("", zap.String("error", err.Error()))
 		return nil

utils/utils.go

@@ -1,13 +1,9 @@
 package utils
 
 import (
-	"asmr-downloader/log"
 	"bufio"
 	"crypto/tls"
 	"fmt"
-	"github.com/melbahja/got"
-	"github.com/xxjwxc/gowp/workpool"
-	"go.uber.org/zap"
 	"io"
 	"math/rand"
 	"net/http"
@@ -16,6 +12,13 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/melbahja/got"
+	"github.com/xxjwxc/gowp/workpool"
+	"go.uber.org/zap"
+	"golang.org/x/text/unicode/norm"
+
+	"asmr-downloader/log"
 )
 
 const FailedDownloadFileName = "failed-download.txt"
@@ -44,16 +47,22 @@ var Client = sync.Pool{
 	},
 }
 
-// FileOrDirExists  判断所给路径文件/文件夹是否存在
+// FileOrDirExists  判断所给路径文件/文件夹是否存在 after unicode normalization
 func FileOrDirExists(path string) bool {
-	_, err := os.Stat(path) //os.Stat获取文件信息
+	path = norm.NFC.String(path)
+	//path = strings.ReplaceAll(path, "/jfs/", "/ASMR/")
+	files, err := os.ReadDir(filepath.Dir(path))
 	if err != nil {
-		if os.IsExist(err) {
+		return false
+	}
+
+	for _, file := range files {
+		if norm.NFC.String(file.Name()) == norm.NFC.String(filepath.Base(path)) {
 			return true
 		}
-		return false
 	}
-	return true
+	return false
+
 }
 
 // PromotForInput 获取用户输入