feat: Add support for Moster Siren.

cmd/fake-server/main.go

@@ -24,6 +24,13 @@ var akAnnoTests = [6]string{
 }
 var akAnnoIdx = 0
 
+var sirenTests = [6]string{
+	"tests/siren/00.json",
+	"tests/siren/01.json",
+	"tests/siren/750450.json",
+}
+var sirenIdx = 0
+
 func weiboHandler(w http.ResponseWriter, r *http.Request) {
 	data, _ := ioutil.ReadFile(weiboTests[weiboIdx])
 	log.Printf("Deliverd %v\n", weiboTests[weiboIdx])
@@ -38,10 +45,18 @@ func akAnnoHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write(data)
 }
 
+func sirenHandler(w http.ResponseWriter, r *http.Request) {
+	data, _ := ioutil.ReadFile(sirenTests[sirenIdx])
+	log.Printf("Deliverd %v\n", sirenTests[sirenIdx])
+	sirenIdx = (sirenIdx + 1) % 3
+	w.Write(data)
+}
+
 func main() {
 	listenAddr := ":8088"
 	http.HandleFunc("/weibo", weiboHandler)
 	http.HandleFunc("/akanno", akAnnoHandler)
+	http.HandleFunc("/siren", sirenHandler)
 
 	log.Printf("Listen at %v\n", listenAddr)
 	log.Fatal(http.ListenAndServe(listenAddr, nil))

dr-feeder.go

@@ -14,7 +14,7 @@ import (
 )
 
 // Version is current `git describe --tags` infomation.
-var Version string = "v2.2.0"
+var Version string = "v2.3.0"
 
 func consume(ch chan common.NotifyPayload, notifiers []notifier.Notifier) {
 	for {

tests/siren/00.json

@@ -0,0 +1 @@
+{"code":0,"msg":"","data":{"list":[{"cid":"750451","title":"Loyal to the beat正式上架","cate":1,"date":"2021-04-02"},{"cid":"241306","title":"#EMPEROR","cate":8,"date":"2021-04-01"},{"cid":"992679","title":"#4 主流音乐","cate":5,"date":"2021-03-29"},{"cid":"605964","title":"#3 混音技巧","cate":5,"date":"2021-03-28"},{"cid":"336215","title":"#2 音色偏好","cate":5,"date":"2021-03-27"},{"cid":"578837","title":"歌曲上线公告","cate":1,"date":"2021-03-26"},{"cid":"863542","title":"#1 试音环节","cate":5,"date":"2021-03-26"},{"cid":"114093","title":"官方致辞","cate":5,"date":"2021-03-23"}],"end":true}}

tests/siren/01.json

@@ -0,0 +1 @@
+{"code":0,"msg":"","data":{"list":[{"cid":"750450","title":"#1","cate":7,"date":"2021-04-02"},{"cid":"750451","title":"Loyal to the beat正式上架","cate":1,"date":"2021-04-02"},{"cid":"241306","title":"#EMPEROR","cate":8,"date":"2021-04-01"},{"cid":"992679","title":"#4 主流音乐","cate":5,"date":"2021-03-29"},{"cid":"605964","title":"#3 混音技巧","cate":5,"date":"2021-03-28"},{"cid":"336215","title":"#2 音色偏好","cate":5,"date":"2021-03-27"},{"cid":"578837","title":"歌曲上线公告","cate":1,"date":"2021-03-26"},{"cid":"863542","title":"#1 试音环节","cate":5,"date":"2021-03-26"},{"cid":"114093","title":"官方致辞","cate":5,"date":"2021-03-23"}],"end":true}}

tests/siren/750450.json

@@ -0,0 +1 @@
+{"code":0,"msg":"","data":{"cid":"750450","title":"#1","cate":7,"author":"塞壬唱片","content":"<p><span style=\"font-size:0.875em\">乐章裹暮色，约定在次月的余晖里。 </span></p><p><span style=\"font-size:0.875em\">待黑夜浸入大海，幕布开启，故人自远方归来。 </span></p><p><span style=\"font-size:0.875em\">使黑白染上彩色的，是与风浪合唱的，启航时的歌声。 </span></p><p></p><div class=\"media-wrap image-wrap\"><img id=\"e670b8fc5dbf7a9931185848cdbcab3c\" title=\"4.2.2-showcase交响预告.jpg\" alt=\"4.2.2-showcase交响预告.jpg\" poster=\"\" src=\"https://web.hycdn.cn/siren/pic/20210401/e670b8fc5dbf7a9931185848cdbcab3c.jpg\"/></div><p></p>","date":"2021-04-02"}}

watcher/siren.go

@@ -0,0 +1,186 @@
+package watcher
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/antchfx/htmlquery"
+	"github.com/gocolly/colly/v2"
+	"github.com/hguandl/dr-feeder/v2/common"
+	"github.com/mitchellh/mapstructure"
+
+	bolt "go.etcd.io/bbolt"
+)
+
+type sirenWatcher struct {
+	name       string
+	latestNews sirenNewsData
+	debugURL   string
+	db         *bolt.DB
+}
+
+// NewSirenWatcher creates a Watcher of news from Monster Siren.
+func NewSirenWatcher(dbPath string, debugURL string) (Watcher, error) {
+	var err error = nil
+
+	watcher := new(sirenWatcher)
+	watcher.name = "塞壬唱片"
+	watcher.debugURL = debugURL
+
+	watcher.db, err = bolt.Open(dbPath, 0666, nil)
+	if err != nil {
+		return watcher, err
+	}
+
+	err = watcher.setup()
+	return watcher, err
+}
+
+func (watcher sirenWatcher) apiURL(newsID string) string {
+	if watcher.debugURL != "" {
+		return watcher.debugURL
+	}
+	return fmt.Sprintf("%s%s",
+		"https://monster-siren.hypergryph.com/api/news/",
+		newsID,
+	)
+}
+
+func (watcher sirenWatcher) fetchAPI(newID string) (sirenAPIPayload, error) {
+	var err error = nil
+	var data sirenAPIPayload
+
+	c := colly.NewCollector(
+		colly.UserAgent(safariUA),
+	)
+
+	c.OnError(func(_ *colly.Response, e error) {
+		err = e
+	})
+
+	c.OnResponse(func(r *colly.Response) {
+		err = json.Unmarshal(r.Body, &data)
+	})
+
+	c.Visit(watcher.apiURL(newID))
+	c.Wait()
+
+	return data, err
+}
+
+func (watcher sirenWatcher) getNewsList() ([]sirenNewsData, error) {
+	data, err := watcher.fetchAPI("")
+	if err != nil {
+		return nil, err
+	}
+
+	var content sirenListData
+	err = mapstructure.Decode(data.Data, &content)
+	if err != nil {
+		return nil, err
+	}
+
+	return content.List, nil
+}
+
+func (watcher sirenWatcher) getNews(newsID string) (sirenNewsData, error) {
+	var content sirenNewsData
+	data, err := watcher.fetchAPI(newsID)
+	if err != nil {
+		return content, err
+	}
+
+	err = mapstructure.Decode(data.Data, &content)
+	if err != nil {
+		return content, err
+	}
+
+	return content, nil
+}
+
+func (watcher *sirenWatcher) setup() error {
+	newsList, err := watcher.getNewsList()
+	if err != nil {
+		return err
+	}
+
+	watcher.storeNews(newsList)
+
+	return nil
+}
+
+func (watcher *sirenWatcher) storeNews(newsList []sirenNewsData) error {
+	err := watcher.db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte("Siren"))
+		for _, news := range newsList {
+			err = b.Put([]byte(news.Cid), []byte(news.Title))
+		}
+		return err
+	})
+
+	return err
+}
+
+func (watcher *sirenWatcher) update() bool {
+	newsList, err := watcher.getNewsList()
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+
+	ret := false
+	err = watcher.db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte("Siren"))
+		for _, news := range newsList {
+			v := b.Get([]byte(news.Cid))
+			if v == nil {
+				watcher.latestNews = news
+				ret = true
+				err = b.Put([]byte(news.Cid), []byte(news.Title))
+				break
+			}
+		}
+		return err
+	})
+
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+
+	return ret
+}
+
+func (watcher sirenWatcher) parseContent() common.NotifyPayload {
+	news := watcher.latestNews
+	fullNews, err := watcher.getNews(news.Cid)
+	texts := news.Title + "\n"
+	if err == nil {
+		doc, _ := htmlquery.Parse(
+			strings.NewReader(fullNews.Content),
+		)
+		nodes, _ := htmlquery.QueryAll(doc, "//text()")
+
+		for _, node := range nodes {
+			texts += "\n"
+			texts += strings.Trim(node.Data, " \n")
+		}
+	}
+
+	return common.NotifyPayload{
+		Title: watcher.name,
+		Body:  texts,
+		URL:   fmt.Sprintf("%s%s", "https://monster-siren.hypergryph.com/info/", news.Cid),
+	}
+}
+
+func (watcher *sirenWatcher) Produce(ch chan common.NotifyPayload) {
+	if watcher.update() {
+		log.Printf("New post from \"%s\"...\n", watcher.name)
+		ch <- watcher.parseContent()
+	} else {
+		log.Printf("Waiting for post \"%s\"...\n", watcher.name)
+	}
+}

watcher/sirenmodels.go

@@ -0,0 +1,21 @@
+package watcher
+
+type sirenAPIPayload struct {
+	Code int                    `json:"code"`
+	Msg  string                 `json:"msg"`
+	Data map[string]interface{} `json:"data"`
+}
+
+type sirenListData struct {
+	List []sirenNewsData `mapstructure:"list"`
+	End  bool            `mapstructure:"end"`
+}
+
+type sirenNewsData struct {
+	Cid     string `mapstructure:"cid"`
+	Title   string `mapstructure:"title"`
+	Cate    int    `mapstructure:"cate"`
+	Author  string `mapstructure:"author,omitempty"`
+	Content string `mapstructure:"content,omitempty"`
+	Date    string `mapstructure:"date"`
+}

watcher/watcher.go

@@ -24,6 +24,10 @@ type akAnnoConfig struct {
 	DebugURL string `mapstructure:"debug_url"`
 }
 
+type sirenConfig struct {
+	DebugURL string `mapstructure:"debug_url"`
+}
+
 func wrapDebug(debugURL string, debugMode bool) string {
 	if debugURL != "" {
 		if debugMode {
@@ -65,6 +69,12 @@ func ParseWatchers(configs []map[string]interface{}, dataPath string, debugMode
 				break
 			}
 			ret[idx], err = NewAkAnnounceWatcher(path.Join(dataPath, "akanno.db"), wrapDebug(akConfig.DebugURL, debugMode))
+		case "siren":
+			var akConfig sirenConfig
+			if err = mapstructure.Decode(config, &akConfig); err != nil {
+				break
+			}
+			ret[idx], err = NewSirenWatcher(path.Join(dataPath, "siren.db"), wrapDebug(akConfig.DebugURL, debugMode))
 		default:
 			err = fmt.Errorf("unknown watcher #%d with type \"%s\"", idx, watcherType)
 		}