added missing cfg.TelegramScraper check

masa-finance · Aug 1, 2024 · e03d9fc · e03d9fc
1 parent 36cf87d
commit e03d9fc
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 13 deletions.
diff --git a/pkg/scrapers/twitter/tweets.go b/pkg/scrapers/twitter/tweets.go
@@ -85,9 +85,17 @@ func ScrapeTweetsForSentiment(query string, count int, model string) (string, st
 
 	// Perform the search with the specified query and count
 	for tweetResult := range scraper.SearchTweets(context.Background(), query, count) {
-		tweet := TweetResult{
-			Tweet: &tweetResult.Tweet,
-			Error: tweetResult.Error,
+		var tweet TweetResult
+		if tweetResult.Error != nil {
+			tweet = TweetResult{
+				Tweet: nil,
+				Error: tweetResult.Error,
+			}
+		} else {
+			tweet = TweetResult{
+				Tweet: tweetResult.Tweet,
+				Error: nil,
+			}
 		}
 		tweets = append(tweets, &tweet)
 	}
@@ -96,7 +104,7 @@ func ScrapeTweetsForSentiment(query string, count int, model string) (string, st
 	twitterScraperTweets := make([]*twitterscraper.TweetResult, len(tweets))
 	for i, tweet := range tweets {
 		twitterScraperTweets[i] = &twitterscraper.TweetResult{
-			Tweet: *tweet.Tweet,
+			Tweet: tweet.Tweet,
 			Error: tweet.Error,
 		}
 	}
@@ -129,9 +137,17 @@ func ScrapeTweetsByQuery(query string, count int) ([]*TweetResult, error) {
 
 	// Perform the search with the specified query and count
 	for tweetResult := range scraper.SearchTweets(context.Background(), query, count) {
-		tweet := TweetResult{
-			Tweet: &tweetResult.Tweet,
-			Error: tweetResult.Error,
+		var tweet TweetResult
+		if tweetResult.Error != nil {
+			tweet = TweetResult{
+				Tweet: nil,
+				Error: tweetResult.Error,
+			}
+		} else {
+			tweet = TweetResult{
+				Tweet: tweetResult.Tweet,
+				Error: nil,
+			}
 		}
 		tweets = append(tweets, &tweet)
 	}

diff --git a/pkg/tests/scrape_test.go b/pkg/tests/scrape_test.go
@@ -162,7 +162,7 @@ func TestScrapeTweetsWithSentimentByQuery(t *testing.T) {
 	twitterScraperTweets := make([]*twitterscraper.TweetResult, len(deserializedTweets))
 	for i, tweet := range deserializedTweets {
 		twitterScraperTweets[i] = &twitterscraper.TweetResult{
-			Tweet: *tweet,
+			Tweet: tweet,
 			Error: nil,
 		}
 	}

diff --git a/pkg/workers/handler.go b/pkg/workers/handler.go
@@ -187,7 +187,7 @@ func (a *Worker) HandleWork(ctx actor.Context, m *messages.Work, node *masa.Orac
 			return
 		}
 		cfg := config.GetInstance()
-		if cfg.TwitterScraper || cfg.DiscordScraper || cfg.TwitterScraper || cfg.WebScraper {
+		if cfg.TelegramScraper || cfg.DiscordScraper || cfg.TwitterScraper || cfg.WebScraper {
 			ctx.Respond(&messages.Response{RequestId: workData["request_id"], Value: string(jsn)})
 		}
 		for _, pid := range getPeers(node) {

diff --git a/pkg/workers/workers.go b/pkg/workers/workers.go
@@ -272,10 +272,6 @@ func SendWork(node *masa.OracleNode, m *pubsub2.Message) {
 	for _, p := range peers {
 		for _, addr := range p.Multiaddrs {
 			ipAddr, _ := addr.ValueForProtocol(multiaddr.P_IP4)
-
-			// @TODO add a limiter if a node 429s add them to a list and don't call them for n number of minutes
-			// re: Add feature timeout timestamps to nodeData and helper methods to do timeout checks
-
 			if (p.PeerId.String() != node.Host.ID().String()) &&
 				p.IsStaked &&
 				node.NodeTracker.GetNodeData(p.PeerId.String()).CanDoWork(pubsub.WorkerCategory(message.Type)) {
@@ -321,6 +317,8 @@ func SendWork(node *masa.OracleNode, m *pubsub2.Message) {
 					}
 					responseCollector <- msg
 					n++
+					// @TODO add a limiter if a node 429s add them to a list and don't call them for n number of minutes
+					// re: Add feature timeout timestamps to nodeData and helper methods to do timeout checks
 					// @note need to handle if we have thousands of workers this could take a very long time to complete
 					// here we cap n
 					if n == len(peers) || n == 100 {