CMC verification changes (#237)

go.mod

@@ -36,7 +36,7 @@ require (
 	cloud.google.com/go/compute/metadata v0.5.2 // indirect
 	cloud.google.com/go/firestore v1.17.0 // indirect
 	cloud.google.com/go/iam v1.2.2 // indirect
-	cloud.google.com/go/longrunning v0.6.2 // indirect
+	cloud.google.com/go/longrunning v0.6.3 // indirect
 	cloud.google.com/go/monitoring v1.21.2 // indirect
 	cloud.google.com/go/storage v1.47.0 // indirect
 	firebase.google.com/go/v4 v4.15.0 // indirect
@@ -184,7 +184,7 @@ require (
 	golang.org/x/text v0.20.0 // indirect
 	golang.org/x/time v0.8.0 // indirect
 	golang.org/x/tools v0.27.0 // indirect
-	google.golang.org/api v0.206.0 // indirect
+	google.golang.org/api v0.207.0 // indirect
 	google.golang.org/appengine/v2 v2.0.6 // indirect
 	google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect

go.sum

@@ -15,8 +15,8 @@ cloud.google.com/go/iam v1.2.2 h1:ozUSofHUGf/F4tCNy/mu9tHLTaxZFLOUiKzjcgWHGIA=
 cloud.google.com/go/iam v1.2.2/go.mod h1:0Ys8ccaZHdI1dEUilwzqng/6ps2YB6vRsjIe00/+6JY=
 cloud.google.com/go/logging v1.12.0 h1:ex1igYcGFd4S/RZWOCU51StlIEuey5bjqwH9ZYjHibk=
 cloud.google.com/go/logging v1.12.0/go.mod h1:wwYBt5HlYP1InnrtYI0wtwttpVU1rifnMT7RejksUAM=
-cloud.google.com/go/longrunning v0.6.2 h1:xjDfh1pQcWPEvnfjZmwjKQEcHnpz6lHjfy7Fo0MK+hc=
-cloud.google.com/go/longrunning v0.6.2/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI=
+cloud.google.com/go/longrunning v0.6.3 h1:A2q2vuyXysRcwzqDpMMLSI6mb6o39miS52UEG/Rd2ng=
+cloud.google.com/go/longrunning v0.6.3/go.mod h1:k/vIs83RN4bE3YCswdXC5PFfWVILjm3hpEUlSko4PiI=
 cloud.google.com/go/monitoring v1.21.2 h1:FChwVtClH19E7pJ+e0xUhJPGksctZNVOk2UhMmblmdU=
 cloud.google.com/go/monitoring v1.21.2/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU=
 cloud.google.com/go/storage v1.47.0 h1:ajqgt30fnOMmLfWfu1PWcb+V9Dxz6n+9WKjdNg5R4HM=
@@ -563,8 +563,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.206.0 h1:A27GClesCSheW5P2BymVHjpEeQ2XHH8DI8Srs2HI2L8=
-google.golang.org/api v0.206.0/go.mod h1:BtB8bfjTYIrai3d8UyvPmV9REGgox7coh+ZRwm0b+W8=
+google.golang.org/api v0.207.0 h1:Fvt6IGCYjf7YLcQ+GCegeAI2QSQCfIWhRkmrMPj3JRM=
+google.golang.org/api v0.207.0/go.mod h1:I53S168Yr/PNDNMi5yPnDc0/LGRZO6o7PoEbl/HY3CM=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine/v2 v2.0.6 h1:LvPZLGuchSBslPBp+LAhihBeGSiRh1myRoYK4NtuBIw=

kyc/scraper/cmc.go

@@ -21,47 +21,57 @@ func newCMCVerifier(sc webScraper, countries []string) *cmcVerifierImpl {
 }
 
 func (c *cmcVerifierImpl) VerifyPost(ctx context.Context, meta *Metadata) (username string, err error) {
-	if !validateProfileURL(meta.PostURL) {
+	if !validatePostURL(meta.PostURL) {
 		return "", errors.Wrapf(ErrInvalidURL, "invalid url: %v", meta.PostURL)
 	}
-	postURL := normalizeProfileURL(meta.PostURL)
-	oe, err := c.Scrape(ctx, postURL)
+	oe, err := c.Scrape(ctx, meta.PostURL)
 	if err != nil {
-		return "", errors.Wrapf(err, "can't scrape: %v", postURL)
+		return "", errors.Wrapf(err, "can't scrape: %v", meta.PostURL)
 	}
 
-	return "", errors.Wrapf(verifyFollowing(oe.Content, meta), "can't verify following: %v", postURL)
+	return "", errors.Wrapf(verifyPost(oe.Content), "can't verify cmc link: %v", meta.PostURL)
 }
 
-func verifyFollowing(html []byte, meta *Metadata) (err error) {
+//nolint:funlen // .
+func verifyPost(html []byte) (err error) {
 	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(html))
 	if err != nil {
 		return multierror.Append(ErrInvalidPageContent, err)
 	}
-	found := false
-	doc.Find("div.name div.nickName").EachWithBreak(func(_ int, s *goquery.Selection) bool {
-		found = strings.EqualFold(s.Find("span").Text(), meta.ExpectedPostText)
+	var (
+		foundIceCoin    = false
+		foundPairedCoin = false
+	)
+	doc.Find("#post-detail a.coin-link").EachWithBreak(func(_ int, s *goquery.Selection) bool {
+		const coinPrefix = "$"
+		txt := s.Find("span.real-text").Text()
+		txt, foundPrefix := strings.CutPrefix(txt, coinPrefix)
+		if !foundPrefix {
+			return false
+		}
+		if foundIce := strings.EqualFold(txt, iceCoinName); foundIce {
+			foundIceCoin = true
+		} else {
+			for _, coinName := range supportedCMCCoinNameList {
+				if foundPaired := strings.EqualFold(txt, coinName); foundPaired {
+					foundPairedCoin = true
+
+					break
+				}
+			}
+		}
 
-		return !found
+		return !foundIceCoin || !foundPairedCoin
 	})
-	if !found {
-		return errors.Wrapf(ErrTextNotFound, "text %v not found", meta.ExpectedPostText)
+	if !foundIceCoin || !foundPairedCoin {
+		return errors.Wrap(ErrTextNotFound, "ice coin or paired coin not found")
 	}
 
 	return nil
 }
 
-func validateProfileURL(url string) bool {
-	return strings.HasPrefix(url, "https://coinmarketcap.com/community/profile")
-}
-
-func normalizeProfileURL(url string) string {
-	res := strings.TrimSuffix(url, "/")
-	if !strings.HasSuffix(res, "following") {
-		res += "/following"
-	}
-
-	return res
+func validatePostURL(url string) bool {
+	return strings.HasPrefix(url, "https://coinmarketcap.com/community/post")
 }
 
 func (c *cmcVerifierImpl) Scrape(ctx context.Context, target string) (result *webScraperResult, err error) { //nolint:funlen // .
@@ -76,8 +86,6 @@ func (c *cmcVerifierImpl) Scrape(ctx context.Context, target string) (result *we
 				Retry: twitterRetryFn,
 				Options: func(options map[string]string) map[string]string {
 					options["country"] = country
-					options["wait_for_css"] = ".user-card__container"
-					options["auto_solve"] = trueVal
 					options["block_resources"] = falseVal
 					options["load_iframes"] = trueVal
 					options["load_shadowroots"] = falseVal

kyc/scraper/contract.go

@@ -15,6 +15,7 @@ const (
 	applicationYAMLKey = "kyc/social"
 	scraperV1Suffix    = "v1"
 	scraperV2Suffix    = "v2"
+	iceCoinName        = "ICE"
 )
 
 type (
@@ -154,6 +155,7 @@ const (
 	StrategyCMC      StrategyType = "cmc"
 )
 
+//nolint:gochecknoglobals // .
 var (
 	ErrInvalidPageContent = errors.New("invalid page content")
 	ErrTextNotFound       = errors.New("expected text not found")
@@ -165,4 +167,8 @@ var (
 	ErrScrapeFailed       = errors.New("cannot scrape target")
 	ErrInvalidToken       = errors.New("invalid token")
 	ErrTweetPrivate       = errors.New("tweet is private or does not exist")
+
+	supportedCMCCoinNameList = []string{
+		"BTC", "ETH", "USDT", "SOL", "BNB", "XRP", "DOGE", "USDC", "ADA", "TRX", "SHIB", "AVAX", "TON", "SUI", "LINK", "DOT", "BCH", "PEPE", "LEO", "XLM", "NEAR", "LTC", "APT", "UNI", "DAI", "CRO", "HBAR", "ICP", "RENDER", "BONK", "KAS", "ETC", "OM", "TAO", "POL", "WIF", "FET", "XMR", "ARB", "STX", "OKB", "VET", "FIL", "ATOM", "MNT", "AAVE", "FDUSD", "INJ", "FLOKI", "IMX", //nolint:lll // .
+	}
 )

kyc/verification_scenarios/contract.go

@@ -33,8 +33,7 @@ const (
 	CoinDistributionScenarioSignUpDoctorx    TenantScenario = "signup_doctorx"
 	CoinDistributionScenarioSignUpTokero     TenantScenario = "signup_tokero"
 
-	singUpPrefix         = "signup"
-	iceOpenNetworkHandle = "IceOpenNetwork"
+	singUpPrefix = "signup"
 )
 
 // .

kyc/verification_scenarios/verification_scenarios.go

@@ -62,7 +62,7 @@ func (r *repository) VerifyScenarios(ctx context.Context, metadata *Verification
 	}
 	switch metadata.ScenarioEnum {
 	case CoinDistributionScenarioCmc:
-		if vErr := r.VerifyCMCProfile(ctx, metadata); vErr != nil {
+		if vErr := r.VerifyCMC(ctx, metadata); vErr != nil {
 			return nil, errors.Wrapf(vErr, "haven't passed the CMC verification for userID:%v", metadata.UserID)
 		}
 	case CoinDistributionScenarioTwitter:
@@ -393,15 +393,14 @@ func (r *repository) VerifyTwitterPost(ctx context.Context, metadata *Verificati
 	return nil
 }
 
-func (r *repository) VerifyCMCProfile(ctx context.Context, metadata *VerificationMetadata) error {
+func (r *repository) VerifyCMC(ctx context.Context, metadata *VerificationMetadata) error {
 	pvm := &social.Metadata{
-		PostURL:          metadata.CMCProfileLink,
-		ExpectedPostText: iceOpenNetworkHandle,
+		PostURL: metadata.CMCProfileLink,
 	}
 	_, err := r.cmcVerifier.VerifyPost(ctx, pvm)
 	if err != nil {
 		return errors.Wrapf(ErrVerificationNotPassed,
-			"can't verify post for cmc verifier userID:%v,reason:%v", metadata.UserID, social.DetectReason(err))
+			"can't verify post for cmc verifier userID:%v,reason:%v", metadata.UserID, err.Error())
 	}
 
 	return nil