Merge pull request #131 from nanos/no-bot

Do not backfill users that have opted out

find_posts.py

@@ -112,7 +112,20 @@ def add_post_with_context(post, server, access_token, seen_urls, seen_hosts):
 
     return False
 
+def user_has_opted_out(user):
+    if 'note' in user and isinstance(user['note'], str) and (' nobot' in user['note'].lower() or '/tags/nobot' in user['note'].lower()):
+        return True
+    if 'indexable' in user and not user['indexable']:
+        return True
+    if 'discoverable' in user and not user['discoverable']:
+        return True
+    return False
+
+
 def get_user_posts(user, known_followings, server, seen_hosts):
+    if user_has_opted_out(user):
+        logger.debug(f"User {user} has opted out of backfilling")
+        return None
     parsed_url = parse_user_url(user['url'])
 
     if parsed_url == None: