chore: 🔥 update to new Allo Indexer Data API

Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:3
+FROM mcr.microsoft.com/devcontainers/python:3.11
 
 # Install Quarto
 RUN curl -sL $(curl https://quarto.org/docs/download/_download.json | grep -oP "(?<=\"download_url\":\s\")https.*${ARCH}\.deb") --output /tmp/quarto.deb \

README.md

@@ -14,7 +14,7 @@ Open source, serverless, and local-first data hub for Gitcoin Grants data to imp
 
 ## 📖 Overview
 
-The repository contains code and artifacts to help process Gitcoin Grants data from the [Allo Indexer Data API](https://indexer-grants-stack.gitcoin.co/data/). It is an instance of [Datadex](https://github.com/davidgasquez/datadex) allowing you and everyone else to:
+The repository contains code and artifacts to help process Gitcoin Grants data from the [Allo Indexer Data API](https://indexer-production.fly.dev/data/). It is an instance of [Datadex](https://github.com/davidgasquez/datadex) allowing you and everyone else to:
 
 - Add new data sources to the portal, collaborate on better models (ala Dune) or submit an interesting analysis.
 - All in a permissionless way. Don't ask, fork it and improve the models, add a new source or update any script.

ggdp/__init__.py

@@ -1,9 +1,8 @@
 import os
 
 from dagster import Definitions, load_assets_from_modules
-from dagster_duckdb_pandas import DuckDBPandasIOManager
-
 from dagster_dbt import dbt_cli_resource, load_assets_from_dbt_project
+from dagster_duckdb_pandas import DuckDBPandasIOManager
 
 from . import assets
 
@@ -18,7 +17,9 @@
 
 resources = {
     "dbt": dbt_resource,
-    "io_manager": DuckDBPandasIOManager(database="data/dbt.duckdb"),
+    "io_manager": DuckDBPandasIOManager(
+        database="data/dbt.duckdb",
+    ),
 }
 
 defs = Definitions(assets=[*dbt_assets, *all_assets], resources=resources)

ggdp/assets.py

@@ -2,7 +2,7 @@
 from dagster import asset
 from fsspec.implementations.http import HTTPFileSystem
 
-ALLO_INDEXER_URL = "https://indexer-grants-stack.gitcoin.co/data"
+ALLO_INDEXER_URL = "https://indexer-production.fly.dev/data"
 
 
 def chain_file_aggregator(json_name):
@@ -12,22 +12,33 @@ def chain_file_aggregator(json_name):
     df = pd.DataFrame()
     for path in paths:
         chain_id = int(path.split("/")[-1])
-        df_chain = pd.read_json(f"{path}/{json_name}")
-        df_chain["chainId"] = chain_id
+        try:
+            df_chain = pd.read_json(f"{path}/{json_name}")
+            df_chain["chainId"] = chain_id
+        except Exception as e:
+            print(f"Error reading {path}/{json_name}")
+            print(f"Error: {e}")
+            df_chain = pd.DataFrame()
+            continue
         df = pd.concat([df, df_chain])
     return df
 
 
 def round_file_aggregator(json_name):
     fs = HTTPFileSystem(simple_links=True)
-    paths = fs.ls("https://indexer-grants-stack.gitcoin.co/data/")
+    paths = fs.ls(ALLO_INDEXER_URL)
     paths = [path["name"] for path in paths if path["name"].split("/")[-1].isdigit()]
 
     df = pd.DataFrame()
 
     for path in paths:
         chain_id = int(path.split("/")[-1])
-        chain_rounds = fs.ls(f"{path}/rounds/")
+        try:
+            chain_rounds = fs.ls(f"{path}/rounds/")
+        except Exception as e:
+            print(f"Error reading {path}/rounds/")
+            print(f"Error: {e}")
+            continue
         chain_rounds = [
             round["name"]
             for round in chain_rounds
@@ -53,10 +64,11 @@ def round_file_aggregator(json_name):
 def raw_passport_scores() -> pd.DataFrame:
     file_url = f"{ALLO_INDEXER_URL}/passport_scores.json"
     df = pd.read_json(file_url)
-    df = df.drop(columns=["error"])
+    df = df.drop(columns=["error", "stamp_scores"])
     df["last_score_timestamp"] = pd.to_datetime(
         df["last_score_timestamp"], errors="coerce"
     )
+
     return df[df["address"].str.startswith("0x")]