diff --git a/.github/workflows/jupyter-book.yml b/.github/workflows/jupyter-book.yml index 4e5f545..1221fe7 100644 --- a/.github/workflows/jupyter-book.yml +++ b/.github/workflows/jupyter-book.yml @@ -9,18 +9,16 @@ on: # This job installs dependencies, build the book, and pushes it to `gh-pages` jobs: deploy-book: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - # Install dependencies + # Install dependencies and build the book - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: 3.7 - - run: pip install jupyter-book==0.15.1, docutils==0.17.1 - - # Build the book - - name: Build the book - run: | + - run: | + cd docs + pip install -r requirements.txt jupyter-book build . # Push the book's HTML to github-pages @@ -28,8 +26,8 @@ jobs: uses: peaceiris/actions-gh-pages@v3 with: personal_token: ${{ secrets.PERSONAL_TOKEN }} - publish_dir: ./_build/html - exclude_assets: '_sources' external_repository: openbenchmark/openbenchmark.github.io publish_branch: main + publish_dir: ./docs/_build/html + exclude_assets: '_sources' destination_dir: BARS diff --git a/README.md b/README.md index 6abc89d..fe9619c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ If you find our benchmarks helpful in your research, please kindly cite the foll ## Contributing -We welcome any contribution that could help improve the BARS benchmark. Check the [start guide on how to contribute](./contribution.md). +We welcome any contribution that could help improve the BARS benchmark. Check the [start guide on how to contribute](./docs/contribution.md). ## Discussion diff --git a/docs/CTR/leaderboard/avazu_x4.md b/docs/CTR/leaderboard/avazu_x4.md index 8b9caa7..2ddede4 100644 --- a/docs/CTR/leaderboard/avazu_x4.md +++ b/docs/CTR/leaderboard/avazu_x4.md @@ -31,7 +31,7 @@ Please use the following evaluation settings for this benchmark: ```{admonition} Avazu_x4_002 :class: note Please use the following evaluation settings for this benchmark: -+ Dataset split: [avazu_x4](https://github.com/reczoo/Datasets/tree/main/Avazu/Avazu_x4) ++ Dataset split: [Avazu_x4](https://github.com/reczoo/Datasets/tree/main/Avazu/Avazu_x4) + Rare features filtering: min_categr_count=1 + Embedding size: 40 ``` diff --git a/docs/CTR/leaderboard/movielenslatest_x1.csv b/docs/CTR/leaderboard/movielenslatest_x1.csv index 1ca4a0c..c9bc468 100644 --- a/docs/CTR/leaderboard/movielenslatest_x1.csv +++ b/docs/CTR/leaderboard/movielenslatest_x1.csv @@ -17,7 +17,7 @@ Year,Publication,Model,Paper URL,AUC,Logloss,Running Steps,Contributor 2021,WWW'21,DCN-V2,https://arxiv.org/abs/2008.13535,0.9691,0.2147,https://github.com/reczoo/BARS/tree/main/ranking/ctr/DCNv2/DCNv2_movielenslatest_x1,"Zhu et al." 2018,KDD'18,CIN,https://arxiv.org/pdf/1803.05170.pdf,0.9674,0.2085,https://github.com/reczoo/BARS/tree/main/ranking/ctr/xDeepFM/CIN_movielenslatest_x1,"Zhu et al." 2018,KDD'18,xDeepFM,https://arxiv.org/pdf/1803.05170.pdf,0.9697,0.2409,https://github.com/reczoo/BARS/tree/main/ranking/ctr/xDeepFM/xDeepFM_movielenslatest_x1,"Zhu et al." -2019,CIKM'19,AutoInt,https://arxiv.org/abs/1810.11921,0.9662,0.2228,https://github.com/reczoo/BARS/tree/main/ranking/ctr/AutoInt/AutoInt_movielenslatest_x1,"Zhu et al." +2019,CIKM'19,AutoInt,https://arxiv.org/abs/1810.11921,0.9663,0.2228,https://github.com/reczoo/BARS/tree/main/ranking/ctr/AutoInt/AutoInt_movielenslatest_x1,"Zhu et al." 2019,CIKM'19,AutoInt+,https://arxiv.org/abs/1810.11921,0.9692,0.2148,https://github.com/reczoo/BARS/tree/main/ranking/ctr/AutoInt/AutoInt+_movielenslatest_x1,"Zhu et al." 2019,CIKM'19,FiGNN,https://arxiv.org/abs/1910.05552,0.9525,0.2559,https://github.com/reczoo/BARS/tree/main/ranking/ctr/FiGNN/FiGNN_movielenslatest_x1,"Zhu et al." 2019,RecSys'19,FiBiNET,https://arxiv.org/abs/1905.09433,0.9576,0.2518,https://github.com/reczoo/BARS/tree/main/ranking/ctr/FiBiNET/FiBiNET_movielenslatest_x1,"Zhu et al." diff --git a/docs/CTR/leaderboard/plots.py b/docs/CTR/leaderboard/plots.py index b48669b..9e28339 100644 --- a/docs/CTR/leaderboard/plots.py +++ b/docs/CTR/leaderboard/plots.py @@ -12,6 +12,7 @@ def show_table(csv_path): df["Running Steps"] = df["Running Steps"].map(lambda x: f"🔗") df = df.sort_values(by=["AUC"], ascending=False).reset_index(drop=True) df.insert(0, "Rank", range(1, len(df) + 1)) + df[['AUC', 'Logloss']] = df[['AUC', 'Logloss']].applymap('{:.4f}'.format) show(df, lengthMenu=[10, 20, 50, 100], classes="display") def show_plot(csv_path): @@ -19,12 +20,12 @@ def show_plot(csv_path): fig = make_subplots(specs=[[{"secondary_y": True}]]) fig.add_trace( go.Scatter(x=df["Model"], y=df["AUC"], name="AUC", mode='lines+markers', - line=dict(color="#0071a7"), marker=dict(size=7)), + line=dict(color="#0071a7", shape="spline", smoothing=1.3), marker=dict(size=7)), secondary_y=False, ) fig.add_trace( go.Scatter(x=df["Model"], y=df["Logloss"], name="Logloss", mode='lines+markers', - line=dict(color="#ff404e"), marker=dict(size=7)), + line=dict(color="#ff404e", shape="spline", smoothing=1.3), marker=dict(size=7)), secondary_y=True, ) diff --git a/docs/Matching/leaderboard/plots.py b/docs/Matching/leaderboard/plots.py index c2f7a07..7936c00 100644 --- a/docs/Matching/leaderboard/plots.py +++ b/docs/Matching/leaderboard/plots.py @@ -6,13 +6,14 @@ opt.columnDefs = [{"className": "dt-center", "targets": "_all"}] def show_table(csv_path): - df = pd.read_csv(csv_path) - df = df.fillna("") + df = pd.read_csv(csv_path).fillna("") df["Model"] = df.apply(lambda x: f"{x['Model']}", axis=1) del df["Paper URL"], df["Recall@50"], df["NDCG@50"], df["HitRate@50"] df["Running Steps"] = df["Running Steps"].map(lambda x: f"🔗") df = df.sort_values(by=["Recall@20"], ascending=False).reset_index(drop=True) df.insert(0, "Rank", range(1, len(df) + 1)) + df[['Recall@20', 'NDCG@20', 'HitRate@20']] = df[['Recall@20', 'NDCG@20', 'HitRate@20']].applymap( + lambda x: '{:.4f}'.format(x) if x else "") show(df, lengthMenu=[10, 20, 50, 100], classes="display") def show_plot(csv_path): @@ -20,12 +21,12 @@ def show_plot(csv_path): fig = make_subplots(specs=[[{"secondary_y": True}]]) fig.add_trace( go.Scatter(x=df["Model"], y=df["Recall@20"], name="Recall@20", mode='lines+markers', - line=dict(color="#0071a7"), marker=dict(size=7)), + line=dict(color="#0071a7", shape="spline", smoothing=1.3), marker=dict(size=7)), secondary_y=False, ) fig.add_trace( go.Scatter(x=df["Model"], y=df["NDCG@20"], name="NDCG@20", mode='lines+markers', - line=dict(color="#ff404e"), marker=dict(size=7)), + line=dict(color="#ff404e", shape="spline", smoothing=1.3), marker=dict(size=7)), secondary_y=True, ) diff --git a/docs/_toc.yml b/docs/_toc.yml index bd4a4d5..5917182 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -23,12 +23,13 @@ parts: - file: CTR/leaderboard/criteo_x1.md - file: CTR/leaderboard/criteo_x4.md - file: CTR/leaderboard/frappe_x1.md + - file: CTR/leaderboard/movielenslatest_x1.md - file: CTR/leaderboard/kkbox_x1.md + - file: CTR/leaderboard/amazonelectronics_x1.md + - file: CTR/leaderboard/taobaoad_x1.md - file: CTR/leaderboard/kuaivideo_x1.md - file: CTR/leaderboard/microvideo1.7m_x1.md - - file: CTR/leaderboard/movielenslatest_x1.md - - file: CTR/leaderboard/taobaoad_x1.md - + - caption: "CANDIDATE MATCHING" chapters: - file: Matching/index.md diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..594c3e5 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +jupyter-book==0.15.1 +docutils==0.17.1 +itables==1.6.3 +pandas==1.0.1 +plotly==5.18.0 \ No newline at end of file