NicoCommentCleaner.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q3orYxJibJzj"
      },
      "outputs": [],
      "source": [
        "import glob\n",
        "\n",
        "zips = glob.glob(\"/content/*.zip\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from zipfile import ZipFile\n",
        "from json import loads\n",
        "from tqdm import tqdm\n",
        "from random import sample\n",
        "\n",
        "data = []\n",
        "\n",
        "for zip_path in tqdm(sample(zips, 4)):\n",
        "  zip = ZipFile(zip_path)\n",
        "\n",
        "  file_list = zip.namelist()\n",
        "\n",
        "  for file in file_list:\n",
        "    if (not file.endswith(\".jsonl\")):\n",
        "      continue\n",
        "\n",
        "    with zip.open(file) as f:\n",
        "      raw = f.read().decode(\"utf8\")\n",
        "\n",
        "    for line in raw.split(\"\\n\")[:-1]:\n",
        "      j = loads(line)\n",
        "\n",
        "      data.append((j[\"date\"], j[\"content\"]))\n",
        "\n",
        "  zip.close()\n",
        "\n",
        "len(data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0wl7aEakcW9W",
        "outputId": "199d83ef-f73a-4794-a360-4a4696429d80"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4/4 [02:01<00:00, 30.26s/it]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "15632920"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_sampled = sample(data, int(1e6))"
      ],
      "metadata": {
        "id": "9ocWdnAbggyo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"/content/df.csv\", \"w\", encoding=\"utf-8\") as f:\n",
        "  strs = []\n",
        "  for d in data_sampled:\n",
        "    this_str = d[0]\n",
        "    this_str += \",\"\n",
        "    this_str += d[1].replace(\",\", \" \").replace(\"\\n\", \" \")\n",
        "    strs.append(this_str)\n",
        "\n",
        "  f.write(\"\\n\".join(strs))"
      ],
      "metadata": {
        "id": "44N8-QCOgwjh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "oylWYFlriTBl"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}