lang_detect_mschloegl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "source": [
    "from langdetect import detect\n",
    "import pandas as pd\n",
    "import langid\n",
    "import os"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "source": [
    "a = [1, 2]\n",
    "d = {\"a\": 1, \"b\": 2, \"c\": [1, 2, 3]}\n",
    "print(d[\"c\"])\n",
    "for x in d[\"c\"]:\n",
    "    print(x)"
   ],
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[1, 2, 3]\n",
      "1\n",
      "2\n",
      "3\n"
     ]
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "source": [
    "def open_regest(f_name):\n",
    "    with open(f_name, \"r\") as f:\n",
    "        txt = f.read()\n",
    "        res = {\"length\": len(txt), \"file_name\": f_name.split(\"/\")[-1], \"txt\": txt}\n",
    "    return res"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "source": [
    "rootdir ='data/'\n",
    "final = []\n",
    "for subdir, dirs, files in os.walk(rootdir):\n",
    "    for file in files:\n",
    "        reg = open_regest(os.path.join(subdir, file))\n",
    "        lang = detect(reg[\"txt\"])\n",
    "        lang_id = langid.classify(reg[\"txt\"])\n",
    "        reg[\"langdetect\"] = lang\n",
    "        reg[\"lang_id\"] = lang_id\n",
    "        final.append(reg)\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "source": [
    "df = pd.DataFrame(final)"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "source": [
    "df"
   ],
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "       length                                  file_name  \\\n",
       "0         277       AUR_1366_X_27__13661027-13661027.txt   \n",
       "1         268      AUR_1522_VI_26__15220626-15220626.txt   \n",
       "2         171    AUR_1370_XI_05.1__13701105-13701105.txt   \n",
       "3         204     AUR_1553_III_20__15530320-15530320.txt   \n",
       "4         146    AUR_1431_IX_24.1__14310924-14310924.txt   \n",
       "...       ...                                        ...   \n",
       "11595      92    AUR_1488-1490.30__14890622-14890622.txt   \n",
       "11596     133      AUR_1479_IX_09__14790909-14790909.txt   \n",
       "11597     108  AUR_1387-1390_GF.3__13910415-13910415.txt   \n",
       "11598      71     AUR_1462_I_15.2__14620115-14620115.txt   \n",
       "11599     139      AUR_1436_XI_06__14361106-14361106.txt   \n",
       "\n",
       "                                                     txt langdetect  \\\n",
       "0      ﻿Revers vom Abt und Kloster zu St. Peter auf E...         de   \n",
       "1      ﻿Benigna Reichenberger, Witwe nach Virgil Reic...         de   \n",
       "2      ﻿Hans Trauner (Trawner) bezeugt, dass er von E...         de   \n",
       "3      ﻿Simon Zopf und seine Hausfrau Brigitta verkau...         de   \n",
       "4      ﻿Quittbrief über Sold des Leonhard Sweithard (...         de   \n",
       "...                                                  ...        ...   \n",
       "11595  ﻿Quittung des Georg Schneeweis (Schneweis) übe...         de   \n",
       "11596  ﻿Dienstrevers des Rupert Truchseß (drugsess), ...         de   \n",
       "11597  ﻿Schreiben des Bischof von Alet, päpstlicher K...         de   \n",
       "11598  ﻿Papst Pius II. bestätigt die Wahl des Eb Burk...         de   \n",
       "11599  ﻿Andre Plab resigniert den Zehent auf einer Sc...         de   \n",
       "\n",
       "                         lang_id  \n",
       "0       (de, -877.7149858474731)  \n",
       "1       (de, -1102.709720134735)  \n",
       "2        (de, -618.595380783081)  \n",
       "3       (de, -769.3758821487427)  \n",
       "4       (de, -502.6080741882324)  \n",
       "...                          ...  \n",
       "11595   (de, -344.1782875061035)  \n",
       "11596  (de, -478.03086376190186)  \n",
       "11597   (de, -310.8915982246399)  \n",
       "11598  (de, -246.18044567108154)  \n",
       "11599  (de, -401.29517793655396)  \n",
       "\n",
       "[11600 rows x 5 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>length</th>\n",
       "      <th>file_name</th>\n",
       "      <th>txt</th>\n",
       "      <th>langdetect</th>\n",
       "      <th>lang_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>277</td>\n",
       "      <td>AUR_1366_X_27__13661027-13661027.txt</td>\n",
       "      <td>﻿Revers vom Abt und Kloster zu St. Peter auf E...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -877.7149858474731)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>268</td>\n",
       "      <td>AUR_1522_VI_26__15220626-15220626.txt</td>\n",
       "      <td>﻿Benigna Reichenberger, Witwe nach Virgil Reic...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -1102.709720134735)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>171</td>\n",
       "      <td>AUR_1370_XI_05.1__13701105-13701105.txt</td>\n",
       "      <td>﻿Hans Trauner (Trawner) bezeugt, dass er von E...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -618.595380783081)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>204</td>\n",
       "      <td>AUR_1553_III_20__15530320-15530320.txt</td>\n",
       "      <td>﻿Simon Zopf und seine Hausfrau Brigitta verkau...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -769.3758821487427)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>146</td>\n",
       "      <td>AUR_1431_IX_24.1__14310924-14310924.txt</td>\n",
       "      <td>﻿Quittbrief über Sold des Leonhard Sweithard (...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -502.6080741882324)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11595</th>\n",
       "      <td>92</td>\n",
       "      <td>AUR_1488-1490.30__14890622-14890622.txt</td>\n",
       "      <td>﻿Quittung des Georg Schneeweis (Schneweis) übe...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -344.1782875061035)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11596</th>\n",
       "      <td>133</td>\n",
       "      <td>AUR_1479_IX_09__14790909-14790909.txt</td>\n",
       "      <td>﻿Dienstrevers des Rupert Truchseß (drugsess), ...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -478.03086376190186)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11597</th>\n",
       "      <td>108</td>\n",
       "      <td>AUR_1387-1390_GF.3__13910415-13910415.txt</td>\n",
       "      <td>﻿Schreiben des Bischof von Alet, päpstlicher K...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -310.8915982246399)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11598</th>\n",
       "      <td>71</td>\n",
       "      <td>AUR_1462_I_15.2__14620115-14620115.txt</td>\n",
       "      <td>﻿Papst Pius II. bestätigt die Wahl des Eb Burk...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -246.18044567108154)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11599</th>\n",
       "      <td>139</td>\n",
       "      <td>AUR_1436_XI_06__14361106-14361106.txt</td>\n",
       "      <td>﻿Andre Plab resigniert den Zehent auf einer Sc...</td>\n",
       "      <td>de</td>\n",
       "      <td>(de, -401.29517793655396)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11600 rows × 5 columns</p>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "source": [
    "df.to_csv(\"language_regesten_8-21.csv\")"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "source": [],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.9.5",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.5 64-bit ('mon-ner-P6_0KPNV-py3.9': poetry)"
  },
  "interpreter": {
   "hash": "f9016cd046cc9741956e0d5730b6a834802915467a404575811866851ec558a2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}