scratch/01_generate_table_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "alternate-stanford",
   "metadata": {},
   "source": [
    "# Test Creating Fake Allometry Data\n",
    "\n",
    "Please note that this is just an experiment and is not expected to work in the future or be used directly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "toxic-toronto",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.290582Z",
     "start_time": "2021-02-25T22:55:50.167300Z"
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "import string\n",
    "from collections import defaultdict, deque\n",
    "from datetime import datetime, timedelta\n",
    "from functools import partial\n",
    "from random import choice, randint, random, randrange, seed\n",
    "from os.path import basename, splitext\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from PIL import Image, ImageDraw, ImageFilter, ImageFont"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "complimentary-helen",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.296192Z",
     "start_time": "2021-02-25T22:55:50.292873Z"
    }
   },
   "outputs": [],
   "source": [
    "DATA_DIR = Path('..') / 'data'\n",
    "ITIS_DIR = DATA_DIR / 'itis'\n",
    "CLEAN_DIR = DATA_DIR / 'clean'\n",
    "DIRTY_DIR = DATA_DIR / 'dirty'\n",
    "TEXT_DIR = DATA_DIR / 'text'\n",
    "\n",
    "FONTS_DIR = Path('..') / 'fonts'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "hybrid-sheffield",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.308058Z",
     "start_time": "2021-02-25T22:55:50.297346Z"
    }
   },
   "outputs": [],
   "source": [
    "SEED = 981\n",
    "seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "three-album",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.317774Z",
     "start_time": "2021-02-25T22:55:50.309358Z"
    }
   },
   "outputs": [],
   "source": [
    "GUTTER = ' ' * 6  # Spaces between columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "proof-quarter",
   "metadata": {},
   "source": [
    "Get a list of all of the fonts we will be using."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "superb-rider",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.325311Z",
     "start_time": "2021-02-25T22:55:50.318916Z"
    }
   },
   "outputs": [],
   "source": [
    "FONTS = FONTS_DIR.glob('*/*.ttf')\n",
    "FONTS = sorted([str(f) for f in FONTS])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "declared-certificate",
   "metadata": {},
   "source": [
    "## Columns for the fake data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tired-glance",
   "metadata": {},
   "source": [
    "Get some genus and species names to use as fake IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "rubber-afghanistan",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.333005Z",
     "start_time": "2021-02-25T22:55:50.326980Z"
    }
   },
   "outputs": [],
   "source": [
    "with open(ITIS_DIR / 'species.txt') as in_file:\n",
    "    SPECIES = [ln.strip().upper() for ln in in_file.readlines()]\n",
    "\n",
    "with open(ITIS_DIR / 'genera.txt') as in_file:\n",
    "    GENERA = [ln.strip().upper() + ' SP' for ln in in_file.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "least-jonathan",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.337850Z",
     "start_time": "2021-02-25T22:55:50.334319Z"
    }
   },
   "outputs": [],
   "source": [
    "def fake_id(width=20):\n",
    "    id_ = choice(SPECIES) if random() < 0.8 else choice(GENERA)\n",
    "    return id_[:width].ljust(width)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "confident-cream",
   "metadata": {},
   "source": [
    "The UF ID is just a string of letters and digits between 5 and 6 characters long."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "certified-karma",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.343991Z",
     "start_time": "2021-02-25T22:55:50.339359Z"
    }
   },
   "outputs": [],
   "source": [
    "UF_CHARS = string.ascii_uppercase + string.digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "confused-intake",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.350404Z",
     "start_time": "2021-02-25T22:55:50.345089Z"
    }
   },
   "outputs": [],
   "source": [
    "def fake_uf(width=6):\n",
    "    length = randint(5, width)\n",
    "    uf = [choice(UF_CHARS) for i in range(length + 1)]\n",
    "    uf = ''.join(uf)\n",
    "    return uf[:width].ljust(width)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "stable-offering",
   "metadata": {},
   "source": [
    "The numbers are all similar with NA being represented as a lone decimal point. 1 in 10 numbers will be NA. The numbers are returned as fixed length stings.\n",
    "\n",
    "after = The number of digits after the decimal point.\n",
    "\n",
    "neg = Does the number allow negative values?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "native-raising",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.356855Z",
     "start_time": "2021-02-25T22:55:50.351729Z"
    }
   },
   "outputs": [],
   "source": [
    "HIGH = 9.999999\n",
    "SUB = HIGH / 2.0\n",
    "BEFORE_NEG = 3  # sign + before digits + point\n",
    "BEFORE_POS = 2  # before digits + point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "seventh-increase",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.363691Z",
     "start_time": "2021-02-25T22:55:50.357961Z"
    }
   },
   "outputs": [],
   "source": [
    "def fake_float(after=4, neg=False):\n",
    "    num = random() * HIGH\n",
    "    num = num if not neg else (num - SUB)\n",
    "\n",
    "    if neg:\n",
    "        formatter = f'{{: {BEFORE_NEG + after}.{after}f}}'\n",
    "        na = f'  .{\" \" * after}'\n",
    "    else:\n",
    "        formatter = f'{{:{BEFORE_POS + after}.{after}f}}'\n",
    "        na = f' .{\" \" * after}'\n",
    "\n",
    "    formatted = formatter.format(num)\n",
    "\n",
    "    # How missing numbers are reported\n",
    "    if random() < 0.1:\n",
    "        formatted = na\n",
    "\n",
    "    return formatted"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "random-better",
   "metadata": {},
   "source": [
    "## Generate fake page text"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "inside-designer",
   "metadata": {},
   "source": [
    "Generate a fake table.\n",
    "\n",
    "**Note that there will be other table formats.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "empty-athletics",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.369805Z",
     "start_time": "2021-02-25T22:55:50.364678Z"
    }
   },
   "outputs": [],
   "source": [
    "MIN_ROWS = 50\n",
    "MAX_ROWS = 76"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "sunset-commander",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.376757Z",
     "start_time": "2021-02-25T22:55:50.370740Z"
    }
   },
   "outputs": [],
   "source": [
    "def fake_data():\n",
    "    count = randint(MIN_ROWS, MAX_ROWS) + 1\n",
    "    rows = []\n",
    "    for i in range(1, count):\n",
    "        rows.append({\n",
    "            'OBS': str(i).rjust(3),\n",
    "            'ID': fake_id(),\n",
    "            'UF': fake_uf(),\n",
    "            'TFW': fake_float(),\n",
    "            'SW': fake_float(),\n",
    "            'WDV': fake_float(),\n",
    "            'TBW': fake_float(after=6),\n",
    "            'USW': fake_float(),\n",
    "            'PFW': fake_float(after=2),\n",
    "            'OCW': fake_float(after=2),\n",
    "            'AGW': fake_float(after=2),\n",
    "            'ASTLOG': fake_float(after=5, neg=True),\n",
    "            'BIOLOG': fake_float(neg=True),\n",
    "        })\n",
    "    return rows"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "jewish-inside",
   "metadata": {},
   "source": [
    "Add the rows first so we can easily calculate the page width in characters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "voluntary-stake",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.385434Z",
     "start_time": "2021-02-25T22:55:50.377755Z"
    }
   },
   "outputs": [],
   "source": [
    "def fake_lines(rows):\n",
    "    lines = []\n",
    "    for row in rows:\n",
    "        lines.append((GUTTER.join(v for v in row.values())))\n",
    "    return lines"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "collect-canvas",
   "metadata": {},
   "source": [
    "Generate fake page header."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "subjective-collaboration",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.392352Z",
     "start_time": "2021-02-25T22:55:50.386658Z"
    }
   },
   "outputs": [],
   "source": [
    "def fake_header(line_len):\n",
    "    header = ' '.join(list('STATISTICAL ANANYSIS SYSTEM'))\n",
    "    header = header.center(line_len)\n",
    "    page_no = randint(1, 9)\n",
    "    header += '  ' + str(page_no)\n",
    "    return header, page_no"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "amber-lotus",
   "metadata": {},
   "source": [
    "Generate a fake date line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "decent-treaty",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.399155Z",
     "start_time": "2021-02-25T22:55:50.393392Z"
    }
   },
   "outputs": [],
   "source": [
    "def fake_date(line_len):\n",
    "    date = datetime(randint(1970, 1990), 1, 1)\n",
    "    date += timedelta(days=randint(0, 365))\n",
    "    date += timedelta(seconds=randint(0, 24 * 60 * 60))\n",
    "    date = date.strftime('%H:%M %A, %B %m, %Y')\n",
    "    date_line = date.rjust(line_len + 3) + '\\n'\n",
    "    return date_line, date"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "simplified-course",
   "metadata": {},
   "source": [
    "Generate fake column headers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "legal-smith",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.405732Z",
     "start_time": "2021-02-25T22:55:50.400074Z"
    }
   },
   "outputs": [],
   "source": [
    "COLUMNS = {\n",
    "    'OBS': {'width': 3, 'just': 'left'},\n",
    "    'ID': {'width': 20, 'just': 'center'},\n",
    "    'UF': {'width': 6, 'just': 'left'},\n",
    "    'TFW': {'width': 6, 'just': 'right'},\n",
    "    'SW': {'width': 6, 'just': 'left'},\n",
    "    'WDV': {'width': 6, 'just': 'left'},\n",
    "    'TBW': {'width': 8, 'just': 'left'},\n",
    "    'USW': {'width': 6, 'just': 'left'},\n",
    "    'PFW': {'width': 4, 'just': 'left'},\n",
    "    'OCW': {'width': 4, 'just': 'left'},\n",
    "    'AGW': {'width': 4, 'just': 'left'},\n",
    "    'ASTLOG': {'width': 8, 'just': 'right'},\n",
    "    'BIOLOG': {'width': 7, 'just': 'right'},\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "governmental-christopher",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.412233Z",
     "start_time": "2021-02-25T22:55:50.406812Z"
    }
   },
   "outputs": [],
   "source": [
    "def fake_column_headers():\n",
    "    headers = []\n",
    "    for name, data in COLUMNS.items():\n",
    "        width = data['width']\n",
    "        just = data['just']\n",
    "\n",
    "        if just == 'center':\n",
    "            header = name.center(width)\n",
    "        elif just == 'right':\n",
    "            header = name.rjust(width)\n",
    "        else:\n",
    "            header = name.ljust(width)\n",
    "\n",
    "        headers.append(header[:width])\n",
    "\n",
    "    column_headers = GUTTER.join(h for h in headers) + '\\n'\n",
    "    return column_headers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "blank-insurance",
   "metadata": {},
   "source": [
    "Generate the page."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "paperback-society",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.421314Z",
     "start_time": "2021-02-25T22:55:50.413358Z"
    }
   },
   "outputs": [],
   "source": [
    "def fake_page():\n",
    "    rows = fake_data()\n",
    "    lines = fake_lines(rows)\n",
    "    line_len = len(lines[0])\n",
    "    page_header, page_no = fake_header(line_len)\n",
    "    date_line, date = fake_date(line_len)\n",
    "    column_headers = fake_column_headers()\n",
    "\n",
    "    page = [page_header, date_line, column_headers] + lines\n",
    "    page = page = '\\n'.join(page) + '\\n'\n",
    "\n",
    "    data = {\n",
    "        'rows': rows,\n",
    "        'page_no': page_no,\n",
    "        'date': date,\n",
    "        'page': page,\n",
    "    }\n",
    "\n",
    "    return page, data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "liberal-efficiency",
   "metadata": {},
   "source": [
    "Randomly translate the image left/right and up/down."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "occupied-atlantic",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.427776Z",
     "start_time": "2021-02-25T22:55:50.422330Z"
    }
   },
   "outputs": [],
   "source": [
    "def translate(image, text, up_down=None, left_right=None):\n",
    "    x = int((WIDTH - size[0]) / 4)\n",
    "    y = int((HEIGHT - size[1]) / 4)\n",
    "\n",
    "    if random() < 0.5:\n",
    "        x = -x\n",
    "    if random() < 0.5:\n",
    "        y = -y\n",
    "\n",
    "    image.translate()\n",
    "    \n",
    "    return image"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "revolutionary-hawaii",
   "metadata": {},
   "source": [
    "## Generate clean fake page image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "affiliated-marker",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.434156Z",
     "start_time": "2021-02-25T22:55:50.428859Z"
    }
   },
   "outputs": [],
   "source": [
    "WIDTH = 4500\n",
    "HEIGHT = 3440"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "modified-developer",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.440694Z",
     "start_time": "2021-02-25T22:55:50.435113Z"
    }
   },
   "outputs": [],
   "source": [
    "def y_image(page, clean_params):\n",
    "    font = clean_params['font']\n",
    "    font_size = clean_params['font_size']\n",
    "\n",
    "    font = ImageFont.truetype(font=font, size=font_size)\n",
    "    \n",
    "    size = font.getsize_multiline(page)\n",
    "    clean_params['text_size'] = list(size)\n",
    "\n",
    "    image = Image.new(mode='L', size=(WIDTH, HEIGHT), color='white')\n",
    "\n",
    "    draw = ImageDraw.Draw(image)\n",
    "    \n",
    "    dx = WIDTH - size[0]\n",
    "    dy = HEIGHT - size[1]\n",
    "\n",
    "    x = (dx // 2) + (randint(0, dx // 4) * choice([1, -1]))\n",
    "    y = (dy // 2) + (randint(0, dy // 4) * choice([1, -1]))\n",
    "\n",
    "    draw.text((x, y), page, font=font, fill='black')\n",
    "\n",
    "    return image, size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "exposed-effects",
   "metadata": {},
   "source": [
    "## Dirty the image"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "exotic-phenomenon",
   "metadata": {},
   "source": [
    "A function to change random values in a numpy array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "traditional-flashing",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.447129Z",
     "start_time": "2021-02-25T22:55:50.441724Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_snow(data, dirty_params, low=128, high=255):\n",
    "    snow_fract = dirty_params['snow_fract']\n",
    "    dirty_params['snow_low'] = low\n",
    "    dirty_params['snow_high'] = high\n",
    "\n",
    "    shape = data.shape\n",
    "    data = data.flatten()\n",
    "    how_many = int(data.size * snow_fract)\n",
    "    mask = np.random.choice(data.size, how_many)\n",
    "    data[mask] = np.random.randint(low, high)\n",
    "    data = data.reshape(shape)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "underlying-technical",
   "metadata": {},
   "source": [
    "Filter the image to make the snow more destructive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "social-edinburgh",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.456405Z",
     "start_time": "2021-02-25T22:55:50.450305Z"
    }
   },
   "outputs": [],
   "source": [
    "def filter_image(image, dirty_params):\n",
    "    image = image.filter(ImageFilter.UnsharpMask())\n",
    "\n",
    "    image_filter = dirty_params.get('filter')\n",
    "\n",
    "    if image_filter == 'max':\n",
    "        image = image.filter(ImageFilter.MaxFilter())\n",
    "    elif image_filter == 'median':\n",
    "        image = image.filter(ImageFilter.MedianFilter())\n",
    "    elif image_filter == 'mode':\n",
    "        image = image.filter(ImageFilter.ModeFilter())\n",
    "\n",
    "    return image"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "internal-liberty",
   "metadata": {},
   "source": [
    "Rotate the image slightly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "sealed-mirror",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.463376Z",
     "start_time": "2021-02-25T22:55:50.458145Z"
    }
   },
   "outputs": [],
   "source": [
    "def rotate_image(image, dirty_params):\n",
    "    if (theta := dirty_params.get('theta')) is None:\n",
    "        if random() < 0.5:\n",
    "            theta = randrange(0.0, 2.0, 1.0)\n",
    "        else:\n",
    "            theta = randrange(358.0, 360.0, 1.0)\n",
    "        dirty_params['theta'] = theta\n",
    "\n",
    "    return image.rotate(theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "rolled-awareness",
   "metadata": {},
   "source": [
    "Dirty the image randomly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "municipal-quebec",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.469542Z",
     "start_time": "2021-02-25T22:55:50.464357Z"
    }
   },
   "outputs": [],
   "source": [
    "def x_image(image, dirty_params):\n",
    "    dirty = np.array(image)\n",
    "    dirty = add_snow(dirty, dirty_params)\n",
    "    dirty = Image.fromarray(dirty)\n",
    "\n",
    "    dirty = filter_image(dirty, dirty_params)\n",
    "\n",
    "    dirty = rotate_image(dirty, dirty_params)\n",
    "\n",
    "    dirty = dirty.convert('RGB')\n",
    "\n",
    "    return dirty"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "entertaining-netherlands",
   "metadata": {},
   "source": [
    "## Put it all together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "excited-adolescent",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.476130Z",
     "start_time": "2021-02-25T22:55:50.470570Z"
    }
   },
   "outputs": [],
   "source": [
    "def build_page(base_name, clean_params, dirty_params):\n",
    "    name = base_name + '.jpg'\n",
    "\n",
    "    page, data = fake_page()\n",
    "\n",
    "    clean, size = y_image(page, clean_params)\n",
    "    clean.save(CLEAN_DIR / name, 'JPEG')\n",
    "\n",
    "    dirty = x_image(clean, dirty_params)\n",
    "    dirty.save(DIRTY_DIR / name, 'JPEG')\n",
    "\n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "documented-nurse",
   "metadata": {},
   "source": [
    "Fonts sometimes require special treatment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "accepting-honor",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:55:50.482706Z",
     "start_time": "2021-02-25T22:55:50.477087Z"
    }
   },
   "outputs": [],
   "source": [
    "FONT_PARAMS = {\n",
    "    'B612Mono-Bold': {},\n",
    "    'B612Mono-BoldItalic': {},\n",
    "    'B612Mono-Italic': {'filter': 'median'},\n",
    "    'B612Mono-Regular': {'filter': 'median'},\n",
    "    'CourierPrime-Bold': {},\n",
    "    'CourierPrime-BoldItalic': {},\n",
    "    'CourierPrime-Italic': {'filter': 'median'},\n",
    "    'CourierPrime-Regular': {'filter': 'median'},\n",
    "    'CutiveMono-Regular': {'filter': 'median'},\n",
    "    'RobotoMono-Italic-VariableFont_wght': {'filter': 'median', 'size': 32},\n",
    "    'RobotoMono-VariableFont_wght': {'filter': 'median', 'size': 32},\n",
    "    'SyneMono-Regular': {'filter': 'median'},\n",
    "    'VT323-Regular': {'filter': 'median'},\n",
    "    'XanhMono-Italic': {'filter': 'median'},\n",
    "    'XanhMono-Regular': {'filter': 'median'},\n",
    "    'Kingthings_Trypewriter_2': {'filter': 'median'},\n",
    "    'OCRB_Medium': {'filter': 'median'},\n",
    "    'OCRB_Regular': {},\n",
    "    'OcrB2': {},\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "broken-pulse",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-25T22:59:57.512174Z",
     "start_time": "2021-02-25T22:55:50.483609Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 1800130.47it/s]\n"
     ]
    }
   ],
   "source": [
    "def create_pages():\n",
    "    font_file = choice(FONTS)\n",
    "\n",
    "    font_name = splitext(basename(font_file))[0]\n",
    "    base_name = f'table_{str(i).zfill(3)}'\n",
    "\n",
    "    params = FONT_PARAMS.get(font_name, {})\n",
    "    clean_params = {\n",
    "        'font': font_file,\n",
    "        'font_size': params.get('size', 36),\n",
    "    }\n",
    "    dirty_params = {\n",
    "        'snow_fract': params.get('snow', 0.05),\n",
    "        'image_filter': params.get('filter', 'max'),\n",
    "    }\n",
    "\n",
    "    data = build_page(\n",
    "        base_name,\n",
    "        clean_params=clean_params,\n",
    "        dirty_params=dirty_params)\n",
    "\n",
    "    data['clean'] = clean_params\n",
    "    data['dirty'] = dirty_params\n",
    "\n",
    "    with open(TEXT_DIR / (base_name + '.json'), 'w') as json_file:\n",
    "        json.dump(data, json_file, indent='  ')\n",
    "\n",
    "\n",
    "# for i in tqdm(range(100)):\n",
    "#     create_pages()\n",
    "#     pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "humanitarian-healing",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0+"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}