ben_dev_4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3326, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-1-b97d0e4af443>\", line 2, in <module>\n",
      "    import tensorflow as tf\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/__init__.py\", line 98, in <module>\n",
      "    from tensorflow_core import *\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/__init__.py\", line 40, in <module>\n",
      "    from tensorflow.python.tools import module_util as _module_util\n",
      "  File \"<frozen importlib._bootstrap>\", line 983, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 959, in _find_and_load_unlocked\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/__init__.py\", line 50, in __getattr__\n",
      "    module = self._load()\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/__init__.py\", line 44, in _load\n",
      "    module = _importlib.import_module(self.__name__)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/importlib/__init__.py\", line 127, in import_module\n",
      "    return _bootstrap._gcd_import(name[level:], package, level)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/__init__.py\", line 52, in <module>\n",
      "    from tensorflow.core.framework.graph_pb2 import *\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/core/framework/graph_pb2.py\", line 7, in <module>\n",
      "    from google.protobuf import descriptor as _descriptor\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/google/protobuf/__init__.py\", line 37, in <module>\n",
      "    __import__('pkg_resources').declare_namespace(__name__)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 3250, in <module>\n",
      "    @_call_aside\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 3234, in _call_aside\n",
      "    f(*args, **kwargs)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 3263, in _initialize_master_working_set\n",
      "    working_set = WorkingSet._build_master()\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 574, in _build_master\n",
      "    ws = cls()\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 567, in __init__\n",
      "    self.add_entry(entry)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 623, in add_entry\n",
      "    for dist in find_distributions(entry, True):\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 2065, in find_on_path\n",
      "    for dist in factory(fullpath):\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 2135, in distributions_from_metadata\n",
      "    root, entry, metadata, precedence=DEVELOP_DIST,\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 2592, in from_location\n",
      "    py_version=py_version, platform=platform, **kw\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 2994, in _reload_version\n",
      "    md_version = self._get_version()\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 2772, in _get_version\n",
      "    version = _version_from_file(lines)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 2556, in _version_from_file\n",
      "    line = next(iter(version_lines), '')\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 2767, in _get_metadata\n",
      "    for line in self.get_metadata_lines(name):\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 1432, in get_metadata_lines\n",
      "    return yield_lines(self.get_metadata(name))\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 1420, in get_metadata\n",
      "    value = self._get(path)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/pkg_resources/__init__.py\", line 1617, in _get\n",
      "    return stream.read()\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2040, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1101, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 319, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 353, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/inspect.py\", line 1502, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/inspect.py\", line 1460, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/inspect.py\", line 733, in getmodule\n",
      "    if ismodule(module) and hasattr(module, '__file__'):\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/__init__.py\", line 50, in __getattr__\n",
      "    module = self._load()\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/__init__.py\", line 44, in _load\n",
      "    module = _importlib.import_module(self.__name__)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/importlib/__init__.py\", line 127, in import_module\n",
      "    return _bootstrap._gcd_import(name[level:], package, level)\n",
      "  File \"<frozen importlib._bootstrap>\", line 1006, in _gcd_import\n",
      "  File \"<frozen importlib._bootstrap>\", line 983, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 953, in _find_and_load_unlocked\n",
      "  File \"<frozen importlib._bootstrap>\", line 219, in _call_with_frames_removed\n",
      "  File \"<frozen importlib._bootstrap>\", line 1006, in _gcd_import\n",
      "  File \"<frozen importlib._bootstrap>\", line 983, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 967, in _find_and_load_unlocked\n",
      "  File \"<frozen importlib._bootstrap>\", line 677, in _load_unlocked\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 728, in exec_module\n",
      "  File \"<frozen importlib._bootstrap>\", line 219, in _call_with_frames_removed\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/__init__.py\", line 42, in <module>\n",
      "    from . _api.v2 import audio\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/_api/v2/audio/__init__.py\", line 10, in <module>\n",
      "    from tensorflow.python.ops.gen_audio_ops import decode_wav\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/ops/gen_audio_ops.py\", line 11, in <module>\n",
      "    from tensorflow.python.eager import context as _context\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/eager/context.py\", line 32, in <module>\n",
      "    from tensorflow.python import tf2\n",
      "  File \"<frozen importlib._bootstrap>\", line 1032, in _handle_fromlist\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/__init__.py\", line 50, in __getattr__\n",
      "    module = self._load()\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow/__init__.py\", line 44, in _load\n",
      "    module = _importlib.import_module(self.__name__)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/importlib/__init__.py\", line 127, in import_module\n",
      "    return _bootstrap._gcd_import(name[level:], package, level)\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/__init__.py\", line 63, in <module>\n",
      "    from tensorflow.python.framework.framework_lib import *  # pylint: disable=redefined-builtin\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/framework/framework_lib.py\", line 25, in <module>\n",
      "    from tensorflow.python.framework.ops import Graph\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/framework/ops.py\", line 41, in <module>\n",
      "    from tensorflow.python.eager import core\n",
      "  File \"/home/BenChoi/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/eager/core.py\", line 22, in <module>\n",
      "    from tensorflow.python.framework import errors\n",
      "  File \"<frozen importlib._bootstrap>\", line 983, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 967, in _find_and_load_unlocked\n",
      "  File \"<frozen importlib._bootstrap>\", line 677, in _load_unlocked\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 724, in exec_module\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 818, in get_code\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 917, in get_data\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "\n",
    "from modules.run import load_config\n",
    "from modules.run import Trainer\n",
    "from modules.data import DataManager, processing\n",
    "from modules.models import pretrained_cnn, pretrained_cnn_multichannel\n",
    "\n",
    "## Testing imports\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config=load_config(\"cls_cropped_all\")\n",
    "data_manager = DataManager(config)\n",
    "train_generator, val_generator, dataframe = data_manager.generate_kenya()\n",
    "# convnet = pretrained_cnn_multichannel(config, image_size=config[\"image_size\"], n_channels=config[\"n_channels\"])\n",
    "# trainer = Trainer(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>98087</th>\n",
       "      <td>98440_419864.jpg</td>\n",
       "      <td>minor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129880</th>\n",
       "      <td>130263_108532.jpg</td>\n",
       "      <td>two-track</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50229</th>\n",
       "      <td>50570_231962.jpg</td>\n",
       "      <td>minor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50042</th>\n",
       "      <td>50383_231008.jpg</td>\n",
       "      <td>minor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148516</th>\n",
       "      <td>148901_192826.jpg</td>\n",
       "      <td>two-track</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4909</th>\n",
       "      <td>4912_20148.jpg</td>\n",
       "      <td>minor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163225</th>\n",
       "      <td>163610_297043.jpg</td>\n",
       "      <td>two-track</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71041</th>\n",
       "      <td>71392_387218.jpg</td>\n",
       "      <td>minor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40875</th>\n",
       "      <td>41216_177898.jpg</td>\n",
       "      <td>minor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67345</th>\n",
       "      <td>67696_382700.jpg</td>\n",
       "      <td>minor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 filename      class\n",
       "98087    98440_419864.jpg      minor\n",
       "129880  130263_108532.jpg  two-track\n",
       "50229    50570_231962.jpg      minor\n",
       "50042    50383_231008.jpg      minor\n",
       "148516  148901_192826.jpg  two-track\n",
       "...                   ...        ...\n",
       "4909       4912_20148.jpg      minor\n",
       "163225  163610_297043.jpg  two-track\n",
       "71041    71392_387218.jpg      minor\n",
       "40875    41216_177898.jpg      minor\n",
       "67345    67696_382700.jpg      minor\n",
       "\n",
       "[150000 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                filename\n",
      "0       68690_383909.jpg\n",
      "1      149273_194688.jpg\n",
      "2      162784_257777.jpg\n",
      "3      158284_236778.jpg\n",
      "4      102449_425384.jpg\n",
      "...                  ...\n",
      "19266  156999_225867.jpg\n",
      "19267  162555_256729.jpg\n",
      "19268   83519_401216.jpg\n",
      "19269   117816_57879.jpg\n",
      "19270   63402_372278.jpg\n",
      "\n",
      "[19271 rows x 1 columns]\n",
      "                 filename      class\n",
      "98087    98440_419864.jpg      minor\n",
      "129880  130263_108532.jpg  two-track\n",
      "50229    50570_231962.jpg      minor\n",
      "148516  148901_192826.jpg  two-track\n",
      "80776    81128_398494.jpg      minor\n",
      "...                   ...        ...\n",
      "14866     14913_26183.jpg      minor\n",
      "92881    93233_412708.jpg      minor\n",
      "163225  163610_297043.jpg  two-track\n",
      "40875    41216_177898.jpg      minor\n",
      "67345    67696_382700.jpg      minor\n",
      "\n",
      "[133169 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('data/kenya/cloudy.txt', sep=\" \", header=None)\n",
    "data.columns = [\"filename\"]\n",
    "data[\"filename\"] = data.filename.str.slice(16)\n",
    "data[\"filename\"] = data.filename.str.slice(0, -4) + \".jpg\"\n",
    "\n",
    "print(data)\n",
    "\n",
    "dataframe = dataframe[~dataframe['filename'].isin(data.filename)]\n",
    "\n",
    "\n",
    "print(dataframe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image, ImageFile\n",
    "ImageFile.LOAD_TRUNCATED_IMAGES = True\n",
    "import os\n",
    "from modules.data import util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed 0 file descriptors.\n",
      "Processed 1000 file descriptors.\n",
      "Processed 2000 file descriptors.\n",
      "Processed 3000 file descriptors.\n",
      "Processed 4000 file descriptors.\n",
      "Processed 5000 file descriptors.\n",
      "Processed 6000 file descriptors.\n",
      "Processed 7000 file descriptors.\n",
      "Processed 8000 file descriptors.\n",
      "Processed 9000 file descriptors.\n",
      "Processed 10000 file descriptors.\n",
      "Processed 11000 file descriptors.\n",
      "Processed 12000 file descriptors.\n",
      "Processed 13000 file descriptors.\n",
      "Processed 14000 file descriptors.\n",
      "Processed 15000 file descriptors.\n",
      "Processed 16000 file descriptors.\n",
      "Processed 17000 file descriptors.\n",
      "Processed 18000 file descriptors.\n",
      "Processed 19000 file descriptors.\n",
      "Processed 20000 file descriptors.\n",
      "Processed 21000 file descriptors.\n",
      "Processed 22000 file descriptors.\n",
      "Processed 23000 file descriptors.\n",
      "Processed 24000 file descriptors.\n",
      "Processed 25000 file descriptors.\n",
      "Processed 26000 file descriptors.\n",
      "Processed 27000 file descriptors.\n",
      "Processed 28000 file descriptors.\n",
      "Processed 29000 file descriptors.\n",
      "Processed 30000 file descriptors.\n",
      "Processed 31000 file descriptors.\n",
      "Processed 32000 file descriptors.\n",
      "Processed 33000 file descriptors.\n",
      "Processed 34000 file descriptors.\n",
      "Processed 35000 file descriptors.\n",
      "Processed 36000 file descriptors.\n",
      "Processed 37000 file descriptors.\n",
      "Processed 38000 file descriptors.\n",
      "Processed 39000 file descriptors.\n",
      "Processed 40000 file descriptors.\n",
      "Processed 41000 file descriptors.\n",
      "Processed 42000 file descriptors.\n",
      "Processed 43000 file descriptors.\n",
      "Processed 44000 file descriptors.\n",
      "Processed 45000 file descriptors.\n",
      "Processed 46000 file descriptors.\n",
      "Processed 47000 file descriptors.\n",
      "Processed 48000 file descriptors.\n",
      "Processed 49000 file descriptors.\n",
      "Processed 50000 file descriptors.\n",
      "Processed 51000 file descriptors.\n",
      "Processed 52000 file descriptors.\n",
      "Processed 53000 file descriptors.\n",
      "Processed 54000 file descriptors.\n",
      "Processed 55000 file descriptors.\n",
      "Processed 56000 file descriptors.\n",
      "Processed 57000 file descriptors.\n",
      "Processed 58000 file descriptors.\n",
      "Processed 59000 file descriptors.\n",
      "Processed 60000 file descriptors.\n",
      "Processed 61000 file descriptors.\n",
      "Processed 62000 file descriptors.\n",
      "Processed 63000 file descriptors.\n",
      "Processed 64000 file descriptors.\n",
      "Processed 65000 file descriptors.\n",
      "Processed 66000 file descriptors.\n",
      "Processed 67000 file descriptors.\n",
      "Processed 68000 file descriptors.\n",
      "Processed 69000 file descriptors.\n",
      "Processed 70000 file descriptors.\n",
      "Processed 71000 file descriptors.\n",
      "Processed 72000 file descriptors.\n",
      "Processed 73000 file descriptors.\n",
      "Processed 74000 file descriptors.\n",
      "Processed 75000 file descriptors.\n",
      "Processed 76000 file descriptors.\n",
      "Processed 77000 file descriptors.\n",
      "Processed 78000 file descriptors.\n",
      "Processed 79000 file descriptors.\n",
      "Processed 80000 file descriptors.\n",
      "Processed 81000 file descriptors.\n",
      "Processed 82000 file descriptors.\n",
      "Processed 83000 file descriptors.\n",
      "Processed 84000 file descriptors.\n",
      "Processed 85000 file descriptors.\n",
      "Processed 86000 file descriptors.\n",
      "Processed 87000 file descriptors.\n",
      "Processed 88000 file descriptors.\n",
      "Processed 89000 file descriptors.\n",
      "Processed 90000 file descriptors.\n",
      "Processed 91000 file descriptors.\n",
      "Processed 92000 file descriptors.\n",
      "Processed 93000 file descriptors.\n",
      "Processed 94000 file descriptors.\n",
      "Processed 95000 file descriptors.\n",
      "Processed 96000 file descriptors.\n",
      "Processed 97000 file descriptors.\n",
      "Processed 98000 file descriptors.\n",
      "Processed 99000 file descriptors.\n",
      "Processed 100000 file descriptors.\n",
      "Processed 101000 file descriptors.\n",
      "Processed 102000 file descriptors.\n",
      "Processed 103000 file descriptors.\n",
      "Processed 104000 file descriptors.\n",
      "Processed 105000 file descriptors.\n",
      "Processed 106000 file descriptors.\n",
      "Processed 107000 file descriptors.\n",
      "Processed 108000 file descriptors.\n",
      "Processed 109000 file descriptors.\n"
     ]
    }
   ],
   "source": [
    "i_path = \"/home/BenChoi/roadtype/peru_road_images\"\n",
    "country = \"peru\"\n",
    "D = 224\n",
    "subsampling=0\n",
    "quality=90\n",
    "\n",
    "o_path = os.path.join(util.root(), country, f\"{D}\", \"cropped\")\n",
    "\n",
    "if not os.path.exists(o_path):\n",
    "    os.makedirs(o_path)\n",
    "\n",
    "greater = 1000 // 2 + D // 2\n",
    "smaller = 1000 // 2 - D // 2\n",
    "\n",
    "errors = []\n",
    "\n",
    "for i, fname in enumerate(os.listdir(i_path)):\n",
    "    if i % 1000 == 0:\n",
    "        print(f\"Processed {i} file descriptors.\")\n",
    "    if i < 81001:\n",
    "        continue\n",
    "    if os.path.isfile(os.path.join(i_path, fname)):\n",
    "        try:\n",
    "            im = Image.open(os.path.join(i_path, fname))\n",
    "        except:\n",
    "            errors.append(fname)\n",
    "            continue\n",
    "\n",
    "        name, _ = os.path.splitext(fname)\n",
    "        head = name.split(\"_\")[-1]\n",
    "        first = name.split(\"_\")[-2]\n",
    "        \n",
    "        im = im.crop((smaller, smaller, greater, greater)).convert(\"RGB\")\n",
    "\n",
    "        im.save(os.path.join(o_path, f\"{first}_{head}.jpg\"), \"JPEG\", subsampling=subsampling, quality=quality)\n",
    "\n",
    "\n",
    "with open(os.path.join(o_path, \"errors.txt\"), \"w\") as o_err:\n",
    "    for e in errors:\n",
    "        o_err.write(f\"{e}\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}