final_run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "\n",
    "from modules.run import load_config\n",
    "from modules.run import Trainer, Metrics\n",
    "from modules.data import DataManager\n",
    "from modules.models import pretrained_cnn, pretrained_cnn_multichannel\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score, accuracy_score\n",
    "\n",
    "## Testing imports\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_experiment_from_config(config_file, country):\n",
    "\n",
    "    config = load_config(config_file)\n",
    "    data_manager = DataManager(config)\n",
    "\n",
    "    class_weight = None\n",
    "    train_generator = None\n",
    "    val_generator = None\n",
    "\n",
    "    if country == 'kenya':\n",
    "        train_generator, val_generator, dataframe = data_manager.generate_kenya()\n",
    "        class_weight = data_manager.class_weight(\"kenya\")\n",
    "    elif country == 'peru':\n",
    "        train_generator, val_generator, dataframe = data_manager.generate_peru()\n",
    "        class_weight = class_weight=[1.64, 1, 2]\n",
    "    \n",
    "    convnet = pretrained_cnn_multichannel(config, image_size=config[\"image_size\"], n_channels=config[\"n_channels\"])\n",
    "#     return convnet, val_generator\n",
    "    val_steps = config[\"sample\"][\"size\"] * (config[\"validation_split\"]) // config[\"batch_size\"] + 1\n",
    "\n",
    "    labels = None\n",
    "    if config['mask'] is not None:\n",
    "        epochs = 0\n",
    "        labels = []\n",
    "        for data, label in val_generator:\n",
    "            if epochs >= val_steps:\n",
    "                break\n",
    "            labels.extend(np.argmax(label, axis=1))\n",
    "            epochs += 1\n",
    "        labels = np.array(labels)\n",
    "    trainer = Trainer(config)\n",
    "    metrics_callback = Metrics(val_generator, trainer.tensorboard_dir, labels, val_steps)\n",
    "    trainer.callbacks.append(metrics_callback)\n",
    "\n",
    "    convnet.compile(loss=trainer.loss, optimizer=trainer.optimizer, metrics=config[\"weighted_metrics\"])\n",
    "\n",
    "    convnet.fit_generator(\n",
    "        train_generator, \n",
    "        config[\"sample\"][\"size\"] * (1 - config[\"validation_split\"]) // config[\"batch_size\"] + 1,\n",
    "        epochs=config[\"n_epochs\"],\n",
    "        callbacks=trainer.callbacks,\n",
    "        validation_data=val_generator, \n",
    "        validation_steps=val_steps,\n",
    "        class_weight=class_weight,\n",
    "        use_multiprocessing=True\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Declouded dataframe length: 152505\n"
     ]
    }
   ],
   "source": [
    "run_experiment_from_config(\"cls_final_xception_kenya_rgb\", \"kenya\")\n",
    "run_experiment_from_config(\"cls_final_xception_peru_rgb\", \"peru\")\n",
    "run_experiment_from_config(\"cls_final_xception_kenya_masked\", \"kenya\")\n",
    "run_experiment_from_config(\"cls_final_xception_kenya_masked-inverted\", \"kenya\")\n",
    "run_experiment_from_config(\"cls_final_xception_kenya_two_with_mask\", \"kenya\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def setup_cross_domain_from_config(config_file, country):\n",
    "\n",
    "    config = load_config(config_file)\n",
    "    data_manager = DataManager(config)\n",
    "\n",
    "    class_weight = None\n",
    "    train_generator = None\n",
    "    val_generator = None\n",
    "\n",
    "    if country == 'kenya':\n",
    "        train_generator, val_generator, dataframe = data_manager.generate_kenya()\n",
    "        class_weight = data_manager.class_weight(\"kenya\")\n",
    "    elif country == 'peru':\n",
    "        train_generator, val_generator, dataframe = data_manager.generate_peru()\n",
    "        class_weight = class_weight=[1.64, 1, 2]\n",
    "    \n",
    "    convnet = pretrained_cnn_multichannel(config, image_size=config[\"image_size\"], n_channels=config[\"n_channels\"])\n",
    "    return convnet, val_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Declouded dataframe length: 97281\n",
      "Found 10800 validated image filenames belonging to 3 classes.\n",
      "Found 1200 validated image filenames belonging to 3 classes.\n",
      "0.455\n"
     ]
    }
   ],
   "source": [
    "model, val_gen = setup_cross_domain_from_config(\"cls_final_xception_peru_balanced\", \"peru\")\n",
    "model.load_weights(\"./data/cls_final_xception_kenya_rgb/checkpoints/weights.02-0.46.hdf5\")\n",
    "val_predict = np.argmax(model.predict(val_gen), axis=1)\n",
    "print(accuracy_score(val_gen.classes, val_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Declouded dataframe length: 152505\n",
      "Found 10800 validated image filenames belonging to 3 classes.\n",
      "Found 1200 validated image filenames belonging to 3 classes.\n",
      "0.5975\n"
     ]
    }
   ],
   "source": [
    "model, val_gen = setup_cross_domain_from_config(\"cls_final_xception_kenya_balanced\", \"kenya\")\n",
    "model.load_weights(\"./data/cls_final_xception_peru_rgb/checkpoints/weights.02-0.71.hdf5\")\n",
    "val_predict = np.argmax(model.predict(val_gen), axis=1)\n",
    "print(accuracy_score(val_gen.classes, val_predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1, 2]), array([391, 423, 386]))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(val_gen.classes, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 15000 validated image filenames belonging to 3 classes.\n",
      "Found 15000 validated image filenames belonging to 3 classes.\n"
     ]
    }
   ],
   "source": [
    "# pred = np.argmax(val_predict, axis=1)\n",
    "val_steps = 10\n",
    "labels = None\n",
    "epochs = 0\n",
    "labels = []\n",
    "pred = []\n",
    "for data, label in val:\n",
    "    if epochs >= val_steps:\n",
    "        break\n",
    "    labels.extend(np.argmax(label, axis=1))\n",
    "    pred.extend(np.argmax(model.predict(data), axis=1))\n",
    "    epochs += 1\n",
    "labels = np.array(labels)\n",
    "pred = np.array(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 2 1 1 1 1 1 1 1 2 1 1 0 1 1 1 0 0 1 2 1 2 2 2 2 1 1 1 1 1 1 2 1 1 1 2\n",
      " 2 1 1 1 1 1 1 1 2 2 2 2 1 2 1 1 1 1 2 1 2 1 1 1 2 2 1 1 1 2 2 1 1 2 2 1 1\n",
      " 1 0 1 1 1 2 2 1 1 1 2 1 1 0 1 1 1 1 1 1 2 2 1 1 2 2 2 1 1 1 2 2 1 2 1 1 1\n",
      " 1 1 1 1 1 1 1 2 1 1 1 1 2 1 2 1 1 2 1 2 2 2 1 1 2 1 2 1 2 1 1 2 1 2 0 1 1\n",
      " 2 2 1 2 2 1 1 1 1 1 2 1 1 2 2 1 1 2 2 2 2 0 1 2 1 2 1 1 1 2 1 1 1 1 1 1 2\n",
      " 1 1 1 2 1 1 2 1 1 2 1 1 1 1 1 1 1 1 1 1 1 1 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1\n",
      " 2 1 1 2 1 2 1 1 1 1 1 2 1 1 1 1 1 1 2 1 2 2 1 1 1 1 2 1 1 2 2 1 1 2 1 1 2\n",
      " 1 1 2 2 2 1 2 1 1 1 1 1 2 1 1 1 2 2 1 1 2 1 1 2 1 1 1 1 2 1 2 2 1 1 1 1 1\n",
      " 0 1 2 1 1 1 1 1 1 1 1 2 2 1 1 1 1 1 2 1 1 1 0 1]\n",
      "[1 1 1 1 1 1 1 1 2 1 2 2 1 1 1 1 2 0 0 1 2 1 2 2 1 2 1 1 1 2 1 1 2 1 1 1 2\n",
      " 1 1 2 1 1 1 1 1 2 1 2 2 1 2 1 1 2 1 1 1 2 1 1 1 2 2 1 1 1 2 2 1 1 2 2 1 1\n",
      " 1 0 1 1 1 2 2 1 1 1 2 1 1 0 1 1 1 1 1 1 1 2 1 1 2 1 2 1 2 1 2 1 1 2 1 1 1\n",
      " 1 1 1 1 1 1 2 2 1 1 1 1 2 1 1 2 1 2 1 2 2 1 1 1 2 1 2 1 1 1 1 2 1 2 1 1 1\n",
      " 2 2 1 2 1 1 2 2 2 1 2 1 1 1 2 1 1 2 2 2 1 0 1 1 1 2 1 1 1 1 1 2 1 1 1 1 2\n",
      " 1 1 1 2 2 1 2 1 1 1 1 1 1 1 1 1 1 1 1 1 2 1 1 1 1 1 1 1 1 2 1 2 1 1 1 1 1\n",
      " 1 1 1 2 1 1 1 1 1 2 1 1 1 1 1 1 2 1 2 2 2 1 1 1 1 1 2 2 1 2 2 1 1 2 1 1 2\n",
      " 1 0 1 2 2 1 2 1 1 1 1 1 2 1 2 1 1 1 2 1 2 2 1 2 1 1 1 1 2 1 2 1 1 1 2 1 1\n",
      " 1 0 1 1 1 1 2 1 1 1 1 2 1 1 2 1 1 1 2 1 1 1 0 2]\n",
      "0.809375\n",
      "0.7432604228942901\n"
     ]
    }
   ],
   "source": [
    "print(pred)\n",
    "print(labels)\n",
    "print(accuracy_score(labels, pred))\n",
    "print(f1_score(labels, pred, average='macro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 2 1 2 2 2 1 1 1 2 1 2 1 2 1 1 1 2 2 2 2 2 2 1 1 0 2 1 1 1 1 1 2 2 1 1 1\n",
      " 1 1 2 1 2 2 2 2 1 2 1 1 1 1 2 1 2 1 2 1 2 1 1 1 2 1 2]\n",
      "[1 1 1 1 1 1 1 1 1 1 2 2 1 1 1 1 1 0 1 1 2 1 2 1 2 2 1 1 1 1 1 1 2 1 1 1 1\n",
      " 1 1 0 1 1 1 1 1 2 2 2 2 1 2 1 1 1 1 1 1 2 1 1 1 2 2 1]\n",
      "0.3282828282828283\n",
      "0.53125\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score, accuracy_score\n",
    "print(labels)\n",
    "print(np.argmax(val_predict, axis=1))\n",
    "_val_f1 = f1_score(labels, np.argmax(val_predict, axis=1), average='macro')\n",
    "print(_val_f1)\n",
    "print(accuracy_score(labels, np.argmax(val_predict, axis=1)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}