2-Weakly-Supervised-Subtyping/Evaluation-Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Base / Native\n",
    "import math\n",
    "import os\n",
    "from os.path import join\n",
    "import pickle\n",
    "import re\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# Numerical / Array\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy\n",
    "from scipy import interp\n",
    "from scipy.stats import ttest_ind\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllll}\n",
      "\\toprule\n",
      "{} &       BRCA (25\\%) &      BRCA (100\\%) &       Lung (25\\%) &      Lung (100\\%) &     Kidney (25\\%) &    Kidney (100\\%) \\\\\n",
      "\\midrule\n",
      "MIL                         &  0.673 +/- 0.112 &  0.778 +/- 0.091 &  0.857 +/- 0.059 &  0.892 +/- 0.042 &  0.904 +/- 0.055 &  0.959 +/- 0.015 \\\\\n",
      "CLAM                        &  0.796 +/- 0.063 &  0.858 +/- 0.067 &  0.852 +/- 0.034 &  0.928 +/- 0.021 &  0.957 +/- 0.012 &  0.973 +/- 0.017 \\\\\n",
      "DeepAttnMISL                &  0.685 +/- 0.110 &  0.784 +/- 0.061 &  0.663 +/- 0.077 &  0.778 +/- 0.045 &  0.904 +/- 0.024 &  0.943 +/- 0.016 \\\\\n",
      "DS-MIL                      &  0.760 +/- 0.088 &  0.838 +/- 0.074 &  0.787 +/- 0.073 &  0.920 +/- 0.024 &  0.949 +/- 0.028 &  0.971 +/- 0.016 \\\\\n",
      "DGCN                        &  0.727 +/- 0.076 &  0.840 +/- 0.073 &  0.748 +/- 0.050 &  0.831 +/- 0.034 &  0.923 +/- 0.012 &  0.957 +/- 0.012 \\\\\n",
      "HIPT (ViT-256-F + ViT-4096) &  0.821 +/- 0.069 &  0.874 +/- 0.060 &  0.923 +/- 0.020 &  0.952 +/- 0.021 &  0.974 +/- 0.012 &  0.980 +/- 0.013 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BRCA (25%)</th>\n",
       "      <th>BRCA (100%)</th>\n",
       "      <th>Lung (25%)</th>\n",
       "      <th>Lung (100%)</th>\n",
       "      <th>Kidney (25%)</th>\n",
       "      <th>Kidney (100%)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MIL</th>\n",
       "      <td>0.673 +/- 0.112</td>\n",
       "      <td>0.778 +/- 0.091</td>\n",
       "      <td>0.857 +/- 0.059</td>\n",
       "      <td>0.892 +/- 0.042</td>\n",
       "      <td>0.904 +/- 0.055</td>\n",
       "      <td>0.959 +/- 0.015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLAM</th>\n",
       "      <td>0.796 +/- 0.063</td>\n",
       "      <td>0.858 +/- 0.067</td>\n",
       "      <td>0.852 +/- 0.034</td>\n",
       "      <td>0.928 +/- 0.021</td>\n",
       "      <td>0.957 +/- 0.012</td>\n",
       "      <td>0.973 +/- 0.017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DeepAttnMISL</th>\n",
       "      <td>0.685 +/- 0.110</td>\n",
       "      <td>0.784 +/- 0.061</td>\n",
       "      <td>0.663 +/- 0.077</td>\n",
       "      <td>0.778 +/- 0.045</td>\n",
       "      <td>0.904 +/- 0.024</td>\n",
       "      <td>0.943 +/- 0.016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DS-MIL</th>\n",
       "      <td>0.760 +/- 0.088</td>\n",
       "      <td>0.838 +/- 0.074</td>\n",
       "      <td>0.787 +/- 0.073</td>\n",
       "      <td>0.920 +/- 0.024</td>\n",
       "      <td>0.949 +/- 0.028</td>\n",
       "      <td>0.971 +/- 0.016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DGCN</th>\n",
       "      <td>0.727 +/- 0.076</td>\n",
       "      <td>0.840 +/- 0.073</td>\n",
       "      <td>0.748 +/- 0.050</td>\n",
       "      <td>0.831 +/- 0.034</td>\n",
       "      <td>0.923 +/- 0.012</td>\n",
       "      <td>0.957 +/- 0.012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HIPT (ViT-256-F + ViT-4096)</th>\n",
       "      <td>0.821 +/- 0.069</td>\n",
       "      <td>0.874 +/- 0.060</td>\n",
       "      <td>0.923 +/- 0.020</td>\n",
       "      <td>0.952 +/- 0.021</td>\n",
       "      <td>0.974 +/- 0.012</td>\n",
       "      <td>0.980 +/- 0.013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  BRCA (25%)      BRCA (100%)  \\\n",
       "MIL                          0.673 +/- 0.112  0.778 +/- 0.091   \n",
       "CLAM                         0.796 +/- 0.063  0.858 +/- 0.067   \n",
       "DeepAttnMISL                 0.685 +/- 0.110  0.784 +/- 0.061   \n",
       "DS-MIL                       0.760 +/- 0.088  0.838 +/- 0.074   \n",
       "DGCN                         0.727 +/- 0.076  0.840 +/- 0.073   \n",
       "HIPT (ViT-256-F + ViT-4096)  0.821 +/- 0.069  0.874 +/- 0.060   \n",
       "\n",
       "                                  Lung (25%)      Lung (100%)  \\\n",
       "MIL                          0.857 +/- 0.059  0.892 +/- 0.042   \n",
       "CLAM                         0.852 +/- 0.034  0.928 +/- 0.021   \n",
       "DeepAttnMISL                 0.663 +/- 0.077  0.778 +/- 0.045   \n",
       "DS-MIL                       0.787 +/- 0.073  0.920 +/- 0.024   \n",
       "DGCN                         0.748 +/- 0.050  0.831 +/- 0.034   \n",
       "HIPT (ViT-256-F + ViT-4096)  0.923 +/- 0.020  0.952 +/- 0.021   \n",
       "\n",
       "                                Kidney (25%)    Kidney (100%)  \n",
       "MIL                          0.904 +/- 0.055  0.959 +/- 0.015  \n",
       "CLAM                         0.957 +/- 0.012  0.973 +/- 0.017  \n",
       "DeepAttnMISL                 0.904 +/- 0.024  0.943 +/- 0.016  \n",
       "DS-MIL                       0.949 +/- 0.028  0.971 +/- 0.016  \n",
       "DGCN                         0.923 +/- 0.012  0.957 +/- 0.012  \n",
       "HIPT (ViT-256-F + ViT-4096)  0.974 +/- 0.012  0.980 +/- 0.013  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_dir = './results_cvpr2022_class/'\n",
    "summary_df_all = []\n",
    "for task in ['tcga_brca_subtype', 'tcga_lung_subtype', 'tcga_kidney_subtype']: # os.listdir(os.path.join(results_dir))\n",
    "    props = [0.25, 1.0]\n",
    "    for prop in props:\n",
    "        models = ['mil', 'clam_sb', 'mi_fcn', 'dsmil', 'dgcn',\n",
    "                  'hipt_lgp[vit4k_xs_dino]_freeze_[None]']\n",
    "        model_names = ['MIL', 'CLAM', 'DeepAttnMISL', 'DSMIL', 'DGCN',\n",
    "                       'HIPT (SA-DINO-Freeze, SA-None)']\n",
    "        features = ['vits_tcga_pancancer_dino']\n",
    "\n",
    "        exps = []\n",
    "        for feature in features:\n",
    "            for model in models:\n",
    "                exp_name = '%s_%s_%s_%0.2f_none_s1' % (task, model, feature, prop)\n",
    "                exps.append(exp_name)\n",
    "        \n",
    "        summary_df = []\n",
    "        for exp in exps:\n",
    "            summary_path = os.path.join(results_dir, task, exp, 'summary.csv')\n",
    "            if os.path.isfile(summary_path):\n",
    "                results_df = pd.read_csv(summary_path, index_col=0)\n",
    "                exp_auc = '%0.3f +/- %0.3f' % (results_df['test_auc'].mean(), results_df['test_auc'].std())\n",
    "                summary_df.append(exp_auc)\n",
    "            else:\n",
    "                summary_df.append('-')\n",
    "        summary_df_all.append(summary_df)\n",
    "\n",
    "summary_df_all = pd.DataFrame(summary_df_all).T\n",
    "summary_df_all.columns = ['BRCA (25%)', 'BRCA (100%)', 'Lung (25%)', 'Lung (100%)',\n",
    "                          'Kidney (25%)', 'Kidney (100%)']\n",
    "model_names = [ 'MIL', 'CLAM',  'DeepAttnMISL', 'DS-MIL', 'DGCN',\n",
    "               'HIPT (ViT-256-F + ViT-4096)']\n",
    "summary_df_all.index = model_names\n",
    "print(summary_df_all.to_latex())\n",
    "summary_df_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllll}\n",
      "\\toprule\n",
      "{} & BRCA (25\\%) & BRCA (100\\%) & Lung (25\\%) & Lung (100\\%) & Kidney (25\\%) & Kidney (100\\%) \\\\\n",
      "\\midrule\n",
      "MIL                         &      0.640 &       0.830 &      0.822 &       0.831 &        0.904 &         0.953 \\\\\n",
      "CLAM                        &      0.850 &       0.943 &      0.832 &       0.936 &        0.947 &         0.969 \\\\\n",
      "DeepAttnMISL                &      0.702 &       0.817 &      0.752 &       0.800 &        0.867 &         0.941 \\\\\n",
      "DS-MIL                      &      0.778 &       0.904 &      0.760 &       0.916 &        0.937 &         0.940 \\\\\n",
      "DGCN                        &      0.698 &       0.923 &      0.673 &       0.841 &        0.915 &         0.951 \\\\\n",
      "HIPT (ViT-256-F + ViT-4096) &      0.843 &       0.902 &      0.920 &       0.942 &        0.975 &         0.977 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BRCA (25%)</th>\n",
       "      <th>BRCA (100%)</th>\n",
       "      <th>Lung (25%)</th>\n",
       "      <th>Lung (100%)</th>\n",
       "      <th>Kidney (25%)</th>\n",
       "      <th>Kidney (100%)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MIL</th>\n",
       "      <td>0.640</td>\n",
       "      <td>0.830</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.831</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLAM</th>\n",
       "      <td>0.850</td>\n",
       "      <td>0.943</td>\n",
       "      <td>0.832</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.947</td>\n",
       "      <td>0.969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DeepAttnMISL</th>\n",
       "      <td>0.702</td>\n",
       "      <td>0.817</td>\n",
       "      <td>0.752</td>\n",
       "      <td>0.800</td>\n",
       "      <td>0.867</td>\n",
       "      <td>0.941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DS-MIL</th>\n",
       "      <td>0.778</td>\n",
       "      <td>0.904</td>\n",
       "      <td>0.760</td>\n",
       "      <td>0.916</td>\n",
       "      <td>0.937</td>\n",
       "      <td>0.940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DGCN</th>\n",
       "      <td>0.698</td>\n",
       "      <td>0.923</td>\n",
       "      <td>0.673</td>\n",
       "      <td>0.841</td>\n",
       "      <td>0.915</td>\n",
       "      <td>0.951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HIPT (ViT-256-F + ViT-4096)</th>\n",
       "      <td>0.843</td>\n",
       "      <td>0.902</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.975</td>\n",
       "      <td>0.977</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            BRCA (25%) BRCA (100%) Lung (25%) Lung (100%)  \\\n",
       "MIL                              0.640       0.830      0.822       0.831   \n",
       "CLAM                             0.850       0.943      0.832       0.936   \n",
       "DeepAttnMISL                     0.702       0.817      0.752       0.800   \n",
       "DS-MIL                           0.778       0.904      0.760       0.916   \n",
       "DGCN                             0.698       0.923      0.673       0.841   \n",
       "HIPT (ViT-256-F + ViT-4096)      0.843       0.902      0.920       0.942   \n",
       "\n",
       "                            Kidney (25%) Kidney (100%)  \n",
       "MIL                                0.904         0.953  \n",
       "CLAM                               0.947         0.969  \n",
       "DeepAttnMISL                       0.867         0.941  \n",
       "DS-MIL                             0.937         0.940  \n",
       "DGCN                               0.915         0.951  \n",
       "HIPT (ViT-256-F + ViT-4096)        0.975         0.977  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_dir = './results_cvpr2022_class/'\n",
    "summary_df_all = []\n",
    "for task in ['tcga_brca_subtype', 'tcga_lung_subtype', 'tcga_kidney_subtype']: # os.listdir(os.path.join(results_dir))\n",
    "    props = [0.25, 1.0]\n",
    "    for prop in props:\n",
    "        models = ['mil', 'clam_sb', 'mi_fcn', 'dsmil', 'dgcn',\n",
    "                  'hipt_lgp[vit4k_xs_dino]_freeze_[None]']\n",
    "        model_names = ['MIL', 'CLAM', 'DeepAttnMISL', 'DSMIL', 'DGCN',\n",
    "                       'HIPT (SA-DINO-Freeze, SA-None)']\n",
    "        features = ['vits_tcga_pancancer_dino']\n",
    "\n",
    "        exps = []\n",
    "        for feature in features:\n",
    "            for model in models:\n",
    "                exp_name = '%s_%s_%s_%0.2f_none_s1' % (task, model, feature, prop)\n",
    "                exps.append(exp_name)\n",
    "        \n",
    "        summary_df = []\n",
    "        for exp in exps:\n",
    "            summary_path = os.path.join(results_dir, task, exp, 'summary.csv')\n",
    "            if os.path.isfile(summary_path):\n",
    "                results_df = pd.read_csv(summary_path, index_col=0)\n",
    "                exp_auc = '%0.3f' % (results_df['test_auc'][0])\n",
    "                summary_df.append(exp_auc)\n",
    "            else:\n",
    "                summary_df.append('-')\n",
    "        summary_df_all.append(summary_df)\n",
    "\n",
    "summary_df_all = pd.DataFrame(summary_df_all).T\n",
    "summary_df_all.columns = ['BRCA (25%)', 'BRCA (100%)', 'Lung (25%)', 'Lung (100%)',\n",
    "                          'Kidney (25%)', 'Kidney (100%)']\n",
    "model_names = [ 'MIL', 'CLAM',  'DeepAttnMISL', 'DS-MIL', 'DGCN',\n",
    "               'HIPT (ViT-256-F + ViT-4096)']\n",
    "summary_df_all.index = model_names\n",
    "print(summary_df_all.to_latex())\n",
    "summary_df_all"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllll}\n",
      "\\toprule\n",
      "{} &       BRCA (25\\%) &      BRCA (100\\%) &       Lung (25\\%) &      Lung (100\\%) &     Kidney (25\\%) &    Kidney (100\\%) \\\\\n",
      "\\midrule\n",
      "HIPT (No ViT)                &  0.784 +/- 0.061 &  0.837 +/- 0.062 &  0.835 +/- 0.050 &  0.928 +/- 0.023 &  0.955 +/- 0.016 &  0.965 +/- 0.013 \\\\\n",
      "HIPT (ViT-256 + ViT-4096)    &  0.758 +/- 0.076 &  0.823 +/- 0.071 &  0.695 +/- 0.069 &  0.786 +/- 0.096 &                - &  0.956 +/- 0.016 \\\\\n",
      "HIPT (ViT-256-P + ViT-4096)  &  0.762 +/- 0.089 &  0.827 +/- 0.069 &  0.652 +/- 0.076 &  0.820 +/- 0.047 &  0.935 +/- 0.022 &  0.956 +/- 0.013 \\\\\n",
      "HIPT (ViT-256-PF + ViT-4096) &  0.821 +/- 0.069 &  0.874 +/- 0.060 &  0.923 +/- 0.020 &  0.952 +/- 0.021 &  0.974 +/- 0.012 &  0.980 +/- 0.013 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BRCA (25%)</th>\n",
       "      <th>BRCA (100%)</th>\n",
       "      <th>Lung (25%)</th>\n",
       "      <th>Lung (100%)</th>\n",
       "      <th>Kidney (25%)</th>\n",
       "      <th>Kidney (100%)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>HIPT (No ViT)</th>\n",
       "      <td>0.784 +/- 0.061</td>\n",
       "      <td>0.837 +/- 0.062</td>\n",
       "      <td>0.835 +/- 0.050</td>\n",
       "      <td>0.928 +/- 0.023</td>\n",
       "      <td>0.955 +/- 0.016</td>\n",
       "      <td>0.965 +/- 0.013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HIPT (ViT-256 + ViT-4096)</th>\n",
       "      <td>0.758 +/- 0.076</td>\n",
       "      <td>0.823 +/- 0.071</td>\n",
       "      <td>0.695 +/- 0.069</td>\n",
       "      <td>0.786 +/- 0.096</td>\n",
       "      <td>-</td>\n",
       "      <td>0.956 +/- 0.016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HIPT (ViT-256-P + ViT-4096)</th>\n",
       "      <td>0.762 +/- 0.089</td>\n",
       "      <td>0.827 +/- 0.069</td>\n",
       "      <td>0.652 +/- 0.076</td>\n",
       "      <td>0.820 +/- 0.047</td>\n",
       "      <td>0.935 +/- 0.022</td>\n",
       "      <td>0.956 +/- 0.013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HIPT (ViT-256-PF + ViT-4096)</th>\n",
       "      <td>0.821 +/- 0.069</td>\n",
       "      <td>0.874 +/- 0.060</td>\n",
       "      <td>0.923 +/- 0.020</td>\n",
       "      <td>0.952 +/- 0.021</td>\n",
       "      <td>0.974 +/- 0.012</td>\n",
       "      <td>0.980 +/- 0.013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   BRCA (25%)      BRCA (100%)  \\\n",
       "HIPT (No ViT)                 0.784 +/- 0.061  0.837 +/- 0.062   \n",
       "HIPT (ViT-256 + ViT-4096)     0.758 +/- 0.076  0.823 +/- 0.071   \n",
       "HIPT (ViT-256-P + ViT-4096)   0.762 +/- 0.089  0.827 +/- 0.069   \n",
       "HIPT (ViT-256-PF + ViT-4096)  0.821 +/- 0.069  0.874 +/- 0.060   \n",
       "\n",
       "                                   Lung (25%)      Lung (100%)  \\\n",
       "HIPT (No ViT)                 0.835 +/- 0.050  0.928 +/- 0.023   \n",
       "HIPT (ViT-256 + ViT-4096)     0.695 +/- 0.069  0.786 +/- 0.096   \n",
       "HIPT (ViT-256-P + ViT-4096)   0.652 +/- 0.076  0.820 +/- 0.047   \n",
       "HIPT (ViT-256-PF + ViT-4096)  0.923 +/- 0.020  0.952 +/- 0.021   \n",
       "\n",
       "                                 Kidney (25%)    Kidney (100%)  \n",
       "HIPT (No ViT)                 0.955 +/- 0.016  0.965 +/- 0.013  \n",
       "HIPT (ViT-256 + ViT-4096)                   -  0.956 +/- 0.016  \n",
       "HIPT (ViT-256-P + ViT-4096)   0.935 +/- 0.022  0.956 +/- 0.013  \n",
       "HIPT (ViT-256-PF + ViT-4096)  0.974 +/- 0.012  0.980 +/- 0.013  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_dir = './results_cvpr2022_class/'\n",
    "summary_df_all = []\n",
    "for task in ['tcga_brca_subtype', 'tcga_lung_subtype', 'tcga_kidney_subtype']: # os.listdir(os.path.join(results_dir))\n",
    "    props = [0.25, 1.0]\n",
    "    for prop in props:\n",
    "        models = [\n",
    "                  'hipt_n',\n",
    "                  'hipt_lgp[None]_[None]',\n",
    "                  'hipt_lgp[vit4k_xs_dino]_[None]',\n",
    "                  'hipt_lgp[vit4k_xs_dino]_freeze_[None]']\n",
    "        model_names = [\n",
    "                       'HIPT (No ViT)',\n",
    "                       'HIPT (ViT-256 + ViT-4096)',\n",
    "                       'HIPT (ViT-256-P + ViT-4096)',\n",
    "                       'HIPT (ViT-256-PF + ViT-4096)']\n",
    "        features = ['vits_tcga_pancancer_dino']\n",
    "\n",
    "        exps = []\n",
    "        for feature in features:\n",
    "            for model in models:\n",
    "                exp_name = '%s_%s_%s_%0.2f_none_s1' % (task, model, feature, prop)\n",
    "                exps.append(exp_name)\n",
    "        \n",
    "        summary_df = []\n",
    "        for exp in exps:\n",
    "            summary_path = os.path.join(results_dir, task, exp, 'summary.csv')\n",
    "            if os.path.isfile(summary_path):\n",
    "                results_df = pd.read_csv(summary_path, index_col=0)\n",
    "                exp_auc = '%0.3f +/- %0.3f' % (results_df['test_auc'].mean(), results_df['test_auc'].std())\n",
    "                summary_df.append(exp_auc)\n",
    "            else:\n",
    "                summary_df.append('-')\n",
    "        summary_df_all.append(summary_df)\n",
    "\n",
    "summary_df_all = pd.DataFrame(summary_df_all).T\n",
    "summary_df_all.columns = ['BRCA (25%)', 'BRCA (100%)', 'Lung (25%)', 'Lung (100%)',\n",
    "                          'Kidney (25%)', 'Kidney (100%)']\n",
    "model_names = [\n",
    "              'HIPT (No ViT)',\n",
    "               'HIPT (ViT-256 + ViT-4096)',\n",
    "               'HIPT (ViT-256-P + ViT-4096)',\n",
    "               'HIPT (ViT-256-PF + ViT-4096)']\n",
    "summary_df_all.index = model_names\n",
    "print(summary_df_all.to_latex())\n",
    "summary_df_all"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc-autonumbering": true,
  "toc-showcode": false,
  "toc-showmarkdowntxt": false,
  "toc-showtags": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}